deploy: SQL-only two-phase install with server-side fan-out and per-node verification (#692 part 2)

.github/workflows/ci.yml

@@ -28,6 +28,7 @@ jobs:
       build: ${{ steps.filter.outputs.build }}
       bundle: ${{ steps.filter.outputs.bundle }}
       docker: ${{ steps.filter.outputs.docker }}
+      deploy: ${{ steps.filter.outputs.deploy }}
       e2e: ${{ steps.filter.outputs.e2e }}
     steps:
       - uses: actions/checkout@v7
@@ -98,6 +99,15 @@ jobs:
               - 'package-lock.json'
               - '.github/workflows/**'
               - 'packages/**'
+            # #692: the installer integration test (real Keeper + 2x2 ClickHouse
+            # cluster in Docker). The installer, its tests, and the workflow.
+            deploy:
+              - 'deploy/**'
+              - 'tests/deploy/**'
+              - 'tests/helpers/deploy-*'
+              - 'tests/unit/*-script.test.js'
+              - 'package.json'
+              - '.github/workflows/**'
             docker:
               - 'Dockerfile'
               - 'deploy/caddy/**'
@@ -239,7 +249,7 @@ jobs:
           set -euo pipefail
           tmp=$(mktemp -d)
           tar -C "$tmp" -xzf dist/altinity-sql-browser.tar.gz --strip-components=1
-          for f in sql.html local.py sql-browser.xml run.sh VERSION; do
+          for f in sql.html local.py sql-browser.xml run.sh VERSION deploy/install.sh deploy/uninstall.sh deploy/lib/asb-ch.sh deploy/http_handlers.xml; do
             test -f "$tmp/$f" || { echo "missing $f in bundle" >&2; exit 1; }
           done
           python3 -c "import ast,sys; ast.parse(open(sys.argv[1]).read())" "$tmp/local.py"
@@ -253,13 +263,18 @@ jobs:
           curl -fsS "http://localhost:8901/config.json" \
             | python3 -c "import sys,json; assert json.load(sys.stdin)['hosts'], 'no hosts parsed'"
           kill "$pid"
+          # The shipped installer, from the extracted bundle, with no build tree: --artifact is the
+          # whole contract. --dry-run renders config + http_handlers.xml; the (unreachable) OIDC
+          # discovery must be a soft failure.
+          "$tmp/deploy/install.sh" --dry-run --artifact "$tmp/sql.html" --client-id x --issuer http://idp.invalid
+          test -s "$tmp/dist/http_handlers.xml"
       - name: Lint the installer (shellcheck)
         run: |
           sudo apt-get update -qq && sudo apt-get install -y -qq shellcheck
           # #630 Phase 8 modifies both build wrappers (each now composes the
           # package build explicitly) — lint both alongside the existing root
           # installer shell script.
-          shellcheck install.sh build/bundle.sh deploy/install.sh
+          shellcheck install.sh build/bundle.sh deploy/install.sh deploy/uninstall.sh deploy/lib/asb-ch.sh tests/deploy/run.sh
 
   # Exercise the published runtime shape, including Caddy's static
   # Content-Encoding negotiation. Node decodes each selected sidecar, so the
@@ -407,12 +422,52 @@ jobs:
           path: test-results/
           retention-days: 14
 
+  # #692: run the real installer against a Keeper + 2-shard x 2-replica
+  # ClickHouse cluster (with <secret>) and assert, per node and byte-exact, what
+  # /sql and /sql/config.json serve after install, upgrade, repair, partial
+  # failure, and uninstall. Matrix over the supported server flavours; pin exact
+  # tags so a moving image cannot turn this red unrelated to the diff.
+  deploy-integration:
+    needs: changes
+    if: >-
+      github.event_name == 'push' ||
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'pull_request' && needs.changes.outputs.deploy == 'true')
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        image:
+          - clickhouse/clickhouse-server:26.3.18.32
+          - altinity/clickhouse-server:25.8.33.20001.altinityantalya
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-node@v7
+        with:
+          node-version: '22'
+          cache: npm
+      - run: npm ci --no-audit --no-fund
+      - name: Build the artifact
+        run: npm run build
+      - name: Installer integration test
+        env:
+          CH_IMAGE: ${{ matrix.image }}
+        run: npm run test:deploy
+      - uses: actions/upload-artifact@v7
+        if: failure()
+        with:
+          name: deploy-test-logs-${{ strategy.job-index }}
+          path: dist/deploy-test-logs/
+          if-no-files-found: ignore
+          retention-days: 7
+
   # Require this stable check in branch protection. It runs even when some
   # expensive jobs were correctly skipped and fails if any applicable job did.
   gate:
     name: CI gate
     if: always()
-    needs: [changes, examples, test, size, bundle, docker-smoke, e2e]
+    needs: [changes, examples, test, size, bundle, docker-smoke, deploy-integration, e2e]
     runs-on: ubuntu-latest
     steps:
       - name: Verify applicable jobs

.wiki/Deployment-and-Security.md

@@ -9,6 +9,17 @@ The build produces `dist/sql.html`. Deployment copies it to ClickHouse
 Cluster distribution options and tradeoffs are documented in
 [`docs/ASSET-DISTRIBUTION.md`](../docs/ASSET-DISTRIBUTION.md).
 
+`deploy/install.sh --cluster` (#692) fans the files out SQL-only in two phases
+(`_asb_next_*` staged and verified per node, then promoted locally and verified;
+a failed `CREATE ... AS SELECT` truncates its target, so the live file is never
+the target of a remote-source statement). Needs a cluster `<secret>` and the
+documented minimal grants. `npm run test:deploy` (`tests/deploy/`, CI job
+`deploy-integration`) runs it against Keeper + 2x2 ClickHouse on exact tags
+(26.3.18.32, Antalya 25.8.33) and asserts byte-exact files on every node,
+including a node killed mid-run. The release bundle ships `deploy/` (install.sh,
+uninstall.sh, lib/) for use with `--artifact`. One installer at a time (no lock);
+`--ddl-timeout` must be >= 1; the password reaches only `clickhouse-client`.
+
 ## Authentication modes
 
 - Native/Bearer OAuth for ClickHouse variants with token processors and ephemeral

CHANGELOG.md

@@ -9,6 +9,38 @@ auto-generated per-PR notes; this file is the curated, human-readable history.
 
 ## [Unreleased]
 
+### Added
+- **#692 (part 2): `deploy/install.sh --cluster` now works on multi-replica clusters.**
+  SQL-only, two-phase install with server-side fan-out: preflight, pre-clean,
+  phase 1 to `_asb_next_*` (upload once and copy, or `--pull-url`), per-node length
+  and SHA-256 verification, phase 2 local promote, verify live, always-cleanup. Live
+  files are only written from a node-local verified copy (a failed
+  `CREATE ... AS SELECT` truncates its target). New flags `--artifact`, `--pull-url`,
+  `--transport native|http`, `--http-port`, `--ddl-timeout`, `--verify-only`;
+  `deploy/uninstall.sh` empties the files on every node. A hung `--pull-url`
+  download is bounded (`max_execution_time` and HTTP timeouts from `--ddl-timeout`).
+  Single-node mode writes with `INSERT INTO FUNCTION file` so it needs only `FILE`
+  (a CTAS also demanded `INSERT` on the temp table). New integration suite
+  `tests/deploy/` (`npm run test:deploy`, CI job `deploy-integration`, matrix
+  ClickHouse 26.3 and Antalya 25.8.33) and the release bundle now ships `deploy/`.
+  `docs/ASSET-DISTRIBUTION.md` documents the procedure, grants and limits.
+  Review hardening: `--ddl-timeout` must be a decimal integer 1..999999 (0 means asynchronous DDL; no octal or overflow) and
+  `distributed_ddl_output_mode=throw` is pinned on both transports; pre-clean and
+  `uninstall.sh` empty the `_asb_next_*` files on every node; verification demands
+  one row per node, as many distinct nodes (`serverUUID()`, port-independent, so several
+  instances per host work) as the cluster has, and `count() = 1`
+  per row (uninstall's 0-byte check alone accepts `n = 0`); a zero-byte artifact is
+  refused before any write; cleanup ignores INT/TERM, is bounded (DDL timeout capped at 30 s plus client-side
+  timeouts on every call)
+  and always removes the credential header file; the password is no longer exported
+  (only `clickhouse-client` sees it; never `npm`/`node`/OIDC `curl`); `curl -q`
+  ignores `~/.curlrc`; no `--artifact` and no build tree is a clear error. Concurrent
+  runs are documented as unsupported. The integration suite gains a node killed
+  mid-run and a minimal `deploy_verify` user (a verify-only user needs
+  `CREATE TEMPORARY TABLE`; the install/uninstall user does not), checks all four nodes with positive controls, pins exact
+  image tags, and dumps container logs on any failure; the bundle CI step now runs
+  the shipped installer with `--dry-run --artifact`.
+
 ### Fixed
 - **#692 (part 1): deployment docs and installer output corrections.**
   `deploy/install.sh` printed `https://` even without `--secure` and omitted the

README.md

@@ -653,12 +653,18 @@ CLICKHOUSE_PASSWORD=… ./deploy/install.sh \
   [--issuer https://accounts.google.com] \
   [--audience <api-audience>] \   # audience-gated CH → also sends the access_token
   [--ch-auth basic] \             # OSS CH + ch-jwt-verify → JWT as Basic password
-  [--cluster <cluster-name>]      # BROKEN on multi-replica clusters (#692); omit
+  [--cluster <cluster-name>] \    # fan out to every node (server-side, verified per node)
+  [--artifact sql.html] \         # prebuilt SPA (release asset): no npm/node needed
+  [--pull-url <url>] [--transport native|http] [--http-port N] \
+  [--ddl-timeout 180] [--verify-only] [--secure]
 ```
 
-`--cluster` currently fails on multi-replica clusters (tracked in #692); for a
-single node run the installer without it, and for multi-node clusters follow
-[the interim manual procedure](docs/ASSET-DISTRIBUTION.md#multi-node-clusters-interim-manual).
+With `--cluster` the installer writes `sql.html` and `sql-config.json` to every node's
+`user_files/` using SQL only, in two phases (stage and verify, then promote and
+verify), and checks each node's length and SHA-256; re-run it after a scale-out.
+`deploy/uninstall.sh` empties the files. Cross-node reads need the cluster's `<secret>`
+(or per-replica credentials), run one installer at a time, and the deploy user needs a few grants — see
+[docs/ASSET-DISTRIBUTION.md](docs/ASSET-DISTRIBUTION.md#installing-with-deployinstallsh).
 
 With **no** `--audience`, the IdP returns an **id_token** (its `aud` is the
 client_id) and the browser sends that as the bearer — so ClickHouse's

build/bundle.sh

@@ -9,6 +9,7 @@
 #     sql-browser.xml     — sample public-demo connections (merged with the
 #                           user's ~/.clickhouse-client/config.xml by the runner)
 #     run.sh              — self-resolving launcher (python3 local.py)
+#     deploy/             — cluster installer: install.sh, uninstall.sh, lib/, http_handlers.xml
 #     VERSION
 #     README.txt
 #
@@ -35,6 +36,11 @@ mkdir -p "$STAGE"
 cp "$OUT/sql.html"                  "$STAGE/sql.html"
 cp "$ROOT/build/local.py"           "$STAGE/local.py"
 cp "$ROOT/deploy/sql-browser.xml"   "$STAGE/sql-browser.xml"
+# The cluster installer (deploy/install.sh sources deploy/lib/asb-ch.sh; with
+# --artifact sql.html it needs no node/npm). Keep the same relative layout.
+mkdir -p "$STAGE/deploy/lib"
+cp "$ROOT/deploy/install.sh" "$ROOT/deploy/uninstall.sh" "$ROOT/deploy/http_handlers.xml" "$STAGE/deploy/"
+cp "$ROOT/deploy/lib/asb-ch.sh" "$STAGE/deploy/lib/"
 printf '%s\n' "$VERSION" > "$STAGE/VERSION"
 
 cat > "$STAGE/run.sh" <<'EOF'
@@ -60,6 +66,13 @@ sql-browser.xml (public demos) and offers each in the login picker — your own
 config wins on a name clash. Run ./install.sh (curl|sh) to also copy
 sql-browser.xml into ~/.clickhouse-client/.
 
+Installing on a ClickHouse server or cluster: deploy/ holds install.sh,
+uninstall.sh and http_handlers.xml. This bundle has no build tree, so pass the
+bundled SPA explicitly (no node/npm needed):
+
+  CLICKHOUSE_PASSWORD=... ./deploy/install.sh --artifact ./sql.html \
+    --ch-host HOST --ch-user USER --client-id OAUTH_CLIENT_ID [--cluster NAME]
+
 Env: PORT (default 8900), LOCAL_CH_CONFIG (override with one explicit file),
 SQL_BROWSER_SPA.