diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31a6a31c..086a9b00 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,7 @@ jobs: build: ${{ steps.filter.outputs.build }} bundle: ${{ steps.filter.outputs.bundle }} docker: ${{ steps.filter.outputs.docker }} + deploy: ${{ steps.filter.outputs.deploy }} e2e: ${{ steps.filter.outputs.e2e }} steps: - uses: actions/checkout@v7 @@ -98,6 +99,15 @@ jobs: - 'package-lock.json' - '.github/workflows/**' - 'packages/**' + # #692: the installer integration test (real Keeper + 2x2 ClickHouse + # cluster in Docker). The installer, its tests, and the workflow. + deploy: + - 'deploy/**' + - 'tests/deploy/**' + - 'tests/helpers/deploy-*' + - 'tests/unit/*-script.test.js' + - 'package.json' + - '.github/workflows/**' docker: - 'Dockerfile' - 'deploy/caddy/**' @@ -239,7 +249,7 @@ jobs: set -euo pipefail tmp=$(mktemp -d) tar -C "$tmp" -xzf dist/altinity-sql-browser.tar.gz --strip-components=1 - for f in sql.html local.py sql-browser.xml run.sh VERSION; do + for f in sql.html local.py sql-browser.xml run.sh VERSION deploy/install.sh deploy/uninstall.sh deploy/lib/asb-ch.sh deploy/http_handlers.xml; do test -f "$tmp/$f" || { echo "missing $f in bundle" >&2; exit 1; } done python3 -c "import ast,sys; ast.parse(open(sys.argv[1]).read())" "$tmp/local.py" @@ -253,13 +263,18 @@ jobs: curl -fsS "http://localhost:8901/config.json" \ | python3 -c "import sys,json; assert json.load(sys.stdin)['hosts'], 'no hosts parsed'" kill "$pid" + # The shipped installer, from the extracted bundle, with no build tree: --artifact is the + # whole contract. --dry-run renders config + http_handlers.xml; the (unreachable) OIDC + # discovery must be a soft failure. + "$tmp/deploy/install.sh" --dry-run --artifact "$tmp/sql.html" --client-id x --issuer http://idp.invalid + test -s "$tmp/dist/http_handlers.xml" - name: Lint the installer (shellcheck) run: | sudo apt-get update -qq && sudo apt-get install -y -qq shellcheck # #630 Phase 8 modifies both build wrappers (each now composes the # package build explicitly) — lint both alongside the existing root # installer shell script. - shellcheck install.sh build/bundle.sh deploy/install.sh + shellcheck install.sh build/bundle.sh deploy/install.sh deploy/uninstall.sh deploy/lib/asb-ch.sh tests/deploy/run.sh # Exercise the published runtime shape, including Caddy's static # Content-Encoding negotiation. Node decodes each selected sidecar, so the @@ -407,12 +422,52 @@ jobs: path: test-results/ retention-days: 14 + # #692: run the real installer against a Keeper + 2-shard x 2-replica + # ClickHouse cluster (with ) and assert, per node and byte-exact, what + # /sql and /sql/config.json serve after install, upgrade, repair, partial + # failure, and uninstall. Matrix over the supported server flavours; pin exact + # tags so a moving image cannot turn this red unrelated to the diff. + deploy-integration: + needs: changes + if: >- + github.event_name == 'push' || + github.event_name == 'workflow_dispatch' || + (github.event_name == 'pull_request' && needs.changes.outputs.deploy == 'true') + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + image: + - clickhouse/clickhouse-server:26.3.18.32 + - altinity/clickhouse-server:25.8.33.20001.altinityantalya + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: '22' + cache: npm + - run: npm ci --no-audit --no-fund + - name: Build the artifact + run: npm run build + - name: Installer integration test + env: + CH_IMAGE: ${{ matrix.image }} + run: npm run test:deploy + - uses: actions/upload-artifact@v7 + if: failure() + with: + name: deploy-test-logs-${{ strategy.job-index }} + path: dist/deploy-test-logs/ + if-no-files-found: ignore + retention-days: 7 + # Require this stable check in branch protection. It runs even when some # expensive jobs were correctly skipped and fails if any applicable job did. gate: name: CI gate if: always() - needs: [changes, examples, test, size, bundle, docker-smoke, e2e] + needs: [changes, examples, test, size, bundle, docker-smoke, deploy-integration, e2e] runs-on: ubuntu-latest steps: - name: Verify applicable jobs diff --git a/.wiki/Deployment-and-Security.md b/.wiki/Deployment-and-Security.md index 08c4524c..02b4e6e6 100644 --- a/.wiki/Deployment-and-Security.md +++ b/.wiki/Deployment-and-Security.md @@ -9,6 +9,17 @@ The build produces `dist/sql.html`. Deployment copies it to ClickHouse Cluster distribution options and tradeoffs are documented in [`docs/ASSET-DISTRIBUTION.md`](../docs/ASSET-DISTRIBUTION.md). +`deploy/install.sh --cluster` (#692) fans the files out SQL-only in two phases +(`_asb_next_*` staged and verified per node, then promoted locally and verified; +a failed `CREATE ... AS SELECT` truncates its target, so the live file is never +the target of a remote-source statement). Needs a cluster `` and the +documented minimal grants. `npm run test:deploy` (`tests/deploy/`, CI job +`deploy-integration`) runs it against Keeper + 2x2 ClickHouse on exact tags +(26.3.18.32, Antalya 25.8.33) and asserts byte-exact files on every node, +including a node killed mid-run. The release bundle ships `deploy/` (install.sh, +uninstall.sh, lib/) for use with `--artifact`. One installer at a time (no lock); +`--ddl-timeout` must be >= 1; the password reaches only `clickhouse-client`. + ## Authentication modes - Native/Bearer OAuth for ClickHouse variants with token processors and ephemeral diff --git a/CHANGELOG.md b/CHANGELOG.md index ff8b59db..2bcc3f15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,38 @@ auto-generated per-PR notes; this file is the curated, human-readable history. ## [Unreleased] +### Added +- **#692 (part 2): `deploy/install.sh --cluster` now works on multi-replica clusters.** + SQL-only, two-phase install with server-side fan-out: preflight, pre-clean, + phase 1 to `_asb_next_*` (upload once and copy, or `--pull-url`), per-node length + and SHA-256 verification, phase 2 local promote, verify live, always-cleanup. Live + files are only written from a node-local verified copy (a failed + `CREATE ... AS SELECT` truncates its target). New flags `--artifact`, `--pull-url`, + `--transport native|http`, `--http-port`, `--ddl-timeout`, `--verify-only`; + `deploy/uninstall.sh` empties the files on every node. A hung `--pull-url` + download is bounded (`max_execution_time` and HTTP timeouts from `--ddl-timeout`). + Single-node mode writes with `INSERT INTO FUNCTION file` so it needs only `FILE` + (a CTAS also demanded `INSERT` on the temp table). New integration suite + `tests/deploy/` (`npm run test:deploy`, CI job `deploy-integration`, matrix + ClickHouse 26.3 and Antalya 25.8.33) and the release bundle now ships `deploy/`. + `docs/ASSET-DISTRIBUTION.md` documents the procedure, grants and limits. + Review hardening: `--ddl-timeout` must be a decimal integer 1..999999 (0 means asynchronous DDL; no octal or overflow) and + `distributed_ddl_output_mode=throw` is pinned on both transports; pre-clean and + `uninstall.sh` empty the `_asb_next_*` files on every node; verification demands + one row per node, as many distinct nodes (`serverUUID()`, port-independent, so several + instances per host work) as the cluster has, and `count() = 1` + per row (uninstall's 0-byte check alone accepts `n = 0`); a zero-byte artifact is + refused before any write; cleanup ignores INT/TERM, is bounded (DDL timeout capped at 30 s plus client-side + timeouts on every call) + and always removes the credential header file; the password is no longer exported + (only `clickhouse-client` sees it; never `npm`/`node`/OIDC `curl`); `curl -q` + ignores `~/.curlrc`; no `--artifact` and no build tree is a clear error. Concurrent + runs are documented as unsupported. The integration suite gains a node killed + mid-run and a minimal `deploy_verify` user (a verify-only user needs + `CREATE TEMPORARY TABLE`; the install/uninstall user does not), checks all four nodes with positive controls, pins exact + image tags, and dumps container logs on any failure; the bundle CI step now runs + the shipped installer with `--dry-run --artifact`. + ### Fixed - **#692 (part 1): deployment docs and installer output corrections.** `deploy/install.sh` printed `https://` even without `--secure` and omitted the diff --git a/README.md b/README.md index 6ddba27f..6121d704 100644 --- a/README.md +++ b/README.md @@ -653,12 +653,18 @@ CLICKHOUSE_PASSWORD=… ./deploy/install.sh \ [--issuer https://accounts.google.com] \ [--audience ] \ # audience-gated CH → also sends the access_token [--ch-auth basic] \ # OSS CH + ch-jwt-verify → JWT as Basic password - [--cluster ] # BROKEN on multi-replica clusters (#692); omit + [--cluster ] \ # fan out to every node (server-side, verified per node) + [--artifact sql.html] \ # prebuilt SPA (release asset): no npm/node needed + [--pull-url ] [--transport native|http] [--http-port N] \ + [--ddl-timeout 180] [--verify-only] [--secure] ``` -`--cluster` currently fails on multi-replica clusters (tracked in #692); for a -single node run the installer without it, and for multi-node clusters follow -[the interim manual procedure](docs/ASSET-DISTRIBUTION.md#multi-node-clusters-interim-manual). +With `--cluster` the installer writes `sql.html` and `sql-config.json` to every node's +`user_files/` using SQL only, in two phases (stage and verify, then promote and +verify), and checks each node's length and SHA-256; re-run it after a scale-out. +`deploy/uninstall.sh` empties the files. Cross-node reads need the cluster's `` +(or per-replica credentials), run one installer at a time, and the deploy user needs a few grants — see +[docs/ASSET-DISTRIBUTION.md](docs/ASSET-DISTRIBUTION.md#installing-with-deployinstallsh). With **no** `--audience`, the IdP returns an **id_token** (its `aud` is the client_id) and the browser sends that as the bearer — so ClickHouse's diff --git a/build/bundle.sh b/build/bundle.sh index e15fe41f..7f061d6c 100755 --- a/build/bundle.sh +++ b/build/bundle.sh @@ -9,6 +9,7 @@ # sql-browser.xml — sample public-demo connections (merged with the # user's ~/.clickhouse-client/config.xml by the runner) # run.sh — self-resolving launcher (python3 local.py) +# deploy/ — cluster installer: install.sh, uninstall.sh, lib/, http_handlers.xml # VERSION # README.txt # @@ -35,6 +36,11 @@ mkdir -p "$STAGE" cp "$OUT/sql.html" "$STAGE/sql.html" cp "$ROOT/build/local.py" "$STAGE/local.py" cp "$ROOT/deploy/sql-browser.xml" "$STAGE/sql-browser.xml" +# The cluster installer (deploy/install.sh sources deploy/lib/asb-ch.sh; with +# --artifact sql.html it needs no node/npm). Keep the same relative layout. +mkdir -p "$STAGE/deploy/lib" +cp "$ROOT/deploy/install.sh" "$ROOT/deploy/uninstall.sh" "$ROOT/deploy/http_handlers.xml" "$STAGE/deploy/" +cp "$ROOT/deploy/lib/asb-ch.sh" "$STAGE/deploy/lib/" printf '%s\n' "$VERSION" > "$STAGE/VERSION" cat > "$STAGE/run.sh" <<'EOF' @@ -60,6 +66,13 @@ sql-browser.xml (public demos) and offers each in the login picker — your own config wins on a name clash. Run ./install.sh (curl|sh) to also copy sql-browser.xml into ~/.clickhouse-client/. +Installing on a ClickHouse server or cluster: deploy/ holds install.sh, +uninstall.sh and http_handlers.xml. This bundle has no build tree, so pass the +bundled SPA explicitly (no node/npm needed): + + CLICKHOUSE_PASSWORD=... ./deploy/install.sh --artifact ./sql.html \ + --ch-host HOST --ch-user USER --client-id OAUTH_CLIENT_ID [--cluster NAME] + Env: PORT (default 8900), LOCAL_CH_CONFIG (override with one explicit file), SQL_BROWSER_SPA. diff --git a/deploy/install.sh b/deploy/install.sh index 19782a9b..7c0e9264 100755 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -1,19 +1,35 @@ #!/usr/bin/env bash -# Install the Altinity SQL Browser onto a ClickHouse cluster: -# 1. build the single-file SPA (dist/sql.html) +# Install the Altinity SQL Browser onto a ClickHouse server or cluster, SQL-only +# (no ssh, no per-node upload): +# 1. build the single-file SPA (dist/sql.html), or take --artifact # 2. render config.json from the OAuth args # 3. render dist/http_handlers.xml (CSP connect-src filled from OIDC discovery) -# 4. upload the SPA + config into ClickHouse user_files/ (sql.html, sql-config.json) +# 4. distribute sql.html + sql-config.json into every node's user_files/ with +# server-side fan-out, in two phases so a failure never half-writes the +# live files (see "How it works" below) # 5. print the next step to enable /sql with the rendered http_handlers.xml # -# The password is read from the CLICKHOUSE_PASSWORD env var or prompted — never -# passed on the command line (it would leak via `ps`/shell history). +# The password is read from the CLICKHOUSE_PASSWORD env var or prompted - never +# passed on the command line (it would leak via `ps`/shell history). The script +# takes it out of its environment at once and hands it only to clickhouse-client +# (as an env prefix on that one call); npm, node and the OIDC curl never see it. +# With --transport http it travels in a 0600 header file handed to curl, not argv. +# HTTP(S)_PROXY / NO_PROXY apply to every curl call (curl's own rules, no --noproxy): +# in plain-HTTP mode a configured proxy would see the credentials; use --secure or +# unset the proxy variables. ~/.curlrc is ignored (curl -q). # -# Prerequisites: bash, node 22+ and npm (the installer builds the SPA from source), -# curl (only for OIDC discovery; it falls back to the Google connect-src without -# it), and clickhouse-client (not needed for --dry-run). To skip the build, take -# the prebuilt sql.html from the GitHub release and upload it yourself; see -# docs/ASSET-DISTRIBUTION.md. +# Concurrent runs against the same cluster are UNSUPPORTED (no lock): two installs +# share the _asb_t_* temp tables and _asb_next_* files and can corrupt each other. +# Run one installer (or uninstaller) at a time. +# +# Internal test hooks (not a user interface; used only by tests/deploy/run.sh and the unit suite): +# ASB_STAGE_TOKEN fixes the stage-file token; ASB_TEST_BARRIER[_POINT|_TIMEOUT] parks the run at a +# defined point (see "TEST HOOK" below). Both are validated; unset => no behavior change. +# +# Prerequisites: bash, node 22+ and npm (unless --artifact), curl (OIDC +# discovery; and the HTTP transport), clickhouse-client (native transport; not +# needed for --dry-run). The prebuilt sql.html from a GitHub release works as +# --artifact; see docs/ASSET-DISTRIBUTION.md. # # Usage: # CLICKHOUSE_PASSWORD=... ./deploy/install.sh \ @@ -21,18 +37,43 @@ # --ch-user admin \ # --client-id \ # [--issuer https://accounts.google.com] \ -# [--audience ] \ # audience-gated CH → also sends access_token -# [--ch-auth basic] \ # OSS CH + ch-jwt-verify → JWT as Basic password -# [--cluster my_cluster] \ # BROKEN on multi-replica clusters (#692); omit +# [--audience ] \ # audience-gated CH -> also sends access_token +# [--ch-auth basic] \ # OSS CH + ch-jwt-verify -> JWT as Basic password +# [--cluster my_cluster] \ # fan out to every node of this cluster +# [--artifact dist/sql.html] \ # skip the build; use this prebuilt SPA +# [--pull-url ] \ # cluster mode: nodes fetch the SPA from this URL +# # (default: upload once, nodes copy from the stage) +# [--transport native|http] [--http-port N] \ +# [--ddl-timeout 180] \ # distributed_ddl_task_timeout, seconds (>= 1) +# [--verify-only] \ # only compare every node's live files to the artifact # [--secure] \ # [--dry-run] # build + render config.json + http_handlers.xml, print, no CH contact +# +# How it works (cluster mode). Facts proven in #692: a failed CREATE TABLE ... +# AS SELECT truncates its target file, so a remote source is never CTAS'd into +# the live file directly. +# preflight every node reachable, cross-node reads authenticate +# pre-clean DROP TABLE IF EXISTS default._asb_* ON CLUSTER ... SYNC, and EMPTY the +# _asb_next_* files on every node (a stale one can never pass phase-1 verify) +# phase 1 fan out to _asb_next_* on every node (stage upload to the +# connected node, or url() pull), then verify length + sha256 +# phase 2 only if every node verified: each node copies its own _asb_next_* +# over the live sql.html / sql-config.json, then verify again +# always truncate the stage file, drop the temp tables (live files untouched) set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=deploy/lib/asb-ch.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/asb-ch.sh" CH_HOST="" CH_USER="default" ISSUER="https://accounts.google.com" CLIENT_ID="" AUDIENCE="" CLUSTER="" SECURE=0 CH_AUTH="" DRY_RUN=0 +ARTIFACT="" PULL_URL="" TRANSPORT="native" HTTP_PORT="" DDL_TIMEOUT=180 VERIFY_ONLY=0 while [[ $# -gt 0 ]]; do + case "$1" in + --ch-host|--ch-user|--client-id|--issuer|--audience|--ch-auth|--cluster|--artifact|--pull-url|--transport|--http-port|--ddl-timeout) + [[ $# -ge 2 ]] || { echo "missing value for $1" >&2; exit 2; } ;; + esac case "$1" in --ch-host) CH_HOST="$2"; shift 2 ;; --ch-user) CH_USER="$2"; shift 2 ;; @@ -41,6 +82,12 @@ while [[ $# -gt 0 ]]; do --audience) AUDIENCE="$2"; shift 2 ;; --ch-auth) CH_AUTH="$2"; shift 2 ;; --cluster) CLUSTER="$2"; shift 2 ;; + --artifact) ARTIFACT="$2"; shift 2 ;; + --pull-url) PULL_URL="$2"; shift 2 ;; + --transport) TRANSPORT="$2"; shift 2 ;; + --http-port) HTTP_PORT="$2"; shift 2 ;; + --ddl-timeout) DDL_TIMEOUT="$2"; shift 2 ;; + --verify-only) VERIFY_ONLY=1; shift ;; --secure) SECURE=1; shift ;; --dry-run) DRY_RUN=1; shift ;; # build + render config.json + http_handlers.xml, print, no ClickHouse contact *) echo "unknown arg: $1" >&2; exit 2 ;; @@ -50,53 +97,105 @@ done [[ -n "$CLIENT_ID" ]] || { echo "--client-id is required" >&2; exit 2; } # --ch-host is only needed to reach ClickHouse; a --dry-run just renders artifacts. [[ -n "$CH_HOST" || "$DRY_RUN" == 1 ]] || { echo "--ch-host is required" >&2; exit 2; } +case "$TRANSPORT" in native|http) ;; *) echo "--transport must be native or http" >&2; exit 2 ;; esac +[[ -z "$CLUSTER" ]] || validate_cluster_name "$CLUSTER" +validate_pos --ddl-timeout "$DDL_TIMEOUT" +[[ -z "$HTTP_PORT" ]] || validate_uint --http-port "$HTTP_PORT" +if [[ -n "$PULL_URL" && "$VERIFY_ONLY" == 1 ]]; then echo "--pull-url and --verify-only are mutually exclusive" >&2; exit 2; fi +if [[ -n "$ARTIFACT" && ! -f "$ARTIFACT" ]]; then echo "--artifact: no such file: $ARTIFACT" >&2; exit 2; fi +BUILD_DIR="$ROOT/build"; BUILD_ENTRY="$BUILD_DIR/build.mjs" # (spelled in pieces: see the build-order policy test) +if [[ -z "$ARTIFACT" && ! ( -f "$ROOT/package.json" && -f "$BUILD_ENTRY" ) ]]; then + echo "no build tree next to this script (package.json or the SPA build entry point is missing); pass --artifact " >&2 + exit 2 +fi if [[ "$DRY_RUN" != 1 && -z "${CLICKHOUSE_PASSWORD:-}" ]]; then read -r -s -p "ClickHouse password for $CH_USER@$CH_HOST: " CLICKHOUSE_PASSWORD echo fi -export CLICKHOUSE_PASSWORD - -CH=(clickhouse-client --host "$CH_HOST" --user "$CH_USER") -[[ "$SECURE" == 1 ]] && CH+=(--secure) - -# user_files is node-local. NOTE: --cluster is currently broken (#692): the -# clusterAllReplicas INSERT below fails on any cluster with more than one -# replica (STORAGE_REQUIRES_PARAMETER). The guard fails CLOSED, before building: -# proceed only when the count query succeeds and prints exactly 1. -if [[ "$DRY_RUN" != 1 && -n "$CLUSTER" ]]; then - GUARD_ERR_FILE="$(mktemp)" - if ! NODES=$("${CH[@]}" --query "SELECT count() FROM system.clusters WHERE cluster = '${CLUSTER}'" 2>"$GUARD_ERR_FILE"); then - GUARD_ERR="$(head -n 1 "$GUARD_ERR_FILE")" - rm -f "$GUARD_ERR_FILE" - echo "ERROR: could not query system.clusters on ${CH_HOST} to check cluster" >&2 - echo "'${CLUSTER}': ${GUARD_ERR:-no error output}" >&2 - exit 2 - fi - rm -f "$GUARD_ERR_FILE" - NODES="${NODES//[[:space:]]/}" - if [[ ! "$NODES" =~ ^[0-9]+$ ]]; then - echo "ERROR: unexpected system.clusters output for cluster '${CLUSTER}':" >&2 - echo "'${NODES}' (expected a node count). Refusing to continue." >&2 - exit 2 - elif (( NODES == 0 )); then - echo "ERROR: cluster '${CLUSTER}' was not found in system.clusters on ${CH_HOST}" >&2 - echo "(0 replicas). Check the cluster name and the node you connected to." >&2 - exit 2 - elif (( NODES > 1 )); then - echo "ERROR: cluster '${CLUSTER}' has ${NODES} replicas. --cluster is currently" >&2 - echo "broken on multi-node clusters (#692); user_files is node-local. Use the" >&2 - echo "manual procedure: 'Multi-node clusters (interim, manual)' in" >&2 - echo "docs/ASSET-DISTRIBUTION.md." >&2 - exit 2 +# Keep the password out of the environment of everything but clickhouse-client. +CH_PASSWORD="${CLICKHOUSE_PASSWORD:-}" +unset CLICKHOUSE_PASSWORD + +# Stage token: unique per run so concurrent/aborted runs never share a stage file. +STAGE_TOKEN="${ASB_STAGE_TOKEN:-$(date +%s)${RANDOM}${RANDOM}}" +validate_token "$STAGE_TOKEN" +STAGE_FILE="_asb_stage_${STAGE_TOKEN}.html" + +# TEST HOOK (internal; used only by tests/deploy/run.sh, like ASB_STAGE_TOKEN): with ASB_TEST_BARRIER= +# the run pauses at ASB_TEST_BARRIER_POINT (pre_fanout = after stage upload+verify, before the phase-1 fan-out; +# pre_promote = after phase-1 verify, before phase 2), creates ".reached" and waits for ".go" +# (hard cap ASB_TEST_BARRIER_TIMEOUT seconds, default 120, then fails). Unset => no behavior change at all. +BARRIER="${ASB_TEST_BARRIER:-}" BARRIER_POINT="${ASB_TEST_BARRIER_POINT:-pre_fanout}" +BARRIER_TIMEOUT="${ASB_TEST_BARRIER_TIMEOUT:-120}" +if [[ -n "$BARRIER" ]]; then + [[ "$BARRIER" =~ ^/[A-Za-z0-9_./-]+$ && "$BARRIER" != *..* ]] || fail "invalid ASB_TEST_BARRIER '$BARRIER' (absolute path; allowed: A-Z a-z 0-9 _ . / -)" + echo "WARNING: test hook ASB_TEST_BARRIER active (${BARRIER}, point ${BARRIER_POINT}); never set this in production" >&2 + [[ "$BARRIER_POINT" == pre_fanout || "$BARRIER_POINT" == pre_promote ]] || fail "invalid ASB_TEST_BARRIER_POINT '$BARRIER_POINT' (pre_fanout|pre_promote)" + [[ "$BARRIER_TIMEOUT" =~ ^[1-9][0-9]{0,3}$ ]] || fail "invalid ASB_TEST_BARRIER_TIMEOUT '$BARRIER_TIMEOUT'" +fi +unset ASB_TEST_BARRIER ASB_TEST_BARRIER_POINT ASB_TEST_BARRIER_TIMEOUT # never inherited by child processes +barrier() { + [[ -n "$BARRIER" && "$BARRIER_POINT" == "$1" ]] || return 0 + : > "${BARRIER}.reached" + local i + for (( i = 0; i < BARRIER_TIMEOUT * 5; i++ )); do + [[ -e "${BARRIER}.go" ]] && return 0 + sleep 0.2 + done + fail "test barrier ${BARRIER}.go not released within ${BARRIER_TIMEOUT}s" +} +NEXT_HTML="_asb_next_sql.html" NEXT_CFG="_asb_next_sql-config.json" +# Table names are independent of the file names they point at. +T_NEXT_HTML="default._asb_t_next_html" T_NEXT_CFG="default._asb_t_next_cfg" +T_LIVE_HTML="default._asb_t_live_html" T_LIVE_CFG="default._asb_t_live_cfg" +ALL_TABLES=("$T_NEXT_HTML" "$T_NEXT_CFG" "$T_LIVE_HTML" "$T_LIVE_CFG") +EXPECTED_NODES=1 ON_CLUSTER="" +CLEANUP_ARMED=0 STAGE_USED=0 +CH=() + +TMP="$(mktemp -d)" +cleanup() { + local rc=$? + trap '' INT TERM # a second ^C must not abandon the cleanup half-done + trap - EXIT + set +e # nothing below may stop us reaching `rm -rf "$TMP"` (it holds the credentials) + if [[ "$CLEANUP_ARMED" == 1 ]]; then + limit_ddl_timeout 30 # bounded: every cleanup call ends within min(--ddl-timeout,30)+10 s, even on a dead node + echo "==> Cleanup (stage file emptied, temp tables dropped; live files are never touched here)" + if [[ "$STAGE_USED" == 1 ]]; then + ch_query "INSERT INTO FUNCTION file('${STAGE_FILE}','RawBLOB') SETTINGS engine_file_truncate_on_insert=1 SELECT ''" >/dev/null \ + || echo "WARN: could not empty stage file ${STAGE_FILE} on ${CH_HOST}; remove its contents by hand" >&2 + fi + local t + for t in "${ALL_TABLES[@]}"; do + ch_query "DROP TABLE IF EXISTS ${t}${ON_CLUSTER} SYNC" >/dev/null \ + || echo "WARN: could not drop ${t}; drop it by hand before re-running" >&2 + done fi + rm -rf "$TMP" + exit "$rc" +} +trap cleanup EXIT +trap 'exit 130' INT TERM +set_cluster_clause + +# ---- reachability first, so a bad host/cluster fails before the slow build ---- +if [[ "$DRY_RUN" != 1 ]]; then + setup_transport + if [[ -n "$CLUSTER" ]]; then preflight_cluster; else info "==> Single-node mode: only ${CH_HOST} is updated (behind a load balancer, pass --cluster to reach every node)"; fi fi -echo "==> Building @altinity/clickhouse-http" -npm --prefix "$ROOT" run build:clickhouse-http +if [[ -z "$ARTIFACT" ]]; then + echo "==> Building @altinity/clickhouse-http" + npm --prefix "$ROOT" run build:clickhouse-http -echo "==> Building dist/sql.html" -node "$ROOT/build/build.mjs" + echo "==> Building dist/sql.html" + node "$ROOT/build/build.mjs" + ARTIFACT="$ROOT/dist/sql.html" +else + echo "==> Using prebuilt artifact $ARTIFACT (no build)" +fi echo "==> Rendering config.json" CONFIG_JSON="{\"issuer\":\"${ISSUER}\",\"client_id\":\"${CLIENT_ID}\"" @@ -112,9 +211,8 @@ if [[ "$CH_AUTH" == "basic" ]]; then CONFIG_JSON+=",\"ch_auth\":\"basic\"" fi CONFIG_JSON+="}" -CONFIG_FILE="$(mktemp)" -trap 'rm -f "$CONFIG_FILE"' EXIT -printf '%s\n' "$CONFIG_JSON" > "$CONFIG_FILE" +CONFIG_FILE="$TMP/config.json" +printf '%s' "$CONFIG_JSON" > "$CONFIG_FILE" echo "==> Rendering http_handlers.xml (CSP connect-src from OIDC discovery)" # The CSP connect-src must allow same-origin ('self', for ClickHouse + config.json) @@ -125,7 +223,7 @@ echo "==> Rendering http_handlers.xml (CSP connect-src from OIDC discovery)" ISSUER_ORIGIN="$(printf '%s' "$ISSUER" | grep -oiE '^https?://[^/]+' || true)" CONNECT_HOSTS="https://accounts.google.com https://oauth2.googleapis.com" DISC_URL="${ISSUER%/}/.well-known/openid-configuration" -if DISC_JSON="$(curl -fsS --max-time 10 "$DISC_URL" 2>/dev/null)"; then +if DISC_JSON="$(curl -q -fsS --max-time 10 "$DISC_URL" 2>/dev/null)"; then # Pull the origin (scheme://host[:port]) of token/authorization endpoints, add # the issuer origin, dedupe. Tolerates whitespace variations in the JSON. EP_ORIGINS="$(printf '%s' "$DISC_JSON" \ @@ -141,13 +239,10 @@ fi # Rewrite only the connect-src host list in the committed template; everything else # (the rest of the CSP, the other headers) is copied verbatim. HANDLERS_OUT="$ROOT/dist/http_handlers.xml" +mkdir -p "$ROOT/dist" sed -E "s#(connect-src 'self')[^;]*#\1 ${CONNECT_HOSTS}#" \ "$ROOT/deploy/http_handlers.xml" > "$HANDLERS_OUT" -# Print the post-install instructions. Called after upload, and also by --dry-run -# so the URL/scheme logic is testable without a ClickHouse. The native port -# --ch-host is used with (9000/9440) differs from the HTTP(S) port the browser -# needs, so print the HTTP defaults (8123 plain / 8443 TLS) explicitly. print_next_steps() { local scheme="http" port="8123" host="${CH_HOST:-}" if [[ "$SECURE" == 1 ]]; then scheme="https"; port="8443"; fi @@ -172,43 +267,105 @@ http_authentication verifier. See README. EOT } + if [[ "$DRY_RUN" == 1 ]]; then echo - echo "==> DRY RUN — no ClickHouse contact. Rendered artifacts:" + echo "==> DRY RUN - no ClickHouse contact. Rendered artifacts:" echo "--- config.json ---" - cat "$CONFIG_FILE" + cat "$CONFIG_FILE"; echo echo "--- dist/http_handlers.xml ---" cat "$HANDLERS_OUT" print_next_steps exit 0 fi -# Upload raw bytes via FORMAT RawBLOB on stdin — no base64, no command-line -# length limit, written as the clickhouse process so perms are correct. -upload() { # upload - local src="$1" - local fname="$2" - local tbl - tbl="default._asb_$(echo "$fname" | tr '.-' '__')" - local on_cluster="" - [[ -n "$CLUSTER" ]] && on_cluster="ON CLUSTER '${CLUSTER}'" - "${CH[@]}" --query "CREATE TABLE IF NOT EXISTS ${tbl} ${on_cluster} (content String) - ENGINE = File('RawBLOB', '/var/lib/clickhouse/user_files/${fname}')" - if [[ -n "$CLUSTER" ]]; then - "${CH[@]}" --query "INSERT INTO FUNCTION clusterAllReplicas('${CLUSTER}','${tbl%.*}','${tbl#*.}') - SETTINGS engine_file_truncate_on_insert = 1 FORMAT RawBLOB" < "$src" +[[ -f "$ARTIFACT" ]] || fail "artifact not found: $ARTIFACT (the build did not produce it)" +ART_LEN="$(size_of "$ARTIFACT")" ART_SHA="$(sha256_of "$ARTIFACT")" +CFG_LEN="$(size_of "$CONFIG_FILE")" CFG_SHA="$(sha256_of "$CONFIG_FILE")" +# A 0-byte file is what a failed write leaves behind and what the static handler serves as an +# empty 200; installing one (or "verifying" it) would be a silent outage. Refuse before any write. +(( ART_LEN > 0 )) || fail "artifact ${ARTIFACT} is empty (0 bytes); refusing to install it" +(( CFG_LEN > 0 )) || fail "rendered config.json is empty; refusing to install" +info "==> Artifact: ${ART_LEN} bytes, sha256 ${ART_SHA}" + +verify_live() { verify_files "live files" one sql.html "$ART_LEN" "$ART_SHA" sql-config.json "$CFG_LEN" "$CFG_SHA"; } + +if [[ "$VERIFY_ONLY" == 1 ]]; then + verify_live || fail "live files differ from the artifact" + info "==> Verified: every node serves the expected files." + exit 0 +fi + +# ---------------------------------------------------------------- phases +CLEANUP_ARMED=1 + +pre_clean() { + info "==> Pre-clean (drop leftover temp tables, empty stale ${NEXT_HTML} / ${NEXT_CFG})" + local t + for t in "${ALL_TABLES[@]}"; do ddl "drop ${t}" "DROP TABLE IF EXISTS ${t}${ON_CLUSTER} SYNC"; done + # Phase-1 verify must only ever see bytes written by THIS run, never a leftover next file. + ddl "empty ${NEXT_HTML} on every node" "$(write_file_stmt "$T_NEXT_HTML" "$NEXT_HTML" "SELECT '' AS c SETTINGS engine_file_truncate_on_insert=1")" + ddl "empty ${NEXT_CFG} on every node" "$(write_file_stmt "$T_NEXT_CFG" "$NEXT_CFG" "SELECT '' AS c SETTINGS engine_file_truncate_on_insert=1")" + ddl "drop ${T_NEXT_HTML}" "DROP TABLE IF EXISTS ${T_NEXT_HTML}${ON_CLUSTER} SYNC" + ddl "drop ${T_NEXT_CFG}" "DROP TABLE IF EXISTS ${T_NEXT_CFG}${ON_CLUSTER} SYNC" +} + +# next-file source for the SPA, as the SELECT of the fan-out statement. +# A hung or slow url() is bounded: per-query max_execution_time plus HTTP +# connect/receive timeouts, all derived from --ddl-timeout (>= 1). +PULL_TIMEOUT="$DDL_TIMEOUT" +phase1_html_source() { + if [[ -n "$PULL_URL" ]]; then + printf "SELECT c FROM url('%s','RawBLOB','c String') SETTINGS max_http_get_redirects=10, engine_file_truncate_on_insert=1, max_execution_time=%s, http_connection_timeout=10, http_receive_timeout=%s" \ + "$(sql_str "$PULL_URL")" "$PULL_TIMEOUT" "$PULL_TIMEOUT" else - "${CH[@]}" --query "INSERT INTO ${tbl} - SETTINGS engine_file_truncate_on_insert = 1 FORMAT RawBLOB" < "$src" + printf "SELECT c FROM clusterAllReplicas('%s', file('%s','RawBLOB','c String')) SETTINGS engine_file_empty_if_not_exists=1, engine_file_truncate_on_insert=1" "$CLUSTER" "$STAGE_FILE" + fi +} + +phase1() { + info "==> Phase 1: distribute to ${NEXT_HTML} / ${NEXT_CFG}" + local up_name="$STAGE_FILE" rc + [[ -n "$CLUSTER" ]] || up_name="$NEXT_HTML" + if [[ -z "$PULL_URL" ]]; then + info " uploading $(basename "$ARTIFACT") to ${CH_HOST}:${up_name}" + if [[ -n "$CLUSTER" ]]; then STAGE_USED=1; fi + ch_query "INSERT INTO FUNCTION file('${up_name}','RawBLOB') SETTINGS engine_file_truncate_on_insert=1 FORMAT RawBLOB" "$ARTIFACT" >/dev/null \ + || fail "upload of the artifact failed" + if [[ -n "$CLUSTER" ]]; then + if check_rows "$STAGE_FILE" "$ART_LEN" "$ART_SHA" 0; then rc=0; else rc=$?; fi + (( rc == 2 )) && fail "could not verify the staged upload" + # The stage lives on the connected node only: exactly one node must hold it, and every node must answer. + [[ "$ROWS" == "$EXPECTED_NODES" && "$HOSTS" == "$EXPECTED_NODES" ]] \ + || fail "stage verification returned ${ROWS} rows / ${HOSTS} distinct nodes; expected ${EXPECTED_NODES}" + [[ "$MATCH" == 1 ]] || fail "staged upload not found with the expected length/sha256 on exactly one node (matches: ${MATCH}, rows: ${ROWS})" + fi fi - "${CH[@]}" --query "DROP TABLE IF EXISTS ${tbl} ${on_cluster}" + barrier pre_fanout + if [[ -n "$CLUSTER" || -n "$PULL_URL" ]]; then + ddl "fan out sql.html to every node" "$(write_file_stmt "$T_NEXT_HTML" "$NEXT_HTML" "$(phase1_html_source)")" + fi + ddl "write sql-config.json to every node" "$(write_file_stmt "$T_NEXT_CFG" "$NEXT_CFG" "SELECT '$(sql_str "$(cat "$CONFIG_FILE")")' AS c SETTINGS engine_file_truncate_on_insert=1")" + ddl "drop ${T_NEXT_HTML}" "DROP TABLE IF EXISTS ${T_NEXT_HTML}${ON_CLUSTER} SYNC" + ddl "drop ${T_NEXT_CFG}" "DROP TABLE IF EXISTS ${T_NEXT_CFG}${ON_CLUSTER} SYNC" +} + +phase2() { + info "==> Phase 2: promote ${NEXT_HTML} / ${NEXT_CFG} over the live files (local copy on each node)" + ddl "promote sql.html" "$(write_file_stmt "$T_LIVE_HTML" sql.html "SELECT c FROM file('${NEXT_HTML}','RawBLOB','c String') SETTINGS engine_file_truncate_on_insert=1")" + ddl "promote sql-config.json" "$(write_file_stmt "$T_LIVE_CFG" sql-config.json "SELECT c FROM file('${NEXT_CFG}','RawBLOB','c String') SETTINGS engine_file_truncate_on_insert=1")" + ddl "drop ${T_LIVE_HTML}" "DROP TABLE IF EXISTS ${T_LIVE_HTML}${ON_CLUSTER} SYNC" + ddl "drop ${T_LIVE_CFG}" "DROP TABLE IF EXISTS ${T_LIVE_CFG}${ON_CLUSTER} SYNC" } -echo "==> Uploading sql.html" -upload "$ROOT/dist/sql.html" "sql.html" -echo "==> Uploading sql-config.json" -upload "$CONFIG_FILE" "sql-config.json" +pre_clean +phase1 +verify_files "phase 1 (next files)" one "$NEXT_HTML" "$ART_LEN" "$ART_SHA" "$NEXT_CFG" "$CFG_LEN" "$CFG_SHA" \ + || fail "phase 1 verification failed; live files were NOT touched" +barrier pre_promote +phase2 +verify_live || fail "phase 2 verification failed: some live files do not match; re-run the installer" echo -echo "==> Assets uploaded to ClickHouse user_files/." +echo "==> Assets uploaded to ClickHouse user_files/ and verified on ${EXPECTED_NODES} node(s)." print_next_steps diff --git a/deploy/lib/asb-ch.sh b/deploy/lib/asb-ch.sh new file mode 100644 index 00000000..3ef4ce7d --- /dev/null +++ b/deploy/lib/asb-ch.sh @@ -0,0 +1,234 @@ +# shellcheck shell=bash +# Shared ClickHouse plumbing for deploy/install.sh and deploy/uninstall.sh (#692). +# Sourced, never executed. Callers set (before using any function): +# CH_HOST CH_USER SECURE TRANSPORT(native|http) HTTP_PORT DDL_TIMEOUT CLUSTER TMP +# and CH_PASSWORD (a plain shell variable, NOT exported: the caller moves it out of the +# environment so npm/node/curl never inherit it). It reaches clickhouse-client only as an +# env prefix on that one invocation, and curl only inside a 0600 header file; it is never +# in argv. + +# ---------------------------------------------------------------- basics +fail() { echo "ERROR: $*" >&2; exit 1; } +info() { echo "$*"; } + +# SQL single-quoted-literal escaping: backslash first, then the quote. +sql_str() { printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e "s/'/\\\\'/g"; } + +sha256_of() { # sha256_of -> lowercase hex + local out + if command -v sha256sum >/dev/null 2>&1; then out="$(sha256sum "$1")" + else out="$(shasum -a 256 "$1")"; fi + printf '%s' "${out%% *}" +} +size_of() { local n; n="$(wc -c < "$1")"; printf '%s' "${n//[[:space:]]/}"; } + +validate_cluster_name() { # injection guard: the name lands in SQL + [[ "$1" =~ ^[A-Za-z0-9_.-]+$ ]] || fail "invalid --cluster name '$1' (allowed: A-Z a-z 0-9 _ . -)" +} +validate_token() { [[ "$1" =~ ^[A-Za-z0-9_]+$ ]] || fail "invalid stage token '$1' (allowed: A-Z a-z 0-9 _)"; } +validate_uint() { [[ "$2" =~ ^[0-9]+$ ]] || fail "$1 must be a non-negative integer (got '$2')"; } +# 0 would make ClickHouse run the DDL asynchronously (no wait, no error surfaced): refuse it. +# Digits only, no leading zero (bash arithmetic would read 09 as a bad octal and 010 as 8), at most 6. +validate_pos() { [[ "$2" =~ ^[1-9][0-9]{0,5}$ ]] || fail "$1 must be a decimal integer from 1 to 999999 without leading zeros (got '$2'; 0 means asynchronous DDL in ClickHouse)"; } + +# ---------------------------------------------------------------- transport +setup_transport() { + if [[ "$TRANSPORT" == http ]]; then + command -v curl >/dev/null 2>&1 || fail "curl is required for --transport http" + local scheme=http port="${HTTP_PORT:-8123}" + if [[ "$SECURE" == 1 ]]; then scheme=https; port="${HTTP_PORT:-8443}"; fi + HTTP_URL="${scheme}://${CH_HOST}:${port}/" + case "$CH_USER$CH_PASSWORD" in *$'\n'*|*$'\r'*) fail "user/password must not contain newlines (--transport http)" ;; esac + # Credentials travel in headers read from a 0600 file (curl -H @file): never argv. + ( umask 077; printf 'X-ClickHouse-User: %s\nX-ClickHouse-Key: %s\n' "$CH_USER" "$CH_PASSWORD" > "$TMP/auth" ) + chmod 600 "$TMP/auth" + else + command -v clickhouse-client >/dev/null 2>&1 || fail "clickhouse-client is required for --transport native" + build_native_args + fi + return 0 +} + +# distributed_ddl_output_mode is pinned: a profile/user default of never_throw or +# *_only_active would turn a dead node's DDL failure into a silent success. +build_native_args() { + CH=(clickhouse-client --host "$CH_HOST" --user "$CH_USER" "--distributed_ddl_task_timeout=${DDL_TIMEOUT}" --distributed_ddl_output_mode=throw --format TSV) + # Cleanup only: bound every socket wait too, so an unreachable node cannot stall it for minutes. + if (( CLIENT_SLACK < 100 )); then + CH+=("--connect_timeout=${CONNECT_TIMEOUT}" "--send_timeout=$((DDL_TIMEOUT + CLIENT_SLACK))" "--receive_timeout=$((DDL_TIMEOUT + CLIENT_SLACK))") + fi + [[ "$SECURE" == 1 ]] && CH+=(--secure) + return 0 +} + +# Client-side bounds. A normal call may legitimately take --ddl-timeout plus a long tail +# (fan-out of a large file), so it gets 300 s of slack; cleanup gets 10 s and a 10 s connect +# timeout. limit_ddl_timeout (cleanup only) therefore makes every cleanup call finish within +# min(--ddl-timeout, ) + 10 s, whatever the server or network does. +CLIENT_SLACK=300 CONNECT_TIMEOUT=30 +limit_ddl_timeout() { # limit_ddl_timeout + (( DDL_TIMEOUT <= $1 )) || DDL_TIMEOUT="$1" + CLIENT_SLACK=10 CONNECT_TIMEOUT=10 + [[ "$TRANSPORT" == http ]] || build_native_args + return 0 +} + +# Print a failure's text plus a targeted hint. $1 = file holding the text. +explain_failure() { + head -n 20 "$1" >&2 + local t; t="$(cat "$1")" + case "$t" in + *AUTHENTICATION_FAILED*|*"Code: 516"*) + echo "HINT: authentication failed. If the first connection works but cross-node reads fail, the cluster" >&2 + echo " needs (or per-replica user/password) in remote_servers, and the deploy user must" >&2 + echo " exist on every node. Otherwise check the user/password." >&2 ;; + *ACCESS_DENIED*|*NOT_ENOUGH_PRIVILEGES*|*"Code: 497"*) + echo "HINT: missing grants - see 'Grants' in docs/ASSET-DISTRIBUTION.md (FILE, CLUSTER, REMOTE," >&2 + echo " CREATE/DROP TABLE on default.*, SELECT on system.clusters; URL for --pull-url; a --verify-only-only" >&2 + echo " user also needs CREATE TEMPORARY TABLE)." >&2 ;; + *TIMEOUT_EXCEEDED*|*"Code: 159"*) + echo "HINT: a node did not answer the ON CLUSTER DDL within ${DDL_TIMEOUT}s (--ddl-timeout). It may be down;" >&2 + echo " its queued DDL replays when it returns. Bring it back and re-run." >&2 ;; + *"Code: 209"*|*"Code: 210"*|*NETWORK_ERROR*|*"Connection refused"*|*"Could not resolve"*|*"Failed to connect"*|*"Couldn't connect"*) + echo "HINT: node unreachable - check the host/port and that every cluster node is up." >&2 ;; + esac +} + +# ch_query [stdin-file] -> TSV rows on stdout. Failure: text on stderr, rc 1. +ch_query() { + if [[ "$TRANSPORT" == http ]]; then ch_query_http "$@"; else ch_query_native "$@"; fi +} + +ch_query_native() { + local sql="$1" in="${2:-/dev/null}" wrap=() + # Cleanup only: clickhouse-client's send/receive timeouts are IDLE timeouts, not a total bound, so + # wrap the call in coreutils `timeout` when it exists (absent on stock macOS: idle timeouts only). + if (( CLIENT_SLACK < 100 )) && command -v timeout >/dev/null 2>&1; then wrap=(timeout "$((DDL_TIMEOUT + CLIENT_SLACK))"); fi + if ! CLICKHOUSE_PASSWORD="$CH_PASSWORD" "${wrap[@]}" "${CH[@]}" --query "$sql" < "$in" 2>"$TMP/err"; then + explain_failure "$TMP/err"; return 1 + fi +} + +ch_query_http() { + local sql="$1" in="${2:-}" code + : > "$TMP/body"; : > "$TMP/hdrs" + if [[ -n "$in" ]]; then { printf '%s\n' "$sql"; cat "$in"; } > "$TMP/req"; else printf '%s' "$sql" > "$TMP/req"; fi + if ! code="$(curl -q -sS --connect-timeout "$CONNECT_TIMEOUT" --max-time "$((DDL_TIMEOUT + CLIENT_SLACK))" -o "$TMP/body" -D "$TMP/hdrs" -w '%{http_code}' \ + -H "@$TMP/auth" -H 'Content-Type: text/plain' --data-binary "@$TMP/req" \ + "${HTTP_URL}?wait_end_of_query=1&distributed_ddl_task_timeout=${DDL_TIMEOUT}&distributed_ddl_output_mode=throw" 2>"$TMP/err")"; then + explain_failure "$TMP/err"; return 1 + fi + local line body; body="$(cat "$TMP/body")" + local bad=0 + [[ "$code" =~ ^2[0-9][0-9]$ ]] || bad=1 + # A header LINE, not a substring: Access-Control-Expose-Headers lists the name on every response. + while IFS= read -r line; do + line="$(printf '%s' "$line" | tr '[:upper:]' '[:lower:]')" + case "$line" in x-clickhouse-exception-code:*) bad=1 ;; esac + done < "$TMP/hdrs" + case "$body" in *__exception__*) bad=1 ;; esac + if (( bad )); then + { echo "HTTP ${code}"; printf '%s\n' "$body"; } > "$TMP/err" + explain_failure "$TMP/err"; return 1 + fi + printf '%s' "$body"; [[ -z "$body" ]] || echo + return 0 +} + +# ddl