Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/previous-release.env
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Exact server image from the release immediately preceding this candidate.
# Update this file as part of each release-preparation change.
# Versioned published upgrade fixture; not the mutable newest-release pointer.
# Advance deliberately during release preparation; publication alone cannot invalidate CI.
MDBASE_CONNECT_PREVIOUS_RELEASE=v0.1.0-beta.102
MDBASE_CONNECT_PREVIOUS_RELEASE_COMMIT=ea65ae4f13d0e0621fe9654618e13b505cdebf21
MDBASE_CONNECT_PREVIOUS_SERVER_IMAGE=ghcr.io/mdbase-dev/mdbase-connect-server@sha256:bfb2ca0beb8e63005b89354f7b57e3568fb438f40f4948963a723c1aeb815f1e
Expand Down
35 changes: 33 additions & 2 deletions .github/workflows/desktop-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,15 @@ on:
pull_request:
paths:
- ".github/workflows/desktop-release.yml"
- "scripts/ci/desktop-test-plan.mjs"
- "scripts/lib/ci-work-selection.test.mjs"
- "Cargo.toml"
- "Cargo.lock"
- "crates/**"
- ".cargo/**"
- "rust-toolchain*"
- "deploy/docker/mdbase-rs-revision"
- "package.json"
- "apps/editor/**"
- "apps/desktop/**"
- "packages/**"
Expand All @@ -31,6 +40,26 @@ concurrency:
cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
select-tests:
if: github.event_name == 'pull_request'
runs-on: ubuntu-24.04
outputs:
headless: ${{ steps.plan.outputs.headless }}
windows: ${{ steps.plan.outputs.windows }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 24
- id: plan
env:
BASE: ${{ github.event.pull_request.base.sha }}
HEAD: ${{ github.event.pull_request.head.sha }}
run: node scripts/ci/desktop-test-plan.mjs "$BASE" "$HEAD"

build:
if: github.event_name == 'workflow_dispatch' && startsWith(github.ref, 'refs/tags/v')
name: ${{ matrix.label }}
Expand Down Expand Up @@ -569,7 +598,8 @@ jobs:
--input -

headless-package-smoke:
if: github.event_name != 'push'
needs: select-tests
if: ${{ !cancelled() && (github.event_name == 'workflow_dispatch' || (needs.select-tests.result == 'success' && needs.select-tests.outputs.headless == 'true')) }}
name: ${{ matrix.label }}
permissions:
contents: read
Expand Down Expand Up @@ -695,7 +725,8 @@ jobs:
run: pnpm --filter mdbase-editor test

windows-package-smoke:
if: github.event_name != 'push'
needs: select-tests
if: ${{ !cancelled() && (github.event_name == 'workflow_dispatch' || (needs.select-tests.result == 'success' && needs.select-tests.outputs.windows == 'true')) }}
name: Windows Store package smoke test
runs-on: windows-2025
permissions:
Expand Down
52 changes: 41 additions & 11 deletions .github/workflows/server-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ jobs:
GH_TOKEN: ${{ github.token }}
run: |
if [[ "$GITHUB_EVENT_NAME" == pull_request ]]; then
if jq -e 'any(.pull_request.labels[]?; .name == "ci:full")' \
if jq -e 'any(.pull_request.labels[]?; .name == "ci:full" or .name == "ci:benchmark-binaries")' \
"$GITHUB_EVENT_PATH" >/dev/null
then
echo 'run_full=true' >>"$GITHUB_OUTPUT"
Expand Down Expand Up @@ -425,12 +425,49 @@ jobs:
key: hosted-provider-${{ steps.mdbase_revision.outputs.revision }}
cache-on-failure: true
- run: cargo fmt --all --check
- run: scripts/check-cargo-features
- run: cargo clippy --locked --workspace --all-targets -- -D warnings
- run: cargo test --locked --workspace
env:
MDBASE_CONNECT_ENV: test
MDBASE_CONNECT_SECRET_BACKEND: insecure-test-file
- run: cargo build --locked --workspace
- name: Archive runtime binaries for transfer measurement
if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:benchmark-binaries')
run: |
tar -cf "$RUNNER_TEMP/binaries.tar" -C target/debug mdbase mdbase-connect-hosted-provider
du -b "$RUNNER_TEMP/binaries.tar" >>"$GITHUB_STEP_SUMMARY"
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:benchmark-binaries')
with:
name: benchmark-linux-binaries
path: ${{ runner.temp }}/binaries.tar
compression-level: 1
retention-days: 1
if-no-files-found: error

# Measurement only: does not supply qualification inputs or replace builds.
# Compare producer wait + transfer with the parallel shard build timings.
binary-transfer-benchmark:
name: Binary transfer benchmark (${{ matrix.shard }})
needs: hosted-provider-rust
if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:benchmark-binaries')
runs-on: ubuntu-latest
timeout-minutes: 10
strategy:
fail-fast: false
matrix:
shard: [local-relay, sync, provider, files, files-adversarial, desktop]
steps:
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: benchmark-linux-binaries
- name: Extract and exercise transferred binaries
run: |
tar -xf binaries.tar
./mdbase --help >/dev/null
./mdbase-connect-hosted-provider --help >/dev/null
du -b binaries.tar >>"$GITHUB_STEP_SUMMARY"

hosted-provider-system:
name: System suite (${{ matrix.shard }})
Expand Down Expand Up @@ -504,16 +541,9 @@ jobs:
cache-on-failure: true
- run: pnpm install --frozen-lockfile
- run: pnpm build
- run: cargo fmt --all --check
- run: pnpm check:cargo-features
- run: cargo clippy --locked --workspace --all-targets -- -D warnings
# Compile and test every Rust crate against the exact SDK revision used
# by the production provider image. This keeps path-only developer
# dependencies from making a clean checkout unreproducible.
- run: cargo test --locked --workspace
env:
MDBASE_CONNECT_ENV: test
MDBASE_CONNECT_SECRET_BACKEND: insecure-test-file
# The hosted-provider-rust job checks this exact lockfile/SDK/toolchain
# configuration once; Qualification requires it as well as every shard.
# Keep builds local until measured artifact fan-out beats cached builds.
- run: cargo build --locked --workspace
- name: Restore Playwright cache
if: matrix.browser
Expand Down
61 changes: 61 additions & 0 deletions docs/ci-qualification.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,67 @@ JSON manifest containing the commit, Git tree, package and Cargo lock hashes,
the mdbase engine revision, and the Server CI workflow hash. A reused main run
also records the exact upstream merge-queue run.

## Desktop PR selection

`Desktop Release` keeps cross-platform editor/release regression tests on every
matching PR. Editor-only changes skip the standalone headless CLI build matrix;
CSS-only editor changes also skip Windows Store packaging. Mixed changes,
shared inputs, unknown paths, and empty diffs retain native checks. The selector
uses the complete base-to-head merge-base diff, with renames expanded to both
paths. Explicit release dispatches still run all native checks.

## Rust checks and binary transfer measurement

The `hosted-provider-rust` job owns formatting, resolved feature validation,
workspace Clippy, and workspace unit tests for the hosted-provider lockfile,
pinned engine revision, and Linux toolchain configuration. System shards build
that same configuration locally and run their own suites, without repeating
those workspace checks. `Qualification` still requires both the Rust job and
all shards. macOS/Windows coverage remains separate.

To measure binary fan-out, label a PR `ci:benchmark-binaries`. This opts into
full Server CI and adds a measurement-only upload of the CLI/provider runtime
binaries, followed by six matching Linux runner downloads. The tar preserves
executable permissions; each consumer extracts it and exercises both binaries.
Artifacts expire after one day. This never replaces a shard build, contributes
no qualification evidence, and is not a release artifact. Remove the label
when the experiment is complete.

Compare the archive/upload/download/extraction steps and producer readiness
against the same run's per-shard `cargo build` steps using `CI timings`. Include
producer dependency wait in wall-clock estimates and all six downloads in
runner-minute estimates. Do not compare transfer time with the old combined
Clippy/unit-test/build cost: those duplicate checks have already been removed.
The `files-adversarial` suite invokes Cargo tests directly, so shipping only
runtime binaries cannot remove its compilation requirements. A dedicated
build-only producer would need its own end-to-end measurement before adoption.

### Initial measurement (2026-09-16)

[Run 35100242511](https://github.com/mdbase-dev/mdbase-connect/actions/runs/35100242511),
source `dd7506cead8e98fe1da192dd2826669ba8add07b`, completed all six transfer
probes successfully:

- Compressed runtime artifact: 247,093,162 bytes (about 236 MiB).
- Archive step: under the API's one-second timing resolution; upload including
compression: 12 seconds.
- Download steps: 7, 7, 12, 9, 13, 19 seconds; extraction and both `--help`
probes: 0–1 seconds each.
- Independent shard builds: desktop 179s, files 156s, local-relay 189s,
provider 175s, files-adversarial 181s, sync 132s (1,012 runner-seconds total).
- The existing Rust qualification producer completed after 454 seconds;
consumers started two seconds later. Its unit tests and Clippy are on that
dependency path. Waiting for that job would delay runtime tests versus the
independent builds, despite cheap transfers.

Decision: retain independent builds for now. Transfer is inexpensive enough to
justify a future **build-first** producer experiment, not a dependency on the
existing qualification job. This is one run, not a cold/warm-cache study or an
end-to-end artifact-fed system-suite qualification. Runtime probes do not prove
all suite dependencies portable, and files-adversarial still compiles tests.
The run's two upgrade jobs failed on the then-existing mutable-newest-release
policy; that failure does not invalidate the completed transfer measurements.

## Windows daemon task qualification

Full Server CI calls `windows-daemon-lifecycle.yml` and requires its result in
Expand Down
28 changes: 19 additions & 9 deletions docs/release-contract.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,13 +52,23 @@ components.

## Upgrade predecessor and historical regressions

`.github/previous-release.env` identifies the immediate published predecessor.
Update its annotated tag, full commit and immutable server/provider digests as
part of release preparation; ordinary upgrade qualification still requires it to
be the unique newest non-draft GitHub release. Following beta102 publication,
qualification uses beta102 (`ea65ae4f13d0e0621fe9654618e13b505cdebf21`) and its
exact signed server/provider images from verified publication run `35059159880`. Advancing this fixture preserves the
unique-newest-release check; the retained beta95 and beta94 lanes do not move. This fixture refresh does not itself publish or deploy a release.
`.github/previous-release.env` identifies a versioned published upgrade fixture,
not a live newest-release pointer. Advance its annotated tag, full commit and
immutable server/provider digests deliberately during release preparation.
Ordinary CI queries the exact pinned release, requires non-draft published
metadata, verifies its annotated origin tag resolves to the pinned commit, and
checks pulled image source/revision labels. A newer publication cannot invalidate
an unchanged candidate or retroactively change the tested upgrade pair. Missing,
draft, malformed or mismatched fixture identities still fail; no latest-release
fallback or verification bypass exists.

These checks prove compatibility with the recorded fixture, not the currently
deployed predecessor. The actual deployment pair remains independently bound
and qualified by private release preparation and staging below. The retained
beta95 and beta94 lanes do not move. A fixture refresh itself neither publishes
nor deploys a release.
The current fixture is beta102 (`ea65ae4f13d0e0621fe9654618e13b505cdebf21`),
with server/provider images from verified publication run `35059159880`.

Historical regressions and candidate qualification are separate. The beta94
schema-38→41 prelude and beta95 retained-v2 provider rollback scenarios use the
Expand All @@ -71,8 +81,8 @@ metadata and annotated origin tags are checked. These tests do not qualify the
current candidate or authorize restoring beta95/beta99 in production.

The same required provider CI job separately runs `--current-upgrade` against
`.github/previous-release.env`, retaining the unique-newest-published-release
check. It exercises predecessor-issued v1/v2 authority and exact receipts through
`.github/previous-release.env`, retaining exact published-fixture identity
verification. It exercises predecessor-issued v1/v2 authority and exact receipts through
candidate migration and restart, verifies an unchanged historical ledger prefix
and canonical authority, installs a cancellation fence, and actually attempts
predecessor startup on the resulting database. Same-schema predecessors must
Expand Down
25 changes: 25 additions & 0 deletions scripts/ci/desktop-test-plan.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
import { execFileSync } from "node:child_process";
import { appendFileSync } from "node:fs";
import { pathToFileURL } from "node:url";

// Deliberately use narrow exclusions, not an incomplete dependency allowlist.
// Unknown/shared inputs and mixed changes retain all native checks.
export function desktopTestPlan(paths) {
return {
headless: paths.length === 0 || paths.some((path) => !path.startsWith("apps/editor/")),
windows: paths.length === 0 || paths.some((path) => !/^apps\/editor\/.*\.css$/.test(path))
};
}

if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
const [base, head] = process.argv.slice(2);
if (!base || !head) throw new Error("Expected base and head revisions");
// --no-renames includes both sides of moves so native inputs cannot disappear.
const paths = execFileSync("git", ["diff", "--no-renames", "--name-only", "-z", `${base}...${head}`], {
encoding: "utf8"
}).split("\0").filter(Boolean);
const plan = desktopTestPlan(paths);
console.log(JSON.stringify({ paths, ...plan }, null, 2));
appendFileSync(process.env.GITHUB_OUTPUT, Object.entries(plan)
.map(([key, value]) => `${key}=${value}\n`).join(""));
}
68 changes: 68 additions & 0 deletions scripts/lib/ci-work-selection.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import test from "node:test";
import { desktopTestPlan } from "../ci/desktop-test-plan.mjs";

const workflow = (name) => readFileSync(new URL(`../../.github/workflows/${name}.yml`, import.meta.url), "utf8");

test("editor styling skips native CLI and Windows packaging, not release regressions", () => {
assert.deepEqual(desktopTestPlan(["apps/editor/src/styles.css"]), { headless: false, windows: false });
assert.deepEqual(desktopTestPlan(["apps/editor/src/main.ts"]), { headless: false, windows: true });
assert.deepEqual(desktopTestPlan(["apps/editor/package.json"]), { headless: false, windows: true });
assert.match(workflow("desktop-release"), /cross-platform-release-tests:\n if: github.event_name != 'push'/);
});

test("shared, native, unknown, empty and mixed inputs retain native checks", () => {
for (const path of ["Cargo.lock", "Cargo.toml", "crates/cli/src/main.rs", "deploy/docker/mdbase-rs-revision", "apps/desktop/src/main.ts", "packages/sdk/src/index.ts", "pnpm-lock.yaml", ".github/workflows/desktop-release.yml", "scripts/package-headless-cli.mjs", "unknown/input"]) {
for (const paths of [[path], ["apps/editor/src/styles.css", path]]) {
assert.deepEqual(desktopTestPlan(paths), { headless: true, windows: true });
}
}
assert.deepEqual(desktopTestPlan([]), { headless: true, windows: true });
});

test("native selection retains explicit dispatch checks and includes Rust input triggers", () => {
const desktop = workflow("desktop-release");
for (const job of ["headless-package-smoke", "windows-package-smoke"]) {
const block = desktop.split(` ${job}:\n`)[1].split(/\n [a-z][\w-]+:\n/)[0];
assert.match(block, /needs: select-tests/);
assert.match(block, /!cancelled\(\) && \(github.event_name == 'workflow_dispatch' \|\|/);
assert.doesNotMatch(block, /always\(\)/);
assert.match(block, /needs.select-tests.result == 'success'/);
}
for (const path of ["Cargo.toml", "Cargo.lock", "crates/**", "deploy/docker/mdbase-rs-revision", "scripts/ci/desktop-test-plan.mjs"]) {
assert.ok(desktop.includes(`- "${path}"`));
}
});

test("same-configuration Rust checks run once and remain required for qualification", () => {
const server = workflow("server-ci");
const rust = server.split(" hosted-provider-rust:\n")[1].split(" binary-transfer-benchmark:\n")[0];
const shards = server.split(" hosted-provider-system:\n")[1].split(" qualification:\n")[0];
for (const command of ["cargo fmt --all --check", "scripts/check-cargo-features", "cargo clippy --locked --workspace --all-targets -- -D warnings", "cargo test --locked --workspace"]) {
assert.ok(rust.includes(command), command);
assert.ok(!shards.includes(command), command);
}
for (const block of [rust, shards]) {
assert.ok(block.includes("cp deploy/docker/Cargo.lock.hosted-provider Cargo.lock"));
assert.ok(block.includes("cargo build --locked --workspace"));
assert.ok(block.includes("1.94.0"));
}
const gate = server.split(" qualification:\n")[1];
assert.match(gate, /- hosted-provider-rust/);
assert.match(gate, /- hosted-provider-system/);
assert.match(gate, /"\$PROVIDER_RUST"/);
assert.match(gate, /"\$PROVIDER_SYSTEM"/);
assert.doesNotMatch(shards, /needs: hosted-provider-rust/);
});

test("binary transfer remains opt-in and outside the qualification gate", () => {
const server = workflow("server-ci");
assert.match(server, /\.name == "ci:full" or \.name == "ci:benchmark-binaries"/);
assert.match(server, /compression-level: 1/);
const benchmark = server.split(" binary-transfer-benchmark:\n")[1].split(" hosted-provider-system:\n")[0];
assert.match(benchmark, /needs: hosted-provider-rust/);
assert.match(benchmark, /github.event_name == 'pull_request' && contains/);
assert.match(benchmark, /\.\/mdbase --help/);
assert.doesNotMatch(server.split(" qualification:\n")[1], /binary-transfer-benchmark/);
});
Loading
Loading