From b9bc7e5809cdf45869d137578780d2db05e4db25 Mon Sep 17 00:00:00 2001 From: Bozhidar Batsov Date: Tue, 3 Nov 2026 11:03:34 +0200 Subject: [PATCH 1/3] Show which version of a server proof checked describe's versions can't be trusted for this: nbb says TODO, let-go says 1.0 while it's 1.13.0, and jank says jank-0.1-alpha whatever the build. So a profile can have a version snippet now, which proof evaluates in a session of its own before the checks. The text and JSON reports and the Markdown matrix show what it evaluates to after the name of the server. The bundled profiles that install the latest release (or main) get one. nREPL and ClojureCLR keep the version they pin in their names, and jank has no way to tell its version from the code. --- CHANGELOG.md | 4 ++++ cmd/proof/main.go | 41 +++++++++++++++++++++++++++++++++- cmd/proof/main_test.go | 35 +++++++++++++++++++++++++++++ doc/hacking.md | 2 ++ doc/profiles.md | 8 ++++++- doc/usage.md | 4 +++- internal/report/matrix.go | 7 +++--- internal/report/report.go | 24 +++++++++++++++----- internal/report/report_test.go | 33 ++++++++++++++++++++------- profiles/babashka.toml | 4 ++++ profiles/basilisp.toml | 4 ++++ profiles/dialtone.toml | 4 ++++ profiles/repartee.toml | 5 +++++ 13 files changed, 155 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0653a51..d6c04ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## main (unreleased) +### Features + +- [#18](https://github.com/nrepl/proof/pull/18): The `version` snippet, so reports and the compatibility matrix show which version of a server proof checked. + ## 0.1.0 (2026-10-26) Initial release. diff --git a/cmd/proof/main.go b/cmd/proof/main.go index 8683184..2952c0d 100644 --- a/cmd/proof/main.go +++ b/cmd/proof/main.go @@ -12,6 +12,7 @@ import ( "os/signal" "path" "runtime/debug" + "strconv" "strings" "syscall" "time" @@ -22,6 +23,7 @@ import ( "github.com/nrepl/proof/internal/report" "github.com/nrepl/proof/internal/serve" "github.com/nrepl/proof/internal/server" + "github.com/nrepl/proof/nrepl" ) // version is set for releases, with -ldflags "-X main.version=...". @@ -208,12 +210,17 @@ func run(args []string) int { addr = srv.Addr } + ver, err := serverVersion(p, addr) + if err != nil { + fmt.Fprintln(os.Stderr, "proof: couldn't get the version of the server:", err) + } env := &check.Env{Profile: p, Addr: addr} started := time.Now() results := check.Run(env, selected, checks.WireRules()) outcome := check.Apply(results, p.ExpectedFailures, opts.only == "") - r := report.Run{Proof: proofVersion(), Server: p.Name, Address: addr, Started: started, Results: results} + r := report.Run{Proof: proofVersion(), Server: p.Name, ServerVersion: ver, Address: addr, Started: started, + Results: results} report.Text(os.Stdout, r, opts.verbose) for _, id := range outcome.Stale { fmt.Printf("%s is listed in expected-failures but didn't fail; remove it from the profile\n", id) @@ -260,6 +267,38 @@ func startServer(p *profile.Profile) (*server.Server, error) { return srv, err } +// serverVersion evaluates the version snippet of the profile, if it has +// one, in a session of its own. A string comes back without its quotes, +// and on a single line. +func serverVersion(p *profile.Profile, addr string) (string, error) { + sn, ok := p.Snippets["version"] + if !ok { + return "", nil + } + c, err := nrepl.Dial(addr, 5*time.Second) + if err != nil { + return "", err + } + defer c.Close() + clone, err := c.Request(nrepl.Message{"op": "clone"}, p.Timeout) + if err != nil { + return "", err + } + resp, err := c.Request(nrepl.Message{"op": "eval", "code": sn.Code, "session": clone.Str("new-session")}, p.Timeout) + if err != nil { + return "", err + } + values := resp.Values() + if len(values) == 0 { + return "", fmt.Errorf("the version snippet gave no value (status %v)", resp.Status()) + } + v := values[len(values)-1] + if s, err := strconv.Unquote(v); err == nil { + v = s + } + return strings.Join(strings.Fields(v), " "), nil +} + // showDeath shows the output of a server that exited while proof was // using it, as that explains whatever went wrong afterwards. func showDeath(w io.Writer, srv *server.Server) { diff --git a/cmd/proof/main_test.go b/cmd/proof/main_test.go index e350a23..a88e437 100644 --- a/cmd/proof/main_test.go +++ b/cmd/proof/main_test.go @@ -3,6 +3,10 @@ package main import ( "runtime/debug" "testing" + "time" + + "github.com/nrepl/proof/internal/profile" + "github.com/nrepl/proof/internal/serve" ) func TestVersion(t *testing.T) { @@ -24,3 +28,34 @@ func TestVersion(t *testing.T) { } } } + +func TestServerVersion(t *testing.T) { + srv, err := serve.Listen("127.0.0.1:0", "0.1.0-dev", nil) + if err != nil { + t.Fatal(err) + } + go srv.Serve() + t.Cleanup(func() { srv.Stop(0) }) + cases := []struct { + code, want string + }{ + {`"1.2.3"`, "1.2.3"}, + {`"1.2.3\nSNAPSHOT"`, "1.2.3 SNAPSHOT"}, + {"(+ 1 2)", "3"}, + {"", ""}, + } + for _, c := range cases { + p := &profile.Profile{Timeout: time.Second} + if c.code != "" { + p.Snippets = map[string]profile.Snippet{"version": {Code: c.code}} + } + got, err := serverVersion(p, srv.Addr()) + if err != nil || got != c.want { + t.Errorf("version snippet %q gave %q (%v), want %q", c.code, got, err, c.want) + } + } + p := &profile.Profile{Timeout: time.Second, Snippets: map[string]profile.Snippet{"version": {Code: "(throw (ex-info \"no\" {}))"}}} + if _, err := serverVersion(p, srv.Addr()); err == nil { + t.Error("expected an error for a snippet that throws") + } +} diff --git a/doc/hacking.md b/doc/hacking.md index 86facd5..1e78536 100644 --- a/doc/hacking.md +++ b/doc/hacking.md @@ -308,6 +308,8 @@ To add a server to the compatibility matrix: 1. Create `profiles/.toml`. Start it with a comment explaining what needs to be installed. [Profiles](profiles.md) covers the format. + Unless the profile always starts the same version of the server, give + it a `version` snippet, so the matrix shows which version it checked. 2. Run it locally and make sure the results make sense. It's fine for a server to fail checks (that's what the matrix is for), but make sure that it's really the server that's failing and not the profile. Running diff --git a/doc/profiles.md b/doc/profiles.md index 09d03f5..265e68b 100644 --- a/doc/profiles.md +++ b/doc/profiles.md @@ -72,6 +72,9 @@ value = "println" [snippets.lookup] code = "println" +[snippets.version] +code = "fizz.core/version" + [expected-failures] "eval.multiple-forms" = "only the last value is returned, see #42" ``` @@ -84,7 +87,7 @@ code = "println" | Option | Required | Description | |---|---|---| -| `name` | yes | The name of the server in reports and in the compatibility matrix. It's a good idea to include the version you're testing. | +| `name` | yes | The name of the server in reports and in the compatibility matrix. If the profile always starts the same version of the server, it's a good idea to include it here (otherwise see the `version` snippet). | | `homepage` | no | The homepage of the server. | | `address` | no | The `host:port` of a server that's already running. When it's set proof connects to the server instead of starting it, and `[launch]` is ignored. The `-address` command-line option does the same. | | `timeout` | no | How long proof waits for each request to finish (i.e. for its `done`). It's a duration like `"10s"` (which is also the default). | @@ -148,6 +151,7 @@ snippet is missing, all the checks that need it are skipped. | `sleep` | `code`, `out` | print `out` and then keep running for longer than the `timeout` (e.g. a minute), so there's something to interrupt | all the `interrupt.*` checks, `robust.disconnect`, `robust.sessions-at-once` | | `completion` | `code`, `value` | `code` is the start of a name, which completes to `value` | all the `completion.*` checks | | `lookup` | `code` | `code` is the name of a function | `lookup.lookup`, `lookup.info`, `lookup.eldoc` | +| `version` | `code` | evaluate to the version of the server | none, proof shows the version in the report and in the compatibility matrix | A few tips for writing snippets: @@ -178,6 +182,8 @@ A few tips for writing snippets: - The checks of completions and lookups evaluate `value` first, and send the `ns` that comes with its value (if any) along with their requests, just like clients send the namespace of the code they complete. +- proof evaluates `version` before the checks, on a connection of its + own. If the version is a string, the quotes are dropped. - proof interrupts `sleep` as soon as its `out` arrives, so make sure your server sends the output right away (in Clojure that takes a `flush`, as there's no newline). diff --git a/doc/usage.md b/doc/usage.md index 8f31038..11f0675 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -225,7 +225,9 @@ $ proof matrix clojure.json my-server.json > matrix.md ``` The columns appear in the order of the reports on the command line, so -you'll probably want to put the reference implementation first. +you'll probably want to put the reference implementation first. The +version of each server comes from the `version` snippet of its profile, +if it has one (see [Profiles](profiles.md#snippets)). > [!TIP] > proof's own CI builds such a matrix for all the profiles in the diff --git a/internal/report/matrix.go b/internal/report/matrix.go index ebae8a3..bde0e13 100644 --- a/internal/report/matrix.go +++ b/internal/report/matrix.go @@ -10,9 +10,10 @@ import ( "github.com/nrepl/proof/internal/check" ) -// Report is a JSON report read back in. +// Report is a JSON report read back in. Its results are the ones JSON +// wrote, rather than the Results of Run. type Report struct { - Server string `json:"server"` + Run Results []jsonResult `json:"results"` } @@ -60,7 +61,7 @@ func Matrix(w io.Writer, reports []Report) { header := []string{"Check"} rule := []string{"---"} for _, r := range reports { - header = append(header, escape(r.Server)) + header = append(header, escape(r.Title())) rule = append(rule, "---") } fmt.Fprintf(w, "| %s |\n|%s|\n", strings.Join(header, " | "), strings.Join(rule, "|")) diff --git a/internal/report/report.go b/internal/report/report.go index 4aa098f..829eb01 100644 --- a/internal/report/report.go +++ b/internal/report/report.go @@ -14,11 +14,23 @@ import ( // Run is everything known about one run. type Run struct { - Proof string `json:"proof"` - Server string `json:"server"` - Address string `json:"address"` - Started time.Time `json:"started"` - Results []check.Result `json:"-"` + Proof string `json:"proof"` + Server string `json:"server"` + // ServerVersion is what the version snippet of the server's profile + // evaluated to, if it has one. + ServerVersion string `json:"server-version,omitempty"` + Address string `json:"address"` + Started time.Time `json:"started"` + Results []check.Result `json:"-"` +} + +// Title is the name of the server, followed by its version when it's +// known. +func (r Run) Title() string { + if r.ServerVersion == "" { + return r.Server + } + return r.Server + " " + r.ServerVersion } // Counts tallies verdicts. @@ -41,7 +53,7 @@ var labels = map[check.Verdict]string{ // Text writes a human-readable report. With verbose set it includes // notes for every check and transcripts for the ones that didn't pass. func Text(w io.Writer, run Run, verbose bool) { - fmt.Fprintf(w, "proof %s: %s at %s\n", run.Proof, run.Server, run.Address) + fmt.Fprintf(w, "proof %s: %s at %s\n", run.Proof, run.Title(), run.Address) group := "" for _, res := range run.Results { g, _, _ := strings.Cut(res.ID, ".") diff --git a/internal/report/report_test.go b/internal/report/report_test.go index 1267ad1..f23fe6f 100644 --- a/internal/report/report_test.go +++ b/internal/report/report_test.go @@ -21,14 +21,18 @@ func sampleRun(server string, verdicts ...check.Verdict) Run { return r } -// The matrix is built from JSON reports, so go through the real encoder. -func TestMatrixFromJSONReports(t *testing.T) { +// sampleReports are the reports of two servers, the second of which +// failed and skipped a check. The matrix is built from JSON reports, so +// they go through the real encoder. +func sampleReports(t *testing.T) []Report { + t.Helper() + beta := sampleRun("Beta | Two", check.Pass, check.Failed, check.Skipped) + beta.ServerVersion = "2.0" + beta.Started = time.Date(2026, 11, 3, 5, 0, 0, 0, time.UTC) + beta.Results[1].Details = []string{"gave no value"} dir := t.TempDir() var reports []Report - for i, run := range []Run{ - sampleRun("Alpha", check.Pass, check.Pass, check.Pass), - sampleRun("Beta | Two", check.Pass, check.Failed, check.Skipped), - } { + for i, run := range []Run{sampleRun("Alpha", check.Pass, check.Pass, check.Pass), beta} { path := filepath.Join(dir, string(rune('a'+i))+".json") f, err := os.Create(path) if err != nil { @@ -44,12 +48,15 @@ func TestMatrixFromJSONReports(t *testing.T) { } reports = append(reports, r) } + return reports +} +func TestMatrixFromJSONReports(t *testing.T) { var buf bytes.Buffer - Matrix(&buf, reports) + Matrix(&buf, sampleReports(t)) got := buf.String() for _, want := range []string{ - "| Check | Alpha | Beta \\| Two |", + "| Check | Alpha | Beta \\| Two 2.0 |", "| `eval.value` title eval.value | pass | **fail** |", "| `eval.ns` title eval.ns | pass | skip |", "| **Passed** | 3/3 | 1/2 |", @@ -60,6 +67,16 @@ func TestMatrixFromJSONReports(t *testing.T) { } } +func TestTextShowsTheServerVersion(t *testing.T) { + run := sampleRun("Alpha", check.Pass) + run.ServerVersion = "1.2.3" + var buf bytes.Buffer + Text(&buf, run, false) + if out := buf.String(); !strings.HasPrefix(out, "proof test: Alpha 1.2.3 at localhost:1") { + t.Errorf("no version in the report:\n%s", out) + } +} + func TestReadJSONRejectsOtherFiles(t *testing.T) { path := filepath.Join(t.TempDir(), "x.json") os.WriteFile(path, []byte(`{"name": "not a report"}`), 0o644) diff --git a/profiles/babashka.toml b/profiles/babashka.toml index 66406e9..adb9c8c 100644 --- a/profiles/babashka.toml +++ b/profiles/babashka.toml @@ -61,3 +61,7 @@ value = "println" # A function to look up. [snippets.lookup] code = "println" + +# The version of bb, for the report. +[snippets.version] +code = '(System/getProperty "babashka.version")' diff --git a/profiles/basilisp.toml b/profiles/basilisp.toml index f0c50cd..f05f1cd 100644 --- a/profiles/basilisp.toml +++ b/profiles/basilisp.toml @@ -55,3 +55,7 @@ value = "println" # A function to look up. [snippets.lookup] code = "println" + +# The version of Basilisp, for the report. +[snippets.version] +code = '(do (import importlib.metadata) (importlib.metadata/version "basilisp"))' diff --git a/profiles/dialtone.toml b/profiles/dialtone.toml index 6d07d24..a9b82f8 100644 --- a/profiles/dialtone.toml +++ b/profiles/dialtone.toml @@ -57,3 +57,7 @@ value = "lists:map" # A function to look up. [snippets.lookup] code = "lists:map" + +# The version of dialtone, for the report. +[snippets.version] +code = "element(2, application:get_key(dialtone, vsn))." diff --git a/profiles/repartee.toml b/profiles/repartee.toml index 5cfd5ff..f232d30 100644 --- a/profiles/repartee.toml +++ b/profiles/repartee.toml @@ -53,3 +53,8 @@ value = "Enum.map" # A function to look up. [snippets.lookup] code = "Enum.map" + +# The version of repartee, for the report. Its application isn't loaded +# until something asks for it. +[snippets.version] +code = "Application.load(:repartee); to_string(Application.spec(:repartee, :vsn))" From 0fb0f7d034ff48f5c5e27e3aaaa68d6ddbfeeffc Mon Sep 17 00:00:00 2001 From: Bozhidar Batsov Date: Tue, 3 Nov 2026 11:03:50 +0200 Subject: [PATCH 2/3] Publish the compatibility matrix on GitHub Pages So far the matrix lived in the summary of the Compatibility workflow, which few people would find. Now proof matrix -html builds it as a web page, where hovering over a check shows why it matters and hovering over a verdict shows what went wrong, and the workflow publishes it from main. The page gets published only when every server's job succeeded, as a page that's a day old beats one that's missing a server. The runs build proof with the latest Go and the whole history, so the reports (and the page) say which commit of proof made them. The table is written without html/template, which would make the binary 2 MB bigger. A check that only a later report has now goes with the rest of its group, in the Markdown matrix as well. --- .github/workflows/compat.yml | 36 ++++++- CHANGELOG.md | 1 + README.md | 3 +- cmd/proof/main.go | 17 +++- doc/hacking.md | 7 +- doc/usage.md | 14 ++- internal/report/matrix.go | 170 +++++++++++++++++++++++++++------ internal/report/matrix.html | 115 ++++++++++++++++++++++ internal/report/report_test.go | 58 ++++++++++- 9 files changed, 378 insertions(+), 43 deletions(-) create mode 100644 internal/report/matrix.html diff --git a/.github/workflows/compat.yml b/.github/workflows/compat.yml index e39f078..07d7bea 100644 --- a/.github/workflows/compat.yml +++ b/.github/workflows/compat.yml @@ -1,6 +1,7 @@ # Runs proof against every server with a profile and publishes the -# compatibility matrix in the job summary. Servers failing checks is the -# expected state of the world, so only harness problems fail this workflow. +# compatibility matrix in the job summary and on GitHub Pages. Servers +# failing checks is the expected state of the world, so only harness +# problems fail this workflow. name: Compatibility on: @@ -22,10 +23,14 @@ jobs: matrix: profile: [clojure, babashka, clojure-clr, basilisp, jank, dialtone, repartee] steps: + # The reports say which version of proof made them. Go 1.24 and newer + # work it out from the tags and the commit, like for go install. - uses: actions/checkout@v7 + with: + fetch-depth: 0 - uses: actions/setup-go@v7 with: - go-version-file: go.mod + go-version: stable - run: go build -o bin/proof ./cmd/proof - if: matrix.profile == 'clojure' @@ -127,7 +132,32 @@ jobs: done bin/proof matrix "${reports[@]}" > matrix.md cat matrix.md >> "$GITHUB_STEP_SUMMARY" + mkdir site + bin/proof matrix -html "${reports[@]}" > site/index.html - uses: actions/upload-artifact@v7 with: name: matrix path: matrix.md + - if: github.ref == 'refs/heads/main' && needs.run.result == 'success' + uses: actions/upload-pages-artifact@v5 + with: + path: site + + pages: + # Only when every server's job succeeded, as a page that's a day old + # beats one that's missing a server. + needs: [run, matrix] + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + pages: write + id-token: write + concurrency: + group: pages + environment: + name: github-pages + url: ${{ steps.deploy.outputs.page_url }} + steps: + - id: deploy + uses: actions/deploy-pages@v5 diff --git a/CHANGELOG.md b/CHANGELOG.md index d6c04ce..0493130 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Features - [#18](https://github.com/nrepl/proof/pull/18): The `version` snippet, so reports and the compatibility matrix show which version of a server proof checked. +- [#18](https://github.com/nrepl/proof/pull/18): `proof matrix -html`, which builds the compatibility matrix as a web page, and the daily matrix of the bundled profiles on [GitHub Pages](https://nrepl.github.io/proof/). ## 0.1.0 (2026-10-26) diff --git a/README.md b/README.md index 431f97d..db43df7 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,8 @@ The [profiles](profiles) folder has complete profiles for nREPL itself, Babashka, Basilisp, ClojureCLR, jank, dialtone (Erlang) and repartee (Elixir), which you can use as a starting point. They come with proof as well, so `proof run babashka` will check Babashka (provided you have it -installed). +installed). The [compatibility matrix](https://nrepl.github.io/proof/) +shows how all of them do, and it's updated every day. Now you can run the checks: diff --git a/cmd/proof/main.go b/cmd/proof/main.go index 2952c0d..22b3e07 100644 --- a/cmd/proof/main.go +++ b/cmd/proof/main.go @@ -54,7 +54,7 @@ Usage: proof run [flags] PROFILE run the checks against the server a profile describes proof proxy [flags] [PROFILE] check what a client sends to a server, by sitting between them proof serve [flags] [SCENARIO...] be a server for a client's tests, acting like other servers where asked - proof matrix REPORT... build a Markdown compatibility matrix from JSON reports + proof matrix [-html] REPORT... build a compatibility matrix (Markdown or a web page) from JSON reports proof list list every check, rule and scenario proof version @@ -350,7 +350,13 @@ func allIDs() []string { return ids } -func matrix(paths []string) int { +func matrix(args []string) int { + fs := newFlags("matrix", "REPORT...", os.Stderr) + html := fs.Bool("html", false, "write a web page instead of a Markdown table") + paths, err := parseArgs(fs, args) + if err != nil { + return flagStatus(err) + } if len(paths) == 0 { fmt.Fprintln(os.Stderr, "proof matrix: expected one or more JSON reports (from proof run -json)") return 2 @@ -364,6 +370,13 @@ func matrix(paths []string) int { } reports = append(reports, r) } + if *html { + if err := report.HTML(os.Stdout, reports); err != nil { + fmt.Fprintln(os.Stderr, "proof:", err) + return 2 + } + return 0 + } report.Matrix(os.Stdout, reports) return 0 } diff --git a/doc/hacking.md b/doc/hacking.md index 1e78536..4b7fe1e 100644 --- a/doc/hacking.md +++ b/doc/hacking.md @@ -364,9 +364,10 @@ proof uses a few GitHub Actions workflows: the way a server's CI would use it. - Compatibility runs all the profiles against their servers every day (and on every push to `main`) and builds the compatibility matrix in - the job summary. As servers failing checks is perfectly normal, this - workflow fails only when proof itself can't do its job (e.g. a server - can't be installed or doesn't start). + the job summary. When every server ran, it also publishes the matrix + on [GitHub Pages](https://nrepl.github.io/proof/). As servers failing + checks is perfectly normal, this workflow fails only when proof itself + can't do its job (e.g. a server can't be installed or doesn't start). - Release builds the binaries for every platform when a version gets tagged and publishes them in a GitHub release. diff --git a/doc/usage.md b/doc/usage.md index 11f0675..eae880b 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -229,11 +229,17 @@ you'll probably want to put the reference implementation first. The version of each server comes from the `version` snippet of its profile, if it has one (see [Profiles](profiles.md#snippets)). +With `-html` you'll get a web page instead, where hovering over a check +shows why it matters and hovering over a verdict shows its details: + +```shell +$ proof matrix -html clojure.json my-server.json > matrix.html +``` + > [!TIP] -> proof's own CI builds such a matrix for all the profiles in the -> repository on a daily basis. You can find it in the summary of the -> Compatibility workflow on the -> [Actions tab](https://github.com/nrepl/proof/actions). +> proof's own CI builds such a page for all the profiles in the +> repository on a daily basis. You can find it at +> [nrepl.github.io/proof](https://nrepl.github.io/proof/). ## Checking Your Client diff --git a/internal/report/matrix.go b/internal/report/matrix.go index bde0e13..5a443da 100644 --- a/internal/report/matrix.go +++ b/internal/report/matrix.go @@ -1,11 +1,15 @@ package report import ( + _ "embed" "encoding/json" "fmt" + "html" "io" "os" + "slices" "strings" + "time" "github.com/nrepl/proof/internal/check" ) @@ -33,6 +37,59 @@ func ReadJSON(path string) (Report, error) { return r, nil } +// table is a row per check, in the order of the first report that has +// it, and a column per report. +type table struct { + ids []string + // checks has the first result of each check, for its title and why. + checks map[string]jsonResult + results []map[string]jsonResult +} + +func newTable(reports []Report) table { + t := table{checks: map[string]jsonResult{}, results: make([]map[string]jsonResult, len(reports))} + for i, r := range reports { + t.results[i] = map[string]jsonResult{} + for _, res := range r.Results { + if _, ok := t.checks[res.ID]; !ok { + t.ids = slices.Insert(t.ids, t.place(res.ID), res.ID) + t.checks[res.ID] = res + } + t.results[i][res.ID] = res + } + } + return t +} + +// place is where a check that isn't in the table yet goes: after the last +// one in its group (e.g. eval), so a check only a later report has (e.g. +// one made by a newer proof) still ends up with the rest of its group. +func (t table) place(id string) int { + group, _, _ := strings.Cut(id, ".") + for i := len(t.ids) - 1; i >= 0; i-- { + if g, _, _ := strings.Cut(t.ids[i], "."); g == group { + return i + 1 + } + } + return len(t.ids) +} + +// passed returns how many checks the server of the i-th report passed, +// out of the ones that ran. +func (t table) passed(i int) string { + passed, ran := 0, 0 + for _, res := range t.results[i] { + if res.Verdict == check.Skipped { + continue + } + ran++ + if res.Verdict == check.Pass { + passed++ + } + } + return fmt.Sprintf("%d/%d", passed, ran) +} + var cells = map[check.Verdict]string{ check.Pass: "pass", check.Failed: "**fail**", @@ -42,22 +99,10 @@ var cells = map[check.Verdict]string{ } // Matrix writes a Markdown table with a row per check and a column per -// server. Checks appear in the order of the first report that has them. +// server. Checks appear in the order of the first report that has them, +// with the rest of their group. func Matrix(w io.Writer, reports []Report) { - var ids []string - titles := map[string]string{} - verdicts := make([]map[string]check.Verdict, len(reports)) - for i, r := range reports { - verdicts[i] = map[string]check.Verdict{} - for _, res := range r.Results { - if _, ok := titles[res.ID]; !ok { - ids = append(ids, res.ID) - titles[res.ID] = res.Title - } - verdicts[i][res.ID] = res.Verdict - } - } - + t := newTable(reports) header := []string{"Check"} rule := []string{"---"} for _, r := range reports { @@ -65,27 +110,17 @@ func Matrix(w io.Writer, reports []Report) { rule = append(rule, "---") } fmt.Fprintf(w, "| %s |\n|%s|\n", strings.Join(header, " | "), strings.Join(rule, "|")) - for _, id := range ids { - row := []string{fmt.Sprintf("`%s` %s", id, escape(titles[id]))} + for _, id := range t.ids { + row := []string{fmt.Sprintf("`%s` %s", id, escape(t.checks[id].Title))} for i := range reports { - row = append(row, cells[verdicts[i][id]]) + row = append(row, cells[t.results[i][id].Verdict]) } fmt.Fprintf(w, "| %s |\n", strings.Join(row, " | ")) } totals := []string{"**Passed**"} for i := range reports { - passed, ran := 0, 0 - for _, v := range verdicts[i] { - if v == check.Skipped { - continue - } - ran++ - if v == check.Pass { - passed++ - } - } - totals = append(totals, fmt.Sprintf("%d/%d", passed, ran)) + totals = append(totals, t.passed(i)) } fmt.Fprintf(w, "| %s |\n", strings.Join(totals, " | ")) } @@ -93,3 +128,80 @@ func Matrix(w io.Writer, reports []Report) { func escape(s string) string { return strings.ReplaceAll(s, "|", `\|`) } + +// The page comes from matrix.html, with the table and the versions of +// proof where its markers are. +// +//go:embed matrix.html +var matrixPage string + +// HTML writes the same matrix as Matrix as a web page, with the checks +// grouped like in the text report. The why of a check and the details of +// a verdict show up when hovering over them. +func HTML(w io.Writer, reports []Report) error { + t := newTable(reports) + esc := html.EscapeString + var b strings.Builder + b.WriteString("\nCheck") + for _, r := range reports { + fmt.Fprintf(&b, "%s", esc(r.Server)) + if r.ServerVersion != "" { + fmt.Fprintf(&b, `%s`, esc(r.ServerVersion)) + } + b.WriteString("") + } + b.WriteString("\n\n\n") + group := "" + for _, id := range t.ids { + if g, _, _ := strings.Cut(id, "."); g != group { + group = g + fmt.Fprintf(&b, "%s\n", len(reports)+1, esc(g)) + } + res := t.checks[id] + fmt.Fprintf(&b, `%s%s`, + titleAttr(res.Why), esc(id), esc(res.Title)) + for i := range reports { + res := t.results[i][id] + notes := res.Details + if res.Expected != "" { + notes = append([]string{"expected: " + res.Expected}, notes...) + } + fmt.Fprintf(&b, `%s`, res.Verdict, titleAttr(strings.Join(notes, "\n")), res.Verdict) + } + b.WriteString("\n") + } + b.WriteString("\n\nPassed") + for i := range reports { + fmt.Fprintf(&b, "%s", t.passed(i)) + } + b.WriteString("\n\n") + + var proofs []string + var latest time.Time + for _, r := range reports { + if r.Proof != "" && !slices.Contains(proofs, r.Proof) { + proofs = append(proofs, r.Proof) + } + if r.Started.After(latest) { + latest = r.Started + } + } + made := "proof" + if len(proofs) > 0 { + made += " " + strings.Join(proofs, ", ") + } + if !latest.IsZero() { + made += " on " + latest.UTC().Format(time.DateOnly) + } + _, err := strings.NewReplacer("\n", b.String(), "", esc(made)).WriteString(w, matrixPage) + return err +} + +// titleAttr is a title attribute with the given text, or nothing when +// there's no text. +func titleAttr(text string) string { + if text == "" { + return "" + } + return ` title="` + html.EscapeString(text) + `"` +} diff --git a/internal/report/matrix.html b/internal/report/matrix.html new file mode 100644 index 0000000..0d218cb --- /dev/null +++ b/internal/report/matrix.html @@ -0,0 +1,115 @@ + + + + + +nREPL Compatibility Matrix + + + +
+

nREPL Compatibility Matrix

+

Here's how the servers below fare against the checks of +proof, which are mostly about +what the clients people use (CIDER, Calva, Conjure, vim-fireplace, REPLy and +neorepl) need from a server.

+
+
pass
The server does what the clients need.
+
fail
Some client won't work properly with the server, or users lose data.
+
warn
The server does something differently from nREPL or the spec, but no client we know of minds.
+
skip
The check doesn't apply to the server (e.g. it needs Java interop).
+
error
proof couldn't run the check.
+
+

Hover over a check to see why it matters, and over a verdict for the details.

+
+
+ + +
+
+
+

Made with .

+
+ + diff --git a/internal/report/report_test.go b/internal/report/report_test.go index f23fe6f..fc57c22 100644 --- a/internal/report/report_test.go +++ b/internal/report/report_test.go @@ -16,7 +16,7 @@ func sampleRun(server string, verdicts ...check.Verdict) Run { ids := []string{"describe.reply", "eval.value", "eval.ns"} r := Run{Proof: "test", Server: server, Address: "localhost:1", Started: time.Unix(0, 0)} for i, v := range verdicts { - r.Results = append(r.Results, check.Result{ID: ids[i], Title: "title " + ids[i], Verdict: v}) + r.Results = append(r.Results, check.Result{ID: ids[i], Title: "title " + ids[i], Why: "why " + ids[i], Verdict: v}) } return r } @@ -67,6 +67,62 @@ func TestMatrixFromJSONReports(t *testing.T) { } } +func TestMatrixAsAPage(t *testing.T) { + var buf bytes.Buffer + if err := HTML(&buf, sampleReports(t)); err != nil { + t.Fatal(err) + } + got := buf.String() + for _, want := range []string{ + `Beta | Two2.0`, + `eval`, + `eval.valuetitle eval.value`, + `passfail`, + `skip`, + `3/31/2`, + "Made with proof test on 2026-11-03.", + } { + if !strings.Contains(got, want) { + t.Errorf("page is missing %q:\n%s", want, got) + } + } +} + +// A check the first report doesn't have still goes in its group. +func TestChecksFromLaterReportsJoinTheirGroup(t *testing.T) { + reports := []Report{ + {Run: Run{Server: "Alpha"}, Results: []jsonResult{{ID: "eval.value"}, {ID: "describe.reply"}}}, + {Run: Run{Server: "Beta"}, Results: []jsonResult{{ID: "eval.value"}, {ID: "describe.reply"}, {ID: "eval.ns"}}}, + } + var buf bytes.Buffer + if err := HTML(&buf, reports); err != nil { + t.Fatal(err) + } + got := buf.String() + if n := strings.Count(got, `eval`); n != 1 { + t.Errorf("eval has %d groups:\n%s", n, got) + } + if i, j := strings.Index(got, "eval.ns"), strings.Index(got, "describe.reply"); i > j { + t.Errorf("eval.ns isn't with the other eval checks:\n%s", got) + } + if !strings.Contains(got, "Made with proof.") { + t.Errorf("reports without a proof version still get one:\n%s", got) + } + reports[0].Proof, reports[1].Proof = "0.1.0", "0.2.0" + buf.Reset() + if err := HTML(&buf, reports); err != nil { + t.Fatal(err) + } + if got := buf.String(); !strings.Contains(got, "Made with proof 0.1.0, 0.2.0.") { + t.Errorf("the versions of proof aren't listed:\n%s", got) + } + buf.Reset() + Matrix(&buf, reports) + if got := buf.String(); strings.Index(got, "`eval.ns`") > strings.Index(got, "`describe.reply`") { + t.Errorf("eval.ns isn't with the other eval checks:\n%s", got) + } +} + func TestTextShowsTheServerVersion(t *testing.T) { run := sampleRun("Alpha", check.Pass) run.ServerVersion = "1.2.3" From 32688f7c629e3ed05379b7e1d0ba985a18337372 Mon Sep 17 00:00:00 2001 From: Bozhidar Batsov Date: Tue, 3 Nov 2026 11:03:50 +0200 Subject: [PATCH 3/3] Plan 0.2.0 The matrix gets a page of its own, along with nbb, let-go and two more clients. The upstream section catches up with what got merged and reported since 0.1.0. --- PLAN.md | 61 +++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/PLAN.md b/PLAN.md index c80c95c..841a67e 100644 --- a/PLAN.md +++ b/PLAN.md @@ -142,16 +142,18 @@ and it covers the rest of what every editor sends: `interrupt`, - [Spec Changes](doc/spec-changes.md) has everything the new checks found. - The docs cover the new checks and the release. -## After 0.1.0 +## 0.2.0 -This is what's next for proof. +The compatibility matrix gets a page of its own, and more servers and +clients. ### The Matrix -- Publish the compatibility matrix on GitHub Pages from the - Compatibility workflow, with the version of each server. -- Add nbb and let-go, which are easy to install in CI, and then - shadow-cljs. +- Done: the Compatibility workflow publishes the matrix on + [GitHub Pages](https://nrepl.github.io/proof/), with the version of + each server, which comes from the new `version` snippet. `proof matrix + -html` builds the same page out of any reports. +- nbb and let-go, which are easy to install in CI. ### More Clients @@ -159,6 +161,17 @@ This is what's next for proof. new client of the Clojure team, which doesn't handle `need-input` as of 0.1.1. - `bb repl --connect`. + +## After 0.2.0 + +This is what's next for proof. + +### The Matrix + +- shadow-cljs. + +### More Clients + - The client in `nrepl.cmdline`, which expects an exact list of statuses when interrupting an idle session. - vim-iced and rebel-readline. @@ -173,23 +186,41 @@ This is what's next for proof. ### Upstream -- The open pull requests based on proof's findings: - [spec.nrepl.org#6-#9](https://github.com/nrepl/spec.nrepl.org/pulls), - [nrepl-beam#1](https://github.com/nrepl/nrepl-beam/pull/1) and - [#2](https://github.com/nrepl/nrepl-beam/pull/2), - [neat#2](https://github.com/nrepl/neat/pull/2) and - [mezcaml#2](https://github.com/nrepl/mezcaml/pull/2). +- The pull requests for the spec based on proof's findings, + [spec.nrepl.org#6-#9](https://github.com/nrepl/spec.nrepl.org/pulls). + The fixes for nrepl-beam, neat and mezcaml got merged on 2026-10-02, + except for [nrepl-beam#2](https://github.com/nrepl/nrepl-beam/pull/2), + which was closed when the branch it was based on went away and has to + be opened again. - Issues for the spec about the rest of [Spec Changes](doc/spec-changes.md), once the spec pull requests are settled. - nREPL: a release with [nrepl#498](https://github.com/nrepl/nrepl/pull/498), then [nrepl#477](https://github.com/nrepl/nrepl/issues/477) and [nrepl#183](https://github.com/nrepl/nrepl/issues/183), which has the - same cause as #498. + same cause as #498. The next eval after a client goes away in the + middle of one loses its `done` for the same reason as + [nrepl#509](https://github.com/nrepl/nrepl/issues/509). Babashka has + the same problem, which is why it fails `robust.disconnect` in a full + run, but not on its own. - CIDER and Calva accepting `ops` as a list, after which `describe.ops-dict` can allow both formats. -- Issues for what the matrix finds in Basilisp and ClojureCLR, a few at - a time. jank's can go in a reply to jank#1104. +- Reported on 2026-10-29: + [basilisp#1356](https://github.com/basilisp-lang/basilisp/issues/1356) + (no `no-info` for unknown symbols), + [basilisp#1357](https://github.com/basilisp-lang/basilisp/issues/1357) + (columns from 0), + [jank#1169](https://github.com/jank-lang/jank/issues/1169) (a request + that throws stops its connection from answering), + [conjure#802](https://github.com/Olical/conjure/issues/802) (no + `interrupt-id` in `interrupt`) and + [cider#4224](https://github.com/clojure-emacs/cider/issues/4224) (async + eldoc ignores `no-eldoc`). +- ClojureCLR doesn't say `no-info` for unknown symbols either, but + clr.tools.nrepl takes bug reports on Ask Clojure rather than GitHub. +- Calva waits for an eval of `java.version` in the main session before + it interrupts, so it looks like it can't interrupt an eval there. That + needs a run in VS Code before it gets reported. ## Further Down the Road