diff --git a/.github/workflows/compat.yml b/.github/workflows/compat.yml index e39f078..07d7bea 100644 --- a/.github/workflows/compat.yml +++ b/.github/workflows/compat.yml @@ -1,6 +1,7 @@ # Runs proof against every server with a profile and publishes the -# compatibility matrix in the job summary. Servers failing checks is the -# expected state of the world, so only harness problems fail this workflow. +# compatibility matrix in the job summary and on GitHub Pages. Servers +# failing checks is the expected state of the world, so only harness +# problems fail this workflow. name: Compatibility on: @@ -22,10 +23,14 @@ jobs: matrix: profile: [clojure, babashka, clojure-clr, basilisp, jank, dialtone, repartee] steps: + # The reports say which version of proof made them. Go 1.24 and newer + # work it out from the tags and the commit, like for go install. - uses: actions/checkout@v7 + with: + fetch-depth: 0 - uses: actions/setup-go@v7 with: - go-version-file: go.mod + go-version: stable - run: go build -o bin/proof ./cmd/proof - if: matrix.profile == 'clojure' @@ -127,7 +132,32 @@ jobs: done bin/proof matrix "${reports[@]}" > matrix.md cat matrix.md >> "$GITHUB_STEP_SUMMARY" + mkdir site + bin/proof matrix -html "${reports[@]}" > site/index.html - uses: actions/upload-artifact@v7 with: name: matrix path: matrix.md + - if: github.ref == 'refs/heads/main' && needs.run.result == 'success' + uses: actions/upload-pages-artifact@v5 + with: + path: site + + pages: + # Only when every server's job succeeded, as a page that's a day old + # beats one that's missing a server. + needs: [run, matrix] + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + pages: write + id-token: write + concurrency: + group: pages + environment: + name: github-pages + url: ${{ steps.deploy.outputs.page_url }} + steps: + - id: deploy + uses: actions/deploy-pages@v5 diff --git a/CHANGELOG.md b/CHANGELOG.md index 0653a51..0493130 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## main (unreleased) +### Features + +- [#18](https://github.com/nrepl/proof/pull/18): The `version` snippet, so reports and the compatibility matrix show which version of a server proof checked. +- [#18](https://github.com/nrepl/proof/pull/18): `proof matrix -html`, which builds the compatibility matrix as a web page, and the daily matrix of the bundled profiles on [GitHub Pages](https://nrepl.github.io/proof/). + ## 0.1.0 (2026-10-26) Initial release. diff --git a/PLAN.md b/PLAN.md index c80c95c..841a67e 100644 --- a/PLAN.md +++ b/PLAN.md @@ -142,16 +142,18 @@ and it covers the rest of what every editor sends: `interrupt`, - [Spec Changes](doc/spec-changes.md) has everything the new checks found. - The docs cover the new checks and the release. -## After 0.1.0 +## 0.2.0 -This is what's next for proof. +The compatibility matrix gets a page of its own, and more servers and +clients. ### The Matrix -- Publish the compatibility matrix on GitHub Pages from the - Compatibility workflow, with the version of each server. -- Add nbb and let-go, which are easy to install in CI, and then - shadow-cljs. +- Done: the Compatibility workflow publishes the matrix on + [GitHub Pages](https://nrepl.github.io/proof/), with the version of + each server, which comes from the new `version` snippet. `proof matrix + -html` builds the same page out of any reports. +- nbb and let-go, which are easy to install in CI. ### More Clients @@ -159,6 +161,17 @@ This is what's next for proof. new client of the Clojure team, which doesn't handle `need-input` as of 0.1.1. - `bb repl --connect`. + +## After 0.2.0 + +This is what's next for proof. + +### The Matrix + +- shadow-cljs. + +### More Clients + - The client in `nrepl.cmdline`, which expects an exact list of statuses when interrupting an idle session. - vim-iced and rebel-readline. @@ -173,23 +186,41 @@ This is what's next for proof. ### Upstream -- The open pull requests based on proof's findings: - [spec.nrepl.org#6-#9](https://github.com/nrepl/spec.nrepl.org/pulls), - [nrepl-beam#1](https://github.com/nrepl/nrepl-beam/pull/1) and - [#2](https://github.com/nrepl/nrepl-beam/pull/2), - [neat#2](https://github.com/nrepl/neat/pull/2) and - [mezcaml#2](https://github.com/nrepl/mezcaml/pull/2). +- The pull requests for the spec based on proof's findings, + [spec.nrepl.org#6-#9](https://github.com/nrepl/spec.nrepl.org/pulls). + The fixes for nrepl-beam, neat and mezcaml got merged on 2026-10-02, + except for [nrepl-beam#2](https://github.com/nrepl/nrepl-beam/pull/2), + which was closed when the branch it was based on went away and has to + be opened again. - Issues for the spec about the rest of [Spec Changes](doc/spec-changes.md), once the spec pull requests are settled. - nREPL: a release with [nrepl#498](https://github.com/nrepl/nrepl/pull/498), then [nrepl#477](https://github.com/nrepl/nrepl/issues/477) and [nrepl#183](https://github.com/nrepl/nrepl/issues/183), which has the - same cause as #498. + same cause as #498. The next eval after a client goes away in the + middle of one loses its `done` for the same reason as + [nrepl#509](https://github.com/nrepl/nrepl/issues/509). Babashka has + the same problem, which is why it fails `robust.disconnect` in a full + run, but not on its own. - CIDER and Calva accepting `ops` as a list, after which `describe.ops-dict` can allow both formats. -- Issues for what the matrix finds in Basilisp and ClojureCLR, a few at - a time. jank's can go in a reply to jank#1104. +- Reported on 2026-10-29: + [basilisp#1356](https://github.com/basilisp-lang/basilisp/issues/1356) + (no `no-info` for unknown symbols), + [basilisp#1357](https://github.com/basilisp-lang/basilisp/issues/1357) + (columns from 0), + [jank#1169](https://github.com/jank-lang/jank/issues/1169) (a request + that throws stops its connection from answering), + [conjure#802](https://github.com/Olical/conjure/issues/802) (no + `interrupt-id` in `interrupt`) and + [cider#4224](https://github.com/clojure-emacs/cider/issues/4224) (async + eldoc ignores `no-eldoc`). +- ClojureCLR doesn't say `no-info` for unknown symbols either, but + clr.tools.nrepl takes bug reports on Ask Clojure rather than GitHub. +- Calva waits for an eval of `java.version` in the main session before + it interrupts, so it looks like it can't interrupt an eval there. That + needs a run in VS Code before it gets reported. ## Further Down the Road diff --git a/README.md b/README.md index 431f97d..db43df7 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,8 @@ The [profiles](profiles) folder has complete profiles for nREPL itself, Babashka, Basilisp, ClojureCLR, jank, dialtone (Erlang) and repartee (Elixir), which you can use as a starting point. They come with proof as well, so `proof run babashka` will check Babashka (provided you have it -installed). +installed). The [compatibility matrix](https://nrepl.github.io/proof/) +shows how all of them do, and it's updated every day. Now you can run the checks: diff --git a/cmd/proof/main.go b/cmd/proof/main.go index 8683184..22b3e07 100644 --- a/cmd/proof/main.go +++ b/cmd/proof/main.go @@ -12,6 +12,7 @@ import ( "os/signal" "path" "runtime/debug" + "strconv" "strings" "syscall" "time" @@ -22,6 +23,7 @@ import ( "github.com/nrepl/proof/internal/report" "github.com/nrepl/proof/internal/serve" "github.com/nrepl/proof/internal/server" + "github.com/nrepl/proof/nrepl" ) // version is set for releases, with -ldflags "-X main.version=...". @@ -52,7 +54,7 @@ Usage: proof run [flags] PROFILE run the checks against the server a profile describes proof proxy [flags] [PROFILE] check what a client sends to a server, by sitting between them proof serve [flags] [SCENARIO...] be a server for a client's tests, acting like other servers where asked - proof matrix REPORT... build a Markdown compatibility matrix from JSON reports + proof matrix [-html] REPORT... build a compatibility matrix (Markdown or a web page) from JSON reports proof list list every check, rule and scenario proof version @@ -208,12 +210,17 @@ func run(args []string) int { addr = srv.Addr } + ver, err := serverVersion(p, addr) + if err != nil { + fmt.Fprintln(os.Stderr, "proof: couldn't get the version of the server:", err) + } env := &check.Env{Profile: p, Addr: addr} started := time.Now() results := check.Run(env, selected, checks.WireRules()) outcome := check.Apply(results, p.ExpectedFailures, opts.only == "") - r := report.Run{Proof: proofVersion(), Server: p.Name, Address: addr, Started: started, Results: results} + r := report.Run{Proof: proofVersion(), Server: p.Name, ServerVersion: ver, Address: addr, Started: started, + Results: results} report.Text(os.Stdout, r, opts.verbose) for _, id := range outcome.Stale { fmt.Printf("%s is listed in expected-failures but didn't fail; remove it from the profile\n", id) @@ -260,6 +267,38 @@ func startServer(p *profile.Profile) (*server.Server, error) { return srv, err } +// serverVersion evaluates the version snippet of the profile, if it has +// one, in a session of its own. A string comes back without its quotes, +// and on a single line. +func serverVersion(p *profile.Profile, addr string) (string, error) { + sn, ok := p.Snippets["version"] + if !ok { + return "", nil + } + c, err := nrepl.Dial(addr, 5*time.Second) + if err != nil { + return "", err + } + defer c.Close() + clone, err := c.Request(nrepl.Message{"op": "clone"}, p.Timeout) + if err != nil { + return "", err + } + resp, err := c.Request(nrepl.Message{"op": "eval", "code": sn.Code, "session": clone.Str("new-session")}, p.Timeout) + if err != nil { + return "", err + } + values := resp.Values() + if len(values) == 0 { + return "", fmt.Errorf("the version snippet gave no value (status %v)", resp.Status()) + } + v := values[len(values)-1] + if s, err := strconv.Unquote(v); err == nil { + v = s + } + return strings.Join(strings.Fields(v), " "), nil +} + // showDeath shows the output of a server that exited while proof was // using it, as that explains whatever went wrong afterwards. func showDeath(w io.Writer, srv *server.Server) { @@ -311,7 +350,13 @@ func allIDs() []string { return ids } -func matrix(paths []string) int { +func matrix(args []string) int { + fs := newFlags("matrix", "REPORT...", os.Stderr) + html := fs.Bool("html", false, "write a web page instead of a Markdown table") + paths, err := parseArgs(fs, args) + if err != nil { + return flagStatus(err) + } if len(paths) == 0 { fmt.Fprintln(os.Stderr, "proof matrix: expected one or more JSON reports (from proof run -json)") return 2 @@ -325,6 +370,13 @@ func matrix(paths []string) int { } reports = append(reports, r) } + if *html { + if err := report.HTML(os.Stdout, reports); err != nil { + fmt.Fprintln(os.Stderr, "proof:", err) + return 2 + } + return 0 + } report.Matrix(os.Stdout, reports) return 0 } diff --git a/cmd/proof/main_test.go b/cmd/proof/main_test.go index e350a23..a88e437 100644 --- a/cmd/proof/main_test.go +++ b/cmd/proof/main_test.go @@ -3,6 +3,10 @@ package main import ( "runtime/debug" "testing" + "time" + + "github.com/nrepl/proof/internal/profile" + "github.com/nrepl/proof/internal/serve" ) func TestVersion(t *testing.T) { @@ -24,3 +28,34 @@ func TestVersion(t *testing.T) { } } } + +func TestServerVersion(t *testing.T) { + srv, err := serve.Listen("127.0.0.1:0", "0.1.0-dev", nil) + if err != nil { + t.Fatal(err) + } + go srv.Serve() + t.Cleanup(func() { srv.Stop(0) }) + cases := []struct { + code, want string + }{ + {`"1.2.3"`, "1.2.3"}, + {`"1.2.3\nSNAPSHOT"`, "1.2.3 SNAPSHOT"}, + {"(+ 1 2)", "3"}, + {"", ""}, + } + for _, c := range cases { + p := &profile.Profile{Timeout: time.Second} + if c.code != "" { + p.Snippets = map[string]profile.Snippet{"version": {Code: c.code}} + } + got, err := serverVersion(p, srv.Addr()) + if err != nil || got != c.want { + t.Errorf("version snippet %q gave %q (%v), want %q", c.code, got, err, c.want) + } + } + p := &profile.Profile{Timeout: time.Second, Snippets: map[string]profile.Snippet{"version": {Code: "(throw (ex-info \"no\" {}))"}}} + if _, err := serverVersion(p, srv.Addr()); err == nil { + t.Error("expected an error for a snippet that throws") + } +} diff --git a/doc/hacking.md b/doc/hacking.md index 86facd5..4b7fe1e 100644 --- a/doc/hacking.md +++ b/doc/hacking.md @@ -308,6 +308,8 @@ To add a server to the compatibility matrix: 1. Create `profiles/.toml`. Start it with a comment explaining what needs to be installed. [Profiles](profiles.md) covers the format. + Unless the profile always starts the same version of the server, give + it a `version` snippet, so the matrix shows which version it checked. 2. Run it locally and make sure the results make sense. It's fine for a server to fail checks (that's what the matrix is for), but make sure that it's really the server that's failing and not the profile. Running @@ -362,9 +364,10 @@ proof uses a few GitHub Actions workflows: the way a server's CI would use it. - Compatibility runs all the profiles against their servers every day (and on every push to `main`) and builds the compatibility matrix in - the job summary. As servers failing checks is perfectly normal, this - workflow fails only when proof itself can't do its job (e.g. a server - can't be installed or doesn't start). + the job summary. When every server ran, it also publishes the matrix + on [GitHub Pages](https://nrepl.github.io/proof/). As servers failing + checks is perfectly normal, this workflow fails only when proof itself + can't do its job (e.g. a server can't be installed or doesn't start). - Release builds the binaries for every platform when a version gets tagged and publishes them in a GitHub release. diff --git a/doc/profiles.md b/doc/profiles.md index 09d03f5..265e68b 100644 --- a/doc/profiles.md +++ b/doc/profiles.md @@ -72,6 +72,9 @@ value = "println" [snippets.lookup] code = "println" +[snippets.version] +code = "fizz.core/version" + [expected-failures] "eval.multiple-forms" = "only the last value is returned, see #42" ``` @@ -84,7 +87,7 @@ code = "println" | Option | Required | Description | |---|---|---| -| `name` | yes | The name of the server in reports and in the compatibility matrix. It's a good idea to include the version you're testing. | +| `name` | yes | The name of the server in reports and in the compatibility matrix. If the profile always starts the same version of the server, it's a good idea to include it here (otherwise see the `version` snippet). | | `homepage` | no | The homepage of the server. | | `address` | no | The `host:port` of a server that's already running. When it's set proof connects to the server instead of starting it, and `[launch]` is ignored. The `-address` command-line option does the same. | | `timeout` | no | How long proof waits for each request to finish (i.e. for its `done`). It's a duration like `"10s"` (which is also the default). | @@ -148,6 +151,7 @@ snippet is missing, all the checks that need it are skipped. | `sleep` | `code`, `out` | print `out` and then keep running for longer than the `timeout` (e.g. a minute), so there's something to interrupt | all the `interrupt.*` checks, `robust.disconnect`, `robust.sessions-at-once` | | `completion` | `code`, `value` | `code` is the start of a name, which completes to `value` | all the `completion.*` checks | | `lookup` | `code` | `code` is the name of a function | `lookup.lookup`, `lookup.info`, `lookup.eldoc` | +| `version` | `code` | evaluate to the version of the server | none, proof shows the version in the report and in the compatibility matrix | A few tips for writing snippets: @@ -178,6 +182,8 @@ A few tips for writing snippets: - The checks of completions and lookups evaluate `value` first, and send the `ns` that comes with its value (if any) along with their requests, just like clients send the namespace of the code they complete. +- proof evaluates `version` before the checks, on a connection of its + own. If the version is a string, the quotes are dropped. - proof interrupts `sleep` as soon as its `out` arrives, so make sure your server sends the output right away (in Clojure that takes a `flush`, as there's no newline). diff --git a/doc/usage.md b/doc/usage.md index 8f31038..eae880b 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -225,13 +225,21 @@ $ proof matrix clojure.json my-server.json > matrix.md ``` The columns appear in the order of the reports on the command line, so -you'll probably want to put the reference implementation first. +you'll probably want to put the reference implementation first. The +version of each server comes from the `version` snippet of its profile, +if it has one (see [Profiles](profiles.md#snippets)). + +With `-html` you'll get a web page instead, where hovering over a check +shows why it matters and hovering over a verdict shows its details: + +```shell +$ proof matrix -html clojure.json my-server.json > matrix.html +``` > [!TIP] -> proof's own CI builds such a matrix for all the profiles in the -> repository on a daily basis. You can find it in the summary of the -> Compatibility workflow on the -> [Actions tab](https://github.com/nrepl/proof/actions). +> proof's own CI builds such a page for all the profiles in the +> repository on a daily basis. You can find it at +> [nrepl.github.io/proof](https://nrepl.github.io/proof/). ## Checking Your Client diff --git a/internal/report/matrix.go b/internal/report/matrix.go index ebae8a3..5a443da 100644 --- a/internal/report/matrix.go +++ b/internal/report/matrix.go @@ -1,18 +1,23 @@ package report import ( + _ "embed" "encoding/json" "fmt" + "html" "io" "os" + "slices" "strings" + "time" "github.com/nrepl/proof/internal/check" ) -// Report is a JSON report read back in. +// Report is a JSON report read back in. Its results are the ones JSON +// wrote, rather than the Results of Run. type Report struct { - Server string `json:"server"` + Run Results []jsonResult `json:"results"` } @@ -32,6 +37,59 @@ func ReadJSON(path string) (Report, error) { return r, nil } +// table is a row per check, in the order of the first report that has +// it, and a column per report. +type table struct { + ids []string + // checks has the first result of each check, for its title and why. + checks map[string]jsonResult + results []map[string]jsonResult +} + +func newTable(reports []Report) table { + t := table{checks: map[string]jsonResult{}, results: make([]map[string]jsonResult, len(reports))} + for i, r := range reports { + t.results[i] = map[string]jsonResult{} + for _, res := range r.Results { + if _, ok := t.checks[res.ID]; !ok { + t.ids = slices.Insert(t.ids, t.place(res.ID), res.ID) + t.checks[res.ID] = res + } + t.results[i][res.ID] = res + } + } + return t +} + +// place is where a check that isn't in the table yet goes: after the last +// one in its group (e.g. eval), so a check only a later report has (e.g. +// one made by a newer proof) still ends up with the rest of its group. +func (t table) place(id string) int { + group, _, _ := strings.Cut(id, ".") + for i := len(t.ids) - 1; i >= 0; i-- { + if g, _, _ := strings.Cut(t.ids[i], "."); g == group { + return i + 1 + } + } + return len(t.ids) +} + +// passed returns how many checks the server of the i-th report passed, +// out of the ones that ran. +func (t table) passed(i int) string { + passed, ran := 0, 0 + for _, res := range t.results[i] { + if res.Verdict == check.Skipped { + continue + } + ran++ + if res.Verdict == check.Pass { + passed++ + } + } + return fmt.Sprintf("%d/%d", passed, ran) +} + var cells = map[check.Verdict]string{ check.Pass: "pass", check.Failed: "**fail**", @@ -41,50 +99,28 @@ var cells = map[check.Verdict]string{ } // Matrix writes a Markdown table with a row per check and a column per -// server. Checks appear in the order of the first report that has them. +// server. Checks appear in the order of the first report that has them, +// with the rest of their group. func Matrix(w io.Writer, reports []Report) { - var ids []string - titles := map[string]string{} - verdicts := make([]map[string]check.Verdict, len(reports)) - for i, r := range reports { - verdicts[i] = map[string]check.Verdict{} - for _, res := range r.Results { - if _, ok := titles[res.ID]; !ok { - ids = append(ids, res.ID) - titles[res.ID] = res.Title - } - verdicts[i][res.ID] = res.Verdict - } - } - + t := newTable(reports) header := []string{"Check"} rule := []string{"---"} for _, r := range reports { - header = append(header, escape(r.Server)) + header = append(header, escape(r.Title())) rule = append(rule, "---") } fmt.Fprintf(w, "| %s |\n|%s|\n", strings.Join(header, " | "), strings.Join(rule, "|")) - for _, id := range ids { - row := []string{fmt.Sprintf("`%s` %s", id, escape(titles[id]))} + for _, id := range t.ids { + row := []string{fmt.Sprintf("`%s` %s", id, escape(t.checks[id].Title))} for i := range reports { - row = append(row, cells[verdicts[i][id]]) + row = append(row, cells[t.results[i][id].Verdict]) } fmt.Fprintf(w, "| %s |\n", strings.Join(row, " | ")) } totals := []string{"**Passed**"} for i := range reports { - passed, ran := 0, 0 - for _, v := range verdicts[i] { - if v == check.Skipped { - continue - } - ran++ - if v == check.Pass { - passed++ - } - } - totals = append(totals, fmt.Sprintf("%d/%d", passed, ran)) + totals = append(totals, t.passed(i)) } fmt.Fprintf(w, "| %s |\n", strings.Join(totals, " | ")) } @@ -92,3 +128,80 @@ func Matrix(w io.Writer, reports []Report) { func escape(s string) string { return strings.ReplaceAll(s, "|", `\|`) } + +// The page comes from matrix.html, with the table and the versions of +// proof where its markers are. +// +//go:embed matrix.html +var matrixPage string + +// HTML writes the same matrix as Matrix as a web page, with the checks +// grouped like in the text report. The why of a check and the details of +// a verdict show up when hovering over them. +func HTML(w io.Writer, reports []Report) error { + t := newTable(reports) + esc := html.EscapeString + var b strings.Builder + b.WriteString("\nCheck") + for _, r := range reports { + fmt.Fprintf(&b, "%s", esc(r.Server)) + if r.ServerVersion != "" { + fmt.Fprintf(&b, `%s`, esc(r.ServerVersion)) + } + b.WriteString("") + } + b.WriteString("\n\n\n") + group := "" + for _, id := range t.ids { + if g, _, _ := strings.Cut(id, "."); g != group { + group = g + fmt.Fprintf(&b, "%s\n", len(reports)+1, esc(g)) + } + res := t.checks[id] + fmt.Fprintf(&b, `%s%s`, + titleAttr(res.Why), esc(id), esc(res.Title)) + for i := range reports { + res := t.results[i][id] + notes := res.Details + if res.Expected != "" { + notes = append([]string{"expected: " + res.Expected}, notes...) + } + fmt.Fprintf(&b, `%s`, res.Verdict, titleAttr(strings.Join(notes, "\n")), res.Verdict) + } + b.WriteString("\n") + } + b.WriteString("\n\nPassed") + for i := range reports { + fmt.Fprintf(&b, "%s", t.passed(i)) + } + b.WriteString("\n\n") + + var proofs []string + var latest time.Time + for _, r := range reports { + if r.Proof != "" && !slices.Contains(proofs, r.Proof) { + proofs = append(proofs, r.Proof) + } + if r.Started.After(latest) { + latest = r.Started + } + } + made := "proof" + if len(proofs) > 0 { + made += " " + strings.Join(proofs, ", ") + } + if !latest.IsZero() { + made += " on " + latest.UTC().Format(time.DateOnly) + } + _, err := strings.NewReplacer("\n", b.String(), "", esc(made)).WriteString(w, matrixPage) + return err +} + +// titleAttr is a title attribute with the given text, or nothing when +// there's no text. +func titleAttr(text string) string { + if text == "" { + return "" + } + return ` title="` + html.EscapeString(text) + `"` +} diff --git a/internal/report/matrix.html b/internal/report/matrix.html new file mode 100644 index 0000000..0d218cb --- /dev/null +++ b/internal/report/matrix.html @@ -0,0 +1,115 @@ + + + + + +nREPL Compatibility Matrix + + + +
+

nREPL Compatibility Matrix

+

Here's how the servers below fare against the checks of +proof, which are mostly about +what the clients people use (CIDER, Calva, Conjure, vim-fireplace, REPLy and +neorepl) need from a server.

+
+
pass
The server does what the clients need.
+
fail
Some client won't work properly with the server, or users lose data.
+
warn
The server does something differently from nREPL or the spec, but no client we know of minds.
+
skip
The check doesn't apply to the server (e.g. it needs Java interop).
+
error
proof couldn't run the check.
+
+

Hover over a check to see why it matters, and over a verdict for the details.

+
+
+ + +
+
+ + + diff --git a/internal/report/report.go b/internal/report/report.go index 4aa098f..829eb01 100644 --- a/internal/report/report.go +++ b/internal/report/report.go @@ -14,11 +14,23 @@ import ( // Run is everything known about one run. type Run struct { - Proof string `json:"proof"` - Server string `json:"server"` - Address string `json:"address"` - Started time.Time `json:"started"` - Results []check.Result `json:"-"` + Proof string `json:"proof"` + Server string `json:"server"` + // ServerVersion is what the version snippet of the server's profile + // evaluated to, if it has one. + ServerVersion string `json:"server-version,omitempty"` + Address string `json:"address"` + Started time.Time `json:"started"` + Results []check.Result `json:"-"` +} + +// Title is the name of the server, followed by its version when it's +// known. +func (r Run) Title() string { + if r.ServerVersion == "" { + return r.Server + } + return r.Server + " " + r.ServerVersion } // Counts tallies verdicts. @@ -41,7 +53,7 @@ var labels = map[check.Verdict]string{ // Text writes a human-readable report. With verbose set it includes // notes for every check and transcripts for the ones that didn't pass. func Text(w io.Writer, run Run, verbose bool) { - fmt.Fprintf(w, "proof %s: %s at %s\n", run.Proof, run.Server, run.Address) + fmt.Fprintf(w, "proof %s: %s at %s\n", run.Proof, run.Title(), run.Address) group := "" for _, res := range run.Results { g, _, _ := strings.Cut(res.ID, ".") diff --git a/internal/report/report_test.go b/internal/report/report_test.go index 1267ad1..fc57c22 100644 --- a/internal/report/report_test.go +++ b/internal/report/report_test.go @@ -16,19 +16,23 @@ func sampleRun(server string, verdicts ...check.Verdict) Run { ids := []string{"describe.reply", "eval.value", "eval.ns"} r := Run{Proof: "test", Server: server, Address: "localhost:1", Started: time.Unix(0, 0)} for i, v := range verdicts { - r.Results = append(r.Results, check.Result{ID: ids[i], Title: "title " + ids[i], Verdict: v}) + r.Results = append(r.Results, check.Result{ID: ids[i], Title: "title " + ids[i], Why: "why " + ids[i], Verdict: v}) } return r } -// The matrix is built from JSON reports, so go through the real encoder. -func TestMatrixFromJSONReports(t *testing.T) { +// sampleReports are the reports of two servers, the second of which +// failed and skipped a check. The matrix is built from JSON reports, so +// they go through the real encoder. +func sampleReports(t *testing.T) []Report { + t.Helper() + beta := sampleRun("Beta | Two", check.Pass, check.Failed, check.Skipped) + beta.ServerVersion = "2.0" + beta.Started = time.Date(2026, 11, 3, 5, 0, 0, 0, time.UTC) + beta.Results[1].Details = []string{"gave no value"} dir := t.TempDir() var reports []Report - for i, run := range []Run{ - sampleRun("Alpha", check.Pass, check.Pass, check.Pass), - sampleRun("Beta | Two", check.Pass, check.Failed, check.Skipped), - } { + for i, run := range []Run{sampleRun("Alpha", check.Pass, check.Pass, check.Pass), beta} { path := filepath.Join(dir, string(rune('a'+i))+".json") f, err := os.Create(path) if err != nil { @@ -44,12 +48,15 @@ func TestMatrixFromJSONReports(t *testing.T) { } reports = append(reports, r) } + return reports +} +func TestMatrixFromJSONReports(t *testing.T) { var buf bytes.Buffer - Matrix(&buf, reports) + Matrix(&buf, sampleReports(t)) got := buf.String() for _, want := range []string{ - "| Check | Alpha | Beta \\| Two |", + "| Check | Alpha | Beta \\| Two 2.0 |", "| `eval.value` title eval.value | pass | **fail** |", "| `eval.ns` title eval.ns | pass | skip |", "| **Passed** | 3/3 | 1/2 |", @@ -60,6 +67,72 @@ func TestMatrixFromJSONReports(t *testing.T) { } } +func TestMatrixAsAPage(t *testing.T) { + var buf bytes.Buffer + if err := HTML(&buf, sampleReports(t)); err != nil { + t.Fatal(err) + } + got := buf.String() + for _, want := range []string{ + `Beta | Two2.0`, + `eval`, + `eval.valuetitle eval.value`, + `passfail`, + `skip`, + `3/31/2`, + "Made with proof test on 2026-11-03.", + } { + if !strings.Contains(got, want) { + t.Errorf("page is missing %q:\n%s", want, got) + } + } +} + +// A check the first report doesn't have still goes in its group. +func TestChecksFromLaterReportsJoinTheirGroup(t *testing.T) { + reports := []Report{ + {Run: Run{Server: "Alpha"}, Results: []jsonResult{{ID: "eval.value"}, {ID: "describe.reply"}}}, + {Run: Run{Server: "Beta"}, Results: []jsonResult{{ID: "eval.value"}, {ID: "describe.reply"}, {ID: "eval.ns"}}}, + } + var buf bytes.Buffer + if err := HTML(&buf, reports); err != nil { + t.Fatal(err) + } + got := buf.String() + if n := strings.Count(got, `eval`); n != 1 { + t.Errorf("eval has %d groups:\n%s", n, got) + } + if i, j := strings.Index(got, "eval.ns"), strings.Index(got, "describe.reply"); i > j { + t.Errorf("eval.ns isn't with the other eval checks:\n%s", got) + } + if !strings.Contains(got, "Made with proof.") { + t.Errorf("reports without a proof version still get one:\n%s", got) + } + reports[0].Proof, reports[1].Proof = "0.1.0", "0.2.0" + buf.Reset() + if err := HTML(&buf, reports); err != nil { + t.Fatal(err) + } + if got := buf.String(); !strings.Contains(got, "Made with proof 0.1.0, 0.2.0.") { + t.Errorf("the versions of proof aren't listed:\n%s", got) + } + buf.Reset() + Matrix(&buf, reports) + if got := buf.String(); strings.Index(got, "`eval.ns`") > strings.Index(got, "`describe.reply`") { + t.Errorf("eval.ns isn't with the other eval checks:\n%s", got) + } +} + +func TestTextShowsTheServerVersion(t *testing.T) { + run := sampleRun("Alpha", check.Pass) + run.ServerVersion = "1.2.3" + var buf bytes.Buffer + Text(&buf, run, false) + if out := buf.String(); !strings.HasPrefix(out, "proof test: Alpha 1.2.3 at localhost:1") { + t.Errorf("no version in the report:\n%s", out) + } +} + func TestReadJSONRejectsOtherFiles(t *testing.T) { path := filepath.Join(t.TempDir(), "x.json") os.WriteFile(path, []byte(`{"name": "not a report"}`), 0o644) diff --git a/profiles/babashka.toml b/profiles/babashka.toml index 66406e9..adb9c8c 100644 --- a/profiles/babashka.toml +++ b/profiles/babashka.toml @@ -61,3 +61,7 @@ value = "println" # A function to look up. [snippets.lookup] code = "println" + +# The version of bb, for the report. +[snippets.version] +code = '(System/getProperty "babashka.version")' diff --git a/profiles/basilisp.toml b/profiles/basilisp.toml index f0c50cd..f05f1cd 100644 --- a/profiles/basilisp.toml +++ b/profiles/basilisp.toml @@ -55,3 +55,7 @@ value = "println" # A function to look up. [snippets.lookup] code = "println" + +# The version of Basilisp, for the report. +[snippets.version] +code = '(do (import importlib.metadata) (importlib.metadata/version "basilisp"))' diff --git a/profiles/dialtone.toml b/profiles/dialtone.toml index 6d07d24..a9b82f8 100644 --- a/profiles/dialtone.toml +++ b/profiles/dialtone.toml @@ -57,3 +57,7 @@ value = "lists:map" # A function to look up. [snippets.lookup] code = "lists:map" + +# The version of dialtone, for the report. +[snippets.version] +code = "element(2, application:get_key(dialtone, vsn))." diff --git a/profiles/repartee.toml b/profiles/repartee.toml index 5cfd5ff..f232d30 100644 --- a/profiles/repartee.toml +++ b/profiles/repartee.toml @@ -53,3 +53,8 @@ value = "Enum.map" # A function to look up. [snippets.lookup] code = "Enum.map" + +# The version of repartee, for the report. Its application isn't loaded +# until something asks for it. +[snippets.version] +code = "Application.load(:repartee); to_string(Application.spec(:repartee, :vsn))"