diff --git a/README.md b/README.md index 89671c0..9528efe 100644 --- a/README.md +++ b/README.md @@ -254,10 +254,20 @@ verdict claims the documented step works: tool itself that kibble extracts. The detail names the README, how many install documents were read, and the routes seen. It is not a `GAP`, since the documents may describe routes kibble does not run, such as apt or brew. +- An example row whose detail opens with "built from the checkout, the documented + install was not tested" ran against the tool kibble built itself, because no + documented install gave the session a binary. kibble builds only a tool it can name: + a Go main package at the root or under `cmd/`, a Rust package's one binary, or a + Python project's one console script, or whichever of these is named for the + repository. The examples' verdicts stand, but a release can differ from the checkout, + and the install rows keep their own verdicts. A checkout that does not build reads + `BLOCKED`, since the failure is kibble's stand-in and not the document. - `BLOCKED` with the reason `inherited-state` means a line that would have been a plain `FAIL` ran in a directory an earlier section entered with `cd`, `pushd`, or `popd`. Documents `cd` once in setup and stay there, so the failure may come from what that - directory holds rather than from what the line says. + directory holds rather than from what the line says. Each collapsed `
` element + is a section of its own, since a README offers alternative setups that way under one + heading and a reader opens only one. - `BLOCKED` with the reason `checkout-state` means a line that would have been a plain `FAIL` ran in the repository's own checkout and its output says the target already exists or is already initialized. The collision is with the checkout kibble chose, so @@ -266,7 +276,16 @@ verdict claims the documented step works: already held the other. So does a documented tool saying the top of the checkout is not a repository of its own kind and naming its own init command, when no earlier line of the document that passed made or cloned one. That line was not tested, and it is not - counted as passing. + counted as passing. When a scaffold line such as `goreleaser init` is stopped this way, + the same tool's later lines in that document read the checkout's project and not the + one the document creates, so their failures and their documented nonzero exits read + `checkout-state` as well. A usage error or a crash still stands. +- `BLOCKED` with the reason `environment-changed` means a line crashed and afterwards + the documented tool failed `--version`, though it passed when the session began. + Something the session ran changed the environment the tool runs from, as a project + command can remove the tool's own dependencies, so the crash says nothing about the + line. A crash the tool survives stays `FAIL`. A tool whose `--version` fails from the + start is never probed. - A passing line that writes a titled fence to its file reads `VERIFIED` "wrote X as the document shows it". The document supplied the content, so only the write is claimed. - An install whose product is a shell function the documented sourcing defines is diff --git a/checkoutbuild.go b/checkoutbuild.go new file mode 100644 index 0000000..7b6f9b9 --- /dev/null +++ b/checkoutbuild.go @@ -0,0 +1,197 @@ +package main + +import ( + "go/parser" + "go/token" + "os" + "path" + "path/filepath" + "strings" + + kplan "github.com/dcadolph/kibble/internal/plan" +) + +// checkoutBuild returns an install that builds the repository's own tool from +// the checkout, for a repository whose documented installs give an example +// session nothing to run against. Without it one missing install route cost +// every example in the repository, and on the 11-11 sweeps that silence was +// the largest single reason a repository came back with nothing tested. +// +// It builds only what it can name without guessing: a Go main package at the +// root or under cmd/, a Rust package with one binary or one named for the +// repository, or a Python project with one console script or one named for it. +// When two ecosystems both offer a tool, the one named for the repository wins, +// and with no such name nothing is built, since running the examples against +// the wrong binary would convict a document of someone else's failure. +func checkoutBuild(dir, repo string) (kplan.PlanInstall, bool) { + var found []kplan.PlanInstall + for _, try := range []func(dir, repo string) (kplan.PlanInstall, bool){ + goCheckoutBuild, rustCheckoutBuild, pythonCheckoutBuild, + } { + if in, ok := try(dir, repo); ok { + found = append(found, in) + } + } + if len(found) == 1 { + return found[0], true + } + for _, in := range found { + if normalizeName(in.Binary) == normalizeName(repo) { + return in, true + } + } + return kplan.PlanInstall{}, false +} + +// goCheckoutBuild builds a Go module's main package with go install, from the +// root when the root is one, else from the cmd/ directory pickBinary selects. +func goCheckoutBuild(dir, repo string) (kplan.PlanInstall, bool) { + body, err := os.ReadFile(filepath.Join(dir, "go.mod")) + if err != nil { + return kplan.PlanInstall{}, false + } + m := reGoModule.FindSubmatch(body) + if m == nil { + return kplan.PlanInstall{}, false + } + base := path.Base(stripMajor(string(m[1]))) + if isMainPackage(dir) { + if !reBinName.MatchString(base) { + return kplan.PlanInstall{}, false + } + return kplan.PlanInstall{Cmd: "go install .", Ecosystem: "go", Binary: base, FromCheckout: true}, true + } + ents, err := os.ReadDir(filepath.Join(dir, "cmd")) + if err != nil { + return kplan.PlanInstall{}, false + } + var mains []string + for _, e := range ents { + if e.IsDir() && isMainPackage(filepath.Join(dir, "cmd", e.Name())) { + mains = append(mains, e.Name()) + } + } + name, ok := pickBinary(mains, repo, base) + if !ok { + return kplan.PlanInstall{}, false + } + return kplan.PlanInstall{Cmd: "go install ./cmd/" + name, Ecosystem: "go", Binary: name, FromCheckout: true}, true +} + +// isMainPackage reports whether dir holds a non-test Go file of package main. +func isMainPackage(dir string) bool { + ents, err := os.ReadDir(dir) + if err != nil { + return false + } + fset := token.NewFileSet() + for _, e := range ents { + name := e.Name() + if e.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + f, err := parser.ParseFile(fset, filepath.Join(dir, name), nil, parser.PackageClauseOnly) + if err == nil && f.Name.Name == "main" { + return true + } + } + return false +} + +// rustCheckoutBuild builds a Rust package's binary with cargo install --path, +// locked when the checkout carries a lock file, as a release build would be. A +// workspace root with no package of its own is left alone. +func rustCheckoutBuild(dir, repo string) (kplan.PlanInstall, bool) { + body, err := os.ReadFile(filepath.Join(dir, "Cargo.toml")) + if err != nil { + return kplan.PlanInstall{}, false + } + pkgs := sectionValues(body, map[string]string{"package": "name"}) + if len(pkgs) == 0 { + return kplan.PlanInstall{}, false + } + bins := sectionValues(body, map[string]string{"[bin]": "name"}) + if len(bins) == 0 { + if _, err := os.Stat(filepath.Join(dir, "src", "main.rs")); err != nil { + return kplan.PlanInstall{}, false + } + bins = pkgs[:1] + } + name, ok := pickBinary(bins, repo, pkgs[0]) + if !ok { + return kplan.PlanInstall{}, false + } + cmd := "cargo install --path . --bin " + name + if _, err := os.Stat(filepath.Join(dir, "Cargo.lock")); err == nil { + cmd += " --locked" + } + return kplan.PlanInstall{Cmd: cmd, Ecosystem: "rust", Binary: name, FromCheckout: true}, true +} + +// checkoutVenv is the virtual environment a Python tool built from the +// checkout is installed into. +const checkoutVenv = "/opt/kibble-tool" + +// pythonCheckoutBuild installs a Python project from the checkout with pip, +// when its pyproject.toml declares the console script to run. The install goes +// into an environment of its own with the script linked onto PATH, the way pipx +// does it. Installed into the image's Python, the tool shared an environment +// with whatever its examples manage: poetry's `poetry sync --only dev`, run in +// poetry's checkout, removed poetry's own dependencies, and the 39 lines after +// it crashed on a missing module. +func pythonCheckoutBuild(dir, repo string) (kplan.PlanInstall, bool) { + body, err := os.ReadFile(filepath.Join(dir, "pyproject.toml")) + if err != nil { + return kplan.PlanInstall{}, false + } + scripts := sectionValues(body, map[string]string{"project.scripts": "*", "tool.poetry.scripts": "*"}) + names := sectionValues(body, map[string]string{"project": "name", "tool.poetry": "name"}) + names = append(names, "") + name, ok := pickBinary(scripts, repo, names[0]) + if !ok { + return kplan.PlanInstall{}, false + } + cmd := "python3 -m venv " + checkoutVenv + " && " + checkoutVenv + "/bin/pip install --quiet . && " + + "ln -sf " + checkoutVenv + "/bin/" + name + " /usr/local/bin/" + name + return kplan.PlanInstall{Cmd: cmd, Ecosystem: "python", Binary: name, FromCheckout: true}, true +} + +// pickBinary returns the candidate named for the repository or its package, +// else the only candidate, else nothing. A name that cannot sit safely in a +// shell command is never returned. +func pickBinary(cands []string, names ...string) (string, bool) { + pick := "" + for _, c := range cands { + for _, n := range names { + if n != "" && normalizeName(c) == normalizeName(n) { + pick = c + break + } + } + if pick != "" { + break + } + } + if pick == "" && len(cands) == 1 { + pick = cands[0] + } + if pick == "" || !reBinName.MatchString(pick) { + return "", false + } + return pick, true +} + +// checkoutNote labels every example result run against a tool kibble built +// from the checkout, since a reader installs a release and the two can differ. +const checkoutNote = "built from the checkout, the documented install was not tested" + +// fromCheckout reports whether a plan's tool is one kibble built from the +// checkout rather than one a documented install provided. +func fromCheckout(plan *kplan.Plan) bool { + for _, in := range plan.Installs { + if in.FromCheckout { + return true + } + } + return false +} diff --git a/checkoutbuild_test.go b/checkoutbuild_test.go new file mode 100644 index 0000000..6fd9a08 --- /dev/null +++ b/checkoutbuild_test.go @@ -0,0 +1,186 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + + kplan "github.com/dcadolph/kibble/internal/plan" +) + +// writeTree writes files under dir, creating their directories. +func writeTree(t *testing.T, dir string, files map[string]string) { + t.Helper() + for name, body := range files { + p := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } +} + +// TestCheckoutBuild checks which checkouts kibble builds its own tool from, +// and that it builds only a binary it can name without guessing. +func TestCheckoutBuild(t *testing.T) { + t.Parallel() + const goMain = "package main\n\nfunc main() {}\n" + tests := []struct { + Repo string + Files map[string]string + WantCmd string + WantBinary string + Want bool + }{{ // Test 0: a root main package installs from the root under the module's name. + Repo: "glance", + Files: map[string]string{"go.mod": "module github.com/glanceapp/glance\n", "main.go": goMain}, + WantCmd: "go install .", WantBinary: "glance", Want: true, + }, { // Test 1: a major-version suffix is not the binary's name. + Repo: "tool", + Files: map[string]string{"go.mod": "module example.com/tool/v2\n", "main.go": goMain}, + WantCmd: "go install .", WantBinary: "tool", Want: true, + }, { // Test 2: the cmd directory named for the repository wins. + Repo: "headscale", + Files: map[string]string{"go.mod": "module github.com/juanfont/headscale\n", + "cmd/headscale/main.go": goMain, "cmd/hi/main.go": goMain}, + WantCmd: "go install ./cmd/headscale", WantBinary: "headscale", Want: true, + }, { // Test 3: a single cmd directory is the tool whatever its name. + Repo: "opentofu", + Files: map[string]string{"go.mod": "module github.com/opentofu/opentofu\n", + "cmd/tofu/main.go": goMain, "cmd/tofu/main_test.go": "package main\n"}, + WantCmd: "go install ./cmd/tofu", WantBinary: "tofu", Want: true, + }, { // Test 4: two cmd directories, neither named for the repository, build nothing. + Repo: "repo", + Files: map[string]string{"go.mod": "module example.com/repo\n", + "cmd/a/main.go": goMain, "cmd/b/main.go": goMain}, + }, { // Test 5: a library builds nothing. + Repo: "lib", + Files: map[string]string{"go.mod": "module example.com/lib\n", "lib.go": "package lib\n"}, + }, { // Test 6: a test file alone does not make a main package. + Repo: "lib", + Files: map[string]string{"go.mod": "module example.com/lib\n", "main_test.go": goMain}, + }, { // Test 7: a Rust package with src/main.rs installs locked when it has a lock file. + Repo: "navi", + Files: map[string]string{"Cargo.toml": "[package]\nname = \"navi\"\nversion = \"2.0.0\"\n", + "src/main.rs": "fn main() {}\n", "Cargo.lock": ""}, + WantCmd: "cargo install --path . --bin navi --locked", WantBinary: "navi", Want: true, + }, { // Test 8: a single declared binary is the tool, unlocked without a lock file. + Repo: "nushell", + Files: map[string]string{"Cargo.toml": "[package]\nname = \"nu\"\n\n[[bin]]\nname = \"nu\"\npath = \"src/main.rs\"\n"}, + WantCmd: "cargo install --path . --bin nu", WantBinary: "nu", Want: true, + }, { // Test 9: a workspace root with no package of its own builds nothing. + Repo: "ws", + Files: map[string]string{"Cargo.toml": "[workspace]\nmembers = [\"crates/*\"]\n"}, + }, { // Test 10: a Rust library builds nothing. + Repo: "lib", + Files: map[string]string{"Cargo.toml": "[package]\nname = \"lib\"\n", "src/lib.rs": ""}, + }, { // Test 11: a Python project with one console script installs with pip. + Repo: "fastapi", + Files: map[string]string{"pyproject.toml": "[project]\nname = \"fastapi\"\n\n" + + "[project.scripts]\nfastapi = \"fastapi.cli:main\"\n"}, + WantCmd: "python3 -m venv /opt/kibble-tool && /opt/kibble-tool/bin/pip install --quiet . && " + + "ln -sf /opt/kibble-tool/bin/fastapi /usr/local/bin/fastapi", + WantBinary: "fastapi", Want: true, + }, { // Test 12: a Python project with no script builds nothing. + Repo: "lib", + Files: map[string]string{"pyproject.toml": "[project]\nname = \"lib\"\n"}, + }, { // Test 13: of two ecosystems, the tool named for the repository wins. + Repo: "tool", + Files: map[string]string{"go.mod": "module example.com/helper\n", "main.go": goMain, + "pyproject.toml": "[project]\nname = \"tool\"\n\n[project.scripts]\ntool = \"tool:main\"\n"}, + WantCmd: "python3 -m venv /opt/kibble-tool && /opt/kibble-tool/bin/pip install --quiet . && " + + "ln -sf /opt/kibble-tool/bin/tool /usr/local/bin/tool", + WantBinary: "tool", Want: true, + }, { // Test 14: of two ecosystems with neither named for the repository, nothing builds. + Repo: "repo", + Files: map[string]string{"go.mod": "module example.com/helper\n", "main.go": goMain, + "pyproject.toml": "[project]\nname = \"other\"\n\n[project.scripts]\nother = \"o:main\"\n"}, + }, { // Test 15: a script name that cannot sit in a shell command is not built. + Repo: "repo", + Files: map[string]string{"pyproject.toml": "[project.scripts]\n\"bad;name\" = \"x:main\"\n"}, + }} + for testNum, test := range tests { + t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + writeTree(t, dir, test.Files) + got, ok := checkoutBuild(dir, test.Repo) + if ok != test.Want { + t.Fatalf("ok = %v, want %v (install %+v)", ok, test.Want, got) + } + if !ok { + return + } + if diff := cmp.Diff(test.WantCmd, got.Cmd); diff != "" { + t.Errorf("cmd mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(test.WantBinary, got.Binary); diff != "" { + t.Errorf("binary mismatch (-want +got):\n%s", diff) + } + if !got.FromCheckout { + t.Error("install is not marked as built from the checkout") + } + }) + } +} + +// TestCheckoutBuildSession checks that examples run against a checkout build +// say so, and that a failed checkout build blocks rather than skips. +func TestCheckoutBuildSession(t *testing.T) { + t.Parallel() + tests := []struct { + Out string + Checkout bool + WantStatus Status + WantDetail string + WantLabel bool + }{{ // Test 0: passing examples on a checkout build are labeled. + Out: sessionOut("KIBBLE-BUILD CODE=0", "KIBBLE-STEP b1 START", "KIBBLE-LINE b1:0 CODE=0", + "KIBBLE-LINE b1:1 CODE=0", "KIBBLE-LINE b1:2 CODE=0", "KIBBLE-DONE"), + Checkout: true, WantStatus: StatusVerified, WantLabel: true, + }, { // Test 1: a failing example on a checkout build is labeled too. + Out: sessionOut("KIBBLE-BUILD CODE=0", "KIBBLE-STEP b1 START", "KIBBLE-LINE b1:0 CODE=0", + "error: unknown command", "KIBBLE-LINE b1:1 CODE=2", "KIBBLE-LINE b1:2 CODE=0", "KIBBLE-DONE"), + Checkout: true, WantStatus: StatusFail, WantLabel: true, + }, { // Test 2: a failed checkout build blocks. + Out: sessionOut("KIBBLE-BUILD CODE=1", "go: go.mod requires go >= 1.27", "KIBBLE-ABORT"), + Checkout: true, WantStatus: StatusBlocked, WantLabel: true, WantDetail: "requires go >= 1.27", + }, { // Test 2a: a timed-out checkout build says so. + Out: sessionOut("KIBBLE-BUILD CODE=124", "KIBBLE-ABORT"), + Checkout: true, WantStatus: StatusBlocked, WantLabel: true, WantDetail: "timed out", + }, { // Test 3: a failed documented install still skips, unlabeled. + Out: sessionOut("KIBBLE-BUILD CODE=1", "build error tail", "KIBBLE-ABORT"), + WantStatus: StatusSkipped, + }, { // Test 4: passing examples on a documented install carry no label. + Out: sessionOut("KIBBLE-BUILD CODE=0", "KIBBLE-STEP b1 START", "KIBBLE-LINE b1:0 CODE=0", + "KIBBLE-LINE b1:1 CODE=0", "KIBBLE-LINE b1:2 CODE=0", "KIBBLE-DONE"), + WantStatus: StatusVerified, + }} + for testNum, test := range tests { + t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) { + t.Parallel() + plan := examplePlan() + if test.Checkout { + plan.Installs = []kplan.PlanInstall{{Cmd: "go install .", Ecosystem: "go", Binary: "tool", + FromCheckout: true}} + } + step := InstallStep{Repo: "repo", Kind: "example", plan: plan} + res := classifyExample(step, plan, test.Out, nil, 0, lineTimeout, sessionEnd{}) + if diff := cmp.Diff(test.WantStatus, res.Status); diff != "" { + t.Errorf("status mismatch (-want +got):\n%s\ndetail: %s", diff, res.Detail) + } + if !strings.Contains(res.Detail, test.WantDetail) { + t.Errorf("detail %q does not contain %q", res.Detail, test.WantDetail) + } + if got := strings.Contains(res.Detail, checkoutNote); got != test.WantLabel { + t.Errorf("labeled = %v, want %v (detail %q)", got, test.WantLabel, res.Detail) + } + }) + } +} diff --git a/corpus/recall.tsv b/corpus/recall.tsv index ed6a5cd..da669bc 100644 --- a/corpus/recall.tsv +++ b/corpus/recall.tsv @@ -20,7 +20,7 @@ # The first four came from the 2026-10-22 hand check of ten repositories # kibble had run nothing on. Columns are tab-separated: # name, clone URL, pinned commit, document, line, wanted status, proof, caught. -jq https://github.com/jqlang/jq.git 94b00aa807a51a8f97bc9345c03e32982e344678 README.md 56 FAIL libjq.so.1 no +jq https://github.com/jqlang/jq.git 94b00aa807a51a8f97bc9345c03e32982e344678 README.md 56 FAIL libjq.so.1 yes goreleaser https://github.com/goreleaser/goreleaser.git a9fb415389a2e55906ae457d9f76bb77b3cb6056 www/content/getting-started/quick-start.md 138 FAIL doesn't contain any tags no pyenv https://github.com/pyenv/pyenv.git 7d65a4683c91b29f7501da4c8737d4e786f8cad2 README.md 187 DRIFT|FAIL --install no shellcheck https://github.com/koalaman/shellcheck.git cd9072643932ee2008e454e2cd990e25c1dc9d49 README.md 412 FAIL exec foo; echo "Done!" no diff --git a/dependent.go b/dependent.go index c751ff2..c7199f7 100644 --- a/dependent.go +++ b/dependent.go @@ -4,6 +4,7 @@ import ( "fmt" "path" "regexp" + "slices" "strings" "github.com/dcadolph/kibble/internal/docblock" @@ -53,9 +54,12 @@ func resolveMissingBinaries(run *exampleRun, plan *kplan.Plan, have map[string]b // the session is no longer a clean test of this line, which is why these // become blocked rather than skipped: the cascade stops being reported as // several broken lines without any of them being called fine. A gap counts as -// not having run, since the document's own hole stopped the line. A Go build +// not having run, since the document's own hole stopped the line. So does a +// command in a block the plan left out: headscale's `make test` fails telling +// the reader to run `nix develop`, which the README shows and no container has. A Go build // that finds the document's file beside the checkout's own package is blocked // here too, since it takes the earlier file writes of the run to see it. So is +// a failure after a scaffold line the checkout's own files stopped. So is // a documented tool saying the top of the checkout is not a repository of its // own kind, since it takes the document's earlier lines to see none made one. func resolveDependentFailures(run *exampleRun, plan *kplan.Plan) { @@ -107,6 +111,22 @@ func resolveDependentFailures(run *exampleRun, plan *kplan.Plan) { } for li := range s.Lines { l := &s.Lines[li] + if l.Status == StatusVerified && l.Code > 0 { + // A nonzero exit the documents bless says the tool answered the + // project it read. After a stopped scaffold that is the + // checkout's project, so the blessing tested nothing: + // goreleaser's `check` exits 1 on the checkout's own Pro + // configuration and passed as documented behavior. + if init := stoppedScaffold(l, prior[s.Doc], bins); init != "" { + l.Status = StatusBlocked + l.Reason = ReasonCheckoutState + l.Detail = fmt.Sprintf("follows `%s`, which the repository's own files stopped, "+ + "so its exit %d answered the checkout's project and not the one the document "+ + "creates", init, l.Code) + prior[s.Doc] = append(prior[s.Doc], *l) + continue + } + } if l.Status != StatusFail { m := reFileWrite.FindStringSubmatch(l.Cmd) if m != nil && l.Status == StatusVerified && l.cwd == sessionRepoDir { @@ -130,12 +150,27 @@ func resolveDependentFailures(run *exampleRun, plan *kplan.Plan) { "not tested as written: %s", bin, bin, l.Detail) continue } + if init := stoppedScaffold(l, prior[s.Doc], bins); init != "" { + l.Status = StatusBlocked + l.Reason = ReasonCheckoutState + l.Detail = fmt.Sprintf("follows `%s`, which the repository's own files stopped, so it "+ + "read the checkout's project and not the one the document creates: %s", + init, l.Detail) + continue + } if cited := citedSkipped(l.output, skippedCmds); cited != "" { l.Status = StatusBlocked l.Reason = ReasonDependsOnSkipped l.Detail = fmt.Sprintf("failed naming `%s`, which did not run", cited) continue } + if cited := citedSkipped(l.output, plan.Unrun); cited != "" { + l.Status = StatusBlocked + l.Reason = ReasonDependsOnSkipped + l.Detail = fmt.Sprintf("failed naming `%s`, which the documents show in a block kibble "+ + "did not run", cited) + continue + } if word := skippedProvider(s.Lines[:li], l.output); word != "" { l.Status = StatusBlocked l.Reason = ReasonMissingFixture @@ -156,6 +191,11 @@ func resolveDependentFailures(run *exampleRun, plan *kplan.Plan) { l.Status = StatusBlocked l.Reason = ReasonDependsOnSkipped l.Detail = fmt.Sprintf("names %s, which `%s` also named and did not pass", op, earlier) + } else if host, earlier := unresolvedHostNotRun(prior[s.Doc], l.Cmd, l.output); host != "" { + l.Status = StatusBlocked + l.Reason = ReasonDependsOnSkipped + l.Detail = fmt.Sprintf("could not resolve host %s, which `%s` also named and did not pass", + host, earlier) } prior[s.Doc] = append(prior[s.Doc], *l) } @@ -237,7 +277,7 @@ func notItsRepository(l *lineResult, earlier []lineResult, bins map[string]bool) return "" } } - if reUsageError.MatchString(out) || reCrash.MatchString(out) { + if reUsageError.MatchString(out) || crashed(out) { return "" } for _, p := range earlier { @@ -248,6 +288,31 @@ func notItsRepository(l *lineResult, earlier []lineResult, bins map[string]bool) return bin } +// stoppedScaffold returns the command of an earlier line of the same document +// that was to create what the document's later lines work on and that the +// checkout stopped by already holding it, when the failing line runs the same +// documented binary. goreleaser's quick start runs `goreleaser init` and then +// builds from the configuration that wrote. In the checkout the init refuses, +// since goreleaser's own configuration is there, and every build after it +// reads that file instead. A usage error is the tool's answer to the line as +// written, and a crash convicts whatever the line read, so both stand. +func stoppedScaffold(l *lineResult, earlier []lineResult, bins map[string]bool) string { + bin, _ := kplan.InvokedBinary(l.Cmd, bins) + if bin == "" { + return "" + } + if out := stripANSI(l.output); reUsageError.MatchString(out) || crashed(out) { + return "" + } + for _, p := range earlier { + if p.Status == StatusBlocked && p.Reason == ReasonCheckoutState && + reAlreadyExists.MatchString(p.output) && runsBinary(p.Cmd, bin) { + return p.Cmd + } + } + return "" +} + // runsBinary reports whether any command of a line runs bin, stepping over // wrappers such as env as InvokedBinary does. func runsBinary(cmd, bin string) bool { @@ -313,9 +378,12 @@ func skippedProvider(earlier []lineResult, output string) string { } // sharedOperandNotPassed returns a file-shaped operand of a failing line and -// the first earlier line of the same document that also named it and was -// skipped, blocked, or stopped by a gap. A query on a database whose load was -// blocked is the same hole surfacing again. An earlier failure does not count, +// the first earlier line of the same document that also named it, or wrote it +// through a redirect, and was skipped, blocked, or stopped by a gap. A query on +// a database whose load was blocked is the same hole surfacing again, and so is +// trivy's `trivy sbom ./rpms-res.cdx.json` after the `jq ... > rpms-res.cdx.json` +// that makes the file could not run. Names are compared as paths, so a leading +// ./ on one side does not hide the match. An earlier failure does not count, // since a break repeated on several lines of one file is still several // reports. Like the other rules here it is not causal: the earlier line still // reports, and the later ones stop counting as separate breaks. @@ -325,8 +393,8 @@ func sharedOperandNotPassed(prior []lineResult, cmd string) (operand, earlier st if p.Status == StatusVerified || p.Status == StatusFail { continue } - for _, pop := range fileOperands(p.Cmd) { - if pop == op { + for _, pop := range append(fileOperands(p.Cmd), shell.Writes(p.Cmd)...) { + if path.Clean(pop) == path.Clean(op) { return op, docblock.Flatten(p.Cmd) } } @@ -335,6 +403,44 @@ func sharedOperandNotPassed(prior []lineResult, cmd string) (operand, earlier st return "", "" } +var ( + // reUnresolvedHost matches a client reporting that a host name did not + // resolve. + reUnresolvedHost = regexp.MustCompile(`(?i)could not resolve host|unable to resolve host|` + + `no such host|name or service not known|temporary failure in name resolution|` + + `name does not resolve`) + // reURLHost matches an http or https URL and captures its host. + reURLHost = regexp.MustCompile(`\bhttps?://([A-Za-z0-9][A-Za-z0-9._-]*)`) +) + +// unresolvedHostNotRun returns the host of a URL the failing line could not +// resolve and the first earlier line of the same document that named that +// host as a word of its own and was skipped, blocked, or stopped by a gap. +// tailscale's Kubernetes guide runs `curl http://nginx` against a service its +// skipped kubectl and make lines set up, so the name has nothing to resolve +// to. A host no earlier line names, or one named only by lines that passed, +// keeps the failure, since a dead domain in a document is a real break. +func unresolvedHostNotRun(prior []lineResult, cmd, output string) (host, earlier string) { + if !reUnresolvedHost.MatchString(output) { + return "", "" + } + for _, m := range reURLHost.FindAllStringSubmatch(cmd, -1) { + h := m[1] + if !strings.Contains(output, h) { + continue + } + for _, p := range prior { + if p.Status == StatusVerified || p.Status == StatusFail { + continue + } + if words := strings.Fields(docblock.Flatten(p.Cmd)); len(words) > 1 && slices.Contains(words[1:], h) { + return h, docblock.Flatten(p.Cmd) + } + } + } + return "", "" +} + // reScriptFile matches a source file an interpreter runs, which is the program // of a line such as python3 scripts/bench.py rather than data it reads. var reScriptFile = regexp.MustCompile(`\.(py|js|mjs|cjs|ts|sh|bash|rb|pl|php|lua)$`) diff --git a/dependent_adversarial_test.go b/dependent_adversarial_test.go index 9049401..123ca75 100644 --- a/dependent_adversarial_test.go +++ b/dependent_adversarial_test.go @@ -187,6 +187,25 @@ func TestSharedOperandAfterLineNotPassed(t *testing.T) { {Cmd: "python3 scripts/bench.py --engine one", Status: StatusFail, output: "error"}, }}}, WantStatus: []Status{StatusBlocked, StatusFail}, + }, { // Test 7: A file an earlier blocked line would have written pairs, ./ or not. + Steps: []exampleStep{{ID: "b1", Doc: "README.md", Lines: []lineResult{ + {Cmd: `jq '.name = "x"' in.cdx.json > out.cdx.json`, Status: StatusBlocked}, + {Cmd: "dbtool sbom ./out.cdx.json", Status: StatusFail, output: "error: unable to open"}, + }}}, + WantStatus: []Status{StatusBlocked, StatusBlocked}, + WantDetail: "names ./out.cdx.json, which `jq '.name = \"x\"' in.cdx.json > out.cdx.json` also named and did not pass", + }, { // Test 8: The same reader after the writer passed stays a failure. + Steps: []exampleStep{{ID: "b1", Doc: "README.md", Lines: []lineResult{ + {Cmd: `jq '.name = "x"' in.cdx.json > out.cdx.json`, Status: StatusVerified}, + {Cmd: "dbtool sbom ./out.cdx.json", Status: StatusFail, output: "error: unable to open"}, + }}}, + WantStatus: []Status{StatusVerified, StatusFail}, + }, { // Test 9: A file the earlier line only read from standard input does not pair. + Steps: []exampleStep{{ID: "b1", Doc: "README.md", Lines: []lineResult{ + {Cmd: "dbtool load < other.json", Status: StatusBlocked}, + {Cmd: "dbtool sbom ./out.cdx.json", Status: StatusFail, output: "error: unable to open"}, + }}}, + WantStatus: []Status{StatusBlocked, StatusFail}, }} for testNum, test := range tests { t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) { diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 631a140..86a26bf 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -41,6 +41,16 @@ reports which steps a brand-new user could actually complete. `LD_LIBRARY_PATH` outside the block, the same error is `BUILT`: the reader was told, and kibble ran only the block. Any other failed smoke test stays `BUILT`. A block that also clones, installs system packages, or pipes a remote script is not a source build. +- Builds the tool from the checkout when no documented install gives the example session + a binary, so one missing install route no longer costs every example in the + repository. Go uses `go install` of the root or of the `cmd/` directory named for the + repository or the only one there, Rust uses `cargo install --path .` of the one + binary or the one named for the repository, `--locked` when a lock file exists, and + Python uses `pip install .` for the one console script or the one named for it, into + a virtual environment of its own with the script linked onto PATH, so an example that + manages packages cannot remove the tool's own dependencies. A checkout that offers no + such name builds nothing. Every example result says it was built from the checkout and + the documented install was not tested, and a build that fails reads `BLOCKED`. - Picks the image from the toolchain the step assumes, so a Rust project builds with `cargo` and a Node project with `npm`, and finds the installed binary wherever that toolchain puts it, even when it is not named after the package. @@ -167,7 +177,13 @@ same reason. A repository too large to stream whole is an `ERROR` too: a file ki failed to deliver is indistinguishable from one the docs never create, and guessing between them is the one mistake this tool must not make. Generated directories such as `node_modules`, `vendor`, `build`, and `target` are left out of that stream, so the -budget goes to source a document might actually name. +budget goes to source a document might actually name. The name is only a guess, so a +directory that holds files git tracks is streamed whatever it is called: goreleaser keeps +source in `internal/pipe/dist` and k6 in `internal/build`. The git index says what is +tracked, and a checkout without one falls back to the names alone. A session that builds +the tool from the checkout also takes every tracked file whatever its size, where a file +over 2 MB is otherwise left out: k6 vendors a 4.9 MB table and does not compile without it. +If that pushes the tree past the stream cap, the size limit comes back. ## What the container is, and is not @@ -237,7 +253,10 @@ with one exception. A tool that reports a file the line passed as missing is rea `inherited-state` too, since the file's absence from that directory says nothing about the directory the document meant. Resetting the working directory at every heading was rejected, because it would break every document that changes directory once in setup and -invent failures. +invent failures. A section is a heading, or the outermost collapsed `
` element +under it, since a README offers alternative setups as sibling details under one heading: +glance's template setup enters a directory, and its manual setup's `mkdir config` then met +the template's own `config` there. A `cd` the planner skips, because its target is a placeholder or a name the reader fills in, loses the directory for the rest of its section. A document keeps one shell, so every @@ -279,6 +298,23 @@ says "No solution found" when it fails. The same holds for installers such as pi whose resolver speaks the same words. Output that is a vocabulary error, an unknown or unexpected flag, option, argument, or subcommand, stays `FAIL`. +A scaffold line the checkout stopped, such as `goreleaser init` refusing because +goreleaser's own configuration is already there, taints the same tool's later lines in +that document: they read the checkout's project, not the one the document creates. Their +failures are `checkout-state`, and so is a nonzero exit the documents bless, since that +exit answered the wrong project. goreleaser's `check` exits 1 on the checkout's GoReleaser +Pro configuration and would otherwise pass as documented behavior. A usage error and a +crash keep the failure, as does a scaffold stopped in another document. + +The session asks each documented binary for `--version` once at the start. After a line +that ran one of them exited nonzero, it asks again, but only for a binary that passed the +first time. A crash followed by a failed probe is `BLOCKED` with the reason +`environment-changed`: the session changed the environment the tool runs from, and the +crash shows that change rather than the line. poetry's documents turn off virtualenv +creation, and a later `poetry sync --only dev` then removed poetry's own dependencies from +the environment it ran in, after which every poetry line crashed on a missing module. A +crash the tool survives still convicts, and a plain nonzero exit is never read this way. + A file a line reads that the planner sees no earlier line create no longer skips the line at plan time. The plan records it as the line's need, and the session checks whether the file exists just before the line runs. A line that passes without the file is classified @@ -311,6 +347,13 @@ lines stays several reports, and an operand ending in a script extension such as `.sh` is the program an interpreter runs rather than data, so two runs of one script never pair. Like the other causal downgrades it suppresses the cascade without claiming cause. +A request that fails because a URL's host did not resolve is read the same way when an +earlier line of the same document named that host as a word of its own and was skipped, +blocked, or a gap. tailscale's Kubernetes guide runs `curl http://nginx` against a service +its skipped `kubectl` and `make` lines set up, so the name has nothing to resolve to. A +host named only by lines that passed, or by none, keeps the `FAIL`, since a dead domain in +a document is a real break. + A titled fence whose title is a plain relative file path is a file the document writes. The planner turns it into a line that writes the block's content, so a later line reading that file does not need it, and a path the checkout already holds is never overwritten. A caption @@ -407,10 +450,18 @@ check mode for that document. The inference is by subcommand, never by binary, s linter run over a bad configuration exits nonzero too and blessing the binary would turn that into a hollow pass. A flag the page says implies one of the check-mode flags, as in "`--format sarif` implies `--fix=false`", is a check-mode flag for that document, read with -its value as written, and an explicit `--fix` beside it turns fixing back on. A nonzero -exit from either is the tool working. A crash stays a +its value as written, and an explicit `--fix` beside it turns fixing back on. A flag that +sets the exit code for a finding, written `--exit-` and a name with a nonzero number as +its value, as in trivy's `--exit-code 1` and `--exit-on-eol 1`, asks for that exit in so +many words and is read the same way. A nonzero +exit from any of these is the tool working. A crash stays a failure, and the classifier now enforces it: a check-mode line whose output holds a -traceback or a panic falls through to the crash rules. Output the document shows under a +traceback or a panic falls through to the crash rules. A crash is read only from lines the +tool printed bare. A runtime writes its panic or traceback straight to the terminal, so a +crash word on a line that opens with a vertical bar, a table row or a quoted block, is the +tool showing something it read: a vulnerability table whose cell says "Segmentation fault +in SSL_check_chain", or a package manager quoting the traceback of a build it ran before +reporting the failure in its own words. Output the document shows under a command as a comment, which the tool then printed, is documented behavior as well: a nonzero ordinary exit whose output contains a shown line of at least twelve characters, with or without its leading program prefix, is `VERIFIED`, and the match needs the tool to diff --git a/examplerun.go b/examplerun.go index 382e8c7..f393c61 100644 --- a/examplerun.go +++ b/examplerun.go @@ -124,7 +124,8 @@ func (d *DockerRunner) runExample(ctx context.Context, step InstallStep) Result cmd.Cancel = sandbox.RemoveFunc(cmd, name) // Decided before the container starts, by a stat-only walk, so a repository // that cannot be streamed whole never gets a partial one. - if !repoFits(step.dir) { + spec := sessionSpec(plan, step.dir) + if !treeFits(step.dir, spec) { return Result{ Step: step, Status: StatusError, Duration: time.Since(start), Detail: fmt.Sprintf("working tree exceeds the %d MB kibble will stream, so the examples have no verdict", @@ -135,7 +136,7 @@ func (d *DockerRunner) runExample(ctx context.Context, step InstallStep) Result // kept the cap small enough to exclude ordinary projects. pr, pw := io.Pipe() go func() { - _, err := repoTarTo(pw, step.dir) + _, err := treeTarTo(pw, step.dir, spec) _ = pw.CloseWithError(err) }() defer func() { _ = pr.Close() }() @@ -218,6 +219,8 @@ type treeSpec struct { skip map[string]bool // maxFile is the size above which a file is left out, 0 for no limit. maxFile int64 + // whole reports whether a file git tracks is carried whatever its size. + whole bool // dirs reports whether directories get entries of their own, so that an // empty one, such as a git ref directory, exists after extraction. dirs bool @@ -227,27 +230,74 @@ type treeSpec struct { } // leftOut reports whether the entry at path, under the walked directory root, -// is outside what the spec carries. -func (s treeSpec) leftOut(root, path string, d fs.DirEntry) bool { - if d.IsDir() && s.skip[d.Name()] { - return true - } - if s.only == nil || path == root { +// is outside what the spec carries. kept holds what git tracks, whose +// directories are carried whatever they are named. The root itself is always +// carried: a checkout cloned into a directory called build is still the +// checkout. +func (s treeSpec) leftOut(root, path string, d fs.DirEntry, kept trackedSet) bool { + if path == root { return false } rel, err := filepath.Rel(root, path) if err != nil { return true } - top, _, _ := strings.Cut(filepath.ToSlash(rel), "/") + rel = filepath.ToSlash(rel) + if d.IsDir() && s.skip[d.Name()] && !kept.dirs[rel] { + return true + } + if s.only == nil { + return false + } + top, _, _ := strings.Cut(rel, "/") return !s.only[top] } +// kept returns what git tracks under dir, for a spec with a skip list. Such +// a spec carries a skip-named directory anyway when git tracks files in it. +// The skip list names what a build produces, and a name is only a guess: +// goreleaser keeps source in internal/pipe/dist and k6 in internal/build, and +// leaving those out made the tool unbuildable in the session. What git tracks +// is source by definition. +func (s treeSpec) kept(dir string) trackedSet { + if len(s.skip) == 0 || s.sub != "" { + return trackedSet{} + } + return tracked(dir) +} + +// tooBig reports whether the spec leaves a file out for its size. rel is the +// file's slash-separated path under the checkout. +func (s treeSpec) tooBig(rel string, size int64, kept trackedSet) bool { + if s.maxFile <= 0 || size <= s.maxFile { + return false + } + return !s.whole || !kept.files[rel] +} + // sessionTree is what an example session receives: the working tree without // generated directories and without files over 2 MB, so documented example // files exist in the container. var sessionTree = treeSpec{skip: tarSkipDirs, maxFile: 2 << 20} +// checkoutTree is what a session that builds the tool from the checkout +// receives: the session tree, with every file git tracks whatever its size. +// The size limit keeps stray output out of the budget, and a build needs the +// source whole: k6 vendors a 4.9 MB table of golang.org/x/text and does not +// compile without it. +var checkoutTree = treeSpec{skip: tarSkipDirs, maxFile: 2 << 20, whole: true} + +// sessionSpec returns the tree an example session of plan receives from dir. +// A session that builds from the checkout takes the tracked files whole when +// that fits under [repoStreamCap], and the plain session tree otherwise, so a +// repository with heavy tracked assets still gets a session. +func sessionSpec(plan *kplan.Plan, dir string) treeSpec { + if fromCheckout(plan) && treeFits(dir, checkoutTree) { + return checkoutTree + } + return sessionTree +} + // gitTree is what a source build receives: the objects and refs of the // checkout's git directory, from which the container restores the committed // tree. Nothing else in that directory is sent. Its config is where a CI @@ -275,11 +325,12 @@ func treeFits(dir string, spec treeSpec) bool { total := int64(0) fits := true root := filepath.Join(dir, spec.sub) + kept := spec.kept(dir) _ = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil { return nil } - if spec.leftOut(root, path, d) { + if spec.leftOut(root, path, d, kept) { if d.IsDir() { return filepath.SkipDir } @@ -292,7 +343,8 @@ func treeFits(dir string, spec treeSpec) bool { return nil } info, err := d.Info() - if err != nil || (spec.maxFile > 0 && info.Size() > spec.maxFile) { + rel, rerr := filepath.Rel(dir, path) + if err != nil || rerr != nil || spec.tooBig(filepath.ToSlash(rel), info.Size(), kept) { return nil } if total += info.Size(); total > repoStreamCap { @@ -329,6 +381,7 @@ func treeTarTo(w io.Writer, dir string, spec treeSpec) (bool, error) { tw := tar.NewWriter(w) total := int64(0) root := filepath.Join(dir, spec.sub) + kept := spec.kept(dir) _ = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil { return nil @@ -337,7 +390,7 @@ func treeTarTo(w io.Writer, dir string, spec treeSpec) (bool, error) { if rerr != nil { return nil } - if spec.leftOut(root, path, d) { + if spec.leftOut(root, path, d, kept) { if d.IsDir() { return filepath.SkipDir } @@ -357,7 +410,7 @@ func treeTarTo(w io.Writer, dir string, spec treeSpec) (bool, error) { return nil } info, err := d.Info() - if err != nil || (spec.maxFile > 0 && info.Size() > spec.maxFile) { + if err != nil || spec.tooBig(filepath.ToSlash(rel), info.Size(), kept) { return nil } if total += info.Size(); total > repoStreamCap { @@ -411,7 +464,8 @@ func (zeroReader) Read(p []byte) (int, error) { // tarSkipDirs are directories a build produces or a package manager fills. // Streaming them wastes the budget on output nobody documents, and on a big -// repository it pushes the source a document needs past the cap. +// repository it pushes the source a document needs past the cap. One that +// holds files git tracks is carried all the same, see kept. var tarSkipDirs = map[string]bool{ ".git": true, "node_modules": true, "vendor": true, "dist": true, "build": true, "target": true, "out": true, ".gradle": true, ".idea": true, diff --git a/examplerun_test.go b/examplerun_test.go index e7609b3..f68e071 100644 --- a/examplerun_test.go +++ b/examplerun_test.go @@ -5,6 +5,7 @@ import ( "bytes" "fmt" "os" + "os/exec" "path/filepath" "sort" "strings" @@ -668,6 +669,145 @@ func TestRepoTar(t *testing.T) { } } +// TestRepoTarKeepsTrackedDirs checks that a directory named like build output +// is streamed when git tracks files in it, since goreleaser keeps source in +// internal/pipe/dist and k6 in internal/build, and that the same names stay out +// when nothing in them is tracked. A checkout cloned into a directory that is +// itself named like build output is still streamed. +func TestRepoTarKeepsTrackedDirs(t *testing.T) { + t.Parallel() + dir := filepath.Join(t.TempDir(), "build") + tracked := map[string]string{ + "main.go": "package main", + "internal/build/build.go": "package build", + "internal/pipe/dist/d.go": "package dist", + "vendor/dep/dep.go": "package dep", + "docs/guide.md": "# G", + "internal/build/sub/s.txt": "s", + } + untracked := map[string]string{ + "dist/app.tar.gz": "bin", + "node_modules/pkg/index.js": "x", + "cmd/tool/build/out.o": "obj", + "internal/build/cache.bin": "generated beside tracked source", + } + write := func(files map[string]string) { + for name, body := range files { + full := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(full), 0o750); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(full, []byte(body), 0o600); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + } + write(tracked) + for _, args := range [][]string{ + {"init", "-q"}, {"add", "-A"}, + {"-c", "user.name=t", "-c", "user.email=t@example.invalid", "commit", "-qm", "init"}, + } { + if out, err := exec.Command("git", append([]string{"-C", dir}, args...)...).CombinedOutput(); err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } + } + write(untracked) + names := func() []string { + data, _ := repoTar(dir) + var got []string + tr := tar.NewReader(bytes.NewReader(data)) + for { + h, err := tr.Next() + if err != nil { + break + } + got = append(got, h.Name) + } + sort.Strings(got) + return got + } + want := []string{ + "docs/guide.md", "internal/build/build.go", "internal/build/cache.bin", + "internal/build/sub/s.txt", "internal/pipe/dist/d.go", "main.go", "vendor/dep/dep.go", + } + if diff := cmp.Diff(want, names()); diff != "" { + t.Errorf("streamed files mismatch (-want +got):\n%s", diff) + } + if !repoFits(dir) { + t.Error("small tree reported as not fitting") + } + // Without an index there is nothing to say a name is source, so the skip + // list decides alone, as it does for a source tarball. + if err := os.RemoveAll(filepath.Join(dir, ".git")); err != nil { + t.Fatal(err) + } + if diff := cmp.Diff([]string{"docs/guide.md", "main.go"}, names()); diff != "" { + t.Errorf("streamed files without an index mismatch (-want +got):\n%s", diff) + } +} + +// TestCheckoutTreeCarriesTrackedFilesWhole checks that a session building from +// the checkout receives a tracked file over the size limit, since k6 vendors a +// 4.9 MB source table and does not compile without it, that an untracked file +// of the same size stays out, and that a session with a documented install +// keeps the limit for both. +func TestCheckoutTreeCarriesTrackedFilesWhole(t *testing.T) { + t.Parallel() + dir := t.TempDir() + big := bytes.Repeat([]byte("x"), 3<<20) + write := func(name string, body []byte) { + full := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(full), 0o750); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(full, body, 0o600); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + write("main.go", []byte("package main")) + write("vendor/dep/tables.go", big) + for _, args := range [][]string{ + {"init", "-q"}, {"add", "-A"}, + {"-c", "user.name=t", "-c", "user.email=t@example.invalid", "commit", "-qm", "init"}, + } { + if out, err := exec.Command("git", append([]string{"-C", dir}, args...)...).CombinedOutput(); err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } + } + write("core.dump", big) + built := &kplan.Plan{Installs: []kplan.PlanInstall{{FromCheckout: true}}} + tests := []struct { + Plan *kplan.Plan + WantNames []string + }{{ // Test 0: a checkout build takes the tracked file whole and no stray one. + Plan: built, WantNames: []string{"main.go", "vendor/dep/tables.go"}, + }, { // Test 1: a documented install keeps the size limit. + Plan: &kplan.Plan{Installs: []kplan.PlanInstall{{}}}, WantNames: []string{"main.go"}, + }} + for testNum, test := range tests { + t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) { + t.Parallel() + var buf bytes.Buffer + if _, err := treeTarTo(&buf, dir, sessionSpec(test.Plan, dir)); err != nil { + t.Fatalf("treeTarTo: %v", err) + } + var got []string + tr := tar.NewReader(&buf) + for { + h, err := tr.Next() + if err != nil { + break + } + got = append(got, h.Name) + } + sort.Strings(got) + if diff := cmp.Diff(test.WantNames, got); diff != "" { + t.Errorf("streamed files mismatch (-want +got):\n%s", diff) + } + }) + } +} + // TestEmptyPipelineSkips checks that a consumer reporting empty input reads // as the search matching nothing, not as the document being wrong. The exit // code it arrives with depends only on which stage the pipeline reports, so diff --git a/go.mod b/go.mod index 5c77b9c..27a42d7 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,12 @@ require ( require gopkg.in/yaml.v3 v3.0.1 -require mvdan.cc/sh/v3 v3.14.1 +require ( + github.com/go-git/go-git/v5 v5.19.3 + mvdan.cc/sh/v3 v3.14.1 +) + +require github.com/pjbgf/sha1cd v0.7.0 // indirect require ( github.com/google/jsonschema-go v0.4.3 // indirect diff --git a/go.sum b/go.sum index b692e41..460a8da 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,13 @@ +github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= +github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-git/go-billy/v5 v5.9.2 h1:OXFSRyz4g20upsGDJgQG9Bak1l/ZEv8GHVYB52O71sE= +github.com/go-git/go-billy/v5 v5.9.2/go.mod h1:ExsU+jcGwXTBOnyilvAnEM1wug1IxHr4yP2ZXsNRtV0= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git/v5 v5.19.3 h1:qHttbZ+Am7wEjdTpR1XMQ4rl42FK9SIXzZ4o9x7s6M4= +github.com/go-git/go-git/v5 v5.19.3/go.mod h1:Ye4C8dVigkqrv9UsB4NMdSLV4yAIkLiIhW61gcOyhl4= github.com/go-quicktest/qt v1.102.0 h1:HSQxCeh5YZH3EL3W39ixjtyaEhcWSXQHtHnMBzSs474= github.com/go-quicktest/qt v1.102.0/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= @@ -12,12 +22,18 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/modelcontextprotocol/go-sdk v1.7.0 h1:yqjY2dsbKAC0LSuWZVBMrHgiG8ukXv6NRo0JiALay44= github.com/modelcontextprotocol/go-sdk v1.7.0/go.mod h1:dL7u98E/zjJTGzEq+j30jQ8K2k1mb6LeAH4inEcSGts= +github.com/pjbgf/sha1cd v0.7.0 h1:ZRNPKHj+gfkLBf0KJv/p1Hmz+7bqCT5o311YX0nq+DA= +github.com/pjbgf/sha1cd v0.7.0/go.mod h1:pKR5Li+qTCo+ebqITZfwPlJGoCFCvSO00qxjbNtTUFQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0= github.com/segmentio/encoding v0.5.4/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE= @@ -32,8 +48,9 @@ golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= mvdan.cc/sh/v3 v3.14.1 h1:bXkhQWNHCs0KZEChF8hYS6FC+T2N9mUZLbQv9blditI= diff --git a/internal/docblock/docblock.go b/internal/docblock/docblock.go index bb20fb3..b224a47 100644 --- a/internal/docblock/docblock.go +++ b/internal/docblock/docblock.go @@ -54,12 +54,21 @@ type Block struct { // Item is the 1-based index of the outermost list's item the block sits // in, and is 0 outside a list. Item int + // Details numbers the outermost HTML details element the block sits in, + // in document order from 1, and is 0 outside one. A README offers + // alternative setups as collapsed details under one heading, and each is + // its own section to a reader who opens only one. + Details int } // reFenceTitle matches a title attribute in a fence's info string, in double // or single quotes. var reFenceTitle = regexp.MustCompile(`\btitle=(?:"([^"]*)"|'([^']*)')`) +// reDetailsTag matches an opening or closing HTML details tag and captures +// the slash of a closing one. +var reDetailsTag = regexp.MustCompile(`(?i)<(/?)details\b`) + // admonitionMarker matches the first line of a MkDocs admonition, whose indented // body is prose that markdown parsers read as an indented code block. var admonitionMarker = regexp.MustCompile(`^(!!!|\?\?\?\+?)\s`) @@ -94,7 +103,24 @@ func CodeBlocks(markdown string) []Block { var heading, intro, lead string leadEnd := 0 lists, depth, list, item := 0, 0, 0, 0 + detailsSeen, detailsDepth, details := 0, 0, 0 _ = ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + if h, ok := n.(*ast.HTMLBlock); ok && entering { + for _, m := range reDetailsTag.FindAllStringSubmatch(htmlBlockText(h, src), -1) { + switch { + case m[1] == "": + if detailsDepth == 0 { + detailsSeen++ + details = detailsSeen + } + detailsDepth++ + case detailsDepth > 0: + if detailsDepth--; detailsDepth == 0 { + details = 0 + } + } + } + } switch n.(type) { case *ast.List: if entering { @@ -122,7 +148,7 @@ func CodeBlocks(markdown string) []Block { } return ast.WalkContinue, nil } - b := Block{Heading: heading, Intro: intro, List: list, Item: item} + b := Block{Heading: heading, Intro: intro, List: list, Item: item, Details: details} switch node := n.(type) { case *ast.Heading: heading = spanText(node, src) @@ -147,7 +173,11 @@ func CodeBlocks(markdown string) []Block { case *ast.CodeBlock: lead = "" if admonitionMarker.MatchString(intro) { - blocks = append(blocks, admonitionBlocks(node, src, heading, intro)...) + nested := admonitionBlocks(node, src, heading, intro) + for i := range nested { + nested[i].Details = details + } + blocks = append(blocks, nested...) return ast.WalkContinue, nil } b.Lines = strings.Split(BlockText(node, src), "\n") @@ -165,6 +195,20 @@ func CodeBlocks(markdown string) []Block { return blocks } +// htmlBlockText returns the raw text of an HTML block, its closing line +// included. +func htmlBlockText(h *ast.HTMLBlock, src []byte) string { + var b strings.Builder + for i := 0; i < h.Lines().Len(); i++ { + seg := h.Lines().At(i) + b.Write(seg.Value(src)) + } + if h.HasClosure() { + b.Write(h.ClosureLine.Value(src)) + } + return b.String() +} + // admonitionBlocks parses an admonition body as markdown and returns the code // blocks nested in it, with lines mapped back to the outer source and the outer // heading and intro applied where the nested block has none. diff --git a/internal/docblock/docblock_test.go b/internal/docblock/docblock_test.go index ff72a39..678443a 100644 --- a/internal/docblock/docblock_test.go +++ b/internal/docblock/docblock_test.go @@ -366,3 +366,31 @@ func TestCodeBlocksList(t *testing.T) { t.Errorf("list positions mismatch (-want +got):\n%s", diff) } } + +// TestCodeBlocksDetails checks that each block carries the number of the +// outermost details element it sits in, that a nested details element keeps +// its parent's number, and that a block after the close is outside any. +func TestCodeBlocksDetails(t *testing.T) { + t.Parallel() + in := "`make`\n\n" + + "
\nTemplate\n\n```bash\ncd glance\n```\n\n" + + "
\nMore\n\n```bash\nvi glance.yml\n```\n\n
\n\n" + + "```bash\ndocker compose up\n```\n\n
\n\n" + + "
\nManual\n\n```bash\nmkdir config\n```\n\n
\n\n" + + "```bash\ngo build .\n```\n" + type pos struct { + Text string + Details int + } + want := []pos{ + {"make", 0}, {"cd glance", 1}, {"vi glance.yml", 1}, {"docker compose up", 1}, + {"mkdir config", 2}, {"go build .", 0}, + } + var got []pos + for _, b := range CodeBlocks(in) { + got = append(got, pos{strings.TrimSpace(strings.Join(b.Lines, " ")), b.Details}) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("details positions mismatch (-want +got):\n%s", diff) + } +} diff --git a/internal/plan/plan.go b/internal/plan/plan.go index 7f8cc32..579996f 100644 --- a/internal/plan/plan.go +++ b/internal/plan/plan.go @@ -46,6 +46,11 @@ type Plan struct { // nothing, such as witr's "2 not found". A documented binary exiting with // one of them reports an empty result rather than a broken command. NotFoundCodes []int `json:"notFoundCodes,omitempty"` + // Unrun are the commands of code blocks the plan leaves out, of two words + // or more. A failure that tells the reader to run one of them first, as + // headscale's Makefile says to run `nix develop`, failed for want of a + // step the documents give and the session could not take. + Unrun []string `json:"unrun,omitempty"` // RootGoFiles are the Go files at the top of the checkout before the // session starts. A Go build that finds a document's file beside one of // them met the repository's own package, which the document never wrote. @@ -68,6 +73,10 @@ type PlanInstall struct { Binary string // Bootstrap installs the package manager itself when the image lacks it. Bootstrap string + // FromCheckout marks an install kibble made up because no documented one + // gave the session a binary: the tool built from the checkout. Examples + // run against it say so, and the documented install stays untested. + FromCheckout bool `json:"fromCheckout,omitempty"` } // PlanStep is one documented code block prepared for execution. @@ -83,6 +92,10 @@ type PlanStep struct { Doc string `json:"doc,omitempty"` // Heading is the section heading the block appears under. Heading string `json:"heading,omitempty"` + // Details numbers the outermost HTML details element the block sits in, + // 0 outside one. Collapsed details under one heading are separate + // sections, often alternative setups. + Details int `json:"details,omitempty"` // Lines are the logical shell lines of the block. Lines []PlanLine `json:"lines"` // Background runs the step behind the session and kills it at the end. @@ -411,6 +424,7 @@ func (pl *planner) addBlock(block docblock.Block) { } if !pl.qualifies(lines) { pl.plan.Excluded++ + pl.noteUnrun(lines) if pl.unplanned == "" && !packageManagerOnly(lines) { pl.unplanned = blockLocation(block) } @@ -419,6 +433,7 @@ func (pl *planner) addBlock(block docblock.Block) { step := PlanStep{ ID: fmt.Sprintf("b%d", len(pl.plan.Steps)+1), Heading: block.Heading, + Details: block.Details, } scopedTo := otherPlatform(block.Intro) if scopedTo == "" { @@ -763,6 +778,27 @@ func (pl *planner) addFileBlock(block docblock.Block, path string) { pl.created[path] = true } +// maxUnrun bounds the commands recorded from blocks the plan leaves out. +const maxUnrun = 200 + +// noteUnrun records the commands of a block the plan leaves out. A one-word +// command is not recorded, since a bare name such as make turns up in a +// tool's output for reasons that have nothing to do with a missing step, and +// neither is a line holding a placeholder. +func (pl *planner) noteUnrun(lines []string) { + for _, ln := range lines { + if len(pl.plan.Unrun) >= maxUnrun { + return + } + flat := strings.TrimSpace(docblock.StripComment(docblock.Flatten(ln))) + if !strings.Contains(flat, " ") || len(flat) > 80 || docblock.HasPlaceholder(flat) || + slices.Contains(pl.plan.Unrun, flat) { + continue + } + pl.plan.Unrun = append(pl.plan.Unrun, flat) + } +} + // resolvedToolPaths returns a block's raw lines with a stand-in path to a // documented binary in command position replaced by the binary, so // `path/to/pyenv --install bash` plans as the pyenv line it documents. The @@ -881,6 +917,9 @@ func (p *Plan) Merge(doc string, other *Plan) { } slices.Sort(p.NotFoundCodes) p.RootGoFiles = mergeUnique(p.RootGoFiles, other.RootGoFiles) + if p.Unrun = mergeUnique(p.Unrun, other.Unrun); len(p.Unrun) > maxUnrun { + p.Unrun = p.Unrun[:maxUnrun] + } if p.Env == nil && len(other.Env) > 0 { p.Env = map[string]string{} } diff --git a/internal/plan/plan_test.go b/internal/plan/plan_test.go index ee01480..7b9a24a 100644 --- a/internal/plan/plan_test.go +++ b/internal/plan/plan_test.go @@ -553,6 +553,16 @@ func TestFindingFlagNonzeroOK(t *testing.T) { }, { // Test 5: a flag marks its own line and does not spread. Markdown: "```sh\ntool run --check\n```\ntext\n```sh\ntool run\n```\n", Binary: "tool", Want: []bool{true, false}, + }, { // Test 6: a flag that sets a nonzero exit on a finding asks for that exit. + Markdown: "```sh\ntool image --exit-code 1 alpine\n```\n", Binary: "tool", Want: []bool{true}, + }, { // Test 7: the same flag written with an equals sign counts. + Markdown: "```sh\ntool image --exit-on-eol=2 alpine\n```\n", Binary: "tool", Want: []bool{true}, + }, { // Test 8: an exit code of zero asks for nothing. + Markdown: "```sh\ntool image --exit-code 0 alpine\n```\n", Binary: "tool", Want: []bool{false}, + }, { // Test 9: a flag of the family with no number asks for nothing. + Markdown: "```sh\ntool scan --exit-zero src\n```\n", Binary: "tool", Want: []bool{false}, + }, { // Test 10: a number that belongs to another flag is not the exit code. + Markdown: "```sh\ntool image --exit-code=0 --depth 1 alpine\n```\n", Binary: "tool", Want: []bool{false}, }} for testNum, test := range tests { t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) { @@ -572,6 +582,34 @@ func TestFindingFlagNonzeroOK(t *testing.T) { } } +// TestPlanRecordsUnrunBlocks checks that the commands of a block the plan +// leaves out are recorded, so a failure asking for one can be read as a step +// the session could not take, and that a block the plan runs records nothing. +func TestPlanRecordsUnrunBlocks(t *testing.T) { + t.Parallel() + tests := []struct { + Markdown string + Want []string + }{{ // Test 0: a block of a command kibble does not know is recorded. + Markdown: "```sh\nnix develop\n```\n\n```sh\ntool run\n```\n", Want: []string{"nix develop"}, + }, { // Test 1: a one-word command, a placeholder line, and a repeat are not. + Markdown: "```sh\nnixshell\nnix develop \nnix develop # enter the shell\n```\n\n" + + "```sh\nnix develop\n```\n", + Want: []string{"nix develop"}, + }, { // Test 2: a block the plan runs records nothing. + Markdown: "```sh\ntool run --fast\n```\n", Want: nil, + }} + for testNum, test := range tests { + t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) { + t.Parallel() + plan := BuildPlan("repo", "", test.Markdown, []string{"tool"}, nil, nil, testRecognizer()) + if diff := cmp.Diff(test.Want, plan.Unrun, cmpopts.EquateEmpty()); diff != "" { + t.Errorf("unrun mismatch (-want +got):\n%s", diff) + } + }) + } +} + // TestWatcherInvocation checks that a tool described as a watcher skips only // the invocations that actually reach its watching or serving mode, so a // one-shot subcommand of the same tool still runs. @@ -944,6 +982,20 @@ func TestSkipReasonReaderFile(t *testing.T) { } } +// TestMergeKeepsUnrun checks that the commands of blocks a document's plan +// left out survive the merge into the session plan, once each, since a +// failure in one document may name a command only another one shows. +func TestMergeKeepsUnrun(t *testing.T) { + t.Parallel() + merged := &Plan{} + merged.Merge("README.md", &Plan{Unrun: []string{"nix develop"}}) + merged.Merge("docs/dev.md", &Plan{Unrun: []string{"nix develop", "direnv allow"}}) + merged.Merge("docs/none.md", &Plan{}) + if diff := cmp.Diff([]string{"nix develop", "direnv allow"}, merged.Unrun); diff != "" { + t.Errorf("unrun mismatch (-want +got):\n%s", diff) + } +} + // TestReaderNamedOperand checks the operands the reader is expected to // replace, and the ones that are real: example values such as foo, file // names, output paths, flags, and names the repo provides. @@ -976,6 +1028,10 @@ func TestReaderNamedOperand(t *testing.T) { Tree: map[string]bool{"my-app": true}, Line: `tool my-app`, WantResult: "", }, { // Test 10: an ordinary word is not a stand-in. Line: `tool build`, WantResult: "", + }, { // Test 11: a your-prefixed name is a stand-in. + Line: `trivy plugin run kubectl pod your-pod -- --exit-code 1`, WantResult: "your-pod", + }, { // Test 12: a word that only starts with your is not one. + Line: `tool greet yourself`, WantResult: "", }} for testNum, test := range tests { diff --git a/internal/plan/planskip.go b/internal/plan/planskip.go index d5f9073..d1237d7 100644 --- a/internal/plan/planskip.go +++ b/internal/plan/planskip.go @@ -374,12 +374,13 @@ func firstWord(flat string) string { } // reReaderNamed matches an operand docs use as a stand-in name the reader -// replaces: a my-prefixed name such as my-project, or a noun with a number -// such as repo1. Both are a naming convention that says "yours goes here". +// replaces: a my-prefixed name such as my-project, a your-prefixed one such +// as your-pod, or a noun with a number such as repo1. Each is a naming +// convention that says "yours goes here". // Words such as foo are not in the class, since docs use them as real example // values, and a token with an extension is not either. var reReaderNamed = regexp.MustCompile( - `^(my[-_]?[\w-]+|(repo|file|dir|group|project|plugin|package|app)\d+)$`) + `^(my[-_]?[\w-]+|your[-_][\w-]+|(repo|file|dir|group|project|plugin|package|app)\d+)$`) // readerNamedOperand returns the first operand of a line that reads as a // name the reader supplies, or empty when none does. A name the repo or an @@ -442,6 +443,13 @@ func (pl *planner) findingFlagged(flat string) bool { if findingFlags[f] { return true } + next := "" + if i+1 < len(words) { + next = words[i+1] + } + if setsFindingExit(f, next) { + return true + } if fixFlags[f] { return false } @@ -455,6 +463,28 @@ func (pl *planner) findingFlagged(flat string) bool { return implied } +// reExitFlag matches a flag that sets the code a tool exits with when it finds +// something, such as trivy's --exit-code and --exit-on-eol, and captures a +// value written after an equals sign. +var reExitFlag = regexp.MustCompile(`^--exit-[a-z][a-z-]*(?:=(\d+))?$`) + +// setsFindingExit reports whether a flag, with the word after it, tells the +// tool to exit with a nonzero code on a finding. `trivy image --exit-code 1` +// asks for exit 1 when it finds a vulnerability, so that exit is the line +// working. A flag of the same family with no number, or with zero, asks for +// nothing of the kind. +func setsFindingExit(flag, next string) bool { + m := reExitFlag.FindStringSubmatch(flag) + if m == nil { + return false + } + value := m[1] + if value == "" && !strings.Contains(flag, "=") { + value = next + } + return value != "" && strings.Trim(value, "0123456789") == "" && strings.Trim(value, "0") != "" +} + // fixFlags are flags that turn fixing back on, so a line carrying one next to // a flag the document says implies check mode is not check mode after all. var fixFlags = map[string]bool{"--fix": true, "--fix=true": true} diff --git a/internal/shell/shell.go b/internal/shell/shell.go index 21fa157..02ac42a 100644 --- a/internal/shell/shell.go +++ b/internal/shell/shell.go @@ -266,6 +266,39 @@ func ArgWords(cmd string) ([]string, bool) { return out, true } +// Writes returns the files a line redirects output into, as written, wherever +// in the line the redirect sits. A target whose name depends on an expansion +// is left out, since its real value is unknown until it runs. +func Writes(cmd string) []string { + parser := syntax.NewParser(syntax.KeepComments(false), syntax.Variant(syntax.LangBash)) + file, err := parser.Parse(strings.NewReader(cmd), "") + if err != nil { + return nil + } + var out []string + syntax.Walk(file, func(node syntax.Node) bool { + stmt, ok := node.(*syntax.Stmt) + if !ok { + return true + } + for _, r := range stmt.Redirs { + switch r.Op { + case syntax.RdrOut, syntax.AppOut, syntax.RdrAll, syntax.AppAll, syntax.RdrClob: + default: + continue + } + if r.Word == nil { + continue + } + if text, literal := wordText(r.Word); literal && text != "" { + out = append(out, text) + } + } + return true + }) + return out +} + // Operands returns every word of a line that is not a command name, so // a rule about a line's arguments never trips over the program itself. A // tool named `pattern` is the command in `pattern build` and a placeholder in diff --git a/internal/shell/shell_test.go b/internal/shell/shell_test.go index 6359262..597a6c0 100644 --- a/internal/shell/shell_test.go +++ b/internal/shell/shell_test.go @@ -164,3 +164,32 @@ func TestMatchesWords(t *testing.T) { }) } } + +// TestWrites checks which files a line is read as redirecting output into. +func TestWrites(t *testing.T) { + t.Parallel() + tests := []struct { + In string + WantResult []string + }{{ // Test 0: A plain redirect names its target. + In: "jq . in.json > out.json", WantResult: []string{"out.json"}, + }, { // Test 1: An append and a redirect inside a pipeline both count. + In: "tool list >> all.txt | sort > sorted.txt", WantResult: []string{"all.txt", "sorted.txt"}, + }, { // Test 2: An input redirect writes nothing. + In: "tool load < in.json", WantResult: nil, + }, { // Test 3: A target that depends on an expansion is unknown. + In: `tool dump > "$(tool path)"`, WantResult: nil, + }, { // Test 4: A greater-than sign inside quotes is not a redirect. + In: `tool query "a > b"`, WantResult: nil, + }, { // Test 5: A line bash cannot parse writes nothing kibble can name. + In: "tool 'unterminated > x", WantResult: nil, + }} + for testNum, test := range tests { + t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) { + t.Parallel() + if diff := cmp.Diff(test.WantResult, Writes(test.In), cmpopts.EquateEmpty()); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/internal/verdict/verdict.go b/internal/verdict/verdict.go index 25b53e5..979cc21 100644 --- a/internal/verdict/verdict.go +++ b/internal/verdict/verdict.go @@ -167,4 +167,9 @@ const ( // exists, a project manager read the checkout's project, or a Go build // found a file the document wrote beside the checkout's own package. ReasonCheckoutState Reason = "checkout-state" + // ReasonEnvironmentChanged marks a crash after which the documented tool no + // longer starts, though it started when the session began. Something the + // session ran changed the environment the tool runs from, so the crash says + // nothing about the line as written. + ReasonEnvironmentChanged Reason = "environment-changed" ) diff --git a/lineverdict.go b/lineverdict.go index 9f8af3f..77bd7fd 100644 --- a/lineverdict.go +++ b/lineverdict.go @@ -73,7 +73,15 @@ func classifyLineResult(lr lineResult, l kplan.PlanLine, o lineOutcome, wrapped lr.Detail = "ran against " + strings.Join(lr.Synthetic, ", ") + ", which kibble fabricated because no documented step creates it" } - case l.NonzeroOK && documentedNonzeroCode(o.code) && !reCrash.MatchString(o.output): + case rePlatformGone.MatchString(o.output): + // Read before a documented exit is honored: a tool that found nothing + // to run on this platform never did the work whose exit the document + // describes. The words are the tool's own, so they excuse nothing. + lr.Status = StatusBlocked + lr.Reason = ReasonOtherPlatform + lr.Detail = "found nothing for this platform, which the container's architecture " + + "may explain or the document may misname: " + tail + case l.NonzeroOK && documentedNonzeroCode(o.code) && !crashed(o.output): // A crash stays a failure even where the document blesses a nonzero exit. lr.Status = StatusVerified lr.Detail = fmt.Sprintf("exit %d is documented behavior", o.code) @@ -83,11 +91,11 @@ func classifyLineResult(lr lineResult, l kplan.PlanLine, o lineOutcome, wrapped lr.Status = StatusVerified lr.Detail = fmt.Sprintf("exit %d is documented behavior: the document shows this output", o.code) - case reCrash.MatchString(o.output) && crashWantsInput(o.output): + case crashed(o.output) && crashWantsInput(o.output): lr.Status = StatusSkipped lr.Reason = ReasonInteractive lr.Detail = "waited for input, and the session's stdin is closed: " + tail - case reCrash.MatchString(o.output) && crashMissingName(o.output) != "": + case crashed(o.output) && crashMissingName(o.output) != "": // The one crash that is not unexplained. A traceback whose last line // names the file or program that was missing is the tool saying what // it lacked, so it is settled the way a plain exit naming it would be. @@ -95,7 +103,17 @@ func classifyLineResult(lr lineResult, l kplan.PlanLine, o lineOutcome, wrapped // traceback did not end the process, and a crash ending any other way // still convicts below. lr = settleMissing(lr, crashMissingName(o.output), tail, o) - case reCrash.MatchString(o.output): + case crashed(o.output) && o.toolGone: + // The tool started when the session began and no longer does, so the + // session changed what it runs from, and this crash is that change + // showing rather than the line as written. A crash the tool survives + // still convicts below. + lr.Status = StatusBlocked + lr.Reason = ReasonEnvironmentChanged + lr.Detail = fmt.Sprintf("crashed with exit %d, and afterwards the tool no longer starts, "+ + "though it did when the session began, so something the session ran changed its "+ + "environment: %s", o.code, tail) + case crashed(o.output): // Checked before every excuse: a process that died did not report a // condition kibble can forgive, whatever else it printed first. lr.Status = StatusFail @@ -374,9 +392,12 @@ var ( `|no .{0,20}configured|please set|must set)\b` + `|\bset [A-Z][A-Z0-9_*]{2,}`) // reNetErr matches errors that mean the command needed a network - // service the container does not run. + // service the container does not run. A client that waits out its own + // deadline says the same thing more slowly: headscale's commands answer + // "context deadline exceeded" when no server is listening. reNetErr = regexp.MustCompile( - `(?i)connection refused|no such host|dial tcp|network is unreachable|could not connect|cannot connect`) + `(?i)connection refused|no such host|dial tcp|network is unreachable|could not connect|cannot connect` + + `|context deadline exceeded|i/o timeout`) // reNoData matches a query that ran correctly and found nothing, which a // fresh session often cannot avoid: the docs query dates and terms that // have no entries yet. @@ -412,9 +433,11 @@ var ( // name after a colon is not one they read. reUnknownNameless = regexp.MustCompile( `(?i)\bunknown (` + registeredNouns + `)\b( +'?[^\s'"]|\s*:\s*'?[^\s'"])?`) - // reRegisteredNotFound matches the same refusal worded as a name not found. + // reRegisteredNotFound matches the same refusal worded as a name not found + // or one that does not exist, in either quote: trivy answers a cluster name + // with `context "cluster" does not exist`. reRegisteredNotFound = regexp.MustCompile( - `(?i)\b(` + registeredNouns + `) '([^']+)' not found`) + `(?i)\b(` + registeredNouns + `) ['"]([^'"]+)['"] (?:not found|does not exist)`) // reNoSuchRegistered matches the same refusal worded as no such noun, with // the name after it optional and optionally quoted. reNoSuchRegistered = regexp.MustCompile( @@ -497,10 +520,13 @@ var ( // installed, such as vhs requiring ffmpeg. The wording is not proof of // the container's gap: a tool says the same words about its own plugins, // extensions, and optional components, which are steps the document owes - // the reader. The line is therefore blocked rather than skipped. + // the reader. The line is therefore blocked rather than skipped. navi + // words it "Please make sure it's correctly installed" about fzf, and + // poetry's installer answers an uninstall with "Poetry is not currently + // installed." when no documented install ran before it. reMissingDep = regexp.MustCompile( - `(?i)\bis not installed\b|\b(please|must) install\b` + - `|\brequires? \S+ to be installed\b`) + `(?i)\bis not (?:currently )?installed\b|\b(please|must) install\b` + + `|\brequires? \S+ to be installed\b|\bmake sure [^.\n]{0,40}\binstalled\b`) // reCrash matches a process dying rather than reporting. A crash convicts // whatever else the output happens to say, so it is checked before any // rule that excuses a line: a tool that panicked after printing "no @@ -509,6 +535,13 @@ var ( `(?im)(^(panic|fatal error):|(^|\s)traceback \(most recent call last\):)` + `|\bsegmentation fault\b|\bassertion failed\b|\bstack overflow\b` + `|\bunhandled exception\b|\bthread '.*' panicked at\b`) + // reFramed matches a line a tool drew a frame around, a table row or a + // quoted block, which opens with a vertical bar after any indentation. + reFramed = regexp.MustCompile(`^\s*[|│┃║]`) + // rePlatformGone matches a tool saying what it was asked to fetch offers + // nothing for the platform it runs on, as trivy says "platform not found" + // of a plugin with no build for the container's architecture. + rePlatformGone = regexp.MustCompile(`(?i)\bplatform not found\b`) // reGitNoHistory matches git reporting that the repository it was handed // has no commits, in the three wordings git uses for an unborn HEAD. reGitNoHistory = regexp.MustCompile( @@ -644,6 +677,25 @@ func reportedMissing(output string) string { return "" } +// crashed reports whether the output holds a process dying in its own words. +// A runtime writes a panic or a traceback bare, so a crash word on a line the +// tool framed is the tool showing something it read, alive and reporting. +// trivy prints "openssl: Segmentation fault in SSL_check_chain" in a table +// cell, and poetry quotes the traceback of a build backend it ran behind a +// bar before saying in its own words that the build failed. +func crashed(output string) bool { + if !reCrash.MatchString(output) { + return false + } + var bare []string + for _, line := range strings.Split(output, "\n") { + if !reFramed.MatchString(line) { + bare = append(bare, line) + } + } + return reCrash.MatchString(strings.Join(bare, "\n")) +} + // crashMissingName returns the file or program a crash's last line names as // missing, or empty when the crash ended any other way. Only the last line // counts, since that is the error that ended the process. diff --git a/lineverdict_reach_test.go b/lineverdict_reach_test.go new file mode 100644 index 0000000..e12ec12 --- /dev/null +++ b/lineverdict_reach_test.go @@ -0,0 +1,435 @@ +package main + +import ( + "fmt" + "strings" + "testing" + + kplan "github.com/dcadolph/kibble/internal/plan" + "github.com/google/go-cmp/cmp" +) + +// TestClassifyNewlyReachedLines pins the rules for lines kibble only began to +// run once it could build a tool from its checkout. The outputs are what +// navi, headscale, trivy, and poetry printed in those runs. Each excuse is +// paired with a failure of the same shape that it must not reach. +func TestClassifyNewlyReachedLines(t *testing.T) { + t.Parallel() + tests := []struct { + WantStatus Status + WantReason Reason + WantDetail string + Cmd string + Output string + Code int + NonzeroOK bool + OwnProject bool + }{{ // Test 0: navi saying fzf must be installed is something not installed. + Cmd: "navi --query \"change branch\" --best-match", + Output: "navi was unable to call fzf.\n Please make sure it's correctly installed.\n" + + " Refer to https://github.com/junegunn/fzf for more info.\n", + Code: 33, WantStatus: StatusBlocked, WantReason: ReasonMissingDependency, + WantDetail: "reports something not installed", + }, { // Test 1: a client that waited out its deadline could not reach a service. + Cmd: "headscale nodes list", + Output: "Error: listing nodes: Get \"http://local/api/v1/node\": context deadline exceeded\n", + Code: 1, WantStatus: StatusBlocked, WantReason: ReasonMissingDependency, + WantDetail: "could not reach a service", + }, { // Test 2: a socket read that timed out is the same refusal. + Cmd: "tool status", + Output: "error: read tcp 10.0.0.2:4411->10.0.0.9:443: i/o timeout\n", + Code: 1, WantStatus: StatusBlocked, WantReason: ReasonMissingDependency, + }, { // Test 3: a tool that crashed after a deadline message still crashed. + Cmd: "headscale nodes list", + Output: "context deadline exceeded\npanic: runtime error: invalid memory address or nil pointer dereference\n", + Code: 2, WantStatus: StatusFail, + }, { // Test 4: a flag the tool does not know is not excused by the word installed. + Cmd: "navi --best-macth", + Output: "error: unexpected argument '--best-macth' found\n", + Code: 2, WantStatus: StatusFail, + }, { // Test 5: a crash word in a table cell is a report, so the documented exit stands. + Cmd: "trivy image --exit-code 1 python:3.4-alpine3.9", Output: trivyTable, + Code: 1, NonzeroOK: true, WantStatus: StatusVerified, + WantDetail: "exit 1 is documented behavior", + }, { // Test 6: the same table with no documented exit fails, and not as a crash. + Cmd: "trivy image python:3.4-alpine3.9", Output: trivyTable, + Code: 1, WantStatus: StatusFail, WantDetail: "exited 1", + }, { // Test 7: a panic printed bare after the table is the tool dying. + Cmd: "trivy image --exit-code 1 python:3.4-alpine3.9", + Output: trivyTable + "panic: runtime error: index out of range [3] with length 3\n", + Code: 2, NonzeroOK: true, WantStatus: StatusFail, WantDetail: "crashed with exit 2", + }, { // Test 8: the shell reporting a segfault bare is a crash whatever the document blesses. + Cmd: "trivy image --exit-code 1 alpine", Output: "Segmentation fault (core dumped)\n", + Code: 1, NonzeroOK: true, WantStatus: StatusFail, WantDetail: "crashed with exit 1", + }, { // Test 9: a traceback a tool quotes behind a bar is the build it ran, not its own death. + Cmd: "poetry add pendulum@^2.0.5", Output: poetryQuotedBuild, + Code: 1, OwnProject: true, WantStatus: StatusBlocked, WantReason: ReasonCheckoutState, + WantDetail: "ran in the repository's own checkout", + }, { // Test 10: the same traceback printed bare is the tool's own crash. + Cmd: "poetry add pendulum@^2.0.5", Output: strings.ReplaceAll(poetryQuotedBuild, " | ", ""), + Code: 1, OwnProject: true, WantStatus: StatusFail, WantDetail: "crashed with exit 1", + }, { // Test 11: a cluster context the tool does not have is a name the reader defines. + Cmd: "trivy k8s --report summary cluster", + Output: "2026-11-15T21:30:22Z\tFATAL\tFatal error\tfailed getting k8s cluster: " + + "context \"cluster\" does not exist\n", + Code: 1, WantStatus: StatusBlocked, WantReason: ReasonMissingFixture, + WantDetail: "names a context the reader defines first", + }, { // Test 12: a subcommand that does not exist is the tool's own vocabulary. + Cmd: "trivy k9s cluster", Output: "Error: subcommand \"k9s\" does not exist\n", + Code: 1, WantStatus: StatusFail, + }, { // Test 13: a plugin with no build for this platform settles nothing. + Cmd: "trivy plugin install github.com/aquasecurity/trivy-plugin-kubectl", Output: noPlatform, + Code: 1, WantStatus: StatusBlocked, WantReason: ReasonOtherPlatform, + WantDetail: "found nothing for this platform", + }, { // Test 14: and is not read as the exit the document asked for. + Cmd: "trivy plugin run kubectl pod web -- --exit-code 1", Output: noPlatform, + Code: 1, NonzeroOK: true, WantStatus: StatusBlocked, WantReason: ReasonOtherPlatform, + }, { // Test 15: a plugin that is simply not found is still a failure. + Cmd: "trivy plugin install github.com/aquasecurity/trivy-plugin-kubectl", + Output: "FATAL\tFatal error\tplugin install error: plugin not found\n", + Code: 1, WantStatus: StatusFail, + }, { // Test 16: an uninstaller finding nothing to remove reports something not installed. + Cmd: "curl -sSL https://install.python-poetry.org | POETRY_UNINSTALL=1 python3 -", + Output: "Poetry is not currently installed.\n", + Code: 1, WantStatus: StatusBlocked, WantReason: ReasonMissingDependency, + WantDetail: "reports something not installed", + }} + for testNum, test := range tests { + t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) { + t.Parallel() + lr := lineResult{Cmd: test.Cmd, Code: -1} + o := lineOutcome{code: test.Code, output: test.Output, ownProject: test.OwnProject} + if test.OwnProject { + o.cwd = sessionRepoDir + } + line := kplan.PlanLine{Cmd: test.Cmd, NonzeroOK: test.NonzeroOK} + got := classifyLineResult(lr, line, o, false, nil, lineTimeout) + if diff := cmp.Diff(test.WantStatus, got.Status); diff != "" { + t.Errorf("status mismatch (-want +got):\n%s\ndetail: %s", diff, got.Detail) + } + if diff := cmp.Diff(test.WantReason, got.Reason); diff != "" { + t.Errorf("reason mismatch (-want +got):\n%s", diff) + } + if !strings.Contains(got.Detail, test.WantDetail) { + t.Errorf("detail = %q, want it to contain %q", got.Detail, test.WantDetail) + } + }) + } +} + +// Outputs of the newly reached lines, cut to the rows that decide them. +const ( + // trivyTable is a vulnerability table whose one row quotes a crash word. + trivyTable = "python:3.4-alpine3.9 (alpine 3.9.2)\n" + + "┌─────────┬───────────────┬──────────────────────────────────────────────────────────────┐\n" + + "│ openssl │ CVE-2020-1967 │ openssl: Segmentation fault in SSL_check_chain causes denial │\n" + + "│ │ │ https://avd.aquasec.com/nvd/cve-2020-1967 │\n" + + "└─────────┴───────────────┴──────────────────────────────────────────────────────────────┘\n" + // poetryQuotedBuild is poetry reporting a dependency whose build backend died. + poetryQuotedBuild = " - Installing pendulum (2.1.2)\n\nPEP517 build of a dependency failed\n\n" + + "Backend subprocess exited when trying to invoke build_wheel\n\n" + + " | Traceback (most recent call last):\n" + + " | File \"/tmp/tmp3i3k32w4/pendulum-2.1.2/build.py\", line 5, in \n" + + " | from distutils.command.build_ext import build_ext\n" + + " | ModuleNotFoundError: No module named 'distutils'\n\n" + + "Note: This error originates from the build backend, and is likely not a problem with poetry\n\n" + + "You can verify this by running pip wheel --no-cache-dir --use-pep517 \"pendulum (==2.1.2)\".\n" + // noPlatform is trivy refusing a plugin that has no build for the container. + noPlatform = "2026-11-15T21:33:58Z\tFATAL\tFatal error\tplugin install error: failed to install " + + "the plugin: platform selection error: platform not found\n" +) + +// TestFailureNamingUnrunBlock checks that a failure telling the reader to run +// a command the documents show, in a block the plan left out, is blocked +// behind that command, and that nothing weaker than the tool asking for it +// pairs the two. +func TestFailureNamingUnrunBlock(t *testing.T) { + t.Parallel() + const nix = "Warning: golangci-lint not found. Run 'nix develop' to ensure all dependencies are available." + tests := []struct { + Unrun []string + Output string + WantStatus Status + WantDetail string + }{{ // Test 0: the tool asks for a command the documents show and kibble left out. + Unrun: []string{"nix develop"}, Output: nix, WantStatus: StatusBlocked, + WantDetail: "failed naming `nix develop`, which the documents show in a block kibble did not run", + }, { // Test 1: the same output with no such block in the documents is a failure. + Output: nix, WantStatus: StatusFail, + }, { // Test 2: a different command left out does not pair. + Unrun: []string{"nix build"}, Output: nix, WantStatus: StatusFail, + }, { // Test 3: a hint pointing at help is advice, not a prerequisite. + Unrun: []string{"nix develop"}, Output: "error: bad target. See nix develop --help for usage.", + WantStatus: StatusFail, + }, { // Test 4: the command inside a longer token is not the command. + Unrun: []string{"nix develop"}, Output: "error: must run my-nix developer-mode first", + WantStatus: StatusFail, + }} + for testNum, test := range tests { + t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) { + t.Parallel() + run := &exampleRun{Steps: []exampleStep{{ID: "b1", Doc: "README.md", Lines: []lineResult{ + {Cmd: "make test", Status: StatusFail, Detail: "exited 2", output: test.Output}, + }}}} + resolveDependentFailures(run, &kplan.Plan{Binaries: []string{"tool"}, Unrun: test.Unrun}) + got := run.Steps[0].Lines[0] + if diff := cmp.Diff(test.WantStatus, got.Status); diff != "" { + t.Errorf("status mismatch (-want +got):\n%s\ndetail: %s", diff, got.Detail) + } + if test.WantDetail != "" && got.Detail != test.WantDetail { + t.Errorf("detail = %q, want %q", got.Detail, test.WantDetail) + } + }) + } +} + +// TestFailureAfterStoppedScaffold checks that a failure of a documented tool +// is blocked when an earlier line of its document was to create the project +// and the checkout stopped it by already holding one, as goreleaser's quick +// start does, and that nothing short of that pairs the two. +func TestFailureAfterStoppedScaffold(t *testing.T) { + t.Parallel() + const pro = "you are using a GoReleaser Pro configuration file with GoReleaser OSS" + stopped := lineResult{ + Cmd: "goreleaser init", Status: StatusBlocked, Reason: ReasonCheckoutState, + output: "command failed error=.goreleaser.yaml already exists", + } + tests := []struct { + Earlier lineResult + EarlierDoc string + Cmd string + Output string + Status Status + Code int + WantStatus Status + WantDetail string + }{{ // Test 0: a build after the stopped init read the checkout's own project. + Earlier: stopped, Cmd: "goreleaser build --single-target --clean", Output: pro, + WantStatus: StatusBlocked, WantDetail: "follows `goreleaser init`, which the repository's own files stopped", + }, { // Test 1: a flag the tool does not know is its answer to the line as written. + Earlier: stopped, Cmd: "goreleaser build --single-traget", Output: "unknown flag: --single-traget", + WantStatus: StatusFail, + }, { // Test 2: a crash stands whatever the line read. + Earlier: stopped, Cmd: "goreleaser build", Output: "panic: runtime error: nil map\n", + WantStatus: StatusFail, + }, { // Test 3: another program's failure is not the scaffolded tool's. + Earlier: stopped, Cmd: "make release", Output: pro, WantStatus: StatusFail, + }, { // Test 4: an init that passed set the project up, so the failure stands. + Earlier: lineResult{Cmd: "goreleaser init", Status: StatusVerified}, + Cmd: "goreleaser build", Output: pro, WantStatus: StatusFail, + }, { // Test 5: an init blocked for another reason is not a stopped scaffold. + Earlier: lineResult{ + Cmd: "goreleaser init", Status: StatusBlocked, Reason: ReasonMissingFixture, + output: "needs git history", + }, + Cmd: "goreleaser build", Output: pro, WantStatus: StatusFail, + }, { // Test 6: an init stopped in another document says nothing about this one. + Earlier: stopped, EarlierDoc: "docs/other.md", Cmd: "goreleaser build", Output: pro, + WantStatus: StatusFail, + }, { // Test 7: a documented nonzero exit after the stopped init answered the checkout's project. + Earlier: stopped, Cmd: "goreleaser check", Output: pro, Status: StatusVerified, Code: 1, + WantStatus: StatusBlocked, WantDetail: "so its exit 1 answered the checkout's project", + }, { // Test 8: the same documented exit after an init that passed stands. + Earlier: lineResult{Cmd: "goreleaser init", Status: StatusVerified}, Cmd: "goreleaser check", + Output: pro, Status: StatusVerified, Code: 1, WantStatus: StatusVerified, + }, { // Test 9: a zero exit after the stopped init is a pass the rule does not reach. + Earlier: stopped, Cmd: "goreleaser --help", Status: StatusVerified, Code: 0, + WantStatus: StatusVerified, + }} + for testNum, test := range tests { + t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) { + t.Parallel() + doc := "docs/quick-start.md" + if test.Status == "" { + test.Status, test.Code = StatusFail, 1 + } + if test.EarlierDoc == "" { + test.EarlierDoc = doc + } + run := &exampleRun{Steps: []exampleStep{ + {ID: "b1", Doc: test.EarlierDoc, Lines: []lineResult{test.Earlier}}, + {ID: "b2", Doc: doc, Lines: []lineResult{ + {Cmd: test.Cmd, Status: test.Status, Code: test.Code, Detail: "exited 1", output: test.Output}, + }}, + }} + resolveDependentFailures(run, &kplan.Plan{Binaries: []string{"goreleaser"}}) + got := run.Steps[1].Lines[0] + if diff := cmp.Diff(test.WantStatus, got.Status); diff != "" { + t.Errorf("status mismatch (-want +got):\n%s\ndetail: %s", diff, got.Detail) + } + if !strings.Contains(got.Detail, test.WantDetail) { + t.Errorf("detail = %q, want it to contain %q", got.Detail, test.WantDetail) + } + }) + } +} + +// TestCrashAfterToolGone checks that a crash after which the documented tool no +// longer starts is blocked as an environment change, read end to end from the +// session's markers, and that a crash the tool survives still fails. The +// output is poetry's after an earlier document turned off virtualenv creation +// and `poetry sync --only dev` removed its own dependencies. +func TestCrashAfterToolGone(t *testing.T) { + t.Parallel() + const cleo = "Traceback (most recent call last):\n File \"/opt/kibble-tool/bin/poetry\", line 5, in \n" + + "ModuleNotFoundError: No module named 'cleo'" + tests := []struct { + Probe string + Output string + WantStatus Status + WantReason Reason + }{{ // Test 0: the probe failed after the crash, so the environment changed. + Probe: "KIBBLE-PROBE b1:1 CODE=1", Output: cleo, + WantStatus: StatusBlocked, WantReason: ReasonEnvironmentChanged, + }, { // Test 1: the tool still starts, so the crash convicts. + Probe: "KIBBLE-PROBE b1:1 CODE=0", Output: cleo, WantStatus: StatusFail, + }, { // Test 2: no probe ran, as for a tool whose --version failed at the start. + Output: cleo, WantStatus: StatusFail, + }, { // Test 3: a plain nonzero exit with the tool gone is not a crash, and fails. + Probe: "KIBBLE-PROBE b1:1 CODE=1", Output: "error: the lock file is out of date", + WantStatus: StatusFail, + }, { // Test 4: a probe naming another line leaves this one alone. + Probe: "KIBBLE-PROBE b1:0 CODE=1", Output: cleo, WantStatus: StatusFail, + }} + for testNum, test := range tests { + t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) { + t.Parallel() + out := []string{"KIBBLE-BUILD CODE=0", "KIBBLE-STEP b1 START", "KIBBLE-LINE b1:0 CODE=0", + test.Output, "KIBBLE-LINE b1:1 CODE=1"} + if test.Probe != "" { + out = append(out, test.Probe) + } + out = append(out, "KIBBLE-LINE b1:2 CODE=0", "KIBBLE-DONE") + plan := examplePlan() + res := classifyExample(InstallStep{Repo: "repo", Kind: "example", plan: plan}, plan, + sessionOut(out...), nil, 0, lineTimeout, sessionEnd{}) + if res.example == nil || len(res.example.Steps) == 0 || len(res.example.Steps[0].Lines) < 2 { + t.Fatalf("no line results: %+v", res) + } + got := res.example.Steps[0].Lines[1] + if diff := cmp.Diff(test.WantStatus, got.Status); diff != "" { + t.Errorf("status mismatch (-want +got):\n%s\ndetail: %s", diff, got.Detail) + } + if diff := cmp.Diff(test.WantReason, got.Reason); diff != "" { + t.Errorf("reason mismatch (-want +got):\n%s", diff) + } + }) + } +} + +// TestFailureOnUnresolvedHost checks that a request to a host that did not +// resolve is blocked when an earlier line of its document named that host and +// did not pass, as tailscale's Kubernetes guide does with services its skipped +// kubectl lines set up, and that a dead host with no such line still fails. +func TestFailureOnUnresolvedHost(t *testing.T) { + t.Parallel() + const curl = "curl: (6) Could not resolve host: nginx" + skipped := lineResult{Cmd: "kubectl logs nginx ts-sidecar", Status: StatusSkipped, + Reason: ReasonMissingDependency} + tests := []struct { + Earlier lineResult + EarlierDoc string + Cmd string + Output string + WantStatus Status + WantDetail string + }{{ // Test 0: the service a skipped line named has nothing to resolve to. + Earlier: skipped, Cmd: "curl http://nginx", Output: curl, WantStatus: StatusBlocked, + WantDetail: "could not resolve host nginx, which `kubectl logs nginx ts-sidecar` also named", + }, { // Test 1: a Go client's lookup error with a port reads the same. + Earlier: lineResult{Cmd: "make proxy | kubectl apply -f-", Status: StatusSkipped}, + Cmd: "tool get http://proxy:8080/health", Output: "dial tcp: lookup proxy: no such host", + WantStatus: StatusBlocked, WantDetail: "could not resolve host proxy", + }, { // Test 2: a line that named the host and passed leaves the failure standing. + Earlier: lineResult{Cmd: "kubectl logs nginx ts-sidecar", Status: StatusVerified}, + Cmd: "curl http://nginx", Output: curl, WantStatus: StatusFail, + }, { // Test 3: a dead domain no earlier line names is a real break. + Earlier: skipped, Cmd: "curl https://downloads.example.org/tool.tgz", + Output: "curl: (6) Could not resolve host: downloads.example.org", + WantStatus: StatusFail, + }, { // Test 4: a refused connection is not a name that failed to resolve. + Earlier: skipped, Cmd: "curl http://nginx", + Output: "curl: (7) Failed to connect to nginx port 80: Connection refused", + WantStatus: StatusFail, + }, { // Test 5: the host inside a longer word is not the host. + Earlier: lineResult{Cmd: "kubectl logs nginx-proxy", Status: StatusSkipped}, + Cmd: "curl http://nginx", Output: curl, WantStatus: StatusFail, + }, { // Test 6: a skipped line in another document says nothing about this one. + Earlier: skipped, EarlierDoc: "docs/other.md", Cmd: "curl http://nginx", Output: curl, + WantStatus: StatusFail, + }, { // Test 7: the earlier line's program is not a name it set up. + Earlier: lineResult{Cmd: "nginx -t", Status: StatusSkipped}, + Cmd: "curl http://nginx", Output: curl, WantStatus: StatusFail, + }} + for testNum, test := range tests { + t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) { + t.Parallel() + doc := "docs/k8s/README.md" + if test.EarlierDoc == "" { + test.EarlierDoc = doc + } + run := &exampleRun{Steps: []exampleStep{ + {ID: "b1", Doc: test.EarlierDoc, Lines: []lineResult{test.Earlier}}, + {ID: "b2", Doc: doc, Lines: []lineResult{ + {Cmd: test.Cmd, Status: StatusFail, Code: 6, Detail: "exited 6", output: test.Output}, + }}, + }} + resolveDependentFailures(run, &kplan.Plan{Binaries: []string{"tailscale"}}) + got := run.Steps[1].Lines[0] + if diff := cmp.Diff(test.WantStatus, got.Status); diff != "" { + t.Errorf("status mismatch (-want +got):\n%s\ndetail: %s", diff, got.Detail) + } + if !strings.Contains(got.Detail, test.WantDetail) { + t.Errorf("detail = %q, want it to contain %q", got.Detail, test.WantDetail) + } + }) + } +} + +// TestInheritedAcrossDetails checks that a collapsed details element is a +// section of its own, so a setup that ran in the directory a sibling setup +// under the same heading entered is blocked, read end to end from the +// session's markers. glance's README offers a template setup that enters a +// directory beside a manual one that then met the template's files. +func TestInheritedAcrossDetails(t *testing.T) { + t.Parallel() + const exists = "mkdir: cannot create directory 'config': File exists" + tests := []struct { + First, Second int + WantStatus Status + WantReason Reason + }{{ // Test 0: sibling details under one heading are separate sections. + First: 1, Second: 2, WantStatus: StatusBlocked, WantReason: ReasonInheritedState, + }, { // Test 1: both blocks in one details element are one section that entered it. + First: 1, Second: 1, WantStatus: StatusFail, + }, { // Test 2: outside any details, one heading is one section as before. + First: 0, Second: 0, WantStatus: StatusFail, + }} + for testNum, test := range tests { + t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) { + t.Parallel() + plan := &kplan.Plan{Repo: "glance", Steps: []kplan.PlanStep{ + {ID: "b1", Doc: "README.md", Heading: "Installation", Details: test.First, + Lines: []kplan.PlanLine{{Cmd: "mkdir glance && cd glance"}}}, + {ID: "b2", Doc: "README.md", Heading: "Installation", Details: test.Second, + Lines: []kplan.PlanLine{{Cmd: "mkdir config"}}}, + }} + out := sessionOut("KIBBLE-BUILD CODE=0", + "KIBBLE-STEP b1 START", "KIBBLE-LINE b1:0 CODE=0 CWD=/work/repo/glance", + "KIBBLE-STEP b2 START", exists, "KIBBLE-LINE b2:0 CODE=1 CWD=/work/repo/glance", + "KIBBLE-DONE") + res := classifyExample(InstallStep{Repo: "glance", Kind: "example"}, plan, out, nil, 0, + lineTimeout, sessionEnd{}) + if res.example == nil { + t.Fatalf("no per-line outcomes, status %s: %s", res.Status, res.Detail) + } + got := res.example.Steps[1].Lines[0] + if diff := cmp.Diff(test.WantStatus, got.Status); diff != "" { + t.Errorf("status mismatch (-want +got):\n%s\ndetail: %s", diff, got.Detail) + } + if diff := cmp.Diff(test.WantReason, got.Reason); diff != "" { + t.Errorf("reason mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/main.go b/main.go index a3f9c17..db097ca 100644 --- a/main.go +++ b/main.go @@ -314,6 +314,17 @@ func collect(paths []string, examples bool) ([]InstallStep, []*kplan.Plan, []Res // where install and usage steps go once the front page fills up, and // it rots faster because nobody reads it on the way past, so every // document that walks a reader through commands gets its own session. + // No documented install gave the session a binary, so the tool is + // built from the checkout and the examples run against that, labeled. + // Skipping them cost every example in the repository for want of one + // install route, and the documented install rows keep their own + // statuses either way. + if len(installs) == 0 { + if in, ok := checkoutBuild(p, repo); ok { + installs = []kplan.PlanInstall{in} + bins = append(bins, in.Binary) + } + } sessionBins := sessionBinaries(repo, bins, installs) // Settings are documented once for a repository, often in the README, // and every document's session inherits them. Reading only the diff --git a/ownership.go b/ownership.go index f2ff423..1f5d299 100644 --- a/ownership.go +++ b/ownership.go @@ -226,6 +226,18 @@ func (rn repoNames) npmBinary(name string) (string, bool) { // section is a name, as in a scripts table. An array-of-tables header such as // [[bin]] is looked up with its inner brackets, as "[bin]". func (rn *repoNames) addSections(body []byte, keys map[string]string) { + for _, v := range sectionValues(body, keys) { + rn.declare(v) + } +} + +// sectionValues returns, in manifest order and exactly as written, the values +// keys selects from a TOML manifest: for a section mapped to a key name, that +// key's value, and for a section mapped to "*", every key in it, as in a +// scripts table. An array-of-tables header such as [[bin]] is looked up with +// its inner brackets, as "[bin]". +func sectionValues(body []byte, keys map[string]string) []string { + var out []string section := "" sc := bufio.NewScanner(strings.NewReader(string(body))) for sc.Scan() { @@ -246,11 +258,12 @@ func (rn *repoNames) addSections(body []byte, keys map[string]string) { v = strings.Trim(strings.TrimSpace(v), `"'`) switch want { case "*": - rn.declare(k) + out = append(out, k) case k: - rn.declare(v) + out = append(out, v) } } + return out } // owns reports whether a documented install puts the repository's own tool on diff --git a/session.go b/session.go index 8738341..2cc1746 100644 --- a/session.go +++ b/session.go @@ -198,7 +198,12 @@ for kb in %s; do if command -v "$kb" >/dev/null 2>&1; then have=1; printf '\nKIBBLE-HAVE %%s\n' "$kb"; fi done if [ "$have" -eq 0 ]; then printf '\nKIBBLE-NOBIN\n'; exit 0; fi -`, strings.Join(quoted, " ")) +__kprobe=" " +for kb in %s; do + command -v "$kb" >/dev/null 2>&1 || continue + timeout %d "$kb" --version >/dev/null 2>&1 && __kprobe="$__kprobe$kb " +done +`, strings.Join(quoted, " "), strings.Join(quoted, " "), probeSecs) // The copy is only of use at a document boundary, so a session with // one document is left exactly as it was. Only a regular file on an // absolute path is kept. A function's name is not a path, and a link @@ -240,6 +245,10 @@ done // repository, and each document after the first starts with a repository // as new as the one the first was given, in a working tree as the install // and the fixtures left it. + bins := map[string]bool{} + for _, b := range plan.Binaries { + bins[b] = true + } doc, opened := "", false for _, s := range plan.Steps { if !opened || s.Doc != doc { @@ -305,6 +314,9 @@ done fmt.Fprintf(&b, "printf '"+markerLead+"KIBBLE-LINE %s:%d CODE=%%d CWD=%%s\\n' "+ "\"$(__pipecode \"${__ps[@]}\")\" \"$__cwd\"\n", s.ID, i) + if bin, _ := kplan.InvokedBinary(docblock.Flatten(l.Cmd), bins); bin != "" { + writeProbe(&b, s.ID, i, bin) + } } } if opened { @@ -315,6 +327,26 @@ done return b.String(), wrapped } +// probeSecs bounds one `--version` probe of a documented binary. +const probeSecs = 20 + +// writeProbe asks, after a line that ran bin and exited nonzero, whether bin +// still starts. Only a binary whose `--version` passed when the session began +// is asked, so a tool without that flag is never read as broken. poetry's +// `poetry sync --only dev`, run with virtualenv creation turned off by an +// earlier document, removed poetry's own dependencies, and 39 lines after it +// crashed on a missing module for a reason no document wrote. +func writeProbe(b *strings.Builder, step string, i int, bin string) { + q := shellSafe(bin) + fmt.Fprintf(b, `if [ "$(__pipecode "${__ps[@]}")" -ne 0 ]; then + case "${__kprobe:-}" in *' %s '*) + timeout %d '%s' --version >/dev/null 2>&1 + printf '`+markerLead+`KIBBLE-PROBE %s:%d CODE=%%d\n' "$?" ;; + esac +fi +`, q, probeSecs, q, step, i) +} + // documentCount returns how many documents the plan's steps come from. func documentCount(plan *kplan.Plan) int { n, doc := 0, "" diff --git a/session_test.go b/session_test.go index c499f08..7c103b1 100644 --- a/session_test.go +++ b/session_test.go @@ -533,3 +533,78 @@ func TestSessionScriptFile(t *testing.T) { t.Errorf("content mismatch (-want +got):\n%s", diff) } } + +// TestSessionScriptProbe runs the generated `--version` probe in bash with a +// stand-in tool that breaks on cue, so the marker is checked by a shell. Only +// a line that ran a documented binary and exited nonzero is probed, and only +// for a binary whose probe passed when the session began. +func TestSessionScriptProbe(t *testing.T) { + t.Parallel() + for _, tool := range []string{"bash", "timeout"} { + if _, err := exec.LookPath(tool); err != nil { + t.Skipf("%s is not installed", tool) + } + } + plan := &kplan.Plan{ + Repo: "repo", Binaries: []string{"tool"}, + Steps: []kplan.PlanStep{{ID: "b1", Doc: "README.md", Lines: []kplan.PlanLine{ + {Cmd: "tool sync --only dev"}, {Cmd: "make all"}, + }}}, + } + script, _ := sessionScript(plan, 240, 90) + var probe strings.Builder + writeProbe(&probe, "b1", 0, "tool") + if !strings.Contains(script, probe.String()) { + t.Fatalf("script lacks the probe after the tool's line:\n%s", script) + } + if strings.Contains(script, "KIBBLE-PROBE b1:1") { + t.Errorf("script probes a line that runs no documented binary:\n%s", script) + } + pipecode := regexp.MustCompile(`(?s)__pipecode\(\) \{\n.*?\n\}\n`).FindString(script) + baseline := regexp.MustCompile(`(?s)__kprobe=" "\n.*?\ndone\n`).FindString(script) + if pipecode == "" || baseline == "" { + t.Fatalf("script lacks __pipecode or the baseline probe:\n%s", script) + } + tests := []struct { + WantResult string + BrokenAtStart bool + BreakAfter bool + Code string + }{{ // Test 0: the tool stopped starting after the line, so the probe fails. + BreakAfter: true, Code: "1", WantResult: "KIBBLE-PROBE b1:0 CODE=1", + }, { // Test 1: the tool still starts, so the probe passes. + Code: "1", WantResult: "KIBBLE-PROBE b1:0 CODE=0", + }, { // Test 2: a tool that never passed its probe is never asked again. + BrokenAtStart: true, BreakAfter: true, Code: "1", WantResult: "", + }, { // Test 3: a line that passed is not probed. + BreakAfter: true, Code: "0", WantResult: "", + }} + for testNum, test := range tests { + t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + stub := "#!/bin/sh\n[ -e \"" + dir + "/broken\" ] && exit 1\nexit 0\n" + if err := os.WriteFile(filepath.Join(dir, "tool"), []byte(stub), 0o755); err != nil { + t.Fatal(err) + } + body := pipecode + if test.BrokenAtStart { + body += "touch '" + dir + "/broken'\n" + } + body += baseline + if test.BreakAfter { + body += "touch '" + dir + "/broken'\n" + } + body += "__ps=(" + test.Code + ")\n" + probe.String() + cmd := exec.Command("bash", "-c", body) + cmd.Env = append(os.Environ(), "PATH="+dir+":"+os.Getenv("PATH")) + out, err := cmd.Output() + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(test.WantResult, strings.TrimSpace(string(out))); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/sessionparse.go b/sessionparse.go index 98f7e6d..9e9572a 100644 --- a/sessionparse.go +++ b/sessionparse.go @@ -63,6 +63,9 @@ type lineOutcome struct { // creators maps each path an earlier line of the same document was to // create, and did not pass, to that line's command. creators map[string]string + // toolGone marks a line after which the documented binary it ran failed + // its `--version` probe, though the probe passed when the session began. + toolGone bool } // sessionRepoDir is the directory every document's shell starts in. @@ -75,6 +78,9 @@ var ( // reInitializedAt matches the line uv prints for a new project, capturing // the project name and the directory it was created in. reInitializedAt = regexp.MustCompile("Initialized project `([^`]+)` at `([^`]+)`") + // reProbeMarker parses a KIBBLE-PROBE marker into step ID, line index, and + // the exit code of the `--version` probe that followed the line. + reProbeMarker = regexp.MustCompile(`KIBBLE-PROBE (\S+):(\d+) CODE=(-?\d+)$`) ) // markerTail splits a line on a session marker, returning the output that @@ -112,7 +118,8 @@ func classifyExample(step InstallStep, plan *kplan.Plan, out string, wrapped map absent := map[string]bool{} var chunk []string aborted, done, noBin := false, false, false - pkgCode := 0 + pkgCode, buildCode := 0, 0 + var buildTail []string cut := false keep := func(s string) { switch { @@ -141,13 +148,24 @@ func classifyExample(step InstallStep, plan *kplan.Plan, out string, wrapped map pkgCode, _ = strconv.Atoi(strings.TrimSpace(code)) chunk, cut = nil, false case strings.Contains(line, "KIBBLE-BUILD CODE="): + _, code, _ := markerTail(line, "KIBBLE-BUILD CODE=") + buildCode, _ = strconv.Atoi(strings.TrimSpace(code)) chunk, cut = nil, false case strings.Contains(line, "KIBBLE-ABORT"): aborted = true + buildTail = chunk case strings.Contains(line, "KIBBLE-STEP "): chunk, cut = nil, false case strings.Contains(line, "KIBBLE-DONE"): done = true + case reProbeMarker.MatchString(line): + m := reProbeMarker.FindStringSubmatch(line) + keep(line[:len(line)-len(m[0])]) + if o, ok := outcomes[m[1]+":"+m[2]]; ok { + o.toolGone = m[3] != "0" + outcomes[m[1]+":"+m[2]] = o + } + chunk, cut = nil, false case reLineMarker.MatchString(line): m := reLineMarker.FindStringSubmatch(line) keep(line[:len(line)-len(m[0])]) @@ -168,12 +186,30 @@ func classifyExample(step InstallStep, plan *kplan.Plan, out string, wrapped map keep(line) } } - if aborted { + built := fromCheckout(plan) + switch { + case aborted && built: + // The build is kibble's stand-in for an install the docs never let it + // run, so its failure says nothing about the document. + res.Status = StatusBlocked + res.Reason = ReasonMissingDependency + res.Detail = "building the tool from the checkout failed, so the examples did not run; " + checkoutNote + switch { + case buildCode == 124: + res.Detail += ": the build timed out" + case len(buildTail) > 0: + res.Detail += ": " + util.Truncate(buildFailLine(buildTail), 160) + } + return res + case aborted: res.Status = StatusSkipped res.Detail = "documented install failed in the session; examples not run" return res - } - if noBin { + case noBin && built: + res.Status = StatusSkipped + res.Detail = "the tool built from the checkout is not on PATH under the expected name; examples not run" + return res + case noBin: res.Status = StatusSkipped res.Detail = "installed tool is not on PATH under any documented name; examples not run" return res @@ -196,6 +232,9 @@ func classifyExample(step InstallStep, plan *kplan.Plan, out string, wrapped map res.Detail += ": " + util.Truncate(first, 120) } } + if built && res.Status != StatusError { + res.Detail = strings.TrimSuffix(checkoutNote+"; "+res.Detail, "; ") + } return res } @@ -223,9 +262,12 @@ func buildOutcomes(plan *kplan.Plan, outcomes map[string]lineOutcome, wrapped ma for _, b := range plan.Binaries { bins[b] = true } - // entered holds the sections, by document and heading, that ran a cd. A - // document keeps one shell, so a directory one section enters is where - // every later section of that document runs. + // entered holds the sections, by document, heading, and details element, + // that ran a cd. A document keeps one shell, so a directory one section + // enters is where every later section of that document runs. Collapsed + // details under one heading are sections of their own: glance's README + // offers a template setup that enters a directory and a manual setup + // beside it, and the manual one then ran inside the template's. entered := map[string]bool{} // pending holds, per document, the paths earlier lines were to create and // did not, each with the first such line's command. @@ -235,7 +277,7 @@ func buildOutcomes(plan *kplan.Plan, outcomes map[string]lineOutcome, wrapped ma members := map[string]bool{} for _, s := range plan.Steps { es := exampleStep{ID: s.ID, Doc: s.Doc, Heading: s.Heading} - section := s.Doc + "\x00" + s.Heading + section := fmt.Sprintf("%s\x00%s\x00%d", s.Doc, s.Heading, s.Details) for i, l := range s.Lines { key := fmt.Sprintf("%s:%d", s.ID, i) lr := lineResult{Cmd: docblock.Flatten(l.Cmd), Code: -1, Line: l.Line, Synthetic: l.Synthetic} diff --git a/tracked.go b/tracked.go new file mode 100644 index 0000000..0bc390a --- /dev/null +++ b/tracked.go @@ -0,0 +1,42 @@ +package main + +import ( + "os" + "path" + "path/filepath" + + "github.com/go-git/go-git/v5/plumbing/format/index" +) + +// trackedSet is what the git index says a checkout tracks. +type trackedSet struct { + // dirs holds every directory with at least one tracked file under it, as a + // slash-separated path relative to the checkout. + dirs map[string]bool + // files holds every tracked file, named the same way. + files map[string]bool +} + +// tracked returns the files of a checkout that git tracks and the directories +// that hold them. It reads the git index and nothing else. A checkout with no +// index of its own, such as a source tarball or a linked worktree, returns an +// empty set. +func tracked(dir string) trackedSet { + f, err := os.Open(filepath.Join(dir, ".git", "index")) + if err != nil { + return trackedSet{} + } + defer func() { _ = f.Close() }() + var idx index.Index + if err := index.NewDecoder(f).Decode(&idx); err != nil { + return trackedSet{} + } + set := trackedSet{dirs: map[string]bool{}, files: map[string]bool{}} + for _, e := range idx.Entries { + set.files[e.Name] = true + for d := path.Dir(e.Name); d != "." && !set.dirs[d]; d = path.Dir(d) { + set.dirs[d] = true + } + } + return set +} diff --git a/verdict_alias.go b/verdict_alias.go index 014fb2f..ef2b614 100644 --- a/verdict_alias.go +++ b/verdict_alias.go @@ -25,6 +25,7 @@ const ( ReasonAlreadyProven = verdict.ReasonAlreadyProven ReasonCheckoutState = verdict.ReasonCheckoutState ReasonDependsOnSkipped = verdict.ReasonDependsOnSkipped + ReasonEnvironmentChanged = verdict.ReasonEnvironmentChanged ReasonInheritedState = verdict.ReasonInheritedState ReasonInteractive = verdict.ReasonInteractive ReasonLongRunning = verdict.ReasonLongRunning