From db644d384679b910556dc52d683b64b3bbdb95c9 Mon Sep 17 00:00:00 2001 From: Douglas Adolph <116844282+dcadolph@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:43:50 -0600 Subject: [PATCH 1/4] Exit 3 when a run tests no documented claim --- .github/workflows/corpus.yml | 2 +- README.md | 9 +++-- docs/DESIGN.md | 9 +++-- main.go | 48 ++++++++++++++++++++------ main_test.go | 35 +++++++++++++++++-- report.go | 33 ++++++++++++++++++ report_test.go | 66 ++++++++++++++++++++++++++++++++++++ 7 files changed, 183 insertions(+), 19 deletions(-) diff --git a/.github/workflows/corpus.yml b/.github/workflows/corpus.yml index e274606..f5cee69 100644 --- a/.github/workflows/corpus.yml +++ b/.github/workflows/corpus.yml @@ -50,7 +50,7 @@ jobs: rc=0 /tmp/kibble -json "$dir" > "/tmp/${name}.json" || rc=$? # Exit 1 is kibble reporting failures and exit 3 is kibble - # reporting that every row is a skip. Both are verdicts with a + # reporting that it tested no claim. Both are verdicts with a # JSON array behind them, and a 0,0,0 pin is exactly the shape # that exits 3. Any other nonzero is kibble unable to do its job, # and blaming the corpus for that would be the same mistake as diff --git a/README.md b/README.md index ccaa4f4..456fd0a 100644 --- a/README.md +++ b/README.md @@ -189,12 +189,15 @@ it, so a CI log can always say which kibble produced its verdict. The exit code tells a script what happened: -- `0` means at least one check ran and none failed. +- `0` means at least one documented claim was tested and none failed. - `1` means the run failed. `-strict` widens what counts as a failure. - `2` means a usage error, such as no path and no README in the current directory, Docker unavailable, or less than 2 GiB free in Docker. -- `3` means nothing was checked: every row is a skip, so no install ran and no example - could run either. The action fails on purpose, since a check that ran nothing must not pass. +- `3` means nothing was tested: no row and no example line reached `VERIFIED`, `FAIL`, + `GAP`, or `DRIFT`. A brew formula lookup, a recipe with no binary to smoke-test, a + timeout, and a skip all leave the documents unchecked, and the summary names up to three + of those causes. The action fails on purpose, since a check that tested nothing must not + pass. - `130` means the run was interrupted. ## What it has found diff --git a/docs/DESIGN.md b/docs/DESIGN.md index e6367fa..9606181 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -81,8 +81,13 @@ honestly rather than as if nothing were documented. When the documents hold no install of the tool itself that kibble extracts, the install check is a `SKIP` with the reason `no-install-route`. The detail names the README, how many install documents were read, and the routes seen, such as brew, apt, or none. It is -not a `GAP`, because the documents may describe routes kibble does not run. When every -row is a skip, the run exits 3, so a check that ran nothing does not read as a pass. +not a `GAP`, because the documents may describe routes kibble does not run. When no row +and no example line reached `VERIFIED`, `FAIL`, `GAP`, or `DRIFT`, the run exits 3 and +the summary says nothing was tested and why, so a check that tested nothing does not read +as a pass. A brew formula lookup (`EXISTS`), a recipe with no binary (`RAN`), and a +`TIMEOUT` count as nothing tested: each leaves every documented claim unsettled. Before +this rule only an all-skip run exited 3, and a repository whose rows were a formula lookup +and skipped examples exited 0 having tested nothing. A package install runs the documented line verbatim, so what kibble verifies is the command a reader would actually type, flags and all. The bin directory is compared before diff --git a/main.go b/main.go index 8c2740d..fd9ed08 100644 --- a/main.go +++ b/main.go @@ -763,24 +763,52 @@ func gib(n int64) string { } // exitCode is the process exit status for a finished run: 1 when anyFail says -// the run failed, 3 when nothing was checked, else 0. Nothing was checked when -// every row is a skip: no install ran or was looked up, so no example could -// run either, whether the documents hold no install route at all or only -// routes kibble does not run. A check that ran nothing must not read as a -// pass, and it is not a failure of the documents either. +// the run failed, 3 when nothing was tested, else 0. Nothing was tested when no +// row and no example line reached a verdict on a documented claim. A brew +// formula lookup, a recipe that ran with no binary to smoke-test, a timeout, +// and a skip each leave every claim unsettled, and a run made only of those +// used to read as a pass. A check that tested nothing +// must not read as a pass, and it is not a failure of the documents either. func exitCode(results []Result, strict bool) int { if anyFail(results, strict) { return 1 } + if len(results) == 0 || anyTested(results) { + return 0 + } + return 3 +} + +// anyTested reports whether any result, or any example line inside one, +// settled a documented claim. +func anyTested(results []Result) bool { for _, r := range results { - if r.Status != StatusSkipped { - return 0 + if tested(r.Status) { + return true + } + if r.example == nil { + continue + } + for _, s := range r.example.Steps { + for _, l := range s.Lines { + if tested(l.Status) { + return true + } + } } } - if len(results) == 0 { - return 0 + return false +} + +// tested reports whether a status is a verdict on a documented claim: it +// worked, it failed, the document left a gap, or the document drifted from the +// binary. Every other status says the claim went unsettled. +func tested(s Status) bool { + switch s { + case StatusVerified, StatusFail, StatusGap, StatusDrift: + return true } - return 3 + return false } // anyFail reports whether the run should exit non-zero. A build failure always diff --git a/main_test.go b/main_test.go index 28fd8f0..104fe85 100644 --- a/main_test.go +++ b/main_test.go @@ -54,10 +54,20 @@ func TestAnyFail(t *testing.T) { } // TestExitCode checks the process exit status, including the code that says a -// run checked nothing because no repository had an install route kibble runs. +// run tested nothing: no row and no example line settled a documented claim. func TestExitCode(t *testing.T) { t.Parallel() res := func(s Status) Result { return Result{Status: s} } + exampleWith := func(lines ...Status) Result { + step := exampleStep{} + for _, s := range lines { + step.Lines = append(step.Lines, lineResult{Status: s}) + } + return Result{ + Step: InstallStep{Kind: "example"}, Status: StatusBlocked, + example: &exampleRun{Steps: []exampleStep{step}}, + } + } noRoute := Result{Step: InstallStep{Kind: "install"}, Status: StatusSkipped, Reason: ReasonNoInstallRoute} exampleSkip := Result{Step: InstallStep{Kind: "example"}, Status: StatusSkipped} aptSkip := Result{Step: InstallStep{Kind: "os-package"}, Status: StatusSkipped, Reason: ReasonUnsupportedMethod} @@ -84,12 +94,31 @@ func TestExitCode(t *testing.T) { Results: []Result{aptSkip, exampleSkip}, WantCode: 3, }, { // Test 8: an error elsewhere outranks nothing checked. Results: []Result{noRoute, res(StatusError)}, WantCode: 1, - }, { // Test 9: a formula lookup is a check, so a brew row beside skips passes. - Results: []Result{res(StatusExists), exampleSkip}, WantCode: 0, + }, { // Test 9: a formula lookup tests no claim, so a brew row beside skips is nothing tested. + Results: []Result{res(StatusExists), exampleSkip}, WantCode: 3, }, { // Test 10: strict does not turn nothing checked into a failure. Results: []Result{noRoute, exampleSkip}, Strict: true, WantCode: 3, }, { // Test 11: an empty run exits zero. WantCode: 0, + }, { // Test 12: a brew lookup alone is nothing tested. + Results: []Result{res(StatusExists)}, WantCode: 3, + }, { // Test 13: a recipe that ran with no binary is nothing tested. + Results: []Result{res(StatusRan), exampleSkip}, WantCode: 3, + }, { // Test 14: a timeout beside skips is nothing tested. + Results: []Result{res(StatusTimeout), exampleSkip}, WantCode: 3, + }, { // Test 15: built without a passing smoke test is nothing tested. + Results: []Result{res(StatusBuilt), res(StatusCrossArch), res(StatusBlocked)}, WantCode: 3, + }, { // Test 16: drift settles a claim, so the run tested something. + Results: []Result{res(StatusExists), res(StatusDrift)}, WantCode: 0, + }, { // Test 17: a gap settles a claim too. + Results: []Result{res(StatusRan), res(StatusGap)}, WantCode: 0, + }, { // Test 18: one verified example line inside a blocked session is a tested claim. + Results: []Result{res(StatusRan), exampleWith(StatusBlocked, StatusVerified)}, WantCode: 0, + }, { // Test 19: an example session whose lines all went unsettled is nothing tested. + Results: []Result{res(StatusRan), exampleWith(StatusBlocked, StatusSkipped)}, + WantCode: 3, + }, { // Test 20: an error outranks nothing tested. + Results: []Result{res(StatusExists), res(StatusError)}, WantCode: 1, }} for testNum, test := range tests { t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) { diff --git a/report.go b/report.go index f5d0efe..dd36b3b 100644 --- a/report.go +++ b/report.go @@ -212,6 +212,9 @@ func reportTable(w io.Writer, results []Result) { countVerdict(r.Status, &pass, &fail, &gap, &other) } _, _ = fmt.Fprintf(w, "\n%s\n", verdictLine(c, fail, gap, other+blocked)) + if len(results) > 0 && !anyTested(results) { + _, _ = fmt.Fprintf(w, "%s\n", c.yellow(untestedLine(results))) + } _, _ = fmt.Fprintf(w, "%s\n", summaryLine(c, pass, fail, gap, other, len(results), total)) _, _ = fmt.Fprintf(w, "%s\n", evidenceLine(c, results)) } @@ -266,6 +269,36 @@ func verdictLine(c palette, fail, gap, other int) string { } } +// untestedLine says that a run tested nothing and why, naming up to three +// distinct causes from the row details. Install rows come first, since an +// example skipped for want of a binary is the consequence of an install that +// did not run. Each cause is cut at its first semicolon so the causes can be +// joined on one line without running together. +func untestedLine(results []Result) string { + var installs, examples []string + for _, r := range results { + cause := r.Detail + if r.Step.doc != "" { + cause = strings.TrimPrefix(cause, r.Step.doc+": ") + } + cause, _, _ = strings.Cut(cause, ";") + cause = strings.TrimSpace(cause) + if cause == "" { + cause = strings.ToLower(string(r.Status)) + " " + r.Step.Kind + } + if r.Step.Kind == "example" { + examples = append(examples, cause) + } else { + installs = append(installs, cause) + } + } + causes := dedupe(append(installs, examples...)) + if len(causes) > 3 { + causes = causes[:3] + } + return "Nothing was tested: " + strings.Join(causes, "; ") + "." +} + // blockedLines counts the documented lines of a result that ran without // settling. They are counted apart from the result's own status because a // session that verified some lines is reported as verified, and the lines it diff --git a/report_test.go b/report_test.go index 1b7514d..2934fde 100644 --- a/report_test.go +++ b/report_test.go @@ -241,6 +241,72 @@ func TestReportNoInstallRoute(t *testing.T) { } } +// TestUntestedLine checks the summary line a run prints when it tested no +// claim: install causes before the example skips they cause, each cut at its +// first semicolon, no cause twice, and at most three. +func TestUntestedLine(t *testing.T) { + t.Parallel() + row := func(kind string, s Status, detail string) Result { + return Result{Step: InstallStep{Repo: "r", Kind: kind, doc: "README.md"}, Status: s, Detail: detail} + } + examples := row("example", StatusSkipped, "no documented install puts a binary on PATH; examples not run") + apt := row("os-package", StatusSkipped, "kibble does not run system package installs; verify this one by hand") + brew := row("brew", StatusExists, "README.md: formula exists (install not attempted)") + ran := row("git-clone", StatusRan, "recipe exited 0 but produced no binary to smoke-test") + tests := []struct { + Results []Result + WantResult string + }{{ // Test 0: shellcheck's shape, the example skip listed after the installs. + Results: []Result{brew, brew, examples, apt, apt}, + WantResult: "Nothing was tested: formula exists (install not attempted); " + + "kibble does not run system package installs; " + + "no documented install puts a binary on PATH.", + }, { // Test 1: four causes are cut to three, installs first. + Results: []Result{examples, apt, brew, ran}, + WantResult: "Nothing was tested: kibble does not run system package installs; " + + "formula exists (install not attempted); " + + "recipe exited 0 but produced no binary to smoke-test.", + }, { // Test 2: a row with no detail is named by its status and kind. + Results: []Result{{Step: InstallStep{Kind: "cargo-install"}, Status: StatusTimeout}}, + WantResult: "Nothing was tested: timeout cargo-install.", + }} + for testNum, test := range tests { + t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) { + t.Parallel() + if diff := cmp.Diff(test.WantResult, untestedLine(test.Results)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} + +// TestReportTableUntested checks the table prints the nothing-tested line only +// when no row settled a claim. +func TestReportTableUntested(t *testing.T) { + t.Parallel() + brew := Result{Step: InstallStep{Repo: "r", Kind: "brew"}, Status: StatusExists, + Detail: "formula exists (install not attempted)"} + tests := []struct { + Results []Result + WantLine bool + }{{ // Test 0: a brew lookup alone tested nothing. + Results: []Result{brew}, WantLine: true, + }, { // Test 1: a verified row beside it is a tested claim. + Results: []Result{brew, {Step: InstallStep{Repo: "r", Kind: "go-install"}, Status: StatusVerified}}, + }, { // Test 2: an empty run prints no cause line. + }} + for testNum, test := range tests { + t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) { + t.Parallel() + var b bytes.Buffer + reportTable(&b, test.Results) + if got := strings.Contains(b.String(), "Nothing was tested"); got != test.WantLine { + t.Errorf("line printed = %v, want %v:\n%s", got, test.WantLine, b.String()) + } + }) + } +} + // TestSharedCauseKeyIgnoresVersion checks the version fields do not split a shared cause. func TestSharedCauseKeyIgnoresVersion(t *testing.T) { t.Parallel() From 0a91a6854bf05ab70523d2d0e0d0402dd8b06451 Mon Sep 17 00:00:00 2001 From: Douglas Adolph <116844282+dcadolph@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:43:52 -0600 Subject: [PATCH 2/4] Add a recall corpus of bugs kibble must catch --- .github/workflows/corpus.yml | 63 ++++++++++++++++++++++++++++++++++++ corpus/recall.tsv | 27 ++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 corpus/recall.tsv diff --git a/.github/workflows/corpus.yml b/.github/workflows/corpus.yml index f5cee69..c2aa7a4 100644 --- a/.github/workflows/corpus.yml +++ b/.github/workflows/corpus.yml @@ -172,3 +172,66 @@ jobs: echo "::endgroup::" done < corpus/mutations.tsv exit "$status" + # Each recall row is a real bug a person found where kibble tested + # nothing. A caught=no row is a score, printed and never fatal, so the + # misses stay visible without turning the job red for work not yet + # built. A caught=yes row is a regression guard: losing that catch fails + # the job. KIBBLE and WORK let the same loop run on a laptop. + - name: Run kibble against the recall corpus + if: always() + shell: bash + run: | + set -uo pipefail + KIBBLE="${KIBBLE:-/tmp/kibble}" + WORK="${WORK:-/tmp/recall}" + test -x "$KIBBLE" || go build -o "$KIBBLE" . + mkdir -p "$WORK" + status=0 + rows=0 + caught=0 + while IFS=$'\t' read -r name url sha doc line want proof expect; do + case "$name" in ""|\#*) continue ;; esac + rows=$((rows + 1)) + dir="$WORK/${name}-${sha:0:12}" + out="$WORK/${name}-${sha:0:12}.json" + # Two rows can name one repository. It runs once. + if [ ! -s "$out" ]; then + echo "::group::${name} at ${sha:0:12}" + if [ ! -d "$dir/.git" ]; then + git init -q "$dir" + git -C "$dir" fetch -q --depth 1 "$url" "$sha" + git -C "$dir" checkout -q FETCH_HEAD + fi + rc=0 + "$KIBBLE" -json "$dir" > "$out" || rc=$? + # Exit 1 reports failures and exit 3 reports nothing tested. + # Both are verdicts. Anything else is kibble unable to run. + if { [ "$rc" -gt 1 ] && [ "$rc" -ne 3 ]; } || ! jq -e 'type=="array"' "$out" >/dev/null 2>&1; then + echo "${name}: kibble itself failed (exit ${rc})" + rm -f "$out" + fi + echo "::endgroup::" + fi + hit=0 + if [ -s "$out" ]; then + hit=$(jq -r --arg doc "$doc" --arg want "$want" --arg proof "$proof" ' + ($want | split("|")) as $ws + | [ (.[] | select((.status as $s | $ws | index($s)) and ((.detail // "") | contains($proof)))), + (.[] | .steps[]? | select((.doc // "") | endswith($doc)) | .lines[] + | select((.status as $s | $ws | index($s)) + and (((.cmd // "") + "\n" + (.detail // "")) | contains($proof)))) ] + | length' "$out") + fi + where="${name} ${doc}:${line} ${want} naming \"${proof}\"" + if [ "$hit" -gt 0 ]; then + caught=$((caught + 1)) + echo "${where}: caught" + elif [ "$expect" = "yes" ]; then + echo "::error::${where}: no longer caught, and the row says it was" + status=1 + else + echo "${where}: not caught yet" + fi + done < corpus/recall.tsv + echo "${caught} of ${rows} recall rows caught" + exit "$status" diff --git a/corpus/recall.tsv b/corpus/recall.tsv new file mode 100644 index 0000000..ed6a5cd --- /dev/null +++ b/corpus/recall.tsv @@ -0,0 +1,27 @@ +# The recall corpus: real documentation bugs found by hand in repositories +# where kibble tested nothing, each pinned to the commit the bug was found at +# and to the line a reader trips on. repos.tsv proves kibble does not invent +# failures and mutations.tsv proves it catches rot planted on purpose. This +# file holds what neither can: rot that was already there and that kibble +# walked past, reading the repository as fine. +# +# A row is caught when the run reports the wanted status on that bug: an +# example line in the named document whose status is wanted and whose +# command or detail contains the proof, or a top-level row whose status is +# wanted and whose detail contains the proof. The JSON carries no line +# number, so the line column says where a person should look and the proof +# pins identity, as in mutations.tsv. Wanted statuses are separated by "|". +# +# caught=no rows are the scoreboard: the job prints whether each is caught now +# and never fails on them. Once a change catches a row, flip it to caught=yes, +# and from then on the job fails if the line stops reaching its status. Every +# bug found by hand becomes a row here before any fix for it is built. +# +# The first four came from the 2026-10-22 hand check of ten repositories +# kibble had run nothing on. Columns are tab-separated: +# name, clone URL, pinned commit, document, line, wanted status, proof, caught. +jq https://github.com/jqlang/jq.git 94b00aa807a51a8f97bc9345c03e32982e344678 README.md 56 FAIL libjq.so.1 no +goreleaser https://github.com/goreleaser/goreleaser.git a9fb415389a2e55906ae457d9f76bb77b3cb6056 www/content/getting-started/quick-start.md 138 FAIL doesn't contain any tags no +pyenv https://github.com/pyenv/pyenv.git 7d65a4683c91b29f7501da4c8737d4e786f8cad2 README.md 187 DRIFT|FAIL --install no +shellcheck https://github.com/koalaman/shellcheck.git cd9072643932ee2008e454e2cd990e25c1dc9d49 README.md 412 FAIL exec foo; echo "Done!" no +shellcheck https://github.com/koalaman/shellcheck.git cd9072643932ee2008e454e2cd990e25c1dc9d49 README.md 451 FAIL cat file | grep foo no From 660c7e2c0be60157c233ec2e85b8087c230bd90d Mon Sep 17 00:00:00 2001 From: Douglas Adolph <116844282+dcadolph@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:43:57 -0600 Subject: [PATCH 3/4] Resolve stand-in paths to the tool's own binary --- flagcheck.go | 5 ++++ flagcheck_test.go | 16 ++++++++++ internal/docblock/docblock.go | 33 ++++++++++++++++++++ internal/docblock/docblock_test.go | 48 ++++++++++++++++++++++++++++++ internal/plan/plan.go | 25 ++++++++++++++++ internal/plan/plan_test.go | 15 ++++++++++ 6 files changed, 142 insertions(+) diff --git a/flagcheck.go b/flagcheck.go index 40fb5ef..b645f3f 100644 --- a/flagcheck.go +++ b/flagcheck.go @@ -77,9 +77,14 @@ func extractUsage(binaries []string, markdown string) map[string]*Usage { } codeLines = append(codeLines, b.Lines...) } + owned := func(name string) bool { return slices.Contains(binaries, name) } for _, line := range codeLines { line = stripComment(line) for _, seg := range splitSegments(line) { + // A stand-in path to the binary, such as pyenv's own + // ` --install`, cites the binary. Resolved first, + // its angle brackets would otherwise read as a synopsis below. + seg = docblock.ResolveToolPath(strings.TrimSpace(seg), owned) // A synopsis such as `tool schedule [add --every ]` // is a signature, not an invocation. Splitting it on pipes strands // its flags on the parent command, and that mis-attribution once diff --git a/flagcheck_test.go b/flagcheck_test.go index 939a0de..252d7d4 100644 --- a/flagcheck_test.go +++ b/flagcheck_test.go @@ -103,6 +103,22 @@ func TestExtractUsage(t *testing.T) { In: "```sh\nrclone -vvv --stats-one-line --max-stats-groups 10\n```\n", Binaries: []string{"rclone"}, WantFlags: []string{"vvv", "stats-one-line", "max-stats-groups"}, + }, { // Test 17: a stand-in path to the binary in inline code cites its flag. + In: "To set up the shell, run ` --install`.\n", + Binaries: []string{"pyenv"}, + WantFlags: []string{"install"}, + }, { // Test 18: a bare stand-in path to the binary in a block cites its flag. + In: "```sh\npath/to/pyenv --install\n```\n", + Binaries: []string{"pyenv"}, + WantFlags: []string{"install"}, + }, { // Test 19: a stand-in path to another tool cites nothing. + In: "```sh\npath/to/other-tool --install\n```\n", + Binaries: []string{"pyenv"}, + WantNone: true, + }, { // Test 20: a resolved line that still holds a slot is a synopsis. + In: "```sh\npath/to/pyenv --install \n```\n", + Binaries: []string{"pyenv"}, + WantNone: true, }} for testNum, test := range tests { t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) { diff --git a/internal/docblock/docblock.go b/internal/docblock/docblock.go index ee07f50..bb20fb3 100644 --- a/internal/docblock/docblock.go +++ b/internal/docblock/docblock.go @@ -504,6 +504,39 @@ var rePlaceholder = regexp.MustCompile( `|deadbeef|1234567|123abc)\b` + `|(^|[^/])\.\.\.($|[\s'"./])|\b[A-Z][A-Z]+_[0-9]+\b`) +// ResolveToolPath replaces a stand-in path to a program in command position, +// such as path/to/tool, /path/to/tool, , or >, +// with the program's bare name when owned reports that the session installs +// it. pyenv documents its own flag as ` --install`, and reading +// that as a placeholder left the one wrong line of its README untested. A +// stand-in path to anything else, such as path/to/file.txt or a tool the +// session does not install, is returned unchanged and stays a placeholder. +// Leading indentation and a "$ " prompt are kept. +func ResolveToolPath(line string, owned func(string) bool) string { + body := strings.TrimLeft(line, " \t") + lead := line[:len(line)-len(body)] + if strings.HasPrefix(body, "$ ") { + lead += "$ " + body = body[2:] + } + word, rest, spaced := strings.Cut(body, " ") + if spaced { + rest = " " + rest + } + if strings.Count(word, "<") != strings.Count(word, ">") { + return line + } + name, ok := strings.CutPrefix(strings.TrimPrefix(strings.NewReplacer("<", "", ">", "").Replace(word), "/"), + "path/to/") + if !ok || !reToolName.MatchString(name) || !owned(name) { + return line + } + return lead + name + rest +} + +// reToolName matches a program name with no path in it. +var reToolName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._+-]*$`) + // HasPlaceholder reports whether s holds a token the reader must replace // before running it. The planner asks it of an example line and the extractor // asks it of an install line, so both read placeholders the same way. diff --git a/internal/docblock/docblock_test.go b/internal/docblock/docblock_test.go index 1584195..ff72a39 100644 --- a/internal/docblock/docblock_test.go +++ b/internal/docblock/docblock_test.go @@ -124,6 +124,54 @@ func TestHasPlaceholder(t *testing.T) { } } +// TestResolveToolPath checks that a stand-in path to an installed program in +// command position becomes the program, and every other stand-in path stays. +func TestResolveToolPath(t *testing.T) { + t.Parallel() + + owned := func(name string) bool { return name == "pyenv" } + tests := []struct { + In string + WantResult string + }{{ // Test 0: A bare stand-in path resolves. + In: "path/to/pyenv --install bash", WantResult: "pyenv --install bash", + }, { // Test 1: An angle-bracketed stand-in path resolves. + In: " --install", WantResult: "pyenv --install", + }, { // Test 2: A rooted stand-in path resolves. + In: "/path/to/pyenv --install", WantResult: "pyenv --install", + }, { // Test 3: A nested slot resolves. + In: "> --install", WantResult: "pyenv --install", + }, { // Test 4: A slot on the program name alone resolves. + In: "path/to/ init", WantResult: "pyenv init", + }, { // Test 5: A prompt and indentation are kept. + In: " $ path/to/pyenv --install", WantResult: " $ pyenv --install", + }, { // Test 6: The command alone resolves. + In: "path/to/pyenv", WantResult: "pyenv", + }, { // Test 7: A file path stays a placeholder. + In: "path/to/file.txt --x", WantResult: "path/to/file.txt --x", + }, { // Test 8: A tool the session does not install stays a placeholder. + In: "path/to/other-tool --x", WantResult: "path/to/other-tool --x", + }, { // Test 9: A stand-in path past command position is left alone. + In: "pyenv exec path/to/pyenv", WantResult: "pyenv exec path/to/pyenv", + }, { // Test 10: A deeper path is not the program. + In: "path/to/bin/pyenv --install", WantResult: "path/to/bin/pyenv --install", + }, { // Test 11: Unbalanced brackets are left alone. + In: " --install bash\n```\n", + WantSteps: [][]planLine{{{Cmd: "tool --install bash"}}}, + }, { // Test 23: a resolved line that still holds a slot stays a placeholder. + Markdown: "```sh\npath/to/tool --install \n```\n", + WantSteps: [][]planLine{{{Cmd: "tool --install ", Skip: true}}}, + }, { // Test 24: a stand-in path to another program is not an example. + Markdown: "```sh\npath/to/other-tool --x\n```\n", + WantExcluded: 1, + }, { // Test 25: a stand-in path to a file stays a placeholder. + Markdown: "```sh\ntool run path/to/file.txt\n```\n", + WantSteps: [][]planLine{{{Cmd: "tool run path/to/file.txt", Skip: true}}}, }} for testNum, test := range tests { t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) { From d83483fdb27be188bc251e6f3b1c5d96def693ca Mon Sep 17 00:00:00 2001 From: Douglas Adolph <116844282+dcadolph@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:44:03 -0600 Subject: [PATCH 4/4] Run source builds documented as configure and make --- docs/DESIGN.md | 13 +++ docs/SECURITY.md | 8 ++ examplerun.go | 142 +++++++++++++++++++++++++----- extract.go | 110 +++++++++++++++++++++++ extract_test.go | 139 +++++++++++++++++++++++++++++ main.go | 2 +- run.go | 43 ++++++++- run_test.go | 220 ++++++++++++++++++++++++++++++++++++++++++++++ runclassify.go | 19 ++++ runscript.go | 57 ++++++++++++ toolchain.go | 6 ++ toolchain_test.go | 10 +++ 12 files changed, 745 insertions(+), 24 deletions(-) diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 9606181..631a140 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -28,6 +28,19 @@ reports which steps a brand-new user could actually complete. `directories.bin` are not read. - Runs each `git clone` recipe too: the clone and the build lines that follow it in the same code block, with GitHub SSH remotes rewritten to HTTPS for the keyless container. +- Runs a source build when a block configures the checkout (`./configure` or a cmake + generate step) and installs it (`make install`, `ninja install`, or `cmake --install`). + The recipe is the block as written, from its first command through the install and on + to the first blank line, comment, or teardown, with a leading `sudo` dropped. It runs in + `buildpack-deps:stable` on the committed tree, restored from the objects and refs of + the checkout's `.git` directory, which are streamed in so submodules resolve. The git + config, hooks, and reflogs stay on the host, since a config can hold a token. The + installed binary is smoke-tested in its own process. A loader error there, such as a + library `make install` placed where the loader does not look until `ldconfig` runs, is + a `FAIL`, because no argument starts that binary. When the document names `ldconfig` or + `LD_LIBRARY_PATH` outside the block, the same error is `BUILT`: the reader was told, and + kibble ran only the block. Any other failed smoke test stays `BUILT`. A block that also + clones, installs system packages, or pipes a remote script is not a source build. - Picks the image from the toolchain the step assumes, so a Rust project builds with `cargo` and a Node project with `npm`, and finds the installed binary wherever that toolchain puts it, even when it is not named after the package. diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 26ad79a..fd991cb 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -9,6 +9,14 @@ memory and process counts, `no-new-privileges`, and every Linux capability dropp the handful `apt` needs to install the packages the docs depend on. Nothing from the host is mounted in; the repository is streamed in as a tar of its working tree. +A documented source build (`./configure`, `make`, `make install`) is the one case that +receives more than the working tree. It is sent the objects and refs of the checkout's +`.git` directory, so the container can restore the committed tree and resolve submodules +the way a fresh clone does. That is the repository's local history, including commits and +stashes that were never pushed. Nothing else in `.git` is sent: not `config`, where a CI +checkout stores the token it fetched with and a remote URL can carry a password, not +hooks, not reflogs, and not a submodule's git directory. + The boundary that stays open is the network, because verifying an install *is* fetching it: `go install`, `cargo install`, and `npm install -g` are network operations. A malicious documented command can therefore reach out from inside the container. diff --git a/examplerun.go b/examplerun.go index 1d7da6b..382e8c7 100644 --- a/examplerun.go +++ b/examplerun.go @@ -11,6 +11,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "time" kplan "github.com/dcadolph/kibble/internal/plan" @@ -208,31 +209,90 @@ func sessionBudget(plan *kplan.Plan, install time.Duration) time.Duration { // for a small cap, and what is left is a runaway guard. const repoStreamCap = 512 << 20 +// treeSpec says which part of a checkout an archive carries. +type treeSpec struct { + // sub is the directory under the checkout that is walked, empty for all of + // it. Archive names stay relative to the checkout either way. + sub string + // skip holds the directory names left out. + skip map[string]bool + // maxFile is the size above which a file is left out, 0 for no limit. + maxFile int64 + // dirs reports whether directories get entries of their own, so that an + // empty one, such as a git ref directory, exists after extraction. + dirs bool + // only holds the names directly under the walked directory that are + // carried, nil for all of them. + only map[string]bool +} + +// leftOut reports whether the entry at path, under the walked directory root, +// is outside what the spec carries. +func (s treeSpec) leftOut(root, path string, d fs.DirEntry) bool { + if d.IsDir() && s.skip[d.Name()] { + return true + } + if s.only == nil || path == root { + return false + } + rel, err := filepath.Rel(root, path) + if err != nil { + return true + } + top, _, _ := strings.Cut(filepath.ToSlash(rel), "/") + return !s.only[top] +} + +// sessionTree is what an example session receives: the working tree without +// generated directories and without files over 2 MB, so documented example +// files exist in the container. +var sessionTree = treeSpec{skip: tarSkipDirs, maxFile: 2 << 20} + +// gitTree is what a source build receives: the objects and refs of the +// checkout's git directory, from which the container restores the committed +// tree. Nothing else in that directory is sent. Its config is where a CI +// checkout keeps the token it fetched with and where a remote URL can hold a +// password, and its hooks are programs from the host, so neither may reach a +// container that runs documented commands with the network open. +var gitTree = treeSpec{sub: ".git", dirs: true, only: map[string]bool{ + "HEAD": true, "objects": true, "refs": true, "packed-refs": true, "shallow": true, +}} + // repoFits reports whether the working tree is small enough to stream, and is // a stat-only walk so the answer is known before a container starts. Deciding // after the fact is not an option: a partially streamed repository makes a // document look broken when the missing piece is kibble's. func repoFits(dir string) bool { + return treeFits(dir, sessionTree) +} + +// treeFits reports whether the part of dir that spec selects fits under +// [repoStreamCap], walking stat-only. +func treeFits(dir string, spec treeSpec) bool { if dir == "" { return true } total := int64(0) fits := true - _ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + root := filepath.Join(dir, spec.sub) + _ = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil { return nil } - if d.IsDir() { - if tarSkipDirs[d.Name()] { + if spec.leftOut(root, path, d) { + if d.IsDir() { return filepath.SkipDir } return nil } + if d.IsDir() { + return nil + } if !d.Type().IsRegular() { return nil } info, err := d.Info() - if err != nil || info.Size() > 2<<20 { + if err != nil || (spec.maxFile > 0 && info.Size() > spec.maxFile) { return nil } if total += info.Size(); total > repoStreamCap { @@ -244,63 +304,80 @@ func repoFits(dir string) bool { return fits } -// repoTar packs the repo working tree for the session, without generated -// directories and without files over 2 MB, so documented example files exist -// in the container. Call [repoFits] first: this writes whatever it walks and -// does not decide whether the result is complete. +// repoTar packs the repo working tree for the session. Call [repoFits] first: +// this writes whatever it walks and does not decide whether the result is +// complete. func repoTar(dir string) ([]byte, bool) { var buf bytes.Buffer truncated, _ := repoTarTo(&buf, dir) return buf.Bytes(), truncated } -// repoTarTo writes the same archive to w as it walks, so nothing larger than -// one file is held at once. +// repoTarTo writes the session archive to w as it walks, so nothing larger +// than one file is held at once. func repoTarTo(w io.Writer, dir string) (bool, error) { + return treeTarTo(w, dir, sessionTree) +} + +// treeTarTo writes the part of dir that spec selects to w as a tar stream, +// one file at a time, and reports whether [repoStreamCap] cut it short. +func treeTarTo(w io.Writer, dir string, spec treeSpec) (bool, error) { if dir == "" { return false, nil } truncated := false tw := tar.NewWriter(w) total := int64(0) - _ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + root := filepath.Join(dir, spec.sub) + _ = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil { return nil } - if d.IsDir() { - if tarSkipDirs[d.Name()] { + rel, rerr := filepath.Rel(dir, path) + if rerr != nil { + return nil + } + if spec.leftOut(root, path, d) { + if d.IsDir() { return filepath.SkipDir } return nil } + if d.IsDir() { + if spec.dirs && rel != "." { + if err := tw.WriteHeader(&tar.Header{ + Typeflag: tar.TypeDir, Name: filepath.ToSlash(rel) + "/", Mode: 0o755, + }); err != nil { + return filepath.SkipAll + } + } + return nil + } if !d.Type().IsRegular() { return nil } info, err := d.Info() - if err != nil || info.Size() > 2<<20 { + if err != nil || (spec.maxFile > 0 && info.Size() > spec.maxFile) { return nil } if total += info.Size(); total > repoStreamCap { truncated = true return filepath.SkipAll } - rel, err := filepath.Rel(dir, path) - if err != nil { - return nil - } - b, err := os.ReadFile(path) + f, err := os.Open(path) if err != nil { return nil } + defer func() { _ = f.Close() }() hdr := &tar.Header{ Name: filepath.ToSlash(rel), Mode: int64(info.Mode().Perm()), - Size: int64(len(b)), + Size: info.Size(), } if err := tw.WriteHeader(hdr); err != nil { return filepath.SkipAll } - if _, err := tw.Write(b); err != nil { + if err := copyPadded(tw, f, info.Size()); err != nil { return filepath.SkipAll } return nil @@ -309,6 +386,29 @@ func repoTarTo(w io.Writer, dir string) (bool, error) { return truncated, err } +// copyPadded writes exactly size bytes to w, taken from r and then from zeros +// when r runs short. A file that shrank after its tar header was written still +// owes the archive its declared length, and stopping there would cut every +// later file out of the stream and leave a partial repository behind one file +// that changed mid-walk. +func copyPadded(w io.Writer, r io.Reader, size int64) error { + n, err := io.CopyN(w, r, size) + if err == nil { + return nil + } + _, err = io.CopyN(w, zeroReader{}, size-n) + return err +} + +// zeroReader reads as an endless run of zero bytes. +type zeroReader struct{} + +// Read fills p with zeros. +func (zeroReader) Read(p []byte) (int, error) { + clear(p) + return len(p), nil +} + // tarSkipDirs are directories a build produces or a package manager fills. // Streaming them wastes the budget on output nobody documents, and on a big // repository it pushes the source a document needs past the cap. diff --git a/extract.go b/extract.go index 8dd4b74..f5ca3ba 100644 --- a/extract.go +++ b/extract.go @@ -50,6 +50,9 @@ type InstallStep struct { // doc is the document an example step replays, relative to the repository // root. Empty on every other kind of step. doc string + // loaderNote is the loader step, such as ldconfig, that the document names + // outside a source-build recipe. Empty when it names none there. + loaderNote string } // Extractor finds install steps in README markdown. @@ -197,6 +200,16 @@ func ExtractorWith(owned repoNames) Extractor { } steps = append(steps, s) } + if s, ok := sourceBuildStep(repo, block); ok { + // Two builds can end on the same install line, one from git + // and one from a tarball, so the whole recipe is the key. + key := repo + "|source|" + strings.Join(s.Block, "\n") + if !seen[key] { + seen[key] = true + s.loaderNote = loaderNoteOutside(markdown, s.Block) + steps = append(steps, s) + } + } } // A tap documented anywhere in the document covers every brew step, // since a reader adds it once and installs many formulas from it. @@ -287,6 +300,103 @@ func installRecipe(lines []string) []string { return out } +var ( + // reConfigure matches the configure step of an autotools build. + reConfigure = regexp.MustCompile(`^(?:sh\s+)?\./configure(\s|$)`) + // reCMakeGenerate matches a cmake call that generates a build tree, which + // is any cmake call that does not build, install, or run a script. + reCMakeGenerate = regexp.MustCompile(`^cmake\s+(?:[^-]|-[^-EP])`) + // reBuildInstall matches the step that installs what a configured build + // produced. The space before install keeps make uninstall out. + reBuildInstall = regexp.MustCompile(`^(?:make|ninja)\b.*\sinstall\b|^cmake\s+--install\b`) + // reLeadingSudo matches a sudo opening a command, which the container, + // already root and without sudo installed, drops. + reLeadingSudo = regexp.MustCompile(`(^|&&\s*|;\s*)sudo\s+(?:-E\s+)?`) +) + +// reLoaderStep matches the ways a document tells a reader to make an installed +// shared library findable. +var reLoaderStep = regexp.MustCompile(`\bldconfig\b|\bLD_LIBRARY_PATH\b`) + +// loaderNoteOutside returns the loader step, such as ldconfig, that a document +// names somewhere other than in the recipe, or empty when the recipe runs one +// itself or the document names none. A recipe that ran the step and still left +// a binary unable to start has no excuse. One that left it for a later +// sentence was run without it, and that is kibble's gap, not the document's. +func loaderNoteOutside(markdown string, recipe []string) string { + if reLoaderStep.MatchString(strings.Join(recipe, "\n")) { + return "" + } + return reLoaderStep.FindString(markdown) +} + +// sourceBuildStep returns the install route a block documents by building the +// checkout from source: an autotools `./configure` or a cmake generate step, +// followed in the same block by `make install`, `ninja install`, or `cmake +// --install`. jq documents only this route, so kibble tested nothing in jq and +// missed that the binary `make install` leaves cannot find its own library +// until ldconfig runs. The recipe is the block from its first command through +// the install line, and on past it to the first blank line, comment, or +// teardown, since a block that follows the install with `sudo ldconfig` +// documents a working install and must be run whole. Every line runs as +// written apart from a leading sudo. A block that clones is the clone route's +// recipe, and one that installs system packages or pipes a remote script needs +// the routes kibble refuses, so neither is a source build. A recipe holding a +// placeholder is recorded unrun. +func sourceBuildStep(repo string, block docblock.Block) (InstallStep, bool) { + if block.Span || !docblock.ShellLangs[block.Lang] { + return InstallStep{}, false + } + var recipe []string + var step *InstallStep + configured := false + for _, raw := range docblock.PrepareLines(block.Lines) { + raw = strings.TrimPrefix(strings.TrimSpace(raw), "$ ") + written := strings.TrimSpace(stripComment(raw)) + refused := reGitClone.MatchString(written) || reOSPackage.MatchString(written) || + reScriptInstall.MatchString(written) + if step != nil && (written == "" || strings.HasPrefix(written, "#") || + reTeardown.MatchString(written) || refused) { + break + } + if written == "" || strings.HasPrefix(written, "#") { + continue + } + if refused { + return InstallStep{}, false + } + l := reLeadingSudo.ReplaceAllString(written, "$1") + recipe = append(recipe, l) + if step != nil { + continue + } + for _, seg := range splitSegments(l) { + seg = strings.TrimSpace(seg) + if reConfigure.MatchString(seg) || reCMakeGenerate.MatchString(seg) { + configured = true + continue + } + if configured && reBuildInstall.MatchString(seg) { + step = &InstallStep{Repo: repo, Kind: "source-build", Raw: written} + if block.Line > 0 { + step.Line = docblock.SourceLineIndex(block)(raw) + } + break + } + } + } + if step == nil { + return InstallStep{}, false + } + step.Block, step.Run = recipe, true + for _, r := range recipe { + if docblock.HasPlaceholder(docblock.CommandHead(r)) { + step.Run = false + } + } + return *step, true +} + // classifyLine matches one code line against the known install patterns. A // line holding a placeholder is recorded with Run false, since running it // would test the stand-in rather than the document. diff --git a/extract_test.go b/extract_test.go index ec80289..734d486 100644 --- a/extract_test.go +++ b/extract_test.go @@ -5,6 +5,8 @@ import ( "testing" "github.com/google/go-cmp/cmp" + + "github.com/dcadolph/kibble/internal/docblock" ) // TestClassifyLine checks that install commands are recognized and classified. @@ -598,3 +600,140 @@ func TestExtractListRecipe(t *testing.T) { }) } } + +// TestSourceBuildStep checks which blocks document building the checkout from +// source, and that the recipe is the block as written apart from sudo. +func TestSourceBuildStep(t *testing.T) { + t.Parallel() + tests := []struct { + In string + WantRaw string + WantRecipe []string + WantRun bool + Want bool + }{{ // Test 0: jq's block is a route, sudo dropped and make clean kept as written. + In: "```console\ngit submodule update --init # if building from git\nautoreconf -i\n" + + "./configure --with-oniguruma=builtin\nmake clean # if upgrading\nmake -j8\nmake check\n" + + "sudo make install\n```\n", + WantRaw: "sudo make install", + WantRecipe: []string{"git submodule update --init", "autoreconf -i", + "./configure --with-oniguruma=builtin", "make clean", "make -j8", "make check", "make install"}, + WantRun: true, Want: true, + }, { // Test 1: configure with no install is not a route. + In: "```sh\n./configure\nmake\n```\n", + }, { // Test 2: make install with no configure step is not a route. + In: "```sh\nmake\nsudo make install\n```\n", + }, { // Test 3: a cmake generate, build, and install is a route. + In: "```sh\ncmake -B build\ncmake --build build\nsudo cmake --install build\n```\n", + WantRaw: "sudo cmake --install build", + WantRecipe: []string{"cmake -B build", "cmake --build build", "cmake --install build"}, + WantRun: true, Want: true, + }, { // Test 4: a block that clones belongs to the clone route. + In: "```sh\ngit clone https://github.com/o/r\ncd r\n./configure\nmake install\n```\n", + }, { // Test 5: a block that installs system packages needs a refused route. + In: "```sh\nsudo apt-get install -y libfoo-dev\n./configure\nmake install\n```\n", + }, { // Test 6: lines after the install run, up to a blank line or a teardown. + In: "```sh\n./configure\nmake\nsudo make install\nsudo ldconfig\n\nsudo make uninstall\n```\n", + WantRaw: "sudo make install", + WantRecipe: []string{"./configure", "make", "make install", "ldconfig"}, + WantRun: true, Want: true, + }, { // Test 7: a placeholder in the recipe records the route unrun. + In: "```sh\n./configure --prefix=\nmake install\n```\n", + WantRaw: "make install", + WantRecipe: []string{"./configure --prefix=", "make install"}, + Want: true, + }, { // Test 8: inline code is not a recipe. + In: "Run `./configure && make && sudo make install` to build.\n", + }, { // Test 9: make uninstall is not an install. + In: "```sh\n./configure\nmake uninstall\n```\n", + }, { // Test 10: a one-line chain is a route. + In: "```sh\n./configure && make && sudo make install\n```\n", + WantRaw: "./configure && make && sudo make install", + WantRecipe: []string{"./configure && make && make install"}, + WantRun: true, Want: true, + }, { // Test 11: a cmake build-only call does not count as the generate step. + In: "```sh\ncmake --build build\ncmake --install build\n```\n", + }, { // Test 12: a prose block is not shell. + In: "```yaml\n./configure\nmake install\n```\n", + }} + for testNum, test := range tests { + t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) { + t.Parallel() + var got InstallStep + found := false + for _, b := range docblock.CodeBlocks(test.In) { + if s, ok := sourceBuildStep("repo", b); ok { + got, found = s, true + } + } + if found != test.Want { + t.Fatalf("found = %v, want %v (step %+v)", found, test.Want, got) + } + if !found { + return + } + if diff := cmp.Diff(test.WantRaw, got.Raw); diff != "" { + t.Errorf("raw mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(test.WantRecipe, got.Block); diff != "" { + t.Errorf("recipe mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(test.WantRun, got.Run); diff != "" { + t.Errorf("run mismatch (-want +got):\n%s", diff) + } + }) + } +} + +// TestExtractSourceBuildRecipes checks that two builds ending on the same +// install line are two routes, and that a loader step the document gives +// outside a recipe is recorded on it. +func TestExtractSourceBuildRecipes(t *testing.T) { + t.Parallel() + fromGit := "```sh\nautoreconf -i\n./configure\nmake\nsudo make install\n```\n" + fromTar := "```sh\n./configure\nmake\nsudo make install\n```\n" + tests := []struct { + In string + WantNotes []string + }{{ // Test 0: two recipes with one install line are both kept. + In: fromGit + "\nOr from a tarball:\n\n" + fromTar, WantNotes: []string{"", ""}, + }, { // Test 1: the same recipe written twice is one route. + In: fromTar + "\nAgain:\n\n" + fromTar, WantNotes: []string{""}, + }, { // Test 2: ldconfig in a later sentence is noted. + In: fromTar + "\nOn Linux run `sudo ldconfig` afterwards.\n", WantNotes: []string{"ldconfig"}, + }, { // Test 3: a library path variable in a later block is noted. + In: fromTar + "\n```sh\nexport LD_LIBRARY_PATH=/usr/local/lib\n```\n", + WantNotes: []string{"LD_LIBRARY_PATH"}, + }, { // Test 4: a recipe that runs ldconfig itself has nothing left to note. + In: "```sh\n./configure\nmake\nsudo make install\nsudo ldconfig\n```\n\nRun ldconfig again if needed.\n", + WantNotes: []string{""}, + }} + for testNum, test := range tests { + t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) { + t.Parallel() + var notes []string + for _, s := range ExtractorWith(repoNames{}).Extract("repo", test.In) { + if s.Kind == "source-build" { + notes = append(notes, s.loaderNote) + } + } + if diff := cmp.Diff(test.WantNotes, notes); diff != "" { + t.Errorf("loader notes mismatch (-want +got):\n%s", diff) + } + }) + } +} + +// TestExtractSourceBuildLine checks that the route points at the README line +// of its install command, sudo and all. +func TestExtractSourceBuildLine(t *testing.T) { + t.Parallel() + md := "# T\n\n```sh\n./configure\nmake\nsudo make install\n```\n" + steps := ExtractorWith(repoNames{}).Extract("repo", md) + if len(steps) != 1 || steps[0].Kind != "source-build" { + t.Fatalf("steps = %+v, want one source-build", steps) + } + if diff := cmp.Diff(6, steps[0].Line); diff != "" { + t.Errorf("line mismatch (-want +got):\n%s", diff) + } +} diff --git a/main.go b/main.go index fd9ed08..a3f9c17 100644 --- a/main.go +++ b/main.go @@ -706,7 +706,7 @@ func unrunResult(step InstallStep) Result { Detail: "kibble does not run system package installs; verify this one by hand", } case step.Kind == "git-clone" || step.Kind == "go-install" || step.Kind == "brew" || - pkgKinds[step.Kind].Ecosystem != "": + step.Kind == "source-build" || pkgKinds[step.Kind].Ecosystem != "": // These kinds run unless the line holds a placeholder, which is the // only reason one is recorded unrun. res = Result{ diff --git a/run.go b/run.go index dae97ea..205b174 100644 --- a/run.go +++ b/run.go @@ -4,7 +4,10 @@ import ( "context" "errors" "fmt" + "io" + "os" "os/exec" + "path/filepath" "regexp" "strings" "time" @@ -135,6 +138,11 @@ func (d *DockerRunner) Run(ctx context.Context, step InstallStep) Result { return res case "git-clone": script = cloneScriptFor(step, int(d.Timeout.Seconds())) + case "source-build": + if blocked := sourceBlocked(step.dir); blocked != "" { + return Result{Step: step, Status: StatusSkipped, Reason: ReasonMissingDependency, Detail: blocked} + } + script = sourceScriptFor(step, int(d.Timeout.Seconds())) default: if _, isPkg := pkgKinds[step.Kind]; isPkg { script = pkgScriptFor(step, int(d.Timeout.Seconds())) + helpProbe(step) @@ -156,10 +164,22 @@ func (d *DockerRunner) Run(ctx context.Context, step InstallStep) Result { // The official Go images pin GOTOOLCHAIN=local, so a module asking for a // newer Go than the image ships fails to install. A reader running the // default fetches that toolchain, so the session matches them instead. - args := append([]string{"run", "--rm", "--name", name}, - append(sandbox.Args(), "-e", "GOTOOLCHAIN=auto", image, "bash", "-c", script)...) + head := []string{"run", "--rm", "--name", name} + if step.Kind == "source-build" { + head = append(head, "-i") + } + args := append(head, append(sandbox.Args(), "-e", "GOTOOLCHAIN=auto", image, "bash", "-c", script)...) cmd := exec.CommandContext(ctx, sandbox.Bin(), args...) cmd.Cancel = sandbox.RemoveFunc(cmd, name) + if step.Kind == "source-build" { + pr, pw := io.Pipe() + go func() { + _, err := treeTarTo(pw, step.dir, gitTree) + _ = pw.CloseWithError(err) + }() + defer func() { _ = pr.Close() }() + cmd.Stdin = pr + } out, _ := cmd.CombinedOutput() if ctx.Err() != nil && errors.Is(context.Cause(ctx), context.Canceled) { return Result{ @@ -225,6 +245,22 @@ var reNetworkError = regexp.MustCompile( `\b(EAI_AGAIN|ENOTFOUND|ECONNRESET|ETIMEDOUT|ECONNREFUSED|ENETUNREACH|EHOSTUNREACH)\b|` + `getaddrinfo|socket hang up|request to \S+ failed, reason`) +// sourceBlocked returns why a source build cannot be handed the checkout it +// builds, or empty when it can. The build restores the committed tree from the +// git directory, so a checkout without one, or a worktree whose .git is only a +// pointer to another repository's, has nothing to send. +func sourceBlocked(dir string) string { + info, err := os.Stat(filepath.Join(dir, ".git")) + switch { + case err != nil || !info.IsDir(): + return "builds the checkout from its git history, and this checkout has no .git directory to send" + case !treeFits(dir, gitTree): + return fmt.Sprintf("builds the checkout from its git history, which exceeds the %d MB kibble will stream", + repoStreamCap>>20) + } + return "" +} + // imageFor returns the container image a step runs in. A clone recipe runs in // the image that provides the toolchain its commands assume, so a Rust or Node // project builds with the tools its docs were written for. Everything else runs @@ -236,6 +272,9 @@ func (d *DockerRunner) imageFor(step InstallStep) (image, blocked string) { } return d.Image, "" } + if step.Kind == "source-build" { + return sourceBuildImage, "" + } if step.Kind != "git-clone" { return d.Image, "" } diff --git a/run_test.go b/run_test.go index a59050b..fdf181e 100644 --- a/run_test.go +++ b/run_test.go @@ -1,8 +1,11 @@ package main import ( + "archive/tar" + "bytes" "context" "fmt" + "io" "os" "os/exec" "path/filepath" @@ -1216,3 +1219,220 @@ func TestClassifyBinKind(t *testing.T) { }) } } + +// TestClassifySourceBuildLoader checks that a loader refusing to start the +// binary convicts a source build, and only a source build. +func TestClassifySourceBuildLoader(t *testing.T) { + t.Parallel() + loader := "BUILDCODE=0\nSMOKECODE=127\nSMOKELINE=/usr/local/bin/jq: error while loading shared libraries: " + + "libjq.so.1: cannot open shared object file: No such file or directory\n" + tests := []struct { + Kind string + In string + Note string + WantStatus Status + }{{ // Test 0: a source build whose binary cannot load its library fails. + Kind: "source-build", In: loader, WantStatus: StatusFail, + }, { // Test 1: the same smoke line after a go install stays BUILT. + Kind: "go-install", In: loader, WantStatus: StatusBuilt, + }, { // Test 2: any other failed smoke test after a source build stays BUILT. + Kind: "source-build", In: "BUILDCODE=0\nSMOKECODE=2\nSMOKELINE=usage: tool [-x]\n", WantStatus: StatusBuilt, + }, { // Test 3: a passing smoke test after a source build is VERIFIED. + Kind: "source-build", In: "BUILDCODE=0\nSMOKECODE=0\nSMOKELINE=jq-1.8.1\n", WantStatus: StatusVerified, + }, { // Test 4: a checkout the container could not restore is kibble's error. + Kind: "source-build", In: "kibble could not restore the checkout from its git history\n", + WantStatus: StatusError, + }, { // Test 5: a loader step the document gives outside the recipe keeps it BUILT. + Kind: "source-build", In: loader, Note: "ldconfig", WantStatus: StatusBuilt, + }} + for testNum, test := range tests { + t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) { + t.Parallel() + got := classify(InstallStep{Repo: "jq", Kind: test.Kind, loaderNote: test.Note}, test.In, time.Second) + if diff := cmp.Diff(test.WantStatus, got.Status); diff != "" { + t.Errorf("status mismatch (-want +got):\n%s", diff) + } + if test.WantStatus == StatusFail && !strings.Contains(got.Detail, "libjq.so.1") { + t.Errorf("detail %q does not name the library", got.Detail) + } + if test.Note != "" && !strings.Contains(got.Detail, test.Note) { + t.Errorf("detail %q does not name the step the document gives", got.Detail) + } + }) + } +} + +// TestSourceBlocked checks that a source build needs a git directory to send. +func TestSourceBlocked(t *testing.T) { + t.Parallel() + withDir := t.TempDir() + if err := os.MkdirAll(filepath.Join(withDir, ".git", "refs"), 0o755); err != nil { + t.Fatal(err) + } + withFile := t.TempDir() + if err := os.WriteFile(filepath.Join(withFile, ".git"), []byte("gitdir: /elsewhere\n"), 0o600); err != nil { + t.Fatal(err) + } + tests := []struct { + Dir string + WantNone bool + }{{ // Test 0: a git directory can be sent. + Dir: withDir, WantNone: true, + }, { // Test 1: a worktree pointer cannot. + Dir: withFile, + }, { // Test 2: no git at all cannot. + Dir: t.TempDir(), + }} + for testNum, test := range tests { + t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) { + t.Parallel() + if got := sourceBlocked(test.Dir); (got == "") != test.WantNone { + t.Errorf("sourceBlocked = %q, want none %v", got, test.WantNone) + } + }) + } +} + +// TestGitTreeStream checks that a source build's stream carries the objects +// and refs of the git directory, empty directories and large packs included, +// and nothing else: not the config a token or a remote password lives in, not +// the host's hooks, not a submodule's git directory, and not the working tree. +func TestGitTreeStream(t *testing.T) { + t.Parallel() + dir := t.TempDir() + for _, d := range []string{".git/refs/tags", ".git/objects/pack", ".git/hooks", ".git/logs", + ".git/modules/dep/objects"} { + if err := os.MkdirAll(filepath.Join(dir, d), 0o755); err != nil { + t.Fatal(err) + } + } + for name, body := range map[string]string{ + ".git/HEAD": "ref: refs/heads/main\n", + ".git/packed-refs": "# pack-refs\n", + ".git/shallow": "0000\n", + ".git/config": "[http \"https://github.com/\"]\n\textraheader = AUTHORIZATION: basic c2VjcmV0\n", + ".git/hooks/post-commit": "#!/bin/sh\n", + ".git/logs/HEAD": "clone: from https://user:secret@example.test/r\n", + ".git/FETCH_HEAD": "https://user:secret@example.test/r\n", + ".git/modules/dep/config": "[remote \"origin\"]\n\turl = https://user:secret@example.test/d\n", + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + big := make([]byte, 3<<20) + if err := os.WriteFile(filepath.Join(dir, ".git/objects/pack/p.pack"), big, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "main.c"), []byte("int main(){}\n"), 0o600); err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + if _, err := treeTarTo(&buf, dir, gitTree); err != nil { + t.Fatal(err) + } + got := map[string]int64{} + tr := tar.NewReader(&buf) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + got[hdr.Name] = hdr.Size + } + want := map[string]int64{ + ".git/": 0, ".git/refs/": 0, ".git/refs/tags/": 0, ".git/objects/": 0, + ".git/objects/pack/": 0, ".git/objects/pack/p.pack": 3 << 20, + ".git/HEAD": 21, ".git/packed-refs": 12, ".git/shallow": 5, + } + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("archive mismatch (-want +got):\n%s", diff) + } +} + +// TestCopyPadded checks that a source running short of its declared size is +// filled out with zeros, so the tar entry it feeds stays whole. +func TestCopyPadded(t *testing.T) { + t.Parallel() + tests := []struct { + In string + Size int64 + WantResult string + }{{ // Test 0: a source of the declared size is copied as is. + In: "abcd", Size: 4, WantResult: "abcd", + }, { // Test 1: a source that shrank is padded to the declared size. + In: "ab", Size: 4, WantResult: "ab\x00\x00", + }, { // Test 2: a source that grew is cut at the declared size. + In: "abcdef", Size: 4, WantResult: "abcd", + }, { // Test 3: an empty source of declared size zero writes nothing. + In: "", Size: 0, WantResult: "", + }} + for testNum, test := range tests { + t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) { + t.Parallel() + var buf bytes.Buffer + if err := copyPadded(&buf, strings.NewReader(test.In), test.Size); err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(test.WantResult, buf.String()); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} + +// TestDockerSourceBuild builds a checkout whose installed binary needs its own +// shared library. Without ldconfig the documented install leaves a binary that +// cannot start, and with it the same project verifies. +func TestDockerSourceBuild(t *testing.T) { + if testing.Short() { + t.Skip("container test skipped in short mode") + } + if os.Getenv("KIBBLE_INTEGRATION") == "" { + t.Skip("set KIBBLE_INTEGRATION=1 to run container tests") + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + if err := sandbox.Available(ctx); err != nil { + t.Skipf("docker is not reachable: %v", err) + } + dir := t.TempDir() + files := map[string]string{ + "configure": "#!/bin/sh\necho configured\n", + "lib.c": "const char *greeting(void) { return \"hello 1.0\"; }\n", + "main.c": "#include \nconst char *greeting(void);\nint main(void) { puts(greeting()); return 0; }\n", + "Makefile": "all: hello\nlibhello.so: lib.c\n\tcc -shared -fPIC -o $@ lib.c\n" + + "hello: main.c libhello.so\n\tcc -o $@ main.c -L. -lhello\ninstall: all\n" + + "\tinstall -m755 hello /usr/local/bin/hello\n\tinstall -m644 libhello.so /usr/local/lib/libhello.so\n", + } + for name, body := range files { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o755); err != nil { + t.Fatal(err) + } + } + for _, args := range [][]string{ + {"init", "-q"}, {"add", "-A"}, + {"-c", "user.name=t", "-c", "user.email=t@example.invalid", "commit", "-qm", "init"}, + {"config", "--local", "http.https://example.invalid/.extraheader", "AUTHORIZATION: kibble-marker"}, + } { + if out, err := exec.Command("git", append([]string{"-C", dir}, args...)...).CombinedOutput(); err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } + } + runner := &DockerRunner{Image: "golang:1.26", Timeout: 120 * time.Second} + // The first line fails the recipe if the token planted in the host's git + // config reached the container, so neither result below can be met then. + recipe := []string{"if grep -rqs kibble-marker .git; then exit 9; fi", "./configure", "make", "make install"} + res := runner.Run(ctx, InstallStep{Repo: "hello", Kind: "source-build", Run: true, Block: recipe, dir: dir}) + if res.Status != StatusFail || !strings.Contains(res.Detail, "libhello.so") { + t.Errorf("without ldconfig: status %s detail %q, want FAIL naming libhello.so", res.Status, res.Detail) + } + recipe = append(recipe, "ldconfig") + res = runner.Run(ctx, InstallStep{Repo: "hello", Kind: "source-build", Run: true, Block: recipe, dir: dir}) + if res.Status != StatusVerified || res.SmokeLine != "hello 1.0" { + t.Errorf("with ldconfig: status %s smoke %q detail %q, want VERIFIED", res.Status, res.SmokeLine, res.Detail) + } +} diff --git a/runclassify.go b/runclassify.go index 9bade32..e7fc091 100644 --- a/runclassify.go +++ b/runclassify.go @@ -111,6 +111,21 @@ func classify(step InstallStep, out string, dur time.Duration) Result { res.Status = StatusCrossArch res.SmokeLine = "" res.Detail = "installed, but the binary targets another architecture, smoke test not possible here" + case step.Kind == "source-build" && reLoaderError.MatchString(res.SmokeLine) && step.loaderNote != "": + // The document covers the loader in a place the recipe did not reach, + // so the reader who follows all of it may well have a working binary. + // kibble ran only the block, and its own shortfall is not a failure. + res.Status = StatusBuilt + res.Detail = fmt.Sprintf("installed, but the binary cannot start until the %s step the document "+ + "gives outside this block, which kibble did not run", step.loaderNote) + case step.Kind == "source-build" && reLoaderError.MatchString(res.SmokeLine): + // A failed smoke test is normally only BUILT, since --version and --help + // can fail on a working tool. A dynamic loader refusing to start it + // cannot: no argument gets that binary to run. After a source build + // every library it linked against was present at link time, so the + // install the document gave is what left it unable to find one. + res.Status = StatusFail + res.Detail = "the documented install leaves a binary that cannot start: " + res.SmokeLine default: res.Status = StatusBuilt res.Detail = fmt.Sprintf("binary built but smoke exit=%d", smokeCode) @@ -131,6 +146,10 @@ var reFlagMarker = regexp.MustCompile(`^KIBBLE-FLAG (--?\S+) CODE=(-?\d+)$`) // failing its smoke test. var reArchMismatch = regexp.MustCompile(`qemu-\w+: |Exec format error|cannot execute binary file`) +// reLoaderError matches the dynamic loader refusing to start a binary because +// a shared library it needs cannot be found or loaded. +var reLoaderError = regexp.MustCompile(`error while loading shared libraries|Library not loaded`) + // reANSI matches the escape sequences a program writes to color or reposition // its own output. Both the color form and the wider set of control sequences // are covered, since a progress bar redrawing itself is as unwelcome in a diff --git a/runscript.go b/runscript.go index bdb64e8..d36ac41 100644 --- a/runscript.go +++ b/runscript.go @@ -256,6 +256,63 @@ printf 'SMOKECODE=%%d\n' "$scode" printf 'SMOKELINE=%%s\n' "$(printf '%%s' "$sout" | head -n1 | cut -c1-70)" ` +// sourceScriptFor builds the container script for a source-build recipe. The +// checkout's git history arrives on stdin, so the build runs on the committed +// tree with its submodules reachable, as a reader's clone would, rather than +// on whatever build output the host's working tree holds. +func sourceScriptFor(step InstallStep, timeoutSecs int) string { + body := strings.ReplaceAll(strings.Join(step.Block, "\n"), "'", `'\''`) + name := "" + if kplan.IsSimpleWord(step.Repo) { + name = step.Repo + } + return fmt.Sprintf(sourceScript, timeoutSecs, body, name) +} + +// sourceScript restores the checkout from the git history on stdin, runs the +// documented build and install, and smoke-tests the binary the install put on +// PATH, in a process of its own after the recipe's shell has exited. That is +// what a reader does next, and it is where jq's installed binary fails to find +// the library `make install` placed beside it. Only a binary written during +// the recipe counts, so a same-named tool the image already ships is never the +// one tested. The stream holds objects and refs only, so the repository they +// land in is one the container made itself, with a config of its own. It +// prints the same markers as installScript. +const sourceScript = `set -u +mkdir -p /work/repo +cd /work/repo +if ! git init -q . 2>/dev/null || ! tar -xf - 2>/dev/null || + ! git config --global --add safe.directory '*' || ! git reset -q --hard HEAD 2>/dev/null; then + printf 'kibble could not restore the checkout from its git history\n' + exit 0 +fi +touch /tmp/kibble-mark +out=$(timeout %[1]d bash -ec '%[2]s' 2>&1); code=$? +if [ "$code" -ne 0 ]; then + printf 'BUILDCODE=%%d\n' "$code" + printf '%%s\n' "$out" | grep -iE 'error(\[[A-Z]|:| )' | head -n 4 + printf '%%s\n' "$out" | tail -n 10 + exit 0 +fi +printf 'BUILDCODE=0\n' +bin='' +if [ -n '%[3]s' ]; then + bin=$(command -v '%[3]s' 2>/dev/null || true) + case "$bin" in /*) [ "$bin" -nt /tmp/kibble-mark ] || bin='' ;; *) bin='' ;; esac +fi +if [ -z "$bin" ]; then + bin=$(IFS=:; find $PATH -maxdepth 1 -type f -perm -u+x -newer /tmp/kibble-mark 2>/dev/null | head -n1) +fi +if [ -z "$bin" ]; then + printf 'NOBIN=1\n' + exit 0 +fi +sout=$(cd / && timeout 15 "$bin" --version 2>&1); scode=$? +if [ "$scode" -ne 0 ]; then sout=$(cd / && timeout 15 "$bin" --help 2>&1); scode=$?; fi +printf 'SMOKECODE=%%d\n' "$scode" +printf 'SMOKELINE=%%s\n' "$(printf '%%s' "$sout" | head -n1 | cut -c1-160)" +` + // installScript builds, then smoke-tests, one module inside the container. It // prints BUILDCODE, SMOKECODE, and SMOKELINE markers for the parent to read. A // build timeout surfaces as BUILDCODE=124 so it is not mistaken for a failure. diff --git a/toolchain.go b/toolchain.go index 7f54b91..3652b73 100644 --- a/toolchain.go +++ b/toolchain.go @@ -29,6 +29,12 @@ var toolchains = map[string]toolchain{ "go": {Name: "go", Image: ""}, } +// sourceBuildImage is the image a source build runs in. It is Debian's own +// build image, with a C and C++ compiler, make, autoconf, automake, libtool, +// pkg-config, git, and the common development libraries, which is the machine +// a configure-and-make README assumes its reader has. +const sourceBuildImage = "buildpack-deps:stable" + // pkgKind describes how one package manager installs a documented tool and // where the binaries it produces land. type pkgKind struct { diff --git a/toolchain_test.go b/toolchain_test.go index 4555911..20945da 100644 --- a/toolchain_test.go +++ b/toolchain_test.go @@ -261,3 +261,13 @@ func TestPythonStepImage(t *testing.T) { t.Errorf("session image = %q ok=%v, want python:3.14", image, ok) } } + +// TestSourceBuildImage checks that a source build runs in the build image +// whatever the runner's default is. +func TestSourceBuildImage(t *testing.T) { + t.Parallel() + d := &DockerRunner{Image: "golang:1.26"} + if image, _ := d.imageFor(InstallStep{Kind: "source-build", dir: t.TempDir()}); image != sourceBuildImage { + t.Errorf("image = %q, want %q", image, sourceBuildImage) + } +}