diff --git a/.github/workflows/corpus.yml b/.github/workflows/corpus.yml
index e274606..c2aa7a4 100644
--- a/.github/workflows/corpus.yml
+++ b/.github/workflows/corpus.yml
@@ -50,7 +50,7 @@ jobs:
rc=0
/tmp/kibble -json "$dir" > "/tmp/${name}.json" || rc=$?
# Exit 1 is kibble reporting failures and exit 3 is kibble
- # reporting that every row is a skip. Both are verdicts with a
+ # reporting that it tested no claim. Both are verdicts with a
# JSON array behind them, and a 0,0,0 pin is exactly the shape
# that exits 3. Any other nonzero is kibble unable to do its job,
# and blaming the corpus for that would be the same mistake as
@@ -172,3 +172,66 @@ jobs:
echo "::endgroup::"
done < corpus/mutations.tsv
exit "$status"
+ # Each recall row is a real bug a person found where kibble tested
+ # nothing. A caught=no row is a score, printed and never fatal, so the
+ # misses stay visible without turning the job red for work not yet
+ # built. A caught=yes row is a regression guard: losing that catch fails
+ # the job. KIBBLE and WORK let the same loop run on a laptop.
+ - name: Run kibble against the recall corpus
+ if: always()
+ shell: bash
+ run: |
+ set -uo pipefail
+ KIBBLE="${KIBBLE:-/tmp/kibble}"
+ WORK="${WORK:-/tmp/recall}"
+ test -x "$KIBBLE" || go build -o "$KIBBLE" .
+ mkdir -p "$WORK"
+ status=0
+ rows=0
+ caught=0
+ while IFS=$'\t' read -r name url sha doc line want proof expect; do
+ case "$name" in ""|\#*) continue ;; esac
+ rows=$((rows + 1))
+ dir="$WORK/${name}-${sha:0:12}"
+ out="$WORK/${name}-${sha:0:12}.json"
+ # Two rows can name one repository. It runs once.
+ if [ ! -s "$out" ]; then
+ echo "::group::${name} at ${sha:0:12}"
+ if [ ! -d "$dir/.git" ]; then
+ git init -q "$dir"
+ git -C "$dir" fetch -q --depth 1 "$url" "$sha"
+ git -C "$dir" checkout -q FETCH_HEAD
+ fi
+ rc=0
+ "$KIBBLE" -json "$dir" > "$out" || rc=$?
+ # Exit 1 reports failures and exit 3 reports nothing tested.
+ # Both are verdicts. Anything else is kibble unable to run.
+ if { [ "$rc" -gt 1 ] && [ "$rc" -ne 3 ]; } || ! jq -e 'type=="array"' "$out" >/dev/null 2>&1; then
+ echo "${name}: kibble itself failed (exit ${rc})"
+ rm -f "$out"
+ fi
+ echo "::endgroup::"
+ fi
+ hit=0
+ if [ -s "$out" ]; then
+ hit=$(jq -r --arg doc "$doc" --arg want "$want" --arg proof "$proof" '
+ ($want | split("|")) as $ws
+ | [ (.[] | select((.status as $s | $ws | index($s)) and ((.detail // "") | contains($proof)))),
+ (.[] | .steps[]? | select((.doc // "") | endswith($doc)) | .lines[]
+ | select((.status as $s | $ws | index($s))
+ and (((.cmd // "") + "\n" + (.detail // "")) | contains($proof)))) ]
+ | length' "$out")
+ fi
+ where="${name} ${doc}:${line} ${want} naming \"${proof}\""
+ if [ "$hit" -gt 0 ]; then
+ caught=$((caught + 1))
+ echo "${where}: caught"
+ elif [ "$expect" = "yes" ]; then
+ echo "::error::${where}: no longer caught, and the row says it was"
+ status=1
+ else
+ echo "${where}: not caught yet"
+ fi
+ done < corpus/recall.tsv
+ echo "${caught} of ${rows} recall rows caught"
+ exit "$status"
diff --git a/README.md b/README.md
index ccaa4f4..456fd0a 100644
--- a/README.md
+++ b/README.md
@@ -189,12 +189,15 @@ it, so a CI log can always say which kibble produced its verdict.
The exit code tells a script what happened:
-- `0` means at least one check ran and none failed.
+- `0` means at least one documented claim was tested and none failed.
- `1` means the run failed. `-strict` widens what counts as a failure.
- `2` means a usage error, such as no path and no README in the current directory,
Docker unavailable, or less than 2 GiB free in Docker.
-- `3` means nothing was checked: every row is a skip, so no install ran and no example
- could run either. The action fails on purpose, since a check that ran nothing must not pass.
+- `3` means nothing was tested: no row and no example line reached `VERIFIED`, `FAIL`,
+ `GAP`, or `DRIFT`. A brew formula lookup, a recipe with no binary to smoke-test, a
+ timeout, and a skip all leave the documents unchecked, and the summary names up to three
+ of those causes. The action fails on purpose, since a check that tested nothing must not
+ pass.
- `130` means the run was interrupted.
## What it has found
diff --git a/corpus/recall.tsv b/corpus/recall.tsv
new file mode 100644
index 0000000..ed6a5cd
--- /dev/null
+++ b/corpus/recall.tsv
@@ -0,0 +1,27 @@
+# The recall corpus: real documentation bugs found by hand in repositories
+# where kibble tested nothing, each pinned to the commit the bug was found at
+# and to the line a reader trips on. repos.tsv proves kibble does not invent
+# failures and mutations.tsv proves it catches rot planted on purpose. This
+# file holds what neither can: rot that was already there and that kibble
+# walked past, reading the repository as fine.
+#
+# A row is caught when the run reports the wanted status on that bug: an
+# example line in the named document whose status is wanted and whose
+# command or detail contains the proof, or a top-level row whose status is
+# wanted and whose detail contains the proof. The JSON carries no line
+# number, so the line column says where a person should look and the proof
+# pins identity, as in mutations.tsv. Wanted statuses are separated by "|".
+#
+# caught=no rows are the scoreboard: the job prints whether each is caught now
+# and never fails on them. Once a change catches a row, flip it to caught=yes,
+# and from then on the job fails if the line stops reaching its status. Every
+# bug found by hand becomes a row here before any fix for it is built.
+#
+# The first four came from the 2026-10-22 hand check of ten repositories
+# kibble had run nothing on. Columns are tab-separated:
+# name, clone URL, pinned commit, document, line, wanted status, proof, caught.
+jq https://github.com/jqlang/jq.git 94b00aa807a51a8f97bc9345c03e32982e344678 README.md 56 FAIL libjq.so.1 no
+goreleaser https://github.com/goreleaser/goreleaser.git a9fb415389a2e55906ae457d9f76bb77b3cb6056 www/content/getting-started/quick-start.md 138 FAIL doesn't contain any tags no
+pyenv https://github.com/pyenv/pyenv.git 7d65a4683c91b29f7501da4c8737d4e786f8cad2 README.md 187 DRIFT|FAIL --install no
+shellcheck https://github.com/koalaman/shellcheck.git cd9072643932ee2008e454e2cd990e25c1dc9d49 README.md 412 FAIL exec foo; echo "Done!" no
+shellcheck https://github.com/koalaman/shellcheck.git cd9072643932ee2008e454e2cd990e25c1dc9d49 README.md 451 FAIL cat file | grep foo no
diff --git a/docs/DESIGN.md b/docs/DESIGN.md
index e6367fa..631a140 100644
--- a/docs/DESIGN.md
+++ b/docs/DESIGN.md
@@ -28,6 +28,19 @@ reports which steps a brand-new user could actually complete.
`directories.bin` are not read.
- Runs each `git clone` recipe too: the clone and the build lines that follow it in the
same code block, with GitHub SSH remotes rewritten to HTTPS for the keyless container.
+- Runs a source build when a block configures the checkout (`./configure` or a cmake
+ generate step) and installs it (`make install`, `ninja install`, or `cmake --install`).
+ The recipe is the block as written, from its first command through the install and on
+ to the first blank line, comment, or teardown, with a leading `sudo` dropped. It runs in
+ `buildpack-deps:stable` on the committed tree, restored from the objects and refs of
+ the checkout's `.git` directory, which are streamed in so submodules resolve. The git
+ config, hooks, and reflogs stay on the host, since a config can hold a token. The
+ installed binary is smoke-tested in its own process. A loader error there, such as a
+ library `make install` placed where the loader does not look until `ldconfig` runs, is
+ a `FAIL`, because no argument starts that binary. When the document names `ldconfig` or
+ `LD_LIBRARY_PATH` outside the block, the same error is `BUILT`: the reader was told, and
+ kibble ran only the block. Any other failed smoke test stays `BUILT`. A block that also
+ clones, installs system packages, or pipes a remote script is not a source build.
- Picks the image from the toolchain the step assumes, so a Rust project builds with
`cargo` and a Node project with `npm`, and finds the installed binary wherever that
toolchain puts it, even when it is not named after the package.
@@ -81,8 +94,13 @@ honestly rather than as if nothing were documented.
When the documents hold no install of the tool itself that kibble extracts, the install
check is a `SKIP` with the reason `no-install-route`. The detail names the README, how
many install documents were read, and the routes seen, such as brew, apt, or none. It is
-not a `GAP`, because the documents may describe routes kibble does not run. When every
-row is a skip, the run exits 3, so a check that ran nothing does not read as a pass.
+not a `GAP`, because the documents may describe routes kibble does not run. When no row
+and no example line reached `VERIFIED`, `FAIL`, `GAP`, or `DRIFT`, the run exits 3 and
+the summary says nothing was tested and why, so a check that tested nothing does not read
+as a pass. A brew formula lookup (`EXISTS`), a recipe with no binary (`RAN`), and a
+`TIMEOUT` count as nothing tested: each leaves every documented claim unsettled. Before
+this rule only an all-skip run exited 3, and a repository whose rows were a formula lookup
+and skipped examples exited 0 having tested nothing.
A package install runs the documented line verbatim, so what kibble verifies is the
command a reader would actually type, flags and all. The bin directory is compared before
diff --git a/docs/SECURITY.md b/docs/SECURITY.md
index 26ad79a..fd991cb 100644
--- a/docs/SECURITY.md
+++ b/docs/SECURITY.md
@@ -9,6 +9,14 @@ memory and process counts, `no-new-privileges`, and every Linux capability dropp
the handful `apt` needs to install the packages the docs depend on. Nothing from the host
is mounted in; the repository is streamed in as a tar of its working tree.
+A documented source build (`./configure`, `make`, `make install`) is the one case that
+receives more than the working tree. It is sent the objects and refs of the checkout's
+`.git` directory, so the container can restore the committed tree and resolve submodules
+the way a fresh clone does. That is the repository's local history, including commits and
+stashes that were never pushed. Nothing else in `.git` is sent: not `config`, where a CI
+checkout stores the token it fetched with and a remote URL can carry a password, not
+hooks, not reflogs, and not a submodule's git directory.
+
The boundary that stays open is the network, because verifying an install *is* fetching
it: `go install`, `cargo install`, and `npm install -g` are network operations. A
malicious documented command can therefore reach out from inside the container.
diff --git a/examplerun.go b/examplerun.go
index 1d7da6b..382e8c7 100644
--- a/examplerun.go
+++ b/examplerun.go
@@ -11,6 +11,7 @@ import (
"os"
"os/exec"
"path/filepath"
+ "strings"
"time"
kplan "github.com/dcadolph/kibble/internal/plan"
@@ -208,31 +209,90 @@ func sessionBudget(plan *kplan.Plan, install time.Duration) time.Duration {
// for a small cap, and what is left is a runaway guard.
const repoStreamCap = 512 << 20
+// treeSpec says which part of a checkout an archive carries.
+type treeSpec struct {
+ // sub is the directory under the checkout that is walked, empty for all of
+ // it. Archive names stay relative to the checkout either way.
+ sub string
+ // skip holds the directory names left out.
+ skip map[string]bool
+ // maxFile is the size above which a file is left out, 0 for no limit.
+ maxFile int64
+ // dirs reports whether directories get entries of their own, so that an
+ // empty one, such as a git ref directory, exists after extraction.
+ dirs bool
+ // only holds the names directly under the walked directory that are
+ // carried, nil for all of them.
+ only map[string]bool
+}
+
+// leftOut reports whether the entry at path, under the walked directory root,
+// is outside what the spec carries.
+func (s treeSpec) leftOut(root, path string, d fs.DirEntry) bool {
+ if d.IsDir() && s.skip[d.Name()] {
+ return true
+ }
+ if s.only == nil || path == root {
+ return false
+ }
+ rel, err := filepath.Rel(root, path)
+ if err != nil {
+ return true
+ }
+ top, _, _ := strings.Cut(filepath.ToSlash(rel), "/")
+ return !s.only[top]
+}
+
+// sessionTree is what an example session receives: the working tree without
+// generated directories and without files over 2 MB, so documented example
+// files exist in the container.
+var sessionTree = treeSpec{skip: tarSkipDirs, maxFile: 2 << 20}
+
+// gitTree is what a source build receives: the objects and refs of the
+// checkout's git directory, from which the container restores the committed
+// tree. Nothing else in that directory is sent. Its config is where a CI
+// checkout keeps the token it fetched with and where a remote URL can hold a
+// password, and its hooks are programs from the host, so neither may reach a
+// container that runs documented commands with the network open.
+var gitTree = treeSpec{sub: ".git", dirs: true, only: map[string]bool{
+ "HEAD": true, "objects": true, "refs": true, "packed-refs": true, "shallow": true,
+}}
+
// repoFits reports whether the working tree is small enough to stream, and is
// a stat-only walk so the answer is known before a container starts. Deciding
// after the fact is not an option: a partially streamed repository makes a
// document look broken when the missing piece is kibble's.
func repoFits(dir string) bool {
+ return treeFits(dir, sessionTree)
+}
+
+// treeFits reports whether the part of dir that spec selects fits under
+// [repoStreamCap], walking stat-only.
+func treeFits(dir string, spec treeSpec) bool {
if dir == "" {
return true
}
total := int64(0)
fits := true
- _ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
+ root := filepath.Join(dir, spec.sub)
+ _ = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return nil
}
- if d.IsDir() {
- if tarSkipDirs[d.Name()] {
+ if spec.leftOut(root, path, d) {
+ if d.IsDir() {
return filepath.SkipDir
}
return nil
}
+ if d.IsDir() {
+ return nil
+ }
if !d.Type().IsRegular() {
return nil
}
info, err := d.Info()
- if err != nil || info.Size() > 2<<20 {
+ if err != nil || (spec.maxFile > 0 && info.Size() > spec.maxFile) {
return nil
}
if total += info.Size(); total > repoStreamCap {
@@ -244,63 +304,80 @@ func repoFits(dir string) bool {
return fits
}
-// repoTar packs the repo working tree for the session, without generated
-// directories and without files over 2 MB, so documented example files exist
-// in the container. Call [repoFits] first: this writes whatever it walks and
-// does not decide whether the result is complete.
+// repoTar packs the repo working tree for the session. Call [repoFits] first:
+// this writes whatever it walks and does not decide whether the result is
+// complete.
func repoTar(dir string) ([]byte, bool) {
var buf bytes.Buffer
truncated, _ := repoTarTo(&buf, dir)
return buf.Bytes(), truncated
}
-// repoTarTo writes the same archive to w as it walks, so nothing larger than
-// one file is held at once.
+// repoTarTo writes the session archive to w as it walks, so nothing larger
+// than one file is held at once.
func repoTarTo(w io.Writer, dir string) (bool, error) {
+ return treeTarTo(w, dir, sessionTree)
+}
+
+// treeTarTo writes the part of dir that spec selects to w as a tar stream,
+// one file at a time, and reports whether [repoStreamCap] cut it short.
+func treeTarTo(w io.Writer, dir string, spec treeSpec) (bool, error) {
if dir == "" {
return false, nil
}
truncated := false
tw := tar.NewWriter(w)
total := int64(0)
- _ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
+ root := filepath.Join(dir, spec.sub)
+ _ = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return nil
}
- if d.IsDir() {
- if tarSkipDirs[d.Name()] {
+ rel, rerr := filepath.Rel(dir, path)
+ if rerr != nil {
+ return nil
+ }
+ if spec.leftOut(root, path, d) {
+ if d.IsDir() {
return filepath.SkipDir
}
return nil
}
+ if d.IsDir() {
+ if spec.dirs && rel != "." {
+ if err := tw.WriteHeader(&tar.Header{
+ Typeflag: tar.TypeDir, Name: filepath.ToSlash(rel) + "/", Mode: 0o755,
+ }); err != nil {
+ return filepath.SkipAll
+ }
+ }
+ return nil
+ }
if !d.Type().IsRegular() {
return nil
}
info, err := d.Info()
- if err != nil || info.Size() > 2<<20 {
+ if err != nil || (spec.maxFile > 0 && info.Size() > spec.maxFile) {
return nil
}
if total += info.Size(); total > repoStreamCap {
truncated = true
return filepath.SkipAll
}
- rel, err := filepath.Rel(dir, path)
- if err != nil {
- return nil
- }
- b, err := os.ReadFile(path)
+ f, err := os.Open(path)
if err != nil {
return nil
}
+ defer func() { _ = f.Close() }()
hdr := &tar.Header{
Name: filepath.ToSlash(rel),
Mode: int64(info.Mode().Perm()),
- Size: int64(len(b)),
+ Size: info.Size(),
}
if err := tw.WriteHeader(hdr); err != nil {
return filepath.SkipAll
}
- if _, err := tw.Write(b); err != nil {
+ if err := copyPadded(tw, f, info.Size()); err != nil {
return filepath.SkipAll
}
return nil
@@ -309,6 +386,29 @@ func repoTarTo(w io.Writer, dir string) (bool, error) {
return truncated, err
}
+// copyPadded writes exactly size bytes to w, taken from r and then from zeros
+// when r runs short. A file that shrank after its tar header was written still
+// owes the archive its declared length, and stopping there would cut every
+// later file out of the stream and leave a partial repository behind one file
+// that changed mid-walk.
+func copyPadded(w io.Writer, r io.Reader, size int64) error {
+ n, err := io.CopyN(w, r, size)
+ if err == nil {
+ return nil
+ }
+ _, err = io.CopyN(w, zeroReader{}, size-n)
+ return err
+}
+
+// zeroReader reads as an endless run of zero bytes.
+type zeroReader struct{}
+
+// Read fills p with zeros.
+func (zeroReader) Read(p []byte) (int, error) {
+ clear(p)
+ return len(p), nil
+}
+
// tarSkipDirs are directories a build produces or a package manager fills.
// Streaming them wastes the budget on output nobody documents, and on a big
// repository it pushes the source a document needs past the cap.
diff --git a/extract.go b/extract.go
index 8dd4b74..f5ca3ba 100644
--- a/extract.go
+++ b/extract.go
@@ -50,6 +50,9 @@ type InstallStep struct {
// doc is the document an example step replays, relative to the repository
// root. Empty on every other kind of step.
doc string
+ // loaderNote is the loader step, such as ldconfig, that the document names
+ // outside a source-build recipe. Empty when it names none there.
+ loaderNote string
}
// Extractor finds install steps in README markdown.
@@ -197,6 +200,16 @@ func ExtractorWith(owned repoNames) Extractor {
}
steps = append(steps, s)
}
+ if s, ok := sourceBuildStep(repo, block); ok {
+ // Two builds can end on the same install line, one from git
+ // and one from a tarball, so the whole recipe is the key.
+ key := repo + "|source|" + strings.Join(s.Block, "\n")
+ if !seen[key] {
+ seen[key] = true
+ s.loaderNote = loaderNoteOutside(markdown, s.Block)
+ steps = append(steps, s)
+ }
+ }
}
// A tap documented anywhere in the document covers every brew step,
// since a reader adds it once and installs many formulas from it.
@@ -287,6 +300,103 @@ func installRecipe(lines []string) []string {
return out
}
+var (
+ // reConfigure matches the configure step of an autotools build.
+ reConfigure = regexp.MustCompile(`^(?:sh\s+)?\./configure(\s|$)`)
+ // reCMakeGenerate matches a cmake call that generates a build tree, which
+ // is any cmake call that does not build, install, or run a script.
+ reCMakeGenerate = regexp.MustCompile(`^cmake\s+(?:[^-]|-[^-EP])`)
+ // reBuildInstall matches the step that installs what a configured build
+ // produced. The space before install keeps make uninstall out.
+ reBuildInstall = regexp.MustCompile(`^(?:make|ninja)\b.*\sinstall\b|^cmake\s+--install\b`)
+ // reLeadingSudo matches a sudo opening a command, which the container,
+ // already root and without sudo installed, drops.
+ reLeadingSudo = regexp.MustCompile(`(^|&&\s*|;\s*)sudo\s+(?:-E\s+)?`)
+)
+
+// reLoaderStep matches the ways a document tells a reader to make an installed
+// shared library findable.
+var reLoaderStep = regexp.MustCompile(`\bldconfig\b|\bLD_LIBRARY_PATH\b`)
+
+// loaderNoteOutside returns the loader step, such as ldconfig, that a document
+// names somewhere other than in the recipe, or empty when the recipe runs one
+// itself or the document names none. A recipe that ran the step and still left
+// a binary unable to start has no excuse. One that left it for a later
+// sentence was run without it, and that is kibble's gap, not the document's.
+func loaderNoteOutside(markdown string, recipe []string) string {
+ if reLoaderStep.MatchString(strings.Join(recipe, "\n")) {
+ return ""
+ }
+ return reLoaderStep.FindString(markdown)
+}
+
+// sourceBuildStep returns the install route a block documents by building the
+// checkout from source: an autotools `./configure` or a cmake generate step,
+// followed in the same block by `make install`, `ninja install`, or `cmake
+// --install`. jq documents only this route, so kibble tested nothing in jq and
+// missed that the binary `make install` leaves cannot find its own library
+// until ldconfig runs. The recipe is the block from its first command through
+// the install line, and on past it to the first blank line, comment, or
+// teardown, since a block that follows the install with `sudo ldconfig`
+// documents a working install and must be run whole. Every line runs as
+// written apart from a leading sudo. A block that clones is the clone route's
+// recipe, and one that installs system packages or pipes a remote script needs
+// the routes kibble refuses, so neither is a source build. A recipe holding a
+// placeholder is recorded unrun.
+func sourceBuildStep(repo string, block docblock.Block) (InstallStep, bool) {
+ if block.Span || !docblock.ShellLangs[block.Lang] {
+ return InstallStep{}, false
+ }
+ var recipe []string
+ var step *InstallStep
+ configured := false
+ for _, raw := range docblock.PrepareLines(block.Lines) {
+ raw = strings.TrimPrefix(strings.TrimSpace(raw), "$ ")
+ written := strings.TrimSpace(stripComment(raw))
+ refused := reGitClone.MatchString(written) || reOSPackage.MatchString(written) ||
+ reScriptInstall.MatchString(written)
+ if step != nil && (written == "" || strings.HasPrefix(written, "#") ||
+ reTeardown.MatchString(written) || refused) {
+ break
+ }
+ if written == "" || strings.HasPrefix(written, "#") {
+ continue
+ }
+ if refused {
+ return InstallStep{}, false
+ }
+ l := reLeadingSudo.ReplaceAllString(written, "$1")
+ recipe = append(recipe, l)
+ if step != nil {
+ continue
+ }
+ for _, seg := range splitSegments(l) {
+ seg = strings.TrimSpace(seg)
+ if reConfigure.MatchString(seg) || reCMakeGenerate.MatchString(seg) {
+ configured = true
+ continue
+ }
+ if configured && reBuildInstall.MatchString(seg) {
+ step = &InstallStep{Repo: repo, Kind: "source-build", Raw: written}
+ if block.Line > 0 {
+ step.Line = docblock.SourceLineIndex(block)(raw)
+ }
+ break
+ }
+ }
+ }
+ if step == nil {
+ return InstallStep{}, false
+ }
+ step.Block, step.Run = recipe, true
+ for _, r := range recipe {
+ if docblock.HasPlaceholder(docblock.CommandHead(r)) {
+ step.Run = false
+ }
+ }
+ return *step, true
+}
+
// classifyLine matches one code line against the known install patterns. A
// line holding a placeholder is recorded with Run false, since running it
// would test the stand-in rather than the document.
diff --git a/extract_test.go b/extract_test.go
index ec80289..734d486 100644
--- a/extract_test.go
+++ b/extract_test.go
@@ -5,6 +5,8 @@ import (
"testing"
"github.com/google/go-cmp/cmp"
+
+ "github.com/dcadolph/kibble/internal/docblock"
)
// TestClassifyLine checks that install commands are recognized and classified.
@@ -598,3 +600,140 @@ func TestExtractListRecipe(t *testing.T) {
})
}
}
+
+// TestSourceBuildStep checks which blocks document building the checkout from
+// source, and that the recipe is the block as written apart from sudo.
+func TestSourceBuildStep(t *testing.T) {
+ t.Parallel()
+ tests := []struct {
+ In string
+ WantRaw string
+ WantRecipe []string
+ WantRun bool
+ Want bool
+ }{{ // Test 0: jq's block is a route, sudo dropped and make clean kept as written.
+ In: "```console\ngit submodule update --init # if building from git\nautoreconf -i\n" +
+ "./configure --with-oniguruma=builtin\nmake clean # if upgrading\nmake -j8\nmake check\n" +
+ "sudo make install\n```\n",
+ WantRaw: "sudo make install",
+ WantRecipe: []string{"git submodule update --init", "autoreconf -i",
+ "./configure --with-oniguruma=builtin", "make clean", "make -j8", "make check", "make install"},
+ WantRun: true, Want: true,
+ }, { // Test 1: configure with no install is not a route.
+ In: "```sh\n./configure\nmake\n```\n",
+ }, { // Test 2: make install with no configure step is not a route.
+ In: "```sh\nmake\nsudo make install\n```\n",
+ }, { // Test 3: a cmake generate, build, and install is a route.
+ In: "```sh\ncmake -B build\ncmake --build build\nsudo cmake --install build\n```\n",
+ WantRaw: "sudo cmake --install build",
+ WantRecipe: []string{"cmake -B build", "cmake --build build", "cmake --install build"},
+ WantRun: true, Want: true,
+ }, { // Test 4: a block that clones belongs to the clone route.
+ In: "```sh\ngit clone https://github.com/o/r\ncd r\n./configure\nmake install\n```\n",
+ }, { // Test 5: a block that installs system packages needs a refused route.
+ In: "```sh\nsudo apt-get install -y libfoo-dev\n./configure\nmake install\n```\n",
+ }, { // Test 6: lines after the install run, up to a blank line or a teardown.
+ In: "```sh\n./configure\nmake\nsudo make install\nsudo ldconfig\n\nsudo make uninstall\n```\n",
+ WantRaw: "sudo make install",
+ WantRecipe: []string{"./configure", "make", "make install", "ldconfig"},
+ WantRun: true, Want: true,
+ }, { // Test 7: a placeholder in the recipe records the route unrun.
+ In: "```sh\n./configure --prefix=
\nmake install\n```\n",
+ WantRaw: "make install",
+ WantRecipe: []string{"./configure --prefix=", "make install"},
+ Want: true,
+ }, { // Test 8: inline code is not a recipe.
+ In: "Run `./configure && make && sudo make install` to build.\n",
+ }, { // Test 9: make uninstall is not an install.
+ In: "```sh\n./configure\nmake uninstall\n```\n",
+ }, { // Test 10: a one-line chain is a route.
+ In: "```sh\n./configure && make && sudo make install\n```\n",
+ WantRaw: "./configure && make && sudo make install",
+ WantRecipe: []string{"./configure && make && make install"},
+ WantRun: true, Want: true,
+ }, { // Test 11: a cmake build-only call does not count as the generate step.
+ In: "```sh\ncmake --build build\ncmake --install build\n```\n",
+ }, { // Test 12: a prose block is not shell.
+ In: "```yaml\n./configure\nmake install\n```\n",
+ }}
+ for testNum, test := range tests {
+ t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) {
+ t.Parallel()
+ var got InstallStep
+ found := false
+ for _, b := range docblock.CodeBlocks(test.In) {
+ if s, ok := sourceBuildStep("repo", b); ok {
+ got, found = s, true
+ }
+ }
+ if found != test.Want {
+ t.Fatalf("found = %v, want %v (step %+v)", found, test.Want, got)
+ }
+ if !found {
+ return
+ }
+ if diff := cmp.Diff(test.WantRaw, got.Raw); diff != "" {
+ t.Errorf("raw mismatch (-want +got):\n%s", diff)
+ }
+ if diff := cmp.Diff(test.WantRecipe, got.Block); diff != "" {
+ t.Errorf("recipe mismatch (-want +got):\n%s", diff)
+ }
+ if diff := cmp.Diff(test.WantRun, got.Run); diff != "" {
+ t.Errorf("run mismatch (-want +got):\n%s", diff)
+ }
+ })
+ }
+}
+
+// TestExtractSourceBuildRecipes checks that two builds ending on the same
+// install line are two routes, and that a loader step the document gives
+// outside a recipe is recorded on it.
+func TestExtractSourceBuildRecipes(t *testing.T) {
+ t.Parallel()
+ fromGit := "```sh\nautoreconf -i\n./configure\nmake\nsudo make install\n```\n"
+ fromTar := "```sh\n./configure\nmake\nsudo make install\n```\n"
+ tests := []struct {
+ In string
+ WantNotes []string
+ }{{ // Test 0: two recipes with one install line are both kept.
+ In: fromGit + "\nOr from a tarball:\n\n" + fromTar, WantNotes: []string{"", ""},
+ }, { // Test 1: the same recipe written twice is one route.
+ In: fromTar + "\nAgain:\n\n" + fromTar, WantNotes: []string{""},
+ }, { // Test 2: ldconfig in a later sentence is noted.
+ In: fromTar + "\nOn Linux run `sudo ldconfig` afterwards.\n", WantNotes: []string{"ldconfig"},
+ }, { // Test 3: a library path variable in a later block is noted.
+ In: fromTar + "\n```sh\nexport LD_LIBRARY_PATH=/usr/local/lib\n```\n",
+ WantNotes: []string{"LD_LIBRARY_PATH"},
+ }, { // Test 4: a recipe that runs ldconfig itself has nothing left to note.
+ In: "```sh\n./configure\nmake\nsudo make install\nsudo ldconfig\n```\n\nRun ldconfig again if needed.\n",
+ WantNotes: []string{""},
+ }}
+ for testNum, test := range tests {
+ t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) {
+ t.Parallel()
+ var notes []string
+ for _, s := range ExtractorWith(repoNames{}).Extract("repo", test.In) {
+ if s.Kind == "source-build" {
+ notes = append(notes, s.loaderNote)
+ }
+ }
+ if diff := cmp.Diff(test.WantNotes, notes); diff != "" {
+ t.Errorf("loader notes mismatch (-want +got):\n%s", diff)
+ }
+ })
+ }
+}
+
+// TestExtractSourceBuildLine checks that the route points at the README line
+// of its install command, sudo and all.
+func TestExtractSourceBuildLine(t *testing.T) {
+ t.Parallel()
+ md := "# T\n\n```sh\n./configure\nmake\nsudo make install\n```\n"
+ steps := ExtractorWith(repoNames{}).Extract("repo", md)
+ if len(steps) != 1 || steps[0].Kind != "source-build" {
+ t.Fatalf("steps = %+v, want one source-build", steps)
+ }
+ if diff := cmp.Diff(6, steps[0].Line); diff != "" {
+ t.Errorf("line mismatch (-want +got):\n%s", diff)
+ }
+}
diff --git a/flagcheck.go b/flagcheck.go
index 40fb5ef..b645f3f 100644
--- a/flagcheck.go
+++ b/flagcheck.go
@@ -77,9 +77,14 @@ func extractUsage(binaries []string, markdown string) map[string]*Usage {
}
codeLines = append(codeLines, b.Lines...)
}
+ owned := func(name string) bool { return slices.Contains(binaries, name) }
for _, line := range codeLines {
line = stripComment(line)
for _, seg := range splitSegments(line) {
+ // A stand-in path to the binary, such as pyenv's own
+ // ` --install`, cites the binary. Resolved first,
+ // its angle brackets would otherwise read as a synopsis below.
+ seg = docblock.ResolveToolPath(strings.TrimSpace(seg), owned)
// A synopsis such as `tool schedule [add --every ]`
// is a signature, not an invocation. Splitting it on pipes strands
// its flags on the parent command, and that mis-attribution once
diff --git a/flagcheck_test.go b/flagcheck_test.go
index 939a0de..252d7d4 100644
--- a/flagcheck_test.go
+++ b/flagcheck_test.go
@@ -103,6 +103,22 @@ func TestExtractUsage(t *testing.T) {
In: "```sh\nrclone -vvv --stats-one-line --max-stats-groups 10\n```\n",
Binaries: []string{"rclone"},
WantFlags: []string{"vvv", "stats-one-line", "max-stats-groups"},
+ }, { // Test 17: a stand-in path to the binary in inline code cites its flag.
+ In: "To set up the shell, run ` --install`.\n",
+ Binaries: []string{"pyenv"},
+ WantFlags: []string{"install"},
+ }, { // Test 18: a bare stand-in path to the binary in a block cites its flag.
+ In: "```sh\npath/to/pyenv --install\n```\n",
+ Binaries: []string{"pyenv"},
+ WantFlags: []string{"install"},
+ }, { // Test 19: a stand-in path to another tool cites nothing.
+ In: "```sh\npath/to/other-tool --install\n```\n",
+ Binaries: []string{"pyenv"},
+ WantNone: true,
+ }, { // Test 20: a resolved line that still holds a slot is a synopsis.
+ In: "```sh\npath/to/pyenv --install \n```\n",
+ Binaries: []string{"pyenv"},
+ WantNone: true,
}}
for testNum, test := range tests {
t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) {
diff --git a/internal/docblock/docblock.go b/internal/docblock/docblock.go
index ee07f50..bb20fb3 100644
--- a/internal/docblock/docblock.go
+++ b/internal/docblock/docblock.go
@@ -504,6 +504,39 @@ var rePlaceholder = regexp.MustCompile(
`|deadbeef|1234567|123abc)\b` +
`|(^|[^/])\.\.\.($|[\s'"./])|\b[A-Z][A-Z]+_[0-9]+\b`)
+// ResolveToolPath replaces a stand-in path to a program in command position,
+// such as path/to/tool, /path/to/tool, , or >,
+// with the program's bare name when owned reports that the session installs
+// it. pyenv documents its own flag as ` --install`, and reading
+// that as a placeholder left the one wrong line of its README untested. A
+// stand-in path to anything else, such as path/to/file.txt or a tool the
+// session does not install, is returned unchanged and stays a placeholder.
+// Leading indentation and a "$ " prompt are kept.
+func ResolveToolPath(line string, owned func(string) bool) string {
+ body := strings.TrimLeft(line, " \t")
+ lead := line[:len(line)-len(body)]
+ if strings.HasPrefix(body, "$ ") {
+ lead += "$ "
+ body = body[2:]
+ }
+ word, rest, spaced := strings.Cut(body, " ")
+ if spaced {
+ rest = " " + rest
+ }
+ if strings.Count(word, "<") != strings.Count(word, ">") {
+ return line
+ }
+ name, ok := strings.CutPrefix(strings.TrimPrefix(strings.NewReplacer("<", "", ">", "").Replace(word), "/"),
+ "path/to/")
+ if !ok || !reToolName.MatchString(name) || !owned(name) {
+ return line
+ }
+ return lead + name + rest
+}
+
+// reToolName matches a program name with no path in it.
+var reToolName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._+-]*$`)
+
// HasPlaceholder reports whether s holds a token the reader must replace
// before running it. The planner asks it of an example line and the extractor
// asks it of an install line, so both read placeholders the same way.
diff --git a/internal/docblock/docblock_test.go b/internal/docblock/docblock_test.go
index 1584195..ff72a39 100644
--- a/internal/docblock/docblock_test.go
+++ b/internal/docblock/docblock_test.go
@@ -124,6 +124,54 @@ func TestHasPlaceholder(t *testing.T) {
}
}
+// TestResolveToolPath checks that a stand-in path to an installed program in
+// command position becomes the program, and every other stand-in path stays.
+func TestResolveToolPath(t *testing.T) {
+ t.Parallel()
+
+ owned := func(name string) bool { return name == "pyenv" }
+ tests := []struct {
+ In string
+ WantResult string
+ }{{ // Test 0: A bare stand-in path resolves.
+ In: "path/to/pyenv --install bash", WantResult: "pyenv --install bash",
+ }, { // Test 1: An angle-bracketed stand-in path resolves.
+ In: " --install", WantResult: "pyenv --install",
+ }, { // Test 2: A rooted stand-in path resolves.
+ In: "/path/to/pyenv --install", WantResult: "pyenv --install",
+ }, { // Test 3: A nested slot resolves.
+ In: "> --install", WantResult: "pyenv --install",
+ }, { // Test 4: A slot on the program name alone resolves.
+ In: "path/to/ init", WantResult: "pyenv init",
+ }, { // Test 5: A prompt and indentation are kept.
+ In: " $ path/to/pyenv --install", WantResult: " $ pyenv --install",
+ }, { // Test 6: The command alone resolves.
+ In: "path/to/pyenv", WantResult: "pyenv",
+ }, { // Test 7: A file path stays a placeholder.
+ In: "path/to/file.txt --x", WantResult: "path/to/file.txt --x",
+ }, { // Test 8: A tool the session does not install stays a placeholder.
+ In: "path/to/other-tool --x", WantResult: "path/to/other-tool --x",
+ }, { // Test 9: A stand-in path past command position is left alone.
+ In: "pyenv exec path/to/pyenv", WantResult: "pyenv exec path/to/pyenv",
+ }, { // Test 10: A deeper path is not the program.
+ In: "path/to/bin/pyenv --install", WantResult: "path/to/bin/pyenv --install",
+ }, { // Test 11: Unbalanced brackets are left alone.
+ In: " --install bash\n```\n",
+ WantSteps: [][]planLine{{{Cmd: "tool --install bash"}}},
+ }, { // Test 23: a resolved line that still holds a slot stays a placeholder.
+ Markdown: "```sh\npath/to/tool --install \n```\n",
+ WantSteps: [][]planLine{{{Cmd: "tool --install ", Skip: true}}},
+ }, { // Test 24: a stand-in path to another program is not an example.
+ Markdown: "```sh\npath/to/other-tool --x\n```\n",
+ WantExcluded: 1,
+ }, { // Test 25: a stand-in path to a file stays a placeholder.
+ Markdown: "```sh\ntool run path/to/file.txt\n```\n",
+ WantSteps: [][]planLine{{{Cmd: "tool run path/to/file.txt", Skip: true}}},
}}
for testNum, test := range tests {
t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) {
diff --git a/main.go b/main.go
index 8c2740d..a3f9c17 100644
--- a/main.go
+++ b/main.go
@@ -706,7 +706,7 @@ func unrunResult(step InstallStep) Result {
Detail: "kibble does not run system package installs; verify this one by hand",
}
case step.Kind == "git-clone" || step.Kind == "go-install" || step.Kind == "brew" ||
- pkgKinds[step.Kind].Ecosystem != "":
+ step.Kind == "source-build" || pkgKinds[step.Kind].Ecosystem != "":
// These kinds run unless the line holds a placeholder, which is the
// only reason one is recorded unrun.
res = Result{
@@ -763,24 +763,52 @@ func gib(n int64) string {
}
// exitCode is the process exit status for a finished run: 1 when anyFail says
-// the run failed, 3 when nothing was checked, else 0. Nothing was checked when
-// every row is a skip: no install ran or was looked up, so no example could
-// run either, whether the documents hold no install route at all or only
-// routes kibble does not run. A check that ran nothing must not read as a
-// pass, and it is not a failure of the documents either.
+// the run failed, 3 when nothing was tested, else 0. Nothing was tested when no
+// row and no example line reached a verdict on a documented claim. A brew
+// formula lookup, a recipe that ran with no binary to smoke-test, a timeout,
+// and a skip each leave every claim unsettled, and a run made only of those
+// used to read as a pass. A check that tested nothing
+// must not read as a pass, and it is not a failure of the documents either.
func exitCode(results []Result, strict bool) int {
if anyFail(results, strict) {
return 1
}
+ if len(results) == 0 || anyTested(results) {
+ return 0
+ }
+ return 3
+}
+
+// anyTested reports whether any result, or any example line inside one,
+// settled a documented claim.
+func anyTested(results []Result) bool {
for _, r := range results {
- if r.Status != StatusSkipped {
- return 0
+ if tested(r.Status) {
+ return true
+ }
+ if r.example == nil {
+ continue
+ }
+ for _, s := range r.example.Steps {
+ for _, l := range s.Lines {
+ if tested(l.Status) {
+ return true
+ }
+ }
}
}
- if len(results) == 0 {
- return 0
+ return false
+}
+
+// tested reports whether a status is a verdict on a documented claim: it
+// worked, it failed, the document left a gap, or the document drifted from the
+// binary. Every other status says the claim went unsettled.
+func tested(s Status) bool {
+ switch s {
+ case StatusVerified, StatusFail, StatusGap, StatusDrift:
+ return true
}
- return 3
+ return false
}
// anyFail reports whether the run should exit non-zero. A build failure always
diff --git a/main_test.go b/main_test.go
index 28fd8f0..104fe85 100644
--- a/main_test.go
+++ b/main_test.go
@@ -54,10 +54,20 @@ func TestAnyFail(t *testing.T) {
}
// TestExitCode checks the process exit status, including the code that says a
-// run checked nothing because no repository had an install route kibble runs.
+// run tested nothing: no row and no example line settled a documented claim.
func TestExitCode(t *testing.T) {
t.Parallel()
res := func(s Status) Result { return Result{Status: s} }
+ exampleWith := func(lines ...Status) Result {
+ step := exampleStep{}
+ for _, s := range lines {
+ step.Lines = append(step.Lines, lineResult{Status: s})
+ }
+ return Result{
+ Step: InstallStep{Kind: "example"}, Status: StatusBlocked,
+ example: &exampleRun{Steps: []exampleStep{step}},
+ }
+ }
noRoute := Result{Step: InstallStep{Kind: "install"}, Status: StatusSkipped, Reason: ReasonNoInstallRoute}
exampleSkip := Result{Step: InstallStep{Kind: "example"}, Status: StatusSkipped}
aptSkip := Result{Step: InstallStep{Kind: "os-package"}, Status: StatusSkipped, Reason: ReasonUnsupportedMethod}
@@ -84,12 +94,31 @@ func TestExitCode(t *testing.T) {
Results: []Result{aptSkip, exampleSkip}, WantCode: 3,
}, { // Test 8: an error elsewhere outranks nothing checked.
Results: []Result{noRoute, res(StatusError)}, WantCode: 1,
- }, { // Test 9: a formula lookup is a check, so a brew row beside skips passes.
- Results: []Result{res(StatusExists), exampleSkip}, WantCode: 0,
+ }, { // Test 9: a formula lookup tests no claim, so a brew row beside skips is nothing tested.
+ Results: []Result{res(StatusExists), exampleSkip}, WantCode: 3,
}, { // Test 10: strict does not turn nothing checked into a failure.
Results: []Result{noRoute, exampleSkip}, Strict: true, WantCode: 3,
}, { // Test 11: an empty run exits zero.
WantCode: 0,
+ }, { // Test 12: a brew lookup alone is nothing tested.
+ Results: []Result{res(StatusExists)}, WantCode: 3,
+ }, { // Test 13: a recipe that ran with no binary is nothing tested.
+ Results: []Result{res(StatusRan), exampleSkip}, WantCode: 3,
+ }, { // Test 14: a timeout beside skips is nothing tested.
+ Results: []Result{res(StatusTimeout), exampleSkip}, WantCode: 3,
+ }, { // Test 15: built without a passing smoke test is nothing tested.
+ Results: []Result{res(StatusBuilt), res(StatusCrossArch), res(StatusBlocked)}, WantCode: 3,
+ }, { // Test 16: drift settles a claim, so the run tested something.
+ Results: []Result{res(StatusExists), res(StatusDrift)}, WantCode: 0,
+ }, { // Test 17: a gap settles a claim too.
+ Results: []Result{res(StatusRan), res(StatusGap)}, WantCode: 0,
+ }, { // Test 18: one verified example line inside a blocked session is a tested claim.
+ Results: []Result{res(StatusRan), exampleWith(StatusBlocked, StatusVerified)}, WantCode: 0,
+ }, { // Test 19: an example session whose lines all went unsettled is nothing tested.
+ Results: []Result{res(StatusRan), exampleWith(StatusBlocked, StatusSkipped)},
+ WantCode: 3,
+ }, { // Test 20: an error outranks nothing tested.
+ Results: []Result{res(StatusExists), res(StatusError)}, WantCode: 1,
}}
for testNum, test := range tests {
t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) {
diff --git a/report.go b/report.go
index f5d0efe..dd36b3b 100644
--- a/report.go
+++ b/report.go
@@ -212,6 +212,9 @@ func reportTable(w io.Writer, results []Result) {
countVerdict(r.Status, &pass, &fail, &gap, &other)
}
_, _ = fmt.Fprintf(w, "\n%s\n", verdictLine(c, fail, gap, other+blocked))
+ if len(results) > 0 && !anyTested(results) {
+ _, _ = fmt.Fprintf(w, "%s\n", c.yellow(untestedLine(results)))
+ }
_, _ = fmt.Fprintf(w, "%s\n", summaryLine(c, pass, fail, gap, other, len(results), total))
_, _ = fmt.Fprintf(w, "%s\n", evidenceLine(c, results))
}
@@ -266,6 +269,36 @@ func verdictLine(c palette, fail, gap, other int) string {
}
}
+// untestedLine says that a run tested nothing and why, naming up to three
+// distinct causes from the row details. Install rows come first, since an
+// example skipped for want of a binary is the consequence of an install that
+// did not run. Each cause is cut at its first semicolon so the causes can be
+// joined on one line without running together.
+func untestedLine(results []Result) string {
+ var installs, examples []string
+ for _, r := range results {
+ cause := r.Detail
+ if r.Step.doc != "" {
+ cause = strings.TrimPrefix(cause, r.Step.doc+": ")
+ }
+ cause, _, _ = strings.Cut(cause, ";")
+ cause = strings.TrimSpace(cause)
+ if cause == "" {
+ cause = strings.ToLower(string(r.Status)) + " " + r.Step.Kind
+ }
+ if r.Step.Kind == "example" {
+ examples = append(examples, cause)
+ } else {
+ installs = append(installs, cause)
+ }
+ }
+ causes := dedupe(append(installs, examples...))
+ if len(causes) > 3 {
+ causes = causes[:3]
+ }
+ return "Nothing was tested: " + strings.Join(causes, "; ") + "."
+}
+
// blockedLines counts the documented lines of a result that ran without
// settling. They are counted apart from the result's own status because a
// session that verified some lines is reported as verified, and the lines it
diff --git a/report_test.go b/report_test.go
index 1b7514d..2934fde 100644
--- a/report_test.go
+++ b/report_test.go
@@ -241,6 +241,72 @@ func TestReportNoInstallRoute(t *testing.T) {
}
}
+// TestUntestedLine checks the summary line a run prints when it tested no
+// claim: install causes before the example skips they cause, each cut at its
+// first semicolon, no cause twice, and at most three.
+func TestUntestedLine(t *testing.T) {
+ t.Parallel()
+ row := func(kind string, s Status, detail string) Result {
+ return Result{Step: InstallStep{Repo: "r", Kind: kind, doc: "README.md"}, Status: s, Detail: detail}
+ }
+ examples := row("example", StatusSkipped, "no documented install puts a binary on PATH; examples not run")
+ apt := row("os-package", StatusSkipped, "kibble does not run system package installs; verify this one by hand")
+ brew := row("brew", StatusExists, "README.md: formula exists (install not attempted)")
+ ran := row("git-clone", StatusRan, "recipe exited 0 but produced no binary to smoke-test")
+ tests := []struct {
+ Results []Result
+ WantResult string
+ }{{ // Test 0: shellcheck's shape, the example skip listed after the installs.
+ Results: []Result{brew, brew, examples, apt, apt},
+ WantResult: "Nothing was tested: formula exists (install not attempted); " +
+ "kibble does not run system package installs; " +
+ "no documented install puts a binary on PATH.",
+ }, { // Test 1: four causes are cut to three, installs first.
+ Results: []Result{examples, apt, brew, ran},
+ WantResult: "Nothing was tested: kibble does not run system package installs; " +
+ "formula exists (install not attempted); " +
+ "recipe exited 0 but produced no binary to smoke-test.",
+ }, { // Test 2: a row with no detail is named by its status and kind.
+ Results: []Result{{Step: InstallStep{Kind: "cargo-install"}, Status: StatusTimeout}},
+ WantResult: "Nothing was tested: timeout cargo-install.",
+ }}
+ for testNum, test := range tests {
+ t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) {
+ t.Parallel()
+ if diff := cmp.Diff(test.WantResult, untestedLine(test.Results)); diff != "" {
+ t.Errorf("mismatch (-want +got):\n%s", diff)
+ }
+ })
+ }
+}
+
+// TestReportTableUntested checks the table prints the nothing-tested line only
+// when no row settled a claim.
+func TestReportTableUntested(t *testing.T) {
+ t.Parallel()
+ brew := Result{Step: InstallStep{Repo: "r", Kind: "brew"}, Status: StatusExists,
+ Detail: "formula exists (install not attempted)"}
+ tests := []struct {
+ Results []Result
+ WantLine bool
+ }{{ // Test 0: a brew lookup alone tested nothing.
+ Results: []Result{brew}, WantLine: true,
+ }, { // Test 1: a verified row beside it is a tested claim.
+ Results: []Result{brew, {Step: InstallStep{Repo: "r", Kind: "go-install"}, Status: StatusVerified}},
+ }, { // Test 2: an empty run prints no cause line.
+ }}
+ for testNum, test := range tests {
+ t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) {
+ t.Parallel()
+ var b bytes.Buffer
+ reportTable(&b, test.Results)
+ if got := strings.Contains(b.String(), "Nothing was tested"); got != test.WantLine {
+ t.Errorf("line printed = %v, want %v:\n%s", got, test.WantLine, b.String())
+ }
+ })
+ }
+}
+
// TestSharedCauseKeyIgnoresVersion checks the version fields do not split a shared cause.
func TestSharedCauseKeyIgnoresVersion(t *testing.T) {
t.Parallel()
diff --git a/run.go b/run.go
index dae97ea..205b174 100644
--- a/run.go
+++ b/run.go
@@ -4,7 +4,10 @@ import (
"context"
"errors"
"fmt"
+ "io"
+ "os"
"os/exec"
+ "path/filepath"
"regexp"
"strings"
"time"
@@ -135,6 +138,11 @@ func (d *DockerRunner) Run(ctx context.Context, step InstallStep) Result {
return res
case "git-clone":
script = cloneScriptFor(step, int(d.Timeout.Seconds()))
+ case "source-build":
+ if blocked := sourceBlocked(step.dir); blocked != "" {
+ return Result{Step: step, Status: StatusSkipped, Reason: ReasonMissingDependency, Detail: blocked}
+ }
+ script = sourceScriptFor(step, int(d.Timeout.Seconds()))
default:
if _, isPkg := pkgKinds[step.Kind]; isPkg {
script = pkgScriptFor(step, int(d.Timeout.Seconds())) + helpProbe(step)
@@ -156,10 +164,22 @@ func (d *DockerRunner) Run(ctx context.Context, step InstallStep) Result {
// The official Go images pin GOTOOLCHAIN=local, so a module asking for a
// newer Go than the image ships fails to install. A reader running the
// default fetches that toolchain, so the session matches them instead.
- args := append([]string{"run", "--rm", "--name", name},
- append(sandbox.Args(), "-e", "GOTOOLCHAIN=auto", image, "bash", "-c", script)...)
+ head := []string{"run", "--rm", "--name", name}
+ if step.Kind == "source-build" {
+ head = append(head, "-i")
+ }
+ args := append(head, append(sandbox.Args(), "-e", "GOTOOLCHAIN=auto", image, "bash", "-c", script)...)
cmd := exec.CommandContext(ctx, sandbox.Bin(), args...)
cmd.Cancel = sandbox.RemoveFunc(cmd, name)
+ if step.Kind == "source-build" {
+ pr, pw := io.Pipe()
+ go func() {
+ _, err := treeTarTo(pw, step.dir, gitTree)
+ _ = pw.CloseWithError(err)
+ }()
+ defer func() { _ = pr.Close() }()
+ cmd.Stdin = pr
+ }
out, _ := cmd.CombinedOutput()
if ctx.Err() != nil && errors.Is(context.Cause(ctx), context.Canceled) {
return Result{
@@ -225,6 +245,22 @@ var reNetworkError = regexp.MustCompile(
`\b(EAI_AGAIN|ENOTFOUND|ECONNRESET|ETIMEDOUT|ECONNREFUSED|ENETUNREACH|EHOSTUNREACH)\b|` +
`getaddrinfo|socket hang up|request to \S+ failed, reason`)
+// sourceBlocked returns why a source build cannot be handed the checkout it
+// builds, or empty when it can. The build restores the committed tree from the
+// git directory, so a checkout without one, or a worktree whose .git is only a
+// pointer to another repository's, has nothing to send.
+func sourceBlocked(dir string) string {
+ info, err := os.Stat(filepath.Join(dir, ".git"))
+ switch {
+ case err != nil || !info.IsDir():
+ return "builds the checkout from its git history, and this checkout has no .git directory to send"
+ case !treeFits(dir, gitTree):
+ return fmt.Sprintf("builds the checkout from its git history, which exceeds the %d MB kibble will stream",
+ repoStreamCap>>20)
+ }
+ return ""
+}
+
// imageFor returns the container image a step runs in. A clone recipe runs in
// the image that provides the toolchain its commands assume, so a Rust or Node
// project builds with the tools its docs were written for. Everything else runs
@@ -236,6 +272,9 @@ func (d *DockerRunner) imageFor(step InstallStep) (image, blocked string) {
}
return d.Image, ""
}
+ if step.Kind == "source-build" {
+ return sourceBuildImage, ""
+ }
if step.Kind != "git-clone" {
return d.Image, ""
}
diff --git a/run_test.go b/run_test.go
index a59050b..fdf181e 100644
--- a/run_test.go
+++ b/run_test.go
@@ -1,8 +1,11 @@
package main
import (
+ "archive/tar"
+ "bytes"
"context"
"fmt"
+ "io"
"os"
"os/exec"
"path/filepath"
@@ -1216,3 +1219,220 @@ func TestClassifyBinKind(t *testing.T) {
})
}
}
+
+// TestClassifySourceBuildLoader checks that a loader refusing to start the
+// binary convicts a source build, and only a source build.
+func TestClassifySourceBuildLoader(t *testing.T) {
+ t.Parallel()
+ loader := "BUILDCODE=0\nSMOKECODE=127\nSMOKELINE=/usr/local/bin/jq: error while loading shared libraries: " +
+ "libjq.so.1: cannot open shared object file: No such file or directory\n"
+ tests := []struct {
+ Kind string
+ In string
+ Note string
+ WantStatus Status
+ }{{ // Test 0: a source build whose binary cannot load its library fails.
+ Kind: "source-build", In: loader, WantStatus: StatusFail,
+ }, { // Test 1: the same smoke line after a go install stays BUILT.
+ Kind: "go-install", In: loader, WantStatus: StatusBuilt,
+ }, { // Test 2: any other failed smoke test after a source build stays BUILT.
+ Kind: "source-build", In: "BUILDCODE=0\nSMOKECODE=2\nSMOKELINE=usage: tool [-x]\n", WantStatus: StatusBuilt,
+ }, { // Test 3: a passing smoke test after a source build is VERIFIED.
+ Kind: "source-build", In: "BUILDCODE=0\nSMOKECODE=0\nSMOKELINE=jq-1.8.1\n", WantStatus: StatusVerified,
+ }, { // Test 4: a checkout the container could not restore is kibble's error.
+ Kind: "source-build", In: "kibble could not restore the checkout from its git history\n",
+ WantStatus: StatusError,
+ }, { // Test 5: a loader step the document gives outside the recipe keeps it BUILT.
+ Kind: "source-build", In: loader, Note: "ldconfig", WantStatus: StatusBuilt,
+ }}
+ for testNum, test := range tests {
+ t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) {
+ t.Parallel()
+ got := classify(InstallStep{Repo: "jq", Kind: test.Kind, loaderNote: test.Note}, test.In, time.Second)
+ if diff := cmp.Diff(test.WantStatus, got.Status); diff != "" {
+ t.Errorf("status mismatch (-want +got):\n%s", diff)
+ }
+ if test.WantStatus == StatusFail && !strings.Contains(got.Detail, "libjq.so.1") {
+ t.Errorf("detail %q does not name the library", got.Detail)
+ }
+ if test.Note != "" && !strings.Contains(got.Detail, test.Note) {
+ t.Errorf("detail %q does not name the step the document gives", got.Detail)
+ }
+ })
+ }
+}
+
+// TestSourceBlocked checks that a source build needs a git directory to send.
+func TestSourceBlocked(t *testing.T) {
+ t.Parallel()
+ withDir := t.TempDir()
+ if err := os.MkdirAll(filepath.Join(withDir, ".git", "refs"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ withFile := t.TempDir()
+ if err := os.WriteFile(filepath.Join(withFile, ".git"), []byte("gitdir: /elsewhere\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ tests := []struct {
+ Dir string
+ WantNone bool
+ }{{ // Test 0: a git directory can be sent.
+ Dir: withDir, WantNone: true,
+ }, { // Test 1: a worktree pointer cannot.
+ Dir: withFile,
+ }, { // Test 2: no git at all cannot.
+ Dir: t.TempDir(),
+ }}
+ for testNum, test := range tests {
+ t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) {
+ t.Parallel()
+ if got := sourceBlocked(test.Dir); (got == "") != test.WantNone {
+ t.Errorf("sourceBlocked = %q, want none %v", got, test.WantNone)
+ }
+ })
+ }
+}
+
+// TestGitTreeStream checks that a source build's stream carries the objects
+// and refs of the git directory, empty directories and large packs included,
+// and nothing else: not the config a token or a remote password lives in, not
+// the host's hooks, not a submodule's git directory, and not the working tree.
+func TestGitTreeStream(t *testing.T) {
+ t.Parallel()
+ dir := t.TempDir()
+ for _, d := range []string{".git/refs/tags", ".git/objects/pack", ".git/hooks", ".git/logs",
+ ".git/modules/dep/objects"} {
+ if err := os.MkdirAll(filepath.Join(dir, d), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ }
+ for name, body := range map[string]string{
+ ".git/HEAD": "ref: refs/heads/main\n",
+ ".git/packed-refs": "# pack-refs\n",
+ ".git/shallow": "0000\n",
+ ".git/config": "[http \"https://github.com/\"]\n\textraheader = AUTHORIZATION: basic c2VjcmV0\n",
+ ".git/hooks/post-commit": "#!/bin/sh\n",
+ ".git/logs/HEAD": "clone: from https://user:secret@example.test/r\n",
+ ".git/FETCH_HEAD": "https://user:secret@example.test/r\n",
+ ".git/modules/dep/config": "[remote \"origin\"]\n\turl = https://user:secret@example.test/d\n",
+ } {
+ if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ }
+ big := make([]byte, 3<<20)
+ if err := os.WriteFile(filepath.Join(dir, ".git/objects/pack/p.pack"), big, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(dir, "main.c"), []byte("int main(){}\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ var buf bytes.Buffer
+ if _, err := treeTarTo(&buf, dir, gitTree); err != nil {
+ t.Fatal(err)
+ }
+ got := map[string]int64{}
+ tr := tar.NewReader(&buf)
+ for {
+ hdr, err := tr.Next()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ got[hdr.Name] = hdr.Size
+ }
+ want := map[string]int64{
+ ".git/": 0, ".git/refs/": 0, ".git/refs/tags/": 0, ".git/objects/": 0,
+ ".git/objects/pack/": 0, ".git/objects/pack/p.pack": 3 << 20,
+ ".git/HEAD": 21, ".git/packed-refs": 12, ".git/shallow": 5,
+ }
+ if diff := cmp.Diff(want, got); diff != "" {
+ t.Errorf("archive mismatch (-want +got):\n%s", diff)
+ }
+}
+
+// TestCopyPadded checks that a source running short of its declared size is
+// filled out with zeros, so the tar entry it feeds stays whole.
+func TestCopyPadded(t *testing.T) {
+ t.Parallel()
+ tests := []struct {
+ In string
+ Size int64
+ WantResult string
+ }{{ // Test 0: a source of the declared size is copied as is.
+ In: "abcd", Size: 4, WantResult: "abcd",
+ }, { // Test 1: a source that shrank is padded to the declared size.
+ In: "ab", Size: 4, WantResult: "ab\x00\x00",
+ }, { // Test 2: a source that grew is cut at the declared size.
+ In: "abcdef", Size: 4, WantResult: "abcd",
+ }, { // Test 3: an empty source of declared size zero writes nothing.
+ In: "", Size: 0, WantResult: "",
+ }}
+ for testNum, test := range tests {
+ t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) {
+ t.Parallel()
+ var buf bytes.Buffer
+ if err := copyPadded(&buf, strings.NewReader(test.In), test.Size); err != nil {
+ t.Fatal(err)
+ }
+ if diff := cmp.Diff(test.WantResult, buf.String()); diff != "" {
+ t.Errorf("mismatch (-want +got):\n%s", diff)
+ }
+ })
+ }
+}
+
+// TestDockerSourceBuild builds a checkout whose installed binary needs its own
+// shared library. Without ldconfig the documented install leaves a binary that
+// cannot start, and with it the same project verifies.
+func TestDockerSourceBuild(t *testing.T) {
+ if testing.Short() {
+ t.Skip("container test skipped in short mode")
+ }
+ if os.Getenv("KIBBLE_INTEGRATION") == "" {
+ t.Skip("set KIBBLE_INTEGRATION=1 to run container tests")
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+ defer cancel()
+ if err := sandbox.Available(ctx); err != nil {
+ t.Skipf("docker is not reachable: %v", err)
+ }
+ dir := t.TempDir()
+ files := map[string]string{
+ "configure": "#!/bin/sh\necho configured\n",
+ "lib.c": "const char *greeting(void) { return \"hello 1.0\"; }\n",
+ "main.c": "#include \nconst char *greeting(void);\nint main(void) { puts(greeting()); return 0; }\n",
+ "Makefile": "all: hello\nlibhello.so: lib.c\n\tcc -shared -fPIC -o $@ lib.c\n" +
+ "hello: main.c libhello.so\n\tcc -o $@ main.c -L. -lhello\ninstall: all\n" +
+ "\tinstall -m755 hello /usr/local/bin/hello\n\tinstall -m644 libhello.so /usr/local/lib/libhello.so\n",
+ }
+ for name, body := range files {
+ if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ }
+ for _, args := range [][]string{
+ {"init", "-q"}, {"add", "-A"},
+ {"-c", "user.name=t", "-c", "user.email=t@example.invalid", "commit", "-qm", "init"},
+ {"config", "--local", "http.https://example.invalid/.extraheader", "AUTHORIZATION: kibble-marker"},
+ } {
+ if out, err := exec.Command("git", append([]string{"-C", dir}, args...)...).CombinedOutput(); err != nil {
+ t.Fatalf("git %v: %v: %s", args, err, out)
+ }
+ }
+ runner := &DockerRunner{Image: "golang:1.26", Timeout: 120 * time.Second}
+ // The first line fails the recipe if the token planted in the host's git
+ // config reached the container, so neither result below can be met then.
+ recipe := []string{"if grep -rqs kibble-marker .git; then exit 9; fi", "./configure", "make", "make install"}
+ res := runner.Run(ctx, InstallStep{Repo: "hello", Kind: "source-build", Run: true, Block: recipe, dir: dir})
+ if res.Status != StatusFail || !strings.Contains(res.Detail, "libhello.so") {
+ t.Errorf("without ldconfig: status %s detail %q, want FAIL naming libhello.so", res.Status, res.Detail)
+ }
+ recipe = append(recipe, "ldconfig")
+ res = runner.Run(ctx, InstallStep{Repo: "hello", Kind: "source-build", Run: true, Block: recipe, dir: dir})
+ if res.Status != StatusVerified || res.SmokeLine != "hello 1.0" {
+ t.Errorf("with ldconfig: status %s smoke %q detail %q, want VERIFIED", res.Status, res.SmokeLine, res.Detail)
+ }
+}
diff --git a/runclassify.go b/runclassify.go
index 9bade32..e7fc091 100644
--- a/runclassify.go
+++ b/runclassify.go
@@ -111,6 +111,21 @@ func classify(step InstallStep, out string, dur time.Duration) Result {
res.Status = StatusCrossArch
res.SmokeLine = ""
res.Detail = "installed, but the binary targets another architecture, smoke test not possible here"
+ case step.Kind == "source-build" && reLoaderError.MatchString(res.SmokeLine) && step.loaderNote != "":
+ // The document covers the loader in a place the recipe did not reach,
+ // so the reader who follows all of it may well have a working binary.
+ // kibble ran only the block, and its own shortfall is not a failure.
+ res.Status = StatusBuilt
+ res.Detail = fmt.Sprintf("installed, but the binary cannot start until the %s step the document "+
+ "gives outside this block, which kibble did not run", step.loaderNote)
+ case step.Kind == "source-build" && reLoaderError.MatchString(res.SmokeLine):
+ // A failed smoke test is normally only BUILT, since --version and --help
+ // can fail on a working tool. A dynamic loader refusing to start it
+ // cannot: no argument gets that binary to run. After a source build
+ // every library it linked against was present at link time, so the
+ // install the document gave is what left it unable to find one.
+ res.Status = StatusFail
+ res.Detail = "the documented install leaves a binary that cannot start: " + res.SmokeLine
default:
res.Status = StatusBuilt
res.Detail = fmt.Sprintf("binary built but smoke exit=%d", smokeCode)
@@ -131,6 +146,10 @@ var reFlagMarker = regexp.MustCompile(`^KIBBLE-FLAG (--?\S+) CODE=(-?\d+)$`)
// failing its smoke test.
var reArchMismatch = regexp.MustCompile(`qemu-\w+: |Exec format error|cannot execute binary file`)
+// reLoaderError matches the dynamic loader refusing to start a binary because
+// a shared library it needs cannot be found or loaded.
+var reLoaderError = regexp.MustCompile(`error while loading shared libraries|Library not loaded`)
+
// reANSI matches the escape sequences a program writes to color or reposition
// its own output. Both the color form and the wider set of control sequences
// are covered, since a progress bar redrawing itself is as unwelcome in a
diff --git a/runscript.go b/runscript.go
index bdb64e8..d36ac41 100644
--- a/runscript.go
+++ b/runscript.go
@@ -256,6 +256,63 @@ printf 'SMOKECODE=%%d\n' "$scode"
printf 'SMOKELINE=%%s\n' "$(printf '%%s' "$sout" | head -n1 | cut -c1-70)"
`
+// sourceScriptFor builds the container script for a source-build recipe. The
+// checkout's git history arrives on stdin, so the build runs on the committed
+// tree with its submodules reachable, as a reader's clone would, rather than
+// on whatever build output the host's working tree holds.
+func sourceScriptFor(step InstallStep, timeoutSecs int) string {
+ body := strings.ReplaceAll(strings.Join(step.Block, "\n"), "'", `'\''`)
+ name := ""
+ if kplan.IsSimpleWord(step.Repo) {
+ name = step.Repo
+ }
+ return fmt.Sprintf(sourceScript, timeoutSecs, body, name)
+}
+
+// sourceScript restores the checkout from the git history on stdin, runs the
+// documented build and install, and smoke-tests the binary the install put on
+// PATH, in a process of its own after the recipe's shell has exited. That is
+// what a reader does next, and it is where jq's installed binary fails to find
+// the library `make install` placed beside it. Only a binary written during
+// the recipe counts, so a same-named tool the image already ships is never the
+// one tested. The stream holds objects and refs only, so the repository they
+// land in is one the container made itself, with a config of its own. It
+// prints the same markers as installScript.
+const sourceScript = `set -u
+mkdir -p /work/repo
+cd /work/repo
+if ! git init -q . 2>/dev/null || ! tar -xf - 2>/dev/null ||
+ ! git config --global --add safe.directory '*' || ! git reset -q --hard HEAD 2>/dev/null; then
+ printf 'kibble could not restore the checkout from its git history\n'
+ exit 0
+fi
+touch /tmp/kibble-mark
+out=$(timeout %[1]d bash -ec '%[2]s' 2>&1); code=$?
+if [ "$code" -ne 0 ]; then
+ printf 'BUILDCODE=%%d\n' "$code"
+ printf '%%s\n' "$out" | grep -iE 'error(\[[A-Z]|:| )' | head -n 4
+ printf '%%s\n' "$out" | tail -n 10
+ exit 0
+fi
+printf 'BUILDCODE=0\n'
+bin=''
+if [ -n '%[3]s' ]; then
+ bin=$(command -v '%[3]s' 2>/dev/null || true)
+ case "$bin" in /*) [ "$bin" -nt /tmp/kibble-mark ] || bin='' ;; *) bin='' ;; esac
+fi
+if [ -z "$bin" ]; then
+ bin=$(IFS=:; find $PATH -maxdepth 1 -type f -perm -u+x -newer /tmp/kibble-mark 2>/dev/null | head -n1)
+fi
+if [ -z "$bin" ]; then
+ printf 'NOBIN=1\n'
+ exit 0
+fi
+sout=$(cd / && timeout 15 "$bin" --version 2>&1); scode=$?
+if [ "$scode" -ne 0 ]; then sout=$(cd / && timeout 15 "$bin" --help 2>&1); scode=$?; fi
+printf 'SMOKECODE=%%d\n' "$scode"
+printf 'SMOKELINE=%%s\n' "$(printf '%%s' "$sout" | head -n1 | cut -c1-160)"
+`
+
// installScript builds, then smoke-tests, one module inside the container. It
// prints BUILDCODE, SMOKECODE, and SMOKELINE markers for the parent to read. A
// build timeout surfaces as BUILDCODE=124 so it is not mistaken for a failure.
diff --git a/toolchain.go b/toolchain.go
index 7f54b91..3652b73 100644
--- a/toolchain.go
+++ b/toolchain.go
@@ -29,6 +29,12 @@ var toolchains = map[string]toolchain{
"go": {Name: "go", Image: ""},
}
+// sourceBuildImage is the image a source build runs in. It is Debian's own
+// build image, with a C and C++ compiler, make, autoconf, automake, libtool,
+// pkg-config, git, and the common development libraries, which is the machine
+// a configure-and-make README assumes its reader has.
+const sourceBuildImage = "buildpack-deps:stable"
+
// pkgKind describes how one package manager installs a documented tool and
// where the binaries it produces land.
type pkgKind struct {
diff --git a/toolchain_test.go b/toolchain_test.go
index 4555911..20945da 100644
--- a/toolchain_test.go
+++ b/toolchain_test.go
@@ -261,3 +261,13 @@ func TestPythonStepImage(t *testing.T) {
t.Errorf("session image = %q ok=%v, want python:3.14", image, ok)
}
}
+
+// TestSourceBuildImage checks that a source build runs in the build image
+// whatever the runner's default is.
+func TestSourceBuildImage(t *testing.T) {
+ t.Parallel()
+ d := &DockerRunner{Image: "golang:1.26"}
+ if image, _ := d.imageFor(InstallStep{Kind: "source-build", dir: t.TempDir()}); image != sourceBuildImage {
+ t.Errorf("image = %q, want %q", image, sourceBuildImage)
+ }
+}