diff --git a/.gitignore b/.gitignore index 5bcd671cc..6ff23bb5c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .claude/scheduled_tasks.lock .claude/worktrees/ .superset/ +.correctness/ .idea/ /rshell diff --git a/Makefile b/Makefile index f3a158121..233826877 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,8 @@ -.PHONY: build fmt test test_all test_against_bash compliance +.PHONY: build fmt test test_all test_against_bash compliance correctness-scan + +TARGET ?= changed +DEPTH ?= quick +CORRECTNESS_SCAN_ARGS ?= build: go build -o rshell ./cmd/rshell @@ -17,3 +21,6 @@ test_against_bash: compliance: RSHELL_COMPLIANCE_TEST=1 go test -v ./tests/ -run TestCompliance -count=1 + +correctness-scan: + go run ./tools/correctness-scan --target "$(TARGET)" --depth "$(DEPTH)" $(CORRECTNESS_SCAN_ARGS) diff --git a/README.md b/README.md index f01f80c9e..bcbc63b31 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,8 @@ The interpreter supports Linux, macOS, and Windows. Some host-inspection builtin See [CONTRIBUTING.md](CONTRIBUTING.md) for setup, testing, and pull request guidance. Security-sensitive builtin implementation rules live in [docs/RULES.md](docs/RULES.md). +Maintainers can run the AI-assisted, behavior-only correctness scanner locally; see [Correctness scanner](docs/CORRECTNESS_SCANNER.md). It is separate from the repository's security-review tooling and reports one GitHub issue per publishable finding. + ## License [Apache License 2.0](LICENSE) diff --git a/docs/BEHAVIOR_CONTRACTS.md b/docs/BEHAVIOR_CONTRACTS.md new file mode 100644 index 000000000..1998d4727 --- /dev/null +++ b/docs/BEHAVIOR_CONTRACTS.md @@ -0,0 +1,108 @@ +# rshell Behavior Contracts + +These contracts define the user-visible behavior examined by the correctness +scanner. They are compatibility contracts, not security invariants. The scan +must not investigate authorization bypasses, sandbox escapes, secret exposure, +resource-exhaustion attacks, privileged execution, or other threat-model +concerns covered by the repository's security tooling. + +## Sources of truth + +Apply these sources in order: + +1. Explicit accepted design decisions and intentional divergences. +2. `SHELL_FEATURES.md` for the supported rshell surface. +3. User-visible flag, help, and output requirements in `docs/RULES.md`. +4. Bash and GNU behavior for functionality rshell claims to support without a + documented divergence. +5. Builtin help and command descriptions. +6. Existing tests as evidence of intent, but not as final authority. + +The current implementation is the subject under examination and is never an +authority merely because the checked-in tests agree with it. When authoritative +sources disagree, report a documentation ambiguity instead of choosing one. + +## Contracts + +### BEH-001: Supported shell semantics + +For supported syntax and ordinary bounded inputs, parsing, expansion, +execution order, and exit-status propagation match Bash unless an intentional +divergence is documented. + +### BEH-002: Builtin output + +For supported flags and operands, deterministic builtins produce the expected +stdout bytes, including separators, headers, ordering, line endings, and +trailing-newline behavior. + +### BEH-003: Errors and status + +Ordinary invalid usage produces the documented stderr and exit status. This +contract covers user-visible behavior only; whether blocked functionality can +cross a security boundary is outside the scan. + +### BEH-004: Input sources + +Standard input, `-`, file operands, multiple files, pipes, and command +substitutions interact as documented and, where applicable, like the reference +command. + +### BEH-005: Flag interactions + +Supported short, long, combined, repeated, and reordered flags retain their +documented meaning. Attached values, separate values, and `--` termination +follow the documented or reference behavior. + +### BEH-006: Shell state + +Variables, working directories, `$?`, loops, and subshell isolation have the +documented lifetime and propagation behavior. Inline assignments remain scoped +correctly, and subshell changes do not escape to their parent. + +### BEH-007: Formatting and ordering + +Deterministic formatting, numeric rendering, sorting, labeling, and record +boundaries match the selected oracle for supported inputs. + +### BEH-008: Intentional differences + +Documented rshell-specific behavior matches the documentation even when Bash +or a GNU command behaves differently. A reference mismatch is not a defect +when the divergence is explicit and the implementation follows it. + +### BEH-009: Surface consistency + +Builtin registration, command descriptions, `--help`, `README.md`, and +`SHELL_FEATURES.md` agree about the supported user-visible command and flag +surface. + +## Initial scan scope + +The first scanner version focuses on deterministic text-processing builtins +and positive shell semantics: + +- `cat`, `cut`, `echo`, `grep`, `head`, `printf`, `sed`, `sort`, `tail`, + `test`, `tr`, `uniq`, and `wc`. +- Variables, quoting, expansion, pipes, substitutions, AND/OR lists, loops, + conditionals, and subshells. + +Other targets may be requested, but a scan must mark a claim inconclusive when +the target depends on live host state, privileged access, or a security-policy +decision rather than a stable behavioral oracle. + +## Excluded security concerns + +The correctness scanner must not report findings about: + +- `AllowedCommands` or `AllowedPaths` bypasses. +- Remediation or system-service authorization. +- Privileged-helper behavior. +- Secret, environment, process, or telemetry exposure. +- Host-observation trust boundaries. +- Resource-exhaustion attacks or security limits. +- Cross-tenant isolation. +- Side effects occurring before authorization. + +Those concerns belong to the security threat pack and its dedicated review +workflows. diff --git a/docs/CORRECTNESS_SCANNER.md b/docs/CORRECTNESS_SCANNER.md new file mode 100644 index 000000000..11aa552d2 --- /dev/null +++ b/docs/CORRECTNESS_SCANNER.md @@ -0,0 +1,111 @@ +# Correctness scanner + +The correctness scanner is a local, AI-assisted review tool for rshell's +user-visible behavior. It uses Codex to inspect one bounded target, generate +temporary tests in a detached worktree, and validate suspected mismatches with +the repository's existing scenario and Bash-comparison infrastructure. + +It is intentionally separate from security review. The scanner reads +`docs/BEHAVIOR_CONTRACTS.md` and does not investigate sandbox escapes, +authorization bypasses, secret exposure, or other threat-model concerns. + +## Prerequisites + +- A clean, committed Git ref to scan. The current checkout may contain + unrelated untracked files; the scanner always examines the committed ref. +- Go and Git. +- Codex CLI authenticated for local use. +- Docker when a Bash comparison is required. +- GitHub CLI authenticated for issue publication. + +The scanner uses the Codex CLI's configured default model. Model selection is +not exposed by V0. + +## Run locally + +Scan changed behavior relative to `origin/main`: + +```bash +make correctness-scan TARGET=changed DEPTH=quick +``` + +Scan one builtin: + +```bash +make correctness-scan TARGET=cat DEPTH=standard +``` + +Issue publication is enabled by default. For a calibration or offline run: + +```bash +make correctness-scan TARGET=cat CORRECTNESS_SCAN_ARGS=--no-publish +``` + +Local runs print each scanner phase. When the investigation begins, the +scanner prints an exact `tail -f` command for following the full Codex log in +another terminal without flooding the command's normal output. Use `--quiet` +to suppress the phase messages while still writing the log files: + +```bash +make correctness-scan TARGET=cat CORRECTNESS_SCAN_ARGS="--no-publish --quiet" +``` + +The underlying command also accepts `--ref`, `--base-ref`, `--repo`, and +`--output-dir`: + +```bash +go run ./tools/correctness-scan \ + --target shell:variables \ + --depth quick \ + --ref HEAD \ + --no-publish +``` + +## Artifacts + +Each run writes to the gitignored `.correctness/runs//` directory: + +- `prompt.md` — the exact investigator prompt. +- `schema.json` — the structured output contract passed to Codex. +- `codex.stdout.log` and `codex.stderr.log` — complete Codex process output. +- `findings.json` — the structured final response. +- `report.md` — the human-readable scan report. +- `issues/*.md` — one publication preview per publishable finding, including + when `--no-publish` is used. +- `issue-links.md` — issues created, updated, or reopened by the run. + +The temporary Git worktree is removed after the scan unless +`--keep-worktree` is supplied for debugging. + +## Publishing rules + +The scanner publishes one issue per finding for these classifications: + +- Confirmed correctness defect. +- Actionable documentation ambiguity. +- Actionable missing coverage. + +Inconclusive observations remain in the local report. A stable fingerprint in +each issue body prevents duplicate issues. A repeat finding comments on the +existing issue; if the issue was closed, the scanner reopens it first. + +The planned V1 automation will invoke this same command and publication path +on an Actions runner. Only scheduling and credential sourcing move into the +workflow, so local and automated scans use identical finding classifications, +validation, fingerprints, and issue bodies. + +Use `--no-publish` whenever a run should not mutate GitHub. Publishing uses the +developer's existing `gh` authentication and never passes GitHub credentials +to Codex or generated test programs. + +## Isolation and evidence + +Codex receives workspace-write access only to the detached temporary worktree. +It may add temporary scenario or Go tests, but it must not modify production +code, commit, push, or create issues. Issue publication happens afterward in +the parent scanner process. + +A confirmed defect must include an exact reproducer, expected and actual +stdout/stderr/status, at least two successful reproductions, source references, +and implementation references. Claims without that evidence are rejected or +reported as inconclusive. diff --git a/interp/expansion_limits_test.go b/interp/expansion_limits_test.go index f6f820c80..36f55a817 100644 --- a/interp/expansion_limits_test.go +++ b/interp/expansion_limits_test.go @@ -9,6 +9,7 @@ import ( "bytes" "context" "errors" + "fmt" "strings" "sync/atomic" "testing" @@ -94,6 +95,28 @@ func TestExpandedArgumentCountLimitAllowsBoundary(t *testing.T) { } } +func TestNonWhitespaceIFSFieldCountLimit(t *testing.T) { + // Empty fields consume argument slots even though they consume no bytes. + for _, count := range []int{MaxExpandedArgumentsPerCommand, MaxExpandedArgumentsPerCommand + 1} { + t.Run(fmt.Sprint(count), func(t *testing.T) { + stdout, stderr, err := runExpansionScript(t, "IFS=:\ntrue $X\necho done", + Env("X="+strings.Repeat(":", count)), + AllowedCommands([]string{"rshell:true", "rshell:echo"}), + ) + if count == MaxExpandedArgumentsPerCommand { + if err != nil || stderr != "" || stdout != "done\n" { + t.Fatalf("boundary: stdout=%q stderr=%q err=%v", stdout, stderr, err) + } + return + } + requireExitStatus(t, err, 1) + if stdout != "" || !strings.Contains(stderr, "expansion exceeds maximum field count") { + t.Fatalf("overflow: stdout=%q stderr=%q", stdout, stderr) + } + }) + } +} + func TestExpandedArgumentByteLimit(t *testing.T) { value := strings.Repeat("x", 1<<20) script := "true " + strings.TrimSpace(strings.Repeat("$X ", 11)) diff --git a/interp/field_splitting.go b/interp/field_splitting.go new file mode 100644 index 000000000..d869c7787 --- /dev/null +++ b/interp/field_splitting.go @@ -0,0 +1,144 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package interp + +import ( + "iter" + "strconv" + "strings" + "unicode/utf8" + + "mvdan.cc/sh/v3/expand" + "mvdan.cc/sh/v3/syntax" +) + +// fieldsSeq compensates for mvdan's coalescing of non-whitespace IFS +// delimiters. Keep its expansion path when IFS contains only whitespace. +func (r *Runner) fieldsSeq(word *syntax.Word) iter.Seq2[string, error] { + ifs := r.ecfg.Env.Get("IFS").String() + if strings.Trim(ifs, " \t\n") == "" { + return expand.FieldsSeq(r.ecfg, word) + } + return func(yield func(string, error) bool) { + word := *word + syntax.SplitBraces(&word) + for w, err := range expand.BracesSeq(r.ecfg, &word) { + if err != nil { + yield("", err) + return + } + if !r.splitWordFields(w, ifs, yield) { + return + } + } + } +} + +func (r *Runner) splitWordFields(word *syntax.Word, ifs string, yield func(string, error) bool) bool { + // Evaluate every expansion once, in source order, before pathname + // expansion. A nil part marks unquoted data subject to field splitting. + type expandedPart struct { + part syntax.WordPart + value string + } + parts := make([]expandedPart, 0, len(word.Parts)) + for _, part := range word.Parts { + if r.stop(r.ectx) { + return false + } + switch part.(type) { + case *syntax.ParamExp, *syntax.CmdSubst, *syntax.SglQuoted, *syntax.DblQuoted: + value, err := expand.Literal(r.ecfg, &syntax.Word{Parts: []syntax.WordPart{part}}) + if err != nil { + yield("", err) + return false + } + switch part.(type) { + case *syntax.SglQuoted, *syntax.DblQuoted: + parts = append(parts, expandedPart{part: &syntax.SglQuoted{Value: value}}) + default: + parts = append(parts, expandedPart{value: value}) + } + default: + parts = append(parts, expandedPart{part: part}) + } + } + + // Feed already-split unquoted data back as parameters with IFS disabled. + // Unlike synthetic literals, parameters cannot reinterpret backslashes, + // tildes or braces in expansion results. All original parameters and + // quoted substitutions were evaluated above using the real environment. + env := &overlayEnviron{parent: r.ecfg.Env, values: make(map[string]expand.Variable)} + cfg := *r.ecfg + cfg.Env = env + var field syntax.Word + reset := func() { + clear(env.values) + env.values["IFS"] = expand.Variable{Set: true, Kind: expand.String} + // This anchor preserves explicit empty fields and prevents a literal + // following a split expansion from becoming a tilde expansion. + field.Parts = []syntax.WordPart{&syntax.SglQuoted{}} + } + reset() + flush := func(force bool) bool { + if !force && len(field.Parts) == 1 { + return true + } + for value, err := range expand.FieldsSeq(&cfg, &field) { + if !yield(value, err) || err != nil { + return false + } + } + reset() + return true + } + addValue := func(value string) { + name := "field" + strconv.Itoa(len(field.Parts)) + env.values[name] = expand.Variable{Set: true, Kind: expand.String, Str: value} + field.Parts = append(field.Parts, &syntax.ParamExp{Param: &syntax.Lit{Value: name}}) + } + + // Defer whitespace separators so an adjacent non-whitespace delimiter + // (even in the next expansion) can join them into a single separator. + whitespace := false + for _, part := range parts { + if part.part != nil { + if whitespace && !flush(false) { + return false + } + whitespace = false + field.Parts = append(field.Parts, part.part) + continue + } + start := 0 + for i, ch := range part.value { + if !strings.ContainsRune(ifs, ch) { + if whitespace && !flush(false) { + return false + } + whitespace = false + continue + } + if i > start { + addValue(part.value[start:i]) + } + _, size := utf8.DecodeRuneInString(part.value[i:]) + start = i + size + if ch == ' ' || ch == '\t' || ch == '\n' { + whitespace = true + } else { + if !flush(true) { + return false + } + whitespace = false + } + } + if start < len(part.value) { + addValue(part.value[start:]) + } + } + return flush(false) +} diff --git a/interp/runner_expand.go b/interp/runner_expand.go index f4d46602e..052501f72 100644 --- a/interp/runner_expand.go +++ b/interp/runner_expand.go @@ -316,7 +316,7 @@ func (c *fieldCollector) add(words ...*syntax.Word) bool { return false } - for field, err := range expand.FieldsSeq(c.r.ecfg, word) { + for field, err := range c.r.fieldsSeq(word) { if err != nil { if strings.HasPrefix(err.Error(), braceExpansionLimitErrorPrefix) { err = &expansionLimitError{message: err.Error()} diff --git a/tests/scenarios/shell/field_splitting/consecutive_nonwhitespace_ifs.yaml b/tests/scenarios/shell/field_splitting/consecutive_nonwhitespace_ifs.yaml index d2671a6d8..d6aef8018 100644 --- a/tests/scenarios/shell/field_splitting/consecutive_nonwhitespace_ifs.yaml +++ b/tests/scenarios/shell/field_splitting/consecutive_nonwhitespace_ifs.yaml @@ -1,18 +1,38 @@ -description: > - Consecutive non-whitespace IFS delimiters should produce empty fields. - In bash, "a::b" with IFS=: produces three fields: "a", "", "b". - The upstream mvdan.cc/sh library does not produce the empty middle field. - skip_assert_against_bash because of this known upstream limitation. -# skip: known upstream library limitation with consecutive non-whitespace IFS delimiters -skip_assert_against_bash: true +description: Leading and consecutive non-whitespace IFS delimiters preserve empty fields; a trailing delimiter only terminates the last field. input: script: |+ IFS=: - A="a::b" - for w in $A; do echo "[$w]"; done + for A in '' ':' '::' ':a::b:' 'a::b' 'a::'; do + printf 'value=<%s>\n' "$A" + for w in $A; do printf '[%s]\n' "$w"; done + done + x=':a::b:' + printf '<%s>\n' $x + IFS=; x='a b'; printf '<%s>\n' $x expect: - # bash produces: [a]\n[]\n[b] - # rshell currently produces: [a]\n[b] (upstream library limitation) - stdout: "[a]\n[b]\n" - stderr: "" + stdout: |+ + value=<> + value=<:> + [] + value=<::> + [] + [] + value=<:a::b:> + [] + [a] + [] + [b] + value= + [a] + [] + [b] + value= + [a] + [] + <> + + <> + + + stderr: |+ exit_code: 0 diff --git a/tests/scenarios/shell/field_splitting/nonwhitespace_ifs_globbing.yaml b/tests/scenarios/shell/field_splitting/nonwhitespace_ifs_globbing.yaml new file mode 100644 index 000000000..34d9814da --- /dev/null +++ b/tests/scenarios/shell/field_splitting/nonwhitespace_ifs_globbing.yaml @@ -0,0 +1,35 @@ +description: Non-whitespace IFS fields retain globbing and quote protection without reinterpreting expansion results as shell syntax. +setup: + files: + - path: a.txt + content: "" + - path: b.txt + content: "" +input: + allowed_paths: ["$DIR"] + script: |+ + IFS=: + x=':*.txt::missing*:' + printf '<%s>\n' $x + printf 'quoted=<%s>\n' "$x" + x=':a:' + printf 'suffix=<%s>\n' $x'*' + x=':{a,b}:~:\q:' + printf 'data=<%s>\n' $x +expect: + stdout: |+ + <> + + + <> + + quoted=<:*.txt::missing*:> + suffix=<> + suffix= + suffix=<*> + data=<> + data=<{a,b}> + data=<~> + data=<\q> + stderr: |+ + exit_code: 0 diff --git a/tests/scenarios/shell/field_splitting/nonwhitespace_ifs_word_parts.yaml b/tests/scenarios/shell/field_splitting/nonwhitespace_ifs_word_parts.yaml new file mode 100644 index 000000000..719a51a92 --- /dev/null +++ b/tests/scenarios/shell/field_splitting/nonwhitespace_ifs_word_parts.yaml @@ -0,0 +1,61 @@ +description: Non-whitespace IFS splitting respects adjacent expansions, IFS whitespace, quotes, literals, braces, and command substitutions. +input: + script: |+ + IFS=' :,' + x=' :a: :b,, c : ' + printf '<%s>\n' $x + IFS=' :' + x='a '; y=':b' + printf 'joined=<%s>\n' $x$y + printf 'quoted=<%s>\n' $x""$y + x='a:'; y=' :b' + printf 'joined=<%s>\n' $x$y + IFS=: + x=':a::b:' + printf 'literal=<%s>\n' pre${x}post a::b + printf 'quoted=<%s>\n' "$x" '$x' + printf 'braces=<%s>\n' {p,q}${x} + printf 'substitution=<%s>\n' $(printf ':a::b:\n\n'; echo once >&2) + x='a:' + printf 'suffix=<%s>\n' $x"" +expect: + stdout: |+ + <> + + <> + + <> + + joined= + joined= + quoted= + quoted=<> + quoted= + joined= + joined=<> + joined= + literal=
+    literal=
+    literal=<>
+    literal=
+    literal=
+    literal=
+    quoted=<:a::b:>
+    quoted=<$x>
+    braces=

+ braces= + braces=<> + braces= + braces= + braces= + braces=<> + braces= + substitution=<> + substitution= + substitution=<> + substitution= + suffix= + suffix=<> + stderr: |+ + once + exit_code: 0 diff --git a/tools/correctness-scan/findings.schema.json b/tools/correctness-scan/findings.schema.json new file mode 100644 index 000000000..c3e83db54 --- /dev/null +++ b/tools/correctness-scan/findings.schema.json @@ -0,0 +1,161 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": false, + "properties": { + "summary": { "type": "string" }, + "components_examined": { + "type": "array", + "items": { "type": "string" } + }, + "hypotheses_tested": { "type": "integer", "minimum": 0 }, + "findings": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "classification": { + "type": "string", + "enum": [ + "confirmed-defect", + "documentation-ambiguity", + "missing-coverage", + "inconclusive" + ] + }, + "contract_id": { + "type": "string", + "enum": [ + "BEH-001", + "BEH-002", + "BEH-003", + "BEH-004", + "BEH-005", + "BEH-006", + "BEH-007", + "BEH-008", + "BEH-009" + ] + }, + "target": { "type": "string" }, + "title": { "type": "string" }, + "impact": { + "type": "string", + "enum": ["high", "medium", "low"] + }, + "description": { "type": "string" }, + "source_references": { + "type": "array", + "items": { "$ref": "#/$defs/reference" } + }, + "implementation_references": { + "type": "array", + "items": { "$ref": "#/$defs/reference" } + }, + "reproducer": { "$ref": "#/$defs/reproducer" }, + "expected": { "$ref": "#/$defs/outcome" }, + "actual": { "$ref": "#/$defs/outcome" }, + "verification": { "$ref": "#/$defs/verification" }, + "suggested_test_location": { "type": "string" } + }, + "required": [ + "classification", + "contract_id", + "target", + "title", + "impact", + "description", + "source_references", + "implementation_references", + "reproducer", + "expected", + "actual", + "verification", + "suggested_test_location" + ] + } + }, + "coverage_notes": { + "type": "object", + "additionalProperties": false, + "properties": { + "files_read": { + "type": "array", + "items": { "type": "string" } + }, + "tests_reviewed": { + "type": "array", + "items": { "type": "string" } + }, + "cases_attempted": { + "type": "array", + "items": { "type": "string" } + }, + "areas_not_examined": { + "type": "array", + "items": { "type": "string" } + } + }, + "required": [ + "files_read", + "tests_reviewed", + "cases_attempted", + "areas_not_examined" + ] + } + }, + "required": [ + "summary", + "components_examined", + "hypotheses_tested", + "findings", + "coverage_notes" + ], + "$defs": { + "reference": { + "type": "object", + "additionalProperties": false, + "properties": { + "location": { "type": "string" }, + "detail": { "type": "string" } + }, + "required": ["location", "detail"] + }, + "reproducer": { + "type": "object", + "additionalProperties": false, + "properties": { + "script": { "type": "string" }, + "fixture_setup": { "type": "string" }, + "commands": { + "type": "array", + "items": { "type": "string" } + } + }, + "required": ["script", "fixture_setup", "commands"] + }, + "outcome": { + "type": "object", + "additionalProperties": false, + "properties": { + "stdout": { "type": "string" }, + "stderr": { "type": "string" }, + "exit_code": { + "anyOf": [{ "type": "integer" }, { "type": "null" }] + } + }, + "required": ["stdout", "stderr", "exit_code"] + }, + "verification": { + "type": "object", + "additionalProperties": false, + "properties": { + "runs": { "type": "integer", "minimum": 0 }, + "reference": { "type": "string" }, + "temporary_test": { "type": "string" } + }, + "required": ["runs", "reference", "temporary_test"] + } + } +} diff --git a/tools/correctness-scan/investigator.md b/tools/correctness-scan/investigator.md new file mode 100644 index 000000000..de790379c --- /dev/null +++ b/tools/correctness-scan/investigator.md @@ -0,0 +1,59 @@ +You are running a behavior-only correctness scan of the rshell repository. + +Read `docs/BEHAVIOR_CONTRACTS.md` completely before investigating. Treat it, +`SHELL_FEATURES.md`, command help, and the applicable user-visible portions of +`docs/RULES.md` as the contract. Bash or a GNU command is authoritative only +for supported behavior without a documented rshell divergence. Existing tests +are evidence, not ground truth, and current implementation output is never +ground truth merely because a test expects it. + +## Scope + +- Target: {{TARGET}} +- Depth: {{DEPTH}} +- Scanned commit: {{COMMIT}} +- Comparison base commit: {{BASE_COMMIT}} +- Maximum hypotheses: {{MAX_HYPOTHESES}} + +Focus only on the requested target and directly related tests and +documentation. Prefer a few carefully verified hypotheses over a broad review. + +This is not a security review. Do not investigate or report authorization +bypasses, sandbox escapes, remediation policy, privileged execution, secret or +telemetry exposure, resource-exhaustion attacks, or cross-tenant isolation. + +## Workflow + +1. Read the relevant implementation, documentation, and existing tests. +2. Identify behavior within the supported surface that is missing meaningful + coverage or appears inconsistent with its oracle. +3. Prefer temporary scenario YAML files under `tests/scenarios/`. Always use + YAML `|+` block scalars for scripts and exact stdout/stderr values. Use a + temporary Go test only when the scenario framework cannot express the case. +4. Prefix every temporary file with `zz_correctness_scan_`. Do not modify or + delete existing files. +5. Run focused rshell tests. For eligible Bash-compatible behavior, run the + existing Bash comparison with `RSHELL_BASH_TEST=1`. Do not use Bash as the + oracle for AWK or an intentional divergence. +6. Reproduce a claimed implementation defect at least twice from clean test + fixtures. Capture exact stdout, stderr, and exit status. +7. Return the structured result required by the supplied JSON schema. + +Do not modify production code or checked-in expectations. Do not commit, push, +create GitHub issues, or access production systems. Leave temporary test files +in place; the parent process destroys this worktree after collecting results. + +Classify findings as follows: + +- `confirmed-defect`: deterministic mismatch against an authoritative + behavioral oracle, reproduced at least twice. +- `documentation-ambiguity`: authoritative user-visible sources conflict and + the conflict is actionable. +- `missing-coverage`: a meaningful contract lacks a regression test, but no + implementation defect was demonstrated. +- `inconclusive`: the claim lacks a stable oracle or could not be reproduced. + +For a confirmed defect, include the exact script and fixture setup, commands +used, expected and actual outputs, reproduction count, source references, and +implementation references. If any required evidence is unavailable, classify +the observation as inconclusive instead. diff --git a/tools/correctness-scan/main.go b/tools/correctness-scan/main.go new file mode 100644 index 000000000..36a257d76 --- /dev/null +++ b/tools/correctness-scan/main.go @@ -0,0 +1,1005 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package main + +import ( + "bytes" + "context" + "crypto/sha256" + _ "embed" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net/url" + "os" + "os/exec" + "os/signal" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "time" +) + +//go:embed investigator.md +var investigatorTemplate string + +//go:embed findings.schema.json +var findingsSchema []byte + +const ( + defaultOutputDir = ".correctness/runs" + fingerprintLabel = "rshell-correctness-fingerprint" +) + +var ( + targetPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9:._/-]{0,127}$`) + refPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]{0,255}$`) + repoPattern = regexp.MustCompile(`^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$`) + markerPattern = regexp.MustCompile(``) +) + +type config struct { + target string + depth string + ref string + baseRef string + repository string + outputDir string + codexBinary string + ghBinary string + publish bool + dryRun bool + quiet bool + keepWorktree bool +} + +type scanResult struct { + Summary string `json:"summary"` + ComponentsExamined []string `json:"components_examined"` + HypothesesTested int `json:"hypotheses_tested"` + Findings []finding `json:"findings"` + CoverageNotes coverageNotes `json:"coverage_notes"` +} + +type finding struct { + Classification string `json:"classification"` + ContractID string `json:"contract_id"` + Target string `json:"target"` + Title string `json:"title"` + Impact string `json:"impact"` + Description string `json:"description"` + SourceReferences []reference `json:"source_references"` + ImplementationReferences []reference `json:"implementation_references"` + Reproducer reproducer `json:"reproducer"` + Expected outcome `json:"expected"` + Actual outcome `json:"actual"` + Verification verification `json:"verification"` + SuggestedTestLocation string `json:"suggested_test_location"` +} + +type reference struct { + Location string `json:"location"` + Detail string `json:"detail"` +} + +type reproducer struct { + Script string `json:"script"` + FixtureSetup string `json:"fixture_setup"` + Commands []string `json:"commands"` +} + +type outcome struct { + Stdout string `json:"stdout"` + Stderr string `json:"stderr"` + ExitCode *int `json:"exit_code"` +} + +type verification struct { + Runs int `json:"runs"` + Reference string `json:"reference"` + TemporaryTest string `json:"temporary_test"` +} + +type coverageNotes struct { + FilesRead []string `json:"files_read"` + TestsReviewed []string `json:"tests_reviewed"` + CasesAttempted []string `json:"cases_attempted"` + AreasNotExamined []string `json:"areas_not_examined"` +} + +type runMetadata struct { + RunID string + Target string + Depth string + Commit string + BaseCommit string + Repository string + StartedAt time.Time + WorkflowURL string +} + +type existingIssue struct { + Number int `json:"number"` + State string `json:"state"` + URL string `json:"url"` + Title string `json:"title"` + Body string `json:"body"` +} + +type issueLink struct { + Action string + URL string + Title string +} + +type issuePreview struct { + Finding finding + Fingerprint string + BodyPath string +} + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + + if err := run(ctx, os.Args[1:], os.Stdout, os.Stderr); err != nil { + if errors.Is(err, flag.ErrHelp) { + return + } + fmt.Fprintln(os.Stderr, "correctness-scan:", err) + os.Exit(1) + } +} + +func run(ctx context.Context, args []string, stdout, stderr io.Writer) error { + cfg, err := parseConfig(args, stderr) + if err != nil { + return err + } + + repoRoot, err := commandOutput(ctx, "", "git", "rev-parse", "--show-toplevel") + if err != nil { + return fmt.Errorf("locate repository root: %w", err) + } + repoRoot = strings.TrimSpace(repoRoot) + + commit, err := resolveCommit(ctx, repoRoot, cfg.ref) + if err != nil { + return fmt.Errorf("resolve --ref %q: %w", cfg.ref, err) + } + baseCommit := "" + if cfg.target == "changed" { + baseCommit, err = resolveCommit(ctx, repoRoot, cfg.baseRef) + if err != nil { + return fmt.Errorf("resolve --base-ref %q: %w", cfg.baseRef, err) + } + } + + repository := cfg.repository + if repository == "" { + repository, err = repositoryFromRemote(ctx, repoRoot) + if err != nil && cfg.publish { + return err + } + } + + startedAt := time.Now().UTC() + runID := startedAt.Format("20060102T150405.000000000Z") + "-" + commit[:12] + outputRoot := cfg.outputDir + if !filepath.IsAbs(outputRoot) { + outputRoot = filepath.Join(repoRoot, outputRoot) + } + runDir := filepath.Join(outputRoot, runID) + if err := os.MkdirAll(runDir, 0o755); err != nil { + return fmt.Errorf("create run directory: %w", err) + } + + meta := runMetadata{ + RunID: runID, + Target: cfg.target, + Depth: cfg.depth, + Commit: commit, + BaseCommit: baseCommit, + Repository: repository, + StartedAt: startedAt, + WorkflowURL: workflowURL(), + } + prompt := buildPrompt(cfg, commit, baseCommit) + if err := os.WriteFile(filepath.Join(runDir, "prompt.md"), []byte(prompt), 0o644); err != nil { + return fmt.Errorf("write prompt: %w", err) + } + schemaPath := filepath.Join(runDir, "schema.json") + if err := os.WriteFile(schemaPath, findingsSchema, 0o644); err != nil { + return fmt.Errorf("write output schema: %w", err) + } + + if cfg.dryRun { + fmt.Fprintf(stdout, "Prepared correctness scan at %s\n", runDir) + fmt.Fprintln(stdout, "Dry run requested; Codex and GitHub were not invoked.") + return nil + } + logProgress(cfg, stderr, "run directory: %s", runDir) + + tempRoot, err := os.MkdirTemp("", "rshell-correctness-scan-") + if err != nil { + return fmt.Errorf("create temporary directory: %w", err) + } + worktreeDir := filepath.Join(tempRoot, "repo") + logProgress(cfg, stderr, "preparing detached worktree for %s", commit[:12]) + if _, err := commandOutput(ctx, repoRoot, "git", "worktree", "add", "--detach", worktreeDir, commit); err != nil { + _ = os.RemoveAll(tempRoot) + return fmt.Errorf("create detached worktree: %w", err) + } + defer cleanupWorktree(ctx, repoRoot, tempRoot, worktreeDir, cfg.keepWorktree, cfg.quiet, stderr) + + resultPath := filepath.Join(runDir, "findings.json") + logProgress(cfg, stderr, "investigation started") + logProgress(cfg, stderr, "follow along: tail -f %s", shellQuote(filepath.Join(runDir, "codex.stderr.log"))) + if err := invokeCodex(ctx, cfg, worktreeDir, runDir, schemaPath, resultPath, prompt); err != nil { + return err + } + + logProgress(cfg, stderr, "Codex finished; validating structured findings") + result, err := readResult(resultPath) + if err != nil { + return err + } + if err := validateResult(result, cfg); err != nil { + return fmt.Errorf("validate Codex result: %w", err) + } + if err := writePrettyJSON(resultPath, result); err != nil { + return err + } + + reportPath := filepath.Join(runDir, "report.md") + if err := os.WriteFile(reportPath, []byte(renderReport(meta, result)), 0o644); err != nil { + return fmt.Errorf("write report: %w", err) + } + logProgress(cfg, stderr, "report written: %s", reportPath) + + previews, err := prepareIssuePreviews(runDir, meta, result) + if err != nil { + return err + } + var links []issueLink + if cfg.publish { + logProgress(cfg, stderr, "publishing %d finding(s) to GitHub", len(previews)) + links, err = publishFindings(ctx, cfg, repoRoot, meta, previews) + if err != nil { + _ = writeIssueLinks(filepath.Join(runDir, "issue-links.md"), links, err) + return fmt.Errorf("publish findings: %w (report: %s)", err, reportPath) + } + } else { + logProgress(cfg, stderr, "publication disabled; wrote %d issue preview(s)", len(previews)) + } + issueLinksPath := filepath.Join(runDir, "issue-links.md") + if err := writeIssueLinks(issueLinksPath, links, nil); err != nil { + return err + } + + fmt.Fprintf(stdout, "Correctness scan complete: %s\n", reportPath) + if cfg.publish { + printIssueSummary(stdout, links) + } else { + fmt.Fprintln(stdout, "GitHub publication disabled by --no-publish.") + } + return nil +} + +func parseConfig(args []string, stderr io.Writer) (config, error) { + fs := flag.NewFlagSet("correctness-scan", flag.ContinueOnError) + fs.SetOutput(stderr) + var cfg config + var noPublish bool + fs.StringVar(&cfg.target, "target", "changed", "builtin, shell feature, or changed") + fs.StringVar(&cfg.depth, "depth", "quick", "scan depth: quick or standard") + fs.StringVar(&cfg.ref, "ref", "HEAD", "committed Git ref to scan") + fs.StringVar(&cfg.baseRef, "base-ref", "origin/main", "comparison ref used by target=changed") + fs.StringVar(&cfg.repository, "repo", "", "GitHub owner/repository override") + fs.StringVar(&cfg.outputDir, "output-dir", defaultOutputDir, "directory for run artifacts") + fs.StringVar(&cfg.codexBinary, "codex-bin", "codex", "Codex CLI executable") + fs.StringVar(&cfg.ghBinary, "gh-bin", "gh", "GitHub CLI executable") + fs.BoolVar(&noPublish, "no-publish", false, "write issue previews without mutating GitHub") + fs.BoolVar(&cfg.dryRun, "dry-run", false, "prepare artifacts without invoking Codex or GitHub") + fs.BoolVar(&cfg.quiet, "quiet", false, "suppress phase messages") + fs.BoolVar(&cfg.keepWorktree, "keep-worktree", false, "keep the temporary worktree for debugging") + if err := fs.Parse(args); err != nil { + return config{}, err + } + if fs.NArg() != 0 { + return config{}, fmt.Errorf("unexpected positional arguments: %s", strings.Join(fs.Args(), " ")) + } + cfg.publish = !noPublish + if !targetPattern.MatchString(cfg.target) || strings.Contains(cfg.target, "..") { + return config{}, fmt.Errorf("invalid --target %q", cfg.target) + } + if cfg.depth != "quick" && cfg.depth != "standard" { + return config{}, fmt.Errorf("invalid --depth %q: want quick or standard", cfg.depth) + } + if err := validateRef(cfg.ref); err != nil { + return config{}, fmt.Errorf("invalid --ref: %w", err) + } + if err := validateRef(cfg.baseRef); err != nil { + return config{}, fmt.Errorf("invalid --base-ref: %w", err) + } + if cfg.repository != "" && !repoPattern.MatchString(cfg.repository) { + return config{}, fmt.Errorf("invalid --repo %q: want owner/repository", cfg.repository) + } + return cfg, nil +} + +func validateRef(ref string) error { + if !refPattern.MatchString(ref) || strings.Contains(ref, "..") || strings.Contains(ref, "@{") { + return fmt.Errorf("unsupported Git ref %q", ref) + } + return nil +} + +func resolveCommit(ctx context.Context, dir, ref string) (string, error) { + out, err := commandOutput(ctx, dir, "git", "rev-parse", "--verify", ref+"^{commit}") + if err != nil { + return "", err + } + commit := strings.TrimSpace(out) + if len(commit) != 40 { + return "", fmt.Errorf("unexpected commit ID %q", commit) + } + return commit, nil +} + +func repositoryFromRemote(ctx context.Context, repoRoot string) (string, error) { + remote, err := commandOutput(ctx, repoRoot, "git", "remote", "get-url", "origin") + if err != nil { + return "", fmt.Errorf("resolve GitHub repository: %w", err) + } + repository, err := parseGitHubRepository(strings.TrimSpace(remote)) + if err != nil { + return "", fmt.Errorf("resolve GitHub repository: %w", err) + } + return repository, nil +} + +func parseGitHubRepository(remote string) (string, error) { + var path string + switch { + case strings.HasPrefix(remote, "git@github.com:"): + path = strings.TrimPrefix(remote, "git@github.com:") + default: + parsed, err := url.Parse(remote) + if err != nil { + return "", err + } + if parsed.Hostname() != "github.com" { + return "", fmt.Errorf("origin is not github.com: %q", remote) + } + path = strings.TrimPrefix(parsed.Path, "/") + } + path = strings.TrimSuffix(path, ".git") + if !repoPattern.MatchString(path) { + return "", fmt.Errorf("cannot parse owner/repository from %q", remote) + } + return path, nil +} + +func buildPrompt(cfg config, commit, baseCommit string) string { + maxHypotheses := 3 + if cfg.depth == "standard" { + maxHypotheses = 8 + } + if baseCommit == "" { + baseCommit = "not applicable" + } + prompt := strings.NewReplacer( + "{{TARGET}}", cfg.target, + "{{DEPTH}}", cfg.depth, + "{{COMMIT}}", commit, + "{{BASE_COMMIT}}", baseCommit, + "{{MAX_HYPOTHESES}}", strconv.Itoa(maxHypotheses), + ).Replace(investigatorTemplate) + if cfg.target == "changed" { + prompt += fmt.Sprintf("\nFor the changed target, begin with `git diff --stat %s...%s` and map only the relevant changed paths to behavior contracts.\n", baseCommit, commit) + } + return prompt +} + +func invokeCodex(ctx context.Context, cfg config, worktreeDir, runDir, schemaPath, resultPath, prompt string) error { + timeout := 30 * time.Minute + if cfg.depth == "standard" { + timeout = 60 * time.Minute + } + scanCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + stdoutFile, err := os.Create(filepath.Join(runDir, "codex.stdout.log")) + if err != nil { + return fmt.Errorf("create Codex stdout log: %w", err) + } + defer stdoutFile.Close() + stderrFile, err := os.Create(filepath.Join(runDir, "codex.stderr.log")) + if err != nil { + return fmt.Errorf("create Codex stderr log: %w", err) + } + defer stderrFile.Close() + + cmd := exec.CommandContext(scanCtx, cfg.codexBinary, + "exec", + "--approve-for-me", + "--ephemeral", + "--color", "never", + "--output-schema", schemaPath, + "--output-last-message", resultPath, + "-", + ) + cmd.Dir = worktreeDir + cmd.Env = codexEnvironment(os.Environ()) + cmd.Stdin = strings.NewReader(prompt) + cmd.Stdout = stdoutFile + cmd.Stderr = stderrFile + if err := cmd.Run(); err != nil { + if errors.Is(scanCtx.Err(), context.DeadlineExceeded) { + return fmt.Errorf("Codex scan exceeded %s; logs: %s", timeout, runDir) + } + return fmt.Errorf("Codex exited unsuccessfully: %w; logs: %s", err, runDir) + } + return nil +} + +func readResult(path string) (scanResult, error) { + f, err := os.Open(path) + if err != nil { + return scanResult{}, fmt.Errorf("open Codex result: %w", err) + } + defer f.Close() + var result scanResult + decoder := json.NewDecoder(f) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&result); err != nil { + return scanResult{}, fmt.Errorf("decode Codex result: %w", err) + } + if decoder.Decode(&struct{}{}) != io.EOF { + return scanResult{}, errors.New("decode Codex result: trailing JSON data") + } + return result, nil +} + +func validateResult(result scanResult, cfg config) error { + maxHypotheses := 3 + if cfg.depth == "standard" { + maxHypotheses = 8 + } + if strings.TrimSpace(result.Summary) == "" { + return errors.New("summary is empty") + } + if result.HypothesesTested < 0 || result.HypothesesTested > maxHypotheses { + return fmt.Errorf("hypotheses_tested is %d, want 0..%d", result.HypothesesTested, maxHypotheses) + } + if len(result.Findings) > maxHypotheses { + return fmt.Errorf("findings contains %d entries, exceeds hypothesis budget %d", len(result.Findings), maxHypotheses) + } + if len(result.Findings) > result.HypothesesTested { + return fmt.Errorf("findings contains %d entries but only %d hypotheses were tested", len(result.Findings), result.HypothesesTested) + } + validContracts := map[string]bool{ + "BEH-001": true, "BEH-002": true, "BEH-003": true, + "BEH-004": true, "BEH-005": true, "BEH-006": true, + "BEH-007": true, "BEH-008": true, "BEH-009": true, + } + seenFingerprints := make(map[string]bool) + for i, f := range result.Findings { + if !validContracts[f.ContractID] { + return fmt.Errorf("finding %d has invalid contract_id %q", i+1, f.ContractID) + } + if strings.TrimSpace(f.Target) == "" || strings.TrimSpace(f.Title) == "" || strings.TrimSpace(f.Description) == "" { + return fmt.Errorf("finding %d is missing target, title, or description", i+1) + } + if f.Impact != "high" && f.Impact != "medium" && f.Impact != "low" { + return fmt.Errorf("finding %d has invalid impact %q", i+1, f.Impact) + } + switch f.Classification { + case "confirmed-defect": + if f.Verification.Runs < 2 { + return fmt.Errorf("finding %d confirmed-defect has %d verification runs, want at least 2", i+1, f.Verification.Runs) + } + if strings.TrimSpace(f.Reproducer.Script) == "" || len(f.Reproducer.Commands) == 0 { + return fmt.Errorf("finding %d confirmed-defect lacks an executable reproducer", i+1) + } + if len(f.SourceReferences) == 0 || len(f.ImplementationReferences) == 0 { + return fmt.Errorf("finding %d confirmed-defect lacks source or implementation references", i+1) + } + if outcomesEqual(f.Expected, f.Actual) { + return fmt.Errorf("finding %d confirmed-defect has identical expected and actual outcomes", i+1) + } + case "documentation-ambiguity": + if len(f.SourceReferences) < 2 { + return fmt.Errorf("finding %d documentation-ambiguity needs at least two conflicting sources", i+1) + } + case "missing-coverage": + if strings.TrimSpace(f.SuggestedTestLocation) == "" { + return fmt.Errorf("finding %d missing-coverage lacks a suggested test location", i+1) + } + case "inconclusive": + default: + return fmt.Errorf("finding %d has invalid classification %q", i+1, f.Classification) + } + fingerprint := findingFingerprint(f) + if seenFingerprints[fingerprint] { + return fmt.Errorf("finding %d duplicates an earlier finding", i+1) + } + seenFingerprints[fingerprint] = true + } + return nil +} + +func outcomesEqual(a, b outcome) bool { + if a.Stdout != b.Stdout || a.Stderr != b.Stderr { + return false + } + if a.ExitCode == nil || b.ExitCode == nil { + return a.ExitCode == nil && b.ExitCode == nil + } + return *a.ExitCode == *b.ExitCode +} + +func writePrettyJSON(path string, result scanResult) error { + data, err := json.MarshalIndent(result, "", " ") + if err != nil { + return fmt.Errorf("encode normalized result: %w", err) + } + data = append(data, '\n') + if err := os.WriteFile(path, data, 0o644); err != nil { + return fmt.Errorf("write normalized result: %w", err) + } + return nil +} + +func renderReport(meta runMetadata, result scanResult) string { + var b strings.Builder + fmt.Fprintln(&b, "# rshell correctness scan") + fmt.Fprintln(&b) + fmt.Fprintf(&b, "- Run: `%s`\n", meta.RunID) + fmt.Fprintf(&b, "- Commit: `%s`\n", meta.Commit) + if meta.BaseCommit != "" { + fmt.Fprintf(&b, "- Comparison base: `%s`\n", meta.BaseCommit) + } + fmt.Fprintf(&b, "- Target: `%s`\n", meta.Target) + fmt.Fprintf(&b, "- Depth: `%s`\n", meta.Depth) + fmt.Fprintf(&b, "- Started: `%s`\n", meta.StartedAt.Format(time.RFC3339)) + if meta.WorkflowURL != "" { + fmt.Fprintf(&b, "- Workflow: %s\n", meta.WorkflowURL) + } + fmt.Fprintln(&b) + fmt.Fprintln(&b, "## Summary") + fmt.Fprintln(&b) + fmt.Fprintln(&b, result.Summary) + fmt.Fprintln(&b) + fmt.Fprintf(&b, "Hypotheses tested: %d\n\n", result.HypothesesTested) + writeStringList(&b, "Components examined", result.ComponentsExamined) + + classifications := []struct { + value string + title string + }{ + {"confirmed-defect", "Confirmed defects"}, + {"documentation-ambiguity", "Documentation ambiguities"}, + {"missing-coverage", "Missing coverage"}, + {"inconclusive", "Inconclusive observations"}, + } + for _, class := range classifications { + fmt.Fprintf(&b, "## %s\n\n", class.title) + count := 0 + for i, f := range result.Findings { + if f.Classification != class.value { + continue + } + count++ + writeFinding(&b, i+1, f) + } + if count == 0 { + fmt.Fprintln(&b, "None.") + fmt.Fprintln(&b) + } + } + + fmt.Fprintln(&b, "## Coverage notes") + fmt.Fprintln(&b) + writeStringList(&b, "Files read", result.CoverageNotes.FilesRead) + writeStringList(&b, "Tests reviewed", result.CoverageNotes.TestsReviewed) + writeStringList(&b, "Cases attempted", result.CoverageNotes.CasesAttempted) + writeStringList(&b, "Areas not examined", result.CoverageNotes.AreasNotExamined) + return b.String() +} + +func writeFinding(b *strings.Builder, number int, f finding) { + fmt.Fprintf(b, "### %d. %s\n\n", number, f.Title) + fmt.Fprintf(b, "- Classification: `%s`\n", f.Classification) + fmt.Fprintf(b, "- Contract: `%s`\n", f.ContractID) + fmt.Fprintf(b, "- Target: `%s`\n", f.Target) + fmt.Fprintf(b, "- Impact: `%s`\n\n", f.Impact) + fmt.Fprintln(b, f.Description) + fmt.Fprintln(b) + writeReferences(b, "Contract sources", f.SourceReferences) + writeReferences(b, "Implementation references", f.ImplementationReferences) + if f.Reproducer.Script != "" || f.Reproducer.FixtureSetup != "" || len(f.Reproducer.Commands) > 0 { + fmt.Fprintln(b, "#### Reproducer") + fmt.Fprintln(b) + if f.Reproducer.FixtureSetup != "" { + fmt.Fprintln(b, "Fixture setup:") + writeCodeBlock(b, "text", f.Reproducer.FixtureSetup) + } + if f.Reproducer.Script != "" { + fmt.Fprintln(b, "Script:") + writeCodeBlock(b, "bash", f.Reproducer.Script) + } + if len(f.Reproducer.Commands) > 0 { + fmt.Fprintln(b, "Commands:") + for _, command := range f.Reproducer.Commands { + fmt.Fprintf(b, "- `%s`\n", strings.ReplaceAll(command, "`", "'")) + } + fmt.Fprintln(b) + } + } + if f.Classification == "confirmed-defect" { + writeOutcome(b, "Expected", f.Expected) + writeOutcome(b, "Actual", f.Actual) + } + if f.Verification.Runs > 0 || f.Verification.Reference != "" || f.Verification.TemporaryTest != "" { + fmt.Fprintln(b, "#### Verification") + fmt.Fprintln(b) + fmt.Fprintf(b, "- Runs: %d\n", f.Verification.Runs) + fmt.Fprintf(b, "- Reference: %s\n", emptyAsNone(f.Verification.Reference)) + fmt.Fprintf(b, "- Temporary test: %s\n\n", emptyAsNone(f.Verification.TemporaryTest)) + } + if f.SuggestedTestLocation != "" { + fmt.Fprintf(b, "Suggested regression-test location: `%s`\n\n", f.SuggestedTestLocation) + } +} + +func writeReferences(b *strings.Builder, title string, refs []reference) { + if len(refs) == 0 { + return + } + fmt.Fprintf(b, "#### %s\n\n", title) + for _, ref := range refs { + fmt.Fprintf(b, "- `%s`: %s\n", ref.Location, ref.Detail) + } + fmt.Fprintln(b) +} + +func writeOutcome(b *strings.Builder, title string, value outcome) { + fmt.Fprintf(b, "#### %s\n\n", title) + fmt.Fprintf(b, "Exit code: %s\n\n", formatExitCode(value.ExitCode)) + fmt.Fprintln(b, "stdout:") + writeCodeBlock(b, "text", value.Stdout) + fmt.Fprintln(b, "stderr:") + writeCodeBlock(b, "text", value.Stderr) +} + +func writeCodeBlock(b *strings.Builder, language, value string) { + fmt.Fprintf(b, "```%s\n%s", language, value) + if !strings.HasSuffix(value, "\n") { + fmt.Fprintln(b) + } + fmt.Fprintln(b, "```") + fmt.Fprintln(b) +} + +func writeStringList(b *strings.Builder, title string, values []string) { + fmt.Fprintf(b, "### %s\n\n", title) + if len(values) == 0 { + fmt.Fprintln(b, "None.") + fmt.Fprintln(b) + return + } + for _, value := range values { + fmt.Fprintf(b, "- %s\n", value) + } + fmt.Fprintln(b) +} + +func formatExitCode(code *int) string { + if code == nil { + return "not applicable" + } + return strconv.Itoa(*code) +} + +func emptyAsNone(value string) string { + if strings.TrimSpace(value) == "" { + return "not provided" + } + return value +} + +func prepareIssuePreviews(runDir string, meta runMetadata, result scanResult) ([]issuePreview, error) { + var previews []issuePreview + for _, f := range result.Findings { + if f.Classification == "inconclusive" { + continue + } + fingerprint := findingFingerprint(f) + previews = append(previews, issuePreview{ + Finding: f, + Fingerprint: fingerprint, + BodyPath: filepath.Join(runDir, "issues", fingerprint+".md"), + }) + } + if len(previews) == 0 { + return nil, nil + } + if err := os.MkdirAll(filepath.Join(runDir, "issues"), 0o755); err != nil { + return nil, fmt.Errorf("create issue previews directory: %w", err) + } + for _, preview := range previews { + body := renderIssueBody(meta, preview.Finding, preview.Fingerprint) + if err := os.WriteFile(preview.BodyPath, []byte(body), 0o644); err != nil { + return nil, fmt.Errorf("write issue preview: %w", err) + } + } + return previews, nil +} + +func publishFindings(ctx context.Context, cfg config, repoRoot string, meta runMetadata, previews []issuePreview) ([]issueLink, error) { + if len(previews) == 0 { + return nil, nil + } + if meta.Repository == "" { + return nil, errors.New("GitHub repository is unknown; pass --repo owner/repository") + } + if _, err := commandOutput(ctx, repoRoot, cfg.ghBinary, "auth", "status", "--hostname", "github.com"); err != nil { + return nil, fmt.Errorf("GitHub CLI authentication failed: %w; use --no-publish to keep the report local", err) + } + existing, err := loadExistingIssues(ctx, cfg, repoRoot, meta.Repository) + if err != nil { + return nil, err + } + byFingerprint := make(map[string]existingIssue) + for _, issue := range existing { + if match := markerPattern.FindStringSubmatch(issue.Body); len(match) == 2 { + byFingerprint[match[1]] = issue + } + } + + links := make([]issueLink, 0, len(previews)) + for _, preview := range previews { + f := preview.Finding + if issue, ok := byFingerprint[preview.Fingerprint]; ok { + action := "updated" + if strings.EqualFold(issue.State, "closed") { + if _, err := commandOutput(ctx, repoRoot, cfg.ghBinary, "issue", "reopen", strconv.Itoa(issue.Number), "--repo", meta.Repository); err != nil { + return links, fmt.Errorf("reopen issue %d: %w", issue.Number, err) + } + action = "reopened" + } + if _, err := commandOutput(ctx, repoRoot, cfg.ghBinary, "issue", "comment", strconv.Itoa(issue.Number), "--repo", meta.Repository, "--body-file", preview.BodyPath); err != nil { + return links, fmt.Errorf("comment on issue %d: %w", issue.Number, err) + } + links = append(links, issueLink{Action: action, URL: issue.URL, Title: f.Title}) + continue + } + + title := issueTitle(f) + issueURL, err := commandOutput(ctx, repoRoot, cfg.ghBinary, "issue", "create", "--repo", meta.Repository, "--title", title, "--body-file", preview.BodyPath) + if err != nil { + return links, fmt.Errorf("create issue %q: %w", title, err) + } + issueURL = strings.TrimSpace(issueURL) + links = append(links, issueLink{Action: "created", URL: issueURL, Title: f.Title}) + byFingerprint[preview.Fingerprint] = existingIssue{State: "OPEN", URL: issueURL, Title: title} + } + return links, nil +} + +func loadExistingIssues(ctx context.Context, cfg config, repoRoot, repository string) ([]existingIssue, error) { + out, err := commandOutput(ctx, repoRoot, cfg.ghBinary, "issue", "list", "--repo", repository, "--state", "all", "--limit", "1000", "--json", "number,state,url,title,body") + if err != nil { + return nil, fmt.Errorf("list existing correctness issues: %w", err) + } + var issues []existingIssue + if err := json.Unmarshal([]byte(out), &issues); err != nil { + return nil, fmt.Errorf("decode existing issues: %w", err) + } + return issues, nil +} + +func findingFingerprint(f finding) string { + parts := []string{ + strings.ToLower(strings.TrimSpace(f.Classification)), + strings.ToUpper(strings.TrimSpace(f.ContractID)), + strings.ToLower(strings.TrimSpace(f.Target)), + normalizeFingerprintText(f.Title), + normalizeFingerprintText(f.Reproducer.Script), + } + sum := sha256.Sum256([]byte(strings.Join(parts, "\x00"))) + return hex.EncodeToString(sum[:]) +} + +func normalizeFingerprintText(value string) string { + return strings.Join(strings.Fields(strings.ToLower(value)), " ") +} + +func issueTitle(f finding) string { + title := fmt.Sprintf("[correctness][%s][%s] %s", f.Classification, f.Target, strings.TrimSpace(f.Title)) + runes := []rune(title) + if len(runes) > 240 { + title = string(runes[:240]) + } + return title +} + +func renderIssueBody(meta runMetadata, f finding, fingerprint string) string { + var b strings.Builder + fmt.Fprintf(&b, "\n", fingerprintLabel, fingerprint) + fmt.Fprintln(&b, "Generated by the rshell behavior correctness scanner.") + fmt.Fprintln(&b) + fmt.Fprintf(&b, "- Contract: `%s`\n", f.ContractID) + fmt.Fprintf(&b, "- Classification: `%s`\n", f.Classification) + fmt.Fprintf(&b, "- Impact: `%s`\n", f.Impact) + fmt.Fprintf(&b, "- Target: `%s`\n", f.Target) + fmt.Fprintf(&b, "- Scanned commit: `%s`\n", meta.Commit) + if meta.WorkflowURL != "" { + fmt.Fprintf(&b, "- Workflow run: %s\n", meta.WorkflowURL) + } + fmt.Fprintln(&b) + fmt.Fprintln(&b, f.Description) + fmt.Fprintln(&b) + writeReferences(&b, "Contract sources", f.SourceReferences) + writeReferences(&b, "Implementation references", f.ImplementationReferences) + if f.Reproducer.FixtureSetup != "" { + fmt.Fprintln(&b, "## Fixture setup") + fmt.Fprintln(&b) + writeCodeBlock(&b, "text", f.Reproducer.FixtureSetup) + } + if f.Reproducer.Script != "" { + fmt.Fprintln(&b, "## Reproducer") + fmt.Fprintln(&b) + writeCodeBlock(&b, "bash", f.Reproducer.Script) + } + if len(f.Reproducer.Commands) > 0 { + fmt.Fprintln(&b, "Commands used:") + fmt.Fprintln(&b) + for _, command := range f.Reproducer.Commands { + fmt.Fprintf(&b, "- `%s`\n", strings.ReplaceAll(command, "`", "'")) + } + fmt.Fprintln(&b) + } + if f.Classification == "confirmed-defect" { + writeOutcome(&b, "Expected", f.Expected) + writeOutcome(&b, "Actual", f.Actual) + } + fmt.Fprintln(&b, "## Verification") + fmt.Fprintln(&b) + fmt.Fprintf(&b, "- Reproduction runs: %d\n", f.Verification.Runs) + fmt.Fprintf(&b, "- Reference: %s\n", emptyAsNone(f.Verification.Reference)) + fmt.Fprintf(&b, "- Temporary test: %s\n", emptyAsNone(f.Verification.TemporaryTest)) + if f.SuggestedTestLocation != "" { + fmt.Fprintf(&b, "- Suggested regression-test location: `%s`\n", f.SuggestedTestLocation) + } + fmt.Fprintln(&b) + fmt.Fprintf(&b, "Fingerprint: `%s`\n", fingerprint) + return b.String() +} + +func writeIssueLinks(path string, links []issueLink, publishErr error) error { + var b strings.Builder + fmt.Fprintln(&b, "# Correctness scan issues") + fmt.Fprintln(&b) + if len(links) == 0 { + fmt.Fprintln(&b, "No issues were created or updated.") + } else { + sort.SliceStable(links, func(i, j int) bool { + if links[i].Action == links[j].Action { + return links[i].Title < links[j].Title + } + return links[i].Action < links[j].Action + }) + current := "" + for _, link := range links { + if link.Action != current { + current = link.Action + fmt.Fprintf(&b, "## %s\n\n", strings.ToUpper(current[:1])+current[1:]) + } + fmt.Fprintf(&b, "- [%s](%s)\n", link.Title, link.URL) + } + } + if publishErr != nil { + fmt.Fprintln(&b) + fmt.Fprintln(&b, "## Publication error") + fmt.Fprintln(&b) + fmt.Fprintln(&b, publishErr) + } + if err := os.WriteFile(path, []byte(b.String()), 0o644); err != nil { + return fmt.Errorf("write issue links: %w", err) + } + return nil +} + +func printIssueSummary(w io.Writer, links []issueLink) { + if len(links) == 0 { + fmt.Fprintln(w, "No publishable findings; no GitHub issues were changed.") + return + } + fmt.Fprintln(w, "GitHub issues:") + for _, link := range links { + fmt.Fprintf(w, "- %s: %s (%s)\n", link.Action, link.Title, link.URL) + } +} + +func workflowURL() string { + server := os.Getenv("GITHUB_SERVER_URL") + repository := os.Getenv("GITHUB_REPOSITORY") + runID := os.Getenv("GITHUB_RUN_ID") + if server == "" || repository == "" || runID == "" { + return "" + } + return strings.TrimSuffix(server, "/") + "/" + repository + "/actions/runs/" + runID +} + +func codexEnvironment(env []string) []string { + filtered := make([]string, 0, len(env)) + for _, entry := range env { + key, _, _ := strings.Cut(entry, "=") + if strings.HasPrefix(key, "GITHUB_") || key == "GH_TOKEN" || key == "GH_ENTERPRISE_TOKEN" || + key == "ACTIONS_ID_TOKEN_REQUEST_TOKEN" || key == "ACTIONS_RUNTIME_TOKEN" { + continue + } + filtered = append(filtered, entry) + } + return filtered +} + +func logProgress(cfg config, stderr io.Writer, format string, args ...any) { + if cfg.quiet { + return + } + fmt.Fprintf(stderr, "correctness-scan: "+format+"\n", args...) +} + +func shellQuote(value string) string { + return "'" + strings.ReplaceAll(value, "'", "'\\''") + "'" +} + +func cleanupWorktree(ctx context.Context, repoRoot, tempRoot, worktreeDir string, keep, quiet bool, stderr io.Writer) { + if keep { + fmt.Fprintf(stderr, "correctness-scan: kept worktree at %s\n", worktreeDir) + return + } + if !quiet { + fmt.Fprintln(stderr, "correctness-scan: removing temporary worktree") + } + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + if _, err := commandOutput(cleanupCtx, repoRoot, "git", "worktree", "remove", "--force", worktreeDir); err != nil { + fmt.Fprintf(stderr, "correctness-scan: remove worktree: %v\n", err) + return + } + if err := os.RemoveAll(tempRoot); err != nil { + fmt.Fprintf(stderr, "correctness-scan: remove temporary directory: %v\n", err) + } +} + +func commandOutput(ctx context.Context, dir, name string, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Dir = dir + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + message := strings.TrimSpace(stderr.String()) + if message == "" { + message = strings.TrimSpace(stdout.String()) + } + if message != "" { + return "", fmt.Errorf("%w: %s", err, message) + } + return "", err + } + return stdout.String(), nil +} diff --git a/tools/correctness-scan/main_test.go b/tools/correctness-scan/main_test.go new file mode 100644 index 000000000..419db275c --- /dev/null +++ b/tools/correctness-scan/main_test.go @@ -0,0 +1,333 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-present Datadog, Inc. + +package main + +import ( + "context" + "encoding/json" + "io" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "unicode/utf8" +) + +func TestParseConfigPublishesByDefault(t *testing.T) { + cfg, err := parseConfig([]string{"--target", "cat"}, io.Discard) + if err != nil { + t.Fatal(err) + } + if !cfg.publish { + t.Fatal("publish should be enabled by default") + } + + cfg, err = parseConfig([]string{"--target", "cat", "--no-publish"}, io.Discard) + if err != nil { + t.Fatal(err) + } + if cfg.publish { + t.Fatal("--no-publish should disable publication") + } +} + +func TestParseConfigQuiet(t *testing.T) { + cfg, err := parseConfig([]string{"--quiet"}, io.Discard) + if err != nil { + t.Fatal(err) + } + if !cfg.quiet { + t.Fatal("--quiet should suppress phase messages") + } +} + +func TestParseConfigRejectsInvalidInput(t *testing.T) { + for _, args := range [][]string{ + {"--target", "../cat"}, + {"--depth", "exhaustive"}, + {"--ref", "HEAD@{1}"}, + {"--repo", "rshell"}, + } { + if _, err := parseConfig(args, io.Discard); err == nil { + t.Fatalf("parseConfig(%q) succeeded", args) + } + } +} + +func TestParseGitHubRepository(t *testing.T) { + tests := map[string]string{ + "git@github.com:DataDog/rshell.git": "DataDog/rshell", + "https://github.com/DataDog/rshell.git": "DataDog/rshell", + "ssh://git@github.com/DataDog/rshell": "DataDog/rshell", + } + for remote, want := range tests { + got, err := parseGitHubRepository(remote) + if err != nil { + t.Fatalf("parseGitHubRepository(%q): %v", remote, err) + } + if got != want { + t.Fatalf("parseGitHubRepository(%q) = %q, want %q", remote, got, want) + } + } + if _, err := parseGitHubRepository("https://example.com/DataDog/rshell.git"); err == nil { + t.Fatal("non-GitHub remote succeeded") + } +} + +func TestBuildPrompt(t *testing.T) { + cfg := config{target: "changed", depth: "standard"} + prompt := buildPrompt(cfg, strings.Repeat("a", 40), strings.Repeat("b", 40)) + for _, want := range []string{"Target: changed", "Depth: standard", "Maximum hypotheses: 8", "git diff --stat"} { + if !strings.Contains(prompt, want) { + t.Errorf("prompt does not contain %q", want) + } + } + if strings.Contains(prompt, "{{") { + t.Fatal("prompt contains an unreplaced template variable") + } +} + +func TestValidateResult(t *testing.T) { + wantCode, gotCode := 0, 1 + valid := scanResult{ + Summary: "Found one mismatch.", + HypothesesTested: 1, + Findings: []finding{{ + Classification: "confirmed-defect", + ContractID: "BEH-003", + Target: "cat", + Title: "status differs", + Impact: "medium", + Description: "The exit status differs from the documented behavior.", + SourceReferences: []reference{{ + Location: "SHELL_FEATURES.md:1", + Detail: "documents the behavior", + }}, + ImplementationReferences: []reference{{ + Location: "builtins/cat/cat.go:1", + Detail: "returns the observed status", + }}, + Reproducer: reproducer{ + Script: "cat missing\n", + Commands: []string{"go test ./tests"}, + }, + Expected: outcome{ExitCode: &wantCode}, + Actual: outcome{ExitCode: &gotCode}, + Verification: verification{Runs: 2}, + }}, + } + if err := validateResult(valid, config{depth: "quick"}); err != nil { + t.Fatal(err) + } + + invalid := valid + invalid.Findings = append([]finding(nil), valid.Findings...) + invalid.Findings[0].Verification.Runs = 1 + if err := validateResult(invalid, config{depth: "quick"}); err == nil { + t.Fatal("confirmed defect with one verification run succeeded") + } +} + +func TestFindingFingerprintNormalizesPresentation(t *testing.T) { + a := finding{Classification: "missing-coverage", ContractID: "BEH-004", Target: "cat", Title: "Missing stdin case", Reproducer: reproducer{Script: "cat -\n"}} + b := a + b.Title = " MISSING stdin CASE " + b.Reproducer.Script = " CAT - " + if findingFingerprint(a) != findingFingerprint(b) { + t.Fatal("presentation-only changes altered the fingerprint") + } +} + +func TestPrepareIssuePreviewsSkipsInconclusive(t *testing.T) { + runDir := t.TempDir() + result := scanResult{Findings: []finding{ + {Classification: "missing-coverage", ContractID: "BEH-004", Target: "cat", Title: "stdin lacks coverage", Impact: "low", Description: "No scenario covers stdin.", SuggestedTestLocation: "tests/scenarios/cat.yaml"}, + {Classification: "inconclusive", ContractID: "BEH-002", Target: "cat", Title: "unstable output", Impact: "low", Description: "Could not reproduce."}, + }} + previews, err := prepareIssuePreviews(runDir, runMetadata{Commit: strings.Repeat("a", 40)}, result) + if err != nil { + t.Fatal(err) + } + if len(previews) != 1 { + t.Fatalf("got %d previews, want 1", len(previews)) + } + body, err := os.ReadFile(previews[0].BodyPath) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(body), "") { + t.Fatal("preview does not contain its fingerprint marker") + } + if filepath.Dir(previews[0].BodyPath) != filepath.Join(runDir, "issues") { + t.Fatalf("preview written outside issues directory: %s", previews[0].BodyPath) + } +} + +func TestIssueTitleTruncatesOnRuneBoundary(t *testing.T) { + title := issueTitle(finding{Classification: "missing-coverage", Target: "cat", Title: strings.Repeat("é", 300)}) + if !utf8.ValidString(title) { + t.Fatal("truncated title is invalid UTF-8") + } + if utf8.RuneCountInString(title) != 240 { + t.Fatalf("title contains %d runes, want 240", utf8.RuneCountInString(title)) + } +} + +func TestCodexEnvironmentExcludesGitHubCredentials(t *testing.T) { + env := codexEnvironment([]string{ + "PATH=/bin", + "OPENAI_API_KEY=keep", + "GH_TOKEN=drop", + "GITHUB_TOKEN=drop", + "GITHUB_REPOSITORY=drop", + "ACTIONS_ID_TOKEN_REQUEST_TOKEN=drop", + "ACTIONS_RUNTIME_TOKEN=drop", + }) + got := strings.Join(env, "\n") + if got != "PATH=/bin\nOPENAI_API_KEY=keep" { + t.Fatalf("unexpected environment:\n%s", got) + } +} + +func TestInvokeCodexWritesProcessLogs(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("test fixture uses a POSIX shell") + } + root := t.TempDir() + codexBinary := filepath.Join(root, "codex") + if err := os.WriteFile(codexBinary, []byte(`#!/bin/sh +printf 'stdout event\n' +printf 'live progress\n' >&2 +`), 0o755); err != nil { + t.Fatal(err) + } + + runDir := t.TempDir() + cfg := config{codexBinary: codexBinary, depth: "quick"} + if err := invokeCodex(context.Background(), cfg, root, runDir, "schema.json", "findings.json", "prompt"); err != nil { + t.Fatal(err) + } + for name, want := range map[string]string{ + "codex.stdout.log": "stdout event\n", + "codex.stderr.log": "live progress\n", + } { + data, err := os.ReadFile(filepath.Join(runDir, name)) + if err != nil { + t.Fatal(err) + } + if string(data) != want { + t.Fatalf("unexpected %s: %q", name, data) + } + } +} + +func TestShellQuote(t *testing.T) { + if got, want := shellQuote("/tmp/scan log's"), `'/tmp/scan log'\''s'`; got != want { + t.Fatalf("shellQuote() = %q, want %q", got, want) + } +} + +func TestPublishFindingsCreatesOrReopensIssue(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("test fixture uses a POSIX shell") + } + ghBinary := filepath.Join(t.TempDir(), "gh") + if err := os.WriteFile(ghBinary, []byte(`#!/bin/sh +printf '%s\n' "$*" >> "$FAKE_GH_LOG" +if [ "$1 $2" = "issue list" ]; then + printf '%s\n' "$FAKE_GH_ISSUES" +fi +if [ "$1 $2" = "issue create" ]; then + printf '%s\n' "https://github.com/DataDog/rshell/issues/42" +fi +`), 0o755); err != nil { + t.Fatal(err) + } + f := finding{ + Classification: "missing-coverage", + ContractID: "BEH-004", + Target: "cat", + Title: "stdin lacks coverage", + Impact: "low", + Description: "No scenario covers stdin.", + SuggestedTestLocation: "tests/scenarios/cat.yaml", + } + meta := runMetadata{Repository: "DataDog/rshell", Commit: strings.Repeat("a", 40)} + cfg := config{ghBinary: ghBinary} + + t.Run("create", func(t *testing.T) { + runDir := t.TempDir() + logPath := filepath.Join(runDir, "gh.log") + t.Setenv("FAKE_GH_LOG", logPath) + t.Setenv("FAKE_GH_ISSUES", "[]") + previews, err := prepareIssuePreviews(runDir, meta, scanResult{Findings: []finding{f}}) + if err != nil { + t.Fatal(err) + } + links, err := publishFindings(context.Background(), cfg, runDir, meta, previews) + if err != nil { + t.Fatal(err) + } + if len(links) != 1 || links[0].Action != "created" { + t.Fatalf("unexpected links: %#v", links) + } + assertCommandLogContains(t, logPath, "issue create") + }) + + t.Run("reopen", func(t *testing.T) { + runDir := t.TempDir() + logPath := filepath.Join(runDir, "gh.log") + t.Setenv("FAKE_GH_LOG", logPath) + fingerprint := findingFingerprint(f) + issues, err := json.Marshal([]existingIssue{{ + Number: 7, + State: "CLOSED", + URL: "https://github.com/DataDog/rshell/issues/7", + Body: "", + }}) + if err != nil { + t.Fatal(err) + } + t.Setenv("FAKE_GH_ISSUES", string(issues)) + previews, err := prepareIssuePreviews(runDir, meta, scanResult{Findings: []finding{f}}) + if err != nil { + t.Fatal(err) + } + links, err := publishFindings(context.Background(), cfg, runDir, meta, previews) + if err != nil { + t.Fatal(err) + } + if len(links) != 1 || links[0].Action != "reopened" { + t.Fatalf("unexpected links: %#v", links) + } + assertCommandLogContains(t, logPath, "issue reopen 7") + assertCommandLogContains(t, logPath, "issue comment 7") + assertCommandLogExcludes(t, logPath, "issue create") + }) +} + +func assertCommandLogContains(t *testing.T, path, want string) { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), want) { + t.Fatalf("command log does not contain %q:\n%s", want, data) + } +} + +func assertCommandLogExcludes(t *testing.T, path, unwanted string) { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(data), unwanted) { + t.Fatalf("command log contains %q:\n%s", unwanted, data) + } +}