From df14f59bc7ebf3c64024d9b1ee99b21e134fa11d Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 20:24:22 +0000 Subject: [PATCH 01/10] ci: use precompiled caches, cache them, and precompile explicitly The --compiled-modules=no flag made every Julia process ignore the precompiled caches. Remove it from the test steps, the mutation harness and the docs. Cover the project only (--code-coverage=@) so dependencies keep native code. Cache ~/.julia/compiled in both Julia jobs, keyed on the Manifest, the Julia pin and the R apt pin. Run an explicit Pkg.precompile() after the RCall rebuild so a failure shows up there. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_0136P7ZmNJPhB7wGCRymdu1D --- .github/workflows/ci.yml | 66 +++++++++++++++++++++----- docs/statistics/validation-evidence.md | 2 +- scripts/process-ci-coverage.jl | 11 +++++ scripts/statistics/mutation_check.jl | 2 +- test/integration/test_server.jl | 3 +- 5 files changed, 69 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd8ebfe9..f388e197 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -208,15 +208,6 @@ jobs: - name: Parse Julia sources before dependency setup run: julia scripts/check-julia-syntax.jl - - name: Cache Julia packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - ~/.julia/artifacts - ~/.julia/packages - ~/.julia/registries - key: julia-${{ runner.os }}-${{ hashFiles('Manifest.toml', 'config/defaults/tool_versions.yml') }} - # Every external version CI installs is read from the committed pin file, so CI # and a developer's machine cannot drift apart. A temporary environment is used # because the project environment cannot be instantiated until R is present: @@ -230,6 +221,7 @@ jobs: using YAML pins = YAML.load_file("config/defaults/tool_versions.yml") open(ENV["GITHUB_ENV"], "a") do io + println(io, "JULIA_VERSION=", pins["runtimes"]["julia"]["version"]) println(io, "R_APT_VERSION=", pins["runtimes"]["r"]["apt_version"]) println(io, "BUN_VERSION=", pins["toolchain"]["bun"]["version"]) println(io, "CUTADAPT_SPEC=cutadapt==", pins["tools"]["cutadapt"]["version"]) @@ -245,6 +237,21 @@ jobs: end end' + # Packages, artifacts and the precompiled caches. The compiled caches are + # valid only for the Julia that wrote them, and RCall's image is built + # against the installed R, so the key names the Julia pin, the R apt pin + # and the Manifest. The job's own name is in the key so the standards job + # does not race this one to save the same entry. + - name: Cache Julia packages and compiled caches + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.julia/artifacts + ~/.julia/packages + ~/.julia/registries + ~/.julia/compiled + key: julia-test-${{ runner.os }}-J${{ env.JULIA_VERSION }}-R${{ env.R_APT_VERSION }}-${{ hashFiles('Manifest.toml') }} + # R + required packages (needed by RCall) # Install R directly via apt to avoid r-lib/actions/setup-r mangling # R_HOME and stripping default packages (utils, methods, stats, etc.) @@ -359,7 +366,11 @@ jobs: sudo ln -sf /opt/FastQC/fastqc /usr/local/bin/fastqc fastqc --version + # No precompile here: RCall is rebuilt below and that invalidates its image + # and everything that loads it, so precompiling now would be done twice. - name: Instantiate Julia project + env: + JULIA_PKG_PRECOMPILE_AUTO: "0" run: julia --project=. -e 'import Pkg; Pkg.instantiate()' - name: Set up Bun @@ -389,6 +400,12 @@ jobs: - name: Rebuild RCall against installed R run: julia --project=. -e 'import Pkg; Pkg.build("RCall")' + # Explicit, so a precompile failure (an RCall image that does not match the + # installed R, for one) fails here with its own error and not later as a + # slow load inside the test run. + - name: Precompile Julia project + run: julia --project=. -e 'import Pkg; Pkg.precompile()' + - name: Verify R packages visible from RCall run: | julia --project=. -e ' @@ -408,7 +425,7 @@ jobs: shell: bash run: | set +e - julia --project=. -t2 --compiled-modules=no test/runtests.jl validated_statistics numeric_policy exact_summaries high_precision derived_analyses method_catalogue statistics_boundaries statistics_adversarial statistics_properties statistics_ui 2>&1 | tee /tmp/validated-statistics-tests.log + julia --project=. -t2 test/runtests.jl validated_statistics numeric_policy exact_summaries high_precision derived_analyses method_catalogue statistics_boundaries statistics_adversarial statistics_properties statistics_ui 2>&1 | tee /tmp/validated-statistics-tests.log # Preserve both exits immediately: a successful Julia run must not # hide a failed log write (e.g. a full runner filesystem). statuses=("${PIPESTATUS[@]}") @@ -462,7 +479,7 @@ jobs: run: | command -v vsearch # The full suite must run the occupancy experiment, not skip it. set +e - julia --project=. -t 2 --code-coverage=user --compiled-modules=no test/runtests.jl --integration --server 2>&1 | tee /tmp/julia-tests.log + julia --project=. -t 2 --code-coverage=@ test/runtests.jl --integration --server 2>&1 | tee /tmp/julia-tests.log # Preserve both exits immediately: a successful Julia run must not # hide a failed log write (e.g. a full runner filesystem). statuses=("${PIPESTATUS[@]}") @@ -538,6 +555,25 @@ jobs: - name: Set up Julia run: bash scripts/install-ci-julia.sh + - name: Read pinned runtime versions + run: | + { + echo "JULIA_VERSION=$(yq -r '.runtimes.julia.version' config/defaults/tool_versions.yml)" + echo "R_APT_VERSION=$(yq -r '.runtimes.r.apt_version' config/defaults/tool_versions.yml)" + } >> "$GITHUB_ENV" + + # Same reasoning as the Julia tests job: the compiled caches depend on the + # Julia pin and on the R that RCall was built against. + - name: Cache Julia packages and compiled caches + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.julia/artifacts + ~/.julia/packages + ~/.julia/registries + ~/.julia/compiled + key: julia-standards-${{ runner.os }}-J${{ env.JULIA_VERSION }}-R${{ env.R_APT_VERSION }}-${{ hashFiles('Manifest.toml') }} + # The versions, the HTTPS restriction and the exact-commit fetches live in # the script (one reviewed place), not interpolated into this workflow. - name: Install pinned standards tools @@ -561,8 +597,14 @@ jobs: # statistics modules, so the environment must be instantiated after R. - name: Instantiate Julia project id: instantiate + env: + JULIA_PKG_PRECOMPILE_AUTO: "0" run: julia --project=. -e 'import Pkg; Pkg.instantiate()' + # Explicit, so a precompile failure fails here and names itself. + - name: Precompile Julia project + id: precompile + run: julia --project=. -e 'import Pkg; Pkg.precompile()' # YAML-POLICY Y-3 / D307. --check exits 1 on drift and 2 when yq is @@ -614,7 +656,7 @@ jobs: # study, the gates or an earlier evidence step failed) without inventing a # failure that cannot distinguish a broken mutant from an absent project. - name: Mutation campaign - if: always() && steps.instantiate.outcome == 'success' + if: always() && steps.instantiate.outcome == 'success' && steps.precompile.outcome == 'success' env: RUN_LOG_TITLE: Mutation campaign run: | diff --git a/docs/statistics/validation-evidence.md b/docs/statistics/validation-evidence.md index 7f717091..f6d83402 100644 --- a/docs/statistics/validation-evidence.md +++ b/docs/statistics/validation-evidence.md @@ -175,7 +175,7 @@ issue #2 remains blocked until this layer is approved. ```bash # Focused -julia --project=. -t2 --compiled-modules=no test/runtests.jl \ +julia --project=. -t2 test/runtests.jl \ validated_statistics numeric_policy exact_summaries high_precision \ derived_analyses method_catalogue statistics_boundaries \ statistics_adversarial statistics_properties statistics_ui diff --git a/scripts/process-ci-coverage.jl b/scripts/process-ci-coverage.jl index 2e4b69c3..ccbfd809 100644 --- a/scripts/process-ci-coverage.jl +++ b/scripts/process-ci-coverage.jl @@ -5,3 +5,14 @@ Pkg.activate(; temp=true) Pkg.add(PackageSpec(name="CoverageTools", version="1.4.2")) using CoverageTools LCOV.writefile("lcov.info", process_folder("src")) + +# Report which src/ files reached lcov.info, so a change of coverage mode is +# visible in the log: a file missing here is a file Codecov will not see. +let files = sort([m.captures[1] for l in eachline("lcov.info") if (m = match(r"^SF:(.*)$", l)) !== nothing]) + println("lcov.info covers ", length(files), " source files") + foreach(f -> println(" ", f), files) + on_disk = sort(replace.(filter(endswith(".jl"), [joinpath(r, f) for (r, _, fs) in walkdir("src") for f in fs]), "\\" => "/")) + missing_files = setdiff(on_disk, files) + println("src/ files absent from lcov.info: ", length(missing_files)) + foreach(f -> println(" ", f), missing_files) +end diff --git a/scripts/statistics/mutation_check.jl b/scripts/statistics/mutation_check.jl index 11699341..b5f5fbeb 100755 --- a/scripts/statistics/mutation_check.jl +++ b/scripts/statistics/mutation_check.jl @@ -96,7 +96,7 @@ function _sha256(path::AbstractString) end function _run_tests(julia::String, tests::Vector{String}, log::String) - command = Cmd(`$julia --project=. -t2 --compiled-modules=no test/runtests.jl $(tests...)`; + command = Cmd(`$julia --project=. -t2 test/runtests.jl $(tests...)`; dir=ROOT) open(log, "w") do io println(io, "\$ ", command) diff --git a/test/integration/test_server.jl b/test/integration/test_server.jl index 7e4430a5..8ee3c44f 100644 --- a/test/integration/test_server.jl +++ b/test/integration/test_server.jl @@ -25,7 +25,8 @@ "JULIA_METAMANIFOLD_PORT" => string(port)))); wait=false) - # CI loads the server with --compiled-modules=no, which takes minutes. + # The server subprocess loads from the precompiled caches. A cold cache + # makes the first start slow, so the wait below is generous. ready = false for _ in 1:1200 process_running(proc) || break From c6eed43389b502e8e7a0266ea13f00c89dc288fa Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 20:51:23 +0000 Subject: [PATCH 02/10] ci: fix the coverage file listing added with the coverage change The comprehension used a variable bound in its own filter, which is undefined in Julia. Use a loop. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_0136P7ZmNJPhB7wGCRymdu1D --- scripts/process-ci-coverage.jl | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/scripts/process-ci-coverage.jl b/scripts/process-ci-coverage.jl index ccbfd809..952bbe76 100644 --- a/scripts/process-ci-coverage.jl +++ b/scripts/process-ci-coverage.jl @@ -8,11 +8,19 @@ LCOV.writefile("lcov.info", process_folder("src")) # Report which src/ files reached lcov.info, so a change of coverage mode is # visible in the log: a file missing here is a file Codecov will not see. -let files = sort([m.captures[1] for l in eachline("lcov.info") if (m = match(r"^SF:(.*)$", l)) !== nothing]) +let files = String[] + for line in eachline("lcov.info") + m = match(r"^SF:(.*)$", line) + m === nothing || push!(files, String(m.captures[1])) + end + sort!(files) println("lcov.info covers ", length(files), " source files") foreach(f -> println(" ", f), files) - on_disk = sort(replace.(filter(endswith(".jl"), [joinpath(r, f) for (r, _, fs) in walkdir("src") for f in fs]), "\\" => "/")) - missing_files = setdiff(on_disk, files) - println("src/ files absent from lcov.info: ", length(missing_files)) - foreach(f -> println(" ", f), missing_files) + on_disk = String[] + for (root, _, names) in walkdir("src"), name in names + endswith(name, ".jl") && push!(on_disk, replace(joinpath(root, name), "\\" => "/")) + end + absent = setdiff(sort!(on_disk), files) + println("src/ files absent from lcov.info: ", length(absent)) + foreach(f -> println(" ", f), absent) end From c563be4b76ac6823630e54dfa8395a0072307aa0 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 20:54:22 +0000 Subject: [PATCH 03/10] Port the Python oracles and the failure reporter The high-precision and exact-summary oracles are GNU bc driven by bash, the ordination oracle is awk, and the failure reporter is Julia. None of the oracles share maths code with Julia. Callers and docs are updated and the old .py files are removed. The python-test-oracles proposal is marked decided. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_0136P7ZmNJPhB7wGCRymdu1D --- .../contractiles/must/must.k9.ncl | 10 +- VALIDATED_STATISTICS.md | 10 +- VALIDATED_STATISTICS_VALIDATION.md | 6 +- bench/null_calibration/report_failures.jl | 166 +++ bench/null_calibration/report_failures.py | 94 -- config/defaults/validated_statistics.kyaml | 4 +- docs/proposals/python-test-oracles.md | 9 +- docs/statistics/constrained-ordinations.md | 2 +- docs/statistics/issue-1-ultraplan.md | 8 +- docs/statistics/method-catalogue-v1.md | 4 +- docs/statistics/numeric-contracts.md | 2 +- docs/statistics/validation-evidence.md | 8 +- .../tests/test_null_calibration_failures.jl | 97 ++ .../tests/test_null_calibration_failures.py | 100 -- .../check_high_precision_references.py | 75 -- .../check_high_precision_references.sh | 85 ++ .../check_ordination_references.awk | 1093 +++++++++++++++++ .../references/check_ordination_references.py | 868 ------------- .../references/check_ordination_references.sh | 682 ++++++++++ test/references/exact_summary_reference.py | 62 - test/references/exact_summary_reference.sh | 108 ++ test/references/high_precision_references.bc | 99 ++ test/unit/test_exact_summaries.jl | 11 +- test/unit/test_null_calibration_csv.R | 12 +- test/unit/test_ordination.jl | 8 +- 25 files changed, 2379 insertions(+), 1244 deletions(-) create mode 100644 bench/null_calibration/report_failures.jl delete mode 100644 bench/null_calibration/report_failures.py create mode 100644 scripts/tests/test_null_calibration_failures.jl delete mode 100644 scripts/tests/test_null_calibration_failures.py delete mode 100644 test/references/check_high_precision_references.py create mode 100755 test/references/check_high_precision_references.sh create mode 100644 test/references/check_ordination_references.awk delete mode 100755 test/references/check_ordination_references.py create mode 100755 test/references/check_ordination_references.sh delete mode 100644 test/references/exact_summary_reference.py create mode 100755 test/references/exact_summary_reference.sh create mode 100644 test/references/high_precision_references.bc diff --git a/.machine_readable/contractiles/must/must.k9.ncl b/.machine_readable/contractiles/must/must.k9.ncl index b3d94d94..bc35cafe 100644 --- a/.machine_readable/contractiles/must/must.k9.ncl +++ b/.machine_readable/contractiles/must/must.k9.ncl @@ -60,15 +60,15 @@ K9! }, { id = "high-precision-reference-oracle", - target = "test/references/check_high_precision_references.py", - check_command = "python3 test/references/check_high_precision_references.py --check test/unit/test_high_precision.jl", + target = "test/references/check_high_precision_references.sh", + check_command = "bash test/references/check_high_precision_references.sh --check test/unit/test_high_precision.jl", rationale = "Independent arbitrary-precision reference constants must match the Julia test suite.", }, { id = "constrained-ordination-reference-oracle", - target = "test/references/check_ordination_references.py", - check_command = "python3 test/references/check_ordination_references.py --check test/fixtures/ordination_reference.json", - rationale = "Independent Python reference constants for RDA eigenvalues, Ezekiel adjusted-R2, two-way variance partitioning, and canonical JCS schema/fixtures must match within 1e-6.", + target = "test/references/check_ordination_references.sh", + check_command = "bash test/references/check_ordination_references.sh --check test/fixtures/ordination_reference.json", + rationale = "Independent awk reference constants for RDA eigenvalues, Ezekiel adjusted-R2, two-way variance partitioning, and canonical JCS schema/fixtures must match within 1e-6.", }, { id = "upstream-license-untouched", diff --git a/VALIDATED_STATISTICS.md b/VALIDATED_STATISTICS.md index 9f208657..4830ba18 100644 --- a/VALIDATED_STATISTICS.md +++ b/VALIDATED_STATISTICS.md @@ -285,16 +285,16 @@ Full suite (with the repository's integration and server gates): julia --project=. -t2 test/runtests.jl --integration --server ``` -The suite includes known answers, independent Python `fractions` and -high-precision `decimal` references, zero/negative/nonfinite/missing controls, values beyond `2^53`, rational budget +The suite includes known answers, independent GNU `bc` exact-fraction and +high-precision references, zero/negative/nonfinite/missing controls, values beyond `2^53`, rational budget failures, SQL-sum overflow controls, feature-permutation and scale invariance, generative count matrices, malformed JSON fuzz cases, serialization round trips, concurrent precision scopes, concurrent immutable publication, and tamper -recovery. Reproduce the Python `decimal` references independently with: +recovery. Reproduce the `bc` references independently with: ```bash -python3 test/references/check_high_precision_references.py -python3 test/references/check_high_precision_references.py --check \ +bash test/references/check_high_precision_references.sh +bash test/references/check_high_precision_references.sh --check \ test/unit/test_high_precision.jl ``` diff --git a/VALIDATED_STATISTICS_VALIDATION.md b/VALIDATED_STATISTICS_VALIDATION.md index 788974f1..4567a621 100644 --- a/VALIDATED_STATISTICS_VALIDATION.md +++ b/VALIDATED_STATISTICS_VALIDATION.md @@ -13,7 +13,7 @@ independent scientific review. | Exact sample proportions | Non-negative counts and positive total; zero total gives undefined values | `[1,1,1]` gives three `1/3`; Python `fractions.Fraction` compares full sample results | Values beyond `2^53`, zero-total samples, scale invariance, feature permutations, random integer matrices with proportions summing to one | Implemented | | Exact group proportions | Same count contract; members and group labels are bound to the input fingerprint | Hand-computed two-group fixture | Empty groups and aligned-label failures; group zero-total state | Implemented | | Observed richness | Number of strictly positive exact counts; empty sample richness is zero | Uniform three-taxon and single-taxon fixtures | Empty and extreme-scale samples | Implemented | -| High-precision Shannon | Positive total; scoped MPFR precision above 53 bits; undefined at zero total | Python `decimal` references for `log(2)`, `log(3)`, and counts `[1,2,3,4]` at a 220-digit context | Extreme rare-taxon scale that underflows `Float64`; precision round-trip and concurrent 128/256/512/1024-bit scopes | Implemented; no exactness or sampling-error claim | +| High-precision Shannon | Positive total; scoped MPFR precision above 53 bits; undefined at zero total | GNU `bc` references for `log(2)`, `log(3)`, and counts `[1,2,3,4]` at a 220-digit context | Extreme rare-taxon scale that underflows `Float64`; precision round-trip and concurrent 128/256/512/1024-bit scopes | Implemented; no exactness or sampling-error claim | | High-precision Gini–Simpson | Positive total; exact `(N²-Σc²)/N²` intermediate within the configured rational budget; undefined at zero total | Exact known answers `2/3`, `0`, and `7/10`, converted once to MPFR | Extreme scale, budget exhaustion, serialization and concurrency tests | Implemented | | Fixed-decimal display | Display only; exact/storage values remain separate | `1/3`, `1/2`, and negative half-way fixtures | Huge `BigFloat` display avoids `Float64` overflow; ties-away-from-zero check | Implemented | | Direct JSON boundary | JSON object, explicit validated policy, features-by-samples matrix, 16 MiB / 100,000-cell limits | In-process and real-listener request fixtures | Deterministic malformed-input fuzz, wrong shapes, unsafe integers, float representations and unknown policies | Implemented | @@ -21,8 +21,8 @@ independent scientific review. | Immutable derived analysis | Canonical JSON-safe record; content ID recomputed on load; no replacement of an existing destination | Repeat save/load and canonical-byte round trip | Concurrent publishers, same-ID tampering, invalid record, failed publication cleanup and source/run ownership checks | Implemented | The high-precision reference generator is -`test/references/check_high_precision_references.py`; it uses Python's `decimal` -module and does not read output from the Julia implementation. The CI job checks +`test/references/check_high_precision_references.sh`; it uses GNU `bc` +(not Julia's MPFR or GMP) and does not read output from the Julia implementation. The CI job checks that the constants embedded in the Julia tests still match that independent generator. diff --git a/bench/null_calibration/report_failures.jl b/bench/null_calibration/report_failures.jl new file mode 100644 index 00000000..91aaaa6b --- /dev/null +++ b/bench/null_calibration/report_failures.jl @@ -0,0 +1,166 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# Publish distinct native-calibration failure notes as Actions warnings. +# +# Usage: julia --startup-file=no report_failures.jl CELL < RESULTS.csv +# Standard library only. The CSV reader follows the rules of the Python csv module +# (default dialect) that this replaces: quoted fields may hold commas, quotes +# and line breaks, a quote inside an unquoted field is literal, and blank lines +# are skipped. + +const RESULT_FIELDS = [ + "truth", "shape", "offset", "n_per", "taxa", "sim", "taxon", + "true_log2fc", "test", "status", "pvalue", "note", +] +const LEGACY_FIELDS = RESULT_FIELDS[1:end-1] +const CELL_LABEL = r"^[A-Za-z0-9][A-Za-z0-9_.-]*$" + +# Escape data characters reserved by the GitHub Actions command protocol. +escape_annotation(text::AbstractString) = + replace(text, "%" => "%25", "\r" => "%0D", "\n" => "%0A") + +# Keep untrusted cell labels from injecting extra Actions commands. The label +# is matched as a whole: a trailing newline must not slip past an end anchor. +function validate_cell_label(cell::AbstractString) + (occursin(CELL_LABEL, cell) && !endswith(cell, "\n")) || + throw(ArgumentError("unsafe cell label for Actions annotation")) + String(cell) +end + +""" + parse_csv(text) -> Vector{Vector{String}} + +Split `text` into records of fields. Blank records come back as empty vectors. +""" +function parse_csv(text::AbstractString) + records = Vector{Vector{String}}() + fields = String[] + field = IOBuffer() + # :start_record, :start_field, :in_field, :in_quoted, :quote_in_quoted, :eat_newlines + state = :start_record + finish_field!() = (push!(fields, String(take!(field))); nothing) + finish_record!() = (push!(records, fields); fields = String[]; nothing) + for c in text + if state === :eat_newlines + (c == '\n' || c == '\r') && continue + state = :start_record + end + if state === :start_record + if c == '\n' || c == '\r' + finish_record!() # a blank line: an empty record + state = :eat_newlines + continue + end + state = :start_field + end + if state === :start_field + if c == '\n' || c == '\r' + finish_field!(); finish_record!(); state = :eat_newlines + elseif c == '"' + state = :in_quoted + elseif c == ',' + finish_field!() + else + write(field, c); state = :in_field + end + elseif state === :in_field + if c == '\n' || c == '\r' + finish_field!(); finish_record!(); state = :eat_newlines + elseif c == ',' + finish_field!(); state = :start_field + else + write(field, c) + end + elseif state === :in_quoted + if c == '"' + state = :quote_in_quoted + else + write(field, c) + end + elseif state === :quote_in_quoted + if c == '"' + write(field, '"'); state = :in_quoted + elseif c == ',' + finish_field!(); state = :start_field + elseif c == '\n' || c == '\r' + finish_field!(); finish_record!(); state = :eat_newlines + else + write(field, c); state = :in_field # lenient, as Python's default + end + end + end + # End of input: a field still open is closed, as the Python reader does. + if state === :in_field || state === :in_quoted || state === :quote_in_quoted + finish_field!(); finish_record!() + elseif state === :start_field + finish_field!(); finish_record!() + end + records +end + +# Python's repr of a list of strings, as the header error message showed it. +function py_repr(values) + quote_one(s) = begin + q = occursin('\'', s) && !occursin('"', s) ? '"' : '\'' + body = replace(s, "\\" => "\\\\", "\n" => "\\n", "\r" => "\\r", "\t" => "\\t") + q == '\'' && (body = replace(body, "'" => "\\'")) + string(q, body, q) + end + "[" * join(quote_one.(values), ", ") * "]" +end + +""" + failure_annotations(io, cell) -> Vector{String} + +Warnings for each distinct non-empty (test, failure-note) pair of the failed rows. +""" +function failure_annotations(io::IO, cell::AbstractString) + cell = validate_cell_label(cell) + text = read(io, String) + isvalid(text) || throw(ArgumentError("input is not valid UTF-8")) + records = parse_csv(text) + annotations = String[] + # The header is the very first record, blank or not, as with Python's DictReader. + isempty(records) && throw(ArgumentError("unexpected CSV header None")) + header = records[1] + header == RESULT_FIELDS || header == LEGACY_FIELDS || + throw(ArgumentError("unexpected CSV header $(py_repr(header))")) + seen = Set{Tuple{String,String}}() + for (offset, row) in enumerate(filter(!isempty, records[2:end])) + record_number = offset + 1 # the header is record 1; the first data record is 2 + length(row) == length(header) || + throw(ArgumentError("malformed CSV record $record_number")) + values = Dict(zip(header, row)) + get(values, "status", nothing) == "failed" || continue + note = get(values, "note", nothing) + (note === nothing || isempty(note) || note == "NA") && continue + key = (get(values, "test", ""), note) + key in seen && continue + push!(seen, key) + message = escape_annotation("$(key[1]) :: $(key[2])") + push!(annotations, "::warning title=$(cell): a requested test failed::$(message)") + end + annotations +end + +function main(args = ARGS) + if length(args) != 1 + println(stderr, "usage: ", basename(PROGRAM_FILE), " CELL < RESULTS.csv") + return 2 + end + cell = args[1] + try + for annotation in failure_annotations(stdin, cell) + println(annotation) + end + catch err + err isa ArgumentError && (println(stderr, cell, ": ", err.msg); return 2) + err isa Union{SystemError,Base.IOError} && (println(stderr, cell, ": ", sprint(showerror, err)); return 2) + rethrow() + end + 0 +end + +if abspath(PROGRAM_FILE) == @__FILE__ + exit(main()) +end diff --git a/bench/null_calibration/report_failures.py b/bench/null_calibration/report_failures.py deleted file mode 100644 index bc3e62fa..00000000 --- a/bench/null_calibration/report_failures.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: AGPL-3.0-only -# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) -"""Publish distinct native-calibration failure notes as Actions warnings.""" - -from __future__ import annotations - -import csv -import io -import re -import sys -from pathlib import Path -from typing import TextIO - -RESULT_FIELDS = [ - "truth", - "shape", - "offset", - "n_per", - "taxa", - "sim", - "taxon", - "true_log2fc", - "test", - "status", - "pvalue", - "note", -] -LEGACY_FIELDS = RESULT_FIELDS[:-1] -CELL_LABEL = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*\Z") - - -def _escape_annotation(text: str) -> str: - """Escape data characters reserved by the GitHub Actions command protocol.""" - return text.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") - - -def _validate_cell_label(cell: str) -> str: - """Keep untrusted cell labels from injecting extra Actions commands.""" - if not CELL_LABEL.fullmatch(cell): - raise ValueError("unsafe cell label for Actions annotation") - return cell - - -def failure_annotations(stream: TextIO, cell: str) -> list[str]: - """Return warnings for each distinct non-empty (test, failure-note) pair.""" - cell = _validate_cell_label(cell) - annotations: list[str] = [] - seen: set[tuple[str, str]] = set() - reader = csv.DictReader(stream) - if reader.fieldnames not in (RESULT_FIELDS, LEGACY_FIELDS): - raise ValueError(f"unexpected CSV header {reader.fieldnames!r}") - for record_number, row in enumerate(reader, start=2): - if None in row or any(value is None for value in row.values()): - raise ValueError(f"malformed CSV record {record_number}") - if row.get("status") != "failed": - continue - note = row.get("note") - if not note or note == "NA": - continue - key = (row.get("test", ""), note) - if key in seen: - continue - seen.add(key) - message = _escape_annotation(f"{key[0]} :: {note}") - annotations.append( - f"::warning title={cell}: a requested test failed::{message}" - ) - return annotations - - -def main(argv: list[str] | None = None) -> int: - args = sys.argv[1:] if argv is None else argv - if len(args) != 1: - print( - f"usage: {Path(sys.argv[0]).name} CELL < RESULTS.csv", - file=sys.stderr, - ) - return 2 - cell = args[0] - try: - # The workflow redirects a known result file to stdin. Read the raw - # stream with newline="" so csv preserves embedded CR/LF characters. - stream = io.TextIOWrapper(sys.stdin.buffer, encoding="utf-8", newline="") - for annotation in failure_annotations(stream, cell): - print(annotation) - except (OSError, csv.Error, ValueError) as error: - print(f"{cell}: {error}", file=sys.stderr) - return 2 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/config/defaults/validated_statistics.kyaml b/config/defaults/validated_statistics.kyaml index 3e122764..5d04a361 100644 --- a/config/defaults/validated_statistics.kyaml +++ b/config/defaults/validated_statistics.kyaml @@ -61,7 +61,7 @@ "test/unit/test_statistics_boundaries.jl", "test/unit/test_statistics_adversarial.jl", "test/unit/test_statistics_properties.jl", - "test/references/exact_summary_reference.py", + "test/references/exact_summary_reference.sh", "docs/statistics/numeric-contracts.md", ], }, @@ -114,7 +114,7 @@ ], evidence: [ "test/unit/test_high_precision.jl", - "test/references/check_high_precision_references.py", + "test/references/check_high_precision_references.sh", "test/unit/test_statistics_boundaries.jl", "test/unit/test_statistics_adversarial.jl", "test/unit/test_statistics_properties.jl", diff --git a/docs/proposals/python-test-oracles.md b/docs/proposals/python-test-oracles.md index 12fbf236..c80d7c5d 100644 --- a/docs/proposals/python-test-oracles.md +++ b/docs/proposals/python-test-oracles.md @@ -3,8 +3,13 @@ # Proposal: where the no-new-Python rule stops -Status: **needs an owner decision.** Nothing here is applied; the rule in -`.github/workflows/language-policy.yml` is unchanged. +Status: **decided.** The owner ruled to port all Python in the repository and to +write the oracles in GNU `bc` and awk, so that they share no maths code with +Julia (MPFR, GMP, LAPACK). The no-new-Python rule in +`.github/workflows/language-policy.yml` is unchanged and needs no exception. +The oracles are `test/references/*.sh`, `*.bc` and `*.awk`; the harness and report +code is Julia. Third-party pipeline tools installed with pip (cutadapt, multiqc) +are not estate code and stay. ## The question diff --git a/docs/statistics/constrained-ordinations.md b/docs/statistics/constrained-ordinations.md index dc8895cb..863cff0a 100644 --- a/docs/statistics/constrained-ordinations.md +++ b/docs/statistics/constrained-ordinations.md @@ -85,7 +85,7 @@ When `variance_partitioning = true` and 2 to 4 constraining covariates vary: - Capability & compatibility contract (KYAML): `config/defaults/ordination.kyaml` (`GET /api/v1/ordination/contract`) - Canonical response schema (RFC 7493 I-JSON + RFC 8785 JCS): `docs/schemas/constrained-ordination.schema.json` -- Independent Python stdlib reference oracle & canonical fixture: `test/references/check_ordination_references.py` and `test/fixtures/ordination_reference.json` +- Independent awk reference oracle & canonical fixture: `test/references/check_ordination_references.sh` and `test/fixtures/ordination_reference.json` --- diff --git a/docs/statistics/issue-1-ultraplan.md b/docs/statistics/issue-1-ultraplan.md index fe50ced2..47a3d0b8 100644 --- a/docs/statistics/issue-1-ultraplan.md +++ b/docs/statistics/issue-1-ultraplan.md @@ -40,7 +40,7 @@ symbolic gate (#2) remains blocked. | --- | --- | --- | | Approve and publish the bounded method catalogue | Approval recorded in an issue comment; the catalogue document is absent from this checkout | Publish the approved scope and a machine-readable catalogue | | Publish numeric conversion/storage contracts | Code implements the rules; the contract document is absent | Restore the boundary-by-boundary contract and test it | -| Known-answer and independent references | Python `fractions`/`decimal` references exist for two paths | Add cross-language boundary and round-trip evidence | +| Known-answer and independent references | GNU `bc` references exist for two paths | Add cross-language boundary and round-trip evidence | | Adversarial/precision stability | Some cases exist | Add a dedicated adversarial/precision suite | | Property/generative tests and fuzzing | Some generative cases exist | Add invariants, metamorphic properties, config fuzzing | | Negative controls and mutation evidence | Not retained | Add a mutation driver, CI evidence, and recorded commands | @@ -75,7 +75,7 @@ symbolic gate (#2) remains blocked. ### 1.4 Environment limits recorded honestly The agent sandbox has no Julia, R, bun, cargo, Go, yq or `ijson-jcs` binary. -Local work can therefore run Python reference checks, static inspection and +Local work can therefore run the bc and awk reference checks, static inspection and JavaScript-runtime boundary probes only. Julia tests and the standards tools must run in GitHub Actions; the branch is pushed and CI is iterated until the required checks are green. No local Julia result is claimed. @@ -242,9 +242,9 @@ julia --project=. -t2 test/runtests.jl numeric_policy exact_summaries \ statistics_ui # references -python3 test/references/check_high_precision_references.py --check \ +bash test/references/check_high_precision_references.sh --check \ test/unit/test_high_precision.jl -python3 test/references/exact_summary_reference.py +bash test/references/exact_summary_reference.sh # benchmark (CI retains the JSON) julia --project=. -t2 bench/validated_statistics.jl \ diff --git a/docs/statistics/method-catalogue-v1.md b/docs/statistics/method-catalogue-v1.md index a71417ce..959848fd 100644 --- a/docs/statistics/method-catalogue-v1.md +++ b/docs/statistics/method-catalogue-v1.md @@ -58,7 +58,7 @@ proportions. Integer counts are summed in Julia `BigInt`; proportions are `test/unit/test_statistics_boundaries.jl`, `test/unit/test_statistics_adversarial.jl`, `test/unit/test_statistics_properties.jl`, - `test/references/exact_summary_reference.py`, + `test/references/exact_summary_reference.sh`, `docs/statistics/numeric-contracts.md`. ### `high-precision-descriptive-diversity` — supported @@ -70,7 +70,7 @@ an explicit MPFR precision above 53 bits. - Not claimed: exactness, uncertainty intervals, group summaries (refused in this mode rather than silently ignored). - Evidence: `test/unit/test_high_precision.jl`, - `test/references/check_high_precision_references.py`, + `test/references/check_high_precision_references.sh`, `test/unit/test_statistics_boundaries.jl`, `test/unit/test_statistics_adversarial.jl`, `test/unit/test_statistics_properties.jl`, diff --git a/docs/statistics/numeric-contracts.md b/docs/statistics/numeric-contracts.md index 6b66d003..ff15a4cf 100644 --- a/docs/statistics/numeric-contracts.md +++ b/docs/statistics/numeric-contracts.md @@ -86,7 +86,7 @@ cross a boundary as decimal strings; floating point never carries a count.** | Julia `Int64` → `Float64` | Exact through `2^53`; `2^53 + 1` rounds down silently | `MAX_EXACT_FLOAT`; refusal of `> 2^53 - 1` JSON numbers | | JavaScript `JSON.parse` | `{"n":9007199254740993}` becomes `9007199254740992` with no error; the string form is byte-exact | Node check in `test_statistics_boundaries.jl` (skipped and announced when `node` is absent) | | R `numeric` | R has no integer type wider than 32 bits; every number is a double, so `9007199254740993` arrives one short | Rscript check in `test_statistics_boundaries.jl` (skipped and announced when `Rscript` is absent) | -| Python | `int` is arbitrary precision; `decimal`/`fractions` are the independent reference implementations | `test/references/exact_summary_reference.py`, `check_high_precision_references.py` | +| GNU bc | arbitrary-precision integers and decimals; the independent reference implementation | `test/references/exact_summary_reference.sh`, `check_high_precision_references.py` | | DuckDB | `TINYINT`/`SMALLINT`/`INTEGER`/`BIGINT`/`HUGEINT` are accepted; `FLOAT`/`DOUBLE`/`DECIMAL` are refused with `422` | `_validated_integer_sample_columns`; boundary test builds one table per type | | CSV/text | There is no CSV writer for this layer. Display strings are for humans and are labelled rounded; they are not a data interchange format | `to_display`, `summary_to_display` | diff --git a/docs/statistics/validation-evidence.md b/docs/statistics/validation-evidence.md index f6d83402..c0ea1170 100644 --- a/docs/statistics/validation-evidence.md +++ b/docs/statistics/validation-evidence.md @@ -35,9 +35,9 @@ never counted as a pass. ## 3. Known-answer and independent references **Met.** Julia known-answer tests in `test_exact_summaries.jl` and -`test_high_precision.jl`; independent Python `fractions` and `decimal` -references in `test/references/exact_summary_reference.py` and -`test/references/check_high_precision_references.py`, with the CI step +`test_high_precision.jl`; independent GNU `bc` fraction and high-precision +references in `test/references/exact_summary_reference.sh` and +`test/references/check_high_precision_references.sh`, with the CI step `Check independent high-precision references` verifying the constants embedded in the Julia tests. The reference implementations share no code with the layer. @@ -181,7 +181,7 @@ julia --project=. -t2 test/runtests.jl \ statistics_adversarial statistics_properties statistics_ui # Independent references -python3 test/references/check_high_precision_references.py --check \ +bash test/references/check_high_precision_references.sh --check \ test/unit/test_high_precision.jl # Simulation, mutation, benchmark diff --git a/scripts/tests/test_null_calibration_failures.jl b/scripts/tests/test_null_calibration_failures.jl new file mode 100644 index 00000000..6fd205b0 --- /dev/null +++ b/scripts/tests/test_null_calibration_failures.jl @@ -0,0 +1,97 @@ +# SPDX-License-Identifier: MPL-2.0 +# CSV parsing and annotation escaping for null-calibration package failures. +# +# Run: julia --startup-file=no scripts/tests/test_null_calibration_failures.jl +using Test + +include(joinpath(@__DIR__, "..", "..", "bench", "null_calibration", "report_failures.jl")) + +# One CSV field as Python's csv.writer (QUOTE_MINIMAL) writes it. +csv_field(v) = v === nothing ? "" : begin + s = string(v) + occursin(r"[\",\r\n]", s) ? "\"" * replace(s, "\"" => "\"\"") * "\"" : s +end + +write_rows(path, fields, rows) = open(path, "w") do io + write(io, join(fields, ","), "\n") + for row in rows + write(io, join((csv_field(get(row, f, nothing)) for f in fields), ","), "\n") + end +end + +annotations_for(path, cell) = open(io -> failure_annotations(io, cell), path) + +function result_row(; test, status, note = nothing) + Dict("truth" => "null", "shape" => "original", "offset" => "tss", "n_per" => 4, + "taxa" => 40, "sim" => 1, "taxon" => 1, "true_log2fc" => 0, + "test" => test, "status" => status, + "pvalue" => status == "failed" ? "NA" : 0.01, "note" => note) +end + +@testset "failure annotations" begin + @testset "distinct failures are CSV-aware, deduplicated and escaped" begin + note_one = "comma, quote \" and newline\ncontinued" + note_two = "second 50% and carriage\rreturn\ncontinued" + mktempdir() do directory + path = joinpath(directory, "pvalues_smoke.csv") + write_rows(path, RESULT_FIELDS, [ + result_row(test = "duplicate_test", status = "failed", note = note_one), + result_row(test = "duplicate_test", status = "failed", note = note_one), + result_row(test = "duplicate_test", status = "failed", note = note_two), + result_row(test = "missing_note", status = "failed", note = "NA"), + result_row(test = "ok_test", status = "ok"), + ]) + cell = "null_original_tss_n4_t40_smoke" + @test annotations_for(path, cell) == [ + "::warning title=$cell: a requested test failed::duplicate_test :: " * + "comma, quote \" and newline%0Acontinued", + "::warning title=$cell: a requested test failed::duplicate_test :: " * + "second 50%25 and carriage%0Dreturn%0Acontinued", + ] + end + end + + @testset "an unsafe cell label is rejected" begin + io = IOBuffer(join(RESULT_FIELDS, ",") * "\n") + @test_throws ArgumentError("unsafe cell label for Actions annotation") failure_annotations(io, "bad\n::notice") + @test_throws ArgumentError failure_annotations(IOBuffer(""), "trailing\n") + @test_throws ArgumentError failure_annotations(IOBuffer(""), "") + end + + @testset "a legacy header without notes yields nothing" begin + mktempdir() do directory + path = joinpath(directory, "legacy.csv") + write_rows(path, LEGACY_FIELDS, [result_row(test = "old_test", status = "failed")]) + @test annotations_for(path, "legacy_cell") == String[] + end + end + + @testset "an extra CSV field is rejected" begin + mktempdir() do directory + path = joinpath(directory, "malformed.csv") + write(path, join(RESULT_FIELDS, ",") * "\n" * + "null,original,tss,4,40,1,1,0,edger_ql,failed,NA,reason,extra\n") + err = try annotations_for(path, "bad_cell"); nothing catch e; e end + @test err isa ArgumentError + @test occursin("malformed CSV record", err.msg) + end + end + + @testset "other boundaries" begin + @test_throws ArgumentError failure_annotations(IOBuffer(""), "cell") + @test_throws ArgumentError failure_annotations(IOBuffer("a,b\n1,2\n"), "cell") + # Blank lines are skipped and do not shift the record numbers of real rows. + text = join(RESULT_FIELDS, ",") * "\n\n" * + "null,original,tss,4,40,1,1,0,edger_ql,failed,NA,reason\n\n" + @test failure_annotations(IOBuffer(text), "c") == + ["::warning title=c: a requested test failed::edger_ql :: reason"] + # CRLF files read the same as LF files. + crlf = replace(text, "\n" => "\r\n") + @test failure_annotations(IOBuffer(crlf), "c") == failure_annotations(IOBuffer(text), "c") + # A short row is malformed. + short = join(RESULT_FIELDS, ",") * "\nnull,original\n" + @test_throws ArgumentError failure_annotations(IOBuffer(short), "c") + # Invalid UTF-8 is an error, not a silent replacement. + @test_throws ArgumentError failure_annotations(IOBuffer(UInt8[0xff, 0xfe]), "c") + end +end diff --git a/scripts/tests/test_null_calibration_failures.py b/scripts/tests/test_null_calibration_failures.py deleted file mode 100644 index 00209c28..00000000 --- a/scripts/tests/test_null_calibration_failures.py +++ /dev/null @@ -1,100 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -"""CSV parsing and annotation escaping for null-calibration package failures.""" -import csv -import io -from pathlib import Path -import sys -import tempfile -import unittest - -ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT / "bench/null_calibration")) -from report_failures import LEGACY_FIELDS, RESULT_FIELDS, failure_annotations - - -class FailureAnnotationTests(unittest.TestCase): - def write_rows(self, path, fields, rows): - with path.open("w", encoding="utf-8", newline="") as stream: - writer = csv.DictWriter(stream, fieldnames=fields, lineterminator="\n") - writer.writeheader() - writer.writerows( - {field: row.get(field) for field in fields} for row in rows - ) - - def annotations_for(self, path, cell): - with path.open(encoding="utf-8", newline="") as stream: - return failure_annotations(stream, cell) - - def row(self, *, test, status, note=None): - return { - "truth": "null", - "shape": "original", - "offset": "tss", - "n_per": 4, - "taxa": 40, - "sim": 1, - "taxon": 1, - "true_log2fc": 0, - "test": test, - "status": status, - "pvalue": "NA" if status == "failed" else 0.01, - "note": note, - } - - def test_distinct_failures_are_csv_aware_deduplicated_and_escaped(self): - note_one = 'comma, quote " and newline\ncontinued' - note_two = "second 50% and carriage\rreturn\ncontinued" - with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "pvalues_smoke.csv" - self.write_rows( - path, - RESULT_FIELDS, - [ - self.row(test="duplicate_test", status="failed", note=note_one), - self.row(test="duplicate_test", status="failed", note=note_one), - self.row(test="duplicate_test", status="failed", note=note_two), - self.row(test="missing_note", status="failed", note="NA"), - self.row(test="ok_test", status="ok"), - ], - ) - cell = "null_original_tss_n4_t40_smoke" - self.assertEqual( - self.annotations_for(path, cell), - [ - f"::warning title={cell}: a requested test failed::duplicate_test :: " - 'comma, quote " and newline%0Acontinued', - f"::warning title={cell}: a requested test failed::duplicate_test :: " - "second 50%25 and carriage%0Dreturn%0Acontinued", - ], - ) - - def test_unsafe_cell_label_is_rejected(self): - stream = io.StringIO(",".join(RESULT_FIELDS) + "\n") - with self.assertRaisesRegex(ValueError, "unsafe cell label"): - failure_annotations(stream, "bad\n::notice") - - def test_legacy_header_without_notes_and_missing_note_are_ignored(self): - with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "legacy.csv" - self.write_rows( - path, - LEGACY_FIELDS, - [self.row(test="old_test", status="failed")], - ) - self.assertEqual(self.annotations_for(path, "legacy_cell"), []) - - def test_extra_csv_field_is_rejected(self): - with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "malformed.csv" - path.write_text( - ",".join(RESULT_FIELDS) + "\n" - + "null,original,tss,4,40,1,1,0,edger_ql,failed,NA,reason,extra\n", - encoding="utf-8", - ) - with path.open(encoding="utf-8", newline="") as stream: - with self.assertRaisesRegex(ValueError, "malformed CSV record"): - failure_annotations(stream, "bad_cell") - - -if __name__ == "__main__": - unittest.main() diff --git a/test/references/check_high_precision_references.py b/test/references/check_high_precision_references.py deleted file mode 100644 index 4f8ec7db..00000000 --- a/test/references/check_high_precision_references.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python3 -"""Reproduce the independent high-precision known answers used by Julia tests. - -Python's decimal module is deliberately independent of Julia/MPFR. No result -produced by the implementation is read here. With --check (the default when a -path is supplied), verify the constants embedded in test_high_precision.jl. -""" -from __future__ import annotations - -import argparse -from decimal import Decimal, localcontext -import json -from pathlib import Path -import re -import sys - -PRECISION = 460 - - -def references(precision: int = PRECISION) -> dict[str, str]: - with localcontext() as context: - context.prec = precision - counts = (1, 2, 3, 4) - total = sum(counts) - h = sum( - -(Decimal(count) / total) * (Decimal(count) / total).ln() - for count in counts - ) - return { - "REF_LN2": str(Decimal(2).ln()), - "REF_LN3": str(Decimal(3).ln()), - "REF_TWO_THIRDS": str(Decimal(2) / Decimal(3)), - "REF_H_1234": str(h), - } - - -def check(path: Path, expected: dict[str, str]) -> list[str]: - text = path.read_text(encoding="utf-8") - failures = [] - for name in expected: - match = re.search(rf'^const {name} = "([^"]+)"$', text, re.MULTILINE) - if not match: - failures.append(f"{name}: constant not found") - continue - embedded = match.group(1) - # REF_LN2 deliberately carries enough digits for the 1024-bit test; - # the other references currently use 220. Verify each value at the - # precision actually embedded instead of rejecting extra correct digits. - precision = len(Decimal(embedded).as_tuple().digits) - if embedded != references(precision)[name]: - failures.append(f"{name}: embedded reference differs from Python decimal") - return failures - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("julia_test", nargs="?", type=Path) - parser.add_argument("--check", action="store_true") - args = parser.parse_args() - expected = references() - if args.check or args.julia_test: - if not args.julia_test: - parser.error("--check requires the Julia test path") - failures = check(args.julia_test, expected) - if failures: - print("\n".join(failures), file=sys.stderr) - return 1 - print("Independent decimal references agree at their embedded precisions.") - return 0 - print(json.dumps({"decimal_precision": PRECISION, "references": expected}, indent=2)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/test/references/check_high_precision_references.sh b/test/references/check_high_precision_references.sh new file mode 100755 index 00000000..a7313c22 --- /dev/null +++ b/test/references/check_high_precision_references.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# Reproduce the independent high-precision known answers used by the Julia tests. +# +# GNU bc shares no code with Julia (MPFR, GMP), which is the point: the answers +# come from a different arithmetic library. No result produced by the +# implementation is read here. The maths is in high_precision_references.bc. +# +# Usage: +# check_high_precision_references.sh print the references as JSON +# check_high_precision_references.sh --check TEST.jl compare the constants in TEST.jl +# check_high_precision_references.sh TEST.jl same as --check +# Exit: 0 agree, 1 a constant differs or is missing, 2 usage or tool error. +set -euo pipefail + +here="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +bc_source="$here/high_precision_references.bc" +PRECISION=460 +names=(REF_LN2 REF_LN3 REF_TWO_THIRDS REF_H_1234) + +command -v bc >/dev/null || { echo "bc is required (apt-get install bc)" >&2; exit 2; } + +# Print the references at N significant digits, one "NAME text" line each. +references() { + local n="$1" out name m e text + out="$(printf 'n=%s\n' "$n" | cat - "$bc_source" | BC_LINE_LENGTH=0 bc -l)" || return 2 + for name in "${names[@]}"; do + m="$(awk -v k="$name" '$1 == k { print $2 }' <<<"$out")" + e="$(awk -v k="EX_${name#REF_}" '$1 == k { print $2 }' <<<"$out")" + [[ "$m" =~ ^[0-9]+$ && "$e" =~ ^-?[0-9]+$ && ${#m} -eq $n ]] || { echo "bc gave no $name at $n digits" >&2; return 2; } + if (( e < 0 )); then + text="0.$(printf '%*s' $((-e - 1)) '' | tr ' ' 0)$m" + elif (( e >= n - 1 )); then + text="$m" + else + text="${m:0:e+1}.${m:e+1}" + fi + printf '%s %s\n' "$name" "$text" + done +} + +# Significant digits of a decimal string: drop the point and leading zeros. +significant_digits() { + local d="${1/./}" + d="${d#"${d%%[!0]*}"}" + echo "${#d}" +} + +check() { + local file="$1" failures=0 name embedded n expected + [[ -r "$file" ]] || { echo "cannot read $file" >&2; return 2; } + for name in "${names[@]}"; do + embedded="$(sed -n -E "s/^const $name = \"([^\"]+)\"\$/\\1/p" "$file" | head -n 1)" + if [[ -z "$embedded" ]]; then + echo "$name: constant not found" >&2; failures=1; continue + fi + # Each constant is verified at the precision actually embedded, so extra + # correct digits are not rejected and a wrong digit is. + n="$(significant_digits "$embedded")" + expected="$(references "$n" | awk -v k="$name" '$1 == k { print $2 }')" + if [[ "$embedded" != "$expected" ]]; then + echo "$name: embedded reference differs from the bc reference" >&2; failures=1 + fi + done + if (( failures )); then return 1; fi + echo "Independent decimal references agree at their embedded precisions." +} + +if (( $# == 0 )); then + refs="$(references "$PRECISION")" + printf '{\n "decimal_precision": %s,\n "references": {\n' "$PRECISION" + last="${names[${#names[@]}-1]}" + for name in "${names[@]}"; do + sep=","; [[ "$name" == "$last" ]] && sep="" + printf ' "%s": "%s"%s\n' "$name" "$(awk -v k="$name" '$1 == k { print $2 }' <<<"$refs")" "$sep" + done + printf ' }\n}\n' + exit 0 +fi + +case "$1" in + --check) [[ $# -eq 2 ]] || { echo "--check requires the Julia test path" >&2; exit 2; }; check "$2" ;; + -*) echo "usage: $0 [--check] [TEST.jl]" >&2; exit 2 ;; + *) check "$1" ;; +esac diff --git a/test/references/check_ordination_references.awk b/test/references/check_ordination_references.awk new file mode 100644 index 00000000..3532f700 --- /dev/null +++ b/test/references/check_ordination_references.awk @@ -0,0 +1,1093 @@ +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# Independent awk reference oracle for Issue #19 constrained ordinations +# (RDA, partial RDA with Condition(...), CLR/Aitchison and Hellinger geometries, +# Ezekiel adjusted-R2, Jacobi eigenvalues, and Peres-Neto two-way variance +# partitioning), plus RFC 7493 I-JSON / RFC 8785 JCS canonical verification for +# the constrained-ordination schema and reference fixture. +# +# This is a port of check_ordination_references.py. All the linear algebra is +# hand-written here in plain double precision awk and shares no maths code with +# Julia (no LAPACK, no MPFR). It runs under POSIX awk and under mawk 1.3.4. +# It is driven by check_ordination_references.sh, which passes settings in the +# environment (ORD_*). Messages go to stderr; success prints one line. +# +# Floating point order of operations is kept the same as the Python, including +# the compensated sum() of CPython 3.12 and later (see padd below). + +# ---------------------------------------------------------------- utilities + +function die(msg) { + print "error: " msg > "/dev/stderr" + exit 1 +} + +function abs(x) { return x < 0 ? -x : x } + +# Python float division raises on a zero divisor, so do the same here. +function fdiv(a, b) { + if (b == 0) die("float division by zero") + return a / b +} + +function mlog(x) { + if (x <= 0) die("math domain error: log of non-positive value") + return log(x) +} + +function msqrt(x) { + if (x < 0) die("math domain error: sqrt of negative value") + return sqrt(x) +} + +# Python round(x, d) for finite x: correctly rounded decimal text, read back. +function rnd(x, d) { return sprintf("%." d "f", x) + 0 } + +function zeros(c, z) { + if (c <= 0) return "" + z = sprintf("%" c "s", "") + gsub(/ /, "0", z) + return z +} + +function init_tables( i) { + for (i = 1; i < 256; i++) ORD[sprintf("%c", i)] = i + NULCH = sprintf("%c", 0) + MAXF = 1.7976931348623157e308 +} + +# ------------------------------------------------- compensated sum (CPython) +# CPython 3.12+ sum(): ints are added exactly while no float has been seen. +# The first float switches to float mode (plain add). From then on floats use +# Neumaier compensation and ints are added plainly. The caller says whether +# each item is an int (isint) so mixed int and float lists match CPython. + +function pinit() { PS_M = 0; PS_F = 0; PS_C = 0 } + +function padd(x, isint, t) { + if (PS_M == 0) { + if (isint) { PS_F += x; return } + PS_M = 1 + PS_F = PS_F + x + return + } + if (isint) { PS_F += x; return } + t = PS_F + x + if (abs(PS_F) >= abs(x)) PS_C += (PS_F - t) + x + else PS_C += (x - t) + PS_F + PS_F = t +} + +function pres() { + if (PS_M == 1 && PS_C != 0) return PS_F + PS_C + return PS_F +} + +# ------------------------------------------------------------ JSON parsing +# Tree storage. Node types: o a s i f T F N (object, array, string, int, +# float, true, false, null). Kids of o and a are KID[id,k], k = 1..NK[id]. +# Object keys: KEY (JCS escaped form), SK (UTF-16 code unit hex, for sorting +# and lookup), KBAD (I-JSON problem). Strings: V escaped form, D decoded bytes, +# BAD problem text. Numbers: V value, BAD problem text. + +function newnode(t) { + NID++ + T[NID] = t + NK[NID] = 0 + return NID +} + +function jerr(msg) { die("JSON parse error: " msg " (char " POS ")") } + +function skipws( c) { + while (POS <= TL) { + c = substr(TXT, POS, 1) + if (c == " " || c == "\t" || c == "\n" || c == "\r") POS++ + else break + } +} + +function hex2num(h, i, n, d) { + n = 0 + for (i = 1; i <= length(h); i++) { + d = index("0123456789abcdef", tolower(substr(h, i, 1))) + if (d == 0) return -1 + n = n * 16 + d - 1 + } + return n +} + +function askhex(s, i, r) { + r = "" + for (i = 1; i <= length(s); i++) r = r sprintf("%04x", ORD[substr(s, i, 1)]) + return r +} + +function u16hex(cp) { + if (cp < 65536) return sprintf("%04x", cp) + cp -= 65536 + return sprintf("%04x%04x", 55296 + int(cp / 1024), 56320 + cp % 1024) +} + +function utf8enc(cp) { + if (cp < 128) return sprintf("%c", cp) + if (cp < 2048) return sprintf("%c%c", 192 + int(cp / 64), 128 + cp % 64) + if (cp < 65536) return sprintf("%c%c%c", 224 + int(cp / 4096), 128 + int(cp / 64) % 64, 128 + cp % 64) + return sprintf("%c%c%c%c", 240 + int(cp / 262144), 128 + int(cp / 4096) % 64, 128 + int(cp / 64) % 64, 128 + cp % 64) +} + +# Append one code point to the string being built in S_ESC, S_DEC, S_SK. +function emitcp(cp, wantsk, ch) { + if ((cp >= 64976 && cp <= 65007) || (cp % 65536) >= 65534) { + if (S_BAD == "") S_BAD = sprintf("Unicode noncharacter U+%04X forbidden in I-JSON", cp) + } + if (wantsk) S_SK = S_SK u16hex(cp) + if (cp == 34) { S_ESC = S_ESC "\\\""; S_DEC = S_DEC "\""; return } + if (cp == 92) { S_ESC = S_ESC "\\\\"; S_DEC = S_DEC "\\"; return } + if (cp < 32) { + if (cp == 8) S_ESC = S_ESC "\\b" + else if (cp == 9) S_ESC = S_ESC "\\t" + else if (cp == 10) S_ESC = S_ESC "\\n" + else if (cp == 12) S_ESC = S_ESC "\\f" + else if (cp == 13) S_ESC = S_ESC "\\r" + else S_ESC = S_ESC sprintf("\\u%04x", cp) + S_DEC = S_DEC sprintf("%c", cp) + return + } + ch = utf8enc(cp) + S_ESC = S_ESC ch + S_DEC = S_DEC ch +} + +function u4(p, h) { + h = substr(TXT, p, 4) + if (length(h) < 4) return -1 + return hex2num(h) +} + +# Parse a JSON string at POS (on the opening quote). Strict UTF-8 decoding. +function pstr(wantsk, w, n, run, c, e, cp, lo, b1, need, k, b, loB, hiB) { + S_ESC = "\""; S_DEC = ""; S_SK = ""; S_BAD = "" + POS++ + while (1) { + if (POS > TL) jerr("Unterminated string") + w = substr(TXT, POS, 96) + if (match(w, /^[^"\\\001-\037\200-\377]+/)) { + n = RLENGTH + run = substr(w, 1, n) + S_ESC = S_ESC run + S_DEC = S_DEC run + if (wantsk) S_SK = S_SK askhex(run) + POS += n + continue + } + c = substr(w, 1, 1) + if (c == "\"") { POS++; S_ESC = S_ESC "\""; return } + if (c == "\\") { + e = substr(TXT, POS + 1, 1) + if (e == "u") { + cp = u4(POS + 2) + if (cp < 0) jerr("Invalid \\uXXXX escape") + POS += 6 + if (cp >= 55296 && cp <= 56319 && substr(TXT, POS, 2) == "\\u") { + lo = u4(POS + 2) + if (lo < 0) jerr("Invalid \\uXXXX escape") + if (lo >= 56320 && lo <= 57343) { + cp = 65536 + (cp - 55296) * 1024 + (lo - 56320) + POS += 6 + } + } + if (cp >= 55296 && cp <= 57343) { + if (S_BAD == "") S_BAD = sprintf("Lone surrogate U+%04X forbidden in I-JSON", cp) + if (wantsk) S_SK = S_SK sprintf("%04x", cp) + S_DEC = S_DEC "?" + } else emitcp(cp, wantsk) + continue + } + if (e == "\"") emitcp(34, wantsk) + else if (e == "\\") emitcp(92, wantsk) + else if (e == "/") emitcp(47, wantsk) + else if (e == "b") emitcp(8, wantsk) + else if (e == "f") emitcp(12, wantsk) + else if (e == "n") emitcp(10, wantsk) + else if (e == "r") emitcp(13, wantsk) + else if (e == "t") emitcp(9, wantsk) + else jerr("Invalid \\escape") + POS += 2 + continue + } + b1 = ORD[c] + if (b1 < 32) jerr("Invalid control character in string") + if (b1 >= 194 && b1 <= 223) { need = 1; cp = b1 - 192; loB = 128; hiB = 191 } + else if (b1 >= 224 && b1 <= 239) { need = 2; cp = b1 - 224; loB = (b1 == 224) ? 160 : 128; hiB = (b1 == 237) ? 159 : 191 } + else if (b1 >= 240 && b1 <= 244) { need = 3; cp = b1 - 240; loB = (b1 == 240) ? 144 : 128; hiB = (b1 == 244) ? 143 : 191 } + else jerr("invalid UTF-8 byte") + for (k = 1; k <= need; k++) { + if (POS + k > TL) jerr("truncated UTF-8 sequence") + b = ORD[substr(TXT, POS + k, 1)] + if (k > 1) { loB = 128; hiB = 191 } + if (b < loB || b > hiB) jerr("invalid UTF-8 sequence") + cp = cp * 64 + (b - 128) + } + POS += need + 1 + emitcp(cp, wantsk) + } +} + +function addmember(o, key, sk, kbad, v, i) { + if ((o, sk) in KIDX) { + i = KIDX[o, sk] + KID[o, i] = v + return + } + i = ++NK[o] + KIDX[o, sk] = i + KID[o, i] = v + KEY[o, i] = key + SK[o, i] = sk + KBAD[o, i] = kbad +} + +function pobj( id, k, sk, kb, v, c) { + id = newnode("o") + POS++ + skipws() + if (substr(TXT, POS, 1) == "}") { POS++; return id } + while (1) { + skipws() + if (substr(TXT, POS, 1) != "\"") jerr("Expecting property name enclosed in double quotes") + pstr(1) + k = S_ESC; sk = S_SK; kb = S_BAD + skipws() + if (substr(TXT, POS, 1) != ":") jerr("Expecting ':' delimiter") + POS++ + v = pval() + addmember(id, k, sk, kb, v) + skipws() + c = substr(TXT, POS, 1) + if (c == "}") { POS++; return id } + if (c != ",") jerr("Expecting ',' delimiter") + POS++ + } +} + +function parr( id, v, c) { + id = newnode("a") + POS++ + skipws() + if (substr(TXT, POS, 1) == "]") { POS++; return id } + while (1) { + v = pval() + NK[id]++ + KID[id, NK[id]] = v + skipws() + c = substr(TXT, POS, 1) + if (c == "]") { POS++; return id } + if (c != ",") jerr("Expecting ',' delimiter") + POS++ + } +} + +function pval( c, id, w, win, txt, ds) { + skipws() + if (POS > TL) jerr("Expecting value") + c = substr(TXT, POS, 1) + if (c == "{") return pobj() + if (c == "[") return parr() + if (c == "\"") { + pstr(0) + id = newnode("s") + V[id] = S_ESC; D[id] = S_DEC; BAD[id] = S_BAD + return id + } + if (substr(TXT, POS, 4) == "true") { POS += 4; return newnode("T") } + if (substr(TXT, POS, 5) == "false") { POS += 5; return newnode("F") } + if (substr(TXT, POS, 4) == "null") { POS += 4; return newnode("N") } + win = 64 + while (1) { + w = substr(TXT, POS, win) + match(w, /^-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/) + if (RLENGTH < length(w) || length(w) < win) break + win *= 4 + } + if (RSTART == 0 || RLENGTH <= 0) jerr("Expecting value") + txt = substr(w, 1, RLENGTH) + POS += RLENGTH + if (txt ~ /[.eE]/) { + id = newnode("f") + V[id] = txt + 0 + BAD[id] = "" + if (V[id] > MAXF || V[id] < -MAXF) BAD[id] = "Non-finite float forbidden in I-JSON/JCS" + } else { + id = newnode("i") + V[id] = txt + 0 + BAD[id] = "" + ds = txt + sub(/^-/, "", ds) + if (length(ds) > 16 || (length(ds) == 16 && (ds "") > "9007199254740991")) + BAD[id] = "Integer " txt " outside I-JSON safe range" + } + return id +} + +function parse_json(text) { + TXT = text + TL = length(text) + POS = 1 + ROOT = pval() + skipws() + if (POS <= TL) jerr("Extra data") + return ROOT +} + +# Object lookup by ASCII key name. Returns the kid id or 0. +function getk(o, name, sk) { + sk = askhex(name) + if ((o, sk) in KIDX) return KID[o, KIDX[o, sk]] + return 0 +} + +# ------------------------------------------------------------ JCS (RFC 8785) + +function fmtfloat(x, neg, ax, nd, s, ep, dg, ex, k, n, body, mant, sg) { + if (x == 0) return "0" + neg = (x < 0) + ax = neg ? -x : x + # shortest digit count that reads back to the same double + for (nd = 1; nd <= 17; nd++) { + s = sprintf("%." (nd - 1) "e", ax) + if ((s + 0) == ax) break + } + ep = index(s, "e") + dg = substr(s, 1, ep - 1) + ex = substr(s, ep + 1) + 0 + gsub(/\./, "", dg) + sub(/0+$/, "", dg) + k = length(dg) + n = ex + 1 + if (k <= n && n <= 21) body = dg zeros(n - k) + else if (0 < n && n <= 21) body = substr(dg, 1, n) "." substr(dg, n + 1) + else if (-5 <= n && n <= 0) body = "0." zeros(-n) dg + else { + sg = (n - 1 >= 0) ? "+" : "-" + mant = (k == 1) ? dg : (substr(dg, 1, 1) "." substr(dg, 2)) + body = mant "e" sg abs(n - 1) + } + return neg ? ("-" body) : body +} + +function jcs(id, t, i, n, s, ix, j, tmp) { + t = T[id] + if (t == "N") return "null" + if (t == "T") return "true" + if (t == "F") return "false" + if (t == "i") { + if (BAD[id] != "") die(BAD[id]) + return sprintf("%d", V[id]) + } + if (t == "f") { + if (BAD[id] != "") die(BAD[id]) + return fmtfloat(V[id]) + } + if (t == "s") { + if (BAD[id] != "") die(BAD[id]) + return V[id] + } + if (t == "a") { + s = "[" + for (i = 1; i <= NK[id]; i++) s = s (i > 1 ? "," : "") jcs(KID[id, i]) + return s "]" + } + n = NK[id] + for (i = 1; i <= n; i++) ix[i] = i + for (i = 2; i <= n; i++) { + tmp = ix[i] + j = i - 1 + while (j >= 1 && ((SK[id, ix[j]] "") > (SK[id, tmp] ""))) { ix[j + 1] = ix[j]; j-- } + ix[j + 1] = tmp + } + s = "{" + for (i = 1; i <= n; i++) { + if (KBAD[id, ix[i]] != "") die(KBAD[id, ix[i]]) + s = s (i > 1 ? "," : "") KEY[id, ix[i]] ":" jcs(KID[id, ix[i]]) + } + return s "}" +} + +# ----------------------------------------------------------------- file I/O + +# Python-style whitespace for str.strip(). +function lstrip(s) { + sub(/^([ \t\r\013\014\034-\037]|\302[\205\240]|\341\232\200|\342\200[\200-\212\250\251\257]|\342\201\237|\343\200\200)+/, "", s) + return s +} +function rstrip(s) { + sub(/([ \t\r\013\014\034-\037]|\302[\205\240]|\341\232\200|\342\200[\200-\212\250\251\257]|\342\201\237|\343\200\200)+$/, "", s) + return s +} +function pystrip(s) { return rstrip(lstrip(s)) } + +function utf8valid(s) { + return s ~ /^([\001-\177]|[\302-\337][\200-\277]|\340[\240-\277][\200-\277]|[\341-\354][\200-\277][\200-\277]|\355[\200-\237][\200-\277]|[\356\357][\200-\277][\200-\277]|\360[\220-\277][\200-\277][\200-\277]|[\361-\363][\200-\277][\200-\277][\200-\277]|\364[\200-\217][\200-\277][\200-\277])*$/ +} + +# Whole file as text, lines joined by LF (no final LF). Dies if unreadable. +function slurp(path, line, r, text, first) { + text = ""; first = 1 + while ((r = (getline line < path)) > 0) { + if (index(line, NULCH)) die(path ": NUL byte in file") + text = first ? line : (text "\n" line) + first = 0 + } + close(path) + if (r < 0) die("cannot read " path) + return text +} + +# Lines as str.splitlines() would give them. Sets RL_TEXT to the whole text. +function readlines(path, L, line, r, n, parts, np, i, tmp, c) { + n = 0; c = 0; RL_TEXT = "" + delete L + while ((r = (getline line < path)) > 0) { + if (index(line, NULCH)) die(path ": NUL byte in file") + if (!utf8valid(line)) die(path ": invalid UTF-8") + RL_TEXT = c ? (RL_TEXT "\n" line) : line + c = 1 + sub(/\r$/, "", line) + tmp = line + gsub(/[\013\014\015\034-\036]|\302\205|\342\200[\250\251]/, "\n", tmp) + np = split(tmp, parts, "\n") + for (i = 1; i <= np; i++) L[++n] = parts[i] + } + close(path) + if (r < 0) die("cannot read " path) + return n +} + +# verify_jcs_file: LF check done by the driver (lfok), then parse and compare. +function verify_jcs_file(path, lfok, text, root, canon) { + if (!lfok) die(path ": must end with exactly one trailing LF") + text = slurp(path) + if (!utf8valid(text)) die(path ": invalid UTF-8") + root = parse_json(text) + canon = jcs(root) + if ((text "") != (canon "")) die(path ": content is not byte-for-byte RFC 8785 JCS") + return root +} + +function check_oracle(path, probe, L, n, i, root, inp, ex_, out) { + if ((getline probe < path) < 0) return + close(path) + n = readlines(path, L) + for (i = 1; i <= n; i++) { + if (pystrip(L[i]) == "") continue + root = parse_json(L[i]) + if (T[root] != "o") die("jcs_oracle.jsonl line " i ": not an object") + inp = getk(root, "input"); ex_ = getk(root, "expected") + if (!inp || !ex_) die("jcs_oracle.jsonl line " i ": missing input or expected") + out = jcs(inp) + if (T[ex_] != "s" || BAD[ex_] != "" || (out "") != (D[ex_] "")) + die("jcs_oracle.jsonl line " i " mismatch") + } +} + +# ------------------------------------------------------------ KYAML contract + +function verify_kyaml(path, L, n, i, s, B, nb, toks, nt, text, t, g, val, m) { + n = readlines(path, L) + text = RL_TEXT + nb = 0 + for (i = 1; i <= n; i++) { + s = pystrip(L[i]) + if (s != "" && substr(s, 1, 1) != "#") B[++nb] = s + } + if (nb == 0 || B[1] != "---") die(path ": KYAML must open with explicit '---' after SPDX comments") + if (nb < 2 || B[2] != "{") die(path ": KYAML top-level mapping must use explicit '{' flow syntax") + if (B[nb] != "}") die(path ": KYAML top-level mapping must close with '}'") + nt = split("rda: cca: cap: dbrda: \"capscale\" \"aitchison\" \"chisq\" 99 9999", toks, " ") + for (i = 1; i <= nt; i++) + if (index(text, toks[i]) == 0) die(path ": missing expected contract token " toks[i]) + for (i = 1; i <= n; i++) { + s = pystrip(L[i]) + if (s == "" || substr(s, 1, 1) == "#") continue + if (s == "---" || s == "{" || s == "}" || s == "}," || s == "],") continue + if (match(s, /^[A-Za-z0-9_"']+:/)) { + t = lstrip(substr(s, RLENGTH + 1)) + if (t == "") continue + g = t + if (substr(t, length(t), 1) == ",") { + g = rstrip(substr(t, 1, length(t) - 1)) + if (g == "") g = t + } + val = g + sub(/,+$/, "", val) + val = pystrip(val) + if (val == "{" || val == "[" || substr(val, 1, 1) == "\"" || val == "true" || val == "false" || val == "null") continue + if (val ~ /^-?[0-9]+(\.[0-9]+)?$/) continue + die(path ":" i ": unquoted scalar value '" val "' violates KYAML") + } + } +} + +# --------------------------------------------------- numeric reference engine +# Matrices are arrays M[i,j] (0-based), with int flags MI[i,j] (1 when the +# value came from a JSON integer) and dimensions passed in n and p. + +function centre(M, MI, n, p, OUT, i, j, mean) { + for (j = 0; j < p; j++) { + pinit() + for (i = 0; i < n; i++) padd(M[i, j], MI[i, j]) + mean = fdiv(pres(), n) + for (i = 0; i < n; i++) OUT[i, j] = M[i, j] - mean + } +} + +# Modified Gram-Schmidt with one re-orthogonalisation. Basis vectors go to +# Q[b,i]. Returns the number of vectors kept. +# MI flags JSON ints: while no basis vector exists the column is still int +# typed, so its squared norm is summed as exact ints (as CPython does). +function mgs(M, MI, n, m, Q, j, i, b, nb, rep, proj, norm) { + nb = 0 + for (j = 0; j < m; j++) { + for (i = 0; i < n; i++) VV[i] = M[i, j] + for (rep = 0; rep < 2; rep++) { + for (b = 0; b < nb; b++) { + pinit() + for (i = 0; i < n; i++) padd(Q[b, i] * VV[i], 0) + proj = pres() + for (i = 0; i < n; i++) VV[i] -= proj * Q[b, i] + } + } + pinit() + for (i = 0; i < n; i++) padd(VV[i] * VV[i], nb == 0 ? MI[i, j] : 0) + norm = msqrt(pres()) + if (norm > 1e-11) { + for (i = 0; i < n; i++) Q[nb, i] = VV[i] / norm + nb++ + } + } + return nb +} + +function project(Q, nb, M, n, p, OUT, b, i, j, dot) { + for (i = 0; i < n; i++) for (j = 0; j < p; j++) OUT[i, j] = 0 + for (b = 0; b < nb; b++) { + for (j = 0; j < p; j++) { + pinit() + for (i = 0; i < n; i++) padd(Q[b, i] * M[i, j], 0) + dot = pres() + for (i = 0; i < n; i++) OUT[i, j] += Q[b, i] * dot + } + } +} + +function sub_mat(A, B, n, p, OUT, i, j) { + for (i = 0; i < n; i++) for (j = 0; j < p; j++) OUT[i, j] = A[i, j] - B[i, j] +} + +function frob_sq(M, n, p, i, j) { + pinit() + for (i = 0; i < n; i++) for (j = 0; j < p; j++) padd(M[i, j] * M[i, j], 0) + return pres() +} + +# Eigenvalues of a symmetric m x m matrix by cyclic Jacobi rotations, sorted +# descending, tiny ones dropped. Result in E[0..count-1]; returns count. +function jacobi(S, m, E, tol, i, j, p, q, r, sweep, maxoff, apq, tau, t, c, s, app, aqq, arp, arq, cnt, v, cutoff, h, tmp, vals) { + tol = 1e-13 + if (m == 0) return 0 + if (m == 1) { + if (S[0, 0] > tol) { E[0] = S[0, 0]; return 1 } + return 0 + } + for (i = 0; i < m; i++) for (j = 0; j < m; j++) JA[i, j] = 0.5 * (S[i, j] + S[j, i]) + for (sweep = 0; sweep < 90; sweep++) { + maxoff = 0 + for (p = 0; p < m - 1; p++) + for (q = p + 1; q < m; q++) + if (abs(JA[p, q]) > maxoff) maxoff = abs(JA[p, q]) + if (maxoff <= tol) break + for (p = 0; p < m - 1; p++) { + for (q = p + 1; q < m; q++) { + apq = JA[p, q] + if (abs(apq) <= tol) continue + tau = (JA[q, q] - JA[p, p]) / (2.0 * apq) + if (tau >= 0) t = 1.0 / (tau + sqrt(1.0 + tau * tau)) + else t = -1.0 / (-tau + sqrt(1.0 + tau * tau)) + c = 1.0 / sqrt(1.0 + t * t) + s = t * c + app = JA[p, p] + aqq = JA[q, q] + JA[p, p] = app - t * apq + JA[q, q] = aqq + t * apq + JA[p, q] = 0 + JA[q, p] = 0 + for (r = 0; r < m; r++) { + if (r == p || r == q) continue + arp = JA[r, p] + arq = JA[r, q] + JA[r, p] = c * arp - s * arq + JA[p, r] = JA[r, p] + JA[r, q] = s * arp + c * arq + JA[q, r] = JA[r, q] + } + } + } + } + for (i = 0; i < m; i++) vals[i] = JA[i, i] + for (i = 1; i < m; i++) { + tmp = vals[i] + j = i - 1 + while (j >= 0 && vals[j] < tmp) { vals[j + 1] = vals[j]; j-- } + vals[j + 1] = tmp + } + cutoff = 1e-12 * (vals[0] > 1.0 ? vals[0] : 1.0) + if (!(cutoff > tol)) cutoff = tol + cnt = 0 + for (i = 0; i < m; i++) if (vals[i] > cutoff) E[cnt++] = vals[i] + return cnt +} + +function gram_eigs(M, n, p, dof, E, i, j, k, m) { + if (n <= p) { + m = n + for (i = 0; i < n; i++) for (j = 0; j < n; j++) { + pinit() + for (k = 0; k < p; k++) padd(M[i, k] * M[j, k], 0) + GG[i, j] = fdiv(pres(), dof) + } + } else { + m = p + for (i = 0; i < p; i++) for (j = 0; j < p; j++) { + pinit() + for (k = 0; k < n; k++) padd(M[k, i] * M[k, j], 0) + GG[i, j] = fdiv(pres(), dof) + } + } + return jacobi(GG, m, E) +} + +function ezekiel(r2, n, m) { + return 1.0 - fdiv((1.0 - r2) * (n - 1), n - 1 - m) +} + +function clr(C, n, p, OUT, i, j, ml, L) { + for (i = 0; i < n; i++) { + pinit() + for (j = 0; j < p; j++) { L[j] = mlog(C[i, j]); padd(L[j], 0) } + ml = fdiv(pres(), p) + for (j = 0; j < p; j++) OUT[i, j] = L[j] - ml + } +} + +function hellinger(C, CI, n, p, OUT, i, j, tot) { + for (i = 0; i < n; i++) { + pinit() + for (j = 0; j < p; j++) padd(C[i, j], CI[i, j]) + tot = pres() + for (j = 0; j < p; j++) OUT[i, j] = msqrt(fdiv(C[i, j], tot)) + } +} + +# reference_linear_rda. Results go to R_* globals (rounded as in the Python) +# and U_* globals (unrounded, used by the numeric test driver only). +function rda(Y, YI, n, p, X, XI, kx, Z, ZI, kz, hasz, i, j, dof, tot, cond, q, k, dfres, con, unc, r2, adj, f, r2xz, r2z, ne, nu, v) { + if (n < 1) die("empty matrix") + dof = n - 1 + centre(Y, YI, n, p, YC) + centre(X, XI, n, kx, XC) + tot = fdiv(frob_sq(YC, n, p), dof) + cond = 0; q = 0 + for (i = 0; i < n; i++) for (j = 0; j < p; j++) YW[i, j] = YC[i, j] + for (i = 0; i < n; i++) for (j = 0; j < kx; j++) XW[i, j] = XC[i, j] + if (hasz) { + centre(Z, ZI, n, kz, ZC) + q = mgs(ZC, NOZ, n, kz, QZ) + if (q > 0) { + project(QZ, q, YW, n, p, YZ) + cond = fdiv(frob_sq(YZ, n, p), dof) + sub_mat(YW, YZ, n, p, YW) + project(QZ, q, XW, n, kx, XZ) + sub_mat(XW, XZ, n, kx, XW) + } + } + k = mgs(XW, NOZ, n, kx, QX) + dfres = n - 1 - q - k + project(QX, k, YW, n, p, YH) + sub_mat(YW, YH, n, p, YR) + con = fdiv(frob_sq(YH, n, p), dof) + unc = fdiv(frob_sq(YR, n, p), dof) + r2 = fdiv(con, tot) + if (q == 0) adj = ezekiel(r2, n, k) + else { + r2xz = fdiv(con + cond, tot) + r2z = fdiv(cond, tot) + adj = ezekiel(r2xz, n, k + q) - ezekiel(r2z, n, q) + } + f = fdiv(fdiv(con, k), fdiv(unc, dfres)) + delete R_CE; delete R_UE; delete U_CE; delete U_UE + ne = gram_eigs(YH, n, p, dof, U_CE) + for (i = 0; i < ne; i++) R_CE[i] = rnd(U_CE[i], 12) + nu = gram_eigs(YR, n, p, dof, U_UE) + for (i = 0; i < nu; i++) R_UE[i] = rnd(U_UE[i], 12) + R_NCE = ne; R_NUE = nu + R_adj = rnd(adj, 12); R_cond = rnd(cond, 12); R_con = rnd(con, 12) + R_q = q; R_k = k; R_dfres = dfres + R_f = rnd(f, 12); R_r2 = rnd(r2, 12); R_tot = rnd(tot, 12); R_unc = rnd(unc, 12) + U_adj = adj; U_cond = cond; U_con = con; U_f = f; U_r2 = r2; U_tot = tot; U_unc = unc +} + +# reference_varpart_twoway. Results in VP[name]. +function varpart(Y, YI, n, p, X1, X1I, k1, X2, X2I, k2, i, j, ra, rb, rab) { + rda(Y, YI, n, p, X1, X1I, k1, NOZ, NOZ, 0, 0) + ra = R_adj + rda(Y, YI, n, p, X2, X2I, k2, NOZ, NOZ, 0, 0) + rb = R_adj + for (i = 0; i < n; i++) { + for (j = 0; j < k1; j++) { X12[i, j] = X1[i, j]; X12I[i, j] = X1I[i, j] } + for (j = 0; j < k2; j++) { X12[i, k1 + j] = X2[i, j]; X12I[i, k1 + j] = X2I[i, j] } + } + rda(Y, YI, n, p, X12, X12I, k1 + k2, NOZ, NOZ, 0, 0) + rab = R_adj + VP["gross_x1_adj_r2"] = rnd(ra, 12) + VP["gross_x12_adj_r2"] = rnd(rab, 12) + VP["gross_x2_adj_r2"] = rnd(rb, 12) + VP["residual"] = rnd(1.0 - rab, 12) + VP["shared"] = rnd(ra + rb - rab, 12) + VP["unique_x1"] = rnd(rab - rb, 12) + VP["unique_x2"] = rnd(rab - ra, 12) +} + +# ---------------------------------------------------------- fixture checking + +# Load a JSON matrix node into M and MI. Sets MN (rows) and MP (columns). +function loadmat(id, M, MI, what, i, j, r, e) { + delete M; delete MI + if (T[id] != "a" || NK[id] < 1) die(what ": expected a non-empty array of rows") + MN = NK[id]; MP = -1 + for (i = 1; i <= MN; i++) { + r = KID[id, i] + if (T[r] != "a" || NK[r] < 1) die(what ": row " i " is not a non-empty array") + if (MP < 0) MP = NK[r] + else if (NK[r] != MP) die(what ": ragged rows") + for (j = 1; j <= MP; j++) { + e = KID[r, j] + if (T[e] != "i" && T[e] != "f") die(what ": non-numeric entry") + M[i - 1, j - 1] = V[e] + MI[i - 1, j - 1] = (T[e] == "i") ? 1 : 0 + } + } +} + +function needk(o, name, what, id) { + id = getk(o, name) + if (!id) die(what ": missing key " name) + return id +} + +function numval(id, what) { + if (T[id] != "i" && T[id] != "f") die(what ": expected a number") + return V[id] +} + +function check_dataset(d, dsid, yn, y, tr, trs, rawn, i, j, mn, mp, z, zn, xn, key, keys, nk, a, b, ex_, k, vk, exv, ev, anum, enum_, isum, vpsum, parr_, nex, r) { + if (T[d] != "o") die("dataset entry is not an object") + dsid = D[needk(d, "id", "dataset")] + y = needk(d, "y", dsid) + loadmat(y, MY, MYI, dsid ".y"); YN = MN; YP = MP + tr = getk(d, "transform") + trs = (tr && T[tr] == "s") ? V[tr] : "" + if (trs == "\"clr\"" || trs == "\"hellinger\"") { + loadmat(needk(d, "raw_counts", dsid), MC, MCI, dsid ".raw_counts"); RN = MN; RP = MP + if (trs == "\"clr\"") clr(MC, RN, RP, MR) + else hellinger(MC, MCI, RN, RP, MR) + for (i = 0; i < RN; i++) for (j = 0; j < RP; j++) MRI[i, j] = 0 + mn = RN; mp = RP + a = (YN < RN) ? YN : RN; b = (YP < RP) ? YP : RP + for (i = 0; i < a; i++) for (j = 0; j < b; j++) + if (abs(MY[i, j] - MR[i, j]) > 1e-10) die(dsid ": stored transformed matrix drifted from raw_counts") + } else { + delete MR; delete MRI + for (i = 0; i < YN; i++) for (j = 0; j < YP; j++) { MR[i, j] = MY[i, j]; MRI[i, j] = MYI[i, j] } + mn = YN; mp = YP + } + loadmat(needk(d, "x", dsid), MX, MXI, dsid ".x"); xn = MN; MXP = MP + if (xn != mn) die(dsid ": x row count differs from y") + z = needk(d, "z", dsid) + zn = 0 + if (T[z] != "N") { + loadmat(z, MZ, MZI, dsid ".z"); zn = 1 + if (MN != mn) die(dsid ": z row count differs from y") + MZP = MP + } + rda(MR, MRI, mn, mp, MX, MXI, MXP, MZ, MZI, MZP, zn) + ex_ = needk(d, "expected_rda", dsid) + if (T[ex_] != "o") die(dsid ": expected_rda is not an object") + nk = split("total_inertia conditioned_inertia constrained_inertia unconstrained_inertia r2 adj_r2 f_statistic", keys, " ") + RV["total_inertia"] = R_tot; RV["conditioned_inertia"] = R_cond; RV["constrained_inertia"] = R_con + RV["unconstrained_inertia"] = R_unc; RV["r2"] = R_r2; RV["adj_r2"] = R_adj; RV["f_statistic"] = R_f + for (i = 1; i <= nk; i++) { + key = keys[i] + ev = numval(needk(ex_, key, dsid ".expected_rda"), dsid "." key) + if (abs(RV[key] - ev) > 1e-9) die(sprintf("%s.%s: expected %s, recomputed %s", dsid, key, ev, RV[key])) + } + for (k = 1; k <= 2; k++) { + key = (k == 1) ? "constrained_eigenvalues" : "unconstrained_eigenvalues" + parr_ = needk(ex_, key, dsid ".expected_rda") + if (T[parr_] != "a") die(dsid "." key ": expected an array") + nex = (k == 1) ? R_NCE : R_NUE + if (nex != NK[parr_]) die(dsid "." key ": length mismatch") + for (i = 0; i < nex; i++) { + anum = (k == 1) ? R_CE[i] : R_UE[i] + enum_ = numval(KID[parr_, i + 1], dsid "." key) + if (abs(anum - enum_) > 1e-9) die(sprintf("%s.%s: %s != %s", dsid, key, anum, enum_)) + } + } + isum = R_cond + R_con + R_unc + if (abs(R_tot - isum) > 1e-10) die(dsid ": inertia partition does not sum to total_inertia") + vk = needk(d, "expected_varpart", dsid) + if (T[vk] != "N") { + if (T[vk] != "o") die(dsid ": expected_varpart is not an object") + loadmat(needk(d, "x1", dsid), MX1, MX1I, dsid ".x1"); a = MN; MX1P = MP + loadmat(needk(d, "x2", dsid), MX2, MX2I, dsid ".x2"); b = MN; MX2P = MP + if (a != mn || b != mn) die(dsid ": x1 or x2 row count differs from y") + varpart(MR, MRI, mn, mp, MX1, MX1I, MX1P, MX2, MX2I, MX2P) + for (i = 1; i <= NK[vk]; i++) { + # KEY holds the escaped, quoted form; strip the quotes for lookup + key = KEY[vk, i] + key = substr(key, 2, length(key) - 2) + if (!(key in VP)) die(dsid ".varpart: unknown key " key) + ev = numval(KID[vk, i], dsid ".varpart." key) + if (abs(VP[key] - ev) > 1e-9) die(sprintf("%s.varpart.%s: expected %s, got %s", dsid, key, ev, VP[key])) + } + vpsum = VP["unique_x1"] + VP["unique_x2"] + VP["shared"] + VP["residual"] + if (abs(vpsum - 1.0) > 1e-10) die(sprintf("%s: varpart fractions sum to %s, expected 1.0", dsid, vpsum)) + } +} + +function verify_fixture_math(root, ds, nds, i) { + if (T[root] != "o") die("fixture root must be an object") + ds = getk(root, "datasets") + nds = 0 + if (ds) { + if (T[ds] != "a") die("datasets must be an array") + nds = NK[ds] + } + if (nds < 4) die("Expected >= 4 reference datasets, found " nds) + for (i = 1; i <= nds; i++) check_dataset(KID[ds, i]) + return nds +} + +# ------------------------------------------------------------- generation +# Builders for output trees. + +function mkobj() { return newnode("o") } +function mkarr() { return newnode("a") } +function mknum(x, id) { id = newnode("f"); V[id] = x; BAD[id] = ""; return id } +function mkint(x, id) { id = newnode("i"); V[id] = x; BAD[id] = ""; return id } +function mknull() { return newnode("N") } +function mkstr(s, id) { + id = newnode("s") + gsub(/\\/, "\\\\", s) + gsub(/"/, "\\\"", s) + V[id] = "\"" s "\""; D[id] = s; BAD[id] = "" + return id +} +function apush(a, v) { NK[a]++; KID[a, NK[a]] = v } +function oset(o, name, v, sk) { + sk = askhex(name) + addmember(o, "\"" name "\"", sk, "", v) +} + +# Matrix as a node, rounding to d digits when d > 0. +function matnode(M, n, p, d, a, r, i, j, x) { + a = mkarr() + for (i = 0; i < n; i++) { + r = mkarr() + for (j = 0; j < p; j++) { + x = M[i, j] + if (d > 0) x = rnd(x, d) + apush(r, mknum(x)) + } + apush(a, r) + } + return a +} + +function rdanode( o, a, i) { + o = mkobj() + oset(o, "adj_r2", mknum(R_adj)) + oset(o, "conditioned_inertia", mknum(R_cond)) + a = mkarr(); for (i = 0; i < R_NCE; i++) apush(a, mknum(R_CE[i])) + oset(o, "constrained_eigenvalues", a) + oset(o, "constrained_inertia", mknum(R_con)) + oset(o, "df_conditioned", mkint(R_q)) + oset(o, "df_model", mkint(R_k)) + oset(o, "df_residual", mkint(R_dfres)) + oset(o, "f_statistic", mknum(R_f)) + oset(o, "r2", mknum(R_r2)) + oset(o, "total_inertia", mknum(R_tot)) + a = mkarr(); for (i = 0; i < R_NUE; i++) apush(a, mknum(R_UE[i])) + oset(o, "unconstrained_eigenvalues", a) + oset(o, "unconstrained_inertia", mknum(R_unc)) + return o +} + +function vpnode( o, k) { + o = mkobj() + for (k in VP) oset(o, k, mknum(VP[k])) + return o +} + +# Fill M from rows "a b c;d e f". Sets MN, MP. Flags all zero (floats). +function parse_rows(str, M, MI, rows, cols, nr, nc, i, j) { + delete M; delete MI + nr = split(str, rows, ";") + for (i = 1; i <= nr; i++) { + nc = split(rows[i], cols, " ") + for (j = 1; j <= nc; j++) { M[i - 1, j - 1] = cols[j] + 0; MI[i - 1, j - 1] = 0 } + } + MN = nr; MP = nc +} + +# Column vector from "a b c ...". +function parse_col(str, M, MI, cols, n, i) { + delete M; delete MI + n = split(str, cols, " ") + for (i = 1; i <= n; i++) { M[i - 1, 0] = cols[i] + 0; MI[i - 1, 0] = 0 } + MN = n; MP = 1 +} + +function hcat(A, n, pa, B, pb, OUT, i, j) { + for (i = 0; i < n; i++) { + for (j = 0; j < pa; j++) OUT[i, j] = A[i, j] + for (j = 0; j < pb; j++) OUT[i, pa + j] = B[i, j] + } +} + +function build_reference_document( doc, dss, ds, n, p, i, j) { + # 1. Ecological benchmark: first 10 sites x 8 species of the vegan dune data + parse_rows("1 0 0 0 0 0 0 0;3 0 0 2 0 3 4 0;0 4 0 7 0 2 0 0;0 8 0 2 0 2 3 0;2 0 0 0 4 2 2 0;2 0 0 0 3 0 0 0;2 0 0 0 2 0 2 0;0 4 0 5 0 0 0 0;0 3 0 3 0 0 0 1;4 0 0 0 4 2 4 0", DY, DYI) + DN = MN; DP = MP + parse_col("2.8 3.5 4.3 4.2 6.3 4.3 2.8 4.2 3.7 3.3", DA, DAI) + parse_col("1 1 2 2 1 1 1 5 4 2", DM, DMI) + hcat(DA, DN, 1, DM, 1, DX) + # 2. Gut amplicon counts, CLR (Aitchison geometry) + parse_rows("420 310 120 85 45 20;395 330 115 90 48 22;450 290 110 80 50 20;410 305 130 82 49 24;140 160 290 210 130 70;125 175 310 195 125 70;155 150 280 215 135 65;135 165 305 205 120 70;75 95 220 270 210 130;80 90 210 280 215 125;70 100 230 265 205 130;85 88 215 275 212 125", GC, GCI) + GN = MN; GP = MP + clr(GC, GN, GP, GY) + parse_col("0 0 0 0 1 1 1 1 1 1 1 1", GA, GAI) + parse_col("28 26 31 29 18 19 17 20 9 8 10 9.5", GF, GFI) + hcat(GA, GN, 1, GF, 1, GX) + # 3. Oral microbiome, CLR with Condition(batch) partial RDA + parse_rows("500 250 120 70 40 20;480 270 115 72 42 21;520 240 110 68 41 21;190 180 260 190 110 70;205 175 250 195 108 67;185 185 268 188 106 68;310 380 150 85 50 25;295 395 155 80 52 23;320 370 148 86 49 27;110 240 310 185 95 60;118 232 305 190 97 58;105 248 315 180 92 60", OC, OCI) + ON = MN; OP = MP + clr(OC, ON, OP, OY) + parse_col("0 0 0 1 1 1 0 0 0 1 1 1", OS, OSI) + parse_col("0 0 0 0 0 0 1 1 1 1 1 1", OB, OBI) + # 4. Marine Hellinger community across salinity, depth and interaction + parse_rows("610 220 90 40 20 12 8;580 245 95 38 22 11 9;430 310 140 65 30 15 10;390 330 155 70 28 16 11;240 290 230 125 65 30 20;210 275 250 140 70 33 22;130 195 270 195 115 58 37;115 180 280 205 120 62 38;65 110 240 255 175 95 60;55 102 232 265 182 100 64", MC2, MC2I) + MNN = MN; MPP = MP + hellinger(MC2, MC2I, MNN, MPP, MY2) + parse_col("30.1 30.4 31.8 32.1 33.5 33.9 34.8 35.0 35.7 36.0", MS, MSI) + parse_col("5 8 15 18 35 42 75 85 140 160", MD, MDI) + for (i = 0; i < MNN; i++) { MX3[i, 0] = MS[i, 0]; MX3[i, 1] = MD[i, 0]; MX3[i, 2] = MS[i, 0] * MD[i, 0] } + for (i = 0; i < MNN; i++) for (j = 0; j < 3; j++) MX3I[i, j] = 0 + + doc = mkobj() + dss = mkarr() + + ds = mkobj() + oset(ds, "description", mkstr("vegan dune meadow 10-site x 8-species benchmark with A1 and Moisture")) + rda(DY, DYI, DN, DP, DX, DYI, 2, NOZ, NOZ, 0, 0); oset(ds, "expected_rda", rdanode()) + varpart(DY, DYI, DN, DP, DA, DAI, 1, DM, DMI, 1); oset(ds, "expected_varpart", vpnode()) + oset(ds, "id", mkstr("dune_meadow_subset")) + oset(ds, "x", matnode(DX, DN, 2, 10)) + oset(ds, "x1", matnode(DA, DN, 1, 0)) + oset(ds, "x2", matnode(DM, DN, 1, 0)) + oset(ds, "y", matnode(DY, DN, DP, 0)) + oset(ds, "z", mknull()) + apush(dss, ds) + + ds = mkobj() + oset(ds, "description", mkstr("Gut microbiome 12-sample x 6-taxon CLR (Aitchison) ordination across antibiotic exposure and fibre")) + rda(GY, GCI, GN, GP, GX, GCI, 2, NOZ, NOZ, 0, 0); oset(ds, "expected_rda", rdanode()) + varpart(GY, GCI, GN, GP, GA, GAI, 1, GF, GFI, 1); oset(ds, "expected_varpart", vpnode()) + oset(ds, "id", mkstr("microbiome_gut_clr_antibiotic")) + oset(ds, "raw_counts", matnode(GC, GN, GP, 0)) + oset(ds, "transform", mkstr("clr")) + oset(ds, "x", matnode(GX, GN, 2, 0)) + oset(ds, "x1", matnode(GA, GN, 1, 0)) + oset(ds, "x2", matnode(GF, GN, 1, 0)) + oset(ds, "y", matnode(GY, GN, GP, 12)) + oset(ds, "z", mknull()) + apush(dss, ds) + + ds = mkobj() + oset(ds, "description", mkstr("Oral microbiome 12-sample x 6-taxon partial CLR RDA with Condition(batch)")) + rda(OY, OCI, ON, OP, OS, OSI, 1, OB, OBI, 1, 1); oset(ds, "expected_rda", rdanode()) + oset(ds, "expected_varpart", mknull()) + oset(ds, "id", mkstr("microbiome_oral_partial_batch")) + oset(ds, "raw_counts", matnode(OC, ON, OP, 0)) + oset(ds, "transform", mkstr("clr")) + oset(ds, "x", matnode(OS, ON, 1, 0)) + oset(ds, "y", matnode(OY, ON, OP, 12)) + oset(ds, "z", matnode(OB, ON, 1, 0)) + apush(dss, ds) + + ds = mkobj() + oset(ds, "description", mkstr("Marine microbiome 10-sample x 7-taxon Hellinger RDA with salinity, depth, and interaction")) + rda(MY2, MC2I, MNN, MPP, MX3, MX3I, 3, NOZ, NOZ, 0, 0); oset(ds, "expected_rda", rdanode()) + varpart(MY2, MC2I, MNN, MPP, MS, MSI, 1, MD, MDI, 1); oset(ds, "expected_varpart", vpnode()) + oset(ds, "id", mkstr("microbiome_marine_salinity_depth")) + oset(ds, "raw_counts", matnode(MC2, MNN, MPP, 0)) + oset(ds, "transform", mkstr("hellinger")) + oset(ds, "x", matnode(MX3, MNN, 3, 10)) + oset(ds, "x1", matnode(MS, MNN, 1, 0)) + oset(ds, "x2", matnode(MD, MNN, 1, 0)) + oset(ds, "y", matnode(MY2, MNN, MPP, 12)) + oset(ds, "z", mknull()) + apush(dss, ds) + + oset(doc, "datasets", dss) + oset(doc, "schema_version", mkstr("1.0.0")) + oset(doc, "tolerance", mknum(0.000001)) + return doc +} + +# The schema source is plain JSON read from ORD_SCHEMA_SRC. Its key order does +# not matter: the JCS serialiser sorts it. +function build_schema_document( text) { + text = slurp(ENVIRON["ORD_SCHEMA_SRC"]) + return parse_json(text) +} + +function write_file(path, s) { + printf "%s\n", s > path + close(path) +} + +function do_generate( doc, sch) { + sch = build_schema_document() + write_file(ENVIRON["ORD_SCHEMA_OUT"], jcs(sch)) + doc = build_reference_document() + write_file(ENVIRON["ORD_FIXTURE_OUT"], jcs(doc)) +} + +function do_check( fix, nds) { + check_oracle(ENVIRON["ORD_ORACLE"]) + verify_kyaml(ENVIRON["ORD_KYAML"]) + verify_jcs_file(ENVIRON["ORD_SCHEMA"], ENVIRON["ORD_SCHEMA_LF"] == "1") + fix = verify_jcs_file(ENVIRON["ORD_FIXTURE"], ENVIRON["ORD_FIXTURE_LF"] == "1") + nds = verify_fixture_math(fix) + printf "OK: verified %d ordination reference datasets, JCS schema (%s), and KYAML contract (%s).\n", nds, ENVIRON["ORD_SCHEMA_NAME"], ENVIRON["ORD_KYAML_NAME"] +} + +BEGIN { + init_tables() + if (ENVIRON["ORD_LIB"] == "") { + if (ENVIRON["ORD_STAGE"] == "generate") do_generate() + else do_check() + } +} diff --git a/test/references/check_ordination_references.py b/test/references/check_ordination_references.py deleted file mode 100755 index 8cdedaae..00000000 --- a/test/references/check_ordination_references.py +++ /dev/null @@ -1,868 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MPL-2.0 -# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) -""" -Independent Python stdlib reference oracle for Issue #19 constrained ordinations -(RDA, partial RDA with Condition(...), CLR/Aitchison and Hellinger geometries, -Ezekiel adjusted-R2, Jacobi eigenvalues, and Peres-Neto two-way variance -partitioning), plus RFC 7493 I-JSON / RFC 8785 JCS canonical verification for -the constrained-ordination schema and reference fixture. - -Zero third-party dependencies: uses only Python 3 standard library modules. -""" - -from __future__ import annotations - -import argparse -import json -import math -import pathlib -import re -import sys -from typing import Any - -SAFE_INT_MAX = (1 << 53) - 1 -REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] -DEFAULT_FIXTURE = REPO_ROOT / "test" / "fixtures" / "ordination_reference.json" -DEFAULT_SCHEMA = REPO_ROOT / "docs" / "schemas" / "constrained-ordination.schema.json" -DEFAULT_KYAML = REPO_ROOT / "config" / "defaults" / "ordination.kyaml" - - -def _utf16_units(s: str) -> tuple[int, ...]: - raw = s.encode("utf-16-be") - return tuple((raw[i] << 8) | raw[i + 1] for i in range(0, len(raw), 2)) - - -def _check_ijson_str(s: str) -> None: - for ch in s: - cp = ord(ch) - if 0xD800 <= cp <= 0xDFFF: - raise ValueError(f"Lone surrogate U+{cp:04X} forbidden in I-JSON") - if 0xFDD0 <= cp <= 0xFDEF or (cp & 0xFFFE) == 0xFFFE: - raise ValueError(f"Unicode noncharacter U+{cp:04X} forbidden in I-JSON") - - -def _jcs_escape_str(s: str) -> str: - _check_ijson_str(s) - out = ['"'] - for ch in s: - cp = ord(ch) - if ch == '"': - out.append('\\"') - elif ch == "\\": - out.append("\\\\") - elif cp == 0x08: - out.append("\\b") - elif cp == 0x09: - out.append("\\t") - elif cp == 0x0A: - out.append("\\n") - elif cp == 0x0C: - out.append("\\f") - elif cp == 0x0D: - out.append("\\r") - elif cp < 0x20: - out.append(f"\\u{cp:04x}") - else: - out.append(ch) - out.append('"') - return "".join(out) - - -def _jcs_format_float(x: float) -> str: - if not math.isfinite(x): - raise ValueError("Non-finite float forbidden in I-JSON/JCS") - if x == 0.0: - return "0" - neg = x < 0.0 - ax = -x if neg else x - # Python's repr(float) uses David Gay / Ryū shortest round-trip digits - s = repr(ax) - if "e" in s or "E" in s: - mant, exp_s = s.lower().split("e") - digits = mant.replace(".", "") - exp10 = int(exp_s) - else: - if "." in s: - int_p, frac_p = s.split(".") - if int_p == "0": - stripped = frac_p.lstrip("0") - zeros = len(frac_p) - len(stripped) - digits = stripped.rstrip("0") - exp10 = -(zeros + 1) - else: - digits = (int_p + frac_p).rstrip("0") - exp10 = len(int_p) - 1 - else: - digits = s.rstrip("0") - exp10 = len(s) - 1 - - k = len(digits) - n = exp10 + 1 # ECMAScript Number::toString n (10^(n-1) <= ax < 10^n) - if k <= n <= 21: - body = digits + ("0" * (n - k)) - elif 0 < n <= 21: - body = digits[:n] + "." + digits[n:] - elif -5 <= n <= 0: - body = "0." + ("0" * (-n)) + digits - else: - sign_e = "+" if (n - 1) >= 0 else "-" - mant_e = digits if k == 1 else (digits[0] + "." + digits[1:]) - body = f"{mant_e}e{sign_e}{abs(n - 1)}" - return ("-" + body) if neg else body - - -def jcs_dumps(val: Any) -> str: - if val is None: - return "null" - if val is True: - return "true" - if val is False: - return "false" - if isinstance(val, int): - if not (-SAFE_INT_MAX <= val <= SAFE_INT_MAX): - raise ValueError(f"Integer {val} outside I-JSON safe range") - return str(val) - if isinstance(val, float): - return _jcs_format_float(val) - if isinstance(val, str): - return _jcs_escape_str(val) - if isinstance(val, (list, tuple)): - return "[" + ",".join(jcs_dumps(item) for item in val) + "]" - if isinstance(val, dict): - items: list[tuple[tuple[int, ...], str, Any]] = [] - for k, v in val.items(): - if not isinstance(k, str): - raise ValueError("JSON object keys must be strings") - items.append((_utf16_units(k), k, v)) - items.sort(key=lambda t: t[0]) - return "{" + ",".join(f"{_jcs_escape_str(k)}:{jcs_dumps(v)}" for _, k, v in items) + "}" - raise TypeError(f"Unsupported type for JCS: {type(val)!r}") - - -# --- Pure-Python statistical reference engine --- - -def _centre_cols(mat: list[list[float]]) -> list[list[float]]: - n = len(mat) - p = len(mat[0]) - means = [sum(mat[i][j] for i in range(n)) / n for j in range(p)] - return [[mat[i][j] - means[j] for j in range(p)] for i in range(n)] - - -def _mgs_orthonormal(mat: list[list[float]], tol: float = 1e-11) -> list[list[float]]: - """Return orthonormal basis columns (as list of column vectors of length n).""" - n = len(mat) - m = len(mat[0]) - basis: list[list[float]] = [] - for j in range(m): - v = [mat[i][j] for i in range(n)] - for _ in range(2): - for q in basis: - proj = sum(q[i] * v[i] for i in range(n)) - for i in range(n): - v[i] -= proj * q[i] - norm = math.sqrt(sum(x * x for x in v)) - if norm > tol: - basis.append([x / norm for x in v]) - return basis - - -def _project(basis: list[list[float]], mat: list[list[float]]) -> list[list[float]]: - n = len(mat) - p = len(mat[0]) - out = [[0.0] * p for _ in range(n)] - for q in basis: - for j in range(p): - dot = sum(q[i] * mat[i][j] for i in range(n)) - for i in range(n): - out[i][j] += q[i] * dot - return out - - -def _sub_mat(a: list[list[float]], b: list[list[float]]) -> list[list[float]]: - return [[a[i][j] - b[i][j] for j in range(len(a[0]))] for i in range(len(a))] - - -def _frob_sq(mat: list[list[float]]) -> float: - return sum(x * x for row in mat for x in row) - - -def _jacobi_sym_eigenvalues(sym: list[list[float]], tol: float = 1e-13) -> list[float]: - m = len(sym) - if m == 0: - return [] - if m == 1: - return [sym[0][0]] if sym[0][0] > tol else [] - a = [[0.5 * (sym[i][j] + sym[j][i]) for j in range(m)] for i in range(m)] - for _ in range(90): - max_off = 0.0 - for p in range(m - 1): - for q in range(p + 1, m): - if abs(a[p][q]) > max_off: - max_off = abs(a[p][q]) - if max_off <= tol: - break - for p in range(m - 1): - for q in range(p + 1, m): - apq = a[p][q] - if abs(apq) <= tol: - continue - tau = (a[q][q] - a[p][p]) / (2.0 * apq) - t = (1.0 / (tau + math.sqrt(1.0 + tau * tau))) if tau >= 0.0 else (-1.0 / (-tau + math.sqrt(1.0 + tau * tau))) - c = 1.0 / math.sqrt(1.0 + t * t) - s = t * c - app = a[p][p] - aqq = a[q][q] - a[p][p] = app - t * apq - a[q][q] = aqq + t * apq - a[p][q] = 0.0 - a[q][p] = 0.0 - for r in range(m): - if r == p or r == q: - continue - arp = a[r][p] - arq = a[r][q] - a[r][p] = c * arp - s * arq - a[p][r] = a[r][p] - a[r][q] = s * arp + c * arq - a[q][r] = a[r][q] - vals = sorted((a[i][i] for i in range(m)), reverse=True) - cutoff = max(tol, 1e-12 * max(1.0, vals[0])) if vals else tol - return [v for v in vals if v > cutoff] - - -def _gram_eigenvalues(mat: list[list[float]], dof: float) -> list[float]: - n = len(mat) - p = len(mat[0]) - if n <= p: - g = [[sum(mat[i][k] * mat[j][k] for k in range(p)) / dof for j in range(n)] for i in range(n)] - else: - g = [[sum(mat[k][i] * mat[k][j] for k in range(n)) / dof for j in range(p)] for i in range(p)] - return _jacobi_sym_eigenvalues(g) - - -def ezekiel_adj_r2(r2: float, n: int, m: int) -> float: - return 1.0 - (1.0 - r2) * (n - 1) / (n - 1 - m) - - -def clr_transform(counts: list[list[float]]) -> list[list[float]]: - out: list[list[float]] = [] - for row in counts: - logs = [math.log(v) for v in row] - mean_log = sum(logs) / len(logs) - out.append([lv - mean_log for lv in logs]) - return out - - -def hellinger_transform(counts: list[list[float]]) -> list[list[float]]: - out: list[list[float]] = [] - for row in counts: - tot = sum(row) - out.append([math.sqrt(v / tot) for v in row]) - return out - - -def reference_linear_rda( - y: list[list[float]], - x: list[list[float]], - z: list[list[float]] | None = None, -) -> dict[str, Any]: - n = len(y) - dof = float(n - 1) - yc = _centre_cols(y) - xc = _centre_cols(x) - tot_inertia = _frob_sq(yc) / dof - - cond_inertia = 0.0 - q_rank = 0 - y_work = [row[:] for row in yc] - x_work = [row[:] for row in xc] - if z is not None: - zc = _centre_cols(z) - qz = _mgs_orthonormal(zc) - q_rank = len(qz) - if q_rank > 0: - yz = _project(qz, y_work) - cond_inertia = _frob_sq(yz) / dof - y_work = _sub_mat(y_work, yz) - xz = _project(qz, x_work) - x_work = _sub_mat(x_work, xz) - - qx = _mgs_orthonormal(x_work) - k_rank = len(qx) - df_res = n - 1 - q_rank - k_rank - y_hat = _project(qx, y_work) - y_res = _sub_mat(y_work, y_hat) - con_inertia = _frob_sq(y_hat) / dof - unc_inertia = _frob_sq(y_res) / dof - r2 = con_inertia / tot_inertia - if q_rank == 0: - adj_r2 = ezekiel_adj_r2(r2, n, k_rank) - else: - r2_xz = (con_inertia + cond_inertia) / tot_inertia - r2_z = cond_inertia / tot_inertia - adj_r2 = ezekiel_adj_r2(r2_xz, n, k_rank + q_rank) - ezekiel_adj_r2(r2_z, n, q_rank) - f_stat = (con_inertia / k_rank) / (unc_inertia / df_res) - con_eigs = [round(v, 12) for v in _gram_eigenvalues(y_hat, dof)] - unc_eigs = [round(v, 12) for v in _gram_eigenvalues(y_res, dof)] - - return { - "adj_r2": round(adj_r2, 12), - "conditioned_inertia": round(cond_inertia, 12), - "constrained_eigenvalues": con_eigs, - "constrained_inertia": round(con_inertia, 12), - "df_conditioned": q_rank, - "df_model": k_rank, - "df_residual": df_res, - "f_statistic": round(f_stat, 12), - "r2": round(r2, 12), - "total_inertia": round(tot_inertia, 12), - "unconstrained_eigenvalues": unc_eigs, - "unconstrained_inertia": round(unc_inertia, 12), - } - - -def reference_varpart_twoway( - y: list[list[float]], - x1: list[list[float]], - x2: list[list[float]], -) -> dict[str, float]: - r_a = reference_linear_rda(y, x1)["adj_r2"] - r_b = reference_linear_rda(y, x2)["adj_r2"] - x12 = [r1 + r2 for r1, r2 in zip(x1, x2)] - r_ab = reference_linear_rda(y, x12)["adj_r2"] - uniq_x1 = r_ab - r_b - uniq_x2 = r_ab - r_a - shared = r_a + r_b - r_ab - resid = 1.0 - r_ab - return { - "gross_x1_adj_r2": round(r_a, 12), - "gross_x12_adj_r2": round(r_ab, 12), - "gross_x2_adj_r2": round(r_b, 12), - "residual": round(resid, 12), - "shared": round(shared, 12), - "unique_x1": round(uniq_x1, 12), - "unique_x2": round(uniq_x2, 12), - } - - -def build_reference_document() -> dict[str, Any]: - # 1. Ecological benchmark: first 10 sites x 8 species of vegan's dune meadow dataset - dune_y = [ - [1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], - [3.0, 0.0, 0.0, 2.0, 0.0, 3.0, 4.0, 0.0], - [0.0, 4.0, 0.0, 7.0, 0.0, 2.0, 0.0, 0.0], - [0.0, 8.0, 0.0, 2.0, 0.0, 2.0, 3.0, 0.0], - [2.0, 0.0, 0.0, 0.0, 4.0, 2.0, 2.0, 0.0], - [2.0, 0.0, 0.0, 0.0, 3.0, 0.0, 0.0, 0.0], - [2.0, 0.0, 0.0, 0.0, 2.0, 0.0, 2.0, 0.0], - [0.0, 4.0, 0.0, 5.0, 0.0, 0.0, 0.0, 0.0], - [0.0, 3.0, 0.0, 3.0, 0.0, 0.0, 0.0, 1.0], - [4.0, 0.0, 0.0, 0.0, 4.0, 2.0, 4.0, 0.0], - ] - dune_a1 = [[2.8], [3.5], [4.3], [4.2], [6.3], [4.3], [2.8], [4.2], [3.7], [3.3]] - dune_moist = [[1.0], [1.0], [2.0], [2.0], [1.0], [1.0], [1.0], [5.0], [4.0], [2.0]] - dune_x = [[a[0], m[0]] for a, m in zip(dune_a1, dune_moist)] - - # 2. Microbiome dataset 1: Gut amplicon counts -> CLR (Aitchison geometry) - gut_counts = [ - [420.0, 310.0, 120.0, 85.0, 45.0, 20.0], - [395.0, 330.0, 115.0, 90.0, 48.0, 22.0], - [450.0, 290.0, 110.0, 80.0, 50.0, 20.0], - [410.0, 305.0, 130.0, 82.0, 49.0, 24.0], - [140.0, 160.0, 290.0, 210.0, 130.0, 70.0], - [125.0, 175.0, 310.0, 195.0, 125.0, 70.0], - [155.0, 150.0, 280.0, 215.0, 135.0, 65.0], - [135.0, 165.0, 305.0, 205.0, 120.0, 70.0], - [75.0, 95.0, 220.0, 270.0, 210.0, 130.0], - [80.0, 90.0, 210.0, 280.0, 215.0, 125.0], - [70.0, 100.0, 230.0, 265.0, 205.0, 130.0], - [85.0, 88.0, 215.0, 275.0, 212.0, 125.0], - ] - gut_y = clr_transform(gut_counts) - gut_abx = [[0.0], [0.0], [0.0], [0.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0]] - gut_fibre = [[28.0], [26.0], [31.0], [29.0], [18.0], [19.0], [17.0], [20.0], [9.0], [8.0], [10.0], [9.5]] - gut_x = [[a[0], f[0]] for a, f in zip(gut_abx, gut_fibre)] - - # 3. Microbiome dataset 2: Oral microbiome CLR with Condition(batch) partial RDA - oral_counts = [ - [500.0, 250.0, 120.0, 70.0, 40.0, 20.0], - [480.0, 270.0, 115.0, 72.0, 42.0, 21.0], - [520.0, 240.0, 110.0, 68.0, 41.0, 21.0], - [190.0, 180.0, 260.0, 190.0, 110.0, 70.0], - [205.0, 175.0, 250.0, 195.0, 108.0, 67.0], - [185.0, 185.0, 268.0, 188.0, 106.0, 68.0], - [310.0, 380.0, 150.0, 85.0, 50.0, 25.0], - [295.0, 395.0, 155.0, 80.0, 52.0, 23.0], - [320.0, 370.0, 148.0, 86.0, 49.0, 27.0], - [110.0, 240.0, 310.0, 185.0, 95.0, 60.0], - [118.0, 232.0, 305.0, 190.0, 97.0, 58.0], - [105.0, 248.0, 315.0, 180.0, 92.0, 60.0], - ] - oral_y = clr_transform(oral_counts) - oral_status = [[0.0], [0.0], [0.0], [1.0], [1.0], [1.0], [0.0], [0.0], [0.0], [1.0], [1.0], [1.0]] - oral_batch = [[0.0], [0.0], [0.0], [0.0], [0.0], [0.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0]] - - # 4. Microbiome dataset 3: Marine Hellinger-transformed community across salinity, depth & interaction - marine_counts = [ - [610.0, 220.0, 90.0, 40.0, 20.0, 12.0, 8.0], - [580.0, 245.0, 95.0, 38.0, 22.0, 11.0, 9.0], - [430.0, 310.0, 140.0, 65.0, 30.0, 15.0, 10.0], - [390.0, 330.0, 155.0, 70.0, 28.0, 16.0, 11.0], - [240.0, 290.0, 230.0, 125.0, 65.0, 30.0, 20.0], - [210.0, 275.0, 250.0, 140.0, 70.0, 33.0, 22.0], - [130.0, 195.0, 270.0, 195.0, 115.0, 58.0, 37.0], - [115.0, 180.0, 280.0, 205.0, 120.0, 62.0, 38.0], - [65.0, 110.0, 240.0, 255.0, 175.0, 95.0, 60.0], - [55.0, 102.0, 232.0, 265.0, 182.0, 100.0, 64.0], - ] - marine_y = hellinger_transform(marine_counts) - marine_sal = [[30.1], [30.4], [31.8], [32.1], [33.5], [33.9], [34.8], [35.0], [35.7], [36.0]] - marine_dep = [[5.0], [8.0], [15.0], [18.0], [35.0], [42.0], [75.0], [85.0], [140.0], [160.0]] - marine_x = [[s[0], d[0], s[0] * d[0]] for s, d in zip(marine_sal, marine_dep)] - - return { - "datasets": [ - { - "description": "vegan dune meadow 10-site x 8-species benchmark with A1 and Moisture", - "expected_rda": reference_linear_rda(dune_y, dune_x), - "expected_varpart": reference_varpart_twoway(dune_y, dune_a1, dune_moist), - "id": "dune_meadow_subset", - "x": [[round(v, 10) for v in row] for row in dune_x], - "x1": dune_a1, - "x2": dune_moist, - "y": dune_y, - "z": None, - }, - { - "description": "Gut microbiome 12-sample x 6-taxon CLR (Aitchison) ordination across antibiotic exposure and fibre", - "expected_rda": reference_linear_rda(gut_y, gut_x), - "expected_varpart": reference_varpart_twoway(gut_y, gut_abx, gut_fibre), - "id": "microbiome_gut_clr_antibiotic", - "raw_counts": gut_counts, - "transform": "clr", - "x": gut_x, - "x1": gut_abx, - "x2": gut_fibre, - "y": [[round(v, 12) for v in row] for row in gut_y], - "z": None, - }, - { - "description": "Oral microbiome 12-sample x 6-taxon partial CLR RDA with Condition(batch)", - "expected_rda": reference_linear_rda(oral_y, oral_status, oral_batch), - "expected_varpart": None, - "id": "microbiome_oral_partial_batch", - "raw_counts": oral_counts, - "transform": "clr", - "x": oral_status, - "y": [[round(v, 12) for v in row] for row in oral_y], - "z": oral_batch, - }, - { - "description": "Marine microbiome 10-sample x 7-taxon Hellinger RDA with salinity, depth, and interaction", - "expected_rda": reference_linear_rda(marine_y, marine_x), - "expected_varpart": reference_varpart_twoway(marine_y, marine_sal, marine_dep), - "id": "microbiome_marine_salinity_depth", - "raw_counts": marine_counts, - "transform": "hellinger", - "x": [[round(v, 10) for v in row] for row in marine_x], - "x1": marine_sal, - "x2": marine_dep, - "y": [[round(v, 12) for v in row] for row in marine_y], - "z": None, - }, - ], - "schema_version": "1.0.0", - "tolerance": 0.000001, - } - - -def build_schema_document() -> dict[str, Any]: - return { - "$id": "https://github.com/hyperpolymath/MetaManifold-WebUI/blob/main/docs/schemas/constrained-ordination.schema.json", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "additionalProperties": False, - "description": "Canonical RFC 7493 I-JSON / RFC 8785 JCS response schema for MetaManifold constrained ordinations (RDA, CCA, CAP, dbRDA).", - "properties": { - "adj_r2": {"type": ["number", "null"]}, - "artifact": { - "additionalProperties": False, - "properties": { - "analysis_id": {"pattern": "^[0-9a-f]{64}$", "type": "string"}, - "created": {"type": "boolean"}, - }, - "required": ["analysis_id", "created"], - "type": "object", - }, - "available_covariates": {"items": {"type": "string"}, "type": "array"}, - "axis_labels": {"items": {"type": "string"}, "maxItems": 2, "minItems": 2, "type": "array"}, - "axis_names": {"items": {"type": "string"}, "maxItems": 2, "minItems": 2, "type": "array"}, - "axis_tests": {"$ref": "#/$defs/termTestArray"}, - "biplot": { - "additionalProperties": False, - "properties": { - "data": {"items": {"type": "object"}, "type": "array"}, - "layout": {"type": "object"}, - }, - "required": ["data", "layout"], - "type": "object", - }, - "biplot_arrows": { - "items": { - "additionalProperties": False, - "properties": { - "axis1": {"type": "number"}, - "axis2": {"type": "number"}, - "label": {"type": "string"}, - "length": {"minimum": 0, "type": "number"}, - }, - "required": ["label", "axis1", "axis2", "length"], - "type": "object", - }, - "type": "array", - }, - "cautions": {"items": {"type": "string"}, "type": "array"}, - "centroids": { - "items": { - "additionalProperties": False, - "properties": { - "axis1": {"type": "number"}, - "axis2": {"type": "number"}, - "label": {"type": "string"}, - }, - "required": ["label", "axis1", "axis2"], - "type": "object", - }, - "type": "array", - }, - "condition_terms": {"items": {"type": "string"}, "type": "array"}, - "conditioned_inertia": {"minimum": 0, "type": "number"}, - "conditioned_proportion": {"minimum": 0, "type": "number"}, - "constrained_eigenvalues": {"$ref": "#/$defs/eigenvalueArray"}, - "constrained_inertia": {"minimum": 0, "type": "number"}, - "constrained_proportion": {"minimum": 0, "type": "number"}, - "constrained_terms": {"items": {"type": "string"}, "minItems": 1, "type": "array"}, - "distance": { - "enum": [ - "bray", - "jaccard", - "euclidean", - "aitchison", - "chisq", - "unweighted_unifrac", - "weighted_unifrac", - ], - "type": "string", - }, - "formula": {"minLength": 2, "type": "string"}, - "imaginary_inertia": {"minimum": 0, "type": "number"}, - "marginal_tests": {"$ref": "#/$defs/termTestArray"}, - "method": {"enum": ["rda", "cca", "cap", "dbrda"], "type": "string"}, - "n_features": {"minimum": 0, "type": "integer"}, - "n_samples": {"minimum": 3, "type": "integer"}, - "omnibus_test": { - "additionalProperties": False, - "properties": { - "df_model": {"minimum": 1, "type": "integer"}, - "df_residual": {"minimum": 1, "type": "integer"}, - "f_statistic": {"type": ["number", "null"]}, - "min_p_value": {"exclusiveMinimum": 0, "maximum": 1, "type": "number"}, - "p_value": {"type": ["number", "null"]}, - "permutations": {"maximum": 9999, "minimum": 1, "type": "integer"}, - "text": {"type": "string"}, - "untestable_reason": {"type": ["string", "null"]}, - "variance_model": {"type": ["number", "null"]}, - "variance_residual": {"type": ["number", "null"]}, - }, - "required": [ - "df_model", - "df_residual", - "variance_model", - "variance_residual", - "f_statistic", - "p_value", - "untestable_reason", - "permutations", - "min_p_value", - "text", - ], - "type": "object", - }, - "provenance": { - "additionalProperties": True, - "properties": { - "beta_normalisation": {"type": "string"}, - "blocked": {"type": "boolean"}, - "condition_terms": {"items": {"type": "string"}, "type": "array"}, - "constrained_terms": {"items": {"type": "string"}, "type": "array"}, - "distance": {"type": "string"}, - "dropped_zero_features": {"minimum": 0, "type": "integer"}, - "formula": {"type": "string"}, - "method": {"enum": ["rda", "cca", "cap", "dbrda"], "type": "string"}, - "multiple_testing_policy": {"type": "string"}, - "n_features": {"minimum": 0, "type": "integer"}, - "n_samples": {"minimum": 3, "type": "integer"}, - "permutations_performed": {"minimum": 1, "type": "integer"}, - "permutations_requested": {"maximum": 9999, "minimum": 99, "type": "integer"}, - "rng_kind": {"type": "string"}, - "scaling": {"enum": [1, 2], "type": "integer"}, - "scaling_meaning": {"enum": ["distance", "correlation"], "type": "string"}, - "seed": {"type": "integer"}, - "software": { - "additionalProperties": False, - "properties": { - "r_package": {"const": "vegan", "type": "string"}, - "vegan_version": {"type": "string"}, - }, - "required": ["r_package", "vegan_version"], - "type": "object", - }, - "zero_replacement": {"type": ["object", "null"]}, - }, - "required": [ - "method", - "distance", - "scaling", - "scaling_meaning", - "formula", - "permutations_requested", - "permutations_performed", - "seed", - "multiple_testing_policy", - "software", - ], - "type": "object", - }, - "r2": {"type": ["number", "null"]}, - "runtime_estimate": { - "additionalProperties": False, - "properties": { - "estimated_seconds": {"minimum": 0, "type": "number"}, - "tier": {"enum": ["fast", "moderate", "heavy"], "type": "string"}, - "warn_five_minutes": {"type": "boolean"}, - }, - "required": ["estimated_seconds", "tier", "warn_five_minutes"], - "type": "object", - }, - "scaling": {"enum": [1, 2], "type": "integer"}, - "sites": { - "items": { - "additionalProperties": False, - "properties": { - "axis1": {"type": "number"}, - "axis2": {"type": "number"}, - "sample": {"type": "string"}, - }, - "required": ["sample", "axis1", "axis2"], - "type": "object", - }, - "minItems": 3, - "type": "array", - }, - "term_tests": {"$ref": "#/$defs/termTestArray"}, - "top_feature_scores": {"items": {"type": "object"}, "type": "array"}, - "total_inertia": {"minimum": 0, "type": "number"}, - "unconstrained_eigenvalues": {"$ref": "#/$defs/eigenvalueArray"}, - "unconstrained_inertia": {"minimum": 0, "type": "number"}, - "unconstrained_proportion": {"minimum": 0, "type": "number"}, - "variance_partitioning": {"type": ["object", "null"]}, - }, - "required": [ - "method", - "distance", - "scaling", - "formula", - "constrained_terms", - "condition_terms", - "n_samples", - "n_features", - "total_inertia", - "constrained_inertia", - "unconstrained_inertia", - "r2", - "adj_r2", - "sites", - "biplot_arrows", - "omnibus_test", - "term_tests", - "provenance", - "biplot", - ], - "title": "ConstrainedOrdinationResult", - "type": "object", - "$defs": { - "eigenvalueArray": { - "items": { - "additionalProperties": False, - "properties": { - "axis": {"type": "string"}, - "eigenvalue": {"minimum": 0, "type": "number"}, - "proportion_constrained": {"minimum": 0, "type": "number"}, - "proportion_total": {"minimum": 0, "type": "number"}, - }, - "required": ["axis", "eigenvalue", "proportion_total"], - "type": "object", - }, - "type": "array", - }, - "termTestArray": { - "items": { - "additionalProperties": False, - "properties": { - "df": {"minimum": 0, "type": "integer"}, - "f_statistic": {"type": ["number", "null"]}, - "p_value": {"type": ["number", "null"]}, - "term": {"type": "string"}, - "untestable_reason": {"type": ["string", "null"]}, - "variance": {"type": ["number", "null"]}, - }, - "required": ["term", "df", "variance", "f_statistic", "p_value", "untestable_reason"], - "type": "object", - }, - "type": "array", - }, - }, - } - - -def verify_jcs_file(path: pathlib.Path) -> Any: - raw = path.read_bytes() - if not raw.endswith(b"\n") or raw.endswith(b"\n\n"): - raise AssertionError(f"{path}: must end with exactly one trailing LF") - text = raw[:-1].decode("utf-8") - parsed = json.loads(text) - canonical = jcs_dumps(parsed) - if text != canonical: - raise AssertionError(f"{path}: content is not byte-for-byte RFC 8785 JCS") - return parsed - - -def verify_kyaml_contract(path: pathlib.Path) -> None: - text = path.read_text(encoding="utf-8") - lines = text.splitlines() - body = [ln for ln in lines if ln.strip() and not ln.strip().startswith("#")] - if not body or body[0].strip() != "---": - raise AssertionError(f"{path}: KYAML must open with explicit '---' after SPDX comments") - if body[1].strip() != "{": - raise AssertionError(f"{path}: KYAML top-level mapping must use explicit '{{' flow syntax") - if body[-1].strip() != "}": - raise AssertionError(f"{path}: KYAML top-level mapping must close with '}}'") - for req_token in ("rda:", "cca:", "cap:", "dbrda:", '"capscale"', '"aitchison"', '"chisq"', "99", "9999"): - if req_token not in text: - raise AssertionError(f"{path}: missing expected contract token {req_token}") - # Check there are no bare unquoted string values after ':' - for idx, ln in enumerate(lines, start=1): - stripped = ln.strip() - if not stripped or stripped.startswith("#") or stripped in ("---", "{", "}", "},", "],"): - continue - m = re.match(r"^[A-Za-z0-9_\"']+:\s*(.+?)\s*,?$", stripped) - if m: - val = m.group(1).rstrip(",").strip() - if val in ("{", "[") or val.startswith('"') or val in ("true", "false", "null"): - continue - if re.fullmatch(r"-?[0-9]+(\.[0-9]+)?", val): - continue - raise AssertionError(f"{path}:{idx}: unquoted scalar value {val!r} violates KYAML") - - -def verify_fixture_math(doc: dict[str, Any], tol: float = 1e-9) -> None: - datasets = doc.get("datasets", []) - if len(datasets) < 4: - raise AssertionError(f"Expected >= 4 reference datasets, found {len(datasets)}") - for ds in datasets: - ds_id = ds["id"] - y = ds["y"] - if ds.get("transform") == "clr": - y_recomp = clr_transform(ds["raw_counts"]) - elif ds.get("transform") == "hellinger": - y_recomp = hellinger_transform(ds["raw_counts"]) - else: - y_recomp = y - for r1, r2 in zip(y, y_recomp): - for a, b in zip(r1, r2): - if abs(a - b) > 1e-10: - raise AssertionError(f"{ds_id}: stored transformed matrix drifted from raw_counts") - - actual_rda = reference_linear_rda(y_recomp, ds["x"], ds["z"]) - exp_rda = ds["expected_rda"] - for key in ( - "total_inertia", - "conditioned_inertia", - "constrained_inertia", - "unconstrained_inertia", - "r2", - "adj_r2", - "f_statistic", - ): - if abs(actual_rda[key] - exp_rda[key]) > tol: - raise AssertionError( - f"{ds_id}.{key}: expected {exp_rda[key]}, recomputed {actual_rda[key]}" - ) - for eig_key in ("constrained_eigenvalues", "unconstrained_eigenvalues"): - if len(actual_rda[eig_key]) != len(exp_rda[eig_key]): - raise AssertionError(f"{ds_id}.{eig_key}: length mismatch") - for a, b in zip(actual_rda[eig_key], exp_rda[eig_key]): - if abs(a - b) > tol: - raise AssertionError(f"{ds_id}.{eig_key}: {a} != {b}") - - # Verify inertia conservation: total == conditioned + constrained + unconstrained - inertia_sum = ( - actual_rda["conditioned_inertia"] - + actual_rda["constrained_inertia"] - + actual_rda["unconstrained_inertia"] - ) - if abs(actual_rda["total_inertia"] - inertia_sum) > 1e-10: - raise AssertionError(f"{ds_id}: inertia partition does not sum to total_inertia") - - if ds["expected_varpart"] is not None: - actual_vp = reference_varpart_twoway(y_recomp, ds["x1"], ds["x2"]) - exp_vp = ds["expected_varpart"] - for k, v in exp_vp.items(): - if abs(actual_vp[k] - v) > tol: - raise AssertionError(f"{ds_id}.varpart.{k}: expected {v}, got {actual_vp[k]}") - # Verify Peres-Neto identity: [a] + [b] + [c] + [d] == 1 - part_sum = ( - actual_vp["unique_x1"] - + actual_vp["unique_x2"] - + actual_vp["shared"] - + actual_vp["residual"] - ) - if abs(part_sum - 1.0) > 1e-10: - raise AssertionError(f"{ds_id}: varpart fractions sum to {part_sum}, expected 1.0") - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("--generate", action="store_true", help="Write canonical JCS schema and reference fixture") - ap.add_argument("--check", type=pathlib.Path, default=DEFAULT_FIXTURE, help="Fixture path to verify") - args = ap.parse_args() - - if args.generate: - DEFAULT_SCHEMA.parent.mkdir(parents=True, exist_ok=True) - DEFAULT_FIXTURE.parent.mkdir(parents=True, exist_ok=True) - DEFAULT_SCHEMA.write_bytes((jcs_dumps(build_schema_document()) + "\n").encode("utf-8")) - DEFAULT_FIXTURE.write_bytes((jcs_dumps(build_reference_document()) + "\n").encode("utf-8")) - - # Cross-check our JCS serializer against the repo's jcs_oracle.jsonl first - oracle_path = REPO_ROOT / "test" / "fixtures" / "jcs_oracle.jsonl" - if oracle_path.exists(): - for idx, line in enumerate(oracle_path.read_text(encoding="utf-8").splitlines(), start=1): - if not line.strip(): - continue - parsed = json.loads(line) - if jcs_dumps(parsed["input"]) != parsed["expected"]: - raise AssertionError(f"jcs_oracle.jsonl line {idx} mismatch") - - verify_kyaml_contract(DEFAULT_KYAML) - verify_jcs_file(DEFAULT_SCHEMA) - fixture_doc = verify_jcs_file(args.check) - verify_fixture_math(fixture_doc) - print( - f"OK: verified {len(fixture_doc['datasets'])} ordination reference datasets, " - f"JCS schema ({DEFAULT_SCHEMA.name}), and KYAML contract ({DEFAULT_KYAML.name})." - ) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/test/references/check_ordination_references.sh b/test/references/check_ordination_references.sh new file mode 100755 index 00000000..e92fc950 --- /dev/null +++ b/test/references/check_ordination_references.sh @@ -0,0 +1,682 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# Independent awk reference oracle for Issue #19 constrained ordinations. +# Bash driver for check_ordination_references.awk (a port of the former +# check_ordination_references.py). All the maths is hand-written in plain +# double precision awk and shares no code with Julia. +# +# Usage: +# check_ordination_references.sh [--check PATH] [--generate] +# +# --check PATH fixture to verify (default test/fixtures/ordination_reference.json) +# --generate first write the canonical JCS schema and fixture +# +# Environment: +# AWK awk program to use (default: awk) +# ORDINATION_GENERATE_DIR write generated files here instead of the repo +# (constrained-ordination.schema.json and +# ordination_reference.json); checks still read the +# repo files +# +# Exit: 0 verified, 1 a check failed, 2 usage error. +set -euo pipefail +export LC_ALL=C + +here="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +repo_root="$(cd -- "$here/../.." && pwd -P)" +awk_prog="$here/check_ordination_references.awk" +default_fixture="$repo_root/test/fixtures/ordination_reference.json" +schema="$repo_root/docs/schemas/constrained-ordination.schema.json" +kyaml="$repo_root/config/defaults/ordination.kyaml" +oracle="$repo_root/test/fixtures/jcs_oracle.jsonl" + +usage() { + echo "usage: check_ordination_references.sh [-h] [--generate] [--check PATH]" +} + +fixture="$default_fixture" +generate=0 +while (( $# > 0 )); do + case "$1" in + -h|--h|--he|--hel|--help) + usage + echo " --generate Write canonical JCS schema and reference fixture" + echo " --check PATH Fixture path to verify" + exit 0 ;; + --g|--ge|--gen|--gene|--gener|--genera|--generat|--generate) generate=1; shift ;; + --c|--ch|--che|--chec|--check) + if (( $# < 2 )); then echo "error: argument --check: expected one argument" >&2; usage >&2; exit 2; fi + fixture="$2"; shift 2 ;; + --c=*|--ch=*|--che=*|--chec=*|--check=*) fixture="${1#*=}"; shift ;; + *) echo "error: unrecognized arguments: $1" >&2; usage >&2; exit 2 ;; + esac +done + +AWK="${AWK:-awk}" +command -v "$AWK" >/dev/null || { echo "error: awk is required" >&2; exit 2; } + +# 1 when the file ends with exactly one LF (the JCS file rule), else 0. +lf_ok() { + local tail2 + [[ -f "$1" ]] || { echo 0; return; } + tail2="$(tail -c 2 -- "$1" | od -An -v -tx1 | tr -d ' \n')" + case "$tail2" in + *0a0a) echo 0 ;; + *0a) echo 1 ;; + *) echo 0 ;; + esac +} + +# The schema as plain JSON. Key order is irrelevant: the awk JCS serialiser +# sorts keys, so this is the content of build_schema_document(). +schema_source() { + cat <<'SCHEMA_JSON' +{ + "$id": "https://github.com/hyperpolymath/MetaManifold-WebUI/blob/main/docs/schemas/constrained-ordination.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "description": "Canonical RFC 7493 I-JSON / RFC 8785 JCS response schema for MetaManifold constrained ordinations (RDA, CCA, CAP, dbRDA).", + "properties": { + "adj_r2": { + "type": [ + "number", + "null" + ] + }, + "artifact": { + "additionalProperties": false, + "properties": { + "analysis_id": { + "pattern": "^[0-9a-f]{64}$", + "type": "string" + }, + "created": { + "type": "boolean" + } + }, + "required": [ + "analysis_id", + "created" + ], + "type": "object" + }, + "available_covariates": { + "items": { + "type": "string" + }, + "type": "array" + }, + "axis_labels": { + "items": { + "type": "string" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + "axis_names": { + "items": { + "type": "string" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + "axis_tests": { + "$ref": "#/$defs/termTestArray" + }, + "biplot": { + "additionalProperties": false, + "properties": { + "data": { + "items": { + "type": "object" + }, + "type": "array" + }, + "layout": { + "type": "object" + } + }, + "required": [ + "data", + "layout" + ], + "type": "object" + }, + "biplot_arrows": { + "items": { + "additionalProperties": false, + "properties": { + "axis1": { + "type": "number" + }, + "axis2": { + "type": "number" + }, + "label": { + "type": "string" + }, + "length": { + "minimum": 0, + "type": "number" + } + }, + "required": [ + "label", + "axis1", + "axis2", + "length" + ], + "type": "object" + }, + "type": "array" + }, + "cautions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "centroids": { + "items": { + "additionalProperties": false, + "properties": { + "axis1": { + "type": "number" + }, + "axis2": { + "type": "number" + }, + "label": { + "type": "string" + } + }, + "required": [ + "label", + "axis1", + "axis2" + ], + "type": "object" + }, + "type": "array" + }, + "condition_terms": { + "items": { + "type": "string" + }, + "type": "array" + }, + "conditioned_inertia": { + "minimum": 0, + "type": "number" + }, + "conditioned_proportion": { + "minimum": 0, + "type": "number" + }, + "constrained_eigenvalues": { + "$ref": "#/$defs/eigenvalueArray" + }, + "constrained_inertia": { + "minimum": 0, + "type": "number" + }, + "constrained_proportion": { + "minimum": 0, + "type": "number" + }, + "constrained_terms": { + "items": { + "type": "string" + }, + "minItems": 1, + "type": "array" + }, + "distance": { + "enum": [ + "bray", + "jaccard", + "euclidean", + "aitchison", + "chisq", + "unweighted_unifrac", + "weighted_unifrac" + ], + "type": "string" + }, + "formula": { + "minLength": 2, + "type": "string" + }, + "imaginary_inertia": { + "minimum": 0, + "type": "number" + }, + "marginal_tests": { + "$ref": "#/$defs/termTestArray" + }, + "method": { + "enum": [ + "rda", + "cca", + "cap", + "dbrda" + ], + "type": "string" + }, + "n_features": { + "minimum": 0, + "type": "integer" + }, + "n_samples": { + "minimum": 3, + "type": "integer" + }, + "omnibus_test": { + "additionalProperties": false, + "properties": { + "df_model": { + "minimum": 1, + "type": "integer" + }, + "df_residual": { + "minimum": 1, + "type": "integer" + }, + "f_statistic": { + "type": [ + "number", + "null" + ] + }, + "min_p_value": { + "exclusiveMinimum": 0, + "maximum": 1, + "type": "number" + }, + "p_value": { + "type": [ + "number", + "null" + ] + }, + "permutations": { + "maximum": 9999, + "minimum": 1, + "type": "integer" + }, + "text": { + "type": "string" + }, + "untestable_reason": { + "type": [ + "string", + "null" + ] + }, + "variance_model": { + "type": [ + "number", + "null" + ] + }, + "variance_residual": { + "type": [ + "number", + "null" + ] + } + }, + "required": [ + "df_model", + "df_residual", + "variance_model", + "variance_residual", + "f_statistic", + "p_value", + "untestable_reason", + "permutations", + "min_p_value", + "text" + ], + "type": "object" + }, + "provenance": { + "additionalProperties": true, + "properties": { + "beta_normalisation": { + "type": "string" + }, + "blocked": { + "type": "boolean" + }, + "condition_terms": { + "items": { + "type": "string" + }, + "type": "array" + }, + "constrained_terms": { + "items": { + "type": "string" + }, + "type": "array" + }, + "distance": { + "type": "string" + }, + "dropped_zero_features": { + "minimum": 0, + "type": "integer" + }, + "formula": { + "type": "string" + }, + "method": { + "enum": [ + "rda", + "cca", + "cap", + "dbrda" + ], + "type": "string" + }, + "multiple_testing_policy": { + "type": "string" + }, + "n_features": { + "minimum": 0, + "type": "integer" + }, + "n_samples": { + "minimum": 3, + "type": "integer" + }, + "permutations_performed": { + "minimum": 1, + "type": "integer" + }, + "permutations_requested": { + "maximum": 9999, + "minimum": 99, + "type": "integer" + }, + "rng_kind": { + "type": "string" + }, + "scaling": { + "enum": [ + 1, + 2 + ], + "type": "integer" + }, + "scaling_meaning": { + "enum": [ + "distance", + "correlation" + ], + "type": "string" + }, + "seed": { + "type": "integer" + }, + "software": { + "additionalProperties": false, + "properties": { + "r_package": { + "const": "vegan", + "type": "string" + }, + "vegan_version": { + "type": "string" + } + }, + "required": [ + "r_package", + "vegan_version" + ], + "type": "object" + }, + "zero_replacement": { + "type": [ + "object", + "null" + ] + } + }, + "required": [ + "method", + "distance", + "scaling", + "scaling_meaning", + "formula", + "permutations_requested", + "permutations_performed", + "seed", + "multiple_testing_policy", + "software" + ], + "type": "object" + }, + "r2": { + "type": [ + "number", + "null" + ] + }, + "runtime_estimate": { + "additionalProperties": false, + "properties": { + "estimated_seconds": { + "minimum": 0, + "type": "number" + }, + "tier": { + "enum": [ + "fast", + "moderate", + "heavy" + ], + "type": "string" + }, + "warn_five_minutes": { + "type": "boolean" + } + }, + "required": [ + "estimated_seconds", + "tier", + "warn_five_minutes" + ], + "type": "object" + }, + "scaling": { + "enum": [ + 1, + 2 + ], + "type": "integer" + }, + "sites": { + "items": { + "additionalProperties": false, + "properties": { + "axis1": { + "type": "number" + }, + "axis2": { + "type": "number" + }, + "sample": { + "type": "string" + } + }, + "required": [ + "sample", + "axis1", + "axis2" + ], + "type": "object" + }, + "minItems": 3, + "type": "array" + }, + "term_tests": { + "$ref": "#/$defs/termTestArray" + }, + "top_feature_scores": { + "items": { + "type": "object" + }, + "type": "array" + }, + "total_inertia": { + "minimum": 0, + "type": "number" + }, + "unconstrained_eigenvalues": { + "$ref": "#/$defs/eigenvalueArray" + }, + "unconstrained_inertia": { + "minimum": 0, + "type": "number" + }, + "unconstrained_proportion": { + "minimum": 0, + "type": "number" + }, + "variance_partitioning": { + "type": [ + "object", + "null" + ] + } + }, + "required": [ + "method", + "distance", + "scaling", + "formula", + "constrained_terms", + "condition_terms", + "n_samples", + "n_features", + "total_inertia", + "constrained_inertia", + "unconstrained_inertia", + "r2", + "adj_r2", + "sites", + "biplot_arrows", + "omnibus_test", + "term_tests", + "provenance", + "biplot" + ], + "title": "ConstrainedOrdinationResult", + "type": "object", + "$defs": { + "eigenvalueArray": { + "items": { + "additionalProperties": false, + "properties": { + "axis": { + "type": "string" + }, + "eigenvalue": { + "minimum": 0, + "type": "number" + }, + "proportion_constrained": { + "minimum": 0, + "type": "number" + }, + "proportion_total": { + "minimum": 0, + "type": "number" + } + }, + "required": [ + "axis", + "eigenvalue", + "proportion_total" + ], + "type": "object" + }, + "type": "array" + }, + "termTestArray": { + "items": { + "additionalProperties": false, + "properties": { + "df": { + "minimum": 0, + "type": "integer" + }, + "f_statistic": { + "type": [ + "number", + "null" + ] + }, + "p_value": { + "type": [ + "number", + "null" + ] + }, + "term": { + "type": "string" + }, + "untestable_reason": { + "type": [ + "string", + "null" + ] + }, + "variance": { + "type": [ + "number", + "null" + ] + } + }, + "required": [ + "term", + "df", + "variance", + "f_statistic", + "p_value", + "untestable_reason" + ], + "type": "object" + }, + "type": "array" + } + } +} +SCHEMA_JSON +} + +if (( generate )); then + out_schema="$schema" + out_fixture="$default_fixture" + if [[ -n "${ORDINATION_GENERATE_DIR:-}" ]]; then + out_schema="$ORDINATION_GENERATE_DIR/constrained-ordination.schema.json" + out_fixture="$ORDINATION_GENERATE_DIR/ordination_reference.json" + fi + mkdir -p -- "$(dirname -- "$out_schema")" "$(dirname -- "$out_fixture")" + env ORD_STAGE=generate ORD_SCHEMA_SRC=<(schema_source) \ + ORD_SCHEMA_OUT="$out_schema" ORD_FIXTURE_OUT="$out_fixture" \ + "$AWK" -f "$awk_prog" +fi + +ORD_STAGE=check \ +ORD_ORACLE="$oracle" ORD_KYAML="$kyaml" ORD_SCHEMA="$schema" ORD_FIXTURE="$fixture" \ +ORD_SCHEMA_LF="$(lf_ok "$schema")" ORD_FIXTURE_LF="$(lf_ok "$fixture")" \ +ORD_SCHEMA_NAME="$(basename -- "$schema")" ORD_KYAML_NAME="$(basename -- "$kyaml")" \ + "$AWK" -f "$awk_prog" diff --git a/test/references/exact_summary_reference.py b/test/references/exact_summary_reference.py deleted file mode 100644 index f40c3c26..00000000 --- a/test/references/exact_summary_reference.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 -"""Independent exact-summary reference using Python fractions.Fraction. - -Usage: python3 exact_summary_reference.py INPUT.json -Input/output counts are decimal strings; zero-total proportions are JSON null. -This program never imports or reads output from the Julia implementation. -""" -from __future__ import annotations - -from collections import OrderedDict -from fractions import Fraction -import json -from pathlib import Path -import sys - - -def proportion_text(count: int, total: int) -> str | None: - if total == 0: - return None - value = Fraction(count, total) - return f"{value.numerator}/{value.denominator}" - - -def summarise(label: str, columns: list[list[int]], features: list[str]) -> dict: - counts = [sum(column[i] for column in columns) for i in range(len(features))] - total = sum(counts) - return { - "label": label, - "total": str(total), - "counts": [str(count) for count in counts], - "proportions": [proportion_text(count, total) for count in counts], - } - - -def main() -> int: - if len(sys.argv) != 2: - raise SystemExit("usage: exact_summary_reference.py INPUT.json") - payload = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) - labels = payload["sample_labels"] - features = payload["feature_labels"] - rows = payload["counts"] - assert len(rows) == len(features) - columns = [[int(row[j]) for row in rows] for j in range(len(labels))] - result = { - "samples": [summarise(label, [column], features) - for label, column in zip(labels, columns)], - "groups": [], - } - groups = payload.get("groups") - if groups is not None: - assert len(groups) == len(labels) - by_group = OrderedDict() - for label, column in zip(groups, columns): - by_group.setdefault(label, []).append(column) - result["groups"] = [summarise(label, members, features) - for label, members in by_group.items()] - print(json.dumps(result, separators=(",", ":"))) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/test/references/exact_summary_reference.sh b/test/references/exact_summary_reference.sh new file mode 100755 index 00000000..6e487a3f --- /dev/null +++ b/test/references/exact_summary_reference.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# Independent exact-summary reference: integer sums and reduced fractions in GNU bc. +# +# Usage: exact_summary_reference.sh INPUT.json +# Input and output counts are decimal strings; zero-total proportions are JSON null. +# bc shares no code with the Julia implementation (GMP), and this program never +# reads output from it. jq only moves labels and digit strings around; every +# sum, gcd and division is done by bc. +set -euo pipefail + +[[ $# -eq 1 ]] || { echo "usage: exact_summary_reference.sh INPUT.json" >&2; exit 2; } +for tool in bc jq; do + command -v "$tool" >/dev/null || { echo "$tool is required (apt-get install $tool)" >&2; exit 2; } +done +input="$1" + +# One block per summary: "Ljson label", then one "Fvalues" line per +# feature holding that feature's value in every member column, then "E". +# Samples first, then groups in order of first appearance. +blocks() { + jq -r ' + . as $p + | ($p.counts | length) as $nf + | ($p.feature_labels | length) as $nl + | if $nf != $nl then error("counts rows do not match feature_labels") else . end + | def block($label; $cols): + "L\t" + ($label | @json), + (range(0; $nf) as $i | "F\t" + ([$cols[] as $j | $p.counts[$i][$j] | tostring] | join(" "))), + "E"; + ($p.sample_labels | length) as $ns + | (range(0; $ns) as $j | block($p.sample_labels[$j]; [$j])), + (if $p.groups == null then empty else + if ($p.groups | length) != $ns then error("groups do not match sample_labels") else . end + | ($p.groups | reduce .[] as $g ([]; if index([$g]) == null then . + [$g] else . end)) as $order + | ($order[] as $g | block($g; [range(0; $ns) | select($p.groups[.] == $g)])) + end) + ' "$input" +} + +# Print "total", the counts and the reduced proportions of one summary, one per +# line, from the value lists on stdin-style arguments. +summarise() { + local -a lists=("$@") + local program="define g(a, b) { auto t; if (a < 0) a = -a; if (b < 0) b = -b; while (b != 0) { t = a % b; a = b; b = t }; return (a) } +t = 0 +" + local i=0 value list + for list in "${lists[@]}"; do + program+="c[$i] = 0" + for value in $list; do + [[ "$value" =~ ^-?[0-9]+$ ]] || { echo "not a decimal integer: $value" >&2; return 2; } + program+=" + ($value)" + done + program+=$'\n'"t = t + c[$i]"$'\n' + i=$((i + 1)) + done + program+="t +" + for ((k = 0; k < i; k++)); do program+="c[$k]"$'\n'; done + for ((k = 0; k < i; k++)); do + program+="if (t == 0) { \"null"$'\n'"\"; \"null"$'\n'"\" } else { d = g(c[$k], t); n = c[$k] / d; m = t / d; if (m < 0) { n = -n; m = -m }; n; m }"$'\n' + done + # Each feature prints its numerator and denominator on two lines, or null twice. + printf '%s\nquit\n' "$program" | BC_LINE_LENGTH=0 bc +} + +emit() { + local label="$1"; shift + local -a lists=("$@") + local n=${#lists[@]} out + out="$(summarise "${lists[@]}")" || exit 2 + local -a lines + mapfile -t lines <<<"$out" + local total="${lines[0]}" counts="" props="" k pos + for ((k = 0; k < n; k++)); do + counts+="${counts:+,}\"${lines[1 + k]}\"" + done + pos=$((1 + n)) + for ((k = 0; k < n; k++)); do + if [[ "${lines[pos]}" == null ]]; then + props+="${props:+,}null" + else + props+="${props:+,}\"${lines[pos]}/${lines[pos + 1]}\"" + fi + pos=$((pos + 2)) + done + printf '{"label":%s,"total":"%s","counts":[%s],"proportions":[%s]}\n' "$label" "$total" "$counts" "$props" +} + +samples=() groups=() +section=samples +label="" ; lists=() +n_samples="$(jq '.sample_labels | length' "$input")" +count=0 +while IFS=$'\t' read -r tag rest; do + case "$tag" in + L) label="$rest"; lists=() ;; + F) lists+=("$rest") ;; + E) + line="$(emit "$label" "${lists[@]}")" + if (( count < n_samples )); then samples+=("$line"); else groups+=("$line"); fi + count=$((count + 1)) ;; + esac +done < <(blocks) + +join() { local IFS=,; echo "$*"; } +printf '{"samples":[%s],"groups":[%s]}\n' "$(join "${samples[@]}")" "$(join "${groups[@]}")" | jq -ac . diff --git a/test/references/high_precision_references.bc b/test/references/high_precision_references.bc new file mode 100644 index 00000000..f72a55af --- /dev/null +++ b/test/references/high_precision_references.bc @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: AGPL-3.0-only + * + * Independent high-precision known answers for test/unit/test_high_precision.jl. + * + * GNU bc shares no code with Julia (MPFR, GMP). Every operation is rounded to + * n significant digits, half to even, one operation at a time, which is how the + * Python decimal reference this replaces rounded, so the digit strings match. + * + * The driver sets n (significant digits) before loading this file and prints + * the four mantissas with their decimal exponents. + */ + +scale = n + 120 + +/* ln(x) to n + 40 digits; callers round the result. */ +define lg(x) { + auto s, v + s = scale + scale = n + 100 + v = l(x) + scale = s + return (v) +} + +/* Decimal exponent e of x > 0: 10^e <= x < 10^(e+1). */ +define ex(x) { + auto e + e = 0 + while (x >= 10) { x = x / 10; e = e + 1 } + while (x < 1) { x = x * 10; e = e - 1 } + return (e) +} + +/* x > 0 rounded to n significant digits, half to even. */ +define rn(x) { + auto e, y, i, f, s + if (x == 0) return (0) + if (x < 0) return (-rn(-x)) + s = scale + scale = n + 60 + e = ex(x) + y = x * 10 ^ (n - 1 - e) + scale = 0 + i = y / 1 + scale = n + 60 + f = y - i + if (f > 0.5) i = i + 1 + if (f == 0.5) { + /* % is exact only at scale 0, so the parity test is done there. */ + scale = 0 + if (i % 2 == 1) i = i + 1 + scale = n + 60 + } + y = i / 10 ^ (n - 1 - e) + scale = s + return (y) +} + +/* The mantissa integer of a value already rounded by rn(): n digits. */ +define mt(x) { + auto e, s, v + s = scale + scale = n + 60 + e = ex(x) + v = x * 10 ^ (n - 1 - e) + scale = 0 + v = v / 1 + scale = s + return (v) +} + +ln2 = rn(lg(2)) +ln3 = rn(lg(3)) +tt = rn(2 / 3) + +h = 0 +for (c = 1; c <= 4; c++) { + q = rn(c / 10) + t = rn(q * rn(lg(q))) + h = rn(h - t) +} + +"REF_LN2 "; mt(ln2); " +" +"EX_LN2 "; ex(ln2); " +" +"REF_LN3 "; mt(ln3); " +" +"EX_LN3 "; ex(ln3); " +" +"REF_TWO_THIRDS "; mt(tt); " +" +"EX_TWO_THIRDS "; ex(tt); " +" +"REF_H_1234 "; mt(h); " +" +"EX_H_1234 "; ex(h); " +" +quit diff --git a/test/unit/test_exact_summaries.jl b/test/unit/test_exact_summaries.jl index d78f23e3..c9c6347b 100644 --- a/test/unit/test_exact_summaries.jl +++ b/test/unit/test_exact_summaries.jl @@ -139,10 +139,9 @@ end end end -@testset "Independent Python fractions reference" begin - python = Sys.which("python3") - if isnothing(python) - @info "Skipping Independent Python fractions reference: python3 is unavailable" +@testset "Independent bc fractions reference" begin + if isnothing(Sys.which("bc")) || isnothing(Sys.which("jq")) || isnothing(Sys.which("bash")) + @info "Skipping Independent bc fractions reference: bash, bc or jq is unavailable" else payloads = [ Dict{String,Any}( @@ -158,7 +157,7 @@ end "counts" => [[string(big(2)^80 + 17), "1"], ["1", "3"]], ), ] - script = joinpath(@__DIR__, "..", "references", "exact_summary_reference.py") + script = joinpath(@__DIR__, "..", "references", "exact_summary_reference.sh") strings(values) = [isnothing(value) ? nothing : string(value) for value in values] for payload in payloads labels = payload["sample_labels"] @@ -171,7 +170,7 @@ end mktempdir() do directory input = joinpath(directory, "input.json") write(input, JSON3.write(payload)) - reference = JSON3.read(read(`$python $script $input`, String)) + reference = JSON3.read(read(`bash $script $input`, String)) for kind in ("samples", "groups") for (i, actual) in enumerate(stored[kind]) expected = reference[kind][i] diff --git a/test/unit/test_null_calibration_csv.R b/test/unit/test_null_calibration_csv.R index 6c6882b2..bd7fa942 100644 --- a/test/unit/test_null_calibration_csv.R +++ b/test/unit/test_null_calibration_csv.R @@ -60,14 +60,14 @@ read_error <- tryCatch({ }, error = function(e) conditionMessage(e)) stopifnot(grepl("unreadable result CSV", read_error, fixed = TRUE)) -# The workflow calls this same CSV-aware Python reader. It emits distinct +# The workflow calls this same CSV-aware Julia reader. It emits distinct # reasons once, skips NA notes and escapes annotation control characters. -python <- Sys.which("python3") -stopifnot(nzchar(python)) -reader_script <- "bench/null_calibration/report_failures.py" +julia <- Sys.which("julia") +stopifnot(nzchar(julia)) +reader_script <- "bench/null_calibration/report_failures.jl" annotations <- suppressWarnings(system2( - python, - args = c(shQuote(reader_script), shQuote(cell)), + julia, + args = c("--startup-file=no", shQuote(reader_script), shQuote(cell)), stdout = TRUE, stderr = TRUE, stdin = csv_path )) reader_status <- attr(annotations, "status") diff --git a/test/unit/test_ordination.jl b/test/unit/test_ordination.jl index 580c999e..32a3c2c9 100644 --- a/test/unit/test_ordination.jl +++ b/test/unit/test_ordination.jl @@ -18,7 +18,7 @@ using RCall const _ORD_SV = MetaManifold.Server const _ORD_REPO_ROOT = normpath(joinpath(@__DIR__, "..", "..")) const _ORD_REF_PATH = joinpath(_ORD_REPO_ROOT, "test", "fixtures", "ordination_reference.json") -const _ORD_PY_ORACLE = joinpath(_ORD_REPO_ROOT, "test", "references", "check_ordination_references.py") +const _ORD_PY_ORACLE = joinpath(_ORD_REPO_ROOT, "test", "references", "check_ordination_references.sh") function _ord_http(method::String, path::String, body = nothing; raw::Bool = false) payload = isnothing(body) ? UInt8[] : (raw ? body : jcs_bytes(body)) @@ -85,9 +85,9 @@ end @testset "Constrained ordinations (Issue #19)" begin - @testset "Independent Python reference oracle, KYAML contract, and JCS schema" begin - if !isnothing(Sys.which("python3")) - proc = run(`python3 $_ORD_PY_ORACLE --check $_ORD_REF_PATH`) + @testset "Independent awk reference oracle, KYAML contract, and JCS schema" begin + if !isnothing(Sys.which("awk")) + proc = run(`bash $_ORD_PY_ORACLE --check $_ORD_REF_PATH`) @test proc.exitcode == 0 end contract = load_ordination_contract() From d97b7ef83f782e71685554e399f5d3d14be32345 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 20:54:22 +0000 Subject: [PATCH 04/10] ci: fail early, skip repeated work, time every step Read pins with yq, run Bun and the frontend before R, run the validated statistics files once (--exclude), time every test file, share one change classifier between the two relevance gates, add timeouts to network installs, and reuse an Agda pass on pull requests when its inputs are unchanged (the summary names the run reused). The DOI lane's inline Python is now bash. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_0136P7ZmNJPhB7wGCRymdu1D --- .github/workflows/ci.yml | 358 ++++++++++++------ .github/workflows/doi.yml | 41 +- .../null-calibration-comparators.yml | 9 +- scripts/ci/classify-change.sh | 76 ++++ scripts/tests/test-classify-change.sh | 51 +++ test/runtests.jl | 106 +++++- test/unit/test_install_pins.jl | 5 +- 7 files changed, 486 insertions(+), 160 deletions(-) create mode 100644 scripts/ci/classify-change.sh create mode 100644 scripts/tests/test-classify-change.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f388e197..cdac63eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,48 +42,31 @@ jobs: persist-credentials: false # nothing here pushes fetch-depth: 0 - # --no-renames, so a file renamed out of a listed path is reported as a - # deletion under its original name and still counts. -z with a - # NUL-delimited read, so a pathname Git would quote or that contains a - # newline arrives raw and matches the pattern. The base SHA travels - # through env, not template interpolation into the script. + # The diff logic is scripts/ci/classify-change.sh, shared with the occupancy + # gate: a diff to a file, --no-renames -z, and a failed diff fails the gate + # (an unreadable comparison must never read as "nothing relevant", because + # a skipped job satisfies a required check). The base SHA travels through + # env, not template interpolation into the script. Every pattern is quoted, + # so the glob fires at match time, against the diff, and is not expanded + # against the checkout the runner happens to be standing in. - name: Does the change touch anything the suite exercises? id: pr if: github.event_name == 'pull_request' env: BASE_SHA: ${{ github.event.pull_request.base.sha }} run: | - relevant=false - # Every pattern is quoted, so the glob survives array assignment and - # fires at `case` time, against the diff -- an unquoted `src/*` would - # expand against the checkout the runner happens to be standing in. - patterns=('src/*' 'test/*' 'scripts/*' 'frontend/*' 'config/*' - 'bench/*' 'data/*' 'proofs/*' 'R/*' 'renv/*' 'Project.toml' - 'Manifest.toml' 'renv.lock' '.Rprofile' 'install.sh' 'install.jl' - 'start.sh' 'docs/occupancy/*' 'docs/occupancy-2026-10.adoc' - '.github/workflows/*') - # The diff goes to a file first because a process substitution hides - # git diff's exit status: an unavailable comparison SHA (a force-push - # on the base branch, for instance) or a repository error would read - # as "no changes", and the gate would post relevant=false -- skipping - # the Julia suite and the proof checker, whose `skipped` conclusion - # satisfies a required check -- so a relevant PR would pass silently. - # A failed diff must fail the gate itself: that is the one failure - # the required-check design cannot afford to skip. As in the - # occupancy gate, which this mirrors. - changed="$RUNNER_TEMP/ci-changed-files" - if ! git diff --no-renames --name-only -z "$BASE_SHA...HEAD" > "$changed"; then - echo "ERROR: git diff $BASE_SHA...HEAD failed" >&2 - exit 1 - fi - while IFS= read -r -d '' file; do - for pattern in "${patterns[@]}"; do - case "$file" in - $pattern) relevant=true; break 2 ;; - esac - done - done < "$changed" - echo "relevant=$relevant" >> "$GITHUB_OUTPUT" + bash "$GITHUB_WORKSPACE/scripts/ci/classify-change.sh" \ + 'src/*' 'test/*' 'scripts/*' 'frontend/*' 'config/*' \ + 'bench/*' 'data/*' 'proofs/*' 'R/*' 'renv/*' 'Project.toml' \ + 'Manifest.toml' 'renv.lock' '.Rprofile' 'install.sh' 'install.jl' \ + 'start.sh' 'docs/occupancy/*' 'docs/occupancy-2026-10.adoc' \ + '.github/workflows/*' + + # The classifier's own fail-closed controls, so a change to it cannot + # quietly turn a failed diff into a pass. + - name: Check the classifier fails closed + if: github.event_name == 'pull_request' + run: bash scripts/tests/test-classify-change.sh occupancy_gate: name: Occupancy relevance @@ -108,45 +91,25 @@ jobs: persist-credentials: false # nothing here pushes fetch-depth: 0 + # Same classifier as the code gate (scripts/ci/classify-change.sh). The base + # SHA is GitHub-supplied, but it travels through env all the same: zizmor + # flags template expressions interpolated into run scripts, and the + # script is easier to run locally with the value in the environment. - name: Does the change touch the occupancy reading? id: pr if: github.event_name == 'pull_request' env: - # zizmor flags template expressions interpolated into run scripts - # (template-injection), and the gate script is easier to run locally - # with the value in an environment variable in any case. The base SHA - # is GitHub-supplied, but the indirection costs nothing. BASE_SHA: ${{ github.event.pull_request.base.sha }} run: | - relevant=false - patterns=('src/core/memory_probe.jl' 'src/core/provenance.jl' 'src/pipeline/tools.jl' - 'src/server/routes/pipeline.jl' 'test/occupancy_disagreement_test.jl' - 'test/unit/test_occupancy_grade.jl' 'scripts/check-occupancy-grades.jl' - 'scripts/lint-occupancy-readings.sh' 'docs/occupancy/*' - 'docs/occupancy-2026-10.adoc' 'scripts/check-occupancy.sh' - 'scripts/fetch-siblings.sh' 'scripts/install-ci-julia.sh' - 'Project.toml' 'Manifest.toml' 'renv.lock' 'config/defaults/tool_versions.yml' - '.github/workflows/*') - # The diff goes to a file first because a process substitution hides - # git diff's exit status: an unavailable comparison SHA or a repository - # error would read as "no changes", and the gate would post - # relevant=false -- skipping the two occupancy jobs, whose `skipped` - # conclusion satisfies a required check -- so a relevant PR would - # pass silently. A failed diff must fail the gate itself: that is - # the one failure the required-check design cannot afford to skip. - changed="$RUNNER_TEMP/occupancy-changed-files" - if ! git diff --no-renames --name-only -z "$BASE_SHA...HEAD" > "$changed"; then - echo "ERROR: git diff $BASE_SHA...HEAD failed" >&2 - exit 1 - fi - while IFS= read -r -d '' file; do - for pattern in "${patterns[@]}"; do - case "$file" in - $pattern) relevant=true; break 2 ;; - esac - done - done < "$changed" - echo "relevant=$relevant" >> "$GITHUB_OUTPUT" + bash "$GITHUB_WORKSPACE/scripts/ci/classify-change.sh" \ + 'src/core/memory_probe.jl' 'src/core/provenance.jl' 'src/pipeline/tools.jl' \ + 'src/server/routes/pipeline.jl' 'test/occupancy_disagreement_test.jl' \ + 'test/unit/test_occupancy_grade.jl' 'scripts/check-occupancy-grades.jl' \ + 'scripts/lint-occupancy-readings.sh' 'docs/occupancy/*' \ + 'docs/occupancy-2026-10.adoc' 'scripts/check-occupancy.sh' \ + 'scripts/fetch-siblings.sh' 'scripts/install-ci-julia.sh' \ + 'Project.toml' 'Manifest.toml' 'renv.lock' 'config/defaults/tool_versions.yml' \ + '.github/workflows/*' test: # The name is a fixed string, it carries no version, and the job has NO matrix. @@ -189,6 +152,12 @@ jobs: # the environment, so those commands pass --no-init-file instead. env: RENV_CONFIG_AUTOLOADER_ENABLED: "FALSE" + # The unit files the early step runs. The full suite skips exactly these + # (--exclude), so no file runs twice; the early step still fails first. + VALIDATED_STATISTICS_TESTS: >- + validated_statistics numeric_policy exact_summaries high_precision + derived_analyses method_catalogue statistics_boundaries + statistics_adversarial statistics_properties statistics_ui steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -203,39 +172,82 @@ jobs: run: bash scripts/lint-sonar-reliability.sh - name: Set up Julia + timeout-minutes: 10 run: bash scripts/install-ci-julia.sh - name: Parse Julia sources before dependency setup run: julia scripts/check-julia-syntax.jl # Every external version CI installs is read from the committed pin file, so CI - # and a developer's machine cannot drift apart. A temporary environment is used - # because the project environment cannot be instantiated until R is present: - # RCall's build step requires it. + # and a developer's machine cannot drift apart. yq is already on the runner, so + # no Julia (and no package install) is needed just to read a YAML file. A pin + # that is missing or null fails the step here instead of exporting an empty + # value that a later step would install around. - name: Read pinned versions run: | - julia -e ' - using Pkg - Pkg.activate(temp = true) - Pkg.add("YAML") - using YAML - pins = YAML.load_file("config/defaults/tool_versions.yml") - open(ENV["GITHUB_ENV"], "a") do io - println(io, "JULIA_VERSION=", pins["runtimes"]["julia"]["version"]) - println(io, "R_APT_VERSION=", pins["runtimes"]["r"]["apt_version"]) - println(io, "BUN_VERSION=", pins["toolchain"]["bun"]["version"]) - println(io, "CUTADAPT_SPEC=cutadapt==", pins["tools"]["cutadapt"]["version"]) - println(io, "MULTIQC_SPEC=multiqc==", pins["tools"]["multiqc"]["version"]) - fastqc = pins["tools"]["fastqc"]["archives"]["any"] - println(io, "FASTQC_URL=", fastqc["url"]) - println(io, "FASTQC_SHA256=", fastqc["sha256"]) - for tool in ("vsearch", "swarm") - archive = pins["tools"][tool]["archives"]["linux-x86_64"] - println(io, uppercase(tool), "_VERSION=", pins["tools"][tool]["version"]) - println(io, uppercase(tool), "_URL=", archive["url"]) - println(io, uppercase(tool), "_SHA256=", archive["sha256"]) - end - end' + set -euo pipefail + pins=config/defaults/tool_versions.yml + pin() { + local value + value="$(yq -r "$1" "$pins")" + if [[ -z "$value" || "$value" == null ]]; then + echo "::error::no value for $1 in $pins" >&2 + return 1 + fi + printf '%s' "$value" + } + julia_version="$(pin '.runtimes.julia.version')" + r_apt_version="$(pin '.runtimes.r.apt_version')" + bun_version="$(pin '.toolchain.bun.version')" + cutadapt_version="$(pin '.tools.cutadapt.version')" + multiqc_version="$(pin '.tools.multiqc.version')" + fastqc_url="$(pin '.tools.fastqc.archives.any.url')" + fastqc_sha256="$(pin '.tools.fastqc.archives.any.sha256')" + vsearch_version="$(pin '.tools.vsearch.version')" + vsearch_url="$(pin '.tools.vsearch.archives."linux-x86_64".url')" + vsearch_sha256="$(pin '.tools.vsearch.archives."linux-x86_64".sha256')" + swarm_version="$(pin '.tools.swarm.version')" + swarm_url="$(pin '.tools.swarm.archives."linux-x86_64".url')" + swarm_sha256="$(pin '.tools.swarm.archives."linux-x86_64".sha256')" + { + echo "JULIA_VERSION=$julia_version" + echo "R_APT_VERSION=$r_apt_version" + echo "BUN_VERSION=$bun_version" + echo "CUTADAPT_SPEC=cutadapt==$cutadapt_version" + echo "MULTIQC_SPEC=multiqc==$multiqc_version" + echo "FASTQC_URL=$fastqc_url" + echo "FASTQC_SHA256=$fastqc_sha256" + echo "VSEARCH_VERSION=$vsearch_version" + echo "VSEARCH_URL=$vsearch_url" + echo "VSEARCH_SHA256=$vsearch_sha256" + echo "SWARM_VERSION=$swarm_version" + echo "SWARM_URL=$swarm_url" + echo "SWARM_SHA256=$swarm_sha256" + } >> "$GITHUB_ENV" + + # Bun and the frontend need nothing from R or Julia, so they run first: a + # broken frontend fails in about a minute, not after the R and Julia setup. + - name: Set up Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: ${{ env.BUN_VERSION }} + + - name: Build frontend + working-directory: frontend + timeout-minutes: 10 + run: | + bun install --frozen-lockfile --ignore-scripts + bun run build + + - name: Audit frontend dependencies + working-directory: frontend + # Moderate advisories remain visible in `bun audit`; upgrading Router + # across a major version needs a separate routing/SSR review. + run: bun audit --audit-level high + + - name: Frontend unit tests + working-directory: frontend + run: bun run test # Packages, artifacts and the precompiled caches. The compiled caches are # valid only for the Julia that wrote them, and RCall's image is built @@ -259,6 +271,7 @@ jobs: # R.Version in renv.lock; CRAN's Ubuntu repository retains older revisions, so # this resolves rather than merely requesting the newest. - name: Set up R + timeout-minutes: 10 run: bash scripts/ci/install-r-runtime.sh full # The R packages are restored from renv.lock, the same pin a developer restores @@ -288,6 +301,7 @@ jobs: # R_CLI_DYNAMIC and TERM make renv print progress one line at a time in the log. - name: Install R packages + timeout-minutes: 25 env: RENV_PATHS_CACHE: /opt/renv-cache run: | @@ -304,7 +318,7 @@ jobs: run: Rscript --vanilla test/unit/test_null_calibration_csv.R - name: Check null-calibration failure annotations - run: python3 -m unittest discover -s scripts/tests -p 'test_null_calibration_failures.py' -v + run: julia --startup-file=no scripts/tests/test_null_calibration_failures.jl # Saved on failure too, since the packages built before a failure are the ones # the next attempt should reuse. Root writes the cache, so it is made readable first. @@ -320,17 +334,20 @@ jobs: key: renv-${{ runner.os }}-R${{ env.R_APT_VERSION }}-${{ hashFiles('renv.lock') }}-${{ github.run_id }} - name: Install cutadapt + timeout-minutes: 10 run: pip install "$CUTADAPT_SPEC" # The one tool CI does not pin. Ubuntu ships no versioned cd-hit revision worth # naming, and 24.04 is frozen, so its cd-hit does not move under us; the version # that arrives is recorded at preflight, where a discrepancy is visible. - name: Install cd-hit + timeout-minutes: 5 run: sudo apt-get install -y cd-hit # Downloaded against the pinned URL and refused unless it hashes to the pinned # SHA256, so CI runs the same bytes install.jl puts on a developer's machine. - name: Install vsearch + timeout-minutes: 5 run: | source scripts/ci/fetch_pinned.sh fetch_pinned "$VSEARCH_URL" vsearch.tar.gz @@ -341,6 +358,7 @@ jobs: vsearch --version - name: Install swarm + timeout-minutes: 5 run: | source scripts/ci/fetch_pinned.sh fetch_pinned "$SWARM_URL" swarm.tar.gz @@ -351,11 +369,13 @@ jobs: swarm --version - name: Install multiqc + timeout-minutes: 10 run: pip install "$MULTIQC_SPEC" # FastQC ships as a Java launcher script; `fastqc --version` checks that Java # is on PATH as well. - name: Install fastqc + timeout-minutes: 5 run: | source scripts/ci/fetch_pinned.sh fetch_pinned "$FASTQC_URL" fastqc.zip @@ -373,28 +393,8 @@ jobs: JULIA_PKG_PRECOMPILE_AUTO: "0" run: julia --project=. -e 'import Pkg; Pkg.instantiate()' - - name: Set up Bun - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 - with: - bun-version: ${{ env.BUN_VERSION }} - - - name: Build frontend - working-directory: frontend - run: | - bun install --frozen-lockfile --ignore-scripts - bun run build - - - name: Audit frontend dependencies - working-directory: frontend - # Moderate advisories remain visible in `bun audit`; upgrading Router - # across a major version needs a separate routing/SSR review. - run: bun audit --audit-level high - - - name: Frontend unit tests - working-directory: frontend - run: bun run test - - name: Download PR2 databases + timeout-minutes: 10 run: julia --project=. config/ci/download_databases.jl - name: Rebuild RCall against installed R @@ -416,16 +416,30 @@ jobs: end ' + # The independent oracles are GNU bc (high precision, exact summaries) and awk + # (ordination): none shares maths code with Julia. bc is installed explicitly, + # and the step fails if a tool is missing, so a test cannot skip itself green. + - name: Install the independent oracle tools + timeout-minutes: 5 + run: | + sudo apt-get install -y --no-install-recommends bc jq + command -v bc jq awk + - name: Check independent high-precision references - run: python3 test/references/check_high_precision_references.py --check test/unit/test_high_precision.jl + run: bash test/references/check_high_precision_references.sh --check test/unit/test_high_precision.jl + # Same coverage flag as the full suite: the .cov files of both runs sit side by + # side and the coverage step below merges them. - name: Run validated-statistics tests first env: R_LIBS_SITE: /usr/local/lib/R/site-library:/usr/lib/R/site-library:/usr/lib/R/library + TEST_TIMING_TITLE: Validated-statistics tests (early step) shell: bash run: | set +e - julia --project=. -t2 test/runtests.jl validated_statistics numeric_policy exact_summaries high_precision derived_analyses method_catalogue statistics_boundaries statistics_adversarial statistics_properties statistics_ui 2>&1 | tee /tmp/validated-statistics-tests.log + # The list is word-split on purpose: each name is a separate argument. + # shellcheck disable=SC2086 + julia --project=. -t2 --code-coverage=@ test/runtests.jl $VALIDATED_STATISTICS_TESTS 2>&1 | tee /tmp/validated-statistics-tests.log # Preserve both exits immediately: a successful Julia run must not # hide a failed log write (e.g. a full runner filesystem). statuses=("${PIPESTATUS[@]}") @@ -471,6 +485,7 @@ jobs: env: OCCUPANCY_RESULTS: ${{ runner.temp }}/occupancy-results.yml CI_SKIP_TAXONOMY: "1" + TEST_TIMING_TITLE: Full suite (less the early step's files) R_LIBS_SITE: /usr/local/lib/R/site-library:/usr/lib/R/site-library:/usr/lib/R/library # A failure also posts annotations, which the checks API serves without the full # log. Annotations are capped at 4096 bytes, so each failing assertion gets its own @@ -479,7 +494,7 @@ jobs: run: | command -v vsearch # The full suite must run the occupancy experiment, not skip it. set +e - julia --project=. -t 2 --code-coverage=@ test/runtests.jl --integration --server 2>&1 | tee /tmp/julia-tests.log + julia --project=. -t 2 --code-coverage=@ test/runtests.jl --exclude="${VALIDATED_STATISTICS_TESTS// /,}" --integration --server 2>&1 | tee /tmp/julia-tests.log # Preserve both exits immediately: a successful Julia run must not # hide a failed log write (e.g. a full runner filesystem). statuses=("${PIPESTATUS[@]}") @@ -527,8 +542,17 @@ jobs: julia --project=. scripts/check-occupancy-grades.jl "$RUNNER_TEMP/occupancy-results.yml" bash scripts/lint-occupancy-readings.sh + # Both test runs write .cov files named for their process id. Fewer than two + # processes means one run's coverage never reached this step. - name: Process coverage - run: julia scripts/process-ci-coverage.jl + run: | + find src -name '*.cov' | sed -E 's/.*\.([0-9]+)\.cov$/\1/' | sort -u > "$RUNNER_TEMP/coverage-pids" + echo "coverage files: $(find src -name '*.cov' | wc -l); processes that wrote them: $(wc -l < "$RUNNER_TEMP/coverage-pids")" + if [ "$(wc -l < "$RUNNER_TEMP/coverage-pids")" -lt 2 ]; then + echo "::error::coverage from the early step or the full suite is missing" + exit 1 + fi + julia scripts/process-ci-coverage.jl - name: Upload coverage to Codecov uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 @@ -553,6 +577,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Julia + timeout-minutes: 10 run: bash scripts/install-ci-julia.sh - name: Read pinned runtime versions @@ -577,12 +602,14 @@ jobs: # The versions, the HTTPS restriction and the exact-commit fetches live in # the script (one reviewed place), not interpolated into this workflow. - name: Install pinned standards tools + timeout-minutes: 10 run: bash scripts/ci/fetch-statistics-tools.sh # MetaManifold includes RCall at module load, and RCall's build step needs # an R installation even though this job runs no R code. The version comes # from the committed pin file, not from whatever apt calls current. - name: Install the pinned R runtime + timeout-minutes: 10 run: | R_APT_VERSION="$(yq -r '.runtimes.r.apt_version' config/defaults/tool_versions.yml)" R_APT_VERSION="$R_APT_VERSION" bash scripts/ci/install-r-runtime.sh core @@ -696,13 +723,77 @@ jobs: timeout-minutes: 30 env: DEBIAN_FRONTEND: noninteractive + # The pins of this job, in one place. They are part of the input hash below, + # so changing one re-runs the proofs; so does any change under proofs/. + AGDA_BIN_VERSION: 2.6.4.3-1+b2 + AGDA_STDLIB_VERSION: 2.1-4 + AGDA_FLAGS: --safe --without-K --cubical-compatible steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # A pass is memoised on its inputs: the proofs, the checker pins and the + # invocation. Pull requests reuse a pass whose inputs are unchanged, which + # is the usual case for a PR that does not touch proofs/ (a workflow-only or + # script change is "relevant" but cannot change a proof). The step summary + # always names the run reused. A push to main never reuses: it re-checks in + # full, so the memo can only ever shorten a pull request, never main. + - name: Hash the proof inputs + id: inputs + env: + IMAGE: ${{ job.container.image }} + run: | + set -euo pipefail + hash="$( { + printf '%s\n' "$IMAGE" "$AGDA_BIN_VERSION" "$AGDA_STDLIB_VERSION" "$AGDA_FLAGS" + find proofs -type f -print0 | LC_ALL=C sort -z | xargs -0 sha256sum + } | sha256sum | cut -d' ' -f1 )" + echo "hash=$hash" >> "$GITHUB_OUTPUT" + echo "Input hash: $hash" + + # lookup-only: nothing is downloaded, the matched key carries the run id. + - name: Look for a reusable pass + id: lookup + if: github.event_name == 'pull_request' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + lookup-only: true + path: ${{ runner.temp }}/agda-pass-marker + key: agda-pass-${{ steps.inputs.outputs.hash }}-none + restore-keys: | + agda-pass-${{ steps.inputs.outputs.hash }}- + + - name: Reuse the earlier pass + id: reuse + if: github.event_name == 'pull_request' && steps.lookup.outputs.cache-matched-key != '' + env: + MATCHED: ${{ steps.lookup.outputs.cache-matched-key }} + HASH: ${{ steps.inputs.outputs.hash }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + # The key ends in .: a re-run of a run must not try to + # save over the key its first attempt wrote. + run_id="${MATCHED##*-}" + run_id="${run_id%%.*}" + [[ "$run_id" =~ ^[0-9]+$ ]] || { echo "ERROR: cannot read a run id from $MATCHED" >&2; exit 1; } + msg="Reused pass from run $run_id for input hash $HASH" + echo "$msg" + { + echo "### Agda proofs: reused pass" + echo + echo "$msg" + echo + echo "Earlier run: https://github.com/$REPO/actions/runs/$run_id" + echo "The proofs, the checker pins and the invocation are unchanged, so the type-check was not repeated. A push to main always re-checks in full." + } >> "$GITHUB_STEP_SUMMARY" + echo "reused=true" >> "$GITHUB_OUTPUT" + - name: Install Agda and the standard library + if: steps.reuse.outputs.reused != 'true' + timeout-minutes: 10 run: | apt-get update -qq - apt-get install -y --no-install-recommends agda-bin=2.6.4.3-1+b2 agda-stdlib=2.1-4 + apt-get install -y --no-install-recommends "agda-bin=$AGDA_BIN_VERSION" "agda-stdlib=$AGDA_STDLIB_VERSION" agda --version | grep -qx 'Agda version 2.6.4.3' grep -qx 'name: standard-library-2.1' /usr/share/agda-stdlib/standard-library.agda-lib @@ -711,6 +802,7 @@ jobs: # module that MetaManifold/All.agda does not import (an unimported module # is never type-checked at all). - name: Axiom audit + if: steps.reuse.outputs.reused != 'true' run: bash proofs/tests/axiom-audit.sh # The invocation is the estate PROOF-POLICY §2.1 one, written out rather @@ -718,6 +810,7 @@ jobs: # Unicode, and under the image's default POSIX locale it then exits 42 # instead of reporting, so the locale is UTF-8. - name: Type-check + if: steps.reuse.outputs.reused != 'true' env: AGDA_DIR: /tmp/agda-dir LC_ALL: C.UTF-8 @@ -725,7 +818,9 @@ jobs: mkdir -p "$AGDA_DIR" echo /usr/share/agda-stdlib/standard-library.agda-lib > "$AGDA_DIR/libraries" cd proofs/agda - if ! agda --safe --without-K --cubical-compatible MetaManifold/All.agda >/tmp/agda.log 2>&1; then + # AGDA_FLAGS is word-split on purpose: each flag is its own argument. + # shellcheck disable=SC2086 + if ! agda $AGDA_FLAGS MetaManifold/All.agda >/tmp/agda.log 2>&1; then cat /tmp/agda.log msg="$(awk 'NR > 1 { printf "%%0A" } { gsub(/%/, "%25"); printf "%s", $0 }' /tmp/agda.log | tail -c 3500)" echo "::error file=proofs/agda/MetaManifold/All.agda,title=Agda type-check failed::$msg" @@ -733,6 +828,23 @@ jobs: fi cat /tmp/agda.log + # Only a full pass leaves a marker, keyed on the inputs and the run id. + # Pull requests and main both leave one; only pull requests read it. + - name: Record the pass + if: steps.reuse.outputs.reused != 'true' + env: + HASH: ${{ steps.inputs.outputs.hash }} + RUN_ID: ${{ github.run_id }} + MARKER: ${{ runner.temp }}/agda-pass-marker + run: printf 'run=%s\nhash=%s\n' "$RUN_ID" "$HASH" > "$MARKER" + + - name: Save the pass marker + if: steps.reuse.outputs.reused != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/agda-pass-marker + key: agda-pass-${{ steps.inputs.outputs.hash }}-${{ github.run_id }}.${{ github.run_attempt }} + occupancy: name: Occupancy typed peak needs: occupancy_gate @@ -743,10 +855,12 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install proof tools + timeout-minutes: 10 run: | apt-get update -qq apt-get install -y --no-install-recommends ca-certificates curl git agda-bin=2.6.4.3-1+b2 agda-stdlib=2.1-4 - name: Set up Julia + timeout-minutes: 10 run: bash scripts/install-ci-julia.sh - name: Check Session IR and occupancy proofs run: bash scripts/check-occupancy.sh diff --git a/.github/workflows/doi.yml b/.github/workflows/doi.yml index 17088cb3..ca425699 100644 --- a/.github/workflows/doi.yml +++ b/.github/workflows/doi.yml @@ -75,31 +75,28 @@ jobs: - name: Citation metadata check run: | - python3 - <<'PY' - from pathlib import Path - import sys - - cff = Path("CITATION.cff").read_text(encoding="utf-8") - required_prefixes = ("cff-version:", "title:", "authors:", "license:") - missing = [p for p in required_prefixes if not any(line.startswith(p) for line in cff.splitlines())] - if missing: - sys.exit(f"CITATION.cff missing required top-level keys: {', '.join(missing)}") - PY + missing=() + for key in cff-version title authors license; do + grep -q "^${key}:" CITATION.cff || missing+=("${key}:") + done + if [ "${#missing[@]}" -ne 0 ]; then + echo "CITATION.cff missing required top-level keys: ${missing[*]}" >&2 + exit 1 + fi - name: Receipt schema shape run: | - python3 - <<'PY' - from pathlib import Path - import sys - - source = Path("src/core/doi.jl").read_text(encoding="utf-8") - keys = ("schema", "study", "run", "origin", "deposition_id", "doi", - "published", "files", "created_at", "published_at", "checksum") - missing = [k for k in keys if f'"{k}"' not in source and f":{k}" not in source] - if missing: - sys.exit(f"src/core/doi.jl missing receipt schema keys: {', '.join(missing)}") - print("verified DOI receipt schema keys in src/core/doi.jl") - PY + missing=() + for key in schema study run origin deposition_id doi published files created_at published_at checksum; do + if ! grep -qF -e "\"${key}\"" -e ":${key}" src/core/doi.jl; then + missing+=("${key}") + fi + done + if [ "${#missing[@]}" -ne 0 ]; then + echo "src/core/doi.jl missing receipt schema keys: ${missing[*]}" >&2 + exit 1 + fi + echo "verified DOI receipt schema keys in src/core/doi.jl" # Julia lifecycle and real-HTTP checks require a provisioned machine with the # Julia runtime and are a documented blocking acceptance gate, not part of diff --git a/.github/workflows/null-calibration-comparators.yml b/.github/workflows/null-calibration-comparators.yml index e49c20c2..cfa1476b 100644 --- a/.github/workflows/null-calibration-comparators.yml +++ b/.github/workflows/null-calibration-comparators.yml @@ -236,6 +236,13 @@ jobs: # A test that produced no usable p-value leaves its reason only in the # per-cell CSV, which is an artifact a reader may not be able to open. # The distinct reasons are therefore published as annotations too. + # The reporter is Julia (standard library only), installed from the same + # pin as the CI jobs. Always run, so the notes still appear after a failure. + - name: Set up Julia + if: always() + timeout-minutes: 10 + run: bash scripts/install-ci-julia.sh + - name: Report any test that could not run if: always() run: | @@ -243,7 +250,7 @@ jobs: shopt -s nullglob for csv in bench/null_calibration/results/raw/pvalues_*.csv; do cell="$(basename "$csv" .csv)" - python3 bench/null_calibration/report_failures.py "$cell" < "$csv" + julia --startup-file=no bench/null_calibration/report_failures.jl "$cell" < "$csv" done - name: Publish the summaries as annotations diff --git a/scripts/ci/classify-change.sh b/scripts/ci/classify-change.sh new file mode 100644 index 00000000..968762bb --- /dev/null +++ b/scripts/ci/classify-change.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# Decide whether a pull request touches any path in a list of glob patterns. +# +# Used by both relevance gates in ci.yml, so the diff logic lives here once. +# +# Usage: +# classify-change.sh [--changed-file FILE] [--patterns-file FILE] [PATTERN...] +# +# The base commit comes from BASE_SHA. The patterns are shell globs matched with +# `case`, so `src/*` also matches `src/a/b.jl`. Each pattern is an argument or a +# line of --patterns-file (blank lines and lines starting with # are ignored). +# Result: "relevant=true" or "relevant=false", appended to GITHUB_OUTPUT when it +# is set and printed otherwise. +# +# The diff goes to a file first because a process substitution hides git diff's +# exit status. An unavailable base commit (a force-push on the base branch, for +# instance) or a repository error would then read as "no changes" and post +# relevant=false, which skips the jobs a required check stands for, and a +# relevant change would pass silently. A failed diff fails this script, and so +# the gate itself: the one failure the required-check design cannot afford to +# skip. +# +# --no-renames, so a file renamed out of a listed path is reported as a deletion +# under its original name and still counts. -z with a NUL-delimited read, so a +# pathname git would quote, or that contains a newline, arrives raw and matches. +set -euo pipefail + +changed="${RUNNER_TEMP:-${TMPDIR:-/tmp}}/classify-change-files.$$" +patterns=() +trap 'rm -f "$changed"' EXIT + +while (($#)); do + case "$1" in + --changed-file) + changed="${2:?--changed-file needs a path}"; trap - EXIT; shift 2 ;; + --patterns-file) + file="${2:?--patterns-file needs a path}" + [[ -r "$file" ]] || { echo "ERROR: cannot read patterns file $file" >&2; exit 2; } + while IFS= read -r line || [[ -n "$line" ]]; do + [[ -z "$line" || "$line" == \#* ]] && continue + patterns+=("$line") + done < "$file" + shift 2 ;; + --) shift; patterns+=("$@"); break ;; + -*) echo "ERROR: unknown option $1" >&2; exit 2 ;; + *) patterns+=("$1"); shift ;; + esac +done + +: "${BASE_SHA:?BASE_SHA must name the comparison commit}" +((${#patterns[@]})) || { echo "ERROR: no patterns given" >&2; exit 2; } + +if ! git diff --no-renames --name-only -z "$BASE_SHA...HEAD" > "$changed"; then + echo "ERROR: git diff $BASE_SHA...HEAD failed" >&2 + exit 1 +fi + +relevant=false +while IFS= read -r -d '' file; do + for pattern in "${patterns[@]}"; do + # The pattern is deliberately unquoted: it is a glob and must fire at case + # time, against the diff. It came in as an argument, so it is never + # expanded against the checkout the runner happens to be standing in. + # shellcheck disable=SC2254 + case "$file" in + $pattern) relevant=true; break 2 ;; + esac + done +done < "$changed" + +if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + echo "relevant=$relevant" >> "$GITHUB_OUTPUT" +else + echo "relevant=$relevant" +fi diff --git a/scripts/tests/test-classify-change.sh b/scripts/tests/test-classify-change.sh new file mode 100644 index 00000000..772da62e --- /dev/null +++ b/scripts/tests/test-classify-change.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# The change classifier must fail closed: a diff that cannot be computed is an +# error, never "nothing relevant". Also a positive and a negative control. +set -euo pipefail +script="$(cd "$(dirname "$0")/.." && pwd)/ci/classify-change.sh" +scratch="$(mktemp -d)" +trap 'rm -rf "$scratch"' EXIT +cd "$scratch" +git init -q +git config user.name 'Classifier Test' +git config user.email 'classifier@example.invalid' +mkdir src +echo base > README.md +git add . && git commit -qm base +base="$(git rev-parse HEAD)" + +run() { BASE_SHA="$1" GITHUB_OUTPUT="$scratch/out" bash "$script" 'src/*' 'Project.toml'; } +: > out + +# A diff that cannot be computed must exit non-zero and write no verdict. +if BASE_SHA=not-a-commit GITHUB_OUTPUT="$scratch/out" bash "$script" 'src/*' > /dev/null 2>&1; then + echo 'ERROR: a failed diff exited zero' >&2; exit 1 +fi +if BASE_SHA=0000000000000000000000000000000000000000 GITHUB_OUTPUT="$scratch/out" bash "$script" 'src/*' > /dev/null 2>&1; then + echo 'ERROR: a missing base commit exited zero' >&2; exit 1 +fi +[[ ! -s out ]] || { echo 'ERROR: a failed diff still wrote a verdict' >&2; cat out >&2; exit 1; } + +# No patterns, or no base, is a usage error, not a verdict. +if BASE_SHA="$base" bash "$script" > /dev/null 2>&1; then echo 'ERROR: no patterns exited zero' >&2; exit 1; fi +if env -u BASE_SHA bash "$script" 'src/*' > /dev/null 2>&1; then echo 'ERROR: no base exited zero' >&2; exit 1; fi + +# Controls: an unrelated change is not relevant, a listed one is, a rename out +# of a listed path counts, and --patterns-file gives the same answer. +echo docs > NOTES.md && git add . && git commit -qm docs +: > out; run "$base"; [[ "$(cat out)" == relevant=false ]] || { echo 'ERROR: unrelated change was relevant' >&2; exit 1; } +echo x > src/a.jl && git add . && git commit -qm src +: > out; run "$base"; [[ "$(cat out)" == relevant=true ]] || { echo 'ERROR: listed change was not relevant' >&2; exit 1; } +printf '# comment\n\nsrc/*\n' > patterns.txt +: > out; BASE_SHA="$base" GITHUB_OUTPUT="$scratch/out" bash "$script" --patterns-file patterns.txt +[[ "$(cat out)" == relevant=true ]] || { echo 'ERROR: patterns file disagreed' >&2; exit 1; } +git reset -q --hard "$base" +mkdir -p src && echo y > src/b.jl && git add . && git commit -qm add +sub="$(git rev-parse HEAD)" +git mv src/b.jl elsewhere.jl && git commit -qm rename +: > out; BASE_SHA="$base" GITHUB_OUTPUT="$scratch/out" bash "$script" 'src/*' +[[ "$(cat out)" == relevant=false ]] || { echo 'ERROR: add then move back out was relevant' >&2; exit 1; } +: > out; BASE_SHA="$sub" GITHUB_OUTPUT="$scratch/out" bash "$script" 'src/*' +[[ "$(cat out)" == relevant=true ]] || { echo 'ERROR: rename out of a listed path was not relevant' >&2; exit 1; } +echo 'classify-change: fail-closed and classification controls PASS' diff --git a/test/runtests.jl b/test/runtests.jl index 4763ea0a..e3421517 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,4 +1,6 @@ #!/usr/bin/env julia +# Time before the first test file: package loading and the run's set-up. +const _SCRIPT_START = time() # MetaManifold test suite # # Run with: @@ -9,6 +11,13 @@ # # Name unit test files to run only those, e.g. # julia --project=. test/runtests.jl test_routes.jl trees +# +# Leave files out of a full run, e.g. the ones an earlier step already ran: +# julia --project=. test/runtests.jl --exclude=trees,test_routes.jl +# +# Every run ends with a table of where the time went, sorted by time. When +# GITHUB_STEP_SUMMARY is set the table is appended there as Markdown, under the +# title in TEST_TIMING_TITLE. using Test const RUN_INTEGRATION = "--integration" in ARGS @@ -72,19 +81,71 @@ const UNIT_FILES = [ "test_doi.jl", ] -# The files named on the command line ("test_routes.jl", "routes" or -# "unit/test_routes.jl"), or every unit file when none is named. +# A name as given on the command line ("test_routes.jl", "routes" or +# "unit/test_routes.jl") as the file name it stands for. +function _unit_file_name(a) + b = basename(a) + b = startswith(b, "test_") ? b : "test_" * b + endswith(b, ".jl") ? b : b * ".jl" +end + +# The comma-separated names of every --exclude=a,b option, as file names. +function _excluded_files(args = ARGS) + names = String[] + for a in args + startswith(a, "--exclude=") || continue + append!(names, filter(!isempty, strip.(split(a[length("--exclude=")+1:end], ',')))) + end + files = _unit_file_name.(names) + unknown = filter(f -> !isfile(joinpath(@__DIR__, "unit", f)), files) + isempty(unknown) || error("No such unit test file to exclude: $(join(unknown, ", "))") + files +end + +# The files named on the command line, or every unit file when none is named, +# less the ones --exclude names. function _selected_files() named = filter(a -> !startswith(a, "--"), ARGS) - isempty(named) && return UNIT_FILES - files = map(named) do a - b = basename(a) - b = startswith(b, "test_") ? b : "test_" * b - endswith(b, ".jl") ? b : b * ".jl" - end + files = isempty(named) ? UNIT_FILES : _unit_file_name.(named) unknown = filter(f -> !isfile(joinpath(@__DIR__, "unit", f)), files) isempty(unknown) || error("No such unit test file: $(join(unknown, ", "))") - files + excluded = _excluded_files() + filter(f -> !(f in excluded), files) +end + +# Where the time went: one row per unit file and per block, then the table. +const _TIMINGS = Tuple{String,Float64}[] + +function _timed(f, label) + t0 = time() + try + f() + finally + push!(_TIMINGS, (label, time() - t0)) + end +end + +function _timing_report() + rows = sort(_TIMINGS; by = last, rev = true) + total = sum(last, rows; init = 0.0) + fmt(s) = s >= 60 ? string(floor(Int, s / 60), "m", lpad(round(Int, s % 60), 2, '0'), "s") : + string(round(s; digits = 1), "s") + title = get(ENV, "TEST_TIMING_TITLE", "Julia test timings") + println("\n", title, " (", fmt(total), " measured)") + for (label, s) in rows + println(rpad(label, 48), lpad(fmt(s), 9), lpad(string(round(Int, 100 * s / max(total, eps())), "%"), 6)) + end + summary = get(ENV, "GITHUB_STEP_SUMMARY", "") + if !isempty(summary) + open(summary, "a") do io + println(io, "### ", title, "\n") + println(io, "| Step | Time | Share |\n| --- | ---: | ---: |") + for (label, s) in rows + println(io, "| `", label, "` | ", fmt(s), " | ", round(Int, 100 * s / max(total, eps())), "% |") + end + println(io, "| **Measured total** | **", fmt(total), "** | |\n") + end + end end using MetaManifold @@ -96,7 +157,13 @@ using MetaManifold.Tools, MetaManifold.TaxonomyTableTools, MetaManifold.ProjectS using MetaManifold.DiversityMetrics, MetaManifold.Analysis using MetaManifold.Categories, MetaManifold.CompositionLibrary +# The load is everything above: the packages and MetaManifold itself. +push!(_TIMINGS, ("load packages and MetaManifold", time() - _SCRIPT_START)) + ## Unit tests (always run) +# The table is printed even when a test fails, so the cost of a failing run is +# visible too. The failure still propagates: finally does not swallow it. +try @testset "MetaManifold" begin # Each file gets its own testset so that a throw at a file's top level -- outside @@ -105,21 +172,27 @@ using MetaManifold.Categories, MetaManifold.CompositionLibrary # skipped every remaining unit file (and the occupancy probe below) while the # summary still looked like a single stray error. for f in _selected_files() - @testset "$f" begin - include(joinpath(@__DIR__, "unit", f)) + _timed(f) do + @testset "$f" begin + include(joinpath(@__DIR__, "unit", f)) + end end end ## Occupancy probe and the two-thread disagreement test (MMW-01), with the ## full suite. Its vsearch part skips, saying so, when vsearch is not on PATH. if isempty(filter(a -> !startswith(a, "--"), ARGS)) - include("occupancy_disagreement_test.jl") + _timed("occupancy_disagreement_test.jl") do + include("occupancy_disagreement_test.jl") + end end ## Integration tests (opt-in) if RUN_INTEGRATION using MetaManifold.DADA2, MetaManifold.OTUPipeline - include("integration/test_pipeline.jl") + _timed("integration/test_pipeline.jl") do + include("integration/test_pipeline.jl") + end else @info "Skipping integration tests (pass --integration to enable)" end @@ -127,9 +200,14 @@ using MetaManifold.Categories, MetaManifold.CompositionLibrary ## Server smoke tests (opt-in, starts a Julia subprocess - slow on first run) if RUN_SERVER using HTTP - include("integration/test_server.jl") + _timed("integration/test_server.jl") do + include("integration/test_server.jl") + end else @info "Skipping server smoke tests (pass --server to enable)" end end +finally + _timing_report() +end diff --git a/test/unit/test_install_pins.jl b/test/unit/test_install_pins.jl index 8066ca7d..a14ad2ec 100644 --- a/test/unit/test_install_pins.jl +++ b/test/unit/test_install_pins.jl @@ -278,7 +278,10 @@ end out = joinpath(dir, "gate-output") write(out, "") cmd = addenv(Cmd(Cmd(["bash", "-e", "-o", "pipefail", "-c", script]); dir=dir), - "BASE_SHA" => comparison, "RUNNER_TEMP" => dir, "GITHUB_OUTPUT" => out) + "BASE_SHA" => comparison, "RUNNER_TEMP" => dir, "GITHUB_OUTPUT" => out, + # The step calls scripts/ci/classify-change.sh from the checkout + # while git runs in the scratch repository, as on a runner. + "GITHUB_WORKSPACE" => REPO_ROOT) ok = success(pipeline(cmd; stdout=devnull, stderr=devnull)) (ok, strip(read(out, String))) end From 190bf0132bd3a7643f90fce7c85513f3e020fe10 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 20:55:23 +0000 Subject: [PATCH 05/10] ci: make the Julia syntax gate able to fail The failure flag was assigned in a top-level loop of a script, which makes it a new local, so the gate printed failures and exited 0. Count failures in a function, add a planted-negative test, and fix the one expression in the seven bench files it had been hiding (a macro call before &&). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_0136P7ZmNJPhB7wGCRymdu1D --- .github/workflows/ci.yml | 5 ++ bench/advanced_zero_handling/benchmark.jl | 2 +- bench/cladistics.jl | 2 +- bench/compositional_differential/benchmark.jl | 2 +- bench/layer1_mock_recovery/evaluate.jl | 2 +- bench/layer1_mock_recovery/fetch.jl | 2 +- bench/layer1_mock_recovery/report.jl | 2 +- bench/layer1_mock_recovery/runner.jl | 2 +- scripts/check-julia-syntax.jl | 74 ++++++++++--------- scripts/tests/test-check-julia-syntax.sh | 23 ++++++ 10 files changed, 76 insertions(+), 40 deletions(-) create mode 100644 scripts/tests/test-check-julia-syntax.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cdac63eb..c86629b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -178,6 +178,11 @@ jobs: - name: Parse Julia sources before dependency setup run: julia scripts/check-julia-syntax.jl + # The gate once printed "Julia parse failed" and exited 0. This plants an + # unparsable file and requires the gate to fail on it. + - name: Check the syntax gate can fail + run: bash scripts/tests/test-check-julia-syntax.sh + # Every external version CI installs is read from the committed pin file, so CI # and a developer's machine cannot drift apart. yq is already on the runner, so # no Julia (and no package install) is needed just to read a YAML file. A pin diff --git a/bench/advanced_zero_handling/benchmark.jl b/bench/advanced_zero_handling/benchmark.jl index a385aa17..3821e7c2 100644 --- a/bench/advanced_zero_handling/benchmark.jl +++ b/bench/advanced_zero_handling/benchmark.jl @@ -115,4 +115,4 @@ function main() nothing end -abspath(PROGRAM_FILE) == @__FILE__ && main() +abspath(PROGRAM_FILE) == (@__FILE__) && main() diff --git a/bench/cladistics.jl b/bench/cladistics.jl index c417b393..7a98548d 100644 --- a/bench/cladistics.jl +++ b/bench/cladistics.jl @@ -34,4 +34,4 @@ function main() println("Includes one cumulative roll-up, Faith PD, and both UniFrac matrices; no I/O.") end -abspath(PROGRAM_FILE) == @__FILE__ && main() +abspath(PROGRAM_FILE) == (@__FILE__) && main() diff --git a/bench/compositional_differential/benchmark.jl b/bench/compositional_differential/benchmark.jl index 7a7154dd..29f8292c 100644 --- a/bench/compositional_differential/benchmark.jl +++ b/bench/compositional_differential/benchmark.jl @@ -174,4 +174,4 @@ function main(args::Vector{String}) end end -abspath(PROGRAM_FILE) == @__FILE__ && main(ARGS) +abspath(PROGRAM_FILE) == (@__FILE__) && main(ARGS) diff --git a/bench/layer1_mock_recovery/evaluate.jl b/bench/layer1_mock_recovery/evaluate.jl index 13810703..2f743a6c 100644 --- a/bench/layer1_mock_recovery/evaluate.jl +++ b/bench/layer1_mock_recovery/evaluate.jl @@ -199,4 +199,4 @@ function main() return metrics end -abspath(PROGRAM_FILE) == @__FILE__ && main() +abspath(PROGRAM_FILE) == (@__FILE__) && main() diff --git a/bench/layer1_mock_recovery/fetch.jl b/bench/layer1_mock_recovery/fetch.jl index 3293a30e..7ef2caa9 100644 --- a/bench/layer1_mock_recovery/fetch.jl +++ b/bench/layer1_mock_recovery/fetch.jl @@ -73,4 +73,4 @@ function main() @info "Fetch complete" data=DATA_DIR expected=EXPECTED_DIR end -abspath(PROGRAM_FILE) == @__FILE__ && main() +abspath(PROGRAM_FILE) == (@__FILE__) && main() diff --git a/bench/layer1_mock_recovery/report.jl b/bench/layer1_mock_recovery/report.jl index 39f23848..1c35bc97 100644 --- a/bench/layer1_mock_recovery/report.jl +++ b/bench/layer1_mock_recovery/report.jl @@ -60,4 +60,4 @@ function main() @info "Report written" out_path end -abspath(PROGRAM_FILE) == @__FILE__ && main() +abspath(PROGRAM_FILE) == (@__FILE__) && main() diff --git a/bench/layer1_mock_recovery/runner.jl b/bench/layer1_mock_recovery/runner.jl index 39214d74..1c6c3ea0 100644 --- a/bench/layer1_mock_recovery/runner.jl +++ b/bench/layer1_mock_recovery/runner.jl @@ -105,4 +105,4 @@ function main(args::Vector{String}=String[]) return runs end -abspath(PROGRAM_FILE) == @__FILE__ && main(ARGS) +abspath(PROGRAM_FILE) == (@__FILE__) && main(ARGS) diff --git a/scripts/check-julia-syntax.jl b/scripts/check-julia-syntax.jl index f4a93eaf..7fbd87c5 100644 --- a/scripts/check-julia-syntax.jl +++ b/scripts/check-julia-syntax.jl @@ -8,41 +8,49 @@ # Parse each expression explicitly and reject both error and incomplete results. files = sort(filter(f -> endswith(f, ".jl"), split(read(`git ls-files -z`, String), '\0'))) -failed = false -for file in files - issue = nothing - try - source = read(file, String) - position = 1 - while position <= ncodeunits(source) - ex, next_position = Meta.parse(source, position; filename=file) - if Meta.isexpr(ex, :error) || Meta.isexpr(ex, :incomplete) - issue = ex - break +# The loop is in a function so the failure count is a real local: a bare +# `failed = true` in a top-level loop of a script assigns a new local, and the +# gate could never fail. +function count_failures(files) + failures = 0 + for file in files + issue = nothing + try + source = read(file, String) + position = 1 + while position <= ncodeunits(source) + ex, next_position = Meta.parse(source, position; filename=file) + if Meta.isexpr(ex, :error) || Meta.isexpr(ex, :incomplete) + issue = ex + break + end + # A bare `end` at the top level parses without error and then fails + # at lowering (`syntax: "end" at top level`), which is how a file + # that lost its `module Foo` opener reaches CI looking syntactically + # fine. An `end` balances exactly one opener, so top level cannot + # hold one: this is a missing or duplicated opener, and the file + # would not load. + if ex === :end + issue = "stray `end` at top level (a `module`/`function`/... opener " * + "is missing or was duplicated in this file)" + break + end + next_position > position || + error("Julia parser made no progress at code unit $position") + position = next_position end - # A bare `end` at the top level parses without error and then fails - # at lowering (`syntax: "end" at top level`), which is how a file - # that lost its `module Foo` opener reaches CI looking syntactically - # fine. An `end` balances exactly one opener, so top level cannot - # hold one: this is a missing or duplicated opener, and the file - # would not load. - if ex === :end - issue = "stray `end` at top level (a `module`/`function`/... opener " * - "is missing or was duplicated in this file)" - break - end - next_position > position || - error("Julia parser made no progress at code unit $position") - position = next_position + catch err + issue = err + end + if !isnothing(issue) + failures += 1 + println(stderr, "Julia parse failed in $file:") + println(stderr, issue isa AbstractString ? issue : sprint(showerror, issue)) end - catch err - issue = err - end - if !isnothing(issue) - failed = true - println(stderr, "Julia parse failed in $file:") - println(stderr, issue isa AbstractString ? issue : sprint(showerror, issue)) end + return failures end -failed && exit(1) + +failures = count_failures(files) +failures == 0 || (println(stderr, "$failures Julia source file(s) failed to parse."); exit(1)) println("Julia parser accepted $(length(files)) tracked source files.") diff --git a/scripts/tests/test-check-julia-syntax.sh b/scripts/tests/test-check-julia-syntax.sh new file mode 100644 index 00000000..578f7f88 --- /dev/null +++ b/scripts/tests/test-check-julia-syntax.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# The syntax gate must be able to fail: an unparsable tracked file exits 1, a +# clean tree exits 0. A gate that prints a failure and exits 0 is a hole. +set -euo pipefail +script="$(cd "$(dirname "$0")/.." && pwd)/check-julia-syntax.jl" +scratch="$(mktemp -d)" +trap 'rm -rf "$scratch"' EXIT +cd "$scratch" +git init -q +printf 'x = 1\n' > ok.jl +git add . +julia --startup-file=no "$script" > /dev/null 2>&1 || { echo 'ERROR: a clean tree failed the gate' >&2; exit 1; } +printf 'y = 1 &&\n' > bad.jl +git add bad.jl +if julia --startup-file=no "$script" > /dev/null 2>&1; then + echo 'ERROR: an unparsable file passed the gate' >&2; exit 1 +fi +printf 'function f()\n' > bad.jl +if julia --startup-file=no "$script" > /dev/null 2>&1; then + echo 'ERROR: an unterminated block passed the gate' >&2; exit 1 +fi +echo 'check-julia-syntax: planted-negative and clean controls PASS' From 52c94834433ab1c2c06324fae863114fc0e12c2e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 20:56:37 +0000 Subject: [PATCH 06/10] ci: address the three SonarCloud findings Add default branches to two case statements and use a local variable for a positional parameter. Behaviour is unchanged. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_0136P7ZmNJPhB7wGCRymdu1D --- scripts/ci/classify-change.sh | 1 + test/references/check_ordination_references.sh | 6 +++--- test/references/exact_summary_reference.sh | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/ci/classify-change.sh b/scripts/ci/classify-change.sh index 968762bb..520b8ba8 100644 --- a/scripts/ci/classify-change.sh +++ b/scripts/ci/classify-change.sh @@ -65,6 +65,7 @@ while IFS= read -r -d '' file; do # shellcheck disable=SC2254 case "$file" in $pattern) relevant=true; break 2 ;; + *) ;; esac done done < "$changed" diff --git a/test/references/check_ordination_references.sh b/test/references/check_ordination_references.sh index e92fc950..da5cf9c1 100755 --- a/test/references/check_ordination_references.sh +++ b/test/references/check_ordination_references.sh @@ -59,9 +59,9 @@ command -v "$AWK" >/dev/null || { echo "error: awk is required" >&2; exit 2; } # 1 when the file ends with exactly one LF (the JCS file rule), else 0. lf_ok() { - local tail2 - [[ -f "$1" ]] || { echo 0; return; } - tail2="$(tail -c 2 -- "$1" | od -An -v -tx1 | tr -d ' \n')" + local tail2 path="$1" + [[ -f "$path" ]] || { echo 0; return; } + tail2="$(tail -c 2 -- "$path" | od -An -v -tx1 | tr -d ' \n')" case "$tail2" in *0a0a) echo 0 ;; *0a) echo 1 ;; diff --git a/test/references/exact_summary_reference.sh b/test/references/exact_summary_reference.sh index 6e487a3f..613981e3 100755 --- a/test/references/exact_summary_reference.sh +++ b/test/references/exact_summary_reference.sh @@ -101,6 +101,7 @@ while IFS=$'\t' read -r tag rest; do line="$(emit "$label" "${lists[@]}")" if (( count < n_samples )); then samples+=("$line"); else groups+=("$line"); fi count=$((count + 1)) ;; + *) ;; esac done < <(blocks) From 8721b446c92944a65703fe003ebb6d418a1acdb1 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 20:59:45 +0000 Subject: [PATCH 07/10] test: start the Julia reader without R's LD_LIBRARY_PATH and print its output on failure The R test that calls the Julia failure reporter saw a non-zero exit in CI while the reporter passes on the same CSV from a shell. Print what it said and start it without the library path R sets. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_0136P7ZmNJPhB7wGCRymdu1D --- test/unit/test_null_calibration_csv.R | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test/unit/test_null_calibration_csv.R b/test/unit/test_null_calibration_csv.R index bd7fa942..0c677a7c 100644 --- a/test/unit/test_null_calibration_csv.R +++ b/test/unit/test_null_calibration_csv.R @@ -65,12 +65,19 @@ stopifnot(grepl("unreadable result CSV", read_error, fixed = TRUE)) julia <- Sys.which("julia") stopifnot(nzchar(julia)) reader_script <- "bench/null_calibration/report_failures.jl" +# R puts its own libraries on LD_LIBRARY_PATH, and Julia ships its own copies of +# some of them, so Julia is started without the variable. annotations <- suppressWarnings(system2( - julia, - args = c("--startup-file=no", shQuote(reader_script), shQuote(cell)), + "env", + args = c("-u", "LD_LIBRARY_PATH", julia, "--startup-file=no", + shQuote(reader_script), shQuote(cell)), stdout = TRUE, stderr = TRUE, stdin = csv_path )) reader_status <- attr(annotations, "status") +if (!(is.null(reader_status) || identical(reader_status, 0L))) { + cat("report_failures.jl exited with status", reader_status, "\n", + paste(annotations, collapse = "\n"), "\n") +} expected_annotations <- c( paste0("::warning title=", cell, ": a requested test failed::", duplicate_test, " :: comma, quote ", double_quote, From 8ac3af2c988584aa3dbf322ff445287799e78250 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 21:01:12 +0000 Subject: [PATCH 08/10] ci: run the Agda memo steps under bash and hash the real image Container job steps default to sh, where [[ is not available, so the reuse step failed. The container image is not readable from the step context, so keep it as a literal in env, checked against the container line. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_0136P7ZmNJPhB7wGCRymdu1D --- .github/workflows/ci.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c86629b5..b639cf09 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -733,6 +733,9 @@ jobs: AGDA_BIN_VERSION: 2.6.4.3-1+b2 AGDA_STDLIB_VERSION: 2.1-4 AGDA_FLAGS: --safe --without-K --cubical-compatible + # The image of the container: above, the one literal that `container:` + # cannot read from env. The hash step refuses to run if the two differ. + AGDA_IMAGE: debian:trixie-20250929-slim@sha256:1caf1c703c8f7e15dcf2e7769b35000c764e6f50e4d7401c355fb0248f3ddfdb steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -744,12 +747,15 @@ jobs: # full, so the memo can only ever shorten a pull request, never main. - name: Hash the proof inputs id: inputs - env: - IMAGE: ${{ job.container.image }} + shell: bash run: | set -euo pipefail + # A container job's steps cannot read the container image from the + # context, so the image is a literal in env; it must be the one in use. + grep -qF "container: $AGDA_IMAGE" .github/workflows/ci.yml \ + || { echo "ERROR: AGDA_IMAGE differs from the container image" >&2; exit 1; } hash="$( { - printf '%s\n' "$IMAGE" "$AGDA_BIN_VERSION" "$AGDA_STDLIB_VERSION" "$AGDA_FLAGS" + printf '%s\n' "$AGDA_IMAGE" "$AGDA_BIN_VERSION" "$AGDA_STDLIB_VERSION" "$AGDA_FLAGS" find proofs -type f -print0 | LC_ALL=C sort -z | xargs -0 sha256sum } | sha256sum | cut -d' ' -f1 )" echo "hash=$hash" >> "$GITHUB_OUTPUT" @@ -769,6 +775,7 @@ jobs: - name: Reuse the earlier pass id: reuse + shell: bash if: github.event_name == 'pull_request' && steps.lookup.outputs.cache-matched-key != '' env: MATCHED: ${{ steps.lookup.outputs.cache-matched-key }} From 20b1a46e1d6f26a57530a0cb016aa4c630646695 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 21:13:57 +0000 Subject: [PATCH 09/10] Port the format-readiness harness from Python to Julia The five scripts under scripts/formats and their tests are Julia, standard library only. Supervision uses setsid, process groups, a subreaper and a procfs walk through ccall. The workflow installs the pinned Julia first and reads the ijson-jcs pin with a Julia script. The Python files are removed. Equivalence with the Python was checked on the repository inventory, a synthetic repository, probe_kyq error paths, the real pinned ijson-jcs corpus and the pin reader. Eighteen deliberate breaks each fail the Julia tests. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_0136P7ZmNJPhB7wGCRymdu1D --- .github/workflows/format-readiness.yml | 28 +- docs/format-migration.md | 8 +- scripts/formats/conformance.jl | 132 +++++++ scripts/formats/conformance.py | 103 ------ scripts/formats/descendants.jl | 163 +++++++++ scripts/formats/descendants.py | 115 ------ scripts/formats/inventory.jl | 172 +++++++++ scripts/formats/inventory.py | 104 ------ scripts/formats/probe_kyq.jl | 69 ++++ scripts/formats/probe_kyq.py | 53 --- scripts/formats/process.jl | 485 +++++++++++++++++++++++++ scripts/formats/process.py | 165 --------- scripts/formats/read_pins.jl | 20 + scripts/tests/test_format_readiness.jl | 368 +++++++++++++++++++ scripts/tests/test_format_readiness.py | 247 ------------- 15 files changed, 1420 insertions(+), 812 deletions(-) create mode 100644 scripts/formats/conformance.jl delete mode 100644 scripts/formats/conformance.py create mode 100644 scripts/formats/descendants.jl delete mode 100644 scripts/formats/descendants.py create mode 100644 scripts/formats/inventory.jl delete mode 100644 scripts/formats/inventory.py create mode 100644 scripts/formats/probe_kyq.jl delete mode 100644 scripts/formats/probe_kyq.py create mode 100644 scripts/formats/process.jl delete mode 100644 scripts/formats/process.py create mode 100644 scripts/formats/read_pins.jl create mode 100644 scripts/tests/test_format_readiness.jl delete mode 100644 scripts/tests/test_format_readiness.py diff --git a/.github/workflows/format-readiness.yml b/.github/workflows/format-readiness.yml index fa40fe50..9a4e39df 100644 --- a/.github/workflows/format-readiness.yml +++ b/.github/workflows/format-readiness.yml @@ -25,28 +25,16 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: Set up Julia + timeout-minutes: 10 + run: bash scripts/install-ci-julia.sh - name: Test harness failure boundaries - env: - PYTHONDONTWRITEBYTECODE: '1' - run: python3 -m unittest discover -s scripts/tests -p 'test_format*.py' -v + run: julia --startup-file=no scripts/tests/test_format_readiness.jl - name: Inventory candidates without declaring compliance - run: python3 scripts/formats/inventory.py > "$RUNNER_TEMP/format-boundaries.tsv" + run: julia --startup-file=no scripts/formats/inventory.jl > "$RUNNER_TEMP/format-boundaries.tsv" - name: Read immutable JSON reference pins id: pins - run: | - python3 - <<'PY' - import os, re, tomllib - with open('config/format-tools.toml', 'rb') as stream: - pin = tomllib.load(stream)['ijson_jcs'] - if pin['repository'] != 'https://github.com/hyperpolymath/ijson-jcs': - raise SystemExit('unexpected reference repository') - if not re.fullmatch(r'[0-9a-f]{40}', pin['revision']): - raise SystemExit('reference must be an immutable commit') - if not re.fullmatch(r'[0-9]+\.[0-9]+\.[0-9]+', pin['rust_toolchain']): - raise SystemExit('Rust must be an explicit release') - with open(os.environ['GITHUB_OUTPUT'], 'a') as stream: - stream.write(f"revision={pin['revision']}\nrust={pin['rust_toolchain']}\n") - PY + run: julia --startup-file=no scripts/formats/read_pins.jl - name: Checkout pinned reference source uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -64,10 +52,8 @@ jobs: rustup toolchain install "$RUST_VERSION" --profile minimal cargo "+${RUST_VERSION}" install --path .format-reference --locked --root "$RUNNER_TEMP/format-reference" - name: Exercise original bytes against the installed reference - env: - PYTHONDONTWRITEBYTECODE: '1' run: | - PATH="$RUNNER_TEMP/format-reference/bin:$PATH" python3 scripts/formats/conformance.py --adapter reference + PATH="$RUNNER_TEMP/format-reference/bin:$PATH" julia --startup-file=no scripts/formats/conformance.jl --adapter reference { echo '## Format migration evidence' echo 'Pinned ijson-jcs raw-byte corpus passed. This is not application-wide conformance.' diff --git a/docs/format-migration.md b/docs/format-migration.md index 0dbc7c8d..52f64e57 100644 --- a/docs/format-migration.md +++ b/docs/format-migration.md @@ -48,7 +48,7 @@ replace it with another language or publish another tool under the kyq name. Reproduce using a clean clone at the pinned commit: ```sh -PATH="/path/to/trusted/just/bin:$PATH" python3 scripts/formats/probe_kyq.py --source /path/to/kyq +PATH="/path/to/trusted/just/bin:$PATH" julia --startup-file=no scripts/formats/probe_kyq.jl --source /path/to/kyq ``` Probe exit codes: 0 means only the three source recipes returned success; @@ -61,7 +61,7 @@ conformance. The tool prints those missing obligations explicitly. Generate locations without printing source lines or study/credential values: ```sh -python3 scripts/formats/inventory.py > /tmp/format-boundaries.tsv +julia --startup-file=no scripts/formats/inventory.jl > /tmp/format-boundaries.tsv ``` The inventory includes tracked and non-ignored new files, hidden workflows and @@ -105,8 +105,8 @@ escaped and raw controls, integer boundaries, beyond-uint64 counts, exponent thresholds, negative zero, array order, trailing input and truncation. ```sh -PATH="/path/to/trusted/reference/bin:$PATH" python3 scripts/formats/conformance.py --adapter reference -python3 -m unittest discover -s scripts/tests -p 'test_format*.py' -v +PATH="/path/to/trusted/reference/bin:$PATH" julia --startup-file=no scripts/formats/conformance.jl --adapter reference +julia --startup-file=no scripts/tests/test_format_readiness.jl ``` The protocol is the existing ijson-jcs `canon`: original bytes on stdin, exact diff --git a/scripts/formats/conformance.jl b/scripts/formats/conformance.jl new file mode 100644 index 00000000..64682c01 --- /dev/null +++ b/scripts/formats/conformance.jl @@ -0,0 +1,132 @@ +#!/usr/bin/env julia +# SPDX-License-Identifier: MPL-2.0 +# Characterize an installed ijson-jcs-compatible `canon` adapter. +# +# Tests original bytes, exact stdout and documented exit codes. No parsing or +# canonicalization is implemented here. Exit 0 = corpus passed, 1 = mismatch, +# 2 = infrastructure/configuration failure. This is NOT kyq readiness approval. +isdefined(Main, :FormatProcess) || Base.include(Main, joinpath(@__DIR__, "process.jl")) + +module Conformance + +using TOML +using Main.FormatProcess: ToolError, ValueError, run_probe, pyname, CliOption, parse_cli + +const ROOT = dirname(dirname(dirname(realpath(@__FILE__)))) +const DESCRIPTION = "Characterize an installed ijson-jcs-compatible `canon` adapter." +const ALLOWED = Set(["id", "input", "hex", "valid", "canonical"]) + +# One corpus case: id, original bytes, expected verdict, expected canonical bytes. +const Case = NamedTuple{(:name, :data, :valid, :expected), + Tuple{String,Vector{UInt8},Bool,Vector{UInt8}}} + +# bytes.fromhex: pairs of hex digits, ASCII whitespace allowed between bytes. +function fromhex(text::AbstractString) + out = UInt8[] + chars = collect(text) + i = 1 + while i <= length(chars) + if chars[i] in (' ', '\t', '\n', '\r', '\f', '\v') + i += 1 + continue + end + (i + 1 <= length(chars) && isascii(chars[i]) && isascii(chars[i + 1]) && + isxdigit(chars[i]) && isxdigit(chars[i + 1])) || + throw(ValueError("non-hexadecimal number found in fromhex() arg")) + push!(out, parse(UInt8, string(chars[i], chars[i + 1]); base=16)) + i += 2 + end + return out +end + +function case_input(case) + haskey(case, "input") == haskey(case, "hex") && throw(ValueError("case needs exactly one input")) + raw = haskey(case, "input") ? case["input"] : case["hex"] + raw isa AbstractString || throw(ValueError("input must be a string or hex string")) + return haskey(case, "input") ? Vector{UInt8}(codeunits(raw)) : fromhex(raw) +end + +function parse_case(case) + if !(case isa AbstractDict) || !isempty(setdiff(Set(keys(case)), ALLOWED)) + throw(ValueError("invalid case fields")) + end + name = get(case, "id", "") + if !(name isa AbstractString) || !occursin(r"^[a-z0-9-]+\z", name) + throw(ValueError("invalid case id")) + end + get(case, "valid", nothing) isa Bool || throw(ValueError("case needs an explicit verdict")) + case["valid"] != haskey(case, "canonical") && + throw(ValueError("only valid cases must have canonical bytes")) + expected = get(case, "canonical", "") + expected isa AbstractString || throw(ValueError("canonical bytes must be a UTF-8 string")) + data, encoded = case_input(case), Vector{UInt8}(codeunits(expected)) + if length(data) > 131072 || length(encoded) > 131072 + throw(ValueError("case exceeds the corpus budget")) + end + return Case((String(name), data, case["valid"], encoded)) +end + +function load_cases(path) + # Paths come from this repo's constant fixture location or test fixtures, + # never a CLI-supplied filesystem path. No arbitrary-file reader CLI. + document = TOML.parse(String(read(path))) + (get(document, "schema", nothing) isa Int && document["schema"] == 1) || + throw(ValueError("unsupported corpus schema")) + Set(keys(document)) == Set(["schema", "case"]) || + throw(ValueError("unsupported corpus fields")) + cases = document["case"] + (cases isa AbstractVector && !isempty(cases)) || throw(ValueError("empty or invalid corpus")) + seen = Set{String}() + result = Case[] + for raw in cases + case = parse_case(raw) + case.name in seen && throw(ValueError("duplicate case id")) + push!(seen, case.name) + push!(result, case) + end + return result +end + +function check(adapter, cases; timeout=5.0) + operations = Dict("reference" => "ijson-canon", "candidate" => "kyq-canon") + operation = operations[adapter] + passed = true + for case in cases + result = run_probe(operation, case.data; timeout=timeout) + # Rejection must be the documented invalid-input exit, not a crash, + # missing library, usage failure from a wrapper, or arbitrary nonzero. + wanted = case.valid ? 0 : 2 + ok = result.code == wanted && result.stdout == case.expected + println(ok ? "PASS" : "FAIL", " ", case.name) + passed = passed && ok + end + return passed +end + +function main(argv=ARGS) + args = parse_cli("conformance.jl", DESCRIPTION, argv, [ + CliOption("adapter", "ADAPTER"; default="reference", choices=["reference", "candidate"]), + CliOption("timeout", "TIMEOUT"; default=5.0, convert=s -> parse(Float64, strip(s))), + ]) + try + # Validate the entire corpus before invoking the first child. + cases = load_cases(joinpath(ROOT, "test/fixtures/formats/ijson-jcs.toml")) + ok = check(args["adapter"], cases; timeout=args["timeout"]) + println("Corpus: $(length(cases)) cases; runtime migration remains separately gated") + return ok ? 0 : 1 + catch error + known = error isa ToolError || error isa ValueError || error isa Base.IOError || + error isa SystemError || error isa TOML.ParserError + known || rethrow() + # Do not echo paths, source text, child stderr or exception values. + name = error isa TOML.ParserError ? "TOMLDecodeError" : pyname(error) + println(stderr, "INFRASTRUCTURE: $name; no conformance pass") + return 2 + end +end + +end # module + +if abspath(PROGRAM_FILE) == @__FILE__ + exit(Conformance.main(ARGS)) +end diff --git a/scripts/formats/conformance.py b/scripts/formats/conformance.py deleted file mode 100644 index 6107ef48..00000000 --- a/scripts/formats/conformance.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MPL-2.0 -"""Characterize an installed ijson-jcs-compatible `canon` adapter. - -Tests original bytes, exact stdout and documented exit codes. No parsing or -canonicalization is implemented in Python. Exit 0 = corpus passed, 1 = mismatch, -2 = infrastructure/configuration failure. This is NOT kyq readiness approval. -""" -import argparse -from pathlib import Path -import re -import sys -import tomllib - -from process import ToolError, run - -ROOT = Path(__file__).resolve().parents[2] - - -def _input(case): - if ("input" in case) == ("hex" in case): - raise ValueError("case needs exactly one input") - raw = case.get("input", case.get("hex")) - if not isinstance(raw, str): - raise ValueError("input must be a string or hex string") - return raw.encode("utf-8") if "input" in case else bytes.fromhex(raw) - - -def _case(case): - allowed = {"id", "input", "hex", "valid", "canonical"} - if not isinstance(case, dict) or set(case) - allowed: - raise ValueError("invalid case fields") - name = case.get("id", "") - if not isinstance(name, str) or not re.fullmatch(r"[a-z0-9-]+", name): - raise ValueError("invalid case id") - if type(case.get("valid")) is not bool: - raise ValueError("case needs an explicit verdict") - if case["valid"] != ("canonical" in case): - raise ValueError("only valid cases must have canonical bytes") - expected = case.get("canonical", "") - if not isinstance(expected, str): - raise ValueError("canonical bytes must be a UTF-8 string") - data, encoded = _input(case), expected.encode("utf-8") - if len(data) > 131072 or len(encoded) > 131072: - raise ValueError("case exceeds the corpus budget") - return name, data, case["valid"], encoded - - -def load_cases(path): - # Paths come from this repo's constant fixture location or test fixtures, - # never a CLI-supplied filesystem path. No arbitrary-file reader CLI. - with Path(path).open("rb") as stream: - document = tomllib.load(stream) - if type(document.get("schema")) is not int or document["schema"] != 1: - raise ValueError("unsupported corpus schema") - if set(document) != {"schema", "case"}: - raise ValueError("unsupported corpus fields") - cases = document["case"] - if not isinstance(cases, list) or not cases: - raise ValueError("empty or invalid corpus") - seen = set() - for raw in cases: - case = _case(raw) - if case[0] in seen: - raise ValueError("duplicate case id") - seen.add(case[0]) - yield case - - -def check(adapter, cases, *, timeout=5.0): - operation = {"reference": "ijson-canon", "candidate": "kyq-canon"}[adapter] - passed = True - for name, data, valid, expected in cases: - result = run(operation, data, timeout=timeout) - # Rejection must be the documented invalid-input exit, not a crash, - # missing library, usage failure from a wrapper, or arbitrary nonzero. - wanted = 0 if valid else 2 - ok = result.code == wanted and result.stdout == expected - print(f"{'PASS' if ok else 'FAIL'} {name}") - passed = passed and ok - return passed - - -def main(argv=None): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--adapter", choices=("reference", "candidate"), default="reference", - help="named installed adapter on trusted PATH, never a command/path") - parser.add_argument("--timeout", type=float, default=5.0) - args = parser.parse_args(argv) - try: - # Validate the entire corpus before invoking the first child. - cases = list(load_cases(ROOT / "test/fixtures/formats/ijson-jcs.toml")) - ok = check(args.adapter, cases, timeout=args.timeout) - print(f"Corpus: {len(cases)} cases; runtime migration remains separately gated") - return 0 if ok else 1 - except (ToolError, ValueError, TypeError, OSError) as error: - # Do not echo paths, source text, child stderr or exception values. - print(f"INFRASTRUCTURE: {type(error).__name__}; no conformance pass", file=sys.stderr) - return 2 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/formats/descendants.jl b/scripts/formats/descendants.jl new file mode 100644 index 00000000..6ee008e3 --- /dev/null +++ b/scripts/formats/descendants.jl @@ -0,0 +1,163 @@ +# SPDX-License-Identifier: MPL-2.0 +# Linux-only, single-threaded supervision of trusted probe subprocesses. +# +# A subreaper adopts orphaned descendants even if they call setsid/double-fork. +# Restore the previous process-wide setting; refuse pre-existing child contexts. +# This is lifecycle management, not a sandbox for adversarial executables. +module FormatDescendants + +export SupervisionError, ProcessLookupError, supervise + +const PR_SET_CHILD_SUBREAPER = 36 +const PR_GET_CHILD_SUBREAPER = 37 +const SIGSTOP = 19 +const SIGKILL = 9 +const ENOENT = 2 +const ESRCH = 3 +const ECHILD = 10 +const WNOHANG = 1 + +struct SupervisionError <: Exception + msg::String +end +Base.showerror(io::IO, e::SupervisionError) = print(io, "SupervisionError: ", e.msg) + +# Stands in for Python's ProcessLookupError (errno ESRCH). +struct ProcessLookupError <: Exception end +Base.showerror(io::IO, ::ProcessLookupError) = print(io, "ProcessLookupError") + +# Filesystem seams. Tests replace these to simulate a missing procfs. +const LISTDIR = Ref{Function}(readdir) +const READTEXT = Ref{Function}(path -> read(path, String)) +const CHILDREN = Ref{Function}(pid -> _children(pid)) + +is_enoent(e) = (e isa Base.IOError && e.code == Base.UV_ENOENT) || + (e isa SystemError && e.errnum == ENOENT) + +children(pid) = CHILDREN[](pid) + +function _children(pid::Integer) + result = Set{Int}() + local tasks + try + tasks = LISTDIR[]("/proc/$pid/task") + catch error + is_enoent(error) || rethrow() + pid == getpid() && throw(SupervisionError("procfs child enumeration unavailable")) + return result + end + for task in tasks + try + for value in split(READTEXT[]("/proc/$pid/task/$task/children")) + push!(result, parse(Int, value)) + end + catch error + is_enoent(error) || rethrow() + if pid == getpid() && task == string(pid) + throw(SupervisionError("procfs children interface unavailable")) + end + # A different process/thread exited during enumeration. + end + end + return result +end + +# Returns true when the signal was delivered, false for ESRCH. +function send_signal(pid::Integer, signum::Integer) + ccall(:kill, Cint, (Cint, Cint), pid, signum) == 0 && return true + errno = Libc.errno() + errno == ESRCH && return false + throw(SystemError("kill", errno)) +end + +function kill_tree(pid::Integer) + pending, stopped = Int[pid], Int[] + while !isempty(pending) + current = pop!(pending) + # The process may exit after its parent list was read; it then has no + # live descendants to freeze. + send_signal(current, SIGSTOP) || continue + push!(stopped, current) + append!(pending, children(current)) + end + for current in reverse(stopped) + # A frozen process can still disappear if it was reaped concurrently. + send_signal(current, SIGKILL) + end +end + +# waitpid(WNOHANG); ECHILD means it was already reaped elsewhere. +function reap_nowait(pid::Integer) + status = Ref{Cint}(0) + result = ccall(:waitpid, Cint, (Cint, Ptr{Cint}, Cint), pid, status, WNOHANG) + if result == -1 && Libc.errno() != ECHILD + throw(SystemError("waitpid", Libc.errno())) + end + return nothing +end + +now_s() = time_ns() * 1e-9 + +function drain() + deadline = now_s() + 0.5 + while now_s() < deadline + found = children(getpid()) + isempty(found) && return + for pid in sort!(collect(found)) + kill_tree(pid) + reap_nowait(pid) + end + remaining = deadline - now_s() + # Bounded cleanup; also reap newly adopted orphans. + remaining > 0 && sleep(min(0.005, remaining)) + end + + # A final census lets a child that exited at the deadline be reaped, + # and catches anything newly adopted by the subreaper during cleanup. + found = children(getpid()) + isempty(found) && return + for pid in sort!(collect(found)) + kill_tree(pid) + reap_nowait(pid) + end + isempty(children(getpid())) || + throw(SupervisionError("descendants did not exit within cleanup budget")) + return +end + +function set_subreaper(value::Integer) + ccall(:prctl, Cint, (Cint, Culong, Culong, Culong, Culong), + PR_SET_CHILD_SUBREAPER, value, 0, 0, 0) == 0 || + throw(SupervisionError("cannot set subreaper state")) + return nothing +end + +""" + supervise(f) + +Run `f()` with this process as a child subreaper. Afterwards every adopted +descendant is stopped, killed and reaped, and the previous setting restored. +""" +function supervise(f) + if !Sys.islinux() || Threads.nthreads() != 1 + throw(SupervisionError("probe requires single-threaded Linux supervision")) + end + previous = Ref{Cint}(0) + ccall(:prctl, Cint, (Cint, Ptr{Cint}, Culong, Culong, Culong), + PR_GET_CHILD_SUBREAPER, previous, 0, 0, 0) == 0 || + throw(SupervisionError("cannot inspect subreaper state")) + isempty(children(getpid())) || + throw(SupervisionError("probe requires no pre-existing child processes")) + set_subreaper(1) + try + return f() + finally + try + drain() + finally + set_subreaper(previous[]) + end + end +end + +end # module diff --git a/scripts/formats/descendants.py b/scripts/formats/descendants.py deleted file mode 100644 index a49b9f43..00000000 --- a/scripts/formats/descendants.py +++ /dev/null @@ -1,115 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -"""Linux-only, single-threaded supervision of trusted probe subprocesses. - -A subreaper adopts orphaned descendants even if they call setsid/double-fork. -Restore the previous process-wide setting; refuse pre-existing child contexts. -This is lifecycle management, not a sandbox for adversarial executables. -""" -import ctypes -import os -from pathlib import Path -import signal -import sys -import threading -import time - -PR_SET_CHILD_SUBREAPER = 36 -PR_GET_CHILD_SUBREAPER = 37 - - -class SupervisionError(RuntimeError): - pass - - -def _children(pid): - result = set() - try: - tasks = list(Path(f"/proc/{pid}/task").iterdir()) - except FileNotFoundError as error: - if pid == os.getpid(): - raise SupervisionError("procfs child enumeration unavailable") from error - return result - for task in tasks: - try: - result.update(int(value) for value in (task / "children").read_text().split()) - except FileNotFoundError as error: - if pid == os.getpid() and task.name == str(pid): - raise SupervisionError("procfs children interface unavailable") from error - # A different process/thread exited during enumeration. - return result - - -def _kill_tree(pid): - pending, stopped = [pid], [] - while pending: - current = pending.pop() - try: - os.kill(current, signal.SIGSTOP) - except ProcessLookupError: - # The process exited after its parent list was read; it has no live descendants to freeze. - continue - stopped.append(current) - pending.extend(_children(current)) - for current in reversed(stopped): - try: - os.kill(current, signal.SIGKILL) - except ProcessLookupError: - # A frozen process can still disappear if it was reaped concurrently. - continue - - -class Supervise: - def __enter__(self): - if sys.platform != "linux" or threading.active_count() != 1: - raise SupervisionError("probe requires single-threaded Linux supervision") - self.libc = ctypes.CDLL(None, use_errno=True) - self.previous = ctypes.c_int() - if self.libc.prctl(PR_GET_CHILD_SUBREAPER, ctypes.byref(self.previous), 0, 0, 0) != 0: - raise SupervisionError("cannot inspect subreaper state") - if _children(os.getpid()): - raise SupervisionError("probe requires no pre-existing child processes") - self._set(1) - return self - - def _set(self, value): - if self.libc.prctl(PR_SET_CHILD_SUBREAPER, ctypes.c_ulong(value), 0, 0, 0) != 0: - raise SupervisionError("cannot set subreaper state") - - def _drain(self): - deadline = time.monotonic() + 0.5 - while time.monotonic() < deadline: - children = _children(os.getpid()) - if not children: - return - for pid in children: - _kill_tree(pid) - try: - os.waitpid(pid, os.WNOHANG) - except ChildProcessError: - # Popen.wait or an earlier nonblocking wait already reaped this child. - continue - remaining = deadline - time.monotonic() - if remaining > 0: - # Bounded cleanup; also reap newly adopted orphans. - time.sleep(min(0.005, remaining)) - - # A final census lets a child that exited at the deadline be reaped, - # and catches anything newly adopted by the subreaper during cleanup. - children = _children(os.getpid()) - if not children: - return - for pid in children: - _kill_tree(pid) - try: - os.waitpid(pid, os.WNOHANG) - except ChildProcessError: - # Another wait already reaped it during the final census. - continue - if _children(os.getpid()): - raise SupervisionError("descendants did not exit within cleanup budget") - - def __exit__(self, *_): - try: - self._drain() - finally: - self._set(self.previous.value) diff --git a/scripts/formats/inventory.jl b/scripts/formats/inventory.jl new file mode 100644 index 00000000..a1e05ff3 --- /dev/null +++ b/scripts/formats/inventory.jl @@ -0,0 +1,172 @@ +#!/usr/bin/env julia +# SPDX-License-Identifier: MPL-2.0 +# Inventory lexical JSON/YAML candidates; NEVER a compliance verdict. +# +# Includes tracked and non-ignored new files, including hidden workflow/hooks. +# Search hits can be comments/test strings. Aliases/dynamic dispatch require human +# review. Prints locations/operation names, not source lines or document contents. +isdefined(Main, :FormatProcess) || Base.include(Main, joinpath(@__DIR__, "process.jl")) + +module Inventory + +using Main.FormatProcess: CliOption, parse_cli + +const ROOT = dirname(dirname(dirname(realpath(@__FILE__)))) +const DESCRIPTION = "Inventory lexical JSON/YAML candidates; NEVER a compliance verdict." +const CODE = Set([".yaml", ".yml", ".kyaml", ".jl", ".ts", ".tsx", ".js", ".mjs", ".py", + ".sh", ".R", ".r", ".rs"]) +const DATA = Set([".json", ".jsonl", ".yaml", ".yml", ".kyaml"]) +# (*UCP) makes \b, \w and \s Unicode-aware, as they are for Python str patterns. +const PATTERNS = [ + "JSON3.read" => r"(*UCP)\bJSON3\.read\s*\(", + "JSON3.write" => r"(*UCP)\bJSON3\.write\s*\(", + "YAML IO" => r"(*UCP)\bYAML\.(?:load_file|load|write_file|write)\s*\(", + "strict/JCS adapter" => r"(*UCP)\b(?:ijson_read|jcs_bytes|jcs_sha256|ijsonJcsStringify|_request_json)\s*\(", + "JSON.parse" => r"(*UCP)\bJSON\.parse\s*\(", + "JSON.stringify" => r"(*UCP)\bJSON\.stringify\s*\(", + "response.json candidate" => r"(*UCP)\.json\s*\(", + "Python JSON IO" => r"(*UCP)\bjson\.(?:load|loads|dump|dumps)\s*\(", + "Oxygen JSON candidate" => r"(*UCP)(? r"(*UCP)\b(?:jsonlite::)?(?:fromJSON|toJSON|read_json|write_json)\s*\(", + "shell jq/yq candidate" => r"(*UCP)(?m)(? "\n"), "\r" => "\n") +end + +function file_rows(emit, root, name) + path = joinpath(root, name) + bucket = category(name) + if islink(path) + emit(Row(("symlink", bucket, name, 0, "target-not-followed"))) + return + end + if !isfile(path) + emit(Row(("missing", bucket, name, 0, "missing-worktree-file"))) + return + end + base = basename(name) + suffix = suffix_of(base) + if suffix in DATA || base == "renv.lock" || base == "bun.lock" + emit(Row(("document", bucket, name, 0, "format-owner-needs-review"))) + end + suffix in CODE || return + local text + try + text = read_text(path) + catch error + error isa InvalidUTF8 || rethrow() + emit(Row(("unreadable", bucket, name, 0, "invalid-utf8-source"))) + return + end + codeunit_text = codeunits(text) + for (label, pattern) in PATTERNS + for match in eachmatch(pattern, text) + line = count(==(0x0a), view(codeunit_text, 1:(match.offset - 1))) + 1 + emit(Row(("candidate", bucket, name, line, label))) + end + end +end + +# Streams rows to `emit`, like the generator it replaces: an error part-way +# leaves the earlier rows already emitted. +function each_row(emit, root) + raw = read(pipeline(Cmd(`git ls-files --cached --others --exclude-standard -z`; dir=root); + stderr=devnull)) + names = String[] + start = 1 + for stop in vcat(findall(==(0x00), raw), length(raw) + 1) + piece = raw[start:(stop - 1)] + start = stop + 1 + isempty(piece) && continue + isvalid(String, piece) || throw(InvalidUTF8()) + push!(names, String(piece)) + end + for name in sort!(unique!(names)) + file_rows(emit, root, name) + end +end + +function inventory(root) + rows = Row[] + each_row(row -> push!(rows, row), root) + return rows +end + +# csv.writer(delimiter="\t", lineterminator="\n"), QUOTE_MINIMAL. +function csv_field(value) + text = string(value) + if occursin(r"[\t\"\n]", text) + return "\"" * replace(text, "\"" => "\"\"") * "\"" + end + return text +end +write_row(io, values) = print(io, join(map(csv_field, Tuple(values)), '\t'), '\n') + +function main(argv=ARGS) + args = parse_cli("inventory.jl", DESCRIPTION, argv, [ + CliOption("root", "ROOT"; default=ROOT), + ]) + root = try + realpath(args["root"]) + catch + abspath(args["root"]) + end + write_row(stdout, ("kind", "routing-bucket-not-verdict", "path", "line", "operation")) + try + each_row(row -> write_row(stdout, row), root) + return 0 + catch error + known = error isa Base.IOError || error isa SystemError || error isa InvalidUTF8 || + error isa ProcessFailedException + known || rethrow() + println(stderr, "inventory failed; no completeness or compliance claim") + return 2 + end +end + +end # module + +if abspath(PROGRAM_FILE) == @__FILE__ + exit(Inventory.main(ARGS)) +end diff --git a/scripts/formats/inventory.py b/scripts/formats/inventory.py deleted file mode 100644 index e637c8fa..00000000 --- a/scripts/formats/inventory.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MPL-2.0 -"""Inventory lexical JSON/YAML candidates; NEVER a compliance verdict. - -Includes tracked and non-ignored new files, including hidden workflow/hooks. -Search hits can be comments/test strings. Aliases/dynamic dispatch require human -review. Prints locations/operation names, not source lines or document contents. -""" -import argparse -import csv -from pathlib import Path -import re -import subprocess -import sys - -ROOT = Path(__file__).resolve().parents[2] -CODE = {".yaml", ".yml", ".kyaml", ".jl", ".ts", ".tsx", ".js", ".mjs", ".py", ".sh", ".R", ".r", ".rs"} -DATA = {".json", ".jsonl", ".yaml", ".yml", ".kyaml"} -PATTERNS = { - "JSON3.read": r"\bJSON3\.read\s*\(", - "JSON3.write": r"\bJSON3\.write\s*\(", - "YAML IO": r"\bYAML\.(?:load_file|load|write_file|write)\s*\(", - "strict/JCS adapter": r"\b(?:ijson_read|jcs_bytes|jcs_sha256|ijsonJcsStringify|_request_json)\s*\(", - "JSON.parse": r"\bJSON\.parse\s*\(", - "JSON.stringify": r"\bJSON\.stringify\s*\(", - "response.json candidate": r"\.json\s*\(", - "Python JSON IO": r"\bjson\.(?:load|loads|dump|dumps)\s*\(", - "Oxygen JSON candidate": r"(? parse(Float64, strip(s))), + ]) + try + return probe(args["source"], args["timeout"]) + catch error + known = error isa ToolError || error isa ValueError || error isa Base.IOError || + error isa SystemError || error isa KeyError || error isa TOML.ParserError + known || rethrow() + name = error isa TOML.ParserError ? "TOMLDecodeError" : pyname(error) + println(stderr, "INFRASTRUCTURE: $name; kyq readiness not established") + return 2 + end +end + +end # module + +if abspath(PROGRAM_FILE) == @__FILE__ + exit(ProbeKyq.main(ARGS)) +end diff --git a/scripts/formats/probe_kyq.py b/scripts/formats/probe_kyq.py deleted file mode 100644 index 33a481e2..00000000 --- a/scripts/formats/probe_kyq.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MPL-2.0 -"""Probe pinned kyq source recipes without installing or rewriting anything. - -Exit 0 means ONLY that build/wasm/test returned success. It does not establish -parser conformance, an installed executable, or readiness for application use. -Exit 1 = recipe failure; 2 = missing tools, wrong source pin, or other infra. -""" -import argparse -from pathlib import Path -import sys -import tomllib - -from process import ToolError, run - -ROOT = Path(__file__).resolve().parents[2] - - -def probe(source, timeout): - with (ROOT / "config/format-tools.toml").open("rb") as stream: - pin = tomllib.load(stream)["kyq"] - actual = run("git-head", cwd=source) - clean = run("git-status", cwd=source) - if actual.code or actual.stdout.decode().strip() != pin["revision"] or clean.code or clean.stdout: - raise ToolError("source is not the clean pinned revision") - version = run("just-version") - if version.code or version.stdout.decode().strip() != f"just {pin['just_version']}": - raise ToolError("unexpected just version") - ok = True - for recipe in ("build", "wasm", "test"): - result = run(f"just-{recipe}", cwd=source, timeout=timeout) - print(f"{recipe}: exit={result.code}") - ok = ok and result.code == 0 - print("install: NOT_RUN (no isolated-prefix contract has been established)") - print("parser/CLI conformance: NOT_RUN (separate evidence required)") - print("MetaManifold migration: BLOCKED pending runtime conformance and isolated installation") - return 0 if ok else 1 - - -def main(argv=None): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--source", type=Path, required=True) - parser.add_argument("--timeout", type=float, default=120.0) - args = parser.parse_args(argv) - try: - return probe(args.source, args.timeout) - except (ToolError, OSError, ValueError, KeyError) as error: - print(f"INFRASTRUCTURE: {type(error).__name__}; kyq readiness not established", file=sys.stderr) - return 2 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/formats/process.jl b/scripts/formats/process.jl new file mode 100644 index 00000000..6a34ed70 --- /dev/null +++ b/scripts/formats/process.jl @@ -0,0 +1,485 @@ +# SPDX-License-Identifier: MPL-2.0 +# Bounded Linux IO for a deliberately closed set of format-probe commands. +# +# Installed tools are trusted executables, not sandboxed plugins. A trusted PATH +# provides their location; no program paths or argument strings come from a CLI. +# Child diagnostics can contain secrets and are not printed by the probe CLIs. +isdefined(Main, :FormatDescendants) || Base.include(Main, joinpath(@__DIR__, "descendants.jl")) + +module FormatProcess + +using Main.FormatDescendants: SupervisionError, ProcessLookupError, supervise +import Main.FormatDescendants + +export ToolError, ValueError, Result, run_probe, pyname, CliOption, parse_cli + +const TIMEOUT = "tool exceeded its time budget" +const COMMANDS = Dict{String,Vector{String}}( + "ijson-canon" => ["ijson-jcs", "canon"], + "kyq-canon" => ["kyq-json-adapter", "canon"], + "git-head" => ["git", "rev-parse", "HEAD"], + "git-status" => ["git", "status", "--porcelain", "--untracked-files=all"], + "just-version" => ["just", "--version"], + "just-build" => ["just", "build"], + "just-wasm" => ["just", "wasm"], + "just-test" => ["just", "test"], +) + +const SIGKILL = 9 +const EAGAIN = 11 +const EINTR = 4 +const EPIPE = 32 +const O_CLOEXEC = 0o2000000 +const O_NONBLOCK = 0o4000 +const F_GETFL = 3 +const F_SETFL = 4 +const POLLIN = 0x0001 +const POLLOUT = 0x0004 +const CHUNK = 16384 + +"Infrastructure failure; distinct from a tool rejecting a document." +struct ToolError <: Exception + msg::String +end +Base.showerror(io::IO, e::ToolError) = print(io, "ToolError: ", e.msg) + +"Stands in for Python's ValueError family; `name` is the Python class name reported." +struct ValueError <: Exception + msg::String + name::String +end +ValueError(msg::AbstractString) = ValueError(String(msg), "ValueError") +Base.showerror(io::IO, e::ValueError) = print(io, e.name, ": ", e.msg) + +struct Result + code::Int + stdout::Vector{UInt8} + stderr::Vector{UInt8} +end + +struct WaitTimeout <: Exception end + +# The Python exception class name that the original CLIs print in diagnostics. +function pyname(e) + e isa ToolError && return "ToolError" + e isa ValueError && return e.name + e isa Base.IOError && return e.code == Base.UV_ENOENT ? "FileNotFoundError" : "OSError" + e isa SystemError && return e.errnum == 2 ? "FileNotFoundError" : "OSError" + e isa ProcessLookupError && return "ProcessLookupError" + e isa KeyError && return "KeyError" + return string(nameof(typeof(e))) +end + +# ---- command resolution ------------------------------------------------- + +# Same lookup rules as Python's shutil.which on POSIX. +function which(name::AbstractString) + path = get(ENV, "PATH", nothing) + path === nothing && (path = "/bin:/usr/bin") + isempty(path) && return nothing + seen = Set{String}() + for dir in split(path, ':'; keepempty=true) + dir in seen && continue + push!(seen, dir) + candidate = joinpath(dir, name) + if ispath(candidate) && Base.Filesystem.isexecutable(candidate) && !isdir(candidate) + return candidate + end + end + return nothing +end + +function command(name) + if !(name isa AbstractString) || !haskey(COMMANDS, name) + throw(ToolError("unknown probe operation")) + end + # Only select complete, literal commands. Neither program paths nor argv + # strings from a CLI are passed through to process creation. + argv = COMMANDS[name] + location = which(argv[1]) + location === nothing && throw(ToolError("tool not found on trusted PATH")) + basename(realpath(location)) == argv[1] || + throw(ToolError("tool resolves to an unrelated executable")) + return argv +end + +# ---- raw descriptor helpers --------------------------------------------- + +close_fd(fd::Integer) = fd >= 0 && ccall(:close, Cint, (Cint,), fd) + +function make_pipe() + fds = Vector{Cint}(undef, 2) + ccall(:pipe2, Cint, (Ptr{Cint}, Cint), fds, O_CLOEXEC) == 0 || + throw(SystemError("pipe2", Libc.errno())) + return Int(fds[1]), Int(fds[2]) +end + +function set_nonblocking(fd::Integer) + flags = ccall(:fcntl, Cint, (Cint, Cint, Cint), fd, F_GETFL, 0) + flags < 0 && throw(SystemError("fcntl", Libc.errno())) + ccall(:fcntl, Cint, (Cint, Cint, Cint), fd, F_SETFL, flags | O_NONBLOCK) < 0 && + throw(SystemError("fcntl", Libc.errno())) + return nothing +end + +# ---- child abstraction (the tests substitute a mock for the cleanup cases) -- + +mutable struct Child + proc::Base.Process + pid::Int + stdin_fd::Int + stdout_fd::Int + stderr_fd::Int +end + +child_pid(c::Child) = c.pid + +function exit_code(c::Child) + p = c.proc + return p.termsignal != 0 ? -Int(p.termsignal) : Int(p.exitcode) +end + +# Non-blocking exit check; the exit is only observed while libuv's loop runs. +function child_poll(c::Child) + Base.process_exited(c.proc) || sleep(0.0005) + return Base.process_exited(c.proc) ? exit_code(c) : nothing +end + +function child_kill(c::Child) + child_poll(c) === nothing || return nothing + if ccall(:kill, Cint, (Cint, Cint), c.pid, SIGKILL) != 0 + Libc.errno() == 3 || throw(SystemError("kill", Libc.errno())) + end + return nothing +end + +function child_wait(c::Child, timeout::Real) + ok = timedwait(() -> Base.process_exited(c.proc), Float64(timeout); pollint=0.002) + ok === :ok || throw(WaitTimeout()) + return exit_code(c) +end + +function close_streams(c::Child) + for field in (:stdin_fd, :stdout_fd, :stderr_fd) + fd = getfield(c, field) + if fd >= 0 + setfield!(c, field, -1) + close_fd(fd) + end + end + return nothing +end + +function spawn_child(argv::Vector{String}, cwd) + in_r, in_w = make_pipe() + local out_r, out_w, err_r, err_w + out_r = out_w = err_r = err_w = -1 + local proc + try + out_r, out_w = make_pipe() + err_r, err_w = make_pipe() + # detach => the child calls setsid(): its own session and process group. + cmd = Cmd(Cmd(argv); detach=true, dir=cwd === nothing ? "" : String(cwd)) + proc = Base.run(pipeline(cmd; stdin=RawFD(in_r), stdout=RawFD(out_w), + stderr=RawFD(err_w)); wait=false) + catch + for fd in (in_r, in_w, out_r, out_w, err_r, err_w) + close_fd(fd) + end + rethrow() + end + close_fd(in_r); close_fd(out_w); close_fd(err_w) + set_nonblocking(in_w); set_nonblocking(out_r); set_nonblocking(err_r) + return Child(proc, Int(getpid(proc)), in_w, out_r, err_r) +end + +# Seam so a test can prove nothing is launched. +const SPAWN = Ref{Function}(spawn_child) + +# ---- IO pump ------------------------------------------------------------- + +struct PollFD + fd::Cint + events::Cshort + revents::Cshort +end + +mutable struct Pump + child::Child + data::Vector{UInt8} + limit::Int + offset::Int + total::Int + out::Vector{UInt8} + err::Vector{UInt8} +end +Pump(child, data, limit) = Pump(child, data, limit, 0, 0, UInt8[], UInt8[]) + +now_s() = time_ns() * 1e-9 + +function remaining(deadline) + left = deadline - now_s() + left <= 0 && throw(ToolError(TIMEOUT)) + return left +end + +function close_field!(c::Child, field::Symbol) + fd = getfield(c, field) + setfield!(c, field, -1) + close_fd(fd) +end + +function write_input!(p::Pump) + c = p.child + n = min(CHUNK, length(p.data) - p.offset) + data = p.data + written = GC.@preserve data ccall(:write, Cssize_t, (Cint, Ptr{UInt8}, Csize_t), + c.stdin_fd, pointer(data, p.offset + 1), n) + if written < 0 + errno = Libc.errno() + if errno == EPIPE + p.offset = length(p.data) + elseif errno == EAGAIN || errno == EINTR + return + else + throw(SystemError("write", errno)) + end + else + p.offset += written + end + p.offset == length(p.data) && close_field!(c, :stdin_fd) + return +end + +function read_output!(p::Pump, field::Symbol, sink::Vector{UInt8}) + c = p.child + buffer = Vector{UInt8}(undef, CHUNK) + got = GC.@preserve buffer ccall(:read, Cssize_t, (Cint, Ptr{UInt8}, Csize_t), + getfield(c, field), pointer(buffer), CHUNK) + if got < 0 + errno = Libc.errno() + (errno == EAGAIN || errno == EINTR) && return + throw(SystemError("read", errno)) + elseif got == 0 + close_field!(c, field) + return + end + p.total += got + p.total > p.limit && throw(ToolError("tool exceeded its output budget")) + append!(sink, view(buffer, 1:got)) + return +end + +function collect_output(p::Pump, deadline) + c = p.child + isempty(p.data) && close_field!(c, :stdin_fd) + while c.stdout_fd >= 0 || c.stderr_fd >= 0 || c.stdin_fd >= 0 + left = remaining(deadline) + polls = PollFD[] + fields = Symbol[] + for (field, event) in ((:stdout_fd, POLLIN), (:stderr_fd, POLLIN), (:stdin_fd, POLLOUT)) + fd = getfield(c, field) + fd >= 0 || continue + push!(polls, PollFD(fd, event, 0)) + push!(fields, field) + end + millis = clamp(ceil(Int, left * 1000), 1, typemax(Int32)) + ready = ccall(:poll, Cint, (Ptr{PollFD}, Culong, Cint), polls, length(polls), millis) + if ready < 0 + errno = Libc.errno() + errno == EINTR && continue + throw(SystemError("poll", errno)) + end + for (entry, field) in zip(polls, fields) + entry.revents == 0 && continue + if field === :stdin_fd + write_input!(p) + elseif field === :stdout_fd + read_output!(p, field, p.out) + else + read_output!(p, field, p.err) + end + end + end + code = try + child_wait(c, remaining(deadline)) + catch error + error isa WaitTimeout ? throw(ToolError(TIMEOUT)) : rethrow() + end + return Result(code, copy(p.out), copy(p.err)) +end + +# ---- cleanup ------------------------------------------------------------ + +function killpg_real(pid::Integer) + ccall(:killpg, Cint, (Cint, Cint), pid, SIGKILL) == 0 && return nothing + errno = Libc.errno() + errno == 3 && throw(ProcessLookupError()) + throw(SystemError("killpg", errno)) +end + +# Fast group cleanup; supervise also adopts/kills detached descendants. +function cleanup(child; killpg=killpg_real) + try + try + killpg(child_pid(child)) + catch error + error isa ProcessLookupError || rethrow() + # A tool can leave its session; do not mistake a missing process + # group for a stopped direct child. + if child_poll(child) === nothing + try + child_kill(child) + catch error2 + error2 isa ProcessLookupError || rethrow() + # The child may exit between poll and kill; verify that + # race instead of silently discarding an unexpected live PID. + if child_poll(child) === nothing + throw(ToolError( + "tool escaped its process group and could not be terminated")) + end + end + end + end + try + child_wait(child, 0.5) + catch error + error isa WaitTimeout || rethrow() + throw(ToolError("tool did not exit within cleanup budget")) + end + finally + close_streams(child) + end + return nothing +end + +# ---- public entry point ------------------------------------------------- + +function run_probe(operation, data::AbstractVector{UInt8}=UInt8[]; cwd=nothing, + timeout::Real=5.0, limit::Integer=131072) + if !isfinite(timeout) || timeout <= 0 || limit <= 0 || length(data) > limit + throw(ToolError("invalid process budget or oversized test input")) + end + bytes = Vector{UInt8}(data) + try + return supervise() do + child = SPAWN[](command(operation), cwd) + try + collect_output(Pump(child, bytes, Int(limit)), now_s() + timeout) + finally + cleanup(child) + end + end + catch error + if error isa SystemError || error isa Base.IOError || error isa SupervisionError || + error isa ProcessLookupError + throw(ToolError("tool startup or descendant supervision failed")) + end + rethrow() + end +end + +# ---- minimal argparse stand-in ------------------------------------------- + +"One `--name VALUE` option. `convert` maps the string or throws; `choices` may be empty." +struct CliOption + name::String + metavar::String + default::Any + required::Bool + convert::Function + choices::Vector{String} +end +CliOption(name, metavar; default=nothing, required=false, convert=identity, choices=String[]) = + CliOption(name, metavar, default, required, convert, choices) + +function cli_usage(prog, options) + parts = String["[-h]"] + for o in options + shown = isempty(o.choices) ? o.metavar : "{" * join(o.choices, ",") * "}" + text = "--$(o.name) $shown" + push!(parts, o.required ? text : "[" * text * "]") + end + return "usage: $prog " * join(parts, " ") +end + +function cli_fail(prog, options, message) + println(stderr, cli_usage(prog, options)) + println(stderr, "$prog: error: $message") + exit(2) +end + +""" + parse_cli(prog, description, argv, options) -> Dict{String,Any} + +Supports `--name value`, `--name=value`, unique long-option prefixes and `-h`. +Exits with status 2 on a usage error, like argparse. +""" +function parse_cli(prog, description, argv, options::Vector{CliOption}) + values = Dict{String,Any}(o.name => o.default for o in options) + given = Set{String}() + unrecognized = String[] + index = 1 + while index <= length(argv) + arg = String(argv[index]) + index += 1 + if arg == "-h" || arg == "--help" + println(cli_usage(prog, options)) + println() + println(description) + exit(0) + end + if startswith(arg, "--") && length(arg) > 2 + key, eq, inline = partition(arg[3:end], '=') + hits = [o for o in options if o.name == key] + if isempty(hits) + hits = [o for o in options if startswith(o.name, key)] + end + if length(hits) > 1 + cli_fail(prog, options, "ambiguous option: --$key could match " * + join(["--" * o.name for o in hits], ", ")) + elseif isempty(hits) + push!(unrecognized, arg) + continue + end + option = hits[1] + if eq + value = inline + else + if index > length(argv) || + (startswith(argv[index], "-") && !occursin(r"^-(\d+\.?\d*|\.\d+)$", argv[index])) + cli_fail(prog, options, "argument --$(option.name): expected one argument") + end + value = String(argv[index]) + index += 1 + end + if !isempty(option.choices) && !(value in option.choices) + cli_fail(prog, options, "argument --$(option.name): invalid choice: '$value' " * + "(choose from " * join(["'$c'" for c in option.choices], ", ") * ")") + end + converted = try + option.convert(value) + catch + cli_fail(prog, options, "argument --$(option.name): invalid value: '$value'") + end + values[option.name] = converted + push!(given, option.name) + else + push!(unrecognized, arg) + end + end + missing_required = ["--" * o.name for o in options if o.required && !(o.name in given)] + isempty(missing_required) || + cli_fail(prog, options, "the following arguments are required: " * join(missing_required, ", ")) + isempty(unrecognized) || + cli_fail(prog, options, "unrecognized arguments: " * join(unrecognized, " ")) + return values +end + +function partition(text, separator) + i = findfirst(separator, text) + i === nothing && return text, false, "" + return text[1:prevind(text, i)], true, text[nextind(text, i):end] +end + +end # module diff --git a/scripts/formats/process.py b/scripts/formats/process.py deleted file mode 100644 index afbf44a3..00000000 --- a/scripts/formats/process.py +++ /dev/null @@ -1,165 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -"""Bounded Linux IO for a deliberately closed set of format-probe commands. - -Installed tools are trusted executables, not sandboxed plugins. A trusted PATH -provides their location; no program paths or argument strings come from a CLI. -Child diagnostics can contain secrets and are not printed by the probe CLIs. -""" -import math -import os -from pathlib import Path -import selectors -import shutil -import signal -import subprocess -import time -from dataclasses import dataclass - -from descendants import Supervise, SupervisionError - -TIMEOUT = "tool exceeded its time budget" -_COMMANDS = { - "ijson-canon": ("ijson-jcs", "canon"), - "kyq-canon": ("kyq-json-adapter", "canon"), - "git-head": ("git", "rev-parse", "HEAD"), - "git-status": ("git", "status", "--porcelain", "--untracked-files=all"), - "just-version": ("just", "--version"), - "just-build": ("just", "build"), - "just-wasm": ("just", "wasm"), - "just-test": ("just", "test"), -} - - -class ToolError(RuntimeError): - """Infrastructure failure; distinct from a tool rejecting a document.""" - - -@dataclass(frozen=True) -class Result: - code: int - stdout: bytes - stderr: bytes - - -def _command(name): - if not isinstance(name, str) or name not in _COMMANDS: - raise ToolError("unknown probe operation") - # Only select complete, literal commands. Neither program paths nor argv - # strings from a CLI are passed through to process creation. - argv = _COMMANDS[name] - location = shutil.which(argv[0]) - if location is None: - raise ToolError("tool not found on trusted PATH") - if Path(location).resolve(strict=True).name != argv[0]: - raise ToolError("tool resolves to an unrelated executable") - return argv - - -class _Pump: - def __init__(self, child, data, limit): - self.child, self.data, self.limit = child, data, limit - self.offset = 0 - self.total = 0 - self.buffers = {"stdout": bytearray(), "stderr": bytearray()} - - def register(self, selector): - for stream, label in ((self.child.stdout, "stdout"), (self.child.stderr, "stderr")): - os.set_blocking(stream.fileno(), False) - selector.register(stream, selectors.EVENT_READ, label) - if not self.data: - self.child.stdin.close() - return - os.set_blocking(self.child.stdin.fileno(), False) - selector.register(self.child.stdin, selectors.EVENT_WRITE, "stdin") - - def write_input(self, selector, stream): - try: - self.offset += os.write(stream.fileno(), self.data[self.offset:self.offset + 16384]) - except BrokenPipeError: - self.offset = len(self.data) - except BlockingIOError: - return - if self.offset == len(self.data): - selector.unregister(stream) - stream.close() - - def read_output(self, selector, stream, label): - try: - chunk = os.read(stream.fileno(), 16384) - except BlockingIOError: - return - if not chunk: - selector.unregister(stream) - stream.close() - return - self.total += len(chunk) - if self.total > self.limit: - raise ToolError("tool exceeded its output budget") - self.buffers[label].extend(chunk) - - def collect(self, deadline): - with selectors.DefaultSelector() as selector: - self.register(selector) - while selector.get_map(): - for key, _ in selector.select(_remaining(deadline)): - if key.data == "stdin": - self.write_input(selector, key.fileobj) - else: - self.read_output(selector, key.fileobj, key.data) - try: - code = self.child.wait(timeout=_remaining(deadline)) - except subprocess.TimeoutExpired as error: - raise ToolError(TIMEOUT) from error - return Result(code, bytes(self.buffers["stdout"]), bytes(self.buffers["stderr"])) - - -def _remaining(deadline): - remaining = deadline - time.monotonic() - if remaining <= 0: - raise ToolError(TIMEOUT) - return remaining - - -def _cleanup(child): - # Fast group cleanup; Supervise also adopts/kills detached descendants. - try: - try: - os.killpg(child.pid, signal.SIGKILL) - except ProcessLookupError: - # A tool can leave its session; do not mistake a missing process - # group for a stopped direct child. - if child.poll() is None: - try: - child.kill() - except ProcessLookupError as error: - # The child may exit between poll() and kill(); verify that - # race instead of silently discarding an unexpected live PID. - if child.poll() is None: - raise ToolError( - "tool escaped its process group and could not be terminated" - ) from error - try: - child.wait(timeout=0.5) - except subprocess.TimeoutExpired as error: - raise ToolError("tool did not exit within cleanup budget") from error - finally: - for stream in (child.stdin, child.stdout, child.stderr): - if not stream.closed: - stream.close() - - -def run(operation, data=b"", *, cwd=None, timeout=5.0, limit=131072): - if not math.isfinite(timeout) or timeout <= 0 or limit <= 0 or len(data) > limit: - raise ToolError("invalid process budget or oversized test input") - try: - with Supervise(): - child = subprocess.Popen( - _command(operation), cwd=cwd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, start_new_session=True, shell=False, - ) - try: - return _Pump(child, data, limit).collect(time.monotonic() + timeout) - finally: - _cleanup(child) - except (OSError, SupervisionError) as error: - raise ToolError("tool startup or descendant supervision failed") from error diff --git a/scripts/formats/read_pins.jl b/scripts/formats/read_pins.jl new file mode 100644 index 00000000..77085e08 --- /dev/null +++ b/scripts/formats/read_pins.jl @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: MPL-2.0 +# Read and validate the immutable ijson-jcs reference pins for the format-readiness +# workflow. Run from the repository root; appends `revision=` and `rust=` lines to +# the file named by GITHUB_OUTPUT. Exits 1 with a short message on any bad pin. +using TOML + +function fail(message) + println(stderr, message) + exit(1) +end + +pin = TOML.parsefile("config/format-tools.toml")["ijson_jcs"] +pin["repository"] == "https://github.com/hyperpolymath/ijson-jcs" || + fail("unexpected reference repository") +occursin(r"^[0-9a-f]{40}\z", pin["revision"]) || fail("reference must be an immutable commit") +occursin(r"^[0-9]+\.[0-9]+\.[0-9]+\z", pin["rust_toolchain"]) || + fail("Rust must be an explicit release") +open(ENV["GITHUB_OUTPUT"], "a") do stream + write(stream, "revision=$(pin["revision"])\nrust=$(pin["rust_toolchain"])\n") +end diff --git a/scripts/tests/test_format_readiness.jl b/scripts/tests/test_format_readiness.jl new file mode 100644 index 00000000..ec08a186 --- /dev/null +++ b/scripts/tests/test_format_readiness.jl @@ -0,0 +1,368 @@ +# SPDX-License-Identifier: MPL-2.0 +# Tests of the harness itself; fake children are NOT parser conformance evidence. +# +# Run: julia --startup-file=no scripts/tests/test_format_readiness.jl +# Fake installed tools are small /bin/sh scripts, never a production interpreter command. +using Test +using TOML + +const ROOT = dirname(dirname(dirname(realpath(@__FILE__)))) +include(joinpath(ROOT, "scripts/formats/conformance.jl")) +include(joinpath(ROOT, "scripts/formats/inventory.jl")) +include(joinpath(ROOT, "scripts/formats/probe_kyq.jl")) +using Main.FormatProcess: ToolError, run_probe +using Main.FormatDescendants: ProcessLookupError + +# ---- helpers ------------------------------------------------------------- + +# Run `f` with stdout and stderr captured; returns (value, stdout, stderr). +function capture(f) + mktempdir() do tmp + out_path, err_path = joinpath(tmp, "out"), joinpath(tmp, "err") + value = open(out_path, "w") do out + open(err_path, "w") do err + redirect_stdout(out) do + redirect_stderr(err) do + f() + end + end + end + end + return value, read(out_path, String), read(err_path, String) + end +end + +function write_tool(directory, body) + path = joinpath(directory, "ijson-jcs") + write(path, "#!/bin/sh\n$body\n") + chmod(path, 0o700) + return path +end + +path_with(directory) = directory * ":" * ENV["PATH"] + +# A fake installed protocol adapter named ijson-jcs, first on PATH. +function run_fake(body; data=UInt8[], kwargs...) + mktempdir(; prefix="format tool ") do tmp + write_tool(tmp, body) + withenv("PATH" => path_with(tmp)) do + run_probe("ijson-canon", data; kwargs...) + end + end +end + +# True when `f` throws a ToolError whose message matches `pattern`. +function tool_error(f, pattern=nothing) + try + f() + catch error + return error isa ToolError && (pattern === nothing || occursin(pattern, error.msg)) + end + return false +end + +# Corpus rejection: the Python ValueError/TypeError family, including TOML decode errors. +function rejects_corpus(f) + try + f() + catch error + return error isa Main.FormatProcess.ValueError || error isa TOML.ParserError + end + return false +end + +mkcase(name, data, valid, expected) = + Conformance.Case((name, Vector{UInt8}(codeunits(data)), valid, Vector{UInt8}(codeunits(expected)))) + +# Stands in for the Python Mock child in the cleanup tests. +mutable struct MockChild + pid::Int + polls::Vector{Any} + poll_calls::Int + kill_error::Bool + kills::Int + waits::Vector{Float64} + wait_result::Int + closed::Bool +end +MockChild(pid; polls=Any[nothing], kill_error=false) = + MockChild(pid, polls, 0, kill_error, 0, Float64[], -9, false) +Main.FormatProcess.child_pid(m::MockChild) = m.pid +function Main.FormatProcess.child_poll(m::MockChild) + m.poll_calls += 1 + return m.polls[min(m.poll_calls, length(m.polls))] +end +function Main.FormatProcess.child_kill(m::MockChild) + m.kills += 1 + m.kill_error && throw(ProcessLookupError()) +end +function Main.FormatProcess.child_wait(m::MockChild, timeout) + push!(m.waits, timeout) + return m.wait_result +end +Main.FormatProcess.close_streams(m::MockChild) = (m.closed = true; nothing) + +# Warm up compilation so timing-sensitive cases measure the child, not the JIT. +run_fake("true") + +# ---- ProcessTests -------------------------------------------------------- + +@testset "ProcessTests" begin + @testset "test_round_trip_binary_io" begin + raw = repeat(UInt8.(0:255), 200) + result = run_fake("cat"; data=raw) + @test (result.code, result.stdout, result.stderr) == (0, raw, UInt8[]) + end + + @testset "test_stderr_separate_and_exit_preserved" begin + result = run_fake("printf private >&2; exit 7") + @test (result.code, result.stdout, result.stderr) == (7, UInt8[], Vector{UInt8}("private")) + end + + @testset "test_missing_binary_is_infrastructure_not_document_rejection" begin + withenv("PATH" => "") do + @test tool_error(() -> run_probe("ijson-canon")) + end + end + + @testset "test_timeout" begin + @test tool_error(() -> run_fake("sleep 10"; timeout=0.15), "time budget") + end + + @testset "test_output_budget_covers_both_streams" begin + @test tool_error(() -> run_fake("head -c 10000 /dev/zero | tr '\\0' x >&2"; limit=100), + "output budget") + end + + @testset "test_cleanup_kills_direct_child_when_its_group_is_gone" begin + child = MockChild(12345; polls=Any[nothing]) + Main.FormatProcess.cleanup(child; killpg=pid -> throw(ProcessLookupError())) + @test child.kills == 1 + @test child.waits == [0.5] + @test child.closed + end + + @testset "test_cleanup_reports_a_live_child_that_cannot_be_killed" begin + child = MockChild(12345; polls=Any[nothing, nothing], kill_error=true) + @test tool_error(() -> Main.FormatProcess.cleanup(child; killpg=pid -> throw(ProcessLookupError())), + "escaped its process group") + @test isempty(child.waits) + @test child.poll_calls == 2 + @test child.closed + end + + @testset "test_input_and_deadline_budgets" begin + for kwargs in (Dict(:timeout => 0), Dict(:timeout => NaN), Dict(:timeout => Inf), + Dict(:limit => 0), Dict(:data => UInt8[0x78, 0x78], :limit => 1)) + @test tool_error(() -> run_fake("true"; kwargs...)) + end + end + + @testset "test_arbitrary_programs_and_arguments_are_refused" begin + for argv in ([joinpath(Sys.BINDIR, "julia"), "-e", "nothing"], + ["git", "config", "--global", "x", "y"]) + @test tool_error(() -> run_probe(argv)) + end + end + + @testset "test_symlink_cannot_disguise_an_interpreter_as_a_canonicalizer" begin + mktempdir() do tmp + symlink(joinpath(Sys.BINDIR, "julia"), joinpath(tmp, "ijson-jcs")) + withenv("PATH" => path_with(tmp)) do + @test tool_error(() -> run_probe("ijson-canon")) + end + end + end + + @testset "test_detached_double_fork_is_killed_and_reaped" begin + mktempdir() do tmp + pid_file = joinpath(tmp, "detached.pid") + body = "( setsid sh -c 'echo \$\$ > \"$pid_file\"; exec sleep 30' & )\nsleep 30" + @test tool_error(() -> run_fake(body; timeout=0.4), "time budget") + pid = parse(Int, strip(read(pid_file, String))) + @test ccall(:kill, Cint, (Cint, Cint), pid, 0) == -1 + @test Libc.errno() == 3 # ESRCH + end + end + + @testset "test_missing_procfs_fails_closed" begin + for hook in (Main.FormatDescendants.LISTDIR, Main.FormatDescendants.READTEXT) + original = hook[] + hook[] = _ -> throw(Base.IOError("procfs gone", Base.UV_ENOENT)) + try + # Stricter than the Python original: the failure must come from + # supervision, not from a coincidentally missing tool. + @test tool_error(() -> run_probe("ijson-canon"), "supervision failed") + finally + hook[] = original + end + end + end + + @testset "test_existing_child_context_is_refused_before_launch" begin + launches = Ref(0) + original_children, original_spawn = Main.FormatDescendants.CHILDREN[], Main.FormatProcess.SPAWN[] + Main.FormatDescendants.CHILDREN[] = _ -> Set([12345]) + Main.FormatProcess.SPAWN[] = (args...) -> (launches[] += 1; error("must not launch")) + try + # A resolvable fake tool is on PATH, so a launch would really be attempted + # (the Python original never reaches Popen here, which makes its assertion vacuous). + mktempdir() do tmp + write_tool(tmp, "true") + withenv("PATH" => path_with(tmp)) do + @test tool_error(() -> run_probe("ijson-canon"), "supervision failed") + end + end + @test launches[] == 0 + finally + Main.FormatDescendants.CHILDREN[] = original_children + Main.FormatProcess.SPAWN[] = original_spawn + end + end + + @testset "test_descendant_keeping_pipe_open_is_bounded" begin + @test tool_error(() -> run_fake("sleep 10 &"; timeout=0.2), "time budget") + end +end + +# ---- CorpusTests --------------------------------------------------------- + +# Run `f(tool_directory)` with a fake ijson-jcs in "tool with spaces" first on PATH. +function with_fake(f, directory, body) + folder = joinpath(directory, "tool with spaces") + mkdir(folder) + write_tool(folder, body) + withenv("PATH" => path_with(folder)) do + f("reference") + end +end + +@testset "CorpusTests" begin + @testset "test_raw_fixtures_survive_loading" begin + cases = Conformance.load_cases(joinpath(ROOT, "test/fixtures/formats/ijson-jcs.toml")) + @test length(cases) >= 30 + by_id = Dict(c.name => c for c in cases) + @test by_id["invalid-utf8"].data == vcat(UInt8['[', '"'], 0xff, UInt8['"', ']']) + @test by_id["escaped-equivalent-key"].data == Vector{UInt8}("{\"a\":1,\"\\u0061\":2}") + @test by_id["exponent-thresholds"].expected == Vector{UInt8}("[0.000001,1e-7,1e+21]") + end + + @testset "test_bad_and_ambiguous_corpora_fail_before_running_tools" begin + for contents in ( + "schema=2\ncase=[]\n", "schema=true\ncase=[]\n", "schema=1\ncase=[]\n", + "schema=1\n[[case]]\nid='a'\ninput='{}'\nvalid=true\n", + "schema=1\n[[case]]\nid='a'\ninput='{}'\nhex='ff'\nvalid=false\n", + "schema=1\n[[case]]\nid='a'\ninput='{}'\nvalid='false'\n", + "schema=1\n[[case]]\nid='a'\ninput='{}'\nvalid=false\nunknown=1\n", + "schema=1\n[[case]]\nid='a'\nhex='not-hex'\nvalid=false\n", + ) + mktempdir() do tmp + path = joinpath(tmp, "cases.toml") + write(path, contents) + @test rejects_corpus(() -> Conformance.load_cases(path)) + end + end + end + + @testset "test_duplicate_ids_fail" begin + case = "\n[[case]]\nid='a'\ninput='{}'\nvalid=false\n" + mktempdir() do tmp + path = joinpath(tmp, "cases.toml") + write(path, "schema=1\n" * case * case) + @test rejects_corpus(() -> Conformance.load_cases(path)) + end + end + + @testset "test_mutants_cannot_pass_as_canonicalizers" begin + cases = [mkcase("valid", " {\"a\":1} ", true, "{\"a\":1}"), + mkcase("invalid", "{\"a\":1,\"a\":2}", false, "")] + # Always-success, always-invalid, crashes, and passthrough all fail. + for body in ("exit 0", "exit 2", "exit 1", "cat") + mktempdir() do tmp + passed, _, _ = capture() do + with_fake(tmp, body) do adapter + Conformance.check(adapter, cases) + end + end + @test passed == false + end + end + end + + @testset "test_correct_adapter_protocol_passes_and_redacts_diagnostics" begin + cases = [mkcase("valid", "true", true, "true"), mkcase("invalid", "broken", false, "")] + body = """ + input=\$(cat) + if [ "\$input" = true ]; then + printf true + else + printf 'private input' >&2 + exit 2 + fi""" + mktempdir() do tmp + passed, out, _ = capture() do + with_fake(tmp, body) do adapter + Conformance.check(adapter, cases) + end + end + @test passed == true + @test !occursin("private input", out) + end + end + + @testset "test_exact_bytes_no_newline_or_wrong_order" begin + cases = [mkcase("order", "{}", true, "{\"a\":1,\"z\":2}")] + for body in ("printf '%s' '{\"z\":2,\"a\":1}'", "printf '%s\\n' '{\"a\":1,\"z\":2}'") + mktempdir() do tmp + passed, _, _ = capture() do + with_fake(tmp, body) do adapter + Conformance.check(adapter, cases) + end + end + @test passed == false + end + end + end + + @testset "test_missing_tool_cli_fails_closed_without_echoing_input" begin + code, out, err = capture() do + withenv("PATH" => "") do + Conformance.main(String[]) + end + end + @test code == 2 + @test occursin("no conformance pass", err) + @test !occursin("9007199254740992", out * err) + end + + @testset "test_kyq_probe_missing_source_is_not_a_pass" begin + code, _, _ = capture() do + ProbeKyq.main(["--source", "/nonexistent/kyq"]) + end + @test code == 2 + end +end + +# ---- InventoryTests ------------------------------------------------------ + +@testset "InventoryTests" begin + @testset "test_inventory_covers_hidden_new_files_and_never_echoes_source" begin + mktempdir() do tmp + run(`git init -q $tmp`) + mkpath(joinpath(tmp, ".github/workflows")) + write(joinpath(tmp, ".github/workflows/probe.yml"), "name: \"private-value\"\nrun: jq -r .value\n") + write(joinpath(tmp, "client.ts"), "res.json();\nJSON.parse(\"private-value\")\n") + symlink("/etc/passwd", joinpath(tmp, "foreign.json")) + write(joinpath(tmp, ".gitignore"), "ignored.py\n") + write(joinpath(tmp, "ignored.py"), "json.loads(\"ignore\")") + rows = Inventory.inventory(realpath(tmp)) + @test any(r -> r.path == ".github/workflows/probe.yml", rows) + @test any(r -> r.kind == "symlink" && r.path == "foreign.json", rows) + @test any(r -> r.line == 2 && r.operation == "JSON.parse", rows) + @test any(r -> r.path == ".github/workflows/probe.yml" && r.line == 2 && + r.operation == "shell jq/yq candidate", rows) + @test !any(r -> r.path == "ignored.py", rows) + @test !occursin("private-value", repr(rows)) + end + end +end diff --git a/scripts/tests/test_format_readiness.py b/scripts/tests/test_format_readiness.py deleted file mode 100644 index 48f3dd9e..00000000 --- a/scripts/tests/test_format_readiness.py +++ /dev/null @@ -1,247 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -"""Tests of the harness itself; fake children are NOT parser conformance evidence.""" -import contextlib -import io -import os -from pathlib import Path -import subprocess -import sys -import tempfile -import unittest -from unittest.mock import Mock, patch - -ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT / "scripts/formats")) -from conformance import check, load_cases, main as conformance_main -from inventory import inventory -from process import ToolError, _cleanup, run -from probe_kyq import main as probe_main - - -def run_python(body, data=b"", **kwargs): - # A fake installed protocol adapter, never a production interpreter command. - with tempfile.TemporaryDirectory(prefix="format tool ") as tmp: - path = Path(tmp) / "ijson-jcs" - path.write_text(f"#!{sys.executable}\n{body}\n") - path.chmod(0o700) - with patch.dict(os.environ, {"PATH": tmp + os.pathsep + os.environ["PATH"]}): - return run("ijson-canon", data, **kwargs) - - -class ProcessTests(unittest.TestCase): - def test_round_trip_binary_io(self): - raw = bytes(range(256)) * 200 - result = run_python("import sys; sys.stdout.buffer.write(sys.stdin.buffer.read())", raw) - self.assertEqual((result.code, result.stdout, result.stderr), (0, raw, b"")) - - def test_stderr_separate_and_exit_preserved(self): - result = run_python("import sys; sys.stderr.write('private'); sys.exit(7)") - self.assertEqual((result.code, result.stdout, result.stderr), (7, b"", b"private")) - - def test_missing_binary_is_infrastructure_not_document_rejection(self): - with patch.dict(os.environ, {"PATH": ""}), self.assertRaises(ToolError): - run("ijson-canon") - - def test_timeout(self): - with self.assertRaisesRegex(ToolError, "time budget"): - run_python("import time; time.sleep(10)", timeout=0.15) - - def test_output_budget_covers_both_streams(self): - with self.assertRaisesRegex(ToolError, "output budget"): - run_python("import sys; sys.stderr.write('x' * 10000)", limit=100) - - def test_cleanup_kills_direct_child_when_its_group_is_gone(self): - child = Mock() - child.pid = 12345 - streams = [io.BytesIO(), io.BytesIO(), io.BytesIO()] - child.stdin, child.stdout, child.stderr = streams - child.poll.return_value = None - child.wait.return_value = -9 - with patch("process.os.killpg", side_effect=ProcessLookupError): - _cleanup(child) - child.kill.assert_called_once_with() - child.wait.assert_called_once_with(timeout=0.5) - self.assertTrue(all(stream.closed for stream in streams)) - - def test_cleanup_reports_a_live_child_that_cannot_be_killed(self): - child = Mock() - child.pid = 12345 - streams = [io.BytesIO(), io.BytesIO(), io.BytesIO()] - child.stdin, child.stdout, child.stderr = streams - child.poll.side_effect = [None, None] - child.kill.side_effect = ProcessLookupError - with patch("process.os.killpg", side_effect=ProcessLookupError): - with self.assertRaisesRegex(ToolError, "escaped its process group"): - _cleanup(child) - child.wait.assert_not_called() - self.assertTrue(all(stream.closed for stream in streams)) - - def test_input_and_deadline_budgets(self): - for kwargs in ({"timeout": 0}, {"timeout": float("nan")}, {"timeout": float("inf")}, {"limit": 0}, {"data": b"xx", "limit": 1}): - with self.assertRaises(ToolError): - run_python("pass", **kwargs) - - def test_arbitrary_programs_and_arguments_are_refused(self): - for argv in ([sys.executable, "-c", "pass"], ["git", "config", "--global", "x", "y"]): - with self.subTest(argv=argv), self.assertRaises(ToolError): - run(argv) - - def test_symlink_cannot_disguise_an_interpreter_as_a_canonicalizer(self): - with tempfile.TemporaryDirectory() as tmp: - path = Path(tmp) / "ijson-jcs" - path.symlink_to(sys.executable) - with patch.dict(os.environ, {"PATH": tmp + os.pathsep + os.environ["PATH"]}): - with self.assertRaises(ToolError): - run("ijson-canon") - - def test_detached_double_fork_is_killed_and_reaped(self): - with tempfile.TemporaryDirectory() as tmp: - pid_file = Path(tmp) / "detached.pid" - body = f"""import os,time -if os.fork() == 0: - os.setsid() - if os.fork() == 0: - with open({str(pid_file)!r}, 'w') as stream: - stream.write(str(os.getpid())) - time.sleep(30) - os._exit(0) -time.sleep(30) -""" - with self.assertRaisesRegex(ToolError, "time budget"): - run_python(body, timeout=0.4) - pid = int(pid_file.read_text()) - with self.assertRaises(ProcessLookupError): - os.kill(pid, 0) - - def test_missing_procfs_fails_closed(self): - for method in ("iterdir", "read_text"): - with self.subTest(method=method): - with patch(f"descendants.Path.{method}", side_effect=FileNotFoundError): - with self.assertRaises(ToolError): - run("ijson-canon") - - def test_existing_child_context_is_refused_before_launch(self): - with patch("descendants._children", return_value={12345}): - with patch("process.subprocess.Popen") as launch: - with self.assertRaises(ToolError): - run("ijson-canon") - launch.assert_not_called() - - def test_descendant_keeping_pipe_open_is_bounded(self): - with self.assertRaisesRegex(ToolError, "time budget"): - run_python("import os,time; p=os.fork(); time.sleep(10) if p==0 else None", timeout=0.2) - - -class CorpusTests(unittest.TestCase): - def test_raw_fixtures_survive_loading(self): - cases = list(load_cases(ROOT / "test/fixtures/formats/ijson-jcs.toml")) - self.assertGreaterEqual(len(cases), 30) - by_id = {c[0]: c for c in cases} - self.assertEqual(by_id["invalid-utf8"][1], b'["\xff"]') - self.assertEqual(by_id["escaped-equivalent-key"][1], b'{"a":1,"\\u0061":2}') - self.assertEqual(by_id["exponent-thresholds"][3], b'[0.000001,1e-7,1e+21]') - - def test_bad_and_ambiguous_corpora_fail_before_running_tools(self): - for contents in ( - "schema=2\ncase=[]\n", "schema=true\ncase=[]\n", "schema=1\ncase=[]\n", - "schema=1\n[[case]]\nid='a'\ninput='{}'\nvalid=true\n", - "schema=1\n[[case]]\nid='a'\ninput='{}'\nhex='ff'\nvalid=false\n", - "schema=1\n[[case]]\nid='a'\ninput='{}'\nvalid='false'\n", - "schema=1\n[[case]]\nid='a'\ninput='{}'\nvalid=false\nunknown=1\n", - "schema=1\n[[case]]\nid='a'\nhex='not-hex'\nvalid=false\n", - ): - with self.subTest(contents=contents), tempfile.TemporaryDirectory() as tmp: - path = Path(tmp) / "cases.toml" - path.write_text(contents) - with self.assertRaises((ValueError, TypeError)): - list(load_cases(path)) - - def test_duplicate_ids_fail(self): - case = "\n[[case]]\nid='a'\ninput='{}'\nvalid=false\n" - with tempfile.TemporaryDirectory() as tmp: - path = Path(tmp) / "cases.toml" - path.write_text("schema=1\n" + case * 2) - with self.assertRaises(ValueError): - list(load_cases(path)) - - @contextlib.contextmanager - def fake(self, directory, body): - folder = Path(directory) / "tool with spaces" - folder.mkdir() - path = folder / "ijson-jcs" - path.write_text(f"#!{sys.executable}\n{body}\n") - path.chmod(0o700) - with patch.dict(os.environ, {"PATH": str(folder) + os.pathsep + os.environ["PATH"]}): - yield "reference" - - def test_mutants_cannot_pass_as_canonicalizers(self): - cases = [("valid", b' {"a":1} ', True, b'{"a":1}'), - ("invalid", b'{"a":1,"a":2}', False, b'')] - # Always-success, always-invalid, crashes, and passthrough all fail. - for body in ("raise SystemExit(0)", "raise SystemExit(2)", - "raise SystemExit(1)", "import sys; sys.stdout.buffer.write(sys.stdin.buffer.read())"): - with self.subTest(body=body), tempfile.TemporaryDirectory() as tmp: - with contextlib.redirect_stdout(io.StringIO()): - with self.fake(tmp, body) as adapter: - self.assertFalse(check(adapter, cases)) - - def test_correct_adapter_protocol_passes_and_redacts_diagnostics(self): - cases = [("valid", b"true", True, b"true"), ("invalid", b"broken", False, b"")] - body = """import sys -raw = sys.stdin.buffer.read() -if raw == b'true': - sys.stdout.buffer.write(raw) -else: - sys.stderr.write('private input') - raise SystemExit(2) -""" - with tempfile.TemporaryDirectory() as tmp, contextlib.redirect_stdout(io.StringIO()) as out: - with self.fake(tmp, body) as adapter: - self.assertTrue(check(adapter, cases)) - self.assertNotIn("private input", out.getvalue()) - - def test_exact_bytes_no_newline_or_wrong_order(self): - cases = [("order", b'{}', True, b'{"a":1,"z":2}')] - for output in (b'{"z":2,"a":1}', b'{"a":1,"z":2}\n'): - with tempfile.TemporaryDirectory() as tmp, contextlib.redirect_stdout(io.StringIO()): - with self.fake(tmp, f"import sys; sys.stdout.buffer.write({output!r})") as adapter: - self.assertFalse(check(adapter, cases)) - - def test_missing_tool_cli_fails_closed_without_echoing_input(self): - out, err = io.StringIO(), io.StringIO() - with patch.dict(os.environ, {"PATH": ""}): - with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): - code = conformance_main([]) - self.assertEqual(code, 2) - self.assertIn("no conformance pass", err.getvalue()) - self.assertNotIn("9007199254740992", out.getvalue() + err.getvalue()) - - def test_kyq_probe_missing_source_is_not_a_pass(self): - with contextlib.redirect_stderr(io.StringIO()): - code = probe_main(["--source", "/nonexistent/kyq"]) - self.assertEqual(code, 2) - - -class InventoryTests(unittest.TestCase): - def test_inventory_covers_hidden_new_files_and_never_echoes_source(self): - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - subprocess.run(["git", "init", "-q", tmp], check=True) - (root / ".github/workflows").mkdir(parents=True) - (root / ".github/workflows/probe.yml").write_text('name: "private-value"\nrun: jq -r .value\n') - (root / "client.ts").write_text('res.json();\nJSON.parse("private-value")\n') - (root / "foreign.json").symlink_to("/etc/passwd") - (root / ".gitignore").write_text('ignored.py\n') - (root / "ignored.py").write_text('json.loads("ignore")') - rows = list(inventory(root)) - self.assertTrue(any(r[2] == ".github/workflows/probe.yml" for r in rows)) - self.assertTrue(any(r[0] == "symlink" and r[2] == "foreign.json" for r in rows)) - self.assertTrue(any(r[3] == 2 and r[4] == "JSON.parse" for r in rows)) - self.assertTrue(any(r[2] == ".github/workflows/probe.yml" and r[3] == 2 - and r[4] == "shell jq/yq candidate" for r in rows)) - self.assertFalse(any(r[2] == "ignored.py" for r in rows)) - self.assertNotIn("private-value", repr(rows)) - - -if __name__ == "__main__": - unittest.main() From 3388cb4a9af399d64259ca6c48f8a9e24f3f7275 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 21:15:08 +0000 Subject: [PATCH 10/10] Address review findings on the Agda memo and the new oracles Compare AGDA_IMAGE with the Agda job's own container line and hash the check target. Print integers with %.0f in the awk oracle so mawk keeps the full safe-integer range, require the stored transformed matrix to have the shape of the one rebuilt from raw_counts, stop the exact-summary reference when its input validation fails, and name the bc checker in the numeric contract table. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_0136P7ZmNJPhB7wGCRymdu1D --- .github/workflows/ci.yml | 11 +++++++---- docs/statistics/numeric-contracts.md | 2 +- test/references/check_ordination_references.awk | 6 ++++-- test/references/exact_summary_reference.sh | 5 ++++- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b639cf09..5061224e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -733,6 +733,7 @@ jobs: AGDA_BIN_VERSION: 2.6.4.3-1+b2 AGDA_STDLIB_VERSION: 2.1-4 AGDA_FLAGS: --safe --without-K --cubical-compatible + AGDA_TARGET: MetaManifold/All.agda # The image of the container: above, the one literal that `container:` # cannot read from env. The hash step refuses to run if the two differ. AGDA_IMAGE: debian:trixie-20250929-slim@sha256:1caf1c703c8f7e15dcf2e7769b35000c764e6f50e4d7401c355fb0248f3ddfdb @@ -752,10 +753,12 @@ jobs: set -euo pipefail # A container job's steps cannot read the container image from the # context, so the image is a literal in env; it must be the one in use. - grep -qF "container: $AGDA_IMAGE" .github/workflows/ci.yml \ - || { echo "ERROR: AGDA_IMAGE differs from the container image" >&2; exit 1; } + # Only this job's container line counts: the occupancy job has its own. + container="$(awk '/^ agda:/ { a = 1; next } a && /^ [a-z_]+:/ { a = 0 } a && /^ container:/ { sub(/^ container: */, ""); print; exit }' .github/workflows/ci.yml)" + [ "$container" = "$AGDA_IMAGE" ] \ + || { echo "ERROR: AGDA_IMAGE differs from the Agda job container" >&2; exit 1; } hash="$( { - printf '%s\n' "$AGDA_IMAGE" "$AGDA_BIN_VERSION" "$AGDA_STDLIB_VERSION" "$AGDA_FLAGS" + printf '%s\n' "$AGDA_IMAGE" "$AGDA_BIN_VERSION" "$AGDA_STDLIB_VERSION" "$AGDA_FLAGS" "$AGDA_TARGET" find proofs -type f -print0 | LC_ALL=C sort -z | xargs -0 sha256sum } | sha256sum | cut -d' ' -f1 )" echo "hash=$hash" >> "$GITHUB_OUTPUT" @@ -832,7 +835,7 @@ jobs: cd proofs/agda # AGDA_FLAGS is word-split on purpose: each flag is its own argument. # shellcheck disable=SC2086 - if ! agda $AGDA_FLAGS MetaManifold/All.agda >/tmp/agda.log 2>&1; then + if ! agda $AGDA_FLAGS "$AGDA_TARGET" >/tmp/agda.log 2>&1; then cat /tmp/agda.log msg="$(awk 'NR > 1 { printf "%%0A" } { gsub(/%/, "%25"); printf "%s", $0 }' /tmp/agda.log | tail -c 3500)" echo "::error file=proofs/agda/MetaManifold/All.agda,title=Agda type-check failed::$msg" diff --git a/docs/statistics/numeric-contracts.md b/docs/statistics/numeric-contracts.md index ff15a4cf..b27456d0 100644 --- a/docs/statistics/numeric-contracts.md +++ b/docs/statistics/numeric-contracts.md @@ -86,7 +86,7 @@ cross a boundary as decimal strings; floating point never carries a count.** | Julia `Int64` → `Float64` | Exact through `2^53`; `2^53 + 1` rounds down silently | `MAX_EXACT_FLOAT`; refusal of `> 2^53 - 1` JSON numbers | | JavaScript `JSON.parse` | `{"n":9007199254740993}` becomes `9007199254740992` with no error; the string form is byte-exact | Node check in `test_statistics_boundaries.jl` (skipped and announced when `node` is absent) | | R `numeric` | R has no integer type wider than 32 bits; every number is a double, so `9007199254740993` arrives one short | Rscript check in `test_statistics_boundaries.jl` (skipped and announced when `Rscript` is absent) | -| GNU bc | arbitrary-precision integers and decimals; the independent reference implementation | `test/references/exact_summary_reference.sh`, `check_high_precision_references.py` | +| GNU bc | arbitrary-precision integers and decimals; the independent reference implementation | `test/references/exact_summary_reference.sh`, `test/references/check_high_precision_references.sh` | | DuckDB | `TINYINT`/`SMALLINT`/`INTEGER`/`BIGINT`/`HUGEINT` are accepted; `FLOAT`/`DOUBLE`/`DECIMAL` are refused with `422` | `_validated_integer_sample_columns`; boundary test builds one table per type | | CSV/text | There is no CSV writer for this layer. Display strings are for humans and are labelled rounded; they are not a data interchange format | `to_display`, `summary_to_display` | diff --git a/test/references/check_ordination_references.awk b/test/references/check_ordination_references.awk index 3532f700..c7bdf592 100644 --- a/test/references/check_ordination_references.awk +++ b/test/references/check_ordination_references.awk @@ -384,7 +384,7 @@ function jcs(id, t, i, n, s, ix, j, tmp) { if (t == "F") return "false" if (t == "i") { if (BAD[id] != "") die(BAD[id]) - return sprintf("%d", V[id]) + return sprintf("%.0f", V[id]) } if (t == "f") { if (BAD[id] != "") die(BAD[id]) @@ -804,7 +804,9 @@ function check_dataset(d, dsid, yn, y, tr, trs, rawn, i, j, mn, mp, z, zn, xn, else hellinger(MC, MCI, RN, RP, MR) for (i = 0; i < RN; i++) for (j = 0; j < RP; j++) MRI[i, j] = 0 mn = RN; mp = RP - a = (YN < RN) ? YN : RN; b = (YP < RP) ? YP : RP + # The stored matrix must have the shape of the one rebuilt from raw_counts. + if (YN != RN || YP != RP) die(dsid ": stored transformed matrix has a different shape from raw_counts") + a = RN; b = RP for (i = 0; i < a; i++) for (j = 0; j < b; j++) if (abs(MY[i, j] - MR[i, j]) > 1e-10) die(dsid ": stored transformed matrix drifted from raw_counts") } else { diff --git a/test/references/exact_summary_reference.sh b/test/references/exact_summary_reference.sh index 613981e3..e87c98fc 100755 --- a/test/references/exact_summary_reference.sh +++ b/test/references/exact_summary_reference.sh @@ -93,6 +93,9 @@ section=samples label="" ; lists=() n_samples="$(jq '.sample_labels | length' "$input")" count=0 +# Captured with a checked assignment, so a failure in blocks stops the script +# instead of reading as empty input. +block_lines="$(blocks)" while IFS=$'\t' read -r tag rest; do case "$tag" in L) label="$rest"; lists=() ;; @@ -103,7 +106,7 @@ while IFS=$'\t' read -r tag rest; do count=$((count + 1)) ;; *) ;; esac -done < <(blocks) +done <<<"$block_lines" join() { local IFS=,; echo "$*"; } printf '{"samples":[%s],"groups":[%s]}\n' "$(join "${samples[@]}")" "$(join "${groups[@]}")" | jq -ac .