Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
427 changes: 299 additions & 128 deletions .github/workflows/ci.yml

Large diffs are not rendered by default.

41 changes: 19 additions & 22 deletions .github/workflows/doi.yml
Original file line number Diff line number Diff line change
Expand Up @@ -75,31 +75,28 @@ jobs:

- name: Citation metadata check
run: |
python3 - <<'PY'
from pathlib import Path
import sys

cff = Path("CITATION.cff").read_text(encoding="utf-8")
required_prefixes = ("cff-version:", "title:", "authors:", "license:")
missing = [p for p in required_prefixes if not any(line.startswith(p) for line in cff.splitlines())]
if missing:
sys.exit(f"CITATION.cff missing required top-level keys: {', '.join(missing)}")
PY
missing=()
for key in cff-version title authors license; do
grep -q "^${key}:" CITATION.cff || missing+=("${key}:")
done
if [ "${#missing[@]}" -ne 0 ]; then
echo "CITATION.cff missing required top-level keys: ${missing[*]}" >&2
exit 1
fi

- name: Receipt schema shape
run: |
python3 - <<'PY'
from pathlib import Path
import sys

source = Path("src/core/doi.jl").read_text(encoding="utf-8")
keys = ("schema", "study", "run", "origin", "deposition_id", "doi",
"published", "files", "created_at", "published_at", "checksum")
missing = [k for k in keys if f'"{k}"' not in source and f":{k}" not in source]
if missing:
sys.exit(f"src/core/doi.jl missing receipt schema keys: {', '.join(missing)}")
print("verified DOI receipt schema keys in src/core/doi.jl")
PY
missing=()
for key in schema study run origin deposition_id doi published files created_at published_at checksum; do
if ! grep -qF -e "\"${key}\"" -e ":${key}" src/core/doi.jl; then
missing+=("${key}")
fi
done
if [ "${#missing[@]}" -ne 0 ]; then
echo "src/core/doi.jl missing receipt schema keys: ${missing[*]}" >&2
exit 1
fi
echo "verified DOI receipt schema keys in src/core/doi.jl"

# Julia lifecycle and real-HTTP checks require a provisioned machine with the
# Julia runtime and are a documented blocking acceptance gate, not part of
Expand Down
28 changes: 7 additions & 21 deletions .github/workflows/format-readiness.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,28 +25,16 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up Julia
timeout-minutes: 10
run: bash scripts/install-ci-julia.sh
- name: Test harness failure boundaries
env:
PYTHONDONTWRITEBYTECODE: '1'
run: python3 -m unittest discover -s scripts/tests -p 'test_format*.py' -v
run: julia --startup-file=no scripts/tests/test_format_readiness.jl
- name: Inventory candidates without declaring compliance
run: python3 scripts/formats/inventory.py > "$RUNNER_TEMP/format-boundaries.tsv"
run: julia --startup-file=no scripts/formats/inventory.jl > "$RUNNER_TEMP/format-boundaries.tsv"
- name: Read immutable JSON reference pins
id: pins
run: |
python3 - <<'PY'
import os, re, tomllib
with open('config/format-tools.toml', 'rb') as stream:
pin = tomllib.load(stream)['ijson_jcs']
if pin['repository'] != 'https://github.com/hyperpolymath/ijson-jcs':
raise SystemExit('unexpected reference repository')
if not re.fullmatch(r'[0-9a-f]{40}', pin['revision']):
raise SystemExit('reference must be an immutable commit')
if not re.fullmatch(r'[0-9]+\.[0-9]+\.[0-9]+', pin['rust_toolchain']):
raise SystemExit('Rust must be an explicit release')
with open(os.environ['GITHUB_OUTPUT'], 'a') as stream:
stream.write(f"revision={pin['revision']}\nrust={pin['rust_toolchain']}\n")
PY
run: julia --startup-file=no scripts/formats/read_pins.jl
- name: Checkout pinned reference source
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
Expand All @@ -64,10 +52,8 @@ jobs:
rustup toolchain install "$RUST_VERSION" --profile minimal
cargo "+${RUST_VERSION}" install --path .format-reference --locked --root "$RUNNER_TEMP/format-reference"
- name: Exercise original bytes against the installed reference
env:
PYTHONDONTWRITEBYTECODE: '1'
run: |
PATH="$RUNNER_TEMP/format-reference/bin:$PATH" python3 scripts/formats/conformance.py --adapter reference
PATH="$RUNNER_TEMP/format-reference/bin:$PATH" julia --startup-file=no scripts/formats/conformance.jl --adapter reference
{
echo '## Format migration evidence'
echo 'Pinned ijson-jcs raw-byte corpus passed. This is not application-wide conformance.'
Expand Down
9 changes: 8 additions & 1 deletion .github/workflows/null-calibration-comparators.yml
Original file line number Diff line number Diff line change
Expand Up @@ -236,14 +236,21 @@ jobs:
# A test that produced no usable p-value leaves its reason only in the
# per-cell CSV, which is an artifact a reader may not be able to open.
# The distinct reasons are therefore published as annotations too.
# The reporter is Julia (standard library only), installed from the same
# pin as the CI jobs. Always run, so the notes still appear after a failure.
- name: Set up Julia
if: always()
timeout-minutes: 10
run: bash scripts/install-ci-julia.sh

- name: Report any test that could not run
if: always()
run: |
set -euo pipefail
shopt -s nullglob
for csv in bench/null_calibration/results/raw/pvalues_*.csv; do
cell="$(basename "$csv" .csv)"
python3 bench/null_calibration/report_failures.py "$cell" < "$csv"
julia --startup-file=no bench/null_calibration/report_failures.jl "$cell" < "$csv"
done

- name: Publish the summaries as annotations
Expand Down
10 changes: 5 additions & 5 deletions .machine_readable/contractiles/must/must.k9.ncl
Original file line number Diff line number Diff line change
Expand Up @@ -60,15 +60,15 @@ K9!
},
{
id = "high-precision-reference-oracle",
target = "test/references/check_high_precision_references.py",
check_command = "python3 test/references/check_high_precision_references.py --check test/unit/test_high_precision.jl",
target = "test/references/check_high_precision_references.sh",
check_command = "bash test/references/check_high_precision_references.sh --check test/unit/test_high_precision.jl",
rationale = "Independent arbitrary-precision reference constants must match the Julia test suite.",
},
{
id = "constrained-ordination-reference-oracle",
target = "test/references/check_ordination_references.py",
check_command = "python3 test/references/check_ordination_references.py --check test/fixtures/ordination_reference.json",
rationale = "Independent Python reference constants for RDA eigenvalues, Ezekiel adjusted-R2, two-way variance partitioning, and canonical JCS schema/fixtures must match within 1e-6.",
target = "test/references/check_ordination_references.sh",
check_command = "bash test/references/check_ordination_references.sh --check test/fixtures/ordination_reference.json",
rationale = "Independent awk reference constants for RDA eigenvalues, Ezekiel adjusted-R2, two-way variance partitioning, and canonical JCS schema/fixtures must match within 1e-6.",
},
{
id = "upstream-license-untouched",
Expand Down
10 changes: 5 additions & 5 deletions VALIDATED_STATISTICS.md
Original file line number Diff line number Diff line change
Expand Up @@ -285,16 +285,16 @@ Full suite (with the repository's integration and server gates):
julia --project=. -t2 test/runtests.jl --integration --server
```

The suite includes known answers, independent Python `fractions` and
high-precision `decimal` references, zero/negative/nonfinite/missing controls, values beyond `2^53`, rational budget
The suite includes known answers, independent GNU `bc` exact-fraction and
high-precision references, zero/negative/nonfinite/missing controls, values beyond `2^53`, rational budget
failures, SQL-sum overflow controls, feature-permutation and scale invariance,
generative count matrices, malformed JSON fuzz cases, serialization round trips,
concurrent precision scopes, concurrent immutable publication, and tamper
recovery. Reproduce the Python `decimal` references independently with:
recovery. Reproduce the `bc` references independently with:

```bash
python3 test/references/check_high_precision_references.py
python3 test/references/check_high_precision_references.py --check \
bash test/references/check_high_precision_references.sh
bash test/references/check_high_precision_references.sh --check \
test/unit/test_high_precision.jl
```

Expand Down
6 changes: 3 additions & 3 deletions VALIDATED_STATISTICS_VALIDATION.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,16 @@ independent scientific review.
| Exact sample proportions | Non-negative counts and positive total; zero total gives undefined values | `[1,1,1]` gives three `1/3`; Python `fractions.Fraction` compares full sample results | Values beyond `2^53`, zero-total samples, scale invariance, feature permutations, random integer matrices with proportions summing to one | Implemented |
| Exact group proportions | Same count contract; members and group labels are bound to the input fingerprint | Hand-computed two-group fixture | Empty groups and aligned-label failures; group zero-total state | Implemented |
| Observed richness | Number of strictly positive exact counts; empty sample richness is zero | Uniform three-taxon and single-taxon fixtures | Empty and extreme-scale samples | Implemented |
| High-precision Shannon | Positive total; scoped MPFR precision above 53 bits; undefined at zero total | Python `decimal` references for `log(2)`, `log(3)`, and counts `[1,2,3,4]` at a 220-digit context | Extreme rare-taxon scale that underflows `Float64`; precision round-trip and concurrent 128/256/512/1024-bit scopes | Implemented; no exactness or sampling-error claim |
| High-precision Shannon | Positive total; scoped MPFR precision above 53 bits; undefined at zero total | GNU `bc` references for `log(2)`, `log(3)`, and counts `[1,2,3,4]` at a 220-digit context | Extreme rare-taxon scale that underflows `Float64`; precision round-trip and concurrent 128/256/512/1024-bit scopes | Implemented; no exactness or sampling-error claim |
| High-precision Gini–Simpson | Positive total; exact `(N²-Σc²)/N²` intermediate within the configured rational budget; undefined at zero total | Exact known answers `2/3`, `0`, and `7/10`, converted once to MPFR | Extreme scale, budget exhaustion, serialization and concurrency tests | Implemented |
| Fixed-decimal display | Display only; exact/storage values remain separate | `1/3`, `1/2`, and negative half-way fixtures | Huge `BigFloat` display avoids `Float64` overflow; ties-away-from-zero check | Implemented |
| Direct JSON boundary | JSON object, explicit validated policy, features-by-samples matrix, 16 MiB / 100,000-cell limits | In-process and real-listener request fixtures | Deterministic malformed-input fuzz, wrong shapes, unsafe integers, float representations and unknown policies | Implemented |
| Run/DuckDB boundary | Read-only database; supported integer column schemas; no missing/negative counts; bounded grouped matrix | Taxon totals computed by hand | `HUGEINT` pair whose sum overflows SQL `SUM`, float schema, NULL and negative controls, unsupported sample-read filter | Implemented |
| Immutable derived analysis | Canonical JSON-safe record; content ID recomputed on load; no replacement of an existing destination | Repeat save/load and canonical-byte round trip | Concurrent publishers, same-ID tampering, invalid record, failed publication cleanup and source/run ownership checks | Implemented |

The high-precision reference generator is
`test/references/check_high_precision_references.py`; it uses Python's `decimal`
module and does not read output from the Julia implementation. The CI job checks
`test/references/check_high_precision_references.sh`; it uses GNU `bc`
(not Julia's MPFR or GMP) and does not read output from the Julia implementation. The CI job checks
that the constants embedded in the Julia tests still match that independent
generator.

Expand Down
2 changes: 1 addition & 1 deletion bench/advanced_zero_handling/benchmark.jl
Original file line number Diff line number Diff line change
Expand Up @@ -115,4 +115,4 @@ function main()
nothing
end

abspath(PROGRAM_FILE) == @__FILE__ && main()
abspath(PROGRAM_FILE) == (@__FILE__) && main()
2 changes: 1 addition & 1 deletion bench/cladistics.jl
Original file line number Diff line number Diff line change
Expand Up @@ -34,4 +34,4 @@ function main()
println("Includes one cumulative roll-up, Faith PD, and both UniFrac matrices; no I/O.")
end

abspath(PROGRAM_FILE) == @__FILE__ && main()
abspath(PROGRAM_FILE) == (@__FILE__) && main()
2 changes: 1 addition & 1 deletion bench/compositional_differential/benchmark.jl
Original file line number Diff line number Diff line change
Expand Up @@ -174,4 +174,4 @@ function main(args::Vector{String})
end
end

abspath(PROGRAM_FILE) == @__FILE__ && main(ARGS)
abspath(PROGRAM_FILE) == (@__FILE__) && main(ARGS)
2 changes: 1 addition & 1 deletion bench/layer1_mock_recovery/evaluate.jl
Original file line number Diff line number Diff line change
Expand Up @@ -199,4 +199,4 @@ function main()
return metrics
end

abspath(PROGRAM_FILE) == @__FILE__ && main()
abspath(PROGRAM_FILE) == (@__FILE__) && main()
2 changes: 1 addition & 1 deletion bench/layer1_mock_recovery/fetch.jl
Original file line number Diff line number Diff line change
Expand Up @@ -73,4 +73,4 @@ function main()
@info "Fetch complete" data=DATA_DIR expected=EXPECTED_DIR
end

abspath(PROGRAM_FILE) == @__FILE__ && main()
abspath(PROGRAM_FILE) == (@__FILE__) && main()
2 changes: 1 addition & 1 deletion bench/layer1_mock_recovery/report.jl
Original file line number Diff line number Diff line change
Expand Up @@ -60,4 +60,4 @@ function main()
@info "Report written" out_path
end

abspath(PROGRAM_FILE) == @__FILE__ && main()
abspath(PROGRAM_FILE) == (@__FILE__) && main()
2 changes: 1 addition & 1 deletion bench/layer1_mock_recovery/runner.jl
Original file line number Diff line number Diff line change
Expand Up @@ -105,4 +105,4 @@ function main(args::Vector{String}=String[])
return runs
end

abspath(PROGRAM_FILE) == @__FILE__ && main(ARGS)
abspath(PROGRAM_FILE) == (@__FILE__) && main(ARGS)
166 changes: 166 additions & 0 deletions bench/null_calibration/report_failures.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
# SPDX-License-Identifier: AGPL-3.0-only
# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) <j.d.a.jewell@open.ac.uk>
# Publish distinct native-calibration failure notes as Actions warnings.
#
# Usage: julia --startup-file=no report_failures.jl CELL < RESULTS.csv
# Standard library only. The CSV reader follows the rules of the Python csv module
# (default dialect) that this replaces: quoted fields may hold commas, quotes
# and line breaks, a quote inside an unquoted field is literal, and blank lines
# are skipped.

const RESULT_FIELDS = [
"truth", "shape", "offset", "n_per", "taxa", "sim", "taxon",
"true_log2fc", "test", "status", "pvalue", "note",
]
const LEGACY_FIELDS = RESULT_FIELDS[1:end-1]
const CELL_LABEL = r"^[A-Za-z0-9][A-Za-z0-9_.-]*$"

# Escape data characters reserved by the GitHub Actions command protocol.
escape_annotation(text::AbstractString) =
replace(text, "%" => "%25", "\r" => "%0D", "\n" => "%0A")

# Keep untrusted cell labels from injecting extra Actions commands. The label
# is matched as a whole: a trailing newline must not slip past an end anchor.
function validate_cell_label(cell::AbstractString)
(occursin(CELL_LABEL, cell) && !endswith(cell, "\n")) ||
throw(ArgumentError("unsafe cell label for Actions annotation"))
String(cell)
end

"""
parse_csv(text) -> Vector{Vector{String}}

Split `text` into records of fields. Blank records come back as empty vectors.
"""
function parse_csv(text::AbstractString)
records = Vector{Vector{String}}()
fields = String[]
field = IOBuffer()
# :start_record, :start_field, :in_field, :in_quoted, :quote_in_quoted, :eat_newlines
state = :start_record
finish_field!() = (push!(fields, String(take!(field))); nothing)
finish_record!() = (push!(records, fields); fields = String[]; nothing)
for c in text
if state === :eat_newlines
(c == '\n' || c == '\r') && continue
state = :start_record
end
if state === :start_record
if c == '\n' || c == '\r'
finish_record!() # a blank line: an empty record
state = :eat_newlines
continue
end
state = :start_field
end
if state === :start_field
if c == '\n' || c == '\r'
finish_field!(); finish_record!(); state = :eat_newlines
elseif c == '"'
state = :in_quoted
elseif c == ','
finish_field!()
else
write(field, c); state = :in_field
end
elseif state === :in_field
if c == '\n' || c == '\r'
finish_field!(); finish_record!(); state = :eat_newlines
elseif c == ','
finish_field!(); state = :start_field
else
write(field, c)
end
elseif state === :in_quoted
if c == '"'
state = :quote_in_quoted
else
write(field, c)
end
elseif state === :quote_in_quoted
if c == '"'
write(field, '"'); state = :in_quoted
elseif c == ','
finish_field!(); state = :start_field
elseif c == '\n' || c == '\r'
finish_field!(); finish_record!(); state = :eat_newlines
else
write(field, c); state = :in_field # lenient, as Python's default
end
end
end
# End of input: a field still open is closed, as the Python reader does.
if state === :in_field || state === :in_quoted || state === :quote_in_quoted
finish_field!(); finish_record!()
elseif state === :start_field
finish_field!(); finish_record!()
end
records
end

# Python's repr of a list of strings, as the header error message showed it.
function py_repr(values)
quote_one(s) = begin
q = occursin('\'', s) && !occursin('"', s) ? '"' : '\''
body = replace(s, "\\" => "\\\\", "\n" => "\\n", "\r" => "\\r", "\t" => "\\t")
q == '\'' && (body = replace(body, "'" => "\\'"))
string(q, body, q)
end
"[" * join(quote_one.(values), ", ") * "]"
end

"""
failure_annotations(io, cell) -> Vector{String}

Warnings for each distinct non-empty (test, failure-note) pair of the failed rows.
"""
function failure_annotations(io::IO, cell::AbstractString)
cell = validate_cell_label(cell)
text = read(io, String)
isvalid(text) || throw(ArgumentError("input is not valid UTF-8"))
records = parse_csv(text)
annotations = String[]
# The header is the very first record, blank or not, as with Python's DictReader.
isempty(records) && throw(ArgumentError("unexpected CSV header None"))
header = records[1]
header == RESULT_FIELDS || header == LEGACY_FIELDS ||
throw(ArgumentError("unexpected CSV header $(py_repr(header))"))
seen = Set{Tuple{String,String}}()
for (offset, row) in enumerate(filter(!isempty, records[2:end]))
record_number = offset + 1 # the header is record 1; the first data record is 2
length(row) == length(header) ||
throw(ArgumentError("malformed CSV record $record_number"))
values = Dict(zip(header, row))
get(values, "status", nothing) == "failed" || continue
note = get(values, "note", nothing)
(note === nothing || isempty(note) || note == "NA") && continue
key = (get(values, "test", ""), note)
key in seen && continue
push!(seen, key)
message = escape_annotation("$(key[1]) :: $(key[2])")
push!(annotations, "::warning title=$(cell): a requested test failed::$(message)")
end
annotations
end

function main(args = ARGS)
if length(args) != 1
println(stderr, "usage: ", basename(PROGRAM_FILE), " CELL < RESULTS.csv")
return 2
end
cell = args[1]
try
for annotation in failure_annotations(stdin, cell)
println(annotation)
end
catch err
err isa ArgumentError && (println(stderr, cell, ": ", err.msg); return 2)
err isa Union{SystemError,Base.IOError} && (println(stderr, cell, ": ", sprint(showerror, err)); return 2)
rethrow()
end
0
end

if abspath(PROGRAM_FILE) == @__FILE__
exit(main())
end
Loading
Loading