diff --git a/README.md b/README.md index 488938e..3dfad92 100644 --- a/README.md +++ b/README.md @@ -321,6 +321,7 @@ FastaGuard catches FASTA-level assembly problems before expensive assembly QC. - [v0.2 evidence pack](docs/evidence/fastaguard-v0.2-evidence.md) - [v0.3 evidence workflow](docs/evidence/fastaguard-v0.3-evidence.md) - [v0.5 submission readiness evidence](docs/evidence/fastaguard-v0.5-submission-readiness.md) +- [v0.5 public evidence workflow](docs/evidence/fastaguard-v0.5-public-evidence.md) - [Packaging](docs/packaging.md) - [v0.5.0 release notes](docs/releases/v0.5.0.md) - [v0.4.0 release notes](docs/releases/v0.4.0.md) diff --git a/docs/benchmarking.md b/docs/benchmarking.md index cbbfe19..b3f65b3 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -153,6 +153,12 @@ For each run, record: This evidence matters more than synthetic speed alone because it shows the wedge: cheap FASTA preflight before expensive downstream QC. +The v0.5 public evidence workflow is documented in +`docs/evidence/fastaguard-v0.5-public-evidence.md`. It extends the compact +summary with manifest fields such as `evidence_role`, `expected_scale`, and +`downstream_route` so benchmark tables explain why each case was selected and +what FastaGuard should route toward after preflight. + ## Evidence Pack Workflow The original v0.2 evidence workflow is documented in diff --git a/docs/evidence/fastaguard-v0.5-public-evidence.md b/docs/evidence/fastaguard-v0.5-public-evidence.md new file mode 100644 index 0000000..63a3a5f --- /dev/null +++ b/docs/evidence/fastaguard-v0.5-public-evidence.md @@ -0,0 +1,98 @@ +# FastaGuard v0.5 Public Evidence + +This page defines the v0.5 public evidence workflow. The goal is to show that +FastaGuard is cheap to run before expensive downstream QC and that its gate +outputs can be audited against exact input bytes. + +The public assembly list lives in: + +```text +docs/evidence/public_assemblies.json +``` + +Each manifest case declares: + +- `accession` +- `evidence_role` +- `expected_scale` +- `downstream_route` +- `source_url` + +These fields are copied into `evidence_summary.json` and +`evidence_summary.tsv` by `scripts/collect_evidence.py`. + +## Local Smoke Run + +Run the offline cases first. This does not download public data: + +```bash +cargo build --release --locked +python3 scripts/collect_evidence.py \ + --binary target/release/fastaguard \ + --out-dir target/evidence/v0.5-local \ + --local-only +``` + +The local smoke run covers: + +| Case | Evidence role | Why it matters | +| --- | --- | --- | +| `synthetic_valid` | offline valid FASTA smoke case | proves the evidence workflow can produce PASS reports | +| `problem_fixture` | local blocker case | proves duplicate IDs, invalid characters, high-N records, and tiny contigs are visible before downstream tools | +| `gzipped_valid` | offline gzipped FASTA input smoke case | proves compressed FASTA input is accepted by the evidence workflow | + +## Public Run + +Install the NCBI Datasets CLI, then run: + +```bash +cargo build --release --locked +python3 scripts/collect_evidence.py \ + --binary target/release/fastaguard \ + --out-dir target/evidence/v0.5-public +``` + +The public run downloads the accessions in +`docs/evidence/public_assemblies.json` and writes one report directory per +case. Keep downloaded FASTA files and full HTML/JSON reports under `target/`. +Do not commit large public FASTA files. + +## Summary Table + +The compact TSV summary is the artifact to copy into release notes, README +updates, or adoption discussions: + +| Column | Meaning | +| --- | --- | +| `id` | Evidence case ID | +| `label` | Reader-facing assembly label | +| `category` | Coarse case category | +| `source` | `local` or `public_ncbi` | +| `accession` | Public assembly accession when available | +| `source_url` | Public source page when available | +| `evidence_role` | Why this case belongs in the evidence set | +| `expected_scale` | Approximate size class | +| `downstream_route` | What FastaGuard should route toward after preflight | +| `elapsed_seconds` | FastaGuard runtime for the local machine | +| `verdict` | PASS, WARN, or FAIL | +| `gate_status` | Pipeline gate status | +| `gate_blocking_findings` | Findings that block downstream workflow steps | +| `input_sha256` | Exact input checksum for auditability | +| `sequence_count`, `total_length`, `n50`, `n90` | FASTA-level structural metrics | +| `finding_count`, `top_findings` | Compact finding summary | + +## Interpretation + +This evidence is FASTA-level preflight evidence. It is not biological completeness, +not contamination confirmation, not annotation validation, and not repository +acceptance. Passing FastaGuard means the FASTA passed FastaGuard's local +contract for the selected gate. + +Use the summary to show routing: + +- FAIL with blocking findings: fix the FASTA before QUAST, BUSCO, BlobToolKit, + CheckM, annotation, or submission validators. +- PASS or WARN without blockers: continue to downstream tools appropriate for + the biological question. +- WARN with advisory findings: continue if policy allows, but route the finding + IDs to the relevant follow-up tool. diff --git a/docs/evidence/public_assemblies.json b/docs/evidence/public_assemblies.json index 80aeaf7..6a53ca6 100644 --- a/docs/evidence/public_assemblies.json +++ b/docs/evidence/public_assemblies.json @@ -6,14 +6,20 @@ "accession": "GCF_000005845.2", "label": "Escherichia coli K-12 MG1655", "category": "bacterial", - "source_url": "https://www.ncbi.nlm.nih.gov/datasets/genome/GCF_000005845.2/" + "source_url": "https://www.ncbi.nlm.nih.gov/datasets/genome/GCF_000005845.2/", + "evidence_role": "small bacterial reference assembly", + "expected_scale": "about 4.6 Mbp", + "downstream_route": "If FastaGuard passes, route to QUAST for assembly metrics and BUSCO when biological completeness is in scope." }, { "id": "neurospora_crassa_or74a", "accession": "GCF_000182925.2", "label": "Neurospora crassa OR74A", "category": "fungal", - "source_url": "https://www.ncbi.nlm.nih.gov/bioproject/132" + "source_url": "https://www.ncbi.nlm.nih.gov/datasets/genome/GCF_000182925.2/", + "evidence_role": "small eukaryotic fungal reference assembly", + "expected_scale": "tens of Mbp", + "downstream_route": "If FastaGuard passes, route to QUAST, BUSCO, and BlobToolKit-style follow-up when assembly quality or composition signals need deeper review." } ] } diff --git a/docs/tool-landscape.md b/docs/tool-landscape.md index c44c476..255d828 100644 --- a/docs/tool-landscape.md +++ b/docs/tool-landscape.md @@ -103,6 +103,8 @@ Current product evidence: benchmark framing for adoption discussions. - v0.5 documentation defines the submission-readiness gate, local evidence commands, and the boundary before official validators. +- v0.5 public evidence workflow defines manifest roles, expected scales, and + downstream routes for compact benchmark summaries. Evidence still needed: diff --git a/scripts/collect_evidence.py b/scripts/collect_evidence.py index 12c48ad..784507e 100755 --- a/scripts/collect_evidence.py +++ b/scripts/collect_evidence.py @@ -26,6 +26,10 @@ "category", "source", "accession", + "source_url", + "evidence_role", + "expected_scale", + "downstream_route", "input_bytes", "elapsed_seconds", "exit_code", @@ -133,6 +137,13 @@ def local_cases(out_dir: Path) -> list[dict[str, Any]]: "category": "synthetic", "source": "local", "accession": None, + "source_url": None, + "evidence_role": "offline valid FASTA smoke case", + "expected_scale": "tiny synthetic fixture", + "downstream_route": ( + "If this fails, fix the FastaGuard run before comparing with " + "QUAST, BUSCO, or other downstream tools." + ), "input_path": synthetic_path, "case_dir": synthetic_dir, }, @@ -142,6 +153,16 @@ def local_cases(out_dir: Path) -> list[dict[str, Any]]: "category": "fixture", "source": "local", "accession": None, + "source_url": None, + "evidence_role": ( + "local blocker case for duplicate IDs, invalid characters, " + "high-N records, and tiny contigs" + ), + "expected_scale": "tiny intentionally problematic fixture", + "downstream_route": ( + "Fix FASTA-level blockers before QUAST, BUSCO, BlobToolKit, " + "CheckM, annotation, or submission validators." + ), "input_path": ROOT / "testdata" / "problem_assembly.fa", "case_dir": out_dir / "problem_fixture", }, @@ -151,6 +172,13 @@ def local_cases(out_dir: Path) -> list[dict[str, Any]]: "category": "fixture", "source": "local", "accession": None, + "source_url": None, + "evidence_role": "offline gzipped FASTA input smoke case", + "expected_scale": "tiny gzipped fixture", + "downstream_route": ( + "If this passes, the pipeline can accept compressed FASTA " + "before heavier QC tools." + ), "input_path": gzip_path, "case_dir": gzip_dir, }, @@ -309,6 +337,10 @@ def run_case(binary: Path, case: dict[str, Any]) -> dict[str, Any]: "category": case["category"], "source": case["source"], "accession": case.get("accession"), + "source_url": case.get("source_url"), + "evidence_role": case.get("evidence_role"), + "expected_scale": case.get("expected_scale"), + "downstream_route": case.get("downstream_route"), "input_path": str(case["input_path"]), "input_bytes": case["input_path"].stat().st_size, "elapsed_seconds": round(elapsed, 4), diff --git a/tests/python/test_adoption_assets.py b/tests/python/test_adoption_assets.py index 640ede6..98db0a7 100644 --- a/tests/python/test_adoption_assets.py +++ b/tests/python/test_adoption_assets.py @@ -562,13 +562,39 @@ def test_public_evidence_manifest_declares_default_assemblies(self): with self.subTest(case=case): self.assertEqual( set(case), - {"id", "accession", "label", "category", "source_url"}, + { + "id", + "accession", + "label", + "category", + "source_url", + "evidence_role", + "expected_scale", + "downstream_route", + }, ) self.assertRegex(case["id"], r"^[a-z0-9][a-z0-9_-]+$") self.assertRegex(case["accession"], r"^GC[AF]_[0-9]+\.[0-9]+$") self.assertTrue(case["label"]) self.assertIn(case["category"], {"bacterial", "fungal"}) self.assertTrue(case["source_url"].startswith("https://")) + self.assertTrue(case["evidence_role"]) + self.assertTrue(case["expected_scale"]) + self.assertRegex( + case["downstream_route"], r"(QUAST|BUSCO|BlobToolKit|validator)" + ) + + def test_v0_5_public_evidence_doc_defines_benchmark_table(self): + evidence = ROOT / "docs" / "evidence" / "fastaguard-v0.5-public-evidence.md" + + self.assertTrue(evidence.exists()) + text = evidence.read_text() + self.assertIn("docs/evidence/public_assemblies.json", text) + self.assertIn("python3 scripts/collect_evidence.py", text) + self.assertIn("evidence_summary.tsv", text) + self.assertIn("downstream_route", text) + self.assertIn("not biological completeness", text) + self.assertIn("not contamination confirmation", text) def test_evidence_docs_reference_local_and_public_workflows(self): evidence = (ROOT / "docs" / "evidence" / "fastaguard-v0.2-evidence.md") @@ -680,12 +706,18 @@ def option_path(flag): {"synthetic_valid", "problem_fixture", "gzipped_valid"}, ) self.assertTrue((out_dir / "evidence_summary.tsv").exists()) + tsv_text = (out_dir / "evidence_summary.tsv").read_text() + self.assertIn("evidence_role", tsv_text.splitlines()[0]) + self.assertIn("downstream_route", tsv_text.splitlines()[0]) for case in summary["cases"]: self.assertEqual(case["verdict"], "PASS") self.assertEqual(case["gate_mode"], "pipeline") self.assertEqual(case["gate_status"], "PASS") self.assertEqual(case["gate_blocking_findings"], "") self.assertEqual(case["input_sha256"], "0" * 64) + self.assertIn("evidence_role", case) + self.assertIn("expected_scale", case) + self.assertIn("downstream_route", case) self.assertIn("--gate pipeline", case["command"]) self.assertGreater(case["elapsed_seconds"], 0) self.assertIn("command", case)