diff --git a/.github/workflows/foundation-ci.yml b/.github/workflows/foundation-ci.yml index 6b475d6f2..a96370987 100644 --- a/.github/workflows/foundation-ci.yml +++ b/.github/workflows/foundation-ci.yml @@ -66,6 +66,7 @@ jobs: PYTHONPATH=packages/offer-approval/src COVERAGE_FILE=/tmp/orgmetra-offer-approval.coverage python -m pytest -c packages/offer-approval/pyproject.toml packages/offer-approval/tests PYTHONPATH=packages/requisition-review/src COVERAGE_FILE=/tmp/orgmetra-requisition-review.coverage python -m pytest -c packages/requisition-review/pyproject.toml packages/requisition-review/tests PYTHONPATH=packages/selection-review/src COVERAGE_FILE=/tmp/orgmetra-selection-review.coverage python -m pytest -c packages/selection-review/pyproject.toml packages/selection-review/tests + PYTHONPATH=packages/validity-analysis/src COVERAGE_FILE=/tmp/orgmetra-validity-analysis.coverage python -m pytest -c packages/validity-analysis/pyproject.toml packages/validity-analysis/tests PYTHONPATH=services/job-analysis-api/src:packages/hris-kernel/src:packages/keyverse-adapter/src COVERAGE_FILE=/tmp/orgmetra-job-analysis-api.coverage python -m pytest -c services/job-analysis-api/pyproject.toml services/job-analysis-api/tests PYTHONPATH=services/people-api/src:packages/hris-kernel/src:packages/keyverse-adapter/src COVERAGE_FILE=/tmp/orgmetra-people-api.coverage python -m pytest -c services/people-api/pyproject.toml services/people-api/tests - name: Run PostgreSQL contracts in isolated containers diff --git a/CHANGELOG.md b/CHANGELOG.md index 16454da3d..44aa89a44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,9 @@ All notable changes to Orgmetra will be documented in this file. ### Added - Accepted ADRs 0001–0003 now include buyer-facing Context, Decision, and Consequences grounded in verified ISO 30400:2022, ISO 30414:2025, Uniform Guidelines (29 C.F.R. Part 1607), SIOP (2018), OpenAPI Specification v3.2.0, OpenID Connect Core 1.0 errata set 2, CloudEvents v1.0.2, Jensen and Snodgrass (1999), Snodgrass (1999), and Allen (1983) records already listed in `docs/doctoring/REFERENCES.md`. ADRs 0004 and 0005 gained APA 7th References pointers to that same bibliography without changing their Decision bodies. -- Active-PR governed Job Analysis persistence/API on the canonical `JobAnalysisSnapshot` model: migration `0013_job_analysis_snapshot.sql` stores immutable tenant-scoped snapshot, Task, KSAO, Task–KSAO, FJA and write-command evidence; `POST /v1/tenants/{tenant_record_id}/job-analysis-snapshots` and matching GET enforce purpose-bound Keyverse scope, authenticated-principal actor authority, bounded/strict JSON handling, transactional Idempotency-Key serialization, parent-scope fail-closed integrity, forced RLS, and atomic audit/outbox evidence. ADR 0014 records the persistence decision while ADR 0007 remains the domain/evidence authority; validated evidence still requires accountable human review and non-LLM provenance, and the service does not make a high-impact employment decision. +- Active-PR governed Job Analysis persistence/API on the canonical `JobAnalysisSnapshot` model: migration `0013_job_analysis_snapshot.sql` stores immutable tenant-scoped snapshot, Task, KSAO, FJA and write-command evidence; `POST /v1/tenants/{tenant_record_id}/job-analysis-snapshots` and matching GET enforce purpose-bound Keyverse scope, authenticated-principal actor authority, bounded/strict JSON handling, transactional Idempotency-Key serialization, parent-scope fail-closed integrity, forced RLS, and atomic audit/outbox evidence. ADR 0014 records the persistence decision while ADR 0007 remains the domain/evidence authority; validated evidence still requires accountable human review and non-LLM provenance, and the service does not make a high-impact employment decision. - Active-PR `orgmetra_selection_review` packet for PII-minimized, evidence-bound human selection review: canonical operational tenant identity, UUID-backed opaque candidate/Job/sealed-evidence/reviewer references, explicit purpose/reason/evidence version, deterministic canonical JSON and SHA-256 correlation, mandatory human decision state, redacted packet repr, and provenance-paired model evidence that remains `untrusted_draft`, with exact 100% owned statement and branch coverage required by its quality gate. +- Active-PR `orgmetra_validity_analysis` handoff/result boundary: exact tenant and evidence references, reviewed fast-mlsirm pin, distinct requester/reviewer actors, aggregate-only scientific evidence, construction-time UTC and finite-number snapshots, canonical JSON/SHA-256 correlation, and human-review-only result authority. - Active performance-criterion scope hardening: `criterion_observation_scope_guard` rejects criterion outcomes for a Job the worker did not effectively hold at the observation date, observations before the relevant assignment, and observations outside the referenced performance cycle while preserving valid multiple-assignment cases and existing bitemporal correction semantics. The guard evaluates current-recorded facts, derives the date coordinate from `observed_at` in UTC so session `TimeZone` cannot alter the result, uses a trusted function search path, and adds no PII or automated employment decision authority. The Foundation PostgreSQL contract also rejects a closed `recorded_to` on each time-coordinate lookup and proves UTC midnight plus non-UTC session `TimeZone` boundaries. - Bitemporal tenant-scoped organization hierarchy validation that rejects visible indirect parent cycles and reuses single-valued recorded-time reconstruction before graph traversal. - Stacked governed job-analysis evidence contract via `JobAnalysisSnapshot`, `TaskEvidence`, `KSAORequirement`, `TaskKSAOLink`, `FunctionalJobAnalysisProfile`, and `EvidenceSource`: tenant/Job-scoped observable tasks, explicit Task-to-KSAO linkage, importance/difficulty/proficiency ratings, source/version/retrieval/SHA-256 provenance, deterministic canonical snapshot bytes, current O*NET evidence support, and historical DOT Data/People/Things compatibility. Validated snapshots require accountable human review and complete non-LLM evidence; LLM-origin material remains `analysis_draft`, and the snapshot is evidence input rather than a hiring, promotion, termination, compensation, or other high-impact employment decision. @@ -18,7 +19,7 @@ All notable changes to Orgmetra will be documented in this file. - `employment_record_version.employment_concurrency_code` constrained to `exclusive` or `concurrent`. - ADR 0005 for exclusive employment and staffable seats. - `orgmetra_hris_kernel` 0.3.0 with identity-scoped bitemporal resolution, assignment-employment coverage, allocation-portfolio checks, and a Memorial Hospital RN correction case at 100% statement and branch coverage. -- `employment_record_version` and `position_record_version` so employment and position identity stay stable across retroactive corrections. +- `employment_record_version` and `position_record_version` so corrections no longer mint a new employment or position identifier. - `assignment_record.employment_record_id` bound to the same person as the covering employment. - `orgmetra_keyverse_adapter` that binds an opaque Keyverse subject to a person and rejects passwords, passkeys, and tokens. - Design tokens for the repeating HR actions: approve, review, correct, request evidence, compare, export, and escalate. diff --git a/docs/adr/0027-governed-selection-validity-analysis-handoff.md b/docs/adr/0027-governed-selection-validity-analysis-handoff.md new file mode 100644 index 000000000..008d97d60 --- /dev/null +++ b/docs/adr/0027-governed-selection-validity-analysis-handoff.md @@ -0,0 +1,85 @@ +# ADR 0027: Govern selection-validity numerical work through an immutable handoff + +- Status: Proposed +- Maturity: Active PR only; not protected-branch truth +- Date: 2026-08-21 +- Owners: Orgmetra Workforce Validation + +## Context + +Protected Orgmetra already preserves exact validation-study cases, sealed selection evidence, candidate-to-worker lineage, and Job/cycle/staffing-scoped criterion observations. The remaining boundary is dangerous if left implicit: a statistical worker could receive an underspecified study, silently use a different dependency revision, or turn a model result into an employment decision. + +The Uniform Guidelines recognize criterion-related validity evidence as empirical evidence relating a selection procedure to important job-performance elements and require validity studies to be accurate, standardized, documented, and periodically reviewed for currency. SIOP's *Principles for the Validation and Use of Personnel Selection Procedures* likewise treats validation as an evidence-and-inference problem rather than a correlation-only shortcut. + +`ContextualWisdomLab/fast-mlsirm` owns numerical psychometric/statistical kernels. Its protected `main` was freshly resolved to commit `04d0bc21a2a20693bcf16108cd76d394fe844d23` on 2026-08-21. Orgmetra must not copy that implementation or write the foreign repository. + +## Decision + +Orgmetra adds a leaf `orgmetra_validity_analysis` package whose `ValidationAnalysisHandoff`: + +- binds the exact tenant, validation study, Job, predictor snapshot, criterion snapshot, population snapshot, decision policy, and analysis plan through opaque references plus SHA-256 evidence digests; +- binds distinct requester and reviewer actor references; +- pins fast-mlsirm to reviewed immutable commit `04d0bc21a2a20693bcf16108cd76d394fe844d23`; +- declares the numerical boundary `read_only_pinned_revision` and the initial strategy `criterion_related`; +- requires downstream result evidence for effect estimate, uncertainty interval, sample size, missingness summary, and convergence diagnostics; +- serializes no raw person-level predictor, criterion, candidate, or worker values; +- remains `not_executed`, `scientific_evidence_only`, and human-review-required; +- produces deterministic canonical JSON and a SHA-256 digest for audit/result correlation. + +Both handoff and result envelopes detach exact timezone-aware timestamps to one built-in UTC instant at construction. Result numeric evidence is converted to finite built-in floats before storage, and conversion failures including numeric overflow are normalized to the package's fail-closed `ValueError` contract, so caller-controlled timezone or numeric runtime behavior cannot rewrite canonical evidence after validation or escape normal malformed-result handling. + +The same package also validates `ValidationAnalysisResult` envelopes returned by the approved offline worker. A result must link to the handoff digest and the same pinned revision, identify a Rust CPU or GPU backend and precision, provide finite effect and interval values, match its sample size to aggregate missingness counts, reject impossible complete-versus-missing count combinations, and include explicit convergence diagnostics. The canonicalization boundary accepts only the exact governed `MissingnessSummary` and `ConvergenceDiagnostics` runtime types so subclass method overrides cannot add unreviewed or person-level fields to immutable result evidence. A nonconverged result remains typed scientific evidence requiring human review; it cannot be treated as a valid selection procedure or an employment decision. + +Released scientific results are append-only. The initial `ValidationAnalysisResult` uses `correction_sequence=1` and carries no predecessor coordinates. Every corrected result uses a new `validation_analysis_result:*` reference and binds the exact predecessor result reference and SHA-256 digest into its own canonical bytes. A changed estimate, uncertainty interval, sample, weight binding, or other material result evidence therefore cannot silently overwrite a previously released result while retaining ambiguous lineage. This leaf contract is correlation evidence rather than durable authority: canonical `workforce_validation` owner persistence must independently resolve predecessor release, exclusive supersession cutover, complete successor reference/correction-sequence/digest/release evidence, and historical half-open authority intervals. + +For #407's weighted design-based inference boundary, the active package adds `FinalAnalysisWeightReceipt`. It binds the exact estimand, target population, analysis window, reference duration, eligible/analytic case-set digests, #404/#405 source/sampling receipt digests, base-weight derivation evidence, an ordered immutable adjustment chain, the final point-weight artifact digest, construction time, and append-only correction lineage without storing row-level weights. Each adjustment must be contiguous and digest-linked from the previous artifact to the declared final artifact. `ValidationAnalysisResult` explicitly distinguishes `unweighted` from `weighted_design_based`; a weighted result fails closed unless it separately binds both the final analysis-weight receipt digest and the #406 variance-design receipt digest. An unweighted result cannot carry either receipt and thereby masquerade as weighted scientific evidence. + +`WeightEligibilityReceipt` makes weight use itself versioned evidence rather than a caller-supplied label. The receipt is closed to `cross_sectional` or `longitudinal` scope and binds the exact target population, reference-duration evidence, eligible-case set, and final point-weight artifact. `FinalAnalysisWeightReceipt` validates that the eligibility receipt belongs to the same tenant and that its scope, target population, reference duration, eligible cases, and weight artifact exactly match the estimand-side receipt. A longitudinal weight therefore cannot silently support a cross-sectional estimand or a different target period. The 2025 SIPP Users' Guide is used only as current primary methodological evidence that weight choice depends on both target population and duration and that longitudinal weights cover explicit multi-year reference periods; SIPP-specific variables or estimators are not imported into Orgmetra. + +The adjustment chain has typed scientific evidence for three high-risk transform families. `NonresponseAdjustmentReceipt` binds a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, explicit treatment of ineligible/unknown/unavailable cases, and input/output weight artifacts. `CalibrationAdjustmentReceipt` binds the target population/window, **versioned purpose-limited auxiliary projection reference/version/digest**, exact scientific-use purpose reference/digest, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and use instant, and a separately versioned calibration benchmark receipt reference/version/digest with its released benchmark owner-contract reference/version/digest and exact benchmark reference instant. It also binds algorithm/version, the reviewed/requested constraint set as `constraints_digest`, the actual generating constraint set as `applied_constraints_digest`, input/output artifacts, and an explicit `converged` or `fallback_applied` termination state. Primary `converged` evidence is valid only when reviewed and applied constraint digests are identical. Any bound relaxation, cell collapse, or other constraint change must cross the explicit fallback path and identify the primary failure reason, immutable fallback-rule reference/digest, actual fallback calibration algorithm/version, and configuration digest that generated the output. Both the scientific-use instant and benchmark reference instant are frozen and cannot be later than receipt construction. This makes the complete opaque correlation tuple expected by canonical Workforce Validation owner resolution part of the scientific receipt without copying protected auxiliary or benchmark values. The leaf package still does not pretend its caller-supplied coordinates are authoritative: the durable `workforce_validation` service must resolve the released auxiliary owner contract, authorization receipt, scientific-use receipt, benchmark receipt, released benchmark owner contract, reviewed constraints, and applied constraints before execution or release. Fallback-only evidence is forbidden when the primary algorithm converged. A failed/nonconverged calibration cannot be labeled as an accepted calibration receipt. `TrimmingBoundingAdjustmentReceipt` binds a versioned trimming/bounding rule, reproducible rule configuration digest, the semantic occurrence set and count of cases actually affected, and input/output weight artifacts; a declared trim/bound transform cannot be a no-op. Known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization `AnalysisWeightAdjustment` codes also carry an `evidence_kind` and fail closed unless it names the matching typed receipt family. + +#407 RED #8 requires more than two different well-formed digests. A final point weight must not be combined with replicate or variance evidence generated from a different eligibility, correction, calibration, analytic case set, or final-weight version. The active package therefore adds `WeightVarianceCompatibilityReceipt`. It binds one exact `FinalAnalysisWeightReceipt` to the separate #406 variance-design receipt reference/version/digest and requires the variance side to identify the exact same analysis-weight receipt digest, analytic-case occurrence set, weight-eligibility receipt digest, correction sequence, and final-weight artifact. The receipt must be constructed no earlier than the point-weight receipt, and its variance-design digest must remain distinct from the point-weight receipt itself. `ValidationAnalysisResult` requires this compatibility receipt for `weighted_design_based` output and verifies that its point-weight and variance-design digests are the same ones recorded on the result. Unweighted results cannot carry compatibility evidence. + +This compatibility receipt is deliberately not a self-authenticating variance owner. Its role is deterministic scientific correlation at the leaf boundary: it prevents a caller from presenting internally inconsistent point/variance lineages as one weighted result. Canonical #235/#248 remain responsible for resolving the released #406 variance-design evidence, proving that the variance-side coordinates actually came from the authoritative released owner contract, and rejecting caller-fabricated correlation data. The same owner boundary continues to apply separately to the calibration auxiliary-authority tuple, calibration benchmark-authority tuple, calibration constraint tuple, and validation-result predecessor/successor correction tuple. + +This executable slice still does not claim #407 complete. The leaf preserves the opaque coordinates needed for durable owner-side calibration auxiliary, benchmark, constraint, point/variance and result-correction correlation and can fail closed when point-weight and variance evidence are internally incongruent. Durable service/API verification that these coordinates resolve to released typed receipts/contracts, verification that released authorization permits the resolved **projection version** for the exact purpose and use, owner-resolved validation-result supersession chronology, and released auxiliary/benchmark/variance owner evidence exchange remain `workforce_validation` scientific truth rather than Talent truth or cross-context SQL reconstruction. + +The NIST Privacy Framework 1.0 is used narrowly as a privacy-risk-management basis for expressing and verifying data-processing requirements across organizational roles and contracts; as of the 2026-09-17 check, NIST still presents Privacy Framework 1.1 as an Initial Public Draft rather than a final replacement. This ADR does not treat the voluntary framework as employment law or infer legal permission from a NIST profile. + +The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, prove requester/reviewer identities are distinct authoritative actors, verify cross-sectional/longitudinal weight eligibility against the estimand, verify typed weight-adjustment receipts plus any calibration auxiliary **projection reference/version/digest** and authority/purpose/released-owner/authorization/scientific-use tuple and calibration benchmark receipt/version/released-owner/reference-time tuple, verify reviewed-versus-applied calibration constraints and any fallback reason/rule/actual fallback algorithm-version-configuration tuple, resolve corrected-result predecessor evidence, and resolve the point/variance compatibility receipt against the released #406 owner evidence rather than trusting caller-supplied evidence labels or digests. + +## Consequences + +### Positive + +- Statistical work cannot silently drift to an unreviewed fast-mlsirm revision. +- A buyer or auditor can identify exactly which governed study evidence a result was supposed to analyze without copying raw worker data into the handoff. +- Aggregate missingness evidence cannot claim all observations are complete while simultaneously reporting predictor- or criterion-missing observations. +- Result canonicalization cannot be extended by an unreviewed subclass to serialize extra decision-like or person-level fields. +- Caller-controlled timestamp and numeric runtime behavior cannot rewrite an accepted canonical digest after construction or turn malformed oversized worker output into an uncaught exception type. +- A corrected scientific result cannot silently replace a predecessor: every correction gets a new result reference and digest-linked predecessor lineage. +- Weighted design-based results can no longer identify only a sample while omitting which final point-weight evidence and variance-design evidence were actually used. +- A variance/replicate construction from a different analytic case set, eligibility receipt, correction sequence, or final point-weight artifact cannot be paired with the point estimate as though the two lineages were congruent. +- Cross-sectional and longitudinal weights cannot be interchanged when target population, reference duration, eligible cases, or final weight artifact differ from the estimand. +- The ordered weight transformation chain is independently digest-correlatable without centralizing case-level weights or auxiliary attributes. +- Nonresponse cannot silently drop refusal/unreachable/ineligible/failure dispositions behind one opaque adjustment label. +- Calibration/raking cannot silently float auxiliary projection **version**, benchmark ownership, benchmark version, benchmark reference time, reviewed constraints, or actually applied constraints; omit the exact scientific-use purpose/released-owner/authorization/use correlation tuple; move its use or benchmark time after receipt construction; or hide a constraint relaxation/fallback as primary-algorithm convergence. +- Trimming/bounding/winsorization cannot silently alter final point weights without an immutable rule/configuration and affected-case receipt. +- Human interpretation remains explicit and separate from numerical output. +- The dedicated-writer boundary remains intact: Orgmetra consumes only a pinned foreign revision/contract boundary and never mutates fast-mlsirm. + +### Limitations + +- This slice does not execute a statistical model, estimate validity, correct for measurement error/range restriction, evaluate adverse impact, or assert legal compliance. +- Sampling design, estimator choice, missing-data treatment, reliability evidence, multiplicity, transportability, fairness analysis, and model diagnostics must be encoded in the referenced analysis plan and reviewed before execution. +- The current #407 slice validates generic ordered weight provenance, weighted-result binding, result correction linkage, point/variance compatibility, cross-sectional/longitudinal eligibility congruence, disposition-aware nonresponse evidence, owner-correlatable calibration auxiliary and benchmark evidence, reviewed-versus-applied calibration constraints, explicit fallback generating-method provenance, and trimming/bounding rule provenance; it does not yet make durable typed-receipt/authorization/benchmark-owner/constraint-owner/variance-owner/result-supersession resolution or released auxiliary/benchmark/variance exchange fully executable. +- The generic result envelope does not invent an estimator-specific minimum complete-case count. Whether a converged estimator is identified under a particular missing-data design belongs to the reviewed analysis plan and numerical-worker contract; Orgmetra fails closed on impossible aggregate counts without silently replacing that foreign scientific contract with complete-case analysis. +- The package validates the result envelope, but a future execution adapter must still re-resolve the handoff references, verify the result provenance artifact, resolve typed adjustment, purpose-authorization, benchmark, calibration-constraint and result-correction coordinates to released owner evidence, resolve point/variance compatibility against released #406 evidence, and attach evidence only after accountable human review. + +## Verification + +The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. #407 adds RED/GREEN coverage for append-only validation-result predecessor linkage, deterministic value-minimized final-weight receipts, exact sampling-receipt binding, contiguous transformation lineage, append-only weight correction linkage, fail-closed weighted-result binding to separate point-weight and variance-design receipts, exact point/variance compatibility across analysis-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence and final-weight artifact, cross-sectional/longitudinal scope and target-population/reference-duration/case-set/final-artifact congruence, explicit nonresponse disposition treatment, calibration **auxiliary projection version**, benchmark receipt version/owner-contract/reference-time provenance, reviewed/requested versus actually applied calibration constraint digests, primary-convergence equality and explicit fallback-only constraint drift, scientific auxiliary-authority/purpose/released-owner/authorization/scientific-use correlation coordinates, owner-contract and use-receipt digest validation, future-use-time and future-benchmark-time rejection, fallback reason/rule/actual algorithm-version-configuration disclosure, converged-state rejection of fallback-only evidence, trimming/bounding rule/configuration and affected-case provenance, no-op trim/bound rejection, and typed adjustment-evidence matching. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. + +## References + +See `docs/doctoring/validation-analysis-handoff-references.md`. diff --git a/docs/doctoring/validation-analysis-handoff-references.md b/docs/doctoring/validation-analysis-handoff-references.md new file mode 100644 index 000000000..1a337f705 --- /dev/null +++ b/docs/doctoring/validation-analysis-handoff-references.md @@ -0,0 +1,37 @@ +# Validation-analysis handoff references + +Material decisions for ADR 0027 were checked against the following primary/authoritative sources on 2026-08-21. Regulatory currency was rechecked on 2026-08-29. The #407 analysis-weight evidence boundary was checked on 2026-09-17 against the primary calibration paper, current U.S. Census methodological/quality documentation, and the current final NIST Privacy Framework. Fixed publication identifiers are retained where possible so an auditor can reproduce the cited text even when agency web pages change. + +## APA 7 references + +Equal Employment Opportunity Commission, Civil Service Commission, Department of Justice, & Department of Labor. (1978). *Uniform Guidelines on Employee Selection Procedures (1978)*, 43 Fed. Reg. 38,290 (August 25, 1978) (codified at 29 C.F.R. pt. 1607). The EEOC continues to list 29 C.F.R. pt. 1607 among its Title VII regulations: https://www.eeoc.gov/regulations-and-guidelines + +Society for Industrial and Organizational Psychology. (2018). Principles for the validation and use of personnel selection procedures. *Industrial and Organizational Psychology, 11*(S1), 1–97. https://doi.org/10.1017/iop.2018.195 + +Deville, J.-C., & Särndal, C.-E. (1992). Calibration estimators in survey sampling. *Journal of the American Statistical Association, 87*(418), 376–382. https://doi.org/10.1080/01621459.1992.10475217 + +U.S. Census Bureau. (2021). *Statistical Quality Standard D1: Producing direct estimates from samples*. https://www.census.gov/about/policies/quality/standards/standardd1.html + +U.S. Census Bureau. (2022, August 18). *Survey of Income and Program Participation: Weighting*. https://www.census.gov/programs-surveys/sipp/methodology/weighting.html + +U.S. Census Bureau. (2026). *2025 Survey of Income and Program Participation users' guide* (August 2026 revision), pp. 156–157. https://www2.census.gov/programs-surveys/sipp/tech-documentation/methodology/2025_SIPP_Users_Guide.pdf + +Boeckl, K., & Lefkovitz, N. (2020). *NIST Privacy Framework: A tool for improving privacy through enterprise risk management, Version 1.0* (NIST CSWP 01162020). National Institute of Standards and Technology. https://doi.org/10.6028/NIST.CSWP.01162020 + +ContextualWisdomLab. (2026). *fast-mlsirm* (Commit 04d0bc21a2a20693bcf16108cd76d394fe844d23) [Computer software]. GitHub. https://github.com/ContextualWisdomLab/fast-mlsirm/tree/04d0bc21a2a20693bcf16108cd76d394fe844d23 + +Tabassi, E. (2023). *Artificial intelligence risk management framework (AI RMF 1.0)* (NIST AI 100-1). National Institute of Standards and Technology. https://doi.org/10.6028/NIST.AI.100-1 + +Office of Personnel Management. (2026). *Removal of references to the Uniform Guidelines on Employee Selection Procedures in federal personnel regulations*, 91 Fed. Reg. 48,234 (July 31, 2026) (interim final rule, RIN 3206-AP20). + +## Decision notes + +- 43 Fed. Reg. 38,290 and the still-listed EEOC 29 C.F.R. pt. 1607 source support keeping criterion-related validity evidence tied to an explicit study design, job relevance, accuracy, reporting, and documentation rather than treating a bare coefficient as sufficient evidence. The fixed Federal Register identifier, not a mutable `/current/` eCFR URL, is the reproducible source for the 1978 text cited by this ADR. +- The July 31, 2026 OPM interim final rule removed UGESP references from specified federal civil-service regulations. Orgmetra therefore does not present UGESP as an undifferentiated government-wide mandate; applicability must be evaluated for the employer, jurisdiction, decision, and governing law at use time. +- The SIOP Principles are the professional validation baseline used for the handoff's evidence-and-human-review posture. The journal citation above fixes volume 11, Supplement S1, pages 1–97, and DOI 10.1017/iop.2018.195. +- Deville and Särndal show that calibrated weights are produced by modifying ordinary inverse-inclusion-probability weights under explicit distance measures and calibration equations. ADR 0027 uses that narrow result to justify treating final adjusted point weights as a separately versioned scientific artifact rather than assuming `1/π_i` and calibrated weights are interchangeable. It does not mandate one calibration estimator for Orgmetra. +- Census Statistical Quality Standard D1 requires estimates and variances to account for sample design and post-sampling weighting adjustments. The SIPP methodology illustrates that final weights can combine base selection, nonresponse, longitudinal/panel, and post-stratification/calibration adjustments. These sources support provenance/reproducibility requirements only; SIPP-specific weights are not imported as Orgmetra rules. +- The August 2026 revision of the 2025 SIPP Users' Guide states that choosing a weight depends on the population to which results apply and the duration of interest, distinguishes cross-sectional monthly analysis from longitudinal multi-year analysis, and identifies explicit two-, three-, and four-year reference periods for longitudinal weights. ADR 0027 uses this only to justify fail-closed target-population/reference-duration congruence for `WeightEligibilityReceipt`; it does not adopt SIPP variable names, cohorts, or estimators as Orgmetra domain truth. +- The final NIST Privacy Framework 1.0 is used narrowly to support explicit, verifiable privacy requirements across roles in a data-processing ecosystem and life-cycle privacy-risk management. As checked on 2026-09-17, NIST's official 1.1 project page still presents Version 1.1 as an Initial Public Draft rather than a final replacement, so this ADR does not cite the draft as settled authority. The framework is voluntary and jurisdiction-agnostic; it is not employment-law permission. Orgmetra uses it only to justify carrying scientific-use purpose, released owner-contract coordinates and authorization evidence without copying sensitive auxiliary values across bounded contexts. +- The fast-mlsirm commit is recorded as a read-only dependency coordinate only. This Orgmetra slice does not modify or duplicate its numerical implementation. +- NIST AI RMF's govern, map, measure, and manage functions support preserving backend, precision, provenance, convergence, and human-review fields as inspectable result evidence rather than treating a model response as an autonomous decision. diff --git a/docs/traceability/validation-analysis-handoff.md b/docs/traceability/validation-analysis-handoff.md new file mode 100644 index 000000000..08833862e --- /dev/null +++ b/docs/traceability/validation-analysis-handoff.md @@ -0,0 +1,45 @@ +# Selection-validity analysis handoff traceability + +## Buyer question + +Can an organization send one exact, reviewable validation study to its statistical engine without copying raw person-level values into a workflow envelope, silently changing the numerical dependency, or treating model output as an employment decision? + +## Active-PR contract + +| Concern | Orgmetra evidence | Verification | +|---|---|---| +| Exact study scope | tenant, validation-study, Job, predictor, criterion, population, decision-policy, and analysis-plan references plus digests | namespace/UUID/digest regressions | +| Dependency integrity | immutable fast-mlsirm commit `04d0bc21a2a20693bcf16108cd76d394fe844d23` | malformed and unreviewed revision rejection | +| Privacy minimization | no raw person-level values in canonical handoff or result; result canonicalization accepts only exact governed missingness/convergence runtime types | canonical-payload/redacted-repr regressions plus subclass-injection rejection | +| Human authority | requester/reviewer references must differ, and the host must re-resolve both within the tenant and prove they resolve to distinct authoritative actor identities before execution | direct-construction fail-closed regression plus `test_next_action_requires_resolved_actor_identity_separation` | +| Scientific evidence | effect estimate, uncertainty interval, sample size, internally possible aggregate missingness, convergence diagnostics | immutable required-result-evidence regression plus impossible-missingness rejection | +| Numerical result boundary | handoff digest, pinned fast-mlsirm revision, Rust CPU/GPU backend, precision, finite estimate/interval, aggregate missingness, explicit convergence state | `ValidationAnalysisResult` invariant/canonicalization regressions, exact-runtime-type checks, and oversized-numeric `ValueError` normalization | +| Result correction lineage | correction sequence plus exact predecessor `validation_analysis_result` reference/digest for every corrected result; sequence 1 has no predecessor and a corrected result must use a new result reference | `test_result_correction_lineage.py` deterministic successor and malformed-lineage regressions | +| Final point-weight provenance | exact estimand/target/window/reference-duration and case-set digests, source/sampling receipt digests, base-weight evidence, ordered digest-linked adjustments, final weight artifact, append-only correction lineage | `FinalAnalysisWeightReceipt` deterministic/value-minimized regressions plus chain/correction fail-closed tests | +| Weight eligibility congruence | governed `cross_sectional` or `longitudinal` scope plus exact target population, reference-duration evidence, eligible case set, and final point-weight artifact | `test_weight_eligibility_receipt.py` plus `FinalAnalysisWeightReceipt` mismatch/longitudinal-match regressions | +| Nonresponse adjustment evidence | versioned response/disposition receipt, exact adjustment population, controlled method/configuration, explicit ineligible/unknown/unavailable treatment, input/output weight artifacts | `NonresponseAdjustmentReceipt` deterministic/value-minimized tests plus undocumented-treatment/no-op/version rejection | +| Calibration/raking evidence | target population/window, **versioned** purpose-bound auxiliary projection reference/version/digest, scientific-use purpose reference/digest, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and use instant, calibration benchmark receipt reference/version/digest, released benchmark owner-contract reference/version/digest, benchmark reference instant, algorithm/version, reviewed/requested `constraints_digest`, actual `applied_constraints_digest`, input/output artifacts, explicit converged/fallback state; fallback output additionally binds the primary failure reason, immutable fallback rule, actual fallback algorithm/version, and fallback configuration digest | `test_calibration_constraint_application.py`, `CalibrationAdjustmentReceipt` auxiliary-projection-version/authority/purpose/owner/authorization/scientific-use/benchmark-owner/benchmark-time/termination regressions, and `test_calibration_fallback_provenance.py`; primary convergence requires reviewed and applied constraints to match, while any constraint drift requires the explicit fallback path | +| Trimming/bounding evidence | versioned trimming/bounding rule, reproducible rule configuration digest, exact affected semantic occurrence set and count, input/output weight artifacts | `test_trimming_bounding_receipt.py` deterministic/value-minimized, no-op, missing-provenance, and typed-kind regressions | +| Typed adjustment congruence | known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization adjustment codes must identify the matching typed receipt family through `evidence_kind` | specialized adjustment evidence-kind regressions | +| Point/variance lineage congruence | #406 variance evidence must name the same final point-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence, and final-weight artifact used by the estimate | `test_weight_variance_compatibility_receipt.py` mismatch regressions | +| Weighted-result congruence | `weighted_design_based` result must bind the exact final point-weight receipt, a separate #406 variance-design receipt, and their exact compatibility receipt; `unweighted` result must bind none | `test_analysis_weight_result_binding.py` | +| Execution boundary | `not_executed`, `scientific_evidence_only`, read-only pinned foreign dependency | immutable governance regressions | +| Reproducibility | construction-time UTC timestamp snapshots, finite numeric snapshots, canonical RFC 3339 time, canonical JSON, SHA-256 handoff/result/weight/adjustment/eligibility/compatibility-receipt digests | mutable timezone/numeric and UTC-boundary regressions plus deterministic serialization/digest tests | +| Decision-record integrity | ADR numbers remain unique repository-wide and any `docs/adr/**` change reaches the consolidated Foundation CI validity regression | ADR uniqueness regression plus Foundation CI workflow-trigger contract regression | +| Quality-evidence freshness | consolidated Foundation CI runs the validity package on every `develop` pull request without a repository path filter, so shared Python/test/clean-checkout configuration cannot silently bypass the package gate | `test_foundation_ci_retriggers_without_path_filter` and `test_foundation_ci_runs_validity_analysis_and_adr_changes`; central required workflows remain separate gates | + +## #407 boundary still open + +The active branch makes final-weight eligibility and three adjustment families executable rather than leaving them as opaque labels. It now also closes the leaf-side form of #407 RED #8: `WeightVarianceCompatibilityReceipt` requires the variance side to identify the exact point-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence, and final-weight artifact. `ValidationAnalysisResult` cannot emit weighted design-based evidence without that compatibility receipt in addition to separate point-weight and #406 variance-design digests. + +RED #10 also requires corrected result evidence to be append-only rather than an in-place rewrite. `ValidationAnalysisResult` now serializes `correction_sequence`; sequence 1 rejects predecessor coordinates, while every later correction must use a new result reference and bind the exact predecessor result reference/digest. This is still leaf correlation evidence. Canonical #235/#248 must resolve predecessor/successor release chronology and authority intervals so a superseded result cannot remain current merely because its immutable bytes still exist. + +Calibration fallback is explicit scientific provenance rather than a success-like label. The leaf now separately commits to the reviewed/requested constraint set and the constraint set actually used to generate the calibrated weights. Primary `converged` evidence requires those digests to be identical. A bound relaxation, cell collapse, or other changed constraint set therefore cannot be serialized as primary convergence; it must use `fallback_applied`, which also requires the reason the primary calibration ceased to be authoritative, the versioned fallback rule, and the actual fallback calibration algorithm/version/configuration that produced the output artifact. + +This is still correlation evidence, not owner authentication. Calibration/raking/post-stratification evidence carries the opaque coordinates needed by canonical Workforce Validation authority resolution: auxiliary authority and **projection reference/version/digest**, exact scientific purpose, released auxiliary owner-contract identity/version/digest, authorization receipt identity/digest, scientific-use receipt identity/digest and owner-correlatable use instant, separately versioned benchmark receipt identity/digest, released benchmark owner-contract identity/version/digest, benchmark reference instant, and reviewed-versus-applied constraint digests. Durable #235/#248 owner-side resolution must prove these coordinates against released/versioned owner evidence and resolve the point/variance compatibility coordinates against released #406 variance evidence rather than trusting leaf-supplied digests. Released auxiliary/benchmark/variance evidence exchange also remains open. `workforce_validation` owns those scientific contracts and their released result; `talent_management` must not compute or reconstruct final analysis weights. + +## Maturity + +`implemented_on_active_pr`. + +Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. The active package validates the returned numerical/provenance envelope plus the current #407 final-weight/result-binding, result-correction lineage, point/variance compatibility, weight-eligibility, typed adjustment-evidence, fallback generating-method, calibration-constraint application, auxiliary correlation, and benchmark correlation contracts, but protected Orgmetra evidence still requires host re-resolution, durable owner evidence exchange, result-artifact verification, terminal checks, independent review, and accountable human interpretation. diff --git a/manifest.json b/manifest.json index f7b6cf55e..082180c23 100644 --- a/manifest.json +++ b/manifest.json @@ -5,9 +5,9 @@ "files": [ { "path": ".github/workflows/foundation-ci.yml", - "sha256": "b6a4365936b66803a8112f034c77d53d33301a7a798ed4f68746a4f2d8b081d7", - "bytes": 6651, - "lines": 125 + "sha256": "69e35b7a12e7ebcbe039ef349e2311702fdbfeee8916d4c78a64cf8bd8a1f430", + "bytes": 6853, + "lines": 126 }, { "path": ".gitignore", @@ -29,9 +29,9 @@ }, { "path": "CHANGELOG.md", - "sha256": "f2d2e0b488c0440533effa821808f2f17e37d92f8fb586174c2fdb594f760ca5", - "bytes": 17539, - "lines": 77 + "sha256": "1985ccaf0dfa0fbf095e88998cfcfdac6f245525ac6ff0131d9f29ed43d97b93", + "bytes": 17846, + "lines": 78 }, { "path": "CLAUDE.md", diff --git a/packages/validity-analysis/CHANGELOG.md b/packages/validity-analysis/CHANGELOG.md new file mode 100644 index 000000000..f25c350db --- /dev/null +++ b/packages/validity-analysis/CHANGELOG.md @@ -0,0 +1,24 @@ +# Changelog + +## 0.1.0 - Unreleased + +- Add a governed, value-minimized criterion-related validity analysis handoff. +- Pin the reviewed read-only fast-mlsirm dependency revision. +- Require separate requester/reviewer references and authoritative tenant-scoped re-resolution proving they resolve to distinct actor identities before execution. +- Require deterministic canonical evidence and 100% owned production statement/branch coverage. +- Validate a digest-linked Rust CPU/GPU result envelope with finite estimates, aggregate missingness, and explicit convergence or nonconvergence diagnostics. +- Make result verification explicit without self-authorizing the scientific leaf: converged completed results serialize as `verification_pending`, while completed nonconverged results serialize as `not_verifiable` and retain their failure diagnostics. +- Version corrected `ValidationAnalysisResult` evidence append-only: the initial result is correction sequence 1 with no predecessor, while every corrected result uses a new result reference and binds the exact predecessor result reference/digest into canonical bytes. +- Reject impossible aggregate missingness where complete observations overlap either predictor-missing or criterion-missing counts beyond the sample total. +- Require exact governed missingness/convergence runtime types so subclass method overrides cannot inject unreviewed or person-level fields into canonical result evidence. +- Freeze exact UTC timestamps and finite numeric values at construction, and reject runtime-type forgery before canonical evidence serialization. +- Add a deterministic, row-value-minimized `FinalAnalysisWeightReceipt` that binds an exact estimand to source/sampling evidence, base-weight provenance, an ordered digest-linked adjustment chain, the final point-weight artifact, and append-only correction lineage. +- Add `WeightEligibilityReceipt` and fail closed unless cross-sectional/longitudinal scope, target population, reference duration, eligible case set, and final point-weight artifact match the estimand-side final-weight receipt exactly. +- Distinguish unweighted from weighted design-based results and fail closed unless a weighted result separately binds the exact final analysis-weight receipt and variance-design receipt used. +- Add `WeightVarianceCompatibilityReceipt` so variance evidence must correlate to the exact final point-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence, and final-weight artifact before a weighted result can be emitted. The receipt is correlation evidence, not self-authenticating #406 owner authority. +- Require `ValidationAnalysisResult` to bind the compatibility receipt in addition to the point-weight and variance-design digests; unweighted results cannot carry compatibility evidence. +- Add typed `NonresponseAdjustmentReceipt` evidence that binds the exact versioned response/disposition receipt reference/version/digest, adjustment population, controlled method/configuration, explicit ineligible/unknown/unavailable treatment, and input/output weight artifacts; add typed `CalibrationAdjustmentReceipt` evidence bound to a versioned purpose-limited auxiliary projection identity/digest, exact scientific-use purpose, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest/use time, and a separately versioned calibration benchmark receipt with released benchmark owner-contract reference/version/digest and exact benchmark reference time. Calibration now records both the reviewed/requested `constraints_digest` and the actual `applied_constraints_digest`; primary `converged` evidence requires equality, while any changed generating constraint set must use `fallback_applied` with the primary failure reason, immutable fallback rule, actual fallback calibration algorithm/version, and fallback configuration digest. Neither the use time nor benchmark reference time may be later than receipt construction. +- Add typed `TrimmingBoundingAdjustmentReceipt` evidence that binds a versioned rule/configuration, exact affected semantic occurrence set and count, and input/output weight artifacts; known trimming/bounding/winsorization adjustments must name that evidence family. +- Require known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization adjustment codes to identify the matching typed evidence kind rather than collapse into a generic opaque adjustment digest. +- Keep durable owner-side typed-receipt, point/variance compatibility, auxiliary-authority, benchmark-authority, calibration-constraint, and corrected-result supersession resolution plus released auxiliary/benchmark/variance evidence exchange explicitly incomplete under #407; the scientific leaf preserves exact correlation coordinates including auxiliary projection version but does not self-authenticate them. +- Make `Validity Analysis Handoff Quality` retrigger on shared repository Python/test/clean-checkout configuration, with an executable regression preventing stale package-quality evidence after shared tooling changes. diff --git a/packages/validity-analysis/README.md b/packages/validity-analysis/README.md new file mode 100644 index 000000000..053b8a8f0 --- /dev/null +++ b/packages/validity-analysis/README.md @@ -0,0 +1,54 @@ +# Orgmetra validity-analysis handoff + +This package creates an immutable **selection-validity analysis handoff**, validates the matching numerical result envelope, and preserves executable #407 point-estimation weight lineage. It is the boundary between Orgmetra's authoritative validation-study evidence and numerical work owned by `ContextualWisdomLab/fast-mlsirm`. + +## What it does + +`build_validation_analysis_handoff(...)` binds one tenant, validation study, Job, predictor snapshot, criterion snapshot, population snapshot, decision policy, analysis plan, requester, reviewer, and the reviewed fast-mlsirm revision `04d0bc21a2a20693bcf16108cd76d394fe844d23`. + +The resulting canonical JSON is digest-addressable, contains no raw person-level predictor or criterion values, and remains `not_executed`. Its timestamp is detached to one UTC instant at construction so later timezone-provider changes cannot rewrite the digest. Required result evidence is explicit: effect estimate, uncertainty interval, sample size, missingness summary, and convergence diagnostics. + +`FinalAnalysisWeightReceipt` binds a weighted estimand to exact target/window/reference-duration/case-set evidence, source/sampling receipt digests, base-weight evidence, an ordered immutable adjustment chain, the final point-weight artifact digest, and append-only correction lineage. It records only identifiers, versions and digests; it does not store row-level weight values or copy calibration attributes into the validity package. + +`WeightEligibilityReceipt` makes cross-sectional versus longitudinal use machine-checkable rather than an opaque weight label. It binds the final weight artifact to one governed scope (`cross_sectional` or `longitudinal`), target population, reference-duration evidence, and eligible-case set. `FinalAnalysisWeightReceipt` fails closed unless those fields match the estimand and the same final point-weight artifact exactly, so a longitudinal weight cannot silently support a cross-sectional estimand or a different reference duration. + +Nonresponse and calibration are no longer allowed to hide behind an undifferentiated adjustment digest. `NonresponseAdjustmentReceipt` binds the adjustment to a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, and explicit treatment codes for ineligible, unknown, and unavailable cases. `CalibrationAdjustmentReceipt` binds the adjustment to a target population/window, **versioned** purpose-bound auxiliary projection identity/digest, exact scientific-use purpose, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and owner-correlatable use instant, plus a separately versioned calibration benchmark receipt with released benchmark owner-contract reference/version/digest and exact benchmark reference instant. It records the reviewed constraint set as `constraints_digest` and the constraint set that actually generated the output as `applied_constraints_digest`. A result labelled primary `converged` fails closed unless those digests are identical; any relaxation or other constraint change must therefore cross the explicit `fallback_applied` path and identify the failure reason, immutable fallback rule, actual fallback calibration algorithm/version, and fallback configuration. Neither the scientific-use instant nor the benchmark reference instant may be later than receipt construction. Fallback evidence is forbidden on a genuinely converged primary algorithm, and a nonconverged calibration cannot masquerade as an accepted calibration receipt. These fields preserve the opaque correlation tuples expected by the canonical Workforce Validation authority boundaries without copying protected auxiliary or benchmark values. The leaf receipt still does not authenticate those coordinates itself: the durable owner service must resolve the released auxiliary and benchmark evidence and prove that the exact projection version, benchmark, reviewed constraints, and applied constraints are authoritative for the scientific use. + +Trimming, bounding, and winsorization have a separate provenance family rather than falling back to a generic adjustment label. `TrimmingBoundingAdjustmentReceipt` binds the exact versioned rule, its reproducible configuration digest, the semantic occurrence set and count of cases actually affected, and the input/output weight artifacts. A declared trim/bound transform must change artifact identity. Known trimming/bounding/winsorization adjustment codes fail closed unless `evidence_kind` names this typed receipt family. + +`AnalysisWeightAdjustment` records an `evidence_kind` in addition to the evidence digest. Known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization transforms fail closed unless their evidence kind is the corresponding typed receipt. Other adjustment families remain open work rather than being silently treated as equivalent. + +`WeightVarianceCompatibilityReceipt` closes the scientific-leaf form of #407 RED #8. It binds one exact `FinalAnalysisWeightReceipt` to the separate #406 variance-design receipt/version/digest and requires the variance side to identify the same analytic-case occurrence set, weight-eligibility receipt, correction sequence, and final-weight artifact. A replicate or variance construction generated from a different eligibility, correction, case set, or final-weight version therefore cannot be paired with the point estimate merely because both digests are well formed. This receipt is correlation evidence only; #235/#248 remain responsible for resolving the released #406 owner evidence rather than trusting these caller-supplied coordinates as authority. + +`ValidationAnalysisResult` accepts only a result linked to the handoff digest and the same reviewed fast-mlsirm revision. It records the Rust CPU/GPU backend, precision, aggregate missingness counts, finite effect and interval values, and explicit convergence or nonconvergence diagnostics. Timestamps and finite numeric values are snapshotted before canonicalization, and the result envelope accepts only the exact governed `MissingnessSummary` and `ConvergenceDiagnostics` runtime types, preventing mutable runtime values or subclass method overrides from adding unreviewed or person-level fields to canonical audit evidence. Missingness counts must be internally possible: complete observations cannot overlap either predictor-missing or criterion-missing observations beyond the declared sample total. + +Point-estimation semantics are explicit. `unweighted` results cannot carry weight receipts. `weighted_design_based` results must separately bind the exact `FinalAnalysisWeightReceipt` digest, the variance-design receipt digest that supports the uncertainty method actually used, and the `WeightVarianceCompatibilityReceipt` proving those two evidence paths refer to the same point-weight basis. A replicate/variance evidence reference therefore cannot silently stand in for the final point-estimation weight, or vice versa. The result remains scientific evidence for accountable human interpretation and never becomes an employment decision. + +Corrected scientific results are append-only. The first `ValidationAnalysisResult` is `correction_sequence=1` and has no predecessor coordinates. Any corrected result must use a new `validation_analysis_result:*` reference and include the exact predecessor result reference and digest in its canonical bytes. This prevents a changed estimate, uncertainty interval, weight binding, sample, or other material result evidence from silently replacing a previously released result in place. The leaf records the predecessor link; canonical #235/#248 still must resolve released predecessor/successor chronology and authority intervals before a corrected result becomes durable owner truth. + +## What it does not do + +- It does **not** run statistics. +- It does **not** run or reproduce the fast-mlsirm numerical kernel. +- It does **not** query fast-mlsirm or any other CWL application's database. +- It does **not** claim that a selection procedure is valid. +- It does **not** interpret adverse impact. +- It does **not** authorize hiring, promotion, termination, compensation, or another employment decision. +- It does **not** yet complete #407. Durable owner-side verification that adjustment, point/variance compatibility, auxiliary-authority, benchmark-authority, constraint-set, and corrected-result predecessor/successor coordinates resolve to the released typed receipts/contracts they claim; enforcement that the resolved auxiliary projection **version** is authorized for the exact scientific purpose/use receipt/time; and released auxiliary/benchmark/variance evidence exchange remain open work in `workforce_validation`. + +The fast-mlsirm repository remains a dedicated-writer dependency. This package records only the immutable revision reviewed for the handoff. Durable Workforce Validation registry/API/persistence remains on the canonical #235/#248 owner path; this package does not create a parallel service. + +## Host obligations + +Before an approved offline validation worker executes the handoff, the Orgmetra host must re-resolve every reference inside `tenant_record_id` and prove that the predictor, criterion, population, and policy evidence belong to the exact validation study and Job. The requester and reviewer must resolve to distinct authoritative actors. For weighted design-based inference, the host must also verify the final point-weight receipt, its matching cross-sectional/longitudinal weight-eligibility receipt, its upstream source/sampling evidence, each typed adjustment receipt identified by the ordered chain, the calibration auxiliary authority/purpose/released owner contract/authorization/scientific-use tuple including the exact auxiliary projection **version**, the calibration benchmark receipt/version/released benchmark owner contract/reference instant, the reviewed and actually applied calibration constraint digests plus any fallback reason/rule/actual fallback algorithm-version-configuration tuple, the separately bound #406 variance-design evidence, and the point/variance compatibility receipt against released owner evidence rather than trusting caller-supplied labels or digests. A corrected result must additionally resolve its predecessor result reference/digest against released owner evidence instead of treating a caller-supplied correction sequence as authority. A numerical result is scientific evidence for accountable human interpretation, never an autonomous employment decision. + +## Verification + +Run: + +```bash +PYTHONPATH=packages/validity-analysis/src \ +python -m pytest -c packages/validity-analysis/pyproject.toml packages/validity-analysis/tests +``` + +The package gate requires exact 100% owned production statement and branch coverage. diff --git a/packages/validity-analysis/pyproject.toml b/packages/validity-analysis/pyproject.toml new file mode 100644 index 000000000..5547eabcb --- /dev/null +++ b/packages/validity-analysis/pyproject.toml @@ -0,0 +1,24 @@ +[build-system] +requires = ["setuptools>=69"] +build-backend = "setuptools.build_meta" + +[project] +name = "orgmetra-validity-analysis" +version = "0.1.0" +description = "Governed criterion-related selection-validity analysis handoff for Orgmetra." +requires-python = ">=3.12" + +[project.optional-dependencies] +test = ["pytest>=8.3", "pytest-cov>=5.0"] + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = [ + "--cov=orgmetra_validity_analysis", + "--cov-branch", + "--cov-report=term-missing", + "--cov-fail-under=100", +] diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py b/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py new file mode 100644 index 000000000..bd0ecf577 --- /dev/null +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py @@ -0,0 +1,33 @@ +"""Public governed selection-validity analysis handoff and result contracts.""" + +from .compatibility import WeightVarianceCompatibilityReceipt +from .handoff import ( + REVIEWED_FAST_MLSIRM_REVISION, + ValidationAnalysisHandoff, + build_validation_analysis_handoff, +) +from .result import ConvergenceDiagnostics, MissingnessSummary, ValidationAnalysisResult +from .weights import ( + AnalysisWeightAdjustment, + CalibrationAdjustmentReceipt, + FinalAnalysisWeightReceipt, + NonresponseAdjustmentReceipt, + TrimmingBoundingAdjustmentReceipt, + WeightEligibilityReceipt, +) + +__all__ = [ + "REVIEWED_FAST_MLSIRM_REVISION", + "ValidationAnalysisHandoff", + "build_validation_analysis_handoff", + "ConvergenceDiagnostics", + "MissingnessSummary", + "ValidationAnalysisResult", + "AnalysisWeightAdjustment", + "CalibrationAdjustmentReceipt", + "FinalAnalysisWeightReceipt", + "NonresponseAdjustmentReceipt", + "TrimmingBoundingAdjustmentReceipt", + "WeightEligibilityReceipt", + "WeightVarianceCompatibilityReceipt", +] diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/compatibility.py b/packages/validity-analysis/src/orgmetra_validity_analysis/compatibility.py new file mode 100644 index 000000000..9d29451aa --- /dev/null +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/compatibility.py @@ -0,0 +1,173 @@ +"""Correlate final point-weight evidence with the variance design that used it. + +This boundary prevents a weighted scientific result from combining a final point +weight with variance or replicate evidence generated from a different analytic +case set, eligibility receipt, correction sequence, or final-weight artifact. +It records only immutable references and digests; durable owner corroboration +remains an application/persistence responsibility. +""" +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from hashlib import sha256 +import json + +from .handoff import ( + _canonical_timestamp, + _freeze_timestamp, + _validate_digest, + _validate_operational_uuid, + _validate_reference, +) +from .weights import FinalAnalysisWeightReceipt + + +def _positive_integer(value: object, field_name: str) -> None: + """Require an exact positive integer without accepting booleans.""" + if type(value) is not int or value <= 0: + raise ValueError(f"{field_name} must be a positive integer") + + +@dataclass(frozen=True, slots=True, repr=False) +class WeightVarianceCompatibilityReceipt: + """Bind variance evidence to the exact final point-weight lineage it accompanies.""" + + tenant_record_id: str + receipt_reference: str + analysis_weight_receipt: FinalAnalysisWeightReceipt + variance_design_receipt_reference: str + variance_design_receipt_version: int + variance_design_receipt_digest: str + variance_analysis_weight_receipt_digest: str + variance_analytic_case_occurrence_set_digest: str + variance_weight_eligibility_receipt_digest: str + variance_weight_correction_sequence: int + variance_final_weight_artifact_digest: str + constructed_at: datetime + evidence_version: int = 1 + + def __post_init__(self) -> None: + """Reject variance evidence produced from any different point-weight basis.""" + _validate_operational_uuid(self.tenant_record_id, "tenant_record_id") + _validate_reference( + self.receipt_reference, + "weight_variance_compatibility_receipt", + "receipt_reference", + ) + if type(self.analysis_weight_receipt) is not FinalAnalysisWeightReceipt: + raise ValueError( + "analysis_weight_receipt must be a FinalAnalysisWeightReceipt" + ) + if self.analysis_weight_receipt.tenant_record_id != self.tenant_record_id: + raise ValueError( + "analysis_weight_receipt tenant_record_id must match the compatibility receipt" + ) + _validate_reference( + self.variance_design_receipt_reference, + "variance_design_receipt", + "variance_design_receipt_reference", + ) + _positive_integer( + self.variance_design_receipt_version, + "variance_design_receipt_version", + ) + for field_name in ( + "variance_design_receipt_digest", + "variance_analysis_weight_receipt_digest", + "variance_analytic_case_occurrence_set_digest", + "variance_weight_eligibility_receipt_digest", + "variance_final_weight_artifact_digest", + ): + _validate_digest(getattr(self, field_name), field_name) + _positive_integer( + self.variance_weight_correction_sequence, + "variance_weight_correction_sequence", + ) + + analysis_weight_digest = self.analysis_weight_receipt.sha256_digest() + if self.variance_design_receipt_digest == analysis_weight_digest: + raise ValueError( + "variance_design_receipt_digest must identify evidence distinct from the analysis weight receipt" + ) + if self.variance_analysis_weight_receipt_digest != analysis_weight_digest: + raise ValueError( + "variance evidence analysis weight receipt must match the exact point-weight receipt" + ) + if ( + self.variance_analytic_case_occurrence_set_digest + != self.analysis_weight_receipt.analytic_case_occurrence_set_digest + ): + raise ValueError( + "variance evidence analytic case occurrence set must match the point-weight receipt" + ) + if ( + self.variance_weight_eligibility_receipt_digest + != self.analysis_weight_receipt.weight_eligibility.sha256_digest() + ): + raise ValueError( + "variance evidence weight eligibility must match the point-weight receipt" + ) + if ( + self.variance_weight_correction_sequence + != self.analysis_weight_receipt.correction_sequence + ): + raise ValueError( + "variance evidence correction sequence must match the point-weight receipt" + ) + if ( + self.variance_final_weight_artifact_digest + != self.analysis_weight_receipt.final_weight_artifact_digest + ): + raise ValueError( + "variance evidence final weight artifact must match the point-weight receipt" + ) + + constructed_at = _freeze_timestamp(self.constructed_at, "constructed_at") + if constructed_at < self.analysis_weight_receipt.constructed_at: + raise ValueError( + "constructed_at cannot precede the analysis weight receipt" + ) + if type(self.evidence_version) is not int or self.evidence_version != 1: + raise ValueError("evidence_version must remain 1") + object.__setattr__(self, "constructed_at", constructed_at) + + def __repr__(self) -> str: + """Return a value-minimized representation suitable for routine logs.""" + return "WeightVarianceCompatibilityReceipt()" + + def canonical_json(self) -> str: + """Return deterministic correlation evidence without weight values.""" + payload = { + "analysis_weight_receipt_digest": self.analysis_weight_receipt.sha256_digest(), + "constructed_at": _canonical_timestamp(self.constructed_at, "constructed_at"), + "evidence_version": self.evidence_version, + "receipt_reference": self.receipt_reference, + "tenant_record_id": self.tenant_record_id, + "variance_analysis_weight_receipt_digest": ( + self.variance_analysis_weight_receipt_digest + ), + "variance_analytic_case_occurrence_set_digest": ( + self.variance_analytic_case_occurrence_set_digest + ), + "variance_design_receipt_digest": self.variance_design_receipt_digest, + "variance_design_receipt_reference": self.variance_design_receipt_reference, + "variance_design_receipt_version": self.variance_design_receipt_version, + "variance_final_weight_artifact_digest": ( + self.variance_final_weight_artifact_digest + ), + "variance_weight_correction_sequence": ( + self.variance_weight_correction_sequence + ), + "variance_weight_eligibility_receipt_digest": ( + self.variance_weight_eligibility_receipt_digest + ), + } + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + def sha256_digest(self) -> str: + """Return SHA-256 over the exact canonical compatibility receipt bytes.""" + return sha256(self.canonical_json().encode("utf-8")).hexdigest() + + +__all__ = ["WeightVarianceCompatibilityReceipt"] diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py b/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py new file mode 100644 index 000000000..d6e3d0b39 --- /dev/null +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py @@ -0,0 +1,305 @@ +"""Governed handoff evidence for criterion-related selection validation. + +This package does not execute statistics, read another service's database, or make an +employment decision. It binds authoritative Orgmetra evidence to one reviewed, +immutable fast-mlsirm revision so an approved offline worker can perform numerical +analysis without silently changing the study definition. +""" +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from hashlib import sha256 +import json +import re +from uuid import UUID + +_CODE_PATTERN = re.compile(r"^[a-z][a-z0-9]*(?:_[a-z0-9]+)+$") +_DIGEST_PATTERN = re.compile(r"^[0-9a-f]{64}$") +_REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$") +_REFERENCE_PATTERN = re.compile( + r"^[a-z][a-z0-9_]{1,63}:[A-Za-z0-9](?:[A-Za-z0-9._-]{0,126}[A-Za-z0-9])?$" +) +_PURPOSE_CODE = "selection_validity_analysis" +_REASON_CODE = "criterion_related_validation" +_VALIDATION_STRATEGY = "criterion_related" +_KERNEL_REPOSITORY = "ContextualWisdomLab/fast-mlsirm" +REVIEWED_FAST_MLSIRM_REVISION = "04d0bc21a2a20693bcf16108cd76d394fe844d23" +_KERNEL_BOUNDARY = "read_only_pinned_revision" +_EXECUTION_STATE = "not_executed" +_RESULT_AUTHORITY = "scientific_evidence_only" +_REQUIRED_RESULT_EVIDENCE = ( + "effect_estimate", + "uncertainty_interval", + "sample_size", + "missingness_summary", + "convergence_diagnostics", +) +_NEXT_ACTION = ( + "Within tenant_record_id, re-resolve the validation study, Job, predictor, criterion, " + "population, decision-policy, analysis-plan, requester, and reviewer references; prove " + "requester and reviewer resolve to distinct authoritative actor identities; prove the " + "predictor/criterion/population cases belong to the exact study and Job; then let an " + "approved offline validation worker invoke only the pinned fast-mlsirm revision. Preserve " + "the resulting model/provenance diagnostics as draft scientific evidence for an " + "accountable human reviewer; never convert the result directly into an employment decision." +) + + +def _validate_operational_uuid(value: object, field_name: str) -> None: + """Require canonical non-sentinel UUID text owned by authoritative Orgmetra.""" + if type(value) is not str: + raise ValueError(f"{field_name} must be canonical UUID text") + try: + parsed = UUID(value) + except (ValueError, AttributeError, TypeError) as exc: + raise ValueError(f"{field_name} must be canonical UUID text") from exc + if str(parsed) != value or parsed.int in (0, (1 << 128) - 1): + raise ValueError(f"{field_name} must be a canonical operational UUID") + + +def _validate_reference(value: object, prefix: str, field_name: str) -> None: + """Require the expected namespace plus a canonical opaque UUIDv4 suffix.""" + error_message = f"{field_name} must be an opaque {prefix}: UUIDv4 reference" + if ( + type(value) is not str + or len(value) > 160 + or not _REFERENCE_PATTERN.fullmatch(value) + or not value.startswith(f"{prefix}:") + ): + raise ValueError(error_message) + suffix = value.split(":", 1)[1] + try: + parsed = UUID(suffix) + except (ValueError, AttributeError, TypeError) as exc: + raise ValueError(error_message) from exc + if str(parsed) != suffix or parsed.version != 4 or parsed.int in (0, (1 << 128) - 1): + raise ValueError(error_message) + + +def _validate_digest(value: object, field_name: str) -> None: + """Require lowercase SHA-256 hexadecimal evidence.""" + if type(value) is not str or not _DIGEST_PATTERN.fullmatch(value): + raise ValueError(f"{field_name} must be lowercase SHA-256 hex") + + +def _validate_code(value: object, field_name: str) -> None: + """Require bounded descriptive lower snake_case governance codes.""" + if type(value) is not str or len(value) > 64 or not _CODE_PATTERN.fullmatch(value): + raise ValueError(f"{field_name} must be bounded two-or-more-word lower snake_case") + + +def _validate_kernel_revision(value: object) -> None: + """Require the exact externally reviewed immutable fast-mlsirm revision.""" + if type(value) is not str or not _REVISION_PATTERN.fullmatch(value): + raise ValueError("fast_mlsirm_revision must be lowercase 40-character Git commit hex") + if value != REVIEWED_FAST_MLSIRM_REVISION: + raise ValueError("fast_mlsirm_revision must equal the reviewed immutable revision") + + +def _freeze_timestamp(value: object, field_name: str) -> datetime: + """Detach caller-controlled timezone behavior and store one immutable UTC instant.""" + if type(value) is not datetime or value.tzinfo is None: + raise ValueError(f"{field_name} must be an exact timezone-aware datetime") + try: + offset = value.utcoffset() + except Exception as exc: # noqa: BLE001 - normalize provider behavior at trust boundary. + raise ValueError(f"{field_name} must be an exact timezone-aware datetime") from exc + if type(offset) is not timedelta: + raise ValueError(f"{field_name} must be an exact timezone-aware datetime") + try: + return (value.replace(tzinfo=None) - offset).replace(tzinfo=timezone.utc) + except OverflowError as exc: + raise ValueError(f"{field_name} must be an exact timezone-aware datetime") from exc + + +def _canonical_timestamp(value: object, field_name: str) -> str: + """Render a previously detached built-in UTC instant as RFC 3339 text.""" + if type(value) is not datetime or value.tzinfo is not timezone.utc: + raise ValueError(f"{field_name} must be timezone-aware") + return value.isoformat().replace("+00:00", "Z") + + +@dataclass(frozen=True, slots=True, repr=False) +class ValidationAnalysisHandoff: + """Immutable evidence for one not-yet-executed criterion-related validity analysis.""" + + tenant_record_id: str + handoff_reference: str + validation_study_reference: str + job_profile_reference: str + predictor_snapshot_reference: str + predictor_snapshot_digest: str + criterion_snapshot_reference: str + criterion_snapshot_digest: str + population_snapshot_reference: str + population_snapshot_digest: str + decision_policy_reference: str + decision_policy_digest: str + analysis_plan_reference: str + analysis_plan_digest: str + actor_reference: str + reviewer_reference: str + fast_mlsirm_revision: str + requested_at: datetime + purpose_code: str = _PURPOSE_CODE + reason_code: str = _REASON_CODE + evidence_version: int = 1 + validation_strategy: str = _VALIDATION_STRATEGY + kernel_repository: str = _KERNEL_REPOSITORY + kernel_boundary: str = _KERNEL_BOUNDARY + execution_state: str = _EXECUTION_STATE + contains_raw_person_level_values: bool = False + human_review_required: bool = True + result_authority: str = _RESULT_AUTHORITY + required_result_evidence: tuple[str, ...] = _REQUIRED_RESULT_EVIDENCE + next_action: str = _NEXT_ACTION + + def __post_init__(self) -> None: + """Fail closed when direct construction drifts from the governed handoff.""" + requested_at = _freeze_timestamp(self.requested_at, "requested_at") + object.__setattr__(self, "requested_at", requested_at) + _validate_operational_uuid(self.tenant_record_id, "tenant_record_id") + for value, prefix, field_name in ( + (self.handoff_reference, "validation_analysis_handoff", "handoff_reference"), + (self.validation_study_reference, "validation_study", "validation_study_reference"), + (self.job_profile_reference, "job_profile", "job_profile_reference"), + (self.predictor_snapshot_reference, "predictor_snapshot", "predictor_snapshot_reference"), + (self.criterion_snapshot_reference, "criterion_snapshot", "criterion_snapshot_reference"), + (self.population_snapshot_reference, "study_population_snapshot", "population_snapshot_reference"), + (self.decision_policy_reference, "decision_policy", "decision_policy_reference"), + (self.analysis_plan_reference, "validation_analysis_plan", "analysis_plan_reference"), + (self.actor_reference, "actor", "actor_reference"), + (self.reviewer_reference, "actor", "reviewer_reference"), + ): + _validate_reference(value, prefix, field_name) + for value, field_name in ( + (self.predictor_snapshot_digest, "predictor_snapshot_digest"), + (self.criterion_snapshot_digest, "criterion_snapshot_digest"), + (self.population_snapshot_digest, "population_snapshot_digest"), + (self.decision_policy_digest, "decision_policy_digest"), + (self.analysis_plan_digest, "analysis_plan_digest"), + ): + _validate_digest(value, field_name) + if self.actor_reference == self.reviewer_reference: + raise ValueError("reviewer_reference must identify a different accountable actor") + _validate_kernel_revision(self.fast_mlsirm_revision) + _canonical_timestamp(self.requested_at, "requested_at") + _validate_code(self.purpose_code, "purpose_code") + if self.purpose_code != _PURPOSE_CODE: + raise ValueError("purpose_code must remain selection_validity_analysis") + _validate_code(self.reason_code, "reason_code") + if self.reason_code != _REASON_CODE: + raise ValueError("reason_code must remain criterion_related_validation") + if type(self.evidence_version) is not int or not 1 <= self.evidence_version <= 2_147_483_647: + raise ValueError("evidence_version must be an integer from 1 through 2147483647") + if type(self.validation_strategy) is not str or self.validation_strategy != _VALIDATION_STRATEGY: + raise ValueError("validation_strategy must remain criterion_related") + if type(self.kernel_repository) is not str or self.kernel_repository != _KERNEL_REPOSITORY: + raise ValueError("kernel_repository must remain ContextualWisdomLab/fast-mlsirm") + if type(self.kernel_boundary) is not str or self.kernel_boundary != _KERNEL_BOUNDARY: + raise ValueError("kernel_boundary must remain read_only_pinned_revision") + if type(self.execution_state) is not str or self.execution_state != _EXECUTION_STATE: + raise ValueError("execution_state must remain not_executed") + if self.contains_raw_person_level_values is not False: + raise ValueError("handoff must not contain raw person-level values") + if self.human_review_required is not True: + raise ValueError("human review is mandatory for selection-validity interpretation") + if type(self.result_authority) is not str or self.result_authority != _RESULT_AUTHORITY: + raise ValueError("result_authority must remain scientific_evidence_only") + if ( + type(self.required_result_evidence) is not tuple + or any(type(item) is not str for item in self.required_result_evidence) + or self.required_result_evidence != _REQUIRED_RESULT_EVIDENCE + ): + raise ValueError("required_result_evidence must remain the reviewed evidence set") + if type(self.next_action) is not str or self.next_action != _NEXT_ACTION: + raise ValueError("next_action must remain the governed validation instruction") + + def __repr__(self) -> str: + """Return a fully redacted representation suitable for routine logs.""" + return "ValidationAnalysisHandoff()" + + def canonical_json(self) -> str: + """Return deterministic canonical JSON for audit and result correlation.""" + payload = { + "actor_reference": self.actor_reference, + "analysis_plan_digest": self.analysis_plan_digest, + "analysis_plan_reference": self.analysis_plan_reference, + "contains_raw_person_level_values": self.contains_raw_person_level_values, + "criterion_snapshot_digest": self.criterion_snapshot_digest, + "criterion_snapshot_reference": self.criterion_snapshot_reference, + "decision_policy_digest": self.decision_policy_digest, + "decision_policy_reference": self.decision_policy_reference, + "evidence_version": self.evidence_version, + "execution_state": self.execution_state, + "fast_mlsirm_revision": self.fast_mlsirm_revision, + "handoff_reference": self.handoff_reference, + "human_review_required": self.human_review_required, + "job_profile_reference": self.job_profile_reference, + "kernel_boundary": self.kernel_boundary, + "kernel_repository": self.kernel_repository, + "next_action": self.next_action, + "population_snapshot_digest": self.population_snapshot_digest, + "population_snapshot_reference": self.population_snapshot_reference, + "predictor_snapshot_digest": self.predictor_snapshot_digest, + "predictor_snapshot_reference": self.predictor_snapshot_reference, + "purpose_code": self.purpose_code, + "reason_code": self.reason_code, + "requested_at": _canonical_timestamp(self.requested_at, "requested_at"), + "required_result_evidence": list(self.required_result_evidence), + "result_authority": self.result_authority, + "reviewer_reference": self.reviewer_reference, + "tenant_record_id": self.tenant_record_id, + "validation_strategy": self.validation_strategy, + "validation_study_reference": self.validation_study_reference, + } + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + def sha256_digest(self) -> str: + """Return SHA-256 over the exact canonical UTF-8 handoff.""" + return sha256(self.canonical_json().encode("utf-8")).hexdigest() + + +def build_validation_analysis_handoff( + *, + tenant_record_id: str, + handoff_reference: str, + validation_study_reference: str, + job_profile_reference: str, + predictor_snapshot_reference: str, + predictor_snapshot_digest: str, + criterion_snapshot_reference: str, + criterion_snapshot_digest: str, + population_snapshot_reference: str, + population_snapshot_digest: str, + decision_policy_reference: str, + decision_policy_digest: str, + analysis_plan_reference: str, + analysis_plan_digest: str, + actor_reference: str, + reviewer_reference: str, + fast_mlsirm_revision: str, + requested_at: datetime, +) -> ValidationAnalysisHandoff: + """Build a governed, non-executing selection-validity analysis handoff.""" + return ValidationAnalysisHandoff( + tenant_record_id=tenant_record_id, + handoff_reference=handoff_reference, + validation_study_reference=validation_study_reference, + job_profile_reference=job_profile_reference, + predictor_snapshot_reference=predictor_snapshot_reference, + predictor_snapshot_digest=predictor_snapshot_digest, + criterion_snapshot_reference=criterion_snapshot_reference, + criterion_snapshot_digest=criterion_snapshot_digest, + population_snapshot_reference=population_snapshot_reference, + population_snapshot_digest=population_snapshot_digest, + decision_policy_reference=decision_policy_reference, + decision_policy_digest=decision_policy_digest, + analysis_plan_reference=analysis_plan_reference, + analysis_plan_digest=analysis_plan_digest, + actor_reference=actor_reference, + reviewer_reference=reviewer_reference, + fast_mlsirm_revision=fast_mlsirm_revision, + requested_at=requested_at, + ) \ No newline at end of file diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py new file mode 100644 index 000000000..0c5f3a8f2 --- /dev/null +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py @@ -0,0 +1,363 @@ +"""Validate one immutable numerical result returned by the approved worker. + +Orgmetra does not fit a model in this package. It accepts only a bounded, +digest-linked result envelope from the pinned ``fast-mlsirm`` worker so that +nonconverged or malformed output cannot be presented as an employment +decision. +""" +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from hashlib import sha256 +import json +from math import isfinite +from numbers import Real + +from .compatibility import WeightVarianceCompatibilityReceipt +from .handoff import ( + _canonical_timestamp, + _freeze_timestamp, + _validate_code, + _validate_digest, + _validate_kernel_revision, + _validate_operational_uuid, + _validate_reference, +) + +_RESULT_AUTHORITY = "scientific_evidence_only" +_EXECUTION_STATE = "completed" +_ALLOWED_BACKENDS = frozenset({"rust_cpu", "rust_gpu"}) +_ALLOWED_PRECISIONS = frozenset({"f64", "f32"}) +_ALLOWED_POINT_ESTIMATION_MODES = frozenset({"unweighted", "weighted_design_based"}) + + +def _validate_nonnegative_integer(value: object, field_name: str) -> None: + """Require a real non-negative integer without accepting booleans.""" + if type(value) is not int or value < 0: + raise ValueError(f"{field_name} must be a non-negative integer") + + +def _validate_positive_integer(value: object, field_name: str) -> None: + """Require a real positive integer without accepting booleans.""" + if type(value) is not int or value <= 0: + raise ValueError(f"{field_name} must be a positive integer") + + +def _finite_number(value: object, field_name: str) -> float: + """Return one finite real number and normalize invalid numerics to ValueError.""" + if isinstance(value, bool) or not isinstance(value, Real): + raise ValueError(f"{field_name} must be a finite number") + try: + number = float(value) + except (OverflowError, TypeError, ValueError) as exc: + raise ValueError(f"{field_name} must be a finite number") from exc + if not isfinite(number): + raise ValueError(f"{field_name} must be a finite number") + return number + + +@dataclass(frozen=True, slots=True) +class MissingnessSummary: + """Describe missingness counts without carrying person-level observations.""" + + total_observations: int + complete_observations: int + missing_predictor_observations: int + missing_criterion_observations: int + + def __post_init__(self) -> None: + """Reject impossible counts before a result can be correlated.""" + for field_name in ( + "total_observations", + "complete_observations", + "missing_predictor_observations", + "missing_criterion_observations", + ): + _validate_nonnegative_integer(getattr(self, field_name), field_name) + if self.total_observations == 0: + raise ValueError("total_observations must be positive") + if self.complete_observations > self.total_observations: + raise ValueError("complete_observations cannot exceed total_observations") + if self.missing_predictor_observations > self.total_observations: + raise ValueError("missing_predictor_observations cannot exceed total_observations") + if self.missing_criterion_observations > self.total_observations: + raise ValueError("missing_criterion_observations cannot exceed total_observations") + if self.complete_observations + self.missing_predictor_observations > self.total_observations: + raise ValueError( + "complete_observations and missing_predictor_observations cannot overlap" + ) + if self.complete_observations + self.missing_criterion_observations > self.total_observations: + raise ValueError( + "complete_observations and missing_criterion_observations cannot overlap" + ) + + def to_dict(self) -> dict[str, int]: + """Return deterministic count fields for the canonical result JSON.""" + return { + "complete_observations": self.complete_observations, + "missing_criterion_observations": self.missing_criterion_observations, + "missing_predictor_observations": self.missing_predictor_observations, + "total_observations": self.total_observations, + } + + +@dataclass(frozen=True, slots=True) +class ConvergenceDiagnostics: + """Record convergence evidence while preserving an explicit failure state.""" + + converged: bool + iterations: int + objective_value: Real + maximum_gradient: Real + failure_code: str | None = None + + def __post_init__(self) -> None: + """Require diagnostics that distinguish convergence from a failed fit.""" + if type(self.converged) is not bool: + raise ValueError("converged must be a boolean") + _validate_positive_integer(self.iterations, "iterations") + objective = _finite_number(self.objective_value, "objective_value") + gradient = _finite_number(self.maximum_gradient, "maximum_gradient") + if gradient < 0: + raise ValueError("maximum_gradient must be non-negative") + if self.converged and self.failure_code is not None: + raise ValueError("failure_code must be absent for a converged result") + if not self.converged and ( + type(self.failure_code) is not str or not self.failure_code + ): + raise ValueError("failure_code is required for a nonconverged result") + if self.failure_code is not None: + _validate_code(self.failure_code, "failure_code") + object.__setattr__(self, "objective_value", objective) + object.__setattr__(self, "maximum_gradient", gradient) + + def to_dict(self) -> dict[str, object]: + """Return deterministic convergence fields for the canonical result JSON.""" + payload: dict[str, object] = { + "converged": self.converged, + "iterations": self.iterations, + "maximum_gradient": float(self.maximum_gradient), + "objective_value": float(self.objective_value), + } + if self.failure_code is not None: + payload["failure_code"] = self.failure_code + return payload + + +@dataclass(frozen=True, slots=True, repr=False) +class ValidationAnalysisResult: + """Immutable, digest-linked scientific evidence returned by the offline worker.""" + + tenant_record_id: str + result_reference: str + handoff_digest: str + provenance_digest: str + fast_mlsirm_revision: str + model_code: str + backend: str + precision: str + effect_estimate: Real + uncertainty_lower: Real + uncertainty_upper: Real + sample_size: int + missingness_summary: MissingnessSummary + convergence_diagnostics: ConvergenceDiagnostics + completed_at: datetime + point_estimation_mode: str = "unweighted" + analysis_weight_receipt_digest: str | None = None + variance_design_receipt_digest: str | None = None + weight_variance_compatibility: WeightVarianceCompatibilityReceipt | None = None + result_authority: str = _RESULT_AUTHORITY + execution_state: str = _EXECUTION_STATE + contains_raw_person_level_values: bool = False + human_review_required: bool = True + evidence_version: int = 1 + correction_sequence: int = 1 + supersedes_result_reference: str | None = None + supersedes_result_digest: str | None = None + + def __post_init__(self) -> None: + """Fail closed on malformed, unlinked, or decision-like result data.""" + _validate_operational_uuid(self.tenant_record_id, "tenant_record_id") + _validate_reference(self.result_reference, "validation_analysis_result", "result_reference") + _validate_digest(self.handoff_digest, "handoff_digest") + _validate_digest(self.provenance_digest, "provenance_digest") + _validate_kernel_revision(self.fast_mlsirm_revision) + _validate_code(self.model_code, "model_code") + if type(self.backend) is not str or self.backend not in _ALLOWED_BACKENDS: + raise ValueError("backend must be rust_cpu or rust_gpu") + if type(self.precision) is not str or self.precision not in _ALLOWED_PRECISIONS: + raise ValueError("precision must be f64 or f32") + estimate = _finite_number(self.effect_estimate, "effect_estimate") + lower = _finite_number(self.uncertainty_lower, "uncertainty_lower") + upper = _finite_number(self.uncertainty_upper, "uncertainty_upper") + if lower > upper: + raise ValueError("uncertainty_lower cannot exceed uncertainty_upper") + if not lower <= estimate <= upper: + raise ValueError("effect_estimate must be inside the uncertainty interval") + _validate_positive_integer(self.sample_size, "sample_size") + if type(self.missingness_summary) is not MissingnessSummary: + raise ValueError("missingness_summary must be a MissingnessSummary") + if type(self.convergence_diagnostics) is not ConvergenceDiagnostics: + raise ValueError("convergence_diagnostics must be a ConvergenceDiagnostics") + if self.sample_size != self.missingness_summary.total_observations: + raise ValueError("sample_size must match total_observations") + completed_at = _freeze_timestamp(self.completed_at, "completed_at") + if ( + type(self.point_estimation_mode) is not str + or self.point_estimation_mode not in _ALLOWED_POINT_ESTIMATION_MODES + ): + raise ValueError( + "point_estimation_mode must be unweighted or weighted_design_based" + ) + if self.point_estimation_mode == "weighted_design_based": + if self.analysis_weight_receipt_digest is None: + raise ValueError( + "analysis_weight_receipt_digest is required for weighted_design_based" + ) + if self.variance_design_receipt_digest is None: + raise ValueError( + "variance_design_receipt_digest is required for weighted_design_based" + ) + _validate_digest( + self.analysis_weight_receipt_digest, "analysis_weight_receipt_digest" + ) + _validate_digest( + self.variance_design_receipt_digest, "variance_design_receipt_digest" + ) + if self.analysis_weight_receipt_digest == self.variance_design_receipt_digest: + raise ValueError( + "analysis_weight_receipt_digest and variance_design_receipt_digest " + "must identify different evidence" + ) + if type(self.weight_variance_compatibility) is not WeightVarianceCompatibilityReceipt: + raise ValueError( + "weight_variance_compatibility is required for weighted_design_based" + ) + if self.weight_variance_compatibility.tenant_record_id != self.tenant_record_id: + raise ValueError( + "weight_variance_compatibility tenant_record_id must match the result" + ) + if ( + self.weight_variance_compatibility.analysis_weight_receipt.sha256_digest() + != self.analysis_weight_receipt_digest + ): + raise ValueError( + "weight_variance_compatibility must identify the result analysis weight receipt" + ) + if ( + self.weight_variance_compatibility.variance_design_receipt_digest + != self.variance_design_receipt_digest + ): + raise ValueError( + "weight_variance_compatibility must identify the result variance design receipt" + ) + if self.weight_variance_compatibility.constructed_at > completed_at: + raise ValueError( + "weight_variance_compatibility cannot be constructed after the result" + ) + elif ( + self.analysis_weight_receipt_digest is not None + or self.variance_design_receipt_digest is not None + or self.weight_variance_compatibility is not None + ): + raise ValueError( + "unweighted result must not bind analysis or variance weight receipts" + ) + if type(self.result_authority) is not str or self.result_authority != _RESULT_AUTHORITY: + raise ValueError("result_authority must remain scientific_evidence_only") + if type(self.execution_state) is not str or self.execution_state != _EXECUTION_STATE: + raise ValueError("execution_state must remain completed") + if self.contains_raw_person_level_values is not False: + raise ValueError("result must not contain raw person-level values") + if self.human_review_required is not True: + raise ValueError("human review is mandatory for validity interpretation") + if type(self.evidence_version) is not int or self.evidence_version != 1: + raise ValueError("evidence_version must remain 1") + _validate_positive_integer(self.correction_sequence, "correction_sequence") + if self.correction_sequence == 1: + if ( + self.supersedes_result_reference is not None + or self.supersedes_result_digest is not None + ): + raise ValueError( + "correction_sequence 1 must not identify superseded result evidence" + ) + else: + if self.supersedes_result_reference is None or self.supersedes_result_digest is None: + raise ValueError( + "corrected result requires supersedes_result_reference and supersedes_result_digest" + ) + _validate_reference( + self.supersedes_result_reference, + "validation_analysis_result", + "supersedes_result_reference", + ) + _validate_digest(self.supersedes_result_digest, "supersedes_result_digest") + if self.supersedes_result_reference == self.result_reference: + raise ValueError("corrected result must use a new result_reference") + object.__setattr__(self, "effect_estimate", estimate) + object.__setattr__(self, "uncertainty_lower", lower) + object.__setattr__(self, "uncertainty_upper", upper) + object.__setattr__(self, "completed_at", completed_at) + + @property + def verification_status(self) -> str: + """Return a non-authorizing leaf verification state for this result.""" + return ( + "verification_pending" + if self.convergence_diagnostics.converged + else "not_verifiable" + ) + + def __repr__(self) -> str: + """Return a redacted representation suitable for routine application logs.""" + return "ValidationAnalysisResult()" + + def canonical_json(self) -> str: + """Return deterministic, non-person-level JSON for audit correlation.""" + payload = { + "backend": self.backend, + "completed_at": _canonical_timestamp(self.completed_at, "completed_at"), + "contains_raw_person_level_values": self.contains_raw_person_level_values, + "convergence_diagnostics": self.convergence_diagnostics.to_dict(), + "correction_sequence": self.correction_sequence, + "effect_estimate": float(self.effect_estimate), + "evidence_version": self.evidence_version, + "execution_state": self.execution_state, + "fast_mlsirm_revision": self.fast_mlsirm_revision, + "handoff_digest": self.handoff_digest, + "human_review_required": self.human_review_required, + "missingness_summary": self.missingness_summary.to_dict(), + "model_code": self.model_code, + "point_estimation_mode": self.point_estimation_mode, + "precision": self.precision, + "provenance_digest": self.provenance_digest, + "result_authority": self.result_authority, + "result_reference": self.result_reference, + "sample_size": self.sample_size, + "tenant_record_id": self.tenant_record_id, + "uncertainty_lower": float(self.uncertainty_lower), + "uncertainty_upper": float(self.uncertainty_upper), + "verification_status": self.verification_status, + } + if self.analysis_weight_receipt_digest is not None: + payload["analysis_weight_receipt_digest"] = self.analysis_weight_receipt_digest + if self.variance_design_receipt_digest is not None: + payload["variance_design_receipt_digest"] = self.variance_design_receipt_digest + if self.weight_variance_compatibility is not None: + payload["weight_variance_compatibility_receipt_digest"] = ( + self.weight_variance_compatibility.sha256_digest() + ) + if self.supersedes_result_reference is not None: + payload["supersedes_result_reference"] = self.supersedes_result_reference + payload["supersedes_result_digest"] = self.supersedes_result_digest + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + def sha256_digest(self) -> str: + """Return SHA-256 over the exact canonical result bytes.""" + return sha256(self.canonical_json().encode("utf-8")).hexdigest() + + +__all__ = ["ConvergenceDiagnostics", "MissingnessSummary", "ValidationAnalysisResult"] diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py b/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py new file mode 100644 index 000000000..9f3cd3bf4 --- /dev/null +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py @@ -0,0 +1,782 @@ +"""Immutable provenance for final point-estimation weight construction. + +This module records only versioned scientific lineage and digests. It never +stores row-level weight values or copies auxiliary calibration attributes into +Orgmetra's validity-analysis boundary. +""" +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from hashlib import sha256 +import json + +from .handoff import ( + _canonical_timestamp, + _freeze_timestamp, + _validate_code, + _validate_digest, + _validate_operational_uuid, + _validate_reference, +) + +_CALIBRATION_TERMINATION_CODES = frozenset({"converged", "fallback_applied"}) +_WEIGHT_SCOPE_CODES = frozenset({"cross_sectional", "longitudinal"}) +_SPECIALIZED_EVIDENCE_KIND_BY_ADJUSTMENT_CODE = { + "nonresponse_adjustment": "nonresponse_adjustment_receipt", + "calibration_adjustment": "calibration_adjustment_receipt", + "raking_adjustment": "calibration_adjustment_receipt", + "poststratification_adjustment": "calibration_adjustment_receipt", + "weight_trimming_adjustment": "trimming_bounding_adjustment_receipt", + "weight_bounding_adjustment": "trimming_bounding_adjustment_receipt", + "weight_winsorization_adjustment": "trimming_bounding_adjustment_receipt", +} + + +def _positive_integer(value: object, field_name: str) -> None: + """Require a strict positive integer without accepting booleans.""" + if type(value) is not int or value <= 0: + raise ValueError(f"{field_name} must be a positive integer") + + +@dataclass(frozen=True, slots=True, repr=False) +class NonresponseAdjustmentReceipt: + """Bind one nonresponse adjustment to explicit disposition-aware evidence.""" + + tenant_record_id: str + receipt_reference: str + response_disposition_receipt_reference: str + response_disposition_receipt_version: int + response_disposition_receipt_digest: str + adjustment_population_digest: str + method_reference: str + method_version: int + configuration_digest: str + ineligible_treatment_code: str + unknown_treatment_code: str + unavailable_treatment_code: str + input_weight_artifact_digest: str + output_weight_artifact_digest: str + constructed_at: datetime + evidence_version: int = 1 + + def __post_init__(self) -> None: + """Reject undocumented filters or mutable nonresponse evidence.""" + _validate_operational_uuid(self.tenant_record_id, "tenant_record_id") + _validate_reference( + self.receipt_reference, + "nonresponse_adjustment_receipt", + "receipt_reference", + ) + _validate_reference( + self.response_disposition_receipt_reference, + "response_disposition_receipt", + "response_disposition_receipt_reference", + ) + _positive_integer( + self.response_disposition_receipt_version, + "response_disposition_receipt_version", + ) + for field_name in ( + "response_disposition_receipt_digest", + "adjustment_population_digest", + "configuration_digest", + "input_weight_artifact_digest", + "output_weight_artifact_digest", + ): + _validate_digest(getattr(self, field_name), field_name) + _validate_reference(self.method_reference, "weight_method", "method_reference") + _positive_integer(self.method_version, "method_version") + for field_name in ( + "ineligible_treatment_code", + "unknown_treatment_code", + "unavailable_treatment_code", + ): + _validate_code(getattr(self, field_name), field_name) + if self.input_weight_artifact_digest == self.output_weight_artifact_digest: + raise ValueError( + "output_weight_artifact_digest must identify the adjusted weight artifact" + ) + constructed_at = _freeze_timestamp(self.constructed_at, "constructed_at") + if type(self.evidence_version) is not int or self.evidence_version != 1: + raise ValueError("evidence_version must remain 1") + object.__setattr__(self, "constructed_at", constructed_at) + + def __repr__(self) -> str: + """Return a value-minimized representation for routine logs.""" + return "NonresponseAdjustmentReceipt()" + + def canonical_json(self) -> str: + """Return deterministic disposition-aware provenance without source attributes.""" + payload = { + "adjustment_population_digest": self.adjustment_population_digest, + "configuration_digest": self.configuration_digest, + "constructed_at": _canonical_timestamp(self.constructed_at, "constructed_at"), + "evidence_version": self.evidence_version, + "ineligible_treatment_code": self.ineligible_treatment_code, + "input_weight_artifact_digest": self.input_weight_artifact_digest, + "method_reference": self.method_reference, + "method_version": self.method_version, + "output_weight_artifact_digest": self.output_weight_artifact_digest, + "receipt_reference": self.receipt_reference, + "response_disposition_receipt_digest": self.response_disposition_receipt_digest, + "response_disposition_receipt_reference": self.response_disposition_receipt_reference, + "response_disposition_receipt_version": self.response_disposition_receipt_version, + "tenant_record_id": self.tenant_record_id, + "unavailable_treatment_code": self.unavailable_treatment_code, + "unknown_treatment_code": self.unknown_treatment_code, + } + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + def sha256_digest(self) -> str: + """Return SHA-256 over the exact canonical receipt bytes.""" + return sha256(self.canonical_json().encode("utf-8")).hexdigest() + + +@dataclass(frozen=True, slots=True, repr=False) +class CalibrationAdjustmentReceipt: + """Bind calibration to owner authority, benchmark authority, and termination evidence.""" + + tenant_record_id: str + receipt_reference: str + target_population_digest: str + analysis_window_reference: str + auxiliary_authority_reference: str + auxiliary_projection_reference: str + auxiliary_projection_version: int + auxiliary_projection_digest: str + auxiliary_purpose_reference: str + auxiliary_purpose_digest: str + auxiliary_owner_contract_reference: str + auxiliary_owner_contract_version: int + auxiliary_owner_contract_digest: str + auxiliary_authorization_receipt_reference: str + auxiliary_authorization_receipt_digest: str + auxiliary_scientific_use_receipt_reference: str + auxiliary_scientific_use_receipt_digest: str + auxiliary_scientific_use_at: datetime + benchmark_receipt_reference: str + benchmark_receipt_version: int + benchmark_receipt_digest: str + benchmark_owner_contract_reference: str + benchmark_owner_contract_version: int + benchmark_owner_contract_digest: str + benchmark_reference_at: datetime + algorithm_reference: str + algorithm_version: int + constraints_digest: str + applied_constraints_digest: str + termination_code: str + input_weight_artifact_digest: str + output_weight_artifact_digest: str + constructed_at: datetime + fallback_reason_code: str | None = None + fallback_rule_reference: str | None = None + fallback_rule_digest: str | None = None + fallback_algorithm_reference: str | None = None + fallback_algorithm_version: int | None = None + fallback_configuration_digest: str | None = None + evidence_version: int = 1 + + def __post_init__(self) -> None: + """Fail closed on floating authority, benchmark time, hidden fallback, or nonconvergence.""" + _validate_operational_uuid(self.tenant_record_id, "tenant_record_id") + _validate_reference( + self.receipt_reference, + "calibration_adjustment_receipt", + "receipt_reference", + ) + _validate_reference( + self.analysis_window_reference, + "analysis_window", + "analysis_window_reference", + ) + _validate_reference( + self.auxiliary_authority_reference, + "scientific_auxiliary_authority", + "auxiliary_authority_reference", + ) + _validate_reference( + self.auxiliary_projection_reference, + "calibration_auxiliary_projection", + "auxiliary_projection_reference", + ) + _positive_integer( + self.auxiliary_projection_version, + "auxiliary_projection_version", + ) + _validate_reference( + self.auxiliary_purpose_reference, + "scientific_data_use_purpose", + "auxiliary_purpose_reference", + ) + _validate_reference( + self.auxiliary_owner_contract_reference, + "released_owner_contract", + "auxiliary_owner_contract_reference", + ) + _positive_integer( + self.auxiliary_owner_contract_version, + "auxiliary_owner_contract_version", + ) + _validate_reference( + self.auxiliary_authorization_receipt_reference, + "scientific_data_authorization", + "auxiliary_authorization_receipt_reference", + ) + _validate_reference( + self.auxiliary_scientific_use_receipt_reference, + "scientific_use_receipt", + "auxiliary_scientific_use_receipt_reference", + ) + _validate_reference( + self.benchmark_receipt_reference, + "calibration_benchmark_receipt", + "benchmark_receipt_reference", + ) + _positive_integer(self.benchmark_receipt_version, "benchmark_receipt_version") + _validate_reference( + self.benchmark_owner_contract_reference, + "released_owner_contract", + "benchmark_owner_contract_reference", + ) + _positive_integer( + self.benchmark_owner_contract_version, + "benchmark_owner_contract_version", + ) + _validate_reference( + self.algorithm_reference, + "calibration_algorithm", + "algorithm_reference", + ) + for field_name in ( + "target_population_digest", + "auxiliary_projection_digest", + "auxiliary_purpose_digest", + "auxiliary_owner_contract_digest", + "auxiliary_authorization_receipt_digest", + "auxiliary_scientific_use_receipt_digest", + "benchmark_receipt_digest", + "benchmark_owner_contract_digest", + "constraints_digest", + "applied_constraints_digest", + "input_weight_artifact_digest", + "output_weight_artifact_digest", + ): + _validate_digest(getattr(self, field_name), field_name) + _positive_integer(self.algorithm_version, "algorithm_version") + if ( + type(self.termination_code) is not str + or self.termination_code not in _CALIBRATION_TERMINATION_CODES + ): + raise ValueError("termination_code must be converged or fallback_applied") + fallback_fields = ( + self.fallback_reason_code, + self.fallback_rule_reference, + self.fallback_rule_digest, + self.fallback_algorithm_reference, + self.fallback_algorithm_version, + self.fallback_configuration_digest, + ) + if self.termination_code == "fallback_applied": + if any(value is None for value in fallback_fields): + raise ValueError( + "fallback reason, rule, algorithm, version, and configuration evidence " + "are required for fallback_applied" + ) + _validate_code(self.fallback_reason_code, "fallback_reason_code") + _validate_reference( + self.fallback_rule_reference, + "calibration_fallback_rule", + "fallback_rule_reference", + ) + _validate_digest(self.fallback_rule_digest, "fallback_rule_digest") + _validate_reference( + self.fallback_algorithm_reference, + "calibration_algorithm", + "fallback_algorithm_reference", + ) + _positive_integer( + self.fallback_algorithm_version, + "fallback_algorithm_version", + ) + _validate_digest( + self.fallback_configuration_digest, + "fallback_configuration_digest", + ) + else: + if self.applied_constraints_digest != self.constraints_digest: + raise ValueError( + "changed applied calibration constraints require explicit fallback provenance" + ) + if any(value is not None for value in fallback_fields): + raise ValueError("fallback evidence must be absent when calibration converged") + if self.input_weight_artifact_digest == self.output_weight_artifact_digest: + raise ValueError( + "output_weight_artifact_digest must identify the calibrated weight artifact" + ) + scientific_use_at = _freeze_timestamp( + self.auxiliary_scientific_use_at, + "auxiliary_scientific_use_at", + ) + benchmark_reference_at = _freeze_timestamp( + self.benchmark_reference_at, + "benchmark_reference_at", + ) + constructed_at = _freeze_timestamp(self.constructed_at, "constructed_at") + if scientific_use_at > constructed_at: + raise ValueError("auxiliary_scientific_use_at cannot be later than constructed_at") + if benchmark_reference_at > constructed_at: + raise ValueError("benchmark_reference_at cannot be later than constructed_at") + if type(self.evidence_version) is not int or self.evidence_version != 1: + raise ValueError("evidence_version must remain 1") + object.__setattr__(self, "auxiliary_scientific_use_at", scientific_use_at) + object.__setattr__(self, "benchmark_reference_at", benchmark_reference_at) + object.__setattr__(self, "constructed_at", constructed_at) + + def __repr__(self) -> str: + """Return a value-minimized representation for routine logs.""" + return "CalibrationAdjustmentReceipt()" + + def canonical_json(self) -> str: + """Return deterministic purpose-bound provenance without auxiliary values.""" + payload: dict[str, object] = { + "algorithm_reference": self.algorithm_reference, + "algorithm_version": self.algorithm_version, + "analysis_window_reference": self.analysis_window_reference, + "applied_constraints_digest": self.applied_constraints_digest, + "auxiliary_authority_reference": self.auxiliary_authority_reference, + "auxiliary_authorization_receipt_digest": self.auxiliary_authorization_receipt_digest, + "auxiliary_authorization_receipt_reference": self.auxiliary_authorization_receipt_reference, + "auxiliary_owner_contract_digest": self.auxiliary_owner_contract_digest, + "auxiliary_owner_contract_reference": self.auxiliary_owner_contract_reference, + "auxiliary_owner_contract_version": self.auxiliary_owner_contract_version, + "auxiliary_projection_digest": self.auxiliary_projection_digest, + "auxiliary_projection_reference": self.auxiliary_projection_reference, + "auxiliary_projection_version": self.auxiliary_projection_version, + "auxiliary_purpose_digest": self.auxiliary_purpose_digest, + "auxiliary_purpose_reference": self.auxiliary_purpose_reference, + "auxiliary_scientific_use_at": _canonical_timestamp( + self.auxiliary_scientific_use_at, + "auxiliary_scientific_use_at", + ), + "auxiliary_scientific_use_receipt_digest": self.auxiliary_scientific_use_receipt_digest, + "auxiliary_scientific_use_receipt_reference": self.auxiliary_scientific_use_receipt_reference, + "benchmark_owner_contract_digest": self.benchmark_owner_contract_digest, + "benchmark_owner_contract_reference": self.benchmark_owner_contract_reference, + "benchmark_owner_contract_version": self.benchmark_owner_contract_version, + "benchmark_receipt_digest": self.benchmark_receipt_digest, + "benchmark_receipt_reference": self.benchmark_receipt_reference, + "benchmark_receipt_version": self.benchmark_receipt_version, + "benchmark_reference_at": _canonical_timestamp( + self.benchmark_reference_at, + "benchmark_reference_at", + ), + "constraints_digest": self.constraints_digest, + "constructed_at": _canonical_timestamp(self.constructed_at, "constructed_at"), + "evidence_version": self.evidence_version, + "input_weight_artifact_digest": self.input_weight_artifact_digest, + "output_weight_artifact_digest": self.output_weight_artifact_digest, + "receipt_reference": self.receipt_reference, + "target_population_digest": self.target_population_digest, + "tenant_record_id": self.tenant_record_id, + "termination_code": self.termination_code, + } + if self.fallback_rule_reference is not None: + payload["fallback_algorithm_reference"] = self.fallback_algorithm_reference + payload["fallback_algorithm_version"] = self.fallback_algorithm_version + payload["fallback_configuration_digest"] = self.fallback_configuration_digest + payload["fallback_reason_code"] = self.fallback_reason_code + payload["fallback_rule_digest"] = self.fallback_rule_digest + payload["fallback_rule_reference"] = self.fallback_rule_reference + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + def sha256_digest(self) -> str: + """Return SHA-256 over the exact canonical receipt bytes.""" + return sha256(self.canonical_json().encode("utf-8")).hexdigest() + + +@dataclass(frozen=True, slots=True, repr=False) +class TrimmingBoundingAdjustmentReceipt: + """Bind trimming or bounding to an immutable rule and affected-case evidence.""" + + tenant_record_id: str + receipt_reference: str + rule_reference: str + rule_version: int + rule_configuration_digest: str + affected_case_occurrence_set_digest: str + affected_case_count: int + input_weight_artifact_digest: str + output_weight_artifact_digest: str + constructed_at: datetime + evidence_version: int = 1 + + def __post_init__(self) -> None: + """Reject hidden thresholds, unknown affected cases, or no-op transforms.""" + _validate_operational_uuid(self.tenant_record_id, "tenant_record_id") + _validate_reference( + self.receipt_reference, + "trimming_bounding_adjustment_receipt", + "receipt_reference", + ) + _validate_reference(self.rule_reference, "weight_trimming_rule", "rule_reference") + _positive_integer(self.rule_version, "rule_version") + _validate_digest(self.rule_configuration_digest, "rule_configuration_digest") + _validate_digest( + self.affected_case_occurrence_set_digest, + "affected_case_occurrence_set_digest", + ) + _positive_integer(self.affected_case_count, "affected_case_count") + _validate_digest(self.input_weight_artifact_digest, "input_weight_artifact_digest") + _validate_digest(self.output_weight_artifact_digest, "output_weight_artifact_digest") + if self.input_weight_artifact_digest == self.output_weight_artifact_digest: + raise ValueError( + "output_weight_artifact_digest must identify the trimmed or bounded weight artifact" + ) + constructed_at = _freeze_timestamp(self.constructed_at, "constructed_at") + if type(self.evidence_version) is not int or self.evidence_version != 1: + raise ValueError("evidence_version must remain 1") + object.__setattr__(self, "constructed_at", constructed_at) + + def __repr__(self) -> str: + """Return a value-minimized representation for routine logs.""" + return "TrimmingBoundingAdjustmentReceipt()" + + def canonical_json(self) -> str: + """Return deterministic rule provenance without case-level weight values.""" + payload = { + "affected_case_count": self.affected_case_count, + "affected_case_occurrence_set_digest": self.affected_case_occurrence_set_digest, + "constructed_at": _canonical_timestamp(self.constructed_at, "constructed_at"), + "evidence_version": self.evidence_version, + "input_weight_artifact_digest": self.input_weight_artifact_digest, + "output_weight_artifact_digest": self.output_weight_artifact_digest, + "receipt_reference": self.receipt_reference, + "rule_configuration_digest": self.rule_configuration_digest, + "rule_reference": self.rule_reference, + "rule_version": self.rule_version, + "tenant_record_id": self.tenant_record_id, + } + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + def sha256_digest(self) -> str: + """Return SHA-256 over the exact canonical receipt bytes.""" + return sha256(self.canonical_json().encode("utf-8")).hexdigest() + + +@dataclass(frozen=True, slots=True, repr=False) +class WeightEligibilityReceipt: + """Bind one final weight artifact to its valid population and reference duration.""" + + tenant_record_id: str + receipt_reference: str + weight_scope_code: str + target_population_reference: str + target_population_digest: str + reference_duration_reference: str + reference_duration_digest: str + eligible_case_set_digest: str + weight_artifact_digest: str + constructed_at: datetime + evidence_version: int = 1 + + def __post_init__(self) -> None: + """Reject ambiguous cross-sectional or longitudinal weight eligibility.""" + _validate_operational_uuid(self.tenant_record_id, "tenant_record_id") + _validate_reference( + self.receipt_reference, + "weight_eligibility_receipt", + "receipt_reference", + ) + if ( + type(self.weight_scope_code) is not str + or self.weight_scope_code not in _WEIGHT_SCOPE_CODES + ): + raise ValueError("weight_scope_code must be cross_sectional or longitudinal") + _validate_reference( + self.target_population_reference, + "analysis_target_population", + "target_population_reference", + ) + _validate_digest(self.target_population_digest, "target_population_digest") + _validate_reference( + self.reference_duration_reference, + "analysis_reference_duration", + "reference_duration_reference", + ) + for field_name in ( + "reference_duration_digest", + "eligible_case_set_digest", + "weight_artifact_digest", + ): + _validate_digest(getattr(self, field_name), field_name) + constructed_at = _freeze_timestamp(self.constructed_at, "constructed_at") + if type(self.evidence_version) is not int or self.evidence_version != 1: + raise ValueError("evidence_version must remain 1") + object.__setattr__(self, "constructed_at", constructed_at) + + def __repr__(self) -> str: + """Return a value-minimized representation suitable for routine logs.""" + return "WeightEligibilityReceipt()" + + def canonical_json(self) -> str: + """Return deterministic eligibility provenance without row-level weight values.""" + payload = { + "constructed_at": _canonical_timestamp(self.constructed_at, "constructed_at"), + "eligible_case_set_digest": self.eligible_case_set_digest, + "evidence_version": self.evidence_version, + "receipt_reference": self.receipt_reference, + "reference_duration_digest": self.reference_duration_digest, + "reference_duration_reference": self.reference_duration_reference, + "target_population_digest": self.target_population_digest, + "target_population_reference": self.target_population_reference, + "tenant_record_id": self.tenant_record_id, + "weight_artifact_digest": self.weight_artifact_digest, + "weight_scope_code": self.weight_scope_code, + } + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + def sha256_digest(self) -> str: + """Return SHA-256 over the exact canonical receipt bytes.""" + return sha256(self.canonical_json().encode("utf-8")).hexdigest() + + +@dataclass(frozen=True, slots=True) +class AnalysisWeightAdjustment: + """Describe one ordered, digest-linked transformation of analysis weights.""" + + sequence_number: int + adjustment_code: str + method_reference: str + method_version: int + input_weight_artifact_digest: str + output_weight_artifact_digest: str + configuration_digest: str + evidence_receipt_digest: str + evidence_kind: str + + def __post_init__(self) -> None: + """Reject unordered, opaque, or unverifiable adjustment evidence.""" + _positive_integer(self.sequence_number, "sequence_number") + _validate_code(self.adjustment_code, "adjustment_code") + _validate_reference(self.method_reference, "weight_method", "method_reference") + _positive_integer(self.method_version, "method_version") + for field_name in ( + "input_weight_artifact_digest", + "output_weight_artifact_digest", + "configuration_digest", + "evidence_receipt_digest", + ): + _validate_digest(getattr(self, field_name), field_name) + _validate_code(self.evidence_kind, "evidence_kind") + required_evidence_kind = _SPECIALIZED_EVIDENCE_KIND_BY_ADJUSTMENT_CODE.get( + self.adjustment_code + ) + if required_evidence_kind is not None and self.evidence_kind != required_evidence_kind: + raise ValueError( + f"{self.adjustment_code} requires evidence_kind {required_evidence_kind}" + ) + if self.input_weight_artifact_digest == self.output_weight_artifact_digest: + raise ValueError( + "output_weight_artifact_digest must identify the transformed weight artifact" + ) + + def to_dict(self) -> dict[str, object]: + """Return canonical adjustment fields without row-level weight values.""" + return { + "adjustment_code": self.adjustment_code, + "configuration_digest": self.configuration_digest, + "evidence_kind": self.evidence_kind, + "evidence_receipt_digest": self.evidence_receipt_digest, + "input_weight_artifact_digest": self.input_weight_artifact_digest, + "method_reference": self.method_reference, + "method_version": self.method_version, + "output_weight_artifact_digest": self.output_weight_artifact_digest, + "sequence_number": self.sequence_number, + } + + +@dataclass(frozen=True, slots=True, repr=False) +class FinalAnalysisWeightReceipt: + """Bind one estimand to the exact final point-estimation weight lineage used.""" + + tenant_record_id: str + receipt_reference: str + estimand_reference: str + estimand_digest: str + estimand_scope_code: str + target_population_reference: str + target_population_digest: str + analysis_unit_code: str + analysis_window_reference: str + reference_duration_reference: str + reference_duration_digest: str + eligible_case_set_digest: str + analytic_case_occurrence_set_digest: str + source_universe_receipt_digest: str + sampling_design_receipt_digest: str + base_weight_method_code: str + base_weight_method_version: int + base_weight_evidence_digest: str + base_weight_artifact_digest: str + adjustments: tuple[AnalysisWeightAdjustment, ...] + final_weight_artifact_digest: str + weight_eligibility: WeightEligibilityReceipt + analytic_case_count: int + constructed_at: datetime + correction_sequence: int = 1 + supersedes_receipt_digest: str | None = None + evidence_version: int = 1 + + def __post_init__(self) -> None: + """Fail closed unless the complete point-weight construction is reproducible.""" + _validate_operational_uuid(self.tenant_record_id, "tenant_record_id") + _validate_reference( + self.receipt_reference, "analysis_weight_receipt", "receipt_reference" + ) + _validate_reference(self.estimand_reference, "validation_estimand", "estimand_reference") + _validate_digest(self.estimand_digest, "estimand_digest") + if ( + type(self.estimand_scope_code) is not str + or self.estimand_scope_code not in _WEIGHT_SCOPE_CODES + ): + raise ValueError("estimand_scope_code must be cross_sectional or longitudinal") + _validate_reference( + self.target_population_reference, + "analysis_target_population", + "target_population_reference", + ) + _validate_digest(self.target_population_digest, "target_population_digest") + _validate_code(self.analysis_unit_code, "analysis_unit_code") + _validate_reference( + self.analysis_window_reference, "analysis_window", "analysis_window_reference" + ) + _validate_reference( + self.reference_duration_reference, + "analysis_reference_duration", + "reference_duration_reference", + ) + _validate_digest(self.reference_duration_digest, "reference_duration_digest") + for field_name in ( + "eligible_case_set_digest", + "analytic_case_occurrence_set_digest", + "source_universe_receipt_digest", + "sampling_design_receipt_digest", + "base_weight_evidence_digest", + "base_weight_artifact_digest", + "final_weight_artifact_digest", + ): + _validate_digest(getattr(self, field_name), field_name) + _validate_code(self.base_weight_method_code, "base_weight_method_code") + _positive_integer(self.base_weight_method_version, "base_weight_method_version") + _positive_integer(self.analytic_case_count, "analytic_case_count") + constructed_at = _freeze_timestamp(self.constructed_at, "constructed_at") + if type(self.weight_eligibility) is not WeightEligibilityReceipt: + raise ValueError("weight_eligibility must be a WeightEligibilityReceipt") + if self.weight_eligibility.tenant_record_id != self.tenant_record_id: + raise ValueError("weight_eligibility tenant_record_id must match the analysis receipt") + if self.weight_eligibility.weight_scope_code != self.estimand_scope_code: + raise ValueError("weight scope must match estimand_scope_code") + if ( + self.weight_eligibility.target_population_reference + != self.target_population_reference + or self.weight_eligibility.target_population_digest != self.target_population_digest + ): + raise ValueError("weight target population must match the estimand target population") + if ( + self.weight_eligibility.reference_duration_reference + != self.reference_duration_reference + or self.weight_eligibility.reference_duration_digest != self.reference_duration_digest + ): + raise ValueError("weight reference duration must match the estimand reference duration") + if self.weight_eligibility.eligible_case_set_digest != self.eligible_case_set_digest: + raise ValueError("weight eligible case set must match the analysis eligible case set") + if type(self.adjustments) is not tuple: + raise ValueError("adjustments must be an immutable tuple") + + expected_input = self.base_weight_artifact_digest + for expected_sequence, adjustment in enumerate(self.adjustments, start=1): + if type(adjustment) is not AnalysisWeightAdjustment: + raise ValueError("adjustments must contain AnalysisWeightAdjustment values") + if adjustment.sequence_number != expected_sequence: + raise ValueError("adjustments must have contiguous sequence_number values") + if adjustment.input_weight_artifact_digest != expected_input: + raise ValueError("adjustment input_weight_artifact_digest breaks the weight chain") + expected_input = adjustment.output_weight_artifact_digest + if expected_input != self.final_weight_artifact_digest: + raise ValueError( + "final_weight_artifact_digest must equal the ordered adjustment chain output" + ) + if self.weight_eligibility.weight_artifact_digest != self.final_weight_artifact_digest: + raise ValueError( + "weight eligibility must identify the final point-estimation weight artifact" + ) + + _positive_integer(self.correction_sequence, "correction_sequence") + if self.correction_sequence == 1: + if self.supersedes_receipt_digest is not None: + raise ValueError( + "supersedes_receipt_digest must be absent for correction_sequence 1" + ) + else: + if self.supersedes_receipt_digest is None: + raise ValueError( + "supersedes_receipt_digest is required when correction_sequence exceeds 1" + ) + _validate_digest(self.supersedes_receipt_digest, "supersedes_receipt_digest") + if type(self.evidence_version) is not int or self.evidence_version != 1: + raise ValueError("evidence_version must remain 1") + object.__setattr__(self, "constructed_at", constructed_at) + + def __repr__(self) -> str: + """Return a value-minimized representation suitable for routine logs.""" + return "FinalAnalysisWeightReceipt()" + + def canonical_json(self) -> str: + """Return deterministic provenance JSON without case-level weight values.""" + payload: dict[str, object] = { + "adjustments": [adjustment.to_dict() for adjustment in self.adjustments], + "analysis_unit_code": self.analysis_unit_code, + "analysis_window_reference": self.analysis_window_reference, + "analytic_case_count": self.analytic_case_count, + "analytic_case_occurrence_set_digest": self.analytic_case_occurrence_set_digest, + "base_weight_artifact_digest": self.base_weight_artifact_digest, + "base_weight_evidence_digest": self.base_weight_evidence_digest, + "base_weight_method_code": self.base_weight_method_code, + "base_weight_method_version": self.base_weight_method_version, + "constructed_at": _canonical_timestamp(self.constructed_at, "constructed_at"), + "correction_sequence": self.correction_sequence, + "eligible_case_set_digest": self.eligible_case_set_digest, + "estimand_digest": self.estimand_digest, + "estimand_reference": self.estimand_reference, + "estimand_scope_code": self.estimand_scope_code, + "evidence_version": self.evidence_version, + "final_weight_artifact_digest": self.final_weight_artifact_digest, + "receipt_reference": self.receipt_reference, + "reference_duration_digest": self.reference_duration_digest, + "reference_duration_reference": self.reference_duration_reference, + "sampling_design_receipt_digest": self.sampling_design_receipt_digest, + "source_universe_receipt_digest": self.source_universe_receipt_digest, + "target_population_digest": self.target_population_digest, + "target_population_reference": self.target_population_reference, + "tenant_record_id": self.tenant_record_id, + "weight_eligibility_receipt_digest": self.weight_eligibility.sha256_digest(), + } + if self.supersedes_receipt_digest is not None: + payload["supersedes_receipt_digest"] = self.supersedes_receipt_digest + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + def sha256_digest(self) -> str: + """Return SHA-256 over the exact canonical receipt bytes.""" + return sha256(self.canonical_json().encode("utf-8")).hexdigest() + + +__all__ = [ + "AnalysisWeightAdjustment", + "CalibrationAdjustmentReceipt", + "FinalAnalysisWeightReceipt", + "NonresponseAdjustmentReceipt", + "TrimmingBoundingAdjustmentReceipt", + "WeightEligibilityReceipt", +] diff --git a/packages/validity-analysis/tests/test_adr_numbering.py b/packages/validity-analysis/tests/test_adr_numbering.py new file mode 100644 index 000000000..8f2ee0828 --- /dev/null +++ b/packages/validity-analysis/tests/test_adr_numbering.py @@ -0,0 +1,17 @@ +"""Regression tests for repository-wide ADR number ownership.""" + +from pathlib import Path + + +def test_adr_numbers_are_unique_across_the_integrated_repository() -> None: + """Every four-digit ADR number must identify exactly one decision record.""" + adr_directory = Path(__file__).resolve().parents[3] / "docs" / "adr" + owners: dict[str, str] = {} + + for adr_path in sorted(adr_directory.glob("[0-9][0-9][0-9][0-9]-*.md")): + adr_number = adr_path.name[:4] + previous_owner = owners.get(adr_number) + assert previous_owner is None, ( + f"ADR {adr_number} is reused by {previous_owner} and {adr_path.name}" + ) + owners[adr_number] = adr_path.name diff --git a/packages/validity-analysis/tests/test_analysis_weight_receipt.py b/packages/validity-analysis/tests/test_analysis_weight_receipt.py new file mode 100644 index 000000000..79e367c32 --- /dev/null +++ b/packages/validity-analysis/tests/test_analysis_weight_receipt.py @@ -0,0 +1,221 @@ +"""Regression contracts for reproducible point-estimation weight lineage.""" + +from datetime import datetime, timezone + +import pytest + +from orgmetra_validity_analysis import ( + AnalysisWeightAdjustment, + FinalAnalysisWeightReceipt, + WeightEligibilityReceipt, +) + +TENANT = "10000000-0000-7000-8000-000000000001" +OTHER_TENANT = "10000000-0000-7000-8000-000000000002" +RECEIPT = "analysis_weight_receipt:11111111-1111-4111-8111-111111111111" +ELIGIBILITY_RECEIPT = "weight_eligibility_receipt:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" +ESTIMAND = "validation_estimand:22222222-2222-4222-8222-222222222222" +TARGET = "analysis_target_population:33333333-3333-4333-8333-333333333333" +OTHER_TARGET = "analysis_target_population:33333333-3333-4333-8333-333333333334" +WINDOW = "analysis_window:44444444-4444-4444-8444-444444444444" +DURATION = "analysis_reference_duration:77777777-7777-4777-8777-777777777777" +OTHER_DURATION = "analysis_reference_duration:77777777-7777-4777-8777-777777777778" +DIGEST_A = "a" * 64 +DIGEST_B = "b" * 64 +DIGEST_C = "c" * 64 +DIGEST_D = "d" * 64 +DIGEST_E = "e" * 64 +DIGEST_F = "f" * 64 +DIGEST_1 = "1" * 64 +DIGEST_2 = "2" * 64 +DIGEST_3 = "3" * 64 +DIGEST_4 = "4" * 64 +DIGEST_5 = "5" * 64 +DIGEST_6 = "6" * 64 + + +def adjustment(**overrides: object) -> AnalysisWeightAdjustment: + """Return one governed adjustment without copying case-level weight values.""" + values: dict[str, object] = { + "sequence_number": 1, + "adjustment_code": "nonresponse_adjustment", + "method_reference": "weight_method:55555555-5555-4555-8555-555555555555", + "method_version": 1, + "input_weight_artifact_digest": DIGEST_D, + "output_weight_artifact_digest": DIGEST_E, + "configuration_digest": DIGEST_F, + "evidence_receipt_digest": DIGEST_1, + "evidence_kind": "nonresponse_adjustment_receipt", + } + values.update(overrides) + return AnalysisWeightAdjustment(**values) + + +def eligibility(**overrides: object) -> WeightEligibilityReceipt: + """Return one weight eligibility receipt aligned to the default estimand.""" + values: dict[str, object] = { + "tenant_record_id": TENANT, + "receipt_reference": ELIGIBILITY_RECEIPT, + "weight_scope_code": "cross_sectional", + "target_population_reference": TARGET, + "target_population_digest": DIGEST_B, + "reference_duration_reference": DURATION, + "reference_duration_digest": DIGEST_6, + "eligible_case_set_digest": DIGEST_C, + "weight_artifact_digest": DIGEST_E, + "constructed_at": datetime(2026, 9, 17, 4, 0, tzinfo=timezone.utc), + } + values.update(overrides) + return WeightEligibilityReceipt(**values) + + +def receipt(**overrides: object) -> FinalAnalysisWeightReceipt: + """Return one exact final point-weight receipt.""" + final_weight = overrides.get("final_weight_artifact_digest", DIGEST_E) + eligibility_receipt = overrides.get( + "weight_eligibility", eligibility(weight_artifact_digest=final_weight) + ) + values: dict[str, object] = { + "tenant_record_id": TENANT, + "receipt_reference": RECEIPT, + "estimand_reference": ESTIMAND, + "estimand_digest": DIGEST_A, + "estimand_scope_code": "cross_sectional", + "target_population_reference": TARGET, + "target_population_digest": DIGEST_B, + "analysis_unit_code": "worker_occurrence", + "analysis_window_reference": WINDOW, + "reference_duration_reference": DURATION, + "reference_duration_digest": DIGEST_6, + "eligible_case_set_digest": DIGEST_C, + "analytic_case_occurrence_set_digest": DIGEST_2, + "source_universe_receipt_digest": DIGEST_3, + "sampling_design_receipt_digest": DIGEST_4, + "base_weight_method_code": "inverse_inclusion_probability", + "base_weight_method_version": 1, + "base_weight_evidence_digest": DIGEST_5, + "base_weight_artifact_digest": DIGEST_D, + "adjustments": (adjustment(),), + "final_weight_artifact_digest": final_weight, + "weight_eligibility": eligibility_receipt, + "analytic_case_count": 12, + "constructed_at": datetime(2026, 9, 17, 4, 0, tzinfo=timezone.utc), + } + values.update(overrides) + return FinalAnalysisWeightReceipt(**values) + + +def test_receipt_is_deterministic_value_minimized_and_redacted() -> None: + """Bind exact estimand and ordered weight lineage without embedding row weights.""" + candidate = receipt() + assert candidate.sha256_digest() == receipt().sha256_digest() + assert candidate.final_weight_artifact_digest == DIGEST_E + assert "person_record" not in candidate.canonical_json() + assert "weight_value" not in candidate.canonical_json() + assert "weight_eligibility_receipt_digest" in candidate.canonical_json() + assert repr(candidate) == "FinalAnalysisWeightReceipt()" + + +def test_no_adjustment_receipt_can_bind_base_weight_as_final_weight() -> None: + """Allow an explicit base-weight-only analysis without inventing a transform.""" + candidate = receipt(adjustments=(), final_weight_artifact_digest=DIGEST_D) + assert '"adjustments":[]' in candidate.canonical_json() + assert candidate.final_weight_artifact_digest == DIGEST_D + + +def test_adjustment_chain_must_reach_final_weight_artifact() -> None: + """Reject a receipt whose declared final weight is not the ordered chain output.""" + with pytest.raises(ValueError, match="final_weight_artifact_digest"): + receipt(final_weight_artifact_digest=DIGEST_F) + + +def test_adjustment_rejects_noop_artifact_identity() -> None: + """Require each declared transform to produce a distinct artifact identity.""" + with pytest.raises(ValueError, match="output_weight_artifact_digest"): + adjustment(output_weight_artifact_digest=DIGEST_D) + + +def test_adjustments_must_be_immutable_exact_and_contiguous() -> None: + """Reject mutable, foreign, skipped, or disconnected adjustment chains.""" + with pytest.raises(ValueError, match="immutable tuple"): + receipt(adjustments=[adjustment()]) + with pytest.raises(ValueError, match="AnalysisWeightAdjustment"): + receipt(adjustments=(object(),)) + with pytest.raises(ValueError, match="sequence_number"): + receipt(adjustments=(adjustment(sequence_number=2),)) + with pytest.raises(ValueError, match="breaks the weight chain"): + receipt(adjustments=(adjustment(input_weight_artifact_digest=DIGEST_C),)) + + +def test_probability_design_receipt_is_required() -> None: + """Do not allow point weights to detach from the sampled design evidence.""" + with pytest.raises(ValueError, match="sampling_design_receipt_digest"): + receipt(sampling_design_receipt_digest="not-a-digest") + + +@pytest.mark.parametrize("scope", [None, "panel"]) +def test_estimand_scope_is_closed_to_cross_sectional_or_longitudinal(scope: object) -> None: + """Reject non-string and caller-defined estimand scope labels.""" + with pytest.raises(ValueError, match="estimand_scope_code"): + receipt(estimand_scope_code=scope) + + +def test_weight_eligibility_must_match_estimand_scope_population_duration_and_cases() -> None: + """Reject cross-sectional/longitudinal or target-window mismatches before release.""" + with pytest.raises(ValueError, match="WeightEligibilityReceipt"): + receipt(weight_eligibility=object()) + with pytest.raises(ValueError, match="tenant_record_id"): + receipt(weight_eligibility=eligibility(tenant_record_id=OTHER_TENANT)) + with pytest.raises(ValueError, match="weight scope"): + receipt(weight_eligibility=eligibility(weight_scope_code="longitudinal")) + with pytest.raises(ValueError, match="target population"): + receipt(weight_eligibility=eligibility(target_population_reference=OTHER_TARGET)) + with pytest.raises(ValueError, match="target population"): + receipt(weight_eligibility=eligibility(target_population_digest=DIGEST_A)) + with pytest.raises(ValueError, match="reference duration"): + receipt(weight_eligibility=eligibility(reference_duration_reference=OTHER_DURATION)) + with pytest.raises(ValueError, match="reference duration"): + receipt(weight_eligibility=eligibility(reference_duration_digest=DIGEST_A)) + with pytest.raises(ValueError, match="eligible case set"): + receipt(weight_eligibility=eligibility(eligible_case_set_digest=DIGEST_A)) + with pytest.raises(ValueError, match="final point-estimation weight artifact"): + receipt(weight_eligibility=eligibility(weight_artifact_digest=DIGEST_D)) + + +def test_longitudinal_weight_is_accepted_only_for_matching_longitudinal_estimand() -> None: + """Allow longitudinal inference when population, duration, cases, and artifact all agree.""" + candidate = receipt( + estimand_scope_code="longitudinal", + weight_eligibility=eligibility(weight_scope_code="longitudinal"), + ) + assert candidate.estimand_scope_code == "longitudinal" + assert candidate.weight_eligibility.weight_scope_code == "longitudinal" + + +def test_positive_versions_and_counts_fail_closed() -> None: + """Reject sentinel sequence, method-version, count, and correction values.""" + with pytest.raises(ValueError, match="sequence_number"): + adjustment(sequence_number=0) + with pytest.raises(ValueError, match="method_version"): + adjustment(method_version=0) + with pytest.raises(ValueError, match="analytic_case_count"): + receipt(analytic_case_count=0) + with pytest.raises(ValueError, match="correction_sequence"): + receipt(correction_sequence=0) + + +def test_correction_lineage_is_append_only_and_canonicalized() -> None: + """Require successor evidence rather than overwriting a prior weight receipt.""" + with pytest.raises(ValueError, match="supersedes_receipt_digest"): + receipt(correction_sequence=2) + with pytest.raises(ValueError, match="must be absent"): + receipt(supersedes_receipt_digest=DIGEST_A) + + corrected = receipt(correction_sequence=2, supersedes_receipt_digest=DIGEST_A) + assert f'"supersedes_receipt_digest":"{DIGEST_A}"' in corrected.canonical_json() + + +def test_evidence_version_is_not_caller_extensible() -> None: + """Prevent callers from inventing a new receipt schema without a reviewed contract.""" + with pytest.raises(ValueError, match="evidence_version"): + receipt(evidence_version=2) diff --git a/packages/validity-analysis/tests/test_analysis_weight_result_binding.py b/packages/validity-analysis/tests/test_analysis_weight_result_binding.py new file mode 100644 index 000000000..84344bcd2 --- /dev/null +++ b/packages/validity-analysis/tests/test_analysis_weight_result_binding.py @@ -0,0 +1,309 @@ +"""Regression contract for binding weighted results to exact weight evidence.""" + +from datetime import datetime, timezone + +import pytest + +from orgmetra_validity_analysis import ( + ConvergenceDiagnostics, + FinalAnalysisWeightReceipt, + MissingnessSummary, + REVIEWED_FAST_MLSIRM_REVISION, + ValidationAnalysisResult, + WeightEligibilityReceipt, + WeightVarianceCompatibilityReceipt, +) + +TENANT = "10000000-0000-7000-8000-000000000001" +RESULT = "validation_analysis_result:11111111-1111-4111-8111-111111111111" +WEIGHT_RECEIPT = "analysis_weight_receipt:22222222-2222-4222-8222-222222222222" +ELIGIBILITY_RECEIPT = "weight_eligibility_receipt:33333333-3333-4333-8333-333333333333" +COMPATIBILITY_RECEIPT = ( + "weight_variance_compatibility_receipt:44444444-4444-4444-8444-444444444444" +) +VARIANCE_RECEIPT = "variance_design_receipt:55555555-5555-4555-8555-555555555555" +ESTIMAND = "validation_estimand:66666666-6666-4666-8666-666666666666" +TARGET = "analysis_target_population:77777777-7777-4777-8777-777777777777" +WINDOW = "analysis_window:88888888-8888-4888-8888-888888888888" +DURATION = "analysis_reference_duration:99999999-9999-4999-8999-999999999999" +HANDOFF_DIGEST = "a" * 64 +PROVENANCE_DIGEST = "b" * 64 +VARIANCE_DIGEST = "d" * 64 +OTHER_VARIANCE_DIGEST = "e" * 64 +DIGEST_1 = "1" * 64 +DIGEST_2 = "2" * 64 +DIGEST_3 = "3" * 64 +DIGEST_4 = "4" * 64 +DIGEST_5 = "5" * 64 +DIGEST_6 = "6" * 64 +DIGEST_7 = "7" * 64 + + +def point_weight(**overrides: object) -> FinalAnalysisWeightReceipt: + """Build one base-weight-only point-estimation receipt for result binding.""" + eligibility = WeightEligibilityReceipt( + tenant_record_id=TENANT, + receipt_reference=ELIGIBILITY_RECEIPT, + weight_scope_code="cross_sectional", + target_population_reference=TARGET, + target_population_digest=DIGEST_1, + reference_duration_reference=DURATION, + reference_duration_digest=DIGEST_2, + eligible_case_set_digest=DIGEST_3, + weight_artifact_digest=DIGEST_7, + constructed_at=datetime(2026, 9, 17, 4, 0, tzinfo=timezone.utc), + ) + values: dict[str, object] = { + "tenant_record_id": TENANT, + "receipt_reference": WEIGHT_RECEIPT, + "estimand_reference": ESTIMAND, + "estimand_digest": DIGEST_4, + "estimand_scope_code": "cross_sectional", + "target_population_reference": TARGET, + "target_population_digest": DIGEST_1, + "analysis_unit_code": "worker_occurrence", + "analysis_window_reference": WINDOW, + "reference_duration_reference": DURATION, + "reference_duration_digest": DIGEST_2, + "eligible_case_set_digest": DIGEST_3, + "analytic_case_occurrence_set_digest": DIGEST_5, + "source_universe_receipt_digest": DIGEST_6, + "sampling_design_receipt_digest": "8" * 64, + "base_weight_method_code": "inverse_inclusion_probability", + "base_weight_method_version": 1, + "base_weight_evidence_digest": "9" * 64, + "base_weight_artifact_digest": DIGEST_7, + "adjustments": (), + "final_weight_artifact_digest": DIGEST_7, + "weight_eligibility": eligibility, + "analytic_case_count": 12, + "constructed_at": datetime(2026, 9, 17, 4, 0, tzinfo=timezone.utc), + } + values.update(overrides) + return FinalAnalysisWeightReceipt(**values) + + +def compatibility( + weight: FinalAnalysisWeightReceipt, variance_digest: str = VARIANCE_DIGEST +) -> WeightVarianceCompatibilityReceipt: + """Build compatibility evidence whose variance lineage names the exact point weight.""" + return WeightVarianceCompatibilityReceipt( + tenant_record_id=TENANT, + receipt_reference=COMPATIBILITY_RECEIPT, + analysis_weight_receipt=weight, + variance_design_receipt_reference=VARIANCE_RECEIPT, + variance_design_receipt_version=1, + variance_design_receipt_digest=variance_digest, + variance_analysis_weight_receipt_digest=weight.sha256_digest(), + variance_analytic_case_occurrence_set_digest=( + weight.analytic_case_occurrence_set_digest + ), + variance_weight_eligibility_receipt_digest=( + weight.weight_eligibility.sha256_digest() + ), + variance_weight_correction_sequence=weight.correction_sequence, + variance_final_weight_artifact_digest=weight.final_weight_artifact_digest, + constructed_at=datetime(2026, 9, 17, 4, 5, tzinfo=timezone.utc), + ) + + +def result(**overrides: object) -> ValidationAnalysisResult: + """Build one bounded validity result with targeted inference overrides.""" + values: dict[str, object] = { + "tenant_record_id": TENANT, + "result_reference": RESULT, + "handoff_digest": HANDOFF_DIGEST, + "provenance_digest": PROVENANCE_DIGEST, + "fast_mlsirm_revision": REVIEWED_FAST_MLSIRM_REVISION, + "model_code": "mlsirm_criterion_related", + "backend": "rust_cpu", + "precision": "f64", + "effect_estimate": 0.42, + "uncertainty_lower": 0.10, + "uncertainty_upper": 0.70, + "sample_size": 12, + "missingness_summary": MissingnessSummary( + total_observations=12, + complete_observations=10, + missing_predictor_observations=1, + missing_criterion_observations=1, + ), + "convergence_diagnostics": ConvergenceDiagnostics( + converged=True, + iterations=42, + objective_value=-12.5, + maximum_gradient=0.0001, + ), + "completed_at": datetime(2026, 9, 17, 4, 10, tzinfo=timezone.utc), + } + values.update(overrides) + return ValidationAnalysisResult(**values) + + +def test_weighted_result_binds_point_variance_and_compatibility_receipts() -> None: + """Require exact point-weight, variance, and cross-lineage compatibility evidence.""" + weight = point_weight() + correlation = compatibility(weight) + candidate = result( + point_estimation_mode="weighted_design_based", + analysis_weight_receipt_digest=weight.sha256_digest(), + variance_design_receipt_digest=VARIANCE_DIGEST, + weight_variance_compatibility=correlation, + ) + payload = candidate.canonical_json() + assert '"point_estimation_mode":"weighted_design_based"' in payload + assert f'"analysis_weight_receipt_digest":"{weight.sha256_digest()}"' in payload + assert f'"variance_design_receipt_digest":"{VARIANCE_DIGEST}"' in payload + assert ( + f'"weight_variance_compatibility_receipt_digest":"{correlation.sha256_digest()}"' + in payload + ) + + +def test_weighted_result_rejects_same_point_and_variance_receipt() -> None: + """A variance-design receipt cannot silently stand in for the point-weight receipt.""" + weight = point_weight() + with pytest.raises(ValueError, match="must identify different evidence"): + result( + point_estimation_mode="weighted_design_based", + analysis_weight_receipt_digest=weight.sha256_digest(), + variance_design_receipt_digest=weight.sha256_digest(), + weight_variance_compatibility=compatibility(weight), + ) + + +def test_weighted_result_requires_compatibility_receipt() -> None: + """Do not release weighted inference without explicit point/variance congruence.""" + weight = point_weight() + with pytest.raises(ValueError, match="weight_variance_compatibility"): + result( + point_estimation_mode="weighted_design_based", + analysis_weight_receipt_digest=weight.sha256_digest(), + variance_design_receipt_digest=VARIANCE_DIGEST, + ) + + +def test_weighted_result_rejects_compatibility_for_other_point_or_variance_evidence() -> None: + """The compatibility receipt must correlate the same point and variance digests.""" + weight = point_weight() + corrected_weight = point_weight( + correction_sequence=2, + supersedes_receipt_digest=weight.sha256_digest(), + ) + with pytest.raises(ValueError, match="analysis weight receipt"): + result( + point_estimation_mode="weighted_design_based", + analysis_weight_receipt_digest=weight.sha256_digest(), + variance_design_receipt_digest=VARIANCE_DIGEST, + weight_variance_compatibility=compatibility(corrected_weight), + ) + with pytest.raises(ValueError, match="variance design receipt"): + result( + point_estimation_mode="weighted_design_based", + analysis_weight_receipt_digest=weight.sha256_digest(), + variance_design_receipt_digest=OTHER_VARIANCE_DIGEST, + weight_variance_compatibility=compatibility(weight), + ) + + +def test_weighted_result_rechecks_compatibility_tenant_and_time_boundary() -> None: + """Forged compatibility state cannot cross tenant or result-time boundaries.""" + weight = point_weight() + correlation = compatibility(weight) + object.__setattr__( + correlation, + "tenant_record_id", + "10000000-0000-7000-8000-000000000002", + ) + with pytest.raises(ValueError, match="tenant_record_id"): + result( + point_estimation_mode="weighted_design_based", + analysis_weight_receipt_digest=weight.sha256_digest(), + variance_design_receipt_digest=VARIANCE_DIGEST, + weight_variance_compatibility=correlation, + ) + + later = compatibility(weight) + object.__setattr__( + later, + "constructed_at", + datetime(2026, 9, 17, 4, 11, tzinfo=timezone.utc), + ) + with pytest.raises(ValueError, match="cannot be constructed after"): + result( + point_estimation_mode="weighted_design_based", + analysis_weight_receipt_digest=weight.sha256_digest(), + variance_design_receipt_digest=VARIANCE_DIGEST, + weight_variance_compatibility=later, + ) + + +@pytest.mark.parametrize( + "overrides,match", + [ + ( + { + "point_estimation_mode": "weighted_design_based", + "analysis_weight_receipt_digest": None, + "variance_design_receipt_digest": VARIANCE_DIGEST, + }, + "analysis_weight_receipt_digest", + ), + ( + { + "point_estimation_mode": "weighted_design_based", + "analysis_weight_receipt_digest": "c" * 64, + "variance_design_receipt_digest": None, + }, + "variance_design_receipt_digest", + ), + ( + { + "point_estimation_mode": "unweighted", + "analysis_weight_receipt_digest": "c" * 64, + }, + "unweighted", + ), + ( + { + "point_estimation_mode": "unweighted", + "variance_design_receipt_digest": VARIANCE_DIGEST, + }, + "unweighted", + ), + ( + { + "point_estimation_mode": "unweighted", + "weight_variance_compatibility": compatibility(point_weight()), + }, + "unweighted", + ), + ({"point_estimation_mode": "opaque_weighted"}, "point_estimation_mode"), + ({"point_estimation_mode": None}, "point_estimation_mode"), + ], +) +def test_result_rejects_unverifiable_weight_binding( + overrides: dict[str, object], match: str +) -> None: + """Fail closed when result and point/variance weight provenance disagree.""" + with pytest.raises(ValueError, match=match): + result(**overrides) + + +def test_weighted_result_rejects_malformed_receipt_digests() -> None: + """Do not permit opaque labels to stand in for immutable weight evidence.""" + weight = point_weight() + with pytest.raises(ValueError, match="analysis_weight_receipt_digest"): + result( + point_estimation_mode="weighted_design_based", + analysis_weight_receipt_digest="weight-v1", + variance_design_receipt_digest=VARIANCE_DIGEST, + weight_variance_compatibility=compatibility(weight), + ) + with pytest.raises(ValueError, match="variance_design_receipt_digest"): + result( + point_estimation_mode="weighted_design_based", + analysis_weight_receipt_digest=weight.sha256_digest(), + variance_design_receipt_digest="variance-v1", + weight_variance_compatibility=compatibility(weight), + ) diff --git a/packages/validity-analysis/tests/test_calibration_constraint_application.py b/packages/validity-analysis/tests/test_calibration_constraint_application.py new file mode 100644 index 000000000..73f85d115 --- /dev/null +++ b/packages/validity-analysis/tests/test_calibration_constraint_application.py @@ -0,0 +1,94 @@ +"""RED contracts for reviewed versus actually applied calibration constraints.""" + +from dataclasses import fields +from datetime import datetime, timedelta, timezone + +import pytest + +from orgmetra_validity_analysis import CalibrationAdjustmentReceipt + +TENANT = "10000000-0000-7000-8000-000000000001" +NOW = datetime(2026, 9, 18, 3, 30, tzinfo=timezone.utc) +DIGEST_A = "a" * 64 +DIGEST_B = "b" * 64 +DIGEST_C = "c" * 64 +DIGEST_D = "d" * 64 +DIGEST_E = "e" * 64 +DIGEST_F = "f" * 64 +DIGEST_1 = "1" * 64 +DIGEST_2 = "2" * 64 +DIGEST_3 = "3" * 64 +DIGEST_4 = "4" * 64 +DIGEST_5 = "5" * 64 + + +def _receipt(**overrides: object) -> CalibrationAdjustmentReceipt: + """Build one calibration receipt with separate reviewed and applied constraints.""" + values: dict[str, object] = { + "tenant_record_id": TENANT, + "receipt_reference": "calibration_adjustment_receipt:33333333-3333-4333-8333-333333333333", + "target_population_digest": DIGEST_A, + "analysis_window_reference": "analysis_window:44444444-4444-4444-8444-444444444444", + "auxiliary_authority_reference": "scientific_auxiliary_authority:55555555-5555-4555-8555-555555555550", + "auxiliary_projection_reference": "calibration_auxiliary_projection:55555555-5555-4555-8555-555555555555", + "auxiliary_projection_version": 4, + "auxiliary_projection_digest": DIGEST_B, + "auxiliary_purpose_reference": "scientific_data_use_purpose:55555555-5555-4555-8555-555555555556", + "auxiliary_purpose_digest": DIGEST_3, + "auxiliary_owner_contract_reference": "released_owner_contract:55555555-5555-4555-8555-555555555557", + "auxiliary_owner_contract_version": 1, + "auxiliary_owner_contract_digest": DIGEST_1, + "auxiliary_authorization_receipt_reference": "scientific_data_authorization:55555555-5555-4555-8555-555555555558", + "auxiliary_authorization_receipt_digest": DIGEST_4, + "auxiliary_scientific_use_receipt_reference": "scientific_use_receipt:55555555-5555-4555-8555-555555555559", + "auxiliary_scientific_use_receipt_digest": DIGEST_2, + "auxiliary_scientific_use_at": NOW, + "benchmark_receipt_reference": "calibration_benchmark_receipt:66666666-6666-4666-8666-666666666666", + "benchmark_receipt_version": 2, + "benchmark_receipt_digest": DIGEST_C, + "benchmark_owner_contract_reference": "released_owner_contract:66666666-6666-4666-8666-666666666667", + "benchmark_owner_contract_version": 3, + "benchmark_owner_contract_digest": DIGEST_5, + "benchmark_reference_at": NOW - timedelta(days=1), + "algorithm_reference": "calibration_algorithm:77777777-7777-4777-8777-777777777777", + "algorithm_version": 1, + "constraints_digest": DIGEST_F, + "applied_constraints_digest": DIGEST_F, + "termination_code": "converged", + "input_weight_artifact_digest": DIGEST_D, + "output_weight_artifact_digest": DIGEST_E, + "constructed_at": NOW, + } + values.update(overrides) + return CalibrationAdjustmentReceipt(**values) + + +def test_calibration_receipt_exposes_actual_applied_constraints() -> None: + """A reviewed constraint set and the generating constraint set must be distinct fields.""" + assert "applied_constraints_digest" in { + field.name for field in fields(CalibrationAdjustmentReceipt) + } + + +def test_primary_convergence_rejects_silent_constraint_relaxation() -> None: + """Changed generating constraints require explicit fallback provenance.""" + with pytest.raises(ValueError, match="fallback"): + _receipt(applied_constraints_digest=DIGEST_1) + + +def test_fallback_commits_to_reviewed_and_applied_constraint_sets() -> None: + """A fallback may change constraints only when its generating path is explicit.""" + candidate = _receipt( + termination_code="fallback_applied", + applied_constraints_digest=DIGEST_1, + fallback_reason_code="constraint_relaxation", + fallback_rule_reference="calibration_fallback_rule:99999999-9999-4999-8999-999999999999", + fallback_rule_digest=DIGEST_2, + fallback_algorithm_reference="calibration_algorithm:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + fallback_algorithm_version=2, + fallback_configuration_digest=DIGEST_4, + ) + payload = candidate.canonical_json() + + assert f'"constraints_digest":"{DIGEST_F}"' in payload + assert f'"applied_constraints_digest":"{DIGEST_1}"' in payload diff --git a/packages/validity-analysis/tests/test_calibration_fallback_provenance.py b/packages/validity-analysis/tests/test_calibration_fallback_provenance.py new file mode 100644 index 000000000..e0f196ed7 --- /dev/null +++ b/packages/validity-analysis/tests/test_calibration_fallback_provenance.py @@ -0,0 +1,132 @@ +"""Regression contracts for explicit calibration fallback provenance.""" + +from datetime import datetime, timedelta, timezone + +import pytest + +from orgmetra_validity_analysis import CalibrationAdjustmentReceipt + +TENANT = "10000000-0000-7000-8000-000000000001" +NOW = datetime(2026, 9, 17, 7, 0, tzinfo=timezone.utc) +DIGEST_A = "a" * 64 +DIGEST_B = "b" * 64 +DIGEST_C = "c" * 64 +DIGEST_D = "d" * 64 +DIGEST_E = "e" * 64 +DIGEST_F = "f" * 64 +DIGEST_1 = "1" * 64 +DIGEST_2 = "2" * 64 +DIGEST_3 = "3" * 64 +DIGEST_4 = "4" * 64 +DIGEST_5 = "5" * 64 + + +def calibration_receipt(**overrides: object) -> CalibrationAdjustmentReceipt: + """Return one fallback-bearing calibration receipt for focused provenance checks.""" + values: dict[str, object] = { + "tenant_record_id": TENANT, + "receipt_reference": "calibration_adjustment_receipt:33333333-3333-4333-8333-333333333333", + "target_population_digest": DIGEST_A, + "analysis_window_reference": "analysis_window:44444444-4444-4444-8444-444444444444", + "auxiliary_authority_reference": "scientific_auxiliary_authority:55555555-5555-4555-8555-555555555550", + "auxiliary_projection_reference": "calibration_auxiliary_projection:55555555-5555-4555-8555-555555555555", + "auxiliary_projection_version": 4, + "auxiliary_projection_digest": DIGEST_B, + "auxiliary_purpose_reference": "scientific_data_use_purpose:55555555-5555-4555-8555-555555555556", + "auxiliary_purpose_digest": DIGEST_3, + "auxiliary_owner_contract_reference": "released_owner_contract:55555555-5555-4555-8555-555555555557", + "auxiliary_owner_contract_version": 1, + "auxiliary_owner_contract_digest": DIGEST_1, + "auxiliary_authorization_receipt_reference": "scientific_data_authorization:55555555-5555-4555-8555-555555555558", + "auxiliary_authorization_receipt_digest": DIGEST_4, + "auxiliary_scientific_use_receipt_reference": "scientific_use_receipt:55555555-5555-4555-8555-555555555559", + "auxiliary_scientific_use_receipt_digest": DIGEST_2, + "auxiliary_scientific_use_at": NOW, + "benchmark_receipt_reference": "calibration_benchmark_receipt:66666666-6666-4666-8666-666666666666", + "benchmark_receipt_version": 2, + "benchmark_receipt_digest": DIGEST_C, + "benchmark_owner_contract_reference": "released_owner_contract:66666666-6666-4666-8666-666666666667", + "benchmark_owner_contract_version": 3, + "benchmark_owner_contract_digest": DIGEST_5, + "benchmark_reference_at": NOW - timedelta(days=1), + "algorithm_reference": "calibration_algorithm:77777777-7777-4777-8777-777777777777", + "algorithm_version": 1, + "constraints_digest": DIGEST_F, + "applied_constraints_digest": DIGEST_F, + "termination_code": "fallback_applied", + "fallback_reason_code": "primary_nonconvergence", + "fallback_rule_reference": "calibration_fallback_rule:99999999-9999-4999-8999-999999999999", + "fallback_rule_digest": DIGEST_2, + "fallback_algorithm_reference": "calibration_algorithm:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + "fallback_algorithm_version": 2, + "fallback_configuration_digest": DIGEST_4, + "input_weight_artifact_digest": DIGEST_D, + "output_weight_artifact_digest": DIGEST_E, + "constructed_at": NOW, + } + values.update(overrides) + return CalibrationAdjustmentReceipt(**values) + + +def test_fallback_identifies_reason_rule_algorithm_and_applied_constraints() -> None: + """Do not label fallback output as if the primary calibration specification succeeded.""" + candidate = calibration_receipt(applied_constraints_digest=DIGEST_1) + canonical = candidate.canonical_json() + + assert '"termination_code":"fallback_applied"' in canonical + assert '"fallback_reason_code":"primary_nonconvergence"' in canonical + assert '"fallback_algorithm_reference":"calibration_algorithm:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"' in canonical + assert '"fallback_algorithm_version":2' in canonical + assert f'"fallback_configuration_digest":"{DIGEST_4}"' in canonical + assert f'"fallback_rule_digest":"{DIGEST_2}"' in canonical + assert f'"constraints_digest":"{DIGEST_F}"' in canonical + assert f'"applied_constraints_digest":"{DIGEST_1}"' in canonical + + +@pytest.mark.parametrize( + ("field_name", "value"), + [ + ("fallback_reason_code", None), + ("fallback_rule_reference", None), + ("fallback_rule_digest", None), + ("fallback_algorithm_reference", None), + ("fallback_algorithm_version", None), + ("fallback_configuration_digest", None), + ], +) +def test_fallback_rejects_incomplete_actual_method_provenance( + field_name: str, + value: object, +) -> None: + """A fallback needs the actual generating method, not only an opaque fallback flag.""" + with pytest.raises((TypeError, ValueError), match="fallback"): + calibration_receipt(**{field_name: value}) + + +@pytest.mark.parametrize( + ("field_name", "value", "message"), + [ + ("fallback_reason_code", "", "fallback_reason_code"), + ("fallback_rule_reference", "fallback-rule-v1", "fallback_rule_reference"), + ("fallback_rule_digest", "not-a-digest", "fallback_rule_digest"), + ("fallback_algorithm_reference", "weight_method:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", "fallback_algorithm_reference"), + ("fallback_algorithm_version", 0, "fallback_algorithm_version"), + ("fallback_algorithm_version", True, "fallback_algorithm_version"), + ("fallback_configuration_digest", "not-a-digest", "fallback_configuration_digest"), + ("applied_constraints_digest", "not-a-digest", "applied_constraints_digest"), + ], +) +def test_fallback_rejects_malformed_generating_method_provenance( + field_name: str, + value: object, + message: str, +) -> None: + """Fallback provenance must be typed and reproducible, not merely present.""" + with pytest.raises(ValueError, match=message): + calibration_receipt(**{field_name: value}) + + +def test_converged_calibration_rejects_fallback_only_fields() -> None: + """Fallback evidence must not contaminate a genuinely converged primary algorithm.""" + with pytest.raises(ValueError, match="fallback"): + calibration_receipt(termination_code="converged") diff --git a/packages/validity-analysis/tests/test_handoff.py b/packages/validity-analysis/tests/test_handoff.py new file mode 100644 index 000000000..64d1ab8c6 --- /dev/null +++ b/packages/validity-analysis/tests/test_handoff.py @@ -0,0 +1,221 @@ +"""Regression tests for governed selection-validity analysis handoffs.""" + +from dataclasses import replace +from datetime import datetime, timedelta, timezone +import json + +import pytest + +from orgmetra_validity_analysis import ( + REVIEWED_FAST_MLSIRM_REVISION, + ValidationAnalysisHandoff, + build_validation_analysis_handoff, +) + +TENANT = "10000000-0000-7000-8000-000000000001" +HANDOFF = "validation_analysis_handoff:11111111-1111-4111-8111-111111111111" +STUDY = "validation_study:22222222-2222-4222-8222-222222222222" +JOB = "job_profile:33333333-3333-4333-8333-333333333333" +PREDICTOR = "predictor_snapshot:44444444-4444-4444-8444-444444444444" +CRITERION = "criterion_snapshot:55555555-5555-4555-8555-555555555555" +POPULATION = "study_population_snapshot:66666666-6666-4666-8666-666666666666" +POLICY = "decision_policy:77777777-7777-4777-8777-777777777777" +PLAN = "validation_analysis_plan:88888888-8888-4888-8888-888888888888" +ACTOR = "actor:99999999-9999-4999-8999-999999999999" +REVIEWER = "actor:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" +DIGEST_A = "a" * 64 +DIGEST_B = "b" * 64 +DIGEST_C = "c" * 64 +DIGEST_D = "d" * 64 +DIGEST_E = "e" * 64 +REQUESTED_AT = datetime(2026, 8, 21, 7, 10, 11, 123456, tzinfo=timezone(timedelta(hours=9))) + + +def valid_kwargs(): + """Return one complete governed handoff input fixture.""" + return { + "tenant_record_id": TENANT, + "handoff_reference": HANDOFF, + "validation_study_reference": STUDY, + "job_profile_reference": JOB, + "predictor_snapshot_reference": PREDICTOR, + "predictor_snapshot_digest": DIGEST_A, + "criterion_snapshot_reference": CRITERION, + "criterion_snapshot_digest": DIGEST_B, + "population_snapshot_reference": POPULATION, + "population_snapshot_digest": DIGEST_C, + "decision_policy_reference": POLICY, + "decision_policy_digest": DIGEST_D, + "analysis_plan_reference": PLAN, + "analysis_plan_digest": DIGEST_E, + "actor_reference": ACTOR, + "reviewer_reference": REVIEWER, + "fast_mlsirm_revision": REVIEWED_FAST_MLSIRM_REVISION, + "requested_at": REQUESTED_AT, + } + + +def handoff(): + """Build the canonical valid handoff fixture.""" + return build_validation_analysis_handoff(**valid_kwargs()) + + +def test_handoff_is_value_minimized_deterministic_and_human_review_only(): + """Bind exact study evidence without exposing raw person-level observations.""" + candidate = handoff() + payload = json.loads(candidate.canonical_json()) + + assert payload["tenant_record_id"] == TENANT + assert payload["validation_study_reference"] == STUDY + assert payload["job_profile_reference"] == JOB + assert payload["fast_mlsirm_revision"] == REVIEWED_FAST_MLSIRM_REVISION + assert payload["requested_at"] == "2026-08-20T22:10:11.123456Z" + assert payload["validation_strategy"] == "criterion_related" + assert payload["kernel_repository"] == "ContextualWisdomLab/fast-mlsirm" + assert payload["kernel_boundary"] == "read_only_pinned_revision" + assert payload["execution_state"] == "not_executed" + assert payload["contains_raw_person_level_values"] is False + assert payload["human_review_required"] is True + assert payload["result_authority"] == "scientific_evidence_only" + assert payload["required_result_evidence"] == [ + "effect_estimate", + "uncertainty_interval", + "sample_size", + "missingness_summary", + "convergence_diagnostics", + ] + assert "person_record" not in candidate.canonical_json() + assert "candidate" not in candidate.canonical_json() + assert repr(candidate) == "ValidationAnalysisHandoff()" + assert len(candidate.sha256_digest()) == 64 + assert candidate.canonical_json() == handoff().canonical_json() + + +@pytest.mark.parametrize( + "bad_tenant", + [ + "not-a-uuid", + "00000000-0000-0000-0000-000000000000", + "ffffffff-ffff-ffff-ffff-ffffffffffff", + "10000000-0000-7000-8000-00000000000A", + 1, + ], +) +def test_tenant_identity_must_follow_protected_operational_uuid_contract(bad_tenant): + """Reject malformed, reserved, non-canonical, and non-text tenant identities.""" + values = valid_kwargs() + values["tenant_record_id"] = bad_tenant + with pytest.raises(ValueError, match="tenant_record_id"): + build_validation_analysis_handoff(**values) + + +@pytest.mark.parametrize( + ("field", "bad", "match"), + [ + ("handoff_reference", "validation_analysis_handoff:not-a-uuid", "handoff_reference"), + ("validation_study_reference", JOB, "validation_study_reference"), + ("job_profile_reference", "job_profile:22222222-2222-7222-8222-222222222222", "job_profile_reference"), + ("predictor_snapshot_reference", 1, "predictor_snapshot_reference"), + ("criterion_snapshot_reference", "criterion_snapshot:" + "a" * 161, "criterion_snapshot_reference"), + ("population_snapshot_reference", "study_population_snapshot:not-a-uuid", "population_snapshot_reference"), + ("decision_policy_reference", "decision_policy:BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB", "decision_policy_reference"), + ("analysis_plan_reference", "validation_analysis_plan:00000000-0000-0000-0000-000000000000", "analysis_plan_reference"), + ("actor_reference", "actor:ffffffff-ffff-ffff-ffff-ffffffffffff", "actor_reference"), + ("reviewer_reference", "actor:bbbbbbbb-bbbb-7bbb-8bbb-bbbbbbbbbbbb", "reviewer_reference"), + ], +) +def test_all_public_references_are_namespaced_opaque_uuid4(field, bad, match): + """Fail closed on wrong namespace, malformed, noncanonical, or non-v4 references.""" + values = valid_kwargs() + values[field] = bad + with pytest.raises(ValueError, match=match): + build_validation_analysis_handoff(**values) + + +@pytest.mark.parametrize( + "field", + [ + "predictor_snapshot_digest", + "criterion_snapshot_digest", + "population_snapshot_digest", + "decision_policy_digest", + "analysis_plan_digest", + ], +) +def test_evidence_digests_are_lowercase_sha256(field): + """Reject weak or noncanonical evidence digests for every source snapshot.""" + values = valid_kwargs() + values[field] = "A" * 64 + with pytest.raises(ValueError, match=field): + build_validation_analysis_handoff(**values) + + +def test_requester_and_reviewer_must_be_distinct(): + """Require accountable independent interpretation instead of self-review.""" + values = valid_kwargs() + values["reviewer_reference"] = ACTOR + with pytest.raises(ValueError, match="different accountable actor"): + build_validation_analysis_handoff(**values) + + +@pytest.mark.parametrize("bad_revision", ["not-a-sha", "A" * 40, "0" * 40]) +def test_fast_mlsirm_revision_is_exactly_the_reviewed_immutable_dependency(bad_revision): + """Reject malformed or unreviewed foreign dependency revisions.""" + values = valid_kwargs() + values["fast_mlsirm_revision"] = bad_revision + with pytest.raises(ValueError, match="fast_mlsirm_revision"): + build_validation_analysis_handoff(**values) + + +@pytest.mark.parametrize( + "bad_requested_at", + [ + datetime(2026, 8, 21, 7, 10), + "2026-08-21T07:10:00+09:00", + ], +) +def test_requested_at_requires_an_aware_datetime(bad_requested_at): + """Reject local-time ambiguity in immutable analysis correlation.""" + values = valid_kwargs() + values["requested_at"] = bad_requested_at + with pytest.raises(ValueError, match="requested_at"): + build_validation_analysis_handoff(**values) + + +@pytest.mark.parametrize( + ("field", "bad", "match"), + [ + ("purpose_code", "other_purpose", "purpose_code"), + ("reason_code", "other_reason", "reason_code"), + ("evidence_version", True, "evidence_version"), + ("evidence_version", 0, "evidence_version"), + ("validation_strategy", "content_validity", "validation_strategy"), + ("kernel_repository", "other/repository", "kernel_repository"), + ("kernel_boundary", "direct_database", "kernel_boundary"), + ("execution_state", "executed", "execution_state"), + ("contains_raw_person_level_values", True, "raw person-level"), + ("human_review_required", False, "human review"), + ("result_authority", "employment_decision", "result_authority"), + ("required_result_evidence", ("effect_estimate",), "required_result_evidence"), + ("next_action", "Auto-approve the result.", "next_action"), + ], +) +def test_direct_construction_cannot_weaken_governance(field, bad, match): + """Keep fixed scientific, privacy, dependency, and human-authority semantics immutable.""" + with pytest.raises(ValueError, match=match): + replace(handoff(), **{field: bad}) + + +def test_codes_must_remain_bounded_descriptive_snake_case_before_fixed_value_check(): + """Exercise code-shape rejection separately from the closed purpose/reason vocabulary.""" + with pytest.raises(ValueError, match="purpose_code"): + replace(handoff(), purpose_code="X") + with pytest.raises(ValueError, match="reason_code"): + replace(handoff(), reason_code="x" * 65) + + +def test_public_dataclass_type_is_constructible_only_with_all_invariants(): + """Document the public immutable type while preserving builder equivalence.""" + values = valid_kwargs() + direct = ValidationAnalysisHandoff(**values) + assert direct == handoff() diff --git a/packages/validity-analysis/tests/test_host_resolution_contract.py b/packages/validity-analysis/tests/test_host_resolution_contract.py new file mode 100644 index 000000000..c87fb1a18 --- /dev/null +++ b/packages/validity-analysis/tests/test_host_resolution_contract.py @@ -0,0 +1,34 @@ +"""Regressions for authoritative requester/reviewer identity separation.""" + +from datetime import datetime, timezone + +from orgmetra_validity_analysis import ( + REVIEWED_FAST_MLSIRM_REVISION, + build_validation_analysis_handoff, +) + + +def test_next_action_requires_resolved_actor_identity_separation() -> None: + """Do not let different opaque actor references masquerade as distinct people.""" + handoff = build_validation_analysis_handoff( + tenant_record_id="10000000-0000-7000-8000-000000000001", + handoff_reference="validation_analysis_handoff:11111111-1111-4111-8111-111111111111", + validation_study_reference="validation_study:22222222-2222-4222-8222-222222222222", + job_profile_reference="job_profile:33333333-3333-4333-8333-333333333333", + predictor_snapshot_reference="predictor_snapshot:44444444-4444-4444-8444-444444444444", + predictor_snapshot_digest="a" * 64, + criterion_snapshot_reference="criterion_snapshot:55555555-5555-4555-8555-555555555555", + criterion_snapshot_digest="b" * 64, + population_snapshot_reference="study_population_snapshot:66666666-6666-4666-8666-666666666666", + population_snapshot_digest="c" * 64, + decision_policy_reference="decision_policy:77777777-7777-4777-8777-777777777777", + decision_policy_digest="d" * 64, + analysis_plan_reference="validation_analysis_plan:88888888-8888-4888-8888-888888888888", + analysis_plan_digest="e" * 64, + actor_reference="actor:99999999-9999-4999-8999-999999999999", + reviewer_reference="actor:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + fast_mlsirm_revision=REVIEWED_FAST_MLSIRM_REVISION, + requested_at=datetime(2026, 8, 21, 1, 0, tzinfo=timezone.utc), + ) + + assert "prove requester and reviewer resolve to distinct authoritative actor identities" in handoff.next_action diff --git a/packages/validity-analysis/tests/test_numeric_overflow_contract.py b/packages/validity-analysis/tests/test_numeric_overflow_contract.py new file mode 100644 index 000000000..a4fb88115 --- /dev/null +++ b/packages/validity-analysis/tests/test_numeric_overflow_contract.py @@ -0,0 +1,18 @@ +"""Regression for malformed worker numerics that overflow float conversion.""" + +import pytest + +from orgmetra_validity_analysis import ConvergenceDiagnostics + + +def test_oversized_worker_numeric_is_rejected_as_value_error() -> None: + """Normalize float-conversion overflow to the package's ValueError contract.""" + oversized_integer = 10**10000 + + with pytest.raises(ValueError, match="finite number"): + ConvergenceDiagnostics( + converged=True, + iterations=1, + objective_value=oversized_integer, + maximum_gradient=0.1, + ) diff --git a/packages/validity-analysis/tests/test_quality_workflow_trigger.py b/packages/validity-analysis/tests/test_quality_workflow_trigger.py new file mode 100644 index 000000000..eaf759550 --- /dev/null +++ b/packages/validity-analysis/tests/test_quality_workflow_trigger.py @@ -0,0 +1,16 @@ +"""Regression tests for validity-analysis coverage by consolidated Foundation CI.""" + +from pathlib import Path + + +_WORKFLOW_PATH = Path(".github/workflows/foundation-ci.yml") + + +def test_foundation_ci_retriggers_without_path_filter() -> None: + """Keep shared repository changes inside the consolidated validity gate surface.""" + workflow = _WORKFLOW_PATH.read_text(encoding="utf-8") + + assert "pull_request:" in workflow + assert " - develop" in workflow + assert "\n paths:" not in workflow + assert "\n paths-ignore:" not in workflow diff --git a/packages/validity-analysis/tests/test_result.py b/packages/validity-analysis/tests/test_result.py new file mode 100644 index 000000000..ad00b95f2 --- /dev/null +++ b/packages/validity-analysis/tests/test_result.py @@ -0,0 +1,211 @@ +"""Regression tests for the bounded numerical result contract.""" + +from dataclasses import asdict, replace +from datetime import datetime, timezone +import json + +import pytest + +from orgmetra_validity_analysis import ( + ConvergenceDiagnostics, + MissingnessSummary, + REVIEWED_FAST_MLSIRM_REVISION, + ValidationAnalysisResult, +) + +TENANT = "10000000-0000-7000-8000-000000000001" +RESULT = "validation_analysis_result:11111111-1111-4111-8111-111111111111" +HANDOFF_DIGEST = "a" * 64 +PROVENANCE_DIGEST = "b" * 64 +COMPLETED_AT = datetime(2026, 8, 21, 7, 10, 11, 123456, tzinfo=timezone.utc) + + +def missingness() -> MissingnessSummary: + """Return one realistic aggregate-only missingness summary.""" + return MissingnessSummary( + total_observations=12, + complete_observations=10, + missing_predictor_observations=1, + missing_criterion_observations=1, + ) + + +def convergence(*, converged: bool = True) -> ConvergenceDiagnostics: + """Return one converged or explicitly nonconverged diagnostic record.""" + return ConvergenceDiagnostics( + converged=converged, + iterations=42, + objective_value=-12.5, + maximum_gradient=0.0001, + failure_code=None if converged else "maximum_iterations", + ) + + +def result(**overrides: object) -> ValidationAnalysisResult: + """Build one valid result envelope and apply targeted test overrides.""" + values: dict[str, object] = { + "tenant_record_id": TENANT, + "result_reference": RESULT, + "handoff_digest": HANDOFF_DIGEST, + "provenance_digest": PROVENANCE_DIGEST, + "fast_mlsirm_revision": REVIEWED_FAST_MLSIRM_REVISION, + "model_code": "mlsirm_criterion_related", + "backend": "rust_cpu", + "precision": "f64", + "effect_estimate": 0.42, + "uncertainty_lower": 0.10, + "uncertainty_upper": 0.70, + "sample_size": 12, + "missingness_summary": missingness(), + "convergence_diagnostics": convergence(), + "completed_at": COMPLETED_AT, + } + values.update(overrides) + return ValidationAnalysisResult(**values) + + +def test_aggregate_evidence_is_deterministic_and_redacted() -> None: + """Serialize only aggregate evidence and preserve exact replay bytes.""" + candidate = result() + payload = json.loads(candidate.canonical_json()) + + assert payload["tenant_record_id"] == TENANT + assert payload["backend"] == "rust_cpu" + assert payload["precision"] == "f64" + assert payload["execution_state"] == "completed" + assert payload["result_authority"] == "scientific_evidence_only" + assert payload["missingness_summary"]["total_observations"] == 12 + assert payload["convergence_diagnostics"]["converged"] is True + assert "person_record" not in candidate.canonical_json() + assert repr(candidate) == "ValidationAnalysisResult()" + assert len(candidate.sha256_digest()) == 64 + assert candidate.canonical_json() == result().canonical_json() + + +def test_gpu_and_nonconverged_result_are_explicitly_typed() -> None: + """Record GPU provenance and a typed nonconvergence state without promotion.""" + candidate = result( + backend="rust_gpu", + precision="f32", + convergence_diagnostics=convergence(converged=False), + ) + payload = json.loads(candidate.canonical_json()) + assert payload["backend"] == "rust_gpu" + assert payload["precision"] == "f32" + assert payload["convergence_diagnostics"]["failure_code"] == "maximum_iterations" + + +@pytest.mark.parametrize( + "bad", + [ + {"total_observations": True}, + {"total_observations": -1}, + {"total_observations": 0}, + {"complete_observations": 13}, + {"missing_predictor_observations": 13}, + {"missing_criterion_observations": 13}, + {"complete_observations": 12, "missing_predictor_observations": 1}, + { + "complete_observations": 12, + "missing_predictor_observations": 0, + "missing_criterion_observations": 1, + }, + ], +) +def test_missingness_rejects_invalid_counts(bad: dict[str, object]) -> None: + """Reject booleans, negative counts, empty samples, and impossible totals.""" + values = asdict(missingness()) + values.update(bad) + with pytest.raises(ValueError): + MissingnessSummary(**values) + + +@pytest.mark.parametrize("bad", [True, 0, -1]) +def test_positive_integer_validation_rejects_nonpositive_values(bad: object) -> None: + """Exercise strict sample and iteration bounds.""" + with pytest.raises(ValueError, match="positive integer"): + ConvergenceDiagnostics(True, bad, -1.0, 0.1) + with pytest.raises(ValueError, match="positive integer"): + result(sample_size=bad) + + +@pytest.mark.parametrize("bad", [True, "0.1", float("nan"), float("inf")]) +def test_numeric_fields_reject_boolean_text_and_nonfinite_values(bad: object) -> None: + """Do not accept values that cannot be represented as finite scientific evidence.""" + with pytest.raises(ValueError, match="finite number"): + ConvergenceDiagnostics(True, 1, bad, 0.1) + with pytest.raises(ValueError, match="finite number"): + result(effect_estimate=bad) + + +def test_negative_gradient_and_invalid_convergence_states_fail_closed() -> None: + """Require explicit and internally consistent convergence diagnostics.""" + with pytest.raises(ValueError, match="non-negative"): + ConvergenceDiagnostics(True, 1, 1.0, -0.1) + with pytest.raises(ValueError, match="boolean"): + ConvergenceDiagnostics(1, 1, 1.0, 0.1) + with pytest.raises(ValueError, match="absent"): + ConvergenceDiagnostics(True, 1, 1.0, 0.1, "failed_fit") + with pytest.raises(ValueError, match="required"): + ConvergenceDiagnostics(False, 1, 1.0, 0.1) + with pytest.raises(ValueError, match="required"): + ConvergenceDiagnostics(False, 1, 1.0, 0.1, "") + + +@pytest.mark.parametrize( + "field,bad,match", + [ + ("backend", "numpy", "backend"), + ("precision", "float16", "precision"), + ("uncertainty_lower", 0.8, "uncertainty_lower"), + ("uncertainty_upper", 0.0, "uncertainty_lower"), + ("effect_estimate", 0.8, "effect_estimate"), + ("sample_size", 11, "sample_size"), + ("result_authority", "employment_decision", "result_authority"), + ("execution_state", "not_executed", "execution_state"), + ("contains_raw_person_level_values", True, "raw person-level"), + ("human_review_required", False, "human review"), + ("evidence_version", 2, "evidence_version"), + ], +) +def test_result_invariants_cannot_be_weakened(field: str, bad: object, match: str) -> None: + """Reject malformed intervals, lineage, or governance flags.""" + with pytest.raises(ValueError, match=match): + replace(result(), **{field: bad}) + + +def test_result_requires_canonical_timestamp_and_aggregate_types() -> None: + """Reject naive times, non-contract objects, and subclass method overrides.""" + + class LeakyMissingnessSummary(MissingnessSummary): + def to_dict(self) -> dict[str, object]: + return {**super().to_dict(), "person_record": "must-not-serialize"} + + class LeakyConvergenceDiagnostics(ConvergenceDiagnostics): + def to_dict(self) -> dict[str, object]: + return {**super().to_dict(), "employment_decision": "auto_reject"} + + with pytest.raises(ValueError, match="completed_at"): + result(completed_at=datetime(2026, 8, 21, 7, 10)) + with pytest.raises(ValueError, match="missingness_summary"): + result(missingness_summary=object()) + with pytest.raises(ValueError, match="convergence_diagnostics"): + result(convergence_diagnostics=object()) + with pytest.raises(ValueError, match="missingness_summary"): + result( + missingness_summary=LeakyMissingnessSummary( + total_observations=12, + complete_observations=10, + missing_predictor_observations=1, + missing_criterion_observations=1, + ) + ) + with pytest.raises(ValueError, match="convergence_diagnostics"): + result( + convergence_diagnostics=LeakyConvergenceDiagnostics( + converged=True, + iterations=42, + objective_value=-12.5, + maximum_gradient=0.0001, + ) + ) diff --git a/packages/validity-analysis/tests/test_result_correction_lineage.py b/packages/validity-analysis/tests/test_result_correction_lineage.py new file mode 100644 index 000000000..fb1f7034f --- /dev/null +++ b/packages/validity-analysis/tests/test_result_correction_lineage.py @@ -0,0 +1,117 @@ +"""Regression tests for append-only validation-result correction lineage.""" + +from datetime import datetime, timezone +import json + +import pytest + +from orgmetra_validity_analysis import ( + ConvergenceDiagnostics, + MissingnessSummary, + REVIEWED_FAST_MLSIRM_REVISION, + ValidationAnalysisResult, +) + +TENANT = "10000000-0000-7000-8000-000000000001" +RESULT = "validation_analysis_result:11111111-1111-4111-8111-111111111111" +SUCCESSOR_RESULT = "validation_analysis_result:22222222-2222-4222-8222-222222222222" +COMPLETED_AT = datetime(2026, 9, 17, 12, 30, tzinfo=timezone.utc) + + +def _result(**overrides: object) -> ValidationAnalysisResult: + """Build one valid aggregate-only result envelope.""" + values: dict[str, object] = { + "tenant_record_id": TENANT, + "result_reference": RESULT, + "handoff_digest": "a" * 64, + "provenance_digest": "b" * 64, + "fast_mlsirm_revision": REVIEWED_FAST_MLSIRM_REVISION, + "model_code": "mlsirm_criterion_related", + "backend": "rust_cpu", + "precision": "f64", + "effect_estimate": 0.42, + "uncertainty_lower": 0.10, + "uncertainty_upper": 0.70, + "sample_size": 12, + "missingness_summary": MissingnessSummary( + total_observations=12, + complete_observations=10, + missing_predictor_observations=1, + missing_criterion_observations=1, + ), + "convergence_diagnostics": ConvergenceDiagnostics( + converged=True, + iterations=42, + objective_value=-12.5, + maximum_gradient=0.0001, + ), + "completed_at": COMPLETED_AT, + } + values.update(overrides) + return ValidationAnalysisResult(**values) + + +def test_successor_result_binds_exact_predecessor_evidence() -> None: + """Serialize correction sequence and exact predecessor identity into result bytes.""" + predecessor = _result() + successor = _result( + result_reference=SUCCESSOR_RESULT, + correction_sequence=2, + supersedes_result_reference=predecessor.result_reference, + supersedes_result_digest=predecessor.sha256_digest(), + effect_estimate=0.48, + ) + + payload = json.loads(successor.canonical_json()) + assert payload["correction_sequence"] == 2 + assert payload["supersedes_result_reference"] == predecessor.result_reference + assert payload["supersedes_result_digest"] == predecessor.sha256_digest() + assert successor.sha256_digest() != predecessor.sha256_digest() + + +def test_initial_result_rejects_predecessor_coordinates() -> None: + """Do not let sequence-one evidence pretend to supersede another result.""" + with pytest.raises(ValueError, match="correction_sequence 1"): + _result( + supersedes_result_reference=SUCCESSOR_RESULT, + supersedes_result_digest="c" * 64, + ) + + +@pytest.mark.parametrize( + "overrides", + [ + {"correction_sequence": True}, + {"correction_sequence": 0}, + {"correction_sequence": 2}, + { + "correction_sequence": 2, + "supersedes_result_reference": SUCCESSOR_RESULT, + }, + { + "correction_sequence": 2, + "supersedes_result_digest": "c" * 64, + }, + { + "correction_sequence": 2, + "supersedes_result_reference": RESULT, + "supersedes_result_digest": "c" * 64, + }, + { + "correction_sequence": 2, + "supersedes_result_reference": "analysis_weight_receipt:22222222-2222-4222-8222-222222222222", + "supersedes_result_digest": "c" * 64, + }, + { + "correction_sequence": 2, + "supersedes_result_reference": SUCCESSOR_RESULT, + "supersedes_result_digest": "not-a-digest", + }, + ], +) +def test_malformed_result_correction_lineage_fails_closed( + overrides: dict[str, object], +) -> None: + """Reject ambiguous, cyclic-looking, or malformed correction coordinates.""" + with pytest.raises(ValueError): + _result(**overrides) diff --git a/packages/validity-analysis/tests/test_result_verification_status.py b/packages/validity-analysis/tests/test_result_verification_status.py new file mode 100644 index 000000000..3459c599d --- /dev/null +++ b/packages/validity-analysis/tests/test_result_verification_status.py @@ -0,0 +1,63 @@ +"""Regression coverage for non-authorizing scientific verification status.""" + +from datetime import datetime, timezone +import json + +from orgmetra_validity_analysis import ( + ConvergenceDiagnostics, + MissingnessSummary, + REVIEWED_FAST_MLSIRM_REVISION, + ValidationAnalysisResult, +) + + +def _result(*, converged: bool) -> ValidationAnalysisResult: + """Build one aggregate-only result with an explicit convergence outcome.""" + return ValidationAnalysisResult( + tenant_record_id="10000000-0000-7000-8000-000000000001", + result_reference="validation_analysis_result:77777777-7777-4777-8777-777777777777", + handoff_digest="a" * 64, + provenance_digest="b" * 64, + fast_mlsirm_revision=REVIEWED_FAST_MLSIRM_REVISION, + model_code="mlsirm_criterion_related", + backend="rust_cpu", + precision="f64", + effect_estimate=0.42, + uncertainty_lower=0.10, + uncertainty_upper=0.70, + sample_size=12, + missingness_summary=MissingnessSummary( + total_observations=12, + complete_observations=10, + missing_predictor_observations=1, + missing_criterion_observations=1, + ), + convergence_diagnostics=ConvergenceDiagnostics( + converged=converged, + iterations=42, + objective_value=-12.5, + maximum_gradient=0.0001, + failure_code=None if converged else "maximum_iterations", + ), + completed_at=datetime(2026, 9, 17, 5, 30, tzinfo=timezone.utc), + ) + + +def test_converged_leaf_result_remains_verification_pending() -> None: + candidate = _result(converged=True) + + assert candidate.verification_status == "verification_pending" + assert ( + json.loads(candidate.canonical_json())["verification_status"] + == "verification_pending" + ) + + +def test_nonconverged_result_is_explicitly_not_verifiable() -> None: + candidate = _result(converged=False) + + assert candidate.verification_status == "not_verifiable" + payload = json.loads(candidate.canonical_json()) + assert payload["verification_status"] == "not_verifiable" + assert payload["execution_state"] == "completed" + assert payload["convergence_diagnostics"]["failure_code"] == "maximum_iterations" diff --git a/packages/validity-analysis/tests/test_temporal_evidence_integrity.py b/packages/validity-analysis/tests/test_temporal_evidence_integrity.py new file mode 100644 index 000000000..0431eee55 --- /dev/null +++ b/packages/validity-analysis/tests/test_temporal_evidence_integrity.py @@ -0,0 +1,279 @@ +"""Regression coverage for selection-validity temporal evidence integrity.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone, tzinfo + +import pytest + +from orgmetra_validity_analysis import ( + ConvergenceDiagnostics, + MissingnessSummary, + REVIEWED_FAST_MLSIRM_REVISION, + ValidationAnalysisHandoff, + ValidationAnalysisResult, + build_validation_analysis_handoff, +) +from test_handoff import valid_kwargs +from test_result import result + + +class ForgedDateTime(datetime): + """Datetime subclass able to forge canonical validation evidence.""" + + def astimezone(self, tz=None): # type: ignore[no-untyped-def] + """Keep the hostile subclass alive across UTC normalization.""" + return self + + def isoformat(self, *args, **kwargs) -> str: # type: ignore[no-untyped-def] + """Return an instant different from the underlying evidence instant.""" + return "2099-12-31T23:59:59+00:00" + + +class ForgedReference(str): + """String subclass able to forge namespace and UUID parsing methods.""" + + def startswith(self, prefix, *args): # type: ignore[no-untyped-def] + """Pretend that an invalid namespace has the expected prefix.""" + return True + + def split(self, separator=None, maxsplit=-1): # type: ignore[no-untyped-def] + """Return a valid UUID suffix while retaining invalid source text.""" + return ["validation_analysis_handoff", "11111111-1111-4111-8111-111111111111"] + + +class ForgedFixedText(str): + """String subclass whose comparisons can forge fixed governance values.""" + + def __eq__(self, other): # type: ignore[no-untyped-def] + """Claim equality with any expected governance text.""" + return True + + def __ne__(self, other): # type: ignore[no-untyped-def] + """Claim inequality with no governance text.""" + return False + + def __hash__(self): + """Use a valid fixed-text hash for set membership forgery tests.""" + return hash("rust_cpu") + + +class MutableOffset(tzinfo): + """Timezone fixture whose offset can change after envelope construction.""" + + def __init__(self) -> None: + self.hours = 1 + + def utcoffset(self, dt: datetime | None) -> timedelta: + """Return the currently configured offset.""" + return timedelta(hours=self.hours) + + def dst(self, dt: datetime | None) -> timedelta: + """Return no daylight-saving offset.""" + return timedelta(0) + + +class UnknownOffset(tzinfo): + """Timezone fixture whose UTC offset cannot be resolved.""" + + def utcoffset(self, dt: datetime | None) -> None: + """Return no offset to exercise fail-closed validation.""" + return None + + def dst(self, dt: datetime | None) -> None: + """Return no daylight-saving offset.""" + return None + + +class ExplodingOffset(tzinfo): + """Timezone fixture whose provider raises during offset resolution.""" + + def utcoffset(self, dt: datetime | None) -> timedelta: + """Raise an untrusted provider error.""" + raise RuntimeError("offset provider failed") + + def dst(self, dt: datetime | None) -> timedelta: + """Return no daylight-saving offset when queried separately.""" + return timedelta(0) + + +class MutableReal(float): + """Numeric subclass whose float conversion changes after construction.""" + + def __new__(cls, value: float): + """Create a float-backed value with a separately mutable conversion.""" + instance = super().__new__(cls, value) + instance.current = value + return instance + + def __float__(self) -> float: + """Expose the mutable conversion used by unsafe canonicalization.""" + return self.current + + +def test_handoff_rejects_datetime_subclass_that_can_forge_requested_at() -> None: + """Handoff canonical evidence must not invoke caller-overridable datetime methods.""" + with pytest.raises(ValueError, match="requested_at"): + build_validation_analysis_handoff( + tenant_record_id="10000000-0000-7000-8000-000000000001", + handoff_reference="validation_analysis_handoff:11111111-1111-4111-8111-111111111111", + validation_study_reference="validation_study:22222222-2222-4222-8222-222222222222", + job_profile_reference="job_profile:33333333-3333-4333-8333-333333333333", + predictor_snapshot_reference="predictor_snapshot:44444444-4444-4444-8444-444444444444", + predictor_snapshot_digest="a" * 64, + criterion_snapshot_reference="criterion_snapshot:55555555-5555-4555-8555-555555555555", + criterion_snapshot_digest="b" * 64, + population_snapshot_reference="study_population_snapshot:66666666-6666-4666-8666-666666666666", + population_snapshot_digest="c" * 64, + decision_policy_reference="decision_policy:77777777-7777-4777-8777-777777777777", + decision_policy_digest="d" * 64, + analysis_plan_reference="validation_analysis_plan:88888888-8888-4888-8888-888888888888", + analysis_plan_digest="e" * 64, + actor_reference="actor:99999999-9999-4999-8999-999999999999", + reviewer_reference="actor:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + fast_mlsirm_revision=REVIEWED_FAST_MLSIRM_REVISION, + requested_at=ForgedDateTime(2026, 8, 21, 4, 45, tzinfo=timezone.utc), + ) + + +def test_result_rejects_datetime_subclass_that_can_forge_completed_at() -> None: + """Result canonical evidence must not invoke caller-overridable datetime methods.""" + with pytest.raises(ValueError, match="completed_at"): + ValidationAnalysisResult( + tenant_record_id="10000000-0000-7000-8000-000000000001", + result_reference="validation_analysis_result:11111111-1111-4111-8111-111111111111", + handoff_digest="a" * 64, + provenance_digest="b" * 64, + fast_mlsirm_revision=REVIEWED_FAST_MLSIRM_REVISION, + model_code="mlsirm_criterion_related", + backend="rust_cpu", + precision="f64", + effect_estimate=0.42, + uncertainty_lower=0.10, + uncertainty_upper=0.70, + sample_size=12, + missingness_summary=MissingnessSummary( + total_observations=12, + complete_observations=10, + missing_predictor_observations=1, + missing_criterion_observations=1, + ), + convergence_diagnostics=ConvergenceDiagnostics( + converged=True, + iterations=42, + objective_value=-12.5, + maximum_gradient=0.0001, + ), + completed_at=ForgedDateTime(2026, 8, 21, 4, 45, tzinfo=timezone.utc), + ) + + +def test_handoff_and_result_detach_mutable_timezone_before_digesting() -> None: + """Freeze one UTC instant so later timezone mutation cannot rewrite evidence.""" + handoff_zone = MutableOffset() + handoff_values = valid_kwargs() + handoff_values["requested_at"] = datetime(2026, 8, 21, 4, 45, tzinfo=handoff_zone) + handoff = build_validation_analysis_handoff(**handoff_values) + handoff_before = handoff.canonical_json(), handoff.sha256_digest() + handoff_zone.hours = 2 + assert (handoff.canonical_json(), handoff.sha256_digest()) == handoff_before + + result_zone = MutableOffset() + candidate = result(completed_at=datetime(2026, 8, 21, 4, 45, tzinfo=result_zone)) + result_before = candidate.canonical_json(), candidate.sha256_digest() + result_zone.hours = 2 + assert (candidate.canonical_json(), candidate.sha256_digest()) == result_before + + +@pytest.mark.parametrize( + "timestamp", + [ + datetime.min.replace(tzinfo=timezone(timedelta(hours=1))), + datetime.max.replace(tzinfo=timezone(-timedelta(hours=1))), + datetime(2026, 8, 21, 4, 45, tzinfo=UnknownOffset()), + datetime(2026, 8, 21, 4, 45, tzinfo=ExplodingOffset()), + ], +) +def test_handoff_rejects_unrepresentable_or_untrusted_timestamp(timestamp: datetime) -> None: + """Normalize timezone-provider failures and UTC arithmetic overflow at the boundary.""" + values = valid_kwargs() + values["requested_at"] = timestamp + with pytest.raises(ValueError, match="requested_at"): + build_validation_analysis_handoff(**values) + + +@pytest.mark.parametrize( + "timestamp", + [ + datetime.min.replace(tzinfo=timezone(timedelta(hours=1))), + datetime.max.replace(tzinfo=timezone(-timedelta(hours=1))), + datetime(2026, 8, 21, 4, 45, tzinfo=UnknownOffset()), + datetime(2026, 8, 21, 4, 45, tzinfo=ExplodingOffset()), + ], +) +def test_result_rejects_unrepresentable_or_untrusted_timestamp(timestamp: datetime) -> None: + """Apply the same fail-closed timestamp contract to completed result evidence.""" + with pytest.raises(ValueError, match="completed_at"): + result(completed_at=timestamp) + + +@pytest.mark.parametrize( + "timestamp", + [ + ForgedDateTime(2026, 8, 21, 4, 45, tzinfo=timezone.utc), + datetime(2026, 8, 21, 4, 45, tzinfo=timezone(timedelta(hours=1))), + ], +) +def test_canonicalization_rejects_low_level_timestamp_reinjection(timestamp: datetime) -> None: + """Keep canonicalization fail-closed even if an object is corrupted after construction.""" + handoff = build_validation_analysis_handoff(**valid_kwargs()) + object.__setattr__(handoff, "requested_at", timestamp) + with pytest.raises(ValueError, match="requested_at"): + handoff.canonical_json() + + candidate = result() + object.__setattr__(candidate, "completed_at", timestamp) + with pytest.raises(ValueError, match="completed_at"): + candidate.canonical_json() + + +def test_handoff_rejects_runtime_text_subclasses_before_serialization() -> None: + """Reject text subclasses that can forge reference, digest, code, or fixed-value checks.""" + for field, value in ( + ("tenant_record_id", ForgedFixedText("10000000-0000-7000-8000-000000000001")), + ("handoff_reference", ForgedReference("wrong_namespace:invalid")), + ("predictor_snapshot_digest", ForgedFixedText("a" * 64)), + ("purpose_code", ForgedFixedText("selection_validity_analysis")), + ("fast_mlsirm_revision", ForgedFixedText(REVIEWED_FAST_MLSIRM_REVISION)), + ("validation_strategy", ForgedFixedText("criterion_related")), + ("next_action", ForgedFixedText("governed")), + ): + values = valid_kwargs() + values[field] = value + with pytest.raises(ValueError, match=field): + ValidationAnalysisHandoff(**values) + + +def test_result_snapshots_numeric_values_before_canonicalization() -> None: + """Detach mutable numeric subclasses before recording scientific evidence bytes.""" + objective = MutableReal(-12.5) + gradient = MutableReal(0.0001) + diagnostics = ConvergenceDiagnostics( + converged=True, + iterations=42, + objective_value=objective, + maximum_gradient=gradient, + ) + estimate = MutableReal(0.42) + candidate = result(effect_estimate=estimate, convergence_diagnostics=diagnostics) + before = candidate.canonical_json(), candidate.sha256_digest() + objective.current = -1.0 + gradient.current = 0.5 + estimate.current = 0.69 + assert (candidate.canonical_json(), candidate.sha256_digest()) == before + + +def test_result_rejects_forged_backend_text() -> None: + """Do not allow a string subclass to forge an allowed backend membership check.""" + with pytest.raises(ValueError, match="backend"): + result(backend=ForgedFixedText("numpy")) diff --git a/packages/validity-analysis/tests/test_trimming_bounding_receipt.py b/packages/validity-analysis/tests/test_trimming_bounding_receipt.py new file mode 100644 index 000000000..9fa981de6 --- /dev/null +++ b/packages/validity-analysis/tests/test_trimming_bounding_receipt.py @@ -0,0 +1,95 @@ +"""RED contracts for reproducible trimming and bounding weight adjustments.""" + +from datetime import datetime, timezone + +import pytest + +from orgmetra_validity_analysis import AnalysisWeightAdjustment, TrimmingBoundingAdjustmentReceipt + +TENANT = "10000000-0000-7000-8000-000000000001" +RECEIPT = "trimming_bounding_adjustment_receipt:11111111-1111-4111-8111-111111111111" +RULE = "weight_trimming_rule:22222222-2222-4222-8222-222222222222" +DIGEST_A = "a" * 64 +DIGEST_B = "b" * 64 +DIGEST_C = "c" * 64 +DIGEST_D = "d" * 64 + + +def trimming_receipt(**overrides: object) -> TrimmingBoundingAdjustmentReceipt: + """Return one value-minimized trimming/bounding provenance receipt.""" + values: dict[str, object] = { + "tenant_record_id": TENANT, + "receipt_reference": RECEIPT, + "rule_reference": RULE, + "rule_version": 1, + "rule_configuration_digest": DIGEST_A, + "affected_case_occurrence_set_digest": DIGEST_B, + "affected_case_count": 3, + "input_weight_artifact_digest": DIGEST_C, + "output_weight_artifact_digest": DIGEST_D, + "constructed_at": datetime(2026, 9, 17, 5, 40, tzinfo=timezone.utc), + } + values.update(overrides) + return TrimmingBoundingAdjustmentReceipt(**values) + + +def test_trimming_receipt_is_deterministic_value_minimized_and_redacted() -> None: + """Preserve threshold/rule and affected-case provenance without row weights.""" + candidate = trimming_receipt() + assert candidate.sha256_digest() == trimming_receipt().sha256_digest() + assert f'"rule_configuration_digest":"{DIGEST_A}"' in candidate.canonical_json() + assert f'"affected_case_occurrence_set_digest":"{DIGEST_B}"' in candidate.canonical_json() + assert "weight_value" not in candidate.canonical_json() + assert repr(candidate) == "TrimmingBoundingAdjustmentReceipt()" + + +def test_trimming_adjustment_requires_typed_receipt_kind() -> None: + """Do not let trimming hide behind a generic opaque adjustment receipt.""" + with pytest.raises(ValueError, match="trimming_bounding_adjustment_receipt"): + AnalysisWeightAdjustment( + sequence_number=1, + adjustment_code="weight_trimming_adjustment", + method_reference="weight_method:33333333-3333-4333-8333-333333333333", + method_version=1, + input_weight_artifact_digest=DIGEST_C, + output_weight_artifact_digest=DIGEST_D, + configuration_digest=DIGEST_A, + evidence_receipt_digest=DIGEST_B, + evidence_kind="generic_adjustment_receipt", + ) + + +def test_bounding_adjustment_accepts_typed_receipt_kind() -> None: + """Allow a bound transform only when its evidence family is explicit.""" + adjustment = AnalysisWeightAdjustment( + sequence_number=1, + adjustment_code="weight_bounding_adjustment", + method_reference="weight_method:33333333-3333-4333-8333-333333333333", + method_version=1, + input_weight_artifact_digest=DIGEST_C, + output_weight_artifact_digest=DIGEST_D, + configuration_digest=DIGEST_A, + evidence_receipt_digest=trimming_receipt().sha256_digest(), + evidence_kind="trimming_bounding_adjustment_receipt", + ) + assert adjustment.evidence_kind == "trimming_bounding_adjustment_receipt" + + +def test_receipt_rejects_missing_or_nonreproducible_rule_evidence() -> None: + """Fail closed when the rule version/configuration or affected cases are not reproducible.""" + with pytest.raises(ValueError, match="rule_version"): + trimming_receipt(rule_version=0) + with pytest.raises(ValueError, match="rule_configuration_digest"): + trimming_receipt(rule_configuration_digest="floating") + with pytest.raises(ValueError, match="affected_case_occurrence_set_digest"): + trimming_receipt(affected_case_occurrence_set_digest="missing") + with pytest.raises(ValueError, match="affected_case_count"): + trimming_receipt(affected_case_count=0) + + +def test_receipt_rejects_noop_or_extensible_schema() -> None: + """A declared transform must change artifact identity under the reviewed schema.""" + with pytest.raises(ValueError, match="output_weight_artifact_digest"): + trimming_receipt(output_weight_artifact_digest=DIGEST_C) + with pytest.raises(ValueError, match="evidence_version"): + trimming_receipt(evidence_version=2) diff --git a/packages/validity-analysis/tests/test_weight_adjustment_semantics.py b/packages/validity-analysis/tests/test_weight_adjustment_semantics.py new file mode 100644 index 000000000..fd2f2d2a3 --- /dev/null +++ b/packages/validity-analysis/tests/test_weight_adjustment_semantics.py @@ -0,0 +1,272 @@ +"""Scientific contracts for nonresponse and calibration weight adjustments.""" + +from datetime import datetime, timedelta, timezone + +import pytest + +from orgmetra_validity_analysis import ( + AnalysisWeightAdjustment, + CalibrationAdjustmentReceipt, + NonresponseAdjustmentReceipt, +) + +TENANT = "10000000-0000-7000-8000-000000000001" +NOW = datetime(2026, 9, 17, 5, 0, tzinfo=timezone.utc) +DIGEST_A = "a" * 64 +DIGEST_B = "b" * 64 +DIGEST_C = "c" * 64 +DIGEST_D = "d" * 64 +DIGEST_E = "e" * 64 +DIGEST_F = "f" * 64 +DIGEST_1 = "1" * 64 +DIGEST_2 = "2" * 64 +DIGEST_3 = "3" * 64 +DIGEST_4 = "4" * 64 +DIGEST_5 = "5" * 64 + + +def nonresponse_receipt(**overrides: object) -> NonresponseAdjustmentReceipt: + """Return one disposition-aware nonresponse adjustment receipt.""" + values: dict[str, object] = { + "tenant_record_id": TENANT, + "receipt_reference": "nonresponse_adjustment_receipt:11111111-1111-4111-8111-111111111111", + "response_disposition_receipt_reference": "response_disposition_receipt:11111111-1111-4111-8111-111111111112", + "response_disposition_receipt_version": 3, + "response_disposition_receipt_digest": DIGEST_A, + "adjustment_population_digest": DIGEST_B, + "method_reference": "weight_method:22222222-2222-4222-8222-222222222222", + "method_version": 1, + "configuration_digest": DIGEST_C, + "ineligible_treatment_code": "exclude_as_ineligible", + "unknown_treatment_code": "retain_in_unknown_class", + "unavailable_treatment_code": "retain_in_unavailable_class", + "input_weight_artifact_digest": DIGEST_D, + "output_weight_artifact_digest": DIGEST_E, + "constructed_at": NOW, + } + values.update(overrides) + return NonresponseAdjustmentReceipt(**values) + + +def calibration_receipt(**overrides: object) -> CalibrationAdjustmentReceipt: + """Return one owner-corroboratable converged calibration receipt.""" + values: dict[str, object] = { + "tenant_record_id": TENANT, + "receipt_reference": "calibration_adjustment_receipt:33333333-3333-4333-8333-333333333333", + "target_population_digest": DIGEST_A, + "analysis_window_reference": "analysis_window:44444444-4444-4444-8444-444444444444", + "auxiliary_authority_reference": "scientific_auxiliary_authority:55555555-5555-4555-8555-555555555550", + "auxiliary_projection_reference": "calibration_auxiliary_projection:55555555-5555-4555-8555-555555555555", + "auxiliary_projection_version": 4, + "auxiliary_projection_digest": DIGEST_B, + "auxiliary_purpose_reference": "scientific_data_use_purpose:55555555-5555-4555-8555-555555555556", + "auxiliary_purpose_digest": DIGEST_3, + "auxiliary_owner_contract_reference": "released_owner_contract:55555555-5555-4555-8555-555555555557", + "auxiliary_owner_contract_version": 1, + "auxiliary_owner_contract_digest": DIGEST_1, + "auxiliary_authorization_receipt_reference": "scientific_data_authorization:55555555-5555-4555-8555-555555555558", + "auxiliary_authorization_receipt_digest": DIGEST_4, + "auxiliary_scientific_use_receipt_reference": "scientific_use_receipt:55555555-5555-4555-8555-555555555559", + "auxiliary_scientific_use_receipt_digest": DIGEST_2, + "auxiliary_scientific_use_at": NOW, + "benchmark_receipt_reference": "calibration_benchmark_receipt:66666666-6666-4666-8666-666666666666", + "benchmark_receipt_version": 2, + "benchmark_receipt_digest": DIGEST_C, + "benchmark_owner_contract_reference": "released_owner_contract:66666666-6666-4666-8666-666666666667", + "benchmark_owner_contract_version": 3, + "benchmark_owner_contract_digest": DIGEST_5, + "benchmark_reference_at": NOW - timedelta(days=1), + "algorithm_reference": "calibration_algorithm:77777777-7777-4777-8777-777777777777", + "algorithm_version": 1, + "constraints_digest": DIGEST_F, + "applied_constraints_digest": DIGEST_F, + "termination_code": "converged", + "input_weight_artifact_digest": DIGEST_D, + "output_weight_artifact_digest": DIGEST_E, + "constructed_at": NOW, + } + values.update(overrides) + return CalibrationAdjustmentReceipt(**values) + + +def adjustment(*, code: str, evidence_kind: str) -> AnalysisWeightAdjustment: + """Return one adjustment linked to typed scientific evidence.""" + return AnalysisWeightAdjustment( + sequence_number=1, + adjustment_code=code, + method_reference="weight_method:88888888-8888-4888-8888-888888888888", + method_version=1, + input_weight_artifact_digest=DIGEST_D, + output_weight_artifact_digest=DIGEST_E, + configuration_digest=DIGEST_F, + evidence_receipt_digest=DIGEST_1, + evidence_kind=evidence_kind, + ) + + +def test_nonresponse_receipt_is_value_minimized_and_disposition_aware() -> None: + """Preserve exact versioned disposition input without copying source attributes.""" + candidate = nonresponse_receipt() + canonical = candidate.canonical_json() + assert candidate.sha256_digest() == nonresponse_receipt().sha256_digest() + assert ( + '"response_disposition_receipt_reference":' + '"response_disposition_receipt:11111111-1111-4111-8111-111111111112"' + in canonical + ) + assert '"response_disposition_receipt_version":3' in canonical + assert f'"response_disposition_receipt_digest":"{DIGEST_A}"' in canonical + assert '"unknown_treatment_code":"retain_in_unknown_class"' in canonical + assert "person_record" not in canonical + assert "protected_attribute" not in canonical + assert repr(candidate) == "NonresponseAdjustmentReceipt()" + + with pytest.raises(ValueError, match="response_disposition_receipt_reference"): + nonresponse_receipt(response_disposition_receipt_reference="dispositions-v3") + with pytest.raises(ValueError, match="response_disposition_receipt_version"): + nonresponse_receipt(response_disposition_receipt_version=0) + with pytest.raises(ValueError, match="response_disposition_receipt_version"): + nonresponse_receipt(response_disposition_receipt_version=True) + with pytest.raises(ValueError, match="unknown_treatment_code"): + nonresponse_receipt(unknown_treatment_code="") + with pytest.raises(ValueError, match="output_weight_artifact_digest"): + nonresponse_receipt(output_weight_artifact_digest=DIGEST_D) + with pytest.raises(ValueError, match="evidence_version"): + nonresponse_receipt(evidence_version=2) + with pytest.raises(ValueError, match="evidence_version"): + nonresponse_receipt(evidence_version=True) + + +def test_calibration_receipt_binds_owner_authority_use_and_termination_state() -> None: + """Keep owner-corroboratable authority, benchmark, and generating constraints explicit.""" + candidate = calibration_receipt() + canonical = candidate.canonical_json() + assert candidate.sha256_digest() == calibration_receipt().sha256_digest() + assert '"benchmark_receipt_version":2' in canonical + assert f'"benchmark_receipt_digest":"{DIGEST_C}"' in canonical + assert f'"benchmark_owner_contract_digest":"{DIGEST_5}"' in canonical + assert '"benchmark_reference_at":"2026-09-16T05:00:00Z"' in canonical + assert '"auxiliary_projection_version":4' in canonical + assert f'"auxiliary_purpose_digest":"{DIGEST_3}"' in canonical + assert f'"auxiliary_owner_contract_digest":"{DIGEST_1}"' in canonical + assert f'"auxiliary_authorization_receipt_digest":"{DIGEST_4}"' in canonical + assert f'"auxiliary_scientific_use_receipt_digest":"{DIGEST_2}"' in canonical + assert '"auxiliary_scientific_use_at":"2026-09-17T05:00:00Z"' in canonical + assert f'"constraints_digest":"{DIGEST_F}"' in canonical + assert f'"applied_constraints_digest":"{DIGEST_F}"' in canonical + assert '"termination_code":"converged"' in canonical + assert "protected_attribute" not in canonical + assert repr(candidate) == "CalibrationAdjustmentReceipt()" + + fallback_reference = "calibration_fallback_rule:99999999-9999-4999-8999-999999999999" + fallback_algorithm_reference = "calibration_algorithm:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" + fallback = calibration_receipt( + termination_code="fallback_applied", + applied_constraints_digest=DIGEST_1, + fallback_reason_code="constraint_relaxation", + fallback_rule_reference=fallback_reference, + fallback_rule_digest=DIGEST_2, + fallback_algorithm_reference=fallback_algorithm_reference, + fallback_algorithm_version=2, + fallback_configuration_digest=DIGEST_4, + ) + fallback_json = fallback.canonical_json() + assert f'"fallback_rule_digest":"{DIGEST_2}"' in fallback_json + assert '"fallback_reason_code":"constraint_relaxation"' in fallback_json + assert f'"fallback_algorithm_reference":"{fallback_algorithm_reference}"' in fallback_json + assert '"fallback_algorithm_version":2' in fallback_json + assert f'"fallback_configuration_digest":"{DIGEST_4}"' in fallback_json + assert f'"applied_constraints_digest":"{DIGEST_1}"' in fallback_json + + with pytest.raises(ValueError, match="auxiliary_authority_reference"): + calibration_receipt(auxiliary_authority_reference="authority-v1") + with pytest.raises(ValueError, match="auxiliary_projection_version"): + calibration_receipt(auxiliary_projection_version=0) + with pytest.raises(ValueError, match="auxiliary_projection_version"): + calibration_receipt(auxiliary_projection_version=True) + with pytest.raises(ValueError, match="auxiliary_purpose_digest"): + calibration_receipt(auxiliary_purpose_digest="purpose-v1") + with pytest.raises(ValueError, match="auxiliary_owner_contract_version"): + calibration_receipt(auxiliary_owner_contract_version=0) + with pytest.raises(ValueError, match="auxiliary_owner_contract_digest"): + calibration_receipt(auxiliary_owner_contract_digest="owner-v1") + with pytest.raises(ValueError, match="auxiliary_authorization_receipt_reference"): + calibration_receipt(auxiliary_authorization_receipt_reference="authorization-v1") + with pytest.raises(ValueError, match="auxiliary_authorization_receipt_digest"): + calibration_receipt(auxiliary_authorization_receipt_digest="authorization-v1") + with pytest.raises(ValueError, match="auxiliary_scientific_use_receipt_reference"): + calibration_receipt(auxiliary_scientific_use_receipt_reference="use-v1") + with pytest.raises(ValueError, match="auxiliary_scientific_use_receipt_digest"): + calibration_receipt(auxiliary_scientific_use_receipt_digest="use-v1") + with pytest.raises(ValueError, match="auxiliary_scientific_use_at"): + calibration_receipt(auxiliary_scientific_use_at="2026-09-17T05:00:00Z") + with pytest.raises(ValueError, match="cannot be later"): + calibration_receipt(auxiliary_scientific_use_at=NOW + timedelta(seconds=1)) + with pytest.raises(ValueError, match="benchmark_receipt_version"): + calibration_receipt(benchmark_receipt_version=0) + with pytest.raises(ValueError, match="benchmark_receipt_version"): + calibration_receipt(benchmark_receipt_version=True) + with pytest.raises(ValueError, match="benchmark_owner_contract_reference"): + calibration_receipt(benchmark_owner_contract_reference="benchmark-owner-v3") + with pytest.raises(ValueError, match="benchmark_owner_contract_version"): + calibration_receipt(benchmark_owner_contract_version=0) + with pytest.raises(ValueError, match="benchmark_owner_contract_digest"): + calibration_receipt(benchmark_owner_contract_digest="owner-v3") + with pytest.raises(ValueError, match="benchmark_reference_at"): + calibration_receipt(benchmark_reference_at="2026-09-16T05:00:00Z") + with pytest.raises(ValueError, match="benchmark_reference_at cannot be later"): + calibration_receipt(benchmark_reference_at=NOW + timedelta(seconds=1)) + with pytest.raises(ValueError, match="termination_code"): + calibration_receipt(termination_code="failed") + with pytest.raises(ValueError, match="termination_code"): + calibration_receipt(termination_code=1) + with pytest.raises(ValueError, match="fallback"): + calibration_receipt(applied_constraints_digest=DIGEST_1) + with pytest.raises(ValueError, match="fallback"): + calibration_receipt(termination_code="fallback_applied") + with pytest.raises(ValueError, match="fallback"): + calibration_receipt( + termination_code="fallback_applied", + fallback_rule_reference=fallback_reference, + ) + with pytest.raises(ValueError, match="fallback"): + calibration_receipt( + fallback_reason_code="primary_nonconvergence", + fallback_rule_reference=fallback_reference, + fallback_rule_digest=DIGEST_2, + fallback_algorithm_reference=fallback_algorithm_reference, + fallback_algorithm_version=2, + fallback_configuration_digest=DIGEST_4, + ) + with pytest.raises(ValueError, match="fallback"): + calibration_receipt(fallback_rule_digest=DIGEST_2) + with pytest.raises(ValueError, match="applied_constraints_digest"): + calibration_receipt(applied_constraints_digest="not-a-digest") + with pytest.raises(ValueError, match="output_weight_artifact_digest"): + calibration_receipt(output_weight_artifact_digest=DIGEST_D) + with pytest.raises(ValueError, match="evidence_version"): + calibration_receipt(evidence_version=2) + with pytest.raises(ValueError, match="evidence_version"): + calibration_receipt(evidence_version=True) + + +def test_specialized_adjustments_require_matching_evidence_kind() -> None: + """Do not let typed nonresponse or calibration semantics collapse into opaque digests.""" + nonresponse = adjustment( + code="nonresponse_adjustment", + evidence_kind="nonresponse_adjustment_receipt", + ) + calibration = adjustment( + code="calibration_adjustment", + evidence_kind="calibration_adjustment_receipt", + ) + assert nonresponse.evidence_kind == "nonresponse_adjustment_receipt" + assert calibration.evidence_kind == "calibration_adjustment_receipt" + + with pytest.raises(ValueError, match="nonresponse_adjustment_receipt"): + adjustment(code="nonresponse_adjustment", evidence_kind="generic_adjustment_receipt") + with pytest.raises(ValueError, match="calibration_adjustment_receipt"): + adjustment(code="raking_adjustment", evidence_kind="generic_adjustment_receipt") + + generic = adjustment(code="trimming_adjustment", evidence_kind="generic_adjustment_receipt") + assert generic.evidence_kind == "generic_adjustment_receipt" diff --git a/packages/validity-analysis/tests/test_weight_eligibility_receipt.py b/packages/validity-analysis/tests/test_weight_eligibility_receipt.py new file mode 100644 index 000000000..4d1ba3f9f --- /dev/null +++ b/packages/validity-analysis/tests/test_weight_eligibility_receipt.py @@ -0,0 +1,70 @@ +"""Regression contracts for target-population and duration-safe weight eligibility.""" + +from datetime import datetime, timezone + +import pytest + +from orgmetra_validity_analysis import WeightEligibilityReceipt + +TENANT = "10000000-0000-7000-8000-000000000001" +RECEIPT = "weight_eligibility_receipt:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" +TARGET = "analysis_target_population:33333333-3333-4333-8333-333333333333" +DURATION = "analysis_reference_duration:77777777-7777-4777-8777-777777777777" +DIGEST_A = "a" * 64 +DIGEST_B = "b" * 64 +DIGEST_C = "c" * 64 +DIGEST_D = "d" * 64 + + +def eligibility(**overrides: object) -> WeightEligibilityReceipt: + """Return one exact longitudinal/cross-sectional weight eligibility receipt.""" + values: dict[str, object] = { + "tenant_record_id": TENANT, + "receipt_reference": RECEIPT, + "weight_scope_code": "cross_sectional", + "target_population_reference": TARGET, + "target_population_digest": DIGEST_A, + "reference_duration_reference": DURATION, + "reference_duration_digest": DIGEST_B, + "eligible_case_set_digest": DIGEST_C, + "weight_artifact_digest": DIGEST_D, + "constructed_at": datetime(2026, 9, 17, 7, 0, tzinfo=timezone.utc), + } + values.update(overrides) + return WeightEligibilityReceipt(**values) + + +def test_weight_eligibility_receipt_is_deterministic_and_value_minimized() -> None: + """Bind weight scope, target population, duration, and eligible cases without row values.""" + candidate = eligibility() + assert candidate.sha256_digest() == eligibility().sha256_digest() + assert '"weight_scope_code":"cross_sectional"' in candidate.canonical_json() + assert "weight_value" not in candidate.canonical_json() + assert repr(candidate) == "WeightEligibilityReceipt()" + + +@pytest.mark.parametrize("scope", [None, "monthly", "panel", "opaque", ""]) +def test_weight_scope_is_closed_to_cross_sectional_or_longitudinal(scope: object) -> None: + """Reject caller-defined labels and non-strings that hide weight-scope semantics.""" + with pytest.raises(ValueError, match="weight_scope_code"): + eligibility(weight_scope_code=scope) + + +def test_weight_eligibility_requires_versioned_population_duration_and_case_evidence() -> None: + """Reject mutable or opaque population, duration, case-set, and artifact identities.""" + with pytest.raises(ValueError, match="target_population_digest"): + eligibility(target_population_digest="target-v1") + with pytest.raises(ValueError, match="reference_duration_reference"): + eligibility(reference_duration_reference="duration-v1") + with pytest.raises(ValueError, match="reference_duration_digest"): + eligibility(reference_duration_digest="duration-v1") + with pytest.raises(ValueError, match="eligible_case_set_digest"): + eligibility(eligible_case_set_digest="cases-v1") + with pytest.raises(ValueError, match="weight_artifact_digest"): + eligibility(weight_artifact_digest="weight-v1") + + +def test_evidence_version_is_not_caller_extensible() -> None: + """Prevent ad hoc receipt schemas from bypassing reviewed weight eligibility semantics.""" + with pytest.raises(ValueError, match="evidence_version"): + eligibility(evidence_version=2) diff --git a/packages/validity-analysis/tests/test_weight_variance_compatibility_receipt.py b/packages/validity-analysis/tests/test_weight_variance_compatibility_receipt.py new file mode 100644 index 000000000..fae9e914a --- /dev/null +++ b/packages/validity-analysis/tests/test_weight_variance_compatibility_receipt.py @@ -0,0 +1,188 @@ +"""Regression contract for point-weight and variance-design compatibility evidence.""" + +from datetime import datetime, timezone + +import pytest + +from orgmetra_validity_analysis import ( + FinalAnalysisWeightReceipt, + WeightEligibilityReceipt, + WeightVarianceCompatibilityReceipt, +) + +TENANT = "10000000-0000-7000-8000-000000000001" +WEIGHT_RECEIPT = "analysis_weight_receipt:11111111-1111-4111-8111-111111111111" +COMPATIBILITY_RECEIPT = ( + "weight_variance_compatibility_receipt:22222222-2222-4222-8222-222222222222" +) +VARIANCE_RECEIPT = "variance_design_receipt:33333333-3333-4333-8333-333333333333" +ELIGIBILITY_RECEIPT = "weight_eligibility_receipt:44444444-4444-4444-8444-444444444444" +ESTIMAND = "validation_estimand:55555555-5555-4555-8555-555555555555" +TARGET = "analysis_target_population:66666666-6666-4666-8666-666666666666" +WINDOW = "analysis_window:77777777-7777-4777-8777-777777777777" +DURATION = "analysis_reference_duration:88888888-8888-4888-8888-888888888888" +DIGEST_A = "a" * 64 +DIGEST_B = "b" * 64 +DIGEST_C = "c" * 64 +DIGEST_D = "d" * 64 +DIGEST_E = "e" * 64 +DIGEST_F = "f" * 64 +DIGEST_1 = "1" * 64 +DIGEST_2 = "2" * 64 +DIGEST_3 = "3" * 64 +DIGEST_4 = "4" * 64 +DIGEST_5 = "5" * 64 + + +def weight_receipt(**overrides: object) -> FinalAnalysisWeightReceipt: + """Build one base-weight-only receipt for compatibility correlation tests.""" + eligibility = WeightEligibilityReceipt( + tenant_record_id=TENANT, + receipt_reference=ELIGIBILITY_RECEIPT, + weight_scope_code="cross_sectional", + target_population_reference=TARGET, + target_population_digest=DIGEST_B, + reference_duration_reference=DURATION, + reference_duration_digest=DIGEST_C, + eligible_case_set_digest=DIGEST_D, + weight_artifact_digest=DIGEST_5, + constructed_at=datetime(2026, 9, 17, 0, 30, tzinfo=timezone.utc), + ) + values: dict[str, object] = { + "tenant_record_id": TENANT, + "receipt_reference": WEIGHT_RECEIPT, + "estimand_reference": ESTIMAND, + "estimand_digest": DIGEST_A, + "estimand_scope_code": "cross_sectional", + "target_population_reference": TARGET, + "target_population_digest": DIGEST_B, + "analysis_unit_code": "worker_occurrence", + "analysis_window_reference": WINDOW, + "reference_duration_reference": DURATION, + "reference_duration_digest": DIGEST_C, + "eligible_case_set_digest": DIGEST_D, + "analytic_case_occurrence_set_digest": DIGEST_E, + "source_universe_receipt_digest": DIGEST_F, + "sampling_design_receipt_digest": DIGEST_1, + "base_weight_method_code": "inverse_inclusion_probability", + "base_weight_method_version": 1, + "base_weight_evidence_digest": DIGEST_2, + "base_weight_artifact_digest": DIGEST_5, + "adjustments": (), + "final_weight_artifact_digest": DIGEST_5, + "weight_eligibility": eligibility, + "analytic_case_count": 12, + "constructed_at": datetime(2026, 9, 17, 0, 30, tzinfo=timezone.utc), + } + values.update(overrides) + return FinalAnalysisWeightReceipt(**values) + + +def compatibility(**overrides: object) -> WeightVarianceCompatibilityReceipt: + """Build owner-correlatable compatibility evidence for one weighted analysis.""" + canonical_point_weight = weight_receipt() + point_weight = overrides.pop("analysis_weight_receipt", canonical_point_weight) + variance_point_weight = ( + point_weight + if type(point_weight) is FinalAnalysisWeightReceipt + else canonical_point_weight + ) + values: dict[str, object] = { + "tenant_record_id": TENANT, + "receipt_reference": COMPATIBILITY_RECEIPT, + "analysis_weight_receipt": point_weight, + "variance_design_receipt_reference": VARIANCE_RECEIPT, + "variance_design_receipt_version": 1, + "variance_design_receipt_digest": DIGEST_3, + "variance_analysis_weight_receipt_digest": variance_point_weight.sha256_digest(), + "variance_analytic_case_occurrence_set_digest": ( + variance_point_weight.analytic_case_occurrence_set_digest + ), + "variance_weight_eligibility_receipt_digest": ( + variance_point_weight.weight_eligibility.sha256_digest() + ), + "variance_weight_correction_sequence": variance_point_weight.correction_sequence, + "variance_final_weight_artifact_digest": ( + variance_point_weight.final_weight_artifact_digest + ), + "constructed_at": datetime(2026, 9, 17, 0, 31, tzinfo=timezone.utc), + } + values.update(overrides) + return WeightVarianceCompatibilityReceipt(**values) + + +def test_compatibility_receipt_binds_exact_point_and_variance_lineage() -> None: + """Require variance evidence to name the exact point-weight basis it accompanies.""" + candidate = compatibility() + payload = candidate.canonical_json() + assert candidate.sha256_digest() == compatibility().sha256_digest() + assert f'"variance_design_receipt_digest":"{DIGEST_3}"' in payload + assert f'"variance_final_weight_artifact_digest":"{DIGEST_5}"' in payload + assert repr(candidate) == "WeightVarianceCompatibilityReceipt()" + + +@pytest.mark.parametrize( + "field,value,match", + [ + ("variance_analysis_weight_receipt_digest", DIGEST_A, "analysis weight receipt"), + ("variance_analytic_case_occurrence_set_digest", DIGEST_A, "analytic case occurrence"), + ("variance_weight_eligibility_receipt_digest", DIGEST_A, "weight eligibility"), + ("variance_weight_correction_sequence", 2, "correction sequence"), + ("variance_final_weight_artifact_digest", DIGEST_A, "final weight artifact"), + ], +) +def test_compatibility_receipt_rejects_point_variance_lineage_mismatch( + field: str, value: object, match: str +) -> None: + """Fail closed when variance evidence was generated from a different weight lineage.""" + with pytest.raises(ValueError, match=match): + compatibility(**{field: value}) + + +def test_compatibility_receipt_rejects_foreign_point_weight_or_tenant() -> None: + """Do not admit an opaque object or a point-weight receipt from another tenant.""" + with pytest.raises(ValueError, match="FinalAnalysisWeightReceipt"): + compatibility(analysis_weight_receipt=object()) + + foreign = weight_receipt( + tenant_record_id="10000000-0000-7000-8000-000000000002", + weight_eligibility=WeightEligibilityReceipt( + tenant_record_id="10000000-0000-7000-8000-000000000002", + receipt_reference=ELIGIBILITY_RECEIPT, + weight_scope_code="cross_sectional", + target_population_reference=TARGET, + target_population_digest=DIGEST_B, + reference_duration_reference=DURATION, + reference_duration_digest=DIGEST_C, + eligible_case_set_digest=DIGEST_D, + weight_artifact_digest=DIGEST_5, + constructed_at=datetime(2026, 9, 17, 0, 30, tzinfo=timezone.utc), + ), + ) + with pytest.raises(ValueError, match="tenant_record_id"): + compatibility(analysis_weight_receipt=foreign) + + +def test_compatibility_receipt_rejects_point_receipt_as_variance_receipt() -> None: + """Point-weight evidence cannot masquerade as the distinct variance-design receipt.""" + point_weight = weight_receipt() + with pytest.raises(ValueError, match="distinct from the analysis weight receipt"): + compatibility( + analysis_weight_receipt=point_weight, + variance_design_receipt_digest=point_weight.sha256_digest(), + ) + + +def test_compatibility_receipt_rejects_time_reversal_and_schema_forgery() -> None: + """Compatibility evidence cannot predate its point weight or invent a schema version.""" + with pytest.raises(ValueError, match="cannot precede"): + compatibility(constructed_at=datetime(2026, 9, 17, 0, 29, tzinfo=timezone.utc)) + with pytest.raises(ValueError, match="evidence_version"): + compatibility(evidence_version=2) + + +@pytest.mark.parametrize("version", [0, True]) +def test_variance_design_receipt_version_is_strictly_positive(version: object) -> None: + """Reject unversioned or boolean variance owner evidence.""" + with pytest.raises(ValueError, match="variance_design_receipt_version"): + compatibility(variance_design_receipt_version=version) diff --git a/packages/validity-analysis/tests/test_workflow_trigger_contract.py b/packages/validity-analysis/tests/test_workflow_trigger_contract.py new file mode 100644 index 000000000..4d69ff802 --- /dev/null +++ b/packages/validity-analysis/tests/test_workflow_trigger_contract.py @@ -0,0 +1,18 @@ +"""Regression for validity-analysis execution in consolidated Foundation CI.""" + +from pathlib import Path + + +def test_foundation_ci_runs_validity_analysis_and_adr_changes() -> None: + """Require the consolidated gate to execute this package for every develop PR.""" + repository_root = Path(__file__).resolve().parents[3] + workflow = (repository_root / ".github" / "workflows" / "foundation-ci.yml").read_text( + encoding="utf-8" + ) + + assert "pull_request:" in workflow + assert " - develop" in workflow + assert "\n paths:" not in workflow + assert "\n paths-ignore:" not in workflow + assert "PYTHONPATH=packages/validity-analysis/src" in workflow + assert "packages/validity-analysis/tests" in workflow