From c28fe1bb5171e8620ca53e857e08b620432c3ae4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 15:46:15 -0700 Subject: [PATCH 001/158] test(validity): require governed analysis handoff --- packages/validity-analysis/pyproject.toml | 24 ++ .../validity-analysis/tests/test_handoff.py | 221 ++++++++++++++++++ 2 files changed, 245 insertions(+) create mode 100644 packages/validity-analysis/pyproject.toml create mode 100644 packages/validity-analysis/tests/test_handoff.py diff --git a/packages/validity-analysis/pyproject.toml b/packages/validity-analysis/pyproject.toml new file mode 100644 index 000000000..5547eabcb --- /dev/null +++ b/packages/validity-analysis/pyproject.toml @@ -0,0 +1,24 @@ +[build-system] +requires = ["setuptools>=69"] +build-backend = "setuptools.build_meta" + +[project] +name = "orgmetra-validity-analysis" +version = "0.1.0" +description = "Governed criterion-related selection-validity analysis handoff for Orgmetra." +requires-python = ">=3.12" + +[project.optional-dependencies] +test = ["pytest>=8.3", "pytest-cov>=5.0"] + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = [ + "--cov=orgmetra_validity_analysis", + "--cov-branch", + "--cov-report=term-missing", + "--cov-fail-under=100", +] diff --git a/packages/validity-analysis/tests/test_handoff.py b/packages/validity-analysis/tests/test_handoff.py new file mode 100644 index 000000000..64d1ab8c6 --- /dev/null +++ b/packages/validity-analysis/tests/test_handoff.py @@ -0,0 +1,221 @@ +"""Regression tests for governed selection-validity analysis handoffs.""" + +from dataclasses import replace +from datetime import datetime, timedelta, timezone +import json + +import pytest + +from orgmetra_validity_analysis import ( + REVIEWED_FAST_MLSIRM_REVISION, + ValidationAnalysisHandoff, + build_validation_analysis_handoff, +) + +TENANT = "10000000-0000-7000-8000-000000000001" +HANDOFF = "validation_analysis_handoff:11111111-1111-4111-8111-111111111111" +STUDY = "validation_study:22222222-2222-4222-8222-222222222222" +JOB = "job_profile:33333333-3333-4333-8333-333333333333" +PREDICTOR = "predictor_snapshot:44444444-4444-4444-8444-444444444444" +CRITERION = "criterion_snapshot:55555555-5555-4555-8555-555555555555" +POPULATION = "study_population_snapshot:66666666-6666-4666-8666-666666666666" +POLICY = "decision_policy:77777777-7777-4777-8777-777777777777" +PLAN = "validation_analysis_plan:88888888-8888-4888-8888-888888888888" +ACTOR = "actor:99999999-9999-4999-8999-999999999999" +REVIEWER = "actor:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" +DIGEST_A = "a" * 64 +DIGEST_B = "b" * 64 +DIGEST_C = "c" * 64 +DIGEST_D = "d" * 64 +DIGEST_E = "e" * 64 +REQUESTED_AT = datetime(2026, 8, 21, 7, 10, 11, 123456, tzinfo=timezone(timedelta(hours=9))) + + +def valid_kwargs(): + """Return one complete governed handoff input fixture.""" + return { + "tenant_record_id": TENANT, + "handoff_reference": HANDOFF, + "validation_study_reference": STUDY, + "job_profile_reference": JOB, + "predictor_snapshot_reference": PREDICTOR, + "predictor_snapshot_digest": DIGEST_A, + "criterion_snapshot_reference": CRITERION, + "criterion_snapshot_digest": DIGEST_B, + "population_snapshot_reference": POPULATION, + "population_snapshot_digest": DIGEST_C, + "decision_policy_reference": POLICY, + "decision_policy_digest": DIGEST_D, + "analysis_plan_reference": PLAN, + "analysis_plan_digest": DIGEST_E, + "actor_reference": ACTOR, + "reviewer_reference": REVIEWER, + "fast_mlsirm_revision": REVIEWED_FAST_MLSIRM_REVISION, + "requested_at": REQUESTED_AT, + } + + +def handoff(): + """Build the canonical valid handoff fixture.""" + return build_validation_analysis_handoff(**valid_kwargs()) + + +def test_handoff_is_value_minimized_deterministic_and_human_review_only(): + """Bind exact study evidence without exposing raw person-level observations.""" + candidate = handoff() + payload = json.loads(candidate.canonical_json()) + + assert payload["tenant_record_id"] == TENANT + assert payload["validation_study_reference"] == STUDY + assert payload["job_profile_reference"] == JOB + assert payload["fast_mlsirm_revision"] == REVIEWED_FAST_MLSIRM_REVISION + assert payload["requested_at"] == "2026-08-20T22:10:11.123456Z" + assert payload["validation_strategy"] == "criterion_related" + assert payload["kernel_repository"] == "ContextualWisdomLab/fast-mlsirm" + assert payload["kernel_boundary"] == "read_only_pinned_revision" + assert payload["execution_state"] == "not_executed" + assert payload["contains_raw_person_level_values"] is False + assert payload["human_review_required"] is True + assert payload["result_authority"] == "scientific_evidence_only" + assert payload["required_result_evidence"] == [ + "effect_estimate", + "uncertainty_interval", + "sample_size", + "missingness_summary", + "convergence_diagnostics", + ] + assert "person_record" not in candidate.canonical_json() + assert "candidate" not in candidate.canonical_json() + assert repr(candidate) == "ValidationAnalysisHandoff()" + assert len(candidate.sha256_digest()) == 64 + assert candidate.canonical_json() == handoff().canonical_json() + + +@pytest.mark.parametrize( + "bad_tenant", + [ + "not-a-uuid", + "00000000-0000-0000-0000-000000000000", + "ffffffff-ffff-ffff-ffff-ffffffffffff", + "10000000-0000-7000-8000-00000000000A", + 1, + ], +) +def test_tenant_identity_must_follow_protected_operational_uuid_contract(bad_tenant): + """Reject malformed, reserved, non-canonical, and non-text tenant identities.""" + values = valid_kwargs() + values["tenant_record_id"] = bad_tenant + with pytest.raises(ValueError, match="tenant_record_id"): + build_validation_analysis_handoff(**values) + + +@pytest.mark.parametrize( + ("field", "bad", "match"), + [ + ("handoff_reference", "validation_analysis_handoff:not-a-uuid", "handoff_reference"), + ("validation_study_reference", JOB, "validation_study_reference"), + ("job_profile_reference", "job_profile:22222222-2222-7222-8222-222222222222", "job_profile_reference"), + ("predictor_snapshot_reference", 1, "predictor_snapshot_reference"), + ("criterion_snapshot_reference", "criterion_snapshot:" + "a" * 161, "criterion_snapshot_reference"), + ("population_snapshot_reference", "study_population_snapshot:not-a-uuid", "population_snapshot_reference"), + ("decision_policy_reference", "decision_policy:BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB", "decision_policy_reference"), + ("analysis_plan_reference", "validation_analysis_plan:00000000-0000-0000-0000-000000000000", "analysis_plan_reference"), + ("actor_reference", "actor:ffffffff-ffff-ffff-ffff-ffffffffffff", "actor_reference"), + ("reviewer_reference", "actor:bbbbbbbb-bbbb-7bbb-8bbb-bbbbbbbbbbbb", "reviewer_reference"), + ], +) +def test_all_public_references_are_namespaced_opaque_uuid4(field, bad, match): + """Fail closed on wrong namespace, malformed, noncanonical, or non-v4 references.""" + values = valid_kwargs() + values[field] = bad + with pytest.raises(ValueError, match=match): + build_validation_analysis_handoff(**values) + + +@pytest.mark.parametrize( + "field", + [ + "predictor_snapshot_digest", + "criterion_snapshot_digest", + "population_snapshot_digest", + "decision_policy_digest", + "analysis_plan_digest", + ], +) +def test_evidence_digests_are_lowercase_sha256(field): + """Reject weak or noncanonical evidence digests for every source snapshot.""" + values = valid_kwargs() + values[field] = "A" * 64 + with pytest.raises(ValueError, match=field): + build_validation_analysis_handoff(**values) + + +def test_requester_and_reviewer_must_be_distinct(): + """Require accountable independent interpretation instead of self-review.""" + values = valid_kwargs() + values["reviewer_reference"] = ACTOR + with pytest.raises(ValueError, match="different accountable actor"): + build_validation_analysis_handoff(**values) + + +@pytest.mark.parametrize("bad_revision", ["not-a-sha", "A" * 40, "0" * 40]) +def test_fast_mlsirm_revision_is_exactly_the_reviewed_immutable_dependency(bad_revision): + """Reject malformed or unreviewed foreign dependency revisions.""" + values = valid_kwargs() + values["fast_mlsirm_revision"] = bad_revision + with pytest.raises(ValueError, match="fast_mlsirm_revision"): + build_validation_analysis_handoff(**values) + + +@pytest.mark.parametrize( + "bad_requested_at", + [ + datetime(2026, 8, 21, 7, 10), + "2026-08-21T07:10:00+09:00", + ], +) +def test_requested_at_requires_an_aware_datetime(bad_requested_at): + """Reject local-time ambiguity in immutable analysis correlation.""" + values = valid_kwargs() + values["requested_at"] = bad_requested_at + with pytest.raises(ValueError, match="requested_at"): + build_validation_analysis_handoff(**values) + + +@pytest.mark.parametrize( + ("field", "bad", "match"), + [ + ("purpose_code", "other_purpose", "purpose_code"), + ("reason_code", "other_reason", "reason_code"), + ("evidence_version", True, "evidence_version"), + ("evidence_version", 0, "evidence_version"), + ("validation_strategy", "content_validity", "validation_strategy"), + ("kernel_repository", "other/repository", "kernel_repository"), + ("kernel_boundary", "direct_database", "kernel_boundary"), + ("execution_state", "executed", "execution_state"), + ("contains_raw_person_level_values", True, "raw person-level"), + ("human_review_required", False, "human review"), + ("result_authority", "employment_decision", "result_authority"), + ("required_result_evidence", ("effect_estimate",), "required_result_evidence"), + ("next_action", "Auto-approve the result.", "next_action"), + ], +) +def test_direct_construction_cannot_weaken_governance(field, bad, match): + """Keep fixed scientific, privacy, dependency, and human-authority semantics immutable.""" + with pytest.raises(ValueError, match=match): + replace(handoff(), **{field: bad}) + + +def test_codes_must_remain_bounded_descriptive_snake_case_before_fixed_value_check(): + """Exercise code-shape rejection separately from the closed purpose/reason vocabulary.""" + with pytest.raises(ValueError, match="purpose_code"): + replace(handoff(), purpose_code="X") + with pytest.raises(ValueError, match="reason_code"): + replace(handoff(), reason_code="x" * 65) + + +def test_public_dataclass_type_is_constructible_only_with_all_invariants(): + """Document the public immutable type while preserving builder equivalence.""" + values = valid_kwargs() + direct = ValidationAnalysisHandoff(**values) + assert direct == handoff() From 5b0a1e5926497ca46a9c12bcc253d555dbf99bc8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 15:47:12 -0700 Subject: [PATCH 002/158] feat(validity): add governed fast-mlsirm handoff --- .../orgmetra_validity_analysis/__init__.py | 13 + .../src/orgmetra_validity_analysis/handoff.py | 280 ++++++++++++++++++ 2 files changed, 293 insertions(+) create mode 100644 packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py create mode 100644 packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py b/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py new file mode 100644 index 000000000..d7d15691c --- /dev/null +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py @@ -0,0 +1,13 @@ +"""Public governed selection-validity analysis handoff contract.""" + +from .handoff import ( + REVIEWED_FAST_MLSIRM_REVISION, + ValidationAnalysisHandoff, + build_validation_analysis_handoff, +) + +__all__ = [ + "REVIEWED_FAST_MLSIRM_REVISION", + "ValidationAnalysisHandoff", + "build_validation_analysis_handoff", +] diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py b/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py new file mode 100644 index 000000000..e3a347723 --- /dev/null +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py @@ -0,0 +1,280 @@ +"""Governed handoff evidence for criterion-related selection validation. + +This package does not execute statistics, read another service's database, or make an +employment decision. It binds authoritative Orgmetra evidence to one reviewed, +immutable fast-mlsirm revision so an approved offline worker can perform numerical +analysis without silently changing the study definition. +""" +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from hashlib import sha256 +import json +import re +from uuid import UUID + +_CODE_PATTERN = re.compile(r"^[a-z][a-z0-9]*(?:_[a-z0-9]+)+$") +_DIGEST_PATTERN = re.compile(r"^[0-9a-f]{64}$") +_REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$") +_REFERENCE_PATTERN = re.compile( + r"^[a-z][a-z0-9_]{1,31}:[A-Za-z0-9](?:[A-Za-z0-9._-]{0,126}[A-Za-z0-9])?$" +) +_PURPOSE_CODE = "selection_validity_analysis" +_REASON_CODE = "criterion_related_validation" +_VALIDATION_STRATEGY = "criterion_related" +_KERNEL_REPOSITORY = "ContextualWisdomLab/fast-mlsirm" +REVIEWED_FAST_MLSIRM_REVISION = "04d0bc21a2a20693bcf16108cd76d394fe844d23" +_KERNEL_BOUNDARY = "read_only_pinned_revision" +_EXECUTION_STATE = "not_executed" +_RESULT_AUTHORITY = "scientific_evidence_only" +_REQUIRED_RESULT_EVIDENCE = ( + "effect_estimate", + "uncertainty_interval", + "sample_size", + "missingness_summary", + "convergence_diagnostics", +) +_NEXT_ACTION = ( + "Within tenant_record_id, re-resolve the validation study, Job, predictor, criterion, " + "population, decision-policy, analysis-plan, requester, and reviewer references; prove " + "the predictor/criterion/population cases belong to the exact study and Job; then let an " + "approved offline validation worker invoke only the pinned fast-mlsirm revision. Preserve " + "the resulting model/provenance diagnostics as draft scientific evidence for an " + "accountable human reviewer; never convert the result directly into an employment decision." +) + + +def _validate_operational_uuid(value: str, field_name: str) -> None: + """Require canonical non-sentinel UUID text owned by authoritative Orgmetra.""" + try: + parsed = UUID(value) + except (ValueError, AttributeError, TypeError) as exc: + raise ValueError(f"{field_name} must be canonical UUID text") from exc + if str(parsed) != value or parsed.int in (0, (1 << 128) - 1): + raise ValueError(f"{field_name} must be a canonical operational UUID") + + +def _validate_reference(value: str, prefix: str, field_name: str) -> None: + """Require the expected namespace plus a canonical opaque UUIDv4 suffix.""" + error_message = f"{field_name} must be an opaque {prefix}: UUIDv4 reference" + if ( + not isinstance(value, str) + or len(value) > 160 + or not _REFERENCE_PATTERN.fullmatch(value) + or not value.startswith(f"{prefix}:") + ): + raise ValueError(error_message) + suffix = value.split(":", 1)[1] + try: + parsed = UUID(suffix) + except (ValueError, AttributeError, TypeError) as exc: + raise ValueError(error_message) from exc + if str(parsed) != suffix or parsed.version != 4 or parsed.int in (0, (1 << 128) - 1): + raise ValueError(error_message) + + +def _validate_digest(value: str, field_name: str) -> None: + """Require lowercase SHA-256 hexadecimal evidence.""" + if not isinstance(value, str) or not _DIGEST_PATTERN.fullmatch(value): + raise ValueError(f"{field_name} must be lowercase SHA-256 hex") + + +def _validate_code(value: str, field_name: str) -> None: + """Require bounded descriptive lower snake_case governance codes.""" + if not isinstance(value, str) or len(value) > 64 or not _CODE_PATTERN.fullmatch(value): + raise ValueError(f"{field_name} must be bounded two-or-more-word lower snake_case") + + +def _validate_kernel_revision(value: str) -> None: + """Require the exact externally reviewed immutable fast-mlsirm revision.""" + if not isinstance(value, str) or not _REVISION_PATTERN.fullmatch(value): + raise ValueError("fast_mlsirm_revision must be lowercase 40-character Git commit hex") + if value != REVIEWED_FAST_MLSIRM_REVISION: + raise ValueError("fast_mlsirm_revision must equal the reviewed immutable revision") + + +def _canonical_timestamp(value: datetime) -> str: + """Render an aware instant as precision-preserving UTC RFC 3339 text.""" + if not isinstance(value, datetime) or value.tzinfo is None or value.utcoffset() is None: + raise ValueError("requested_at must be timezone-aware") + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +@dataclass(frozen=True, slots=True, repr=False) +class ValidationAnalysisHandoff: + """Immutable evidence for one not-yet-executed criterion-related validity analysis.""" + + tenant_record_id: str + handoff_reference: str + validation_study_reference: str + job_profile_reference: str + predictor_snapshot_reference: str + predictor_snapshot_digest: str + criterion_snapshot_reference: str + criterion_snapshot_digest: str + population_snapshot_reference: str + population_snapshot_digest: str + decision_policy_reference: str + decision_policy_digest: str + analysis_plan_reference: str + analysis_plan_digest: str + actor_reference: str + reviewer_reference: str + fast_mlsirm_revision: str + requested_at: datetime + purpose_code: str = _PURPOSE_CODE + reason_code: str = _REASON_CODE + evidence_version: int = 1 + validation_strategy: str = _VALIDATION_STRATEGY + kernel_repository: str = _KERNEL_REPOSITORY + kernel_boundary: str = _KERNEL_BOUNDARY + execution_state: str = _EXECUTION_STATE + contains_raw_person_level_values: bool = False + human_review_required: bool = True + result_authority: str = _RESULT_AUTHORITY + required_result_evidence: tuple[str, ...] = _REQUIRED_RESULT_EVIDENCE + next_action: str = _NEXT_ACTION + + def __post_init__(self) -> None: + """Fail closed when direct construction drifts from the governed handoff.""" + _validate_operational_uuid(self.tenant_record_id, "tenant_record_id") + for value, prefix, field_name in ( + (self.handoff_reference, "validation_analysis_handoff", "handoff_reference"), + (self.validation_study_reference, "validation_study", "validation_study_reference"), + (self.job_profile_reference, "job_profile", "job_profile_reference"), + (self.predictor_snapshot_reference, "predictor_snapshot", "predictor_snapshot_reference"), + (self.criterion_snapshot_reference, "criterion_snapshot", "criterion_snapshot_reference"), + (self.population_snapshot_reference, "study_population_snapshot", "population_snapshot_reference"), + (self.decision_policy_reference, "decision_policy", "decision_policy_reference"), + (self.analysis_plan_reference, "validation_analysis_plan", "analysis_plan_reference"), + (self.actor_reference, "actor", "actor_reference"), + (self.reviewer_reference, "actor", "reviewer_reference"), + ): + _validate_reference(value, prefix, field_name) + for value, field_name in ( + (self.predictor_snapshot_digest, "predictor_snapshot_digest"), + (self.criterion_snapshot_digest, "criterion_snapshot_digest"), + (self.population_snapshot_digest, "population_snapshot_digest"), + (self.decision_policy_digest, "decision_policy_digest"), + (self.analysis_plan_digest, "analysis_plan_digest"), + ): + _validate_digest(value, field_name) + if self.actor_reference == self.reviewer_reference: + raise ValueError("reviewer_reference must identify a different accountable actor") + _validate_kernel_revision(self.fast_mlsirm_revision) + _canonical_timestamp(self.requested_at) + _validate_code(self.purpose_code, "purpose_code") + if self.purpose_code != _PURPOSE_CODE: + raise ValueError("purpose_code must remain selection_validity_analysis") + _validate_code(self.reason_code, "reason_code") + if self.reason_code != _REASON_CODE: + raise ValueError("reason_code must remain criterion_related_validation") + if type(self.evidence_version) is not int or not 1 <= self.evidence_version <= 2_147_483_647: + raise ValueError("evidence_version must be an integer from 1 through 2147483647") + if self.validation_strategy != _VALIDATION_STRATEGY: + raise ValueError("validation_strategy must remain criterion_related") + if self.kernel_repository != _KERNEL_REPOSITORY: + raise ValueError("kernel_repository must remain ContextualWisdomLab/fast-mlsirm") + if self.kernel_boundary != _KERNEL_BOUNDARY: + raise ValueError("kernel_boundary must remain read_only_pinned_revision") + if self.execution_state != _EXECUTION_STATE: + raise ValueError("execution_state must remain not_executed") + if self.contains_raw_person_level_values is not False: + raise ValueError("handoff must not contain raw person-level values") + if self.human_review_required is not True: + raise ValueError("human review is mandatory for selection-validity interpretation") + if self.result_authority != _RESULT_AUTHORITY: + raise ValueError("result_authority must remain scientific_evidence_only") + if self.required_result_evidence != _REQUIRED_RESULT_EVIDENCE: + raise ValueError("required_result_evidence must remain the reviewed evidence set") + if self.next_action != _NEXT_ACTION: + raise ValueError("next_action must remain the governed validation instruction") + + def __repr__(self) -> str: + """Return a fully redacted representation suitable for routine logs.""" + return "ValidationAnalysisHandoff()" + + def canonical_json(self) -> str: + """Return deterministic canonical JSON for audit and result correlation.""" + payload = { + "actor_reference": self.actor_reference, + "analysis_plan_digest": self.analysis_plan_digest, + "analysis_plan_reference": self.analysis_plan_reference, + "contains_raw_person_level_values": self.contains_raw_person_level_values, + "criterion_snapshot_digest": self.criterion_snapshot_digest, + "criterion_snapshot_reference": self.criterion_snapshot_reference, + "decision_policy_digest": self.decision_policy_digest, + "decision_policy_reference": self.decision_policy_reference, + "evidence_version": self.evidence_version, + "execution_state": self.execution_state, + "fast_mlsirm_revision": self.fast_mlsirm_revision, + "handoff_reference": self.handoff_reference, + "human_review_required": self.human_review_required, + "job_profile_reference": self.job_profile_reference, + "kernel_boundary": self.kernel_boundary, + "kernel_repository": self.kernel_repository, + "next_action": self.next_action, + "population_snapshot_digest": self.population_snapshot_digest, + "population_snapshot_reference": self.population_snapshot_reference, + "predictor_snapshot_digest": self.predictor_snapshot_digest, + "predictor_snapshot_reference": self.predictor_snapshot_reference, + "purpose_code": self.purpose_code, + "reason_code": self.reason_code, + "requested_at": _canonical_timestamp(self.requested_at), + "required_result_evidence": list(self.required_result_evidence), + "result_authority": self.result_authority, + "reviewer_reference": self.reviewer_reference, + "tenant_record_id": self.tenant_record_id, + "validation_strategy": self.validation_strategy, + "validation_study_reference": self.validation_study_reference, + } + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + def sha256_digest(self) -> str: + """Return SHA-256 over the exact canonical UTF-8 handoff.""" + return sha256(self.canonical_json().encode("utf-8")).hexdigest() + + +def build_validation_analysis_handoff( + *, + tenant_record_id: str, + handoff_reference: str, + validation_study_reference: str, + job_profile_reference: str, + predictor_snapshot_reference: str, + predictor_snapshot_digest: str, + criterion_snapshot_reference: str, + criterion_snapshot_digest: str, + population_snapshot_reference: str, + population_snapshot_digest: str, + decision_policy_reference: str, + decision_policy_digest: str, + analysis_plan_reference: str, + analysis_plan_digest: str, + actor_reference: str, + reviewer_reference: str, + fast_mlsirm_revision: str, + requested_at: datetime, +) -> ValidationAnalysisHandoff: + """Build a governed, non-executing selection-validity analysis handoff.""" + return ValidationAnalysisHandoff( + tenant_record_id=tenant_record_id, + handoff_reference=handoff_reference, + validation_study_reference=validation_study_reference, + job_profile_reference=job_profile_reference, + predictor_snapshot_reference=predictor_snapshot_reference, + predictor_snapshot_digest=predictor_snapshot_digest, + criterion_snapshot_reference=criterion_snapshot_reference, + criterion_snapshot_digest=criterion_snapshot_digest, + population_snapshot_reference=population_snapshot_reference, + population_snapshot_digest=population_snapshot_digest, + decision_policy_reference=decision_policy_reference, + decision_policy_digest=decision_policy_digest, + analysis_plan_reference=analysis_plan_reference, + analysis_plan_digest=analysis_plan_digest, + actor_reference=actor_reference, + reviewer_reference=reviewer_reference, + fast_mlsirm_revision=fast_mlsirm_revision, + requested_at=requested_at, + ) From 3f99928166e4f6b36714c77557f68ec798256208 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 15:49:08 -0700 Subject: [PATCH 003/158] docs(validity): record governed analysis boundary --- .../workflows/validity-analysis-quality.yml | 59 +++++++++++++++++++ ...ned-selection-validity-analysis-handoff.md | 52 ++++++++++++++++ .../validation-analysis-handoff-references.md | 17 ++++++ .../validation-analysis-handoff.md | 23 ++++++++ packages/validity-analysis/CHANGELOG.md | 7 +++ packages/validity-analysis/README.md | 34 +++++++++++ 6 files changed, 192 insertions(+) create mode 100644 .github/workflows/validity-analysis-quality.yml create mode 100644 docs/adr/0025-governed-selection-validity-analysis-handoff.md create mode 100644 docs/doctoring/validation-analysis-handoff-references.md create mode 100644 docs/traceability/validation-analysis-handoff.md create mode 100644 packages/validity-analysis/CHANGELOG.md create mode 100644 packages/validity-analysis/README.md diff --git a/.github/workflows/validity-analysis-quality.yml b/.github/workflows/validity-analysis-quality.yml new file mode 100644 index 000000000..df8eff10d --- /dev/null +++ b/.github/workflows/validity-analysis-quality.yml @@ -0,0 +1,59 @@ +name: Validity Analysis Handoff Quality + +on: + pull_request: + branches: + - bootstrap + - develop + - main + paths: + - "packages/validity-analysis/**" + - "docs/adr/0025-governed-selection-validity-analysis-handoff.md" + - "docs/doctoring/validation-analysis-handoff-references.md" + - "docs/traceability/validation-analysis-handoff.md" + - ".github/requirements/foundation-test.txt" + - ".github/workflows/validity-analysis-quality.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: validity-analysis-quality-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + unit: + name: Validity handoff and 100% coverage + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout exact candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - name: Prove exact candidate checkout + env: + ORGMETRA_EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: test "$(git rev-parse HEAD)" = "$ORGMETRA_EXPECTED_HEAD_SHA" + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + check-latest: false + - name: Install reviewed test toolchain + run: | + python -m pip install --require-hashes --no-deps --only-binary=:all: -r .github/requirements/foundation-test.txt + python -m pip check + - name: Compile validity-analysis handoff + run: python -m compileall -q packages/validity-analysis/src packages/validity-analysis/tests + - name: Test governed handoff with exact statement and branch coverage + env: + PYTHONPATH: packages/validity-analysis/src + COVERAGE_FILE: /tmp/orgmetra-validity-analysis.coverage + run: python -m pytest -c packages/validity-analysis/pyproject.toml packages/validity-analysis/tests + - name: Require clean checkout + run: | + git diff --exit-code + test -z "$(git status --porcelain)" diff --git a/docs/adr/0025-governed-selection-validity-analysis-handoff.md b/docs/adr/0025-governed-selection-validity-analysis-handoff.md new file mode 100644 index 000000000..a1b06d0e8 --- /dev/null +++ b/docs/adr/0025-governed-selection-validity-analysis-handoff.md @@ -0,0 +1,52 @@ +# ADR 0025: Govern selection-validity numerical work through an immutable handoff + +- Status: Proposed +- Maturity: Active PR only; not protected-branch truth +- Date: 2026-08-21 +- Owners: Orgmetra Workforce Validation + +## Context + +Protected Orgmetra already preserves exact validation-study cases, sealed selection evidence, candidate-to-worker lineage, and Job/cycle/staffing-scoped criterion observations. The remaining boundary is dangerous if left implicit: a statistical worker could receive an underspecified study, silently use a different dependency revision, or turn a model result into an employment decision. + +The Uniform Guidelines recognize criterion-related validity evidence as empirical evidence relating a selection procedure to important job-performance elements and require validity studies to be accurate, standardized, documented, and periodically reviewed for currency. SIOP's *Principles for the Validation and Use of Personnel Selection Procedures* likewise treats validation as an evidence-and-inference problem rather than a correlation-only shortcut. + +`ContextualWisdomLab/fast-mlsirm` owns numerical psychometric/statistical kernels. Its protected `main` was freshly resolved to commit `04d0bc21a2a20693bcf16108cd76d394fe844d23` on 2026-08-21. Orgmetra must not copy that implementation or write the foreign repository. + +## Decision + +Orgmetra adds a leaf `orgmetra_validity_analysis` package whose `ValidationAnalysisHandoff`: + +- binds the exact tenant, validation study, Job, predictor snapshot, criterion snapshot, population snapshot, decision policy, and analysis plan through opaque references plus SHA-256 evidence digests; +- binds distinct requester and reviewer actor references; +- pins fast-mlsirm to reviewed immutable commit `04d0bc21a2a20693bcf16108cd76d394fe844d23`; +- declares the numerical boundary `read_only_pinned_revision` and the initial strategy `criterion_related`; +- requires downstream result evidence for effect estimate, uncertainty interval, sample size, missingness summary, and convergence diagnostics; +- serializes no raw person-level predictor, criterion, candidate, or worker values; +- remains `not_executed`, `scientific_evidence_only`, and human-review-required; +- produces deterministic canonical JSON and a SHA-256 digest for audit/result correlation. + +The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, and prove requester/reviewer identities are distinct authoritative actors. + +## Consequences + +### Positive + +- Statistical work cannot silently drift to an unreviewed fast-mlsirm revision. +- A buyer or auditor can identify exactly which governed study evidence a result was supposed to analyze without copying raw worker data into the handoff. +- Human interpretation remains explicit and separate from numerical output. +- The dedicated-writer boundary remains intact: Orgmetra consumes only a pinned foreign revision/contract boundary and never mutates fast-mlsirm. + +### Limitations + +- This slice does not execute a statistical model, estimate validity, correct for measurement error/range restriction, evaluate adverse impact, or assert legal compliance. +- Sampling design, estimator choice, missing-data treatment, reliability evidence, multiplicity, transportability, fairness analysis, and model diagnostics must be encoded in the referenced analysis plan and reviewed before execution. +- A future execution/result adapter must validate the returned model/provenance schema before any result is attached to an Orgmetra study. + +## Verification + +The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, and 100% owned production statement/branch coverage. + +## References + +See `docs/doctoring/validation-analysis-handoff-references.md`. diff --git a/docs/doctoring/validation-analysis-handoff-references.md b/docs/doctoring/validation-analysis-handoff-references.md new file mode 100644 index 000000000..520b30550 --- /dev/null +++ b/docs/doctoring/validation-analysis-handoff-references.md @@ -0,0 +1,17 @@ +# Validation-analysis handoff references + +Material decisions for ADR 0025 were checked against the following primary/authoritative sources on 2026-08-21. + +## APA 7 references + +Electronic Code of Federal Regulations. (2026). *29 C.F.R. pt. 1607—Uniform Guidelines on Employee Selection Procedures (1978).* Retrieved August 21, 2026, from https://www.ecfr.gov/current/title-29/subtitle-B/chapter-XIV/part-1607 + +Society for Industrial and Organizational Psychology. (2018). *Principles for the validation and use of personnel selection procedures* (5th ed.). Cambridge University Press. https://www.apa.org/ed/accreditation/personnel-selection-procedures.pdf + +ContextualWisdomLab. (2026). *fast-mlsirm* (Commit 04d0bc21a2a20693bcf16108cd76d394fe844d23) [Computer software]. GitHub. https://github.com/ContextualWisdomLab/fast-mlsirm/tree/04d0bc21a2a20693bcf16108cd76d394fe844d23 + +## Decision notes + +- 29 C.F.R. §§ 1607.5 and 1607.14 support keeping criterion-related validity evidence tied to an explicit study design, job relevance, accuracy, reporting, and documentation rather than treating a bare coefficient as sufficient evidence. +- The SIOP Principles are the professional validation baseline used for the handoff's evidence-and-human-review posture. +- The fast-mlsirm commit is recorded as a read-only dependency coordinate only. This Orgmetra slice does not modify or duplicate its numerical implementation. diff --git a/docs/traceability/validation-analysis-handoff.md b/docs/traceability/validation-analysis-handoff.md new file mode 100644 index 000000000..c1d53a51f --- /dev/null +++ b/docs/traceability/validation-analysis-handoff.md @@ -0,0 +1,23 @@ +# Selection-validity analysis handoff traceability + +## Buyer question + +Can an organization send one exact, reviewable validation study to its statistical engine without copying raw person-level values into a workflow envelope, silently changing the numerical dependency, or treating model output as an employment decision? + +## Active-PR contract + +| Concern | Orgmetra evidence | Verification | +|---|---|---| +| Exact study scope | tenant, validation-study, Job, predictor, criterion, population, decision-policy, and analysis-plan references plus digests | namespace/UUID/digest regressions | +| Dependency integrity | immutable fast-mlsirm commit `04d0bc21a2a20693bcf16108cd76d394fe844d23` | malformed and unreviewed revision rejection | +| Privacy minimization | no raw person-level values in canonical handoff | canonical-payload regression and redacted repr | +| Human authority | distinct requester/reviewer and `human_review_required=true` | direct-construction fail-closed regressions | +| Scientific evidence | effect estimate, uncertainty interval, sample size, missingness summary, convergence diagnostics | immutable required-result-evidence regression | +| Execution boundary | `not_executed`, `scientific_evidence_only`, read-only pinned foreign dependency | immutable governance regressions | +| Reproducibility | canonical RFC 3339 time, canonical JSON, SHA-256 handoff digest | deterministic serialization/digest tests | + +## Maturity + +`implemented_on_active_pr`. + +Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. A future worker/result boundary must independently earn tests for the exact returned numerical/provenance contract before the result can become protected Orgmetra evidence. diff --git a/packages/validity-analysis/CHANGELOG.md b/packages/validity-analysis/CHANGELOG.md new file mode 100644 index 000000000..821ff22f6 --- /dev/null +++ b/packages/validity-analysis/CHANGELOG.md @@ -0,0 +1,7 @@ +# Changelog + +## 0.1.0 - Unreleased + +- Add a governed, value-minimized criterion-related validity analysis handoff. +- Pin the reviewed read-only fast-mlsirm dependency revision. +- Require separate requester/reviewer authority, deterministic canonical evidence, and 100% owned production statement/branch coverage. diff --git a/packages/validity-analysis/README.md b/packages/validity-analysis/README.md new file mode 100644 index 000000000..c7e14eb37 --- /dev/null +++ b/packages/validity-analysis/README.md @@ -0,0 +1,34 @@ +# Orgmetra validity-analysis handoff + +This package creates an immutable **selection-validity analysis handoff**. It is the boundary between Orgmetra's authoritative validation-study evidence and numerical work owned by `ContextualWisdomLab/fast-mlsirm`. + +## What it does + +`build_validation_analysis_handoff(...)` binds one tenant, validation study, Job, predictor snapshot, criterion snapshot, population snapshot, decision policy, analysis plan, requester, reviewer, and the reviewed fast-mlsirm revision `04d0bc21a2a20693bcf16108cd76d394fe844d23`. + +The resulting canonical JSON is digest-addressable, contains no raw person-level predictor or criterion values, and remains `not_executed`. Required result evidence is explicit: effect estimate, uncertainty interval, sample size, missingness summary, and convergence diagnostics. + +## What it does not do + +- It does **not** run statistics. +- It does **not** query fast-mlsirm or any other CWL application's database. +- It does **not** claim that a selection procedure is valid. +- It does **not** interpret adverse impact. +- It does **not** authorize hiring, promotion, termination, compensation, or another employment decision. + +The fast-mlsirm repository remains a dedicated-writer dependency. This package records only the immutable revision reviewed for the handoff. + +## Host obligations + +Before an approved offline validation worker executes the handoff, the Orgmetra host must re-resolve every reference inside `tenant_record_id` and prove that the predictor, criterion, population, and policy evidence belong to the exact validation study and Job. The requester and reviewer must resolve to distinct authoritative actors. A numerical result is scientific evidence for accountable human interpretation, never an autonomous employment decision. + +## Verification + +Run: + +```bash +PYTHONPATH=packages/validity-analysis/src \ +python -m pytest -c packages/validity-analysis/pyproject.toml packages/validity-analysis/tests +``` + +The package gate requires exact 100% owned production statement and branch coverage. From a9168a9fde7e6de1b4a066098c5c4c2993daa0ce Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 16:30:38 -0700 Subject: [PATCH 004/158] test(validity): reject duplicate ADR numbers after integration --- .../tests/test_adr_numbering.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 packages/validity-analysis/tests/test_adr_numbering.py diff --git a/packages/validity-analysis/tests/test_adr_numbering.py b/packages/validity-analysis/tests/test_adr_numbering.py new file mode 100644 index 000000000..8f2ee0828 --- /dev/null +++ b/packages/validity-analysis/tests/test_adr_numbering.py @@ -0,0 +1,17 @@ +"""Regression tests for repository-wide ADR number ownership.""" + +from pathlib import Path + + +def test_adr_numbers_are_unique_across_the_integrated_repository() -> None: + """Every four-digit ADR number must identify exactly one decision record.""" + adr_directory = Path(__file__).resolve().parents[3] / "docs" / "adr" + owners: dict[str, str] = {} + + for adr_path in sorted(adr_directory.glob("[0-9][0-9][0-9][0-9]-*.md")): + adr_number = adr_path.name[:4] + previous_owner = owners.get(adr_number) + assert previous_owner is None, ( + f"ADR {adr_number} is reused by {previous_owner} and {adr_path.name}" + ) + owners[adr_number] = adr_path.name From 6814fd9b3c924f9ed3b1f2e4eaea96cfe7191baf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 16:31:16 -0700 Subject: [PATCH 005/158] fix(validity): reserve ADR 0027 for analysis handoff --- ...ned-selection-validity-analysis-handoff.md | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 docs/adr/0027-governed-selection-validity-analysis-handoff.md diff --git a/docs/adr/0027-governed-selection-validity-analysis-handoff.md b/docs/adr/0027-governed-selection-validity-analysis-handoff.md new file mode 100644 index 000000000..14fe92e05 --- /dev/null +++ b/docs/adr/0027-governed-selection-validity-analysis-handoff.md @@ -0,0 +1,52 @@ +# ADR 0027: Govern selection-validity numerical work through an immutable handoff + +- Status: Proposed +- Maturity: Active PR only; not protected-branch truth +- Date: 2026-08-21 +- Owners: Orgmetra Workforce Validation + +## Context + +Protected Orgmetra already preserves exact validation-study cases, sealed selection evidence, candidate-to-worker lineage, and Job/cycle/staffing-scoped criterion observations. The remaining boundary is dangerous if left implicit: a statistical worker could receive an underspecified study, silently use a different dependency revision, or turn a model result into an employment decision. + +The Uniform Guidelines recognize criterion-related validity evidence as empirical evidence relating a selection procedure to important job-performance elements and require validity studies to be accurate, standardized, documented, and periodically reviewed for currency. SIOP's *Principles for the Validation and Use of Personnel Selection Procedures* likewise treats validation as an evidence-and-inference problem rather than a correlation-only shortcut. + +`ContextualWisdomLab/fast-mlsirm` owns numerical psychometric/statistical kernels. Its protected `main` was freshly resolved to commit `04d0bc21a2a20693bcf16108cd76d394fe844d23` on 2026-08-21. Orgmetra must not copy that implementation or write the foreign repository. + +## Decision + +Orgmetra adds a leaf `orgmetra_validity_analysis` package whose `ValidationAnalysisHandoff`: + +- binds the exact tenant, validation study, Job, predictor snapshot, criterion snapshot, population snapshot, decision policy, and analysis plan through opaque references plus SHA-256 evidence digests; +- binds distinct requester and reviewer actor references; +- pins fast-mlsirm to reviewed immutable commit `04d0bc21a2a20693bcf16108cd76d394fe844d23`; +- declares the numerical boundary `read_only_pinned_revision` and the initial strategy `criterion_related`; +- requires downstream result evidence for effect estimate, uncertainty interval, sample size, missingness summary, and convergence diagnostics; +- serializes no raw person-level predictor, criterion, candidate, or worker values; +- remains `not_executed`, `scientific_evidence_only`, and human-review-required; +- produces deterministic canonical JSON and a SHA-256 digest for audit/result correlation. + +The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, and prove requester/reviewer identities are distinct authoritative actors. + +## Consequences + +### Positive + +- Statistical work cannot silently drift to an unreviewed fast-mlsirm revision. +- A buyer or auditor can identify exactly which governed study evidence a result was supposed to analyze without copying raw worker data into the handoff. +- Human interpretation remains explicit and separate from numerical output. +- The dedicated-writer boundary remains intact: Orgmetra consumes only a pinned foreign revision/contract boundary and never mutates fast-mlsirm. + +### Limitations + +- This slice does not execute a statistical model, estimate validity, correct for measurement error/range restriction, evaluate adverse impact, or assert legal compliance. +- Sampling design, estimator choice, missing-data treatment, reliability evidence, multiplicity, transportability, fairness analysis, and model diagnostics must be encoded in the referenced analysis plan and reviewed before execution. +- A future execution/result adapter must validate the returned model/provenance schema before any result is attached to an Orgmetra study. + +## Verification + +The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, and 100% owned production statement/branch coverage. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number. + +## References + +See `docs/doctoring/validation-analysis-handoff-references.md`. From a4876a8d3ce0c4994cb93de9490c3fc3b46d8ec0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 16:31:26 -0700 Subject: [PATCH 006/158] fix(validity): track reserved ADR 0027 in quality gate --- .github/workflows/validity-analysis-quality.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validity-analysis-quality.yml b/.github/workflows/validity-analysis-quality.yml index df8eff10d..5b1ae41fc 100644 --- a/.github/workflows/validity-analysis-quality.yml +++ b/.github/workflows/validity-analysis-quality.yml @@ -8,7 +8,7 @@ on: - main paths: - "packages/validity-analysis/**" - - "docs/adr/0025-governed-selection-validity-analysis-handoff.md" + - "docs/adr/0027-governed-selection-validity-analysis-handoff.md" - "docs/doctoring/validation-analysis-handoff-references.md" - "docs/traceability/validation-analysis-handoff.md" - ".github/requirements/foundation-test.txt" From 7e9cd769cd2b9fad0c994497847255885e0526c8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 16:31:39 -0700 Subject: [PATCH 007/158] docs(validity): bind references to ADR 0027 --- docs/doctoring/validation-analysis-handoff-references.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/doctoring/validation-analysis-handoff-references.md b/docs/doctoring/validation-analysis-handoff-references.md index 520b30550..c2e31dbd2 100644 --- a/docs/doctoring/validation-analysis-handoff-references.md +++ b/docs/doctoring/validation-analysis-handoff-references.md @@ -1,6 +1,6 @@ # Validation-analysis handoff references -Material decisions for ADR 0025 were checked against the following primary/authoritative sources on 2026-08-21. +Material decisions for ADR 0027 were checked against the following primary/authoritative sources on 2026-08-21. ## APA 7 references From a872938b508a42a7010c059789e012beffd5ef60 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 16:31:45 -0700 Subject: [PATCH 008/158] fix(validity): remove conflicting ADR 0025 allocation --- ...ned-selection-validity-analysis-handoff.md | 52 ------------------- 1 file changed, 52 deletions(-) delete mode 100644 docs/adr/0025-governed-selection-validity-analysis-handoff.md diff --git a/docs/adr/0025-governed-selection-validity-analysis-handoff.md b/docs/adr/0025-governed-selection-validity-analysis-handoff.md deleted file mode 100644 index a1b06d0e8..000000000 --- a/docs/adr/0025-governed-selection-validity-analysis-handoff.md +++ /dev/null @@ -1,52 +0,0 @@ -# ADR 0025: Govern selection-validity numerical work through an immutable handoff - -- Status: Proposed -- Maturity: Active PR only; not protected-branch truth -- Date: 2026-08-21 -- Owners: Orgmetra Workforce Validation - -## Context - -Protected Orgmetra already preserves exact validation-study cases, sealed selection evidence, candidate-to-worker lineage, and Job/cycle/staffing-scoped criterion observations. The remaining boundary is dangerous if left implicit: a statistical worker could receive an underspecified study, silently use a different dependency revision, or turn a model result into an employment decision. - -The Uniform Guidelines recognize criterion-related validity evidence as empirical evidence relating a selection procedure to important job-performance elements and require validity studies to be accurate, standardized, documented, and periodically reviewed for currency. SIOP's *Principles for the Validation and Use of Personnel Selection Procedures* likewise treats validation as an evidence-and-inference problem rather than a correlation-only shortcut. - -`ContextualWisdomLab/fast-mlsirm` owns numerical psychometric/statistical kernels. Its protected `main` was freshly resolved to commit `04d0bc21a2a20693bcf16108cd76d394fe844d23` on 2026-08-21. Orgmetra must not copy that implementation or write the foreign repository. - -## Decision - -Orgmetra adds a leaf `orgmetra_validity_analysis` package whose `ValidationAnalysisHandoff`: - -- binds the exact tenant, validation study, Job, predictor snapshot, criterion snapshot, population snapshot, decision policy, and analysis plan through opaque references plus SHA-256 evidence digests; -- binds distinct requester and reviewer actor references; -- pins fast-mlsirm to reviewed immutable commit `04d0bc21a2a20693bcf16108cd76d394fe844d23`; -- declares the numerical boundary `read_only_pinned_revision` and the initial strategy `criterion_related`; -- requires downstream result evidence for effect estimate, uncertainty interval, sample size, missingness summary, and convergence diagnostics; -- serializes no raw person-level predictor, criterion, candidate, or worker values; -- remains `not_executed`, `scientific_evidence_only`, and human-review-required; -- produces deterministic canonical JSON and a SHA-256 digest for audit/result correlation. - -The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, and prove requester/reviewer identities are distinct authoritative actors. - -## Consequences - -### Positive - -- Statistical work cannot silently drift to an unreviewed fast-mlsirm revision. -- A buyer or auditor can identify exactly which governed study evidence a result was supposed to analyze without copying raw worker data into the handoff. -- Human interpretation remains explicit and separate from numerical output. -- The dedicated-writer boundary remains intact: Orgmetra consumes only a pinned foreign revision/contract boundary and never mutates fast-mlsirm. - -### Limitations - -- This slice does not execute a statistical model, estimate validity, correct for measurement error/range restriction, evaluate adverse impact, or assert legal compliance. -- Sampling design, estimator choice, missing-data treatment, reliability evidence, multiplicity, transportability, fairness analysis, and model diagnostics must be encoded in the referenced analysis plan and reviewed before execution. -- A future execution/result adapter must validate the returned model/provenance schema before any result is attached to an Orgmetra study. - -## Verification - -The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, and 100% owned production statement/branch coverage. - -## References - -See `docs/doctoring/validation-analysis-handoff-references.md`. From 22fd3745994e8111d94e1355aceb7a2ec1607d5e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 18:09:44 -0700 Subject: [PATCH 009/158] test(validity): require resolved reviewer identity separation --- .../tests/test_host_resolution_contract.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 packages/validity-analysis/tests/test_host_resolution_contract.py diff --git a/packages/validity-analysis/tests/test_host_resolution_contract.py b/packages/validity-analysis/tests/test_host_resolution_contract.py new file mode 100644 index 000000000..c87fb1a18 --- /dev/null +++ b/packages/validity-analysis/tests/test_host_resolution_contract.py @@ -0,0 +1,34 @@ +"""Regressions for authoritative requester/reviewer identity separation.""" + +from datetime import datetime, timezone + +from orgmetra_validity_analysis import ( + REVIEWED_FAST_MLSIRM_REVISION, + build_validation_analysis_handoff, +) + + +def test_next_action_requires_resolved_actor_identity_separation() -> None: + """Do not let different opaque actor references masquerade as distinct people.""" + handoff = build_validation_analysis_handoff( + tenant_record_id="10000000-0000-7000-8000-000000000001", + handoff_reference="validation_analysis_handoff:11111111-1111-4111-8111-111111111111", + validation_study_reference="validation_study:22222222-2222-4222-8222-222222222222", + job_profile_reference="job_profile:33333333-3333-4333-8333-333333333333", + predictor_snapshot_reference="predictor_snapshot:44444444-4444-4444-8444-444444444444", + predictor_snapshot_digest="a" * 64, + criterion_snapshot_reference="criterion_snapshot:55555555-5555-4555-8555-555555555555", + criterion_snapshot_digest="b" * 64, + population_snapshot_reference="study_population_snapshot:66666666-6666-4666-8666-666666666666", + population_snapshot_digest="c" * 64, + decision_policy_reference="decision_policy:77777777-7777-4777-8777-777777777777", + decision_policy_digest="d" * 64, + analysis_plan_reference="validation_analysis_plan:88888888-8888-4888-8888-888888888888", + analysis_plan_digest="e" * 64, + actor_reference="actor:99999999-9999-4999-8999-999999999999", + reviewer_reference="actor:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + fast_mlsirm_revision=REVIEWED_FAST_MLSIRM_REVISION, + requested_at=datetime(2026, 8, 21, 1, 0, tzinfo=timezone.utc), + ) + + assert "prove requester and reviewer resolve to distinct authoritative actor identities" in handoff.next_action From 8432d1c486c94078a92bd5bd5e0027bc19f7c65e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 18:10:21 -0700 Subject: [PATCH 010/158] fix(validity): require authoritative actor identity separation --- .../src/orgmetra_validity_analysis/handoff.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py b/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py index e3a347723..ad257a098 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py @@ -38,7 +38,8 @@ _NEXT_ACTION = ( "Within tenant_record_id, re-resolve the validation study, Job, predictor, criterion, " "population, decision-policy, analysis-plan, requester, and reviewer references; prove " - "the predictor/criterion/population cases belong to the exact study and Job; then let an " + "requester and reviewer resolve to distinct authoritative actor identities; prove the " + "predictor/criterion/population cases belong to the exact study and Job; then let an " "approved offline validation worker invoke only the pinned fast-mlsirm revision. Preserve " "the resulting model/provenance diagnostics as draft scientific evidence for an " "accountable human reviewer; never convert the result directly into an employment decision." From 68a15c4dcfa4ac38820ac6e902f320f9f38e1e10 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 18:10:43 -0700 Subject: [PATCH 011/158] docs(validity): trace authoritative actor separation --- docs/traceability/validation-analysis-handoff.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/traceability/validation-analysis-handoff.md b/docs/traceability/validation-analysis-handoff.md index c1d53a51f..2fb472dbb 100644 --- a/docs/traceability/validation-analysis-handoff.md +++ b/docs/traceability/validation-analysis-handoff.md @@ -11,7 +11,7 @@ Can an organization send one exact, reviewable validation study to its statistic | Exact study scope | tenant, validation-study, Job, predictor, criterion, population, decision-policy, and analysis-plan references plus digests | namespace/UUID/digest regressions | | Dependency integrity | immutable fast-mlsirm commit `04d0bc21a2a20693bcf16108cd76d394fe844d23` | malformed and unreviewed revision rejection | | Privacy minimization | no raw person-level values in canonical handoff | canonical-payload regression and redacted repr | -| Human authority | distinct requester/reviewer and `human_review_required=true` | direct-construction fail-closed regressions | +| Human authority | requester/reviewer references must differ, and the host must re-resolve both within the tenant and prove they resolve to distinct authoritative actor identities before execution | direct-construction fail-closed regression plus `test_next_action_requires_resolved_actor_identity_separation` | | Scientific evidence | effect estimate, uncertainty interval, sample size, missingness summary, convergence diagnostics | immutable required-result-evidence regression | | Execution boundary | `not_executed`, `scientific_evidence_only`, read-only pinned foreign dependency | immutable governance regressions | | Reproducibility | canonical RFC 3339 time, canonical JSON, SHA-256 handoff digest | deterministic serialization/digest tests | From f0c30c5e6cfd6cb90afcbef1efd0ba8825f3fd52 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 18:10:48 -0700 Subject: [PATCH 012/158] docs(validity): record identity-separation repair --- packages/validity-analysis/CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/validity-analysis/CHANGELOG.md b/packages/validity-analysis/CHANGELOG.md index 821ff22f6..662b49f60 100644 --- a/packages/validity-analysis/CHANGELOG.md +++ b/packages/validity-analysis/CHANGELOG.md @@ -4,4 +4,5 @@ - Add a governed, value-minimized criterion-related validity analysis handoff. - Pin the reviewed read-only fast-mlsirm dependency revision. -- Require separate requester/reviewer authority, deterministic canonical evidence, and 100% owned production statement/branch coverage. +- Require separate requester/reviewer references and authoritative tenant-scoped re-resolution proving they resolve to distinct actor identities before execution. +- Require deterministic canonical evidence and 100% owned production statement/branch coverage. From d0c06d21c5ad6d3a0e6ded11f40b04d220d57cba Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 11:49:12 +0900 Subject: [PATCH 013/158] feat(validity): validate pinned numerical result envelopes --- ...ned-selection-validity-analysis-handoff.md | 4 +- .../validation-analysis-handoff-references.md | 3 + .../validation-analysis-handoff.md | 3 +- packages/validity-analysis/CHANGELOG.md | 1 + packages/validity-analysis/README.md | 5 +- .../orgmetra_validity_analysis/__init__.py | 6 +- .../src/orgmetra_validity_analysis/result.py | 231 ++++++++++++++++++ .../validity-analysis/tests/test_result.py | 178 ++++++++++++++ 8 files changed, 427 insertions(+), 4 deletions(-) create mode 100644 packages/validity-analysis/src/orgmetra_validity_analysis/result.py create mode 100644 packages/validity-analysis/tests/test_result.py diff --git a/docs/adr/0027-governed-selection-validity-analysis-handoff.md b/docs/adr/0027-governed-selection-validity-analysis-handoff.md index 14fe92e05..a27a0b09d 100644 --- a/docs/adr/0027-governed-selection-validity-analysis-handoff.md +++ b/docs/adr/0027-governed-selection-validity-analysis-handoff.md @@ -26,6 +26,8 @@ Orgmetra adds a leaf `orgmetra_validity_analysis` package whose `ValidationAnaly - remains `not_executed`, `scientific_evidence_only`, and human-review-required; - produces deterministic canonical JSON and a SHA-256 digest for audit/result correlation. +The same package also validates `ValidationAnalysisResult` envelopes returned by the approved offline worker. A result must link to the handoff digest and the same pinned revision, identify a Rust CPU or GPU backend and precision, provide finite effect and interval values, match its sample size to aggregate missingness counts, and include explicit convergence diagnostics. A nonconverged result remains typed scientific evidence requiring human review; it cannot be treated as a valid selection procedure or an employment decision. + The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, and prove requester/reviewer identities are distinct authoritative actors. ## Consequences @@ -41,7 +43,7 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - This slice does not execute a statistical model, estimate validity, correct for measurement error/range restriction, evaluate adverse impact, or assert legal compliance. - Sampling design, estimator choice, missing-data treatment, reliability evidence, multiplicity, transportability, fairness analysis, and model diagnostics must be encoded in the referenced analysis plan and reviewed before execution. -- A future execution/result adapter must validate the returned model/provenance schema before any result is attached to an Orgmetra study. +- The package validates the result envelope, but a future execution adapter must still re-resolve the handoff references, verify the result provenance artifact, and attach evidence only after accountable human review. ## Verification diff --git a/docs/doctoring/validation-analysis-handoff-references.md b/docs/doctoring/validation-analysis-handoff-references.md index c2e31dbd2..d5de152f1 100644 --- a/docs/doctoring/validation-analysis-handoff-references.md +++ b/docs/doctoring/validation-analysis-handoff-references.md @@ -10,8 +10,11 @@ Society for Industrial and Organizational Psychology. (2018). *Principles for th ContextualWisdomLab. (2026). *fast-mlsirm* (Commit 04d0bc21a2a20693bcf16108cd76d394fe844d23) [Computer software]. GitHub. https://github.com/ContextualWisdomLab/fast-mlsirm/tree/04d0bc21a2a20693bcf16108cd76d394fe844d23 +Tabassi, E. (2023). *Artificial intelligence risk management framework (AI RMF 1.0)* (NIST AI 100-1). National Institute of Standards and Technology. https://doi.org/10.6028/NIST.AI.100-1 + ## Decision notes - 29 C.F.R. §§ 1607.5 and 1607.14 support keeping criterion-related validity evidence tied to an explicit study design, job relevance, accuracy, reporting, and documentation rather than treating a bare coefficient as sufficient evidence. - The SIOP Principles are the professional validation baseline used for the handoff's evidence-and-human-review posture. - The fast-mlsirm commit is recorded as a read-only dependency coordinate only. This Orgmetra slice does not modify or duplicate its numerical implementation. +- NIST AI RMF's govern, map, measure, and manage functions support preserving backend, precision, provenance, convergence, and human-review fields as inspectable result evidence rather than treating a model response as an autonomous decision. diff --git a/docs/traceability/validation-analysis-handoff.md b/docs/traceability/validation-analysis-handoff.md index 2fb472dbb..068886737 100644 --- a/docs/traceability/validation-analysis-handoff.md +++ b/docs/traceability/validation-analysis-handoff.md @@ -13,6 +13,7 @@ Can an organization send one exact, reviewable validation study to its statistic | Privacy minimization | no raw person-level values in canonical handoff | canonical-payload regression and redacted repr | | Human authority | requester/reviewer references must differ, and the host must re-resolve both within the tenant and prove they resolve to distinct authoritative actor identities before execution | direct-construction fail-closed regression plus `test_next_action_requires_resolved_actor_identity_separation` | | Scientific evidence | effect estimate, uncertainty interval, sample size, missingness summary, convergence diagnostics | immutable required-result-evidence regression | +| Numerical result boundary | handoff digest, pinned fast-mlsirm revision, Rust CPU/GPU backend, precision, finite estimate/interval, aggregate missingness, explicit convergence state | `ValidationAnalysisResult` invariant and canonicalization regressions | | Execution boundary | `not_executed`, `scientific_evidence_only`, read-only pinned foreign dependency | immutable governance regressions | | Reproducibility | canonical RFC 3339 time, canonical JSON, SHA-256 handoff digest | deterministic serialization/digest tests | @@ -20,4 +21,4 @@ Can an organization send one exact, reviewable validation study to its statistic `implemented_on_active_pr`. -Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. A future worker/result boundary must independently earn tests for the exact returned numerical/provenance contract before the result can become protected Orgmetra evidence. +Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. The active package now validates the minimum returned numerical/provenance envelope, but protected Orgmetra evidence still requires host re-resolution, result-artifact verification, terminal checks, independent review, and accountable human interpretation. diff --git a/packages/validity-analysis/CHANGELOG.md b/packages/validity-analysis/CHANGELOG.md index 662b49f60..95cab8f32 100644 --- a/packages/validity-analysis/CHANGELOG.md +++ b/packages/validity-analysis/CHANGELOG.md @@ -6,3 +6,4 @@ - Pin the reviewed read-only fast-mlsirm dependency revision. - Require separate requester/reviewer references and authoritative tenant-scoped re-resolution proving they resolve to distinct actor identities before execution. - Require deterministic canonical evidence and 100% owned production statement/branch coverage. +- Validate a digest-linked Rust CPU/GPU result envelope with finite estimates, aggregate missingness, and explicit convergence or nonconvergence diagnostics. diff --git a/packages/validity-analysis/README.md b/packages/validity-analysis/README.md index c7e14eb37..feec8db9a 100644 --- a/packages/validity-analysis/README.md +++ b/packages/validity-analysis/README.md @@ -1,6 +1,6 @@ # Orgmetra validity-analysis handoff -This package creates an immutable **selection-validity analysis handoff**. It is the boundary between Orgmetra's authoritative validation-study evidence and numerical work owned by `ContextualWisdomLab/fast-mlsirm`. +This package creates an immutable **selection-validity analysis handoff** and validates the matching numerical result envelope. It is the boundary between Orgmetra's authoritative validation-study evidence and numerical work owned by `ContextualWisdomLab/fast-mlsirm`. ## What it does @@ -8,9 +8,12 @@ This package creates an immutable **selection-validity analysis handoff**. It is The resulting canonical JSON is digest-addressable, contains no raw person-level predictor or criterion values, and remains `not_executed`. Required result evidence is explicit: effect estimate, uncertainty interval, sample size, missingness summary, and convergence diagnostics. +`ValidationAnalysisResult` accepts only a result linked to the handoff digest and the same reviewed fast-mlsirm revision. It records the Rust CPU/GPU backend, precision, aggregate missingness counts, finite effect and interval values, and explicit convergence or nonconvergence diagnostics. It never promotes a result to an employment decision; human review remains mandatory. + ## What it does not do - It does **not** run statistics. +- It does **not** run or reproduce the fast-mlsirm numerical kernel. - It does **not** query fast-mlsirm or any other CWL application's database. - It does **not** claim that a selection procedure is valid. - It does **not** interpret adverse impact. diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py b/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py index d7d15691c..81acbc0d1 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py @@ -1,13 +1,17 @@ -"""Public governed selection-validity analysis handoff contract.""" +"""Public governed selection-validity analysis handoff and result contracts.""" from .handoff import ( REVIEWED_FAST_MLSIRM_REVISION, ValidationAnalysisHandoff, build_validation_analysis_handoff, ) +from .result import ConvergenceDiagnostics, MissingnessSummary, ValidationAnalysisResult __all__ = [ "REVIEWED_FAST_MLSIRM_REVISION", "ValidationAnalysisHandoff", "build_validation_analysis_handoff", + "ConvergenceDiagnostics", + "MissingnessSummary", + "ValidationAnalysisResult", ] diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py new file mode 100644 index 000000000..4f4d230e5 --- /dev/null +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py @@ -0,0 +1,231 @@ +"""Validate one immutable numerical result returned by the approved worker. + +Orgmetra does not fit a model in this package. It accepts only a bounded, +digest-linked result envelope from the pinned ``fast-mlsirm`` worker so that +nonconverged or malformed output cannot be presented as an employment +decision. +""" +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from hashlib import sha256 +import json +from math import isfinite +from numbers import Real + +from .handoff import ( + REVIEWED_FAST_MLSIRM_REVISION, + _canonical_timestamp, + _validate_code, + _validate_digest, + _validate_kernel_revision, + _validate_operational_uuid, + _validate_reference, +) + +_RESULT_AUTHORITY = "scientific_evidence_only" +_EXECUTION_STATE = "completed" +_ALLOWED_BACKENDS = frozenset({"rust_cpu", "rust_gpu"}) +_ALLOWED_PRECISIONS = frozenset({"f64", "f32"}) + + +def _validate_nonnegative_integer(value: object, field_name: str) -> None: + """Require a real non-negative integer without accepting booleans.""" + if type(value) is not int or value < 0: + raise ValueError(f"{field_name} must be a non-negative integer") + + +def _validate_positive_integer(value: object, field_name: str) -> None: + """Require a real positive integer without accepting booleans.""" + if type(value) is not int or value <= 0: + raise ValueError(f"{field_name} must be a positive integer") + + +def _finite_number(value: object, field_name: str) -> float: + """Return one finite real number and reject booleans or non-numeric text.""" + if isinstance(value, bool) or not isinstance(value, Real): + raise ValueError(f"{field_name} must be a finite number") + number = float(value) + if not isfinite(number): + raise ValueError(f"{field_name} must be a finite number") + return number + + +@dataclass(frozen=True, slots=True) +class MissingnessSummary: + """Describe missingness counts without carrying person-level observations.""" + + total_observations: int + complete_observations: int + missing_predictor_observations: int + missing_criterion_observations: int + + def __post_init__(self) -> None: + """Reject impossible counts before a result can be correlated.""" + for field_name in ( + "total_observations", + "complete_observations", + "missing_predictor_observations", + "missing_criterion_observations", + ): + _validate_nonnegative_integer(getattr(self, field_name), field_name) + if self.total_observations == 0: + raise ValueError("total_observations must be positive") + if self.complete_observations > self.total_observations: + raise ValueError("complete_observations cannot exceed total_observations") + if self.missing_predictor_observations > self.total_observations: + raise ValueError("missing_predictor_observations cannot exceed total_observations") + if self.missing_criterion_observations > self.total_observations: + raise ValueError("missing_criterion_observations cannot exceed total_observations") + + def to_dict(self) -> dict[str, int]: + """Return deterministic count fields for the canonical result JSON.""" + return { + "complete_observations": self.complete_observations, + "missing_criterion_observations": self.missing_criterion_observations, + "missing_predictor_observations": self.missing_predictor_observations, + "total_observations": self.total_observations, + } + + +@dataclass(frozen=True, slots=True) +class ConvergenceDiagnostics: + """Record convergence evidence while preserving an explicit failure state.""" + + converged: bool + iterations: int + objective_value: Real + maximum_gradient: Real + failure_code: str | None = None + + def __post_init__(self) -> None: + """Require diagnostics that distinguish convergence from a failed fit.""" + if type(self.converged) is not bool: + raise ValueError("converged must be a boolean") + _validate_positive_integer(self.iterations, "iterations") + _finite_number(self.objective_value, "objective_value") + gradient = _finite_number(self.maximum_gradient, "maximum_gradient") + if gradient < 0: + raise ValueError("maximum_gradient must be non-negative") + if self.converged and self.failure_code is not None: + raise ValueError("failure_code must be absent for a converged result") + if not self.converged and ( + not isinstance(self.failure_code, str) or not self.failure_code + ): + raise ValueError("failure_code is required for a nonconverged result") + if self.failure_code is not None: + _validate_code(self.failure_code, "failure_code") + + def to_dict(self) -> dict[str, object]: + """Return deterministic convergence fields for the canonical result JSON.""" + payload: dict[str, object] = { + "converged": self.converged, + "iterations": self.iterations, + "maximum_gradient": float(self.maximum_gradient), + "objective_value": float(self.objective_value), + } + if self.failure_code is not None: + payload["failure_code"] = self.failure_code + return payload + + +@dataclass(frozen=True, slots=True, repr=False) +class ValidationAnalysisResult: + """Immutable, digest-linked scientific evidence returned by the offline worker.""" + + tenant_record_id: str + result_reference: str + handoff_digest: str + provenance_digest: str + fast_mlsirm_revision: str + model_code: str + backend: str + precision: str + effect_estimate: Real + uncertainty_lower: Real + uncertainty_upper: Real + sample_size: int + missingness_summary: MissingnessSummary + convergence_diagnostics: ConvergenceDiagnostics + completed_at: datetime + result_authority: str = _RESULT_AUTHORITY + execution_state: str = _EXECUTION_STATE + contains_raw_person_level_values: bool = False + human_review_required: bool = True + evidence_version: int = 1 + + def __post_init__(self) -> None: + """Fail closed on malformed, unlinked, or decision-like result data.""" + _validate_operational_uuid(self.tenant_record_id, "tenant_record_id") + _validate_reference(self.result_reference, "validation_analysis_result", "result_reference") + _validate_digest(self.handoff_digest, "handoff_digest") + _validate_digest(self.provenance_digest, "provenance_digest") + _validate_kernel_revision(self.fast_mlsirm_revision) + _validate_code(self.model_code, "model_code") + if self.backend not in _ALLOWED_BACKENDS: + raise ValueError("backend must be rust_cpu or rust_gpu") + if self.precision not in _ALLOWED_PRECISIONS: + raise ValueError("precision must be f64 or f32") + estimate = _finite_number(self.effect_estimate, "effect_estimate") + lower = _finite_number(self.uncertainty_lower, "uncertainty_lower") + upper = _finite_number(self.uncertainty_upper, "uncertainty_upper") + if lower > upper: + raise ValueError("uncertainty_lower cannot exceed uncertainty_upper") + if not lower <= estimate <= upper: + raise ValueError("effect_estimate must be inside the uncertainty interval") + _validate_positive_integer(self.sample_size, "sample_size") + if not isinstance(self.missingness_summary, MissingnessSummary): + raise ValueError("missingness_summary must be a MissingnessSummary") + if not isinstance(self.convergence_diagnostics, ConvergenceDiagnostics): + raise ValueError("convergence_diagnostics must be ConvergenceDiagnostics") + if self.sample_size != self.missingness_summary.total_observations: + raise ValueError("sample_size must match total_observations") + _canonical_timestamp(self.completed_at) + if self.result_authority != _RESULT_AUTHORITY: + raise ValueError("result_authority must remain scientific_evidence_only") + if self.execution_state != _EXECUTION_STATE: + raise ValueError("execution_state must remain completed") + if self.contains_raw_person_level_values is not False: + raise ValueError("result must not contain raw person-level values") + if self.human_review_required is not True: + raise ValueError("human review is mandatory for validity interpretation") + if type(self.evidence_version) is not int or self.evidence_version != 1: + raise ValueError("evidence_version must remain 1") + + def __repr__(self) -> str: + """Return a redacted representation suitable for routine application logs.""" + return "ValidationAnalysisResult()" + + def canonical_json(self) -> str: + """Return deterministic, non-person-level JSON for audit correlation.""" + payload = { + "backend": self.backend, + "completed_at": _canonical_timestamp(self.completed_at), + "contains_raw_person_level_values": self.contains_raw_person_level_values, + "convergence_diagnostics": self.convergence_diagnostics.to_dict(), + "effect_estimate": float(self.effect_estimate), + "evidence_version": self.evidence_version, + "execution_state": self.execution_state, + "fast_mlsirm_revision": self.fast_mlsirm_revision, + "handoff_digest": self.handoff_digest, + "human_review_required": self.human_review_required, + "missingness_summary": self.missingness_summary.to_dict(), + "model_code": self.model_code, + "precision": self.precision, + "provenance_digest": self.provenance_digest, + "result_authority": self.result_authority, + "result_reference": self.result_reference, + "sample_size": self.sample_size, + "tenant_record_id": self.tenant_record_id, + "uncertainty_lower": float(self.uncertainty_lower), + "uncertainty_upper": float(self.uncertainty_upper), + } + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + def sha256_digest(self) -> str: + """Return SHA-256 over the exact canonical result bytes.""" + return sha256(self.canonical_json().encode("utf-8")).hexdigest() + + +__all__ = ["ConvergenceDiagnostics", "MissingnessSummary", "ValidationAnalysisResult"] diff --git a/packages/validity-analysis/tests/test_result.py b/packages/validity-analysis/tests/test_result.py new file mode 100644 index 000000000..c1f841080 --- /dev/null +++ b/packages/validity-analysis/tests/test_result.py @@ -0,0 +1,178 @@ +"""Regression tests for the bounded numerical result contract.""" + +from dataclasses import asdict, replace +from datetime import datetime, timezone +import json + +import pytest + +from orgmetra_validity_analysis import ( + ConvergenceDiagnostics, + MissingnessSummary, + REVIEWED_FAST_MLSIRM_REVISION, + ValidationAnalysisResult, +) + +TENANT = "10000000-0000-7000-8000-000000000001" +RESULT = "validation_analysis_result:11111111-1111-4111-8111-111111111111" +HANDOFF_DIGEST = "a" * 64 +PROVENANCE_DIGEST = "b" * 64 +COMPLETED_AT = datetime(2026, 8, 21, 7, 10, 11, 123456, tzinfo=timezone.utc) + + +def missingness() -> MissingnessSummary: + """Return one realistic aggregate-only missingness summary.""" + return MissingnessSummary( + total_observations=12, + complete_observations=10, + missing_predictor_observations=1, + missing_criterion_observations=1, + ) + + +def convergence(*, converged: bool = True) -> ConvergenceDiagnostics: + """Return one converged or explicitly nonconverged diagnostic record.""" + return ConvergenceDiagnostics( + converged=converged, + iterations=42, + objective_value=-12.5, + maximum_gradient=0.0001, + failure_code=None if converged else "maximum_iterations", + ) + + +def result(**overrides: object) -> ValidationAnalysisResult: + """Build one valid result envelope and apply targeted test overrides.""" + values: dict[str, object] = { + "tenant_record_id": TENANT, + "result_reference": RESULT, + "handoff_digest": HANDOFF_DIGEST, + "provenance_digest": PROVENANCE_DIGEST, + "fast_mlsirm_revision": REVIEWED_FAST_MLSIRM_REVISION, + "model_code": "mlsirm_criterion_related", + "backend": "rust_cpu", + "precision": "f64", + "effect_estimate": 0.42, + "uncertainty_lower": 0.10, + "uncertainty_upper": 0.70, + "sample_size": 12, + "missingness_summary": missingness(), + "convergence_diagnostics": convergence(), + "completed_at": COMPLETED_AT, + } + values.update(overrides) + return ValidationAnalysisResult(**values) + + +def test_aggregate_evidence_is_deterministic_and_redacted() -> None: + """Serialize only aggregate evidence and preserve exact replay bytes.""" + candidate = result() + payload = json.loads(candidate.canonical_json()) + + assert payload["tenant_record_id"] == TENANT + assert payload["backend"] == "rust_cpu" + assert payload["precision"] == "f64" + assert payload["execution_state"] == "completed" + assert payload["result_authority"] == "scientific_evidence_only" + assert payload["missingness_summary"]["total_observations"] == 12 + assert payload["convergence_diagnostics"]["converged"] is True + assert "person_record" not in candidate.canonical_json() + assert repr(candidate) == "ValidationAnalysisResult()" + assert len(candidate.sha256_digest()) == 64 + assert candidate.canonical_json() == result().canonical_json() + + +def test_gpu_and_nonconverged_result_are_explicitly_typed() -> None: + """Record GPU provenance and a typed nonconvergence state without promotion.""" + candidate = result( + backend="rust_gpu", + precision="f32", + convergence_diagnostics=convergence(converged=False), + ) + payload = json.loads(candidate.canonical_json()) + assert payload["backend"] == "rust_gpu" + assert payload["precision"] == "f32" + assert payload["convergence_diagnostics"]["failure_code"] == "maximum_iterations" + + +@pytest.mark.parametrize( + "bad", + [ + {"total_observations": True}, + {"total_observations": -1}, + {"total_observations": 0}, + {"complete_observations": 13}, + {"missing_predictor_observations": 13}, + {"missing_criterion_observations": 13}, + ], +) +def test_missingness_rejects_invalid_counts(bad: dict[str, object]) -> None: + """Reject booleans, negative counts, empty samples, and impossible totals.""" + values = asdict(missingness()) + values.update(bad) + with pytest.raises(ValueError): + MissingnessSummary(**values) + + +@pytest.mark.parametrize("bad", [True, 0, -1]) +def test_positive_integer_validation_rejects_nonpositive_values(bad: object) -> None: + """Exercise strict sample and iteration bounds.""" + with pytest.raises(ValueError, match="positive integer"): + ConvergenceDiagnostics(True, bad, -1.0, 0.1) + with pytest.raises(ValueError, match="positive integer"): + result(sample_size=bad) + + +@pytest.mark.parametrize("bad", [True, "0.1", float("nan"), float("inf")]) +def test_numeric_fields_reject_boolean_text_and_nonfinite_values(bad: object) -> None: + """Do not accept values that cannot be represented as finite scientific evidence.""" + with pytest.raises(ValueError, match="finite number"): + ConvergenceDiagnostics(True, 1, bad, 0.1) + with pytest.raises(ValueError, match="finite number"): + result(effect_estimate=bad) + + +def test_negative_gradient_and_invalid_convergence_states_fail_closed() -> None: + """Require explicit and internally consistent convergence diagnostics.""" + with pytest.raises(ValueError, match="non-negative"): + ConvergenceDiagnostics(True, 1, 1.0, -0.1) + with pytest.raises(ValueError, match="boolean"): + ConvergenceDiagnostics(1, 1, 1.0, 0.1) + with pytest.raises(ValueError, match="absent"): + ConvergenceDiagnostics(True, 1, 1.0, 0.1, "failed_fit") + with pytest.raises(ValueError, match="required"): + ConvergenceDiagnostics(False, 1, 1.0, 0.1) + with pytest.raises(ValueError, match="required"): + ConvergenceDiagnostics(False, 1, 1.0, 0.1, "") + + +@pytest.mark.parametrize( + "field,bad,match", + [ + ("backend", "numpy", "backend"), + ("precision", "float16", "precision"), + ("uncertainty_lower", 0.8, "uncertainty_lower"), + ("uncertainty_upper", 0.0, "uncertainty_lower"), + ("effect_estimate", 0.8, "effect_estimate"), + ("sample_size", 11, "sample_size"), + ("result_authority", "employment_decision", "result_authority"), + ("execution_state", "not_executed", "execution_state"), + ("contains_raw_person_level_values", True, "raw person-level"), + ("human_review_required", False, "human review"), + ("evidence_version", 2, "evidence_version"), + ], +) +def test_result_invariants_cannot_be_weakened(field: str, bad: object, match: str) -> None: + """Reject malformed intervals, lineage, or governance flags.""" + with pytest.raises(ValueError, match=match): + replace(result(), **{field: bad}) + + +def test_result_requires_canonical_timestamp_and_aggregate_types() -> None: + """Reject a naive completion time and non-summary diagnostic objects.""" + with pytest.raises(ValueError, match="requested_at"): + result(completed_at=datetime(2026, 8, 21, 7, 10)) + with pytest.raises(ValueError, match="missingness_summary"): + result(missingness_summary=object()) + with pytest.raises(ValueError, match="convergence_diagnostics"): + result(convergence_diagnostics=object()) From 0a3173784eeea69a12e3384fbcc3679b5b9e2b6b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 20:03:48 -0700 Subject: [PATCH 014/158] test(validity): reject impossible complete missingness totals --- packages/validity-analysis/tests/test_result.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/validity-analysis/tests/test_result.py b/packages/validity-analysis/tests/test_result.py index c1f841080..a75518ee8 100644 --- a/packages/validity-analysis/tests/test_result.py +++ b/packages/validity-analysis/tests/test_result.py @@ -104,6 +104,8 @@ def test_gpu_and_nonconverged_result_are_explicitly_typed() -> None: {"complete_observations": 13}, {"missing_predictor_observations": 13}, {"missing_criterion_observations": 13}, + {"complete_observations": 12, "missing_predictor_observations": 1}, + {"complete_observations": 12, "missing_criterion_observations": 1}, ], ) def test_missingness_rejects_invalid_counts(bad: dict[str, object]) -> None: From 2419aa98eeeb67add4ccdd92c5b494479b0cc234 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 20:04:16 -0700 Subject: [PATCH 015/158] fix(validity): reject impossible missingness summaries --- .../src/orgmetra_validity_analysis/result.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py index 4f4d230e5..4ccce8e66 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py @@ -78,6 +78,14 @@ def __post_init__(self) -> None: raise ValueError("missing_predictor_observations cannot exceed total_observations") if self.missing_criterion_observations > self.total_observations: raise ValueError("missing_criterion_observations cannot exceed total_observations") + if self.complete_observations + self.missing_predictor_observations > self.total_observations: + raise ValueError( + "complete_observations and missing_predictor_observations cannot overlap" + ) + if self.complete_observations + self.missing_criterion_observations > self.total_observations: + raise ValueError( + "complete_observations and missing_criterion_observations cannot overlap" + ) def to_dict(self) -> dict[str, int]: """Return deterministic count fields for the canonical result JSON.""" From c17936ce1ebae673208f8c76dcee90cb2ae80285 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 20:05:06 -0700 Subject: [PATCH 016/158] test(validity): reject subclassed result evidence --- .../validity-analysis/tests/test_result.py | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/packages/validity-analysis/tests/test_result.py b/packages/validity-analysis/tests/test_result.py index a75518ee8..4209cb203 100644 --- a/packages/validity-analysis/tests/test_result.py +++ b/packages/validity-analysis/tests/test_result.py @@ -171,10 +171,37 @@ def test_result_invariants_cannot_be_weakened(field: str, bad: object, match: st def test_result_requires_canonical_timestamp_and_aggregate_types() -> None: - """Reject a naive completion time and non-summary diagnostic objects.""" + """Reject naive times, non-contract objects, and subclass method overrides.""" + + class LeakyMissingnessSummary(MissingnessSummary): + def to_dict(self) -> dict[str, object]: + return {**super().to_dict(), "person_record": "must-not-serialize"} + + class LeakyConvergenceDiagnostics(ConvergenceDiagnostics): + def to_dict(self) -> dict[str, object]: + return {**super().to_dict(), "employment_decision": "auto_reject"} + with pytest.raises(ValueError, match="requested_at"): result(completed_at=datetime(2026, 8, 21, 7, 10)) with pytest.raises(ValueError, match="missingness_summary"): result(missingness_summary=object()) with pytest.raises(ValueError, match="convergence_diagnostics"): result(convergence_diagnostics=object()) + with pytest.raises(ValueError, match="missingness_summary"): + result( + missingness_summary=LeakyMissingnessSummary( + total_observations=12, + complete_observations=10, + missing_predictor_observations=1, + missing_criterion_observations=1, + ) + ) + with pytest.raises(ValueError, match="convergence_diagnostics"): + result( + convergence_diagnostics=LeakyConvergenceDiagnostics( + converged=True, + iterations=42, + objective_value=-12.5, + maximum_gradient=0.0001, + ) + ) From f31422a473c59a686e4195e592ae629ad7a7eac9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 20:05:29 -0700 Subject: [PATCH 017/158] fix(validity): require exact result evidence types --- .../src/orgmetra_validity_analysis/result.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py index 4ccce8e66..25ce3410e 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py @@ -183,9 +183,9 @@ def __post_init__(self) -> None: if not lower <= estimate <= upper: raise ValueError("effect_estimate must be inside the uncertainty interval") _validate_positive_integer(self.sample_size, "sample_size") - if not isinstance(self.missingness_summary, MissingnessSummary): + if type(self.missingness_summary) is not MissingnessSummary: raise ValueError("missingness_summary must be a MissingnessSummary") - if not isinstance(self.convergence_diagnostics, ConvergenceDiagnostics): + if type(self.convergence_diagnostics) is not ConvergenceDiagnostics: raise ValueError("convergence_diagnostics must be ConvergenceDiagnostics") if self.sample_size != self.missingness_summary.total_observations: raise ValueError("sample_size must match total_observations") From e11ae2a062364911fac830293906366fc53a3915 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 20:05:47 -0700 Subject: [PATCH 018/158] docs(validity): record result evidence hardening --- packages/validity-analysis/CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/validity-analysis/CHANGELOG.md b/packages/validity-analysis/CHANGELOG.md index 95cab8f32..9e9e67b43 100644 --- a/packages/validity-analysis/CHANGELOG.md +++ b/packages/validity-analysis/CHANGELOG.md @@ -7,3 +7,5 @@ - Require separate requester/reviewer references and authoritative tenant-scoped re-resolution proving they resolve to distinct actor identities before execution. - Require deterministic canonical evidence and 100% owned production statement/branch coverage. - Validate a digest-linked Rust CPU/GPU result envelope with finite estimates, aggregate missingness, and explicit convergence or nonconvergence diagnostics. +- Reject impossible aggregate missingness where complete observations overlap either predictor-missing or criterion-missing counts beyond the sample total. +- Require exact governed missingness/convergence runtime types so subclass method overrides cannot inject unreviewed or person-level fields into canonical result evidence. From e9a7a31fc9066e6ba6e8f81954941c8efa795842 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 20:05:56 -0700 Subject: [PATCH 019/158] docs(validity): define hardened result envelope --- packages/validity-analysis/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/validity-analysis/README.md b/packages/validity-analysis/README.md index feec8db9a..527241d5b 100644 --- a/packages/validity-analysis/README.md +++ b/packages/validity-analysis/README.md @@ -8,7 +8,7 @@ This package creates an immutable **selection-validity analysis handoff** and va The resulting canonical JSON is digest-addressable, contains no raw person-level predictor or criterion values, and remains `not_executed`. Required result evidence is explicit: effect estimate, uncertainty interval, sample size, missingness summary, and convergence diagnostics. -`ValidationAnalysisResult` accepts only a result linked to the handoff digest and the same reviewed fast-mlsirm revision. It records the Rust CPU/GPU backend, precision, aggregate missingness counts, finite effect and interval values, and explicit convergence or nonconvergence diagnostics. It never promotes a result to an employment decision; human review remains mandatory. +`ValidationAnalysisResult` accepts only a result linked to the handoff digest and the same reviewed fast-mlsirm revision. It records the Rust CPU/GPU backend, precision, aggregate missingness counts, finite effect and interval values, and explicit convergence or nonconvergence diagnostics. Missingness counts must be internally possible: complete observations cannot overlap either predictor-missing or criterion-missing observations beyond the declared sample total. The result envelope accepts only the exact governed `MissingnessSummary` and `ConvergenceDiagnostics` runtime types, preventing subclass method overrides from adding unreviewed or person-level fields to canonical audit evidence. It never promotes a result to an employment decision; human review remains mandatory. ## What it does not do From 8b265203c04b24ca3e7526b81914e1ac03776d83 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 20:06:08 -0700 Subject: [PATCH 020/158] docs(validity): trace result integrity regressions --- docs/traceability/validation-analysis-handoff.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/traceability/validation-analysis-handoff.md b/docs/traceability/validation-analysis-handoff.md index 068886737..af8f6b622 100644 --- a/docs/traceability/validation-analysis-handoff.md +++ b/docs/traceability/validation-analysis-handoff.md @@ -10,10 +10,10 @@ Can an organization send one exact, reviewable validation study to its statistic |---|---|---| | Exact study scope | tenant, validation-study, Job, predictor, criterion, population, decision-policy, and analysis-plan references plus digests | namespace/UUID/digest regressions | | Dependency integrity | immutable fast-mlsirm commit `04d0bc21a2a20693bcf16108cd76d394fe844d23` | malformed and unreviewed revision rejection | -| Privacy minimization | no raw person-level values in canonical handoff | canonical-payload regression and redacted repr | +| Privacy minimization | no raw person-level values in canonical handoff or result; result canonicalization accepts only exact governed missingness/convergence runtime types | canonical-payload/redacted-repr regressions plus subclass-injection rejection | | Human authority | requester/reviewer references must differ, and the host must re-resolve both within the tenant and prove they resolve to distinct authoritative actor identities before execution | direct-construction fail-closed regression plus `test_next_action_requires_resolved_actor_identity_separation` | -| Scientific evidence | effect estimate, uncertainty interval, sample size, missingness summary, convergence diagnostics | immutable required-result-evidence regression | -| Numerical result boundary | handoff digest, pinned fast-mlsirm revision, Rust CPU/GPU backend, precision, finite estimate/interval, aggregate missingness, explicit convergence state | `ValidationAnalysisResult` invariant and canonicalization regressions | +| Scientific evidence | effect estimate, uncertainty interval, sample size, internally possible aggregate missingness, convergence diagnostics | immutable required-result-evidence regression plus impossible-missingness rejection | +| Numerical result boundary | handoff digest, pinned fast-mlsirm revision, Rust CPU/GPU backend, precision, finite estimate/interval, aggregate missingness, explicit convergence state | `ValidationAnalysisResult` invariant/canonicalization regressions and exact-runtime-type checks | | Execution boundary | `not_executed`, `scientific_evidence_only`, read-only pinned foreign dependency | immutable governance regressions | | Reproducibility | canonical RFC 3339 time, canonical JSON, SHA-256 handoff digest | deterministic serialization/digest tests | @@ -21,4 +21,4 @@ Can an organization send one exact, reviewable validation study to its statistic `implemented_on_active_pr`. -Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. The active package now validates the minimum returned numerical/provenance envelope, but protected Orgmetra evidence still requires host re-resolution, result-artifact verification, terminal checks, independent review, and accountable human interpretation. +Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. The active package now validates the minimum returned numerical/provenance envelope, including missingness consistency and exact aggregate-evidence runtime types, but protected Orgmetra evidence still requires host re-resolution, result-artifact verification, terminal checks, independent review, and accountable human interpretation. From b33cbdaed094e1ac79f009b193f05debdb55323e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 20:06:25 -0700 Subject: [PATCH 021/158] docs(validity): bind result integrity in ADR --- .../0027-governed-selection-validity-analysis-handoff.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/adr/0027-governed-selection-validity-analysis-handoff.md b/docs/adr/0027-governed-selection-validity-analysis-handoff.md index a27a0b09d..6db6c3a1c 100644 --- a/docs/adr/0027-governed-selection-validity-analysis-handoff.md +++ b/docs/adr/0027-governed-selection-validity-analysis-handoff.md @@ -26,7 +26,7 @@ Orgmetra adds a leaf `orgmetra_validity_analysis` package whose `ValidationAnaly - remains `not_executed`, `scientific_evidence_only`, and human-review-required; - produces deterministic canonical JSON and a SHA-256 digest for audit/result correlation. -The same package also validates `ValidationAnalysisResult` envelopes returned by the approved offline worker. A result must link to the handoff digest and the same pinned revision, identify a Rust CPU or GPU backend and precision, provide finite effect and interval values, match its sample size to aggregate missingness counts, and include explicit convergence diagnostics. A nonconverged result remains typed scientific evidence requiring human review; it cannot be treated as a valid selection procedure or an employment decision. +The same package also validates `ValidationAnalysisResult` envelopes returned by the approved offline worker. A result must link to the handoff digest and the same pinned revision, identify a Rust CPU or GPU backend and precision, provide finite effect and interval values, match its sample size to aggregate missingness counts, reject impossible complete-versus-missing count combinations, and include explicit convergence diagnostics. The canonicalization boundary accepts only the exact governed `MissingnessSummary` and `ConvergenceDiagnostics` runtime types so subclass method overrides cannot add unreviewed or person-level fields to immutable result evidence. A nonconverged result remains typed scientific evidence requiring human review; it cannot be treated as a valid selection procedure or an employment decision. The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, and prove requester/reviewer identities are distinct authoritative actors. @@ -36,6 +36,8 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - Statistical work cannot silently drift to an unreviewed fast-mlsirm revision. - A buyer or auditor can identify exactly which governed study evidence a result was supposed to analyze without copying raw worker data into the handoff. +- Aggregate missingness evidence cannot claim all observations are complete while simultaneously reporting predictor- or criterion-missing observations. +- Result canonicalization cannot be extended by an unreviewed subclass to serialize extra decision-like or person-level fields. - Human interpretation remains explicit and separate from numerical output. - The dedicated-writer boundary remains intact: Orgmetra consumes only a pinned foreign revision/contract boundary and never mutates fast-mlsirm. @@ -47,7 +49,7 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later ## Verification -The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, and 100% owned production statement/branch coverage. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number. +The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number. ## References From 081942a723c2ae2cb9bb98a8ffd0b33b72a8bd74 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 12:11:13 +0900 Subject: [PATCH 022/158] test(validity): cover criterion overlap guard --- packages/validity-analysis/tests/test_result.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/validity-analysis/tests/test_result.py b/packages/validity-analysis/tests/test_result.py index 4209cb203..ab71fdef3 100644 --- a/packages/validity-analysis/tests/test_result.py +++ b/packages/validity-analysis/tests/test_result.py @@ -105,7 +105,11 @@ def test_gpu_and_nonconverged_result_are_explicitly_typed() -> None: {"missing_predictor_observations": 13}, {"missing_criterion_observations": 13}, {"complete_observations": 12, "missing_predictor_observations": 1}, - {"complete_observations": 12, "missing_criterion_observations": 1}, + { + "complete_observations": 12, + "missing_predictor_observations": 0, + "missing_criterion_observations": 1, + }, ], ) def test_missingness_rejects_invalid_counts(bad: dict[str, object]) -> None: From d6bb5f27603efbebe31a14d4a21e919de5d01cce Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 21:22:20 -0700 Subject: [PATCH 023/158] test(validity): reject temporal evidence subclasses --- .../tests/test_temporal_evidence_integrity.py | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 packages/validity-analysis/tests/test_temporal_evidence_integrity.py diff --git a/packages/validity-analysis/tests/test_temporal_evidence_integrity.py b/packages/validity-analysis/tests/test_temporal_evidence_integrity.py new file mode 100644 index 000000000..e47628263 --- /dev/null +++ b/packages/validity-analysis/tests/test_temporal_evidence_integrity.py @@ -0,0 +1,84 @@ +"""Regression coverage for selection-validity temporal evidence integrity.""" + +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest + +from orgmetra_validity_analysis import ( + ConvergenceDiagnostics, + MissingnessSummary, + REVIEWED_FAST_MLSIRM_REVISION, + ValidationAnalysisResult, + build_validation_analysis_handoff, +) + + +class ForgedDateTime(datetime): + """Datetime subclass able to forge canonical validation evidence.""" + + def astimezone(self, tz=None): # type: ignore[no-untyped-def] + """Keep the hostile subclass alive across UTC normalization.""" + return self + + def isoformat(self, *args, **kwargs) -> str: # type: ignore[no-untyped-def] + """Return an instant different from the underlying evidence instant.""" + return "2099-12-31T23:59:59+00:00" + + +def test_handoff_rejects_datetime_subclass_that_can_forge_requested_at() -> None: + """Handoff canonical evidence must not invoke caller-overridable datetime methods.""" + with pytest.raises(ValueError, match="requested_at"): + build_validation_analysis_handoff( + tenant_record_id="10000000-0000-7000-8000-000000000001", + handoff_reference="validation_analysis_handoff:11111111-1111-4111-8111-111111111111", + validation_study_reference="validation_study:22222222-2222-4222-8222-222222222222", + job_profile_reference="job_profile:33333333-3333-4333-8333-333333333333", + predictor_snapshot_reference="predictor_snapshot:44444444-4444-4444-8444-444444444444", + predictor_snapshot_digest="a" * 64, + criterion_snapshot_reference="criterion_snapshot:55555555-5555-4555-8555-555555555555", + criterion_snapshot_digest="b" * 64, + population_snapshot_reference="study_population_snapshot:66666666-6666-4666-8666-666666666666", + population_snapshot_digest="c" * 64, + decision_policy_reference="decision_policy:77777777-7777-4777-8777-777777777777", + decision_policy_digest="d" * 64, + analysis_plan_reference="validation_analysis_plan:88888888-8888-4888-8888-888888888888", + analysis_plan_digest="e" * 64, + actor_reference="actor:99999999-9999-4999-8999-999999999999", + reviewer_reference="actor:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + fast_mlsirm_revision=REVIEWED_FAST_MLSIRM_REVISION, + requested_at=ForgedDateTime(2026, 8, 21, 4, 45, tzinfo=timezone.utc), + ) + + +def test_result_rejects_datetime_subclass_that_can_forge_completed_at() -> None: + """Result canonical evidence must not invoke caller-overridable datetime methods.""" + with pytest.raises(ValueError, match="requested_at"): + ValidationAnalysisResult( + tenant_record_id="10000000-0000-7000-8000-000000000001", + result_reference="validation_analysis_result:11111111-1111-4111-8111-111111111111", + handoff_digest="a" * 64, + provenance_digest="b" * 64, + fast_mlsirm_revision=REVIEWED_FAST_MLSIRM_REVISION, + model_code="mlsirm_criterion_related", + backend="rust_cpu", + precision="f64", + effect_estimate=0.42, + uncertainty_lower=0.10, + uncertainty_upper=0.70, + sample_size=12, + missingness_summary=MissingnessSummary( + total_observations=12, + complete_observations=10, + missing_predictor_observations=1, + missing_criterion_observations=1, + ), + convergence_diagnostics=ConvergenceDiagnostics( + converged=True, + iterations=42, + objective_value=-12.5, + maximum_gradient=0.0001, + ), + completed_at=ForgedDateTime(2026, 8, 21, 4, 45, tzinfo=timezone.utc), + ) From b616a1306bcbf4d77c83708ea9db93158f7d06b3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 21:23:02 -0700 Subject: [PATCH 024/158] fix(validity): require exact temporal evidence type --- .../src/orgmetra_validity_analysis/handoff.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py b/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py index ad257a098..536f96e01 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py @@ -96,8 +96,8 @@ def _validate_kernel_revision(value: str) -> None: def _canonical_timestamp(value: datetime) -> str: - """Render an aware instant as precision-preserving UTC RFC 3339 text.""" - if not isinstance(value, datetime) or value.tzinfo is None or value.utcoffset() is None: + """Render an exact built-in aware instant as precision-preserving UTC RFC 3339 text.""" + if type(value) is not datetime or value.tzinfo is None or value.utcoffset() is None: raise ValueError("requested_at must be timezone-aware") return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") From 219c2e3ba58f4156645317cf5b71a5e3804a8678 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 02:20:20 -0700 Subject: [PATCH 025/158] fix(validity): remove unused kernel revision import --- .../validity-analysis/src/orgmetra_validity_analysis/result.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py index 25ce3410e..cea4c56e3 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py @@ -15,7 +15,6 @@ from numbers import Real from .handoff import ( - REVIEWED_FAST_MLSIRM_REVISION, _canonical_timestamp, _validate_code, _validate_digest, From 89c970772fd2ee8328ffd6df9836c7fc68745ff5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 17:16:46 -0700 Subject: [PATCH 026/158] test(validity): require completed_at diagnostics --- packages/validity-analysis/tests/test_result.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/validity-analysis/tests/test_result.py b/packages/validity-analysis/tests/test_result.py index ab71fdef3..ad00b95f2 100644 --- a/packages/validity-analysis/tests/test_result.py +++ b/packages/validity-analysis/tests/test_result.py @@ -185,7 +185,7 @@ class LeakyConvergenceDiagnostics(ConvergenceDiagnostics): def to_dict(self) -> dict[str, object]: return {**super().to_dict(), "employment_decision": "auto_reject"} - with pytest.raises(ValueError, match="requested_at"): + with pytest.raises(ValueError, match="completed_at"): result(completed_at=datetime(2026, 8, 21, 7, 10)) with pytest.raises(ValueError, match="missingness_summary"): result(missingness_summary=object()) From b72c7a0c267c316b0c0d8189388126df13355d14 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 17:17:23 -0700 Subject: [PATCH 027/158] fix(validity): report field-correct timestamp errors --- .../src/orgmetra_validity_analysis/handoff.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py b/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py index 536f96e01..a77b280c6 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py @@ -95,10 +95,10 @@ def _validate_kernel_revision(value: str) -> None: raise ValueError("fast_mlsirm_revision must equal the reviewed immutable revision") -def _canonical_timestamp(value: datetime) -> str: - """Render an exact built-in aware instant as precision-preserving UTC RFC 3339 text.""" +def _canonical_timestamp(value: datetime, field_name: str) -> str: + """Render an exact built-in aware instant with field-correct diagnostics.""" if type(value) is not datetime or value.tzinfo is None or value.utcoffset() is None: - raise ValueError("requested_at must be timezone-aware") + raise ValueError(f"{field_name} must be timezone-aware") return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") @@ -164,7 +164,7 @@ def __post_init__(self) -> None: if self.actor_reference == self.reviewer_reference: raise ValueError("reviewer_reference must identify a different accountable actor") _validate_kernel_revision(self.fast_mlsirm_revision) - _canonical_timestamp(self.requested_at) + _canonical_timestamp(self.requested_at, "requested_at") _validate_code(self.purpose_code, "purpose_code") if self.purpose_code != _PURPOSE_CODE: raise ValueError("purpose_code must remain selection_validity_analysis") @@ -222,7 +222,7 @@ def canonical_json(self) -> str: "predictor_snapshot_reference": self.predictor_snapshot_reference, "purpose_code": self.purpose_code, "reason_code": self.reason_code, - "requested_at": _canonical_timestamp(self.requested_at), + "requested_at": _canonical_timestamp(self.requested_at, "requested_at"), "required_result_evidence": list(self.required_result_evidence), "result_authority": self.result_authority, "reviewer_reference": self.reviewer_reference, From 5197338f19c3096cf25d757ef66a3d37f4a3ae76 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 17:17:45 -0700 Subject: [PATCH 028/158] fix(validity): bind result timestamp diagnostic --- .../src/orgmetra_validity_analysis/result.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py index cea4c56e3..ae21c3d9d 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py @@ -188,7 +188,7 @@ def __post_init__(self) -> None: raise ValueError("convergence_diagnostics must be ConvergenceDiagnostics") if self.sample_size != self.missingness_summary.total_observations: raise ValueError("sample_size must match total_observations") - _canonical_timestamp(self.completed_at) + _canonical_timestamp(self.completed_at, "completed_at") if self.result_authority != _RESULT_AUTHORITY: raise ValueError("result_authority must remain scientific_evidence_only") if self.execution_state != _EXECUTION_STATE: @@ -208,7 +208,7 @@ def canonical_json(self) -> str: """Return deterministic, non-person-level JSON for audit correlation.""" payload = { "backend": self.backend, - "completed_at": _canonical_timestamp(self.completed_at), + "completed_at": _canonical_timestamp(self.completed_at, "completed_at"), "contains_raw_person_level_values": self.contains_raw_person_level_values, "convergence_diagnostics": self.convergence_diagnostics.to_dict(), "effect_estimate": float(self.effect_estimate), From e7c340495fb057e41e5b0e0338f648480063805e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 17:19:54 -0700 Subject: [PATCH 029/158] test(validity): align temporal diagnostic regression --- .../validity-analysis/tests/test_temporal_evidence_integrity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/validity-analysis/tests/test_temporal_evidence_integrity.py b/packages/validity-analysis/tests/test_temporal_evidence_integrity.py index e47628263..25e4ab83d 100644 --- a/packages/validity-analysis/tests/test_temporal_evidence_integrity.py +++ b/packages/validity-analysis/tests/test_temporal_evidence_integrity.py @@ -54,7 +54,7 @@ def test_handoff_rejects_datetime_subclass_that_can_forge_requested_at() -> None def test_result_rejects_datetime_subclass_that_can_forge_completed_at() -> None: """Result canonical evidence must not invoke caller-overridable datetime methods.""" - with pytest.raises(ValueError, match="requested_at"): + with pytest.raises(ValueError, match="completed_at"): ValidationAnalysisResult( tenant_record_id="10000000-0000-7000-8000-000000000001", result_reference="validation_analysis_result:11111111-1111-4111-8111-111111111111", From a8da84cdd43bca0b1fe4a6bbc4c2207c7cf2072d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 22:04:31 +0900 Subject: [PATCH 030/158] fix(validity): freeze handoff evidence boundaries --- CHANGELOG.md | 1 + ...ned-selection-validity-analysis-handoff.md | 3 + .../validation-analysis-handoff.md | 2 +- manifest.json | 2 +- packages/validity-analysis/CHANGELOG.md | 1 + packages/validity-analysis/README.md | 4 +- .../src/orgmetra_validity_analysis/handoff.py | 66 ++++-- .../src/orgmetra_validity_analysis/result.py | 21 +- .../tests/test_temporal_evidence_integrity.py | 197 +++++++++++++++++- 9 files changed, 264 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99f4752d7..d954dc355 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ All notable changes to Orgmetra will be documented in this file. - Accepted ADRs 0001–0003 now include buyer-facing Context, Decision, and Consequences grounded in verified ISO 30400:2022, ISO 30414:2025, Uniform Guidelines (29 C.F.R. Part 1607), SIOP (2018), OpenAPI Specification v3.2.0, OpenID Connect Core 1.0 errata set 2, CloudEvents v1.0.2, Jensen and Snodgrass (1999), Snodgrass (1999), and Allen (1983) records already listed in `docs/doctoring/REFERENCES.md`. ADRs 0004 and 0005 gained APA 7th References pointers to that same bibliography without changing their Decision bodies. - Active-PR governed Job Analysis persistence/API on the canonical `JobAnalysisSnapshot` model: migration `0013_job_analysis_snapshot.sql` stores immutable tenant-scoped snapshot, Task, KSAO, Task–KSAO, FJA and write-command evidence; `POST /v1/tenants/{tenant_record_id}/job-analysis-snapshots` and matching GET enforce purpose-bound Keyverse scope, authenticated-principal actor authority, bounded/strict JSON handling, transactional Idempotency-Key serialization, parent-scope fail-closed integrity, forced RLS, and atomic audit/outbox evidence. ADR 0014 records the persistence decision while ADR 0007 remains the domain/evidence authority; validated evidence still requires accountable human review and non-LLM provenance, and the service does not make a high-impact employment decision. - Active-PR `orgmetra_selection_review` packet for PII-minimized, evidence-bound human selection review: canonical operational tenant identity, UUID-backed opaque candidate/Job/sealed-evidence/reviewer references, explicit purpose/reason/evidence version, deterministic canonical JSON and SHA-256 correlation, mandatory human decision state, redacted packet repr, and provenance-paired model evidence that remains `untrusted_draft`, with exact 100% owned statement and branch coverage required by its quality gate. +- Active-PR `orgmetra_validity_analysis` handoff/result boundary: exact tenant and evidence references, reviewed fast-mlsirm pin, distinct requester/reviewer actors, aggregate-only scientific evidence, construction-time UTC and finite-number snapshots, canonical JSON/SHA-256 correlation, and human-review-only result authority. - Active performance-criterion scope hardening: `criterion_observation_scope_guard` rejects criterion outcomes for a Job the worker did not effectively hold at the observation date, observations before the relevant assignment, and observations outside the referenced performance cycle while preserving valid multiple-assignment cases and existing bitemporal correction semantics. The guard evaluates current-recorded facts, derives the date coordinate from `observed_at` in UTC so session `TimeZone` cannot alter the result, uses a trusted function search path, and adds no PII or automated employment decision authority. The Foundation PostgreSQL contract also rejects a closed `recorded_to` on each time-coordinate lookup and proves UTC midnight plus non-UTC session `TimeZone` boundaries. - Bitemporal tenant-scoped organization hierarchy validation that rejects visible indirect parent cycles and reuses single-valued recorded-time reconstruction before graph traversal. - Stacked governed job-analysis evidence contract via `JobAnalysisSnapshot`, `TaskEvidence`, `KSAORequirement`, `TaskKSAOLink`, `FunctionalJobAnalysisProfile`, and `EvidenceSource`: tenant/Job-scoped observable tasks, explicit Task-to-KSAO linkage, importance/difficulty/proficiency ratings, source/version/retrieval/SHA-256 provenance, deterministic canonical snapshot bytes, current O*NET evidence support, and historical DOT Data/People/Things compatibility. Validated snapshots require accountable human review and complete non-LLM evidence; LLM-origin material remains `analysis_draft`, and the snapshot is evidence input rather than a hiring, promotion, termination, compensation, or other high-impact employment decision. diff --git a/docs/adr/0027-governed-selection-validity-analysis-handoff.md b/docs/adr/0027-governed-selection-validity-analysis-handoff.md index 6db6c3a1c..c3862a968 100644 --- a/docs/adr/0027-governed-selection-validity-analysis-handoff.md +++ b/docs/adr/0027-governed-selection-validity-analysis-handoff.md @@ -26,6 +26,8 @@ Orgmetra adds a leaf `orgmetra_validity_analysis` package whose `ValidationAnaly - remains `not_executed`, `scientific_evidence_only`, and human-review-required; - produces deterministic canonical JSON and a SHA-256 digest for audit/result correlation. +Both handoff and result envelopes detach exact timezone-aware timestamps to one built-in UTC instant at construction. Result numeric evidence is converted to finite built-in floats before storage, so caller-controlled timezone or numeric runtime behavior cannot rewrite canonical evidence after validation. + The same package also validates `ValidationAnalysisResult` envelopes returned by the approved offline worker. A result must link to the handoff digest and the same pinned revision, identify a Rust CPU or GPU backend and precision, provide finite effect and interval values, match its sample size to aggregate missingness counts, reject impossible complete-versus-missing count combinations, and include explicit convergence diagnostics. The canonicalization boundary accepts only the exact governed `MissingnessSummary` and `ConvergenceDiagnostics` runtime types so subclass method overrides cannot add unreviewed or person-level fields to immutable result evidence. A nonconverged result remains typed scientific evidence requiring human review; it cannot be treated as a valid selection procedure or an employment decision. The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, and prove requester/reviewer identities are distinct authoritative actors. @@ -38,6 +40,7 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - A buyer or auditor can identify exactly which governed study evidence a result was supposed to analyze without copying raw worker data into the handoff. - Aggregate missingness evidence cannot claim all observations are complete while simultaneously reporting predictor- or criterion-missing observations. - Result canonicalization cannot be extended by an unreviewed subclass to serialize extra decision-like or person-level fields. +- Caller-controlled timestamp and numeric runtime behavior cannot rewrite an accepted canonical digest after construction. - Human interpretation remains explicit and separate from numerical output. - The dedicated-writer boundary remains intact: Orgmetra consumes only a pinned foreign revision/contract boundary and never mutates fast-mlsirm. diff --git a/docs/traceability/validation-analysis-handoff.md b/docs/traceability/validation-analysis-handoff.md index af8f6b622..455a8380c 100644 --- a/docs/traceability/validation-analysis-handoff.md +++ b/docs/traceability/validation-analysis-handoff.md @@ -15,7 +15,7 @@ Can an organization send one exact, reviewable validation study to its statistic | Scientific evidence | effect estimate, uncertainty interval, sample size, internally possible aggregate missingness, convergence diagnostics | immutable required-result-evidence regression plus impossible-missingness rejection | | Numerical result boundary | handoff digest, pinned fast-mlsirm revision, Rust CPU/GPU backend, precision, finite estimate/interval, aggregate missingness, explicit convergence state | `ValidationAnalysisResult` invariant/canonicalization regressions and exact-runtime-type checks | | Execution boundary | `not_executed`, `scientific_evidence_only`, read-only pinned foreign dependency | immutable governance regressions | -| Reproducibility | canonical RFC 3339 time, canonical JSON, SHA-256 handoff digest | deterministic serialization/digest tests | +| Reproducibility | construction-time UTC timestamp snapshots, finite numeric snapshots, canonical RFC 3339 time, canonical JSON, SHA-256 handoff digest | mutable timezone/numeric and UTC-boundary regressions plus deterministic serialization/digest tests | ## Maturity diff --git a/manifest.json b/manifest.json index 97f2bab14..df6ee9743 100644 --- a/manifest.json +++ b/manifest.json @@ -1 +1 @@ -{"package":"orgmetra-foundation-pack","version":"0.1.0","generated_for_branch":"feat/audit-outbox-envelope","files":[{"path":".github/workflows/foundation-ci.yml","sha256":"12686a3bbd6445e6fdb202b4137dae118ddeeab1efb0c7f18ea6c8fa19d62537","bytes":4379,"lines":123},{"path":".github/workflows/job-analysis-api-quality.yml","sha256":"352dc78931dd94afea3e88912d38dcc4b562a004112f199f3d7a12d22b6d637a","bytes":4159,"lines":105},{"path":".gitignore","sha256":"145fda644f5209fa1fb3e3b40c9af9258bfac6d1a634bba2520fd08fe6d77a21","bytes":375,"lines":37},{"path":"AGENTS.md","sha256":"28f7b7bc010a7739cfdc3e793fb5d39a0e74b842ea9c190e9a251e2d0cbc3a16","bytes":2246,"lines":34},{"path":"ARCHITECTURE.md","sha256":"52d68786f7359c1a50d804996021e4c70e90accd2fff6f1a27c91de1dd8df850","bytes":7864,"lines":107},{"path":"CHANGELOG.md","sha256":"32cc4ef78d1eca557fa01731026840be01211a043eb0ada552e4e6cb9eace353","bytes":17295,"lines":76},{"path":"CLAUDE.md","sha256":"add33884f466d324e20875388d103de41c6e062938a6e98727dc83a87ffe976f","bytes":1229,"lines":20},{"path":"LICENSE","sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30","bytes":11358,"lines":202},{"path":"NOTICE","sha256":"34b4618e946bdd8d33407d6ac5279f0a0388f5e7c8f79d2e7d8c3c47d0266042","bytes":305,"lines":4},{"path":"README.md","sha256":"1a9fc400d26d8137ae5911488794a6d3fa915957c95f27b36a48cef0fdf823c6","bytes":3785,"lines":81},{"path":"database/migrations/0001_foundation_schema.sql","sha256":"ce2ae52fc66b2f99597ea5285df82c66f90caa46174fef4930d68a8b6177d0dd","bytes":38747,"lines":916},{"path":"database/migrations/0002_sealed_evidence_digest.sql","sha256":"93d659ca8e0e9293a83d5422d043be7b1022c5470a5b22670aa3416fa334a04c","bytes":6649,"lines":202},{"path":"database/migrations/0003_audit_outbox_persistence.sql","sha256":"2aa7bbb8220923ec584537c0cd46f0cba2b692d69d431f097b7df6db75235bfc","bytes":15417,"lines":423},{"path":"database/migrations/0004_outbox_delivery_claim.sql","sha256":"d4504acf7d58528a2a8f4f03d1584b868c8d3ba9046a007b9c2e7cfef993b2ef","bytes":9451,"lines":234},{"path":"database/migrations/0005_outbox_delivery_finalization.sql","sha256":"b7e8790595b288f752d6ef5cc6cbfe4e1b6712248f5b7a3a25fa60016b6a4961","bytes":6125,"lines":170},{"path":"database/migrations/0006_outbox_delivery_dead_letter.sql","sha256":"c1fb91cdf98169fd6684984e86cb0a14fa19c8f1226028d2346a2a069df2b3c7","bytes":24919,"lines":628},{"path":"database/migrations/0007_outbox_retry_exhaustion.sql","sha256":"812f50d70ca5929c7eba964d34a208aedee660d11cc7ffc09d67688c4737e0d5","bytes":19081,"lines":476},{"path":"database/migrations/0008_audit_outbox_review_hardening.sql","sha256":"c3713a12db9d00fdc10005df1f86c07965e9555eefad78ca67e994537a739d9b","bytes":17562,"lines":448},{"path":"database/migrations/0009_candidate_worker_conversion_governance.sql","sha256":"4030666629a6b8deb383b8337ead4f09d6a945969313def2577a38f31f06cda9","bytes":11537,"lines":281},{"path":"database/migrations/0010_validity_study_case_integrity.sql","sha256":"3f594810ac9e1a6747a2bb4838e5ce65b921cb6e3d36fcdc3ff08b4a7579ebd1","bytes":11979,"lines":313},{"path":"database/migrations/0011_criterion_observation_scope.sql","sha256":"f9fe7c35f1ee7b167e1c2ba75a50a84febda9a6ccf8123b4f5726f51968694f9","bytes":7444,"lines":165},{"path":"database/migrations/0012_people_mutation_idempotency.sql","sha256":"52dbbb9ec7f9be5291593ba88f228d7fffd736dcb99547a08c1d6cad076afb69","bytes":3162,"lines":76},{"path":"database/migrations/0013_job_analysis_snapshot.sql","sha256":"b6553a5a4c94c4aa9f341a474e13bbe34db63044eda2446b3ebee178995977ee","bytes":12713,"lines":260},{"path":"docs/API_CONTRACT.md","sha256":"63533dff785da62b89e585d742a158e2aeb05913644f2bf9fb6486f281c2e589","bytes":4555,"lines":76},{"path":"docs/DATA_MODEL.md","sha256":"6ad29731ae7ee7aa5bf3a2d0bfef88894a35a2550edb2be3244d6f143d76444a","bytes":13366,"lines":85},{"path":"docs/ERD.md","sha256":"546001aa85c4fe020e0c39d881dc860daf7f69090596666fdf9092487b0725fe","bytes":6964,"lines":70},{"path":"docs/OPERABILITY.md","sha256":"82b2d3e70cec371ef35e9e0f982ac40fef84351976bc04b863b81d27023d5a62","bytes":11189,"lines":71},{"path":"docs/PRD.md","sha256":"3ad85ae633cce0fc7a93af39b21d7a7c70bb2efa786da6b12f3c5327906e34f1","bytes":5490,"lines":111},{"path":"docs/SECURITY.md","sha256":"01918512d8882060e9cff0c4aa8206e0eccbdfb61cfd7f829331123c7a9fe6ac","bytes":11185,"lines":64},{"path":"docs/STORYBOARD.md","sha256":"6e4ffb0eb03a80343f50d363ffc43b34da9348a44232dd947a9ff416ea92a3d2","bytes":1342,"lines":28},{"path":"docs/STORYBOOK.md","sha256":"82f79029b3c2b7a45393bad5ba8fabe61014d4b6149c7d4e73f70ba447f885e9","bytes":1389,"lines":50},{"path":"docs/TEST_STRATEGY.md","sha256":"d0a0bc3b54ed0fc7973747987f1afb117d6144c390b51ed9370eb571972a33f8","bytes":16534,"lines":135},{"path":"docs/THREAT_MODEL.md","sha256":"f314f375c2e41252536de224c7bc7e4a10ab8f340cb86642724e7399e32f4252","bytes":6736,"lines":23},{"path":"docs/TRACEABILITY.md","sha256":"dbf6fd91375ea28e05456d2a0c9ba629506cbac6f52f5dfda61ae68db2395f7e","bytes":11462,"lines":40},{"path":"docs/TRD.md","sha256":"23697d88a4882698e1a2782b7da3f2ccd0d3cd2d6d1bffe89b6597dc16851077","bytes":9064,"lines":101},{"path":"docs/UML.md","sha256":"fe67c37aa88e5814ceb2db7e8f7d8d85ca27a994802efbb7c75164b387adf0a9","bytes":5528,"lines":122},{"path":"docs/USER_STORIES.md","sha256":"5535b39d8c71a36c81f78e2d6dbd90a2d32e6541790f0d28f6dd4baf3ea7b45f","bytes":2670,"lines":37},{"path":"docs/WIREFRAMES.md","sha256":"b03aa6419aeaf5d42a5698c4d43a434c1633b7ac6fd0b0bd0cda979077adc56e","bytes":2005,"lines":77},{"path":"docs/adr/0001-orgmetra-authoritative-hris-record.md","sha256":"0f8055b73c63d3130321415ad53233588ff952aabd1a88952b39c71747253572","bytes":6108,"lines":53},{"path":"docs/adr/0002-federated-cwl-integration-boundaries.md","sha256":"b77165f2aacfa6f4fde994baf77d5879c6da3e8dae4fd2db0ed912d60ae9b3b2","bytes":4072,"lines":44},{"path":"docs/adr/0003-bitemporal-hris-data-contract.md","sha256":"d7f2660616622c1a7994b28aa66d99d13836bcf755735595f9609a41282ab799","bytes":4453,"lines":47},{"path":"docs/adr/0004-employment-position-version-and-assignment-binding.md","sha256":"fee89e700414abe0b1cffec2acc687e5e014634db8f5ef9e8a92abba5c3cf182","bytes":1872,"lines":30},{"path":"docs/adr/0005-exclusive-employment-and-staffable-seats.md","sha256":"10f0eb409f4fa32d2c5bed2d583d8b43be8e61b5cbef0e927e5bebb5f5c8f85b","bytes":2091,"lines":34},{"path":"docs/adr/0006-governed-audit-outbox-envelope.md","sha256":"827298ddd997b47f78a89e89911ad8ea72e517b7714303637f0329b8cb52cabd","bytes":14100,"lines":66},{"path":"docs/adr/0007-governed-job-analysis-evidence.md","sha256":"953c6d2b9864a78b461b576092ec3f198f0b76709eaaaf7d0ed0182f95182c52","bytes":5653,"lines":57},{"path":"docs/adr/0008-purpose-bound-pii-authorization.md","sha256":"c5157d3bc58f3d8d29e03104dd15eb2911cc1bb66e2c92a935b26d7164648dc7","bytes":5988,"lines":55},{"path":"docs/adr/0009-performance-criterion-observation-scope.md","sha256":"1ac10bb2747b0a5b4d62f627825cfd7f978f3fa88d7575bffc23d56371240a64","bytes":7057,"lines":57},{"path":"docs/adr/0010-naruon-calendar-intent-boundary.md","sha256":"3e1050a964cc4ed76a1a0cf1e699ae5080acf8c9336f0decdd6d5229359db3c9","bytes":3917,"lines":35},{"path":"docs/adr/0011-bitemporal-workforce-composition.md","sha256":"1656ef8b57c836ef7936a8e9cb6a824681eb7563157a1ab0a29deb25849a457b","bytes":5568,"lines":53},{"path":"docs/adr/0012-governed-migration-handoff.md","sha256":"713855d670001d3964ecb36cc653830502fb1d82a58b9e39f564b6992dd2bd80","bytes":5965,"lines":59},{"path":"docs/adr/0013-governed-requisition-review-packet.md","sha256":"70bf2cbdf903a8793d6d8bc116a08331931090118341f42010236e09c6cc1802","bytes":4693,"lines":46},{"path":"docs/adr/0014-job-analysis-snapshot-persistence.md","sha256":"a7ab6fee50aaa63f7f407516a4cb39885faeb0fc6e5035ee8fc352ed73430105","bytes":5365,"lines":49},{"path":"docs/adr/README.md","sha256":"f3b3b5ed3b3b31a40a0a3696abf0065e3c25879b6be50077f38ffae742b9d002","bytes":1838,"lines":18},{"path":"docs/doctoring/REFERENCES.md","sha256":"929f7ee36df16279f028f726fcf039982180deb377746fe3804f3c0d090778d5","bytes":6352,"lines":69},{"path":"docs/superpowers/plans/2026-08-15-orgmetra-foundation-implementation-plan.md","sha256":"b64f21abb19373e780db8b9e64deb8ba9a6219ccf9625a651f25407b8691fcbd","bytes":8227,"lines":226},{"path":"docs/superpowers/specs/2026-08-15-orgmetra-foundation-design.md","sha256":"4a0e1a7943e40d12bd3082db3757045b4085e5a089fea7bc0d8a1565ffcbcf1d","bytes":6237,"lines":187},{"path":"package.json","sha256":"59ae9e3e67c3fba9320cb18439692395cdfd16ae5c24e3c4cf30d77d63ebabb5","bytes":388,"lines":9},{"path":"packages/hris-kernel/src/orgmetra_hris_kernel/audit.py","sha256":"3e5b7190cf857dc8c1fc7e898cef303060f34aabee6c27a9034d4d9650e33190","bytes":7707,"lines":160},{"path":"packages/hris-kernel/tests/test_audit_outbox.py","sha256":"5928dd7b97fe38d6b7472ce62966437e339058a59c3b301a93a7b5c05432b40c","bytes":7556,"lines":200},{"path":"schemas/openapi.yaml","sha256":"09c1e43486779198574fe31b8bcabbd1c1f74beec7bf86245ae578061619838f","bytes":29503,"lines":1020},{"path":"scripts/foundation-contract-core.mjs","sha256":"595e8381dbd62e97093b11eef818af5f04d6473ac592d57e3985ffbc2210d445","bytes":28173,"lines":689},{"path":"scripts/foundation-contract.mjs","sha256":"5242dcdbe0935775edf074462c82600e9bc4927d9fdc50c47727af915fd4b23a","bytes":218,"lines":6},{"path":"tests/dispatcher-inventory.test.mjs","sha256":"09f5e64410e6b7a26bf8d6ce61c50b737da2ea85d955f91eba63aa21f1537261","bytes":1597,"lines":34},{"path":"tests/foundation-contract.test.mjs","sha256":"960306fd7cda7b982a52c4428a432d10a4f570430a5d39fb23aeca0b2ede0615","bytes":14860,"lines":386},{"path":"tests/openapi-contract.test.mjs","sha256":"80c1610ef1c189fa325e55389501e0e51531ddf61ee335bb94d9cb3aa55a9fdc","bytes":6438,"lines":195},{"path":"tests/test_audit_outbox_hardening_postgres.sh","sha256":"518ba2f37ba6292943e5abe22c2599452b2f031a42e453b2493aedf8714421a0","bytes":13396,"lines":333},{"path":"tests/test_audit_outbox_postgres.sh","sha256":"e57a04920a0ba97fa6a06752d15ea150016ab8d44099e998c5c4f4067592b4d2","bytes":13443,"lines":357},{"path":"tests/test_bitemporal_postgres.sh","sha256":"7684b8c2ff52c044c081135515bd5aabbfd00e2daad0d471b0868701af2df6cc","bytes":8209,"lines":230},{"path":"tests/test_candidate_worker_conversion_postgres.sh","sha256":"681cb74d6cfa859ed92c6c2439881ea20c430ef8df94ec662e2807761a377f90","bytes":14673,"lines":344},{"path":"tests/test_criterion_observation_scope_postgres.sh","sha256":"0ee9539ee57f840c27d08009f7868cdc8662669df78a01dbc8be39216b8f1a3d","bytes":17811,"lines":469},{"path":"tests/test_evidence_sealing_postgres.sh","sha256":"57d16b632a0c60ffdcb4842ceb1cfe25d19c54cefeeefb622ff4fa6e83441ad7","bytes":11349,"lines":370},{"path":"tests/test_job_analysis_snapshot_postgres.sh","sha256":"ca9c323a1dd68cfc520277efbbb7495e37fb3ca027890928c8624e5b4f57403f","bytes":13542,"lines":296},{"path":"tests/test_operational_uuid_postgres.sh","sha256":"7378f98f0d4b3000e8ea641d8701f1540dbad71410b3637d81d799969e0f6ff7","bytes":3346,"lines":101},{"path":"tests/test_outbox_claim_postgres.sh","sha256":"1027806d436ebfe34e108c25b6a4001f43b9550f1d70057c6c0d7974323b0c9b","bytes":14817,"lines":429},{"path":"tests/test_outbox_dead_letter_postgres.sh","sha256":"0d728d578e64252e6079f2d141ddaa7fa9cfbf9784e625832273596d69a6e13d","bytes":14008,"lines":377},{"path":"tests/test_people_mutation_idempotency_postgres.sh","sha256":"3f57e12f80bd1b034c9aac54b669d8530106e3e26b3795689671fb53807b3cd5","bytes":16191,"lines":381},{"path":"tests/test_tenant_isolation_postgres.sh","sha256":"dd649435ef8ab9e57f0609c101917e36656a6d40d63de9bcdbdac23d764f6c3a","bytes":15134,"lines":388},{"path":"tests/test_validity_study_case_postgres.sh","sha256":"0070ad58300323c7f9900c5645e0df3106b36ccd245ae686e982c2fd6fa4dc02","bytes":14708,"lines":301},{"path":"tests/validate_repository.py","sha256":"918cf92fd18d81572e9bd5f5daa7f033c32731e2e13f0d00661d1c1de30b12a9","bytes":27291,"lines":638}]} +{"package":"orgmetra-foundation-pack","version":"0.1.0","generated_for_branch":"feat/audit-outbox-envelope","files":[{"path":".github/workflows/foundation-ci.yml","sha256":"12686a3bbd6445e6fdb202b4137dae118ddeeab1efb0c7f18ea6c8fa19d62537","bytes":4379,"lines":123},{"path":".github/workflows/job-analysis-api-quality.yml","sha256":"352dc78931dd94afea3e88912d38dcc4b562a004112f199f3d7a12d22b6d637a","bytes":4159,"lines":105},{"path":".gitignore","sha256":"145fda644f5209fa1fb3e3b40c9af9258bfac6d1a634bba2520fd08fe6d77a21","bytes":375,"lines":37},{"path":"AGENTS.md","sha256":"28f7b7bc010a7739cfdc3e793fb5d39a0e74b842ea9c190e9a251e2d0cbc3a16","bytes":2246,"lines":34},{"path":"ARCHITECTURE.md","sha256":"52d68786f7359c1a50d804996021e4c70e90accd2fff6f1a27c91de1dd8df850","bytes":7864,"lines":107},{"path":"CHANGELOG.md","sha256":"264a8af7f8a324a044317d24ce8f8d7eee65ef3d54a8819e7a934d6b9859aa83","bytes":17624,"lines":77},{"path":"CLAUDE.md","sha256":"add33884f466d324e20875388d103de41c6e062938a6e98727dc83a87ffe976f","bytes":1229,"lines":20},{"path":"LICENSE","sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30","bytes":11358,"lines":202},{"path":"NOTICE","sha256":"34b4618e946bdd8d33407d6ac5279f0a0388f5e7c8f79d2e7d8c3c47d0266042","bytes":305,"lines":4},{"path":"README.md","sha256":"1a9fc400d26d8137ae5911488794a6d3fa915957c95f27b36a48cef0fdf823c6","bytes":3785,"lines":81},{"path":"database/migrations/0001_foundation_schema.sql","sha256":"ce2ae52fc66b2f99597ea5285df82c66f90caa46174fef4930d68a8b6177d0dd","bytes":38747,"lines":916},{"path":"database/migrations/0002_sealed_evidence_digest.sql","sha256":"93d659ca8e0e9293a83d5422d043be7b1022c5470a5b22670aa3416fa334a04c","bytes":6649,"lines":202},{"path":"database/migrations/0003_audit_outbox_persistence.sql","sha256":"2aa7bbb8220923ec584537c0cd46f0cba2b692d69d431f097b7df6db75235bfc","bytes":15417,"lines":423},{"path":"database/migrations/0004_outbox_delivery_claim.sql","sha256":"d4504acf7d58528a2a8f4f03d1584b868c8d3ba9046a007b9c2e7cfef993b2ef","bytes":9451,"lines":234},{"path":"database/migrations/0005_outbox_delivery_finalization.sql","sha256":"b7e8790595b288f752d6ef5cc6cbfe4e1b6712248f5b7a3a25fa60016b6a4961","bytes":6125,"lines":170},{"path":"database/migrations/0006_outbox_delivery_dead_letter.sql","sha256":"c1fb91cdf98169fd6684984e86cb0a14fa19c8f1226028d2346a2a069df2b3c7","bytes":24919,"lines":628},{"path":"database/migrations/0007_outbox_retry_exhaustion.sql","sha256":"812f50d70ca5929c7eba964d34a208aedee660d11cc7ffc09d67688c4737e0d5","bytes":19081,"lines":476},{"path":"database/migrations/0008_audit_outbox_review_hardening.sql","sha256":"c3713a12db9d00fdc10005df1f86c07965e9555eefad78ca67e994537a739d9b","bytes":17562,"lines":448},{"path":"database/migrations/0009_candidate_worker_conversion_governance.sql","sha256":"4030666629a6b8deb383b8337ead4f09d6a945969313def2577a38f31f06cda9","bytes":11537,"lines":281},{"path":"database/migrations/0010_validity_study_case_integrity.sql","sha256":"3f594810ac9e1a6747a2bb4838e5ce65b921cb6e3d36fcdc3ff08b4a7579ebd1","bytes":11979,"lines":313},{"path":"database/migrations/0011_criterion_observation_scope.sql","sha256":"f9fe7c35f1ee7b167e1c2ba75a50a84febda9a6ccf8123b4f5726f51968694f9","bytes":7444,"lines":165},{"path":"database/migrations/0012_people_mutation_idempotency.sql","sha256":"52dbbb9ec7f9be5291593ba88f228d7fffd736dcb99547a08c1d6cad076afb69","bytes":3162,"lines":76},{"path":"database/migrations/0013_job_analysis_snapshot.sql","sha256":"b6553a5a4c94c4aa9f341a474e13bbe34db63044eda2446b3ebee178995977ee","bytes":12713,"lines":260},{"path":"docs/API_CONTRACT.md","sha256":"63533dff785da62b89e585d742a158e2aeb05913644f2bf9fb6486f281c2e589","bytes":4555,"lines":76},{"path":"docs/DATA_MODEL.md","sha256":"6ad29731ae7ee7aa5bf3a2d0bfef88894a35a2550edb2be3244d6f143d76444a","bytes":13366,"lines":85},{"path":"docs/ERD.md","sha256":"546001aa85c4fe020e0c39d881dc860daf7f69090596666fdf9092487b0725fe","bytes":6964,"lines":70},{"path":"docs/OPERABILITY.md","sha256":"82b2d3e70cec371ef35e9e0f982ac40fef84351976bc04b863b81d27023d5a62","bytes":11189,"lines":71},{"path":"docs/PRD.md","sha256":"3ad85ae633cce0fc7a93af39b21d7a7c70bb2efa786da6b12f3c5327906e34f1","bytes":5490,"lines":111},{"path":"docs/SECURITY.md","sha256":"01918512d8882060e9cff0c4aa8206e0eccbdfb61cfd7f829331123c7a9fe6ac","bytes":11185,"lines":64},{"path":"docs/STORYBOARD.md","sha256":"6e4ffb0eb03a80343f50d363ffc43b34da9348a44232dd947a9ff416ea92a3d2","bytes":1342,"lines":28},{"path":"docs/STORYBOOK.md","sha256":"82f79029b3c2b7a45393bad5ba8fabe61014d4b6149c7d4e73f70ba447f885e9","bytes":1389,"lines":50},{"path":"docs/TEST_STRATEGY.md","sha256":"d0a0bc3b54ed0fc7973747987f1afb117d6144c390b51ed9370eb571972a33f8","bytes":16534,"lines":135},{"path":"docs/THREAT_MODEL.md","sha256":"f314f375c2e41252536de224c7bc7e4a10ab8f340cb86642724e7399e32f4252","bytes":6736,"lines":23},{"path":"docs/TRACEABILITY.md","sha256":"dbf6fd91375ea28e05456d2a0c9ba629506cbac6f52f5dfda61ae68db2395f7e","bytes":11462,"lines":40},{"path":"docs/TRD.md","sha256":"23697d88a4882698e1a2782b7da3f2ccd0d3cd2d6d1bffe89b6597dc16851077","bytes":9064,"lines":101},{"path":"docs/UML.md","sha256":"fe67c37aa88e5814ceb2db7e8f7d8d85ca27a994802efbb7c75164b387adf0a9","bytes":5528,"lines":122},{"path":"docs/USER_STORIES.md","sha256":"5535b39d8c71a36c81f78e2d6dbd90a2d32e6541790f0d28f6dd4baf3ea7b45f","bytes":2670,"lines":37},{"path":"docs/WIREFRAMES.md","sha256":"b03aa6419aeaf5d42a5698c4d43a434c1633b7ac6fd0b0bd0cda979077adc56e","bytes":2005,"lines":77},{"path":"docs/adr/0001-orgmetra-authoritative-hris-record.md","sha256":"0f8055b73c63d3130321415ad53233588ff952aabd1a88952b39c71747253572","bytes":6108,"lines":53},{"path":"docs/adr/0002-federated-cwl-integration-boundaries.md","sha256":"b77165f2aacfa6f4fde994baf77d5879c6da3e8dae4fd2db0ed912d60ae9b3b2","bytes":4072,"lines":44},{"path":"docs/adr/0003-bitemporal-hris-data-contract.md","sha256":"d7f2660616622c1a7994b28aa66d99d13836bcf755735595f9609a41282ab799","bytes":4453,"lines":47},{"path":"docs/adr/0004-employment-position-version-and-assignment-binding.md","sha256":"fee89e700414abe0b1cffec2acc687e5e014634db8f5ef9e8a92abba5c3cf182","bytes":1872,"lines":30},{"path":"docs/adr/0005-exclusive-employment-and-staffable-seats.md","sha256":"10f0eb409f4fa32d2c5bed2d583d8b43be8e61b5cbef0e927e5bebb5f5c8f85b","bytes":2091,"lines":34},{"path":"docs/adr/0006-governed-audit-outbox-envelope.md","sha256":"827298ddd997b47f78a89e89911ad8ea72e517b7714303637f0329b8cb52cabd","bytes":14100,"lines":66},{"path":"docs/adr/0007-governed-job-analysis-evidence.md","sha256":"953c6d2b9864a78b461b576092ec3f198f0b76709eaaaf7d0ed0182f95182c52","bytes":5653,"lines":57},{"path":"docs/adr/0008-purpose-bound-pii-authorization.md","sha256":"c5157d3bc58f3d8d29e03104dd15eb2911cc1bb66e2c92a935b26d7164648dc7","bytes":5988,"lines":55},{"path":"docs/adr/0009-performance-criterion-observation-scope.md","sha256":"1ac10bb2747b0a5b4d62f627825cfd7f978f3fa88d7575bffc23d56371240a64","bytes":7057,"lines":57},{"path":"docs/adr/0010-naruon-calendar-intent-boundary.md","sha256":"3e1050a964cc4ed76a1a0cf1e699ae5080acf8c9336f0decdd6d5229359db3c9","bytes":3917,"lines":35},{"path":"docs/adr/0011-bitemporal-workforce-composition.md","sha256":"1656ef8b57c836ef7936a8e9cb6a824681eb7563157a1ab0a29deb25849a457b","bytes":5568,"lines":53},{"path":"docs/adr/0012-governed-migration-handoff.md","sha256":"713855d670001d3964ecb36cc653830502fb1d82a58b9e39f564b6992dd2bd80","bytes":5965,"lines":59},{"path":"docs/adr/0013-governed-requisition-review-packet.md","sha256":"70bf2cbdf903a8793d6d8bc116a08331931090118341f42010236e09c6cc1802","bytes":4693,"lines":46},{"path":"docs/adr/0014-job-analysis-snapshot-persistence.md","sha256":"a7ab6fee50aaa63f7f407516a4cb39885faeb0fc6e5035ee8fc352ed73430105","bytes":5365,"lines":49},{"path":"docs/adr/README.md","sha256":"f3b3b5ed3b3b31a40a0a3696abf0065e3c25879b6be50077f38ffae742b9d002","bytes":1838,"lines":18},{"path":"docs/doctoring/REFERENCES.md","sha256":"929f7ee36df16279f028f726fcf039982180deb377746fe3804f3c0d090778d5","bytes":6352,"lines":69},{"path":"docs/superpowers/plans/2026-08-15-orgmetra-foundation-implementation-plan.md","sha256":"b64f21abb19373e780db8b9e64deb8ba9a6219ccf9625a651f25407b8691fcbd","bytes":8227,"lines":226},{"path":"docs/superpowers/specs/2026-08-15-orgmetra-foundation-design.md","sha256":"4a0e1a7943e40d12bd3082db3757045b4085e5a089fea7bc0d8a1565ffcbcf1d","bytes":6237,"lines":187},{"path":"package.json","sha256":"59ae9e3e67c3fba9320cb18439692395cdfd16ae5c24e3c4cf30d77d63ebabb5","bytes":388,"lines":9},{"path":"packages/hris-kernel/src/orgmetra_hris_kernel/audit.py","sha256":"3e5b7190cf857dc8c1fc7e898cef303060f34aabee6c27a9034d4d9650e33190","bytes":7707,"lines":160},{"path":"packages/hris-kernel/tests/test_audit_outbox.py","sha256":"5928dd7b97fe38d6b7472ce62966437e339058a59c3b301a93a7b5c05432b40c","bytes":7556,"lines":200},{"path":"schemas/openapi.yaml","sha256":"09c1e43486779198574fe31b8bcabbd1c1f74beec7bf86245ae578061619838f","bytes":29503,"lines":1020},{"path":"scripts/foundation-contract-core.mjs","sha256":"595e8381dbd62e97093b11eef818af5f04d6473ac592d57e3985ffbc2210d445","bytes":28173,"lines":689},{"path":"scripts/foundation-contract.mjs","sha256":"5242dcdbe0935775edf074462c82600e9bc4927d9fdc50c47727af915fd4b23a","bytes":218,"lines":6},{"path":"tests/dispatcher-inventory.test.mjs","sha256":"09f5e64410e6b7a26bf8d6ce61c50b737da2ea85d955f91eba63aa21f1537261","bytes":1597,"lines":34},{"path":"tests/foundation-contract.test.mjs","sha256":"960306fd7cda7b982a52c4428a432d10a4f570430a5d39fb23aeca0b2ede0615","bytes":14860,"lines":386},{"path":"tests/openapi-contract.test.mjs","sha256":"80c1610ef1c189fa325e55389501e0e51531ddf61ee335bb94d9cb3aa55a9fdc","bytes":6438,"lines":195},{"path":"tests/test_audit_outbox_hardening_postgres.sh","sha256":"518ba2f37ba6292943e5abe22c2599452b2f031a42e453b2493aedf8714421a0","bytes":13396,"lines":333},{"path":"tests/test_audit_outbox_postgres.sh","sha256":"e57a04920a0ba97fa6a06752d15ea150016ab8d44099e998c5c4f4067592b4d2","bytes":13443,"lines":357},{"path":"tests/test_bitemporal_postgres.sh","sha256":"7684b8c2ff52c044c081135515bd5aabbfd00e2daad0d471b0868701af2df6cc","bytes":8209,"lines":230},{"path":"tests/test_candidate_worker_conversion_postgres.sh","sha256":"681cb74d6cfa859ed92c6c2439881ea20c430ef8df94ec662e2807761a377f90","bytes":14673,"lines":344},{"path":"tests/test_criterion_observation_scope_postgres.sh","sha256":"0ee9539ee57f840c27d08009f7868cdc8662669df78a01dbc8be39216b8f1a3d","bytes":17811,"lines":469},{"path":"tests/test_evidence_sealing_postgres.sh","sha256":"57d16b632a0c60ffdcb4842ceb1cfe25d19c54cefeeefb622ff4fa6e83441ad7","bytes":11349,"lines":370},{"path":"tests/test_job_analysis_snapshot_postgres.sh","sha256":"ca9c323a1dd68cfc520277efbbb7495e37fb3ca027890928c8624e5b4f57403f","bytes":13542,"lines":296},{"path":"tests/test_operational_uuid_postgres.sh","sha256":"7378f98f0d4b3000e8ea641d8701f1540dbad71410b3637d81d799969e0f6ff7","bytes":3346,"lines":101},{"path":"tests/test_outbox_claim_postgres.sh","sha256":"1027806d436ebfe34e108c25b6a4001f43b9550f1d70057c6c0d7974323b0c9b","bytes":14817,"lines":429},{"path":"tests/test_outbox_dead_letter_postgres.sh","sha256":"0d728d578e64252e6079f2d141ddaa7fa9cfbf9784e625832273596d69a6e13d","bytes":14008,"lines":377},{"path":"tests/test_people_mutation_idempotency_postgres.sh","sha256":"3f57e12f80bd1b034c9aac54b669d8530106e3e26b3795689671fb53807b3cd5","bytes":16191,"lines":381},{"path":"tests/test_tenant_isolation_postgres.sh","sha256":"dd649435ef8ab9e57f0609c101917e36656a6d40d63de9bcdbdac23d764f6c3a","bytes":15134,"lines":388},{"path":"tests/test_validity_study_case_postgres.sh","sha256":"0070ad58300323c7f9900c5645e0df3106b36ccd245ae686e982c2fd6fa4dc02","bytes":14708,"lines":301},{"path":"tests/validate_repository.py","sha256":"918cf92fd18d81572e9bd5f5daa7f033c32731e2e13f0d00661d1c1de30b12a9","bytes":27291,"lines":638}]} diff --git a/packages/validity-analysis/CHANGELOG.md b/packages/validity-analysis/CHANGELOG.md index 9e9e67b43..fd7762043 100644 --- a/packages/validity-analysis/CHANGELOG.md +++ b/packages/validity-analysis/CHANGELOG.md @@ -9,3 +9,4 @@ - Validate a digest-linked Rust CPU/GPU result envelope with finite estimates, aggregate missingness, and explicit convergence or nonconvergence diagnostics. - Reject impossible aggregate missingness where complete observations overlap either predictor-missing or criterion-missing counts beyond the sample total. - Require exact governed missingness/convergence runtime types so subclass method overrides cannot inject unreviewed or person-level fields into canonical result evidence. +- Freeze exact UTC timestamps and finite numeric values at construction, and reject runtime-type forgery before canonical evidence serialization. diff --git a/packages/validity-analysis/README.md b/packages/validity-analysis/README.md index 527241d5b..b56e9a040 100644 --- a/packages/validity-analysis/README.md +++ b/packages/validity-analysis/README.md @@ -6,9 +6,9 @@ This package creates an immutable **selection-validity analysis handoff** and va `build_validation_analysis_handoff(...)` binds one tenant, validation study, Job, predictor snapshot, criterion snapshot, population snapshot, decision policy, analysis plan, requester, reviewer, and the reviewed fast-mlsirm revision `04d0bc21a2a20693bcf16108cd76d394fe844d23`. -The resulting canonical JSON is digest-addressable, contains no raw person-level predictor or criterion values, and remains `not_executed`. Required result evidence is explicit: effect estimate, uncertainty interval, sample size, missingness summary, and convergence diagnostics. +The resulting canonical JSON is digest-addressable, contains no raw person-level predictor or criterion values, and remains `not_executed`. Its timestamp is detached to one UTC instant at construction so later timezone-provider changes cannot rewrite the digest. Required result evidence is explicit: effect estimate, uncertainty interval, sample size, missingness summary, and convergence diagnostics. -`ValidationAnalysisResult` accepts only a result linked to the handoff digest and the same reviewed fast-mlsirm revision. It records the Rust CPU/GPU backend, precision, aggregate missingness counts, finite effect and interval values, and explicit convergence or nonconvergence diagnostics. Missingness counts must be internally possible: complete observations cannot overlap either predictor-missing or criterion-missing observations beyond the declared sample total. The result envelope accepts only the exact governed `MissingnessSummary` and `ConvergenceDiagnostics` runtime types, preventing subclass method overrides from adding unreviewed or person-level fields to canonical audit evidence. It never promotes a result to an employment decision; human review remains mandatory. +`ValidationAnalysisResult` accepts only a result linked to the handoff digest and the same reviewed fast-mlsirm revision. It records the Rust CPU/GPU backend, precision, aggregate missingness counts, finite effect and interval values, and explicit convergence or nonconvergence diagnostics. Timestamps and finite numeric values are snapshotted before canonicalization, and the result envelope accepts only the exact governed `MissingnessSummary` and `ConvergenceDiagnostics` runtime types, preventing mutable runtime values or subclass method overrides from adding unreviewed or person-level fields to canonical audit evidence. Missingness counts must be internally possible: complete observations cannot overlap either predictor-missing or criterion-missing observations beyond the declared sample total. It never promotes a result to an employment decision; human review remains mandatory. ## What it does not do diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py b/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py index a77b280c6..5f3fb2f57 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py @@ -8,7 +8,7 @@ from __future__ import annotations from dataclasses import dataclass -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from hashlib import sha256 import json import re @@ -46,8 +46,10 @@ ) -def _validate_operational_uuid(value: str, field_name: str) -> None: +def _validate_operational_uuid(value: object, field_name: str) -> None: """Require canonical non-sentinel UUID text owned by authoritative Orgmetra.""" + if type(value) is not str: + raise ValueError(f"{field_name} must be canonical UUID text") try: parsed = UUID(value) except (ValueError, AttributeError, TypeError) as exc: @@ -56,11 +58,11 @@ def _validate_operational_uuid(value: str, field_name: str) -> None: raise ValueError(f"{field_name} must be a canonical operational UUID") -def _validate_reference(value: str, prefix: str, field_name: str) -> None: +def _validate_reference(value: object, prefix: str, field_name: str) -> None: """Require the expected namespace plus a canonical opaque UUIDv4 suffix.""" error_message = f"{field_name} must be an opaque {prefix}: UUIDv4 reference" if ( - not isinstance(value, str) + type(value) is not str or len(value) > 160 or not _REFERENCE_PATTERN.fullmatch(value) or not value.startswith(f"{prefix}:") @@ -75,31 +77,47 @@ def _validate_reference(value: str, prefix: str, field_name: str) -> None: raise ValueError(error_message) -def _validate_digest(value: str, field_name: str) -> None: +def _validate_digest(value: object, field_name: str) -> None: """Require lowercase SHA-256 hexadecimal evidence.""" - if not isinstance(value, str) or not _DIGEST_PATTERN.fullmatch(value): + if type(value) is not str or not _DIGEST_PATTERN.fullmatch(value): raise ValueError(f"{field_name} must be lowercase SHA-256 hex") -def _validate_code(value: str, field_name: str) -> None: +def _validate_code(value: object, field_name: str) -> None: """Require bounded descriptive lower snake_case governance codes.""" - if not isinstance(value, str) or len(value) > 64 or not _CODE_PATTERN.fullmatch(value): + if type(value) is not str or len(value) > 64 or not _CODE_PATTERN.fullmatch(value): raise ValueError(f"{field_name} must be bounded two-or-more-word lower snake_case") -def _validate_kernel_revision(value: str) -> None: +def _validate_kernel_revision(value: object) -> None: """Require the exact externally reviewed immutable fast-mlsirm revision.""" - if not isinstance(value, str) or not _REVISION_PATTERN.fullmatch(value): + if type(value) is not str or not _REVISION_PATTERN.fullmatch(value): raise ValueError("fast_mlsirm_revision must be lowercase 40-character Git commit hex") if value != REVIEWED_FAST_MLSIRM_REVISION: raise ValueError("fast_mlsirm_revision must equal the reviewed immutable revision") -def _canonical_timestamp(value: datetime, field_name: str) -> str: - """Render an exact built-in aware instant with field-correct diagnostics.""" - if type(value) is not datetime or value.tzinfo is None or value.utcoffset() is None: +def _freeze_timestamp(value: object, field_name: str) -> datetime: + """Detach caller-controlled timezone behavior and store one immutable UTC instant.""" + if type(value) is not datetime or value.tzinfo is None: + raise ValueError(f"{field_name} must be an exact timezone-aware datetime") + try: + offset = value.utcoffset() + except Exception as exc: # noqa: BLE001 - normalize provider behavior at trust boundary. + raise ValueError(f"{field_name} must be an exact timezone-aware datetime") from exc + if type(offset) is not timedelta: + raise ValueError(f"{field_name} must be an exact timezone-aware datetime") + try: + return (value.replace(tzinfo=None) - offset).replace(tzinfo=timezone.utc) + except OverflowError as exc: + raise ValueError(f"{field_name} must be an exact timezone-aware datetime") from exc + + +def _canonical_timestamp(value: object, field_name: str) -> str: + """Render a previously detached built-in UTC instant as RFC 3339 text.""" + if type(value) is not datetime or value.tzinfo is not timezone.utc: raise ValueError(f"{field_name} must be timezone-aware") - return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + return value.isoformat().replace("+00:00", "Z") @dataclass(frozen=True, slots=True, repr=False) @@ -139,6 +157,8 @@ class ValidationAnalysisHandoff: def __post_init__(self) -> None: """Fail closed when direct construction drifts from the governed handoff.""" + requested_at = _freeze_timestamp(self.requested_at, "requested_at") + object.__setattr__(self, "requested_at", requested_at) _validate_operational_uuid(self.tenant_record_id, "tenant_record_id") for value, prefix, field_name in ( (self.handoff_reference, "validation_analysis_handoff", "handoff_reference"), @@ -173,23 +193,27 @@ def __post_init__(self) -> None: raise ValueError("reason_code must remain criterion_related_validation") if type(self.evidence_version) is not int or not 1 <= self.evidence_version <= 2_147_483_647: raise ValueError("evidence_version must be an integer from 1 through 2147483647") - if self.validation_strategy != _VALIDATION_STRATEGY: + if type(self.validation_strategy) is not str or self.validation_strategy != _VALIDATION_STRATEGY: raise ValueError("validation_strategy must remain criterion_related") - if self.kernel_repository != _KERNEL_REPOSITORY: + if type(self.kernel_repository) is not str or self.kernel_repository != _KERNEL_REPOSITORY: raise ValueError("kernel_repository must remain ContextualWisdomLab/fast-mlsirm") - if self.kernel_boundary != _KERNEL_BOUNDARY: + if type(self.kernel_boundary) is not str or self.kernel_boundary != _KERNEL_BOUNDARY: raise ValueError("kernel_boundary must remain read_only_pinned_revision") - if self.execution_state != _EXECUTION_STATE: + if type(self.execution_state) is not str or self.execution_state != _EXECUTION_STATE: raise ValueError("execution_state must remain not_executed") if self.contains_raw_person_level_values is not False: raise ValueError("handoff must not contain raw person-level values") if self.human_review_required is not True: raise ValueError("human review is mandatory for selection-validity interpretation") - if self.result_authority != _RESULT_AUTHORITY: + if type(self.result_authority) is not str or self.result_authority != _RESULT_AUTHORITY: raise ValueError("result_authority must remain scientific_evidence_only") - if self.required_result_evidence != _REQUIRED_RESULT_EVIDENCE: + if ( + type(self.required_result_evidence) is not tuple + or any(type(item) is not str for item in self.required_result_evidence) + or self.required_result_evidence != _REQUIRED_RESULT_EVIDENCE + ): raise ValueError("required_result_evidence must remain the reviewed evidence set") - if self.next_action != _NEXT_ACTION: + if type(self.next_action) is not str or self.next_action != _NEXT_ACTION: raise ValueError("next_action must remain the governed validation instruction") def __repr__(self) -> str: diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py index ae21c3d9d..9970ebb57 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py @@ -16,6 +16,7 @@ from .handoff import ( _canonical_timestamp, + _freeze_timestamp, _validate_code, _validate_digest, _validate_kernel_revision, @@ -111,18 +112,20 @@ def __post_init__(self) -> None: if type(self.converged) is not bool: raise ValueError("converged must be a boolean") _validate_positive_integer(self.iterations, "iterations") - _finite_number(self.objective_value, "objective_value") + objective = _finite_number(self.objective_value, "objective_value") gradient = _finite_number(self.maximum_gradient, "maximum_gradient") if gradient < 0: raise ValueError("maximum_gradient must be non-negative") if self.converged and self.failure_code is not None: raise ValueError("failure_code must be absent for a converged result") if not self.converged and ( - not isinstance(self.failure_code, str) or not self.failure_code + type(self.failure_code) is not str or not self.failure_code ): raise ValueError("failure_code is required for a nonconverged result") if self.failure_code is not None: _validate_code(self.failure_code, "failure_code") + object.__setattr__(self, "objective_value", objective) + object.__setattr__(self, "maximum_gradient", gradient) def to_dict(self) -> dict[str, object]: """Return deterministic convergence fields for the canonical result JSON.""" @@ -170,9 +173,9 @@ def __post_init__(self) -> None: _validate_digest(self.provenance_digest, "provenance_digest") _validate_kernel_revision(self.fast_mlsirm_revision) _validate_code(self.model_code, "model_code") - if self.backend not in _ALLOWED_BACKENDS: + if type(self.backend) is not str or self.backend not in _ALLOWED_BACKENDS: raise ValueError("backend must be rust_cpu or rust_gpu") - if self.precision not in _ALLOWED_PRECISIONS: + if type(self.precision) is not str or self.precision not in _ALLOWED_PRECISIONS: raise ValueError("precision must be f64 or f32") estimate = _finite_number(self.effect_estimate, "effect_estimate") lower = _finite_number(self.uncertainty_lower, "uncertainty_lower") @@ -188,10 +191,10 @@ def __post_init__(self) -> None: raise ValueError("convergence_diagnostics must be ConvergenceDiagnostics") if self.sample_size != self.missingness_summary.total_observations: raise ValueError("sample_size must match total_observations") - _canonical_timestamp(self.completed_at, "completed_at") - if self.result_authority != _RESULT_AUTHORITY: + completed_at = _freeze_timestamp(self.completed_at, "completed_at") + if type(self.result_authority) is not str or self.result_authority != _RESULT_AUTHORITY: raise ValueError("result_authority must remain scientific_evidence_only") - if self.execution_state != _EXECUTION_STATE: + if type(self.execution_state) is not str or self.execution_state != _EXECUTION_STATE: raise ValueError("execution_state must remain completed") if self.contains_raw_person_level_values is not False: raise ValueError("result must not contain raw person-level values") @@ -199,6 +202,10 @@ def __post_init__(self) -> None: raise ValueError("human review is mandatory for validity interpretation") if type(self.evidence_version) is not int or self.evidence_version != 1: raise ValueError("evidence_version must remain 1") + object.__setattr__(self, "effect_estimate", estimate) + object.__setattr__(self, "uncertainty_lower", lower) + object.__setattr__(self, "uncertainty_upper", upper) + object.__setattr__(self, "completed_at", completed_at) def __repr__(self) -> str: """Return a redacted representation suitable for routine application logs.""" diff --git a/packages/validity-analysis/tests/test_temporal_evidence_integrity.py b/packages/validity-analysis/tests/test_temporal_evidence_integrity.py index 25e4ab83d..0431eee55 100644 --- a/packages/validity-analysis/tests/test_temporal_evidence_integrity.py +++ b/packages/validity-analysis/tests/test_temporal_evidence_integrity.py @@ -2,7 +2,7 @@ from __future__ import annotations -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone, tzinfo import pytest @@ -10,9 +10,12 @@ ConvergenceDiagnostics, MissingnessSummary, REVIEWED_FAST_MLSIRM_REVISION, + ValidationAnalysisHandoff, ValidationAnalysisResult, build_validation_analysis_handoff, ) +from test_handoff import valid_kwargs +from test_result import result class ForgedDateTime(datetime): @@ -27,6 +30,87 @@ def isoformat(self, *args, **kwargs) -> str: # type: ignore[no-untyped-def] return "2099-12-31T23:59:59+00:00" +class ForgedReference(str): + """String subclass able to forge namespace and UUID parsing methods.""" + + def startswith(self, prefix, *args): # type: ignore[no-untyped-def] + """Pretend that an invalid namespace has the expected prefix.""" + return True + + def split(self, separator=None, maxsplit=-1): # type: ignore[no-untyped-def] + """Return a valid UUID suffix while retaining invalid source text.""" + return ["validation_analysis_handoff", "11111111-1111-4111-8111-111111111111"] + + +class ForgedFixedText(str): + """String subclass whose comparisons can forge fixed governance values.""" + + def __eq__(self, other): # type: ignore[no-untyped-def] + """Claim equality with any expected governance text.""" + return True + + def __ne__(self, other): # type: ignore[no-untyped-def] + """Claim inequality with no governance text.""" + return False + + def __hash__(self): + """Use a valid fixed-text hash for set membership forgery tests.""" + return hash("rust_cpu") + + +class MutableOffset(tzinfo): + """Timezone fixture whose offset can change after envelope construction.""" + + def __init__(self) -> None: + self.hours = 1 + + def utcoffset(self, dt: datetime | None) -> timedelta: + """Return the currently configured offset.""" + return timedelta(hours=self.hours) + + def dst(self, dt: datetime | None) -> timedelta: + """Return no daylight-saving offset.""" + return timedelta(0) + + +class UnknownOffset(tzinfo): + """Timezone fixture whose UTC offset cannot be resolved.""" + + def utcoffset(self, dt: datetime | None) -> None: + """Return no offset to exercise fail-closed validation.""" + return None + + def dst(self, dt: datetime | None) -> None: + """Return no daylight-saving offset.""" + return None + + +class ExplodingOffset(tzinfo): + """Timezone fixture whose provider raises during offset resolution.""" + + def utcoffset(self, dt: datetime | None) -> timedelta: + """Raise an untrusted provider error.""" + raise RuntimeError("offset provider failed") + + def dst(self, dt: datetime | None) -> timedelta: + """Return no daylight-saving offset when queried separately.""" + return timedelta(0) + + +class MutableReal(float): + """Numeric subclass whose float conversion changes after construction.""" + + def __new__(cls, value: float): + """Create a float-backed value with a separately mutable conversion.""" + instance = super().__new__(cls, value) + instance.current = value + return instance + + def __float__(self) -> float: + """Expose the mutable conversion used by unsafe canonicalization.""" + return self.current + + def test_handoff_rejects_datetime_subclass_that_can_forge_requested_at() -> None: """Handoff canonical evidence must not invoke caller-overridable datetime methods.""" with pytest.raises(ValueError, match="requested_at"): @@ -82,3 +166,114 @@ def test_result_rejects_datetime_subclass_that_can_forge_completed_at() -> None: ), completed_at=ForgedDateTime(2026, 8, 21, 4, 45, tzinfo=timezone.utc), ) + + +def test_handoff_and_result_detach_mutable_timezone_before_digesting() -> None: + """Freeze one UTC instant so later timezone mutation cannot rewrite evidence.""" + handoff_zone = MutableOffset() + handoff_values = valid_kwargs() + handoff_values["requested_at"] = datetime(2026, 8, 21, 4, 45, tzinfo=handoff_zone) + handoff = build_validation_analysis_handoff(**handoff_values) + handoff_before = handoff.canonical_json(), handoff.sha256_digest() + handoff_zone.hours = 2 + assert (handoff.canonical_json(), handoff.sha256_digest()) == handoff_before + + result_zone = MutableOffset() + candidate = result(completed_at=datetime(2026, 8, 21, 4, 45, tzinfo=result_zone)) + result_before = candidate.canonical_json(), candidate.sha256_digest() + result_zone.hours = 2 + assert (candidate.canonical_json(), candidate.sha256_digest()) == result_before + + +@pytest.mark.parametrize( + "timestamp", + [ + datetime.min.replace(tzinfo=timezone(timedelta(hours=1))), + datetime.max.replace(tzinfo=timezone(-timedelta(hours=1))), + datetime(2026, 8, 21, 4, 45, tzinfo=UnknownOffset()), + datetime(2026, 8, 21, 4, 45, tzinfo=ExplodingOffset()), + ], +) +def test_handoff_rejects_unrepresentable_or_untrusted_timestamp(timestamp: datetime) -> None: + """Normalize timezone-provider failures and UTC arithmetic overflow at the boundary.""" + values = valid_kwargs() + values["requested_at"] = timestamp + with pytest.raises(ValueError, match="requested_at"): + build_validation_analysis_handoff(**values) + + +@pytest.mark.parametrize( + "timestamp", + [ + datetime.min.replace(tzinfo=timezone(timedelta(hours=1))), + datetime.max.replace(tzinfo=timezone(-timedelta(hours=1))), + datetime(2026, 8, 21, 4, 45, tzinfo=UnknownOffset()), + datetime(2026, 8, 21, 4, 45, tzinfo=ExplodingOffset()), + ], +) +def test_result_rejects_unrepresentable_or_untrusted_timestamp(timestamp: datetime) -> None: + """Apply the same fail-closed timestamp contract to completed result evidence.""" + with pytest.raises(ValueError, match="completed_at"): + result(completed_at=timestamp) + + +@pytest.mark.parametrize( + "timestamp", + [ + ForgedDateTime(2026, 8, 21, 4, 45, tzinfo=timezone.utc), + datetime(2026, 8, 21, 4, 45, tzinfo=timezone(timedelta(hours=1))), + ], +) +def test_canonicalization_rejects_low_level_timestamp_reinjection(timestamp: datetime) -> None: + """Keep canonicalization fail-closed even if an object is corrupted after construction.""" + handoff = build_validation_analysis_handoff(**valid_kwargs()) + object.__setattr__(handoff, "requested_at", timestamp) + with pytest.raises(ValueError, match="requested_at"): + handoff.canonical_json() + + candidate = result() + object.__setattr__(candidate, "completed_at", timestamp) + with pytest.raises(ValueError, match="completed_at"): + candidate.canonical_json() + + +def test_handoff_rejects_runtime_text_subclasses_before_serialization() -> None: + """Reject text subclasses that can forge reference, digest, code, or fixed-value checks.""" + for field, value in ( + ("tenant_record_id", ForgedFixedText("10000000-0000-7000-8000-000000000001")), + ("handoff_reference", ForgedReference("wrong_namespace:invalid")), + ("predictor_snapshot_digest", ForgedFixedText("a" * 64)), + ("purpose_code", ForgedFixedText("selection_validity_analysis")), + ("fast_mlsirm_revision", ForgedFixedText(REVIEWED_FAST_MLSIRM_REVISION)), + ("validation_strategy", ForgedFixedText("criterion_related")), + ("next_action", ForgedFixedText("governed")), + ): + values = valid_kwargs() + values[field] = value + with pytest.raises(ValueError, match=field): + ValidationAnalysisHandoff(**values) + + +def test_result_snapshots_numeric_values_before_canonicalization() -> None: + """Detach mutable numeric subclasses before recording scientific evidence bytes.""" + objective = MutableReal(-12.5) + gradient = MutableReal(0.0001) + diagnostics = ConvergenceDiagnostics( + converged=True, + iterations=42, + objective_value=objective, + maximum_gradient=gradient, + ) + estimate = MutableReal(0.42) + candidate = result(effect_estimate=estimate, convergence_diagnostics=diagnostics) + before = candidate.canonical_json(), candidate.sha256_digest() + objective.current = -1.0 + gradient.current = 0.5 + estimate.current = 0.69 + assert (candidate.canonical_json(), candidate.sha256_digest()) == before + + +def test_result_rejects_forged_backend_text() -> None: + """Do not allow a string subclass to forge an allowed backend membership check.""" + with pytest.raises(ValueError, match="backend"): + result(backend=ForgedFixedText("numpy")) From 508184420607ccd7213a73cd3e20e3041c9bdd27 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 07:46:35 -0700 Subject: [PATCH 031/158] test(validity): reject overflowing worker numerics --- .../tests/test_numeric_overflow_contract.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 packages/validity-analysis/tests/test_numeric_overflow_contract.py diff --git a/packages/validity-analysis/tests/test_numeric_overflow_contract.py b/packages/validity-analysis/tests/test_numeric_overflow_contract.py new file mode 100644 index 000000000..a4fb88115 --- /dev/null +++ b/packages/validity-analysis/tests/test_numeric_overflow_contract.py @@ -0,0 +1,18 @@ +"""Regression for malformed worker numerics that overflow float conversion.""" + +import pytest + +from orgmetra_validity_analysis import ConvergenceDiagnostics + + +def test_oversized_worker_numeric_is_rejected_as_value_error() -> None: + """Normalize float-conversion overflow to the package's ValueError contract.""" + oversized_integer = 10**10000 + + with pytest.raises(ValueError, match="finite number"): + ConvergenceDiagnostics( + converged=True, + iterations=1, + objective_value=oversized_integer, + maximum_gradient=0.1, + ) From a55f95cee13b58a064abcafe42a159be940d6854 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 07:46:43 -0700 Subject: [PATCH 032/158] test(validity): require ADR-wide quality trigger --- .../tests/test_workflow_trigger_contract.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 packages/validity-analysis/tests/test_workflow_trigger_contract.py diff --git a/packages/validity-analysis/tests/test_workflow_trigger_contract.py b/packages/validity-analysis/tests/test_workflow_trigger_contract.py new file mode 100644 index 000000000..2a0295fc9 --- /dev/null +++ b/packages/validity-analysis/tests/test_workflow_trigger_contract.py @@ -0,0 +1,13 @@ +"""Regression for the validity package's repository-wide ADR numbering trigger.""" + +from pathlib import Path + + +def test_any_adr_change_runs_the_adr_numbering_regression() -> None: + """Keep ADR uniqueness enforcement reachable when any decision record changes.""" + repository_root = Path(__file__).resolve().parents[3] + workflow = (repository_root / ".github" / "workflows" / "validity-analysis-quality.yml").read_text( + encoding="utf-8" + ) + + assert ' - "docs/adr/**"' in workflow From 06db36667eb791ff69b8d43bed85e5746887f9ab Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 07:47:28 -0700 Subject: [PATCH 033/158] fix(validity): normalize numeric conversion failures --- .../src/orgmetra_validity_analysis/result.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py index 9970ebb57..3f684ab10 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py @@ -43,10 +43,13 @@ def _validate_positive_integer(value: object, field_name: str) -> None: def _finite_number(value: object, field_name: str) -> float: - """Return one finite real number and reject booleans or non-numeric text.""" + """Return one finite real number and normalize invalid numerics to ValueError.""" if isinstance(value, bool) or not isinstance(value, Real): raise ValueError(f"{field_name} must be a finite number") - number = float(value) + try: + number = float(value) + except (OverflowError, TypeError, ValueError) as exc: + raise ValueError(f"{field_name} must be a finite number") from exc if not isfinite(number): raise ValueError(f"{field_name} must be a finite number") return number From be52fc7637656fa2560d8572ba80204c59f25cbe Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 07:47:38 -0700 Subject: [PATCH 034/158] fix(validity): run ADR uniqueness check for all ADR changes --- .github/workflows/validity-analysis-quality.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validity-analysis-quality.yml b/.github/workflows/validity-analysis-quality.yml index 5b1ae41fc..5c9c57e8a 100644 --- a/.github/workflows/validity-analysis-quality.yml +++ b/.github/workflows/validity-analysis-quality.yml @@ -8,7 +8,7 @@ on: - main paths: - "packages/validity-analysis/**" - - "docs/adr/0027-governed-selection-validity-analysis-handoff.md" + - "docs/adr/**" - "docs/doctoring/validation-analysis-handoff-references.md" - "docs/traceability/validation-analysis-handoff.md" - ".github/requirements/foundation-test.txt" From 376c2ca2e9d69b0f8e2ed5337b9fb22a9fbcd0ee Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 07:48:07 -0700 Subject: [PATCH 035/158] docs(validity): pin reproducible selection-validation references --- .../validation-analysis-handoff-references.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/doctoring/validation-analysis-handoff-references.md b/docs/doctoring/validation-analysis-handoff-references.md index d5de152f1..cfde9303b 100644 --- a/docs/doctoring/validation-analysis-handoff-references.md +++ b/docs/doctoring/validation-analysis-handoff-references.md @@ -1,20 +1,23 @@ # Validation-analysis handoff references -Material decisions for ADR 0027 were checked against the following primary/authoritative sources on 2026-08-21. +Material decisions for ADR 0027 were checked against the following primary/authoritative sources on 2026-08-21. Regulatory currency was rechecked on 2026-08-29; fixed publication identifiers are retained so an auditor can reproduce the cited text even when agency web pages change. ## APA 7 references -Electronic Code of Federal Regulations. (2026). *29 C.F.R. pt. 1607—Uniform Guidelines on Employee Selection Procedures (1978).* Retrieved August 21, 2026, from https://www.ecfr.gov/current/title-29/subtitle-B/chapter-XIV/part-1607 +Equal Employment Opportunity Commission, Civil Service Commission, Department of Justice, & Department of Labor. (1978). *Uniform Guidelines on Employee Selection Procedures (1978)*, 43 Fed. Reg. 38,290 (August 25, 1978) (codified at 29 C.F.R. pt. 1607). The EEOC continues to list 29 C.F.R. pt. 1607 among its Title VII regulations: https://www.eeoc.gov/regulations-and-guidelines -Society for Industrial and Organizational Psychology. (2018). *Principles for the validation and use of personnel selection procedures* (5th ed.). Cambridge University Press. https://www.apa.org/ed/accreditation/personnel-selection-procedures.pdf +Society for Industrial and Organizational Psychology. (2018). Principles for the validation and use of personnel selection procedures. *Industrial and Organizational Psychology, 11*(S1), 1–97. https://doi.org/10.1017/iop.2018.195 ContextualWisdomLab. (2026). *fast-mlsirm* (Commit 04d0bc21a2a20693bcf16108cd76d394fe844d23) [Computer software]. GitHub. https://github.com/ContextualWisdomLab/fast-mlsirm/tree/04d0bc21a2a20693bcf16108cd76d394fe844d23 Tabassi, E. (2023). *Artificial intelligence risk management framework (AI RMF 1.0)* (NIST AI 100-1). National Institute of Standards and Technology. https://doi.org/10.6028/NIST.AI.100-1 +Office of Personnel Management. (2026). *Removal of references to the Uniform Guidelines on Employee Selection Procedures in federal personnel regulations*, 91 Fed. Reg. 48,234 (July 31, 2026) (interim final rule, RIN 3206-AP20). + ## Decision notes -- 29 C.F.R. §§ 1607.5 and 1607.14 support keeping criterion-related validity evidence tied to an explicit study design, job relevance, accuracy, reporting, and documentation rather than treating a bare coefficient as sufficient evidence. -- The SIOP Principles are the professional validation baseline used for the handoff's evidence-and-human-review posture. +- 43 Fed. Reg. 38,290 and the still-listed EEOC 29 C.F.R. pt. 1607 source support keeping criterion-related validity evidence tied to an explicit study design, job relevance, accuracy, reporting, and documentation rather than treating a bare coefficient as sufficient evidence. The fixed Federal Register identifier, not a mutable `/current/` eCFR URL, is the reproducible source for the 1978 text cited by this ADR. +- The July 31, 2026 OPM interim final rule removed UGESP references from specified federal civil-service regulations. Orgmetra therefore does not present UGESP as an undifferentiated government-wide mandate; applicability must be evaluated for the employer, jurisdiction, decision, and governing law at use time. +- The SIOP Principles are the professional validation baseline used for the handoff's evidence-and-human-review posture. The journal citation above fixes volume 11, Supplement S1, pages 1–97, and DOI 10.1017/iop.2018.195. - The fast-mlsirm commit is recorded as a read-only dependency coordinate only. This Orgmetra slice does not modify or duplicate its numerical implementation. - NIST AI RMF's govern, map, measure, and manage functions support preserving backend, precision, provenance, convergence, and human-review fields as inspectable result evidence rather than treating a model response as an autonomous decision. From 2ff2582709dd03038f5a73816e9cde9605764836 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 07:49:14 -0700 Subject: [PATCH 036/158] docs(validity): trace repaired evidence boundaries --- docs/traceability/validation-analysis-handoff.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/traceability/validation-analysis-handoff.md b/docs/traceability/validation-analysis-handoff.md index 455a8380c..65e22b253 100644 --- a/docs/traceability/validation-analysis-handoff.md +++ b/docs/traceability/validation-analysis-handoff.md @@ -13,9 +13,10 @@ Can an organization send one exact, reviewable validation study to its statistic | Privacy minimization | no raw person-level values in canonical handoff or result; result canonicalization accepts only exact governed missingness/convergence runtime types | canonical-payload/redacted-repr regressions plus subclass-injection rejection | | Human authority | requester/reviewer references must differ, and the host must re-resolve both within the tenant and prove they resolve to distinct authoritative actor identities before execution | direct-construction fail-closed regression plus `test_next_action_requires_resolved_actor_identity_separation` | | Scientific evidence | effect estimate, uncertainty interval, sample size, internally possible aggregate missingness, convergence diagnostics | immutable required-result-evidence regression plus impossible-missingness rejection | -| Numerical result boundary | handoff digest, pinned fast-mlsirm revision, Rust CPU/GPU backend, precision, finite estimate/interval, aggregate missingness, explicit convergence state | `ValidationAnalysisResult` invariant/canonicalization regressions and exact-runtime-type checks | +| Numerical result boundary | handoff digest, pinned fast-mlsirm revision, Rust CPU/GPU backend, precision, finite estimate/interval, aggregate missingness, explicit convergence state | `ValidationAnalysisResult` invariant/canonicalization regressions, exact-runtime-type checks, and oversized-numeric `ValueError` normalization | | Execution boundary | `not_executed`, `scientific_evidence_only`, read-only pinned foreign dependency | immutable governance regressions | | Reproducibility | construction-time UTC timestamp snapshots, finite numeric snapshots, canonical RFC 3339 time, canonical JSON, SHA-256 handoff digest | mutable timezone/numeric and UTC-boundary regressions plus deterministic serialization/digest tests | +| Decision-record integrity | ADR numbers remain unique repository-wide and any `docs/adr/**` change reaches the validity quality gate | ADR uniqueness regression plus workflow-trigger contract regression | ## Maturity From 4359cfbb4cd8ee5885acb9110d7723099d712353 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 07:49:37 -0700 Subject: [PATCH 037/158] docs(validity): record numeric and missingness ownership boundaries --- .../0027-governed-selection-validity-analysis-handoff.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/adr/0027-governed-selection-validity-analysis-handoff.md b/docs/adr/0027-governed-selection-validity-analysis-handoff.md index c3862a968..ac74dbec8 100644 --- a/docs/adr/0027-governed-selection-validity-analysis-handoff.md +++ b/docs/adr/0027-governed-selection-validity-analysis-handoff.md @@ -26,7 +26,7 @@ Orgmetra adds a leaf `orgmetra_validity_analysis` package whose `ValidationAnaly - remains `not_executed`, `scientific_evidence_only`, and human-review-required; - produces deterministic canonical JSON and a SHA-256 digest for audit/result correlation. -Both handoff and result envelopes detach exact timezone-aware timestamps to one built-in UTC instant at construction. Result numeric evidence is converted to finite built-in floats before storage, so caller-controlled timezone or numeric runtime behavior cannot rewrite canonical evidence after validation. +Both handoff and result envelopes detach exact timezone-aware timestamps to one built-in UTC instant at construction. Result numeric evidence is converted to finite built-in floats before storage, and conversion failures including numeric overflow are normalized to the package's fail-closed `ValueError` contract, so caller-controlled timezone or numeric runtime behavior cannot rewrite canonical evidence after validation or escape normal malformed-result handling. The same package also validates `ValidationAnalysisResult` envelopes returned by the approved offline worker. A result must link to the handoff digest and the same pinned revision, identify a Rust CPU or GPU backend and precision, provide finite effect and interval values, match its sample size to aggregate missingness counts, reject impossible complete-versus-missing count combinations, and include explicit convergence diagnostics. The canonicalization boundary accepts only the exact governed `MissingnessSummary` and `ConvergenceDiagnostics` runtime types so subclass method overrides cannot add unreviewed or person-level fields to immutable result evidence. A nonconverged result remains typed scientific evidence requiring human review; it cannot be treated as a valid selection procedure or an employment decision. @@ -40,7 +40,7 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - A buyer or auditor can identify exactly which governed study evidence a result was supposed to analyze without copying raw worker data into the handoff. - Aggregate missingness evidence cannot claim all observations are complete while simultaneously reporting predictor- or criterion-missing observations. - Result canonicalization cannot be extended by an unreviewed subclass to serialize extra decision-like or person-level fields. -- Caller-controlled timestamp and numeric runtime behavior cannot rewrite an accepted canonical digest after construction. +- Caller-controlled timestamp and numeric runtime behavior cannot rewrite an accepted canonical digest after construction or turn malformed oversized worker output into an uncaught exception type. - Human interpretation remains explicit and separate from numerical output. - The dedicated-writer boundary remains intact: Orgmetra consumes only a pinned foreign revision/contract boundary and never mutates fast-mlsirm. @@ -48,11 +48,12 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - This slice does not execute a statistical model, estimate validity, correct for measurement error/range restriction, evaluate adverse impact, or assert legal compliance. - Sampling design, estimator choice, missing-data treatment, reliability evidence, multiplicity, transportability, fairness analysis, and model diagnostics must be encoded in the referenced analysis plan and reviewed before execution. +- The generic result envelope does not invent an estimator-specific minimum complete-case count. Whether a converged estimator is identified under a particular missing-data design belongs to the reviewed analysis plan and numerical-worker contract; Orgmetra fails closed on impossible aggregate counts without silently replacing that foreign scientific contract with complete-case analysis. - The package validates the result envelope, but a future execution adapter must still re-resolve the handoff references, verify the result provenance artifact, and attach evidence only after accountable human review. ## Verification -The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number. +The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. ## References From 2f6d1dca8a6ec2ed1e23453860018b5b58344418 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 15:43:55 -0700 Subject: [PATCH 038/158] test(validity): require shared config quality triggers --- .../tests/test_quality_workflow_trigger.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 packages/validity-analysis/tests/test_quality_workflow_trigger.py diff --git a/packages/validity-analysis/tests/test_quality_workflow_trigger.py b/packages/validity-analysis/tests/test_quality_workflow_trigger.py new file mode 100644 index 000000000..aed4f4060 --- /dev/null +++ b/packages/validity-analysis/tests/test_quality_workflow_trigger.py @@ -0,0 +1,27 @@ +"""Regression tests for the validity-analysis quality-gate trigger surface.""" + +from pathlib import Path + + +_WORKFLOW_PATH = Path(".github/workflows/validity-analysis-quality.yml") +_SHARED_TEST_CONFIGURATION = ( + ".gitignore", + ".python-version", + "conftest.py", + "packages/conftest.py", + "pyproject.toml", + "pytest.ini", + "setup.cfg", + "tox.ini", +) + + +def test_quality_workflow_retriggers_on_shared_test_configuration() -> None: + """Require shared test/runtime configuration changes to retrigger this gate.""" + workflow = _WORKFLOW_PATH.read_text(encoding="utf-8") + + for path in _SHARED_TEST_CONFIGURATION: + assert f'- "{path}"' in workflow, ( + f"{path} can change package test or clean-checkout behavior and must retrigger " + "Validity Analysis Handoff Quality" + ) From 8815f73d478b76a4b54b977811466edd23611a29 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 15:44:50 -0700 Subject: [PATCH 039/158] fix(validity): retrigger quality on shared config --- .github/workflows/validity-analysis-quality.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/validity-analysis-quality.yml b/.github/workflows/validity-analysis-quality.yml index 5c9c57e8a..17d5e8755 100644 --- a/.github/workflows/validity-analysis-quality.yml +++ b/.github/workflows/validity-analysis-quality.yml @@ -13,6 +13,14 @@ on: - "docs/traceability/validation-analysis-handoff.md" - ".github/requirements/foundation-test.txt" - ".github/workflows/validity-analysis-quality.yml" + - ".gitignore" + - ".python-version" + - "conftest.py" + - "packages/conftest.py" + - "pyproject.toml" + - "pytest.ini" + - "setup.cfg" + - "tox.ini" workflow_dispatch: permissions: From 3c2a1dec4cb0668542d73ae71e4605ca6a708f17 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 15:45:03 -0700 Subject: [PATCH 040/158] docs(validity): record shared-config gate integrity --- packages/validity-analysis/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/validity-analysis/CHANGELOG.md b/packages/validity-analysis/CHANGELOG.md index fd7762043..d0ff8bd9f 100644 --- a/packages/validity-analysis/CHANGELOG.md +++ b/packages/validity-analysis/CHANGELOG.md @@ -10,3 +10,4 @@ - Reject impossible aggregate missingness where complete observations overlap either predictor-missing or criterion-missing counts beyond the sample total. - Require exact governed missingness/convergence runtime types so subclass method overrides cannot inject unreviewed or person-level fields into canonical result evidence. - Freeze exact UTC timestamps and finite numeric values at construction, and reject runtime-type forgery before canonical evidence serialization. +- Make `Validity Analysis Handoff Quality` retrigger on shared repository Python/test/clean-checkout configuration, with an executable regression preventing stale package-quality evidence after shared tooling changes. From 6ca554791595d925a76587378b543e7dbc3dc20b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 15:45:16 -0700 Subject: [PATCH 041/158] docs(validity): trace shared-config quality evidence --- docs/traceability/validation-analysis-handoff.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/traceability/validation-analysis-handoff.md b/docs/traceability/validation-analysis-handoff.md index 65e22b253..f9f8c755e 100644 --- a/docs/traceability/validation-analysis-handoff.md +++ b/docs/traceability/validation-analysis-handoff.md @@ -17,6 +17,7 @@ Can an organization send one exact, reviewable validation study to its statistic | Execution boundary | `not_executed`, `scientific_evidence_only`, read-only pinned foreign dependency | immutable governance regressions | | Reproducibility | construction-time UTC timestamp snapshots, finite numeric snapshots, canonical RFC 3339 time, canonical JSON, SHA-256 handoff digest | mutable timezone/numeric and UTC-boundary regressions plus deterministic serialization/digest tests | | Decision-record integrity | ADR numbers remain unique repository-wide and any `docs/adr/**` change reaches the validity quality gate | ADR uniqueness regression plus workflow-trigger contract regression | +| Quality-evidence freshness | package quality reruns whenever shared repository Python/test/clean-checkout configuration can alter execution or tracked-tree cleanliness | `test_quality_workflow_retriggers_on_shared_test_configuration` plus `.github/workflows/validity-analysis-quality.yml`; this supplemental package gate does not replace central required workflows | ## Maturity From 276b395e7bed3beff3a1333c0dab2b5b13736bf6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 04:06:44 +0900 Subject: [PATCH 042/158] test(validity): replay owner contract on protected truth --- ...ned-selection-validity-analysis-handoff.md | 60 ++++ .../validation-analysis-handoff-references.md | 23 ++ .../validation-analysis-handoff.md | 26 ++ packages/validity-analysis/CHANGELOG.md | 13 + packages/validity-analysis/README.md | 37 +++ packages/validity-analysis/pyproject.toml | 24 ++ .../orgmetra_validity_analysis/__init__.py | 17 + .../src/orgmetra_validity_analysis/handoff.py | 305 ++++++++++++++++++ .../src/orgmetra_validity_analysis/result.py | 248 ++++++++++++++ .../tests/test_adr_numbering.py | 17 + .../validity-analysis/tests/test_handoff.py | 221 +++++++++++++ .../tests/test_host_resolution_contract.py | 34 ++ .../tests/test_numeric_overflow_contract.py | 18 ++ .../tests/test_quality_workflow_trigger.py | 16 + .../validity-analysis/tests/test_result.py | 211 ++++++++++++ .../tests/test_temporal_evidence_integrity.py | 279 ++++++++++++++++ .../tests/test_workflow_trigger_contract.py | 18 ++ 17 files changed, 1567 insertions(+) create mode 100644 docs/adr/0027-governed-selection-validity-analysis-handoff.md create mode 100644 docs/doctoring/validation-analysis-handoff-references.md create mode 100644 docs/traceability/validation-analysis-handoff.md create mode 100644 packages/validity-analysis/CHANGELOG.md create mode 100644 packages/validity-analysis/README.md create mode 100644 packages/validity-analysis/pyproject.toml create mode 100644 packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py create mode 100644 packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py create mode 100644 packages/validity-analysis/src/orgmetra_validity_analysis/result.py create mode 100644 packages/validity-analysis/tests/test_adr_numbering.py create mode 100644 packages/validity-analysis/tests/test_handoff.py create mode 100644 packages/validity-analysis/tests/test_host_resolution_contract.py create mode 100644 packages/validity-analysis/tests/test_numeric_overflow_contract.py create mode 100644 packages/validity-analysis/tests/test_quality_workflow_trigger.py create mode 100644 packages/validity-analysis/tests/test_result.py create mode 100644 packages/validity-analysis/tests/test_temporal_evidence_integrity.py create mode 100644 packages/validity-analysis/tests/test_workflow_trigger_contract.py diff --git a/docs/adr/0027-governed-selection-validity-analysis-handoff.md b/docs/adr/0027-governed-selection-validity-analysis-handoff.md new file mode 100644 index 000000000..ac74dbec8 --- /dev/null +++ b/docs/adr/0027-governed-selection-validity-analysis-handoff.md @@ -0,0 +1,60 @@ +# ADR 0027: Govern selection-validity numerical work through an immutable handoff + +- Status: Proposed +- Maturity: Active PR only; not protected-branch truth +- Date: 2026-08-21 +- Owners: Orgmetra Workforce Validation + +## Context + +Protected Orgmetra already preserves exact validation-study cases, sealed selection evidence, candidate-to-worker lineage, and Job/cycle/staffing-scoped criterion observations. The remaining boundary is dangerous if left implicit: a statistical worker could receive an underspecified study, silently use a different dependency revision, or turn a model result into an employment decision. + +The Uniform Guidelines recognize criterion-related validity evidence as empirical evidence relating a selection procedure to important job-performance elements and require validity studies to be accurate, standardized, documented, and periodically reviewed for currency. SIOP's *Principles for the Validation and Use of Personnel Selection Procedures* likewise treats validation as an evidence-and-inference problem rather than a correlation-only shortcut. + +`ContextualWisdomLab/fast-mlsirm` owns numerical psychometric/statistical kernels. Its protected `main` was freshly resolved to commit `04d0bc21a2a20693bcf16108cd76d394fe844d23` on 2026-08-21. Orgmetra must not copy that implementation or write the foreign repository. + +## Decision + +Orgmetra adds a leaf `orgmetra_validity_analysis` package whose `ValidationAnalysisHandoff`: + +- binds the exact tenant, validation study, Job, predictor snapshot, criterion snapshot, population snapshot, decision policy, and analysis plan through opaque references plus SHA-256 evidence digests; +- binds distinct requester and reviewer actor references; +- pins fast-mlsirm to reviewed immutable commit `04d0bc21a2a20693bcf16108cd76d394fe844d23`; +- declares the numerical boundary `read_only_pinned_revision` and the initial strategy `criterion_related`; +- requires downstream result evidence for effect estimate, uncertainty interval, sample size, missingness summary, and convergence diagnostics; +- serializes no raw person-level predictor, criterion, candidate, or worker values; +- remains `not_executed`, `scientific_evidence_only`, and human-review-required; +- produces deterministic canonical JSON and a SHA-256 digest for audit/result correlation. + +Both handoff and result envelopes detach exact timezone-aware timestamps to one built-in UTC instant at construction. Result numeric evidence is converted to finite built-in floats before storage, and conversion failures including numeric overflow are normalized to the package's fail-closed `ValueError` contract, so caller-controlled timezone or numeric runtime behavior cannot rewrite canonical evidence after validation or escape normal malformed-result handling. + +The same package also validates `ValidationAnalysisResult` envelopes returned by the approved offline worker. A result must link to the handoff digest and the same pinned revision, identify a Rust CPU or GPU backend and precision, provide finite effect and interval values, match its sample size to aggregate missingness counts, reject impossible complete-versus-missing count combinations, and include explicit convergence diagnostics. The canonicalization boundary accepts only the exact governed `MissingnessSummary` and `ConvergenceDiagnostics` runtime types so subclass method overrides cannot add unreviewed or person-level fields to immutable result evidence. A nonconverged result remains typed scientific evidence requiring human review; it cannot be treated as a valid selection procedure or an employment decision. + +The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, and prove requester/reviewer identities are distinct authoritative actors. + +## Consequences + +### Positive + +- Statistical work cannot silently drift to an unreviewed fast-mlsirm revision. +- A buyer or auditor can identify exactly which governed study evidence a result was supposed to analyze without copying raw worker data into the handoff. +- Aggregate missingness evidence cannot claim all observations are complete while simultaneously reporting predictor- or criterion-missing observations. +- Result canonicalization cannot be extended by an unreviewed subclass to serialize extra decision-like or person-level fields. +- Caller-controlled timestamp and numeric runtime behavior cannot rewrite an accepted canonical digest after construction or turn malformed oversized worker output into an uncaught exception type. +- Human interpretation remains explicit and separate from numerical output. +- The dedicated-writer boundary remains intact: Orgmetra consumes only a pinned foreign revision/contract boundary and never mutates fast-mlsirm. + +### Limitations + +- This slice does not execute a statistical model, estimate validity, correct for measurement error/range restriction, evaluate adverse impact, or assert legal compliance. +- Sampling design, estimator choice, missing-data treatment, reliability evidence, multiplicity, transportability, fairness analysis, and model diagnostics must be encoded in the referenced analysis plan and reviewed before execution. +- The generic result envelope does not invent an estimator-specific minimum complete-case count. Whether a converged estimator is identified under a particular missing-data design belongs to the reviewed analysis plan and numerical-worker contract; Orgmetra fails closed on impossible aggregate counts without silently replacing that foreign scientific contract with complete-case analysis. +- The package validates the result envelope, but a future execution adapter must still re-resolve the handoff references, verify the result provenance artifact, and attach evidence only after accountable human review. + +## Verification + +The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. + +## References + +See `docs/doctoring/validation-analysis-handoff-references.md`. diff --git a/docs/doctoring/validation-analysis-handoff-references.md b/docs/doctoring/validation-analysis-handoff-references.md new file mode 100644 index 000000000..cfde9303b --- /dev/null +++ b/docs/doctoring/validation-analysis-handoff-references.md @@ -0,0 +1,23 @@ +# Validation-analysis handoff references + +Material decisions for ADR 0027 were checked against the following primary/authoritative sources on 2026-08-21. Regulatory currency was rechecked on 2026-08-29; fixed publication identifiers are retained so an auditor can reproduce the cited text even when agency web pages change. + +## APA 7 references + +Equal Employment Opportunity Commission, Civil Service Commission, Department of Justice, & Department of Labor. (1978). *Uniform Guidelines on Employee Selection Procedures (1978)*, 43 Fed. Reg. 38,290 (August 25, 1978) (codified at 29 C.F.R. pt. 1607). The EEOC continues to list 29 C.F.R. pt. 1607 among its Title VII regulations: https://www.eeoc.gov/regulations-and-guidelines + +Society for Industrial and Organizational Psychology. (2018). Principles for the validation and use of personnel selection procedures. *Industrial and Organizational Psychology, 11*(S1), 1–97. https://doi.org/10.1017/iop.2018.195 + +ContextualWisdomLab. (2026). *fast-mlsirm* (Commit 04d0bc21a2a20693bcf16108cd76d394fe844d23) [Computer software]. GitHub. https://github.com/ContextualWisdomLab/fast-mlsirm/tree/04d0bc21a2a20693bcf16108cd76d394fe844d23 + +Tabassi, E. (2023). *Artificial intelligence risk management framework (AI RMF 1.0)* (NIST AI 100-1). National Institute of Standards and Technology. https://doi.org/10.6028/NIST.AI.100-1 + +Office of Personnel Management. (2026). *Removal of references to the Uniform Guidelines on Employee Selection Procedures in federal personnel regulations*, 91 Fed. Reg. 48,234 (July 31, 2026) (interim final rule, RIN 3206-AP20). + +## Decision notes + +- 43 Fed. Reg. 38,290 and the still-listed EEOC 29 C.F.R. pt. 1607 source support keeping criterion-related validity evidence tied to an explicit study design, job relevance, accuracy, reporting, and documentation rather than treating a bare coefficient as sufficient evidence. The fixed Federal Register identifier, not a mutable `/current/` eCFR URL, is the reproducible source for the 1978 text cited by this ADR. +- The July 31, 2026 OPM interim final rule removed UGESP references from specified federal civil-service regulations. Orgmetra therefore does not present UGESP as an undifferentiated government-wide mandate; applicability must be evaluated for the employer, jurisdiction, decision, and governing law at use time. +- The SIOP Principles are the professional validation baseline used for the handoff's evidence-and-human-review posture. The journal citation above fixes volume 11, Supplement S1, pages 1–97, and DOI 10.1017/iop.2018.195. +- The fast-mlsirm commit is recorded as a read-only dependency coordinate only. This Orgmetra slice does not modify or duplicate its numerical implementation. +- NIST AI RMF's govern, map, measure, and manage functions support preserving backend, precision, provenance, convergence, and human-review fields as inspectable result evidence rather than treating a model response as an autonomous decision. diff --git a/docs/traceability/validation-analysis-handoff.md b/docs/traceability/validation-analysis-handoff.md new file mode 100644 index 000000000..f2e832efa --- /dev/null +++ b/docs/traceability/validation-analysis-handoff.md @@ -0,0 +1,26 @@ +# Selection-validity analysis handoff traceability + +## Buyer question + +Can an organization send one exact, reviewable validation study to its statistical engine without copying raw person-level values into a workflow envelope, silently changing the numerical dependency, or treating model output as an employment decision? + +## Active-PR contract + +| Concern | Orgmetra evidence | Verification | +|---|---|---| +| Exact study scope | tenant, validation-study, Job, predictor, criterion, population, decision-policy, and analysis-plan references plus digests | namespace/UUID/digest regressions | +| Dependency integrity | immutable fast-mlsirm commit `04d0bc21a2a20693bcf16108cd76d394fe844d23` | malformed and unreviewed revision rejection | +| Privacy minimization | no raw person-level values in canonical handoff or result; result canonicalization accepts only exact governed missingness/convergence runtime types | canonical-payload/redacted-repr regressions plus subclass-injection rejection | +| Human authority | requester/reviewer references must differ, and the host must re-resolve both within the tenant and prove they resolve to distinct authoritative actor identities before execution | direct-construction fail-closed regression plus `test_next_action_requires_resolved_actor_identity_separation` | +| Scientific evidence | effect estimate, uncertainty interval, sample size, internally possible aggregate missingness, convergence diagnostics | immutable required-result-evidence regression plus impossible-missingness rejection | +| Numerical result boundary | handoff digest, pinned fast-mlsirm revision, Rust CPU/GPU backend, precision, finite estimate/interval, aggregate missingness, explicit convergence state | `ValidationAnalysisResult` invariant/canonicalization regressions, exact-runtime-type checks, and oversized-numeric `ValueError` normalization | +| Execution boundary | `not_executed`, `scientific_evidence_only`, read-only pinned foreign dependency | immutable governance regressions | +| Reproducibility | construction-time UTC timestamp snapshots, finite numeric snapshots, canonical RFC 3339 time, canonical JSON, SHA-256 handoff digest | mutable timezone/numeric and UTC-boundary regressions plus deterministic serialization/digest tests | +| Decision-record integrity | ADR numbers remain unique repository-wide and any `docs/adr/**` change reaches the consolidated Foundation CI validity regression | ADR uniqueness regression plus Foundation CI workflow-trigger contract regression | +| Quality-evidence freshness | consolidated Foundation CI runs the validity package on every `develop` pull request without a repository path filter, so shared Python/test/clean-checkout configuration cannot silently bypass the package gate | `test_foundation_ci_retriggers_without_path_filter` and `test_foundation_ci_runs_validity_analysis_and_adr_changes`; central required workflows remain separate gates | + +## Maturity + +`implemented_on_active_pr`. + +Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. The active package now validates the minimum returned numerical/provenance envelope, including missingness consistency and exact aggregate-evidence runtime types, but protected Orgmetra evidence still requires host re-resolution, result-artifact verification, terminal checks, independent review, and accountable human interpretation. diff --git a/packages/validity-analysis/CHANGELOG.md b/packages/validity-analysis/CHANGELOG.md new file mode 100644 index 000000000..d0ff8bd9f --- /dev/null +++ b/packages/validity-analysis/CHANGELOG.md @@ -0,0 +1,13 @@ +# Changelog + +## 0.1.0 - Unreleased + +- Add a governed, value-minimized criterion-related validity analysis handoff. +- Pin the reviewed read-only fast-mlsirm dependency revision. +- Require separate requester/reviewer references and authoritative tenant-scoped re-resolution proving they resolve to distinct actor identities before execution. +- Require deterministic canonical evidence and 100% owned production statement/branch coverage. +- Validate a digest-linked Rust CPU/GPU result envelope with finite estimates, aggregate missingness, and explicit convergence or nonconvergence diagnostics. +- Reject impossible aggregate missingness where complete observations overlap either predictor-missing or criterion-missing counts beyond the sample total. +- Require exact governed missingness/convergence runtime types so subclass method overrides cannot inject unreviewed or person-level fields into canonical result evidence. +- Freeze exact UTC timestamps and finite numeric values at construction, and reject runtime-type forgery before canonical evidence serialization. +- Make `Validity Analysis Handoff Quality` retrigger on shared repository Python/test/clean-checkout configuration, with an executable regression preventing stale package-quality evidence after shared tooling changes. diff --git a/packages/validity-analysis/README.md b/packages/validity-analysis/README.md new file mode 100644 index 000000000..b56e9a040 --- /dev/null +++ b/packages/validity-analysis/README.md @@ -0,0 +1,37 @@ +# Orgmetra validity-analysis handoff + +This package creates an immutable **selection-validity analysis handoff** and validates the matching numerical result envelope. It is the boundary between Orgmetra's authoritative validation-study evidence and numerical work owned by `ContextualWisdomLab/fast-mlsirm`. + +## What it does + +`build_validation_analysis_handoff(...)` binds one tenant, validation study, Job, predictor snapshot, criterion snapshot, population snapshot, decision policy, analysis plan, requester, reviewer, and the reviewed fast-mlsirm revision `04d0bc21a2a20693bcf16108cd76d394fe844d23`. + +The resulting canonical JSON is digest-addressable, contains no raw person-level predictor or criterion values, and remains `not_executed`. Its timestamp is detached to one UTC instant at construction so later timezone-provider changes cannot rewrite the digest. Required result evidence is explicit: effect estimate, uncertainty interval, sample size, missingness summary, and convergence diagnostics. + +`ValidationAnalysisResult` accepts only a result linked to the handoff digest and the same reviewed fast-mlsirm revision. It records the Rust CPU/GPU backend, precision, aggregate missingness counts, finite effect and interval values, and explicit convergence or nonconvergence diagnostics. Timestamps and finite numeric values are snapshotted before canonicalization, and the result envelope accepts only the exact governed `MissingnessSummary` and `ConvergenceDiagnostics` runtime types, preventing mutable runtime values or subclass method overrides from adding unreviewed or person-level fields to canonical audit evidence. Missingness counts must be internally possible: complete observations cannot overlap either predictor-missing or criterion-missing observations beyond the declared sample total. It never promotes a result to an employment decision; human review remains mandatory. + +## What it does not do + +- It does **not** run statistics. +- It does **not** run or reproduce the fast-mlsirm numerical kernel. +- It does **not** query fast-mlsirm or any other CWL application's database. +- It does **not** claim that a selection procedure is valid. +- It does **not** interpret adverse impact. +- It does **not** authorize hiring, promotion, termination, compensation, or another employment decision. + +The fast-mlsirm repository remains a dedicated-writer dependency. This package records only the immutable revision reviewed for the handoff. + +## Host obligations + +Before an approved offline validation worker executes the handoff, the Orgmetra host must re-resolve every reference inside `tenant_record_id` and prove that the predictor, criterion, population, and policy evidence belong to the exact validation study and Job. The requester and reviewer must resolve to distinct authoritative actors. A numerical result is scientific evidence for accountable human interpretation, never an autonomous employment decision. + +## Verification + +Run: + +```bash +PYTHONPATH=packages/validity-analysis/src \ +python -m pytest -c packages/validity-analysis/pyproject.toml packages/validity-analysis/tests +``` + +The package gate requires exact 100% owned production statement and branch coverage. diff --git a/packages/validity-analysis/pyproject.toml b/packages/validity-analysis/pyproject.toml new file mode 100644 index 000000000..5547eabcb --- /dev/null +++ b/packages/validity-analysis/pyproject.toml @@ -0,0 +1,24 @@ +[build-system] +requires = ["setuptools>=69"] +build-backend = "setuptools.build_meta" + +[project] +name = "orgmetra-validity-analysis" +version = "0.1.0" +description = "Governed criterion-related selection-validity analysis handoff for Orgmetra." +requires-python = ">=3.12" + +[project.optional-dependencies] +test = ["pytest>=8.3", "pytest-cov>=5.0"] + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = [ + "--cov=orgmetra_validity_analysis", + "--cov-branch", + "--cov-report=term-missing", + "--cov-fail-under=100", +] diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py b/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py new file mode 100644 index 000000000..81acbc0d1 --- /dev/null +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py @@ -0,0 +1,17 @@ +"""Public governed selection-validity analysis handoff and result contracts.""" + +from .handoff import ( + REVIEWED_FAST_MLSIRM_REVISION, + ValidationAnalysisHandoff, + build_validation_analysis_handoff, +) +from .result import ConvergenceDiagnostics, MissingnessSummary, ValidationAnalysisResult + +__all__ = [ + "REVIEWED_FAST_MLSIRM_REVISION", + "ValidationAnalysisHandoff", + "build_validation_analysis_handoff", + "ConvergenceDiagnostics", + "MissingnessSummary", + "ValidationAnalysisResult", +] diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py b/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py new file mode 100644 index 000000000..5f3fb2f57 --- /dev/null +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py @@ -0,0 +1,305 @@ +"""Governed handoff evidence for criterion-related selection validation. + +This package does not execute statistics, read another service's database, or make an +employment decision. It binds authoritative Orgmetra evidence to one reviewed, +immutable fast-mlsirm revision so an approved offline worker can perform numerical +analysis without silently changing the study definition. +""" +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from hashlib import sha256 +import json +import re +from uuid import UUID + +_CODE_PATTERN = re.compile(r"^[a-z][a-z0-9]*(?:_[a-z0-9]+)+$") +_DIGEST_PATTERN = re.compile(r"^[0-9a-f]{64}$") +_REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$") +_REFERENCE_PATTERN = re.compile( + r"^[a-z][a-z0-9_]{1,31}:[A-Za-z0-9](?:[A-Za-z0-9._-]{0,126}[A-Za-z0-9])?$" +) +_PURPOSE_CODE = "selection_validity_analysis" +_REASON_CODE = "criterion_related_validation" +_VALIDATION_STRATEGY = "criterion_related" +_KERNEL_REPOSITORY = "ContextualWisdomLab/fast-mlsirm" +REVIEWED_FAST_MLSIRM_REVISION = "04d0bc21a2a20693bcf16108cd76d394fe844d23" +_KERNEL_BOUNDARY = "read_only_pinned_revision" +_EXECUTION_STATE = "not_executed" +_RESULT_AUTHORITY = "scientific_evidence_only" +_REQUIRED_RESULT_EVIDENCE = ( + "effect_estimate", + "uncertainty_interval", + "sample_size", + "missingness_summary", + "convergence_diagnostics", +) +_NEXT_ACTION = ( + "Within tenant_record_id, re-resolve the validation study, Job, predictor, criterion, " + "population, decision-policy, analysis-plan, requester, and reviewer references; prove " + "requester and reviewer resolve to distinct authoritative actor identities; prove the " + "predictor/criterion/population cases belong to the exact study and Job; then let an " + "approved offline validation worker invoke only the pinned fast-mlsirm revision. Preserve " + "the resulting model/provenance diagnostics as draft scientific evidence for an " + "accountable human reviewer; never convert the result directly into an employment decision." +) + + +def _validate_operational_uuid(value: object, field_name: str) -> None: + """Require canonical non-sentinel UUID text owned by authoritative Orgmetra.""" + if type(value) is not str: + raise ValueError(f"{field_name} must be canonical UUID text") + try: + parsed = UUID(value) + except (ValueError, AttributeError, TypeError) as exc: + raise ValueError(f"{field_name} must be canonical UUID text") from exc + if str(parsed) != value or parsed.int in (0, (1 << 128) - 1): + raise ValueError(f"{field_name} must be a canonical operational UUID") + + +def _validate_reference(value: object, prefix: str, field_name: str) -> None: + """Require the expected namespace plus a canonical opaque UUIDv4 suffix.""" + error_message = f"{field_name} must be an opaque {prefix}: UUIDv4 reference" + if ( + type(value) is not str + or len(value) > 160 + or not _REFERENCE_PATTERN.fullmatch(value) + or not value.startswith(f"{prefix}:") + ): + raise ValueError(error_message) + suffix = value.split(":", 1)[1] + try: + parsed = UUID(suffix) + except (ValueError, AttributeError, TypeError) as exc: + raise ValueError(error_message) from exc + if str(parsed) != suffix or parsed.version != 4 or parsed.int in (0, (1 << 128) - 1): + raise ValueError(error_message) + + +def _validate_digest(value: object, field_name: str) -> None: + """Require lowercase SHA-256 hexadecimal evidence.""" + if type(value) is not str or not _DIGEST_PATTERN.fullmatch(value): + raise ValueError(f"{field_name} must be lowercase SHA-256 hex") + + +def _validate_code(value: object, field_name: str) -> None: + """Require bounded descriptive lower snake_case governance codes.""" + if type(value) is not str or len(value) > 64 or not _CODE_PATTERN.fullmatch(value): + raise ValueError(f"{field_name} must be bounded two-or-more-word lower snake_case") + + +def _validate_kernel_revision(value: object) -> None: + """Require the exact externally reviewed immutable fast-mlsirm revision.""" + if type(value) is not str or not _REVISION_PATTERN.fullmatch(value): + raise ValueError("fast_mlsirm_revision must be lowercase 40-character Git commit hex") + if value != REVIEWED_FAST_MLSIRM_REVISION: + raise ValueError("fast_mlsirm_revision must equal the reviewed immutable revision") + + +def _freeze_timestamp(value: object, field_name: str) -> datetime: + """Detach caller-controlled timezone behavior and store one immutable UTC instant.""" + if type(value) is not datetime or value.tzinfo is None: + raise ValueError(f"{field_name} must be an exact timezone-aware datetime") + try: + offset = value.utcoffset() + except Exception as exc: # noqa: BLE001 - normalize provider behavior at trust boundary. + raise ValueError(f"{field_name} must be an exact timezone-aware datetime") from exc + if type(offset) is not timedelta: + raise ValueError(f"{field_name} must be an exact timezone-aware datetime") + try: + return (value.replace(tzinfo=None) - offset).replace(tzinfo=timezone.utc) + except OverflowError as exc: + raise ValueError(f"{field_name} must be an exact timezone-aware datetime") from exc + + +def _canonical_timestamp(value: object, field_name: str) -> str: + """Render a previously detached built-in UTC instant as RFC 3339 text.""" + if type(value) is not datetime or value.tzinfo is not timezone.utc: + raise ValueError(f"{field_name} must be timezone-aware") + return value.isoformat().replace("+00:00", "Z") + + +@dataclass(frozen=True, slots=True, repr=False) +class ValidationAnalysisHandoff: + """Immutable evidence for one not-yet-executed criterion-related validity analysis.""" + + tenant_record_id: str + handoff_reference: str + validation_study_reference: str + job_profile_reference: str + predictor_snapshot_reference: str + predictor_snapshot_digest: str + criterion_snapshot_reference: str + criterion_snapshot_digest: str + population_snapshot_reference: str + population_snapshot_digest: str + decision_policy_reference: str + decision_policy_digest: str + analysis_plan_reference: str + analysis_plan_digest: str + actor_reference: str + reviewer_reference: str + fast_mlsirm_revision: str + requested_at: datetime + purpose_code: str = _PURPOSE_CODE + reason_code: str = _REASON_CODE + evidence_version: int = 1 + validation_strategy: str = _VALIDATION_STRATEGY + kernel_repository: str = _KERNEL_REPOSITORY + kernel_boundary: str = _KERNEL_BOUNDARY + execution_state: str = _EXECUTION_STATE + contains_raw_person_level_values: bool = False + human_review_required: bool = True + result_authority: str = _RESULT_AUTHORITY + required_result_evidence: tuple[str, ...] = _REQUIRED_RESULT_EVIDENCE + next_action: str = _NEXT_ACTION + + def __post_init__(self) -> None: + """Fail closed when direct construction drifts from the governed handoff.""" + requested_at = _freeze_timestamp(self.requested_at, "requested_at") + object.__setattr__(self, "requested_at", requested_at) + _validate_operational_uuid(self.tenant_record_id, "tenant_record_id") + for value, prefix, field_name in ( + (self.handoff_reference, "validation_analysis_handoff", "handoff_reference"), + (self.validation_study_reference, "validation_study", "validation_study_reference"), + (self.job_profile_reference, "job_profile", "job_profile_reference"), + (self.predictor_snapshot_reference, "predictor_snapshot", "predictor_snapshot_reference"), + (self.criterion_snapshot_reference, "criterion_snapshot", "criterion_snapshot_reference"), + (self.population_snapshot_reference, "study_population_snapshot", "population_snapshot_reference"), + (self.decision_policy_reference, "decision_policy", "decision_policy_reference"), + (self.analysis_plan_reference, "validation_analysis_plan", "analysis_plan_reference"), + (self.actor_reference, "actor", "actor_reference"), + (self.reviewer_reference, "actor", "reviewer_reference"), + ): + _validate_reference(value, prefix, field_name) + for value, field_name in ( + (self.predictor_snapshot_digest, "predictor_snapshot_digest"), + (self.criterion_snapshot_digest, "criterion_snapshot_digest"), + (self.population_snapshot_digest, "population_snapshot_digest"), + (self.decision_policy_digest, "decision_policy_digest"), + (self.analysis_plan_digest, "analysis_plan_digest"), + ): + _validate_digest(value, field_name) + if self.actor_reference == self.reviewer_reference: + raise ValueError("reviewer_reference must identify a different accountable actor") + _validate_kernel_revision(self.fast_mlsirm_revision) + _canonical_timestamp(self.requested_at, "requested_at") + _validate_code(self.purpose_code, "purpose_code") + if self.purpose_code != _PURPOSE_CODE: + raise ValueError("purpose_code must remain selection_validity_analysis") + _validate_code(self.reason_code, "reason_code") + if self.reason_code != _REASON_CODE: + raise ValueError("reason_code must remain criterion_related_validation") + if type(self.evidence_version) is not int or not 1 <= self.evidence_version <= 2_147_483_647: + raise ValueError("evidence_version must be an integer from 1 through 2147483647") + if type(self.validation_strategy) is not str or self.validation_strategy != _VALIDATION_STRATEGY: + raise ValueError("validation_strategy must remain criterion_related") + if type(self.kernel_repository) is not str or self.kernel_repository != _KERNEL_REPOSITORY: + raise ValueError("kernel_repository must remain ContextualWisdomLab/fast-mlsirm") + if type(self.kernel_boundary) is not str or self.kernel_boundary != _KERNEL_BOUNDARY: + raise ValueError("kernel_boundary must remain read_only_pinned_revision") + if type(self.execution_state) is not str or self.execution_state != _EXECUTION_STATE: + raise ValueError("execution_state must remain not_executed") + if self.contains_raw_person_level_values is not False: + raise ValueError("handoff must not contain raw person-level values") + if self.human_review_required is not True: + raise ValueError("human review is mandatory for selection-validity interpretation") + if type(self.result_authority) is not str or self.result_authority != _RESULT_AUTHORITY: + raise ValueError("result_authority must remain scientific_evidence_only") + if ( + type(self.required_result_evidence) is not tuple + or any(type(item) is not str for item in self.required_result_evidence) + or self.required_result_evidence != _REQUIRED_RESULT_EVIDENCE + ): + raise ValueError("required_result_evidence must remain the reviewed evidence set") + if type(self.next_action) is not str or self.next_action != _NEXT_ACTION: + raise ValueError("next_action must remain the governed validation instruction") + + def __repr__(self) -> str: + """Return a fully redacted representation suitable for routine logs.""" + return "ValidationAnalysisHandoff()" + + def canonical_json(self) -> str: + """Return deterministic canonical JSON for audit and result correlation.""" + payload = { + "actor_reference": self.actor_reference, + "analysis_plan_digest": self.analysis_plan_digest, + "analysis_plan_reference": self.analysis_plan_reference, + "contains_raw_person_level_values": self.contains_raw_person_level_values, + "criterion_snapshot_digest": self.criterion_snapshot_digest, + "criterion_snapshot_reference": self.criterion_snapshot_reference, + "decision_policy_digest": self.decision_policy_digest, + "decision_policy_reference": self.decision_policy_reference, + "evidence_version": self.evidence_version, + "execution_state": self.execution_state, + "fast_mlsirm_revision": self.fast_mlsirm_revision, + "handoff_reference": self.handoff_reference, + "human_review_required": self.human_review_required, + "job_profile_reference": self.job_profile_reference, + "kernel_boundary": self.kernel_boundary, + "kernel_repository": self.kernel_repository, + "next_action": self.next_action, + "population_snapshot_digest": self.population_snapshot_digest, + "population_snapshot_reference": self.population_snapshot_reference, + "predictor_snapshot_digest": self.predictor_snapshot_digest, + "predictor_snapshot_reference": self.predictor_snapshot_reference, + "purpose_code": self.purpose_code, + "reason_code": self.reason_code, + "requested_at": _canonical_timestamp(self.requested_at, "requested_at"), + "required_result_evidence": list(self.required_result_evidence), + "result_authority": self.result_authority, + "reviewer_reference": self.reviewer_reference, + "tenant_record_id": self.tenant_record_id, + "validation_strategy": self.validation_strategy, + "validation_study_reference": self.validation_study_reference, + } + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + def sha256_digest(self) -> str: + """Return SHA-256 over the exact canonical UTF-8 handoff.""" + return sha256(self.canonical_json().encode("utf-8")).hexdigest() + + +def build_validation_analysis_handoff( + *, + tenant_record_id: str, + handoff_reference: str, + validation_study_reference: str, + job_profile_reference: str, + predictor_snapshot_reference: str, + predictor_snapshot_digest: str, + criterion_snapshot_reference: str, + criterion_snapshot_digest: str, + population_snapshot_reference: str, + population_snapshot_digest: str, + decision_policy_reference: str, + decision_policy_digest: str, + analysis_plan_reference: str, + analysis_plan_digest: str, + actor_reference: str, + reviewer_reference: str, + fast_mlsirm_revision: str, + requested_at: datetime, +) -> ValidationAnalysisHandoff: + """Build a governed, non-executing selection-validity analysis handoff.""" + return ValidationAnalysisHandoff( + tenant_record_id=tenant_record_id, + handoff_reference=handoff_reference, + validation_study_reference=validation_study_reference, + job_profile_reference=job_profile_reference, + predictor_snapshot_reference=predictor_snapshot_reference, + predictor_snapshot_digest=predictor_snapshot_digest, + criterion_snapshot_reference=criterion_snapshot_reference, + criterion_snapshot_digest=criterion_snapshot_digest, + population_snapshot_reference=population_snapshot_reference, + population_snapshot_digest=population_snapshot_digest, + decision_policy_reference=decision_policy_reference, + decision_policy_digest=decision_policy_digest, + analysis_plan_reference=analysis_plan_reference, + analysis_plan_digest=analysis_plan_digest, + actor_reference=actor_reference, + reviewer_reference=reviewer_reference, + fast_mlsirm_revision=fast_mlsirm_revision, + requested_at=requested_at, + ) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py new file mode 100644 index 000000000..3f684ab10 --- /dev/null +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py @@ -0,0 +1,248 @@ +"""Validate one immutable numerical result returned by the approved worker. + +Orgmetra does not fit a model in this package. It accepts only a bounded, +digest-linked result envelope from the pinned ``fast-mlsirm`` worker so that +nonconverged or malformed output cannot be presented as an employment +decision. +""" +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from hashlib import sha256 +import json +from math import isfinite +from numbers import Real + +from .handoff import ( + _canonical_timestamp, + _freeze_timestamp, + _validate_code, + _validate_digest, + _validate_kernel_revision, + _validate_operational_uuid, + _validate_reference, +) + +_RESULT_AUTHORITY = "scientific_evidence_only" +_EXECUTION_STATE = "completed" +_ALLOWED_BACKENDS = frozenset({"rust_cpu", "rust_gpu"}) +_ALLOWED_PRECISIONS = frozenset({"f64", "f32"}) + + +def _validate_nonnegative_integer(value: object, field_name: str) -> None: + """Require a real non-negative integer without accepting booleans.""" + if type(value) is not int or value < 0: + raise ValueError(f"{field_name} must be a non-negative integer") + + +def _validate_positive_integer(value: object, field_name: str) -> None: + """Require a real positive integer without accepting booleans.""" + if type(value) is not int or value <= 0: + raise ValueError(f"{field_name} must be a positive integer") + + +def _finite_number(value: object, field_name: str) -> float: + """Return one finite real number and normalize invalid numerics to ValueError.""" + if isinstance(value, bool) or not isinstance(value, Real): + raise ValueError(f"{field_name} must be a finite number") + try: + number = float(value) + except (OverflowError, TypeError, ValueError) as exc: + raise ValueError(f"{field_name} must be a finite number") from exc + if not isfinite(number): + raise ValueError(f"{field_name} must be a finite number") + return number + + +@dataclass(frozen=True, slots=True) +class MissingnessSummary: + """Describe missingness counts without carrying person-level observations.""" + + total_observations: int + complete_observations: int + missing_predictor_observations: int + missing_criterion_observations: int + + def __post_init__(self) -> None: + """Reject impossible counts before a result can be correlated.""" + for field_name in ( + "total_observations", + "complete_observations", + "missing_predictor_observations", + "missing_criterion_observations", + ): + _validate_nonnegative_integer(getattr(self, field_name), field_name) + if self.total_observations == 0: + raise ValueError("total_observations must be positive") + if self.complete_observations > self.total_observations: + raise ValueError("complete_observations cannot exceed total_observations") + if self.missing_predictor_observations > self.total_observations: + raise ValueError("missing_predictor_observations cannot exceed total_observations") + if self.missing_criterion_observations > self.total_observations: + raise ValueError("missing_criterion_observations cannot exceed total_observations") + if self.complete_observations + self.missing_predictor_observations > self.total_observations: + raise ValueError( + "complete_observations and missing_predictor_observations cannot overlap" + ) + if self.complete_observations + self.missing_criterion_observations > self.total_observations: + raise ValueError( + "complete_observations and missing_criterion_observations cannot overlap" + ) + + def to_dict(self) -> dict[str, int]: + """Return deterministic count fields for the canonical result JSON.""" + return { + "complete_observations": self.complete_observations, + "missing_criterion_observations": self.missing_criterion_observations, + "missing_predictor_observations": self.missing_predictor_observations, + "total_observations": self.total_observations, + } + + +@dataclass(frozen=True, slots=True) +class ConvergenceDiagnostics: + """Record convergence evidence while preserving an explicit failure state.""" + + converged: bool + iterations: int + objective_value: Real + maximum_gradient: Real + failure_code: str | None = None + + def __post_init__(self) -> None: + """Require diagnostics that distinguish convergence from a failed fit.""" + if type(self.converged) is not bool: + raise ValueError("converged must be a boolean") + _validate_positive_integer(self.iterations, "iterations") + objective = _finite_number(self.objective_value, "objective_value") + gradient = _finite_number(self.maximum_gradient, "maximum_gradient") + if gradient < 0: + raise ValueError("maximum_gradient must be non-negative") + if self.converged and self.failure_code is not None: + raise ValueError("failure_code must be absent for a converged result") + if not self.converged and ( + type(self.failure_code) is not str or not self.failure_code + ): + raise ValueError("failure_code is required for a nonconverged result") + if self.failure_code is not None: + _validate_code(self.failure_code, "failure_code") + object.__setattr__(self, "objective_value", objective) + object.__setattr__(self, "maximum_gradient", gradient) + + def to_dict(self) -> dict[str, object]: + """Return deterministic convergence fields for the canonical result JSON.""" + payload: dict[str, object] = { + "converged": self.converged, + "iterations": self.iterations, + "maximum_gradient": float(self.maximum_gradient), + "objective_value": float(self.objective_value), + } + if self.failure_code is not None: + payload["failure_code"] = self.failure_code + return payload + + +@dataclass(frozen=True, slots=True, repr=False) +class ValidationAnalysisResult: + """Immutable, digest-linked scientific evidence returned by the offline worker.""" + + tenant_record_id: str + result_reference: str + handoff_digest: str + provenance_digest: str + fast_mlsirm_revision: str + model_code: str + backend: str + precision: str + effect_estimate: Real + uncertainty_lower: Real + uncertainty_upper: Real + sample_size: int + missingness_summary: MissingnessSummary + convergence_diagnostics: ConvergenceDiagnostics + completed_at: datetime + result_authority: str = _RESULT_AUTHORITY + execution_state: str = _EXECUTION_STATE + contains_raw_person_level_values: bool = False + human_review_required: bool = True + evidence_version: int = 1 + + def __post_init__(self) -> None: + """Fail closed on malformed, unlinked, or decision-like result data.""" + _validate_operational_uuid(self.tenant_record_id, "tenant_record_id") + _validate_reference(self.result_reference, "validation_analysis_result", "result_reference") + _validate_digest(self.handoff_digest, "handoff_digest") + _validate_digest(self.provenance_digest, "provenance_digest") + _validate_kernel_revision(self.fast_mlsirm_revision) + _validate_code(self.model_code, "model_code") + if type(self.backend) is not str or self.backend not in _ALLOWED_BACKENDS: + raise ValueError("backend must be rust_cpu or rust_gpu") + if type(self.precision) is not str or self.precision not in _ALLOWED_PRECISIONS: + raise ValueError("precision must be f64 or f32") + estimate = _finite_number(self.effect_estimate, "effect_estimate") + lower = _finite_number(self.uncertainty_lower, "uncertainty_lower") + upper = _finite_number(self.uncertainty_upper, "uncertainty_upper") + if lower > upper: + raise ValueError("uncertainty_lower cannot exceed uncertainty_upper") + if not lower <= estimate <= upper: + raise ValueError("effect_estimate must be inside the uncertainty interval") + _validate_positive_integer(self.sample_size, "sample_size") + if type(self.missingness_summary) is not MissingnessSummary: + raise ValueError("missingness_summary must be a MissingnessSummary") + if type(self.convergence_diagnostics) is not ConvergenceDiagnostics: + raise ValueError("convergence_diagnostics must be ConvergenceDiagnostics") + if self.sample_size != self.missingness_summary.total_observations: + raise ValueError("sample_size must match total_observations") + completed_at = _freeze_timestamp(self.completed_at, "completed_at") + if type(self.result_authority) is not str or self.result_authority != _RESULT_AUTHORITY: + raise ValueError("result_authority must remain scientific_evidence_only") + if type(self.execution_state) is not str or self.execution_state != _EXECUTION_STATE: + raise ValueError("execution_state must remain completed") + if self.contains_raw_person_level_values is not False: + raise ValueError("result must not contain raw person-level values") + if self.human_review_required is not True: + raise ValueError("human review is mandatory for validity interpretation") + if type(self.evidence_version) is not int or self.evidence_version != 1: + raise ValueError("evidence_version must remain 1") + object.__setattr__(self, "effect_estimate", estimate) + object.__setattr__(self, "uncertainty_lower", lower) + object.__setattr__(self, "uncertainty_upper", upper) + object.__setattr__(self, "completed_at", completed_at) + + def __repr__(self) -> str: + """Return a redacted representation suitable for routine application logs.""" + return "ValidationAnalysisResult()" + + def canonical_json(self) -> str: + """Return deterministic, non-person-level JSON for audit correlation.""" + payload = { + "backend": self.backend, + "completed_at": _canonical_timestamp(self.completed_at, "completed_at"), + "contains_raw_person_level_values": self.contains_raw_person_level_values, + "convergence_diagnostics": self.convergence_diagnostics.to_dict(), + "effect_estimate": float(self.effect_estimate), + "evidence_version": self.evidence_version, + "execution_state": self.execution_state, + "fast_mlsirm_revision": self.fast_mlsirm_revision, + "handoff_digest": self.handoff_digest, + "human_review_required": self.human_review_required, + "missingness_summary": self.missingness_summary.to_dict(), + "model_code": self.model_code, + "precision": self.precision, + "provenance_digest": self.provenance_digest, + "result_authority": self.result_authority, + "result_reference": self.result_reference, + "sample_size": self.sample_size, + "tenant_record_id": self.tenant_record_id, + "uncertainty_lower": float(self.uncertainty_lower), + "uncertainty_upper": float(self.uncertainty_upper), + } + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + def sha256_digest(self) -> str: + """Return SHA-256 over the exact canonical result bytes.""" + return sha256(self.canonical_json().encode("utf-8")).hexdigest() + + +__all__ = ["ConvergenceDiagnostics", "MissingnessSummary", "ValidationAnalysisResult"] diff --git a/packages/validity-analysis/tests/test_adr_numbering.py b/packages/validity-analysis/tests/test_adr_numbering.py new file mode 100644 index 000000000..8f2ee0828 --- /dev/null +++ b/packages/validity-analysis/tests/test_adr_numbering.py @@ -0,0 +1,17 @@ +"""Regression tests for repository-wide ADR number ownership.""" + +from pathlib import Path + + +def test_adr_numbers_are_unique_across_the_integrated_repository() -> None: + """Every four-digit ADR number must identify exactly one decision record.""" + adr_directory = Path(__file__).resolve().parents[3] / "docs" / "adr" + owners: dict[str, str] = {} + + for adr_path in sorted(adr_directory.glob("[0-9][0-9][0-9][0-9]-*.md")): + adr_number = adr_path.name[:4] + previous_owner = owners.get(adr_number) + assert previous_owner is None, ( + f"ADR {adr_number} is reused by {previous_owner} and {adr_path.name}" + ) + owners[adr_number] = adr_path.name diff --git a/packages/validity-analysis/tests/test_handoff.py b/packages/validity-analysis/tests/test_handoff.py new file mode 100644 index 000000000..64d1ab8c6 --- /dev/null +++ b/packages/validity-analysis/tests/test_handoff.py @@ -0,0 +1,221 @@ +"""Regression tests for governed selection-validity analysis handoffs.""" + +from dataclasses import replace +from datetime import datetime, timedelta, timezone +import json + +import pytest + +from orgmetra_validity_analysis import ( + REVIEWED_FAST_MLSIRM_REVISION, + ValidationAnalysisHandoff, + build_validation_analysis_handoff, +) + +TENANT = "10000000-0000-7000-8000-000000000001" +HANDOFF = "validation_analysis_handoff:11111111-1111-4111-8111-111111111111" +STUDY = "validation_study:22222222-2222-4222-8222-222222222222" +JOB = "job_profile:33333333-3333-4333-8333-333333333333" +PREDICTOR = "predictor_snapshot:44444444-4444-4444-8444-444444444444" +CRITERION = "criterion_snapshot:55555555-5555-4555-8555-555555555555" +POPULATION = "study_population_snapshot:66666666-6666-4666-8666-666666666666" +POLICY = "decision_policy:77777777-7777-4777-8777-777777777777" +PLAN = "validation_analysis_plan:88888888-8888-4888-8888-888888888888" +ACTOR = "actor:99999999-9999-4999-8999-999999999999" +REVIEWER = "actor:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" +DIGEST_A = "a" * 64 +DIGEST_B = "b" * 64 +DIGEST_C = "c" * 64 +DIGEST_D = "d" * 64 +DIGEST_E = "e" * 64 +REQUESTED_AT = datetime(2026, 8, 21, 7, 10, 11, 123456, tzinfo=timezone(timedelta(hours=9))) + + +def valid_kwargs(): + """Return one complete governed handoff input fixture.""" + return { + "tenant_record_id": TENANT, + "handoff_reference": HANDOFF, + "validation_study_reference": STUDY, + "job_profile_reference": JOB, + "predictor_snapshot_reference": PREDICTOR, + "predictor_snapshot_digest": DIGEST_A, + "criterion_snapshot_reference": CRITERION, + "criterion_snapshot_digest": DIGEST_B, + "population_snapshot_reference": POPULATION, + "population_snapshot_digest": DIGEST_C, + "decision_policy_reference": POLICY, + "decision_policy_digest": DIGEST_D, + "analysis_plan_reference": PLAN, + "analysis_plan_digest": DIGEST_E, + "actor_reference": ACTOR, + "reviewer_reference": REVIEWER, + "fast_mlsirm_revision": REVIEWED_FAST_MLSIRM_REVISION, + "requested_at": REQUESTED_AT, + } + + +def handoff(): + """Build the canonical valid handoff fixture.""" + return build_validation_analysis_handoff(**valid_kwargs()) + + +def test_handoff_is_value_minimized_deterministic_and_human_review_only(): + """Bind exact study evidence without exposing raw person-level observations.""" + candidate = handoff() + payload = json.loads(candidate.canonical_json()) + + assert payload["tenant_record_id"] == TENANT + assert payload["validation_study_reference"] == STUDY + assert payload["job_profile_reference"] == JOB + assert payload["fast_mlsirm_revision"] == REVIEWED_FAST_MLSIRM_REVISION + assert payload["requested_at"] == "2026-08-20T22:10:11.123456Z" + assert payload["validation_strategy"] == "criterion_related" + assert payload["kernel_repository"] == "ContextualWisdomLab/fast-mlsirm" + assert payload["kernel_boundary"] == "read_only_pinned_revision" + assert payload["execution_state"] == "not_executed" + assert payload["contains_raw_person_level_values"] is False + assert payload["human_review_required"] is True + assert payload["result_authority"] == "scientific_evidence_only" + assert payload["required_result_evidence"] == [ + "effect_estimate", + "uncertainty_interval", + "sample_size", + "missingness_summary", + "convergence_diagnostics", + ] + assert "person_record" not in candidate.canonical_json() + assert "candidate" not in candidate.canonical_json() + assert repr(candidate) == "ValidationAnalysisHandoff()" + assert len(candidate.sha256_digest()) == 64 + assert candidate.canonical_json() == handoff().canonical_json() + + +@pytest.mark.parametrize( + "bad_tenant", + [ + "not-a-uuid", + "00000000-0000-0000-0000-000000000000", + "ffffffff-ffff-ffff-ffff-ffffffffffff", + "10000000-0000-7000-8000-00000000000A", + 1, + ], +) +def test_tenant_identity_must_follow_protected_operational_uuid_contract(bad_tenant): + """Reject malformed, reserved, non-canonical, and non-text tenant identities.""" + values = valid_kwargs() + values["tenant_record_id"] = bad_tenant + with pytest.raises(ValueError, match="tenant_record_id"): + build_validation_analysis_handoff(**values) + + +@pytest.mark.parametrize( + ("field", "bad", "match"), + [ + ("handoff_reference", "validation_analysis_handoff:not-a-uuid", "handoff_reference"), + ("validation_study_reference", JOB, "validation_study_reference"), + ("job_profile_reference", "job_profile:22222222-2222-7222-8222-222222222222", "job_profile_reference"), + ("predictor_snapshot_reference", 1, "predictor_snapshot_reference"), + ("criterion_snapshot_reference", "criterion_snapshot:" + "a" * 161, "criterion_snapshot_reference"), + ("population_snapshot_reference", "study_population_snapshot:not-a-uuid", "population_snapshot_reference"), + ("decision_policy_reference", "decision_policy:BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB", "decision_policy_reference"), + ("analysis_plan_reference", "validation_analysis_plan:00000000-0000-0000-0000-000000000000", "analysis_plan_reference"), + ("actor_reference", "actor:ffffffff-ffff-ffff-ffff-ffffffffffff", "actor_reference"), + ("reviewer_reference", "actor:bbbbbbbb-bbbb-7bbb-8bbb-bbbbbbbbbbbb", "reviewer_reference"), + ], +) +def test_all_public_references_are_namespaced_opaque_uuid4(field, bad, match): + """Fail closed on wrong namespace, malformed, noncanonical, or non-v4 references.""" + values = valid_kwargs() + values[field] = bad + with pytest.raises(ValueError, match=match): + build_validation_analysis_handoff(**values) + + +@pytest.mark.parametrize( + "field", + [ + "predictor_snapshot_digest", + "criterion_snapshot_digest", + "population_snapshot_digest", + "decision_policy_digest", + "analysis_plan_digest", + ], +) +def test_evidence_digests_are_lowercase_sha256(field): + """Reject weak or noncanonical evidence digests for every source snapshot.""" + values = valid_kwargs() + values[field] = "A" * 64 + with pytest.raises(ValueError, match=field): + build_validation_analysis_handoff(**values) + + +def test_requester_and_reviewer_must_be_distinct(): + """Require accountable independent interpretation instead of self-review.""" + values = valid_kwargs() + values["reviewer_reference"] = ACTOR + with pytest.raises(ValueError, match="different accountable actor"): + build_validation_analysis_handoff(**values) + + +@pytest.mark.parametrize("bad_revision", ["not-a-sha", "A" * 40, "0" * 40]) +def test_fast_mlsirm_revision_is_exactly_the_reviewed_immutable_dependency(bad_revision): + """Reject malformed or unreviewed foreign dependency revisions.""" + values = valid_kwargs() + values["fast_mlsirm_revision"] = bad_revision + with pytest.raises(ValueError, match="fast_mlsirm_revision"): + build_validation_analysis_handoff(**values) + + +@pytest.mark.parametrize( + "bad_requested_at", + [ + datetime(2026, 8, 21, 7, 10), + "2026-08-21T07:10:00+09:00", + ], +) +def test_requested_at_requires_an_aware_datetime(bad_requested_at): + """Reject local-time ambiguity in immutable analysis correlation.""" + values = valid_kwargs() + values["requested_at"] = bad_requested_at + with pytest.raises(ValueError, match="requested_at"): + build_validation_analysis_handoff(**values) + + +@pytest.mark.parametrize( + ("field", "bad", "match"), + [ + ("purpose_code", "other_purpose", "purpose_code"), + ("reason_code", "other_reason", "reason_code"), + ("evidence_version", True, "evidence_version"), + ("evidence_version", 0, "evidence_version"), + ("validation_strategy", "content_validity", "validation_strategy"), + ("kernel_repository", "other/repository", "kernel_repository"), + ("kernel_boundary", "direct_database", "kernel_boundary"), + ("execution_state", "executed", "execution_state"), + ("contains_raw_person_level_values", True, "raw person-level"), + ("human_review_required", False, "human review"), + ("result_authority", "employment_decision", "result_authority"), + ("required_result_evidence", ("effect_estimate",), "required_result_evidence"), + ("next_action", "Auto-approve the result.", "next_action"), + ], +) +def test_direct_construction_cannot_weaken_governance(field, bad, match): + """Keep fixed scientific, privacy, dependency, and human-authority semantics immutable.""" + with pytest.raises(ValueError, match=match): + replace(handoff(), **{field: bad}) + + +def test_codes_must_remain_bounded_descriptive_snake_case_before_fixed_value_check(): + """Exercise code-shape rejection separately from the closed purpose/reason vocabulary.""" + with pytest.raises(ValueError, match="purpose_code"): + replace(handoff(), purpose_code="X") + with pytest.raises(ValueError, match="reason_code"): + replace(handoff(), reason_code="x" * 65) + + +def test_public_dataclass_type_is_constructible_only_with_all_invariants(): + """Document the public immutable type while preserving builder equivalence.""" + values = valid_kwargs() + direct = ValidationAnalysisHandoff(**values) + assert direct == handoff() diff --git a/packages/validity-analysis/tests/test_host_resolution_contract.py b/packages/validity-analysis/tests/test_host_resolution_contract.py new file mode 100644 index 000000000..c87fb1a18 --- /dev/null +++ b/packages/validity-analysis/tests/test_host_resolution_contract.py @@ -0,0 +1,34 @@ +"""Regressions for authoritative requester/reviewer identity separation.""" + +from datetime import datetime, timezone + +from orgmetra_validity_analysis import ( + REVIEWED_FAST_MLSIRM_REVISION, + build_validation_analysis_handoff, +) + + +def test_next_action_requires_resolved_actor_identity_separation() -> None: + """Do not let different opaque actor references masquerade as distinct people.""" + handoff = build_validation_analysis_handoff( + tenant_record_id="10000000-0000-7000-8000-000000000001", + handoff_reference="validation_analysis_handoff:11111111-1111-4111-8111-111111111111", + validation_study_reference="validation_study:22222222-2222-4222-8222-222222222222", + job_profile_reference="job_profile:33333333-3333-4333-8333-333333333333", + predictor_snapshot_reference="predictor_snapshot:44444444-4444-4444-8444-444444444444", + predictor_snapshot_digest="a" * 64, + criterion_snapshot_reference="criterion_snapshot:55555555-5555-4555-8555-555555555555", + criterion_snapshot_digest="b" * 64, + population_snapshot_reference="study_population_snapshot:66666666-6666-4666-8666-666666666666", + population_snapshot_digest="c" * 64, + decision_policy_reference="decision_policy:77777777-7777-4777-8777-777777777777", + decision_policy_digest="d" * 64, + analysis_plan_reference="validation_analysis_plan:88888888-8888-4888-8888-888888888888", + analysis_plan_digest="e" * 64, + actor_reference="actor:99999999-9999-4999-8999-999999999999", + reviewer_reference="actor:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + fast_mlsirm_revision=REVIEWED_FAST_MLSIRM_REVISION, + requested_at=datetime(2026, 8, 21, 1, 0, tzinfo=timezone.utc), + ) + + assert "prove requester and reviewer resolve to distinct authoritative actor identities" in handoff.next_action diff --git a/packages/validity-analysis/tests/test_numeric_overflow_contract.py b/packages/validity-analysis/tests/test_numeric_overflow_contract.py new file mode 100644 index 000000000..a4fb88115 --- /dev/null +++ b/packages/validity-analysis/tests/test_numeric_overflow_contract.py @@ -0,0 +1,18 @@ +"""Regression for malformed worker numerics that overflow float conversion.""" + +import pytest + +from orgmetra_validity_analysis import ConvergenceDiagnostics + + +def test_oversized_worker_numeric_is_rejected_as_value_error() -> None: + """Normalize float-conversion overflow to the package's ValueError contract.""" + oversized_integer = 10**10000 + + with pytest.raises(ValueError, match="finite number"): + ConvergenceDiagnostics( + converged=True, + iterations=1, + objective_value=oversized_integer, + maximum_gradient=0.1, + ) diff --git a/packages/validity-analysis/tests/test_quality_workflow_trigger.py b/packages/validity-analysis/tests/test_quality_workflow_trigger.py new file mode 100644 index 000000000..eaf759550 --- /dev/null +++ b/packages/validity-analysis/tests/test_quality_workflow_trigger.py @@ -0,0 +1,16 @@ +"""Regression tests for validity-analysis coverage by consolidated Foundation CI.""" + +from pathlib import Path + + +_WORKFLOW_PATH = Path(".github/workflows/foundation-ci.yml") + + +def test_foundation_ci_retriggers_without_path_filter() -> None: + """Keep shared repository changes inside the consolidated validity gate surface.""" + workflow = _WORKFLOW_PATH.read_text(encoding="utf-8") + + assert "pull_request:" in workflow + assert " - develop" in workflow + assert "\n paths:" not in workflow + assert "\n paths-ignore:" not in workflow diff --git a/packages/validity-analysis/tests/test_result.py b/packages/validity-analysis/tests/test_result.py new file mode 100644 index 000000000..ad00b95f2 --- /dev/null +++ b/packages/validity-analysis/tests/test_result.py @@ -0,0 +1,211 @@ +"""Regression tests for the bounded numerical result contract.""" + +from dataclasses import asdict, replace +from datetime import datetime, timezone +import json + +import pytest + +from orgmetra_validity_analysis import ( + ConvergenceDiagnostics, + MissingnessSummary, + REVIEWED_FAST_MLSIRM_REVISION, + ValidationAnalysisResult, +) + +TENANT = "10000000-0000-7000-8000-000000000001" +RESULT = "validation_analysis_result:11111111-1111-4111-8111-111111111111" +HANDOFF_DIGEST = "a" * 64 +PROVENANCE_DIGEST = "b" * 64 +COMPLETED_AT = datetime(2026, 8, 21, 7, 10, 11, 123456, tzinfo=timezone.utc) + + +def missingness() -> MissingnessSummary: + """Return one realistic aggregate-only missingness summary.""" + return MissingnessSummary( + total_observations=12, + complete_observations=10, + missing_predictor_observations=1, + missing_criterion_observations=1, + ) + + +def convergence(*, converged: bool = True) -> ConvergenceDiagnostics: + """Return one converged or explicitly nonconverged diagnostic record.""" + return ConvergenceDiagnostics( + converged=converged, + iterations=42, + objective_value=-12.5, + maximum_gradient=0.0001, + failure_code=None if converged else "maximum_iterations", + ) + + +def result(**overrides: object) -> ValidationAnalysisResult: + """Build one valid result envelope and apply targeted test overrides.""" + values: dict[str, object] = { + "tenant_record_id": TENANT, + "result_reference": RESULT, + "handoff_digest": HANDOFF_DIGEST, + "provenance_digest": PROVENANCE_DIGEST, + "fast_mlsirm_revision": REVIEWED_FAST_MLSIRM_REVISION, + "model_code": "mlsirm_criterion_related", + "backend": "rust_cpu", + "precision": "f64", + "effect_estimate": 0.42, + "uncertainty_lower": 0.10, + "uncertainty_upper": 0.70, + "sample_size": 12, + "missingness_summary": missingness(), + "convergence_diagnostics": convergence(), + "completed_at": COMPLETED_AT, + } + values.update(overrides) + return ValidationAnalysisResult(**values) + + +def test_aggregate_evidence_is_deterministic_and_redacted() -> None: + """Serialize only aggregate evidence and preserve exact replay bytes.""" + candidate = result() + payload = json.loads(candidate.canonical_json()) + + assert payload["tenant_record_id"] == TENANT + assert payload["backend"] == "rust_cpu" + assert payload["precision"] == "f64" + assert payload["execution_state"] == "completed" + assert payload["result_authority"] == "scientific_evidence_only" + assert payload["missingness_summary"]["total_observations"] == 12 + assert payload["convergence_diagnostics"]["converged"] is True + assert "person_record" not in candidate.canonical_json() + assert repr(candidate) == "ValidationAnalysisResult()" + assert len(candidate.sha256_digest()) == 64 + assert candidate.canonical_json() == result().canonical_json() + + +def test_gpu_and_nonconverged_result_are_explicitly_typed() -> None: + """Record GPU provenance and a typed nonconvergence state without promotion.""" + candidate = result( + backend="rust_gpu", + precision="f32", + convergence_diagnostics=convergence(converged=False), + ) + payload = json.loads(candidate.canonical_json()) + assert payload["backend"] == "rust_gpu" + assert payload["precision"] == "f32" + assert payload["convergence_diagnostics"]["failure_code"] == "maximum_iterations" + + +@pytest.mark.parametrize( + "bad", + [ + {"total_observations": True}, + {"total_observations": -1}, + {"total_observations": 0}, + {"complete_observations": 13}, + {"missing_predictor_observations": 13}, + {"missing_criterion_observations": 13}, + {"complete_observations": 12, "missing_predictor_observations": 1}, + { + "complete_observations": 12, + "missing_predictor_observations": 0, + "missing_criterion_observations": 1, + }, + ], +) +def test_missingness_rejects_invalid_counts(bad: dict[str, object]) -> None: + """Reject booleans, negative counts, empty samples, and impossible totals.""" + values = asdict(missingness()) + values.update(bad) + with pytest.raises(ValueError): + MissingnessSummary(**values) + + +@pytest.mark.parametrize("bad", [True, 0, -1]) +def test_positive_integer_validation_rejects_nonpositive_values(bad: object) -> None: + """Exercise strict sample and iteration bounds.""" + with pytest.raises(ValueError, match="positive integer"): + ConvergenceDiagnostics(True, bad, -1.0, 0.1) + with pytest.raises(ValueError, match="positive integer"): + result(sample_size=bad) + + +@pytest.mark.parametrize("bad", [True, "0.1", float("nan"), float("inf")]) +def test_numeric_fields_reject_boolean_text_and_nonfinite_values(bad: object) -> None: + """Do not accept values that cannot be represented as finite scientific evidence.""" + with pytest.raises(ValueError, match="finite number"): + ConvergenceDiagnostics(True, 1, bad, 0.1) + with pytest.raises(ValueError, match="finite number"): + result(effect_estimate=bad) + + +def test_negative_gradient_and_invalid_convergence_states_fail_closed() -> None: + """Require explicit and internally consistent convergence diagnostics.""" + with pytest.raises(ValueError, match="non-negative"): + ConvergenceDiagnostics(True, 1, 1.0, -0.1) + with pytest.raises(ValueError, match="boolean"): + ConvergenceDiagnostics(1, 1, 1.0, 0.1) + with pytest.raises(ValueError, match="absent"): + ConvergenceDiagnostics(True, 1, 1.0, 0.1, "failed_fit") + with pytest.raises(ValueError, match="required"): + ConvergenceDiagnostics(False, 1, 1.0, 0.1) + with pytest.raises(ValueError, match="required"): + ConvergenceDiagnostics(False, 1, 1.0, 0.1, "") + + +@pytest.mark.parametrize( + "field,bad,match", + [ + ("backend", "numpy", "backend"), + ("precision", "float16", "precision"), + ("uncertainty_lower", 0.8, "uncertainty_lower"), + ("uncertainty_upper", 0.0, "uncertainty_lower"), + ("effect_estimate", 0.8, "effect_estimate"), + ("sample_size", 11, "sample_size"), + ("result_authority", "employment_decision", "result_authority"), + ("execution_state", "not_executed", "execution_state"), + ("contains_raw_person_level_values", True, "raw person-level"), + ("human_review_required", False, "human review"), + ("evidence_version", 2, "evidence_version"), + ], +) +def test_result_invariants_cannot_be_weakened(field: str, bad: object, match: str) -> None: + """Reject malformed intervals, lineage, or governance flags.""" + with pytest.raises(ValueError, match=match): + replace(result(), **{field: bad}) + + +def test_result_requires_canonical_timestamp_and_aggregate_types() -> None: + """Reject naive times, non-contract objects, and subclass method overrides.""" + + class LeakyMissingnessSummary(MissingnessSummary): + def to_dict(self) -> dict[str, object]: + return {**super().to_dict(), "person_record": "must-not-serialize"} + + class LeakyConvergenceDiagnostics(ConvergenceDiagnostics): + def to_dict(self) -> dict[str, object]: + return {**super().to_dict(), "employment_decision": "auto_reject"} + + with pytest.raises(ValueError, match="completed_at"): + result(completed_at=datetime(2026, 8, 21, 7, 10)) + with pytest.raises(ValueError, match="missingness_summary"): + result(missingness_summary=object()) + with pytest.raises(ValueError, match="convergence_diagnostics"): + result(convergence_diagnostics=object()) + with pytest.raises(ValueError, match="missingness_summary"): + result( + missingness_summary=LeakyMissingnessSummary( + total_observations=12, + complete_observations=10, + missing_predictor_observations=1, + missing_criterion_observations=1, + ) + ) + with pytest.raises(ValueError, match="convergence_diagnostics"): + result( + convergence_diagnostics=LeakyConvergenceDiagnostics( + converged=True, + iterations=42, + objective_value=-12.5, + maximum_gradient=0.0001, + ) + ) diff --git a/packages/validity-analysis/tests/test_temporal_evidence_integrity.py b/packages/validity-analysis/tests/test_temporal_evidence_integrity.py new file mode 100644 index 000000000..0431eee55 --- /dev/null +++ b/packages/validity-analysis/tests/test_temporal_evidence_integrity.py @@ -0,0 +1,279 @@ +"""Regression coverage for selection-validity temporal evidence integrity.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone, tzinfo + +import pytest + +from orgmetra_validity_analysis import ( + ConvergenceDiagnostics, + MissingnessSummary, + REVIEWED_FAST_MLSIRM_REVISION, + ValidationAnalysisHandoff, + ValidationAnalysisResult, + build_validation_analysis_handoff, +) +from test_handoff import valid_kwargs +from test_result import result + + +class ForgedDateTime(datetime): + """Datetime subclass able to forge canonical validation evidence.""" + + def astimezone(self, tz=None): # type: ignore[no-untyped-def] + """Keep the hostile subclass alive across UTC normalization.""" + return self + + def isoformat(self, *args, **kwargs) -> str: # type: ignore[no-untyped-def] + """Return an instant different from the underlying evidence instant.""" + return "2099-12-31T23:59:59+00:00" + + +class ForgedReference(str): + """String subclass able to forge namespace and UUID parsing methods.""" + + def startswith(self, prefix, *args): # type: ignore[no-untyped-def] + """Pretend that an invalid namespace has the expected prefix.""" + return True + + def split(self, separator=None, maxsplit=-1): # type: ignore[no-untyped-def] + """Return a valid UUID suffix while retaining invalid source text.""" + return ["validation_analysis_handoff", "11111111-1111-4111-8111-111111111111"] + + +class ForgedFixedText(str): + """String subclass whose comparisons can forge fixed governance values.""" + + def __eq__(self, other): # type: ignore[no-untyped-def] + """Claim equality with any expected governance text.""" + return True + + def __ne__(self, other): # type: ignore[no-untyped-def] + """Claim inequality with no governance text.""" + return False + + def __hash__(self): + """Use a valid fixed-text hash for set membership forgery tests.""" + return hash("rust_cpu") + + +class MutableOffset(tzinfo): + """Timezone fixture whose offset can change after envelope construction.""" + + def __init__(self) -> None: + self.hours = 1 + + def utcoffset(self, dt: datetime | None) -> timedelta: + """Return the currently configured offset.""" + return timedelta(hours=self.hours) + + def dst(self, dt: datetime | None) -> timedelta: + """Return no daylight-saving offset.""" + return timedelta(0) + + +class UnknownOffset(tzinfo): + """Timezone fixture whose UTC offset cannot be resolved.""" + + def utcoffset(self, dt: datetime | None) -> None: + """Return no offset to exercise fail-closed validation.""" + return None + + def dst(self, dt: datetime | None) -> None: + """Return no daylight-saving offset.""" + return None + + +class ExplodingOffset(tzinfo): + """Timezone fixture whose provider raises during offset resolution.""" + + def utcoffset(self, dt: datetime | None) -> timedelta: + """Raise an untrusted provider error.""" + raise RuntimeError("offset provider failed") + + def dst(self, dt: datetime | None) -> timedelta: + """Return no daylight-saving offset when queried separately.""" + return timedelta(0) + + +class MutableReal(float): + """Numeric subclass whose float conversion changes after construction.""" + + def __new__(cls, value: float): + """Create a float-backed value with a separately mutable conversion.""" + instance = super().__new__(cls, value) + instance.current = value + return instance + + def __float__(self) -> float: + """Expose the mutable conversion used by unsafe canonicalization.""" + return self.current + + +def test_handoff_rejects_datetime_subclass_that_can_forge_requested_at() -> None: + """Handoff canonical evidence must not invoke caller-overridable datetime methods.""" + with pytest.raises(ValueError, match="requested_at"): + build_validation_analysis_handoff( + tenant_record_id="10000000-0000-7000-8000-000000000001", + handoff_reference="validation_analysis_handoff:11111111-1111-4111-8111-111111111111", + validation_study_reference="validation_study:22222222-2222-4222-8222-222222222222", + job_profile_reference="job_profile:33333333-3333-4333-8333-333333333333", + predictor_snapshot_reference="predictor_snapshot:44444444-4444-4444-8444-444444444444", + predictor_snapshot_digest="a" * 64, + criterion_snapshot_reference="criterion_snapshot:55555555-5555-4555-8555-555555555555", + criterion_snapshot_digest="b" * 64, + population_snapshot_reference="study_population_snapshot:66666666-6666-4666-8666-666666666666", + population_snapshot_digest="c" * 64, + decision_policy_reference="decision_policy:77777777-7777-4777-8777-777777777777", + decision_policy_digest="d" * 64, + analysis_plan_reference="validation_analysis_plan:88888888-8888-4888-8888-888888888888", + analysis_plan_digest="e" * 64, + actor_reference="actor:99999999-9999-4999-8999-999999999999", + reviewer_reference="actor:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + fast_mlsirm_revision=REVIEWED_FAST_MLSIRM_REVISION, + requested_at=ForgedDateTime(2026, 8, 21, 4, 45, tzinfo=timezone.utc), + ) + + +def test_result_rejects_datetime_subclass_that_can_forge_completed_at() -> None: + """Result canonical evidence must not invoke caller-overridable datetime methods.""" + with pytest.raises(ValueError, match="completed_at"): + ValidationAnalysisResult( + tenant_record_id="10000000-0000-7000-8000-000000000001", + result_reference="validation_analysis_result:11111111-1111-4111-8111-111111111111", + handoff_digest="a" * 64, + provenance_digest="b" * 64, + fast_mlsirm_revision=REVIEWED_FAST_MLSIRM_REVISION, + model_code="mlsirm_criterion_related", + backend="rust_cpu", + precision="f64", + effect_estimate=0.42, + uncertainty_lower=0.10, + uncertainty_upper=0.70, + sample_size=12, + missingness_summary=MissingnessSummary( + total_observations=12, + complete_observations=10, + missing_predictor_observations=1, + missing_criterion_observations=1, + ), + convergence_diagnostics=ConvergenceDiagnostics( + converged=True, + iterations=42, + objective_value=-12.5, + maximum_gradient=0.0001, + ), + completed_at=ForgedDateTime(2026, 8, 21, 4, 45, tzinfo=timezone.utc), + ) + + +def test_handoff_and_result_detach_mutable_timezone_before_digesting() -> None: + """Freeze one UTC instant so later timezone mutation cannot rewrite evidence.""" + handoff_zone = MutableOffset() + handoff_values = valid_kwargs() + handoff_values["requested_at"] = datetime(2026, 8, 21, 4, 45, tzinfo=handoff_zone) + handoff = build_validation_analysis_handoff(**handoff_values) + handoff_before = handoff.canonical_json(), handoff.sha256_digest() + handoff_zone.hours = 2 + assert (handoff.canonical_json(), handoff.sha256_digest()) == handoff_before + + result_zone = MutableOffset() + candidate = result(completed_at=datetime(2026, 8, 21, 4, 45, tzinfo=result_zone)) + result_before = candidate.canonical_json(), candidate.sha256_digest() + result_zone.hours = 2 + assert (candidate.canonical_json(), candidate.sha256_digest()) == result_before + + +@pytest.mark.parametrize( + "timestamp", + [ + datetime.min.replace(tzinfo=timezone(timedelta(hours=1))), + datetime.max.replace(tzinfo=timezone(-timedelta(hours=1))), + datetime(2026, 8, 21, 4, 45, tzinfo=UnknownOffset()), + datetime(2026, 8, 21, 4, 45, tzinfo=ExplodingOffset()), + ], +) +def test_handoff_rejects_unrepresentable_or_untrusted_timestamp(timestamp: datetime) -> None: + """Normalize timezone-provider failures and UTC arithmetic overflow at the boundary.""" + values = valid_kwargs() + values["requested_at"] = timestamp + with pytest.raises(ValueError, match="requested_at"): + build_validation_analysis_handoff(**values) + + +@pytest.mark.parametrize( + "timestamp", + [ + datetime.min.replace(tzinfo=timezone(timedelta(hours=1))), + datetime.max.replace(tzinfo=timezone(-timedelta(hours=1))), + datetime(2026, 8, 21, 4, 45, tzinfo=UnknownOffset()), + datetime(2026, 8, 21, 4, 45, tzinfo=ExplodingOffset()), + ], +) +def test_result_rejects_unrepresentable_or_untrusted_timestamp(timestamp: datetime) -> None: + """Apply the same fail-closed timestamp contract to completed result evidence.""" + with pytest.raises(ValueError, match="completed_at"): + result(completed_at=timestamp) + + +@pytest.mark.parametrize( + "timestamp", + [ + ForgedDateTime(2026, 8, 21, 4, 45, tzinfo=timezone.utc), + datetime(2026, 8, 21, 4, 45, tzinfo=timezone(timedelta(hours=1))), + ], +) +def test_canonicalization_rejects_low_level_timestamp_reinjection(timestamp: datetime) -> None: + """Keep canonicalization fail-closed even if an object is corrupted after construction.""" + handoff = build_validation_analysis_handoff(**valid_kwargs()) + object.__setattr__(handoff, "requested_at", timestamp) + with pytest.raises(ValueError, match="requested_at"): + handoff.canonical_json() + + candidate = result() + object.__setattr__(candidate, "completed_at", timestamp) + with pytest.raises(ValueError, match="completed_at"): + candidate.canonical_json() + + +def test_handoff_rejects_runtime_text_subclasses_before_serialization() -> None: + """Reject text subclasses that can forge reference, digest, code, or fixed-value checks.""" + for field, value in ( + ("tenant_record_id", ForgedFixedText("10000000-0000-7000-8000-000000000001")), + ("handoff_reference", ForgedReference("wrong_namespace:invalid")), + ("predictor_snapshot_digest", ForgedFixedText("a" * 64)), + ("purpose_code", ForgedFixedText("selection_validity_analysis")), + ("fast_mlsirm_revision", ForgedFixedText(REVIEWED_FAST_MLSIRM_REVISION)), + ("validation_strategy", ForgedFixedText("criterion_related")), + ("next_action", ForgedFixedText("governed")), + ): + values = valid_kwargs() + values[field] = value + with pytest.raises(ValueError, match=field): + ValidationAnalysisHandoff(**values) + + +def test_result_snapshots_numeric_values_before_canonicalization() -> None: + """Detach mutable numeric subclasses before recording scientific evidence bytes.""" + objective = MutableReal(-12.5) + gradient = MutableReal(0.0001) + diagnostics = ConvergenceDiagnostics( + converged=True, + iterations=42, + objective_value=objective, + maximum_gradient=gradient, + ) + estimate = MutableReal(0.42) + candidate = result(effect_estimate=estimate, convergence_diagnostics=diagnostics) + before = candidate.canonical_json(), candidate.sha256_digest() + objective.current = -1.0 + gradient.current = 0.5 + estimate.current = 0.69 + assert (candidate.canonical_json(), candidate.sha256_digest()) == before + + +def test_result_rejects_forged_backend_text() -> None: + """Do not allow a string subclass to forge an allowed backend membership check.""" + with pytest.raises(ValueError, match="backend"): + result(backend=ForgedFixedText("numpy")) diff --git a/packages/validity-analysis/tests/test_workflow_trigger_contract.py b/packages/validity-analysis/tests/test_workflow_trigger_contract.py new file mode 100644 index 000000000..4d69ff802 --- /dev/null +++ b/packages/validity-analysis/tests/test_workflow_trigger_contract.py @@ -0,0 +1,18 @@ +"""Regression for validity-analysis execution in consolidated Foundation CI.""" + +from pathlib import Path + + +def test_foundation_ci_runs_validity_analysis_and_adr_changes() -> None: + """Require the consolidated gate to execute this package for every develop PR.""" + repository_root = Path(__file__).resolve().parents[3] + workflow = (repository_root / ".github" / "workflows" / "foundation-ci.yml").read_text( + encoding="utf-8" + ) + + assert "pull_request:" in workflow + assert " - develop" in workflow + assert "\n paths:" not in workflow + assert "\n paths-ignore:" not in workflow + assert "PYTHONPATH=packages/validity-analysis/src" in workflow + assert "packages/validity-analysis/tests" in workflow From 22022b5af807e04a710e2ac988fc4ddd51abbb76 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 04:09:20 +0900 Subject: [PATCH 043/158] fix(ci): integrate validity owner into consolidated gate --- .github/workflows/foundation-ci.yml | 1 + CHANGELOG.md | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/foundation-ci.yml b/.github/workflows/foundation-ci.yml index 6b475d6f2..a96370987 100644 --- a/.github/workflows/foundation-ci.yml +++ b/.github/workflows/foundation-ci.yml @@ -66,6 +66,7 @@ jobs: PYTHONPATH=packages/offer-approval/src COVERAGE_FILE=/tmp/orgmetra-offer-approval.coverage python -m pytest -c packages/offer-approval/pyproject.toml packages/offer-approval/tests PYTHONPATH=packages/requisition-review/src COVERAGE_FILE=/tmp/orgmetra-requisition-review.coverage python -m pytest -c packages/requisition-review/pyproject.toml packages/requisition-review/tests PYTHONPATH=packages/selection-review/src COVERAGE_FILE=/tmp/orgmetra-selection-review.coverage python -m pytest -c packages/selection-review/pyproject.toml packages/selection-review/tests + PYTHONPATH=packages/validity-analysis/src COVERAGE_FILE=/tmp/orgmetra-validity-analysis.coverage python -m pytest -c packages/validity-analysis/pyproject.toml packages/validity-analysis/tests PYTHONPATH=services/job-analysis-api/src:packages/hris-kernel/src:packages/keyverse-adapter/src COVERAGE_FILE=/tmp/orgmetra-job-analysis-api.coverage python -m pytest -c services/job-analysis-api/pyproject.toml services/job-analysis-api/tests PYTHONPATH=services/people-api/src:packages/hris-kernel/src:packages/keyverse-adapter/src COVERAGE_FILE=/tmp/orgmetra-people-api.coverage python -m pytest -c services/people-api/pyproject.toml services/people-api/tests - name: Run PostgreSQL contracts in isolated containers diff --git a/CHANGELOG.md b/CHANGELOG.md index 16454da3d..44aa89a44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,9 @@ All notable changes to Orgmetra will be documented in this file. ### Added - Accepted ADRs 0001–0003 now include buyer-facing Context, Decision, and Consequences grounded in verified ISO 30400:2022, ISO 30414:2025, Uniform Guidelines (29 C.F.R. Part 1607), SIOP (2018), OpenAPI Specification v3.2.0, OpenID Connect Core 1.0 errata set 2, CloudEvents v1.0.2, Jensen and Snodgrass (1999), Snodgrass (1999), and Allen (1983) records already listed in `docs/doctoring/REFERENCES.md`. ADRs 0004 and 0005 gained APA 7th References pointers to that same bibliography without changing their Decision bodies. -- Active-PR governed Job Analysis persistence/API on the canonical `JobAnalysisSnapshot` model: migration `0013_job_analysis_snapshot.sql` stores immutable tenant-scoped snapshot, Task, KSAO, Task–KSAO, FJA and write-command evidence; `POST /v1/tenants/{tenant_record_id}/job-analysis-snapshots` and matching GET enforce purpose-bound Keyverse scope, authenticated-principal actor authority, bounded/strict JSON handling, transactional Idempotency-Key serialization, parent-scope fail-closed integrity, forced RLS, and atomic audit/outbox evidence. ADR 0014 records the persistence decision while ADR 0007 remains the domain/evidence authority; validated evidence still requires accountable human review and non-LLM provenance, and the service does not make a high-impact employment decision. +- Active-PR governed Job Analysis persistence/API on the canonical `JobAnalysisSnapshot` model: migration `0013_job_analysis_snapshot.sql` stores immutable tenant-scoped snapshot, Task, KSAO, FJA and write-command evidence; `POST /v1/tenants/{tenant_record_id}/job-analysis-snapshots` and matching GET enforce purpose-bound Keyverse scope, authenticated-principal actor authority, bounded/strict JSON handling, transactional Idempotency-Key serialization, parent-scope fail-closed integrity, forced RLS, and atomic audit/outbox evidence. ADR 0014 records the persistence decision while ADR 0007 remains the domain/evidence authority; validated evidence still requires accountable human review and non-LLM provenance, and the service does not make a high-impact employment decision. - Active-PR `orgmetra_selection_review` packet for PII-minimized, evidence-bound human selection review: canonical operational tenant identity, UUID-backed opaque candidate/Job/sealed-evidence/reviewer references, explicit purpose/reason/evidence version, deterministic canonical JSON and SHA-256 correlation, mandatory human decision state, redacted packet repr, and provenance-paired model evidence that remains `untrusted_draft`, with exact 100% owned statement and branch coverage required by its quality gate. +- Active-PR `orgmetra_validity_analysis` handoff/result boundary: exact tenant and evidence references, reviewed fast-mlsirm pin, distinct requester/reviewer actors, aggregate-only scientific evidence, construction-time UTC and finite-number snapshots, canonical JSON/SHA-256 correlation, and human-review-only result authority. - Active performance-criterion scope hardening: `criterion_observation_scope_guard` rejects criterion outcomes for a Job the worker did not effectively hold at the observation date, observations before the relevant assignment, and observations outside the referenced performance cycle while preserving valid multiple-assignment cases and existing bitemporal correction semantics. The guard evaluates current-recorded facts, derives the date coordinate from `observed_at` in UTC so session `TimeZone` cannot alter the result, uses a trusted function search path, and adds no PII or automated employment decision authority. The Foundation PostgreSQL contract also rejects a closed `recorded_to` on each time-coordinate lookup and proves UTC midnight plus non-UTC session `TimeZone` boundaries. - Bitemporal tenant-scoped organization hierarchy validation that rejects visible indirect parent cycles and reuses single-valued recorded-time reconstruction before graph traversal. - Stacked governed job-analysis evidence contract via `JobAnalysisSnapshot`, `TaskEvidence`, `KSAORequirement`, `TaskKSAOLink`, `FunctionalJobAnalysisProfile`, and `EvidenceSource`: tenant/Job-scoped observable tasks, explicit Task-to-KSAO linkage, importance/difficulty/proficiency ratings, source/version/retrieval/SHA-256 provenance, deterministic canonical snapshot bytes, current O*NET evidence support, and historical DOT Data/People/Things compatibility. Validated snapshots require accountable human review and complete non-LLM evidence; LLM-origin material remains `analysis_draft`, and the snapshot is evidence input rather than a hiring, promotion, termination, compensation, or other high-impact employment decision. @@ -18,7 +19,7 @@ All notable changes to Orgmetra will be documented in this file. - `employment_record_version.employment_concurrency_code` constrained to `exclusive` or `concurrent`. - ADR 0005 for exclusive employment and staffable seats. - `orgmetra_hris_kernel` 0.3.0 with identity-scoped bitemporal resolution, assignment-employment coverage, allocation-portfolio checks, and a Memorial Hospital RN correction case at 100% statement and branch coverage. -- `employment_record_version` and `position_record_version` so employment and position identity stay stable across retroactive corrections. +- `employment_record_version` and `position_record_version` so corrections no longer mint a new employment or position identifier. - `assignment_record.employment_record_id` bound to the same person as the covering employment. - `orgmetra_keyverse_adapter` that binds an opaque Keyverse subject to a person and rejects passwords, passkeys, and tokens. - Design tokens for the repeating HR actions: approve, review, correct, request evidence, compare, export, and escalate. From 85ac772b05f1ff0260cdb922c954b50ad3652ab2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 04:12:25 +0900 Subject: [PATCH 044/158] fix(ci): reseal validity foundation manifest --- manifest.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/manifest.json b/manifest.json index f7b6cf55e..431fc6d60 100644 --- a/manifest.json +++ b/manifest.json @@ -5,9 +5,9 @@ "files": [ { "path": ".github/workflows/foundation-ci.yml", - "sha256": "b6a4365936b66803a8112f034c77d53d33301a7a798ed4f68746a4f2d8b081d7", - "bytes": 6651, - "lines": 125 + "sha256": "69e35b7a12e7ebcbe039ef349e2311702fdbfeee8916d4c78a64cf8bd8a1f430", + "bytes": 6853, + "lines": 126 }, { "path": ".gitignore", @@ -29,9 +29,9 @@ }, { "path": "CHANGELOG.md", - "sha256": "f2d2e0b488c0440533effa821808f2f17e37d92f8fb586174c2fdb594f760ca5", - "bytes": 17539, - "lines": 77 + "sha256": "4669c7ab654afce04898baa7af37ced058a14d2d9aa42bc4bee712a45b8038bf", + "bytes": 17868, + "lines": 78 }, { "path": "CLAUDE.md", From a2d98fab95b24766e0482e0135d18f4e92795655 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 04:14:24 +0900 Subject: [PATCH 045/158] test(validity): require final analysis weight lineage --- .../tests/test_analysis_weight_receipt.py | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 packages/validity-analysis/tests/test_analysis_weight_receipt.py diff --git a/packages/validity-analysis/tests/test_analysis_weight_receipt.py b/packages/validity-analysis/tests/test_analysis_weight_receipt.py new file mode 100644 index 000000000..133e9c39c --- /dev/null +++ b/packages/validity-analysis/tests/test_analysis_weight_receipt.py @@ -0,0 +1,93 @@ +"""RED contract for reproducible point-estimation weight lineage.""" + +from datetime import datetime, timezone + +import pytest + +from orgmetra_validity_analysis import AnalysisWeightAdjustment, FinalAnalysisWeightReceipt + +TENANT = "10000000-0000-7000-8000-000000000001" +RECEIPT = "analysis_weight_receipt:11111111-1111-4111-8111-111111111111" +ESTIMAND = "validation_estimand:22222222-2222-4222-8222-222222222222" +TARGET = "analysis_target_population:33333333-3333-4333-8333-333333333333" +WINDOW = "analysis_window:44444444-4444-4444-8444-444444444444" +DIGEST_A = "a" * 64 +DIGEST_B = "b" * 64 +DIGEST_C = "c" * 64 +DIGEST_D = "d" * 64 +DIGEST_E = "e" * 64 +DIGEST_F = "f" * 64 +DIGEST_1 = "1" * 64 +DIGEST_2 = "2" * 64 +DIGEST_3 = "3" * 64 +DIGEST_4 = "4" * 64 +DIGEST_5 = "5" * 64 + + +def adjustment() -> AnalysisWeightAdjustment: + """Return one governed nonresponse adjustment without copying case weights.""" + return AnalysisWeightAdjustment( + sequence_number=1, + adjustment_code="nonresponse_adjustment", + method_reference="weight_method:55555555-5555-4555-8555-555555555555", + method_version=1, + input_weight_artifact_digest=DIGEST_D, + output_weight_artifact_digest=DIGEST_E, + configuration_digest=DIGEST_F, + evidence_receipt_digest=DIGEST_1, + ) + + +def receipt(**overrides: object) -> FinalAnalysisWeightReceipt: + """Return one exact final point-weight receipt.""" + values: dict[str, object] = { + "tenant_record_id": TENANT, + "receipt_reference": RECEIPT, + "estimand_reference": ESTIMAND, + "estimand_digest": DIGEST_A, + "target_population_reference": TARGET, + "target_population_digest": DIGEST_B, + "analysis_unit_code": "worker_occurrence", + "analysis_window_reference": WINDOW, + "eligible_case_set_digest": DIGEST_C, + "analytic_case_occurrence_set_digest": DIGEST_2, + "source_universe_receipt_digest": DIGEST_3, + "sampling_design_receipt_digest": DIGEST_4, + "base_weight_method_code": "inverse_inclusion_probability", + "base_weight_method_version": 1, + "base_weight_evidence_digest": DIGEST_5, + "base_weight_artifact_digest": DIGEST_D, + "adjustments": (adjustment(),), + "final_weight_artifact_digest": DIGEST_E, + "analytic_case_count": 12, + "constructed_at": datetime(2026, 9, 17, 4, 0, tzinfo=timezone.utc), + } + values.update(overrides) + return FinalAnalysisWeightReceipt(**values) + + +def test_receipt_is_deterministic_and_value_minimized() -> None: + """Bind exact estimand and ordered weight lineage without embedding row weights.""" + candidate = receipt() + assert candidate.sha256_digest() == receipt().sha256_digest() + assert candidate.final_weight_artifact_digest == DIGEST_E + assert "person_record" not in candidate.canonical_json() + assert "weight_value" not in candidate.canonical_json() + + +def test_adjustment_chain_must_reach_final_weight_artifact() -> None: + """Reject a receipt whose declared final weight is not the ordered chain output.""" + with pytest.raises(ValueError, match="final_weight_artifact_digest"): + receipt(final_weight_artifact_digest=DIGEST_F) + + +def test_probability_design_receipt_is_required() -> None: + """Do not allow point weights to detach from the sampled design evidence.""" + with pytest.raises(ValueError, match="sampling_design_receipt_digest"): + receipt(sampling_design_receipt_digest="not-a-digest") + + +def test_correction_must_link_to_the_superseded_receipt() -> None: + """Require append-only correction lineage instead of overwriting prior weight evidence.""" + with pytest.raises(ValueError, match="supersedes_receipt_digest"): + receipt(correction_sequence=2) From 9a25f7221a57ec0d141b25f778ecab0464016a7e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 04:24:31 +0900 Subject: [PATCH 046/158] test(validity): bind weighted result to weight evidence --- .../test_analysis_weight_result_binding.py | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 packages/validity-analysis/tests/test_analysis_weight_result_binding.py diff --git a/packages/validity-analysis/tests/test_analysis_weight_result_binding.py b/packages/validity-analysis/tests/test_analysis_weight_result_binding.py new file mode 100644 index 000000000..d3e11947b --- /dev/null +++ b/packages/validity-analysis/tests/test_analysis_weight_result_binding.py @@ -0,0 +1,102 @@ +"""RED contract for binding weighted scientific results to exact weight evidence.""" + +from datetime import datetime, timezone + +import pytest + +from orgmetra_validity_analysis import ( + ConvergenceDiagnostics, + MissingnessSummary, + REVIEWED_FAST_MLSIRM_REVISION, + ValidationAnalysisResult, +) + +TENANT = "10000000-0000-7000-8000-000000000001" +RESULT = "validation_analysis_result:11111111-1111-4111-8111-111111111111" +HANDOFF_DIGEST = "a" * 64 +PROVENANCE_DIGEST = "b" * 64 +WEIGHT_DIGEST = "c" * 64 +VARIANCE_DIGEST = "d" * 64 + + +def result(**overrides: object) -> ValidationAnalysisResult: + """Build one bounded validity result with targeted inference overrides.""" + values: dict[str, object] = { + "tenant_record_id": TENANT, + "result_reference": RESULT, + "handoff_digest": HANDOFF_DIGEST, + "provenance_digest": PROVENANCE_DIGEST, + "fast_mlsirm_revision": REVIEWED_FAST_MLSIRM_REVISION, + "model_code": "mlsirm_criterion_related", + "backend": "rust_cpu", + "precision": "f64", + "effect_estimate": 0.42, + "uncertainty_lower": 0.10, + "uncertainty_upper": 0.70, + "sample_size": 12, + "missingness_summary": MissingnessSummary( + total_observations=12, + complete_observations=10, + missing_predictor_observations=1, + missing_criterion_observations=1, + ), + "convergence_diagnostics": ConvergenceDiagnostics( + converged=True, + iterations=42, + objective_value=-12.5, + maximum_gradient=0.0001, + ), + "completed_at": datetime(2026, 9, 17, 4, 10, tzinfo=timezone.utc), + } + values.update(overrides) + return ValidationAnalysisResult(**values) + + +def test_weighted_result_binds_point_and_variance_receipts_separately() -> None: + """Require exact point-weight and variance evidence on weighted inference.""" + candidate = result( + point_estimation_mode="weighted_design_based", + analysis_weight_receipt_digest=WEIGHT_DIGEST, + variance_design_receipt_digest=VARIANCE_DIGEST, + ) + payload = candidate.canonical_json() + assert '"point_estimation_mode":"weighted_design_based"' in payload + assert f'"analysis_weight_receipt_digest":"{WEIGHT_DIGEST}"' in payload + assert f'"variance_design_receipt_digest":"{VARIANCE_DIGEST}"' in payload + + +@pytest.mark.parametrize( + "overrides,match", + [ + ( + { + "point_estimation_mode": "weighted_design_based", + "analysis_weight_receipt_digest": None, + "variance_design_receipt_digest": VARIANCE_DIGEST, + }, + "analysis_weight_receipt_digest", + ), + ( + { + "point_estimation_mode": "weighted_design_based", + "analysis_weight_receipt_digest": WEIGHT_DIGEST, + "variance_design_receipt_digest": None, + }, + "variance_design_receipt_digest", + ), + ( + { + "point_estimation_mode": "unweighted", + "analysis_weight_receipt_digest": WEIGHT_DIGEST, + }, + "unweighted", + ), + ({"point_estimation_mode": "opaque_weighted"}, "point_estimation_mode"), + ], +) +def test_result_rejects_unverifiable_weight_binding( + overrides: dict[str, object], match: str +) -> None: + """Fail closed when result and point/variance weight provenance disagree.""" + with pytest.raises(ValueError, match=match): + result(**overrides) From adf9816cec84c9abbc31077612cc84a04ba7f92d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 04:25:12 +0900 Subject: [PATCH 047/158] feat(validity): add final analysis weight receipt --- .../src/orgmetra_validity_analysis/weights.py | 207 ++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100644 packages/validity-analysis/src/orgmetra_validity_analysis/weights.py diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py b/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py new file mode 100644 index 000000000..9eb48de96 --- /dev/null +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py @@ -0,0 +1,207 @@ +"""Immutable provenance for final point-estimation weight construction. + +This module records only versioned scientific lineage and digests. It never +stores row-level weight values or copies auxiliary calibration attributes into +Orgmetra's validity-analysis boundary. +""" +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from hashlib import sha256 +import json + +from .handoff import ( + _canonical_timestamp, + _freeze_timestamp, + _validate_code, + _validate_digest, + _validate_operational_uuid, + _validate_reference, +) + + +def _positive_integer(value: object, field_name: str) -> None: + """Require a strict positive integer without accepting booleans.""" + if type(value) is not int or value <= 0: + raise ValueError(f"{field_name} must be a positive integer") + + +@dataclass(frozen=True, slots=True) +class AnalysisWeightAdjustment: + """Describe one ordered, digest-linked transformation of analysis weights.""" + + sequence_number: int + adjustment_code: str + method_reference: str + method_version: int + input_weight_artifact_digest: str + output_weight_artifact_digest: str + configuration_digest: str + evidence_receipt_digest: str + + def __post_init__(self) -> None: + """Reject unordered, opaque, or unverifiable adjustment evidence.""" + _positive_integer(self.sequence_number, "sequence_number") + _validate_code(self.adjustment_code, "adjustment_code") + _validate_reference(self.method_reference, "weight_method", "method_reference") + _positive_integer(self.method_version, "method_version") + for field_name in ( + "input_weight_artifact_digest", + "output_weight_artifact_digest", + "configuration_digest", + "evidence_receipt_digest", + ): + _validate_digest(getattr(self, field_name), field_name) + if self.input_weight_artifact_digest == self.output_weight_artifact_digest: + raise ValueError( + "output_weight_artifact_digest must identify the transformed weight artifact" + ) + + def to_dict(self) -> dict[str, object]: + """Return canonical adjustment fields without row-level weight values.""" + return { + "adjustment_code": self.adjustment_code, + "configuration_digest": self.configuration_digest, + "evidence_receipt_digest": self.evidence_receipt_digest, + "input_weight_artifact_digest": self.input_weight_artifact_digest, + "method_reference": self.method_reference, + "method_version": self.method_version, + "output_weight_artifact_digest": self.output_weight_artifact_digest, + "sequence_number": self.sequence_number, + } + + +@dataclass(frozen=True, slots=True, repr=False) +class FinalAnalysisWeightReceipt: + """Bind one estimand to the exact final point-estimation weight lineage used.""" + + tenant_record_id: str + receipt_reference: str + estimand_reference: str + estimand_digest: str + target_population_reference: str + target_population_digest: str + analysis_unit_code: str + analysis_window_reference: str + eligible_case_set_digest: str + analytic_case_occurrence_set_digest: str + source_universe_receipt_digest: str + sampling_design_receipt_digest: str + base_weight_method_code: str + base_weight_method_version: int + base_weight_evidence_digest: str + base_weight_artifact_digest: str + adjustments: tuple[AnalysisWeightAdjustment, ...] + final_weight_artifact_digest: str + analytic_case_count: int + constructed_at: datetime + correction_sequence: int = 1 + supersedes_receipt_digest: str | None = None + evidence_version: int = 1 + + def __post_init__(self) -> None: + """Fail closed unless the complete point-weight construction is reproducible.""" + _validate_operational_uuid(self.tenant_record_id, "tenant_record_id") + _validate_reference( + self.receipt_reference, "analysis_weight_receipt", "receipt_reference" + ) + _validate_reference(self.estimand_reference, "validation_estimand", "estimand_reference") + _validate_digest(self.estimand_digest, "estimand_digest") + _validate_reference( + self.target_population_reference, + "analysis_target_population", + "target_population_reference", + ) + _validate_digest(self.target_population_digest, "target_population_digest") + _validate_code(self.analysis_unit_code, "analysis_unit_code") + _validate_reference( + self.analysis_window_reference, "analysis_window", "analysis_window_reference" + ) + for field_name in ( + "eligible_case_set_digest", + "analytic_case_occurrence_set_digest", + "source_universe_receipt_digest", + "sampling_design_receipt_digest", + "base_weight_evidence_digest", + "base_weight_artifact_digest", + "final_weight_artifact_digest", + ): + _validate_digest(getattr(self, field_name), field_name) + _validate_code(self.base_weight_method_code, "base_weight_method_code") + _positive_integer(self.base_weight_method_version, "base_weight_method_version") + _positive_integer(self.analytic_case_count, "analytic_case_count") + constructed_at = _freeze_timestamp(self.constructed_at, "constructed_at") + if type(self.adjustments) is not tuple: + raise ValueError("adjustments must be an immutable tuple") + + expected_input = self.base_weight_artifact_digest + for expected_sequence, adjustment in enumerate(self.adjustments, start=1): + if type(adjustment) is not AnalysisWeightAdjustment: + raise ValueError("adjustments must contain AnalysisWeightAdjustment values") + if adjustment.sequence_number != expected_sequence: + raise ValueError("adjustments must have contiguous sequence_number values") + if adjustment.input_weight_artifact_digest != expected_input: + raise ValueError("adjustment input_weight_artifact_digest breaks the weight chain") + expected_input = adjustment.output_weight_artifact_digest + if expected_input != self.final_weight_artifact_digest: + raise ValueError( + "final_weight_artifact_digest must equal the ordered adjustment chain output" + ) + + _positive_integer(self.correction_sequence, "correction_sequence") + if self.correction_sequence == 1: + if self.supersedes_receipt_digest is not None: + raise ValueError( + "supersedes_receipt_digest must be absent for correction_sequence 1" + ) + else: + if self.supersedes_receipt_digest is None: + raise ValueError( + "supersedes_receipt_digest is required when correction_sequence exceeds 1" + ) + _validate_digest(self.supersedes_receipt_digest, "supersedes_receipt_digest") + if type(self.evidence_version) is not int or self.evidence_version != 1: + raise ValueError("evidence_version must remain 1") + object.__setattr__(self, "constructed_at", constructed_at) + + def __repr__(self) -> str: + """Return a value-minimized representation suitable for routine logs.""" + return "FinalAnalysisWeightReceipt()" + + def canonical_json(self) -> str: + """Return deterministic provenance JSON without case-level weight values.""" + payload: dict[str, object] = { + "adjustments": [adjustment.to_dict() for adjustment in self.adjustments], + "analysis_unit_code": self.analysis_unit_code, + "analysis_window_reference": self.analysis_window_reference, + "analytic_case_count": self.analytic_case_count, + "analytic_case_occurrence_set_digest": self.analytic_case_occurrence_set_digest, + "base_weight_artifact_digest": self.base_weight_artifact_digest, + "base_weight_evidence_digest": self.base_weight_evidence_digest, + "base_weight_method_code": self.base_weight_method_code, + "base_weight_method_version": self.base_weight_method_version, + "constructed_at": _canonical_timestamp(self.constructed_at, "constructed_at"), + "correction_sequence": self.correction_sequence, + "eligible_case_set_digest": self.eligible_case_set_digest, + "estimand_digest": self.estimand_digest, + "estimand_reference": self.estimand_reference, + "evidence_version": self.evidence_version, + "final_weight_artifact_digest": self.final_weight_artifact_digest, + "receipt_reference": self.receipt_reference, + "sampling_design_receipt_digest": self.sampling_design_receipt_digest, + "source_universe_receipt_digest": self.source_universe_receipt_digest, + "target_population_digest": self.target_population_digest, + "target_population_reference": self.target_population_reference, + "tenant_record_id": self.tenant_record_id, + } + if self.supersedes_receipt_digest is not None: + payload["supersedes_receipt_digest"] = self.supersedes_receipt_digest + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + def sha256_digest(self) -> str: + """Return SHA-256 over the exact canonical receipt bytes.""" + return sha256(self.canonical_json().encode("utf-8")).hexdigest() + + +__all__ = ["AnalysisWeightAdjustment", "FinalAnalysisWeightReceipt"] From 0fd40c1ab14c9041fd558a9f76234a2ee6d56451 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 04:25:28 +0900 Subject: [PATCH 048/158] feat(validity): export analysis weight provenance --- .../src/orgmetra_validity_analysis/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py b/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py index 81acbc0d1..1d8bc1701 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py @@ -6,6 +6,7 @@ build_validation_analysis_handoff, ) from .result import ConvergenceDiagnostics, MissingnessSummary, ValidationAnalysisResult +from .weights import AnalysisWeightAdjustment, FinalAnalysisWeightReceipt __all__ = [ "REVIEWED_FAST_MLSIRM_REVISION", @@ -14,4 +15,6 @@ "ConvergenceDiagnostics", "MissingnessSummary", "ValidationAnalysisResult", + "AnalysisWeightAdjustment", + "FinalAnalysisWeightReceipt", ] From 318145ff93a0c5e455a2d6b6b4f4a81d6a8d6b4b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 04:26:04 +0900 Subject: [PATCH 049/158] feat(validity): bind weighted results to exact weight evidence --- .../src/orgmetra_validity_analysis/result.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py index 3f684ab10..2cc5f121d 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py @@ -28,6 +28,7 @@ _EXECUTION_STATE = "completed" _ALLOWED_BACKENDS = frozenset({"rust_cpu", "rust_gpu"}) _ALLOWED_PRECISIONS = frozenset({"f64", "f32"}) +_ALLOWED_POINT_ESTIMATION_MODES = frozenset({"unweighted", "weighted_design_based"}) def _validate_nonnegative_integer(value: object, field_name: str) -> None: @@ -162,6 +163,9 @@ class ValidationAnalysisResult: missingness_summary: MissingnessSummary convergence_diagnostics: ConvergenceDiagnostics completed_at: datetime + point_estimation_mode: str = "unweighted" + analysis_weight_receipt_digest: str | None = None + variance_design_receipt_digest: str | None = None result_authority: str = _RESULT_AUTHORITY execution_state: str = _EXECUTION_STATE contains_raw_person_level_values: bool = False @@ -195,6 +199,35 @@ def __post_init__(self) -> None: if self.sample_size != self.missingness_summary.total_observations: raise ValueError("sample_size must match total_observations") completed_at = _freeze_timestamp(self.completed_at, "completed_at") + if ( + type(self.point_estimation_mode) is not str + or self.point_estimation_mode not in _ALLOWED_POINT_ESTIMATION_MODES + ): + raise ValueError( + "point_estimation_mode must be unweighted or weighted_design_based" + ) + if self.point_estimation_mode == "weighted_design_based": + if self.analysis_weight_receipt_digest is None: + raise ValueError( + "analysis_weight_receipt_digest is required for weighted_design_based" + ) + if self.variance_design_receipt_digest is None: + raise ValueError( + "variance_design_receipt_digest is required for weighted_design_based" + ) + _validate_digest( + self.analysis_weight_receipt_digest, "analysis_weight_receipt_digest" + ) + _validate_digest( + self.variance_design_receipt_digest, "variance_design_receipt_digest" + ) + elif ( + self.analysis_weight_receipt_digest is not None + or self.variance_design_receipt_digest is not None + ): + raise ValueError( + "unweighted result must not bind analysis or variance weight receipts" + ) if type(self.result_authority) is not str or self.result_authority != _RESULT_AUTHORITY: raise ValueError("result_authority must remain scientific_evidence_only") if type(self.execution_state) is not str or self.execution_state != _EXECUTION_STATE: @@ -229,6 +262,7 @@ def canonical_json(self) -> str: "human_review_required": self.human_review_required, "missingness_summary": self.missingness_summary.to_dict(), "model_code": self.model_code, + "point_estimation_mode": self.point_estimation_mode, "precision": self.precision, "provenance_digest": self.provenance_digest, "result_authority": self.result_authority, @@ -238,6 +272,10 @@ def canonical_json(self) -> str: "uncertainty_lower": float(self.uncertainty_lower), "uncertainty_upper": float(self.uncertainty_upper), } + if self.analysis_weight_receipt_digest is not None: + payload["analysis_weight_receipt_digest"] = self.analysis_weight_receipt_digest + if self.variance_design_receipt_digest is not None: + payload["variance_design_receipt_digest"] = self.variance_design_receipt_digest return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) def sha256_digest(self) -> str: From 9430466dc00cc6d0c57a5f6c80a4edc8181de6db Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 04:27:09 +0900 Subject: [PATCH 050/158] docs(validity): record analysis weight evidence boundary --- .../0027-governed-selection-validity-analysis-handoff.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/adr/0027-governed-selection-validity-analysis-handoff.md b/docs/adr/0027-governed-selection-validity-analysis-handoff.md index ac74dbec8..b0f062916 100644 --- a/docs/adr/0027-governed-selection-validity-analysis-handoff.md +++ b/docs/adr/0027-governed-selection-validity-analysis-handoff.md @@ -30,6 +30,10 @@ Both handoff and result envelopes detach exact timezone-aware timestamps to one The same package also validates `ValidationAnalysisResult` envelopes returned by the approved offline worker. A result must link to the handoff digest and the same pinned revision, identify a Rust CPU or GPU backend and precision, provide finite effect and interval values, match its sample size to aggregate missingness counts, reject impossible complete-versus-missing count combinations, and include explicit convergence diagnostics. The canonicalization boundary accepts only the exact governed `MissingnessSummary` and `ConvergenceDiagnostics` runtime types so subclass method overrides cannot add unreviewed or person-level fields to immutable result evidence. A nonconverged result remains typed scientific evidence requiring human review; it cannot be treated as a valid selection procedure or an employment decision. +For #407's weighted design-based inference boundary, the active package now adds a first executable `FinalAnalysisWeightReceipt`. It binds the exact estimand, target population, analysis window, eligible/analytic case-set digests, #404/#405 source/sampling receipt digests, base-weight derivation evidence, an ordered immutable adjustment chain, the final point-weight artifact digest, construction time, and append-only correction lineage without storing row-level weights. Each adjustment must be contiguous and digest-linked from the previous artifact to the declared final artifact. `ValidationAnalysisResult` explicitly distinguishes `unweighted` from `weighted_design_based`; a weighted result fails closed unless it separately binds both the final analysis-weight receipt digest and the #406 variance-design receipt digest. An unweighted result cannot carry either receipt and thereby masquerade as weighted scientific evidence. + +This executable slice does not yet claim #407 complete. Adjustment-specific owner receipts still need executable contracts for nonresponse dispositions, calibration/raking benchmark totals and constraints, trimming/bounding rules, longitudinal eligibility, convergence/fallback semantics, and released auxiliary-owner evidence. Those remain `workforce_validation` scientific truth rather than Talent truth or cross-context SQL reconstruction. + The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, and prove requester/reviewer identities are distinct authoritative actors. ## Consequences @@ -41,6 +45,8 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - Aggregate missingness evidence cannot claim all observations are complete while simultaneously reporting predictor- or criterion-missing observations. - Result canonicalization cannot be extended by an unreviewed subclass to serialize extra decision-like or person-level fields. - Caller-controlled timestamp and numeric runtime behavior cannot rewrite an accepted canonical digest after construction or turn malformed oversized worker output into an uncaught exception type. +- Weighted design-based results can no longer identify only a sample while omitting which final point-weight evidence and variance-design evidence were actually used. +- The ordered weight transformation chain is independently digest-correlatable without centralizing case-level weights or auxiliary attributes. - Human interpretation remains explicit and separate from numerical output. - The dedicated-writer boundary remains intact: Orgmetra consumes only a pinned foreign revision/contract boundary and never mutates fast-mlsirm. @@ -48,12 +54,13 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - This slice does not execute a statistical model, estimate validity, correct for measurement error/range restriction, evaluate adverse impact, or assert legal compliance. - Sampling design, estimator choice, missing-data treatment, reliability evidence, multiplicity, transportability, fairness analysis, and model diagnostics must be encoded in the referenced analysis plan and reviewed before execution. +- The first #407 slice validates generic ordered weight provenance and result binding; it does not yet make nonresponse, calibration/raking, trimming, longitudinal, or sensitive auxiliary-owner evidence fully executable. - The generic result envelope does not invent an estimator-specific minimum complete-case count. Whether a converged estimator is identified under a particular missing-data design belongs to the reviewed analysis plan and numerical-worker contract; Orgmetra fails closed on impossible aggregate counts without silently replacing that foreign scientific contract with complete-case analysis. - The package validates the result envelope, but a future execution adapter must still re-resolve the handoff references, verify the result provenance artifact, and attach evidence only after accountable human review. ## Verification -The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. +The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. #407 adds RED/GREEN coverage for deterministic value-minimized final-weight receipts, exact sampling-receipt binding, contiguous transformation lineage, append-only correction linkage, and fail-closed weighted-result binding to separate point-weight and variance-design receipts. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. ## References From d55bdc7b9c620f2220fb93619ac5d476c2e2a640 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 04:27:21 +0900 Subject: [PATCH 051/158] docs(validity): trace final weight lineage --- docs/traceability/validation-analysis-handoff.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/traceability/validation-analysis-handoff.md b/docs/traceability/validation-analysis-handoff.md index f2e832efa..23742ac7d 100644 --- a/docs/traceability/validation-analysis-handoff.md +++ b/docs/traceability/validation-analysis-handoff.md @@ -14,13 +14,19 @@ Can an organization send one exact, reviewable validation study to its statistic | Human authority | requester/reviewer references must differ, and the host must re-resolve both within the tenant and prove they resolve to distinct authoritative actor identities before execution | direct-construction fail-closed regression plus `test_next_action_requires_resolved_actor_identity_separation` | | Scientific evidence | effect estimate, uncertainty interval, sample size, internally possible aggregate missingness, convergence diagnostics | immutable required-result-evidence regression plus impossible-missingness rejection | | Numerical result boundary | handoff digest, pinned fast-mlsirm revision, Rust CPU/GPU backend, precision, finite estimate/interval, aggregate missingness, explicit convergence state | `ValidationAnalysisResult` invariant/canonicalization regressions, exact-runtime-type checks, and oversized-numeric `ValueError` normalization | +| Final point-weight provenance | exact estimand/target/window and case-set digests, source/sampling receipt digests, base-weight evidence, ordered digest-linked adjustments, final weight artifact, append-only correction lineage | `FinalAnalysisWeightReceipt` deterministic/value-minimized regressions plus chain/correction fail-closed tests | +| Weighted-result congruence | `weighted_design_based` result must bind the exact final point-weight receipt and a separate #406 variance-design receipt; `unweighted` result must bind neither | `test_analysis_weight_result_binding.py` | | Execution boundary | `not_executed`, `scientific_evidence_only`, read-only pinned foreign dependency | immutable governance regressions | -| Reproducibility | construction-time UTC timestamp snapshots, finite numeric snapshots, canonical RFC 3339 time, canonical JSON, SHA-256 handoff digest | mutable timezone/numeric and UTC-boundary regressions plus deterministic serialization/digest tests | +| Reproducibility | construction-time UTC timestamp snapshots, finite numeric snapshots, canonical RFC 3339 time, canonical JSON, SHA-256 handoff/result/weight-receipt digests | mutable timezone/numeric and UTC-boundary regressions plus deterministic serialization/digest tests | | Decision-record integrity | ADR numbers remain unique repository-wide and any `docs/adr/**` change reaches the consolidated Foundation CI validity regression | ADR uniqueness regression plus Foundation CI workflow-trigger contract regression | | Quality-evidence freshness | consolidated Foundation CI runs the validity package on every `develop` pull request without a repository path filter, so shared Python/test/clean-checkout configuration cannot silently bypass the package gate | `test_foundation_ci_retriggers_without_path_filter` and `test_foundation_ci_runs_validity_analysis_and_adr_changes`; central required workflows remain separate gates | +## #407 boundary still open + +The current branch implements the first executable weight-lineage slice; it does not complete #407. Nonresponse-adjustment inputs/disposition semantics, calibration/raking benchmark-owner receipts and constraints, trimming/bounding provenance, longitudinal-weight eligibility, convergence/fallback semantics, and sensitive auxiliary-variable handling still require typed executable owner evidence before a weighted result can be considered fully reproducible. `workforce_validation` owns those scientific contracts and their released result; `talent_management` must not compute or reconstruct final analysis weights. + ## Maturity `implemented_on_active_pr`. -Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. The active package now validates the minimum returned numerical/provenance envelope, including missingness consistency and exact aggregate-evidence runtime types, but protected Orgmetra evidence still requires host re-resolution, result-artifact verification, terminal checks, independent review, and accountable human interpretation. +Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. The active package now validates the minimum returned numerical/provenance envelope and the first #407 final-weight/result-binding contract, but protected Orgmetra evidence still requires host re-resolution, adjustment-specific owner evidence, result-artifact verification, terminal checks, independent review, and accountable human interpretation. From fe88b7371fd59e6906d2089ebce5ebaf071c3e50 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 04:27:35 +0900 Subject: [PATCH 052/158] docs(validity): doctor analysis weight provenance --- .../validation-analysis-handoff-references.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/doctoring/validation-analysis-handoff-references.md b/docs/doctoring/validation-analysis-handoff-references.md index cfde9303b..50057cdbc 100644 --- a/docs/doctoring/validation-analysis-handoff-references.md +++ b/docs/doctoring/validation-analysis-handoff-references.md @@ -1,6 +1,6 @@ # Validation-analysis handoff references -Material decisions for ADR 0027 were checked against the following primary/authoritative sources on 2026-08-21. Regulatory currency was rechecked on 2026-08-29; fixed publication identifiers are retained so an auditor can reproduce the cited text even when agency web pages change. +Material decisions for ADR 0027 were checked against the following primary/authoritative sources on 2026-08-21. Regulatory currency was rechecked on 2026-08-29. The #407 analysis-weight evidence boundary was checked on 2026-09-17 against the primary calibration paper and current U.S. Census methodological/quality documentation. Fixed publication identifiers are retained where possible so an auditor can reproduce the cited text even when agency web pages change. ## APA 7 references @@ -8,6 +8,12 @@ Equal Employment Opportunity Commission, Civil Service Commission, Department of Society for Industrial and Organizational Psychology. (2018). Principles for the validation and use of personnel selection procedures. *Industrial and Organizational Psychology, 11*(S1), 1–97. https://doi.org/10.1017/iop.2018.195 +Deville, J.-C., & Särndal, C.-E. (1992). Calibration estimators in survey sampling. *Journal of the American Statistical Association, 87*(418), 376–382. https://doi.org/10.1080/01621459.1992.10475217 + +U.S. Census Bureau. (2021). *Statistical Quality Standard D1: Producing direct estimates from samples*. https://www.census.gov/about/policies/quality/standards/standardd1.html + +U.S. Census Bureau. (2022, August 18). *Survey of Income and Program Participation: Weighting*. https://www.census.gov/programs-surveys/sipp/methodology/weighting.html + ContextualWisdomLab. (2026). *fast-mlsirm* (Commit 04d0bc21a2a20693bcf16108cd76d394fe844d23) [Computer software]. GitHub. https://github.com/ContextualWisdomLab/fast-mlsirm/tree/04d0bc21a2a20693bcf16108cd76d394fe844d23 Tabassi, E. (2023). *Artificial intelligence risk management framework (AI RMF 1.0)* (NIST AI 100-1). National Institute of Standards and Technology. https://doi.org/10.6028/NIST.AI.100-1 @@ -19,5 +25,7 @@ Office of Personnel Management. (2026). *Removal of references to the Uniform Gu - 43 Fed. Reg. 38,290 and the still-listed EEOC 29 C.F.R. pt. 1607 source support keeping criterion-related validity evidence tied to an explicit study design, job relevance, accuracy, reporting, and documentation rather than treating a bare coefficient as sufficient evidence. The fixed Federal Register identifier, not a mutable `/current/` eCFR URL, is the reproducible source for the 1978 text cited by this ADR. - The July 31, 2026 OPM interim final rule removed UGESP references from specified federal civil-service regulations. Orgmetra therefore does not present UGESP as an undifferentiated government-wide mandate; applicability must be evaluated for the employer, jurisdiction, decision, and governing law at use time. - The SIOP Principles are the professional validation baseline used for the handoff's evidence-and-human-review posture. The journal citation above fixes volume 11, Supplement S1, pages 1–97, and DOI 10.1017/iop.2018.195. +- Deville and Särndal show that calibrated weights are produced by modifying ordinary inverse-inclusion-probability weights under explicit distance measures and calibration equations. ADR 0027 uses that narrow result to justify treating final adjusted point weights as a separately versioned scientific artifact rather than assuming `1/π_i` and calibrated weights are interchangeable. It does not mandate one calibration estimator for Orgmetra. +- Census Statistical Quality Standard D1 requires estimates and variances to account for sample design and post-sampling weighting adjustments. The SIPP methodology illustrates that final weights can combine base selection, nonresponse, longitudinal/panel, and post-stratification/calibration adjustments and that the appropriate weight depends on target population and reference duration. These sources support provenance/reproducibility requirements only; SIPP-specific weights are not imported as Orgmetra rules. - The fast-mlsirm commit is recorded as a read-only dependency coordinate only. This Orgmetra slice does not modify or duplicate its numerical implementation. - NIST AI RMF's govern, map, measure, and manage functions support preserving backend, precision, provenance, convergence, and human-review fields as inspectable result evidence rather than treating a model response as an autonomous decision. From c2e6d707f7c87a85331c839c219a927f23a2403f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 04:29:31 +0900 Subject: [PATCH 053/158] test(validity): cover analysis weight receipt edge cases --- .../tests/test_analysis_weight_receipt.py | 83 +++++++++++++++---- 1 file changed, 67 insertions(+), 16 deletions(-) diff --git a/packages/validity-analysis/tests/test_analysis_weight_receipt.py b/packages/validity-analysis/tests/test_analysis_weight_receipt.py index 133e9c39c..e6e5670db 100644 --- a/packages/validity-analysis/tests/test_analysis_weight_receipt.py +++ b/packages/validity-analysis/tests/test_analysis_weight_receipt.py @@ -1,4 +1,4 @@ -"""RED contract for reproducible point-estimation weight lineage.""" +"""Regression contracts for reproducible point-estimation weight lineage.""" from datetime import datetime, timezone @@ -24,18 +24,20 @@ DIGEST_5 = "5" * 64 -def adjustment() -> AnalysisWeightAdjustment: - """Return one governed nonresponse adjustment without copying case weights.""" - return AnalysisWeightAdjustment( - sequence_number=1, - adjustment_code="nonresponse_adjustment", - method_reference="weight_method:55555555-5555-4555-8555-555555555555", - method_version=1, - input_weight_artifact_digest=DIGEST_D, - output_weight_artifact_digest=DIGEST_E, - configuration_digest=DIGEST_F, - evidence_receipt_digest=DIGEST_1, - ) +def adjustment(**overrides: object) -> AnalysisWeightAdjustment: + """Return one governed adjustment without copying case-level weight values.""" + values: dict[str, object] = { + "sequence_number": 1, + "adjustment_code": "nonresponse_adjustment", + "method_reference": "weight_method:55555555-5555-4555-8555-555555555555", + "method_version": 1, + "input_weight_artifact_digest": DIGEST_D, + "output_weight_artifact_digest": DIGEST_E, + "configuration_digest": DIGEST_F, + "evidence_receipt_digest": DIGEST_1, + } + values.update(overrides) + return AnalysisWeightAdjustment(**values) def receipt(**overrides: object) -> FinalAnalysisWeightReceipt: @@ -66,13 +68,21 @@ def receipt(**overrides: object) -> FinalAnalysisWeightReceipt: return FinalAnalysisWeightReceipt(**values) -def test_receipt_is_deterministic_and_value_minimized() -> None: +def test_receipt_is_deterministic_value_minimized_and_redacted() -> None: """Bind exact estimand and ordered weight lineage without embedding row weights.""" candidate = receipt() assert candidate.sha256_digest() == receipt().sha256_digest() assert candidate.final_weight_artifact_digest == DIGEST_E assert "person_record" not in candidate.canonical_json() assert "weight_value" not in candidate.canonical_json() + assert repr(candidate) == "FinalAnalysisWeightReceipt()" + + +def test_no_adjustment_receipt_can_bind_base_weight_as_final_weight() -> None: + """Allow an explicit base-weight-only analysis without inventing a transform.""" + candidate = receipt(adjustments=(), final_weight_artifact_digest=DIGEST_D) + assert '"adjustments":[]' in candidate.canonical_json() + assert candidate.final_weight_artifact_digest == DIGEST_D def test_adjustment_chain_must_reach_final_weight_artifact() -> None: @@ -81,13 +91,54 @@ def test_adjustment_chain_must_reach_final_weight_artifact() -> None: receipt(final_weight_artifact_digest=DIGEST_F) +def test_adjustment_rejects_noop_artifact_identity() -> None: + """Require each declared transform to produce a distinct artifact identity.""" + with pytest.raises(ValueError, match="output_weight_artifact_digest"): + adjustment(output_weight_artifact_digest=DIGEST_D) + + +def test_adjustments_must_be_immutable_exact_and_contiguous() -> None: + """Reject mutable, foreign, skipped, or disconnected adjustment chains.""" + with pytest.raises(ValueError, match="immutable tuple"): + receipt(adjustments=[adjustment()]) + with pytest.raises(ValueError, match="AnalysisWeightAdjustment"): + receipt(adjustments=(object(),)) + with pytest.raises(ValueError, match="sequence_number"): + receipt(adjustments=(adjustment(sequence_number=2),)) + with pytest.raises(ValueError, match="breaks the weight chain"): + receipt(adjustments=(adjustment(input_weight_artifact_digest=DIGEST_C),)) + + def test_probability_design_receipt_is_required() -> None: """Do not allow point weights to detach from the sampled design evidence.""" with pytest.raises(ValueError, match="sampling_design_receipt_digest"): receipt(sampling_design_receipt_digest="not-a-digest") -def test_correction_must_link_to_the_superseded_receipt() -> None: - """Require append-only correction lineage instead of overwriting prior weight evidence.""" +def test_positive_versions_and_counts_fail_closed() -> None: + """Reject sentinel sequence, method-version, count, and correction values.""" + with pytest.raises(ValueError, match="sequence_number"): + adjustment(sequence_number=0) + with pytest.raises(ValueError, match="method_version"): + adjustment(method_version=0) + with pytest.raises(ValueError, match="analytic_case_count"): + receipt(analytic_case_count=0) + with pytest.raises(ValueError, match="correction_sequence"): + receipt(correction_sequence=0) + + +def test_correction_lineage_is_append_only_and_canonicalized() -> None: + """Require successor evidence rather than overwriting a prior weight receipt.""" with pytest.raises(ValueError, match="supersedes_receipt_digest"): receipt(correction_sequence=2) + with pytest.raises(ValueError, match="must be absent"): + receipt(supersedes_receipt_digest=DIGEST_A) + + corrected = receipt(correction_sequence=2, supersedes_receipt_digest=DIGEST_A) + assert f'"supersedes_receipt_digest":"{DIGEST_A}"' in corrected.canonical_json() + + +def test_evidence_version_is_not_caller_extensible() -> None: + """Prevent callers from inventing a new receipt schema without a reviewed contract.""" + with pytest.raises(ValueError, match="evidence_version"): + receipt(evidence_version=2) From efac8be81f862ebd6c5d2f53bfde667e43510790 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 04:29:52 +0900 Subject: [PATCH 054/158] test(validity): close weighted result edge coverage --- .../test_analysis_weight_result_binding.py | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/packages/validity-analysis/tests/test_analysis_weight_result_binding.py b/packages/validity-analysis/tests/test_analysis_weight_result_binding.py index d3e11947b..b35c91bf5 100644 --- a/packages/validity-analysis/tests/test_analysis_weight_result_binding.py +++ b/packages/validity-analysis/tests/test_analysis_weight_result_binding.py @@ -1,4 +1,4 @@ -"""RED contract for binding weighted scientific results to exact weight evidence.""" +"""Regression contract for binding weighted results to exact weight evidence.""" from datetime import datetime, timezone @@ -91,7 +91,15 @@ def test_weighted_result_binds_point_and_variance_receipts_separately() -> None: }, "unweighted", ), + ( + { + "point_estimation_mode": "unweighted", + "variance_design_receipt_digest": VARIANCE_DIGEST, + }, + "unweighted", + ), ({"point_estimation_mode": "opaque_weighted"}, "point_estimation_mode"), + ({"point_estimation_mode": None}, "point_estimation_mode"), ], ) def test_result_rejects_unverifiable_weight_binding( @@ -100,3 +108,19 @@ def test_result_rejects_unverifiable_weight_binding( """Fail closed when result and point/variance weight provenance disagree.""" with pytest.raises(ValueError, match=match): result(**overrides) + + +def test_weighted_result_rejects_malformed_receipt_digests() -> None: + """Do not permit opaque labels to stand in for immutable weight evidence.""" + with pytest.raises(ValueError, match="analysis_weight_receipt_digest"): + result( + point_estimation_mode="weighted_design_based", + analysis_weight_receipt_digest="weight-v1", + variance_design_receipt_digest=VARIANCE_DIGEST, + ) + with pytest.raises(ValueError, match="variance_design_receipt_digest"): + result( + point_estimation_mode="weighted_design_based", + analysis_weight_receipt_digest=WEIGHT_DIGEST, + variance_design_receipt_digest="variance-v1", + ) From 0d56b9a2ece525561a6ebd7bdea5ac8333ea822f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 04:30:05 +0900 Subject: [PATCH 055/158] docs(validity): record weight-lineage executable slice --- packages/validity-analysis/CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/validity-analysis/CHANGELOG.md b/packages/validity-analysis/CHANGELOG.md index d0ff8bd9f..cab9753e4 100644 --- a/packages/validity-analysis/CHANGELOG.md +++ b/packages/validity-analysis/CHANGELOG.md @@ -10,4 +10,7 @@ - Reject impossible aggregate missingness where complete observations overlap either predictor-missing or criterion-missing counts beyond the sample total. - Require exact governed missingness/convergence runtime types so subclass method overrides cannot inject unreviewed or person-level fields into canonical result evidence. - Freeze exact UTC timestamps and finite numeric values at construction, and reject runtime-type forgery before canonical evidence serialization. +- Add a deterministic, row-value-minimized `FinalAnalysisWeightReceipt` that binds an exact estimand to source/sampling evidence, base-weight provenance, an ordered digest-linked adjustment chain, the final point-weight artifact, and append-only correction lineage. +- Distinguish unweighted from weighted design-based results and fail closed unless a weighted result separately binds the exact final analysis-weight receipt and variance-design receipt used. +- Keep adjustment-specific nonresponse, calibration/raking, trimming/bounding, longitudinal and auxiliary-owner contracts explicitly incomplete under #407 rather than promoting the first executable weight-lineage slice to scientific completion. - Make `Validity Analysis Handoff Quality` retrigger on shared repository Python/test/clean-checkout configuration, with an executable regression preventing stale package-quality evidence after shared tooling changes. From 5cbd155c67b2dfc5733acf4c9f4a47ee233b2da7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 04:30:41 +0900 Subject: [PATCH 056/158] docs(validity): explain weighted result evidence --- packages/validity-analysis/README.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/validity-analysis/README.md b/packages/validity-analysis/README.md index b56e9a040..a1cdb6cc3 100644 --- a/packages/validity-analysis/README.md +++ b/packages/validity-analysis/README.md @@ -1,6 +1,6 @@ # Orgmetra validity-analysis handoff -This package creates an immutable **selection-validity analysis handoff** and validates the matching numerical result envelope. It is the boundary between Orgmetra's authoritative validation-study evidence and numerical work owned by `ContextualWisdomLab/fast-mlsirm`. +This package creates an immutable **selection-validity analysis handoff**, validates the matching numerical result envelope, and preserves the first executable #407 point-estimation weight lineage contract. It is the boundary between Orgmetra's authoritative validation-study evidence and numerical work owned by `ContextualWisdomLab/fast-mlsirm`. ## What it does @@ -8,7 +8,11 @@ This package creates an immutable **selection-validity analysis handoff** and va The resulting canonical JSON is digest-addressable, contains no raw person-level predictor or criterion values, and remains `not_executed`. Its timestamp is detached to one UTC instant at construction so later timezone-provider changes cannot rewrite the digest. Required result evidence is explicit: effect estimate, uncertainty interval, sample size, missingness summary, and convergence diagnostics. -`ValidationAnalysisResult` accepts only a result linked to the handoff digest and the same reviewed fast-mlsirm revision. It records the Rust CPU/GPU backend, precision, aggregate missingness counts, finite effect and interval values, and explicit convergence or nonconvergence diagnostics. Timestamps and finite numeric values are snapshotted before canonicalization, and the result envelope accepts only the exact governed `MissingnessSummary` and `ConvergenceDiagnostics` runtime types, preventing mutable runtime values or subclass method overrides from adding unreviewed or person-level fields to canonical audit evidence. Missingness counts must be internally possible: complete observations cannot overlap either predictor-missing or criterion-missing observations beyond the declared sample total. It never promotes a result to an employment decision; human review remains mandatory. +`FinalAnalysisWeightReceipt` binds a weighted estimand to exact target/window/case-set evidence, source/sampling receipt digests, base-weight evidence, an ordered immutable adjustment chain, the final point-weight artifact digest, and append-only correction lineage. It records only identifiers, versions and digests; it does not store row-level weight values or copy calibration attributes into the validity package. + +`ValidationAnalysisResult` accepts only a result linked to the handoff digest and the same reviewed fast-mlsirm revision. It records the Rust CPU/GPU backend, precision, aggregate missingness counts, finite effect and interval values, and explicit convergence or nonconvergence diagnostics. Timestamps and finite numeric values are snapshotted before canonicalization, and the result envelope accepts only the exact governed `MissingnessSummary` and `ConvergenceDiagnostics` runtime types, preventing mutable runtime values or subclass method overrides from adding unreviewed or person-level fields to canonical audit evidence. Missingness counts must be internally possible: complete observations cannot overlap either predictor-missing or criterion-missing observations beyond the declared sample total. + +Point-estimation semantics are explicit. `unweighted` results cannot carry weight receipts. `weighted_design_based` results must separately bind the exact `FinalAnalysisWeightReceipt` digest and the variance-design receipt digest that supports the uncertainty method actually used. A replicate/variance evidence reference therefore cannot silently stand in for the final point-estimation weight, or vice versa. The result remains scientific evidence for accountable human interpretation and never becomes an employment decision. ## What it does not do @@ -18,12 +22,13 @@ The resulting canonical JSON is digest-addressable, contains no raw person-level - It does **not** claim that a selection procedure is valid. - It does **not** interpret adverse impact. - It does **not** authorize hiring, promotion, termination, compensation, or another employment decision. +- It does **not** yet implement all #407 adjustment-specific contracts. Nonresponse-disposition evidence, calibration/raking benchmark-owner receipts and constraints, trimming/bounding rules, longitudinal eligibility, convergence/fallback semantics, and sensitive auxiliary-variable purpose binding remain explicit open work in `workforce_validation`. -The fast-mlsirm repository remains a dedicated-writer dependency. This package records only the immutable revision reviewed for the handoff. +The fast-mlsirm repository remains a dedicated-writer dependency. This package records only the immutable revision reviewed for the handoff. Durable Workforce Validation registry/API/persistence remains on the canonical #235/#248 owner path; this package does not create a parallel service. ## Host obligations -Before an approved offline validation worker executes the handoff, the Orgmetra host must re-resolve every reference inside `tenant_record_id` and prove that the predictor, criterion, population, and policy evidence belong to the exact validation study and Job. The requester and reviewer must resolve to distinct authoritative actors. A numerical result is scientific evidence for accountable human interpretation, never an autonomous employment decision. +Before an approved offline validation worker executes the handoff, the Orgmetra host must re-resolve every reference inside `tenant_record_id` and prove that the predictor, criterion, population, and policy evidence belong to the exact validation study and Job. The requester and reviewer must resolve to distinct authoritative actors. For weighted design-based inference, the host must also verify the final point-weight receipt, its upstream source/sampling evidence, and the separately bound variance-design evidence rather than trusting caller-supplied labels. A numerical result is scientific evidence for accountable human interpretation, never an autonomous employment decision. ## Verification From 7aa5e25d582af026ea528da78d8fe76e20728b0a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 05:04:54 +0900 Subject: [PATCH 057/158] test(validity): add RED for governed weight-adjustment evidence --- .../tests/test_weight_adjustment_semantics.py | 150 ++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 packages/validity-analysis/tests/test_weight_adjustment_semantics.py diff --git a/packages/validity-analysis/tests/test_weight_adjustment_semantics.py b/packages/validity-analysis/tests/test_weight_adjustment_semantics.py new file mode 100644 index 000000000..550987626 --- /dev/null +++ b/packages/validity-analysis/tests/test_weight_adjustment_semantics.py @@ -0,0 +1,150 @@ +"""Scientific contracts for nonresponse and calibration weight adjustments.""" + +from datetime import datetime, timezone + +import pytest + +from orgmetra_validity_analysis import ( + AnalysisWeightAdjustment, + CalibrationAdjustmentReceipt, + NonresponseAdjustmentReceipt, +) + +TENANT = "10000000-0000-7000-8000-000000000001" +NOW = datetime(2026, 9, 17, 5, 0, tzinfo=timezone.utc) +DIGEST_A = "a" * 64 +DIGEST_B = "b" * 64 +DIGEST_C = "c" * 64 +DIGEST_D = "d" * 64 +DIGEST_E = "e" * 64 +DIGEST_F = "f" * 64 +DIGEST_1 = "1" * 64 +DIGEST_2 = "2" * 64 +DIGEST_3 = "3" * 64 +DIGEST_4 = "4" * 64 + + +def nonresponse_receipt(**overrides: object) -> NonresponseAdjustmentReceipt: + """Return one disposition-aware nonresponse adjustment receipt.""" + values: dict[str, object] = { + "tenant_record_id": TENANT, + "receipt_reference": "nonresponse_adjustment_receipt:11111111-1111-4111-8111-111111111111", + "response_disposition_receipt_digest": DIGEST_A, + "adjustment_population_digest": DIGEST_B, + "method_reference": "weight_method:22222222-2222-4222-8222-222222222222", + "method_version": 1, + "configuration_digest": DIGEST_C, + "ineligible_treatment_code": "exclude_as_ineligible", + "unknown_treatment_code": "retain_in_unknown_class", + "unavailable_treatment_code": "retain_in_unavailable_class", + "input_weight_artifact_digest": DIGEST_D, + "output_weight_artifact_digest": DIGEST_E, + "constructed_at": NOW, + } + values.update(overrides) + return NonresponseAdjustmentReceipt(**values) + + +def calibration_receipt(**overrides: object) -> CalibrationAdjustmentReceipt: + """Return one benchmark-bound converged calibration receipt.""" + values: dict[str, object] = { + "tenant_record_id": TENANT, + "receipt_reference": "calibration_adjustment_receipt:33333333-3333-4333-8333-333333333333", + "target_population_digest": DIGEST_A, + "analysis_window_reference": "analysis_window:44444444-4444-4444-8444-444444444444", + "auxiliary_projection_reference": "calibration_auxiliary_projection:55555555-5555-4555-8555-555555555555", + "auxiliary_projection_digest": DIGEST_B, + "benchmark_receipt_reference": "calibration_benchmark_receipt:66666666-6666-4666-8666-666666666666", + "benchmark_receipt_digest": DIGEST_C, + "algorithm_reference": "calibration_algorithm:77777777-7777-4777-8777-777777777777", + "algorithm_version": 1, + "constraints_digest": DIGEST_F, + "termination_code": "converged", + "input_weight_artifact_digest": DIGEST_D, + "output_weight_artifact_digest": DIGEST_E, + "constructed_at": NOW, + } + values.update(overrides) + return CalibrationAdjustmentReceipt(**values) + + +def adjustment(*, code: str, evidence_kind: str) -> AnalysisWeightAdjustment: + """Return one adjustment linked to typed scientific evidence.""" + return AnalysisWeightAdjustment( + sequence_number=1, + adjustment_code=code, + method_reference="weight_method:88888888-8888-4888-8888-888888888888", + method_version=1, + input_weight_artifact_digest=DIGEST_D, + output_weight_artifact_digest=DIGEST_E, + configuration_digest=DIGEST_F, + evidence_receipt_digest=DIGEST_1, + evidence_kind=evidence_kind, + ) + + +def test_nonresponse_receipt_is_value_minimized_and_disposition_aware() -> None: + """Preserve explicit disposition treatment without copying source attributes.""" + candidate = nonresponse_receipt() + assert candidate.sha256_digest() == nonresponse_receipt().sha256_digest() + assert '"unknown_treatment_code":"retain_in_unknown_class"' in candidate.canonical_json() + assert "person_record" not in candidate.canonical_json() + assert "protected_attribute" not in candidate.canonical_json() + + with pytest.raises(ValueError, match="unknown_treatment_code"): + nonresponse_receipt(unknown_treatment_code="") + with pytest.raises(ValueError, match="output_weight_artifact_digest"): + nonresponse_receipt(output_weight_artifact_digest=DIGEST_D) + with pytest.raises(ValueError, match="evidence_version"): + nonresponse_receipt(evidence_version=2) + + +def test_calibration_receipt_binds_owner_benchmark_and_termination_state() -> None: + """Keep benchmark ownership and fallback semantics explicit and immutable.""" + candidate = calibration_receipt() + assert candidate.sha256_digest() == calibration_receipt().sha256_digest() + assert f'"benchmark_receipt_digest":"{DIGEST_C}"' in candidate.canonical_json() + assert '"termination_code":"converged"' in candidate.canonical_json() + + fallback = calibration_receipt( + termination_code="fallback_applied", + fallback_rule_reference="calibration_fallback_rule:99999999-9999-4999-8999-999999999999", + fallback_rule_digest=DIGEST_2, + ) + assert f'"fallback_rule_digest":"{DIGEST_2}"' in fallback.canonical_json() + + with pytest.raises(ValueError, match="termination_code"): + calibration_receipt(termination_code="failed") + with pytest.raises(ValueError, match="fallback_rule"): + calibration_receipt(termination_code="fallback_applied") + with pytest.raises(ValueError, match="must be absent"): + calibration_receipt( + fallback_rule_reference="calibration_fallback_rule:99999999-9999-4999-8999-999999999999", + fallback_rule_digest=DIGEST_2, + ) + with pytest.raises(ValueError, match="output_weight_artifact_digest"): + calibration_receipt(output_weight_artifact_digest=DIGEST_D) + with pytest.raises(ValueError, match="evidence_version"): + calibration_receipt(evidence_version=2) + + +def test_specialized_adjustments_require_matching_evidence_kind() -> None: + """Do not let typed nonresponse or calibration semantics collapse into opaque digests.""" + nonresponse = adjustment( + code="nonresponse_adjustment", + evidence_kind="nonresponse_adjustment_receipt", + ) + calibration = adjustment( + code="calibration_adjustment", + evidence_kind="calibration_adjustment_receipt", + ) + assert nonresponse.evidence_kind == "nonresponse_adjustment_receipt" + assert calibration.evidence_kind == "calibration_adjustment_receipt" + + with pytest.raises(ValueError, match="nonresponse_adjustment_receipt"): + adjustment(code="nonresponse_adjustment", evidence_kind="generic_adjustment_receipt") + with pytest.raises(ValueError, match="calibration_adjustment_receipt"): + adjustment(code="raking_adjustment", evidence_kind="generic_adjustment_receipt") + + generic = adjustment(code="trimming_adjustment", evidence_kind="generic_adjustment_receipt") + assert generic.evidence_kind == "generic_adjustment_receipt" From 2bd7188efbe23de267ab487a0b759403a2f4e5d3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 05:05:37 +0900 Subject: [PATCH 058/158] feat(validity): govern nonresponse and calibration weight evidence --- .../src/orgmetra_validity_analysis/weights.py | 228 +++++++++++++++++- 1 file changed, 227 insertions(+), 1 deletion(-) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py b/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py index 9eb48de96..a0597da76 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py @@ -20,6 +20,14 @@ _validate_reference, ) +_CALIBRATION_TERMINATION_CODES = frozenset({"converged", "fallback_applied"}) +_SPECIALIZED_EVIDENCE_KIND_BY_ADJUSTMENT_CODE = { + "nonresponse_adjustment": "nonresponse_adjustment_receipt", + "calibration_adjustment": "calibration_adjustment_receipt", + "raking_adjustment": "calibration_adjustment_receipt", + "poststratification_adjustment": "calibration_adjustment_receipt", +} + def _positive_integer(value: object, field_name: str) -> None: """Require a strict positive integer without accepting booleans.""" @@ -27,6 +35,209 @@ def _positive_integer(value: object, field_name: str) -> None: raise ValueError(f"{field_name} must be a positive integer") +@dataclass(frozen=True, slots=True, repr=False) +class NonresponseAdjustmentReceipt: + """Bind one nonresponse adjustment to explicit disposition-aware evidence.""" + + tenant_record_id: str + receipt_reference: str + response_disposition_receipt_digest: str + adjustment_population_digest: str + method_reference: str + method_version: int + configuration_digest: str + ineligible_treatment_code: str + unknown_treatment_code: str + unavailable_treatment_code: str + input_weight_artifact_digest: str + output_weight_artifact_digest: str + constructed_at: datetime + evidence_version: int = 1 + + def __post_init__(self) -> None: + """Reject undocumented filters or mutable nonresponse evidence.""" + _validate_operational_uuid(self.tenant_record_id, "tenant_record_id") + _validate_reference( + self.receipt_reference, + "nonresponse_adjustment_receipt", + "receipt_reference", + ) + for field_name in ( + "response_disposition_receipt_digest", + "adjustment_population_digest", + "configuration_digest", + "input_weight_artifact_digest", + "output_weight_artifact_digest", + ): + _validate_digest(getattr(self, field_name), field_name) + _validate_reference(self.method_reference, "weight_method", "method_reference") + _positive_integer(self.method_version, "method_version") + for field_name in ( + "ineligible_treatment_code", + "unknown_treatment_code", + "unavailable_treatment_code", + ): + _validate_code(getattr(self, field_name), field_name) + if self.input_weight_artifact_digest == self.output_weight_artifact_digest: + raise ValueError( + "output_weight_artifact_digest must identify the adjusted weight artifact" + ) + constructed_at = _freeze_timestamp(self.constructed_at, "constructed_at") + if type(self.evidence_version) is not int or self.evidence_version != 1: + raise ValueError("evidence_version must remain 1") + object.__setattr__(self, "constructed_at", constructed_at) + + def __repr__(self) -> str: + """Return a value-minimized representation for routine logs.""" + return "NonresponseAdjustmentReceipt()" + + def canonical_json(self) -> str: + """Return deterministic disposition-aware provenance without source attributes.""" + payload = { + "adjustment_population_digest": self.adjustment_population_digest, + "configuration_digest": self.configuration_digest, + "constructed_at": _canonical_timestamp(self.constructed_at, "constructed_at"), + "evidence_version": self.evidence_version, + "ineligible_treatment_code": self.ineligible_treatment_code, + "input_weight_artifact_digest": self.input_weight_artifact_digest, + "method_reference": self.method_reference, + "method_version": self.method_version, + "output_weight_artifact_digest": self.output_weight_artifact_digest, + "receipt_reference": self.receipt_reference, + "response_disposition_receipt_digest": self.response_disposition_receipt_digest, + "tenant_record_id": self.tenant_record_id, + "unavailable_treatment_code": self.unavailable_treatment_code, + "unknown_treatment_code": self.unknown_treatment_code, + } + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + def sha256_digest(self) -> str: + """Return SHA-256 over the exact canonical receipt bytes.""" + return sha256(self.canonical_json().encode("utf-8")).hexdigest() + + +@dataclass(frozen=True, slots=True, repr=False) +class CalibrationAdjustmentReceipt: + """Bind calibration or raking to immutable owner benchmarks and termination evidence.""" + + tenant_record_id: str + receipt_reference: str + target_population_digest: str + analysis_window_reference: str + auxiliary_projection_reference: str + auxiliary_projection_digest: str + benchmark_receipt_reference: str + benchmark_receipt_digest: str + algorithm_reference: str + algorithm_version: int + constraints_digest: str + termination_code: str + input_weight_artifact_digest: str + output_weight_artifact_digest: str + constructed_at: datetime + fallback_rule_reference: str | None = None + fallback_rule_digest: str | None = None + evidence_version: int = 1 + + def __post_init__(self) -> None: + """Fail closed on floating benchmarks, hidden fallback, or nonconverged output.""" + _validate_operational_uuid(self.tenant_record_id, "tenant_record_id") + _validate_reference( + self.receipt_reference, + "calibration_adjustment_receipt", + "receipt_reference", + ) + _validate_reference( + self.analysis_window_reference, + "analysis_window", + "analysis_window_reference", + ) + _validate_reference( + self.auxiliary_projection_reference, + "calibration_auxiliary_projection", + "auxiliary_projection_reference", + ) + _validate_reference( + self.benchmark_receipt_reference, + "calibration_benchmark_receipt", + "benchmark_receipt_reference", + ) + _validate_reference( + self.algorithm_reference, + "calibration_algorithm", + "algorithm_reference", + ) + for field_name in ( + "target_population_digest", + "auxiliary_projection_digest", + "benchmark_receipt_digest", + "constraints_digest", + "input_weight_artifact_digest", + "output_weight_artifact_digest", + ): + _validate_digest(getattr(self, field_name), field_name) + _positive_integer(self.algorithm_version, "algorithm_version") + if ( + type(self.termination_code) is not str + or self.termination_code not in _CALIBRATION_TERMINATION_CODES + ): + raise ValueError("termination_code must be converged or fallback_applied") + if self.termination_code == "fallback_applied": + if self.fallback_rule_reference is None or self.fallback_rule_digest is None: + raise ValueError( + "fallback_rule_reference and fallback_rule_digest are required for fallback_applied" + ) + _validate_reference( + self.fallback_rule_reference, + "calibration_fallback_rule", + "fallback_rule_reference", + ) + _validate_digest(self.fallback_rule_digest, "fallback_rule_digest") + elif self.fallback_rule_reference is not None or self.fallback_rule_digest is not None: + raise ValueError("fallback_rule evidence must be absent when calibration converged") + if self.input_weight_artifact_digest == self.output_weight_artifact_digest: + raise ValueError( + "output_weight_artifact_digest must identify the calibrated weight artifact" + ) + constructed_at = _freeze_timestamp(self.constructed_at, "constructed_at") + if type(self.evidence_version) is not int or self.evidence_version != 1: + raise ValueError("evidence_version must remain 1") + object.__setattr__(self, "constructed_at", constructed_at) + + def __repr__(self) -> str: + """Return a value-minimized representation for routine logs.""" + return "CalibrationAdjustmentReceipt()" + + def canonical_json(self) -> str: + """Return deterministic owner-benchmark provenance without auxiliary values.""" + payload: dict[str, object] = { + "algorithm_reference": self.algorithm_reference, + "algorithm_version": self.algorithm_version, + "analysis_window_reference": self.analysis_window_reference, + "auxiliary_projection_digest": self.auxiliary_projection_digest, + "auxiliary_projection_reference": self.auxiliary_projection_reference, + "benchmark_receipt_digest": self.benchmark_receipt_digest, + "benchmark_receipt_reference": self.benchmark_receipt_reference, + "constraints_digest": self.constraints_digest, + "constructed_at": _canonical_timestamp(self.constructed_at, "constructed_at"), + "evidence_version": self.evidence_version, + "input_weight_artifact_digest": self.input_weight_artifact_digest, + "output_weight_artifact_digest": self.output_weight_artifact_digest, + "receipt_reference": self.receipt_reference, + "target_population_digest": self.target_population_digest, + "tenant_record_id": self.tenant_record_id, + "termination_code": self.termination_code, + } + if self.fallback_rule_reference is not None: + payload["fallback_rule_digest"] = self.fallback_rule_digest + payload["fallback_rule_reference"] = self.fallback_rule_reference + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + def sha256_digest(self) -> str: + """Return SHA-256 over the exact canonical receipt bytes.""" + return sha256(self.canonical_json().encode("utf-8")).hexdigest() + + @dataclass(frozen=True, slots=True) class AnalysisWeightAdjustment: """Describe one ordered, digest-linked transformation of analysis weights.""" @@ -39,6 +250,7 @@ class AnalysisWeightAdjustment: output_weight_artifact_digest: str configuration_digest: str evidence_receipt_digest: str + evidence_kind: str def __post_init__(self) -> None: """Reject unordered, opaque, or unverifiable adjustment evidence.""" @@ -53,6 +265,14 @@ def __post_init__(self) -> None: "evidence_receipt_digest", ): _validate_digest(getattr(self, field_name), field_name) + _validate_code(self.evidence_kind, "evidence_kind") + required_evidence_kind = _SPECIALIZED_EVIDENCE_KIND_BY_ADJUSTMENT_CODE.get( + self.adjustment_code + ) + if required_evidence_kind is not None and self.evidence_kind != required_evidence_kind: + raise ValueError( + f"{self.adjustment_code} requires evidence_kind {required_evidence_kind}" + ) if self.input_weight_artifact_digest == self.output_weight_artifact_digest: raise ValueError( "output_weight_artifact_digest must identify the transformed weight artifact" @@ -63,6 +283,7 @@ def to_dict(self) -> dict[str, object]: return { "adjustment_code": self.adjustment_code, "configuration_digest": self.configuration_digest, + "evidence_kind": self.evidence_kind, "evidence_receipt_digest": self.evidence_receipt_digest, "input_weight_artifact_digest": self.input_weight_artifact_digest, "method_reference": self.method_reference, @@ -204,4 +425,9 @@ def sha256_digest(self) -> str: return sha256(self.canonical_json().encode("utf-8")).hexdigest() -__all__ = ["AnalysisWeightAdjustment", "FinalAnalysisWeightReceipt"] +__all__ = [ + "AnalysisWeightAdjustment", + "CalibrationAdjustmentReceipt", + "FinalAnalysisWeightReceipt", + "NonresponseAdjustmentReceipt", +] From 892dde27c6d4083ea8573216539eed337ace38c8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 05:05:54 +0900 Subject: [PATCH 059/158] test(validity): bind typed adjustment evidence in weight lineage --- packages/validity-analysis/tests/test_analysis_weight_receipt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/validity-analysis/tests/test_analysis_weight_receipt.py b/packages/validity-analysis/tests/test_analysis_weight_receipt.py index e6e5670db..9603d7c58 100644 --- a/packages/validity-analysis/tests/test_analysis_weight_receipt.py +++ b/packages/validity-analysis/tests/test_analysis_weight_receipt.py @@ -35,6 +35,7 @@ def adjustment(**overrides: object) -> AnalysisWeightAdjustment: "output_weight_artifact_digest": DIGEST_E, "configuration_digest": DIGEST_F, "evidence_receipt_digest": DIGEST_1, + "evidence_kind": "nonresponse_adjustment_receipt", } values.update(overrides) return AnalysisWeightAdjustment(**values) From 5c14ae9a70c81689e9c14357205a65bbe4b6ea81 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 05:06:00 +0900 Subject: [PATCH 060/158] feat(validity): export governed adjustment receipts --- .../src/orgmetra_validity_analysis/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py b/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py index 1d8bc1701..794380594 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py @@ -6,7 +6,12 @@ build_validation_analysis_handoff, ) from .result import ConvergenceDiagnostics, MissingnessSummary, ValidationAnalysisResult -from .weights import AnalysisWeightAdjustment, FinalAnalysisWeightReceipt +from .weights import ( + AnalysisWeightAdjustment, + CalibrationAdjustmentReceipt, + FinalAnalysisWeightReceipt, + NonresponseAdjustmentReceipt, +) __all__ = [ "REVIEWED_FAST_MLSIRM_REVISION", @@ -16,5 +21,7 @@ "MissingnessSummary", "ValidationAnalysisResult", "AnalysisWeightAdjustment", + "CalibrationAdjustmentReceipt", "FinalAnalysisWeightReceipt", + "NonresponseAdjustmentReceipt", ] From 36ba026150cbe56c97b1680de4006d758f567d49 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 05:06:58 +0900 Subject: [PATCH 061/158] test(validity): close adjustment-evidence branch coverage --- .../tests/test_weight_adjustment_semantics.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/validity-analysis/tests/test_weight_adjustment_semantics.py b/packages/validity-analysis/tests/test_weight_adjustment_semantics.py index 550987626..2fd64c96e 100644 --- a/packages/validity-analysis/tests/test_weight_adjustment_semantics.py +++ b/packages/validity-analysis/tests/test_weight_adjustment_semantics.py @@ -90,6 +90,7 @@ def test_nonresponse_receipt_is_value_minimized_and_disposition_aware() -> None: assert '"unknown_treatment_code":"retain_in_unknown_class"' in candidate.canonical_json() assert "person_record" not in candidate.canonical_json() assert "protected_attribute" not in candidate.canonical_json() + assert repr(candidate) == "NonresponseAdjustmentReceipt()" with pytest.raises(ValueError, match="unknown_treatment_code"): nonresponse_receipt(unknown_treatment_code="") @@ -97,6 +98,8 @@ def test_nonresponse_receipt_is_value_minimized_and_disposition_aware() -> None: nonresponse_receipt(output_weight_artifact_digest=DIGEST_D) with pytest.raises(ValueError, match="evidence_version"): nonresponse_receipt(evidence_version=2) + with pytest.raises(ValueError, match="evidence_version"): + nonresponse_receipt(evidence_version=True) def test_calibration_receipt_binds_owner_benchmark_and_termination_state() -> None: @@ -105,27 +108,40 @@ def test_calibration_receipt_binds_owner_benchmark_and_termination_state() -> No assert candidate.sha256_digest() == calibration_receipt().sha256_digest() assert f'"benchmark_receipt_digest":"{DIGEST_C}"' in candidate.canonical_json() assert '"termination_code":"converged"' in candidate.canonical_json() + assert repr(candidate) == "CalibrationAdjustmentReceipt()" + fallback_reference = "calibration_fallback_rule:99999999-9999-4999-8999-999999999999" fallback = calibration_receipt( termination_code="fallback_applied", - fallback_rule_reference="calibration_fallback_rule:99999999-9999-4999-8999-999999999999", + fallback_rule_reference=fallback_reference, fallback_rule_digest=DIGEST_2, ) assert f'"fallback_rule_digest":"{DIGEST_2}"' in fallback.canonical_json() with pytest.raises(ValueError, match="termination_code"): calibration_receipt(termination_code="failed") + with pytest.raises(ValueError, match="termination_code"): + calibration_receipt(termination_code=1) with pytest.raises(ValueError, match="fallback_rule"): calibration_receipt(termination_code="fallback_applied") + with pytest.raises(ValueError, match="fallback_rule"): + calibration_receipt( + termination_code="fallback_applied", + fallback_rule_reference=fallback_reference, + ) with pytest.raises(ValueError, match="must be absent"): calibration_receipt( - fallback_rule_reference="calibration_fallback_rule:99999999-9999-4999-8999-999999999999", + fallback_rule_reference=fallback_reference, fallback_rule_digest=DIGEST_2, ) + with pytest.raises(ValueError, match="must be absent"): + calibration_receipt(fallback_rule_digest=DIGEST_2) with pytest.raises(ValueError, match="output_weight_artifact_digest"): calibration_receipt(output_weight_artifact_digest=DIGEST_D) with pytest.raises(ValueError, match="evidence_version"): calibration_receipt(evidence_version=2) + with pytest.raises(ValueError, match="evidence_version"): + calibration_receipt(evidence_version=True) def test_specialized_adjustments_require_matching_evidence_kind() -> None: From b60f9bb3b4ff0b1df217a4087c6b6631c3f2625f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 05:07:16 +0900 Subject: [PATCH 062/158] docs(validity): document typed nonresponse and calibration receipts --- packages/validity-analysis/README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/validity-analysis/README.md b/packages/validity-analysis/README.md index a1cdb6cc3..0f99839ee 100644 --- a/packages/validity-analysis/README.md +++ b/packages/validity-analysis/README.md @@ -1,6 +1,6 @@ # Orgmetra validity-analysis handoff -This package creates an immutable **selection-validity analysis handoff**, validates the matching numerical result envelope, and preserves the first executable #407 point-estimation weight lineage contract. It is the boundary between Orgmetra's authoritative validation-study evidence and numerical work owned by `ContextualWisdomLab/fast-mlsirm`. +This package creates an immutable **selection-validity analysis handoff**, validates the matching numerical result envelope, and preserves executable #407 point-estimation weight lineage. It is the boundary between Orgmetra's authoritative validation-study evidence and numerical work owned by `ContextualWisdomLab/fast-mlsirm`. ## What it does @@ -10,6 +10,10 @@ The resulting canonical JSON is digest-addressable, contains no raw person-level `FinalAnalysisWeightReceipt` binds a weighted estimand to exact target/window/case-set evidence, source/sampling receipt digests, base-weight evidence, an ordered immutable adjustment chain, the final point-weight artifact digest, and append-only correction lineage. It records only identifiers, versions and digests; it does not store row-level weight values or copy calibration attributes into the validity package. +Nonresponse and calibration are no longer allowed to hide behind an undifferentiated adjustment digest. `NonresponseAdjustmentReceipt` binds the adjustment to a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, and explicit treatment codes for ineligible, unknown, and unavailable cases. `CalibrationAdjustmentReceipt` binds the adjustment to a target population/window, purpose-bound auxiliary projection, authoritative benchmark receipt, algorithm/version, constraints digest, and explicit `converged` or `fallback_applied` termination evidence. A fallback must identify its versioned rule; a nonconverged calibration cannot masquerade as an accepted calibration receipt. + +`AnalysisWeightAdjustment` records an `evidence_kind` in addition to the evidence digest. Known nonresponse and calibration/raking/post-stratification transforms fail closed unless their evidence kind is the corresponding typed receipt. Other adjustment families remain open work rather than being silently treated as equivalent. + `ValidationAnalysisResult` accepts only a result linked to the handoff digest and the same reviewed fast-mlsirm revision. It records the Rust CPU/GPU backend, precision, aggregate missingness counts, finite effect and interval values, and explicit convergence or nonconvergence diagnostics. Timestamps and finite numeric values are snapshotted before canonicalization, and the result envelope accepts only the exact governed `MissingnessSummary` and `ConvergenceDiagnostics` runtime types, preventing mutable runtime values or subclass method overrides from adding unreviewed or person-level fields to canonical audit evidence. Missingness counts must be internally possible: complete observations cannot overlap either predictor-missing or criterion-missing observations beyond the declared sample total. Point-estimation semantics are explicit. `unweighted` results cannot carry weight receipts. `weighted_design_based` results must separately bind the exact `FinalAnalysisWeightReceipt` digest and the variance-design receipt digest that supports the uncertainty method actually used. A replicate/variance evidence reference therefore cannot silently stand in for the final point-estimation weight, or vice versa. The result remains scientific evidence for accountable human interpretation and never becomes an employment decision. @@ -22,13 +26,13 @@ Point-estimation semantics are explicit. `unweighted` results cannot carry weigh - It does **not** claim that a selection procedure is valid. - It does **not** interpret adverse impact. - It does **not** authorize hiring, promotion, termination, compensation, or another employment decision. -- It does **not** yet implement all #407 adjustment-specific contracts. Nonresponse-disposition evidence, calibration/raking benchmark-owner receipts and constraints, trimming/bounding rules, longitudinal eligibility, convergence/fallback semantics, and sensitive auxiliary-variable purpose binding remain explicit open work in `workforce_validation`. +- It does **not** yet complete #407. Trimming/bounding receipts, longitudinal/cross-sectional eligibility, owner-side verification that an adjustment digest resolves to the typed receipt claimed by `evidence_kind`, sensitive auxiliary-variable purpose enforcement at the durable service/API boundary, and released auxiliary evidence exchange remain open work in `workforce_validation`. The fast-mlsirm repository remains a dedicated-writer dependency. This package records only the immutable revision reviewed for the handoff. Durable Workforce Validation registry/API/persistence remains on the canonical #235/#248 owner path; this package does not create a parallel service. ## Host obligations -Before an approved offline validation worker executes the handoff, the Orgmetra host must re-resolve every reference inside `tenant_record_id` and prove that the predictor, criterion, population, and policy evidence belong to the exact validation study and Job. The requester and reviewer must resolve to distinct authoritative actors. For weighted design-based inference, the host must also verify the final point-weight receipt, its upstream source/sampling evidence, and the separately bound variance-design evidence rather than trusting caller-supplied labels. A numerical result is scientific evidence for accountable human interpretation, never an autonomous employment decision. +Before an approved offline validation worker executes the handoff, the Orgmetra host must re-resolve every reference inside `tenant_record_id` and prove that the predictor, criterion, population, and policy evidence belong to the exact validation study and Job. The requester and reviewer must resolve to distinct authoritative actors. For weighted design-based inference, the host must also verify the final point-weight receipt, its upstream source/sampling evidence, each typed adjustment receipt identified by the ordered chain, and the separately bound variance-design evidence rather than trusting caller-supplied labels or digests. A numerical result is scientific evidence for accountable human interpretation, never an autonomous employment decision. ## Verification From 91aee7f77ba9c70c678d3e936a1ceea6527ec882 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 05:07:29 +0900 Subject: [PATCH 063/158] docs(validity): record typed weight-adjustment evidence --- packages/validity-analysis/CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/validity-analysis/CHANGELOG.md b/packages/validity-analysis/CHANGELOG.md index cab9753e4..8fbd56089 100644 --- a/packages/validity-analysis/CHANGELOG.md +++ b/packages/validity-analysis/CHANGELOG.md @@ -12,5 +12,7 @@ - Freeze exact UTC timestamps and finite numeric values at construction, and reject runtime-type forgery before canonical evidence serialization. - Add a deterministic, row-value-minimized `FinalAnalysisWeightReceipt` that binds an exact estimand to source/sampling evidence, base-weight provenance, an ordered digest-linked adjustment chain, the final point-weight artifact, and append-only correction lineage. - Distinguish unweighted from weighted design-based results and fail closed unless a weighted result separately binds the exact final analysis-weight receipt and variance-design receipt used. -- Keep adjustment-specific nonresponse, calibration/raking, trimming/bounding, longitudinal and auxiliary-owner contracts explicitly incomplete under #407 rather than promoting the first executable weight-lineage slice to scientific completion. +- Add typed `NonresponseAdjustmentReceipt` evidence with explicit disposition treatment and typed `CalibrationAdjustmentReceipt` evidence bound to purpose-limited auxiliary projections, authoritative benchmark receipts, constraints, and explicit convergence/fallback state. +- Require known nonresponse and calibration/raking/post-stratification adjustment codes to identify the matching typed evidence kind rather than collapse into a generic opaque adjustment digest. +- Keep trimming/bounding, longitudinal eligibility, durable owner-side typed-receipt resolution, sensitive auxiliary purpose enforcement, and released auxiliary evidence exchange explicitly incomplete under #407. - Make `Validity Analysis Handoff Quality` retrigger on shared repository Python/test/clean-checkout configuration, with an executable regression preventing stale package-quality evidence after shared tooling changes. From 64d4f8b69ba4703cc53aa1eb8bd9f198d4b959d7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 05:07:50 +0900 Subject: [PATCH 064/158] docs(adr): govern nonresponse and calibration weight evidence --- ...verned-selection-validity-analysis-handoff.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/adr/0027-governed-selection-validity-analysis-handoff.md b/docs/adr/0027-governed-selection-validity-analysis-handoff.md index b0f062916..ee03f5d7f 100644 --- a/docs/adr/0027-governed-selection-validity-analysis-handoff.md +++ b/docs/adr/0027-governed-selection-validity-analysis-handoff.md @@ -30,11 +30,13 @@ Both handoff and result envelopes detach exact timezone-aware timestamps to one The same package also validates `ValidationAnalysisResult` envelopes returned by the approved offline worker. A result must link to the handoff digest and the same pinned revision, identify a Rust CPU or GPU backend and precision, provide finite effect and interval values, match its sample size to aggregate missingness counts, reject impossible complete-versus-missing count combinations, and include explicit convergence diagnostics. The canonicalization boundary accepts only the exact governed `MissingnessSummary` and `ConvergenceDiagnostics` runtime types so subclass method overrides cannot add unreviewed or person-level fields to immutable result evidence. A nonconverged result remains typed scientific evidence requiring human review; it cannot be treated as a valid selection procedure or an employment decision. -For #407's weighted design-based inference boundary, the active package now adds a first executable `FinalAnalysisWeightReceipt`. It binds the exact estimand, target population, analysis window, eligible/analytic case-set digests, #404/#405 source/sampling receipt digests, base-weight derivation evidence, an ordered immutable adjustment chain, the final point-weight artifact digest, construction time, and append-only correction lineage without storing row-level weights. Each adjustment must be contiguous and digest-linked from the previous artifact to the declared final artifact. `ValidationAnalysisResult` explicitly distinguishes `unweighted` from `weighted_design_based`; a weighted result fails closed unless it separately binds both the final analysis-weight receipt digest and the #406 variance-design receipt digest. An unweighted result cannot carry either receipt and thereby masquerade as weighted scientific evidence. +For #407's weighted design-based inference boundary, the active package adds `FinalAnalysisWeightReceipt`. It binds the exact estimand, target population, analysis window, eligible/analytic case-set digests, #404/#405 source/sampling receipt digests, base-weight derivation evidence, an ordered immutable adjustment chain, the final point-weight artifact digest, construction time, and append-only correction lineage without storing row-level weights. Each adjustment must be contiguous and digest-linked from the previous artifact to the declared final artifact. `ValidationAnalysisResult` explicitly distinguishes `unweighted` from `weighted_design_based`; a weighted result fails closed unless it separately binds both the final analysis-weight receipt digest and the #406 variance-design receipt digest. An unweighted result cannot carry either receipt and thereby masquerade as weighted scientific evidence. -This executable slice does not yet claim #407 complete. Adjustment-specific owner receipts still need executable contracts for nonresponse dispositions, calibration/raking benchmark totals and constraints, trimming/bounding rules, longitudinal eligibility, convergence/fallback semantics, and released auxiliary-owner evidence. Those remain `workforce_validation` scientific truth rather than Talent truth or cross-context SQL reconstruction. +The adjustment chain now has typed scientific evidence for two high-risk transforms. `NonresponseAdjustmentReceipt` binds a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, explicit treatment of ineligible/unknown/unavailable cases, and input/output weight artifacts. `CalibrationAdjustmentReceipt` binds the target population/window, purpose-limited auxiliary projection, authoritative benchmark receipt, algorithm/version, constraints digest, input/output artifacts, and an explicit `converged` or `fallback_applied` termination state. A fallback must identify its immutable fallback-rule reference/digest. A failed/nonconverged calibration cannot be labeled as an accepted calibration receipt. Known nonresponse and calibration/raking/post-stratification `AnalysisWeightAdjustment` codes also carry an `evidence_kind` and fail closed unless it names the matching typed receipt family. -The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, and prove requester/reviewer identities are distinct authoritative actors. +This executable slice still does not claim #407 complete. Trimming/bounding receipts, longitudinal/cross-sectional eligibility, durable service/API verification that an adjustment digest resolves to the typed receipt claimed by `evidence_kind`, sensitive auxiliary-variable purpose enforcement, and released auxiliary-owner evidence exchange remain `workforce_validation` scientific truth rather than Talent truth or cross-context SQL reconstruction. + +The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, prove requester/reviewer identities are distinct authoritative actors, and verify typed weight-adjustment receipts rather than trusting caller-supplied evidence labels or digests. ## Consequences @@ -47,6 +49,8 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - Caller-controlled timestamp and numeric runtime behavior cannot rewrite an accepted canonical digest after construction or turn malformed oversized worker output into an uncaught exception type. - Weighted design-based results can no longer identify only a sample while omitting which final point-weight evidence and variance-design evidence were actually used. - The ordered weight transformation chain is independently digest-correlatable without centralizing case-level weights or auxiliary attributes. +- Nonresponse cannot silently drop refusal/unreachable/ineligible/failure dispositions behind one opaque adjustment label. +- Calibration/raking cannot silently float benchmark ownership or hide fallback as successful convergence. - Human interpretation remains explicit and separate from numerical output. - The dedicated-writer boundary remains intact: Orgmetra consumes only a pinned foreign revision/contract boundary and never mutates fast-mlsirm. @@ -54,13 +58,13 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - This slice does not execute a statistical model, estimate validity, correct for measurement error/range restriction, evaluate adverse impact, or assert legal compliance. - Sampling design, estimator choice, missing-data treatment, reliability evidence, multiplicity, transportability, fairness analysis, and model diagnostics must be encoded in the referenced analysis plan and reviewed before execution. -- The first #407 slice validates generic ordered weight provenance and result binding; it does not yet make nonresponse, calibration/raking, trimming, longitudinal, or sensitive auxiliary-owner evidence fully executable. +- The current #407 slice validates generic ordered weight provenance, weighted-result binding, disposition-aware nonresponse evidence, and benchmark/termination-aware calibration evidence; it does not yet make trimming/bounding, longitudinal eligibility, durable typed-receipt resolution, or sensitive auxiliary-owner enforcement fully executable. - The generic result envelope does not invent an estimator-specific minimum complete-case count. Whether a converged estimator is identified under a particular missing-data design belongs to the reviewed analysis plan and numerical-worker contract; Orgmetra fails closed on impossible aggregate counts without silently replacing that foreign scientific contract with complete-case analysis. -- The package validates the result envelope, but a future execution adapter must still re-resolve the handoff references, verify the result provenance artifact, and attach evidence only after accountable human review. +- The package validates the result envelope, but a future execution adapter must still re-resolve the handoff references, verify the result provenance artifact, resolve typed adjustment digests to released owner evidence, and attach evidence only after accountable human review. ## Verification -The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. #407 adds RED/GREEN coverage for deterministic value-minimized final-weight receipts, exact sampling-receipt binding, contiguous transformation lineage, append-only correction linkage, and fail-closed weighted-result binding to separate point-weight and variance-design receipts. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. +The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. #407 adds RED/GREEN coverage for deterministic value-minimized final-weight receipts, exact sampling-receipt binding, contiguous transformation lineage, append-only correction linkage, fail-closed weighted-result binding to separate point-weight and variance-design receipts, explicit nonresponse disposition treatment, calibration benchmark/constraint/termination provenance, fallback-rule disclosure, and typed adjustment-evidence matching. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. ## References From 6b4405189edd1b01afc93f92a956f697b7ffc73c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 05:08:09 +0900 Subject: [PATCH 065/158] docs(traceability): bind typed weight-adjustment evidence --- docs/traceability/validation-analysis-handoff.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/traceability/validation-analysis-handoff.md b/docs/traceability/validation-analysis-handoff.md index 23742ac7d..da7815a76 100644 --- a/docs/traceability/validation-analysis-handoff.md +++ b/docs/traceability/validation-analysis-handoff.md @@ -15,18 +15,21 @@ Can an organization send one exact, reviewable validation study to its statistic | Scientific evidence | effect estimate, uncertainty interval, sample size, internally possible aggregate missingness, convergence diagnostics | immutable required-result-evidence regression plus impossible-missingness rejection | | Numerical result boundary | handoff digest, pinned fast-mlsirm revision, Rust CPU/GPU backend, precision, finite estimate/interval, aggregate missingness, explicit convergence state | `ValidationAnalysisResult` invariant/canonicalization regressions, exact-runtime-type checks, and oversized-numeric `ValueError` normalization | | Final point-weight provenance | exact estimand/target/window and case-set digests, source/sampling receipt digests, base-weight evidence, ordered digest-linked adjustments, final weight artifact, append-only correction lineage | `FinalAnalysisWeightReceipt` deterministic/value-minimized regressions plus chain/correction fail-closed tests | +| Nonresponse adjustment evidence | versioned response/disposition receipt, exact adjustment population, controlled method/configuration, explicit ineligible/unknown/unavailable treatment, input/output weight artifacts | `NonresponseAdjustmentReceipt` deterministic/value-minimized tests plus undocumented-treatment/no-op/version rejection | +| Calibration/raking evidence | target population/window, purpose-bound auxiliary projection, authoritative benchmark receipt, algorithm/version, constraints, input/output artifacts, explicit converged/fallback state and immutable fallback rule when used | `CalibrationAdjustmentReceipt` benchmark/termination/fallback regressions | +| Typed adjustment congruence | known nonresponse and calibration/raking/post-stratification adjustment codes must identify the matching typed receipt family through `evidence_kind` | `test_specialized_adjustments_require_matching_evidence_kind` | | Weighted-result congruence | `weighted_design_based` result must bind the exact final point-weight receipt and a separate #406 variance-design receipt; `unweighted` result must bind neither | `test_analysis_weight_result_binding.py` | | Execution boundary | `not_executed`, `scientific_evidence_only`, read-only pinned foreign dependency | immutable governance regressions | -| Reproducibility | construction-time UTC timestamp snapshots, finite numeric snapshots, canonical RFC 3339 time, canonical JSON, SHA-256 handoff/result/weight-receipt digests | mutable timezone/numeric and UTC-boundary regressions plus deterministic serialization/digest tests | +| Reproducibility | construction-time UTC timestamp snapshots, finite numeric snapshots, canonical RFC 3339 time, canonical JSON, SHA-256 handoff/result/weight/adjustment-receipt digests | mutable timezone/numeric and UTC-boundary regressions plus deterministic serialization/digest tests | | Decision-record integrity | ADR numbers remain unique repository-wide and any `docs/adr/**` change reaches the consolidated Foundation CI validity regression | ADR uniqueness regression plus Foundation CI workflow-trigger contract regression | | Quality-evidence freshness | consolidated Foundation CI runs the validity package on every `develop` pull request without a repository path filter, so shared Python/test/clean-checkout configuration cannot silently bypass the package gate | `test_foundation_ci_retriggers_without_path_filter` and `test_foundation_ci_runs_validity_analysis_and_adr_changes`; central required workflows remain separate gates | ## #407 boundary still open -The current branch implements the first executable weight-lineage slice; it does not complete #407. Nonresponse-adjustment inputs/disposition semantics, calibration/raking benchmark-owner receipts and constraints, trimming/bounding provenance, longitudinal-weight eligibility, convergence/fallback semantics, and sensitive auxiliary-variable handling still require typed executable owner evidence before a weighted result can be considered fully reproducible. `workforce_validation` owns those scientific contracts and their released result; `talent_management` must not compute or reconstruct final analysis weights. +The active branch now makes two adjustment families executable rather than leaving them as opaque digests: nonresponse adjustment evidence is disposition-aware, and calibration/raking/post-stratification evidence is bound to an authoritative benchmark, purpose-limited auxiliary projection, constraints, and explicit convergence/fallback state. This still does not complete #407. Trimming/bounding provenance, longitudinal/cross-sectional weight eligibility, durable owner-side verification that each adjustment digest resolves to the typed receipt claimed by `evidence_kind`, sensitive auxiliary-variable purpose enforcement, and released auxiliary evidence exchange remain open. `workforce_validation` owns those scientific contracts and their released result; `talent_management` must not compute or reconstruct final analysis weights. ## Maturity `implemented_on_active_pr`. -Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. The active package now validates the minimum returned numerical/provenance envelope and the first #407 final-weight/result-binding contract, but protected Orgmetra evidence still requires host re-resolution, adjustment-specific owner evidence, result-artifact verification, terminal checks, independent review, and accountable human interpretation. +Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. The active package validates the returned numerical/provenance envelope plus the current #407 final-weight/result-binding and typed adjustment-evidence contracts, but protected Orgmetra evidence still requires host re-resolution, durable owner evidence exchange, result-artifact verification, terminal checks, independent review, and accountable human interpretation. From 05a9476849e039313e4e88b7fb461d21ef40c334 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 05:08:57 +0900 Subject: [PATCH 066/158] test(validity): add RED for point-variance receipt aliasing --- .../tests/test_analysis_weight_result_binding.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/validity-analysis/tests/test_analysis_weight_result_binding.py b/packages/validity-analysis/tests/test_analysis_weight_result_binding.py index b35c91bf5..244867863 100644 --- a/packages/validity-analysis/tests/test_analysis_weight_result_binding.py +++ b/packages/validity-analysis/tests/test_analysis_weight_result_binding.py @@ -65,6 +65,16 @@ def test_weighted_result_binds_point_and_variance_receipts_separately() -> None: assert f'"variance_design_receipt_digest":"{VARIANCE_DIGEST}"' in payload +def test_weighted_result_rejects_same_point_and_variance_receipt() -> None: + """A variance-design receipt cannot silently stand in for the point-weight receipt.""" + with pytest.raises(ValueError, match="must identify different evidence"): + result( + point_estimation_mode="weighted_design_based", + analysis_weight_receipt_digest=WEIGHT_DIGEST, + variance_design_receipt_digest=WEIGHT_DIGEST, + ) + + @pytest.mark.parametrize( "overrides,match", [ From 63c346046e2c2d6032bf833de465af72239b3582 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 05:09:28 +0900 Subject: [PATCH 067/158] fix(validity): reject point-variance receipt aliasing --- .../src/orgmetra_validity_analysis/result.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py index 2cc5f121d..56ec4911d 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py @@ -221,6 +221,11 @@ def __post_init__(self) -> None: _validate_digest( self.variance_design_receipt_digest, "variance_design_receipt_digest" ) + if self.analysis_weight_receipt_digest == self.variance_design_receipt_digest: + raise ValueError( + "analysis_weight_receipt_digest and variance_design_receipt_digest " + "must identify different evidence" + ) elif ( self.analysis_weight_receipt_digest is not None or self.variance_design_receipt_digest is not None From e20208957c849c4d27865f4b6993cdbb3bddf943 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 06:01:20 +0900 Subject: [PATCH 068/158] test(validity): add trimming and bounding provenance RED --- .../tests/test_trimming_bounding_receipt.py | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 packages/validity-analysis/tests/test_trimming_bounding_receipt.py diff --git a/packages/validity-analysis/tests/test_trimming_bounding_receipt.py b/packages/validity-analysis/tests/test_trimming_bounding_receipt.py new file mode 100644 index 000000000..9fa981de6 --- /dev/null +++ b/packages/validity-analysis/tests/test_trimming_bounding_receipt.py @@ -0,0 +1,95 @@ +"""RED contracts for reproducible trimming and bounding weight adjustments.""" + +from datetime import datetime, timezone + +import pytest + +from orgmetra_validity_analysis import AnalysisWeightAdjustment, TrimmingBoundingAdjustmentReceipt + +TENANT = "10000000-0000-7000-8000-000000000001" +RECEIPT = "trimming_bounding_adjustment_receipt:11111111-1111-4111-8111-111111111111" +RULE = "weight_trimming_rule:22222222-2222-4222-8222-222222222222" +DIGEST_A = "a" * 64 +DIGEST_B = "b" * 64 +DIGEST_C = "c" * 64 +DIGEST_D = "d" * 64 + + +def trimming_receipt(**overrides: object) -> TrimmingBoundingAdjustmentReceipt: + """Return one value-minimized trimming/bounding provenance receipt.""" + values: dict[str, object] = { + "tenant_record_id": TENANT, + "receipt_reference": RECEIPT, + "rule_reference": RULE, + "rule_version": 1, + "rule_configuration_digest": DIGEST_A, + "affected_case_occurrence_set_digest": DIGEST_B, + "affected_case_count": 3, + "input_weight_artifact_digest": DIGEST_C, + "output_weight_artifact_digest": DIGEST_D, + "constructed_at": datetime(2026, 9, 17, 5, 40, tzinfo=timezone.utc), + } + values.update(overrides) + return TrimmingBoundingAdjustmentReceipt(**values) + + +def test_trimming_receipt_is_deterministic_value_minimized_and_redacted() -> None: + """Preserve threshold/rule and affected-case provenance without row weights.""" + candidate = trimming_receipt() + assert candidate.sha256_digest() == trimming_receipt().sha256_digest() + assert f'"rule_configuration_digest":"{DIGEST_A}"' in candidate.canonical_json() + assert f'"affected_case_occurrence_set_digest":"{DIGEST_B}"' in candidate.canonical_json() + assert "weight_value" not in candidate.canonical_json() + assert repr(candidate) == "TrimmingBoundingAdjustmentReceipt()" + + +def test_trimming_adjustment_requires_typed_receipt_kind() -> None: + """Do not let trimming hide behind a generic opaque adjustment receipt.""" + with pytest.raises(ValueError, match="trimming_bounding_adjustment_receipt"): + AnalysisWeightAdjustment( + sequence_number=1, + adjustment_code="weight_trimming_adjustment", + method_reference="weight_method:33333333-3333-4333-8333-333333333333", + method_version=1, + input_weight_artifact_digest=DIGEST_C, + output_weight_artifact_digest=DIGEST_D, + configuration_digest=DIGEST_A, + evidence_receipt_digest=DIGEST_B, + evidence_kind="generic_adjustment_receipt", + ) + + +def test_bounding_adjustment_accepts_typed_receipt_kind() -> None: + """Allow a bound transform only when its evidence family is explicit.""" + adjustment = AnalysisWeightAdjustment( + sequence_number=1, + adjustment_code="weight_bounding_adjustment", + method_reference="weight_method:33333333-3333-4333-8333-333333333333", + method_version=1, + input_weight_artifact_digest=DIGEST_C, + output_weight_artifact_digest=DIGEST_D, + configuration_digest=DIGEST_A, + evidence_receipt_digest=trimming_receipt().sha256_digest(), + evidence_kind="trimming_bounding_adjustment_receipt", + ) + assert adjustment.evidence_kind == "trimming_bounding_adjustment_receipt" + + +def test_receipt_rejects_missing_or_nonreproducible_rule_evidence() -> None: + """Fail closed when the rule version/configuration or affected cases are not reproducible.""" + with pytest.raises(ValueError, match="rule_version"): + trimming_receipt(rule_version=0) + with pytest.raises(ValueError, match="rule_configuration_digest"): + trimming_receipt(rule_configuration_digest="floating") + with pytest.raises(ValueError, match="affected_case_occurrence_set_digest"): + trimming_receipt(affected_case_occurrence_set_digest="missing") + with pytest.raises(ValueError, match="affected_case_count"): + trimming_receipt(affected_case_count=0) + + +def test_receipt_rejects_noop_or_extensible_schema() -> None: + """A declared transform must change artifact identity under the reviewed schema.""" + with pytest.raises(ValueError, match="output_weight_artifact_digest"): + trimming_receipt(output_weight_artifact_digest=DIGEST_C) + with pytest.raises(ValueError, match="evidence_version"): + trimming_receipt(evidence_version=2) From 87b284dd5c9cb3efe94c76872ee0ff4c152f6d91 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 06:02:35 +0900 Subject: [PATCH 069/158] feat(validity): bind trimming and bounding weight provenance --- .../src/orgmetra_validity_analysis/weights.py | 75 ++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py b/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py index a0597da76..a556b7939 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py @@ -26,6 +26,9 @@ "calibration_adjustment": "calibration_adjustment_receipt", "raking_adjustment": "calibration_adjustment_receipt", "poststratification_adjustment": "calibration_adjustment_receipt", + "weight_trimming_adjustment": "trimming_bounding_adjustment_receipt", + "weight_bounding_adjustment": "trimming_bounding_adjustment_receipt", + "weight_winsorization_adjustment": "trimming_bounding_adjustment_receipt", } @@ -238,6 +241,75 @@ def sha256_digest(self) -> str: return sha256(self.canonical_json().encode("utf-8")).hexdigest() +@dataclass(frozen=True, slots=True, repr=False) +class TrimmingBoundingAdjustmentReceipt: + """Bind trimming or bounding to an immutable rule and affected-case evidence.""" + + tenant_record_id: str + receipt_reference: str + rule_reference: str + rule_version: int + rule_configuration_digest: str + affected_case_occurrence_set_digest: str + affected_case_count: int + input_weight_artifact_digest: str + output_weight_artifact_digest: str + constructed_at: datetime + evidence_version: int = 1 + + def __post_init__(self) -> None: + """Reject hidden thresholds, unknown affected cases, or no-op transforms.""" + _validate_operational_uuid(self.tenant_record_id, "tenant_record_id") + _validate_reference( + self.receipt_reference, + "trimming_bounding_adjustment_receipt", + "receipt_reference", + ) + _validate_reference(self.rule_reference, "weight_trimming_rule", "rule_reference") + _positive_integer(self.rule_version, "rule_version") + _validate_digest(self.rule_configuration_digest, "rule_configuration_digest") + _validate_digest( + self.affected_case_occurrence_set_digest, + "affected_case_occurrence_set_digest", + ) + _positive_integer(self.affected_case_count, "affected_case_count") + _validate_digest(self.input_weight_artifact_digest, "input_weight_artifact_digest") + _validate_digest(self.output_weight_artifact_digest, "output_weight_artifact_digest") + if self.input_weight_artifact_digest == self.output_weight_artifact_digest: + raise ValueError( + "output_weight_artifact_digest must identify the trimmed or bounded weight artifact" + ) + constructed_at = _freeze_timestamp(self.constructed_at, "constructed_at") + if type(self.evidence_version) is not int or self.evidence_version != 1: + raise ValueError("evidence_version must remain 1") + object.__setattr__(self, "constructed_at", constructed_at) + + def __repr__(self) -> str: + """Return a value-minimized representation for routine logs.""" + return "TrimmingBoundingAdjustmentReceipt()" + + def canonical_json(self) -> str: + """Return deterministic rule provenance without case-level weight values.""" + payload = { + "affected_case_count": self.affected_case_count, + "affected_case_occurrence_set_digest": self.affected_case_occurrence_set_digest, + "constructed_at": _canonical_timestamp(self.constructed_at, "constructed_at"), + "evidence_version": self.evidence_version, + "input_weight_artifact_digest": self.input_weight_artifact_digest, + "output_weight_artifact_digest": self.output_weight_artifact_digest, + "receipt_reference": self.receipt_reference, + "rule_configuration_digest": self.rule_configuration_digest, + "rule_reference": self.rule_reference, + "rule_version": self.rule_version, + "tenant_record_id": self.tenant_record_id, + } + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + def sha256_digest(self) -> str: + """Return SHA-256 over the exact canonical receipt bytes.""" + return sha256(self.canonical_json().encode("utf-8")).hexdigest() + + @dataclass(frozen=True, slots=True) class AnalysisWeightAdjustment: """Describe one ordered, digest-linked transformation of analysis weights.""" @@ -430,4 +502,5 @@ def sha256_digest(self) -> str: "CalibrationAdjustmentReceipt", "FinalAnalysisWeightReceipt", "NonresponseAdjustmentReceipt", -] + "TrimmingBoundingAdjustmentReceipt", +] \ No newline at end of file From 4d8a375997c4d9ea5f3cc642942d15811ae4b938 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 06:02:43 +0900 Subject: [PATCH 070/158] feat(validity): export trimming and bounding receipt --- .../src/orgmetra_validity_analysis/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py b/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py index 794380594..70249e971 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py @@ -11,6 +11,7 @@ CalibrationAdjustmentReceipt, FinalAnalysisWeightReceipt, NonresponseAdjustmentReceipt, + TrimmingBoundingAdjustmentReceipt, ) __all__ = [ @@ -24,4 +25,5 @@ "CalibrationAdjustmentReceipt", "FinalAnalysisWeightReceipt", "NonresponseAdjustmentReceipt", -] + "TrimmingBoundingAdjustmentReceipt", +] \ No newline at end of file From 12b2abcfb8d66101ff2f928721b29b6243012ae8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 06:03:57 +0900 Subject: [PATCH 071/158] docs(validity): document trimming and bounding provenance --- packages/validity-analysis/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/validity-analysis/README.md b/packages/validity-analysis/README.md index 0f99839ee..c662627a5 100644 --- a/packages/validity-analysis/README.md +++ b/packages/validity-analysis/README.md @@ -12,7 +12,9 @@ The resulting canonical JSON is digest-addressable, contains no raw person-level Nonresponse and calibration are no longer allowed to hide behind an undifferentiated adjustment digest. `NonresponseAdjustmentReceipt` binds the adjustment to a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, and explicit treatment codes for ineligible, unknown, and unavailable cases. `CalibrationAdjustmentReceipt` binds the adjustment to a target population/window, purpose-bound auxiliary projection, authoritative benchmark receipt, algorithm/version, constraints digest, and explicit `converged` or `fallback_applied` termination evidence. A fallback must identify its versioned rule; a nonconverged calibration cannot masquerade as an accepted calibration receipt. -`AnalysisWeightAdjustment` records an `evidence_kind` in addition to the evidence digest. Known nonresponse and calibration/raking/post-stratification transforms fail closed unless their evidence kind is the corresponding typed receipt. Other adjustment families remain open work rather than being silently treated as equivalent. +Trimming, bounding, and winsorization now have a separate provenance family rather than falling back to a generic adjustment label. `TrimmingBoundingAdjustmentReceipt` binds the exact versioned rule, its reproducible configuration digest, the semantic occurrence set and count of cases actually affected, and the input/output weight artifacts. A declared trim/bound transform must change artifact identity. Known trimming/bounding/winsorization adjustment codes fail closed unless `evidence_kind` names this typed receipt family. + +`AnalysisWeightAdjustment` records an `evidence_kind` in addition to the evidence digest. Known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization transforms fail closed unless their evidence kind is the corresponding typed receipt. Other adjustment families remain open work rather than being silently treated as equivalent. `ValidationAnalysisResult` accepts only a result linked to the handoff digest and the same reviewed fast-mlsirm revision. It records the Rust CPU/GPU backend, precision, aggregate missingness counts, finite effect and interval values, and explicit convergence or nonconvergence diagnostics. Timestamps and finite numeric values are snapshotted before canonicalization, and the result envelope accepts only the exact governed `MissingnessSummary` and `ConvergenceDiagnostics` runtime types, preventing mutable runtime values or subclass method overrides from adding unreviewed or person-level fields to canonical audit evidence. Missingness counts must be internally possible: complete observations cannot overlap either predictor-missing or criterion-missing observations beyond the declared sample total. @@ -26,7 +28,7 @@ Point-estimation semantics are explicit. `unweighted` results cannot carry weigh - It does **not** claim that a selection procedure is valid. - It does **not** interpret adverse impact. - It does **not** authorize hiring, promotion, termination, compensation, or another employment decision. -- It does **not** yet complete #407. Trimming/bounding receipts, longitudinal/cross-sectional eligibility, owner-side verification that an adjustment digest resolves to the typed receipt claimed by `evidence_kind`, sensitive auxiliary-variable purpose enforcement at the durable service/API boundary, and released auxiliary evidence exchange remain open work in `workforce_validation`. +- It does **not** yet complete #407. Longitudinal/cross-sectional weight eligibility, owner-side verification that an adjustment digest resolves to the typed receipt claimed by `evidence_kind`, sensitive auxiliary-variable purpose enforcement at the durable service/API boundary, and released auxiliary evidence exchange remain open work in `workforce_validation`. The fast-mlsirm repository remains a dedicated-writer dependency. This package records only the immutable revision reviewed for the handoff. Durable Workforce Validation registry/API/persistence remains on the canonical #235/#248 owner path; this package does not create a parallel service. @@ -43,4 +45,4 @@ PYTHONPATH=packages/validity-analysis/src \ python -m pytest -c packages/validity-analysis/pyproject.toml packages/validity-analysis/tests ``` -The package gate requires exact 100% owned production statement and branch coverage. +The package gate requires exact 100% owned production statement and branch coverage. \ No newline at end of file From baf0fb0a2e39c97846383d272e5e68e50aaf2b65 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 06:04:19 +0900 Subject: [PATCH 072/158] docs(adr): record trimming and bounding weight evidence --- ...27-governed-selection-validity-analysis-handoff.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/adr/0027-governed-selection-validity-analysis-handoff.md b/docs/adr/0027-governed-selection-validity-analysis-handoff.md index ee03f5d7f..3d2e3af08 100644 --- a/docs/adr/0027-governed-selection-validity-analysis-handoff.md +++ b/docs/adr/0027-governed-selection-validity-analysis-handoff.md @@ -32,9 +32,9 @@ The same package also validates `ValidationAnalysisResult` envelopes returned by For #407's weighted design-based inference boundary, the active package adds `FinalAnalysisWeightReceipt`. It binds the exact estimand, target population, analysis window, eligible/analytic case-set digests, #404/#405 source/sampling receipt digests, base-weight derivation evidence, an ordered immutable adjustment chain, the final point-weight artifact digest, construction time, and append-only correction lineage without storing row-level weights. Each adjustment must be contiguous and digest-linked from the previous artifact to the declared final artifact. `ValidationAnalysisResult` explicitly distinguishes `unweighted` from `weighted_design_based`; a weighted result fails closed unless it separately binds both the final analysis-weight receipt digest and the #406 variance-design receipt digest. An unweighted result cannot carry either receipt and thereby masquerade as weighted scientific evidence. -The adjustment chain now has typed scientific evidence for two high-risk transforms. `NonresponseAdjustmentReceipt` binds a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, explicit treatment of ineligible/unknown/unavailable cases, and input/output weight artifacts. `CalibrationAdjustmentReceipt` binds the target population/window, purpose-limited auxiliary projection, authoritative benchmark receipt, algorithm/version, constraints digest, input/output artifacts, and an explicit `converged` or `fallback_applied` termination state. A fallback must identify its immutable fallback-rule reference/digest. A failed/nonconverged calibration cannot be labeled as an accepted calibration receipt. Known nonresponse and calibration/raking/post-stratification `AnalysisWeightAdjustment` codes also carry an `evidence_kind` and fail closed unless it names the matching typed receipt family. +The adjustment chain has typed scientific evidence for three high-risk transform families. `NonresponseAdjustmentReceipt` binds a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, explicit treatment of ineligible/unknown/unavailable cases, and input/output weight artifacts. `CalibrationAdjustmentReceipt` binds the target population/window, purpose-limited auxiliary projection, authoritative benchmark receipt, algorithm/version, constraints digest, input/output artifacts, and an explicit `converged` or `fallback_applied` termination state. A fallback must identify its immutable fallback-rule reference/digest. A failed/nonconverged calibration cannot be labeled as an accepted calibration receipt. `TrimmingBoundingAdjustmentReceipt` binds a versioned trimming/bounding rule, reproducible rule configuration digest, the semantic occurrence set and count of cases actually affected, and input/output weight artifacts; a declared trim/bound transform cannot be a no-op. Known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization `AnalysisWeightAdjustment` codes also carry an `evidence_kind` and fail closed unless it names the matching typed receipt family. -This executable slice still does not claim #407 complete. Trimming/bounding receipts, longitudinal/cross-sectional eligibility, durable service/API verification that an adjustment digest resolves to the typed receipt claimed by `evidence_kind`, sensitive auxiliary-variable purpose enforcement, and released auxiliary-owner evidence exchange remain `workforce_validation` scientific truth rather than Talent truth or cross-context SQL reconstruction. +This executable slice still does not claim #407 complete. Longitudinal/cross-sectional eligibility, durable service/API verification that an adjustment digest resolves to the typed receipt claimed by `evidence_kind`, sensitive auxiliary-variable purpose enforcement, and released auxiliary-owner evidence exchange remain `workforce_validation` scientific truth rather than Talent truth or cross-context SQL reconstruction. The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, prove requester/reviewer identities are distinct authoritative actors, and verify typed weight-adjustment receipts rather than trusting caller-supplied evidence labels or digests. @@ -51,6 +51,7 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - The ordered weight transformation chain is independently digest-correlatable without centralizing case-level weights or auxiliary attributes. - Nonresponse cannot silently drop refusal/unreachable/ineligible/failure dispositions behind one opaque adjustment label. - Calibration/raking cannot silently float benchmark ownership or hide fallback as successful convergence. +- Trimming/bounding/winsorization cannot silently alter final point weights without an immutable rule/configuration and affected-case receipt. - Human interpretation remains explicit and separate from numerical output. - The dedicated-writer boundary remains intact: Orgmetra consumes only a pinned foreign revision/contract boundary and never mutates fast-mlsirm. @@ -58,14 +59,14 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - This slice does not execute a statistical model, estimate validity, correct for measurement error/range restriction, evaluate adverse impact, or assert legal compliance. - Sampling design, estimator choice, missing-data treatment, reliability evidence, multiplicity, transportability, fairness analysis, and model diagnostics must be encoded in the referenced analysis plan and reviewed before execution. -- The current #407 slice validates generic ordered weight provenance, weighted-result binding, disposition-aware nonresponse evidence, and benchmark/termination-aware calibration evidence; it does not yet make trimming/bounding, longitudinal eligibility, durable typed-receipt resolution, or sensitive auxiliary-owner enforcement fully executable. +- The current #407 slice validates generic ordered weight provenance, weighted-result binding, disposition-aware nonresponse evidence, benchmark/termination-aware calibration evidence, and trimming/bounding rule provenance; it does not yet make longitudinal eligibility, durable typed-receipt resolution, or sensitive auxiliary-owner enforcement fully executable. - The generic result envelope does not invent an estimator-specific minimum complete-case count. Whether a converged estimator is identified under a particular missing-data design belongs to the reviewed analysis plan and numerical-worker contract; Orgmetra fails closed on impossible aggregate counts without silently replacing that foreign scientific contract with complete-case analysis. - The package validates the result envelope, but a future execution adapter must still re-resolve the handoff references, verify the result provenance artifact, resolve typed adjustment digests to released owner evidence, and attach evidence only after accountable human review. ## Verification -The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. #407 adds RED/GREEN coverage for deterministic value-minimized final-weight receipts, exact sampling-receipt binding, contiguous transformation lineage, append-only correction linkage, fail-closed weighted-result binding to separate point-weight and variance-design receipts, explicit nonresponse disposition treatment, calibration benchmark/constraint/termination provenance, fallback-rule disclosure, and typed adjustment-evidence matching. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. +The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. #407 adds RED/GREEN coverage for deterministic value-minimized final-weight receipts, exact sampling-receipt binding, contiguous transformation lineage, append-only correction linkage, fail-closed weighted-result binding to separate point-weight and variance-design receipts, explicit nonresponse disposition treatment, calibration benchmark/constraint/termination provenance, fallback-rule disclosure, trimming/bounding rule/configuration and affected-case provenance, no-op trim/bound rejection, and typed adjustment-evidence matching. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. ## References -See `docs/doctoring/validation-analysis-handoff-references.md`. +See `docs/doctoring/validation-analysis-handoff-references.md`. \ No newline at end of file From fd8ea5b59a439f6a72da106b870c40a139292498 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 06:04:32 +0900 Subject: [PATCH 073/158] docs(traceability): add trimming and bounding evidence --- docs/traceability/validation-analysis-handoff.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/traceability/validation-analysis-handoff.md b/docs/traceability/validation-analysis-handoff.md index da7815a76..da34a2cbb 100644 --- a/docs/traceability/validation-analysis-handoff.md +++ b/docs/traceability/validation-analysis-handoff.md @@ -17,7 +17,8 @@ Can an organization send one exact, reviewable validation study to its statistic | Final point-weight provenance | exact estimand/target/window and case-set digests, source/sampling receipt digests, base-weight evidence, ordered digest-linked adjustments, final weight artifact, append-only correction lineage | `FinalAnalysisWeightReceipt` deterministic/value-minimized regressions plus chain/correction fail-closed tests | | Nonresponse adjustment evidence | versioned response/disposition receipt, exact adjustment population, controlled method/configuration, explicit ineligible/unknown/unavailable treatment, input/output weight artifacts | `NonresponseAdjustmentReceipt` deterministic/value-minimized tests plus undocumented-treatment/no-op/version rejection | | Calibration/raking evidence | target population/window, purpose-bound auxiliary projection, authoritative benchmark receipt, algorithm/version, constraints, input/output artifacts, explicit converged/fallback state and immutable fallback rule when used | `CalibrationAdjustmentReceipt` benchmark/termination/fallback regressions | -| Typed adjustment congruence | known nonresponse and calibration/raking/post-stratification adjustment codes must identify the matching typed receipt family through `evidence_kind` | `test_specialized_adjustments_require_matching_evidence_kind` | +| Trimming/bounding evidence | versioned trimming/bounding rule, reproducible rule configuration digest, exact affected semantic occurrence set and count, input/output weight artifacts | `test_trimming_bounding_receipt.py` deterministic/value-minimized, no-op, missing-provenance, and typed-kind regressions | +| Typed adjustment congruence | known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization adjustment codes must identify the matching typed receipt family through `evidence_kind` | specialized adjustment evidence-kind regressions | | Weighted-result congruence | `weighted_design_based` result must bind the exact final point-weight receipt and a separate #406 variance-design receipt; `unweighted` result must bind neither | `test_analysis_weight_result_binding.py` | | Execution boundary | `not_executed`, `scientific_evidence_only`, read-only pinned foreign dependency | immutable governance regressions | | Reproducibility | construction-time UTC timestamp snapshots, finite numeric snapshots, canonical RFC 3339 time, canonical JSON, SHA-256 handoff/result/weight/adjustment-receipt digests | mutable timezone/numeric and UTC-boundary regressions plus deterministic serialization/digest tests | @@ -26,10 +27,10 @@ Can an organization send one exact, reviewable validation study to its statistic ## #407 boundary still open -The active branch now makes two adjustment families executable rather than leaving them as opaque digests: nonresponse adjustment evidence is disposition-aware, and calibration/raking/post-stratification evidence is bound to an authoritative benchmark, purpose-limited auxiliary projection, constraints, and explicit convergence/fallback state. This still does not complete #407. Trimming/bounding provenance, longitudinal/cross-sectional weight eligibility, durable owner-side verification that each adjustment digest resolves to the typed receipt claimed by `evidence_kind`, sensitive auxiliary-variable purpose enforcement, and released auxiliary evidence exchange remain open. `workforce_validation` owns those scientific contracts and their released result; `talent_management` must not compute or reconstruct final analysis weights. +The active branch now makes three adjustment families executable rather than leaving them as opaque digests: nonresponse adjustment evidence is disposition-aware; calibration/raking/post-stratification evidence is bound to an authoritative benchmark, purpose-limited auxiliary projection, constraints, and explicit convergence/fallback state; and trimming/bounding/winsorization evidence is bound to a versioned rule/configuration plus the exact affected occurrence set. This still does not complete #407. Longitudinal/cross-sectional weight eligibility, durable owner-side verification that each adjustment digest resolves to the typed receipt claimed by `evidence_kind`, sensitive auxiliary-variable purpose enforcement, and released auxiliary evidence exchange remain open. `workforce_validation` owns those scientific contracts and their released result; `talent_management` must not compute or reconstruct final analysis weights. ## Maturity `implemented_on_active_pr`. -Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. The active package validates the returned numerical/provenance envelope plus the current #407 final-weight/result-binding and typed adjustment-evidence contracts, but protected Orgmetra evidence still requires host re-resolution, durable owner evidence exchange, result-artifact verification, terminal checks, independent review, and accountable human interpretation. +Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. The active package validates the returned numerical/provenance envelope plus the current #407 final-weight/result-binding and typed adjustment-evidence contracts, but protected Orgmetra evidence still requires host re-resolution, durable owner evidence exchange, result-artifact verification, terminal checks, independent review, and accountable human interpretation. \ No newline at end of file From 32934e78bf1f2f01514dbdf1002ada3b4faa12aa Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 06:04:44 +0900 Subject: [PATCH 074/158] docs(validity): record trimming and bounding receipt slice --- packages/validity-analysis/CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/validity-analysis/CHANGELOG.md b/packages/validity-analysis/CHANGELOG.md index 8fbd56089..7ab1f182e 100644 --- a/packages/validity-analysis/CHANGELOG.md +++ b/packages/validity-analysis/CHANGELOG.md @@ -13,6 +13,7 @@ - Add a deterministic, row-value-minimized `FinalAnalysisWeightReceipt` that binds an exact estimand to source/sampling evidence, base-weight provenance, an ordered digest-linked adjustment chain, the final point-weight artifact, and append-only correction lineage. - Distinguish unweighted from weighted design-based results and fail closed unless a weighted result separately binds the exact final analysis-weight receipt and variance-design receipt used. - Add typed `NonresponseAdjustmentReceipt` evidence with explicit disposition treatment and typed `CalibrationAdjustmentReceipt` evidence bound to purpose-limited auxiliary projections, authoritative benchmark receipts, constraints, and explicit convergence/fallback state. -- Require known nonresponse and calibration/raking/post-stratification adjustment codes to identify the matching typed evidence kind rather than collapse into a generic opaque adjustment digest. -- Keep trimming/bounding, longitudinal eligibility, durable owner-side typed-receipt resolution, sensitive auxiliary purpose enforcement, and released auxiliary evidence exchange explicitly incomplete under #407. -- Make `Validity Analysis Handoff Quality` retrigger on shared repository Python/test/clean-checkout configuration, with an executable regression preventing stale package-quality evidence after shared tooling changes. +- Add typed `TrimmingBoundingAdjustmentReceipt` evidence that binds a versioned rule/configuration, exact affected semantic occurrence set and count, and input/output weight artifacts; known trimming/bounding/winsorization adjustments must name that evidence family. +- Require known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization adjustment codes to identify the matching typed evidence kind rather than collapse into a generic opaque adjustment digest. +- Keep longitudinal eligibility, durable owner-side typed-receipt resolution, sensitive auxiliary purpose enforcement, and released auxiliary evidence exchange explicitly incomplete under #407. +- Make `Validity Analysis Handoff Quality` retrigger on shared repository Python/test/clean-checkout configuration, with an executable regression preventing stale package-quality evidence after shared tooling changes. \ No newline at end of file From f71669c4880de1231c8e522e26897ceda4aa98ce Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 07:03:57 +0900 Subject: [PATCH 075/158] test(validity): add RED weight eligibility receipt contract --- .../tests/test_weight_eligibility_receipt.py | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 packages/validity-analysis/tests/test_weight_eligibility_receipt.py diff --git a/packages/validity-analysis/tests/test_weight_eligibility_receipt.py b/packages/validity-analysis/tests/test_weight_eligibility_receipt.py new file mode 100644 index 000000000..45017db25 --- /dev/null +++ b/packages/validity-analysis/tests/test_weight_eligibility_receipt.py @@ -0,0 +1,70 @@ +"""Regression contracts for target-population and duration-safe weight eligibility.""" + +from datetime import datetime, timezone + +import pytest + +from orgmetra_validity_analysis import WeightEligibilityReceipt + +TENANT = "10000000-0000-7000-8000-000000000001" +RECEIPT = "weight_eligibility_receipt:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" +TARGET = "analysis_target_population:33333333-3333-4333-8333-333333333333" +DURATION = "analysis_reference_duration:77777777-7777-4777-8777-777777777777" +DIGEST_A = "a" * 64 +DIGEST_B = "b" * 64 +DIGEST_C = "c" * 64 +DIGEST_D = "d" * 64 + + +def eligibility(**overrides: object) -> WeightEligibilityReceipt: + """Return one exact longitudinal/cross-sectional weight eligibility receipt.""" + values: dict[str, object] = { + "tenant_record_id": TENANT, + "receipt_reference": RECEIPT, + "weight_scope_code": "cross_sectional", + "target_population_reference": TARGET, + "target_population_digest": DIGEST_A, + "reference_duration_reference": DURATION, + "reference_duration_digest": DIGEST_B, + "eligible_case_set_digest": DIGEST_C, + "weight_artifact_digest": DIGEST_D, + "constructed_at": datetime(2026, 9, 17, 7, 0, tzinfo=timezone.utc), + } + values.update(overrides) + return WeightEligibilityReceipt(**values) + + +def test_weight_eligibility_receipt_is_deterministic_and_value_minimized() -> None: + """Bind weight scope, target population, duration, and eligible cases without row values.""" + candidate = eligibility() + assert candidate.sha256_digest() == eligibility().sha256_digest() + assert '"weight_scope_code":"cross_sectional"' in candidate.canonical_json() + assert "weight_value" not in candidate.canonical_json() + assert repr(candidate) == "WeightEligibilityReceipt()" + + +@pytest.mark.parametrize("scope", ["monthly", "panel", "opaque", ""]) +def test_weight_scope_is_closed_to_cross_sectional_or_longitudinal(scope: str) -> None: + """Reject caller-defined labels that hide longitudinal/cross-sectional semantics.""" + with pytest.raises(ValueError, match="weight_scope_code"): + eligibility(weight_scope_code=scope) + + +def test_weight_eligibility_requires_versioned_population_duration_and_case_evidence() -> None: + """Reject mutable or opaque population, duration, case-set, and artifact identities.""" + with pytest.raises(ValueError, match="target_population_digest"): + eligibility(target_population_digest="target-v1") + with pytest.raises(ValueError, match="reference_duration_reference"): + eligibility(reference_duration_reference="duration-v1") + with pytest.raises(ValueError, match="reference_duration_digest"): + eligibility(reference_duration_digest="duration-v1") + with pytest.raises(ValueError, match="eligible_case_set_digest"): + eligibility(eligible_case_set_digest="cases-v1") + with pytest.raises(ValueError, match="weight_artifact_digest"): + eligibility(weight_artifact_digest="weight-v1") + + +def test_evidence_version_is_not_caller_extensible() -> None: + """Prevent ad hoc receipt schemas from bypassing reviewed weight eligibility semantics.""" + with pytest.raises(ValueError, match="evidence_version"): + eligibility(evidence_version=2) From 4201a46717d09e7ed76ebd72b5768c699196a8b3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 07:05:08 +0900 Subject: [PATCH 076/158] feat(validity): bind weight eligibility to estimand scope --- .../src/orgmetra_validity_analysis/weights.py | 124 +++++++++++++++++- 1 file changed, 123 insertions(+), 1 deletion(-) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py b/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py index a556b7939..025f33e7c 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py @@ -21,6 +21,7 @@ ) _CALIBRATION_TERMINATION_CODES = frozenset({"converged", "fallback_applied"}) +_WEIGHT_SCOPE_CODES = frozenset({"cross_sectional", "longitudinal"}) _SPECIALIZED_EVIDENCE_KIND_BY_ADJUSTMENT_CODE = { "nonresponse_adjustment": "nonresponse_adjustment_receipt", "calibration_adjustment": "calibration_adjustment_receipt", @@ -310,6 +311,83 @@ def sha256_digest(self) -> str: return sha256(self.canonical_json().encode("utf-8")).hexdigest() +@dataclass(frozen=True, slots=True, repr=False) +class WeightEligibilityReceipt: + """Bind one final weight artifact to its valid population and reference duration.""" + + tenant_record_id: str + receipt_reference: str + weight_scope_code: str + target_population_reference: str + target_population_digest: str + reference_duration_reference: str + reference_duration_digest: str + eligible_case_set_digest: str + weight_artifact_digest: str + constructed_at: datetime + evidence_version: int = 1 + + def __post_init__(self) -> None: + """Reject ambiguous cross-sectional or longitudinal weight eligibility.""" + _validate_operational_uuid(self.tenant_record_id, "tenant_record_id") + _validate_reference( + self.receipt_reference, + "weight_eligibility_receipt", + "receipt_reference", + ) + if ( + type(self.weight_scope_code) is not str + or self.weight_scope_code not in _WEIGHT_SCOPE_CODES + ): + raise ValueError("weight_scope_code must be cross_sectional or longitudinal") + _validate_reference( + self.target_population_reference, + "analysis_target_population", + "target_population_reference", + ) + _validate_digest(self.target_population_digest, "target_population_digest") + _validate_reference( + self.reference_duration_reference, + "analysis_reference_duration", + "reference_duration_reference", + ) + for field_name in ( + "reference_duration_digest", + "eligible_case_set_digest", + "weight_artifact_digest", + ): + _validate_digest(getattr(self, field_name), field_name) + constructed_at = _freeze_timestamp(self.constructed_at, "constructed_at") + if type(self.evidence_version) is not int or self.evidence_version != 1: + raise ValueError("evidence_version must remain 1") + object.__setattr__(self, "constructed_at", constructed_at) + + def __repr__(self) -> str: + """Return a value-minimized representation for routine logs.""" + return "WeightEligibilityReceipt()" + + def canonical_json(self) -> str: + """Return deterministic eligibility provenance without row-level weight values.""" + payload = { + "constructed_at": _canonical_timestamp(self.constructed_at, "constructed_at"), + "eligible_case_set_digest": self.eligible_case_set_digest, + "evidence_version": self.evidence_version, + "receipt_reference": self.receipt_reference, + "reference_duration_digest": self.reference_duration_digest, + "reference_duration_reference": self.reference_duration_reference, + "target_population_digest": self.target_population_digest, + "target_population_reference": self.target_population_reference, + "tenant_record_id": self.tenant_record_id, + "weight_artifact_digest": self.weight_artifact_digest, + "weight_scope_code": self.weight_scope_code, + } + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + def sha256_digest(self) -> str: + """Return SHA-256 over the exact canonical receipt bytes.""" + return sha256(self.canonical_json().encode("utf-8")).hexdigest() + + @dataclass(frozen=True, slots=True) class AnalysisWeightAdjustment: """Describe one ordered, digest-linked transformation of analysis weights.""" @@ -373,10 +451,13 @@ class FinalAnalysisWeightReceipt: receipt_reference: str estimand_reference: str estimand_digest: str + estimand_scope_code: str target_population_reference: str target_population_digest: str analysis_unit_code: str analysis_window_reference: str + reference_duration_reference: str + reference_duration_digest: str eligible_case_set_digest: str analytic_case_occurrence_set_digest: str source_universe_receipt_digest: str @@ -387,6 +468,7 @@ class FinalAnalysisWeightReceipt: base_weight_artifact_digest: str adjustments: tuple[AnalysisWeightAdjustment, ...] final_weight_artifact_digest: str + weight_eligibility: WeightEligibilityReceipt analytic_case_count: int constructed_at: datetime correction_sequence: int = 1 @@ -401,6 +483,11 @@ def __post_init__(self) -> None: ) _validate_reference(self.estimand_reference, "validation_estimand", "estimand_reference") _validate_digest(self.estimand_digest, "estimand_digest") + if ( + type(self.estimand_scope_code) is not str + or self.estimand_scope_code not in _WEIGHT_SCOPE_CODES + ): + raise ValueError("estimand_scope_code must be cross_sectional or longitudinal") _validate_reference( self.target_population_reference, "analysis_target_population", @@ -411,6 +498,12 @@ def __post_init__(self) -> None: _validate_reference( self.analysis_window_reference, "analysis_window", "analysis_window_reference" ) + _validate_reference( + self.reference_duration_reference, + "analysis_reference_duration", + "reference_duration_reference", + ) + _validate_digest(self.reference_duration_digest, "reference_duration_digest") for field_name in ( "eligible_case_set_digest", "analytic_case_occurrence_set_digest", @@ -425,6 +518,26 @@ def __post_init__(self) -> None: _positive_integer(self.base_weight_method_version, "base_weight_method_version") _positive_integer(self.analytic_case_count, "analytic_case_count") constructed_at = _freeze_timestamp(self.constructed_at, "constructed_at") + if type(self.weight_eligibility) is not WeightEligibilityReceipt: + raise ValueError("weight_eligibility must be a WeightEligibilityReceipt") + if self.weight_eligibility.tenant_record_id != self.tenant_record_id: + raise ValueError("weight_eligibility tenant_record_id must match the analysis receipt") + if self.weight_eligibility.weight_scope_code != self.estimand_scope_code: + raise ValueError("weight scope must match estimand_scope_code") + if ( + self.weight_eligibility.target_population_reference + != self.target_population_reference + or self.weight_eligibility.target_population_digest != self.target_population_digest + ): + raise ValueError("weight target population must match the estimand target population") + if ( + self.weight_eligibility.reference_duration_reference + != self.reference_duration_reference + or self.weight_eligibility.reference_duration_digest != self.reference_duration_digest + ): + raise ValueError("weight reference duration must match the estimand reference duration") + if self.weight_eligibility.eligible_case_set_digest != self.eligible_case_set_digest: + raise ValueError("weight eligible case set must match the analysis eligible case set") if type(self.adjustments) is not tuple: raise ValueError("adjustments must be an immutable tuple") @@ -441,6 +554,10 @@ def __post_init__(self) -> None: raise ValueError( "final_weight_artifact_digest must equal the ordered adjustment chain output" ) + if self.weight_eligibility.weight_artifact_digest != self.final_weight_artifact_digest: + raise ValueError( + "weight eligibility must identify the final point-estimation weight artifact" + ) _positive_integer(self.correction_sequence, "correction_sequence") if self.correction_sequence == 1: @@ -479,14 +596,18 @@ def canonical_json(self) -> str: "eligible_case_set_digest": self.eligible_case_set_digest, "estimand_digest": self.estimand_digest, "estimand_reference": self.estimand_reference, + "estimand_scope_code": self.estimand_scope_code, "evidence_version": self.evidence_version, "final_weight_artifact_digest": self.final_weight_artifact_digest, "receipt_reference": self.receipt_reference, + "reference_duration_digest": self.reference_duration_digest, + "reference_duration_reference": self.reference_duration_reference, "sampling_design_receipt_digest": self.sampling_design_receipt_digest, "source_universe_receipt_digest": self.source_universe_receipt_digest, "target_population_digest": self.target_population_digest, "target_population_reference": self.target_population_reference, "tenant_record_id": self.tenant_record_id, + "weight_eligibility_receipt_digest": self.weight_eligibility.sha256_digest(), } if self.supersedes_receipt_digest is not None: payload["supersedes_receipt_digest"] = self.supersedes_receipt_digest @@ -503,4 +624,5 @@ def sha256_digest(self) -> str: "FinalAnalysisWeightReceipt", "NonresponseAdjustmentReceipt", "TrimmingBoundingAdjustmentReceipt", -] \ No newline at end of file + "WeightEligibilityReceipt", +] From 7afa6ce3473bb2a178d55e1704e848d266d43a30 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 07:05:18 +0900 Subject: [PATCH 077/158] feat(validity): export weight eligibility receipt --- .../src/orgmetra_validity_analysis/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py b/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py index 70249e971..1a64d81cc 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py @@ -12,6 +12,7 @@ FinalAnalysisWeightReceipt, NonresponseAdjustmentReceipt, TrimmingBoundingAdjustmentReceipt, + WeightEligibilityReceipt, ) __all__ = [ @@ -26,4 +27,5 @@ "FinalAnalysisWeightReceipt", "NonresponseAdjustmentReceipt", "TrimmingBoundingAdjustmentReceipt", -] \ No newline at end of file + "WeightEligibilityReceipt", +] From c500f17d8898a7d1e8d7428a73334db4ba3757f2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 07:05:44 +0900 Subject: [PATCH 078/158] test(validity): reject weight estimand eligibility mismatch --- .../tests/test_analysis_weight_receipt.py | 62 ++++++++++++++++++- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/packages/validity-analysis/tests/test_analysis_weight_receipt.py b/packages/validity-analysis/tests/test_analysis_weight_receipt.py index 9603d7c58..c8b022178 100644 --- a/packages/validity-analysis/tests/test_analysis_weight_receipt.py +++ b/packages/validity-analysis/tests/test_analysis_weight_receipt.py @@ -4,13 +4,19 @@ import pytest -from orgmetra_validity_analysis import AnalysisWeightAdjustment, FinalAnalysisWeightReceipt +from orgmetra_validity_analysis import ( + AnalysisWeightAdjustment, + FinalAnalysisWeightReceipt, + WeightEligibilityReceipt, +) TENANT = "10000000-0000-7000-8000-000000000001" RECEIPT = "analysis_weight_receipt:11111111-1111-4111-8111-111111111111" +ELIGIBILITY_RECEIPT = "weight_eligibility_receipt:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" ESTIMAND = "validation_estimand:22222222-2222-4222-8222-222222222222" TARGET = "analysis_target_population:33333333-3333-4333-8333-333333333333" WINDOW = "analysis_window:44444444-4444-4444-8444-444444444444" +DURATION = "analysis_reference_duration:77777777-7777-4777-8777-777777777777" DIGEST_A = "a" * 64 DIGEST_B = "b" * 64 DIGEST_C = "c" * 64 @@ -22,6 +28,7 @@ DIGEST_3 = "3" * 64 DIGEST_4 = "4" * 64 DIGEST_5 = "5" * 64 +DIGEST_6 = "6" * 64 def adjustment(**overrides: object) -> AnalysisWeightAdjustment: @@ -41,17 +48,42 @@ def adjustment(**overrides: object) -> AnalysisWeightAdjustment: return AnalysisWeightAdjustment(**values) +def eligibility(**overrides: object) -> WeightEligibilityReceipt: + """Return one weight eligibility receipt aligned to the default estimand.""" + values: dict[str, object] = { + "tenant_record_id": TENANT, + "receipt_reference": ELIGIBILITY_RECEIPT, + "weight_scope_code": "cross_sectional", + "target_population_reference": TARGET, + "target_population_digest": DIGEST_B, + "reference_duration_reference": DURATION, + "reference_duration_digest": DIGEST_6, + "eligible_case_set_digest": DIGEST_C, + "weight_artifact_digest": DIGEST_E, + "constructed_at": datetime(2026, 9, 17, 4, 0, tzinfo=timezone.utc), + } + values.update(overrides) + return WeightEligibilityReceipt(**values) + + def receipt(**overrides: object) -> FinalAnalysisWeightReceipt: """Return one exact final point-weight receipt.""" + final_weight = overrides.get("final_weight_artifact_digest", DIGEST_E) + eligibility_receipt = overrides.get( + "weight_eligibility", eligibility(weight_artifact_digest=final_weight) + ) values: dict[str, object] = { "tenant_record_id": TENANT, "receipt_reference": RECEIPT, "estimand_reference": ESTIMAND, "estimand_digest": DIGEST_A, + "estimand_scope_code": "cross_sectional", "target_population_reference": TARGET, "target_population_digest": DIGEST_B, "analysis_unit_code": "worker_occurrence", "analysis_window_reference": WINDOW, + "reference_duration_reference": DURATION, + "reference_duration_digest": DIGEST_6, "eligible_case_set_digest": DIGEST_C, "analytic_case_occurrence_set_digest": DIGEST_2, "source_universe_receipt_digest": DIGEST_3, @@ -61,7 +93,8 @@ def receipt(**overrides: object) -> FinalAnalysisWeightReceipt: "base_weight_evidence_digest": DIGEST_5, "base_weight_artifact_digest": DIGEST_D, "adjustments": (adjustment(),), - "final_weight_artifact_digest": DIGEST_E, + "final_weight_artifact_digest": final_weight, + "weight_eligibility": eligibility_receipt, "analytic_case_count": 12, "constructed_at": datetime(2026, 9, 17, 4, 0, tzinfo=timezone.utc), } @@ -76,6 +109,7 @@ def test_receipt_is_deterministic_value_minimized_and_redacted() -> None: assert candidate.final_weight_artifact_digest == DIGEST_E assert "person_record" not in candidate.canonical_json() assert "weight_value" not in candidate.canonical_json() + assert "weight_eligibility_receipt_digest" in candidate.canonical_json() assert repr(candidate) == "FinalAnalysisWeightReceipt()" @@ -116,6 +150,30 @@ def test_probability_design_receipt_is_required() -> None: receipt(sampling_design_receipt_digest="not-a-digest") +def test_weight_eligibility_must_match_estimand_scope_population_duration_and_cases() -> None: + """Reject cross-sectional/longitudinal or target-window mismatches before release.""" + with pytest.raises(ValueError, match="weight scope"): + receipt(weight_eligibility=eligibility(weight_scope_code="longitudinal")) + with pytest.raises(ValueError, match="target population"): + receipt(weight_eligibility=eligibility(target_population_digest=DIGEST_A)) + with pytest.raises(ValueError, match="reference duration"): + receipt(weight_eligibility=eligibility(reference_duration_digest=DIGEST_A)) + with pytest.raises(ValueError, match="eligible case set"): + receipt(weight_eligibility=eligibility(eligible_case_set_digest=DIGEST_A)) + with pytest.raises(ValueError, match="final point-estimation weight artifact"): + receipt(weight_eligibility=eligibility(weight_artifact_digest=DIGEST_D)) + + +def test_longitudinal_weight_is_accepted_only_for_matching_longitudinal_estimand() -> None: + """Allow longitudinal inference when population, duration, cases, and artifact all agree.""" + candidate = receipt( + estimand_scope_code="longitudinal", + weight_eligibility=eligibility(weight_scope_code="longitudinal"), + ) + assert candidate.estimand_scope_code == "longitudinal" + assert candidate.weight_eligibility.weight_scope_code == "longitudinal" + + def test_positive_versions_and_counts_fail_closed() -> None: """Reject sentinel sequence, method-version, count, and correction values.""" with pytest.raises(ValueError, match="sequence_number"): From 6be0ff39b423c9ce74e2f7a2b5b3a86a06544cd3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 07:06:46 +0900 Subject: [PATCH 079/158] docs(validity): document weight eligibility congruence --- packages/validity-analysis/README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/validity-analysis/README.md b/packages/validity-analysis/README.md index c662627a5..a4323b660 100644 --- a/packages/validity-analysis/README.md +++ b/packages/validity-analysis/README.md @@ -8,7 +8,9 @@ This package creates an immutable **selection-validity analysis handoff**, valid The resulting canonical JSON is digest-addressable, contains no raw person-level predictor or criterion values, and remains `not_executed`. Its timestamp is detached to one UTC instant at construction so later timezone-provider changes cannot rewrite the digest. Required result evidence is explicit: effect estimate, uncertainty interval, sample size, missingness summary, and convergence diagnostics. -`FinalAnalysisWeightReceipt` binds a weighted estimand to exact target/window/case-set evidence, source/sampling receipt digests, base-weight evidence, an ordered immutable adjustment chain, the final point-weight artifact digest, and append-only correction lineage. It records only identifiers, versions and digests; it does not store row-level weight values or copy calibration attributes into the validity package. +`FinalAnalysisWeightReceipt` binds a weighted estimand to exact target/window/reference-duration/case-set evidence, source/sampling receipt digests, base-weight evidence, an ordered immutable adjustment chain, the final point-weight artifact digest, and append-only correction lineage. It records only identifiers, versions and digests; it does not store row-level weight values or copy calibration attributes into the validity package. + +`WeightEligibilityReceipt` makes cross-sectional versus longitudinal use machine-checkable rather than an opaque weight label. It binds the final weight artifact to one governed scope (`cross_sectional` or `longitudinal`), target population, reference-duration evidence, and eligible-case set. `FinalAnalysisWeightReceipt` fails closed unless those fields match the estimand and the same final point-weight artifact exactly, so a longitudinal weight cannot silently support a cross-sectional estimand or a different reference duration. Nonresponse and calibration are no longer allowed to hide behind an undifferentiated adjustment digest. `NonresponseAdjustmentReceipt` binds the adjustment to a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, and explicit treatment codes for ineligible, unknown, and unavailable cases. `CalibrationAdjustmentReceipt` binds the adjustment to a target population/window, purpose-bound auxiliary projection, authoritative benchmark receipt, algorithm/version, constraints digest, and explicit `converged` or `fallback_applied` termination evidence. A fallback must identify its versioned rule; a nonconverged calibration cannot masquerade as an accepted calibration receipt. @@ -28,13 +30,13 @@ Point-estimation semantics are explicit. `unweighted` results cannot carry weigh - It does **not** claim that a selection procedure is valid. - It does **not** interpret adverse impact. - It does **not** authorize hiring, promotion, termination, compensation, or another employment decision. -- It does **not** yet complete #407. Longitudinal/cross-sectional weight eligibility, owner-side verification that an adjustment digest resolves to the typed receipt claimed by `evidence_kind`, sensitive auxiliary-variable purpose enforcement at the durable service/API boundary, and released auxiliary evidence exchange remain open work in `workforce_validation`. +- It does **not** yet complete #407. Owner-side verification that an adjustment digest resolves to the typed receipt claimed by `evidence_kind`, sensitive auxiliary-variable purpose enforcement at the durable service/API boundary, and released auxiliary evidence exchange remain open work in `workforce_validation`. The fast-mlsirm repository remains a dedicated-writer dependency. This package records only the immutable revision reviewed for the handoff. Durable Workforce Validation registry/API/persistence remains on the canonical #235/#248 owner path; this package does not create a parallel service. ## Host obligations -Before an approved offline validation worker executes the handoff, the Orgmetra host must re-resolve every reference inside `tenant_record_id` and prove that the predictor, criterion, population, and policy evidence belong to the exact validation study and Job. The requester and reviewer must resolve to distinct authoritative actors. For weighted design-based inference, the host must also verify the final point-weight receipt, its upstream source/sampling evidence, each typed adjustment receipt identified by the ordered chain, and the separately bound variance-design evidence rather than trusting caller-supplied labels or digests. A numerical result is scientific evidence for accountable human interpretation, never an autonomous employment decision. +Before an approved offline validation worker executes the handoff, the Orgmetra host must re-resolve every reference inside `tenant_record_id` and prove that the predictor, criterion, population, and policy evidence belong to the exact validation study and Job. The requester and reviewer must resolve to distinct authoritative actors. For weighted design-based inference, the host must also verify the final point-weight receipt, its matching cross-sectional/longitudinal weight-eligibility receipt, its upstream source/sampling evidence, each typed adjustment receipt identified by the ordered chain, and the separately bound variance-design evidence rather than trusting caller-supplied labels or digests. A numerical result is scientific evidence for accountable human interpretation, never an autonomous employment decision. ## Verification @@ -45,4 +47,4 @@ PYTHONPATH=packages/validity-analysis/src \ python -m pytest -c packages/validity-analysis/pyproject.toml packages/validity-analysis/tests ``` -The package gate requires exact 100% owned production statement and branch coverage. \ No newline at end of file +The package gate requires exact 100% owned production statement and branch coverage. From 003daf97e3e33badd4d72f12d4712eefebf39c32 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 07:07:07 +0900 Subject: [PATCH 080/158] docs(validity): bind estimand to weight eligibility evidence --- ...overned-selection-validity-analysis-handoff.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/adr/0027-governed-selection-validity-analysis-handoff.md b/docs/adr/0027-governed-selection-validity-analysis-handoff.md index 3d2e3af08..8742cf13d 100644 --- a/docs/adr/0027-governed-selection-validity-analysis-handoff.md +++ b/docs/adr/0027-governed-selection-validity-analysis-handoff.md @@ -30,13 +30,15 @@ Both handoff and result envelopes detach exact timezone-aware timestamps to one The same package also validates `ValidationAnalysisResult` envelopes returned by the approved offline worker. A result must link to the handoff digest and the same pinned revision, identify a Rust CPU or GPU backend and precision, provide finite effect and interval values, match its sample size to aggregate missingness counts, reject impossible complete-versus-missing count combinations, and include explicit convergence diagnostics. The canonicalization boundary accepts only the exact governed `MissingnessSummary` and `ConvergenceDiagnostics` runtime types so subclass method overrides cannot add unreviewed or person-level fields to immutable result evidence. A nonconverged result remains typed scientific evidence requiring human review; it cannot be treated as a valid selection procedure or an employment decision. -For #407's weighted design-based inference boundary, the active package adds `FinalAnalysisWeightReceipt`. It binds the exact estimand, target population, analysis window, eligible/analytic case-set digests, #404/#405 source/sampling receipt digests, base-weight derivation evidence, an ordered immutable adjustment chain, the final point-weight artifact digest, construction time, and append-only correction lineage without storing row-level weights. Each adjustment must be contiguous and digest-linked from the previous artifact to the declared final artifact. `ValidationAnalysisResult` explicitly distinguishes `unweighted` from `weighted_design_based`; a weighted result fails closed unless it separately binds both the final analysis-weight receipt digest and the #406 variance-design receipt digest. An unweighted result cannot carry either receipt and thereby masquerade as weighted scientific evidence. +For #407's weighted design-based inference boundary, the active package adds `FinalAnalysisWeightReceipt`. It binds the exact estimand, target population, analysis window, reference duration, eligible/analytic case-set digests, #404/#405 source/sampling receipt digests, base-weight derivation evidence, an ordered immutable adjustment chain, the final point-weight artifact digest, construction time, and append-only correction lineage without storing row-level weights. Each adjustment must be contiguous and digest-linked from the previous artifact to the declared final artifact. `ValidationAnalysisResult` explicitly distinguishes `unweighted` from `weighted_design_based`; a weighted result fails closed unless it separately binds both the final analysis-weight receipt digest and the #406 variance-design receipt digest. An unweighted result cannot carry either receipt and thereby masquerade as weighted scientific evidence. + +`WeightEligibilityReceipt` makes weight use itself versioned evidence rather than a caller-supplied label. The receipt is closed to `cross_sectional` or `longitudinal` scope and binds the exact target population, reference-duration evidence, eligible-case set, and final point-weight artifact. `FinalAnalysisWeightReceipt` validates that the eligibility receipt belongs to the same tenant and that its scope, target population, reference duration, eligible cases, and weight artifact exactly match the estimand-side receipt. A longitudinal weight therefore cannot silently support a cross-sectional estimand or a different target period. The 2025 SIPP Users' Guide is used only as current primary methodological evidence that weight choice depends on both target population and duration and that longitudinal weights cover explicit multi-year reference periods; SIPP-specific variables or estimators are not imported into Orgmetra. The adjustment chain has typed scientific evidence for three high-risk transform families. `NonresponseAdjustmentReceipt` binds a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, explicit treatment of ineligible/unknown/unavailable cases, and input/output weight artifacts. `CalibrationAdjustmentReceipt` binds the target population/window, purpose-limited auxiliary projection, authoritative benchmark receipt, algorithm/version, constraints digest, input/output artifacts, and an explicit `converged` or `fallback_applied` termination state. A fallback must identify its immutable fallback-rule reference/digest. A failed/nonconverged calibration cannot be labeled as an accepted calibration receipt. `TrimmingBoundingAdjustmentReceipt` binds a versioned trimming/bounding rule, reproducible rule configuration digest, the semantic occurrence set and count of cases actually affected, and input/output weight artifacts; a declared trim/bound transform cannot be a no-op. Known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization `AnalysisWeightAdjustment` codes also carry an `evidence_kind` and fail closed unless it names the matching typed receipt family. -This executable slice still does not claim #407 complete. Longitudinal/cross-sectional eligibility, durable service/API verification that an adjustment digest resolves to the typed receipt claimed by `evidence_kind`, sensitive auxiliary-variable purpose enforcement, and released auxiliary-owner evidence exchange remain `workforce_validation` scientific truth rather than Talent truth or cross-context SQL reconstruction. +This executable slice still does not claim #407 complete. Durable service/API verification that an adjustment digest resolves to the typed receipt claimed by `evidence_kind`, sensitive auxiliary-variable purpose enforcement, and released auxiliary-owner evidence exchange remain `workforce_validation` scientific truth rather than Talent truth or cross-context SQL reconstruction. -The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, prove requester/reviewer identities are distinct authoritative actors, and verify typed weight-adjustment receipts rather than trusting caller-supplied evidence labels or digests. +The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, prove requester/reviewer identities are distinct authoritative actors, verify cross-sectional/longitudinal weight eligibility against the estimand, and verify typed weight-adjustment receipts rather than trusting caller-supplied evidence labels or digests. ## Consequences @@ -48,6 +50,7 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - Result canonicalization cannot be extended by an unreviewed subclass to serialize extra decision-like or person-level fields. - Caller-controlled timestamp and numeric runtime behavior cannot rewrite an accepted canonical digest after construction or turn malformed oversized worker output into an uncaught exception type. - Weighted design-based results can no longer identify only a sample while omitting which final point-weight evidence and variance-design evidence were actually used. +- Cross-sectional and longitudinal weights cannot be interchanged when target population, reference duration, eligible cases, or final weight artifact differ from the estimand. - The ordered weight transformation chain is independently digest-correlatable without centralizing case-level weights or auxiliary attributes. - Nonresponse cannot silently drop refusal/unreachable/ineligible/failure dispositions behind one opaque adjustment label. - Calibration/raking cannot silently float benchmark ownership or hide fallback as successful convergence. @@ -59,14 +62,14 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - This slice does not execute a statistical model, estimate validity, correct for measurement error/range restriction, evaluate adverse impact, or assert legal compliance. - Sampling design, estimator choice, missing-data treatment, reliability evidence, multiplicity, transportability, fairness analysis, and model diagnostics must be encoded in the referenced analysis plan and reviewed before execution. -- The current #407 slice validates generic ordered weight provenance, weighted-result binding, disposition-aware nonresponse evidence, benchmark/termination-aware calibration evidence, and trimming/bounding rule provenance; it does not yet make longitudinal eligibility, durable typed-receipt resolution, or sensitive auxiliary-owner enforcement fully executable. +- The current #407 slice validates generic ordered weight provenance, weighted-result binding, cross-sectional/longitudinal eligibility congruence, disposition-aware nonresponse evidence, benchmark/termination-aware calibration evidence, and trimming/bounding rule provenance; it does not yet make durable typed-receipt resolution or sensitive auxiliary-owner enforcement fully executable. - The generic result envelope does not invent an estimator-specific minimum complete-case count. Whether a converged estimator is identified under a particular missing-data design belongs to the reviewed analysis plan and numerical-worker contract; Orgmetra fails closed on impossible aggregate counts without silently replacing that foreign scientific contract with complete-case analysis. - The package validates the result envelope, but a future execution adapter must still re-resolve the handoff references, verify the result provenance artifact, resolve typed adjustment digests to released owner evidence, and attach evidence only after accountable human review. ## Verification -The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. #407 adds RED/GREEN coverage for deterministic value-minimized final-weight receipts, exact sampling-receipt binding, contiguous transformation lineage, append-only correction linkage, fail-closed weighted-result binding to separate point-weight and variance-design receipts, explicit nonresponse disposition treatment, calibration benchmark/constraint/termination provenance, fallback-rule disclosure, trimming/bounding rule/configuration and affected-case provenance, no-op trim/bound rejection, and typed adjustment-evidence matching. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. +The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. #407 adds RED/GREEN coverage for deterministic value-minimized final-weight receipts, exact sampling-receipt binding, contiguous transformation lineage, append-only correction linkage, fail-closed weighted-result binding to separate point-weight and variance-design receipts, cross-sectional/longitudinal scope and target-population/reference-duration/case-set/final-artifact congruence, explicit nonresponse disposition treatment, calibration benchmark/constraint/termination provenance, fallback-rule disclosure, trimming/bounding rule/configuration and affected-case provenance, no-op trim/bound rejection, and typed adjustment-evidence matching. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. ## References -See `docs/doctoring/validation-analysis-handoff-references.md`. \ No newline at end of file +See `docs/doctoring/validation-analysis-handoff-references.md`. From 831945a8736f6e7ef1734ea1ce76a7fb110c984e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 07:07:21 +0900 Subject: [PATCH 081/158] docs(validity): trace weight eligibility congruence --- docs/traceability/validation-analysis-handoff.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/traceability/validation-analysis-handoff.md b/docs/traceability/validation-analysis-handoff.md index da34a2cbb..b954e071c 100644 --- a/docs/traceability/validation-analysis-handoff.md +++ b/docs/traceability/validation-analysis-handoff.md @@ -14,23 +14,24 @@ Can an organization send one exact, reviewable validation study to its statistic | Human authority | requester/reviewer references must differ, and the host must re-resolve both within the tenant and prove they resolve to distinct authoritative actor identities before execution | direct-construction fail-closed regression plus `test_next_action_requires_resolved_actor_identity_separation` | | Scientific evidence | effect estimate, uncertainty interval, sample size, internally possible aggregate missingness, convergence diagnostics | immutable required-result-evidence regression plus impossible-missingness rejection | | Numerical result boundary | handoff digest, pinned fast-mlsirm revision, Rust CPU/GPU backend, precision, finite estimate/interval, aggregate missingness, explicit convergence state | `ValidationAnalysisResult` invariant/canonicalization regressions, exact-runtime-type checks, and oversized-numeric `ValueError` normalization | -| Final point-weight provenance | exact estimand/target/window and case-set digests, source/sampling receipt digests, base-weight evidence, ordered digest-linked adjustments, final weight artifact, append-only correction lineage | `FinalAnalysisWeightReceipt` deterministic/value-minimized regressions plus chain/correction fail-closed tests | +| Final point-weight provenance | exact estimand/target/window/reference-duration and case-set digests, source/sampling receipt digests, base-weight evidence, ordered digest-linked adjustments, final weight artifact, append-only correction lineage | `FinalAnalysisWeightReceipt` deterministic/value-minimized regressions plus chain/correction fail-closed tests | +| Weight eligibility congruence | governed `cross_sectional` or `longitudinal` scope plus exact target population, reference-duration evidence, eligible case set, and final point-weight artifact | `test_weight_eligibility_receipt.py` plus `FinalAnalysisWeightReceipt` mismatch/longitudinal-match regressions | | Nonresponse adjustment evidence | versioned response/disposition receipt, exact adjustment population, controlled method/configuration, explicit ineligible/unknown/unavailable treatment, input/output weight artifacts | `NonresponseAdjustmentReceipt` deterministic/value-minimized tests plus undocumented-treatment/no-op/version rejection | | Calibration/raking evidence | target population/window, purpose-bound auxiliary projection, authoritative benchmark receipt, algorithm/version, constraints, input/output artifacts, explicit converged/fallback state and immutable fallback rule when used | `CalibrationAdjustmentReceipt` benchmark/termination/fallback regressions | | Trimming/bounding evidence | versioned trimming/bounding rule, reproducible rule configuration digest, exact affected semantic occurrence set and count, input/output weight artifacts | `test_trimming_bounding_receipt.py` deterministic/value-minimized, no-op, missing-provenance, and typed-kind regressions | | Typed adjustment congruence | known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization adjustment codes must identify the matching typed receipt family through `evidence_kind` | specialized adjustment evidence-kind regressions | | Weighted-result congruence | `weighted_design_based` result must bind the exact final point-weight receipt and a separate #406 variance-design receipt; `unweighted` result must bind neither | `test_analysis_weight_result_binding.py` | | Execution boundary | `not_executed`, `scientific_evidence_only`, read-only pinned foreign dependency | immutable governance regressions | -| Reproducibility | construction-time UTC timestamp snapshots, finite numeric snapshots, canonical RFC 3339 time, canonical JSON, SHA-256 handoff/result/weight/adjustment-receipt digests | mutable timezone/numeric and UTC-boundary regressions plus deterministic serialization/digest tests | +| Reproducibility | construction-time UTC timestamp snapshots, finite numeric snapshots, canonical RFC 3339 time, canonical JSON, SHA-256 handoff/result/weight/adjustment/eligibility-receipt digests | mutable timezone/numeric and UTC-boundary regressions plus deterministic serialization/digest tests | | Decision-record integrity | ADR numbers remain unique repository-wide and any `docs/adr/**` change reaches the consolidated Foundation CI validity regression | ADR uniqueness regression plus Foundation CI workflow-trigger contract regression | | Quality-evidence freshness | consolidated Foundation CI runs the validity package on every `develop` pull request without a repository path filter, so shared Python/test/clean-checkout configuration cannot silently bypass the package gate | `test_foundation_ci_retriggers_without_path_filter` and `test_foundation_ci_runs_validity_analysis_and_adr_changes`; central required workflows remain separate gates | ## #407 boundary still open -The active branch now makes three adjustment families executable rather than leaving them as opaque digests: nonresponse adjustment evidence is disposition-aware; calibration/raking/post-stratification evidence is bound to an authoritative benchmark, purpose-limited auxiliary projection, constraints, and explicit convergence/fallback state; and trimming/bounding/winsorization evidence is bound to a versioned rule/configuration plus the exact affected occurrence set. This still does not complete #407. Longitudinal/cross-sectional weight eligibility, durable owner-side verification that each adjustment digest resolves to the typed receipt claimed by `evidence_kind`, sensitive auxiliary-variable purpose enforcement, and released auxiliary evidence exchange remain open. `workforce_validation` owns those scientific contracts and their released result; `talent_management` must not compute or reconstruct final analysis weights. +The active branch now makes final-weight eligibility and three adjustment families executable rather than leaving them as opaque labels. Weight eligibility is explicitly cross-sectional or longitudinal and must match the estimand's target population, reference duration, eligible cases, and final point-weight artifact. Nonresponse adjustment evidence is disposition-aware; calibration/raking/post-stratification evidence is bound to an authoritative benchmark, purpose-limited auxiliary projection, constraints, and explicit convergence/fallback state; and trimming/bounding/winsorization evidence is bound to a versioned rule/configuration plus the exact affected occurrence set. This still does not complete #407. Durable owner-side verification that each adjustment digest resolves to the typed receipt claimed by `evidence_kind`, sensitive auxiliary-variable purpose enforcement, and released auxiliary evidence exchange remain open. `workforce_validation` owns those scientific contracts and their released result; `talent_management` must not compute or reconstruct final analysis weights. ## Maturity `implemented_on_active_pr`. -Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. The active package validates the returned numerical/provenance envelope plus the current #407 final-weight/result-binding and typed adjustment-evidence contracts, but protected Orgmetra evidence still requires host re-resolution, durable owner evidence exchange, result-artifact verification, terminal checks, independent review, and accountable human interpretation. \ No newline at end of file +Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. The active package validates the returned numerical/provenance envelope plus the current #407 final-weight/result-binding, weight-eligibility, and typed adjustment-evidence contracts, but protected Orgmetra evidence still requires host re-resolution, durable owner evidence exchange, result-artifact verification, terminal checks, independent review, and accountable human interpretation. From 149966a3fe3d4c355dffa6edb8187d581b85551b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 07:07:35 +0900 Subject: [PATCH 082/158] docs(validity): doctor current SIPP weight eligibility evidence --- docs/doctoring/validation-analysis-handoff-references.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/doctoring/validation-analysis-handoff-references.md b/docs/doctoring/validation-analysis-handoff-references.md index 50057cdbc..f107c70dc 100644 --- a/docs/doctoring/validation-analysis-handoff-references.md +++ b/docs/doctoring/validation-analysis-handoff-references.md @@ -14,6 +14,8 @@ U.S. Census Bureau. (2021). *Statistical Quality Standard D1: Producing direct e U.S. Census Bureau. (2022, August 18). *Survey of Income and Program Participation: Weighting*. https://www.census.gov/programs-surveys/sipp/methodology/weighting.html +U.S. Census Bureau. (2026). *2025 Survey of Income and Program Participation users' guide* (August 2026 revision), pp. 156–157. https://www2.census.gov/programs-surveys/sipp/tech-documentation/methodology/2025_SIPP_Users_Guide.pdf + ContextualWisdomLab. (2026). *fast-mlsirm* (Commit 04d0bc21a2a20693bcf16108cd76d394fe844d23) [Computer software]. GitHub. https://github.com/ContextualWisdomLab/fast-mlsirm/tree/04d0bc21a2a20693bcf16108cd76d394fe844d23 Tabassi, E. (2023). *Artificial intelligence risk management framework (AI RMF 1.0)* (NIST AI 100-1). National Institute of Standards and Technology. https://doi.org/10.6028/NIST.AI.100-1 @@ -26,6 +28,7 @@ Office of Personnel Management. (2026). *Removal of references to the Uniform Gu - The July 31, 2026 OPM interim final rule removed UGESP references from specified federal civil-service regulations. Orgmetra therefore does not present UGESP as an undifferentiated government-wide mandate; applicability must be evaluated for the employer, jurisdiction, decision, and governing law at use time. - The SIOP Principles are the professional validation baseline used for the handoff's evidence-and-human-review posture. The journal citation above fixes volume 11, Supplement S1, pages 1–97, and DOI 10.1017/iop.2018.195. - Deville and Särndal show that calibrated weights are produced by modifying ordinary inverse-inclusion-probability weights under explicit distance measures and calibration equations. ADR 0027 uses that narrow result to justify treating final adjusted point weights as a separately versioned scientific artifact rather than assuming `1/π_i` and calibrated weights are interchangeable. It does not mandate one calibration estimator for Orgmetra. -- Census Statistical Quality Standard D1 requires estimates and variances to account for sample design and post-sampling weighting adjustments. The SIPP methodology illustrates that final weights can combine base selection, nonresponse, longitudinal/panel, and post-stratification/calibration adjustments and that the appropriate weight depends on target population and reference duration. These sources support provenance/reproducibility requirements only; SIPP-specific weights are not imported as Orgmetra rules. +- Census Statistical Quality Standard D1 requires estimates and variances to account for sample design and post-sampling weighting adjustments. The SIPP methodology illustrates that final weights can combine base selection, nonresponse, longitudinal/panel, and post-stratification/calibration adjustments. These sources support provenance/reproducibility requirements only; SIPP-specific weights are not imported as Orgmetra rules. +- The August 2026 revision of the 2025 SIPP Users' Guide states that choosing a weight depends on the population to which results apply and the duration of interest, distinguishes cross-sectional monthly analysis from longitudinal multi-year analysis, and identifies explicit two-, three-, and four-year reference periods for longitudinal weights. ADR 0027 uses this only to justify fail-closed target-population/reference-duration congruence for `WeightEligibilityReceipt`; it does not adopt SIPP variable names, cohorts, or estimators as Orgmetra domain truth. - The fast-mlsirm commit is recorded as a read-only dependency coordinate only. This Orgmetra slice does not modify or duplicate its numerical implementation. - NIST AI RMF's govern, map, measure, and manage functions support preserving backend, precision, provenance, convergence, and human-review fields as inspectable result evidence rather than treating a model response as an autonomous decision. From 827adc09a146618273b23f168c0e794efcb2edce Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 07:07:44 +0900 Subject: [PATCH 083/158] docs(validity): record weight eligibility contract --- packages/validity-analysis/CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/validity-analysis/CHANGELOG.md b/packages/validity-analysis/CHANGELOG.md index 7ab1f182e..5a552d808 100644 --- a/packages/validity-analysis/CHANGELOG.md +++ b/packages/validity-analysis/CHANGELOG.md @@ -11,9 +11,10 @@ - Require exact governed missingness/convergence runtime types so subclass method overrides cannot inject unreviewed or person-level fields into canonical result evidence. - Freeze exact UTC timestamps and finite numeric values at construction, and reject runtime-type forgery before canonical evidence serialization. - Add a deterministic, row-value-minimized `FinalAnalysisWeightReceipt` that binds an exact estimand to source/sampling evidence, base-weight provenance, an ordered digest-linked adjustment chain, the final point-weight artifact, and append-only correction lineage. +- Add `WeightEligibilityReceipt` and fail closed unless cross-sectional/longitudinal scope, target population, reference duration, eligible case set, and final point-weight artifact match the estimand-side final-weight receipt exactly. - Distinguish unweighted from weighted design-based results and fail closed unless a weighted result separately binds the exact final analysis-weight receipt and variance-design receipt used. - Add typed `NonresponseAdjustmentReceipt` evidence with explicit disposition treatment and typed `CalibrationAdjustmentReceipt` evidence bound to purpose-limited auxiliary projections, authoritative benchmark receipts, constraints, and explicit convergence/fallback state. - Add typed `TrimmingBoundingAdjustmentReceipt` evidence that binds a versioned rule/configuration, exact affected semantic occurrence set and count, and input/output weight artifacts; known trimming/bounding/winsorization adjustments must name that evidence family. - Require known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization adjustment codes to identify the matching typed evidence kind rather than collapse into a generic opaque adjustment digest. -- Keep longitudinal eligibility, durable owner-side typed-receipt resolution, sensitive auxiliary purpose enforcement, and released auxiliary evidence exchange explicitly incomplete under #407. -- Make `Validity Analysis Handoff Quality` retrigger on shared repository Python/test/clean-checkout configuration, with an executable regression preventing stale package-quality evidence after shared tooling changes. \ No newline at end of file +- Keep durable owner-side typed-receipt resolution, sensitive auxiliary purpose enforcement, and released auxiliary evidence exchange explicitly incomplete under #407. +- Make `Validity Analysis Handoff Quality` retrigger on shared repository Python/test/clean-checkout configuration, with an executable regression preventing stale package-quality evidence after shared tooling changes. From 1bd4e649b20395f7815122653f8d6bed96880bba Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 07:11:25 +0900 Subject: [PATCH 084/158] test(validity): close eligibility scope type edge --- .../tests/test_weight_eligibility_receipt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/validity-analysis/tests/test_weight_eligibility_receipt.py b/packages/validity-analysis/tests/test_weight_eligibility_receipt.py index 45017db25..4d1ba3f9f 100644 --- a/packages/validity-analysis/tests/test_weight_eligibility_receipt.py +++ b/packages/validity-analysis/tests/test_weight_eligibility_receipt.py @@ -43,9 +43,9 @@ def test_weight_eligibility_receipt_is_deterministic_and_value_minimized() -> No assert repr(candidate) == "WeightEligibilityReceipt()" -@pytest.mark.parametrize("scope", ["monthly", "panel", "opaque", ""]) -def test_weight_scope_is_closed_to_cross_sectional_or_longitudinal(scope: str) -> None: - """Reject caller-defined labels that hide longitudinal/cross-sectional semantics.""" +@pytest.mark.parametrize("scope", [None, "monthly", "panel", "opaque", ""]) +def test_weight_scope_is_closed_to_cross_sectional_or_longitudinal(scope: object) -> None: + """Reject caller-defined labels and non-strings that hide weight-scope semantics.""" with pytest.raises(ValueError, match="weight_scope_code"): eligibility(weight_scope_code=scope) From a72160d3b2fa726bf704a242cf55b35f372af0e9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 07:11:49 +0900 Subject: [PATCH 085/158] test(validity): close estimand eligibility edges --- .../tests/test_analysis_weight_receipt.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/packages/validity-analysis/tests/test_analysis_weight_receipt.py b/packages/validity-analysis/tests/test_analysis_weight_receipt.py index c8b022178..79e367c32 100644 --- a/packages/validity-analysis/tests/test_analysis_weight_receipt.py +++ b/packages/validity-analysis/tests/test_analysis_weight_receipt.py @@ -11,12 +11,15 @@ ) TENANT = "10000000-0000-7000-8000-000000000001" +OTHER_TENANT = "10000000-0000-7000-8000-000000000002" RECEIPT = "analysis_weight_receipt:11111111-1111-4111-8111-111111111111" ELIGIBILITY_RECEIPT = "weight_eligibility_receipt:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" ESTIMAND = "validation_estimand:22222222-2222-4222-8222-222222222222" TARGET = "analysis_target_population:33333333-3333-4333-8333-333333333333" +OTHER_TARGET = "analysis_target_population:33333333-3333-4333-8333-333333333334" WINDOW = "analysis_window:44444444-4444-4444-8444-444444444444" DURATION = "analysis_reference_duration:77777777-7777-4777-8777-777777777777" +OTHER_DURATION = "analysis_reference_duration:77777777-7777-4777-8777-777777777778" DIGEST_A = "a" * 64 DIGEST_B = "b" * 64 DIGEST_C = "c" * 64 @@ -150,12 +153,27 @@ def test_probability_design_receipt_is_required() -> None: receipt(sampling_design_receipt_digest="not-a-digest") +@pytest.mark.parametrize("scope", [None, "panel"]) +def test_estimand_scope_is_closed_to_cross_sectional_or_longitudinal(scope: object) -> None: + """Reject non-string and caller-defined estimand scope labels.""" + with pytest.raises(ValueError, match="estimand_scope_code"): + receipt(estimand_scope_code=scope) + + def test_weight_eligibility_must_match_estimand_scope_population_duration_and_cases() -> None: """Reject cross-sectional/longitudinal or target-window mismatches before release.""" + with pytest.raises(ValueError, match="WeightEligibilityReceipt"): + receipt(weight_eligibility=object()) + with pytest.raises(ValueError, match="tenant_record_id"): + receipt(weight_eligibility=eligibility(tenant_record_id=OTHER_TENANT)) with pytest.raises(ValueError, match="weight scope"): receipt(weight_eligibility=eligibility(weight_scope_code="longitudinal")) + with pytest.raises(ValueError, match="target population"): + receipt(weight_eligibility=eligibility(target_population_reference=OTHER_TARGET)) with pytest.raises(ValueError, match="target population"): receipt(weight_eligibility=eligibility(target_population_digest=DIGEST_A)) + with pytest.raises(ValueError, match="reference duration"): + receipt(weight_eligibility=eligibility(reference_duration_reference=OTHER_DURATION)) with pytest.raises(ValueError, match="reference duration"): receipt(weight_eligibility=eligibility(reference_duration_digest=DIGEST_A)) with pytest.raises(ValueError, match="eligible case set"): From 4d18ade1a35743b4c8b6e115cd1db251012cc52d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 07:13:03 +0900 Subject: [PATCH 086/158] test(validity): add RED purpose-bound calibration evidence --- .../tests/test_weight_adjustment_semantics.py | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/packages/validity-analysis/tests/test_weight_adjustment_semantics.py b/packages/validity-analysis/tests/test_weight_adjustment_semantics.py index 2fd64c96e..769722a75 100644 --- a/packages/validity-analysis/tests/test_weight_adjustment_semantics.py +++ b/packages/validity-analysis/tests/test_weight_adjustment_semantics.py @@ -46,7 +46,7 @@ def nonresponse_receipt(**overrides: object) -> NonresponseAdjustmentReceipt: def calibration_receipt(**overrides: object) -> CalibrationAdjustmentReceipt: - """Return one benchmark-bound converged calibration receipt.""" + """Return one benchmark- and purpose-bound converged calibration receipt.""" values: dict[str, object] = { "tenant_record_id": TENANT, "receipt_reference": "calibration_adjustment_receipt:33333333-3333-4333-8333-333333333333", @@ -54,6 +54,11 @@ def calibration_receipt(**overrides: object) -> CalibrationAdjustmentReceipt: "analysis_window_reference": "analysis_window:44444444-4444-4444-8444-444444444444", "auxiliary_projection_reference": "calibration_auxiliary_projection:55555555-5555-4555-8555-555555555555", "auxiliary_projection_digest": DIGEST_B, + "auxiliary_purpose_reference": "scientific_data_use_purpose:55555555-5555-4555-8555-555555555556", + "auxiliary_purpose_digest": DIGEST_3, + "auxiliary_owner_contract_reference": "released_owner_contract:55555555-5555-4555-8555-555555555557", + "auxiliary_owner_contract_version": 1, + "auxiliary_authorization_receipt_digest": DIGEST_4, "benchmark_receipt_reference": "calibration_benchmark_receipt:66666666-6666-4666-8666-666666666666", "benchmark_receipt_digest": DIGEST_C, "algorithm_reference": "calibration_algorithm:77777777-7777-4777-8777-777777777777", @@ -102,12 +107,15 @@ def test_nonresponse_receipt_is_value_minimized_and_disposition_aware() -> None: nonresponse_receipt(evidence_version=True) -def test_calibration_receipt_binds_owner_benchmark_and_termination_state() -> None: - """Keep benchmark ownership and fallback semantics explicit and immutable.""" +def test_calibration_receipt_binds_owner_benchmark_purpose_and_termination_state() -> None: + """Keep benchmark, purpose authorization, and fallback semantics explicit and immutable.""" candidate = calibration_receipt() assert candidate.sha256_digest() == calibration_receipt().sha256_digest() assert f'"benchmark_receipt_digest":"{DIGEST_C}"' in candidate.canonical_json() + assert f'"auxiliary_purpose_digest":"{DIGEST_3}"' in candidate.canonical_json() + assert f'"auxiliary_authorization_receipt_digest":"{DIGEST_4}"' in candidate.canonical_json() assert '"termination_code":"converged"' in candidate.canonical_json() + assert "protected_attribute" not in candidate.canonical_json() assert repr(candidate) == "CalibrationAdjustmentReceipt()" fallback_reference = "calibration_fallback_rule:99999999-9999-4999-8999-999999999999" @@ -118,6 +126,12 @@ def test_calibration_receipt_binds_owner_benchmark_and_termination_state() -> No ) assert f'"fallback_rule_digest":"{DIGEST_2}"' in fallback.canonical_json() + with pytest.raises(ValueError, match="auxiliary_purpose_digest"): + calibration_receipt(auxiliary_purpose_digest="purpose-v1") + with pytest.raises(ValueError, match="auxiliary_owner_contract_version"): + calibration_receipt(auxiliary_owner_contract_version=0) + with pytest.raises(ValueError, match="auxiliary_authorization_receipt_digest"): + calibration_receipt(auxiliary_authorization_receipt_digest="authorization-v1") with pytest.raises(ValueError, match="termination_code"): calibration_receipt(termination_code="failed") with pytest.raises(ValueError, match="termination_code"): From abdf2e1fc7b871372bb5932417c4609fbfeb9468 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 07:14:10 +0900 Subject: [PATCH 087/158] feat(validity): bind calibration auxiliaries to purpose authority --- .../src/orgmetra_validity_analysis/weights.py | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py b/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py index 025f33e7c..aa99fb122 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py @@ -122,7 +122,7 @@ def sha256_digest(self) -> str: @dataclass(frozen=True, slots=True, repr=False) class CalibrationAdjustmentReceipt: - """Bind calibration or raking to immutable owner benchmarks and termination evidence.""" + """Bind calibration to owner benchmarks, purpose authority, and termination evidence.""" tenant_record_id: str receipt_reference: str @@ -130,6 +130,11 @@ class CalibrationAdjustmentReceipt: analysis_window_reference: str auxiliary_projection_reference: str auxiliary_projection_digest: str + auxiliary_purpose_reference: str + auxiliary_purpose_digest: str + auxiliary_owner_contract_reference: str + auxiliary_owner_contract_version: int + auxiliary_authorization_receipt_digest: str benchmark_receipt_reference: str benchmark_receipt_digest: str algorithm_reference: str @@ -144,7 +149,7 @@ class CalibrationAdjustmentReceipt: evidence_version: int = 1 def __post_init__(self) -> None: - """Fail closed on floating benchmarks, hidden fallback, or nonconverged output.""" + """Fail closed on floating authority, hidden fallback, or nonconverged output.""" _validate_operational_uuid(self.tenant_record_id, "tenant_record_id") _validate_reference( self.receipt_reference, @@ -161,6 +166,20 @@ def __post_init__(self) -> None: "calibration_auxiliary_projection", "auxiliary_projection_reference", ) + _validate_reference( + self.auxiliary_purpose_reference, + "scientific_data_use_purpose", + "auxiliary_purpose_reference", + ) + _validate_reference( + self.auxiliary_owner_contract_reference, + "released_owner_contract", + "auxiliary_owner_contract_reference", + ) + _positive_integer( + self.auxiliary_owner_contract_version, + "auxiliary_owner_contract_version", + ) _validate_reference( self.benchmark_receipt_reference, "calibration_benchmark_receipt", @@ -174,6 +193,8 @@ def __post_init__(self) -> None: for field_name in ( "target_population_digest", "auxiliary_projection_digest", + "auxiliary_purpose_digest", + "auxiliary_authorization_receipt_digest", "benchmark_receipt_digest", "constraints_digest", "input_weight_artifact_digest", @@ -213,13 +234,18 @@ def __repr__(self) -> str: return "CalibrationAdjustmentReceipt()" def canonical_json(self) -> str: - """Return deterministic owner-benchmark provenance without auxiliary values.""" + """Return deterministic purpose-bound provenance without auxiliary values.""" payload: dict[str, object] = { "algorithm_reference": self.algorithm_reference, "algorithm_version": self.algorithm_version, "analysis_window_reference": self.analysis_window_reference, + "auxiliary_authorization_receipt_digest": self.auxiliary_authorization_receipt_digest, + "auxiliary_owner_contract_reference": self.auxiliary_owner_contract_reference, + "auxiliary_owner_contract_version": self.auxiliary_owner_contract_version, "auxiliary_projection_digest": self.auxiliary_projection_digest, "auxiliary_projection_reference": self.auxiliary_projection_reference, + "auxiliary_purpose_digest": self.auxiliary_purpose_digest, + "auxiliary_purpose_reference": self.auxiliary_purpose_reference, "benchmark_receipt_digest": self.benchmark_receipt_digest, "benchmark_receipt_reference": self.benchmark_receipt_reference, "constraints_digest": self.constraints_digest, From 1d93e3bd756937004800ab91959ba122b29d553d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 07:14:42 +0900 Subject: [PATCH 088/158] docs(validity): document purpose-bound calibration authority --- packages/validity-analysis/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/validity-analysis/README.md b/packages/validity-analysis/README.md index a4323b660..6af6d1eb9 100644 --- a/packages/validity-analysis/README.md +++ b/packages/validity-analysis/README.md @@ -12,9 +12,9 @@ The resulting canonical JSON is digest-addressable, contains no raw person-level `WeightEligibilityReceipt` makes cross-sectional versus longitudinal use machine-checkable rather than an opaque weight label. It binds the final weight artifact to one governed scope (`cross_sectional` or `longitudinal`), target population, reference-duration evidence, and eligible-case set. `FinalAnalysisWeightReceipt` fails closed unless those fields match the estimand and the same final point-weight artifact exactly, so a longitudinal weight cannot silently support a cross-sectional estimand or a different reference duration. -Nonresponse and calibration are no longer allowed to hide behind an undifferentiated adjustment digest. `NonresponseAdjustmentReceipt` binds the adjustment to a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, and explicit treatment codes for ineligible, unknown, and unavailable cases. `CalibrationAdjustmentReceipt` binds the adjustment to a target population/window, purpose-bound auxiliary projection, authoritative benchmark receipt, algorithm/version, constraints digest, and explicit `converged` or `fallback_applied` termination evidence. A fallback must identify its versioned rule; a nonconverged calibration cannot masquerade as an accepted calibration receipt. +Nonresponse and calibration are no longer allowed to hide behind an undifferentiated adjustment digest. `NonresponseAdjustmentReceipt` binds the adjustment to a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, and explicit treatment codes for ineligible, unknown, and unavailable cases. `CalibrationAdjustmentReceipt` binds the adjustment to a target population/window, purpose-bound auxiliary projection, explicit scientific-use purpose evidence, a released owner-contract coordinate and authorization-receipt digest, authoritative benchmark receipt, algorithm/version, constraints digest, and explicit `converged` or `fallback_applied` termination evidence. A fallback must identify its versioned rule; a nonconverged calibration cannot masquerade as an accepted calibration receipt. These fields preserve purpose and release coordinates without copying protected auxiliary values. The durable owner service must still resolve those coordinates and prove that the authorization receipt actually permits the projection for the stated scientific purpose. -Trimming, bounding, and winsorization now have a separate provenance family rather than falling back to a generic adjustment label. `TrimmingBoundingAdjustmentReceipt` binds the exact versioned rule, its reproducible configuration digest, the semantic occurrence set and count of cases actually affected, and the input/output weight artifacts. A declared trim/bound transform must change artifact identity. Known trimming/bounding/winsorization adjustment codes fail closed unless `evidence_kind` names this typed receipt family. +Trimming, bounding, and winsorization have a separate provenance family rather than falling back to a generic adjustment label. `TrimmingBoundingAdjustmentReceipt` binds the exact versioned rule, its reproducible configuration digest, the semantic occurrence set and count of cases actually affected, and the input/output weight artifacts. A declared trim/bound transform must change artifact identity. Known trimming/bounding/winsorization adjustment codes fail closed unless `evidence_kind` names this typed receipt family. `AnalysisWeightAdjustment` records an `evidence_kind` in addition to the evidence digest. Known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization transforms fail closed unless their evidence kind is the corresponding typed receipt. Other adjustment families remain open work rather than being silently treated as equivalent. @@ -30,13 +30,13 @@ Point-estimation semantics are explicit. `unweighted` results cannot carry weigh - It does **not** claim that a selection procedure is valid. - It does **not** interpret adverse impact. - It does **not** authorize hiring, promotion, termination, compensation, or another employment decision. -- It does **not** yet complete #407. Owner-side verification that an adjustment digest resolves to the typed receipt claimed by `evidence_kind`, sensitive auxiliary-variable purpose enforcement at the durable service/API boundary, and released auxiliary evidence exchange remain open work in `workforce_validation`. +- It does **not** yet complete #407. Durable owner-side verification that adjustment and auxiliary-authorization digests resolve to the released typed receipts/contracts they claim, enforcement that the resolved auxiliary projection is authorized for the exact scientific purpose, and released auxiliary evidence exchange remain open work in `workforce_validation`. The fast-mlsirm repository remains a dedicated-writer dependency. This package records only the immutable revision reviewed for the handoff. Durable Workforce Validation registry/API/persistence remains on the canonical #235/#248 owner path; this package does not create a parallel service. ## Host obligations -Before an approved offline validation worker executes the handoff, the Orgmetra host must re-resolve every reference inside `tenant_record_id` and prove that the predictor, criterion, population, and policy evidence belong to the exact validation study and Job. The requester and reviewer must resolve to distinct authoritative actors. For weighted design-based inference, the host must also verify the final point-weight receipt, its matching cross-sectional/longitudinal weight-eligibility receipt, its upstream source/sampling evidence, each typed adjustment receipt identified by the ordered chain, and the separately bound variance-design evidence rather than trusting caller-supplied labels or digests. A numerical result is scientific evidence for accountable human interpretation, never an autonomous employment decision. +Before an approved offline validation worker executes the handoff, the Orgmetra host must re-resolve every reference inside `tenant_record_id` and prove that the predictor, criterion, population, and policy evidence belong to the exact validation study and Job. The requester and reviewer must resolve to distinct authoritative actors. For weighted design-based inference, the host must also verify the final point-weight receipt, its matching cross-sectional/longitudinal weight-eligibility receipt, its upstream source/sampling evidence, each typed adjustment receipt identified by the ordered chain, the calibration auxiliary purpose and released owner authorization when applicable, and the separately bound variance-design evidence rather than trusting caller-supplied labels or digests. A numerical result is scientific evidence for accountable human interpretation, never an autonomous employment decision. ## Verification From 23ec330fac57b8e119a627a4e5820a39ced1cc11 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 07:15:08 +0900 Subject: [PATCH 089/158] docs(validity): govern calibration auxiliary purpose evidence --- ...verned-selection-validity-analysis-handoff.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/adr/0027-governed-selection-validity-analysis-handoff.md b/docs/adr/0027-governed-selection-validity-analysis-handoff.md index 8742cf13d..c06326fc1 100644 --- a/docs/adr/0027-governed-selection-validity-analysis-handoff.md +++ b/docs/adr/0027-governed-selection-validity-analysis-handoff.md @@ -34,11 +34,13 @@ For #407's weighted design-based inference boundary, the active package adds `Fi `WeightEligibilityReceipt` makes weight use itself versioned evidence rather than a caller-supplied label. The receipt is closed to `cross_sectional` or `longitudinal` scope and binds the exact target population, reference-duration evidence, eligible-case set, and final point-weight artifact. `FinalAnalysisWeightReceipt` validates that the eligibility receipt belongs to the same tenant and that its scope, target population, reference duration, eligible cases, and weight artifact exactly match the estimand-side receipt. A longitudinal weight therefore cannot silently support a cross-sectional estimand or a different target period. The 2025 SIPP Users' Guide is used only as current primary methodological evidence that weight choice depends on both target population and duration and that longitudinal weights cover explicit multi-year reference periods; SIPP-specific variables or estimators are not imported into Orgmetra. -The adjustment chain has typed scientific evidence for three high-risk transform families. `NonresponseAdjustmentReceipt` binds a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, explicit treatment of ineligible/unknown/unavailable cases, and input/output weight artifacts. `CalibrationAdjustmentReceipt` binds the target population/window, purpose-limited auxiliary projection, authoritative benchmark receipt, algorithm/version, constraints digest, input/output artifacts, and an explicit `converged` or `fallback_applied` termination state. A fallback must identify its immutable fallback-rule reference/digest. A failed/nonconverged calibration cannot be labeled as an accepted calibration receipt. `TrimmingBoundingAdjustmentReceipt` binds a versioned trimming/bounding rule, reproducible rule configuration digest, the semantic occurrence set and count of cases actually affected, and input/output weight artifacts; a declared trim/bound transform cannot be a no-op. Known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization `AnalysisWeightAdjustment` codes also carry an `evidence_kind` and fail closed unless it names the matching typed receipt family. +The adjustment chain has typed scientific evidence for three high-risk transform families. `NonresponseAdjustmentReceipt` binds a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, explicit treatment of ineligible/unknown/unavailable cases, and input/output weight artifacts. `CalibrationAdjustmentReceipt` binds the target population/window, purpose-limited auxiliary projection, exact scientific-use purpose reference/digest, released owner-contract coordinate/version, owner authorization-receipt digest, authoritative benchmark receipt, algorithm/version, constraints digest, input/output artifacts, and an explicit `converged` or `fallback_applied` termination state. This makes purpose and owner release coordinates part of the scientific receipt without copying protected auxiliary values. The leaf package does not pretend a caller-supplied digest is authoritative: the durable `workforce_validation` service must resolve the released owner contract and authorization receipt and prove that the actual projection is permitted for the stated scientific purpose before execution/release. A fallback must identify its immutable fallback-rule reference/digest. A failed/nonconverged calibration cannot be labeled as an accepted calibration receipt. `TrimmingBoundingAdjustmentReceipt` binds a versioned trimming/bounding rule, reproducible rule configuration digest, the semantic occurrence set and count of cases actually affected, and input/output weight artifacts; a declared trim/bound transform cannot be a no-op. Known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization `AnalysisWeightAdjustment` codes also carry an `evidence_kind` and fail closed unless it names the matching typed receipt family. -This executable slice still does not claim #407 complete. Durable service/API verification that an adjustment digest resolves to the typed receipt claimed by `evidence_kind`, sensitive auxiliary-variable purpose enforcement, and released auxiliary-owner evidence exchange remain `workforce_validation` scientific truth rather than Talent truth or cross-context SQL reconstruction. +This executable slice still does not claim #407 complete. Durable service/API verification that adjustment and auxiliary authorization digests resolve to the released typed receipts/contracts claimed, verification that the released authorization permits the resolved projection for the exact purpose, and released auxiliary-owner evidence exchange remain `workforce_validation` scientific truth rather than Talent truth or cross-context SQL reconstruction. -The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, prove requester/reviewer identities are distinct authoritative actors, verify cross-sectional/longitudinal weight eligibility against the estimand, and verify typed weight-adjustment receipts rather than trusting caller-supplied evidence labels or digests. +The NIST Privacy Framework 1.0 is used narrowly as a privacy-risk-management basis for expressing and verifying data-processing requirements across organizational roles and contracts; as of the 2026-09-17 check, NIST still presents Privacy Framework 1.1 as an Initial Public Draft rather than a final replacement. This ADR does not treat the voluntary framework as employment law or infer legal permission from a NIST profile. + +The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, prove requester/reviewer identities are distinct authoritative actors, verify cross-sectional/longitudinal weight eligibility against the estimand, and verify typed weight-adjustment receipts plus any calibration auxiliary purpose/owner authorization rather than trusting caller-supplied evidence labels or digests. ## Consequences @@ -53,7 +55,7 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - Cross-sectional and longitudinal weights cannot be interchanged when target population, reference duration, eligible cases, or final weight artifact differ from the estimand. - The ordered weight transformation chain is independently digest-correlatable without centralizing case-level weights or auxiliary attributes. - Nonresponse cannot silently drop refusal/unreachable/ineligible/failure dispositions behind one opaque adjustment label. -- Calibration/raking cannot silently float benchmark ownership or hide fallback as successful convergence. +- Calibration/raking cannot silently float benchmark ownership, omit the scientific-use purpose/released-owner authorization coordinate, or hide fallback as successful convergence. - Trimming/bounding/winsorization cannot silently alter final point weights without an immutable rule/configuration and affected-case receipt. - Human interpretation remains explicit and separate from numerical output. - The dedicated-writer boundary remains intact: Orgmetra consumes only a pinned foreign revision/contract boundary and never mutates fast-mlsirm. @@ -62,13 +64,13 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - This slice does not execute a statistical model, estimate validity, correct for measurement error/range restriction, evaluate adverse impact, or assert legal compliance. - Sampling design, estimator choice, missing-data treatment, reliability evidence, multiplicity, transportability, fairness analysis, and model diagnostics must be encoded in the referenced analysis plan and reviewed before execution. -- The current #407 slice validates generic ordered weight provenance, weighted-result binding, cross-sectional/longitudinal eligibility congruence, disposition-aware nonresponse evidence, benchmark/termination-aware calibration evidence, and trimming/bounding rule provenance; it does not yet make durable typed-receipt resolution or sensitive auxiliary-owner enforcement fully executable. +- The current #407 slice validates generic ordered weight provenance, weighted-result binding, cross-sectional/longitudinal eligibility congruence, disposition-aware nonresponse evidence, purpose/owner-coordinate/benchmark/termination-aware calibration evidence, and trimming/bounding rule provenance; it does not yet make durable typed-receipt/authorization resolution or released auxiliary-owner exchange fully executable. - The generic result envelope does not invent an estimator-specific minimum complete-case count. Whether a converged estimator is identified under a particular missing-data design belongs to the reviewed analysis plan and numerical-worker contract; Orgmetra fails closed on impossible aggregate counts without silently replacing that foreign scientific contract with complete-case analysis. -- The package validates the result envelope, but a future execution adapter must still re-resolve the handoff references, verify the result provenance artifact, resolve typed adjustment digests to released owner evidence, and attach evidence only after accountable human review. +- The package validates the result envelope, but a future execution adapter must still re-resolve the handoff references, verify the result provenance artifact, resolve typed adjustment and purpose-authorization digests to released owner evidence, and attach evidence only after accountable human review. ## Verification -The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. #407 adds RED/GREEN coverage for deterministic value-minimized final-weight receipts, exact sampling-receipt binding, contiguous transformation lineage, append-only correction linkage, fail-closed weighted-result binding to separate point-weight and variance-design receipts, cross-sectional/longitudinal scope and target-population/reference-duration/case-set/final-artifact congruence, explicit nonresponse disposition treatment, calibration benchmark/constraint/termination provenance, fallback-rule disclosure, trimming/bounding rule/configuration and affected-case provenance, no-op trim/bound rejection, and typed adjustment-evidence matching. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. +The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. #407 adds RED/GREEN coverage for deterministic value-minimized final-weight receipts, exact sampling-receipt binding, contiguous transformation lineage, append-only correction linkage, fail-closed weighted-result binding to separate point-weight and variance-design receipts, cross-sectional/longitudinal scope and target-population/reference-duration/case-set/final-artifact congruence, explicit nonresponse disposition treatment, calibration benchmark/constraint/termination provenance, calibration scientific-purpose/released-owner authorization coordinates, fallback-rule disclosure, trimming/bounding rule/configuration and affected-case provenance, no-op trim/bound rejection, and typed adjustment-evidence matching. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. ## References From bba2ce0148e71df7870367aef69f3a92274e0f28 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 07:15:25 +0900 Subject: [PATCH 090/158] docs(validity): trace calibration purpose authority --- docs/traceability/validation-analysis-handoff.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/traceability/validation-analysis-handoff.md b/docs/traceability/validation-analysis-handoff.md index b954e071c..7a86dcc1e 100644 --- a/docs/traceability/validation-analysis-handoff.md +++ b/docs/traceability/validation-analysis-handoff.md @@ -17,7 +17,7 @@ Can an organization send one exact, reviewable validation study to its statistic | Final point-weight provenance | exact estimand/target/window/reference-duration and case-set digests, source/sampling receipt digests, base-weight evidence, ordered digest-linked adjustments, final weight artifact, append-only correction lineage | `FinalAnalysisWeightReceipt` deterministic/value-minimized regressions plus chain/correction fail-closed tests | | Weight eligibility congruence | governed `cross_sectional` or `longitudinal` scope plus exact target population, reference-duration evidence, eligible case set, and final point-weight artifact | `test_weight_eligibility_receipt.py` plus `FinalAnalysisWeightReceipt` mismatch/longitudinal-match regressions | | Nonresponse adjustment evidence | versioned response/disposition receipt, exact adjustment population, controlled method/configuration, explicit ineligible/unknown/unavailable treatment, input/output weight artifacts | `NonresponseAdjustmentReceipt` deterministic/value-minimized tests plus undocumented-treatment/no-op/version rejection | -| Calibration/raking evidence | target population/window, purpose-bound auxiliary projection, authoritative benchmark receipt, algorithm/version, constraints, input/output artifacts, explicit converged/fallback state and immutable fallback rule when used | `CalibrationAdjustmentReceipt` benchmark/termination/fallback regressions | +| Calibration/raking evidence | target population/window, purpose-bound auxiliary projection, scientific-use purpose reference/digest, released owner-contract coordinate/version, authorization-receipt digest, authoritative benchmark receipt, algorithm/version, constraints, input/output artifacts, explicit converged/fallback state and immutable fallback rule when used | `CalibrationAdjustmentReceipt` purpose/owner/authorization/benchmark/termination/fallback regressions | | Trimming/bounding evidence | versioned trimming/bounding rule, reproducible rule configuration digest, exact affected semantic occurrence set and count, input/output weight artifacts | `test_trimming_bounding_receipt.py` deterministic/value-minimized, no-op, missing-provenance, and typed-kind regressions | | Typed adjustment congruence | known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization adjustment codes must identify the matching typed receipt family through `evidence_kind` | specialized adjustment evidence-kind regressions | | Weighted-result congruence | `weighted_design_based` result must bind the exact final point-weight receipt and a separate #406 variance-design receipt; `unweighted` result must bind neither | `test_analysis_weight_result_binding.py` | @@ -28,7 +28,7 @@ Can an organization send one exact, reviewable validation study to its statistic ## #407 boundary still open -The active branch now makes final-weight eligibility and three adjustment families executable rather than leaving them as opaque labels. Weight eligibility is explicitly cross-sectional or longitudinal and must match the estimand's target population, reference duration, eligible cases, and final point-weight artifact. Nonresponse adjustment evidence is disposition-aware; calibration/raking/post-stratification evidence is bound to an authoritative benchmark, purpose-limited auxiliary projection, constraints, and explicit convergence/fallback state; and trimming/bounding/winsorization evidence is bound to a versioned rule/configuration plus the exact affected occurrence set. This still does not complete #407. Durable owner-side verification that each adjustment digest resolves to the typed receipt claimed by `evidence_kind`, sensitive auxiliary-variable purpose enforcement, and released auxiliary evidence exchange remain open. `workforce_validation` owns those scientific contracts and their released result; `talent_management` must not compute or reconstruct final analysis weights. +The active branch now makes final-weight eligibility and three adjustment families executable rather than leaving them as opaque labels. Weight eligibility is explicitly cross-sectional or longitudinal and must match the estimand's target population, reference duration, eligible cases, and final point-weight artifact. Nonresponse adjustment evidence is disposition-aware; calibration/raking/post-stratification evidence is bound to an authoritative benchmark, purpose-limited auxiliary projection, explicit scientific-use purpose, released owner-contract coordinate and authorization-receipt digest, constraints, and explicit convergence/fallback state; and trimming/bounding/winsorization evidence is bound to a versioned rule/configuration plus the exact affected occurrence set. This still does not complete #407. Durable owner-side resolution must prove that each adjustment and auxiliary-authorization digest resolves to the released typed receipt/contract it claims and that the released authorization actually permits the projection for the stated purpose; released auxiliary evidence exchange also remains open. `workforce_validation` owns those scientific contracts and their released result; `talent_management` must not compute or reconstruct final analysis weights. ## Maturity From 189b8cbd7e35b016241071fc2fa3f085b165e818 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 07:16:02 +0900 Subject: [PATCH 091/158] docs(validity): doctor privacy purpose authority --- docs/doctoring/validation-analysis-handoff-references.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/doctoring/validation-analysis-handoff-references.md b/docs/doctoring/validation-analysis-handoff-references.md index f107c70dc..1a337f705 100644 --- a/docs/doctoring/validation-analysis-handoff-references.md +++ b/docs/doctoring/validation-analysis-handoff-references.md @@ -1,6 +1,6 @@ # Validation-analysis handoff references -Material decisions for ADR 0027 were checked against the following primary/authoritative sources on 2026-08-21. Regulatory currency was rechecked on 2026-08-29. The #407 analysis-weight evidence boundary was checked on 2026-09-17 against the primary calibration paper and current U.S. Census methodological/quality documentation. Fixed publication identifiers are retained where possible so an auditor can reproduce the cited text even when agency web pages change. +Material decisions for ADR 0027 were checked against the following primary/authoritative sources on 2026-08-21. Regulatory currency was rechecked on 2026-08-29. The #407 analysis-weight evidence boundary was checked on 2026-09-17 against the primary calibration paper, current U.S. Census methodological/quality documentation, and the current final NIST Privacy Framework. Fixed publication identifiers are retained where possible so an auditor can reproduce the cited text even when agency web pages change. ## APA 7 references @@ -16,6 +16,8 @@ U.S. Census Bureau. (2022, August 18). *Survey of Income and Program Participati U.S. Census Bureau. (2026). *2025 Survey of Income and Program Participation users' guide* (August 2026 revision), pp. 156–157. https://www2.census.gov/programs-surveys/sipp/tech-documentation/methodology/2025_SIPP_Users_Guide.pdf +Boeckl, K., & Lefkovitz, N. (2020). *NIST Privacy Framework: A tool for improving privacy through enterprise risk management, Version 1.0* (NIST CSWP 01162020). National Institute of Standards and Technology. https://doi.org/10.6028/NIST.CSWP.01162020 + ContextualWisdomLab. (2026). *fast-mlsirm* (Commit 04d0bc21a2a20693bcf16108cd76d394fe844d23) [Computer software]. GitHub. https://github.com/ContextualWisdomLab/fast-mlsirm/tree/04d0bc21a2a20693bcf16108cd76d394fe844d23 Tabassi, E. (2023). *Artificial intelligence risk management framework (AI RMF 1.0)* (NIST AI 100-1). National Institute of Standards and Technology. https://doi.org/10.6028/NIST.AI.100-1 @@ -30,5 +32,6 @@ Office of Personnel Management. (2026). *Removal of references to the Uniform Gu - Deville and Särndal show that calibrated weights are produced by modifying ordinary inverse-inclusion-probability weights under explicit distance measures and calibration equations. ADR 0027 uses that narrow result to justify treating final adjusted point weights as a separately versioned scientific artifact rather than assuming `1/π_i` and calibrated weights are interchangeable. It does not mandate one calibration estimator for Orgmetra. - Census Statistical Quality Standard D1 requires estimates and variances to account for sample design and post-sampling weighting adjustments. The SIPP methodology illustrates that final weights can combine base selection, nonresponse, longitudinal/panel, and post-stratification/calibration adjustments. These sources support provenance/reproducibility requirements only; SIPP-specific weights are not imported as Orgmetra rules. - The August 2026 revision of the 2025 SIPP Users' Guide states that choosing a weight depends on the population to which results apply and the duration of interest, distinguishes cross-sectional monthly analysis from longitudinal multi-year analysis, and identifies explicit two-, three-, and four-year reference periods for longitudinal weights. ADR 0027 uses this only to justify fail-closed target-population/reference-duration congruence for `WeightEligibilityReceipt`; it does not adopt SIPP variable names, cohorts, or estimators as Orgmetra domain truth. +- The final NIST Privacy Framework 1.0 is used narrowly to support explicit, verifiable privacy requirements across roles in a data-processing ecosystem and life-cycle privacy-risk management. As checked on 2026-09-17, NIST's official 1.1 project page still presents Version 1.1 as an Initial Public Draft rather than a final replacement, so this ADR does not cite the draft as settled authority. The framework is voluntary and jurisdiction-agnostic; it is not employment-law permission. Orgmetra uses it only to justify carrying scientific-use purpose, released owner-contract coordinates and authorization evidence without copying sensitive auxiliary values across bounded contexts. - The fast-mlsirm commit is recorded as a read-only dependency coordinate only. This Orgmetra slice does not modify or duplicate its numerical implementation. - NIST AI RMF's govern, map, measure, and manage functions support preserving backend, precision, provenance, convergence, and human-review fields as inspectable result evidence rather than treating a model response as an autonomous decision. From a9e4d0eaf289d3ad4f3226f77843e5a571940ee2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 07:16:23 +0900 Subject: [PATCH 092/158] docs(validity): record purpose-bound calibration authority --- packages/validity-analysis/CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/validity-analysis/CHANGELOG.md b/packages/validity-analysis/CHANGELOG.md index 5a552d808..1739d8c13 100644 --- a/packages/validity-analysis/CHANGELOG.md +++ b/packages/validity-analysis/CHANGELOG.md @@ -13,8 +13,8 @@ - Add a deterministic, row-value-minimized `FinalAnalysisWeightReceipt` that binds an exact estimand to source/sampling evidence, base-weight provenance, an ordered digest-linked adjustment chain, the final point-weight artifact, and append-only correction lineage. - Add `WeightEligibilityReceipt` and fail closed unless cross-sectional/longitudinal scope, target population, reference duration, eligible case set, and final point-weight artifact match the estimand-side final-weight receipt exactly. - Distinguish unweighted from weighted design-based results and fail closed unless a weighted result separately binds the exact final analysis-weight receipt and variance-design receipt used. -- Add typed `NonresponseAdjustmentReceipt` evidence with explicit disposition treatment and typed `CalibrationAdjustmentReceipt` evidence bound to purpose-limited auxiliary projections, authoritative benchmark receipts, constraints, and explicit convergence/fallback state. +- Add typed `NonresponseAdjustmentReceipt` evidence with explicit disposition treatment and typed `CalibrationAdjustmentReceipt` evidence bound to purpose-limited auxiliary projections, explicit scientific-use purpose, released owner-contract coordinates, authorization-receipt digest, authoritative benchmark receipts, constraints, and explicit convergence/fallback state. - Add typed `TrimmingBoundingAdjustmentReceipt` evidence that binds a versioned rule/configuration, exact affected semantic occurrence set and count, and input/output weight artifacts; known trimming/bounding/winsorization adjustments must name that evidence family. - Require known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization adjustment codes to identify the matching typed evidence kind rather than collapse into a generic opaque adjustment digest. -- Keep durable owner-side typed-receipt resolution, sensitive auxiliary purpose enforcement, and released auxiliary evidence exchange explicitly incomplete under #407. +- Keep durable owner-side typed-receipt and auxiliary-authorization resolution plus released auxiliary evidence exchange explicitly incomplete under #407. - Make `Validity Analysis Handoff Quality` retrigger on shared repository Python/test/clean-checkout configuration, with an executable regression preventing stale package-quality evidence after shared tooling changes. From af3c59b86a205daed883e4d769be5c281bb7e5fe Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 09:05:55 +0900 Subject: [PATCH 093/158] test(validity): require owner-corroboratable calibration use evidence --- .../tests/test_weight_adjustment_semantics.py | 42 +++++++++++++++---- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/packages/validity-analysis/tests/test_weight_adjustment_semantics.py b/packages/validity-analysis/tests/test_weight_adjustment_semantics.py index 769722a75..329d4f5a1 100644 --- a/packages/validity-analysis/tests/test_weight_adjustment_semantics.py +++ b/packages/validity-analysis/tests/test_weight_adjustment_semantics.py @@ -1,6 +1,6 @@ """Scientific contracts for nonresponse and calibration weight adjustments.""" -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone import pytest @@ -46,19 +46,25 @@ def nonresponse_receipt(**overrides: object) -> NonresponseAdjustmentReceipt: def calibration_receipt(**overrides: object) -> CalibrationAdjustmentReceipt: - """Return one benchmark- and purpose-bound converged calibration receipt.""" + """Return one owner-corroboratable converged calibration receipt.""" values: dict[str, object] = { "tenant_record_id": TENANT, "receipt_reference": "calibration_adjustment_receipt:33333333-3333-4333-8333-333333333333", "target_population_digest": DIGEST_A, "analysis_window_reference": "analysis_window:44444444-4444-4444-8444-444444444444", + "auxiliary_authority_reference": "scientific_auxiliary_authority:55555555-5555-4555-8555-555555555550", "auxiliary_projection_reference": "calibration_auxiliary_projection:55555555-5555-4555-8555-555555555555", "auxiliary_projection_digest": DIGEST_B, "auxiliary_purpose_reference": "scientific_data_use_purpose:55555555-5555-4555-8555-555555555556", "auxiliary_purpose_digest": DIGEST_3, "auxiliary_owner_contract_reference": "released_owner_contract:55555555-5555-4555-8555-555555555557", "auxiliary_owner_contract_version": 1, + "auxiliary_owner_contract_digest": DIGEST_1, + "auxiliary_authorization_receipt_reference": "scientific_data_authorization:55555555-5555-4555-8555-555555555558", "auxiliary_authorization_receipt_digest": DIGEST_4, + "auxiliary_scientific_use_receipt_reference": "scientific_use_receipt:55555555-5555-4555-8555-555555555559", + "auxiliary_scientific_use_receipt_digest": DIGEST_2, + "auxiliary_scientific_use_at": NOW, "benchmark_receipt_reference": "calibration_benchmark_receipt:66666666-6666-4666-8666-666666666666", "benchmark_receipt_digest": DIGEST_C, "algorithm_reference": "calibration_algorithm:77777777-7777-4777-8777-777777777777", @@ -107,15 +113,19 @@ def test_nonresponse_receipt_is_value_minimized_and_disposition_aware() -> None: nonresponse_receipt(evidence_version=True) -def test_calibration_receipt_binds_owner_benchmark_purpose_and_termination_state() -> None: - """Keep benchmark, purpose authorization, and fallback semantics explicit and immutable.""" +def test_calibration_receipt_binds_owner_authority_use_and_termination_state() -> None: + """Keep owner-corroboratable authority, benchmark, and fallback semantics explicit.""" candidate = calibration_receipt() + canonical = candidate.canonical_json() assert candidate.sha256_digest() == calibration_receipt().sha256_digest() - assert f'"benchmark_receipt_digest":"{DIGEST_C}"' in candidate.canonical_json() - assert f'"auxiliary_purpose_digest":"{DIGEST_3}"' in candidate.canonical_json() - assert f'"auxiliary_authorization_receipt_digest":"{DIGEST_4}"' in candidate.canonical_json() - assert '"termination_code":"converged"' in candidate.canonical_json() - assert "protected_attribute" not in candidate.canonical_json() + assert f'"benchmark_receipt_digest":"{DIGEST_C}"' in canonical + assert f'"auxiliary_purpose_digest":"{DIGEST_3}"' in canonical + assert f'"auxiliary_owner_contract_digest":"{DIGEST_1}"' in canonical + assert f'"auxiliary_authorization_receipt_digest":"{DIGEST_4}"' in canonical + assert f'"auxiliary_scientific_use_receipt_digest":"{DIGEST_2}"' in canonical + assert '"auxiliary_scientific_use_at":"2026-09-17T05:00:00Z"' in canonical + assert '"termination_code":"converged"' in canonical + assert "protected_attribute" not in canonical assert repr(candidate) == "CalibrationAdjustmentReceipt()" fallback_reference = "calibration_fallback_rule:99999999-9999-4999-8999-999999999999" @@ -126,12 +136,26 @@ def test_calibration_receipt_binds_owner_benchmark_purpose_and_termination_state ) assert f'"fallback_rule_digest":"{DIGEST_2}"' in fallback.canonical_json() + with pytest.raises(ValueError, match="auxiliary_authority_reference"): + calibration_receipt(auxiliary_authority_reference="authority-v1") with pytest.raises(ValueError, match="auxiliary_purpose_digest"): calibration_receipt(auxiliary_purpose_digest="purpose-v1") with pytest.raises(ValueError, match="auxiliary_owner_contract_version"): calibration_receipt(auxiliary_owner_contract_version=0) + with pytest.raises(ValueError, match="auxiliary_owner_contract_digest"): + calibration_receipt(auxiliary_owner_contract_digest="owner-v1") + with pytest.raises(ValueError, match="auxiliary_authorization_receipt_reference"): + calibration_receipt(auxiliary_authorization_receipt_reference="authorization-v1") with pytest.raises(ValueError, match="auxiliary_authorization_receipt_digest"): calibration_receipt(auxiliary_authorization_receipt_digest="authorization-v1") + with pytest.raises(ValueError, match="auxiliary_scientific_use_receipt_reference"): + calibration_receipt(auxiliary_scientific_use_receipt_reference="use-v1") + with pytest.raises(ValueError, match="auxiliary_scientific_use_receipt_digest"): + calibration_receipt(auxiliary_scientific_use_receipt_digest="use-v1") + with pytest.raises(ValueError, match="auxiliary_scientific_use_at"): + calibration_receipt(auxiliary_scientific_use_at="2026-09-17T05:00:00Z") + with pytest.raises(ValueError, match="cannot be later"): + calibration_receipt(auxiliary_scientific_use_at=NOW + timedelta(seconds=1)) with pytest.raises(ValueError, match="termination_code"): calibration_receipt(termination_code="failed") with pytest.raises(ValueError, match="termination_code"): From 84fe298ca703fad121a085923aa1c4794b70a4ad Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 09:06:57 +0900 Subject: [PATCH 094/158] fix(validity): bind calibration receipt to exact scientific-use authority --- .../src/orgmetra_validity_analysis/weights.py | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py b/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py index aa99fb122..ad2fa0236 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py @@ -122,19 +122,25 @@ def sha256_digest(self) -> str: @dataclass(frozen=True, slots=True, repr=False) class CalibrationAdjustmentReceipt: - """Bind calibration to owner benchmarks, purpose authority, and termination evidence.""" + """Bind calibration to owner authority, benchmarks, and termination evidence.""" tenant_record_id: str receipt_reference: str target_population_digest: str analysis_window_reference: str + auxiliary_authority_reference: str auxiliary_projection_reference: str auxiliary_projection_digest: str auxiliary_purpose_reference: str auxiliary_purpose_digest: str auxiliary_owner_contract_reference: str auxiliary_owner_contract_version: int + auxiliary_owner_contract_digest: str + auxiliary_authorization_receipt_reference: str auxiliary_authorization_receipt_digest: str + auxiliary_scientific_use_receipt_reference: str + auxiliary_scientific_use_receipt_digest: str + auxiliary_scientific_use_at: datetime benchmark_receipt_reference: str benchmark_receipt_digest: str algorithm_reference: str @@ -161,6 +167,11 @@ def __post_init__(self) -> None: "analysis_window", "analysis_window_reference", ) + _validate_reference( + self.auxiliary_authority_reference, + "scientific_auxiliary_authority", + "auxiliary_authority_reference", + ) _validate_reference( self.auxiliary_projection_reference, "calibration_auxiliary_projection", @@ -180,6 +191,16 @@ def __post_init__(self) -> None: self.auxiliary_owner_contract_version, "auxiliary_owner_contract_version", ) + _validate_reference( + self.auxiliary_authorization_receipt_reference, + "scientific_data_authorization", + "auxiliary_authorization_receipt_reference", + ) + _validate_reference( + self.auxiliary_scientific_use_receipt_reference, + "scientific_use_receipt", + "auxiliary_scientific_use_receipt_reference", + ) _validate_reference( self.benchmark_receipt_reference, "calibration_benchmark_receipt", @@ -194,7 +215,9 @@ def __post_init__(self) -> None: "target_population_digest", "auxiliary_projection_digest", "auxiliary_purpose_digest", + "auxiliary_owner_contract_digest", "auxiliary_authorization_receipt_digest", + "auxiliary_scientific_use_receipt_digest", "benchmark_receipt_digest", "constraints_digest", "input_weight_artifact_digest", @@ -224,9 +247,16 @@ def __post_init__(self) -> None: raise ValueError( "output_weight_artifact_digest must identify the calibrated weight artifact" ) + scientific_use_at = _freeze_timestamp( + self.auxiliary_scientific_use_at, + "auxiliary_scientific_use_at", + ) constructed_at = _freeze_timestamp(self.constructed_at, "constructed_at") + if scientific_use_at > constructed_at: + raise ValueError("auxiliary_scientific_use_at cannot be later than constructed_at") if type(self.evidence_version) is not int or self.evidence_version != 1: raise ValueError("evidence_version must remain 1") + object.__setattr__(self, "auxiliary_scientific_use_at", scientific_use_at) object.__setattr__(self, "constructed_at", constructed_at) def __repr__(self) -> str: @@ -239,13 +269,22 @@ def canonical_json(self) -> str: "algorithm_reference": self.algorithm_reference, "algorithm_version": self.algorithm_version, "analysis_window_reference": self.analysis_window_reference, + "auxiliary_authority_reference": self.auxiliary_authority_reference, "auxiliary_authorization_receipt_digest": self.auxiliary_authorization_receipt_digest, + "auxiliary_authorization_receipt_reference": self.auxiliary_authorization_receipt_reference, + "auxiliary_owner_contract_digest": self.auxiliary_owner_contract_digest, "auxiliary_owner_contract_reference": self.auxiliary_owner_contract_reference, "auxiliary_owner_contract_version": self.auxiliary_owner_contract_version, "auxiliary_projection_digest": self.auxiliary_projection_digest, "auxiliary_projection_reference": self.auxiliary_projection_reference, "auxiliary_purpose_digest": self.auxiliary_purpose_digest, "auxiliary_purpose_reference": self.auxiliary_purpose_reference, + "auxiliary_scientific_use_at": _canonical_timestamp( + self.auxiliary_scientific_use_at, + "auxiliary_scientific_use_at", + ), + "auxiliary_scientific_use_receipt_digest": self.auxiliary_scientific_use_receipt_digest, + "auxiliary_scientific_use_receipt_reference": self.auxiliary_scientific_use_receipt_reference, "benchmark_receipt_digest": self.benchmark_receipt_digest, "benchmark_receipt_reference": self.benchmark_receipt_reference, "constraints_digest": self.constraints_digest, From 44a5425793fdac956d3fdb2b723e48dea9d46e45 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 09:08:29 +0900 Subject: [PATCH 095/158] docs(validity): reconcile calibration authority receipt contract --- packages/validity-analysis/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/validity-analysis/README.md b/packages/validity-analysis/README.md index 6af6d1eb9..9b56302d2 100644 --- a/packages/validity-analysis/README.md +++ b/packages/validity-analysis/README.md @@ -12,7 +12,7 @@ The resulting canonical JSON is digest-addressable, contains no raw person-level `WeightEligibilityReceipt` makes cross-sectional versus longitudinal use machine-checkable rather than an opaque weight label. It binds the final weight artifact to one governed scope (`cross_sectional` or `longitudinal`), target population, reference-duration evidence, and eligible-case set. `FinalAnalysisWeightReceipt` fails closed unless those fields match the estimand and the same final point-weight artifact exactly, so a longitudinal weight cannot silently support a cross-sectional estimand or a different reference duration. -Nonresponse and calibration are no longer allowed to hide behind an undifferentiated adjustment digest. `NonresponseAdjustmentReceipt` binds the adjustment to a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, and explicit treatment codes for ineligible, unknown, and unavailable cases. `CalibrationAdjustmentReceipt` binds the adjustment to a target population/window, purpose-bound auxiliary projection, explicit scientific-use purpose evidence, a released owner-contract coordinate and authorization-receipt digest, authoritative benchmark receipt, algorithm/version, constraints digest, and explicit `converged` or `fallback_applied` termination evidence. A fallback must identify its versioned rule; a nonconverged calibration cannot masquerade as an accepted calibration receipt. These fields preserve purpose and release coordinates without copying protected auxiliary values. The durable owner service must still resolve those coordinates and prove that the authorization receipt actually permits the projection for the stated scientific purpose. +Nonresponse and calibration are no longer allowed to hide behind an undifferentiated adjustment digest. `NonresponseAdjustmentReceipt` binds the adjustment to a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, and explicit treatment codes for ineligible, unknown, and unavailable cases. `CalibrationAdjustmentReceipt` binds the adjustment to a target population/window, purpose-bound auxiliary projection, exact scientific-use purpose, scientific auxiliary-authority reference, released owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and owner-correlatable use instant, authoritative benchmark receipt, algorithm/version, constraints digest, and explicit `converged` or `fallback_applied` termination evidence. Its scientific-use instant cannot be later than receipt construction. A fallback must identify its versioned rule; a nonconverged calibration cannot masquerade as an accepted calibration receipt. These fields preserve the complete opaque correlation tuple expected by the canonical Workforce Validation authority resolver without copying protected auxiliary values. The leaf receipt still does not authenticate those coordinates itself: the durable owner service must resolve the released evidence and prove that the exact projection was authorized for that scientific purpose and use receipt. Trimming, bounding, and winsorization have a separate provenance family rather than falling back to a generic adjustment label. `TrimmingBoundingAdjustmentReceipt` binds the exact versioned rule, its reproducible configuration digest, the semantic occurrence set and count of cases actually affected, and the input/output weight artifacts. A declared trim/bound transform must change artifact identity. Known trimming/bounding/winsorization adjustment codes fail closed unless `evidence_kind` names this typed receipt family. @@ -30,13 +30,13 @@ Point-estimation semantics are explicit. `unweighted` results cannot carry weigh - It does **not** claim that a selection procedure is valid. - It does **not** interpret adverse impact. - It does **not** authorize hiring, promotion, termination, compensation, or another employment decision. -- It does **not** yet complete #407. Durable owner-side verification that adjustment and auxiliary-authorization digests resolve to the released typed receipts/contracts they claim, enforcement that the resolved auxiliary projection is authorized for the exact scientific purpose, and released auxiliary evidence exchange remain open work in `workforce_validation`. +- It does **not** yet complete #407. Durable owner-side verification that adjustment and auxiliary-authority coordinates resolve to the released typed receipts/contracts they claim, enforcement that the resolved auxiliary projection is authorized for the exact scientific purpose/use receipt/time, and released auxiliary evidence exchange remain open work in `workforce_validation`. The fast-mlsirm repository remains a dedicated-writer dependency. This package records only the immutable revision reviewed for the handoff. Durable Workforce Validation registry/API/persistence remains on the canonical #235/#248 owner path; this package does not create a parallel service. ## Host obligations -Before an approved offline validation worker executes the handoff, the Orgmetra host must re-resolve every reference inside `tenant_record_id` and prove that the predictor, criterion, population, and policy evidence belong to the exact validation study and Job. The requester and reviewer must resolve to distinct authoritative actors. For weighted design-based inference, the host must also verify the final point-weight receipt, its matching cross-sectional/longitudinal weight-eligibility receipt, its upstream source/sampling evidence, each typed adjustment receipt identified by the ordered chain, the calibration auxiliary purpose and released owner authorization when applicable, and the separately bound variance-design evidence rather than trusting caller-supplied labels or digests. A numerical result is scientific evidence for accountable human interpretation, never an autonomous employment decision. +Before an approved offline validation worker executes the handoff, the Orgmetra host must re-resolve every reference inside `tenant_record_id` and prove that the predictor, criterion, population, and policy evidence belong to the exact validation study and Job. The requester and reviewer must resolve to distinct authoritative actors. For weighted design-based inference, the host must also verify the final point-weight receipt, its matching cross-sectional/longitudinal weight-eligibility receipt, its upstream source/sampling evidence, each typed adjustment receipt identified by the ordered chain, the calibration auxiliary authority/purpose/released owner contract/authorization/scientific-use tuple when applicable, and the separately bound variance-design evidence rather than trusting caller-supplied labels or digests. A numerical result is scientific evidence for accountable human interpretation, never an autonomous employment decision. ## Verification From 4eca586bde8168eb8839d86319a426375f189c74 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 09:08:54 +0900 Subject: [PATCH 096/158] docs(traceability): bind calibration receipt to owner authority tuple --- docs/traceability/validation-analysis-handoff.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/traceability/validation-analysis-handoff.md b/docs/traceability/validation-analysis-handoff.md index 7a86dcc1e..fc37e7ae3 100644 --- a/docs/traceability/validation-analysis-handoff.md +++ b/docs/traceability/validation-analysis-handoff.md @@ -17,7 +17,7 @@ Can an organization send one exact, reviewable validation study to its statistic | Final point-weight provenance | exact estimand/target/window/reference-duration and case-set digests, source/sampling receipt digests, base-weight evidence, ordered digest-linked adjustments, final weight artifact, append-only correction lineage | `FinalAnalysisWeightReceipt` deterministic/value-minimized regressions plus chain/correction fail-closed tests | | Weight eligibility congruence | governed `cross_sectional` or `longitudinal` scope plus exact target population, reference-duration evidence, eligible case set, and final point-weight artifact | `test_weight_eligibility_receipt.py` plus `FinalAnalysisWeightReceipt` mismatch/longitudinal-match regressions | | Nonresponse adjustment evidence | versioned response/disposition receipt, exact adjustment population, controlled method/configuration, explicit ineligible/unknown/unavailable treatment, input/output weight artifacts | `NonresponseAdjustmentReceipt` deterministic/value-minimized tests plus undocumented-treatment/no-op/version rejection | -| Calibration/raking evidence | target population/window, purpose-bound auxiliary projection, scientific-use purpose reference/digest, released owner-contract coordinate/version, authorization-receipt digest, authoritative benchmark receipt, algorithm/version, constraints, input/output artifacts, explicit converged/fallback state and immutable fallback rule when used | `CalibrationAdjustmentReceipt` purpose/owner/authorization/benchmark/termination/fallback regressions | +| Calibration/raking evidence | target population/window, purpose-bound auxiliary projection, scientific-use purpose reference/digest, scientific auxiliary-authority reference, released owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and use instant, authoritative benchmark receipt, algorithm/version, constraints, input/output artifacts, explicit converged/fallback state and immutable fallback rule when used | `CalibrationAdjustmentReceipt` authority/purpose/owner/authorization/scientific-use/benchmark/termination/fallback regressions; scientific-use time cannot be later than receipt construction | | Trimming/bounding evidence | versioned trimming/bounding rule, reproducible rule configuration digest, exact affected semantic occurrence set and count, input/output weight artifacts | `test_trimming_bounding_receipt.py` deterministic/value-minimized, no-op, missing-provenance, and typed-kind regressions | | Typed adjustment congruence | known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization adjustment codes must identify the matching typed receipt family through `evidence_kind` | specialized adjustment evidence-kind regressions | | Weighted-result congruence | `weighted_design_based` result must bind the exact final point-weight receipt and a separate #406 variance-design receipt; `unweighted` result must bind neither | `test_analysis_weight_result_binding.py` | @@ -28,10 +28,10 @@ Can an organization send one exact, reviewable validation study to its statistic ## #407 boundary still open -The active branch now makes final-weight eligibility and three adjustment families executable rather than leaving them as opaque labels. Weight eligibility is explicitly cross-sectional or longitudinal and must match the estimand's target population, reference duration, eligible cases, and final point-weight artifact. Nonresponse adjustment evidence is disposition-aware; calibration/raking/post-stratification evidence is bound to an authoritative benchmark, purpose-limited auxiliary projection, explicit scientific-use purpose, released owner-contract coordinate and authorization-receipt digest, constraints, and explicit convergence/fallback state; and trimming/bounding/winsorization evidence is bound to a versioned rule/configuration plus the exact affected occurrence set. This still does not complete #407. Durable owner-side resolution must prove that each adjustment and auxiliary-authorization digest resolves to the released typed receipt/contract it claims and that the released authorization actually permits the projection for the stated purpose; released auxiliary evidence exchange also remains open. `workforce_validation` owns those scientific contracts and their released result; `talent_management` must not compute or reconstruct final analysis weights. +The active branch makes final-weight eligibility and three adjustment families executable rather than leaving them as opaque labels. Calibration/raking/post-stratification evidence now carries the complete opaque correlation tuple needed by the canonical Workforce Validation authority resolver: auxiliary authority and projection, exact scientific purpose, released owner-contract identity/version/digest, authorization receipt identity/digest, scientific-use receipt identity/digest and owner-correlatable use instant. This removes the prior gap where owner-contract digest, authorization reference and scientific-use identity/time would have had to be supplied outside the immutable calibration receipt. The leaf still cannot self-authenticate those coordinates. Durable #235/#248 owner-side resolution must prove that they resolve to released/versioned evidence and that the exact projection is authorized for the exact scientific purpose and use receipt; released auxiliary evidence exchange also remains open. `workforce_validation` owns those scientific contracts and their released result; `talent_management` must not compute or reconstruct final analysis weights. ## Maturity `implemented_on_active_pr`. -Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. The active package validates the returned numerical/provenance envelope plus the current #407 final-weight/result-binding, weight-eligibility, and typed adjustment-evidence contracts, but protected Orgmetra evidence still requires host re-resolution, durable owner evidence exchange, result-artifact verification, terminal checks, independent review, and accountable human interpretation. +Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. The active package validates the returned numerical/provenance envelope plus the current #407 final-weight/result-binding, weight-eligibility, typed adjustment-evidence, and calibration-authority correlation contracts, but protected Orgmetra evidence still requires host re-resolution, durable owner evidence exchange, result-artifact verification, terminal checks, independent review, and accountable human interpretation. From ea6e6d6baa90cc2b76a667ccc9afaa1f93999fb1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 09:09:06 +0900 Subject: [PATCH 097/158] docs(changelog): record calibration authority correlation repair --- packages/validity-analysis/CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/validity-analysis/CHANGELOG.md b/packages/validity-analysis/CHANGELOG.md index 1739d8c13..da77bfaa0 100644 --- a/packages/validity-analysis/CHANGELOG.md +++ b/packages/validity-analysis/CHANGELOG.md @@ -13,8 +13,8 @@ - Add a deterministic, row-value-minimized `FinalAnalysisWeightReceipt` that binds an exact estimand to source/sampling evidence, base-weight provenance, an ordered digest-linked adjustment chain, the final point-weight artifact, and append-only correction lineage. - Add `WeightEligibilityReceipt` and fail closed unless cross-sectional/longitudinal scope, target population, reference duration, eligible case set, and final point-weight artifact match the estimand-side final-weight receipt exactly. - Distinguish unweighted from weighted design-based results and fail closed unless a weighted result separately binds the exact final analysis-weight receipt and variance-design receipt used. -- Add typed `NonresponseAdjustmentReceipt` evidence with explicit disposition treatment and typed `CalibrationAdjustmentReceipt` evidence bound to purpose-limited auxiliary projections, explicit scientific-use purpose, released owner-contract coordinates, authorization-receipt digest, authoritative benchmark receipts, constraints, and explicit convergence/fallback state. +- Add typed `NonresponseAdjustmentReceipt` evidence with explicit disposition treatment and typed `CalibrationAdjustmentReceipt` evidence bound to purpose-limited auxiliary projections, exact scientific-use purpose, scientific auxiliary-authority reference, released owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest/use time, authoritative benchmark receipts, constraints, and explicit convergence/fallback state; the use time cannot be later than receipt construction. - Add typed `TrimmingBoundingAdjustmentReceipt` evidence that binds a versioned rule/configuration, exact affected semantic occurrence set and count, and input/output weight artifacts; known trimming/bounding/winsorization adjustments must name that evidence family. - Require known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization adjustment codes to identify the matching typed evidence kind rather than collapse into a generic opaque adjustment digest. -- Keep durable owner-side typed-receipt and auxiliary-authorization resolution plus released auxiliary evidence exchange explicitly incomplete under #407. +- Keep durable owner-side typed-receipt and auxiliary-authority resolution plus released auxiliary evidence exchange explicitly incomplete under #407; the scientific leaf preserves the complete opaque correlation tuple but does not self-authenticate it. - Make `Validity Analysis Handoff Quality` retrigger on shared repository Python/test/clean-checkout configuration, with an executable regression preventing stale package-quality evidence after shared tooling changes. From b810abd28e8f3c6362b6333fedf33dcf5793bdca Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 09:09:46 +0900 Subject: [PATCH 098/158] docs(adr): reconcile exact calibration authority correlation --- ...governed-selection-validity-analysis-handoff.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/adr/0027-governed-selection-validity-analysis-handoff.md b/docs/adr/0027-governed-selection-validity-analysis-handoff.md index c06326fc1..cbbd2aa7b 100644 --- a/docs/adr/0027-governed-selection-validity-analysis-handoff.md +++ b/docs/adr/0027-governed-selection-validity-analysis-handoff.md @@ -34,13 +34,13 @@ For #407's weighted design-based inference boundary, the active package adds `Fi `WeightEligibilityReceipt` makes weight use itself versioned evidence rather than a caller-supplied label. The receipt is closed to `cross_sectional` or `longitudinal` scope and binds the exact target population, reference-duration evidence, eligible-case set, and final point-weight artifact. `FinalAnalysisWeightReceipt` validates that the eligibility receipt belongs to the same tenant and that its scope, target population, reference duration, eligible cases, and weight artifact exactly match the estimand-side receipt. A longitudinal weight therefore cannot silently support a cross-sectional estimand or a different target period. The 2025 SIPP Users' Guide is used only as current primary methodological evidence that weight choice depends on both target population and duration and that longitudinal weights cover explicit multi-year reference periods; SIPP-specific variables or estimators are not imported into Orgmetra. -The adjustment chain has typed scientific evidence for three high-risk transform families. `NonresponseAdjustmentReceipt` binds a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, explicit treatment of ineligible/unknown/unavailable cases, and input/output weight artifacts. `CalibrationAdjustmentReceipt` binds the target population/window, purpose-limited auxiliary projection, exact scientific-use purpose reference/digest, released owner-contract coordinate/version, owner authorization-receipt digest, authoritative benchmark receipt, algorithm/version, constraints digest, input/output artifacts, and an explicit `converged` or `fallback_applied` termination state. This makes purpose and owner release coordinates part of the scientific receipt without copying protected auxiliary values. The leaf package does not pretend a caller-supplied digest is authoritative: the durable `workforce_validation` service must resolve the released owner contract and authorization receipt and prove that the actual projection is permitted for the stated scientific purpose before execution/release. A fallback must identify its immutable fallback-rule reference/digest. A failed/nonconverged calibration cannot be labeled as an accepted calibration receipt. `TrimmingBoundingAdjustmentReceipt` binds a versioned trimming/bounding rule, reproducible rule configuration digest, the semantic occurrence set and count of cases actually affected, and input/output weight artifacts; a declared trim/bound transform cannot be a no-op. Known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization `AnalysisWeightAdjustment` codes also carry an `evidence_kind` and fail closed unless it names the matching typed receipt family. +The adjustment chain has typed scientific evidence for three high-risk transform families. `NonresponseAdjustmentReceipt` binds a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, explicit treatment of ineligible/unknown/unavailable cases, and input/output weight artifacts. `CalibrationAdjustmentReceipt` binds the target population/window, purpose-limited auxiliary projection, exact scientific-use purpose reference/digest, scientific auxiliary-authority reference, released owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and use instant, authoritative benchmark receipt, algorithm/version, constraints digest, input/output artifacts, and an explicit `converged` or `fallback_applied` termination state. The scientific-use instant is frozen and cannot be later than receipt construction. This makes the complete opaque correlation tuple expected by the canonical Workforce Validation owner resolver part of the scientific receipt without copying protected auxiliary values. The leaf package still does not pretend its caller-supplied coordinates are authoritative: the durable `workforce_validation` service must resolve the released owner contract, authorization receipt, and scientific-use receipt and prove that the actual projection is permitted for the exact scientific purpose/use before execution or release. A fallback must identify its immutable fallback-rule reference/digest. A failed/nonconverged calibration cannot be labeled as an accepted calibration receipt. `TrimmingBoundingAdjustmentReceipt` binds a versioned trimming/bounding rule, reproducible rule configuration digest, the semantic occurrence set and count of cases actually affected, and input/output weight artifacts; a declared trim/bound transform cannot be a no-op. Known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization `AnalysisWeightAdjustment` codes also carry an `evidence_kind` and fail closed unless it names the matching typed receipt family. -This executable slice still does not claim #407 complete. Durable service/API verification that adjustment and auxiliary authorization digests resolve to the released typed receipts/contracts claimed, verification that the released authorization permits the resolved projection for the exact purpose, and released auxiliary-owner evidence exchange remain `workforce_validation` scientific truth rather than Talent truth or cross-context SQL reconstruction. +This executable slice still does not claim #407 complete. The leaf can now preserve all opaque coordinates needed for durable owner-side correlation instead of requiring owner-contract digest, authorization identity, or scientific-use identity/time to be supplied out-of-band. Durable service/API verification that those coordinates resolve to the released typed receipts/contracts claimed, verification that the released authorization permits the resolved projection for the exact purpose and use, and released auxiliary-owner evidence exchange remain `workforce_validation` scientific truth rather than Talent truth or cross-context SQL reconstruction. The NIST Privacy Framework 1.0 is used narrowly as a privacy-risk-management basis for expressing and verifying data-processing requirements across organizational roles and contracts; as of the 2026-09-17 check, NIST still presents Privacy Framework 1.1 as an Initial Public Draft rather than a final replacement. This ADR does not treat the voluntary framework as employment law or infer legal permission from a NIST profile. -The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, prove requester/reviewer identities are distinct authoritative actors, verify cross-sectional/longitudinal weight eligibility against the estimand, and verify typed weight-adjustment receipts plus any calibration auxiliary purpose/owner authorization rather than trusting caller-supplied evidence labels or digests. +The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, prove requester/reviewer identities are distinct authoritative actors, verify cross-sectional/longitudinal weight eligibility against the estimand, and verify typed weight-adjustment receipts plus any calibration auxiliary authority/purpose/released-owner/authorization/scientific-use tuple rather than trusting caller-supplied evidence labels or digests. ## Consequences @@ -55,7 +55,7 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - Cross-sectional and longitudinal weights cannot be interchanged when target population, reference duration, eligible cases, or final weight artifact differ from the estimand. - The ordered weight transformation chain is independently digest-correlatable without centralizing case-level weights or auxiliary attributes. - Nonresponse cannot silently drop refusal/unreachable/ineligible/failure dispositions behind one opaque adjustment label. -- Calibration/raking cannot silently float benchmark ownership, omit the scientific-use purpose/released-owner authorization coordinate, or hide fallback as successful convergence. +- Calibration/raking cannot silently float benchmark ownership, omit the exact scientific-use purpose/released-owner/authorization/use correlation tuple, move its use instant after receipt construction, or hide fallback as successful convergence. - Trimming/bounding/winsorization cannot silently alter final point weights without an immutable rule/configuration and affected-case receipt. - Human interpretation remains explicit and separate from numerical output. - The dedicated-writer boundary remains intact: Orgmetra consumes only a pinned foreign revision/contract boundary and never mutates fast-mlsirm. @@ -64,13 +64,13 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - This slice does not execute a statistical model, estimate validity, correct for measurement error/range restriction, evaluate adverse impact, or assert legal compliance. - Sampling design, estimator choice, missing-data treatment, reliability evidence, multiplicity, transportability, fairness analysis, and model diagnostics must be encoded in the referenced analysis plan and reviewed before execution. -- The current #407 slice validates generic ordered weight provenance, weighted-result binding, cross-sectional/longitudinal eligibility congruence, disposition-aware nonresponse evidence, purpose/owner-coordinate/benchmark/termination-aware calibration evidence, and trimming/bounding rule provenance; it does not yet make durable typed-receipt/authorization resolution or released auxiliary-owner exchange fully executable. +- The current #407 slice validates generic ordered weight provenance, weighted-result binding, cross-sectional/longitudinal eligibility congruence, disposition-aware nonresponse evidence, owner-correlatable calibration evidence, and trimming/bounding rule provenance; it does not yet make durable typed-receipt/authorization resolution or released auxiliary-owner exchange fully executable. - The generic result envelope does not invent an estimator-specific minimum complete-case count. Whether a converged estimator is identified under a particular missing-data design belongs to the reviewed analysis plan and numerical-worker contract; Orgmetra fails closed on impossible aggregate counts without silently replacing that foreign scientific contract with complete-case analysis. -- The package validates the result envelope, but a future execution adapter must still re-resolve the handoff references, verify the result provenance artifact, resolve typed adjustment and purpose-authorization digests to released owner evidence, and attach evidence only after accountable human review. +- The package validates the result envelope, but a future execution adapter must still re-resolve the handoff references, verify the result provenance artifact, resolve typed adjustment and purpose-authorization coordinates to released owner evidence, and attach evidence only after accountable human review. ## Verification -The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. #407 adds RED/GREEN coverage for deterministic value-minimized final-weight receipts, exact sampling-receipt binding, contiguous transformation lineage, append-only correction linkage, fail-closed weighted-result binding to separate point-weight and variance-design receipts, cross-sectional/longitudinal scope and target-population/reference-duration/case-set/final-artifact congruence, explicit nonresponse disposition treatment, calibration benchmark/constraint/termination provenance, calibration scientific-purpose/released-owner authorization coordinates, fallback-rule disclosure, trimming/bounding rule/configuration and affected-case provenance, no-op trim/bound rejection, and typed adjustment-evidence matching. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. +The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. #407 adds RED/GREEN coverage for deterministic value-minimized final-weight receipts, exact sampling-receipt binding, contiguous transformation lineage, append-only correction linkage, fail-closed weighted-result binding to separate point-weight and variance-design receipts, cross-sectional/longitudinal scope and target-population/reference-duration/case-set/final-artifact congruence, explicit nonresponse disposition treatment, calibration benchmark/constraint/termination provenance, scientific auxiliary-authority/purpose/released-owner/authorization/scientific-use correlation coordinates, owner-contract and use-receipt digest validation, future-use-time rejection, fallback-rule disclosure, trimming/bounding rule/configuration and affected-case provenance, no-op trim/bound rejection, and typed adjustment-evidence matching. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. ## References From d2b5e8cb8fbe59352f741a878e2d4b25e600689e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 09:32:30 +0900 Subject: [PATCH 099/158] test(validity): require versioned disposition receipt identity --- .../tests/test_weight_adjustment_semantics.py | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/packages/validity-analysis/tests/test_weight_adjustment_semantics.py b/packages/validity-analysis/tests/test_weight_adjustment_semantics.py index 329d4f5a1..fe8d9bf86 100644 --- a/packages/validity-analysis/tests/test_weight_adjustment_semantics.py +++ b/packages/validity-analysis/tests/test_weight_adjustment_semantics.py @@ -29,6 +29,8 @@ def nonresponse_receipt(**overrides: object) -> NonresponseAdjustmentReceipt: values: dict[str, object] = { "tenant_record_id": TENANT, "receipt_reference": "nonresponse_adjustment_receipt:11111111-1111-4111-8111-111111111111", + "response_disposition_receipt_reference": "response_disposition_receipt:11111111-1111-4111-8111-111111111112", + "response_disposition_receipt_version": 3, "response_disposition_receipt_digest": DIGEST_A, "adjustment_population_digest": DIGEST_B, "method_reference": "weight_method:22222222-2222-4222-8222-222222222222", @@ -95,14 +97,28 @@ def adjustment(*, code: str, evidence_kind: str) -> AnalysisWeightAdjustment: def test_nonresponse_receipt_is_value_minimized_and_disposition_aware() -> None: - """Preserve explicit disposition treatment without copying source attributes.""" + """Preserve exact versioned disposition input without copying source attributes.""" candidate = nonresponse_receipt() + canonical = candidate.canonical_json() assert candidate.sha256_digest() == nonresponse_receipt().sha256_digest() - assert '"unknown_treatment_code":"retain_in_unknown_class"' in candidate.canonical_json() - assert "person_record" not in candidate.canonical_json() - assert "protected_attribute" not in candidate.canonical_json() + assert ( + '"response_disposition_receipt_reference":' + '"response_disposition_receipt:11111111-1111-4111-8111-111111111112"' + in canonical + ) + assert '"response_disposition_receipt_version":3' in canonical + assert f'"response_disposition_receipt_digest":"{DIGEST_A}"' in canonical + assert '"unknown_treatment_code":"retain_in_unknown_class"' in canonical + assert "person_record" not in canonical + assert "protected_attribute" not in canonical assert repr(candidate) == "NonresponseAdjustmentReceipt()" + with pytest.raises(ValueError, match="response_disposition_receipt_reference"): + nonresponse_receipt(response_disposition_receipt_reference="dispositions-v3") + with pytest.raises(ValueError, match="response_disposition_receipt_version"): + nonresponse_receipt(response_disposition_receipt_version=0) + with pytest.raises(ValueError, match="response_disposition_receipt_version"): + nonresponse_receipt(response_disposition_receipt_version=True) with pytest.raises(ValueError, match="unknown_treatment_code"): nonresponse_receipt(unknown_treatment_code="") with pytest.raises(ValueError, match="output_weight_artifact_digest"): From 6d63a3c336630fd449e688c1c7ef63169ccdb0ce Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 09:33:53 +0900 Subject: [PATCH 100/158] fix(validity): bind nonresponse to versioned disposition input --- .../src/orgmetra_validity_analysis/weights.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py b/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py index ad2fa0236..9b9394e6e 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py @@ -45,6 +45,8 @@ class NonresponseAdjustmentReceipt: tenant_record_id: str receipt_reference: str + response_disposition_receipt_reference: str + response_disposition_receipt_version: int response_disposition_receipt_digest: str adjustment_population_digest: str method_reference: str @@ -66,6 +68,15 @@ def __post_init__(self) -> None: "nonresponse_adjustment_receipt", "receipt_reference", ) + _validate_reference( + self.response_disposition_receipt_reference, + "response_disposition_receipt", + "response_disposition_receipt_reference", + ) + _positive_integer( + self.response_disposition_receipt_version, + "response_disposition_receipt_version", + ) for field_name in ( "response_disposition_receipt_digest", "adjustment_population_digest", @@ -109,6 +120,8 @@ def canonical_json(self) -> str: "output_weight_artifact_digest": self.output_weight_artifact_digest, "receipt_reference": self.receipt_reference, "response_disposition_receipt_digest": self.response_disposition_receipt_digest, + "response_disposition_receipt_reference": self.response_disposition_receipt_reference, + "response_disposition_receipt_version": self.response_disposition_receipt_version, "tenant_record_id": self.tenant_record_id, "unavailable_treatment_code": self.unavailable_treatment_code, "unknown_treatment_code": self.unknown_treatment_code, From 2bf77d63795217e946cd960c3109e8898db8eb6e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 09:35:12 +0900 Subject: [PATCH 101/158] docs(validity): trace versioned disposition input --- packages/validity-analysis/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/validity-analysis/CHANGELOG.md b/packages/validity-analysis/CHANGELOG.md index da77bfaa0..59d16d658 100644 --- a/packages/validity-analysis/CHANGELOG.md +++ b/packages/validity-analysis/CHANGELOG.md @@ -13,7 +13,7 @@ - Add a deterministic, row-value-minimized `FinalAnalysisWeightReceipt` that binds an exact estimand to source/sampling evidence, base-weight provenance, an ordered digest-linked adjustment chain, the final point-weight artifact, and append-only correction lineage. - Add `WeightEligibilityReceipt` and fail closed unless cross-sectional/longitudinal scope, target population, reference duration, eligible case set, and final point-weight artifact match the estimand-side final-weight receipt exactly. - Distinguish unweighted from weighted design-based results and fail closed unless a weighted result separately binds the exact final analysis-weight receipt and variance-design receipt used. -- Add typed `NonresponseAdjustmentReceipt` evidence with explicit disposition treatment and typed `CalibrationAdjustmentReceipt` evidence bound to purpose-limited auxiliary projections, exact scientific-use purpose, scientific auxiliary-authority reference, released owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest/use time, authoritative benchmark receipts, constraints, and explicit convergence/fallback state; the use time cannot be later than receipt construction. +- Add typed `NonresponseAdjustmentReceipt` evidence that binds the exact versioned response/disposition receipt reference/version/digest, adjustment population, controlled method/configuration, explicit ineligible/unknown/unavailable treatment, and input/output weight artifacts; add typed `CalibrationAdjustmentReceipt` evidence bound to purpose-limited auxiliary projections, exact scientific-use purpose, scientific auxiliary-authority reference, released owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest/use time, authoritative benchmark receipts, constraints, and explicit convergence/fallback state; the use time cannot be later than receipt construction. - Add typed `TrimmingBoundingAdjustmentReceipt` evidence that binds a versioned rule/configuration, exact affected semantic occurrence set and count, and input/output weight artifacts; known trimming/bounding/winsorization adjustments must name that evidence family. - Require known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization adjustment codes to identify the matching typed evidence kind rather than collapse into a generic opaque adjustment digest. - Keep durable owner-side typed-receipt and auxiliary-authority resolution plus released auxiliary evidence exchange explicitly incomplete under #407; the scientific leaf preserves the complete opaque correlation tuple but does not self-authenticate it. From 406efbbfc980b97deed70f91abdde7abb9964563 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 09:59:43 +0900 Subject: [PATCH 102/158] test(validity): require point-variance lineage compatibility evidence --- ...t_weight_variance_compatibility_receipt.py | 162 ++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 packages/validity-analysis/tests/test_weight_variance_compatibility_receipt.py diff --git a/packages/validity-analysis/tests/test_weight_variance_compatibility_receipt.py b/packages/validity-analysis/tests/test_weight_variance_compatibility_receipt.py new file mode 100644 index 000000000..738f84823 --- /dev/null +++ b/packages/validity-analysis/tests/test_weight_variance_compatibility_receipt.py @@ -0,0 +1,162 @@ +"""Regression contract for point-weight and variance-design compatibility evidence.""" + +from datetime import datetime, timezone + +import pytest + +from orgmetra_validity_analysis import ( + FinalAnalysisWeightReceipt, + WeightEligibilityReceipt, + WeightVarianceCompatibilityReceipt, +) + +TENANT = "10000000-0000-7000-8000-000000000001" +WEIGHT_RECEIPT = "analysis_weight_receipt:11111111-1111-4111-8111-111111111111" +COMPATIBILITY_RECEIPT = ( + "weight_variance_compatibility_receipt:22222222-2222-4222-8222-222222222222" +) +VARIANCE_RECEIPT = "variance_design_receipt:33333333-3333-4333-8333-333333333333" +ELIGIBILITY_RECEIPT = "weight_eligibility_receipt:44444444-4444-4444-8444-444444444444" +ESTIMAND = "validation_estimand:55555555-5555-4555-8555-555555555555" +TARGET = "analysis_target_population:66666666-6666-4666-8666-666666666666" +WINDOW = "analysis_window:77777777-7777-4777-8777-777777777777" +DURATION = "analysis_reference_duration:88888888-8888-4888-8888-888888888888" +DIGEST_A = "a" * 64 +DIGEST_B = "b" * 64 +DIGEST_C = "c" * 64 +DIGEST_D = "d" * 64 +DIGEST_E = "e" * 64 +DIGEST_F = "f" * 64 +DIGEST_1 = "1" * 64 +DIGEST_2 = "2" * 64 +DIGEST_3 = "3" * 64 +DIGEST_4 = "4" * 64 +DIGEST_5 = "5" * 64 + + +def weight_receipt(**overrides: object) -> FinalAnalysisWeightReceipt: + """Build one base-weight-only receipt for compatibility correlation tests.""" + eligibility = WeightEligibilityReceipt( + tenant_record_id=TENANT, + receipt_reference=ELIGIBILITY_RECEIPT, + weight_scope_code="cross_sectional", + target_population_reference=TARGET, + target_population_digest=DIGEST_B, + reference_duration_reference=DURATION, + reference_duration_digest=DIGEST_C, + eligible_case_set_digest=DIGEST_D, + weight_artifact_digest=DIGEST_5, + constructed_at=datetime(2026, 9, 17, 0, 30, tzinfo=timezone.utc), + ) + values: dict[str, object] = { + "tenant_record_id": TENANT, + "receipt_reference": WEIGHT_RECEIPT, + "estimand_reference": ESTIMAND, + "estimand_digest": DIGEST_A, + "estimand_scope_code": "cross_sectional", + "target_population_reference": TARGET, + "target_population_digest": DIGEST_B, + "analysis_unit_code": "worker_occurrence", + "analysis_window_reference": WINDOW, + "reference_duration_reference": DURATION, + "reference_duration_digest": DIGEST_C, + "eligible_case_set_digest": DIGEST_D, + "analytic_case_occurrence_set_digest": DIGEST_E, + "source_universe_receipt_digest": DIGEST_F, + "sampling_design_receipt_digest": DIGEST_1, + "base_weight_method_code": "inverse_inclusion_probability", + "base_weight_method_version": 1, + "base_weight_evidence_digest": DIGEST_2, + "base_weight_artifact_digest": DIGEST_5, + "adjustments": (), + "final_weight_artifact_digest": DIGEST_5, + "weight_eligibility": eligibility, + "analytic_case_count": 12, + "constructed_at": datetime(2026, 9, 17, 0, 30, tzinfo=timezone.utc), + } + values.update(overrides) + return FinalAnalysisWeightReceipt(**values) + + +def compatibility(**overrides: object) -> WeightVarianceCompatibilityReceipt: + """Build owner-correlatable compatibility evidence for one weighted analysis.""" + point_weight = overrides.pop("analysis_weight_receipt", weight_receipt()) + values: dict[str, object] = { + "tenant_record_id": TENANT, + "receipt_reference": COMPATIBILITY_RECEIPT, + "analysis_weight_receipt": point_weight, + "variance_design_receipt_reference": VARIANCE_RECEIPT, + "variance_design_receipt_version": 1, + "variance_design_receipt_digest": DIGEST_3, + "variance_analysis_weight_receipt_digest": point_weight.sha256_digest(), + "variance_analytic_case_occurrence_set_digest": ( + point_weight.analytic_case_occurrence_set_digest + ), + "variance_weight_eligibility_receipt_digest": ( + point_weight.weight_eligibility.sha256_digest() + ), + "variance_weight_correction_sequence": point_weight.correction_sequence, + "variance_final_weight_artifact_digest": point_weight.final_weight_artifact_digest, + "constructed_at": datetime(2026, 9, 17, 0, 31, tzinfo=timezone.utc), + } + values.update(overrides) + return WeightVarianceCompatibilityReceipt(**values) + + +def test_compatibility_receipt_binds_exact_point_and_variance_lineage() -> None: + """Require variance evidence to name the exact point-weight basis it accompanies.""" + candidate = compatibility() + payload = candidate.canonical_json() + assert candidate.sha256_digest() == compatibility().sha256_digest() + assert f'"variance_design_receipt_digest":"{DIGEST_3}"' in payload + assert f'"variance_final_weight_artifact_digest":"{DIGEST_5}"' in payload + assert repr(candidate) == "WeightVarianceCompatibilityReceipt()" + + +@pytest.mark.parametrize( + "field,value,match", + [ + ("variance_analysis_weight_receipt_digest", DIGEST_A, "analysis weight receipt"), + ("variance_analytic_case_occurrence_set_digest", DIGEST_A, "analytic case occurrence"), + ("variance_weight_eligibility_receipt_digest", DIGEST_A, "weight eligibility"), + ("variance_weight_correction_sequence", 2, "correction sequence"), + ("variance_final_weight_artifact_digest", DIGEST_A, "final weight artifact"), + ], +) +def test_compatibility_receipt_rejects_point_variance_lineage_mismatch( + field: str, value: object, match: str +) -> None: + """Fail closed when variance evidence was generated from a different weight lineage.""" + with pytest.raises(ValueError, match=match): + compatibility(**{field: value}) + + +def test_compatibility_receipt_rejects_foreign_point_weight_or_tenant() -> None: + """Do not admit an opaque object or a point-weight receipt from another tenant.""" + with pytest.raises(ValueError, match="FinalAnalysisWeightReceipt"): + compatibility(analysis_weight_receipt=object()) + + foreign = weight_receipt( + tenant_record_id="10000000-0000-7000-8000-000000000002", + weight_eligibility=WeightEligibilityReceipt( + tenant_record_id="10000000-0000-7000-8000-000000000002", + receipt_reference=ELIGIBILITY_RECEIPT, + weight_scope_code="cross_sectional", + target_population_reference=TARGET, + target_population_digest=DIGEST_B, + reference_duration_reference=DURATION, + reference_duration_digest=DIGEST_C, + eligible_case_set_digest=DIGEST_D, + weight_artifact_digest=DIGEST_5, + constructed_at=datetime(2026, 9, 17, 0, 30, tzinfo=timezone.utc), + ), + ) + with pytest.raises(ValueError, match="tenant_record_id"): + compatibility(analysis_weight_receipt=foreign) + + +@pytest.mark.parametrize("version", [0, True]) +def test_variance_design_receipt_version_is_strictly_positive(version: object) -> None: + """Reject unversioned or boolean variance owner evidence.""" + with pytest.raises(ValueError, match="variance_design_receipt_version"): + compatibility(variance_design_receipt_version=version) From f0b3f812d7abf4fb936c7e2472ac35801d93e318 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 10:00:01 +0900 Subject: [PATCH 103/158] feat(validity): bind variance evidence to final point-weight lineage --- .../compatibility.py | 173 ++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 packages/validity-analysis/src/orgmetra_validity_analysis/compatibility.py diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/compatibility.py b/packages/validity-analysis/src/orgmetra_validity_analysis/compatibility.py new file mode 100644 index 000000000..9d29451aa --- /dev/null +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/compatibility.py @@ -0,0 +1,173 @@ +"""Correlate final point-weight evidence with the variance design that used it. + +This boundary prevents a weighted scientific result from combining a final point +weight with variance or replicate evidence generated from a different analytic +case set, eligibility receipt, correction sequence, or final-weight artifact. +It records only immutable references and digests; durable owner corroboration +remains an application/persistence responsibility. +""" +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from hashlib import sha256 +import json + +from .handoff import ( + _canonical_timestamp, + _freeze_timestamp, + _validate_digest, + _validate_operational_uuid, + _validate_reference, +) +from .weights import FinalAnalysisWeightReceipt + + +def _positive_integer(value: object, field_name: str) -> None: + """Require an exact positive integer without accepting booleans.""" + if type(value) is not int or value <= 0: + raise ValueError(f"{field_name} must be a positive integer") + + +@dataclass(frozen=True, slots=True, repr=False) +class WeightVarianceCompatibilityReceipt: + """Bind variance evidence to the exact final point-weight lineage it accompanies.""" + + tenant_record_id: str + receipt_reference: str + analysis_weight_receipt: FinalAnalysisWeightReceipt + variance_design_receipt_reference: str + variance_design_receipt_version: int + variance_design_receipt_digest: str + variance_analysis_weight_receipt_digest: str + variance_analytic_case_occurrence_set_digest: str + variance_weight_eligibility_receipt_digest: str + variance_weight_correction_sequence: int + variance_final_weight_artifact_digest: str + constructed_at: datetime + evidence_version: int = 1 + + def __post_init__(self) -> None: + """Reject variance evidence produced from any different point-weight basis.""" + _validate_operational_uuid(self.tenant_record_id, "tenant_record_id") + _validate_reference( + self.receipt_reference, + "weight_variance_compatibility_receipt", + "receipt_reference", + ) + if type(self.analysis_weight_receipt) is not FinalAnalysisWeightReceipt: + raise ValueError( + "analysis_weight_receipt must be a FinalAnalysisWeightReceipt" + ) + if self.analysis_weight_receipt.tenant_record_id != self.tenant_record_id: + raise ValueError( + "analysis_weight_receipt tenant_record_id must match the compatibility receipt" + ) + _validate_reference( + self.variance_design_receipt_reference, + "variance_design_receipt", + "variance_design_receipt_reference", + ) + _positive_integer( + self.variance_design_receipt_version, + "variance_design_receipt_version", + ) + for field_name in ( + "variance_design_receipt_digest", + "variance_analysis_weight_receipt_digest", + "variance_analytic_case_occurrence_set_digest", + "variance_weight_eligibility_receipt_digest", + "variance_final_weight_artifact_digest", + ): + _validate_digest(getattr(self, field_name), field_name) + _positive_integer( + self.variance_weight_correction_sequence, + "variance_weight_correction_sequence", + ) + + analysis_weight_digest = self.analysis_weight_receipt.sha256_digest() + if self.variance_design_receipt_digest == analysis_weight_digest: + raise ValueError( + "variance_design_receipt_digest must identify evidence distinct from the analysis weight receipt" + ) + if self.variance_analysis_weight_receipt_digest != analysis_weight_digest: + raise ValueError( + "variance evidence analysis weight receipt must match the exact point-weight receipt" + ) + if ( + self.variance_analytic_case_occurrence_set_digest + != self.analysis_weight_receipt.analytic_case_occurrence_set_digest + ): + raise ValueError( + "variance evidence analytic case occurrence set must match the point-weight receipt" + ) + if ( + self.variance_weight_eligibility_receipt_digest + != self.analysis_weight_receipt.weight_eligibility.sha256_digest() + ): + raise ValueError( + "variance evidence weight eligibility must match the point-weight receipt" + ) + if ( + self.variance_weight_correction_sequence + != self.analysis_weight_receipt.correction_sequence + ): + raise ValueError( + "variance evidence correction sequence must match the point-weight receipt" + ) + if ( + self.variance_final_weight_artifact_digest + != self.analysis_weight_receipt.final_weight_artifact_digest + ): + raise ValueError( + "variance evidence final weight artifact must match the point-weight receipt" + ) + + constructed_at = _freeze_timestamp(self.constructed_at, "constructed_at") + if constructed_at < self.analysis_weight_receipt.constructed_at: + raise ValueError( + "constructed_at cannot precede the analysis weight receipt" + ) + if type(self.evidence_version) is not int or self.evidence_version != 1: + raise ValueError("evidence_version must remain 1") + object.__setattr__(self, "constructed_at", constructed_at) + + def __repr__(self) -> str: + """Return a value-minimized representation suitable for routine logs.""" + return "WeightVarianceCompatibilityReceipt()" + + def canonical_json(self) -> str: + """Return deterministic correlation evidence without weight values.""" + payload = { + "analysis_weight_receipt_digest": self.analysis_weight_receipt.sha256_digest(), + "constructed_at": _canonical_timestamp(self.constructed_at, "constructed_at"), + "evidence_version": self.evidence_version, + "receipt_reference": self.receipt_reference, + "tenant_record_id": self.tenant_record_id, + "variance_analysis_weight_receipt_digest": ( + self.variance_analysis_weight_receipt_digest + ), + "variance_analytic_case_occurrence_set_digest": ( + self.variance_analytic_case_occurrence_set_digest + ), + "variance_design_receipt_digest": self.variance_design_receipt_digest, + "variance_design_receipt_reference": self.variance_design_receipt_reference, + "variance_design_receipt_version": self.variance_design_receipt_version, + "variance_final_weight_artifact_digest": ( + self.variance_final_weight_artifact_digest + ), + "variance_weight_correction_sequence": ( + self.variance_weight_correction_sequence + ), + "variance_weight_eligibility_receipt_digest": ( + self.variance_weight_eligibility_receipt_digest + ), + } + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + def sha256_digest(self) -> str: + """Return SHA-256 over the exact canonical compatibility receipt bytes.""" + return sha256(self.canonical_json().encode("utf-8")).hexdigest() + + +__all__ = ["WeightVarianceCompatibilityReceipt"] From bdf344b04e18e67c366a5cb72ece1c02222ac85b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 10:00:08 +0900 Subject: [PATCH 104/158] feat(validity): export weight-variance compatibility receipt --- .../src/orgmetra_validity_analysis/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py b/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py index 1a64d81cc..bd0ecf577 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/__init__.py @@ -1,5 +1,6 @@ """Public governed selection-validity analysis handoff and result contracts.""" +from .compatibility import WeightVarianceCompatibilityReceipt from .handoff import ( REVIEWED_FAST_MLSIRM_REVISION, ValidationAnalysisHandoff, @@ -28,4 +29,5 @@ "NonresponseAdjustmentReceipt", "TrimmingBoundingAdjustmentReceipt", "WeightEligibilityReceipt", + "WeightVarianceCompatibilityReceipt", ] From cfdcae4f41abfe5841ce89d74d7d78f444b8b984 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 10:00:45 +0900 Subject: [PATCH 105/158] feat(validity): fail closed on point-variance weight mismatch --- .../src/orgmetra_validity_analysis/result.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py index 56ec4911d..2a516a48f 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py @@ -14,6 +14,7 @@ from math import isfinite from numbers import Real +from .compatibility import WeightVarianceCompatibilityReceipt from .handoff import ( _canonical_timestamp, _freeze_timestamp, @@ -166,6 +167,7 @@ class ValidationAnalysisResult: point_estimation_mode: str = "unweighted" analysis_weight_receipt_digest: str | None = None variance_design_receipt_digest: str | None = None + weight_variance_compatibility: WeightVarianceCompatibilityReceipt | None = None result_authority: str = _RESULT_AUTHORITY execution_state: str = _EXECUTION_STATE contains_raw_person_level_values: bool = False @@ -226,9 +228,36 @@ def __post_init__(self) -> None: "analysis_weight_receipt_digest and variance_design_receipt_digest " "must identify different evidence" ) + if type(self.weight_variance_compatibility) is not WeightVarianceCompatibilityReceipt: + raise ValueError( + "weight_variance_compatibility is required for weighted_design_based" + ) + if self.weight_variance_compatibility.tenant_record_id != self.tenant_record_id: + raise ValueError( + "weight_variance_compatibility tenant_record_id must match the result" + ) + if ( + self.weight_variance_compatibility.analysis_weight_receipt.sha256_digest() + != self.analysis_weight_receipt_digest + ): + raise ValueError( + "weight_variance_compatibility must identify the result analysis weight receipt" + ) + if ( + self.weight_variance_compatibility.variance_design_receipt_digest + != self.variance_design_receipt_digest + ): + raise ValueError( + "weight_variance_compatibility must identify the result variance design receipt" + ) + if self.weight_variance_compatibility.constructed_at > completed_at: + raise ValueError( + "weight_variance_compatibility cannot be constructed after the result" + ) elif ( self.analysis_weight_receipt_digest is not None or self.variance_design_receipt_digest is not None + or self.weight_variance_compatibility is not None ): raise ValueError( "unweighted result must not bind analysis or variance weight receipts" @@ -281,6 +310,10 @@ def canonical_json(self) -> str: payload["analysis_weight_receipt_digest"] = self.analysis_weight_receipt_digest if self.variance_design_receipt_digest is not None: payload["variance_design_receipt_digest"] = self.variance_design_receipt_digest + if self.weight_variance_compatibility is not None: + payload["weight_variance_compatibility_receipt_digest"] = ( + self.weight_variance_compatibility.sha256_digest() + ) return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) def sha256_digest(self) -> str: From 053d0fcc4827ced323dc39f709769532f6f6bc5e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 10:01:19 +0900 Subject: [PATCH 106/158] test(validity): bind weighted results to compatibility evidence --- .../test_analysis_weight_result_binding.py | 161 ++++++++++++++++-- 1 file changed, 151 insertions(+), 10 deletions(-) diff --git a/packages/validity-analysis/tests/test_analysis_weight_result_binding.py b/packages/validity-analysis/tests/test_analysis_weight_result_binding.py index 244867863..25de7e51b 100644 --- a/packages/validity-analysis/tests/test_analysis_weight_result_binding.py +++ b/packages/validity-analysis/tests/test_analysis_weight_result_binding.py @@ -6,17 +6,105 @@ from orgmetra_validity_analysis import ( ConvergenceDiagnostics, + FinalAnalysisWeightReceipt, MissingnessSummary, REVIEWED_FAST_MLSIRM_REVISION, ValidationAnalysisResult, + WeightEligibilityReceipt, + WeightVarianceCompatibilityReceipt, ) TENANT = "10000000-0000-7000-8000-000000000001" RESULT = "validation_analysis_result:11111111-1111-4111-8111-111111111111" +WEIGHT_RECEIPT = "analysis_weight_receipt:22222222-2222-4222-8222-222222222222" +ELIGIBILITY_RECEIPT = "weight_eligibility_receipt:33333333-3333-4333-8333-333333333333" +COMPATIBILITY_RECEIPT = ( + "weight_variance_compatibility_receipt:44444444-4444-4444-8444-444444444444" +) +VARIANCE_RECEIPT = "variance_design_receipt:55555555-5555-4555-8555-555555555555" +ESTIMAND = "validation_estimand:66666666-6666-4666-8666-666666666666" +TARGET = "analysis_target_population:77777777-7777-4777-8777-777777777777" +WINDOW = "analysis_window:88888888-8888-4888-8888-888888888888" +DURATION = "analysis_reference_duration:99999999-9999-4999-8999-999999999999" HANDOFF_DIGEST = "a" * 64 PROVENANCE_DIGEST = "b" * 64 -WEIGHT_DIGEST = "c" * 64 VARIANCE_DIGEST = "d" * 64 +OTHER_VARIANCE_DIGEST = "e" * 64 +DIGEST_1 = "1" * 64 +DIGEST_2 = "2" * 64 +DIGEST_3 = "3" * 64 +DIGEST_4 = "4" * 64 +DIGEST_5 = "5" * 64 +DIGEST_6 = "6" * 64 +DIGEST_7 = "7" * 64 + + +def point_weight(**overrides: object) -> FinalAnalysisWeightReceipt: + """Build one base-weight-only point-estimation receipt for result binding.""" + eligibility = WeightEligibilityReceipt( + tenant_record_id=TENANT, + receipt_reference=ELIGIBILITY_RECEIPT, + weight_scope_code="cross_sectional", + target_population_reference=TARGET, + target_population_digest=DIGEST_1, + reference_duration_reference=DURATION, + reference_duration_digest=DIGEST_2, + eligible_case_set_digest=DIGEST_3, + weight_artifact_digest=DIGEST_7, + constructed_at=datetime(2026, 9, 17, 4, 0, tzinfo=timezone.utc), + ) + values: dict[str, object] = { + "tenant_record_id": TENANT, + "receipt_reference": WEIGHT_RECEIPT, + "estimand_reference": ESTIMAND, + "estimand_digest": DIGEST_4, + "estimand_scope_code": "cross_sectional", + "target_population_reference": TARGET, + "target_population_digest": DIGEST_1, + "analysis_unit_code": "worker_occurrence", + "analysis_window_reference": WINDOW, + "reference_duration_reference": DURATION, + "reference_duration_digest": DIGEST_2, + "eligible_case_set_digest": DIGEST_3, + "analytic_case_occurrence_set_digest": DIGEST_5, + "source_universe_receipt_digest": DIGEST_6, + "sampling_design_receipt_digest": "8" * 64, + "base_weight_method_code": "inverse_inclusion_probability", + "base_weight_method_version": 1, + "base_weight_evidence_digest": "9" * 64, + "base_weight_artifact_digest": DIGEST_7, + "adjustments": (), + "final_weight_artifact_digest": DIGEST_7, + "weight_eligibility": eligibility, + "analytic_case_count": 12, + "constructed_at": datetime(2026, 9, 17, 4, 0, tzinfo=timezone.utc), + } + values.update(overrides) + return FinalAnalysisWeightReceipt(**values) + + +def compatibility( + weight: FinalAnalysisWeightReceipt, variance_digest: str = VARIANCE_DIGEST +) -> WeightVarianceCompatibilityReceipt: + """Build compatibility evidence whose variance lineage names the exact point weight.""" + return WeightVarianceCompatibilityReceipt( + tenant_record_id=TENANT, + receipt_reference=COMPATIBILITY_RECEIPT, + analysis_weight_receipt=weight, + variance_design_receipt_reference=VARIANCE_RECEIPT, + variance_design_receipt_version=1, + variance_design_receipt_digest=variance_digest, + variance_analysis_weight_receipt_digest=weight.sha256_digest(), + variance_analytic_case_occurrence_set_digest=( + weight.analytic_case_occurrence_set_digest + ), + variance_weight_eligibility_receipt_digest=( + weight.weight_eligibility.sha256_digest() + ), + variance_weight_correction_sequence=weight.correction_sequence, + variance_final_weight_artifact_digest=weight.final_weight_artifact_digest, + constructed_at=datetime(2026, 9, 17, 4, 5, tzinfo=timezone.utc), + ) def result(**overrides: object) -> ValidationAnalysisResult: @@ -52,26 +140,69 @@ def result(**overrides: object) -> ValidationAnalysisResult: return ValidationAnalysisResult(**values) -def test_weighted_result_binds_point_and_variance_receipts_separately() -> None: - """Require exact point-weight and variance evidence on weighted inference.""" +def test_weighted_result_binds_point_variance_and_compatibility_receipts() -> None: + """Require exact point-weight, variance, and cross-lineage compatibility evidence.""" + weight = point_weight() + correlation = compatibility(weight) candidate = result( point_estimation_mode="weighted_design_based", - analysis_weight_receipt_digest=WEIGHT_DIGEST, + analysis_weight_receipt_digest=weight.sha256_digest(), variance_design_receipt_digest=VARIANCE_DIGEST, + weight_variance_compatibility=correlation, ) payload = candidate.canonical_json() assert '"point_estimation_mode":"weighted_design_based"' in payload - assert f'"analysis_weight_receipt_digest":"{WEIGHT_DIGEST}"' in payload + assert f'"analysis_weight_receipt_digest":"{weight.sha256_digest()}"' in payload assert f'"variance_design_receipt_digest":"{VARIANCE_DIGEST}"' in payload + assert ( + f'"weight_variance_compatibility_receipt_digest":"{correlation.sha256_digest()}"' + in payload + ) def test_weighted_result_rejects_same_point_and_variance_receipt() -> None: """A variance-design receipt cannot silently stand in for the point-weight receipt.""" + weight = point_weight() with pytest.raises(ValueError, match="must identify different evidence"): result( point_estimation_mode="weighted_design_based", - analysis_weight_receipt_digest=WEIGHT_DIGEST, - variance_design_receipt_digest=WEIGHT_DIGEST, + analysis_weight_receipt_digest=weight.sha256_digest(), + variance_design_receipt_digest=weight.sha256_digest(), + weight_variance_compatibility=compatibility(weight), + ) + + +def test_weighted_result_requires_compatibility_receipt() -> None: + """Do not release weighted inference without explicit point/variance congruence.""" + weight = point_weight() + with pytest.raises(ValueError, match="weight_variance_compatibility"): + result( + point_estimation_mode="weighted_design_based", + analysis_weight_receipt_digest=weight.sha256_digest(), + variance_design_receipt_digest=VARIANCE_DIGEST, + ) + + +def test_weighted_result_rejects_compatibility_for_other_point_or_variance_evidence() -> None: + """The compatibility receipt must correlate the same point and variance digests.""" + weight = point_weight() + corrected_weight = point_weight( + correction_sequence=2, + supersedes_receipt_digest=weight.sha256_digest(), + ) + with pytest.raises(ValueError, match="analysis weight receipt"): + result( + point_estimation_mode="weighted_design_based", + analysis_weight_receipt_digest=weight.sha256_digest(), + variance_design_receipt_digest=VARIANCE_DIGEST, + weight_variance_compatibility=compatibility(corrected_weight), + ) + with pytest.raises(ValueError, match="variance design receipt"): + result( + point_estimation_mode="weighted_design_based", + analysis_weight_receipt_digest=weight.sha256_digest(), + variance_design_receipt_digest=OTHER_VARIANCE_DIGEST, + weight_variance_compatibility=compatibility(weight), ) @@ -89,7 +220,7 @@ def test_weighted_result_rejects_same_point_and_variance_receipt() -> None: ( { "point_estimation_mode": "weighted_design_based", - "analysis_weight_receipt_digest": WEIGHT_DIGEST, + "analysis_weight_receipt_digest": "c" * 64, "variance_design_receipt_digest": None, }, "variance_design_receipt_digest", @@ -97,7 +228,7 @@ def test_weighted_result_rejects_same_point_and_variance_receipt() -> None: ( { "point_estimation_mode": "unweighted", - "analysis_weight_receipt_digest": WEIGHT_DIGEST, + "analysis_weight_receipt_digest": "c" * 64, }, "unweighted", ), @@ -108,6 +239,13 @@ def test_weighted_result_rejects_same_point_and_variance_receipt() -> None: }, "unweighted", ), + ( + { + "point_estimation_mode": "unweighted", + "weight_variance_compatibility": compatibility(point_weight()), + }, + "unweighted", + ), ({"point_estimation_mode": "opaque_weighted"}, "point_estimation_mode"), ({"point_estimation_mode": None}, "point_estimation_mode"), ], @@ -122,15 +260,18 @@ def test_result_rejects_unverifiable_weight_binding( def test_weighted_result_rejects_malformed_receipt_digests() -> None: """Do not permit opaque labels to stand in for immutable weight evidence.""" + weight = point_weight() with pytest.raises(ValueError, match="analysis_weight_receipt_digest"): result( point_estimation_mode="weighted_design_based", analysis_weight_receipt_digest="weight-v1", variance_design_receipt_digest=VARIANCE_DIGEST, + weight_variance_compatibility=compatibility(weight), ) with pytest.raises(ValueError, match="variance_design_receipt_digest"): result( point_estimation_mode="weighted_design_based", - analysis_weight_receipt_digest=WEIGHT_DIGEST, + analysis_weight_receipt_digest=weight.sha256_digest(), variance_design_receipt_digest="variance-v1", + weight_variance_compatibility=compatibility(weight), ) From 215c2b186bbfe0a7d67406132e98ded18a9305a8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 10:02:32 +0900 Subject: [PATCH 107/158] docs(validity): record point-variance compatibility contract --- packages/validity-analysis/CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/validity-analysis/CHANGELOG.md b/packages/validity-analysis/CHANGELOG.md index 59d16d658..60f3eddfd 100644 --- a/packages/validity-analysis/CHANGELOG.md +++ b/packages/validity-analysis/CHANGELOG.md @@ -13,8 +13,10 @@ - Add a deterministic, row-value-minimized `FinalAnalysisWeightReceipt` that binds an exact estimand to source/sampling evidence, base-weight provenance, an ordered digest-linked adjustment chain, the final point-weight artifact, and append-only correction lineage. - Add `WeightEligibilityReceipt` and fail closed unless cross-sectional/longitudinal scope, target population, reference duration, eligible case set, and final point-weight artifact match the estimand-side final-weight receipt exactly. - Distinguish unweighted from weighted design-based results and fail closed unless a weighted result separately binds the exact final analysis-weight receipt and variance-design receipt used. +- Add `WeightVarianceCompatibilityReceipt` so variance evidence must correlate to the exact final point-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence, and final-weight artifact before a weighted result can be emitted. The receipt is correlation evidence, not self-authenticating #406 owner authority. +- Require `ValidationAnalysisResult` to bind the compatibility receipt in addition to the point-weight and variance-design digests; unweighted results cannot carry compatibility evidence. - Add typed `NonresponseAdjustmentReceipt` evidence that binds the exact versioned response/disposition receipt reference/version/digest, adjustment population, controlled method/configuration, explicit ineligible/unknown/unavailable treatment, and input/output weight artifacts; add typed `CalibrationAdjustmentReceipt` evidence bound to purpose-limited auxiliary projections, exact scientific-use purpose, scientific auxiliary-authority reference, released owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest/use time, authoritative benchmark receipts, constraints, and explicit convergence/fallback state; the use time cannot be later than receipt construction. - Add typed `TrimmingBoundingAdjustmentReceipt` evidence that binds a versioned rule/configuration, exact affected semantic occurrence set and count, and input/output weight artifacts; known trimming/bounding/winsorization adjustments must name that evidence family. - Require known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization adjustment codes to identify the matching typed evidence kind rather than collapse into a generic opaque adjustment digest. -- Keep durable owner-side typed-receipt and auxiliary-authority resolution plus released auxiliary evidence exchange explicitly incomplete under #407; the scientific leaf preserves the complete opaque correlation tuple but does not self-authenticate it. +- Keep durable owner-side typed-receipt, point/variance compatibility, and auxiliary-authority resolution plus released auxiliary/variance evidence exchange explicitly incomplete under #407; the scientific leaf preserves exact correlation coordinates but does not self-authenticate them. - Make `Validity Analysis Handoff Quality` retrigger on shared repository Python/test/clean-checkout configuration, with an executable regression preventing stale package-quality evidence after shared tooling changes. From af6bf73e8f4b05b40984ac7dae238f57b3e28541 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 10:02:49 +0900 Subject: [PATCH 108/158] docs(validity): trace point-variance compatibility evidence --- docs/traceability/validation-analysis-handoff.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/traceability/validation-analysis-handoff.md b/docs/traceability/validation-analysis-handoff.md index fc37e7ae3..22c2e3fac 100644 --- a/docs/traceability/validation-analysis-handoff.md +++ b/docs/traceability/validation-analysis-handoff.md @@ -20,18 +20,21 @@ Can an organization send one exact, reviewable validation study to its statistic | Calibration/raking evidence | target population/window, purpose-bound auxiliary projection, scientific-use purpose reference/digest, scientific auxiliary-authority reference, released owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and use instant, authoritative benchmark receipt, algorithm/version, constraints, input/output artifacts, explicit converged/fallback state and immutable fallback rule when used | `CalibrationAdjustmentReceipt` authority/purpose/owner/authorization/scientific-use/benchmark/termination/fallback regressions; scientific-use time cannot be later than receipt construction | | Trimming/bounding evidence | versioned trimming/bounding rule, reproducible rule configuration digest, exact affected semantic occurrence set and count, input/output weight artifacts | `test_trimming_bounding_receipt.py` deterministic/value-minimized, no-op, missing-provenance, and typed-kind regressions | | Typed adjustment congruence | known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization adjustment codes must identify the matching typed receipt family through `evidence_kind` | specialized adjustment evidence-kind regressions | -| Weighted-result congruence | `weighted_design_based` result must bind the exact final point-weight receipt and a separate #406 variance-design receipt; `unweighted` result must bind neither | `test_analysis_weight_result_binding.py` | +| Point/variance lineage congruence | #406 variance evidence must name the same final point-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence, and final-weight artifact used by the estimate | `test_weight_variance_compatibility_receipt.py` mismatch regressions | +| Weighted-result congruence | `weighted_design_based` result must bind the exact final point-weight receipt, a separate #406 variance-design receipt, and their exact compatibility receipt; `unweighted` result must bind none | `test_analysis_weight_result_binding.py` | | Execution boundary | `not_executed`, `scientific_evidence_only`, read-only pinned foreign dependency | immutable governance regressions | -| Reproducibility | construction-time UTC timestamp snapshots, finite numeric snapshots, canonical RFC 3339 time, canonical JSON, SHA-256 handoff/result/weight/adjustment/eligibility-receipt digests | mutable timezone/numeric and UTC-boundary regressions plus deterministic serialization/digest tests | +| Reproducibility | construction-time UTC timestamp snapshots, finite numeric snapshots, canonical RFC 3339 time, canonical JSON, SHA-256 handoff/result/weight/adjustment/eligibility/compatibility-receipt digests | mutable timezone/numeric and UTC-boundary regressions plus deterministic serialization/digest tests | | Decision-record integrity | ADR numbers remain unique repository-wide and any `docs/adr/**` change reaches the consolidated Foundation CI validity regression | ADR uniqueness regression plus Foundation CI workflow-trigger contract regression | | Quality-evidence freshness | consolidated Foundation CI runs the validity package on every `develop` pull request without a repository path filter, so shared Python/test/clean-checkout configuration cannot silently bypass the package gate | `test_foundation_ci_retriggers_without_path_filter` and `test_foundation_ci_runs_validity_analysis_and_adr_changes`; central required workflows remain separate gates | ## #407 boundary still open -The active branch makes final-weight eligibility and three adjustment families executable rather than leaving them as opaque labels. Calibration/raking/post-stratification evidence now carries the complete opaque correlation tuple needed by the canonical Workforce Validation authority resolver: auxiliary authority and projection, exact scientific purpose, released owner-contract identity/version/digest, authorization receipt identity/digest, scientific-use receipt identity/digest and owner-correlatable use instant. This removes the prior gap where owner-contract digest, authorization reference and scientific-use identity/time would have had to be supplied outside the immutable calibration receipt. The leaf still cannot self-authenticate those coordinates. Durable #235/#248 owner-side resolution must prove that they resolve to released/versioned evidence and that the exact projection is authorized for the exact scientific purpose and use receipt; released auxiliary evidence exchange also remains open. `workforce_validation` owns those scientific contracts and their released result; `talent_management` must not compute or reconstruct final analysis weights. +The active branch makes final-weight eligibility and three adjustment families executable rather than leaving them as opaque labels. It now also closes the leaf-side form of #407 RED #8: `WeightVarianceCompatibilityReceipt` requires the variance side to identify the exact point-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence, and final-weight artifact. `ValidationAnalysisResult` cannot emit weighted design-based evidence without that compatibility receipt in addition to separate point-weight and #406 variance-design digests. + +This is still correlation evidence, not owner authentication. Calibration/raking/post-stratification evidence carries the complete opaque correlation tuple needed by the canonical Workforce Validation authority resolver: auxiliary authority and projection, exact scientific purpose, released owner-contract identity/version/digest, authorization receipt identity/digest, scientific-use receipt identity/digest and owner-correlatable use instant. Durable #235/#248 owner-side resolution must prove those coordinates and the point/variance compatibility coordinates against released/versioned owner evidence, prove that the exact projection is authorized for the exact scientific purpose/use receipt, and resolve the released #406 variance evidence rather than trusting leaf-supplied digests. Released auxiliary/variance evidence exchange also remains open. `workforce_validation` owns those scientific contracts and their released result; `talent_management` must not compute or reconstruct final analysis weights. ## Maturity `implemented_on_active_pr`. -Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. The active package validates the returned numerical/provenance envelope plus the current #407 final-weight/result-binding, weight-eligibility, typed adjustment-evidence, and calibration-authority correlation contracts, but protected Orgmetra evidence still requires host re-resolution, durable owner evidence exchange, result-artifact verification, terminal checks, independent review, and accountable human interpretation. +Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. The active package validates the returned numerical/provenance envelope plus the current #407 final-weight/result-binding, point/variance compatibility, weight-eligibility, typed adjustment-evidence, and calibration-authority correlation contracts, but protected Orgmetra evidence still requires host re-resolution, durable owner evidence exchange, result-artifact verification, terminal checks, independent review, and accountable human interpretation. From bd1d992d56a6071fd0cfb65b7061295f8f0dee1b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 10:03:09 +0900 Subject: [PATCH 109/158] docs(validity): document point-variance compatibility boundary --- packages/validity-analysis/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/validity-analysis/README.md b/packages/validity-analysis/README.md index 9b56302d2..fca101271 100644 --- a/packages/validity-analysis/README.md +++ b/packages/validity-analysis/README.md @@ -18,9 +18,11 @@ Trimming, bounding, and winsorization have a separate provenance family rather t `AnalysisWeightAdjustment` records an `evidence_kind` in addition to the evidence digest. Known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization transforms fail closed unless their evidence kind is the corresponding typed receipt. Other adjustment families remain open work rather than being silently treated as equivalent. +`WeightVarianceCompatibilityReceipt` closes the scientific-leaf form of #407 RED #8. It binds one exact `FinalAnalysisWeightReceipt` to the separate #406 variance-design receipt/version/digest and requires the variance side to identify the same analytic-case occurrence set, weight-eligibility receipt, correction sequence, and final point-weight artifact. A replicate or variance construction generated from a different eligibility, correction, case set, or final-weight version therefore cannot be paired with the point estimate merely because both digests are well formed. This receipt is correlation evidence only; #235/#248 remain responsible for resolving the released #406 owner evidence rather than trusting these caller-supplied coordinates as authority. + `ValidationAnalysisResult` accepts only a result linked to the handoff digest and the same reviewed fast-mlsirm revision. It records the Rust CPU/GPU backend, precision, aggregate missingness counts, finite effect and interval values, and explicit convergence or nonconvergence diagnostics. Timestamps and finite numeric values are snapshotted before canonicalization, and the result envelope accepts only the exact governed `MissingnessSummary` and `ConvergenceDiagnostics` runtime types, preventing mutable runtime values or subclass method overrides from adding unreviewed or person-level fields to canonical audit evidence. Missingness counts must be internally possible: complete observations cannot overlap either predictor-missing or criterion-missing observations beyond the declared sample total. -Point-estimation semantics are explicit. `unweighted` results cannot carry weight receipts. `weighted_design_based` results must separately bind the exact `FinalAnalysisWeightReceipt` digest and the variance-design receipt digest that supports the uncertainty method actually used. A replicate/variance evidence reference therefore cannot silently stand in for the final point-estimation weight, or vice versa. The result remains scientific evidence for accountable human interpretation and never becomes an employment decision. +Point-estimation semantics are explicit. `unweighted` results cannot carry weight receipts. `weighted_design_based` results must separately bind the exact `FinalAnalysisWeightReceipt` digest, the variance-design receipt digest that supports the uncertainty method actually used, and the `WeightVarianceCompatibilityReceipt` proving those two evidence paths refer to the same point-weight basis. A replicate/variance evidence reference therefore cannot silently stand in for the final point-estimation weight, or vice versa. The result remains scientific evidence for accountable human interpretation and never becomes an employment decision. ## What it does not do @@ -30,13 +32,13 @@ Point-estimation semantics are explicit. `unweighted` results cannot carry weigh - It does **not** claim that a selection procedure is valid. - It does **not** interpret adverse impact. - It does **not** authorize hiring, promotion, termination, compensation, or another employment decision. -- It does **not** yet complete #407. Durable owner-side verification that adjustment and auxiliary-authority coordinates resolve to the released typed receipts/contracts they claim, enforcement that the resolved auxiliary projection is authorized for the exact scientific purpose/use receipt/time, and released auxiliary evidence exchange remain open work in `workforce_validation`. +- It does **not** yet complete #407. Durable owner-side verification that adjustment, point/variance compatibility, and auxiliary-authority coordinates resolve to the released typed receipts/contracts they claim, enforcement that the resolved auxiliary projection is authorized for the exact scientific purpose/use receipt/time, and released auxiliary/variance evidence exchange remain open work in `workforce_validation`. The fast-mlsirm repository remains a dedicated-writer dependency. This package records only the immutable revision reviewed for the handoff. Durable Workforce Validation registry/API/persistence remains on the canonical #235/#248 owner path; this package does not create a parallel service. ## Host obligations -Before an approved offline validation worker executes the handoff, the Orgmetra host must re-resolve every reference inside `tenant_record_id` and prove that the predictor, criterion, population, and policy evidence belong to the exact validation study and Job. The requester and reviewer must resolve to distinct authoritative actors. For weighted design-based inference, the host must also verify the final point-weight receipt, its matching cross-sectional/longitudinal weight-eligibility receipt, its upstream source/sampling evidence, each typed adjustment receipt identified by the ordered chain, the calibration auxiliary authority/purpose/released owner contract/authorization/scientific-use tuple when applicable, and the separately bound variance-design evidence rather than trusting caller-supplied labels or digests. A numerical result is scientific evidence for accountable human interpretation, never an autonomous employment decision. +Before an approved offline validation worker executes the handoff, the Orgmetra host must re-resolve every reference inside `tenant_record_id` and prove that the predictor, criterion, population, and policy evidence belong to the exact validation study and Job. The requester and reviewer must resolve to distinct authoritative actors. For weighted design-based inference, the host must also verify the final point-weight receipt, its matching cross-sectional/longitudinal weight-eligibility receipt, its upstream source/sampling evidence, each typed adjustment receipt identified by the ordered chain, the calibration auxiliary authority/purpose/released owner contract/authorization/scientific-use tuple when applicable, the separately bound #406 variance-design evidence, and the point/variance compatibility receipt against released owner evidence rather than trusting caller-supplied labels or digests. A numerical result is scientific evidence for accountable human interpretation, never an autonomous employment decision. ## Verification From 0f1d0e9d4ed4e04b6fc5ed9425cb91e6063acb7f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 10:03:41 +0900 Subject: [PATCH 110/158] docs(adr): require point-variance lineage compatibility --- ...overned-selection-validity-analysis-handoff.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/adr/0027-governed-selection-validity-analysis-handoff.md b/docs/adr/0027-governed-selection-validity-analysis-handoff.md index cbbd2aa7b..82a96b0e0 100644 --- a/docs/adr/0027-governed-selection-validity-analysis-handoff.md +++ b/docs/adr/0027-governed-selection-validity-analysis-handoff.md @@ -36,11 +36,15 @@ For #407's weighted design-based inference boundary, the active package adds `Fi The adjustment chain has typed scientific evidence for three high-risk transform families. `NonresponseAdjustmentReceipt` binds a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, explicit treatment of ineligible/unknown/unavailable cases, and input/output weight artifacts. `CalibrationAdjustmentReceipt` binds the target population/window, purpose-limited auxiliary projection, exact scientific-use purpose reference/digest, scientific auxiliary-authority reference, released owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and use instant, authoritative benchmark receipt, algorithm/version, constraints digest, input/output artifacts, and an explicit `converged` or `fallback_applied` termination state. The scientific-use instant is frozen and cannot be later than receipt construction. This makes the complete opaque correlation tuple expected by the canonical Workforce Validation owner resolver part of the scientific receipt without copying protected auxiliary values. The leaf package still does not pretend its caller-supplied coordinates are authoritative: the durable `workforce_validation` service must resolve the released owner contract, authorization receipt, and scientific-use receipt and prove that the actual projection is permitted for the exact scientific purpose/use before execution or release. A fallback must identify its immutable fallback-rule reference/digest. A failed/nonconverged calibration cannot be labeled as an accepted calibration receipt. `TrimmingBoundingAdjustmentReceipt` binds a versioned trimming/bounding rule, reproducible rule configuration digest, the semantic occurrence set and count of cases actually affected, and input/output weight artifacts; a declared trim/bound transform cannot be a no-op. Known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization `AnalysisWeightAdjustment` codes also carry an `evidence_kind` and fail closed unless it names the matching typed receipt family. -This executable slice still does not claim #407 complete. The leaf can now preserve all opaque coordinates needed for durable owner-side correlation instead of requiring owner-contract digest, authorization identity, or scientific-use identity/time to be supplied out-of-band. Durable service/API verification that those coordinates resolve to the released typed receipts/contracts claimed, verification that the released authorization permits the resolved projection for the exact purpose and use, and released auxiliary-owner evidence exchange remain `workforce_validation` scientific truth rather than Talent truth or cross-context SQL reconstruction. +#407 RED #8 requires more than two different well-formed digests. A final point weight must not be combined with replicate or variance evidence generated from a different eligibility, correction, calibration, analytic case set, or final-weight version. The active package therefore adds `WeightVarianceCompatibilityReceipt`. It binds one exact `FinalAnalysisWeightReceipt` to the separate #406 variance-design receipt reference/version/digest and requires the variance side to identify the exact same analysis-weight receipt digest, analytic-case occurrence set, weight-eligibility receipt digest, correction sequence, and final-weight artifact. The receipt must be constructed no earlier than the point-weight receipt, and its variance-design digest must remain distinct from the point-weight receipt itself. `ValidationAnalysisResult` now requires this compatibility receipt for `weighted_design_based` output and verifies that its point-weight and variance-design digests are the same ones recorded on the result. Unweighted results cannot carry compatibility evidence. + +This compatibility receipt is deliberately not a self-authenticating variance owner. Its role is deterministic scientific correlation at the leaf boundary: it prevents a caller from presenting internally inconsistent point/variance lineages as one weighted result. Canonical #235/#248 remain responsible for resolving the released #406 variance-design evidence, proving that the variance-side coordinates actually came from the authoritative released owner contract, and rejecting caller-fabricated correlation data. The same owner boundary continues to apply to the calibration auxiliary-authority tuple. + +This executable slice still does not claim #407 complete. The leaf can now preserve the opaque coordinates needed for durable owner-side calibration correlation and can fail closed when point-weight and variance evidence are internally incongruent. Durable service/API verification that calibration coordinates and point/variance compatibility coordinates resolve to the released typed receipts/contracts claimed, verification that released authorization permits the resolved projection for the exact purpose and use, and released auxiliary/variance owner evidence exchange remain `workforce_validation` scientific truth rather than Talent truth or cross-context SQL reconstruction. The NIST Privacy Framework 1.0 is used narrowly as a privacy-risk-management basis for expressing and verifying data-processing requirements across organizational roles and contracts; as of the 2026-09-17 check, NIST still presents Privacy Framework 1.1 as an Initial Public Draft rather than a final replacement. This ADR does not treat the voluntary framework as employment law or infer legal permission from a NIST profile. -The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, prove requester/reviewer identities are distinct authoritative actors, verify cross-sectional/longitudinal weight eligibility against the estimand, and verify typed weight-adjustment receipts plus any calibration auxiliary authority/purpose/released-owner/authorization/scientific-use tuple rather than trusting caller-supplied evidence labels or digests. +The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, prove requester/reviewer identities are distinct authoritative actors, verify cross-sectional/longitudinal weight eligibility against the estimand, verify typed weight-adjustment receipts plus any calibration auxiliary authority/purpose/released-owner/authorization/scientific-use tuple, and resolve the point/variance compatibility receipt against the released #406 owner evidence rather than trusting caller-supplied evidence labels or digests. ## Consequences @@ -52,6 +56,7 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - Result canonicalization cannot be extended by an unreviewed subclass to serialize extra decision-like or person-level fields. - Caller-controlled timestamp and numeric runtime behavior cannot rewrite an accepted canonical digest after construction or turn malformed oversized worker output into an uncaught exception type. - Weighted design-based results can no longer identify only a sample while omitting which final point-weight evidence and variance-design evidence were actually used. +- A variance/replicate construction from a different analytic case set, eligibility receipt, correction sequence, or final point-weight artifact cannot be paired with the point estimate as though the two lineages were congruent. - Cross-sectional and longitudinal weights cannot be interchanged when target population, reference duration, eligible cases, or final weight artifact differ from the estimand. - The ordered weight transformation chain is independently digest-correlatable without centralizing case-level weights or auxiliary attributes. - Nonresponse cannot silently drop refusal/unreachable/ineligible/failure dispositions behind one opaque adjustment label. @@ -64,13 +69,13 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - This slice does not execute a statistical model, estimate validity, correct for measurement error/range restriction, evaluate adverse impact, or assert legal compliance. - Sampling design, estimator choice, missing-data treatment, reliability evidence, multiplicity, transportability, fairness analysis, and model diagnostics must be encoded in the referenced analysis plan and reviewed before execution. -- The current #407 slice validates generic ordered weight provenance, weighted-result binding, cross-sectional/longitudinal eligibility congruence, disposition-aware nonresponse evidence, owner-correlatable calibration evidence, and trimming/bounding rule provenance; it does not yet make durable typed-receipt/authorization resolution or released auxiliary-owner exchange fully executable. +- The current #407 slice validates generic ordered weight provenance, weighted-result binding, point/variance compatibility, cross-sectional/longitudinal eligibility congruence, disposition-aware nonresponse evidence, owner-correlatable calibration evidence, and trimming/bounding rule provenance; it does not yet make durable typed-receipt/authorization/variance-owner resolution or released auxiliary/variance exchange fully executable. - The generic result envelope does not invent an estimator-specific minimum complete-case count. Whether a converged estimator is identified under a particular missing-data design belongs to the reviewed analysis plan and numerical-worker contract; Orgmetra fails closed on impossible aggregate counts without silently replacing that foreign scientific contract with complete-case analysis. -- The package validates the result envelope, but a future execution adapter must still re-resolve the handoff references, verify the result provenance artifact, resolve typed adjustment and purpose-authorization coordinates to released owner evidence, and attach evidence only after accountable human review. +- The package validates the result envelope, but a future execution adapter must still re-resolve the handoff references, verify the result provenance artifact, resolve typed adjustment and purpose-authorization coordinates to released owner evidence, resolve point/variance compatibility against released #406 evidence, and attach evidence only after accountable human review. ## Verification -The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. #407 adds RED/GREEN coverage for deterministic value-minimized final-weight receipts, exact sampling-receipt binding, contiguous transformation lineage, append-only correction linkage, fail-closed weighted-result binding to separate point-weight and variance-design receipts, cross-sectional/longitudinal scope and target-population/reference-duration/case-set/final-artifact congruence, explicit nonresponse disposition treatment, calibration benchmark/constraint/termination provenance, scientific auxiliary-authority/purpose/released-owner/authorization/scientific-use correlation coordinates, owner-contract and use-receipt digest validation, future-use-time rejection, fallback-rule disclosure, trimming/bounding rule/configuration and affected-case provenance, no-op trim/bound rejection, and typed adjustment-evidence matching. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. +The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. #407 adds RED/GREEN coverage for deterministic value-minimized final-weight receipts, exact sampling-receipt binding, contiguous transformation lineage, append-only correction linkage, fail-closed weighted-result binding to separate point-weight and variance-design receipts, exact point/variance compatibility across analysis-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence and final-weight artifact, cross-sectional/longitudinal scope and target-population/reference-duration/case-set/final-artifact congruence, explicit nonresponse disposition treatment, calibration benchmark/constraint/termination provenance, scientific auxiliary-authority/purpose/released-owner/authorization/scientific-use correlation coordinates, owner-contract and use-receipt digest validation, future-use-time rejection, fallback-rule disclosure, trimming/bounding rule/configuration and affected-case provenance, no-op trim/bound rejection, and typed adjustment-evidence matching. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. ## References From 1259184a4e147c9e3ab095cb866205f17f92c3ca Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 10:04:16 +0900 Subject: [PATCH 111/158] test(validity): cover compatibility receipt trust boundaries --- ...st_weight_variance_compatibility_receipt.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/packages/validity-analysis/tests/test_weight_variance_compatibility_receipt.py b/packages/validity-analysis/tests/test_weight_variance_compatibility_receipt.py index 738f84823..2ae22fc02 100644 --- a/packages/validity-analysis/tests/test_weight_variance_compatibility_receipt.py +++ b/packages/validity-analysis/tests/test_weight_variance_compatibility_receipt.py @@ -155,6 +155,24 @@ def test_compatibility_receipt_rejects_foreign_point_weight_or_tenant() -> None: compatibility(analysis_weight_receipt=foreign) +def test_compatibility_receipt_rejects_point_receipt_as_variance_receipt() -> None: + """Point-weight evidence cannot masquerade as the distinct variance-design receipt.""" + point_weight = weight_receipt() + with pytest.raises(ValueError, match="distinct from the analysis weight receipt"): + compatibility( + analysis_weight_receipt=point_weight, + variance_design_receipt_digest=point_weight.sha256_digest(), + ) + + +def test_compatibility_receipt_rejects_time_reversal_and_schema_forgery() -> None: + """Compatibility evidence cannot predate its point weight or invent a schema version.""" + with pytest.raises(ValueError, match="cannot precede"): + compatibility(constructed_at=datetime(2026, 9, 17, 0, 29, tzinfo=timezone.utc)) + with pytest.raises(ValueError, match="evidence_version"): + compatibility(evidence_version=2) + + @pytest.mark.parametrize("version", [0, True]) def test_variance_design_receipt_version_is_strictly_positive(version: object) -> None: """Reject unversioned or boolean variance owner evidence.""" From 16093c58dff462dd16b685322879ef7340b06280 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 10:04:59 +0900 Subject: [PATCH 112/158] test(validity): cover result compatibility trust boundary --- .../test_analysis_weight_result_binding.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/packages/validity-analysis/tests/test_analysis_weight_result_binding.py b/packages/validity-analysis/tests/test_analysis_weight_result_binding.py index 25de7e51b..84344bcd2 100644 --- a/packages/validity-analysis/tests/test_analysis_weight_result_binding.py +++ b/packages/validity-analysis/tests/test_analysis_weight_result_binding.py @@ -206,6 +206,38 @@ def test_weighted_result_rejects_compatibility_for_other_point_or_variance_evide ) +def test_weighted_result_rechecks_compatibility_tenant_and_time_boundary() -> None: + """Forged compatibility state cannot cross tenant or result-time boundaries.""" + weight = point_weight() + correlation = compatibility(weight) + object.__setattr__( + correlation, + "tenant_record_id", + "10000000-0000-7000-8000-000000000002", + ) + with pytest.raises(ValueError, match="tenant_record_id"): + result( + point_estimation_mode="weighted_design_based", + analysis_weight_receipt_digest=weight.sha256_digest(), + variance_design_receipt_digest=VARIANCE_DIGEST, + weight_variance_compatibility=correlation, + ) + + later = compatibility(weight) + object.__setattr__( + later, + "constructed_at", + datetime(2026, 9, 17, 4, 11, tzinfo=timezone.utc), + ) + with pytest.raises(ValueError, match="cannot be constructed after"): + result( + point_estimation_mode="weighted_design_based", + analysis_weight_receipt_digest=weight.sha256_digest(), + variance_design_receipt_digest=VARIANCE_DIGEST, + weight_variance_compatibility=later, + ) + + @pytest.mark.parametrize( "overrides,match", [ From 7551b373d6a3622154fb174f439ec3226bb46631 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 11:04:43 +0900 Subject: [PATCH 113/158] test(validity): require benchmark authority provenance --- .../tests/test_weight_adjustment_semantics.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/packages/validity-analysis/tests/test_weight_adjustment_semantics.py b/packages/validity-analysis/tests/test_weight_adjustment_semantics.py index fe8d9bf86..9378997c0 100644 --- a/packages/validity-analysis/tests/test_weight_adjustment_semantics.py +++ b/packages/validity-analysis/tests/test_weight_adjustment_semantics.py @@ -22,6 +22,7 @@ DIGEST_2 = "2" * 64 DIGEST_3 = "3" * 64 DIGEST_4 = "4" * 64 +DIGEST_5 = "5" * 64 def nonresponse_receipt(**overrides: object) -> NonresponseAdjustmentReceipt: @@ -68,7 +69,12 @@ def calibration_receipt(**overrides: object) -> CalibrationAdjustmentReceipt: "auxiliary_scientific_use_receipt_digest": DIGEST_2, "auxiliary_scientific_use_at": NOW, "benchmark_receipt_reference": "calibration_benchmark_receipt:66666666-6666-4666-8666-666666666666", + "benchmark_receipt_version": 2, "benchmark_receipt_digest": DIGEST_C, + "benchmark_owner_contract_reference": "released_owner_contract:66666666-6666-4666-8666-666666666667", + "benchmark_owner_contract_version": 3, + "benchmark_owner_contract_digest": DIGEST_5, + "benchmark_reference_at": NOW - timedelta(days=1), "algorithm_reference": "calibration_algorithm:77777777-7777-4777-8777-777777777777", "algorithm_version": 1, "constraints_digest": DIGEST_F, @@ -134,7 +140,10 @@ def test_calibration_receipt_binds_owner_authority_use_and_termination_state() - candidate = calibration_receipt() canonical = candidate.canonical_json() assert candidate.sha256_digest() == calibration_receipt().sha256_digest() + assert '"benchmark_receipt_version":2' in canonical assert f'"benchmark_receipt_digest":"{DIGEST_C}"' in canonical + assert f'"benchmark_owner_contract_digest":"{DIGEST_5}"' in canonical + assert '"benchmark_reference_at":"2026-09-16T05:00:00Z"' in canonical assert f'"auxiliary_purpose_digest":"{DIGEST_3}"' in canonical assert f'"auxiliary_owner_contract_digest":"{DIGEST_1}"' in canonical assert f'"auxiliary_authorization_receipt_digest":"{DIGEST_4}"' in canonical @@ -172,6 +181,22 @@ def test_calibration_receipt_binds_owner_authority_use_and_termination_state() - calibration_receipt(auxiliary_scientific_use_at="2026-09-17T05:00:00Z") with pytest.raises(ValueError, match="cannot be later"): calibration_receipt(auxiliary_scientific_use_at=NOW + timedelta(seconds=1)) + with pytest.raises(TypeError): + calibration_receipt() + with pytest.raises(ValueError, match="benchmark_receipt_version"): + calibration_receipt(benchmark_receipt_version=0) + with pytest.raises(ValueError, match="benchmark_receipt_version"): + calibration_receipt(benchmark_receipt_version=True) + with pytest.raises(ValueError, match="benchmark_owner_contract_reference"): + calibration_receipt(benchmark_owner_contract_reference="benchmark-owner-v3") + with pytest.raises(ValueError, match="benchmark_owner_contract_version"): + calibration_receipt(benchmark_owner_contract_version=0) + with pytest.raises(ValueError, match="benchmark_owner_contract_digest"): + calibration_receipt(benchmark_owner_contract_digest="owner-v3") + with pytest.raises(ValueError, match="benchmark_reference_at"): + calibration_receipt(benchmark_reference_at="2026-09-16T05:00:00Z") + with pytest.raises(ValueError, match="benchmark_reference_at cannot be later"): + calibration_receipt(benchmark_reference_at=NOW + timedelta(seconds=1)) with pytest.raises(ValueError, match="termination_code"): calibration_receipt(termination_code="failed") with pytest.raises(ValueError, match="termination_code"): From a862d12b0488935ee33ef5835f9ef882c6460a8f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 11:05:24 +0900 Subject: [PATCH 114/158] test(validity): keep benchmark provenance RED causal --- .../validity-analysis/tests/test_weight_adjustment_semantics.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/validity-analysis/tests/test_weight_adjustment_semantics.py b/packages/validity-analysis/tests/test_weight_adjustment_semantics.py index 9378997c0..60d6dd229 100644 --- a/packages/validity-analysis/tests/test_weight_adjustment_semantics.py +++ b/packages/validity-analysis/tests/test_weight_adjustment_semantics.py @@ -181,8 +181,6 @@ def test_calibration_receipt_binds_owner_authority_use_and_termination_state() - calibration_receipt(auxiliary_scientific_use_at="2026-09-17T05:00:00Z") with pytest.raises(ValueError, match="cannot be later"): calibration_receipt(auxiliary_scientific_use_at=NOW + timedelta(seconds=1)) - with pytest.raises(TypeError): - calibration_receipt() with pytest.raises(ValueError, match="benchmark_receipt_version"): calibration_receipt(benchmark_receipt_version=0) with pytest.raises(ValueError, match="benchmark_receipt_version"): From 949b562775d47af526655ededf4e5b2de39d3f05 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 11:06:38 +0900 Subject: [PATCH 115/158] fix(validity): bind calibration benchmark authority --- .../src/orgmetra_validity_analysis/weights.py | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py b/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py index 9b9394e6e..78f6386ab 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py @@ -135,7 +135,7 @@ def sha256_digest(self) -> str: @dataclass(frozen=True, slots=True, repr=False) class CalibrationAdjustmentReceipt: - """Bind calibration to owner authority, benchmarks, and termination evidence.""" + """Bind calibration to owner authority, benchmark authority, and termination evidence.""" tenant_record_id: str receipt_reference: str @@ -155,7 +155,12 @@ class CalibrationAdjustmentReceipt: auxiliary_scientific_use_receipt_digest: str auxiliary_scientific_use_at: datetime benchmark_receipt_reference: str + benchmark_receipt_version: int benchmark_receipt_digest: str + benchmark_owner_contract_reference: str + benchmark_owner_contract_version: int + benchmark_owner_contract_digest: str + benchmark_reference_at: datetime algorithm_reference: str algorithm_version: int constraints_digest: str @@ -168,7 +173,7 @@ class CalibrationAdjustmentReceipt: evidence_version: int = 1 def __post_init__(self) -> None: - """Fail closed on floating authority, hidden fallback, or nonconverged output.""" + """Fail closed on floating authority, benchmark time, hidden fallback, or nonconvergence.""" _validate_operational_uuid(self.tenant_record_id, "tenant_record_id") _validate_reference( self.receipt_reference, @@ -219,6 +224,16 @@ def __post_init__(self) -> None: "calibration_benchmark_receipt", "benchmark_receipt_reference", ) + _positive_integer(self.benchmark_receipt_version, "benchmark_receipt_version") + _validate_reference( + self.benchmark_owner_contract_reference, + "released_owner_contract", + "benchmark_owner_contract_reference", + ) + _positive_integer( + self.benchmark_owner_contract_version, + "benchmark_owner_contract_version", + ) _validate_reference( self.algorithm_reference, "calibration_algorithm", @@ -232,6 +247,7 @@ def __post_init__(self) -> None: "auxiliary_authorization_receipt_digest", "auxiliary_scientific_use_receipt_digest", "benchmark_receipt_digest", + "benchmark_owner_contract_digest", "constraints_digest", "input_weight_artifact_digest", "output_weight_artifact_digest", @@ -264,12 +280,19 @@ def __post_init__(self) -> None: self.auxiliary_scientific_use_at, "auxiliary_scientific_use_at", ) + benchmark_reference_at = _freeze_timestamp( + self.benchmark_reference_at, + "benchmark_reference_at", + ) constructed_at = _freeze_timestamp(self.constructed_at, "constructed_at") if scientific_use_at > constructed_at: raise ValueError("auxiliary_scientific_use_at cannot be later than constructed_at") + if benchmark_reference_at > constructed_at: + raise ValueError("benchmark_reference_at cannot be later than constructed_at") if type(self.evidence_version) is not int or self.evidence_version != 1: raise ValueError("evidence_version must remain 1") object.__setattr__(self, "auxiliary_scientific_use_at", scientific_use_at) + object.__setattr__(self, "benchmark_reference_at", benchmark_reference_at) object.__setattr__(self, "constructed_at", constructed_at) def __repr__(self) -> str: @@ -298,8 +321,16 @@ def canonical_json(self) -> str: ), "auxiliary_scientific_use_receipt_digest": self.auxiliary_scientific_use_receipt_digest, "auxiliary_scientific_use_receipt_reference": self.auxiliary_scientific_use_receipt_reference, + "benchmark_owner_contract_digest": self.benchmark_owner_contract_digest, + "benchmark_owner_contract_reference": self.benchmark_owner_contract_reference, + "benchmark_owner_contract_version": self.benchmark_owner_contract_version, "benchmark_receipt_digest": self.benchmark_receipt_digest, "benchmark_receipt_reference": self.benchmark_receipt_reference, + "benchmark_receipt_version": self.benchmark_receipt_version, + "benchmark_reference_at": _canonical_timestamp( + self.benchmark_reference_at, + "benchmark_reference_at", + ), "constraints_digest": self.constraints_digest, "constructed_at": _canonical_timestamp(self.constructed_at, "constructed_at"), "evidence_version": self.evidence_version, From aa97628fe8dd8a9bd03a7a2164a084aae773da87 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 11:07:16 +0900 Subject: [PATCH 116/158] docs(validity): trace benchmark authority provenance --- packages/validity-analysis/CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/validity-analysis/CHANGELOG.md b/packages/validity-analysis/CHANGELOG.md index 60f3eddfd..29533b8db 100644 --- a/packages/validity-analysis/CHANGELOG.md +++ b/packages/validity-analysis/CHANGELOG.md @@ -15,8 +15,8 @@ - Distinguish unweighted from weighted design-based results and fail closed unless a weighted result separately binds the exact final analysis-weight receipt and variance-design receipt used. - Add `WeightVarianceCompatibilityReceipt` so variance evidence must correlate to the exact final point-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence, and final-weight artifact before a weighted result can be emitted. The receipt is correlation evidence, not self-authenticating #406 owner authority. - Require `ValidationAnalysisResult` to bind the compatibility receipt in addition to the point-weight and variance-design digests; unweighted results cannot carry compatibility evidence. -- Add typed `NonresponseAdjustmentReceipt` evidence that binds the exact versioned response/disposition receipt reference/version/digest, adjustment population, controlled method/configuration, explicit ineligible/unknown/unavailable treatment, and input/output weight artifacts; add typed `CalibrationAdjustmentReceipt` evidence bound to purpose-limited auxiliary projections, exact scientific-use purpose, scientific auxiliary-authority reference, released owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest/use time, authoritative benchmark receipts, constraints, and explicit convergence/fallback state; the use time cannot be later than receipt construction. +- Add typed `NonresponseAdjustmentReceipt` evidence that binds the exact versioned response/disposition receipt reference/version/digest, adjustment population, controlled method/configuration, explicit ineligible/unknown/unavailable treatment, and input/output weight artifacts; add typed `CalibrationAdjustmentReceipt` evidence bound to purpose-limited auxiliary projections, exact scientific-use purpose, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest/use time, and a separately versioned calibration benchmark receipt with released benchmark owner-contract reference/version/digest and exact benchmark reference time, plus constraints and explicit convergence/fallback state. Neither the use time nor benchmark reference time may be later than receipt construction. - Add typed `TrimmingBoundingAdjustmentReceipt` evidence that binds a versioned rule/configuration, exact affected semantic occurrence set and count, and input/output weight artifacts; known trimming/bounding/winsorization adjustments must name that evidence family. - Require known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization adjustment codes to identify the matching typed evidence kind rather than collapse into a generic opaque adjustment digest. -- Keep durable owner-side typed-receipt, point/variance compatibility, and auxiliary-authority resolution plus released auxiliary/variance evidence exchange explicitly incomplete under #407; the scientific leaf preserves exact correlation coordinates but does not self-authenticate them. +- Keep durable owner-side typed-receipt, point/variance compatibility, auxiliary-authority, and benchmark-authority resolution plus released auxiliary/benchmark/variance evidence exchange explicitly incomplete under #407; the scientific leaf preserves exact correlation coordinates but does not self-authenticate them. - Make `Validity Analysis Handoff Quality` retrigger on shared repository Python/test/clean-checkout configuration, with an executable regression preventing stale package-quality evidence after shared tooling changes. From bb79d94186f31e38916b1910622aaa82c9aa46af Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 11:07:54 +0900 Subject: [PATCH 117/158] docs(adr): make benchmark authority explicit --- ...verned-selection-validity-analysis-handoff.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/adr/0027-governed-selection-validity-analysis-handoff.md b/docs/adr/0027-governed-selection-validity-analysis-handoff.md index 82a96b0e0..20a2398ed 100644 --- a/docs/adr/0027-governed-selection-validity-analysis-handoff.md +++ b/docs/adr/0027-governed-selection-validity-analysis-handoff.md @@ -34,17 +34,17 @@ For #407's weighted design-based inference boundary, the active package adds `Fi `WeightEligibilityReceipt` makes weight use itself versioned evidence rather than a caller-supplied label. The receipt is closed to `cross_sectional` or `longitudinal` scope and binds the exact target population, reference-duration evidence, eligible-case set, and final point-weight artifact. `FinalAnalysisWeightReceipt` validates that the eligibility receipt belongs to the same tenant and that its scope, target population, reference duration, eligible cases, and weight artifact exactly match the estimand-side receipt. A longitudinal weight therefore cannot silently support a cross-sectional estimand or a different target period. The 2025 SIPP Users' Guide is used only as current primary methodological evidence that weight choice depends on both target population and duration and that longitudinal weights cover explicit multi-year reference periods; SIPP-specific variables or estimators are not imported into Orgmetra. -The adjustment chain has typed scientific evidence for three high-risk transform families. `NonresponseAdjustmentReceipt` binds a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, explicit treatment of ineligible/unknown/unavailable cases, and input/output weight artifacts. `CalibrationAdjustmentReceipt` binds the target population/window, purpose-limited auxiliary projection, exact scientific-use purpose reference/digest, scientific auxiliary-authority reference, released owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and use instant, authoritative benchmark receipt, algorithm/version, constraints digest, input/output artifacts, and an explicit `converged` or `fallback_applied` termination state. The scientific-use instant is frozen and cannot be later than receipt construction. This makes the complete opaque correlation tuple expected by the canonical Workforce Validation owner resolver part of the scientific receipt without copying protected auxiliary values. The leaf package still does not pretend its caller-supplied coordinates are authoritative: the durable `workforce_validation` service must resolve the released owner contract, authorization receipt, and scientific-use receipt and prove that the actual projection is permitted for the exact scientific purpose/use before execution or release. A fallback must identify its immutable fallback-rule reference/digest. A failed/nonconverged calibration cannot be labeled as an accepted calibration receipt. `TrimmingBoundingAdjustmentReceipt` binds a versioned trimming/bounding rule, reproducible rule configuration digest, the semantic occurrence set and count of cases actually affected, and input/output weight artifacts; a declared trim/bound transform cannot be a no-op. Known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization `AnalysisWeightAdjustment` codes also carry an `evidence_kind` and fail closed unless it names the matching typed receipt family. +The adjustment chain has typed scientific evidence for three high-risk transform families. `NonresponseAdjustmentReceipt` binds a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, explicit treatment of ineligible/unknown/unavailable cases, and input/output weight artifacts. `CalibrationAdjustmentReceipt` binds the target population/window, purpose-limited auxiliary projection, exact scientific-use purpose reference/digest, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and use instant, and a separately versioned calibration benchmark receipt reference/version/digest with its released benchmark owner-contract reference/version/digest and exact benchmark reference instant. It also binds algorithm/version, constraints digest, input/output artifacts, and an explicit `converged` or `fallback_applied` termination state. Both the scientific-use instant and benchmark reference instant are frozen and cannot be later than receipt construction. This makes the complete opaque correlation tuple expected by canonical Workforce Validation owner resolution part of the scientific receipt without copying protected auxiliary or benchmark values. The leaf package still does not pretend its caller-supplied coordinates are authoritative: the durable `workforce_validation` service must resolve the released auxiliary owner contract, authorization receipt, scientific-use receipt, benchmark receipt, and released benchmark owner contract and prove that the actual projection and benchmark are the exact released evidence authorized/referenced for the analysis before execution or release. A fallback must identify its immutable fallback-rule reference/digest. A failed/nonconverged calibration cannot be labeled as an accepted calibration receipt. `TrimmingBoundingAdjustmentReceipt` binds a versioned trimming/bounding rule, reproducible rule configuration digest, the semantic occurrence set and count of cases actually affected, and input/output weight artifacts; a declared trim/bound transform cannot be a no-op. Known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization `AnalysisWeightAdjustment` codes also carry an `evidence_kind` and fail closed unless it names the matching typed receipt family. #407 RED #8 requires more than two different well-formed digests. A final point weight must not be combined with replicate or variance evidence generated from a different eligibility, correction, calibration, analytic case set, or final-weight version. The active package therefore adds `WeightVarianceCompatibilityReceipt`. It binds one exact `FinalAnalysisWeightReceipt` to the separate #406 variance-design receipt reference/version/digest and requires the variance side to identify the exact same analysis-weight receipt digest, analytic-case occurrence set, weight-eligibility receipt digest, correction sequence, and final-weight artifact. The receipt must be constructed no earlier than the point-weight receipt, and its variance-design digest must remain distinct from the point-weight receipt itself. `ValidationAnalysisResult` now requires this compatibility receipt for `weighted_design_based` output and verifies that its point-weight and variance-design digests are the same ones recorded on the result. Unweighted results cannot carry compatibility evidence. -This compatibility receipt is deliberately not a self-authenticating variance owner. Its role is deterministic scientific correlation at the leaf boundary: it prevents a caller from presenting internally inconsistent point/variance lineages as one weighted result. Canonical #235/#248 remain responsible for resolving the released #406 variance-design evidence, proving that the variance-side coordinates actually came from the authoritative released owner contract, and rejecting caller-fabricated correlation data. The same owner boundary continues to apply to the calibration auxiliary-authority tuple. +This compatibility receipt is deliberately not a self-authenticating variance owner. Its role is deterministic scientific correlation at the leaf boundary: it prevents a caller from presenting internally inconsistent point/variance lineages as one weighted result. Canonical #235/#248 remain responsible for resolving the released #406 variance-design evidence, proving that the variance-side coordinates actually came from the authoritative released owner contract, and rejecting caller-fabricated correlation data. The same owner boundary continues to apply separately to the calibration auxiliary-authority tuple and calibration benchmark-authority tuple. -This executable slice still does not claim #407 complete. The leaf can now preserve the opaque coordinates needed for durable owner-side calibration correlation and can fail closed when point-weight and variance evidence are internally incongruent. Durable service/API verification that calibration coordinates and point/variance compatibility coordinates resolve to the released typed receipts/contracts claimed, verification that released authorization permits the resolved projection for the exact purpose and use, and released auxiliary/variance owner evidence exchange remain `workforce_validation` scientific truth rather than Talent truth or cross-context SQL reconstruction. +This executable slice still does not claim #407 complete. The leaf can now preserve the opaque coordinates needed for durable owner-side calibration auxiliary and benchmark correlation and can fail closed when point-weight and variance evidence are internally incongruent. Durable service/API verification that auxiliary, benchmark, and point/variance compatibility coordinates resolve to the released typed receipts/contracts claimed, verification that released authorization permits the resolved projection for the exact purpose and use, and released auxiliary/benchmark/variance owner evidence exchange remain `workforce_validation` scientific truth rather than Talent truth or cross-context SQL reconstruction. The NIST Privacy Framework 1.0 is used narrowly as a privacy-risk-management basis for expressing and verifying data-processing requirements across organizational roles and contracts; as of the 2026-09-17 check, NIST still presents Privacy Framework 1.1 as an Initial Public Draft rather than a final replacement. This ADR does not treat the voluntary framework as employment law or infer legal permission from a NIST profile. -The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, prove requester/reviewer identities are distinct authoritative actors, verify cross-sectional/longitudinal weight eligibility against the estimand, verify typed weight-adjustment receipts plus any calibration auxiliary authority/purpose/released-owner/authorization/scientific-use tuple, and resolve the point/variance compatibility receipt against the released #406 owner evidence rather than trusting caller-supplied evidence labels or digests. +The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, prove requester/reviewer identities are distinct authoritative actors, verify cross-sectional/longitudinal weight eligibility against the estimand, verify typed weight-adjustment receipts plus any calibration auxiliary authority/purpose/released-owner/authorization/scientific-use tuple and calibration benchmark receipt/version/released-owner/reference-time tuple, and resolve the point/variance compatibility receipt against the released #406 owner evidence rather than trusting caller-supplied evidence labels or digests. ## Consequences @@ -60,7 +60,7 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - Cross-sectional and longitudinal weights cannot be interchanged when target population, reference duration, eligible cases, or final weight artifact differ from the estimand. - The ordered weight transformation chain is independently digest-correlatable without centralizing case-level weights or auxiliary attributes. - Nonresponse cannot silently drop refusal/unreachable/ineligible/failure dispositions behind one opaque adjustment label. -- Calibration/raking cannot silently float benchmark ownership, omit the exact scientific-use purpose/released-owner/authorization/use correlation tuple, move its use instant after receipt construction, or hide fallback as successful convergence. +- Calibration/raking cannot silently float benchmark ownership, benchmark version, or benchmark reference time; omit the exact scientific-use purpose/released-owner/authorization/use correlation tuple; move its use or benchmark time after receipt construction; or hide fallback as successful convergence. - Trimming/bounding/winsorization cannot silently alter final point weights without an immutable rule/configuration and affected-case receipt. - Human interpretation remains explicit and separate from numerical output. - The dedicated-writer boundary remains intact: Orgmetra consumes only a pinned foreign revision/contract boundary and never mutates fast-mlsirm. @@ -69,13 +69,13 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - This slice does not execute a statistical model, estimate validity, correct for measurement error/range restriction, evaluate adverse impact, or assert legal compliance. - Sampling design, estimator choice, missing-data treatment, reliability evidence, multiplicity, transportability, fairness analysis, and model diagnostics must be encoded in the referenced analysis plan and reviewed before execution. -- The current #407 slice validates generic ordered weight provenance, weighted-result binding, point/variance compatibility, cross-sectional/longitudinal eligibility congruence, disposition-aware nonresponse evidence, owner-correlatable calibration evidence, and trimming/bounding rule provenance; it does not yet make durable typed-receipt/authorization/variance-owner resolution or released auxiliary/variance exchange fully executable. +- The current #407 slice validates generic ordered weight provenance, weighted-result binding, point/variance compatibility, cross-sectional/longitudinal eligibility congruence, disposition-aware nonresponse evidence, owner-correlatable calibration auxiliary and benchmark evidence, and trimming/bounding rule provenance; it does not yet make durable typed-receipt/authorization/benchmark-owner/variance-owner resolution or released auxiliary/benchmark/variance exchange fully executable. - The generic result envelope does not invent an estimator-specific minimum complete-case count. Whether a converged estimator is identified under a particular missing-data design belongs to the reviewed analysis plan and numerical-worker contract; Orgmetra fails closed on impossible aggregate counts without silently replacing that foreign scientific contract with complete-case analysis. -- The package validates the result envelope, but a future execution adapter must still re-resolve the handoff references, verify the result provenance artifact, resolve typed adjustment and purpose-authorization coordinates to released owner evidence, resolve point/variance compatibility against released #406 evidence, and attach evidence only after accountable human review. +- The package validates the result envelope, but a future execution adapter must still re-resolve the handoff references, verify the result provenance artifact, resolve typed adjustment, purpose-authorization, and benchmark coordinates to released owner evidence, resolve point/variance compatibility against released #406 evidence, and attach evidence only after accountable human review. ## Verification -The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. #407 adds RED/GREEN coverage for deterministic value-minimized final-weight receipts, exact sampling-receipt binding, contiguous transformation lineage, append-only correction linkage, fail-closed weighted-result binding to separate point-weight and variance-design receipts, exact point/variance compatibility across analysis-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence and final-weight artifact, cross-sectional/longitudinal scope and target-population/reference-duration/case-set/final-artifact congruence, explicit nonresponse disposition treatment, calibration benchmark/constraint/termination provenance, scientific auxiliary-authority/purpose/released-owner/authorization/scientific-use correlation coordinates, owner-contract and use-receipt digest validation, future-use-time rejection, fallback-rule disclosure, trimming/bounding rule/configuration and affected-case provenance, no-op trim/bound rejection, and typed adjustment-evidence matching. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. +The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. #407 adds RED/GREEN coverage for deterministic value-minimized final-weight receipts, exact sampling-receipt binding, contiguous transformation lineage, append-only correction linkage, fail-closed weighted-result binding to separate point-weight and variance-design receipts, exact point/variance compatibility across analysis-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence and final-weight artifact, cross-sectional/longitudinal scope and target-population/reference-duration/case-set/final-artifact congruence, explicit nonresponse disposition treatment, calibration benchmark receipt version/owner-contract/reference-time provenance, calibration constraint/termination provenance, scientific auxiliary-authority/purpose/released-owner/authorization/scientific-use correlation coordinates, owner-contract and use-receipt digest validation, future-use-time and future-benchmark-time rejection, fallback-rule disclosure, trimming/bounding rule/configuration and affected-case provenance, no-op trim/bound rejection, and typed adjustment-evidence matching. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. ## References From 972cf7af2939a19abe83c84463036eaee30cc64d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 11:08:19 +0900 Subject: [PATCH 118/158] docs(validity): document benchmark owner evidence --- packages/validity-analysis/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/validity-analysis/README.md b/packages/validity-analysis/README.md index fca101271..290da9320 100644 --- a/packages/validity-analysis/README.md +++ b/packages/validity-analysis/README.md @@ -12,7 +12,7 @@ The resulting canonical JSON is digest-addressable, contains no raw person-level `WeightEligibilityReceipt` makes cross-sectional versus longitudinal use machine-checkable rather than an opaque weight label. It binds the final weight artifact to one governed scope (`cross_sectional` or `longitudinal`), target population, reference-duration evidence, and eligible-case set. `FinalAnalysisWeightReceipt` fails closed unless those fields match the estimand and the same final point-weight artifact exactly, so a longitudinal weight cannot silently support a cross-sectional estimand or a different reference duration. -Nonresponse and calibration are no longer allowed to hide behind an undifferentiated adjustment digest. `NonresponseAdjustmentReceipt` binds the adjustment to a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, and explicit treatment codes for ineligible, unknown, and unavailable cases. `CalibrationAdjustmentReceipt` binds the adjustment to a target population/window, purpose-bound auxiliary projection, exact scientific-use purpose, scientific auxiliary-authority reference, released owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and owner-correlatable use instant, authoritative benchmark receipt, algorithm/version, constraints digest, and explicit `converged` or `fallback_applied` termination evidence. Its scientific-use instant cannot be later than receipt construction. A fallback must identify its versioned rule; a nonconverged calibration cannot masquerade as an accepted calibration receipt. These fields preserve the complete opaque correlation tuple expected by the canonical Workforce Validation authority resolver without copying protected auxiliary values. The leaf receipt still does not authenticate those coordinates itself: the durable owner service must resolve the released evidence and prove that the exact projection was authorized for that scientific purpose and use receipt. +Nonresponse and calibration are no longer allowed to hide behind an undifferentiated adjustment digest. `NonresponseAdjustmentReceipt` binds the adjustment to a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, and explicit treatment codes for ineligible, unknown, and unavailable cases. `CalibrationAdjustmentReceipt` binds the adjustment to a target population/window, purpose-bound auxiliary projection, exact scientific-use purpose, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and owner-correlatable use instant, plus a separately versioned calibration benchmark receipt with released benchmark owner-contract reference/version/digest and exact benchmark reference instant. It also binds algorithm/version, constraints digest, and explicit `converged` or `fallback_applied` termination evidence. Neither the scientific-use instant nor the benchmark reference instant may be later than receipt construction. A fallback must identify its versioned rule; a nonconverged calibration cannot masquerade as an accepted calibration receipt. These fields preserve the opaque correlation tuples expected by the canonical Workforce Validation authority boundaries without copying protected auxiliary or benchmark values. The leaf receipt still does not authenticate those coordinates itself: the durable owner service must resolve the released auxiliary and benchmark evidence and prove that the exact projection and benchmark are authoritative for the scientific use. Trimming, bounding, and winsorization have a separate provenance family rather than falling back to a generic adjustment label. `TrimmingBoundingAdjustmentReceipt` binds the exact versioned rule, its reproducible configuration digest, the semantic occurrence set and count of cases actually affected, and the input/output weight artifacts. A declared trim/bound transform must change artifact identity. Known trimming/bounding/winsorization adjustment codes fail closed unless `evidence_kind` names this typed receipt family. @@ -32,13 +32,13 @@ Point-estimation semantics are explicit. `unweighted` results cannot carry weigh - It does **not** claim that a selection procedure is valid. - It does **not** interpret adverse impact. - It does **not** authorize hiring, promotion, termination, compensation, or another employment decision. -- It does **not** yet complete #407. Durable owner-side verification that adjustment, point/variance compatibility, and auxiliary-authority coordinates resolve to the released typed receipts/contracts they claim, enforcement that the resolved auxiliary projection is authorized for the exact scientific purpose/use receipt/time, and released auxiliary/variance evidence exchange remain open work in `workforce_validation`. +- It does **not** yet complete #407. Durable owner-side verification that adjustment, point/variance compatibility, auxiliary-authority, and benchmark-authority coordinates resolve to the released typed receipts/contracts they claim; enforcement that the resolved auxiliary projection is authorized for the exact scientific purpose/use receipt/time; and released auxiliary/benchmark/variance evidence exchange remain open work in `workforce_validation`. The fast-mlsirm repository remains a dedicated-writer dependency. This package records only the immutable revision reviewed for the handoff. Durable Workforce Validation registry/API/persistence remains on the canonical #235/#248 owner path; this package does not create a parallel service. ## Host obligations -Before an approved offline validation worker executes the handoff, the Orgmetra host must re-resolve every reference inside `tenant_record_id` and prove that the predictor, criterion, population, and policy evidence belong to the exact validation study and Job. The requester and reviewer must resolve to distinct authoritative actors. For weighted design-based inference, the host must also verify the final point-weight receipt, its matching cross-sectional/longitudinal weight-eligibility receipt, its upstream source/sampling evidence, each typed adjustment receipt identified by the ordered chain, the calibration auxiliary authority/purpose/released owner contract/authorization/scientific-use tuple when applicable, the separately bound #406 variance-design evidence, and the point/variance compatibility receipt against released owner evidence rather than trusting caller-supplied labels or digests. A numerical result is scientific evidence for accountable human interpretation, never an autonomous employment decision. +Before an approved offline validation worker executes the handoff, the Orgmetra host must re-resolve every reference inside `tenant_record_id` and prove that the predictor, criterion, population, and policy evidence belong to the exact validation study and Job. The requester and reviewer must resolve to distinct authoritative actors. For weighted design-based inference, the host must also verify the final point-weight receipt, its matching cross-sectional/longitudinal weight-eligibility receipt, its upstream source/sampling evidence, each typed adjustment receipt identified by the ordered chain, the calibration auxiliary authority/purpose/released owner contract/authorization/scientific-use tuple when applicable, the calibration benchmark receipt/version/released benchmark owner contract/reference instant, the separately bound #406 variance-design evidence, and the point/variance compatibility receipt against released owner evidence rather than trusting caller-supplied labels or digests. A numerical result is scientific evidence for accountable human interpretation, never an autonomous employment decision. ## Verification From f19fad1aa1172195fa5bc42ddd9bfee4cd0474c0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 11:08:42 +0900 Subject: [PATCH 119/158] docs(traceability): bind benchmark owner coordinates --- docs/traceability/validation-analysis-handoff.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/traceability/validation-analysis-handoff.md b/docs/traceability/validation-analysis-handoff.md index 22c2e3fac..d129d1db4 100644 --- a/docs/traceability/validation-analysis-handoff.md +++ b/docs/traceability/validation-analysis-handoff.md @@ -17,7 +17,7 @@ Can an organization send one exact, reviewable validation study to its statistic | Final point-weight provenance | exact estimand/target/window/reference-duration and case-set digests, source/sampling receipt digests, base-weight evidence, ordered digest-linked adjustments, final weight artifact, append-only correction lineage | `FinalAnalysisWeightReceipt` deterministic/value-minimized regressions plus chain/correction fail-closed tests | | Weight eligibility congruence | governed `cross_sectional` or `longitudinal` scope plus exact target population, reference-duration evidence, eligible case set, and final point-weight artifact | `test_weight_eligibility_receipt.py` plus `FinalAnalysisWeightReceipt` mismatch/longitudinal-match regressions | | Nonresponse adjustment evidence | versioned response/disposition receipt, exact adjustment population, controlled method/configuration, explicit ineligible/unknown/unavailable treatment, input/output weight artifacts | `NonresponseAdjustmentReceipt` deterministic/value-minimized tests plus undocumented-treatment/no-op/version rejection | -| Calibration/raking evidence | target population/window, purpose-bound auxiliary projection, scientific-use purpose reference/digest, scientific auxiliary-authority reference, released owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and use instant, authoritative benchmark receipt, algorithm/version, constraints, input/output artifacts, explicit converged/fallback state and immutable fallback rule when used | `CalibrationAdjustmentReceipt` authority/purpose/owner/authorization/scientific-use/benchmark/termination/fallback regressions; scientific-use time cannot be later than receipt construction | +| Calibration/raking evidence | target population/window, purpose-bound auxiliary projection, scientific-use purpose reference/digest, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and use instant, calibration benchmark receipt reference/version/digest, released benchmark owner-contract reference/version/digest, benchmark reference instant, algorithm/version, constraints, input/output artifacts, explicit converged/fallback state and immutable fallback rule when used | `CalibrationAdjustmentReceipt` authority/purpose/owner/authorization/scientific-use/benchmark-owner/benchmark-time/termination/fallback regressions; neither scientific-use time nor benchmark reference time may be later than receipt construction | | Trimming/bounding evidence | versioned trimming/bounding rule, reproducible rule configuration digest, exact affected semantic occurrence set and count, input/output weight artifacts | `test_trimming_bounding_receipt.py` deterministic/value-minimized, no-op, missing-provenance, and typed-kind regressions | | Typed adjustment congruence | known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization adjustment codes must identify the matching typed receipt family through `evidence_kind` | specialized adjustment evidence-kind regressions | | Point/variance lineage congruence | #406 variance evidence must name the same final point-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence, and final-weight artifact used by the estimate | `test_weight_variance_compatibility_receipt.py` mismatch regressions | @@ -31,10 +31,10 @@ Can an organization send one exact, reviewable validation study to its statistic The active branch makes final-weight eligibility and three adjustment families executable rather than leaving them as opaque labels. It now also closes the leaf-side form of #407 RED #8: `WeightVarianceCompatibilityReceipt` requires the variance side to identify the exact point-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence, and final-weight artifact. `ValidationAnalysisResult` cannot emit weighted design-based evidence without that compatibility receipt in addition to separate point-weight and #406 variance-design digests. -This is still correlation evidence, not owner authentication. Calibration/raking/post-stratification evidence carries the complete opaque correlation tuple needed by the canonical Workforce Validation authority resolver: auxiliary authority and projection, exact scientific purpose, released owner-contract identity/version/digest, authorization receipt identity/digest, scientific-use receipt identity/digest and owner-correlatable use instant. Durable #235/#248 owner-side resolution must prove those coordinates and the point/variance compatibility coordinates against released/versioned owner evidence, prove that the exact projection is authorized for the exact scientific purpose/use receipt, and resolve the released #406 variance evidence rather than trusting leaf-supplied digests. Released auxiliary/variance evidence exchange also remains open. `workforce_validation` owns those scientific contracts and their released result; `talent_management` must not compute or reconstruct final analysis weights. +This is still correlation evidence, not owner authentication. Calibration/raking/post-stratification evidence carries the opaque coordinates needed by canonical Workforce Validation authority resolution: auxiliary authority and projection, exact scientific purpose, released auxiliary owner-contract identity/version/digest, authorization receipt identity/digest, scientific-use receipt identity/digest and owner-correlatable use instant, plus separately versioned benchmark receipt identity/digest, released benchmark owner-contract identity/version/digest, and benchmark reference instant. Durable #235/#248 owner-side resolution must prove both the auxiliary tuple and benchmark tuple against released/versioned owner evidence, prove that the exact projection is authorized for the exact scientific purpose/use receipt, and resolve the point/variance compatibility coordinates against released #406 variance evidence rather than trusting leaf-supplied digests. Released auxiliary/benchmark/variance evidence exchange also remains open. `workforce_validation` owns those scientific contracts and their released result; `talent_management` must not compute or reconstruct final analysis weights. ## Maturity `implemented_on_active_pr`. -Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. The active package validates the returned numerical/provenance envelope plus the current #407 final-weight/result-binding, point/variance compatibility, weight-eligibility, typed adjustment-evidence, and calibration-authority correlation contracts, but protected Orgmetra evidence still requires host re-resolution, durable owner evidence exchange, result-artifact verification, terminal checks, independent review, and accountable human interpretation. +Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. The active package validates the returned numerical/provenance envelope plus the current #407 final-weight/result-binding, point/variance compatibility, weight-eligibility, typed adjustment-evidence, auxiliary correlation, and benchmark correlation contracts, but protected Orgmetra evidence still requires host re-resolution, durable owner evidence exchange, result-artifact verification, terminal checks, independent review, and accountable human interpretation. From 7775fd9ff040deebf2a50b8848b75cb90e03ddf8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 14:14:52 +0900 Subject: [PATCH 120/158] test(validity): RED version auxiliary calibration projection --- ...alibration_auxiliary_projection_version.py | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 packages/validity-analysis/tests/test_calibration_auxiliary_projection_version.py diff --git a/packages/validity-analysis/tests/test_calibration_auxiliary_projection_version.py b/packages/validity-analysis/tests/test_calibration_auxiliary_projection_version.py new file mode 100644 index 000000000..66584423e --- /dev/null +++ b/packages/validity-analysis/tests/test_calibration_auxiliary_projection_version.py @@ -0,0 +1,64 @@ +"""RED contract for versioned calibration auxiliary projection identity.""" + +from datetime import datetime, timedelta, timezone + +import pytest + +from orgmetra_validity_analysis import CalibrationAdjustmentReceipt + +TENANT = "10000000-0000-7000-8000-000000000001" +NOW = datetime(2026, 9, 17, 5, 0, tzinfo=timezone.utc) + + +def receipt(**overrides: object) -> CalibrationAdjustmentReceipt: + """Return one calibration receipt with an exact versioned auxiliary projection.""" + values: dict[str, object] = { + "tenant_record_id": TENANT, + "receipt_reference": "calibration_adjustment_receipt:33333333-3333-4333-8333-333333333333", + "target_population_digest": "a" * 64, + "analysis_window_reference": "analysis_window:44444444-4444-4444-8444-444444444444", + "auxiliary_authority_reference": "scientific_auxiliary_authority:55555555-5555-4555-8555-555555555550", + "auxiliary_projection_reference": "calibration_auxiliary_projection:55555555-5555-4555-8555-555555555555", + "auxiliary_projection_version": 4, + "auxiliary_projection_digest": "b" * 64, + "auxiliary_purpose_reference": "scientific_data_use_purpose:55555555-5555-4555-8555-555555555556", + "auxiliary_purpose_digest": "3" * 64, + "auxiliary_owner_contract_reference": "released_owner_contract:55555555-5555-4555-8555-555555555557", + "auxiliary_owner_contract_version": 1, + "auxiliary_owner_contract_digest": "1" * 64, + "auxiliary_authorization_receipt_reference": "scientific_data_authorization:55555555-5555-4555-8555-555555555558", + "auxiliary_authorization_receipt_digest": "4" * 64, + "auxiliary_scientific_use_receipt_reference": "scientific_use_receipt:55555555-5555-4555-8555-555555555559", + "auxiliary_scientific_use_receipt_digest": "2" * 64, + "auxiliary_scientific_use_at": NOW, + "benchmark_receipt_reference": "calibration_benchmark_receipt:66666666-6666-4666-8666-666666666666", + "benchmark_receipt_version": 2, + "benchmark_receipt_digest": "c" * 64, + "benchmark_owner_contract_reference": "released_owner_contract:66666666-6666-4666-8666-666666666667", + "benchmark_owner_contract_version": 3, + "benchmark_owner_contract_digest": "5" * 64, + "benchmark_reference_at": NOW - timedelta(days=1), + "algorithm_reference": "calibration_algorithm:77777777-7777-4777-8777-777777777777", + "algorithm_version": 1, + "constraints_digest": "f" * 64, + "termination_code": "converged", + "input_weight_artifact_digest": "d" * 64, + "output_weight_artifact_digest": "e" * 64, + "constructed_at": NOW, + } + values.update(overrides) + return CalibrationAdjustmentReceipt(**values) + + +def test_calibration_receipt_binds_projection_version_into_canonical_evidence() -> None: + candidate = receipt() + assert candidate.auxiliary_projection_version == 4 + assert '"auxiliary_projection_version":4' in candidate.canonical_json() + + +@pytest.mark.parametrize("invalid", [0, -1, True, "4", None]) +def test_calibration_receipt_rejects_nonpositive_or_nonnumeric_projection_version( + invalid: object, +) -> None: + with pytest.raises(ValueError, match="auxiliary_projection_version"): + receipt(auxiliary_projection_version=invalid) From f62d4e4d4a36a0a3f0283b9002baa27a6e9f6401 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 14:16:03 +0900 Subject: [PATCH 121/158] fix(validity): version calibration auxiliary projection --- .../src/orgmetra_validity_analysis/weights.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py b/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py index 78f6386ab..2d019dcd3 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py @@ -143,6 +143,7 @@ class CalibrationAdjustmentReceipt: analysis_window_reference: str auxiliary_authority_reference: str auxiliary_projection_reference: str + auxiliary_projection_version: int auxiliary_projection_digest: str auxiliary_purpose_reference: str auxiliary_purpose_digest: str @@ -195,6 +196,10 @@ def __post_init__(self) -> None: "calibration_auxiliary_projection", "auxiliary_projection_reference", ) + _positive_integer( + self.auxiliary_projection_version, + "auxiliary_projection_version", + ) _validate_reference( self.auxiliary_purpose_reference, "scientific_data_use_purpose", @@ -313,6 +318,7 @@ def canonical_json(self) -> str: "auxiliary_owner_contract_version": self.auxiliary_owner_contract_version, "auxiliary_projection_digest": self.auxiliary_projection_digest, "auxiliary_projection_reference": self.auxiliary_projection_reference, + "auxiliary_projection_version": self.auxiliary_projection_version, "auxiliary_purpose_digest": self.auxiliary_purpose_digest, "auxiliary_purpose_reference": self.auxiliary_purpose_reference, "auxiliary_scientific_use_at": _canonical_timestamp( From 26ace2c8ed6b61d2955fd7aa664c5f9d8842d818 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 14:16:44 +0900 Subject: [PATCH 122/158] test(validity): cover auxiliary projection version --- .../tests/test_weight_adjustment_semantics.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/validity-analysis/tests/test_weight_adjustment_semantics.py b/packages/validity-analysis/tests/test_weight_adjustment_semantics.py index 60d6dd229..3ef4f5324 100644 --- a/packages/validity-analysis/tests/test_weight_adjustment_semantics.py +++ b/packages/validity-analysis/tests/test_weight_adjustment_semantics.py @@ -57,6 +57,7 @@ def calibration_receipt(**overrides: object) -> CalibrationAdjustmentReceipt: "analysis_window_reference": "analysis_window:44444444-4444-4444-8444-444444444444", "auxiliary_authority_reference": "scientific_auxiliary_authority:55555555-5555-4555-8555-555555555550", "auxiliary_projection_reference": "calibration_auxiliary_projection:55555555-5555-4555-8555-555555555555", + "auxiliary_projection_version": 4, "auxiliary_projection_digest": DIGEST_B, "auxiliary_purpose_reference": "scientific_data_use_purpose:55555555-5555-4555-8555-555555555556", "auxiliary_purpose_digest": DIGEST_3, @@ -144,6 +145,7 @@ def test_calibration_receipt_binds_owner_authority_use_and_termination_state() - assert f'"benchmark_receipt_digest":"{DIGEST_C}"' in canonical assert f'"benchmark_owner_contract_digest":"{DIGEST_5}"' in canonical assert '"benchmark_reference_at":"2026-09-16T05:00:00Z"' in canonical + assert '"auxiliary_projection_version":4' in canonical assert f'"auxiliary_purpose_digest":"{DIGEST_3}"' in canonical assert f'"auxiliary_owner_contract_digest":"{DIGEST_1}"' in canonical assert f'"auxiliary_authorization_receipt_digest":"{DIGEST_4}"' in canonical @@ -163,6 +165,10 @@ def test_calibration_receipt_binds_owner_authority_use_and_termination_state() - with pytest.raises(ValueError, match="auxiliary_authority_reference"): calibration_receipt(auxiliary_authority_reference="authority-v1") + with pytest.raises(ValueError, match="auxiliary_projection_version"): + calibration_receipt(auxiliary_projection_version=0) + with pytest.raises(ValueError, match="auxiliary_projection_version"): + calibration_receipt(auxiliary_projection_version=True) with pytest.raises(ValueError, match="auxiliary_purpose_digest"): calibration_receipt(auxiliary_purpose_digest="purpose-v1") with pytest.raises(ValueError, match="auxiliary_owner_contract_version"): From 9248019fdc7e35d7075f5709e1b5a5597a613684 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 14:16:55 +0900 Subject: [PATCH 123/158] test(validity): fold projection RED into canonical suite --- ...alibration_auxiliary_projection_version.py | 64 ------------------- 1 file changed, 64 deletions(-) delete mode 100644 packages/validity-analysis/tests/test_calibration_auxiliary_projection_version.py diff --git a/packages/validity-analysis/tests/test_calibration_auxiliary_projection_version.py b/packages/validity-analysis/tests/test_calibration_auxiliary_projection_version.py deleted file mode 100644 index 66584423e..000000000 --- a/packages/validity-analysis/tests/test_calibration_auxiliary_projection_version.py +++ /dev/null @@ -1,64 +0,0 @@ -"""RED contract for versioned calibration auxiliary projection identity.""" - -from datetime import datetime, timedelta, timezone - -import pytest - -from orgmetra_validity_analysis import CalibrationAdjustmentReceipt - -TENANT = "10000000-0000-7000-8000-000000000001" -NOW = datetime(2026, 9, 17, 5, 0, tzinfo=timezone.utc) - - -def receipt(**overrides: object) -> CalibrationAdjustmentReceipt: - """Return one calibration receipt with an exact versioned auxiliary projection.""" - values: dict[str, object] = { - "tenant_record_id": TENANT, - "receipt_reference": "calibration_adjustment_receipt:33333333-3333-4333-8333-333333333333", - "target_population_digest": "a" * 64, - "analysis_window_reference": "analysis_window:44444444-4444-4444-8444-444444444444", - "auxiliary_authority_reference": "scientific_auxiliary_authority:55555555-5555-4555-8555-555555555550", - "auxiliary_projection_reference": "calibration_auxiliary_projection:55555555-5555-4555-8555-555555555555", - "auxiliary_projection_version": 4, - "auxiliary_projection_digest": "b" * 64, - "auxiliary_purpose_reference": "scientific_data_use_purpose:55555555-5555-4555-8555-555555555556", - "auxiliary_purpose_digest": "3" * 64, - "auxiliary_owner_contract_reference": "released_owner_contract:55555555-5555-4555-8555-555555555557", - "auxiliary_owner_contract_version": 1, - "auxiliary_owner_contract_digest": "1" * 64, - "auxiliary_authorization_receipt_reference": "scientific_data_authorization:55555555-5555-4555-8555-555555555558", - "auxiliary_authorization_receipt_digest": "4" * 64, - "auxiliary_scientific_use_receipt_reference": "scientific_use_receipt:55555555-5555-4555-8555-555555555559", - "auxiliary_scientific_use_receipt_digest": "2" * 64, - "auxiliary_scientific_use_at": NOW, - "benchmark_receipt_reference": "calibration_benchmark_receipt:66666666-6666-4666-8666-666666666666", - "benchmark_receipt_version": 2, - "benchmark_receipt_digest": "c" * 64, - "benchmark_owner_contract_reference": "released_owner_contract:66666666-6666-4666-8666-666666666667", - "benchmark_owner_contract_version": 3, - "benchmark_owner_contract_digest": "5" * 64, - "benchmark_reference_at": NOW - timedelta(days=1), - "algorithm_reference": "calibration_algorithm:77777777-7777-4777-8777-777777777777", - "algorithm_version": 1, - "constraints_digest": "f" * 64, - "termination_code": "converged", - "input_weight_artifact_digest": "d" * 64, - "output_weight_artifact_digest": "e" * 64, - "constructed_at": NOW, - } - values.update(overrides) - return CalibrationAdjustmentReceipt(**values) - - -def test_calibration_receipt_binds_projection_version_into_canonical_evidence() -> None: - candidate = receipt() - assert candidate.auxiliary_projection_version == 4 - assert '"auxiliary_projection_version":4' in candidate.canonical_json() - - -@pytest.mark.parametrize("invalid", [0, -1, True, "4", None]) -def test_calibration_receipt_rejects_nonpositive_or_nonnumeric_projection_version( - invalid: object, -) -> None: - with pytest.raises(ValueError, match="auxiliary_projection_version"): - receipt(auxiliary_projection_version=invalid) From 941c35f4b3ff2b1f506b2079e48a2deeb39b3f76 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 14:17:21 +0900 Subject: [PATCH 124/158] docs(validity): document versioned auxiliary projection --- packages/validity-analysis/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/validity-analysis/README.md b/packages/validity-analysis/README.md index 290da9320..95a235f66 100644 --- a/packages/validity-analysis/README.md +++ b/packages/validity-analysis/README.md @@ -12,7 +12,7 @@ The resulting canonical JSON is digest-addressable, contains no raw person-level `WeightEligibilityReceipt` makes cross-sectional versus longitudinal use machine-checkable rather than an opaque weight label. It binds the final weight artifact to one governed scope (`cross_sectional` or `longitudinal`), target population, reference-duration evidence, and eligible-case set. `FinalAnalysisWeightReceipt` fails closed unless those fields match the estimand and the same final point-weight artifact exactly, so a longitudinal weight cannot silently support a cross-sectional estimand or a different reference duration. -Nonresponse and calibration are no longer allowed to hide behind an undifferentiated adjustment digest. `NonresponseAdjustmentReceipt` binds the adjustment to a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, and explicit treatment codes for ineligible, unknown, and unavailable cases. `CalibrationAdjustmentReceipt` binds the adjustment to a target population/window, purpose-bound auxiliary projection, exact scientific-use purpose, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and owner-correlatable use instant, plus a separately versioned calibration benchmark receipt with released benchmark owner-contract reference/version/digest and exact benchmark reference instant. It also binds algorithm/version, constraints digest, and explicit `converged` or `fallback_applied` termination evidence. Neither the scientific-use instant nor the benchmark reference instant may be later than receipt construction. A fallback must identify its versioned rule; a nonconverged calibration cannot masquerade as an accepted calibration receipt. These fields preserve the opaque correlation tuples expected by the canonical Workforce Validation authority boundaries without copying protected auxiliary or benchmark values. The leaf receipt still does not authenticate those coordinates itself: the durable owner service must resolve the released auxiliary and benchmark evidence and prove that the exact projection and benchmark are authoritative for the scientific use. +Nonresponse and calibration are no longer allowed to hide behind an undifferentiated adjustment digest. `NonresponseAdjustmentReceipt` binds the adjustment to a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, and explicit treatment codes for ineligible, unknown, and unavailable cases. `CalibrationAdjustmentReceipt` binds the adjustment to a target population/window, **versioned** purpose-bound auxiliary projection identity/digest, exact scientific-use purpose, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and owner-correlatable use instant, plus a separately versioned calibration benchmark receipt with released benchmark owner-contract reference/version/digest and exact benchmark reference instant. It also binds algorithm/version, constraints digest, and explicit `converged` or `fallback_applied` termination evidence. Neither the scientific-use instant nor the benchmark reference instant may be later than receipt construction. A fallback must identify its versioned rule; a nonconverged calibration cannot masquerade as an accepted calibration receipt. These fields preserve the opaque correlation tuples expected by the canonical Workforce Validation authority boundaries without copying protected auxiliary or benchmark values. The leaf receipt still does not authenticate those coordinates itself: the durable owner service must resolve the released auxiliary and benchmark evidence and prove that the exact projection version and benchmark are authoritative for the scientific use. Trimming, bounding, and winsorization have a separate provenance family rather than falling back to a generic adjustment label. `TrimmingBoundingAdjustmentReceipt` binds the exact versioned rule, its reproducible configuration digest, the semantic occurrence set and count of cases actually affected, and the input/output weight artifacts. A declared trim/bound transform must change artifact identity. Known trimming/bounding/winsorization adjustment codes fail closed unless `evidence_kind` names this typed receipt family. @@ -32,13 +32,13 @@ Point-estimation semantics are explicit. `unweighted` results cannot carry weigh - It does **not** claim that a selection procedure is valid. - It does **not** interpret adverse impact. - It does **not** authorize hiring, promotion, termination, compensation, or another employment decision. -- It does **not** yet complete #407. Durable owner-side verification that adjustment, point/variance compatibility, auxiliary-authority, and benchmark-authority coordinates resolve to the released typed receipts/contracts they claim; enforcement that the resolved auxiliary projection is authorized for the exact scientific purpose/use receipt/time; and released auxiliary/benchmark/variance evidence exchange remain open work in `workforce_validation`. +- It does **not** yet complete #407. Durable owner-side verification that adjustment, point/variance compatibility, auxiliary-authority, and benchmark-authority coordinates resolve to the released typed receipts/contracts they claim; enforcement that the resolved auxiliary projection **version** is authorized for the exact scientific purpose/use receipt/time; and released auxiliary/benchmark/variance evidence exchange remain open work in `workforce_validation`. The fast-mlsirm repository remains a dedicated-writer dependency. This package records only the immutable revision reviewed for the handoff. Durable Workforce Validation registry/API/persistence remains on the canonical #235/#248 owner path; this package does not create a parallel service. ## Host obligations -Before an approved offline validation worker executes the handoff, the Orgmetra host must re-resolve every reference inside `tenant_record_id` and prove that the predictor, criterion, population, and policy evidence belong to the exact validation study and Job. The requester and reviewer must resolve to distinct authoritative actors. For weighted design-based inference, the host must also verify the final point-weight receipt, its matching cross-sectional/longitudinal weight-eligibility receipt, its upstream source/sampling evidence, each typed adjustment receipt identified by the ordered chain, the calibration auxiliary authority/purpose/released owner contract/authorization/scientific-use tuple when applicable, the calibration benchmark receipt/version/released benchmark owner contract/reference instant, the separately bound #406 variance-design evidence, and the point/variance compatibility receipt against released owner evidence rather than trusting caller-supplied labels or digests. A numerical result is scientific evidence for accountable human interpretation, never an autonomous employment decision. +Before an approved offline validation worker executes the handoff, the Orgmetra host must re-resolve every reference inside `tenant_record_id` and prove that the predictor, criterion, population, and policy evidence belong to the exact validation study and Job. The requester and reviewer must resolve to distinct authoritative actors. For weighted design-based inference, the host must also verify the final point-weight receipt, its matching cross-sectional/longitudinal weight-eligibility receipt, its upstream source/sampling evidence, each typed adjustment receipt identified by the ordered chain, the calibration auxiliary authority/purpose/released owner contract/authorization/scientific-use tuple including the exact auxiliary projection **version**, the calibration benchmark receipt/version/released benchmark owner contract/reference instant, the separately bound #406 variance-design evidence, and the point/variance compatibility receipt against released owner evidence rather than trusting caller-supplied labels or digests. A numerical result is scientific evidence for accountable human interpretation, never an autonomous employment decision. ## Verification From 155fe44f465981e65f7ccf8c6618f27bc994c05c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 14:17:30 +0900 Subject: [PATCH 125/158] docs(validity): record projection-version provenance --- packages/validity-analysis/CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/validity-analysis/CHANGELOG.md b/packages/validity-analysis/CHANGELOG.md index 29533b8db..8c9adae89 100644 --- a/packages/validity-analysis/CHANGELOG.md +++ b/packages/validity-analysis/CHANGELOG.md @@ -15,8 +15,8 @@ - Distinguish unweighted from weighted design-based results and fail closed unless a weighted result separately binds the exact final analysis-weight receipt and variance-design receipt used. - Add `WeightVarianceCompatibilityReceipt` so variance evidence must correlate to the exact final point-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence, and final-weight artifact before a weighted result can be emitted. The receipt is correlation evidence, not self-authenticating #406 owner authority. - Require `ValidationAnalysisResult` to bind the compatibility receipt in addition to the point-weight and variance-design digests; unweighted results cannot carry compatibility evidence. -- Add typed `NonresponseAdjustmentReceipt` evidence that binds the exact versioned response/disposition receipt reference/version/digest, adjustment population, controlled method/configuration, explicit ineligible/unknown/unavailable treatment, and input/output weight artifacts; add typed `CalibrationAdjustmentReceipt` evidence bound to purpose-limited auxiliary projections, exact scientific-use purpose, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest/use time, and a separately versioned calibration benchmark receipt with released benchmark owner-contract reference/version/digest and exact benchmark reference time, plus constraints and explicit convergence/fallback state. Neither the use time nor benchmark reference time may be later than receipt construction. +- Add typed `NonresponseAdjustmentReceipt` evidence that binds the exact versioned response/disposition receipt reference/version/digest, adjustment population, controlled method/configuration, explicit ineligible/unknown/unavailable treatment, and input/output weight artifacts; add typed `CalibrationAdjustmentReceipt` evidence bound to a versioned purpose-limited auxiliary projection identity/digest, exact scientific-use purpose, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest/use time, and a separately versioned calibration benchmark receipt with released benchmark owner-contract reference/version/digest and exact benchmark reference time, plus constraints and explicit convergence/fallback state. Neither the use time nor benchmark reference time may be later than receipt construction. - Add typed `TrimmingBoundingAdjustmentReceipt` evidence that binds a versioned rule/configuration, exact affected semantic occurrence set and count, and input/output weight artifacts; known trimming/bounding/winsorization adjustments must name that evidence family. - Require known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization adjustment codes to identify the matching typed evidence kind rather than collapse into a generic opaque adjustment digest. -- Keep durable owner-side typed-receipt, point/variance compatibility, auxiliary-authority, and benchmark-authority resolution plus released auxiliary/benchmark/variance evidence exchange explicitly incomplete under #407; the scientific leaf preserves exact correlation coordinates but does not self-authenticate them. +- Keep durable owner-side typed-receipt, point/variance compatibility, auxiliary-authority, and benchmark-authority resolution plus released auxiliary/benchmark/variance evidence exchange explicitly incomplete under #407; the scientific leaf preserves exact correlation coordinates including auxiliary projection version but does not self-authenticate them. - Make `Validity Analysis Handoff Quality` retrigger on shared repository Python/test/clean-checkout configuration, with an executable regression preventing stale package-quality evidence after shared tooling changes. From d727c8d2eef064777dc8d52035dfa44b30941a0f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 14:18:02 +0900 Subject: [PATCH 126/158] docs(adr): bind calibration projection version --- ...027-governed-selection-validity-analysis-handoff.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/adr/0027-governed-selection-validity-analysis-handoff.md b/docs/adr/0027-governed-selection-validity-analysis-handoff.md index 20a2398ed..8c6c5b902 100644 --- a/docs/adr/0027-governed-selection-validity-analysis-handoff.md +++ b/docs/adr/0027-governed-selection-validity-analysis-handoff.md @@ -34,17 +34,17 @@ For #407's weighted design-based inference boundary, the active package adds `Fi `WeightEligibilityReceipt` makes weight use itself versioned evidence rather than a caller-supplied label. The receipt is closed to `cross_sectional` or `longitudinal` scope and binds the exact target population, reference-duration evidence, eligible-case set, and final point-weight artifact. `FinalAnalysisWeightReceipt` validates that the eligibility receipt belongs to the same tenant and that its scope, target population, reference duration, eligible cases, and weight artifact exactly match the estimand-side receipt. A longitudinal weight therefore cannot silently support a cross-sectional estimand or a different target period. The 2025 SIPP Users' Guide is used only as current primary methodological evidence that weight choice depends on both target population and duration and that longitudinal weights cover explicit multi-year reference periods; SIPP-specific variables or estimators are not imported into Orgmetra. -The adjustment chain has typed scientific evidence for three high-risk transform families. `NonresponseAdjustmentReceipt` binds a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, explicit treatment of ineligible/unknown/unavailable cases, and input/output weight artifacts. `CalibrationAdjustmentReceipt` binds the target population/window, purpose-limited auxiliary projection, exact scientific-use purpose reference/digest, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and use instant, and a separately versioned calibration benchmark receipt reference/version/digest with its released benchmark owner-contract reference/version/digest and exact benchmark reference instant. It also binds algorithm/version, constraints digest, input/output artifacts, and an explicit `converged` or `fallback_applied` termination state. Both the scientific-use instant and benchmark reference instant are frozen and cannot be later than receipt construction. This makes the complete opaque correlation tuple expected by canonical Workforce Validation owner resolution part of the scientific receipt without copying protected auxiliary or benchmark values. The leaf package still does not pretend its caller-supplied coordinates are authoritative: the durable `workforce_validation` service must resolve the released auxiliary owner contract, authorization receipt, scientific-use receipt, benchmark receipt, and released benchmark owner contract and prove that the actual projection and benchmark are the exact released evidence authorized/referenced for the analysis before execution or release. A fallback must identify its immutable fallback-rule reference/digest. A failed/nonconverged calibration cannot be labeled as an accepted calibration receipt. `TrimmingBoundingAdjustmentReceipt` binds a versioned trimming/bounding rule, reproducible rule configuration digest, the semantic occurrence set and count of cases actually affected, and input/output weight artifacts; a declared trim/bound transform cannot be a no-op. Known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization `AnalysisWeightAdjustment` codes also carry an `evidence_kind` and fail closed unless it names the matching typed receipt family. +The adjustment chain has typed scientific evidence for three high-risk transform families. `NonresponseAdjustmentReceipt` binds a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, explicit treatment of ineligible/unknown/unavailable cases, and input/output weight artifacts. `CalibrationAdjustmentReceipt` binds the target population/window, **versioned purpose-limited auxiliary projection reference/version/digest**, exact scientific-use purpose reference/digest, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and use instant, and a separately versioned calibration benchmark receipt reference/version/digest with its released benchmark owner-contract reference/version/digest and exact benchmark reference instant. It also binds algorithm/version, constraints digest, input/output artifacts, and an explicit `converged` or `fallback_applied` termination state. Both the scientific-use instant and benchmark reference instant are frozen and cannot be later than receipt construction. This makes the complete opaque correlation tuple expected by canonical Workforce Validation owner resolution part of the scientific receipt without copying protected auxiliary or benchmark values. The leaf package still does not pretend its caller-supplied coordinates are authoritative: the durable `workforce_validation` service must resolve the released auxiliary owner contract, authorization receipt, scientific-use receipt, benchmark receipt, and released benchmark owner contract and prove that the actual **projection version** and benchmark are the exact released evidence authorized/referenced for the analysis before execution or release. A fallback must identify its immutable fallback-rule reference/digest. A failed/nonconverged calibration cannot be labeled as an accepted calibration receipt. `TrimmingBoundingAdjustmentReceipt` binds a versioned trimming/bounding rule, reproducible rule configuration digest, the semantic occurrence set and count of cases actually affected, and input/output weight artifacts; a declared trim/bound transform cannot be a no-op. Known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization `AnalysisWeightAdjustment` codes also carry an `evidence_kind` and fail closed unless it names the matching typed receipt family. #407 RED #8 requires more than two different well-formed digests. A final point weight must not be combined with replicate or variance evidence generated from a different eligibility, correction, calibration, analytic case set, or final-weight version. The active package therefore adds `WeightVarianceCompatibilityReceipt`. It binds one exact `FinalAnalysisWeightReceipt` to the separate #406 variance-design receipt reference/version/digest and requires the variance side to identify the exact same analysis-weight receipt digest, analytic-case occurrence set, weight-eligibility receipt digest, correction sequence, and final-weight artifact. The receipt must be constructed no earlier than the point-weight receipt, and its variance-design digest must remain distinct from the point-weight receipt itself. `ValidationAnalysisResult` now requires this compatibility receipt for `weighted_design_based` output and verifies that its point-weight and variance-design digests are the same ones recorded on the result. Unweighted results cannot carry compatibility evidence. This compatibility receipt is deliberately not a self-authenticating variance owner. Its role is deterministic scientific correlation at the leaf boundary: it prevents a caller from presenting internally inconsistent point/variance lineages as one weighted result. Canonical #235/#248 remain responsible for resolving the released #406 variance-design evidence, proving that the variance-side coordinates actually came from the authoritative released owner contract, and rejecting caller-fabricated correlation data. The same owner boundary continues to apply separately to the calibration auxiliary-authority tuple and calibration benchmark-authority tuple. -This executable slice still does not claim #407 complete. The leaf can now preserve the opaque coordinates needed for durable owner-side calibration auxiliary and benchmark correlation and can fail closed when point-weight and variance evidence are internally incongruent. Durable service/API verification that auxiliary, benchmark, and point/variance compatibility coordinates resolve to the released typed receipts/contracts claimed, verification that released authorization permits the resolved projection for the exact purpose and use, and released auxiliary/benchmark/variance owner evidence exchange remain `workforce_validation` scientific truth rather than Talent truth or cross-context SQL reconstruction. +This executable slice still does not claim #407 complete. The leaf can now preserve the opaque coordinates needed for durable owner-side calibration auxiliary and benchmark correlation and can fail closed when point-weight and variance evidence are internally incongruent. Durable service/API verification that auxiliary, benchmark, and point/variance compatibility coordinates resolve to the released typed receipts/contracts claimed, verification that released authorization permits the resolved **projection version** for the exact purpose and use, and released auxiliary/benchmark/variance owner evidence exchange remain `workforce_validation` scientific truth rather than Talent truth or cross-context SQL reconstruction. The NIST Privacy Framework 1.0 is used narrowly as a privacy-risk-management basis for expressing and verifying data-processing requirements across organizational roles and contracts; as of the 2026-09-17 check, NIST still presents Privacy Framework 1.1 as an Initial Public Draft rather than a final replacement. This ADR does not treat the voluntary framework as employment law or infer legal permission from a NIST profile. -The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, prove requester/reviewer identities are distinct authoritative actors, verify cross-sectional/longitudinal weight eligibility against the estimand, verify typed weight-adjustment receipts plus any calibration auxiliary authority/purpose/released-owner/authorization/scientific-use tuple and calibration benchmark receipt/version/released-owner/reference-time tuple, and resolve the point/variance compatibility receipt against the released #406 owner evidence rather than trusting caller-supplied evidence labels or digests. +The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, prove requester/reviewer identities are distinct authoritative actors, verify cross-sectional/longitudinal weight eligibility against the estimand, verify typed weight-adjustment receipts plus any calibration auxiliary **projection reference/version/digest** and authority/purpose/released-owner/authorization/scientific-use tuple and calibration benchmark receipt/version/released-owner/reference-time tuple, and resolve the point/variance compatibility receipt against the released #406 owner evidence rather than trusting caller-supplied evidence labels or digests. ## Consequences @@ -60,7 +60,7 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - Cross-sectional and longitudinal weights cannot be interchanged when target population, reference duration, eligible cases, or final weight artifact differ from the estimand. - The ordered weight transformation chain is independently digest-correlatable without centralizing case-level weights or auxiliary attributes. - Nonresponse cannot silently drop refusal/unreachable/ineligible/failure dispositions behind one opaque adjustment label. -- Calibration/raking cannot silently float benchmark ownership, benchmark version, or benchmark reference time; omit the exact scientific-use purpose/released-owner/authorization/use correlation tuple; move its use or benchmark time after receipt construction; or hide fallback as successful convergence. +- Calibration/raking cannot silently float auxiliary projection **version**, benchmark ownership, benchmark version, or benchmark reference time; omit the exact scientific-use purpose/released-owner/authorization/use correlation tuple; move its use or benchmark time after receipt construction; or hide fallback as successful convergence. - Trimming/bounding/winsorization cannot silently alter final point weights without an immutable rule/configuration and affected-case receipt. - Human interpretation remains explicit and separate from numerical output. - The dedicated-writer boundary remains intact: Orgmetra consumes only a pinned foreign revision/contract boundary and never mutates fast-mlsirm. @@ -75,7 +75,7 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later ## Verification -The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. #407 adds RED/GREEN coverage for deterministic value-minimized final-weight receipts, exact sampling-receipt binding, contiguous transformation lineage, append-only correction linkage, fail-closed weighted-result binding to separate point-weight and variance-design receipts, exact point/variance compatibility across analysis-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence and final-weight artifact, cross-sectional/longitudinal scope and target-population/reference-duration/case-set/final-artifact congruence, explicit nonresponse disposition treatment, calibration benchmark receipt version/owner-contract/reference-time provenance, calibration constraint/termination provenance, scientific auxiliary-authority/purpose/released-owner/authorization/scientific-use correlation coordinates, owner-contract and use-receipt digest validation, future-use-time and future-benchmark-time rejection, fallback-rule disclosure, trimming/bounding rule/configuration and affected-case provenance, no-op trim/bound rejection, and typed adjustment-evidence matching. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. +The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. #407 adds RED/GREEN coverage for deterministic value-minimized final-weight receipts, exact sampling-receipt binding, contiguous transformation lineage, append-only correction linkage, fail-closed weighted-result binding to separate point-weight and variance-design receipts, exact point/variance compatibility across analysis-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence and final-weight artifact, cross-sectional/longitudinal scope and target-population/reference-duration/case-set/final-artifact congruence, explicit nonresponse disposition treatment, calibration **auxiliary projection version**, benchmark receipt version/owner-contract/reference-time provenance, calibration constraint/termination provenance, scientific auxiliary-authority/purpose/released-owner/authorization/scientific-use correlation coordinates, owner-contract and use-receipt digest validation, future-use-time and future-benchmark-time rejection, fallback-rule disclosure, trimming/bounding rule/configuration and affected-case provenance, no-op trim/bound rejection, and typed adjustment-evidence matching. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. ## References From 4bc81391b16bc16ce7d851122f5813f55552415e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 14:18:23 +0900 Subject: [PATCH 127/158] docs(traceability): version calibration auxiliary projection --- docs/traceability/validation-analysis-handoff.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/traceability/validation-analysis-handoff.md b/docs/traceability/validation-analysis-handoff.md index d129d1db4..6f8258cc6 100644 --- a/docs/traceability/validation-analysis-handoff.md +++ b/docs/traceability/validation-analysis-handoff.md @@ -17,7 +17,7 @@ Can an organization send one exact, reviewable validation study to its statistic | Final point-weight provenance | exact estimand/target/window/reference-duration and case-set digests, source/sampling receipt digests, base-weight evidence, ordered digest-linked adjustments, final weight artifact, append-only correction lineage | `FinalAnalysisWeightReceipt` deterministic/value-minimized regressions plus chain/correction fail-closed tests | | Weight eligibility congruence | governed `cross_sectional` or `longitudinal` scope plus exact target population, reference-duration evidence, eligible case set, and final point-weight artifact | `test_weight_eligibility_receipt.py` plus `FinalAnalysisWeightReceipt` mismatch/longitudinal-match regressions | | Nonresponse adjustment evidence | versioned response/disposition receipt, exact adjustment population, controlled method/configuration, explicit ineligible/unknown/unavailable treatment, input/output weight artifacts | `NonresponseAdjustmentReceipt` deterministic/value-minimized tests plus undocumented-treatment/no-op/version rejection | -| Calibration/raking evidence | target population/window, purpose-bound auxiliary projection, scientific-use purpose reference/digest, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and use instant, calibration benchmark receipt reference/version/digest, released benchmark owner-contract reference/version/digest, benchmark reference instant, algorithm/version, constraints, input/output artifacts, explicit converged/fallback state and immutable fallback rule when used | `CalibrationAdjustmentReceipt` authority/purpose/owner/authorization/scientific-use/benchmark-owner/benchmark-time/termination/fallback regressions; neither scientific-use time nor benchmark reference time may be later than receipt construction | +| Calibration/raking evidence | target population/window, **versioned** purpose-bound auxiliary projection reference/version/digest, scientific-use purpose reference/digest, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and use instant, calibration benchmark receipt reference/version/digest, released benchmark owner-contract reference/version/digest, benchmark reference instant, algorithm/version, constraints, input/output artifacts, explicit converged/fallback state and immutable fallback rule when used | `CalibrationAdjustmentReceipt` auxiliary-projection-version/authority/purpose/owner/authorization/scientific-use/benchmark-owner/benchmark-time/termination/fallback regressions; neither scientific-use time nor benchmark reference time may be later than receipt construction | | Trimming/bounding evidence | versioned trimming/bounding rule, reproducible rule configuration digest, exact affected semantic occurrence set and count, input/output weight artifacts | `test_trimming_bounding_receipt.py` deterministic/value-minimized, no-op, missing-provenance, and typed-kind regressions | | Typed adjustment congruence | known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization adjustment codes must identify the matching typed receipt family through `evidence_kind` | specialized adjustment evidence-kind regressions | | Point/variance lineage congruence | #406 variance evidence must name the same final point-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence, and final-weight artifact used by the estimate | `test_weight_variance_compatibility_receipt.py` mismatch regressions | @@ -31,7 +31,7 @@ Can an organization send one exact, reviewable validation study to its statistic The active branch makes final-weight eligibility and three adjustment families executable rather than leaving them as opaque labels. It now also closes the leaf-side form of #407 RED #8: `WeightVarianceCompatibilityReceipt` requires the variance side to identify the exact point-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence, and final-weight artifact. `ValidationAnalysisResult` cannot emit weighted design-based evidence without that compatibility receipt in addition to separate point-weight and #406 variance-design digests. -This is still correlation evidence, not owner authentication. Calibration/raking/post-stratification evidence carries the opaque coordinates needed by canonical Workforce Validation authority resolution: auxiliary authority and projection, exact scientific purpose, released auxiliary owner-contract identity/version/digest, authorization receipt identity/digest, scientific-use receipt identity/digest and owner-correlatable use instant, plus separately versioned benchmark receipt identity/digest, released benchmark owner-contract identity/version/digest, and benchmark reference instant. Durable #235/#248 owner-side resolution must prove both the auxiliary tuple and benchmark tuple against released/versioned owner evidence, prove that the exact projection is authorized for the exact scientific purpose/use receipt, and resolve the point/variance compatibility coordinates against released #406 variance evidence rather than trusting leaf-supplied digests. Released auxiliary/benchmark/variance evidence exchange also remains open. `workforce_validation` owns those scientific contracts and their released result; `talent_management` must not compute or reconstruct final analysis weights. +This is still correlation evidence, not owner authentication. Calibration/raking/post-stratification evidence carries the opaque coordinates needed by canonical Workforce Validation authority resolution: auxiliary authority and **projection reference/version/digest**, exact scientific purpose, released auxiliary owner-contract identity/version/digest, authorization receipt identity/digest, scientific-use receipt identity/digest and owner-correlatable use instant, plus separately versioned benchmark receipt identity/digest, released benchmark owner-contract identity/version/digest, and benchmark reference instant. Durable #235/#248 owner-side resolution must prove both the auxiliary tuple and benchmark tuple against released/versioned owner evidence, prove that the exact **projection version** is authorized for the exact scientific purpose/use receipt, and resolve the point/variance compatibility coordinates against released #406 variance evidence rather than trusting leaf-supplied digests. Released auxiliary/benchmark/variance evidence exchange also remains open. `workforce_validation` owns those scientific contracts and their released result; `talent_management` must not compute or reconstruct final analysis weights. ## Maturity From 5eb4b128f1dd48dd9830e21da9de19633690c3ac Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 14:33:44 +0900 Subject: [PATCH 128/158] test(validity): make verification status explicit --- .../tests/test_result_verification_status.py | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 packages/validity-analysis/tests/test_result_verification_status.py diff --git a/packages/validity-analysis/tests/test_result_verification_status.py b/packages/validity-analysis/tests/test_result_verification_status.py new file mode 100644 index 000000000..336a90190 --- /dev/null +++ b/packages/validity-analysis/tests/test_result_verification_status.py @@ -0,0 +1,60 @@ +"""Regression coverage for non-authorizing scientific verification status.""" + +from datetime import datetime, timezone +import json + +from orgmetra_validity_analysis import ( + ConvergenceDiagnostics, + MissingnessSummary, + REVIEWED_FAST_MLSIRM_REVISION, + ValidationAnalysisResult, +) + + +def _result(*, converged: bool) -> ValidationAnalysisResult: + """Build one aggregate-only result with an explicit convergence outcome.""" + return ValidationAnalysisResult( + tenant_record_id="10000000-0000-7000-8000-000000000001", + result_reference="validation_analysis_result:77777777-7777-4777-8777-777777777777", + handoff_digest="a" * 64, + provenance_digest="b" * 64, + fast_mlsirm_revision=REVIEWED_FAST_MLSIRM_REVISION, + model_code="mlsirm_criterion_related", + backend="rust_cpu", + precision="f64", + effect_estimate=0.42, + uncertainty_lower=0.10, + uncertainty_upper=0.70, + sample_size=12, + missingness_summary=MissingnessSummary( + total_observations=12, + complete_observations=10, + missing_predictor_observations=1, + missing_criterion_observations=1, + ), + convergence_diagnostics=ConvergenceDiagnostics( + converged=converged, + iterations=42, + objective_value=-12.5, + maximum_gradient=0.0001, + failure_code=None if converged else "maximum_iterations", + ), + completed_at=datetime(2026, 9, 17, 5, 30, tzinfo=timezone.utc), + ) + + +def test_converged_result_is_explicitly_verified() -> None: + candidate = _result(converged=True) + + assert candidate.verification_status == "verified" + assert json.loads(candidate.canonical_json())["verification_status"] == "verified" + + +def test_nonconverged_result_is_explicitly_not_verifiable() -> None: + candidate = _result(converged=False) + + assert candidate.verification_status == "not_verifiable" + payload = json.loads(candidate.canonical_json()) + assert payload["verification_status"] == "not_verifiable" + assert payload["execution_state"] == "completed" + assert payload["convergence_diagnostics"]["failure_code"] == "maximum_iterations" From c2d89b824f1ea7746b86dd739b287f827bcc644c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 14:34:16 +0900 Subject: [PATCH 129/158] fix(validity): mark nonconverged results not verifiable --- .../src/orgmetra_validity_analysis/result.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py index 2a516a48f..75368a852 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py @@ -277,6 +277,11 @@ def __post_init__(self) -> None: object.__setattr__(self, "uncertainty_upper", upper) object.__setattr__(self, "completed_at", completed_at) + @property + def verification_status(self) -> str: + """Return whether the completed scientific result is currently verifiable.""" + return "verified" if self.convergence_diagnostics.converged else "not_verifiable" + def __repr__(self) -> str: """Return a redacted representation suitable for routine application logs.""" return "ValidationAnalysisResult()" @@ -305,6 +310,7 @@ def canonical_json(self) -> str: "tenant_record_id": self.tenant_record_id, "uncertainty_lower": float(self.uncertainty_lower), "uncertainty_upper": float(self.uncertainty_upper), + "verification_status": self.verification_status, } if self.analysis_weight_receipt_digest is not None: payload["analysis_weight_receipt_digest"] = self.analysis_weight_receipt_digest @@ -321,4 +327,4 @@ def sha256_digest(self) -> str: return sha256(self.canonical_json().encode("utf-8")).hexdigest() -__all__ = ["ConvergenceDiagnostics", "MissingnessSummary", "ValidationAnalysisResult"] +__all__ = ["ConvergenceDiagnostics", "MissingnessSummary", "ValidationAnalysisResult"] \ No newline at end of file From 959d812b0c84ef0cad906f7f3948adf32c5dcb39 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 14:34:29 +0900 Subject: [PATCH 130/158] docs(validity): record explicit verification status --- packages/validity-analysis/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/validity-analysis/CHANGELOG.md b/packages/validity-analysis/CHANGELOG.md index 8c9adae89..d365d5066 100644 --- a/packages/validity-analysis/CHANGELOG.md +++ b/packages/validity-analysis/CHANGELOG.md @@ -7,6 +7,7 @@ - Require separate requester/reviewer references and authoritative tenant-scoped re-resolution proving they resolve to distinct actor identities before execution. - Require deterministic canonical evidence and 100% owned production statement/branch coverage. - Validate a digest-linked Rust CPU/GPU result envelope with finite estimates, aggregate missingness, and explicit convergence or nonconvergence diagnostics. +- Make result verification explicit: converged results serialize as `verified`, while completed nonconverged results serialize as `not_verifiable` and retain their failure diagnostics instead of looking like a scientific GREEN. - Reject impossible aggregate missingness where complete observations overlap either predictor-missing or criterion-missing counts beyond the sample total. - Require exact governed missingness/convergence runtime types so subclass method overrides cannot inject unreviewed or person-level fields into canonical result evidence. - Freeze exact UTC timestamps and finite numeric values at construction, and reject runtime-type forgery before canonical evidence serialization. From b78b90819b463daf8235f2ff184dc14a34df2871 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 14:40:03 +0900 Subject: [PATCH 131/158] test(validity): keep converged leaf evidence non-authorizing --- .../tests/test_result_verification_status.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/validity-analysis/tests/test_result_verification_status.py b/packages/validity-analysis/tests/test_result_verification_status.py index 336a90190..3459c599d 100644 --- a/packages/validity-analysis/tests/test_result_verification_status.py +++ b/packages/validity-analysis/tests/test_result_verification_status.py @@ -43,11 +43,14 @@ def _result(*, converged: bool) -> ValidationAnalysisResult: ) -def test_converged_result_is_explicitly_verified() -> None: +def test_converged_leaf_result_remains_verification_pending() -> None: candidate = _result(converged=True) - assert candidate.verification_status == "verified" - assert json.loads(candidate.canonical_json())["verification_status"] == "verified" + assert candidate.verification_status == "verification_pending" + assert ( + json.loads(candidate.canonical_json())["verification_status"] + == "verification_pending" + ) def test_nonconverged_result_is_explicitly_not_verifiable() -> None: From bb81618176031c9b7d57ca491f8cff6d5228db2f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 14:40:40 +0900 Subject: [PATCH 132/158] fix(validity): keep leaf verification non-authorizing --- .../src/orgmetra_validity_analysis/result.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py index 75368a852..b3bd4ddd8 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py @@ -279,8 +279,12 @@ def __post_init__(self) -> None: @property def verification_status(self) -> str: - """Return whether the completed scientific result is currently verifiable.""" - return "verified" if self.convergence_diagnostics.converged else "not_verifiable" + """Return a non-authorizing leaf verification state for this result.""" + return ( + "verification_pending" + if self.convergence_diagnostics.converged + else "not_verifiable" + ) def __repr__(self) -> str: """Return a redacted representation suitable for routine application logs.""" From dbb0708cc1e59a777ee568397764f53e1ad6898e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 14:40:57 +0900 Subject: [PATCH 133/158] docs(validity): keep verification state non-authorizing --- packages/validity-analysis/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/validity-analysis/CHANGELOG.md b/packages/validity-analysis/CHANGELOG.md index d365d5066..7b0494772 100644 --- a/packages/validity-analysis/CHANGELOG.md +++ b/packages/validity-analysis/CHANGELOG.md @@ -7,7 +7,7 @@ - Require separate requester/reviewer references and authoritative tenant-scoped re-resolution proving they resolve to distinct actor identities before execution. - Require deterministic canonical evidence and 100% owned production statement/branch coverage. - Validate a digest-linked Rust CPU/GPU result envelope with finite estimates, aggregate missingness, and explicit convergence or nonconvergence diagnostics. -- Make result verification explicit: converged results serialize as `verified`, while completed nonconverged results serialize as `not_verifiable` and retain their failure diagnostics instead of looking like a scientific GREEN. +- Make result verification explicit without self-authorizing the scientific leaf: converged completed results serialize as `verification_pending`, while completed nonconverged results serialize as `not_verifiable` and retain their failure diagnostics. - Reject impossible aggregate missingness where complete observations overlap either predictor-missing or criterion-missing counts beyond the sample total. - Require exact governed missingness/convergence runtime types so subclass method overrides cannot inject unreviewed or person-level fields into canonical result evidence. - Freeze exact UTC timestamps and finite numeric values at construction, and reject runtime-type forgery before canonical evidence serialization. From 33e7ce06eb7e8af6ee0afd5564e063d8c2ca3829 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 16:03:10 +0900 Subject: [PATCH 134/158] test(validity): require explicit calibration fallback method provenance --- .../test_calibration_fallback_provenance.py | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 packages/validity-analysis/tests/test_calibration_fallback_provenance.py diff --git a/packages/validity-analysis/tests/test_calibration_fallback_provenance.py b/packages/validity-analysis/tests/test_calibration_fallback_provenance.py new file mode 100644 index 000000000..35948786f --- /dev/null +++ b/packages/validity-analysis/tests/test_calibration_fallback_provenance.py @@ -0,0 +1,106 @@ +"""Regression contracts for explicit calibration fallback provenance.""" + +from datetime import datetime, timedelta, timezone + +import pytest + +from orgmetra_validity_analysis import CalibrationAdjustmentReceipt + +TENANT = "10000000-0000-7000-8000-000000000001" +NOW = datetime(2026, 9, 17, 7, 0, tzinfo=timezone.utc) +DIGEST_A = "a" * 64 +DIGEST_B = "b" * 64 +DIGEST_C = "c" * 64 +DIGEST_D = "d" * 64 +DIGEST_E = "e" * 64 +DIGEST_F = "f" * 64 +DIGEST_1 = "1" * 64 +DIGEST_2 = "2" * 64 +DIGEST_3 = "3" * 64 +DIGEST_4 = "4" * 64 +DIGEST_5 = "5" * 64 + + +def calibration_receipt(**overrides: object) -> CalibrationAdjustmentReceipt: + """Return one fallback-bearing calibration receipt for focused provenance checks.""" + values: dict[str, object] = { + "tenant_record_id": TENANT, + "receipt_reference": "calibration_adjustment_receipt:33333333-3333-4333-8333-333333333333", + "target_population_digest": DIGEST_A, + "analysis_window_reference": "analysis_window:44444444-4444-4444-8444-444444444444", + "auxiliary_authority_reference": "scientific_auxiliary_authority:55555555-5555-4555-8555-555555555550", + "auxiliary_projection_reference": "calibration_auxiliary_projection:55555555-5555-4555-8555-555555555555", + "auxiliary_projection_version": 4, + "auxiliary_projection_digest": DIGEST_B, + "auxiliary_purpose_reference": "scientific_data_use_purpose:55555555-5555-4555-8555-555555555556", + "auxiliary_purpose_digest": DIGEST_3, + "auxiliary_owner_contract_reference": "released_owner_contract:55555555-5555-4555-8555-555555555557", + "auxiliary_owner_contract_version": 1, + "auxiliary_owner_contract_digest": DIGEST_1, + "auxiliary_authorization_receipt_reference": "scientific_data_authorization:55555555-5555-4555-8555-555555555558", + "auxiliary_authorization_receipt_digest": DIGEST_4, + "auxiliary_scientific_use_receipt_reference": "scientific_use_receipt:55555555-5555-4555-8555-555555555559", + "auxiliary_scientific_use_receipt_digest": DIGEST_2, + "auxiliary_scientific_use_at": NOW, + "benchmark_receipt_reference": "calibration_benchmark_receipt:66666666-6666-4666-8666-666666666666", + "benchmark_receipt_version": 2, + "benchmark_receipt_digest": DIGEST_C, + "benchmark_owner_contract_reference": "released_owner_contract:66666666-6666-4666-8666-666666666667", + "benchmark_owner_contract_version": 3, + "benchmark_owner_contract_digest": DIGEST_5, + "benchmark_reference_at": NOW - timedelta(days=1), + "algorithm_reference": "calibration_algorithm:77777777-7777-4777-8777-777777777777", + "algorithm_version": 1, + "constraints_digest": DIGEST_F, + "termination_code": "fallback_applied", + "fallback_reason_code": "primary_nonconvergence", + "fallback_rule_reference": "calibration_fallback_rule:99999999-9999-4999-8999-999999999999", + "fallback_rule_digest": DIGEST_2, + "fallback_algorithm_reference": "calibration_algorithm:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + "fallback_algorithm_version": 2, + "fallback_configuration_digest": DIGEST_4, + "input_weight_artifact_digest": DIGEST_D, + "output_weight_artifact_digest": DIGEST_E, + "constructed_at": NOW, + } + values.update(overrides) + return CalibrationAdjustmentReceipt(**values) + + +def test_fallback_identifies_reason_rule_and_algorithm_that_produced_weights() -> None: + """Do not label fallback output as if the primary calibration algorithm succeeded.""" + candidate = calibration_receipt() + canonical = candidate.canonical_json() + + assert '"termination_code":"fallback_applied"' in canonical + assert '"fallback_reason_code":"primary_nonconvergence"' in canonical + assert '"fallback_algorithm_reference":"calibration_algorithm:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"' in canonical + assert '"fallback_algorithm_version":2' in canonical + assert f'"fallback_configuration_digest":"{DIGEST_4}"' in canonical + assert f'"fallback_rule_digest":"{DIGEST_2}"' in canonical + + +@pytest.mark.parametrize( + ("field_name", "value"), + [ + ("fallback_reason_code", None), + ("fallback_rule_reference", None), + ("fallback_rule_digest", None), + ("fallback_algorithm_reference", None), + ("fallback_algorithm_version", None), + ("fallback_configuration_digest", None), + ], +) +def test_fallback_rejects_incomplete_actual_method_provenance( + field_name: str, + value: object, +) -> None: + """A fallback needs the actual generating method, not only an opaque fallback flag.""" + with pytest.raises((TypeError, ValueError), match="fallback"): + calibration_receipt(**{field_name: value}) + + +def test_converged_calibration_rejects_fallback_only_fields() -> None: + """Fallback evidence must not contaminate a genuinely converged primary algorithm.""" + with pytest.raises(ValueError, match="fallback"): + calibration_receipt(termination_code="converged") From 702dc045c3da2f0fff3b557c06008d4b2d326029 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 16:05:48 +0900 Subject: [PATCH 135/158] fix(validity): bind calibration fallback to actual generating algorithm --- .../src/orgmetra_validity_analysis/weights.py | 39 +++++++++++++++++-- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py b/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py index 2d019dcd3..2ef4f5133 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py @@ -169,8 +169,12 @@ class CalibrationAdjustmentReceipt: input_weight_artifact_digest: str output_weight_artifact_digest: str constructed_at: datetime + fallback_reason_code: str | None = None fallback_rule_reference: str | None = None fallback_rule_digest: str | None = None + fallback_algorithm_reference: str | None = None + fallback_algorithm_version: int | None = None + fallback_configuration_digest: str | None = None evidence_version: int = 1 def __post_init__(self) -> None: @@ -264,19 +268,42 @@ def __post_init__(self) -> None: or self.termination_code not in _CALIBRATION_TERMINATION_CODES ): raise ValueError("termination_code must be converged or fallback_applied") + fallback_fields = ( + self.fallback_reason_code, + self.fallback_rule_reference, + self.fallback_rule_digest, + self.fallback_algorithm_reference, + self.fallback_algorithm_version, + self.fallback_configuration_digest, + ) if self.termination_code == "fallback_applied": - if self.fallback_rule_reference is None or self.fallback_rule_digest is None: + if any(value is None for value in fallback_fields): raise ValueError( - "fallback_rule_reference and fallback_rule_digest are required for fallback_applied" + "fallback reason, rule, algorithm, version, and configuration evidence " + "are required for fallback_applied" ) + _validate_code(self.fallback_reason_code, "fallback_reason_code") _validate_reference( self.fallback_rule_reference, "calibration_fallback_rule", "fallback_rule_reference", ) _validate_digest(self.fallback_rule_digest, "fallback_rule_digest") - elif self.fallback_rule_reference is not None or self.fallback_rule_digest is not None: - raise ValueError("fallback_rule evidence must be absent when calibration converged") + _validate_reference( + self.fallback_algorithm_reference, + "calibration_algorithm", + "fallback_algorithm_reference", + ) + _positive_integer( + self.fallback_algorithm_version, + "fallback_algorithm_version", + ) + _validate_digest( + self.fallback_configuration_digest, + "fallback_configuration_digest", + ) + elif any(value is not None for value in fallback_fields): + raise ValueError("fallback evidence must be absent when calibration converged") if self.input_weight_artifact_digest == self.output_weight_artifact_digest: raise ValueError( "output_weight_artifact_digest must identify the calibrated weight artifact" @@ -348,6 +375,10 @@ def canonical_json(self) -> str: "termination_code": self.termination_code, } if self.fallback_rule_reference is not None: + payload["fallback_algorithm_reference"] = self.fallback_algorithm_reference + payload["fallback_algorithm_version"] = self.fallback_algorithm_version + payload["fallback_configuration_digest"] = self.fallback_configuration_digest + payload["fallback_reason_code"] = self.fallback_reason_code payload["fallback_rule_digest"] = self.fallback_rule_digest payload["fallback_rule_reference"] = self.fallback_rule_reference return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) From 9fdaae46e386e95996b07be28f1fe415dac9d73c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 16:06:28 +0900 Subject: [PATCH 136/158] test(validity): align calibration fallback semantics with generating method --- .../tests/test_weight_adjustment_semantics.py | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/packages/validity-analysis/tests/test_weight_adjustment_semantics.py b/packages/validity-analysis/tests/test_weight_adjustment_semantics.py index 3ef4f5324..75126527d 100644 --- a/packages/validity-analysis/tests/test_weight_adjustment_semantics.py +++ b/packages/validity-analysis/tests/test_weight_adjustment_semantics.py @@ -156,12 +156,22 @@ def test_calibration_receipt_binds_owner_authority_use_and_termination_state() - assert repr(candidate) == "CalibrationAdjustmentReceipt()" fallback_reference = "calibration_fallback_rule:99999999-9999-4999-8999-999999999999" + fallback_algorithm_reference = "calibration_algorithm:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" fallback = calibration_receipt( termination_code="fallback_applied", + fallback_reason_code="primary_nonconvergence", fallback_rule_reference=fallback_reference, fallback_rule_digest=DIGEST_2, + fallback_algorithm_reference=fallback_algorithm_reference, + fallback_algorithm_version=2, + fallback_configuration_digest=DIGEST_4, ) - assert f'"fallback_rule_digest":"{DIGEST_2}"' in fallback.canonical_json() + fallback_json = fallback.canonical_json() + assert f'"fallback_rule_digest":"{DIGEST_2}"' in fallback_json + assert '"fallback_reason_code":"primary_nonconvergence"' in fallback_json + assert f'"fallback_algorithm_reference":"{fallback_algorithm_reference}"' in fallback_json + assert '"fallback_algorithm_version":2' in fallback_json + assert f'"fallback_configuration_digest":"{DIGEST_4}"' in fallback_json with pytest.raises(ValueError, match="auxiliary_authority_reference"): calibration_receipt(auxiliary_authority_reference="authority-v1") @@ -205,19 +215,23 @@ def test_calibration_receipt_binds_owner_authority_use_and_termination_state() - calibration_receipt(termination_code="failed") with pytest.raises(ValueError, match="termination_code"): calibration_receipt(termination_code=1) - with pytest.raises(ValueError, match="fallback_rule"): + with pytest.raises(ValueError, match="fallback"): calibration_receipt(termination_code="fallback_applied") - with pytest.raises(ValueError, match="fallback_rule"): + with pytest.raises(ValueError, match="fallback"): calibration_receipt( termination_code="fallback_applied", fallback_rule_reference=fallback_reference, ) - with pytest.raises(ValueError, match="must be absent"): + with pytest.raises(ValueError, match="fallback"): calibration_receipt( + fallback_reason_code="primary_nonconvergence", fallback_rule_reference=fallback_reference, fallback_rule_digest=DIGEST_2, + fallback_algorithm_reference=fallback_algorithm_reference, + fallback_algorithm_version=2, + fallback_configuration_digest=DIGEST_4, ) - with pytest.raises(ValueError, match="must be absent"): + with pytest.raises(ValueError, match="fallback"): calibration_receipt(fallback_rule_digest=DIGEST_2) with pytest.raises(ValueError, match="output_weight_artifact_digest"): calibration_receipt(output_weight_artifact_digest=DIGEST_D) From 4e4d4b2f6ef0031fbfec5a26f36ad6f53dff9ccb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 16:07:27 +0900 Subject: [PATCH 137/158] docs(validity): record actual calibration fallback algorithm provenance --- packages/validity-analysis/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/validity-analysis/CHANGELOG.md b/packages/validity-analysis/CHANGELOG.md index 7b0494772..2d0d0669b 100644 --- a/packages/validity-analysis/CHANGELOG.md +++ b/packages/validity-analysis/CHANGELOG.md @@ -16,7 +16,7 @@ - Distinguish unweighted from weighted design-based results and fail closed unless a weighted result separately binds the exact final analysis-weight receipt and variance-design receipt used. - Add `WeightVarianceCompatibilityReceipt` so variance evidence must correlate to the exact final point-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence, and final-weight artifact before a weighted result can be emitted. The receipt is correlation evidence, not self-authenticating #406 owner authority. - Require `ValidationAnalysisResult` to bind the compatibility receipt in addition to the point-weight and variance-design digests; unweighted results cannot carry compatibility evidence. -- Add typed `NonresponseAdjustmentReceipt` evidence that binds the exact versioned response/disposition receipt reference/version/digest, adjustment population, controlled method/configuration, explicit ineligible/unknown/unavailable treatment, and input/output weight artifacts; add typed `CalibrationAdjustmentReceipt` evidence bound to a versioned purpose-limited auxiliary projection identity/digest, exact scientific-use purpose, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest/use time, and a separately versioned calibration benchmark receipt with released benchmark owner-contract reference/version/digest and exact benchmark reference time, plus constraints and explicit convergence/fallback state. Neither the use time nor benchmark reference time may be later than receipt construction. +- Add typed `NonresponseAdjustmentReceipt` evidence that binds the exact versioned response/disposition receipt reference/version/digest, adjustment population, controlled method/configuration, explicit ineligible/unknown/unavailable treatment, and input/output weight artifacts; add typed `CalibrationAdjustmentReceipt` evidence bound to a versioned purpose-limited auxiliary projection identity/digest, exact scientific-use purpose, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest/use time, and a separately versioned calibration benchmark receipt with released benchmark owner-contract reference/version/digest and exact benchmark reference time, plus constraints and explicit convergence/fallback state. A fallback now records the primary failure reason, immutable fallback rule, actual fallback calibration algorithm/version, and fallback configuration digest so fallback-produced weights cannot be mislabeled as primary-algorithm convergence. Neither the use time nor benchmark reference time may be later than receipt construction. - Add typed `TrimmingBoundingAdjustmentReceipt` evidence that binds a versioned rule/configuration, exact affected semantic occurrence set and count, and input/output weight artifacts; known trimming/bounding/winsorization adjustments must name that evidence family. - Require known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization adjustment codes to identify the matching typed evidence kind rather than collapse into a generic opaque adjustment digest. - Keep durable owner-side typed-receipt, point/variance compatibility, auxiliary-authority, and benchmark-authority resolution plus released auxiliary/benchmark/variance evidence exchange explicitly incomplete under #407; the scientific leaf preserves exact correlation coordinates including auxiliary projection version but does not self-authenticate them. From 7e69072d6ec1687eaa75c901a0054a4171d2a0fa Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 16:07:47 +0900 Subject: [PATCH 138/158] docs(validity): document calibration fallback generating method --- packages/validity-analysis/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/validity-analysis/README.md b/packages/validity-analysis/README.md index 95a235f66..89399eaf8 100644 --- a/packages/validity-analysis/README.md +++ b/packages/validity-analysis/README.md @@ -12,7 +12,7 @@ The resulting canonical JSON is digest-addressable, contains no raw person-level `WeightEligibilityReceipt` makes cross-sectional versus longitudinal use machine-checkable rather than an opaque weight label. It binds the final weight artifact to one governed scope (`cross_sectional` or `longitudinal`), target population, reference-duration evidence, and eligible-case set. `FinalAnalysisWeightReceipt` fails closed unless those fields match the estimand and the same final point-weight artifact exactly, so a longitudinal weight cannot silently support a cross-sectional estimand or a different reference duration. -Nonresponse and calibration are no longer allowed to hide behind an undifferentiated adjustment digest. `NonresponseAdjustmentReceipt` binds the adjustment to a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, and explicit treatment codes for ineligible, unknown, and unavailable cases. `CalibrationAdjustmentReceipt` binds the adjustment to a target population/window, **versioned** purpose-bound auxiliary projection identity/digest, exact scientific-use purpose, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and owner-correlatable use instant, plus a separately versioned calibration benchmark receipt with released benchmark owner-contract reference/version/digest and exact benchmark reference instant. It also binds algorithm/version, constraints digest, and explicit `converged` or `fallback_applied` termination evidence. Neither the scientific-use instant nor the benchmark reference instant may be later than receipt construction. A fallback must identify its versioned rule; a nonconverged calibration cannot masquerade as an accepted calibration receipt. These fields preserve the opaque correlation tuples expected by the canonical Workforce Validation authority boundaries without copying protected auxiliary or benchmark values. The leaf receipt still does not authenticate those coordinates itself: the durable owner service must resolve the released auxiliary and benchmark evidence and prove that the exact projection version and benchmark are authoritative for the scientific use. +Nonresponse and calibration are no longer allowed to hide behind an undifferentiated adjustment digest. `NonresponseAdjustmentReceipt` binds the adjustment to a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, and explicit treatment codes for ineligible, unknown, and unavailable cases. `CalibrationAdjustmentReceipt` binds the adjustment to a target population/window, **versioned** purpose-bound auxiliary projection identity/digest, exact scientific-use purpose, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and owner-correlatable use instant, plus a separately versioned calibration benchmark receipt with released benchmark owner-contract reference/version/digest and exact benchmark reference instant. It also binds algorithm/version, constraints digest, and explicit `converged` or `fallback_applied` termination evidence. Neither the scientific-use instant nor the benchmark reference instant may be later than receipt construction. A fallback must identify why the primary calibration did not remain authoritative, the immutable fallback rule, and the actual fallback calibration algorithm/version/configuration that produced the output weights; fallback evidence is forbidden on a genuinely converged primary algorithm. A nonconverged calibration cannot masquerade as an accepted calibration receipt. These fields preserve the opaque correlation tuples expected by the canonical Workforce Validation authority boundaries without copying protected auxiliary or benchmark values. The leaf receipt still does not authenticate those coordinates itself: the durable owner service must resolve the released auxiliary and benchmark evidence and prove that the exact projection version and benchmark are authoritative for the scientific use. Trimming, bounding, and winsorization have a separate provenance family rather than falling back to a generic adjustment label. `TrimmingBoundingAdjustmentReceipt` binds the exact versioned rule, its reproducible configuration digest, the semantic occurrence set and count of cases actually affected, and the input/output weight artifacts. A declared trim/bound transform must change artifact identity. Known trimming/bounding/winsorization adjustment codes fail closed unless `evidence_kind` names this typed receipt family. @@ -38,7 +38,7 @@ The fast-mlsirm repository remains a dedicated-writer dependency. This package r ## Host obligations -Before an approved offline validation worker executes the handoff, the Orgmetra host must re-resolve every reference inside `tenant_record_id` and prove that the predictor, criterion, population, and policy evidence belong to the exact validation study and Job. The requester and reviewer must resolve to distinct authoritative actors. For weighted design-based inference, the host must also verify the final point-weight receipt, its matching cross-sectional/longitudinal weight-eligibility receipt, its upstream source/sampling evidence, each typed adjustment receipt identified by the ordered chain, the calibration auxiliary authority/purpose/released owner contract/authorization/scientific-use tuple including the exact auxiliary projection **version**, the calibration benchmark receipt/version/released benchmark owner contract/reference instant, the separately bound #406 variance-design evidence, and the point/variance compatibility receipt against released owner evidence rather than trusting caller-supplied labels or digests. A numerical result is scientific evidence for accountable human interpretation, never an autonomous employment decision. +Before an approved offline validation worker executes the handoff, the Orgmetra host must re-resolve every reference inside `tenant_record_id` and prove that the predictor, criterion, population, and policy evidence belong to the exact validation study and Job. The requester and reviewer must resolve to distinct authoritative actors. For weighted design-based inference, the host must also verify the final point-weight receipt, its matching cross-sectional/longitudinal weight-eligibility receipt, its upstream source/sampling evidence, each typed adjustment receipt identified by the ordered chain, the calibration auxiliary authority/purpose/released owner contract/authorization/scientific-use tuple including the exact auxiliary projection **version**, the calibration benchmark receipt/version/released benchmark owner contract/reference instant, any fallback reason/rule/actual fallback algorithm-version-configuration tuple, the separately bound #406 variance-design evidence, and the point/variance compatibility receipt against released owner evidence rather than trusting caller-supplied labels or digests. A numerical result is scientific evidence for accountable human interpretation, never an autonomous employment decision. ## Verification From 29158ce104d71644f952e75fc91439073673b98c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 16:08:19 +0900 Subject: [PATCH 139/158] docs(adr): require actual calibration fallback algorithm provenance --- ...027-governed-selection-validity-analysis-handoff.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/adr/0027-governed-selection-validity-analysis-handoff.md b/docs/adr/0027-governed-selection-validity-analysis-handoff.md index 8c6c5b902..260c65b32 100644 --- a/docs/adr/0027-governed-selection-validity-analysis-handoff.md +++ b/docs/adr/0027-governed-selection-validity-analysis-handoff.md @@ -34,7 +34,7 @@ For #407's weighted design-based inference boundary, the active package adds `Fi `WeightEligibilityReceipt` makes weight use itself versioned evidence rather than a caller-supplied label. The receipt is closed to `cross_sectional` or `longitudinal` scope and binds the exact target population, reference-duration evidence, eligible-case set, and final point-weight artifact. `FinalAnalysisWeightReceipt` validates that the eligibility receipt belongs to the same tenant and that its scope, target population, reference duration, eligible cases, and weight artifact exactly match the estimand-side receipt. A longitudinal weight therefore cannot silently support a cross-sectional estimand or a different target period. The 2025 SIPP Users' Guide is used only as current primary methodological evidence that weight choice depends on both target population and duration and that longitudinal weights cover explicit multi-year reference periods; SIPP-specific variables or estimators are not imported into Orgmetra. -The adjustment chain has typed scientific evidence for three high-risk transform families. `NonresponseAdjustmentReceipt` binds a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, explicit treatment of ineligible/unknown/unavailable cases, and input/output weight artifacts. `CalibrationAdjustmentReceipt` binds the target population/window, **versioned purpose-limited auxiliary projection reference/version/digest**, exact scientific-use purpose reference/digest, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and use instant, and a separately versioned calibration benchmark receipt reference/version/digest with its released benchmark owner-contract reference/version/digest and exact benchmark reference instant. It also binds algorithm/version, constraints digest, input/output artifacts, and an explicit `converged` or `fallback_applied` termination state. Both the scientific-use instant and benchmark reference instant are frozen and cannot be later than receipt construction. This makes the complete opaque correlation tuple expected by canonical Workforce Validation owner resolution part of the scientific receipt without copying protected auxiliary or benchmark values. The leaf package still does not pretend its caller-supplied coordinates are authoritative: the durable `workforce_validation` service must resolve the released auxiliary owner contract, authorization receipt, scientific-use receipt, benchmark receipt, and released benchmark owner contract and prove that the actual **projection version** and benchmark are the exact released evidence authorized/referenced for the analysis before execution or release. A fallback must identify its immutable fallback-rule reference/digest. A failed/nonconverged calibration cannot be labeled as an accepted calibration receipt. `TrimmingBoundingAdjustmentReceipt` binds a versioned trimming/bounding rule, reproducible rule configuration digest, the semantic occurrence set and count of cases actually affected, and input/output weight artifacts; a declared trim/bound transform cannot be a no-op. Known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization `AnalysisWeightAdjustment` codes also carry an `evidence_kind` and fail closed unless it names the matching typed receipt family. +The adjustment chain has typed scientific evidence for three high-risk transform families. `NonresponseAdjustmentReceipt` binds a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, explicit treatment of ineligible/unknown/unavailable cases, and input/output weight artifacts. `CalibrationAdjustmentReceipt` binds the target population/window, **versioned purpose-limited auxiliary projection reference/version/digest**, exact scientific-use purpose reference/digest, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and use instant, and a separately versioned calibration benchmark receipt reference/version/digest with its released benchmark owner-contract reference/version/digest and exact benchmark reference instant. It also binds algorithm/version, constraints digest, input/output artifacts, and an explicit `converged` or `fallback_applied` termination state. Both the scientific-use instant and benchmark reference instant are frozen and cannot be later than receipt construction. This makes the complete opaque correlation tuple expected by canonical Workforce Validation owner resolution part of the scientific receipt without copying protected auxiliary or benchmark values. The leaf package still does not pretend its caller-supplied coordinates are authoritative: the durable `workforce_validation` service must resolve the released auxiliary owner contract, authorization receipt, scientific-use receipt, benchmark receipt, and released benchmark owner contract and prove that the actual **projection version** and benchmark are the exact released evidence authorized/referenced for the analysis before execution or release. A fallback is not represented by a boolean-like flag or rule digest alone: it must identify the primary failure reason, immutable fallback-rule reference/digest, and the actual fallback calibration algorithm/version plus configuration digest that produced the output weights. Fallback-only evidence is forbidden when the primary algorithm converged. A failed/nonconverged calibration cannot be labeled as an accepted calibration receipt. `TrimmingBoundingAdjustmentReceipt` binds a versioned trimming/bounding rule, reproducible rule configuration digest, the semantic occurrence set and count of cases actually affected, and input/output weight artifacts; a declared trim/bound transform cannot be a no-op. Known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization `AnalysisWeightAdjustment` codes also carry an `evidence_kind` and fail closed unless it names the matching typed receipt family. #407 RED #8 requires more than two different well-formed digests. A final point weight must not be combined with replicate or variance evidence generated from a different eligibility, correction, calibration, analytic case set, or final-weight version. The active package therefore adds `WeightVarianceCompatibilityReceipt`. It binds one exact `FinalAnalysisWeightReceipt` to the separate #406 variance-design receipt reference/version/digest and requires the variance side to identify the exact same analysis-weight receipt digest, analytic-case occurrence set, weight-eligibility receipt digest, correction sequence, and final-weight artifact. The receipt must be constructed no earlier than the point-weight receipt, and its variance-design digest must remain distinct from the point-weight receipt itself. `ValidationAnalysisResult` now requires this compatibility receipt for `weighted_design_based` output and verifies that its point-weight and variance-design digests are the same ones recorded on the result. Unweighted results cannot carry compatibility evidence. @@ -44,7 +44,7 @@ This executable slice still does not claim #407 complete. The leaf can now prese The NIST Privacy Framework 1.0 is used narrowly as a privacy-risk-management basis for expressing and verifying data-processing requirements across organizational roles and contracts; as of the 2026-09-17 check, NIST still presents Privacy Framework 1.1 as an Initial Public Draft rather than a final replacement. This ADR does not treat the voluntary framework as employment law or infer legal permission from a NIST profile. -The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, prove requester/reviewer identities are distinct authoritative actors, verify cross-sectional/longitudinal weight eligibility against the estimand, verify typed weight-adjustment receipts plus any calibration auxiliary **projection reference/version/digest** and authority/purpose/released-owner/authorization/scientific-use tuple and calibration benchmark receipt/version/released-owner/reference-time tuple, and resolve the point/variance compatibility receipt against the released #406 owner evidence rather than trusting caller-supplied evidence labels or digests. +The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, prove requester/reviewer identities are distinct authoritative actors, verify cross-sectional/longitudinal weight eligibility against the estimand, verify typed weight-adjustment receipts plus any calibration auxiliary **projection reference/version/digest** and authority/purpose/released-owner/authorization/scientific-use tuple and calibration benchmark receipt/version/released-owner/reference-time tuple, verify any fallback reason/rule/actual fallback algorithm-version-configuration tuple, and resolve the point/variance compatibility receipt against the released #406 owner evidence rather than trusting caller-supplied evidence labels or digests. ## Consequences @@ -60,7 +60,7 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - Cross-sectional and longitudinal weights cannot be interchanged when target population, reference duration, eligible cases, or final weight artifact differ from the estimand. - The ordered weight transformation chain is independently digest-correlatable without centralizing case-level weights or auxiliary attributes. - Nonresponse cannot silently drop refusal/unreachable/ineligible/failure dispositions behind one opaque adjustment label. -- Calibration/raking cannot silently float auxiliary projection **version**, benchmark ownership, benchmark version, or benchmark reference time; omit the exact scientific-use purpose/released-owner/authorization/use correlation tuple; move its use or benchmark time after receipt construction; or hide fallback as successful convergence. +- Calibration/raking cannot silently float auxiliary projection **version**, benchmark ownership, benchmark version, or benchmark reference time; omit the exact scientific-use purpose/released-owner/authorization/use correlation tuple; move its use or benchmark time after receipt construction; or hide a fallback as primary-algorithm convergence without naming the failure reason and actual fallback algorithm/version/configuration. - Trimming/bounding/winsorization cannot silently alter final point weights without an immutable rule/configuration and affected-case receipt. - Human interpretation remains explicit and separate from numerical output. - The dedicated-writer boundary remains intact: Orgmetra consumes only a pinned foreign revision/contract boundary and never mutates fast-mlsirm. @@ -69,13 +69,13 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - This slice does not execute a statistical model, estimate validity, correct for measurement error/range restriction, evaluate adverse impact, or assert legal compliance. - Sampling design, estimator choice, missing-data treatment, reliability evidence, multiplicity, transportability, fairness analysis, and model diagnostics must be encoded in the referenced analysis plan and reviewed before execution. -- The current #407 slice validates generic ordered weight provenance, weighted-result binding, point/variance compatibility, cross-sectional/longitudinal eligibility congruence, disposition-aware nonresponse evidence, owner-correlatable calibration auxiliary and benchmark evidence, and trimming/bounding rule provenance; it does not yet make durable typed-receipt/authorization/benchmark-owner/variance-owner resolution or released auxiliary/benchmark/variance exchange fully executable. +- The current #407 slice validates generic ordered weight provenance, weighted-result binding, point/variance compatibility, cross-sectional/longitudinal eligibility congruence, disposition-aware nonresponse evidence, owner-correlatable calibration auxiliary and benchmark evidence, explicit fallback generating-method provenance, and trimming/bounding rule provenance; it does not yet make durable typed-receipt/authorization/benchmark-owner/variance-owner resolution or released auxiliary/benchmark/variance exchange fully executable. - The generic result envelope does not invent an estimator-specific minimum complete-case count. Whether a converged estimator is identified under a particular missing-data design belongs to the reviewed analysis plan and numerical-worker contract; Orgmetra fails closed on impossible aggregate counts without silently replacing that foreign scientific contract with complete-case analysis. - The package validates the result envelope, but a future execution adapter must still re-resolve the handoff references, verify the result provenance artifact, resolve typed adjustment, purpose-authorization, and benchmark coordinates to released owner evidence, resolve point/variance compatibility against released #406 evidence, and attach evidence only after accountable human review. ## Verification -The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. #407 adds RED/GREEN coverage for deterministic value-minimized final-weight receipts, exact sampling-receipt binding, contiguous transformation lineage, append-only correction linkage, fail-closed weighted-result binding to separate point-weight and variance-design receipts, exact point/variance compatibility across analysis-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence and final-weight artifact, cross-sectional/longitudinal scope and target-population/reference-duration/case-set/final-artifact congruence, explicit nonresponse disposition treatment, calibration **auxiliary projection version**, benchmark receipt version/owner-contract/reference-time provenance, calibration constraint/termination provenance, scientific auxiliary-authority/purpose/released-owner/authorization/scientific-use correlation coordinates, owner-contract and use-receipt digest validation, future-use-time and future-benchmark-time rejection, fallback-rule disclosure, trimming/bounding rule/configuration and affected-case provenance, no-op trim/bound rejection, and typed adjustment-evidence matching. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. +The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. #407 adds RED/GREEN coverage for deterministic value-minimized final-weight receipts, exact sampling-receipt binding, contiguous transformation lineage, append-only correction linkage, fail-closed weighted-result binding to separate point-weight and variance-design receipts, exact point/variance compatibility across analysis-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence and final-weight artifact, cross-sectional/longitudinal scope and target-population/reference-duration/case-set/final-artifact congruence, explicit nonresponse disposition treatment, calibration **auxiliary projection version**, benchmark receipt version/owner-contract/reference-time provenance, calibration constraint/termination provenance, scientific auxiliary-authority/purpose/released-owner/authorization/scientific-use correlation coordinates, owner-contract and use-receipt digest validation, future-use-time and future-benchmark-time rejection, fallback reason/rule/actual algorithm-version-configuration disclosure, converged-state rejection of fallback-only evidence, trimming/bounding rule/configuration and affected-case provenance, no-op trim/bound rejection, and typed adjustment-evidence matching. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. ## References From 958fbc389be6d7090ba1ca604da8109c0895afd4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 16:08:50 +0900 Subject: [PATCH 140/158] docs(traceability): bind calibration fallback to actual generating method --- docs/traceability/validation-analysis-handoff.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/traceability/validation-analysis-handoff.md b/docs/traceability/validation-analysis-handoff.md index 6f8258cc6..9f5509728 100644 --- a/docs/traceability/validation-analysis-handoff.md +++ b/docs/traceability/validation-analysis-handoff.md @@ -17,7 +17,7 @@ Can an organization send one exact, reviewable validation study to its statistic | Final point-weight provenance | exact estimand/target/window/reference-duration and case-set digests, source/sampling receipt digests, base-weight evidence, ordered digest-linked adjustments, final weight artifact, append-only correction lineage | `FinalAnalysisWeightReceipt` deterministic/value-minimized regressions plus chain/correction fail-closed tests | | Weight eligibility congruence | governed `cross_sectional` or `longitudinal` scope plus exact target population, reference-duration evidence, eligible case set, and final point-weight artifact | `test_weight_eligibility_receipt.py` plus `FinalAnalysisWeightReceipt` mismatch/longitudinal-match regressions | | Nonresponse adjustment evidence | versioned response/disposition receipt, exact adjustment population, controlled method/configuration, explicit ineligible/unknown/unavailable treatment, input/output weight artifacts | `NonresponseAdjustmentReceipt` deterministic/value-minimized tests plus undocumented-treatment/no-op/version rejection | -| Calibration/raking evidence | target population/window, **versioned** purpose-bound auxiliary projection reference/version/digest, scientific-use purpose reference/digest, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and use instant, calibration benchmark receipt reference/version/digest, released benchmark owner-contract reference/version/digest, benchmark reference instant, algorithm/version, constraints, input/output artifacts, explicit converged/fallback state and immutable fallback rule when used | `CalibrationAdjustmentReceipt` auxiliary-projection-version/authority/purpose/owner/authorization/scientific-use/benchmark-owner/benchmark-time/termination/fallback regressions; neither scientific-use time nor benchmark reference time may be later than receipt construction | +| Calibration/raking evidence | target population/window, **versioned** purpose-bound auxiliary projection reference/version/digest, scientific-use purpose reference/digest, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and use instant, calibration benchmark receipt reference/version/digest, released benchmark owner-contract reference/version/digest, benchmark reference instant, algorithm/version, constraints, input/output artifacts, explicit converged/fallback state; fallback output additionally binds the primary failure reason, immutable fallback rule, actual fallback algorithm/version, and fallback configuration digest | `CalibrationAdjustmentReceipt` auxiliary-projection-version/authority/purpose/owner/authorization/scientific-use/benchmark-owner/benchmark-time/termination regressions plus `test_calibration_fallback_provenance.py`; neither scientific-use time nor benchmark reference time may be later than receipt construction, and fallback-only evidence is rejected for primary convergence | | Trimming/bounding evidence | versioned trimming/bounding rule, reproducible rule configuration digest, exact affected semantic occurrence set and count, input/output weight artifacts | `test_trimming_bounding_receipt.py` deterministic/value-minimized, no-op, missing-provenance, and typed-kind regressions | | Typed adjustment congruence | known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization adjustment codes must identify the matching typed receipt family through `evidence_kind` | specialized adjustment evidence-kind regressions | | Point/variance lineage congruence | #406 variance evidence must name the same final point-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence, and final-weight artifact used by the estimate | `test_weight_variance_compatibility_receipt.py` mismatch regressions | @@ -31,10 +31,12 @@ Can an organization send one exact, reviewable validation study to its statistic The active branch makes final-weight eligibility and three adjustment families executable rather than leaving them as opaque labels. It now also closes the leaf-side form of #407 RED #8: `WeightVarianceCompatibilityReceipt` requires the variance side to identify the exact point-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence, and final-weight artifact. `ValidationAnalysisResult` cannot emit weighted design-based evidence without that compatibility receipt in addition to separate point-weight and #406 variance-design digests. +Calibration fallback is also explicit scientific provenance rather than a success-like label. `fallback_applied` requires the reason the primary calibration ceased to be authoritative, the versioned fallback rule, and the actual fallback calibration algorithm/version/configuration that produced the output artifact. Those fields are absent for a primary algorithm that converged. This prevents cell-collapse, bound-relaxation, or alternate-method fallback output from being serialized as though the originally declared algorithm simply converged. + This is still correlation evidence, not owner authentication. Calibration/raking/post-stratification evidence carries the opaque coordinates needed by canonical Workforce Validation authority resolution: auxiliary authority and **projection reference/version/digest**, exact scientific purpose, released auxiliary owner-contract identity/version/digest, authorization receipt identity/digest, scientific-use receipt identity/digest and owner-correlatable use instant, plus separately versioned benchmark receipt identity/digest, released benchmark owner-contract identity/version/digest, and benchmark reference instant. Durable #235/#248 owner-side resolution must prove both the auxiliary tuple and benchmark tuple against released/versioned owner evidence, prove that the exact **projection version** is authorized for the exact scientific purpose/use receipt, and resolve the point/variance compatibility coordinates against released #406 variance evidence rather than trusting leaf-supplied digests. Released auxiliary/benchmark/variance evidence exchange also remains open. `workforce_validation` owns those scientific contracts and their released result; `talent_management` must not compute or reconstruct final analysis weights. ## Maturity `implemented_on_active_pr`. -Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. The active package validates the returned numerical/provenance envelope plus the current #407 final-weight/result-binding, point/variance compatibility, weight-eligibility, typed adjustment-evidence, auxiliary correlation, and benchmark correlation contracts, but protected Orgmetra evidence still requires host re-resolution, durable owner evidence exchange, result-artifact verification, terminal checks, independent review, and accountable human interpretation. +Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. The active package validates the returned numerical/provenance envelope plus the current #407 final-weight/result-binding, point/variance compatibility, weight-eligibility, typed adjustment-evidence, fallback generating-method, auxiliary correlation, and benchmark correlation contracts, but protected Orgmetra evidence still requires host re-resolution, durable owner evidence exchange, result-artifact verification, terminal checks, independent review, and accountable human interpretation. From a78256396c17fc3a4a2ccbe69748e3606b65098c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 16:10:03 +0900 Subject: [PATCH 141/158] test(validity): cover malformed calibration fallback provenance --- .../test_calibration_fallback_provenance.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/packages/validity-analysis/tests/test_calibration_fallback_provenance.py b/packages/validity-analysis/tests/test_calibration_fallback_provenance.py index 35948786f..944f51c17 100644 --- a/packages/validity-analysis/tests/test_calibration_fallback_provenance.py +++ b/packages/validity-analysis/tests/test_calibration_fallback_provenance.py @@ -100,6 +100,28 @@ def test_fallback_rejects_incomplete_actual_method_provenance( calibration_receipt(**{field_name: value}) +@pytest.mark.parametrize( + ("field_name", "value", "message"), + [ + ("fallback_reason_code", "", "fallback_reason_code"), + ("fallback_rule_reference", "fallback-rule-v1", "fallback_rule_reference"), + ("fallback_rule_digest", "not-a-digest", "fallback_rule_digest"), + ("fallback_algorithm_reference", "weight_method:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", "fallback_algorithm_reference"), + ("fallback_algorithm_version", 0, "fallback_algorithm_version"), + ("fallback_algorithm_version", True, "fallback_algorithm_version"), + ("fallback_configuration_digest", "not-a-digest", "fallback_configuration_digest"), + ], +) +def test_fallback_rejects_malformed_generating_method_provenance( + field_name: str, + value: object, + message: str, +) -> None: + """Fallback provenance must be typed and reproducible, not merely present.""" + with pytest.raises(ValueError, match=message): + calibration_receipt(**{field_name: value}) + + def test_converged_calibration_rejects_fallback_only_fields() -> None: """Fallback evidence must not contaminate a genuinely converged primary algorithm.""" with pytest.raises(ValueError, match="fallback"): From befbc8aa5e2d4e865fe3bb40371e426f52cbfb09 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 22:00:15 +0900 Subject: [PATCH 142/158] test(validity): require append-only result correction lineage --- .../tests/test_result_correction_lineage.py | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 packages/validity-analysis/tests/test_result_correction_lineage.py diff --git a/packages/validity-analysis/tests/test_result_correction_lineage.py b/packages/validity-analysis/tests/test_result_correction_lineage.py new file mode 100644 index 000000000..fb1f7034f --- /dev/null +++ b/packages/validity-analysis/tests/test_result_correction_lineage.py @@ -0,0 +1,117 @@ +"""Regression tests for append-only validation-result correction lineage.""" + +from datetime import datetime, timezone +import json + +import pytest + +from orgmetra_validity_analysis import ( + ConvergenceDiagnostics, + MissingnessSummary, + REVIEWED_FAST_MLSIRM_REVISION, + ValidationAnalysisResult, +) + +TENANT = "10000000-0000-7000-8000-000000000001" +RESULT = "validation_analysis_result:11111111-1111-4111-8111-111111111111" +SUCCESSOR_RESULT = "validation_analysis_result:22222222-2222-4222-8222-222222222222" +COMPLETED_AT = datetime(2026, 9, 17, 12, 30, tzinfo=timezone.utc) + + +def _result(**overrides: object) -> ValidationAnalysisResult: + """Build one valid aggregate-only result envelope.""" + values: dict[str, object] = { + "tenant_record_id": TENANT, + "result_reference": RESULT, + "handoff_digest": "a" * 64, + "provenance_digest": "b" * 64, + "fast_mlsirm_revision": REVIEWED_FAST_MLSIRM_REVISION, + "model_code": "mlsirm_criterion_related", + "backend": "rust_cpu", + "precision": "f64", + "effect_estimate": 0.42, + "uncertainty_lower": 0.10, + "uncertainty_upper": 0.70, + "sample_size": 12, + "missingness_summary": MissingnessSummary( + total_observations=12, + complete_observations=10, + missing_predictor_observations=1, + missing_criterion_observations=1, + ), + "convergence_diagnostics": ConvergenceDiagnostics( + converged=True, + iterations=42, + objective_value=-12.5, + maximum_gradient=0.0001, + ), + "completed_at": COMPLETED_AT, + } + values.update(overrides) + return ValidationAnalysisResult(**values) + + +def test_successor_result_binds_exact_predecessor_evidence() -> None: + """Serialize correction sequence and exact predecessor identity into result bytes.""" + predecessor = _result() + successor = _result( + result_reference=SUCCESSOR_RESULT, + correction_sequence=2, + supersedes_result_reference=predecessor.result_reference, + supersedes_result_digest=predecessor.sha256_digest(), + effect_estimate=0.48, + ) + + payload = json.loads(successor.canonical_json()) + assert payload["correction_sequence"] == 2 + assert payload["supersedes_result_reference"] == predecessor.result_reference + assert payload["supersedes_result_digest"] == predecessor.sha256_digest() + assert successor.sha256_digest() != predecessor.sha256_digest() + + +def test_initial_result_rejects_predecessor_coordinates() -> None: + """Do not let sequence-one evidence pretend to supersede another result.""" + with pytest.raises(ValueError, match="correction_sequence 1"): + _result( + supersedes_result_reference=SUCCESSOR_RESULT, + supersedes_result_digest="c" * 64, + ) + + +@pytest.mark.parametrize( + "overrides", + [ + {"correction_sequence": True}, + {"correction_sequence": 0}, + {"correction_sequence": 2}, + { + "correction_sequence": 2, + "supersedes_result_reference": SUCCESSOR_RESULT, + }, + { + "correction_sequence": 2, + "supersedes_result_digest": "c" * 64, + }, + { + "correction_sequence": 2, + "supersedes_result_reference": RESULT, + "supersedes_result_digest": "c" * 64, + }, + { + "correction_sequence": 2, + "supersedes_result_reference": "analysis_weight_receipt:22222222-2222-4222-8222-222222222222", + "supersedes_result_digest": "c" * 64, + }, + { + "correction_sequence": 2, + "supersedes_result_reference": SUCCESSOR_RESULT, + "supersedes_result_digest": "not-a-digest", + }, + ], +) +def test_malformed_result_correction_lineage_fails_closed( + overrides: dict[str, object], +) -> None: + """Reject ambiguous, cyclic-looking, or malformed correction coordinates.""" + with pytest.raises(ValueError): + _result(**overrides) From 3b03baa2519f23559f1a18f1de0c675f8218e92a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 22:01:01 +0900 Subject: [PATCH 143/158] fix(validity): version corrected scientific results append-only --- .../src/orgmetra_validity_analysis/result.py | 33 +++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py index b3bd4ddd8..0c5f3a8f2 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/result.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/result.py @@ -173,6 +173,9 @@ class ValidationAnalysisResult: contains_raw_person_level_values: bool = False human_review_required: bool = True evidence_version: int = 1 + correction_sequence: int = 1 + supersedes_result_reference: str | None = None + supersedes_result_digest: str | None = None def __post_init__(self) -> None: """Fail closed on malformed, unlinked, or decision-like result data.""" @@ -197,7 +200,7 @@ def __post_init__(self) -> None: if type(self.missingness_summary) is not MissingnessSummary: raise ValueError("missingness_summary must be a MissingnessSummary") if type(self.convergence_diagnostics) is not ConvergenceDiagnostics: - raise ValueError("convergence_diagnostics must be ConvergenceDiagnostics") + raise ValueError("convergence_diagnostics must be a ConvergenceDiagnostics") if self.sample_size != self.missingness_summary.total_observations: raise ValueError("sample_size must match total_observations") completed_at = _freeze_timestamp(self.completed_at, "completed_at") @@ -272,6 +275,28 @@ def __post_init__(self) -> None: raise ValueError("human review is mandatory for validity interpretation") if type(self.evidence_version) is not int or self.evidence_version != 1: raise ValueError("evidence_version must remain 1") + _validate_positive_integer(self.correction_sequence, "correction_sequence") + if self.correction_sequence == 1: + if ( + self.supersedes_result_reference is not None + or self.supersedes_result_digest is not None + ): + raise ValueError( + "correction_sequence 1 must not identify superseded result evidence" + ) + else: + if self.supersedes_result_reference is None or self.supersedes_result_digest is None: + raise ValueError( + "corrected result requires supersedes_result_reference and supersedes_result_digest" + ) + _validate_reference( + self.supersedes_result_reference, + "validation_analysis_result", + "supersedes_result_reference", + ) + _validate_digest(self.supersedes_result_digest, "supersedes_result_digest") + if self.supersedes_result_reference == self.result_reference: + raise ValueError("corrected result must use a new result_reference") object.__setattr__(self, "effect_estimate", estimate) object.__setattr__(self, "uncertainty_lower", lower) object.__setattr__(self, "uncertainty_upper", upper) @@ -297,6 +322,7 @@ def canonical_json(self) -> str: "completed_at": _canonical_timestamp(self.completed_at, "completed_at"), "contains_raw_person_level_values": self.contains_raw_person_level_values, "convergence_diagnostics": self.convergence_diagnostics.to_dict(), + "correction_sequence": self.correction_sequence, "effect_estimate": float(self.effect_estimate), "evidence_version": self.evidence_version, "execution_state": self.execution_state, @@ -324,6 +350,9 @@ def canonical_json(self) -> str: payload["weight_variance_compatibility_receipt_digest"] = ( self.weight_variance_compatibility.sha256_digest() ) + if self.supersedes_result_reference is not None: + payload["supersedes_result_reference"] = self.supersedes_result_reference + payload["supersedes_result_digest"] = self.supersedes_result_digest return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) def sha256_digest(self) -> str: @@ -331,4 +360,4 @@ def sha256_digest(self) -> str: return sha256(self.canonical_json().encode("utf-8")).hexdigest() -__all__ = ["ConvergenceDiagnostics", "MissingnessSummary", "ValidationAnalysisResult"] \ No newline at end of file +__all__ = ["ConvergenceDiagnostics", "MissingnessSummary", "ValidationAnalysisResult"] From 8164a3d14b68cad93fddc772303c343744384f0f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 22:01:43 +0900 Subject: [PATCH 144/158] docs(validity): record result correction lineage --- packages/validity-analysis/CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/validity-analysis/CHANGELOG.md b/packages/validity-analysis/CHANGELOG.md index 2d0d0669b..77f684874 100644 --- a/packages/validity-analysis/CHANGELOG.md +++ b/packages/validity-analysis/CHANGELOG.md @@ -8,6 +8,7 @@ - Require deterministic canonical evidence and 100% owned production statement/branch coverage. - Validate a digest-linked Rust CPU/GPU result envelope with finite estimates, aggregate missingness, and explicit convergence or nonconvergence diagnostics. - Make result verification explicit without self-authorizing the scientific leaf: converged completed results serialize as `verification_pending`, while completed nonconverged results serialize as `not_verifiable` and retain their failure diagnostics. +- Version corrected `ValidationAnalysisResult` evidence append-only: the initial result is correction sequence 1 with no predecessor, while every corrected result uses a new result reference and binds the exact predecessor result reference/digest into canonical bytes. - Reject impossible aggregate missingness where complete observations overlap either predictor-missing or criterion-missing counts beyond the sample total. - Require exact governed missingness/convergence runtime types so subclass method overrides cannot inject unreviewed or person-level fields into canonical result evidence. - Freeze exact UTC timestamps and finite numeric values at construction, and reject runtime-type forgery before canonical evidence serialization. @@ -19,5 +20,5 @@ - Add typed `NonresponseAdjustmentReceipt` evidence that binds the exact versioned response/disposition receipt reference/version/digest, adjustment population, controlled method/configuration, explicit ineligible/unknown/unavailable treatment, and input/output weight artifacts; add typed `CalibrationAdjustmentReceipt` evidence bound to a versioned purpose-limited auxiliary projection identity/digest, exact scientific-use purpose, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest/use time, and a separately versioned calibration benchmark receipt with released benchmark owner-contract reference/version/digest and exact benchmark reference time, plus constraints and explicit convergence/fallback state. A fallback now records the primary failure reason, immutable fallback rule, actual fallback calibration algorithm/version, and fallback configuration digest so fallback-produced weights cannot be mislabeled as primary-algorithm convergence. Neither the use time nor benchmark reference time may be later than receipt construction. - Add typed `TrimmingBoundingAdjustmentReceipt` evidence that binds a versioned rule/configuration, exact affected semantic occurrence set and count, and input/output weight artifacts; known trimming/bounding/winsorization adjustments must name that evidence family. - Require known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization adjustment codes to identify the matching typed evidence kind rather than collapse into a generic opaque adjustment digest. -- Keep durable owner-side typed-receipt, point/variance compatibility, auxiliary-authority, and benchmark-authority resolution plus released auxiliary/benchmark/variance evidence exchange explicitly incomplete under #407; the scientific leaf preserves exact correlation coordinates including auxiliary projection version but does not self-authenticate them. +- Keep durable owner-side typed-receipt, point/variance compatibility, auxiliary-authority, benchmark-authority, and corrected-result supersession resolution plus released auxiliary/benchmark/variance evidence exchange explicitly incomplete under #407; the scientific leaf preserves exact correlation coordinates including auxiliary projection version but does not self-authenticate them. - Make `Validity Analysis Handoff Quality` retrigger on shared repository Python/test/clean-checkout configuration, with an executable regression preventing stale package-quality evidence after shared tooling changes. From 0d7c9cb50942d60434d71a8c88733f266d0a4c15 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 22:02:08 +0900 Subject: [PATCH 145/158] docs(validity): explain corrected result provenance --- packages/validity-analysis/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/validity-analysis/README.md b/packages/validity-analysis/README.md index 89399eaf8..594f43189 100644 --- a/packages/validity-analysis/README.md +++ b/packages/validity-analysis/README.md @@ -18,12 +18,14 @@ Trimming, bounding, and winsorization have a separate provenance family rather t `AnalysisWeightAdjustment` records an `evidence_kind` in addition to the evidence digest. Known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization transforms fail closed unless their evidence kind is the corresponding typed receipt. Other adjustment families remain open work rather than being silently treated as equivalent. -`WeightVarianceCompatibilityReceipt` closes the scientific-leaf form of #407 RED #8. It binds one exact `FinalAnalysisWeightReceipt` to the separate #406 variance-design receipt/version/digest and requires the variance side to identify the same analytic-case occurrence set, weight-eligibility receipt, correction sequence, and final point-weight artifact. A replicate or variance construction generated from a different eligibility, correction, case set, or final-weight version therefore cannot be paired with the point estimate merely because both digests are well formed. This receipt is correlation evidence only; #235/#248 remain responsible for resolving the released #406 owner evidence rather than trusting these caller-supplied coordinates as authority. +`WeightVarianceCompatibilityReceipt` closes the scientific-leaf form of #407 RED #8. It binds one exact `FinalAnalysisWeightReceipt` to the separate #406 variance-design receipt/version/digest and requires the variance side to identify the same analytic-case occurrence set, weight-eligibility receipt, correction sequence, and final-weight artifact. A replicate or variance construction generated from a different eligibility, correction, case set, or final-weight version therefore cannot be paired with the point estimate merely because both digests are well formed. This receipt is correlation evidence only; #235/#248 remain responsible for resolving the released #406 owner evidence rather than trusting these caller-supplied coordinates as authority. `ValidationAnalysisResult` accepts only a result linked to the handoff digest and the same reviewed fast-mlsirm revision. It records the Rust CPU/GPU backend, precision, aggregate missingness counts, finite effect and interval values, and explicit convergence or nonconvergence diagnostics. Timestamps and finite numeric values are snapshotted before canonicalization, and the result envelope accepts only the exact governed `MissingnessSummary` and `ConvergenceDiagnostics` runtime types, preventing mutable runtime values or subclass method overrides from adding unreviewed or person-level fields to canonical audit evidence. Missingness counts must be internally possible: complete observations cannot overlap either predictor-missing or criterion-missing observations beyond the declared sample total. Point-estimation semantics are explicit. `unweighted` results cannot carry weight receipts. `weighted_design_based` results must separately bind the exact `FinalAnalysisWeightReceipt` digest, the variance-design receipt digest that supports the uncertainty method actually used, and the `WeightVarianceCompatibilityReceipt` proving those two evidence paths refer to the same point-weight basis. A replicate/variance evidence reference therefore cannot silently stand in for the final point-estimation weight, or vice versa. The result remains scientific evidence for accountable human interpretation and never becomes an employment decision. +Corrected scientific results are append-only. The first `ValidationAnalysisResult` is `correction_sequence=1` and has no predecessor coordinates. Any corrected result must use a new `validation_analysis_result:*` reference and include the exact predecessor result reference and digest in its canonical bytes. This prevents a changed estimate, uncertainty interval, weight binding, sample, or other material result evidence from silently replacing a previously released result in place. The leaf records the predecessor link; canonical #235/#248 still must resolve released predecessor/successor chronology and authority intervals before a corrected result becomes durable owner truth. + ## What it does not do - It does **not** run statistics. @@ -32,13 +34,13 @@ Point-estimation semantics are explicit. `unweighted` results cannot carry weigh - It does **not** claim that a selection procedure is valid. - It does **not** interpret adverse impact. - It does **not** authorize hiring, promotion, termination, compensation, or another employment decision. -- It does **not** yet complete #407. Durable owner-side verification that adjustment, point/variance compatibility, auxiliary-authority, and benchmark-authority coordinates resolve to the released typed receipts/contracts they claim; enforcement that the resolved auxiliary projection **version** is authorized for the exact scientific purpose/use receipt/time; and released auxiliary/benchmark/variance evidence exchange remain open work in `workforce_validation`. +- It does **not** yet complete #407. Durable owner-side verification that adjustment, point/variance compatibility, auxiliary-authority, benchmark-authority, and corrected-result predecessor/successor coordinates resolve to the released typed receipts/contracts they claim; enforcement that the resolved auxiliary projection **version** is authorized for the exact scientific purpose/use receipt/time; and released auxiliary/benchmark/variance evidence exchange remain open work in `workforce_validation`. The fast-mlsirm repository remains a dedicated-writer dependency. This package records only the immutable revision reviewed for the handoff. Durable Workforce Validation registry/API/persistence remains on the canonical #235/#248 owner path; this package does not create a parallel service. ## Host obligations -Before an approved offline validation worker executes the handoff, the Orgmetra host must re-resolve every reference inside `tenant_record_id` and prove that the predictor, criterion, population, and policy evidence belong to the exact validation study and Job. The requester and reviewer must resolve to distinct authoritative actors. For weighted design-based inference, the host must also verify the final point-weight receipt, its matching cross-sectional/longitudinal weight-eligibility receipt, its upstream source/sampling evidence, each typed adjustment receipt identified by the ordered chain, the calibration auxiliary authority/purpose/released owner contract/authorization/scientific-use tuple including the exact auxiliary projection **version**, the calibration benchmark receipt/version/released benchmark owner contract/reference instant, any fallback reason/rule/actual fallback algorithm-version-configuration tuple, the separately bound #406 variance-design evidence, and the point/variance compatibility receipt against released owner evidence rather than trusting caller-supplied labels or digests. A numerical result is scientific evidence for accountable human interpretation, never an autonomous employment decision. +Before an approved offline validation worker executes the handoff, the Orgmetra host must re-resolve every reference inside `tenant_record_id` and prove that the predictor, criterion, population, and policy evidence belong to the exact validation study and Job. The requester and reviewer must resolve to distinct authoritative actors. For weighted design-based inference, the host must also verify the final point-weight receipt, its matching cross-sectional/longitudinal weight-eligibility receipt, its upstream source/sampling evidence, each typed adjustment receipt identified by the ordered chain, the calibration auxiliary authority/purpose/released owner contract/authorization/scientific-use tuple including the exact auxiliary projection **version**, the calibration benchmark receipt/version/released benchmark owner contract/reference instant, any fallback reason/rule/actual fallback algorithm-version-configuration tuple, the separately bound #406 variance-design evidence, and the point/variance compatibility receipt against released owner evidence rather than trusting caller-supplied labels or digests. A corrected result must additionally resolve its predecessor result reference/digest against released owner evidence instead of treating a caller-supplied correction sequence as authority. A numerical result is scientific evidence for accountable human interpretation, never an autonomous employment decision. ## Verification From c4c640f830809a77f8bb4396400762b7217e2d83 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 22:02:39 +0900 Subject: [PATCH 146/158] docs(validity): trace corrected result lineage --- docs/traceability/validation-analysis-handoff.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/traceability/validation-analysis-handoff.md b/docs/traceability/validation-analysis-handoff.md index 9f5509728..8b8d8672d 100644 --- a/docs/traceability/validation-analysis-handoff.md +++ b/docs/traceability/validation-analysis-handoff.md @@ -14,6 +14,7 @@ Can an organization send one exact, reviewable validation study to its statistic | Human authority | requester/reviewer references must differ, and the host must re-resolve both within the tenant and prove they resolve to distinct authoritative actor identities before execution | direct-construction fail-closed regression plus `test_next_action_requires_resolved_actor_identity_separation` | | Scientific evidence | effect estimate, uncertainty interval, sample size, internally possible aggregate missingness, convergence diagnostics | immutable required-result-evidence regression plus impossible-missingness rejection | | Numerical result boundary | handoff digest, pinned fast-mlsirm revision, Rust CPU/GPU backend, precision, finite estimate/interval, aggregate missingness, explicit convergence state | `ValidationAnalysisResult` invariant/canonicalization regressions, exact-runtime-type checks, and oversized-numeric `ValueError` normalization | +| Result correction lineage | correction sequence plus exact predecessor `validation_analysis_result` reference/digest for every corrected result; sequence 1 has no predecessor and a corrected result must use a new result reference | `test_result_correction_lineage.py` deterministic successor and malformed-lineage regressions | | Final point-weight provenance | exact estimand/target/window/reference-duration and case-set digests, source/sampling receipt digests, base-weight evidence, ordered digest-linked adjustments, final weight artifact, append-only correction lineage | `FinalAnalysisWeightReceipt` deterministic/value-minimized regressions plus chain/correction fail-closed tests | | Weight eligibility congruence | governed `cross_sectional` or `longitudinal` scope plus exact target population, reference-duration evidence, eligible case set, and final point-weight artifact | `test_weight_eligibility_receipt.py` plus `FinalAnalysisWeightReceipt` mismatch/longitudinal-match regressions | | Nonresponse adjustment evidence | versioned response/disposition receipt, exact adjustment population, controlled method/configuration, explicit ineligible/unknown/unavailable treatment, input/output weight artifacts | `NonresponseAdjustmentReceipt` deterministic/value-minimized tests plus undocumented-treatment/no-op/version rejection | @@ -31,12 +32,14 @@ Can an organization send one exact, reviewable validation study to its statistic The active branch makes final-weight eligibility and three adjustment families executable rather than leaving them as opaque labels. It now also closes the leaf-side form of #407 RED #8: `WeightVarianceCompatibilityReceipt` requires the variance side to identify the exact point-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence, and final-weight artifact. `ValidationAnalysisResult` cannot emit weighted design-based evidence without that compatibility receipt in addition to separate point-weight and #406 variance-design digests. +RED #10 also requires corrected result evidence to be append-only rather than an in-place rewrite. `ValidationAnalysisResult` now serializes `correction_sequence`; sequence 1 rejects predecessor coordinates, while every later correction must use a new result reference and bind the exact predecessor result reference/digest. This is still leaf correlation evidence. Canonical #235/#248 must resolve predecessor/successor release chronology and authority intervals so a superseded result cannot remain current merely because its immutable bytes still exist. + Calibration fallback is also explicit scientific provenance rather than a success-like label. `fallback_applied` requires the reason the primary calibration ceased to be authoritative, the versioned fallback rule, and the actual fallback calibration algorithm/version/configuration that produced the output artifact. Those fields are absent for a primary algorithm that converged. This prevents cell-collapse, bound-relaxation, or alternate-method fallback output from being serialized as though the originally declared algorithm simply converged. -This is still correlation evidence, not owner authentication. Calibration/raking/post-stratification evidence carries the opaque coordinates needed by canonical Workforce Validation authority resolution: auxiliary authority and **projection reference/version/digest**, exact scientific purpose, released auxiliary owner-contract identity/version/digest, authorization receipt identity/digest, scientific-use receipt identity/digest and owner-correlatable use instant, plus separately versioned benchmark receipt identity/digest, released benchmark owner-contract identity/version/digest, and benchmark reference instant. Durable #235/#248 owner-side resolution must prove both the auxiliary tuple and benchmark tuple against released/versioned owner evidence, prove that the exact **projection version** is authorized for the exact scientific purpose/use receipt, and resolve the point/variance compatibility coordinates against released #406 variance evidence rather than trusting leaf-supplied digests. Released auxiliary/benchmark/variance evidence exchange also remains open. `workforce_validation` owns those scientific contracts and their released result; `talent_management` must not compute or reconstruct final analysis weights. +This is still correlation evidence, not owner authentication. Calibration/raking/post-stratification evidence carries the opaque coordinates needed by canonical Workforce Validation authority resolution: auxiliary authority and **projection reference/version/digest**, exact scientific purpose, released auxiliary owner-contract identity/version/digest, authorization receipt identity/digest, scientific-use receipt identity/digest and owner-correlatable use instant, plus separately versioned benchmark receipt identity/digest, released benchmark owner-contract identity/version/digest, and benchmark reference instant. Durable #235/#248 owner-side resolution must prove both the auxiliary tuple and benchmark tuple against released/versioned owner evidence, prove that the exact **projection version** is authorized for the exact scientific purpose/use receipt, resolve corrected-result predecessor/successor chronology, and resolve the point/variance compatibility coordinates against released #406 variance evidence rather than trusting leaf-supplied digests. Released auxiliary/benchmark/variance evidence exchange also remains open. `workforce_validation` owns those scientific contracts and their released result; `talent_management` must not compute or reconstruct final analysis weights. ## Maturity `implemented_on_active_pr`. -Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. The active package validates the returned numerical/provenance envelope plus the current #407 final-weight/result-binding, point/variance compatibility, weight-eligibility, typed adjustment-evidence, fallback generating-method, auxiliary correlation, and benchmark correlation contracts, but protected Orgmetra evidence still requires host re-resolution, durable owner evidence exchange, result-artifact verification, terminal checks, independent review, and accountable human interpretation. +Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. The active package validates the returned numerical/provenance envelope plus the current #407 final-weight/result-binding, result-correction lineage, point/variance compatibility, weight-eligibility, typed adjustment-evidence, fallback generating-method, auxiliary correlation, and benchmark correlation contracts, but protected Orgmetra evidence still requires host re-resolution, durable owner evidence exchange, result-artifact verification, terminal checks, independent review, and accountable human interpretation. From b4aeec9e2565401ab7b754f4b1b888077d2496cb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 22:07:46 +0900 Subject: [PATCH 147/158] docs(adr): govern append-only corrected validation results --- ...erned-selection-validity-analysis-handoff.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/adr/0027-governed-selection-validity-analysis-handoff.md b/docs/adr/0027-governed-selection-validity-analysis-handoff.md index 260c65b32..7546923fc 100644 --- a/docs/adr/0027-governed-selection-validity-analysis-handoff.md +++ b/docs/adr/0027-governed-selection-validity-analysis-handoff.md @@ -30,21 +30,23 @@ Both handoff and result envelopes detach exact timezone-aware timestamps to one The same package also validates `ValidationAnalysisResult` envelopes returned by the approved offline worker. A result must link to the handoff digest and the same pinned revision, identify a Rust CPU or GPU backend and precision, provide finite effect and interval values, match its sample size to aggregate missingness counts, reject impossible complete-versus-missing count combinations, and include explicit convergence diagnostics. The canonicalization boundary accepts only the exact governed `MissingnessSummary` and `ConvergenceDiagnostics` runtime types so subclass method overrides cannot add unreviewed or person-level fields to immutable result evidence. A nonconverged result remains typed scientific evidence requiring human review; it cannot be treated as a valid selection procedure or an employment decision. +Released scientific results are append-only. The initial `ValidationAnalysisResult` uses `correction_sequence=1` and carries no predecessor coordinates. Every corrected result uses a new `validation_analysis_result:*` reference and binds the exact predecessor result reference and SHA-256 digest into its own canonical bytes. A changed estimate, uncertainty interval, sample, weight binding, or other material result evidence therefore cannot silently overwrite a previously released result while retaining ambiguous lineage. This leaf contract is correlation evidence rather than durable authority: canonical `workforce_validation` owner persistence must independently resolve predecessor release, exclusive supersession cutover, complete successor reference/correction-sequence/digest/release evidence, and historical half-open authority intervals. + For #407's weighted design-based inference boundary, the active package adds `FinalAnalysisWeightReceipt`. It binds the exact estimand, target population, analysis window, reference duration, eligible/analytic case-set digests, #404/#405 source/sampling receipt digests, base-weight derivation evidence, an ordered immutable adjustment chain, the final point-weight artifact digest, construction time, and append-only correction lineage without storing row-level weights. Each adjustment must be contiguous and digest-linked from the previous artifact to the declared final artifact. `ValidationAnalysisResult` explicitly distinguishes `unweighted` from `weighted_design_based`; a weighted result fails closed unless it separately binds both the final analysis-weight receipt digest and the #406 variance-design receipt digest. An unweighted result cannot carry either receipt and thereby masquerade as weighted scientific evidence. `WeightEligibilityReceipt` makes weight use itself versioned evidence rather than a caller-supplied label. The receipt is closed to `cross_sectional` or `longitudinal` scope and binds the exact target population, reference-duration evidence, eligible-case set, and final point-weight artifact. `FinalAnalysisWeightReceipt` validates that the eligibility receipt belongs to the same tenant and that its scope, target population, reference duration, eligible cases, and weight artifact exactly match the estimand-side receipt. A longitudinal weight therefore cannot silently support a cross-sectional estimand or a different target period. The 2025 SIPP Users' Guide is used only as current primary methodological evidence that weight choice depends on both target population and duration and that longitudinal weights cover explicit multi-year reference periods; SIPP-specific variables or estimators are not imported into Orgmetra. The adjustment chain has typed scientific evidence for three high-risk transform families. `NonresponseAdjustmentReceipt` binds a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, explicit treatment of ineligible/unknown/unavailable cases, and input/output weight artifacts. `CalibrationAdjustmentReceipt` binds the target population/window, **versioned purpose-limited auxiliary projection reference/version/digest**, exact scientific-use purpose reference/digest, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and use instant, and a separately versioned calibration benchmark receipt reference/version/digest with its released benchmark owner-contract reference/version/digest and exact benchmark reference instant. It also binds algorithm/version, constraints digest, input/output artifacts, and an explicit `converged` or `fallback_applied` termination state. Both the scientific-use instant and benchmark reference instant are frozen and cannot be later than receipt construction. This makes the complete opaque correlation tuple expected by canonical Workforce Validation owner resolution part of the scientific receipt without copying protected auxiliary or benchmark values. The leaf package still does not pretend its caller-supplied coordinates are authoritative: the durable `workforce_validation` service must resolve the released auxiliary owner contract, authorization receipt, scientific-use receipt, benchmark receipt, and released benchmark owner contract and prove that the actual **projection version** and benchmark are the exact released evidence authorized/referenced for the analysis before execution or release. A fallback is not represented by a boolean-like flag or rule digest alone: it must identify the primary failure reason, immutable fallback-rule reference/digest, and the actual fallback calibration algorithm/version plus configuration digest that produced the output weights. Fallback-only evidence is forbidden when the primary algorithm converged. A failed/nonconverged calibration cannot be labeled as an accepted calibration receipt. `TrimmingBoundingAdjustmentReceipt` binds a versioned trimming/bounding rule, reproducible rule configuration digest, the semantic occurrence set and count of cases actually affected, and input/output weight artifacts; a declared trim/bound transform cannot be a no-op. Known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization `AnalysisWeightAdjustment` codes also carry an `evidence_kind` and fail closed unless it names the matching typed receipt family. -#407 RED #8 requires more than two different well-formed digests. A final point weight must not be combined with replicate or variance evidence generated from a different eligibility, correction, calibration, analytic case set, or final-weight version. The active package therefore adds `WeightVarianceCompatibilityReceipt`. It binds one exact `FinalAnalysisWeightReceipt` to the separate #406 variance-design receipt reference/version/digest and requires the variance side to identify the exact same analysis-weight receipt digest, analytic-case occurrence set, weight-eligibility receipt digest, correction sequence, and final-weight artifact. The receipt must be constructed no earlier than the point-weight receipt, and its variance-design digest must remain distinct from the point-weight receipt itself. `ValidationAnalysisResult` now requires this compatibility receipt for `weighted_design_based` output and verifies that its point-weight and variance-design digests are the same ones recorded on the result. Unweighted results cannot carry compatibility evidence. +#407 RED #8 requires more than two different well-formed digests. A final point weight must not be combined with replicate or variance evidence generated from a different eligibility, correction, calibration, analytic case set, or final-weight version. The active package therefore adds `WeightVarianceCompatibilityReceipt`. It binds one exact `FinalAnalysisWeightReceipt` to the separate #406 variance-design receipt reference/version/digest and requires the variance side to identify the exact same analysis-weight receipt digest, analytic-case occurrence set, weight-eligibility receipt digest, correction sequence, and final-weight artifact. The receipt must be constructed no earlier than the point-weight receipt, and its variance-design digest must remain distinct from the point-weight receipt itself. `ValidationAnalysisResult` requires this compatibility receipt for `weighted_design_based` output and verifies that its point-weight and variance-design digests are the same ones recorded on the result. Unweighted results cannot carry compatibility evidence. -This compatibility receipt is deliberately not a self-authenticating variance owner. Its role is deterministic scientific correlation at the leaf boundary: it prevents a caller from presenting internally inconsistent point/variance lineages as one weighted result. Canonical #235/#248 remain responsible for resolving the released #406 variance-design evidence, proving that the variance-side coordinates actually came from the authoritative released owner contract, and rejecting caller-fabricated correlation data. The same owner boundary continues to apply separately to the calibration auxiliary-authority tuple and calibration benchmark-authority tuple. +This compatibility receipt is deliberately not a self-authenticating variance owner. Its role is deterministic scientific correlation at the leaf boundary: it prevents a caller from presenting internally inconsistent point/variance lineages as one weighted result. Canonical #235/#248 remain responsible for resolving the released #406 variance-design evidence, proving that the variance-side coordinates actually came from the authoritative released owner contract, and rejecting caller-fabricated correlation data. The same owner boundary continues to apply separately to the calibration auxiliary-authority tuple, calibration benchmark-authority tuple, and validation-result predecessor/successor correction tuple. -This executable slice still does not claim #407 complete. The leaf can now preserve the opaque coordinates needed for durable owner-side calibration auxiliary and benchmark correlation and can fail closed when point-weight and variance evidence are internally incongruent. Durable service/API verification that auxiliary, benchmark, and point/variance compatibility coordinates resolve to the released typed receipts/contracts claimed, verification that released authorization permits the resolved **projection version** for the exact purpose and use, and released auxiliary/benchmark/variance owner evidence exchange remain `workforce_validation` scientific truth rather than Talent truth or cross-context SQL reconstruction. +This executable slice still does not claim #407 complete. The leaf preserves the opaque coordinates needed for durable owner-side calibration auxiliary, benchmark, point/variance and result-correction correlation and can fail closed when point-weight and variance evidence are internally incongruent. Durable service/API verification that these coordinates resolve to released typed receipts/contracts, verification that released authorization permits the resolved **projection version** for the exact purpose and use, owner-resolved validation-result supersession chronology, and released auxiliary/benchmark/variance owner evidence exchange remain `workforce_validation` scientific truth rather than Talent truth or cross-context SQL reconstruction. The NIST Privacy Framework 1.0 is used narrowly as a privacy-risk-management basis for expressing and verifying data-processing requirements across organizational roles and contracts; as of the 2026-09-17 check, NIST still presents Privacy Framework 1.1 as an Initial Public Draft rather than a final replacement. This ADR does not treat the voluntary framework as employment law or infer legal permission from a NIST profile. -The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, prove requester/reviewer identities are distinct authoritative actors, verify cross-sectional/longitudinal weight eligibility against the estimand, verify typed weight-adjustment receipts plus any calibration auxiliary **projection reference/version/digest** and authority/purpose/released-owner/authorization/scientific-use tuple and calibration benchmark receipt/version/released-owner/reference-time tuple, verify any fallback reason/rule/actual fallback algorithm-version-configuration tuple, and resolve the point/variance compatibility receipt against the released #406 owner evidence rather than trusting caller-supplied evidence labels or digests. +The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, prove requester/reviewer identities are distinct authoritative actors, verify cross-sectional/longitudinal weight eligibility against the estimand, verify typed weight-adjustment receipts plus any calibration auxiliary **projection reference/version/digest** and authority/purpose/released-owner/authorization/scientific-use tuple and calibration benchmark receipt/version/released-owner/reference-time tuple, verify any fallback reason/rule/actual fallback algorithm-version-configuration tuple, resolve corrected-result predecessor evidence, and resolve the point/variance compatibility receipt against the released #406 owner evidence rather than trusting caller-supplied evidence labels or digests. ## Consequences @@ -55,6 +57,7 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - Aggregate missingness evidence cannot claim all observations are complete while simultaneously reporting predictor- or criterion-missing observations. - Result canonicalization cannot be extended by an unreviewed subclass to serialize extra decision-like or person-level fields. - Caller-controlled timestamp and numeric runtime behavior cannot rewrite an accepted canonical digest after construction or turn malformed oversized worker output into an uncaught exception type. +- A corrected scientific result cannot silently replace a predecessor: every correction gets a new result reference and digest-linked predecessor lineage. - Weighted design-based results can no longer identify only a sample while omitting which final point-weight evidence and variance-design evidence were actually used. - A variance/replicate construction from a different analytic case set, eligibility receipt, correction sequence, or final point-weight artifact cannot be paired with the point estimate as though the two lineages were congruent. - Cross-sectional and longitudinal weights cannot be interchanged when target population, reference duration, eligible cases, or final weight artifact differ from the estimand. @@ -69,13 +72,13 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - This slice does not execute a statistical model, estimate validity, correct for measurement error/range restriction, evaluate adverse impact, or assert legal compliance. - Sampling design, estimator choice, missing-data treatment, reliability evidence, multiplicity, transportability, fairness analysis, and model diagnostics must be encoded in the referenced analysis plan and reviewed before execution. -- The current #407 slice validates generic ordered weight provenance, weighted-result binding, point/variance compatibility, cross-sectional/longitudinal eligibility congruence, disposition-aware nonresponse evidence, owner-correlatable calibration auxiliary and benchmark evidence, explicit fallback generating-method provenance, and trimming/bounding rule provenance; it does not yet make durable typed-receipt/authorization/benchmark-owner/variance-owner resolution or released auxiliary/benchmark/variance exchange fully executable. +- The current #407 slice validates generic ordered weight provenance, weighted-result binding, result correction linkage, point/variance compatibility, cross-sectional/longitudinal eligibility congruence, disposition-aware nonresponse evidence, owner-correlatable calibration auxiliary and benchmark evidence, explicit fallback generating-method provenance, and trimming/bounding rule provenance; it does not yet make durable typed-receipt/authorization/benchmark-owner/variance-owner/result-supersession resolution or released auxiliary/benchmark/variance exchange fully executable. - The generic result envelope does not invent an estimator-specific minimum complete-case count. Whether a converged estimator is identified under a particular missing-data design belongs to the reviewed analysis plan and numerical-worker contract; Orgmetra fails closed on impossible aggregate counts without silently replacing that foreign scientific contract with complete-case analysis. -- The package validates the result envelope, but a future execution adapter must still re-resolve the handoff references, verify the result provenance artifact, resolve typed adjustment, purpose-authorization, and benchmark coordinates to released owner evidence, resolve point/variance compatibility against released #406 evidence, and attach evidence only after accountable human review. +- The package validates the result envelope, but a future execution adapter must still re-resolve the handoff references, verify the result provenance artifact, resolve typed adjustment, purpose-authorization, benchmark and result-correction coordinates to released owner evidence, resolve point/variance compatibility against released #406 evidence, and attach evidence only after accountable human review. ## Verification -The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. #407 adds RED/GREEN coverage for deterministic value-minimized final-weight receipts, exact sampling-receipt binding, contiguous transformation lineage, append-only correction linkage, fail-closed weighted-result binding to separate point-weight and variance-design receipts, exact point/variance compatibility across analysis-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence and final-weight artifact, cross-sectional/longitudinal scope and target-population/reference-duration/case-set/final-artifact congruence, explicit nonresponse disposition treatment, calibration **auxiliary projection version**, benchmark receipt version/owner-contract/reference-time provenance, calibration constraint/termination provenance, scientific auxiliary-authority/purpose/released-owner/authorization/scientific-use correlation coordinates, owner-contract and use-receipt digest validation, future-use-time and future-benchmark-time rejection, fallback reason/rule/actual algorithm-version-configuration disclosure, converged-state rejection of fallback-only evidence, trimming/bounding rule/configuration and affected-case provenance, no-op trim/bound rejection, and typed adjustment-evidence matching. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. +The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. #407 adds RED/GREEN coverage for append-only validation-result predecessor linkage, deterministic value-minimized final-weight receipts, exact sampling-receipt binding, contiguous transformation lineage, append-only weight correction linkage, fail-closed weighted-result binding to separate point-weight and variance-design receipts, exact point/variance compatibility across analysis-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence and final-weight artifact, cross-sectional/longitudinal scope and target-population/reference-duration/case-set/final-artifact congruence, explicit nonresponse disposition treatment, calibration **auxiliary projection version**, benchmark receipt version/owner-contract/reference-time provenance, calibration constraint/termination provenance, scientific auxiliary-authority/purpose/released-owner/authorization/scientific-use correlation coordinates, owner-contract and use-receipt digest validation, future-use-time and future-benchmark-time rejection, fallback reason/rule/actual algorithm-version-configuration disclosure, converged-state rejection of fallback-only evidence, trimming/bounding rule/configuration and affected-case provenance, no-op trim/bound rejection, and typed adjustment-evidence matching. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. ## References From 9c12026b77021f403227dd3dfd22793d6bf2dade Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 18 Sep 2026 13:02:58 +0900 Subject: [PATCH 148/158] test(validity): expose calibration constraint drift --- ...test_calibration_constraint_application.py | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 packages/validity-analysis/tests/test_calibration_constraint_application.py diff --git a/packages/validity-analysis/tests/test_calibration_constraint_application.py b/packages/validity-analysis/tests/test_calibration_constraint_application.py new file mode 100644 index 000000000..73f85d115 --- /dev/null +++ b/packages/validity-analysis/tests/test_calibration_constraint_application.py @@ -0,0 +1,94 @@ +"""RED contracts for reviewed versus actually applied calibration constraints.""" + +from dataclasses import fields +from datetime import datetime, timedelta, timezone + +import pytest + +from orgmetra_validity_analysis import CalibrationAdjustmentReceipt + +TENANT = "10000000-0000-7000-8000-000000000001" +NOW = datetime(2026, 9, 18, 3, 30, tzinfo=timezone.utc) +DIGEST_A = "a" * 64 +DIGEST_B = "b" * 64 +DIGEST_C = "c" * 64 +DIGEST_D = "d" * 64 +DIGEST_E = "e" * 64 +DIGEST_F = "f" * 64 +DIGEST_1 = "1" * 64 +DIGEST_2 = "2" * 64 +DIGEST_3 = "3" * 64 +DIGEST_4 = "4" * 64 +DIGEST_5 = "5" * 64 + + +def _receipt(**overrides: object) -> CalibrationAdjustmentReceipt: + """Build one calibration receipt with separate reviewed and applied constraints.""" + values: dict[str, object] = { + "tenant_record_id": TENANT, + "receipt_reference": "calibration_adjustment_receipt:33333333-3333-4333-8333-333333333333", + "target_population_digest": DIGEST_A, + "analysis_window_reference": "analysis_window:44444444-4444-4444-8444-444444444444", + "auxiliary_authority_reference": "scientific_auxiliary_authority:55555555-5555-4555-8555-555555555550", + "auxiliary_projection_reference": "calibration_auxiliary_projection:55555555-5555-4555-8555-555555555555", + "auxiliary_projection_version": 4, + "auxiliary_projection_digest": DIGEST_B, + "auxiliary_purpose_reference": "scientific_data_use_purpose:55555555-5555-4555-8555-555555555556", + "auxiliary_purpose_digest": DIGEST_3, + "auxiliary_owner_contract_reference": "released_owner_contract:55555555-5555-4555-8555-555555555557", + "auxiliary_owner_contract_version": 1, + "auxiliary_owner_contract_digest": DIGEST_1, + "auxiliary_authorization_receipt_reference": "scientific_data_authorization:55555555-5555-4555-8555-555555555558", + "auxiliary_authorization_receipt_digest": DIGEST_4, + "auxiliary_scientific_use_receipt_reference": "scientific_use_receipt:55555555-5555-4555-8555-555555555559", + "auxiliary_scientific_use_receipt_digest": DIGEST_2, + "auxiliary_scientific_use_at": NOW, + "benchmark_receipt_reference": "calibration_benchmark_receipt:66666666-6666-4666-8666-666666666666", + "benchmark_receipt_version": 2, + "benchmark_receipt_digest": DIGEST_C, + "benchmark_owner_contract_reference": "released_owner_contract:66666666-6666-4666-8666-666666666667", + "benchmark_owner_contract_version": 3, + "benchmark_owner_contract_digest": DIGEST_5, + "benchmark_reference_at": NOW - timedelta(days=1), + "algorithm_reference": "calibration_algorithm:77777777-7777-4777-8777-777777777777", + "algorithm_version": 1, + "constraints_digest": DIGEST_F, + "applied_constraints_digest": DIGEST_F, + "termination_code": "converged", + "input_weight_artifact_digest": DIGEST_D, + "output_weight_artifact_digest": DIGEST_E, + "constructed_at": NOW, + } + values.update(overrides) + return CalibrationAdjustmentReceipt(**values) + + +def test_calibration_receipt_exposes_actual_applied_constraints() -> None: + """A reviewed constraint set and the generating constraint set must be distinct fields.""" + assert "applied_constraints_digest" in { + field.name for field in fields(CalibrationAdjustmentReceipt) + } + + +def test_primary_convergence_rejects_silent_constraint_relaxation() -> None: + """Changed generating constraints require explicit fallback provenance.""" + with pytest.raises(ValueError, match="fallback"): + _receipt(applied_constraints_digest=DIGEST_1) + + +def test_fallback_commits_to_reviewed_and_applied_constraint_sets() -> None: + """A fallback may change constraints only when its generating path is explicit.""" + candidate = _receipt( + termination_code="fallback_applied", + applied_constraints_digest=DIGEST_1, + fallback_reason_code="constraint_relaxation", + fallback_rule_reference="calibration_fallback_rule:99999999-9999-4999-8999-999999999999", + fallback_rule_digest=DIGEST_2, + fallback_algorithm_reference="calibration_algorithm:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + fallback_algorithm_version=2, + fallback_configuration_digest=DIGEST_4, + ) + payload = candidate.canonical_json() + + assert f'"constraints_digest":"{DIGEST_F}"' in payload + assert f'"applied_constraints_digest":"{DIGEST_1}"' in payload From ed6fb63ca5087e4c43f3c79b2b5232633c3d4778 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 18 Sep 2026 13:04:27 +0900 Subject: [PATCH 149/158] fix(validity): bind applied calibration constraints --- .../src/orgmetra_validity_analysis/weights.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py b/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py index 2ef4f5133..9f3cd3bf4 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/weights.py @@ -165,6 +165,7 @@ class CalibrationAdjustmentReceipt: algorithm_reference: str algorithm_version: int constraints_digest: str + applied_constraints_digest: str termination_code: str input_weight_artifact_digest: str output_weight_artifact_digest: str @@ -258,6 +259,7 @@ def __post_init__(self) -> None: "benchmark_receipt_digest", "benchmark_owner_contract_digest", "constraints_digest", + "applied_constraints_digest", "input_weight_artifact_digest", "output_weight_artifact_digest", ): @@ -302,8 +304,13 @@ def __post_init__(self) -> None: self.fallback_configuration_digest, "fallback_configuration_digest", ) - elif any(value is not None for value in fallback_fields): - raise ValueError("fallback evidence must be absent when calibration converged") + else: + if self.applied_constraints_digest != self.constraints_digest: + raise ValueError( + "changed applied calibration constraints require explicit fallback provenance" + ) + if any(value is not None for value in fallback_fields): + raise ValueError("fallback evidence must be absent when calibration converged") if self.input_weight_artifact_digest == self.output_weight_artifact_digest: raise ValueError( "output_weight_artifact_digest must identify the calibrated weight artifact" @@ -337,6 +344,7 @@ def canonical_json(self) -> str: "algorithm_reference": self.algorithm_reference, "algorithm_version": self.algorithm_version, "analysis_window_reference": self.analysis_window_reference, + "applied_constraints_digest": self.applied_constraints_digest, "auxiliary_authority_reference": self.auxiliary_authority_reference, "auxiliary_authorization_receipt_digest": self.auxiliary_authorization_receipt_digest, "auxiliary_authorization_receipt_reference": self.auxiliary_authorization_receipt_reference, @@ -509,7 +517,7 @@ def __post_init__(self) -> None: object.__setattr__(self, "constructed_at", constructed_at) def __repr__(self) -> str: - """Return a value-minimized representation for routine logs.""" + """Return a value-minimized representation suitable for routine logs.""" return "WeightEligibilityReceipt()" def canonical_json(self) -> str: From 305560d06280906f5218c978abb640c6f881ecdc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 18 Sep 2026 13:04:49 +0900 Subject: [PATCH 150/158] test(validity): bind fallback applied constraints --- .../tests/test_calibration_fallback_provenance.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/validity-analysis/tests/test_calibration_fallback_provenance.py b/packages/validity-analysis/tests/test_calibration_fallback_provenance.py index 944f51c17..e0f196ed7 100644 --- a/packages/validity-analysis/tests/test_calibration_fallback_provenance.py +++ b/packages/validity-analysis/tests/test_calibration_fallback_provenance.py @@ -52,6 +52,7 @@ def calibration_receipt(**overrides: object) -> CalibrationAdjustmentReceipt: "algorithm_reference": "calibration_algorithm:77777777-7777-4777-8777-777777777777", "algorithm_version": 1, "constraints_digest": DIGEST_F, + "applied_constraints_digest": DIGEST_F, "termination_code": "fallback_applied", "fallback_reason_code": "primary_nonconvergence", "fallback_rule_reference": "calibration_fallback_rule:99999999-9999-4999-8999-999999999999", @@ -67,9 +68,9 @@ def calibration_receipt(**overrides: object) -> CalibrationAdjustmentReceipt: return CalibrationAdjustmentReceipt(**values) -def test_fallback_identifies_reason_rule_and_algorithm_that_produced_weights() -> None: - """Do not label fallback output as if the primary calibration algorithm succeeded.""" - candidate = calibration_receipt() +def test_fallback_identifies_reason_rule_algorithm_and_applied_constraints() -> None: + """Do not label fallback output as if the primary calibration specification succeeded.""" + candidate = calibration_receipt(applied_constraints_digest=DIGEST_1) canonical = candidate.canonical_json() assert '"termination_code":"fallback_applied"' in canonical @@ -78,6 +79,8 @@ def test_fallback_identifies_reason_rule_and_algorithm_that_produced_weights() - assert '"fallback_algorithm_version":2' in canonical assert f'"fallback_configuration_digest":"{DIGEST_4}"' in canonical assert f'"fallback_rule_digest":"{DIGEST_2}"' in canonical + assert f'"constraints_digest":"{DIGEST_F}"' in canonical + assert f'"applied_constraints_digest":"{DIGEST_1}"' in canonical @pytest.mark.parametrize( @@ -110,6 +113,7 @@ def test_fallback_rejects_incomplete_actual_method_provenance( ("fallback_algorithm_version", 0, "fallback_algorithm_version"), ("fallback_algorithm_version", True, "fallback_algorithm_version"), ("fallback_configuration_digest", "not-a-digest", "fallback_configuration_digest"), + ("applied_constraints_digest", "not-a-digest", "applied_constraints_digest"), ], ) def test_fallback_rejects_malformed_generating_method_provenance( From 2ce954c5716bb7c13dec3468de5959f9f27f49bb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 18 Sep 2026 13:05:26 +0900 Subject: [PATCH 151/158] test(validity): preserve applied calibration constraints --- .../tests/test_weight_adjustment_semantics.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/validity-analysis/tests/test_weight_adjustment_semantics.py b/packages/validity-analysis/tests/test_weight_adjustment_semantics.py index 75126527d..fd2f2d2a3 100644 --- a/packages/validity-analysis/tests/test_weight_adjustment_semantics.py +++ b/packages/validity-analysis/tests/test_weight_adjustment_semantics.py @@ -79,6 +79,7 @@ def calibration_receipt(**overrides: object) -> CalibrationAdjustmentReceipt: "algorithm_reference": "calibration_algorithm:77777777-7777-4777-8777-777777777777", "algorithm_version": 1, "constraints_digest": DIGEST_F, + "applied_constraints_digest": DIGEST_F, "termination_code": "converged", "input_weight_artifact_digest": DIGEST_D, "output_weight_artifact_digest": DIGEST_E, @@ -137,7 +138,7 @@ def test_nonresponse_receipt_is_value_minimized_and_disposition_aware() -> None: def test_calibration_receipt_binds_owner_authority_use_and_termination_state() -> None: - """Keep owner-corroboratable authority, benchmark, and fallback semantics explicit.""" + """Keep owner-corroboratable authority, benchmark, and generating constraints explicit.""" candidate = calibration_receipt() canonical = candidate.canonical_json() assert candidate.sha256_digest() == calibration_receipt().sha256_digest() @@ -151,6 +152,8 @@ def test_calibration_receipt_binds_owner_authority_use_and_termination_state() - assert f'"auxiliary_authorization_receipt_digest":"{DIGEST_4}"' in canonical assert f'"auxiliary_scientific_use_receipt_digest":"{DIGEST_2}"' in canonical assert '"auxiliary_scientific_use_at":"2026-09-17T05:00:00Z"' in canonical + assert f'"constraints_digest":"{DIGEST_F}"' in canonical + assert f'"applied_constraints_digest":"{DIGEST_F}"' in canonical assert '"termination_code":"converged"' in canonical assert "protected_attribute" not in canonical assert repr(candidate) == "CalibrationAdjustmentReceipt()" @@ -159,7 +162,8 @@ def test_calibration_receipt_binds_owner_authority_use_and_termination_state() - fallback_algorithm_reference = "calibration_algorithm:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" fallback = calibration_receipt( termination_code="fallback_applied", - fallback_reason_code="primary_nonconvergence", + applied_constraints_digest=DIGEST_1, + fallback_reason_code="constraint_relaxation", fallback_rule_reference=fallback_reference, fallback_rule_digest=DIGEST_2, fallback_algorithm_reference=fallback_algorithm_reference, @@ -168,10 +172,11 @@ def test_calibration_receipt_binds_owner_authority_use_and_termination_state() - ) fallback_json = fallback.canonical_json() assert f'"fallback_rule_digest":"{DIGEST_2}"' in fallback_json - assert '"fallback_reason_code":"primary_nonconvergence"' in fallback_json + assert '"fallback_reason_code":"constraint_relaxation"' in fallback_json assert f'"fallback_algorithm_reference":"{fallback_algorithm_reference}"' in fallback_json assert '"fallback_algorithm_version":2' in fallback_json assert f'"fallback_configuration_digest":"{DIGEST_4}"' in fallback_json + assert f'"applied_constraints_digest":"{DIGEST_1}"' in fallback_json with pytest.raises(ValueError, match="auxiliary_authority_reference"): calibration_receipt(auxiliary_authority_reference="authority-v1") @@ -215,6 +220,8 @@ def test_calibration_receipt_binds_owner_authority_use_and_termination_state() - calibration_receipt(termination_code="failed") with pytest.raises(ValueError, match="termination_code"): calibration_receipt(termination_code=1) + with pytest.raises(ValueError, match="fallback"): + calibration_receipt(applied_constraints_digest=DIGEST_1) with pytest.raises(ValueError, match="fallback"): calibration_receipt(termination_code="fallback_applied") with pytest.raises(ValueError, match="fallback"): @@ -233,6 +240,8 @@ def test_calibration_receipt_binds_owner_authority_use_and_termination_state() - ) with pytest.raises(ValueError, match="fallback"): calibration_receipt(fallback_rule_digest=DIGEST_2) + with pytest.raises(ValueError, match="applied_constraints_digest"): + calibration_receipt(applied_constraints_digest="not-a-digest") with pytest.raises(ValueError, match="output_weight_artifact_digest"): calibration_receipt(output_weight_artifact_digest=DIGEST_D) with pytest.raises(ValueError, match="evidence_version"): From 76b91e5b38ec5e54f501e7e9b5de4f28762d1937 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 18 Sep 2026 13:06:21 +0900 Subject: [PATCH 152/158] docs(validity): distinguish reviewed and applied calibration constraints --- packages/validity-analysis/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/validity-analysis/README.md b/packages/validity-analysis/README.md index 594f43189..053b8a8f0 100644 --- a/packages/validity-analysis/README.md +++ b/packages/validity-analysis/README.md @@ -12,7 +12,7 @@ The resulting canonical JSON is digest-addressable, contains no raw person-level `WeightEligibilityReceipt` makes cross-sectional versus longitudinal use machine-checkable rather than an opaque weight label. It binds the final weight artifact to one governed scope (`cross_sectional` or `longitudinal`), target population, reference-duration evidence, and eligible-case set. `FinalAnalysisWeightReceipt` fails closed unless those fields match the estimand and the same final point-weight artifact exactly, so a longitudinal weight cannot silently support a cross-sectional estimand or a different reference duration. -Nonresponse and calibration are no longer allowed to hide behind an undifferentiated adjustment digest. `NonresponseAdjustmentReceipt` binds the adjustment to a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, and explicit treatment codes for ineligible, unknown, and unavailable cases. `CalibrationAdjustmentReceipt` binds the adjustment to a target population/window, **versioned** purpose-bound auxiliary projection identity/digest, exact scientific-use purpose, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and owner-correlatable use instant, plus a separately versioned calibration benchmark receipt with released benchmark owner-contract reference/version/digest and exact benchmark reference instant. It also binds algorithm/version, constraints digest, and explicit `converged` or `fallback_applied` termination evidence. Neither the scientific-use instant nor the benchmark reference instant may be later than receipt construction. A fallback must identify why the primary calibration did not remain authoritative, the immutable fallback rule, and the actual fallback calibration algorithm/version/configuration that produced the output weights; fallback evidence is forbidden on a genuinely converged primary algorithm. A nonconverged calibration cannot masquerade as an accepted calibration receipt. These fields preserve the opaque correlation tuples expected by the canonical Workforce Validation authority boundaries without copying protected auxiliary or benchmark values. The leaf receipt still does not authenticate those coordinates itself: the durable owner service must resolve the released auxiliary and benchmark evidence and prove that the exact projection version and benchmark are authoritative for the scientific use. +Nonresponse and calibration are no longer allowed to hide behind an undifferentiated adjustment digest. `NonresponseAdjustmentReceipt` binds the adjustment to a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, and explicit treatment codes for ineligible, unknown, and unavailable cases. `CalibrationAdjustmentReceipt` binds the adjustment to a target population/window, **versioned** purpose-bound auxiliary projection identity/digest, exact scientific-use purpose, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and owner-correlatable use instant, plus a separately versioned calibration benchmark receipt with released benchmark owner-contract reference/version/digest and exact benchmark reference instant. It records the reviewed constraint set as `constraints_digest` and the constraint set that actually generated the output as `applied_constraints_digest`. A result labelled primary `converged` fails closed unless those digests are identical; any relaxation or other constraint change must therefore cross the explicit `fallback_applied` path and identify the failure reason, immutable fallback rule, actual fallback calibration algorithm/version, and fallback configuration. Neither the scientific-use instant nor the benchmark reference instant may be later than receipt construction. Fallback evidence is forbidden on a genuinely converged primary algorithm, and a nonconverged calibration cannot masquerade as an accepted calibration receipt. These fields preserve the opaque correlation tuples expected by the canonical Workforce Validation authority boundaries without copying protected auxiliary or benchmark values. The leaf receipt still does not authenticate those coordinates itself: the durable owner service must resolve the released auxiliary and benchmark evidence and prove that the exact projection version, benchmark, reviewed constraints, and applied constraints are authoritative for the scientific use. Trimming, bounding, and winsorization have a separate provenance family rather than falling back to a generic adjustment label. `TrimmingBoundingAdjustmentReceipt` binds the exact versioned rule, its reproducible configuration digest, the semantic occurrence set and count of cases actually affected, and the input/output weight artifacts. A declared trim/bound transform must change artifact identity. Known trimming/bounding/winsorization adjustment codes fail closed unless `evidence_kind` names this typed receipt family. @@ -34,13 +34,13 @@ Corrected scientific results are append-only. The first `ValidationAnalysisResul - It does **not** claim that a selection procedure is valid. - It does **not** interpret adverse impact. - It does **not** authorize hiring, promotion, termination, compensation, or another employment decision. -- It does **not** yet complete #407. Durable owner-side verification that adjustment, point/variance compatibility, auxiliary-authority, benchmark-authority, and corrected-result predecessor/successor coordinates resolve to the released typed receipts/contracts they claim; enforcement that the resolved auxiliary projection **version** is authorized for the exact scientific purpose/use receipt/time; and released auxiliary/benchmark/variance evidence exchange remain open work in `workforce_validation`. +- It does **not** yet complete #407. Durable owner-side verification that adjustment, point/variance compatibility, auxiliary-authority, benchmark-authority, constraint-set, and corrected-result predecessor/successor coordinates resolve to the released typed receipts/contracts they claim; enforcement that the resolved auxiliary projection **version** is authorized for the exact scientific purpose/use receipt/time; and released auxiliary/benchmark/variance evidence exchange remain open work in `workforce_validation`. The fast-mlsirm repository remains a dedicated-writer dependency. This package records only the immutable revision reviewed for the handoff. Durable Workforce Validation registry/API/persistence remains on the canonical #235/#248 owner path; this package does not create a parallel service. ## Host obligations -Before an approved offline validation worker executes the handoff, the Orgmetra host must re-resolve every reference inside `tenant_record_id` and prove that the predictor, criterion, population, and policy evidence belong to the exact validation study and Job. The requester and reviewer must resolve to distinct authoritative actors. For weighted design-based inference, the host must also verify the final point-weight receipt, its matching cross-sectional/longitudinal weight-eligibility receipt, its upstream source/sampling evidence, each typed adjustment receipt identified by the ordered chain, the calibration auxiliary authority/purpose/released owner contract/authorization/scientific-use tuple including the exact auxiliary projection **version**, the calibration benchmark receipt/version/released benchmark owner contract/reference instant, any fallback reason/rule/actual fallback algorithm-version-configuration tuple, the separately bound #406 variance-design evidence, and the point/variance compatibility receipt against released owner evidence rather than trusting caller-supplied labels or digests. A corrected result must additionally resolve its predecessor result reference/digest against released owner evidence instead of treating a caller-supplied correction sequence as authority. A numerical result is scientific evidence for accountable human interpretation, never an autonomous employment decision. +Before an approved offline validation worker executes the handoff, the Orgmetra host must re-resolve every reference inside `tenant_record_id` and prove that the predictor, criterion, population, and policy evidence belong to the exact validation study and Job. The requester and reviewer must resolve to distinct authoritative actors. For weighted design-based inference, the host must also verify the final point-weight receipt, its matching cross-sectional/longitudinal weight-eligibility receipt, its upstream source/sampling evidence, each typed adjustment receipt identified by the ordered chain, the calibration auxiliary authority/purpose/released owner contract/authorization/scientific-use tuple including the exact auxiliary projection **version**, the calibration benchmark receipt/version/released benchmark owner contract/reference instant, the reviewed and actually applied calibration constraint digests plus any fallback reason/rule/actual fallback algorithm-version-configuration tuple, the separately bound #406 variance-design evidence, and the point/variance compatibility receipt against released owner evidence rather than trusting caller-supplied labels or digests. A corrected result must additionally resolve its predecessor result reference/digest against released owner evidence instead of treating a caller-supplied correction sequence as authority. A numerical result is scientific evidence for accountable human interpretation, never an autonomous employment decision. ## Verification From 11c37a98f6816215d40abb71bbe19d6e983d2c0a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 18 Sep 2026 13:07:19 +0900 Subject: [PATCH 153/158] docs(validity): make calibration constraint drift explicit --- ...verned-selection-validity-analysis-handoff.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/adr/0027-governed-selection-validity-analysis-handoff.md b/docs/adr/0027-governed-selection-validity-analysis-handoff.md index 7546923fc..008d97d60 100644 --- a/docs/adr/0027-governed-selection-validity-analysis-handoff.md +++ b/docs/adr/0027-governed-selection-validity-analysis-handoff.md @@ -36,17 +36,17 @@ For #407's weighted design-based inference boundary, the active package adds `Fi `WeightEligibilityReceipt` makes weight use itself versioned evidence rather than a caller-supplied label. The receipt is closed to `cross_sectional` or `longitudinal` scope and binds the exact target population, reference-duration evidence, eligible-case set, and final point-weight artifact. `FinalAnalysisWeightReceipt` validates that the eligibility receipt belongs to the same tenant and that its scope, target population, reference duration, eligible cases, and weight artifact exactly match the estimand-side receipt. A longitudinal weight therefore cannot silently support a cross-sectional estimand or a different target period. The 2025 SIPP Users' Guide is used only as current primary methodological evidence that weight choice depends on both target population and duration and that longitudinal weights cover explicit multi-year reference periods; SIPP-specific variables or estimators are not imported into Orgmetra. -The adjustment chain has typed scientific evidence for three high-risk transform families. `NonresponseAdjustmentReceipt` binds a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, explicit treatment of ineligible/unknown/unavailable cases, and input/output weight artifacts. `CalibrationAdjustmentReceipt` binds the target population/window, **versioned purpose-limited auxiliary projection reference/version/digest**, exact scientific-use purpose reference/digest, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and use instant, and a separately versioned calibration benchmark receipt reference/version/digest with its released benchmark owner-contract reference/version/digest and exact benchmark reference instant. It also binds algorithm/version, constraints digest, input/output artifacts, and an explicit `converged` or `fallback_applied` termination state. Both the scientific-use instant and benchmark reference instant are frozen and cannot be later than receipt construction. This makes the complete opaque correlation tuple expected by canonical Workforce Validation owner resolution part of the scientific receipt without copying protected auxiliary or benchmark values. The leaf package still does not pretend its caller-supplied coordinates are authoritative: the durable `workforce_validation` service must resolve the released auxiliary owner contract, authorization receipt, scientific-use receipt, benchmark receipt, and released benchmark owner contract and prove that the actual **projection version** and benchmark are the exact released evidence authorized/referenced for the analysis before execution or release. A fallback is not represented by a boolean-like flag or rule digest alone: it must identify the primary failure reason, immutable fallback-rule reference/digest, and the actual fallback calibration algorithm/version plus configuration digest that produced the output weights. Fallback-only evidence is forbidden when the primary algorithm converged. A failed/nonconverged calibration cannot be labeled as an accepted calibration receipt. `TrimmingBoundingAdjustmentReceipt` binds a versioned trimming/bounding rule, reproducible rule configuration digest, the semantic occurrence set and count of cases actually affected, and input/output weight artifacts; a declared trim/bound transform cannot be a no-op. Known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization `AnalysisWeightAdjustment` codes also carry an `evidence_kind` and fail closed unless it names the matching typed receipt family. +The adjustment chain has typed scientific evidence for three high-risk transform families. `NonresponseAdjustmentReceipt` binds a versioned response/disposition receipt, the exact adjustment population, controlled method/configuration evidence, explicit treatment of ineligible/unknown/unavailable cases, and input/output weight artifacts. `CalibrationAdjustmentReceipt` binds the target population/window, **versioned purpose-limited auxiliary projection reference/version/digest**, exact scientific-use purpose reference/digest, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and use instant, and a separately versioned calibration benchmark receipt reference/version/digest with its released benchmark owner-contract reference/version/digest and exact benchmark reference instant. It also binds algorithm/version, the reviewed/requested constraint set as `constraints_digest`, the actual generating constraint set as `applied_constraints_digest`, input/output artifacts, and an explicit `converged` or `fallback_applied` termination state. Primary `converged` evidence is valid only when reviewed and applied constraint digests are identical. Any bound relaxation, cell collapse, or other constraint change must cross the explicit fallback path and identify the primary failure reason, immutable fallback-rule reference/digest, actual fallback calibration algorithm/version, and configuration digest that generated the output. Both the scientific-use instant and benchmark reference instant are frozen and cannot be later than receipt construction. This makes the complete opaque correlation tuple expected by canonical Workforce Validation owner resolution part of the scientific receipt without copying protected auxiliary or benchmark values. The leaf package still does not pretend its caller-supplied coordinates are authoritative: the durable `workforce_validation` service must resolve the released auxiliary owner contract, authorization receipt, scientific-use receipt, benchmark receipt, released benchmark owner contract, reviewed constraints, and applied constraints before execution or release. Fallback-only evidence is forbidden when the primary algorithm converged. A failed/nonconverged calibration cannot be labeled as an accepted calibration receipt. `TrimmingBoundingAdjustmentReceipt` binds a versioned trimming/bounding rule, reproducible rule configuration digest, the semantic occurrence set and count of cases actually affected, and input/output weight artifacts; a declared trim/bound transform cannot be a no-op. Known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization `AnalysisWeightAdjustment` codes also carry an `evidence_kind` and fail closed unless it names the matching typed receipt family. #407 RED #8 requires more than two different well-formed digests. A final point weight must not be combined with replicate or variance evidence generated from a different eligibility, correction, calibration, analytic case set, or final-weight version. The active package therefore adds `WeightVarianceCompatibilityReceipt`. It binds one exact `FinalAnalysisWeightReceipt` to the separate #406 variance-design receipt reference/version/digest and requires the variance side to identify the exact same analysis-weight receipt digest, analytic-case occurrence set, weight-eligibility receipt digest, correction sequence, and final-weight artifact. The receipt must be constructed no earlier than the point-weight receipt, and its variance-design digest must remain distinct from the point-weight receipt itself. `ValidationAnalysisResult` requires this compatibility receipt for `weighted_design_based` output and verifies that its point-weight and variance-design digests are the same ones recorded on the result. Unweighted results cannot carry compatibility evidence. -This compatibility receipt is deliberately not a self-authenticating variance owner. Its role is deterministic scientific correlation at the leaf boundary: it prevents a caller from presenting internally inconsistent point/variance lineages as one weighted result. Canonical #235/#248 remain responsible for resolving the released #406 variance-design evidence, proving that the variance-side coordinates actually came from the authoritative released owner contract, and rejecting caller-fabricated correlation data. The same owner boundary continues to apply separately to the calibration auxiliary-authority tuple, calibration benchmark-authority tuple, and validation-result predecessor/successor correction tuple. +This compatibility receipt is deliberately not a self-authenticating variance owner. Its role is deterministic scientific correlation at the leaf boundary: it prevents a caller from presenting internally inconsistent point/variance lineages as one weighted result. Canonical #235/#248 remain responsible for resolving the released #406 variance-design evidence, proving that the variance-side coordinates actually came from the authoritative released owner contract, and rejecting caller-fabricated correlation data. The same owner boundary continues to apply separately to the calibration auxiliary-authority tuple, calibration benchmark-authority tuple, calibration constraint tuple, and validation-result predecessor/successor correction tuple. -This executable slice still does not claim #407 complete. The leaf preserves the opaque coordinates needed for durable owner-side calibration auxiliary, benchmark, point/variance and result-correction correlation and can fail closed when point-weight and variance evidence are internally incongruent. Durable service/API verification that these coordinates resolve to released typed receipts/contracts, verification that released authorization permits the resolved **projection version** for the exact purpose and use, owner-resolved validation-result supersession chronology, and released auxiliary/benchmark/variance owner evidence exchange remain `workforce_validation` scientific truth rather than Talent truth or cross-context SQL reconstruction. +This executable slice still does not claim #407 complete. The leaf preserves the opaque coordinates needed for durable owner-side calibration auxiliary, benchmark, constraint, point/variance and result-correction correlation and can fail closed when point-weight and variance evidence are internally incongruent. Durable service/API verification that these coordinates resolve to released typed receipts/contracts, verification that released authorization permits the resolved **projection version** for the exact purpose and use, owner-resolved validation-result supersession chronology, and released auxiliary/benchmark/variance owner evidence exchange remain `workforce_validation` scientific truth rather than Talent truth or cross-context SQL reconstruction. The NIST Privacy Framework 1.0 is used narrowly as a privacy-risk-management basis for expressing and verifying data-processing requirements across organizational roles and contracts; as of the 2026-09-17 check, NIST still presents Privacy Framework 1.1 as an Initial Public Draft rather than a final replacement. This ADR does not treat the voluntary framework as employment law or infer legal permission from a NIST profile. -The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, prove requester/reviewer identities are distinct authoritative actors, verify cross-sectional/longitudinal weight eligibility against the estimand, verify typed weight-adjustment receipts plus any calibration auxiliary **projection reference/version/digest** and authority/purpose/released-owner/authorization/scientific-use tuple and calibration benchmark receipt/version/released-owner/reference-time tuple, verify any fallback reason/rule/actual fallback algorithm-version-configuration tuple, resolve corrected-result predecessor evidence, and resolve the point/variance compatibility receipt against the released #406 owner evidence rather than trusting caller-supplied evidence labels or digests. +The package does not invoke fast-mlsirm. An approved offline worker is the later execution boundary. Before execution, the Orgmetra host must re-resolve every reference inside the tenant, verify exact study/Job membership and evidence provenance, prove requester/reviewer identities are distinct authoritative actors, verify cross-sectional/longitudinal weight eligibility against the estimand, verify typed weight-adjustment receipts plus any calibration auxiliary **projection reference/version/digest** and authority/purpose/released-owner/authorization/scientific-use tuple and calibration benchmark receipt/version/released-owner/reference-time tuple, verify reviewed-versus-applied calibration constraints and any fallback reason/rule/actual fallback algorithm-version-configuration tuple, resolve corrected-result predecessor evidence, and resolve the point/variance compatibility receipt against the released #406 owner evidence rather than trusting caller-supplied evidence labels or digests. ## Consequences @@ -63,7 +63,7 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - Cross-sectional and longitudinal weights cannot be interchanged when target population, reference duration, eligible cases, or final weight artifact differ from the estimand. - The ordered weight transformation chain is independently digest-correlatable without centralizing case-level weights or auxiliary attributes. - Nonresponse cannot silently drop refusal/unreachable/ineligible/failure dispositions behind one opaque adjustment label. -- Calibration/raking cannot silently float auxiliary projection **version**, benchmark ownership, benchmark version, or benchmark reference time; omit the exact scientific-use purpose/released-owner/authorization/use correlation tuple; move its use or benchmark time after receipt construction; or hide a fallback as primary-algorithm convergence without naming the failure reason and actual fallback algorithm/version/configuration. +- Calibration/raking cannot silently float auxiliary projection **version**, benchmark ownership, benchmark version, benchmark reference time, reviewed constraints, or actually applied constraints; omit the exact scientific-use purpose/released-owner/authorization/use correlation tuple; move its use or benchmark time after receipt construction; or hide a constraint relaxation/fallback as primary-algorithm convergence. - Trimming/bounding/winsorization cannot silently alter final point weights without an immutable rule/configuration and affected-case receipt. - Human interpretation remains explicit and separate from numerical output. - The dedicated-writer boundary remains intact: Orgmetra consumes only a pinned foreign revision/contract boundary and never mutates fast-mlsirm. @@ -72,13 +72,13 @@ The package does not invoke fast-mlsirm. An approved offline worker is the later - This slice does not execute a statistical model, estimate validity, correct for measurement error/range restriction, evaluate adverse impact, or assert legal compliance. - Sampling design, estimator choice, missing-data treatment, reliability evidence, multiplicity, transportability, fairness analysis, and model diagnostics must be encoded in the referenced analysis plan and reviewed before execution. -- The current #407 slice validates generic ordered weight provenance, weighted-result binding, result correction linkage, point/variance compatibility, cross-sectional/longitudinal eligibility congruence, disposition-aware nonresponse evidence, owner-correlatable calibration auxiliary and benchmark evidence, explicit fallback generating-method provenance, and trimming/bounding rule provenance; it does not yet make durable typed-receipt/authorization/benchmark-owner/variance-owner/result-supersession resolution or released auxiliary/benchmark/variance exchange fully executable. +- The current #407 slice validates generic ordered weight provenance, weighted-result binding, result correction linkage, point/variance compatibility, cross-sectional/longitudinal eligibility congruence, disposition-aware nonresponse evidence, owner-correlatable calibration auxiliary and benchmark evidence, reviewed-versus-applied calibration constraints, explicit fallback generating-method provenance, and trimming/bounding rule provenance; it does not yet make durable typed-receipt/authorization/benchmark-owner/constraint-owner/variance-owner/result-supersession resolution or released auxiliary/benchmark/variance exchange fully executable. - The generic result envelope does not invent an estimator-specific minimum complete-case count. Whether a converged estimator is identified under a particular missing-data design belongs to the reviewed analysis plan and numerical-worker contract; Orgmetra fails closed on impossible aggregate counts without silently replacing that foreign scientific contract with complete-case analysis. -- The package validates the result envelope, but a future execution adapter must still re-resolve the handoff references, verify the result provenance artifact, resolve typed adjustment, purpose-authorization, benchmark and result-correction coordinates to released owner evidence, resolve point/variance compatibility against released #406 evidence, and attach evidence only after accountable human review. +- The package validates the result envelope, but a future execution adapter must still re-resolve the handoff references, verify the result provenance artifact, resolve typed adjustment, purpose-authorization, benchmark, calibration-constraint and result-correction coordinates to released owner evidence, resolve point/variance compatibility against released #406 evidence, and attach evidence only after accountable human review. ## Verification -The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. #407 adds RED/GREEN coverage for append-only validation-result predecessor linkage, deterministic value-minimized final-weight receipts, exact sampling-receipt binding, contiguous transformation lineage, append-only weight correction linkage, fail-closed weighted-result binding to separate point-weight and variance-design receipts, exact point/variance compatibility across analysis-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence and final-weight artifact, cross-sectional/longitudinal scope and target-population/reference-duration/case-set/final-artifact congruence, explicit nonresponse disposition treatment, calibration **auxiliary projection version**, benchmark receipt version/owner-contract/reference-time provenance, calibration constraint/termination provenance, scientific auxiliary-authority/purpose/released-owner/authorization/scientific-use correlation coordinates, owner-contract and use-receipt digest validation, future-use-time and future-benchmark-time rejection, fallback reason/rule/actual algorithm-version-configuration disclosure, converged-state rejection of fallback-only evidence, trimming/bounding rule/configuration and affected-case provenance, no-op trim/bound rejection, and typed adjustment-evidence matching. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. +The package regression suite starts RED when the public handoff contract is absent and covers canonical operational tenant UUIDs, opaque UUIDv4 references, exact evidence digests, distinct human actors, exact dependency pinning, timezone-aware event time, immutable governance constants, value minimization, deterministic canonicalization, SHA-256 correlation, impossible aggregate missingness rejection, oversized numeric conversion rejection, exact governed aggregate-evidence runtime types, and 100% owned production statement/branch coverage. #407 adds RED/GREEN coverage for append-only validation-result predecessor linkage, deterministic value-minimized final-weight receipts, exact sampling-receipt binding, contiguous transformation lineage, append-only weight correction linkage, fail-closed weighted-result binding to separate point-weight and variance-design receipts, exact point/variance compatibility across analysis-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence and final-weight artifact, cross-sectional/longitudinal scope and target-population/reference-duration/case-set/final-artifact congruence, explicit nonresponse disposition treatment, calibration **auxiliary projection version**, benchmark receipt version/owner-contract/reference-time provenance, reviewed/requested versus actually applied calibration constraint digests, primary-convergence equality and explicit fallback-only constraint drift, scientific auxiliary-authority/purpose/released-owner/authorization/scientific-use correlation coordinates, owner-contract and use-receipt digest validation, future-use-time and future-benchmark-time rejection, fallback reason/rule/actual algorithm-version-configuration disclosure, converged-state rejection of fallback-only evidence, trimming/bounding rule/configuration and affected-case provenance, no-op trim/bound rejection, and typed adjustment-evidence matching. The repository-wide ADR numbering regression also fails closed if integration reuses an existing decision number, and the validity quality workflow contract requires any `docs/adr/**` change to execute that regression. ## References From ce399d3d5d188235260ca863e18c4b45e11f8c45 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 18 Sep 2026 13:07:47 +0900 Subject: [PATCH 154/158] docs(validity): trace calibration constraint application --- docs/traceability/validation-analysis-handoff.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/traceability/validation-analysis-handoff.md b/docs/traceability/validation-analysis-handoff.md index 8b8d8672d..08833862e 100644 --- a/docs/traceability/validation-analysis-handoff.md +++ b/docs/traceability/validation-analysis-handoff.md @@ -18,7 +18,7 @@ Can an organization send one exact, reviewable validation study to its statistic | Final point-weight provenance | exact estimand/target/window/reference-duration and case-set digests, source/sampling receipt digests, base-weight evidence, ordered digest-linked adjustments, final weight artifact, append-only correction lineage | `FinalAnalysisWeightReceipt` deterministic/value-minimized regressions plus chain/correction fail-closed tests | | Weight eligibility congruence | governed `cross_sectional` or `longitudinal` scope plus exact target population, reference-duration evidence, eligible case set, and final point-weight artifact | `test_weight_eligibility_receipt.py` plus `FinalAnalysisWeightReceipt` mismatch/longitudinal-match regressions | | Nonresponse adjustment evidence | versioned response/disposition receipt, exact adjustment population, controlled method/configuration, explicit ineligible/unknown/unavailable treatment, input/output weight artifacts | `NonresponseAdjustmentReceipt` deterministic/value-minimized tests plus undocumented-treatment/no-op/version rejection | -| Calibration/raking evidence | target population/window, **versioned** purpose-bound auxiliary projection reference/version/digest, scientific-use purpose reference/digest, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and use instant, calibration benchmark receipt reference/version/digest, released benchmark owner-contract reference/version/digest, benchmark reference instant, algorithm/version, constraints, input/output artifacts, explicit converged/fallback state; fallback output additionally binds the primary failure reason, immutable fallback rule, actual fallback algorithm/version, and fallback configuration digest | `CalibrationAdjustmentReceipt` auxiliary-projection-version/authority/purpose/owner/authorization/scientific-use/benchmark-owner/benchmark-time/termination regressions plus `test_calibration_fallback_provenance.py`; neither scientific-use time nor benchmark reference time may be later than receipt construction, and fallback-only evidence is rejected for primary convergence | +| Calibration/raking evidence | target population/window, **versioned** purpose-bound auxiliary projection reference/version/digest, scientific-use purpose reference/digest, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest and use instant, calibration benchmark receipt reference/version/digest, released benchmark owner-contract reference/version/digest, benchmark reference instant, algorithm/version, reviewed/requested `constraints_digest`, actual `applied_constraints_digest`, input/output artifacts, explicit converged/fallback state; fallback output additionally binds the primary failure reason, immutable fallback rule, actual fallback algorithm/version, and fallback configuration digest | `test_calibration_constraint_application.py`, `CalibrationAdjustmentReceipt` auxiliary-projection-version/authority/purpose/owner/authorization/scientific-use/benchmark-owner/benchmark-time/termination regressions, and `test_calibration_fallback_provenance.py`; primary convergence requires reviewed and applied constraints to match, while any constraint drift requires the explicit fallback path | | Trimming/bounding evidence | versioned trimming/bounding rule, reproducible rule configuration digest, exact affected semantic occurrence set and count, input/output weight artifacts | `test_trimming_bounding_receipt.py` deterministic/value-minimized, no-op, missing-provenance, and typed-kind regressions | | Typed adjustment congruence | known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization adjustment codes must identify the matching typed receipt family through `evidence_kind` | specialized adjustment evidence-kind regressions | | Point/variance lineage congruence | #406 variance evidence must name the same final point-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence, and final-weight artifact used by the estimate | `test_weight_variance_compatibility_receipt.py` mismatch regressions | @@ -34,12 +34,12 @@ The active branch makes final-weight eligibility and three adjustment families e RED #10 also requires corrected result evidence to be append-only rather than an in-place rewrite. `ValidationAnalysisResult` now serializes `correction_sequence`; sequence 1 rejects predecessor coordinates, while every later correction must use a new result reference and bind the exact predecessor result reference/digest. This is still leaf correlation evidence. Canonical #235/#248 must resolve predecessor/successor release chronology and authority intervals so a superseded result cannot remain current merely because its immutable bytes still exist. -Calibration fallback is also explicit scientific provenance rather than a success-like label. `fallback_applied` requires the reason the primary calibration ceased to be authoritative, the versioned fallback rule, and the actual fallback calibration algorithm/version/configuration that produced the output artifact. Those fields are absent for a primary algorithm that converged. This prevents cell-collapse, bound-relaxation, or alternate-method fallback output from being serialized as though the originally declared algorithm simply converged. +Calibration fallback is explicit scientific provenance rather than a success-like label. The leaf now separately commits to the reviewed/requested constraint set and the constraint set actually used to generate the calibrated weights. Primary `converged` evidence requires those digests to be identical. A bound relaxation, cell collapse, or other changed constraint set therefore cannot be serialized as primary convergence; it must use `fallback_applied`, which also requires the reason the primary calibration ceased to be authoritative, the versioned fallback rule, and the actual fallback calibration algorithm/version/configuration that produced the output artifact. -This is still correlation evidence, not owner authentication. Calibration/raking/post-stratification evidence carries the opaque coordinates needed by canonical Workforce Validation authority resolution: auxiliary authority and **projection reference/version/digest**, exact scientific purpose, released auxiliary owner-contract identity/version/digest, authorization receipt identity/digest, scientific-use receipt identity/digest and owner-correlatable use instant, plus separately versioned benchmark receipt identity/digest, released benchmark owner-contract identity/version/digest, and benchmark reference instant. Durable #235/#248 owner-side resolution must prove both the auxiliary tuple and benchmark tuple against released/versioned owner evidence, prove that the exact **projection version** is authorized for the exact scientific purpose/use receipt, resolve corrected-result predecessor/successor chronology, and resolve the point/variance compatibility coordinates against released #406 variance evidence rather than trusting leaf-supplied digests. Released auxiliary/benchmark/variance evidence exchange also remains open. `workforce_validation` owns those scientific contracts and their released result; `talent_management` must not compute or reconstruct final analysis weights. +This is still correlation evidence, not owner authentication. Calibration/raking/post-stratification evidence carries the opaque coordinates needed by canonical Workforce Validation authority resolution: auxiliary authority and **projection reference/version/digest**, exact scientific purpose, released auxiliary owner-contract identity/version/digest, authorization receipt identity/digest, scientific-use receipt identity/digest and owner-correlatable use instant, separately versioned benchmark receipt identity/digest, released benchmark owner-contract identity/version/digest, benchmark reference instant, and reviewed-versus-applied constraint digests. Durable #235/#248 owner-side resolution must prove these coordinates against released/versioned owner evidence and resolve the point/variance compatibility coordinates against released #406 variance evidence rather than trusting leaf-supplied digests. Released auxiliary/benchmark/variance evidence exchange also remains open. `workforce_validation` owns those scientific contracts and their released result; `talent_management` must not compute or reconstruct final analysis weights. ## Maturity `implemented_on_active_pr`. -Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. The active package validates the returned numerical/provenance envelope plus the current #407 final-weight/result-binding, result-correction lineage, point/variance compatibility, weight-eligibility, typed adjustment-evidence, fallback generating-method, auxiliary correlation, and benchmark correlation contracts, but protected Orgmetra evidence still requires host re-resolution, durable owner evidence exchange, result-artifact verification, terminal checks, independent review, and accountable human interpretation. +Protected `develop` does **not** gain numerical validity computation from this slice. The handoff is execution preparation only. The active package validates the returned numerical/provenance envelope plus the current #407 final-weight/result-binding, result-correction lineage, point/variance compatibility, weight-eligibility, typed adjustment-evidence, fallback generating-method, calibration-constraint application, auxiliary correlation, and benchmark correlation contracts, but protected Orgmetra evidence still requires host re-resolution, durable owner evidence exchange, result-artifact verification, terminal checks, independent review, and accountable human interpretation. From 6c02fc0b6595f5cbe1d977bd10f5ad4d6083f741 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 18 Sep 2026 13:08:11 +0900 Subject: [PATCH 155/158] docs(validity): record calibration constraint provenance --- packages/validity-analysis/CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/validity-analysis/CHANGELOG.md b/packages/validity-analysis/CHANGELOG.md index 77f684874..f25c350db 100644 --- a/packages/validity-analysis/CHANGELOG.md +++ b/packages/validity-analysis/CHANGELOG.md @@ -17,8 +17,8 @@ - Distinguish unweighted from weighted design-based results and fail closed unless a weighted result separately binds the exact final analysis-weight receipt and variance-design receipt used. - Add `WeightVarianceCompatibilityReceipt` so variance evidence must correlate to the exact final point-weight receipt, analytic-case occurrence set, weight-eligibility receipt, correction sequence, and final-weight artifact before a weighted result can be emitted. The receipt is correlation evidence, not self-authenticating #406 owner authority. - Require `ValidationAnalysisResult` to bind the compatibility receipt in addition to the point-weight and variance-design digests; unweighted results cannot carry compatibility evidence. -- Add typed `NonresponseAdjustmentReceipt` evidence that binds the exact versioned response/disposition receipt reference/version/digest, adjustment population, controlled method/configuration, explicit ineligible/unknown/unavailable treatment, and input/output weight artifacts; add typed `CalibrationAdjustmentReceipt` evidence bound to a versioned purpose-limited auxiliary projection identity/digest, exact scientific-use purpose, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest/use time, and a separately versioned calibration benchmark receipt with released benchmark owner-contract reference/version/digest and exact benchmark reference time, plus constraints and explicit convergence/fallback state. A fallback now records the primary failure reason, immutable fallback rule, actual fallback calibration algorithm/version, and fallback configuration digest so fallback-produced weights cannot be mislabeled as primary-algorithm convergence. Neither the use time nor benchmark reference time may be later than receipt construction. +- Add typed `NonresponseAdjustmentReceipt` evidence that binds the exact versioned response/disposition receipt reference/version/digest, adjustment population, controlled method/configuration, explicit ineligible/unknown/unavailable treatment, and input/output weight artifacts; add typed `CalibrationAdjustmentReceipt` evidence bound to a versioned purpose-limited auxiliary projection identity/digest, exact scientific-use purpose, scientific auxiliary-authority reference, released auxiliary owner-contract reference/version/digest, authorization receipt reference/digest, scientific-use receipt reference/digest/use time, and a separately versioned calibration benchmark receipt with released benchmark owner-contract reference/version/digest and exact benchmark reference time. Calibration now records both the reviewed/requested `constraints_digest` and the actual `applied_constraints_digest`; primary `converged` evidence requires equality, while any changed generating constraint set must use `fallback_applied` with the primary failure reason, immutable fallback rule, actual fallback calibration algorithm/version, and fallback configuration digest. Neither the use time nor benchmark reference time may be later than receipt construction. - Add typed `TrimmingBoundingAdjustmentReceipt` evidence that binds a versioned rule/configuration, exact affected semantic occurrence set and count, and input/output weight artifacts; known trimming/bounding/winsorization adjustments must name that evidence family. - Require known nonresponse, calibration/raking/post-stratification, and trimming/bounding/winsorization adjustment codes to identify the matching typed evidence kind rather than collapse into a generic opaque adjustment digest. -- Keep durable owner-side typed-receipt, point/variance compatibility, auxiliary-authority, benchmark-authority, and corrected-result supersession resolution plus released auxiliary/benchmark/variance evidence exchange explicitly incomplete under #407; the scientific leaf preserves exact correlation coordinates including auxiliary projection version but does not self-authenticate them. +- Keep durable owner-side typed-receipt, point/variance compatibility, auxiliary-authority, benchmark-authority, calibration-constraint, and corrected-result supersession resolution plus released auxiliary/benchmark/variance evidence exchange explicitly incomplete under #407; the scientific leaf preserves exact correlation coordinates including auxiliary projection version but does not self-authenticate them. - Make `Validity Analysis Handoff Quality` retrigger on shared repository Python/test/clean-checkout configuration, with an executable regression preventing stale package-quality evidence after shared tooling changes. From 9e334df37e9b3f2101f420e92c39098d7efabd68 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 18 Sep 2026 18:22:56 +0900 Subject: [PATCH 156/158] fix(ci): reseal validity changelog evidence --- manifest.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifest.json b/manifest.json index 431fc6d60..082180c23 100644 --- a/manifest.json +++ b/manifest.json @@ -29,8 +29,8 @@ }, { "path": "CHANGELOG.md", - "sha256": "4669c7ab654afce04898baa7af37ced058a14d2d9aa42bc4bee712a45b8038bf", - "bytes": 17868, + "sha256": "1985ccaf0dfa0fbf095e88998cfcfdac6f245525ac6ff0131d9f29ed43d97b93", + "bytes": 17846, "lines": 78 }, { From e22df241d3ef9a64724c76a434ecd94cdd7fc73f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 19 Sep 2026 04:07:22 +0900 Subject: [PATCH 157/158] fix(validity): allow governed long reference namespaces --- .../src/orgmetra_validity_analysis/handoff.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py b/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py index 5f3fb2f57..d6e3d0b39 100644 --- a/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py +++ b/packages/validity-analysis/src/orgmetra_validity_analysis/handoff.py @@ -18,7 +18,7 @@ _DIGEST_PATTERN = re.compile(r"^[0-9a-f]{64}$") _REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$") _REFERENCE_PATTERN = re.compile( - r"^[a-z][a-z0-9_]{1,31}:[A-Za-z0-9](?:[A-Za-z0-9._-]{0,126}[A-Za-z0-9])?$" + r"^[a-z][a-z0-9_]{1,63}:[A-Za-z0-9](?:[A-Za-z0-9._-]{0,126}[A-Za-z0-9])?$" ) _PURPOSE_CODE = "selection_validity_analysis" _REASON_CODE = "criterion_related_validation" @@ -302,4 +302,4 @@ def build_validation_analysis_handoff( reviewer_reference=reviewer_reference, fast_mlsirm_revision=fast_mlsirm_revision, requested_at=requested_at, - ) + ) \ No newline at end of file From 6a3e2aa57e3f27a2a2f50ff4072c5634e949a23a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 20 Sep 2026 16:05:33 +0900 Subject: [PATCH 158/158] test(validity): reach compatibility type guard --- ...t_weight_variance_compatibility_receipt.py | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/packages/validity-analysis/tests/test_weight_variance_compatibility_receipt.py b/packages/validity-analysis/tests/test_weight_variance_compatibility_receipt.py index 2ae22fc02..fae9e914a 100644 --- a/packages/validity-analysis/tests/test_weight_variance_compatibility_receipt.py +++ b/packages/validity-analysis/tests/test_weight_variance_compatibility_receipt.py @@ -80,7 +80,13 @@ def weight_receipt(**overrides: object) -> FinalAnalysisWeightReceipt: def compatibility(**overrides: object) -> WeightVarianceCompatibilityReceipt: """Build owner-correlatable compatibility evidence for one weighted analysis.""" - point_weight = overrides.pop("analysis_weight_receipt", weight_receipt()) + canonical_point_weight = weight_receipt() + point_weight = overrides.pop("analysis_weight_receipt", canonical_point_weight) + variance_point_weight = ( + point_weight + if type(point_weight) is FinalAnalysisWeightReceipt + else canonical_point_weight + ) values: dict[str, object] = { "tenant_record_id": TENANT, "receipt_reference": COMPATIBILITY_RECEIPT, @@ -88,15 +94,17 @@ def compatibility(**overrides: object) -> WeightVarianceCompatibilityReceipt: "variance_design_receipt_reference": VARIANCE_RECEIPT, "variance_design_receipt_version": 1, "variance_design_receipt_digest": DIGEST_3, - "variance_analysis_weight_receipt_digest": point_weight.sha256_digest(), + "variance_analysis_weight_receipt_digest": variance_point_weight.sha256_digest(), "variance_analytic_case_occurrence_set_digest": ( - point_weight.analytic_case_occurrence_set_digest + variance_point_weight.analytic_case_occurrence_set_digest ), "variance_weight_eligibility_receipt_digest": ( - point_weight.weight_eligibility.sha256_digest() + variance_point_weight.weight_eligibility.sha256_digest() + ), + "variance_weight_correction_sequence": variance_point_weight.correction_sequence, + "variance_final_weight_artifact_digest": ( + variance_point_weight.final_weight_artifact_digest ), - "variance_weight_correction_sequence": point_weight.correction_sequence, - "variance_final_weight_artifact_digest": point_weight.final_weight_artifact_digest, "constructed_at": datetime(2026, 9, 17, 0, 31, tzinfo=timezone.utc), } values.update(overrides)