diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 28d7888..50b7aa8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,7 +53,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: fruwehq/determa-state-conformance - ref: c6949440f5b9fd6e06ea5571f4e01681dd59ee04 + ref: 090aa297389e416121c25ef29fb0a75671d83233 path: .pinned/determa-state-conformance - name: Check out pinned specification uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -88,7 +88,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: fruwehq/determa-state-conformance - ref: c6949440f5b9fd6e06ea5571f4e01681dd59ee04 + ref: 090aa297389e416121c25ef29fb0a75671d83233 path: .pinned/determa-state-conformance - name: Check out pinned specification uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -126,7 +126,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: fruwehq/determa-state-conformance - ref: c6949440f5b9fd6e06ea5571f4e01681dd59ee04 + ref: 090aa297389e416121c25ef29fb0a75671d83233 path: .pinned/determa-state-conformance - name: Check out pinned specification uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/AGENTS.md b/AGENTS.md index eba3e97..c056871 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,10 +12,11 @@ The implementation is conformant only when it passes the language-neutral suite. The implementation uses these immutable inputs: - specification: `77c0a2e60cd0771a6d44ae170a079ddd51d7d9f0`; -- conformance: `c6949440f5b9fd6e06ea5571f4e01681dd59ee04` (99 format-1 +- conformance: `090aa297389e416121c25ef29fb0a75671d83233` (reviewed coordinated + draft #114; 99 format-1 core cases, 162 version-1 vectors, 142 durable-host vectors, 49 core inspection vectors, 47 extension negotiation vectors, 7 optional native-provider vectors, - 52 runtime-provider vectors, and 487 JSON artifacts). + 52 runtime-provider vectors, 62 committed-native-effect vectors, and 487 JSON artifacts). The package metadata is `0.3.0`. diff --git a/README.md b/README.md index 596bd4e..cb346f2 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ commit `77c0a2e60cd0771a6d44ae170a079ddd51d7d9f0`. Correctness is determined by 99 format-1 core cases, 162 version-1 vectors, 142 durable-host vectors, 49 core inspection vectors, 47 extension negotiation vectors, 7 native guard-provider inspection vectors, 52 runtime-provider vectors, and 487 manifest artifacts at -conformance commit `c6949440f5b9fd6e06ea5571f4e01681dd59ee04`. The native +conformance commit `090aa297389e416121c25ef29fb0a75671d83233`. The native runtime-provider profile is optional and is exercised through its production adapter. The package metadata is `0.3.0`. Artifact and checkpoint schema version 1 is the diff --git a/conformance/effects_adapter.py b/conformance/effects_adapter.py index 9b59466..a781f0b 100644 --- a/conformance/effects_adapter.py +++ b/conformance/effects_adapter.py @@ -396,6 +396,7 @@ def equal(left: Any, right: Any) -> bool: class _FixtureNativeProvider: def __init__(self, invocation: Any) -> None: self.invocation = invocation + self.retry_checker_available = True def validate_configuration(self, configuration: dict[str, Any]) -> dict[str, Any]: return copy.deepcopy(configuration) @@ -410,9 +411,21 @@ def invoke(self, instance: Any, payload: Any, metadata: Any, attempt: Any) -> An return self.invocation(payload, metadata, attempt) def verify_deduplication_evidence(self, instance: Any, evidence: Any) -> bool: - return instance["destination_binding_digest"] == _sha( - _destination_configuration() - ) and evidence == _destination_proof(evidence["scope_identity"], evidence["effect_id"]) + return ( + self.retry_checker_available + and instance["destination_binding_digest"] == _sha(_destination_configuration()) + and { + key: evidence[key] + for key in ( + "scope_identity", + "effect_id", + "destination_binding_digest", + "first_attempt_receipt_bytes_base64", + "repeat_attempt_receipt_bytes_base64", + ) + } + == _destination_proof(evidence["scope_identity"], evidence["effect_id"]) + ) def _run_operation(payload: dict[str, Any], control: Path | None = None) -> dict[str, Any]: @@ -533,6 +546,8 @@ def verify_native(source: Any, selected: Any) -> bool: }, ) installed_handler = VerifiedNativeHandler(handler_registry, configured_handler) + from determa.state.native_clock import NativeCommitClock + host = SQLiteCommittedEffectHost( _PATH, resolver, @@ -541,6 +556,7 @@ def verify_native(source: Any, selected: Any) -> bool: authority_scope=_AUTHORITY_SCOPE, core_observer=observe_core, trusted_clock=lambda: payload["auth_context"]["trusted_host_now"], + commit_clock=NativeCommitClock.controlled(int(payload["auth_context"]["trusted_host_now"])), ) # Install the vector's pre-existing authority facts as trusted test setup. # The production seed path may restore these claims, but cannot issue them. @@ -581,6 +597,43 @@ def verify_native(source: Any, selected: Any) -> bool: fault = payload["fault"] response: dict[str, Any] | None = None caller_kind = "completed" + retry_evidence = None + original_record = next( + ( + record + for record in journal["effect_records"] + if record["effect_id"] == arguments.get("effect_id") + ), + None, + ) + retry_decision = ( + operation == "submit_result" and arguments["outcome_kind"] == "retryable_failure" + ) or ( + operation == "claim" + and original_record is not None + and original_record["attempt_fence"] != "0" + ) + if retry_decision and configuration["destination_deduplication_proven"]: + _destination_call(_AUTHORITY_SCOPE, arguments["effect_id"], None) + _destination_call(_AUTHORITY_SCOPE, arguments["effect_id"], None) + retry_evidence = { + **SQLiteCommittedEffectHost._retry_context(before, original_record), + **_destination_proof(_AUTHORITY_SCOPE, arguments["effect_id"]), + } + if fault == "retry_safety_missing": + retry_evidence = None + elif fault == "retry_safety_boolean": + retry_evidence = True + elif fault == "retry_safety_fabricated": + retry_evidence["first_attempt_receipt_bytes_base64"] = retry_evidence[ + "repeat_attempt_receipt_bytes_base64" + ] = base64.b64encode(b"fabricated equal receipts").decode() + elif fault == "retry_safety_wrong_work": + retry_evidence["effect_id"] = "sha256:" + "0" * 64 + elif fault == "retry_safety_wrong_destination": + retry_evidence["destination_binding_digest"] = "sha256:" + "0" * 64 + elif fault == "retry_safety_verifier_unavailable": + provider.retry_checker_available = False if operation in {"produce", "produce_replay"}: request = arguments["original_request"] aggregate = checkpoint["root_record"]["aggregate_state"] @@ -641,17 +694,6 @@ def before_commit(candidate: dict[str, Any], proposed_journal: dict[str, Any]) - caller_kind = "aborted" elif operation == "claim": try: - evidence = None - if configuration["destination_deduplication_proven"] and any( - record["effect_id"] == arguments["effect_id"] - and record["invocation_state"] == "ambiguous" - for record in journal["effect_records"] - ): - # Exercise the actual installed destination twice. The configuration - # requests this probe; only native receipt verification permits retry. - _destination_call(_AUTHORITY_SCOPE, arguments["effect_id"], None) - _destination_call(_AUTHORITY_SCOPE, arguments["effect_id"], None) - evidence = _destination_proof(_AUTHORITY_SCOPE, arguments["effect_id"]) claim = host.claim( _ROOT, arguments["effect_id"], @@ -659,7 +701,7 @@ def before_commit(candidate: dict[str, Any], proposed_journal: dict[str, Any]) - context["scope_authority_epoch"], expires_at=str(int(context["trusted_host_now"]) + 1), trusted_now=context["trusted_host_now"], - deduplication_evidence=evidence, + deduplication_evidence=retry_evidence, ) new_claims.append( { @@ -704,6 +746,7 @@ def before_commit(candidate: dict[str, Any], proposed_journal: dict[str, Any]) - scope=context["scope_identity"], epoch=context["scope_authority_epoch"], trusted_now=context["trusted_host_now"], + deduplication_evidence=retry_evidence, ) caller_kind = "response" if fault == "after_outcome_before_admission": @@ -800,6 +843,18 @@ def before_commit(candidate: dict[str, Any], proposed_journal: dict[str, Any]) - "claim_guard_observed": operation in {"claim", "dispatch", "submit_result"}, "native_transaction_id": str(uuid.uuid4()), "destination_call_evidence": destination_call_evidence, + "retry_safety_evidence": ( + copy.deepcopy(after["destination_evidence"][arguments["effect_id"]][-1]["evidence"]) + if retry_decision + and ( + operation == "claim" + and new_claims + or operation == "submit_result" + and response is not None + and response.get("status") == "report_recorded" + ) + else None + ), }, } diff --git a/conformance/pins.py b/conformance/pins.py index 0f9c49f..1b8856f 100644 --- a/conformance/pins.py +++ b/conformance/pins.py @@ -4,7 +4,7 @@ from pathlib import Path -CONFORMANCE_COMMIT = "c6949440f5b9fd6e06ea5571f4e01681dd59ee04" +CONFORMANCE_COMMIT = "090aa297389e416121c25ef29fb0a75671d83233" SPEC_COMMIT = "77c0a2e60cd0771a6d44ae170a079ddd51d7d9f0" ROOT = Path(__file__).resolve().parent.parent diff --git a/reviews/native-effect-creation-v030.md b/reviews/native-effect-creation-v030.md new file mode 100644 index 0000000..c55daca --- /dev/null +++ b/reviews/native-effect-creation-v030.md @@ -0,0 +1,96 @@ +# Native effect creation: bounded acceptance plan + +Status: first independent review requires shared creation/journal tests before +implementation. Python implementation has not started. + +This owner-local optional-host slice adds atomic creation to +`SQLiteCommittedEffectHost`. It does not change the portable core, add event +admission, implement a public transport endpoint, or start example integration. +Version 0.3.0 remains unreleased; schemas stay version 1 and machines `format: 1`. + +## Fixed inputs and prior evidence + +- Specification: `77c0a2e60cd0771a6d44ae170a079ddd51d7d9f0`, SPEC + §§17.3, 18 and 19.1–19.2, 19.5. +- Conformance: `090aa297389e416121c25ef29fb0a75671d83233`, reviewed draft + #114. The committed-native-effects generator exercises an existing admitted + event producing an intent; its 62 vectors do not prove native creation. +- Python baseline: `39def1ac8268de55237f90e5cb8d7110fe27fc78`. The + reviewed ownership boundary prevents ordinary adapters from bypassing native + ownership. `seed` restores existing artifacts; it is not atomic core creation. +- Rust baseline: `cc24d28d4f88790e1bdc046dad01867110dfc2be`. + `tests/native_handler.rs` already exercises actual creation, fresh trusted + source selection, retained replay, changed identity, and SIGKILL fate. Credit + that reviewed evidence; do not rewrite the existing implementation merely to + match Python's internal layout. + +## Contract and executable coverage + +| Obligation | Existing contract/tests | Required Python evidence | +|---|---|---| +| Actual core initialization produces revision 0, creation receipt and every initial external intent | §17.3; portable checkpoint creation cases | Real format-1 initialization with multiple effects; no hand-assembled candidate checkpoint | +| Checkpoint, all pinned effect records, exact original request/first response and native authority role evidence commit together | §§18, 19.1–19.2 | One SQLite transaction; reopened exact bytes and matching root/history inventory | +| Equal creation replay survives later checkpoint changes and unavailable executable source without a core/provider call | §§17.3, 19.2 | Retain original response; check request equality before resolver activation, route lookup or CAS; current authority authorization still applies | +| Changed creation ID, definition or typed bindings cannot reuse an existing root | §17.3 | Exact conflict, unchanged checkpoint/journal/authority bytes and zero core/provider calls | +| Trusted configured executable source and exact pinned route are checked before core and again at commit | §19.2; existing route-change vectors | Refuse untrusted/mismatched source, invalid mapping or changed route generation; post-staging source/route loss rolls back all native rows | +| Every initial intent has its own exact effect ID/digest, token, handler/destination/mapping and runtime incarnation | §§19.1–19.2 | Compare actual initial emissions to all journal records; support empty initial outbox; reject unsupported token/target combinations before commit | +| No hidden bypass through a competing helper-only or ordinary checkpoint row | §§18, 19.2; completed ownership tests | Concurrent root creation serializes; pre-existing ownership is rejected or replayed through retained native evidence, never overwritten | +| Process death preserves one atomic fate | §19.5 | Real child SIGKILL after SQL staging before commit, and after commit before response; reopen, compare bytes, retry exactly once without dispatch | + +The existing shared artifacts specify portable creation and journal semantics +separately; they do not execute their coupling at initialization. Add a shared +executable native-creation case before Python implementation: input is the actual +trusted definition, machine/root/creation identities, typed bindings and configured +route; observed output contains the exact actual creation checkpoint, receipt, +all initial external intents and journal pins, and retained first response. The +runner forwards inputs only. Independent semantic/hash checks must validate the +coupling, rather than accepting a copied expected result. + +The additional acceptance cases must distinguish committed faulted initialization +at revision 0 from rejection before any aggregate exists. A rejection reserves no +root and permits a later valid creation. Preserve creation receipt sequence 0, +initialization lifecycle receipts in defined order, and internal mailbox emissions, +in addition to all initial external effects. Creation identity uses §17.3 normalized +typed-binding digest semantics, including numeric types and defaults, rather than +ad hoc raw request equality. Equal replay and conflict also apply after tombstoning. + +Machine-visible operation tokens must originate in declared bindings/variables and +emitted data. A host-only token may instead use the §19.1 derivation, without +claiming that the machine already knows it. Do not interchange a separate optional +owner-local operation identity with the creation identity or imply §25 transport +semantics. The exact owner-local response contract must be fixed by these cases. + +Fresh creation requires active authorized native authority and permanent allocation +evidence. It must not activate seed/imported artifacts, repair history gaps, or +overwrite pre-existing checkpoint/helper ownership. Recheck source and authorization +at the actual commit boundary. Creation makes zero SDK handler calls; independently +count zero core/provider calls on replay and conflict. + +Language-native transaction, source-race and SIGKILL tests remain additional +required implementation evidence. The existing approved prose suffices; no new +portable behavior, grammar or schema version is proposed. + +## Ordered checkpoints and reviews + +1. Independently review this contract/test mapping before source changes. +2. Add and independently review the shared executable creation/journal cases, + deterministic generator, semantic validator and input-only runner. Push this + conformance checkpoint before Python source changes. Then add meaningful + failing tests against the unchanged Python baseline, using the + actual configured authority and handler fixtures. Record RED separately from + setup failures. Commit and ordinary-push the explicitly unfinished checkpoint; + verify the remote SHA. +3. Implement creation only. Freeze each meaningful checkpoint for focused checks + and independent review; commit and ordinary-push corrections. Keep PR #111 draft. +4. At the final clean Python SHA run static checks, full unit/conformance tests, + actual configured 62-vector effects/authority gate, actual PostgreSQL checks, + and independent review. Parent gates are not current-head evidence. +5. Review Rust parity against that acceptance set. Reuse exact-head existing + evidence where unchanged; run focused additional tests for any new case, and + review/gate any correction separately. No parallel Cargo runs. +6. Mark creation complete in the cross-repository audit only after these proofs. + Choose event admission as its own next slice. Examples and publication remain + blocked on their wider dependencies. + +No force pushes, main writes, tags, package publication, release dispatch or +deployment are part of these checkpoints. diff --git a/scripts/sync_schema.py b/scripts/sync_schema.py index 28250cb..c17377a 100644 --- a/scripts/sync_schema.py +++ b/scripts/sync_schema.py @@ -53,6 +53,7 @@ "runtime-action-output-v1.schema.json", "runtime-provider-descriptor-v1.schema.json", "timer-helper-operation-v1.schema.json", + "timer-record-v1.schema.json", ) SPEC_COMMIT = "77c0a2e60cd0771a6d44ae170a079ddd51d7d9f0" diff --git a/src/determa/state/authority.py b/src/determa/state/authority.py index 850daf7..3ab1e4a 100644 --- a/src/determa/state/authority.py +++ b/src/determa/state/authority.py @@ -313,18 +313,27 @@ def _native_checkpoints( "ORDER BY root_instance_id" ) ) + from .effects import _native_effect_documents, validate_journal + + native_documents = _native_effect_documents(ledger) + if native_documents and "determa_committed_effects" not in tables: + raise ValueError("native effect participant table missing") if "determa_committed_effects" in tables: - from .effects import validate_journal from .wire import canonical_bytes - for root, payload in connection.execute( + native_rows = connection.execute( "SELECT root_instance_id, document FROM determa_committed_effects " "ORDER BY root_instance_id" - ): + ).fetchall() + if {root for root, _payload in native_rows} != set(native_documents): + raise ValueError("native effect participant inventory differs") + for root, payload in native_rows: native = _parse(bytes(payload).decode("utf-8", "strict")) validate_journal(native["checkpoint"], native["journal"]) if native["journal"]["scope_identity"] != ledger["scope_identity"]: raise ValueError("untracked native effect scope") + if native != native_documents[root]: + raise ValueError("native effect private history differs") for record in native["journal"]["effect_records"]: if { "work_identity": record["effect_id"], @@ -1049,6 +1058,21 @@ def _perform( and invocation["authenticated_principal"] != ledger["owner_principal"] ): return _compact(_result(request, ledger, "stale_scope_authority")) + # A new mutation cannot certify a scope around a missing private + # native participant. Historical receipt replay above grants no rights. + if any( + key in ledger + for key in ( + "native_effect_roots", + "native_effect_document_bytes", + "native_effect_journal_bytes", + "native_effect_work", + ) + ): + try: + _native_checkpoints(connection, ledger) + except (ValueError, TypeError, KeyError, sqlite3.Error): + return _compact(_result(request, ledger, "scope_fence_unproven")) new_claim = None if operation == "fence_worker": arguments = request["arguments"] @@ -1396,9 +1420,18 @@ def transaction(self, root_instance_id: str) -> Iterator[ExecutionStoreTransacti if root_instance_id not in ledger["roots"]: raise ExecutionStoreError("unauthorized_scope") try: + from .effects import _native_effect_documents + native_history = _native_checkpoint_history(ledger) - except ValueError as error: + native_effect_roots = set(_native_effect_documents(ledger)) + _native_checkpoints(connection, ledger) + except (ValueError, TypeError, KeyError, sqlite3.Error) as error: raise ExecutionStoreError("scope_fence_unproven") from error + # Checkpoint-only execution cannot enter a joint checkpoint/journal + # participant, including reads through this write transaction API. + # Refuse before core actions or an application callback can run. + if root_instance_id in native_effect_roots: + raise ExecutionStoreError("scope_fence_unproven") transaction = _SQLiteTransaction(connection, root_instance_id) previous = transaction.load() if previous is not None and previous.decode("utf-8", "strict") not in native_history: diff --git a/src/determa/state/data/timer-record-v1.schema.json b/src/determa/state/data/timer-record-v1.schema.json new file mode 100644 index 0000000..06a7f57 --- /dev/null +++ b/src/determa/state/data/timer-record-v1.schema.json @@ -0,0 +1,177 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://determa.dev/state/schema/timer-record-v1.schema.json", + "title": "Separate portable timer records artifact version 1", + "type": "object", + "properties": { + "timer_artifact_format": { + "const": "determa.timer_records" + }, + "timer_artifact_schema_version": { + "const": 1 + }, + "records": { + "type": "array", + "items": { + "type": "object", + "properties": { + "scope_identity": { + "type": "string", + "minLength": 1 + }, + "root_instance_id": { + "type": "string", + "minLength": 1 + }, + "root_runtime_id": { + "type": "string", + "minLength": 1 + }, + "timer_id": { + "type": "string", + "minLength": 1 + }, + "schedule_request_digest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "clock_basis": { + "const": "unix_nanoseconds" + }, + "deadline_at": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*|-[1-9][0-9]*)$" + }, + "event_name": { + "type": "string", + "minLength": 1 + }, + "payload": { + "$ref": "aggregate-state-v1.schema.json#/$defs/typedMap" + }, + "correlation_id": { + "oneOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "null" + } + ] + }, + "event_id": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "state": { + "enum": [ + "pending", + "claimed", + "fired", + "cancelled" + ] + }, + "revision": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)$" + }, + "attempt_fence": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)$" + }, + "worker_principal": { + "oneOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "null" + } + ] + }, + "expires_at": { + "oneOf": [ + { + "type": "string", + "pattern": "^(0|[1-9][0-9]*|-[1-9][0-9]*)$" + }, + { + "type": "null" + } + ] + }, + "admission_receipt_digest": { + "oneOf": [ + { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "scope_identity", + "root_instance_id", + "root_runtime_id", + "timer_id", + "schedule_request_digest", + "clock_basis", + "deadline_at", + "event_name", + "payload", + "correlation_id", + "event_id", + "state", + "revision", + "attempt_fence", + "worker_principal", + "expires_at", + "admission_receipt_digest" + ], + "additionalProperties": false + } + }, + "operation_receipts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "operation_id": { + "type": "string", + "minLength": 1 + }, + "request_digest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "result": { + "$ref": "timer-helper-operation-v1.schema.json#/$defs/result" + } + }, + "required": [ + "operation_id", + "request_digest", + "result" + ], + "additionalProperties": false + } + }, + "timer_artifact_digest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + } + }, + "required": [ + "timer_artifact_format", + "timer_artifact_schema_version", + "records", + "operation_receipts", + "timer_artifact_digest" + ], + "additionalProperties": false +} diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 9691ce7..8a80849 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -27,6 +27,7 @@ from .errors import ArtifactError from .extensions import ConfiguredExtension, ExtensionError, ExtensionRegistry from .host import outbox_intent_digest +from .native_clock import NativeCommitClock, _native_clock_mode, _read_native_clock from .wire import ( ArtifactResolver, _schema_registry, @@ -82,12 +83,21 @@ def _journal_validator() -> Any: def _validate_request(request: Mapping[str, Any], name: str) -> dict[str, Any]: from jsonschema import Draft202012Validator + from referencing import Resource if not isinstance(request, Mapping): raise EffectError("invalid_host_request") normalized = copy.deepcopy(dict(request)) source = json.loads((Path(__file__).parent / "data" / name).read_text()) - if not Draft202012Validator(source, registry=_schema_registry()).is_valid(normalized): + registry = _schema_registry() + for filename in ( + "host-effect-journal-v1.schema.json", + "provider-reference-v1.schema.json", + name, + ): + schema = json.loads((Path(__file__).parent / "data" / filename).read_text()) + registry = registry.with_resource(schema["$id"], Resource.from_contents(schema)) + if not Draft202012Validator(source, registry=registry).is_valid(normalized): raise EffectError("invalid_host_request") return normalized @@ -190,6 +200,13 @@ def _pinned_result_target( machine = definition["machine"] identity = [machine["namespace"], machine["machine_id"], machine["machine_version"]] if origin["kind"] == "root": + root_record = checkpoint["root_record"] + aggregate = root_record.get("aggregate_state") + actual_root_runtime_id = ( + aggregate["root_runtime_id"] if aggregate else root_record["root_runtime_id"] + ) + if origin["root_instance_id"] != root or runtime_id != actual_root_runtime_id: + raise EffectError("invalid_effect_journal") expected_id = hash_value( [ "determa-root-runtime-identity-1", @@ -308,6 +325,13 @@ def validate_journal( for item in checkpoint["pending_outbox_intents"] + checkpoint["terminal_outbox_records"] ] + [item["effect_id"] for item in checkpoint["outbox_effect_tombstones"]] for record in records: + # Validate every pinned incarnation before dispatch or native seeding, + # including nonterminal work. A self-consistent alternate trusted origin + # cannot replace the checkpoint's immutable root runtime identity. + try: + pinned_target = _pinned_result_target(checkpoint, record, resolver) + except (ArtifactError, KeyError, TypeError, ValueError) as exc: + raise EffectError("invalid_effect_journal") from exc intent = intents.get(record["effect_id"]) tombstone = compact.get(record["effect_id"]) if location_ids.count(record["effect_id"]) != 1 or record["intent_digest"] != ( @@ -318,9 +342,53 @@ def validate_journal( raise EffectError("invalid_effect_journal") reports = record["attempt_records"] fences = [int(item["attempt_fence"]) for item in reports] - if fences != sorted(set(fences)) or any(f > int(record["attempt_fence"]) for f in fences): + if fences != sorted(set(fences)) or any( + f == 0 or f > int(record["attempt_fence"]) for f in fences + ): raise EffectError("invalid_effect_journal") state = record["invocation_state"] + outcome = record["outcome"] + current_report = next( + (item for item in reports if item["attempt_fence"] == record["attempt_fence"]), + None, + ) + if state == "unclaimed" and record["attempt_fence"] != "0" and current_report is None: + # A consumed attempt cannot become new work without retaining the + # current safe-failure report. Missing evidence is not retry proof. + raise EffectError("invalid_effect_journal") + terminal_reports = [item for item in reports if item["report_kind"] in _OUTCOMES] + if len(terminal_reports) > 1: + # Terminal outcomes are immutable: no later attempt may supersede + # an earlier committed terminal report, even with valid hashes. + raise EffectError("invalid_effect_journal") + if state in {"leased", "ambiguous"} and record["attempt_fence"] == "0": + raise EffectError("invalid_effect_journal") + if state in {"unclaimed", "leased", "ambiguous"} and any( + item["report_kind"] in _OUTCOMES for item in reports + ): + # A committed terminal report cannot become retryable work merely + # because its outcome/result fields were stripped. + raise EffectError("invalid_effect_journal") + if current_report is not None and ( + state == "leased" + or (state == "unclaimed" and current_report["report_kind"] != "retryable_failure") + or (state == "ambiguous" and current_report["report_kind"] != "ambiguous") + ): + raise EffectError("invalid_effect_journal") + cancellation = record["cancellation"] + if ( + cancellation is not None + and cancellation["state"] == "prevented_start" + and ( + state not in {"outcome_recorded", "result_admitted", "closed"} + or record["attempt_fence"] != "0" + or reports + or outcome is None + or outcome["kind"] != "cancelled" + or outcome["attempt_fence"] != "0" + ) + ): + raise EffectError("invalid_effect_journal") if state in {"unclaimed", "leased", "ambiguous"} and any( record[key] is not None for key in ("outcome", "result_event_id", "admission_receipt") ): @@ -337,8 +405,9 @@ def validate_journal( or record["admission_receipt"] not in checkpoint["operation_receipts"] ): raise EffectError("invalid_effect_journal") - outcome = record["outcome"] if outcome is not None: + if outcome["attempt_fence"] != record["attempt_fence"]: + raise EffectError("invalid_effect_journal") if outcome["digest"] != hash_value( [ "determa-effect-outcome-1", @@ -373,7 +442,7 @@ def validate_journal( record, outcome["kind"], outcome["payload"], - _pinned_result_target(checkpoint, record, resolver), + pinned_target, ) receipt = record["admission_receipt"] if ( @@ -459,7 +528,7 @@ def _rejected_result(effect_id: str, fence: str, code: str) -> dict[str, Any]: def _report_reason(request: Mapping[str, Any]) -> str | None: kind = request["outcome_kind"] if kind == "retryable_failure": - return "no_call_proven" + return "destination_deduplication_proven" if kind == "ambiguous": return "provider_acceptance_unknown" return None @@ -470,22 +539,24 @@ def _retained_report( ) -> dict[str, Any] | None: """Compare complete immutable report evidence even after a process restart.""" fence, kind = request["attempt_fence"], request["outcome_kind"] - digest = hash_value( - [ - "determa-effect-attempt-report-1", - record["effect_id"], - record["operation_token"], - fence, - kind, - request["payload"], - _report_reason(request), - ] - ) return next( ( item for item in record["attempt_records"] - if item["attempt_fence"] == fence and item["report_digest"] == digest + if item["attempt_fence"] == fence + and item["report_kind"] == kind + and item["report_digest"] + == hash_value( + [ + "determa-effect-attempt-report-1", + record["effect_id"], + record["operation_token"], + fence, + kind, + request["payload"], + item["reason"], + ] + ) ), None, ) @@ -627,6 +698,119 @@ def visit(value: Any, pointer: str) -> None: return response +def _native_effect_journals(ledger: Mapping[str, Any]) -> dict[str, dict[str, Any]]: + """Resolve only host-participant history, never generic proposed mutation bytes.""" + from .authority import _parse + + history = ledger.get("native_effect_journal_bytes", []) + if type(history) is not list or any(type(source) is not str for source in history): + raise EffectError("unauthorized_scope") + latest = {} + for source in history: + try: + journal = _parse(source) + if type(journal) is not dict: + raise EffectError("unauthorized_scope") + root = journal["root_instance_id"] + if ( + canonical_bytes(journal).decode() != source + or journal.get("host_effect_journal_format") != "determa.host_effect_journal" + or journal.get("host_effect_journal_schema_version") != 1 + or journal.get("scope_identity") != ledger["scope_identity"] + or journal.get("host_effect_journal_digest") != journal_digest(journal) + or type(root) is not str + or not root + ): + raise EffectError("unauthorized_scope") + latest[root] = journal + except (ValueError, TypeError, KeyError) as error: + raise EffectError("unauthorized_scope") from error + return latest + + +def _native_effect_journal(ledger: Mapping[str, Any], root: str) -> dict[str, Any]: + latest = _native_effect_journals(ledger).get(root) + if latest is None: + raise EffectError("unauthorized_scope") + return latest + + +def _native_effect_documents(ledger: Mapping[str, Any]) -> dict[str, dict[str, Any]]: + """Only independently retained native-role bytes certify private call history.""" + roots = ledger.get("native_effect_roots", []) + if ( + not isinstance(roots, list) + or any(type(root) is not str or not root for root in roots) + or len(set(roots)) != len(roots) + ): + raise EffectError("unauthorized_scope") + journals = _native_effect_journals(ledger) + # Losing both indexes cannot turn retained native journal/work into freshness. + if set(journals) != set(roots): + raise EffectError("unauthorized_scope") + work = ledger.get("native_effect_work", []) + if type(work) is not list or any( + type(item) is not dict + or item.get("participant") != "native_effects" + or item.get("work_kind") != "effect" + or item.get("root_instance_id") not in roots + for item in work + ): + raise EffectError("unauthorized_scope") + history = ledger.get("native_effect_document_bytes", [] if not roots else None) + if not isinstance(history, list): + raise EffectError("unauthorized_scope") + latest = {} + for source in history: + if type(source) is not str: + raise EffectError("unauthorized_scope") + try: + value = json.loads(source) + if not isinstance(value, dict) or canonical_bytes(value).decode() != source: + raise ValueError("invalid native participant bytes") + if ( + value["journal"]["scope_identity"] != ledger["scope_identity"] + or value["journal"]["host_effect_journal_format"] != "determa.host_effect_journal" + or value["journal"]["host_effect_journal_schema_version"] != 1 + or value["journal"]["host_effect_journal_digest"] + != journal_digest(value["journal"]) + ): + raise ValueError("wrong native journal provenance") + if not isinstance(value["invocation_starts"], dict): + raise ValueError("missing private start inventory") + root = value["checkpoint"]["root_instance_id"] + if type(root) is not str or not root or value["journal"]["root_instance_id"] != root: + raise ValueError("wrong native root") + latest[root] = value + except (ValueError, TypeError, KeyError) as error: + raise EffectError("unauthorized_scope") from error + if set(latest) != set(roots) or any( + document["journal"] != journals[root] for root, document in latest.items() + ): + raise EffectError("unauthorized_scope") + expected_work = [ + { + "root_instance_id": root, + "work_kind": "effect", + "work_identity": record["effect_id"], + "operation_token": record["operation_token"], + "participant": "native_effects", + } + for root, document in latest.items() + for record in document["journal"]["effect_records"] + ] + if len(work) != len(expected_work) or any(item not in work for item in expected_work): + raise EffectError("unauthorized_scope") + return latest + + +def _native_effect_document(ledger: Mapping[str, Any], root: str) -> dict[str, Any]: + latest = _native_effect_documents(ledger).get(root) + if latest is None: + raise EffectError("unauthorized_scope") + return latest + + def _validate_authority_pair(ledger: Mapping[str, Any], document: Mapping[str, Any]) -> None: """Check the old native pair before any journal transition can be staged.""" from .authority import _native_checkpoint_history @@ -681,6 +865,10 @@ def _validate_authority_pair(ledger: Mapping[str, Any], document: Mapping[str, A raise EffectError("stale_attempt_fence") elif live is not None: raise EffectError("stale_attempt_fence") + if journal != _native_effect_journal(ledger, journal["root_instance_id"]): + raise EffectError("unauthorized_scope") + if document != _native_effect_document(ledger, journal["root_instance_id"]): + raise EffectError("unauthorized_scope") def _issue_effect_claim( @@ -700,7 +888,7 @@ def _issue_effect_claim( record = _record(journal, effect_id) if record["invocation_state"] not in {"unclaimed", "ambiguous"}: raise EffectError("effect_not_outstanding") - if record["invocation_state"] == "ambiguous" and not ( + if record["attempt_fence"] != "0" and not ( deduplication_proven and record["idempotency_policy"] == "destination_deduplicates" ): raise EffectError("effect_not_outstanding") @@ -798,11 +986,28 @@ def invoke( payload: Any, metadata: Mapping[str, Any], attempt: Mapping[str, Any], + ) -> Mapping[str, Any]: + return VerifiedNativeHandler._invoke_with_guard( + self, reference, payload, metadata, attempt, lambda: None + ) + + def _invoke_with_guard( + self, + reference: Mapping[str, Any], + payload: Any, + metadata: Mapping[str, Any], + attempt: Mapping[str, Any], + guard: Callable[[], None], ) -> Mapping[str, Any]: method, instance = VerifiedNativeHandler.verify( self, reference, metadata["destination_binding_digest"] ) - return cast(Mapping[str, Any], method(instance, payload, metadata, attempt)) + # Provider verification may execute native callbacks. Live host rights and + # trusted time must be checked after those callbacks, immediately before I/O. + guard() + result = method(instance, payload, metadata, attempt) + VerifiedNativeHandler.verify(self, reference, metadata["destination_binding_digest"]) + return cast(Mapping[str, Any], result) class SQLiteCommittedEffectHost: @@ -824,6 +1029,7 @@ def __init__( authority_scope: str | None = None, core_observer: Callable[[str, str, Mapping[str, Any]], None] | None = None, trusted_clock: Callable[[], str] | None = None, + commit_clock: NativeCommitClock | None = None, ) -> None: if handler is not None and type(handler) is not VerifiedNativeHandler: raise EffectError("host_capability_mismatch") @@ -835,6 +1041,41 @@ def __init__( self.authority_scope = authority_scope self.core_observer = core_observer self.trusted_clock = trusted_clock + if commit_clock is not None and type(commit_clock) is not NativeCommitClock: + raise EffectError("host_capability_mismatch") + self.commit_clock = commit_clock + self._installed_commit_clock = commit_clock + try: + self._installed_commit_clock_mode = ( + _native_clock_mode(commit_clock) if commit_clock is not None else None + ) + except ValueError as exc: + raise EffectError("host_capability_mismatch") from exc + + def _retry_commit_clock(self) -> tuple[NativeCommitClock, int, bool]: + clock = self.commit_clock + if type(clock) is not NativeCommitClock or clock is not self._installed_commit_clock: + raise EffectError("host_capability_mismatch") + try: + mode = self._installed_commit_clock_mode + if type(mode) is not bool: + raise ValueError("native clock mode absent") + return clock, _read_native_clock(clock, mode=mode), mode + except ValueError as exc: + raise EffectError("stale_attempt_fence") from exc + + def _check_retry_commit_clock( + self, guard: tuple[NativeCommitClock, int, bool], deadline: int + ) -> None: + clock, start, mode = guard + if clock is not self.commit_clock or clock is not self._installed_commit_clock: + raise EffectError("host_capability_mismatch") + try: + now = _read_native_clock(clock, mode=mode) + except ValueError as exc: + raise EffectError("stale_attempt_fence") from exc + if now < start or now >= deadline: + raise EffectError("stale_attempt_fence") def _verified_handler( self, reference: Mapping[str, Any], destination: str @@ -847,6 +1088,56 @@ def _verified_handler( VerifiedNativeHandler.verify(self.handler, reference, destination) return self.handler + def _verify_retry_evidence( + self, context: Mapping[str, Any], evidence: Mapping[str, Any] | None + ) -> dict[str, Any]: + """Bind a native receipt decision to the original committed invocation.""" + receipt_fields = { + "first_attempt_receipt_bytes_base64", + "repeat_attempt_receipt_bytes_base64", + } + if not isinstance(evidence, Mapping) or set(evidence) != set(context) | receipt_fields: + raise EffectError("host_capability_mismatch") + proof = copy.deepcopy(dict(evidence)) + if any(proof[key] != value for key, value in context.items()): + raise EffectError("host_capability_mismatch") + if any(type(proof[field]) is not str for field in receipt_fields): + raise EffectError("host_capability_mismatch") + try: + first = base64.b64decode(proof["first_attempt_receipt_bytes_base64"], validate=True) + repeated = base64.b64decode(proof["repeat_attempt_receipt_bytes_base64"], validate=True) + if not first or first != repeated: + raise ValueError("destination receipts differ") + except (ValueError, TypeError) as exc: + raise EffectError("host_capability_mismatch") from exc + handler = SQLiteCommittedEffectHost._verified_handler( + self, context["handler_reference"], context["destination_binding_digest"] + ) + VerifiedNativeHandler.verify_deduplication_evidence( + handler, context["handler_reference"], context["destination_binding_digest"], proof + ) + return proof + + @staticmethod + def _retry_context(document: Mapping[str, Any], record: Mapping[str, Any]) -> dict[str, Any]: + if record["idempotency_policy"] != "destination_deduplicates": + raise EffectError("host_capability_mismatch") + return { + "kind": "destination_deduplication", + "scope_identity": document["journal"]["scope_identity"], + "root_instance_id": document["journal"]["root_instance_id"], + **{ + key: copy.deepcopy(record[key]) + for key in ( + "effect_id", + "operation_token", + "attempt_fence", + "handler_reference", + "destination_binding_digest", + ) + }, + } + def _connect(self) -> sqlite3.Connection: connection = sqlite3.connect(self.path, isolation_level=None, timeout=30) connection.execute("PRAGMA journal_mode=WAL") @@ -877,6 +1168,7 @@ def seed( "result_responses": {}, "cancel_requests": {}, "producer_requests": {}, + "invocation_starts": {}, } if claim is not None: document["claims"][claim["work_identity"]] = copy.deepcopy(dict(claim)) @@ -904,6 +1196,23 @@ def seed( or journal["scope_identity"] != self.authority_scope ): raise EffectError("unauthorized_scope") + _native_effect_documents(ledger) + if checkpoint["root_instance_id"] in ledger.get( + "native_effect_roots", [] + ) and journal != _native_effect_journal(ledger, checkpoint["root_instance_id"]): + raise EffectError("unauthorized_scope") + if checkpoint["root_instance_id"] in ledger.get("native_effect_roots", []): + retained = _native_effect_document(ledger, checkpoint["root_instance_id"]) + # Portable equality never restores private call permission. + # Existing starts require explicit native recovery, not seed. + if retained["invocation_starts"] or retained["checkpoint"] != checkpoint: + raise EffectError("unauthorized_scope") + if ( + claim is not None + and retained["claims"].get(claim["work_identity"]) != claim + ): + raise EffectError("stale_attempt_fence") + document = copy.deepcopy(retained) # Restoring an already claimed invocation cannot mint authority. # Its exact claim must already have been issued by this ledger. if claim is not None and ( @@ -955,6 +1264,10 @@ def seed( ): raise EffectError("stale_attempt_fence") self._mirror_authority(ledger, document) + connection.execute( + "UPDATE determa_committed_effects SET document=? WHERE root_instance_id=?", + (canonical_bytes(document), checkpoint["root_instance_id"]), + ) connection.execute( "UPDATE determa_scope_authority SET ledger = ? WHERE scope_identity = ?", ( @@ -969,8 +1282,12 @@ def _mirror_authority( ledger: dict[str, Any], document: dict[str, Any], *, advance_generation: bool = True ) -> None: """Mirror effect fences and exact committed bytes in the colocated authority ledger.""" + _native_effect_documents(ledger) native_roots = ledger.setdefault("native_effect_roots", []) root = document["checkpoint"]["root_instance_id"] + fresh = root not in native_roots + if not fresh: + _native_effect_document(ledger, root) if root not in native_roots: native_roots.append(root) records = document["journal"]["effect_records"] @@ -1026,6 +1343,14 @@ def _mirror_authority( history.append(source) ledger["checkpoint_bytes"].append(source) ledger["mutation_bytes"].append(canonical_bytes(document["journal"]).decode()) + # This private native role history cannot be fabricated by a generic + # guarded_commit payload that happens to resemble a journal. + ledger.setdefault("native_effect_journal_bytes", []).append( + canonical_bytes(document["journal"]).decode() + ) + ledger.setdefault("native_effect_document_bytes", []).append( + canonical_bytes(document).decode() + ) if advance_generation: ledger["scope_generation"] = str(int(ledger["scope_generation"]) + 1) @@ -1050,6 +1375,7 @@ def _transact( expected_epoch: str | None = None, authority_mutation: Callable[[dict[str, Any], dict[str, Any]], None] | None = None, worker_guard: tuple[str, str, str | None, str] | None = None, + before_native_commit: Callable[[], None] | None = None, ) -> _T: with self._connect() as connection: connection.execute("BEGIN IMMEDIATE") @@ -1095,6 +1421,12 @@ def _transact( ): raise EffectError("stale_scope_authority") _validate_authority_pair(ledger, document) + from .authority import _native_checkpoints + + try: + _native_checkpoints(connection, ledger) + except (ValueError, TypeError, KeyError, sqlite3.Error) as error: + raise EffectError("unauthorized_scope") from error if worker_guard is not None: effect_id, principal, fence, trusted_now = worker_guard journal_record = _record(document["journal"], effect_id) @@ -1153,9 +1485,10 @@ def _transact( (canonical_bytes(document), root), ) if expiry_guard is not None: - # Keep the original deadline even when this transaction closes - # or revokes the claim. Worker rights must still hold at commit. + # Arbitrary legacy clock callbacks run before native proof. self._check_live_clock(*expiry_guard) + if before_native_commit is not None: + before_native_commit() connection.commit() return value @@ -1315,47 +1648,32 @@ def claim( trusted_now: str, deduplication_evidence: Mapping[str, Any] | None = None, ) -> dict[str, Any]: + final_proof: tuple[dict[str, Any], dict[str, Any]] | None = None + clock_guard: tuple[NativeCommitClock, int, bool] | None = None + + def verify_before_commit() -> None: + if self.trusted_clock is None: + raise EffectError("stale_attempt_fence") + self._check_live_clock(self.trusted_clock, _now(expires_at)) + if final_proof is not None: + context, proof = final_proof + SQLiteCommittedEffectHost._verify_retry_evidence(self, context, proof) + assert clock_guard is not None + SQLiteCommittedEffectHost._check_retry_commit_clock( + self, clock_guard, _now(expires_at) + ) + def change(document: dict[str, Any]) -> dict[str, Any]: + nonlocal final_proof, clock_guard record = _record(document["journal"], effect_id) proof = None - if deduplication_evidence is not None: - if not isinstance(deduplication_evidence, Mapping): - raise EffectError("host_capability_mismatch") - proof = copy.deepcopy(dict(deduplication_evidence)) - fields = { - "scope_identity", - "effect_id", - "destination_binding_digest", - "first_attempt_receipt_bytes_base64", - "repeat_attempt_receipt_bytes_base64", - } - if ( - set(proof) != fields - or proof["scope_identity"] != document["journal"]["scope_identity"] - or proof["effect_id"] != effect_id - or proof["destination_binding_digest"] != record["destination_binding_digest"] - ): - raise EffectError("host_capability_mismatch") - try: - first = base64.b64decode( - proof["first_attempt_receipt_bytes_base64"], validate=True - ) - repeated = base64.b64decode( - proof["repeat_attempt_receipt_bytes_base64"], validate=True - ) - if not first or first != repeated: - raise ValueError("destination receipts differ") - except (ValueError, TypeError) as exc: - raise EffectError("host_capability_mismatch") from exc - handler = self._verified_handler( - record["handler_reference"], record["destination_binding_digest"] - ) - VerifiedNativeHandler.verify_deduplication_evidence( - handler, - record["handler_reference"], - record["destination_binding_digest"], - proof, + if record["attempt_fence"] != "0" or deduplication_evidence is not None: + context = SQLiteCommittedEffectHost._retry_context(document, record) + clock_guard = SQLiteCommittedEffectHost._retry_commit_clock(self) + proof = SQLiteCommittedEffectHost._verify_retry_evidence( + self, context, deduplication_evidence ) + final_proof = (context, proof) claim = _issue_effect_claim( document, effect_id, @@ -1368,6 +1686,7 @@ def change(document: dict[str, Any]) -> dict[str, Any]: if proof is not None: document.setdefault("destination_evidence", {}).setdefault(effect_id, []).append( { + "decision": "claim", "root_instance_id": root, "operation_token": record["operation_token"], "handler_reference": copy.deepcopy(record["handler_reference"]), @@ -1378,7 +1697,11 @@ def change(document: dict[str, Any]) -> dict[str, Any]: return claim return self._transact( - root, change, expected_epoch=epoch, authority_mutation=self._mirror_authority + root, + change, + expected_epoch=epoch, + authority_mutation=self._mirror_authority, + before_native_commit=verify_before_commit, ) def terminalize_outbox( @@ -1480,8 +1803,73 @@ def dispatch( if clock is None: raise EffectError("stale_attempt_fence") + def start(document: dict[str, Any]) -> str: + record = _record(document["journal"], effect_id) + if record["invocation_state"] != "leased": + raise EffectError("effect_not_outstanding") + self._authorize( + document, + effect_id, + record["operation_token"], + record["attempt_fence"], + principal, + scope, + epoch, + clock(), + ) + if ( + not authorized + or credential is None + or record["handler_reference"] != self.route["handler_reference"] + or record["destination_binding_digest"] != self.route["destination_binding_digest"] + ): + raise EffectError("unauthorized_scope") + self._verified_handler( + record["handler_reference"], record["destination_binding_digest"] + ) + identifier = hash_value( + [ + "determa-native-invocation-start-1", + scope, + root, + effect_id, + record["attempt_fence"], + ] + ) + starts = document["invocation_starts"] + if identifier in starts: + raise EffectError("native_invocation_already_started") + intent = next( + item["intent"] + for item in document["checkpoint"]["pending_outbox_intents"] + + document["checkpoint"]["terminal_outbox_records"] + if item["intent"]["effect_id"] == effect_id + ) + starts[identifier] = { + "claim": copy.deepcopy(document["claims"][effect_id]), + "intent": copy.deepcopy(intent), + "effect_record": copy.deepcopy(record), + } + return identifier + + # Only this successful new native commit reaches the call phase. A saved + # marker is historical evidence, never permission for another invocation. + start_id = self._transact( + root, + start, + expected_epoch=epoch, + authority_mutation=self._mirror_authority, + worker_guard=(effect_id, principal, None, trusted_now), + ) + def call(document: dict[str, Any]) -> Mapping[str, Any]: record = _record(document["journal"], effect_id) + retained_start = document["invocation_starts"].get(start_id) + if not isinstance(retained_start, dict) or ( + retained_start.get("claim") != document["claims"].get(effect_id) + or retained_start.get("effect_record") != record + ): + raise EffectError("invalid_effect_journal") if record["invocation_state"] != "leased": raise EffectError("effect_not_outstanding") self._authorize( @@ -1509,6 +1897,8 @@ def call(document: dict[str, Any]) -> Mapping[str, Any]: ) if item["intent"]["effect_id"] == effect_id ) + if retained_start.get("intent") != intent: + raise EffectError("invalid_effect_journal") metadata = { "scope_identity": scope, "effect_id": effect_id, @@ -1521,12 +1911,26 @@ def call(document: dict[str, Any]) -> Mapping[str, Any]: handler = self._verified_handler( record["handler_reference"], record["destination_binding_digest"] ) - result = VerifiedNativeHandler.invoke( + + def live_call_guard() -> None: + self._authorize( + document, + effect_id, + record["operation_token"], + record["attempt_fence"], + principal, + scope, + epoch, + clock(), + ) + + result = VerifiedNativeHandler._invoke_with_guard( handler, record["handler_reference"], copy.deepcopy(intent["payload"]), metadata, {"attempt_fence": record["attempt_fence"]}, + live_call_guard, ) # External acceptance cannot extend an expired worker's mutation # rights. A refusal leaves its leased journal for reconciliation. @@ -1555,6 +1959,7 @@ def submit_result( scope: str, epoch: str, trusted_now: str, + deduplication_evidence: Mapping[str, Any] | None = None, ) -> dict[str, Any]: if ( not isinstance(request, Mapping) @@ -1574,7 +1979,10 @@ def submit_result( scope=scope, epoch=epoch, trusted_now=trusted_now, + deduplication_evidence=deduplication_evidence, ) + if response["status"] == "committed": + return response if request["outcome_kind"] in _OUTCOMES: # The authenticated outcome is already durable. Admission is a # separate host-owned transaction; its failure cannot erase it. @@ -1607,16 +2015,151 @@ def _record_result( scope: str, epoch: str, trusted_now: str, + deduplication_evidence: Mapping[str, Any] | None = None, ) -> dict[str, Any]: """Private durable-outcome stage; never a completed public result.""" + saved = self._saved_result_response(root, request, principal, scope, epoch) + if saved is not None: + return saved + final_proof: tuple[dict[str, Any], dict[str, Any]] | None = None + clock_guard: tuple[NativeCommitClock, int, bool] | None = None + deadline: int | None = None + + def change(document: dict[str, Any]) -> dict[str, Any]: + nonlocal final_proof, clock_guard, deadline + key = request["effect_id"] + ":" + request["attempt_fence"] + fresh_retry = ( + request["outcome_kind"] == "retryable_failure" + and key not in document["result_requests"] + ) + proof = None + if fresh_retry: + record = self._authorize( + document, + request["effect_id"], + request["operation_token"], + request["attempt_fence"], + principal, + scope, + epoch, + trusted_now, + ) + context = SQLiteCommittedEffectHost._retry_context(document, record) + clock_guard = SQLiteCommittedEffectHost._retry_commit_clock(self) + deadline = _now(document["claims"][request["effect_id"]]["expires_at"]) + proof = SQLiteCommittedEffectHost._verify_retry_evidence( + self, context, deduplication_evidence + ) + final_proof = (context, proof) + response = self._submit( + document, + request, + principal, + scope, + epoch, + trusted_now, + retry_verified=proof is not None, + ) + if proof is not None: + document["result_requests"][key] = copy.deepcopy(dict(request)) + document.setdefault("result_responses", {})[key] = copy.deepcopy(response) + document.setdefault("result_disclosure", {})[key] = { + "principal": principal, + "scope": scope, + "epoch": epoch, + } + document.setdefault("destination_evidence", {}).setdefault( + request["effect_id"], [] + ).append( + { + "decision": "report", + "attempt_fence": request["attempt_fence"], + "evidence": proof, + } + ) + return response + + def verify_before_commit() -> None: + if final_proof is not None: + SQLiteCommittedEffectHost._verify_retry_evidence(self, *final_proof) + assert clock_guard is not None and deadline is not None + SQLiteCommittedEffectHost._check_retry_commit_clock(self, clock_guard, deadline) + return self._transact( root, - lambda document: self._submit(document, request, principal, scope, epoch, trusted_now), + change, expected_epoch=epoch, authority_mutation=self._mirror_authority, worker_guard=(request["effect_id"], principal, request["attempt_fence"], trusted_now), + before_native_commit=verify_before_commit, ) + def _saved_result_response( + self, root: str, request: Mapping[str, Any], principal: str, scope: str, epoch: str + ) -> dict[str, Any] | None: + """Disclose retained native evidence without restoring today's executable. + + This read never grants a lease, invokes a verifier, or changes a receipt. + A retained request conflict must not fall through to a fresh decision. + """ + with self._connect() as connection: + connection.execute("BEGIN") + row = connection.execute( + "SELECT document FROM determa_committed_effects WHERE root_instance_id=?", (root,) + ).fetchone() + if row is None: + raise EffectError("wrong_root") + document = json.loads(row[0]) + key = request["effect_id"] + ":" + request["attempt_fence"] + prior = document["result_requests"].get(key) + response = document.get("result_responses", {}).get(key) + if prior is None or response is None: + return None + if self.authority_scope is not None: + authority = connection.execute( + "SELECT ledger FROM determa_scope_authority WHERE scope_identity=?", + (self.authority_scope,), + ).fetchone() + if authority is None: + raise EffectError("unauthorized_scope") + _validate_authority_pair(json.loads(authority[0]), document) + validate_journal(document["checkpoint"], document["journal"]) + disclosure = document.get("result_disclosure", {}).get(key) + if disclosure is None: + self._authorize( + document, + request["effect_id"], + request["operation_token"], + request["attempt_fence"], + principal, + scope, + epoch, + "0", + allow_replay=True, + ) + elif disclosure != {"principal": principal, "scope": scope, "epoch": epoch}: + raise EffectError("unauthorized_scope") + if prior != dict(request): + raise EffectError("effect_result_conflict") + record = _record(document["journal"], request["effect_id"]) + if _retained_report(record, request) is None: + raise EffectError("effect_result_conflict") + if request["outcome_kind"] == "retryable_failure" and not any( + entry.get("decision") == "report" + and entry.get("attempt_fence") == request["attempt_fence"] + and isinstance(entry.get("evidence"), dict) + and all( + entry["evidence"].get(key) == value + for key, value in { + **self._retry_context(document, record), + "attempt_fence": request["attempt_fence"], + }.items() + ) + for entry in document.get("destination_evidence", {}).get(request["effect_id"], []) + ): + raise EffectError("host_capability_mismatch") + return _validate_request(response, "effect-result-response-v1.schema.json") + def _submit( self, document: dict[str, Any], @@ -1625,6 +2168,8 @@ def _submit( scope: str, epoch: str, trusted_now: str, + *, + retry_verified: bool = False, ) -> dict[str, Any]: journal, checkpoint = document["journal"], document["checkpoint"] effect_id, fence, kind = ( @@ -1672,6 +2217,8 @@ def _submit( ) if record["invocation_state"] != "leased": raise EffectError("effect_not_outstanding") + if kind == "retryable_failure" and not retry_verified: + raise EffectError("host_capability_mismatch") payload = copy.deepcopy(request["payload"]) if kind in _OUTCOMES: self._result_delivery(checkpoint, record, kind, payload) @@ -1694,6 +2241,11 @@ def _submit( } record["attempt_records"].append(report) document["result_requests"][key] = copy.deepcopy(dict(request)) + document.setdefault("result_disclosure", {})[key] = { + "principal": principal, + "scope": scope, + "epoch": epoch, + } if kind not in _OUTCOMES: record["invocation_state"] = "unclaimed" if kind == "retryable_failure" else "ambiguous" _bump(journal) @@ -1822,6 +2374,32 @@ def change(document: dict[str, Any], effect_id: str = effect_id) -> None: ) return self.snapshot(root) + def _validate_cancelled_payload( + self, checkpoint: dict[str, Any], record: dict[str, Any], payload: Any + ) -> None: + """Validate the original declaration without requiring current runtime liveness.""" + from .engine import _normalize_payload + from .wire import _origin_machine + + try: + target = _pinned_result_target(checkpoint, record, self.resolver) + delivery = _result_delivery_evidence(checkpoint, record, "cancelled", payload, target) + bundle, machine = _origin_machine( + self.resolver, record["target"]["runtime_incarnation"] + ) + declarations = dict(bundle.raw.get("events") or {}) + declarations.update(machine.raw.get("events") or {}) + declaration = declarations.get(_mapping(record, "cancelled")["event"]) + if not isinstance(declaration, dict) or declaration.get("direction") != "input": + raise EffectError("invalid_host_request") + logical = decoded_typed_value(delivery["envelope"]["payload"]) + if _normalize_payload(declaration, logical) is None: + raise EffectError("invalid_host_request") + except (ArtifactError, KeyError, TypeError, ValueError) as exc: + if isinstance(exc, EffectError): + raise + raise EffectError("invalid_host_request") from exc + def cancel(self, root: str, request: Mapping[str, Any]) -> dict[str, Any]: if ( not isinstance(request, Mapping) @@ -1890,7 +2468,11 @@ def change(document: dict[str, Any]) -> dict[str, Any]: ): raise EffectError("operation_id_conflict") return replay - if record["invocation_state"] == "unclaimed" and record["attempt_fence"] == "0": + self._validate_cancelled_payload(document["checkpoint"], record, request["payload"]) + if record["outcome"] is not None: + state = "too_late" + outcome = copy.deepcopy(record["outcome"]) + elif record["invocation_state"] == "unclaimed" and record["attempt_fence"] == "0": mapping = _mapping(record, "cancelled") self._result_delivery( document["checkpoint"], record, "cancelled", request["payload"] @@ -1924,7 +2506,8 @@ def change(document: dict[str, Any]) -> dict[str, Any]: "reason": request["reason"], "state": state, } - record["cancellation"] = cancellation + if state != "too_late": + record["cancellation"] = cancellation _bump(journal) response = { "status": "committed" if outcome else "reconciliation_required", diff --git a/src/determa/state/extensions.py b/src/determa/state/extensions.py index b75f01f..3a484b2 100644 --- a/src/determa/state/extensions.py +++ b/src/determa/state/extensions.py @@ -1302,6 +1302,7 @@ def verify(provider: Any, descriptor: Mapping[str, Any]) -> bool: "file": {"restart_persistent"}, "sqlite": { "durable_single_writer", + "shared_application_transaction", "root_identity_retention", "permanent_receipt_retention", "permanent_outbox_terminal_retention", diff --git a/src/determa/state/host.py b/src/determa/state/host.py index 6a362c9..369c65d 100644 --- a/src/determa/state/host.py +++ b/src/determa/state/host.py @@ -1526,6 +1526,7 @@ def __init__(self, host: ExecutionHost, root_instance_id: str) -> None: self.root_instance_id = root_instance_id self._active = True self._response: dict[str, Any] | None = None + self._operation: str | None = None def _stage( self, @@ -1537,11 +1538,16 @@ def _stage( if self._response is not None: raise ExecutionHostError("shared_transaction_operation_conflict") self._response = invoke() + self._operation = operation return StagedExecutionResult(operation) def _finish(self) -> dict[str, Any]: if self._response is None: raise ExecutionHostError("shared_transaction_operation_required") + # Queue-bearing admission returns committed acceptance evidence rather + # than the result discriminator used by create/migration operations. + if self._operation == "admit_v1" and set(self._response) == {"evidence"}: + return self._response if self._response["result"] not in { "committed", "pending", @@ -1554,6 +1560,12 @@ def _finish(self) -> dict[str, Any]: def _deactivate(self) -> None: self._active = False + def read_checkpoint(self) -> RestoredExecutionCheckpoint | None: + """Read and resolver-validate the bound root inside this transaction.""" + if not self._active: + raise ExecutionHostError("shared_transaction_closed") + return self._host.read_checkpoint(self.root_instance_id) + def create_v1( self, bundle: Bundle | BundleSource, diff --git a/src/determa/state/native_clock.py b/src/determa/state/native_clock.py new file mode 100644 index 0000000..6903d83 --- /dev/null +++ b/src/determa/state/native_clock.py @@ -0,0 +1,68 @@ +"""Host-owned nanosecond reads with no user callback at the commit boundary. + +Controlled clocks are explicit native test/host sources. Their installer owns +sample updates and availability; they do not advance or schedule themselves. +""" + +from __future__ import annotations + +from time import time_ns as _system_time_ns + + +class NativeCommitClock: + """Exact native clock handle, separate from portable request timestamps.""" + + __slots__ = ("_sample", "_system") + + def __init__(self) -> None: + self._system = True + self._sample: tuple[int, bool] = (0, False) + + @classmethod + def controlled(cls, now_ns: int) -> NativeCommitClock: + clock = cls() + clock._system = False + clock.set_native_sample(now_ns) + return clock + + def set_native_sample(self, now_ns: int, *, unavailable: bool = False) -> None: + if type(now_ns) is not int or not -(2**63) <= now_ns < 2**63: + raise ValueError("invalid native clock sample") + if type(unavailable) is not bool or self._system: + raise ValueError("controlled native clock required") + self._sample = (now_ns, unavailable) + + +# Retain actual native member descriptors, not a dynamic attribute access which +# could invoke a property installed by a callback after proof verification. +_NATIVE_CLOCK_TYPE = NativeCommitClock +_SYSTEM_SLOT = NativeCommitClock.__dict__["_system"] +_SAMPLE_SLOT = NativeCommitClock.__dict__["_sample"] + + +def _native_clock_mode(clock: NativeCommitClock) -> bool: + if type(clock) is not _NATIVE_CLOCK_TYPE: + raise ValueError("native clock identity differs") + namespace = type.__getattribute__(_NATIVE_CLOCK_TYPE, "__dict__") + if namespace.get("_system") is not _SYSTEM_SLOT or namespace.get("_sample") is not _SAMPLE_SLOT: + raise ValueError("native clock descriptors differ") + mode = _SYSTEM_SLOT.__get__(clock, _NATIVE_CLOCK_TYPE) + if type(mode) is not bool: + raise ValueError("native clock configuration differs") + return mode + + +def _read_native_clock(clock: NativeCommitClock, *, mode: bool) -> int: + system = _native_clock_mode(clock) + if system is not mode: + raise ValueError("native clock source differs") + if system: + now = _system_time_ns() + else: + sample = _SAMPLE_SLOT.__get__(clock, _NATIVE_CLOCK_TYPE) + if type(sample) is not tuple or len(sample) != 2 or sample[1] is not False: + raise ValueError("native clock unavailable") + now = sample[0] + if type(now) is not int or not -(2**63) <= now < 2**63: + raise ValueError("invalid native clock sample") + return now diff --git a/src/determa/state/stores/sqlite.py b/src/determa/state/stores/sqlite.py index 45f684d..842b9e1 100644 --- a/src/determa/state/stores/sqlite.py +++ b/src/determa/state/stores/sqlite.py @@ -4,7 +4,7 @@ import re import sqlite3 -from collections.abc import Iterator, Mapping +from collections.abc import Callable, Iterator, Mapping from contextlib import contextmanager from pathlib import Path from typing import Any @@ -17,6 +17,7 @@ PERMANENT_OUTBOX_TERMINAL_RETENTION, PERMANENT_RECEIPT_RETENTION, ROOT_IDENTITY_RETENTION, + SHARED_APPLICATION_TRANSACTION, ExecutionStore, ExecutionStoreError, ExecutionStoreTransaction, @@ -40,21 +41,49 @@ _IMMUTABLE_MESSAGE = "execution_store_immutable" +def _refuse_native_owned_database(connection: sqlite3.Connection) -> None: + """The base adapter cannot prove native authority or joint journal ownership. + + Check permanent schema markers under the writer lock, before any caller or + core code runs. Even a partial native schema requires the native adapter. + """ + if ( + connection.execute( + "SELECT 1 FROM sqlite_master WHERE name COLLATE NOCASE IN (?, ?, ?) LIMIT 1", + ("determa_scope_authority", "determa_scope_allocations", "determa_committed_effects"), + ).fetchone() + is not None + ): + raise ExecutionStoreError("scope_fence_unproven") + + def _schema_tokens(source: str) -> list[str]: return re.findall(r"[a-z_][a-z0-9_]*|[(),]", source.lower()) class _SQLiteTransaction(ExecutionStoreTransaction): - def __init__(self, connection: sqlite3.Connection, root_instance_id: str) -> None: + def __init__( + self, + connection: sqlite3.Connection, + root_instance_id: str, + *, + application_sql: bool = False, + ) -> None: self._connection = connection self._root_instance_id = root_instance_id + self._application_sql = application_sql + + def _execute(self, statement: str, parameters: Any = ()) -> sqlite3.Cursor: + if self._application_sql: + return _execute_host_owned(self._connection, statement, parameters) + return self._connection.execute(statement, parameters) @property def root_instance_id(self) -> str: return self._root_instance_id def load(self) -> bytes | None: - row = self._connection.execute( + row = self._execute( f"SELECT checkpoint FROM {_TABLE} WHERE root_instance_id = ?", (self._root_instance_id,), ).fetchone() @@ -65,7 +94,7 @@ def insert(self, checkpoint: bytes) -> bool: if root_instance_id != self._root_instance_id: raise ExecutionStoreError("transaction_root_mismatch") try: - self._connection.execute( + self._execute( f""" INSERT INTO {_TABLE} (root_instance_id, revision, checkpoint_digest, checkpoint) @@ -86,7 +115,7 @@ def replace( root_instance_id, revision, digest = checkpoint_metadata(checkpoint) if root_instance_id != self._root_instance_id: raise ExecutionStoreError("transaction_root_mismatch") - cursor = self._connection.execute( + cursor = self._execute( f""" UPDATE {_TABLE} SET revision = ?, checkpoint_digest = ?, checkpoint = ? @@ -106,6 +135,92 @@ def replace( return cursor.rowcount == 1 +class SQLiteApplicationCursor: + """Rows from host-owned SQL without exposing the connection's commit controls.""" + + def __init__(self, cursor: sqlite3.Cursor) -> None: + self._cursor = cursor + + @property + def rowcount(self) -> int: + return self._cursor.rowcount + + def fetchone(self) -> Any: + return self._cursor.fetchone() + + def fetchall(self) -> list[Any]: + return self._cursor.fetchall() + + +class SQLiteApplicationTransaction: + """Foreground SQL resource whose lifetime and commit belong to ExecutionHost. + + Application callbacks receive statement execution and rows, not a second + transaction API. SQL transaction control, attached databases and PRAGMA changes + are refused. The resource is closed before the host commits or rolls back. + """ + + def __init__(self, connection: sqlite3.Connection) -> None: + self._connection = connection + self._active = True + self._commit_checks: list[Callable[[], None]] = [] + + def execute(self, statement: str, parameters: Any = ()) -> SQLiteApplicationCursor: + if not self._active: + raise ExecutionStoreError("shared_transaction_closed") + return SQLiteApplicationCursor(self._connection.execute(statement, parameters)) + + def _close(self) -> None: + self._active = False + + def _execute_host_owned(self, statement: str, parameters: Any = ()) -> SQLiteApplicationCursor: + if not self._active: + raise ExecutionStoreError("shared_transaction_closed") + return SQLiteApplicationCursor(_execute_host_owned(self._connection, statement, parameters)) + + def _before_commit(self, check: Callable[[], None]) -> None: + if not self._active: + raise ExecutionStoreError("shared_transaction_closed") + self._commit_checks.append(check) + + +def _execute_host_owned( + connection: sqlite3.Connection, statement: str, parameters: Any +) -> sqlite3.Cursor: + """Trusted adapter/helper path; application SQL never receives this operation.""" + connection.set_authorizer(None) + try: + return connection.execute(statement, parameters) + finally: + connection.set_authorizer(_application_authorizer) + + +def _application_authorizer( + operation: int, + first: str | None, + second: str | None, + database: str | None, + trigger: str | None, +) -> int: + del database, trigger + if any(name and name.lower().startswith("determa_") for name in (first, second)): + return sqlite3.SQLITE_DENY + if operation == sqlite3.SQLITE_PRAGMA and first == "integrity_check" and second is None: + return sqlite3.SQLITE_OK + return ( + sqlite3.SQLITE_DENY + if operation + in ( + sqlite3.SQLITE_TRANSACTION, + sqlite3.SQLITE_SAVEPOINT, + sqlite3.SQLITE_ATTACH, + sqlite3.SQLITE_DETACH, + sqlite3.SQLITE_PRAGMA, + ) + else sqlite3.SQLITE_OK + ) + + class SQLiteExecutionStore(ExecutionStore): """Single-writer durable SQLite storage under verified PRAGMA settings.""" @@ -118,6 +233,7 @@ def __init__( timeout: float = 30.0, replay_retention: str = "bounded", outbox_retention: str = "none", + shared_application_transactions: bool = False, ) -> None: self.path = str(path) self.journal_mode = journal_mode.upper() @@ -125,6 +241,7 @@ def __init__( self.timeout = timeout self.replay_retention = replay_retention self.outbox_retention = outbox_retention + self.shared_application_transactions = shared_application_transactions if ( not self.path or self.path == ":memory:" @@ -133,6 +250,7 @@ def __init__( or timeout <= 0 or replay_retention not in _REPLAY_RETENTION_MODES or outbox_retention not in _OUTBOX_RETENTION_MODES + or type(shared_application_transactions) is not bool ): raise ExecutionStoreError(AdapterCode.INVALID_ADAPTER_CONFIGURATION) @@ -142,6 +260,8 @@ def capabilities(self) -> frozenset[str]: if not self._policy_is_valid(): return frozenset(capabilities) capabilities.add(ROOT_IDENTITY_RETENTION) + if self.shared_application_transactions: + capabilities.add(SHARED_APPLICATION_TRANSACTION) if self.replay_retention == "permanent": capabilities.add(PERMANENT_RECEIPT_RETENTION) if self.outbox_retention == "strict": @@ -339,6 +459,7 @@ def transaction( try: self._validate_schema(connection) connection.execute("BEGIN IMMEDIATE") + _refuse_native_owned_database(connection) transaction = _SQLiteTransaction(connection, root_instance_id) yield transaction connection.commit() @@ -416,6 +537,40 @@ def setup_schema(self) -> None: finally: connection.close() + @contextmanager + def shared_transaction( + self, + root_instance_id: str, + ) -> Iterator[tuple[SQLiteApplicationTransaction, ExecutionStoreTransaction]]: + if not self.shared_application_transactions: + raise ExecutionStoreError(AdapterCode.ADAPTER_CAPABILITY_MISMATCH) + connection = self._connect() + application = SQLiteApplicationTransaction(connection) + try: + self._validate_schema(connection) + connection.execute("BEGIN IMMEDIATE") + _refuse_native_owned_database(connection) + connection.set_authorizer(_application_authorizer) + yield ( + application, + _SQLiteTransaction(connection, root_instance_id, application_sql=True), + ) + application._close() + connection.set_authorizer(None) + if not connection.in_transaction: + raise ExecutionStoreError(AdapterCode.ADAPTER_CAPABILITY_MISMATCH) + self._validate_schema(connection) + for check in application._commit_checks: + check() + connection.commit() + except BaseException: + application._close() + connection.set_authorizer(None) + connection.rollback() + raise + finally: + connection.close() + def health(self) -> Mapping[str, Any]: try: connection = self._connect() @@ -461,6 +616,7 @@ def sqlite_execution_store_factory(uri: str, configuration: Mapping[str, Any]) - "timeout", "replay_retention", "outbox_retention", + "shared_application_transactions", }: raise ExecutionStoreError(AdapterCode.INVALID_ADAPTER_CONFIGURATION) journal_mode = _single_query(query, "journal_mode", "WAL") @@ -468,6 +624,9 @@ def sqlite_execution_store_factory(uri: str, configuration: Mapping[str, Any]) - timeout_text = _single_query(query, "timeout", "30") replay_retention = _single_query(query, "replay_retention", "bounded") outbox_retention = _single_query(query, "outbox_retention", "none") + shared = _single_query(query, "shared_application_transactions", "false") + if shared not in {"true", "false"}: + raise ExecutionStoreError(AdapterCode.INVALID_ADAPTER_CONFIGURATION) try: timeout = float(timeout_text) except ValueError as exc: @@ -479,4 +638,5 @@ def sqlite_execution_store_factory(uri: str, configuration: Mapping[str, Any]) - timeout=timeout, replay_retention=replay_retention, outbox_retention=outbox_retention, + shared_application_transactions=shared == "true", ) diff --git a/src/determa/state/timers.py b/src/determa/state/timers.py new file mode 100644 index 0000000..2e9d8b6 --- /dev/null +++ b/src/determa/state/timers.py @@ -0,0 +1,974 @@ +"""Explicit foreground SQLite timer records, separate from portable checkpoints. + +This implementation checkpoint supports durable schedule/cancel/read, initial claims +and coordinated SQLite admission with strictly local native-fate reclaim. Verified +provider installation, +archive and recovery integration remain unfinished; no completed profile is advertised. +""" + +from __future__ import annotations + +import copy +import hashlib +import json +import re +import sqlite3 +from collections.abc import Callable, Mapping +from contextlib import closing +from pathlib import Path +from typing import Any + +from .checkpoint_v1 import restore_execution_checkpoint_v1 +from .definition import Bundle +from .engine import _normalize_payload +from .errors import ArtifactError +from .host import ( + ExecutionHost, + ExecutionHostError, + SharedExecutionTransaction, + delivery_request_digest, +) +from .stores.base import ExecutionStoreError +from .stores.sqlite import SQLiteApplicationTransaction, SQLiteExecutionStore +from .wire import _schema_registry, canonical_bytes, decoded_typed_value, hash_value, strict_json + +_TIME = re.compile(r"(?:0|-?[1-9][0-9]*)\Z") +_TABLE = ( + "CREATE TABLE determa_timer_helpers " + "(scope_identity TEXT PRIMARY KEY NOT NULL, document BLOB NOT NULL)" +) +_ORIGIN_TABLE = ( + "CREATE TABLE determa_timer_origin " + "(singleton INTEGER PRIMARY KEY NOT NULL CHECK(singleton=1), configuration BLOB NOT NULL)" +) +_COMMIT_TABLE = ( + "CREATE TABLE determa_timer_commits " + "(scope_identity TEXT NOT NULL, sequence INTEGER NOT NULL, operation_id TEXT NOT NULL, " + "journal BLOB NOT NULL, PRIMARY KEY(scope_identity, sequence), " + "UNIQUE(scope_identity, operation_id))" +) +_NATIVE_TABLES = { + "determa_timer_helpers": _TABLE, + "determa_timer_origin": _ORIGIN_TABLE, + "determa_timer_commits": _COMMIT_TABLE, +} +_NATIVE_TRIGGERS = { + f"{table}_forbid_{operation.lower()}": ( + f"CREATE TRIGGER {table}_forbid_{operation.lower()} BEFORE {operation} ON {table} " + "BEGIN SELECT RAISE(ABORT, 'timer_native_evidence_immutable'); END" + ) + for table in ("determa_timer_origin", "determa_timer_commits") + for operation in ("UPDATE", "DELETE") +} +_IDENTITIES = ( + "operation", + "operation_id", + "scope_identity", + "root_instance_id", + "root_runtime_id", + "timer_id", +) + + +class TimerError(ValueError): + def __init__(self, code: str) -> None: + self.code = code + super().__init__(code) + + +class _TimerReturn(Exception): + """Return a replay/refusal only after rolling back the shared host callback.""" + + def __init__(self, result: dict[str, Any]) -> None: + self.result = result + + +def timer_request_digest(request: Mapping[str, Any]) -> str: + return hash_value( + [ + "determa-timer-request-1", + {key: value for key, value in request.items() if key != "request_digest"}, + ] + ) + + +def seal_timer_records(artifact: Mapping[str, Any]) -> dict[str, Any]: + body = copy.deepcopy( + {key: value for key, value in artifact.items() if key != "timer_artifact_digest"} + ) + return {**body, "timer_artifact_digest": hash_value(["determa-timer-artifact-1", body])} + + +def _validator(filename: str, definition: str | None = None) -> Any: + from jsonschema import Draft202012Validator + from referencing import Resource + + schema = json.loads((Path(__file__).parent / "data" / filename).read_text()) + registry = _schema_registry() + for name in ("timer-helper-operation-v1.schema.json", "timer-record-v1.schema.json"): + resource = json.loads((Path(__file__).parent / "data" / name).read_text()) + registry = registry.with_resource(resource["$id"], Resource.from_contents(resource)) + if definition is not None: + schema = {"$ref": schema["$id"] + "#/$defs/" + definition} + return Draft202012Validator(schema, registry=registry) + + +def _time(value: Any, *, duration: bool = False) -> int: + if type(value) is not str or not _TIME.fullmatch(value): + raise TimerError("invalid_timer_time") + parsed = int(value) + if not (0 if duration else -(2**63)) <= parsed <= 2**63 - 1: + raise TimerError("invalid_timer_time") + return parsed + + +def _execute( + connection: sqlite3.Connection | SQLiteApplicationTransaction, + statement: str, + parameters: Any = (), +) -> Any: + if isinstance(connection, SQLiteApplicationTransaction): + return connection._execute_host_owned(statement, parameters) + return connection.execute(statement, parameters) + + +def _result( + request: Mapping[str, Any] | None, record: dict[str, Any] | None, code: str | None = None +) -> dict[str, Any]: + result: dict[str, Any] = { + "interface": "determa.timer_helper", + "interface_version": 1, + **{key: request[key] if request is not None else None for key in _IDENTITIES}, + "status": "accepted" if code is None else "rejected", + "record_revision": record["revision"] if record else None, + "attempt_fence": record["attempt_fence"] + if record and record["attempt_fence"] != "0" + else None, + "clock_basis": "unix_nanoseconds", + "expires_at": record["expires_at"] + if code is None and request is not None and request["operation"] == "claim_fire" and record + else None, + "event_id": record["event_id"] if record else None, + "delivery_state": "admitted" if record and record["admission_receipt_digest"] else "none", + "error_code": code, + "result_digest": None, + } + if code is None and request is not None: + result["result_digest"] = hash_value( + [ + "determa-timer-result-1", + request["request_digest"], + {key: value for key, value in result.items() if key != "result_digest"}, + ] + ) + return result + + +class SQLiteTimerHelper: + """One configured scope and immutable target incarnation; no background work. + + Principals and the clock are trusted host configuration, never request fields. + Schema setup is explicit. Records and exact successful receipts share one native + SQLite commit; a refusal does not write a receipt or change the artifact. + """ + + def __init__( + self, + path: str | Path, + bundle: Bundle, + *, + scope_identity: str, + root_instance_id: str, + root_runtime_id: str, + principals: frozenset[str], + trusted_clock: Callable[[], str], + worker_principals: frozenset[str] = frozenset(), + claim_lease_nanoseconds: str = "20", + coordinated_host: ExecutionHost | None = None, + ) -> None: + if ( + str(path) == ":memory:" + or not str(path) + or not all((scope_identity, root_instance_id, root_runtime_id)) + or not principals + ): + raise ValueError( + "persistent storage and a configured scope, target and principals are required" + ) + self.path = str(Path(path).resolve()) + self.bundle = bundle + self.scope_identity = scope_identity + self.root_instance_id = root_instance_id + self.root_runtime_id = root_runtime_id + self.principals = frozenset(principals) + self.trusted_clock = trusted_clock + self.coordinated_host = coordinated_host + if coordinated_host is not None and ( + type(coordinated_host.store) is not SQLiteExecutionStore + or Path(coordinated_host.store.path).resolve() != Path(self.path) + or not coordinated_host.store.shared_application_transactions + ): + raise ValueError("coordinated admission requires the same host-owned SQLite database") + self.worker_principals = frozenset(worker_principals) + self.claim_lease_nanoseconds = _time(claim_lease_nanoseconds, duration=True) + if self.claim_lease_nanoseconds == 0 or not self.worker_principals <= self.principals: + raise ValueError( + "workers must be authorized principals and the claim lease must be positive" + ) + + def _connect(self) -> sqlite3.Connection: + connection = sqlite3.connect(self.path, isolation_level=None, timeout=30) + journal = connection.execute("PRAGMA journal_mode=WAL").fetchone() + connection.execute("PRAGMA synchronous=FULL") + if journal != ("wal",) or connection.execute("PRAGMA synchronous").fetchone() != (2,): + connection.close() + raise TimerError("timer_capability_mismatch") + return connection + + def setup_schema(self) -> None: + with closing(self._connect()) as connection: + connection.execute("BEGIN IMMEDIATE") + try: + existing = connection.execute( + "SELECT COUNT(*) FROM sqlite_master WHERE name LIKE 'determa_timer_%'" + ).fetchone()[0] + if existing: + # Never reconstruct origin or retained evidence around prior records. + self._schema(connection) + else: + for statement in (*_NATIVE_TABLES.values(), *_NATIVE_TRIGGERS.values()): + connection.execute(statement) + connection.execute( + "INSERT INTO determa_timer_origin VALUES (1,?)", + (canonical_bytes(self._configuration()),), + ) + self._schema(connection) + connection.commit() + finally: + connection.rollback() + + def _implementation_digest(self) -> str: + # An existing native origin cannot silently change completion implementation. + # Independent loaded-source registration is still required before a profile claim. + directory = Path(__file__).parent + closure = b"".join( + path.relative_to(directory).as_posix().encode() + b"\0" + path.read_bytes() + for path in sorted(directory.rglob("*")) + if path.is_file() and path.suffix in {".py", ".json"} + ) + return "sha256:" + hashlib.sha256(closure).hexdigest() + + def _configuration(self) -> dict[str, Any]: + return { + "native_timer_configuration_version": 1, + "implementation_digest": self._implementation_digest(), + "coordinated_store_configuration": { + "journal_mode": self.coordinated_host.store.journal_mode, + "synchronous": self.coordinated_host.store.synchronous, + "replay_retention": self.coordinated_host.store.replay_retention, + "outbox_retention": self.coordinated_host.store.outbox_retention, + "shared_application_transactions": ( + self.coordinated_host.store.shared_application_transactions + ), + } + if self.coordinated_host is not None + and type(self.coordinated_host.store) is SQLiteExecutionStore + else None, + "storage_path": self.path, + "scope_identity": self.scope_identity, + "root_instance_id": self.root_instance_id, + "root_runtime_id": self.root_runtime_id, + "validated_bundle_fingerprint": self.bundle.fingerprint, + "principals": sorted(self.principals), + "worker_principals": sorted(self.worker_principals), + "claim_lease_nanoseconds": str(self.claim_lease_nanoseconds), + "admission_strategy": "coordinated_sqlite" + if self.coordinated_host is not None + else "records_only", + } + + def _schema(self, connection: sqlite3.Connection | SQLiteApplicationTransaction) -> None: + rows = _execute( + connection, + "SELECT name,type,sql FROM sqlite_master WHERE tbl_name IN " + "('determa_timer_helpers','determa_timer_origin','determa_timer_commits') " + "OR name LIKE 'determa_timer_%' ORDER BY name", + ).fetchall() + actual = {name: (kind, sql) for name, kind, sql in rows if sql is not None} + expected = { + **{name: ("table", sql) for name, sql in _NATIVE_TABLES.items()}, + **{name: ("trigger", sql) for name, sql in _NATIVE_TRIGGERS.items()}, + } + if ( + actual != expected + or len(rows) != 10 + or _execute(connection, "PRAGMA integrity_check").fetchone() != ("ok",) + ): + raise TimerError("timer_capability_mismatch") + origin = _execute(connection, "SELECT configuration FROM determa_timer_origin").fetchall() + if origin != [(canonical_bytes(self._configuration()),)]: + raise TimerError("timer_capability_mismatch") + for table in ("determa_timer_helpers", "determa_timer_commits"): + if _execute( + connection, + f"SELECT COUNT(*) FROM {table} WHERE scope_identity!=?", + (self.scope_identity,), + ).fetchone() != (0,): + raise TimerError("timer_capability_mismatch") + + def validate_schema(self) -> None: + with closing(self._connect()) as connection: + connection.execute("BEGIN") + try: + self._schema(connection) + finally: + connection.rollback() + + def _load( + self, connection: sqlite3.Connection | SQLiteApplicationTransaction + ) -> dict[str, Any]: + row = _execute( + connection, + "SELECT document FROM determa_timer_helpers WHERE scope_identity=?", + (self.scope_identity,), + ).fetchone() + if row is None: + artifact = seal_timer_records( + { + "timer_artifact_format": "determa.timer_records", + "timer_artifact_schema_version": 1, + "records": [], + "operation_receipts": [], + } + ) + else: + try: + parsed, raw = strict_json(bytes(row[0])) + if not isinstance(parsed, dict) or raw != canonical_bytes(parsed): + raise TimerError("timer_capability_mismatch") + artifact = parsed + except (ArtifactError, TypeError, ValueError) as error: + raise TimerError("timer_capability_mismatch") from error + if ( + not _validator("timer-record-v1.schema.json").is_valid(artifact) + or seal_timer_records(artifact) != artifact + ): + raise TimerError("timer_capability_mismatch") + if any(record["scope_identity"] != self.scope_identity for record in artifact["records"]): + raise TimerError("timer_capability_mismatch") + self._history(connection, artifact) + return artifact + + def _history( + self, + connection: sqlite3.Connection | SQLiteApplicationTransaction, + artifact: dict[str, Any], + ) -> None: + rows = _execute( + connection, + "SELECT sequence,operation_id,journal FROM determa_timer_commits " + "WHERE scope_identity=? ORDER BY sequence", + (self.scope_identity,), + ).fetchall() + if len(rows) != len(artifact["operation_receipts"]): + raise TimerError("timer_capability_mismatch") + if not rows and artifact["records"]: + raise TimerError("timer_capability_mismatch") + last_digest = None + for sequence, (native_sequence, operation_id, raw) in enumerate(rows, start=1): + try: + journal, original = strict_json(bytes(raw)) + if ( + native_sequence != sequence + or not isinstance(journal, dict) + or original != canonical_bytes(journal) + or set(journal) != {"request", "result", "timer_artifact_digest"} + ): + raise TimerError("timer_capability_mismatch") + request = journal["request"] + result = journal["result"] + receipt = artifact["operation_receipts"][sequence - 1] + if ( + not _validator("timer-helper-operation-v1.schema.json", "request").is_valid( + request + ) + or request["request_digest"] != timer_request_digest(request) + or request["operation_id"] != operation_id + or request["scope_identity"] != self.scope_identity + or request["root_instance_id"] != self.root_instance_id + or request["root_runtime_id"] != self.root_runtime_id + or result["status"] != "accepted" + or any(result[key] != request[key] for key in _IDENTITIES) + or result["result_digest"] + != hash_value( + [ + "determa-timer-result-1", + request["request_digest"], + {key: value for key, value in result.items() if key != "result_digest"}, + ] + ) + or receipt + != { + "operation_id": operation_id, + "request_digest": request["request_digest"], + "result": result, + } + ): + raise TimerError("timer_capability_mismatch") + last_digest = journal["timer_artifact_digest"] + except (ArtifactError, TypeError, ValueError, KeyError) as error: + raise TimerError("timer_capability_mismatch") from error + if last_digest is not None and last_digest != artifact["timer_artifact_digest"]: + raise TimerError("timer_capability_mismatch") + + def _record_commit( + self, + connection: sqlite3.Connection | SQLiteApplicationTransaction, + request: Mapping[str, Any], + result: dict[str, Any], + artifact: dict[str, Any], + ) -> None: + self._persist(connection, artifact) + sealed = seal_timer_records(artifact) + _execute( + connection, + "INSERT INTO determa_timer_commits VALUES (?,?,?,?)", + ( + self.scope_identity, + len(artifact["operation_receipts"]), + request["operation_id"], + canonical_bytes( + { + "request": request, + "result": result, + "timer_artifact_digest": sealed["timer_artifact_digest"], + } + ), + ), + ) + + def snapshot(self) -> dict[str, Any]: + with closing(self._connect()) as connection: + connection.execute("BEGIN") + try: + self._schema(connection) + return self._load(connection) + finally: + connection.rollback() + + def _persist( + self, + connection: sqlite3.Connection | SQLiteApplicationTransaction, + artifact: dict[str, Any], + ) -> None: + artifact["records"].sort( + key=lambda record: ( + record["scope_identity"], + record["root_instance_id"], + record["root_runtime_id"], + record["timer_id"], + ) + ) + artifact = seal_timer_records(artifact) + if not _validator("timer-record-v1.schema.json").is_valid(artifact): + raise TimerError("timer_capability_mismatch") + _execute( + connection, + "INSERT INTO determa_timer_helpers VALUES (?,?) " + "ON CONFLICT(scope_identity) DO UPDATE SET document=excluded.document", + (self.scope_identity, canonical_bytes(artifact)), + ) + + def _event(self, arguments: Mapping[str, Any]) -> None: + declaration = self.bundle.raw.get("events", {}).get(arguments["event_name"]) + try: + payload = decoded_typed_value(arguments["payload"]) + valid = ( + declaration + and declaration.get("direction") == "input" + and _normalize_payload(declaration, payload) is not None + ) + except (TypeError, ValueError, KeyError): + valid = False + if not valid: + raise TimerError("invalid_timer_request") + + def _now(self) -> int: + try: + return _time(self.trusted_clock()) + except Exception as error: + raise TimerError("timer_clock_unavailable") from error + + def _prove_local_uncommitted_fire( + self, connection: sqlite3.Connection, artifact: dict[str, Any], record: dict[str, Any] + ) -> None: + """Resolve the previous native transaction before committing a replacement fence. + + Caller time, absence of an event, and portable claims are not this proof. + The caller owns BEGIN IMMEDIATE, and _load has validated the complete immutable + native commit history against this storage-bound origin. Every completion by + this exact coordinated implementation commits admission and fired evidence + together. The preceding writer has therefore resolved before this decision. + """ + host = self.coordinated_host + if ( + type(self) is not SQLiteTimerHelper + or type(host) is not ExecutionHost + or type(host.store) is not SQLiteExecutionStore + or Path(host.store.path).resolve() != Path(self.path) + or not host.store.shared_application_transactions + or host.store.journal_mode != "WAL" + or host.store.synchronous != "FULL" + or host.store.replay_retention != "permanent" + or not connection.in_transaction + or any( + getattr(SharedExecutionTransaction, name) is not method + or getattr(method, "__code__", None) is not code + for name, method, code in _FATE_SHARED_METHODS + ) + or any( + getattr(getattr(instance, name), "__func__", None) is not method + or getattr(method, "__code__", None) is not code + for instance, methods in ( + (host, _FATE_HOST_METHODS), + (host.store, _FATE_STORE_METHODS), + ) + for name, method, code in methods + ) + or any( + getattr(getattr(self, name), "__func__", None) is not method + or getattr(method, "__code__", None) is not code + for name, method, code in _FATE_METHODS + ) + ): + raise TimerError("delivery_ambiguous") + retained_claim = any( + receipt["result"]["operation"] == "claim_fire" + and receipt["result"]["timer_id"] == record["timer_id"] + and receipt["result"]["record_revision"] == record["revision"] + and receipt["result"]["attempt_fence"] == record["attempt_fence"] + and receipt["result"]["expires_at"] == record["expires_at"] + for receipt in artifact["operation_receipts"] + ) + if not retained_claim or any( + receipt["result"]["operation"] == "complete_fire" + and receipt["result"]["timer_id"] == record["timer_id"] + for receipt in artifact["operation_receipts"] + ): + raise TimerError("delivery_ambiguous") + try: + host.store._validate_schema(connection) + row = connection.execute( + "SELECT revision,checkpoint_digest,checkpoint FROM determa_execution_checkpoints " + "WHERE root_instance_id=?", + (self.root_instance_id,), + ).fetchone() + if row is None: + raise TimerError("delivery_ambiguous") + checkpoint = restore_execution_checkpoint_v1( + bytes(row[2]), host.artifact_resolver + ).document + aggregate = checkpoint["root_record"].get("aggregate_state") + if ( + checkpoint["revision"] != row[0] + or checkpoint["execution_checkpoint_digest"] != row[1] + or checkpoint["root_instance_id"] != self.root_instance_id + or not aggregate + or aggregate["root_runtime_id"] != self.root_runtime_id + or aggregate["validated_bundle_fingerprint"] != self.bundle.fingerprint + or checkpoint["replay_retention"]["mode"] != "permanent" + or checkpoint["replay_retention"]["pruned_through_receipt_sequence"] is not None + # An independently admitted matching event is not helper completion; + # never promote it or infer safe recovery from its receipt or absence. + or any( + receipt.get("event_id") == record["event_id"] + for receipt in checkpoint["operation_receipts"] + ) + ): + raise TimerError("delivery_ambiguous") + except (ArtifactError, ExecutionStoreError, sqlite3.Error) as error: + raise TimerError("delivery_ambiguous") from error + + def _complete(self, request: Mapping[str, Any], principal: str) -> dict[str, Any]: + host = self.coordinated_host + assert host is not None + if ( + type(host.store) is not SQLiteExecutionStore + or Path(host.store.path).resolve() != Path(self.path) + or not host.store.shared_application_transactions + ): + return _result(request, None, "timer_capability_mismatch") + committed: list[dict[str, Any]] = [] + + def callback( + sql: SQLiteApplicationTransaction, execution: SharedExecutionTransaction + ) -> None: + record = None + claimed_record = None + try: + self._schema(sql) + artifact = self._load(sql) + record = next( + ( + item + for item in artifact["records"] + if item["timer_id"] == request["timer_id"] + and item["root_instance_id"] == self.root_instance_id + and item["root_runtime_id"] == self.root_runtime_id + ), + None, + ) + claimed_record = copy.deepcopy(record) + if request["arguments"]["worker_principal"] != principal: + raise TimerError("timer_worker_mismatch") + previous = next( + ( + item + for item in artifact["operation_receipts"] + if item["operation_id"] == request["operation_id"] + ), + None, + ) + if previous is not None: + raise _TimerReturn( + copy.deepcopy(previous["result"]) + if previous["request_digest"] == request["request_digest"] + else _result(request, record, "timer_operation_conflict") + ) + if record is None: + raise TimerError("timer_not_found") + claimed_record = copy.deepcopy(record) + arguments = request["arguments"] + if record["attempt_fence"] != arguments["attempt_fence"]: + raise TimerError("timer_stale_fence") + if record["revision"] != arguments["expected_revision"]: + raise TimerError("timer_revision_conflict") + if record["state"] != "claimed": + raise TimerError( + "timer_already_fired" + if record["state"] == "fired" + else "timer_cancelled" + if record["state"] == "cancelled" + else "timer_stale_fence" + ) + if record["worker_principal"] != principal: + raise TimerError("timer_worker_mismatch") + if record["event_id"] != arguments["event_id"]: + raise TimerError("timer_event_conflict") + claim_expiry = _time(record["expires_at"]) + if self._now() >= claim_expiry: + raise TimerError("timer_stale_fence") + restored = execution.read_checkpoint() + if restored is None: + raise TimerError("timer_admission_rejected") + before = restored.document + aggregate = before["root_record"].get("aggregate_state") + if ( + not aggregate + or aggregate["root_runtime_id"] != self.root_runtime_id + or (aggregate["validated_bundle_fingerprint"] != self.bundle.fingerprint) + ): + raise TimerError("unauthorized_timer_scope") + envelope = { + "event": record["event_name"], + "event_id": record["event_id"], + "cause_id": record["event_id"], + "source": {"host": True}, + "target": { + "root": { + "root_instance_id": self.root_instance_id, + "root_runtime_id": self.root_runtime_id, + } + }, + "payload": record["payload"], + } + if record["correlation_id"] is not None: + envelope["correlation_id"] = record["correlation_id"] + envelope_digest = delivery_request_digest(self.root_instance_id, "input", envelope) + execution.admit_v1( + [ + { + "delivery_mode": "input", + "envelope": envelope, + "envelope_digest": envelope_digest, + } + ], + expected_revision=before["revision"], + expected_checkpoint_digest=before["execution_checkpoint_digest"], + ) + candidate = execution.read_checkpoint() + assert candidate is not None + receipt = next( + ( + item + for item in candidate.document["operation_receipts"] + if item["operation_kind"] == "acceptance" + and item["event_id"] == record["event_id"] + ), + None, + ) + if receipt is None or receipt["request_digest"] != envelope_digest: + raise TimerError("timer_admission_rejected") + receipt_digest = hash_value(["determa-timer-admission-receipt-1", receipt]) + if receipt_digest != arguments["admission_receipt_digest"]: + raise TimerError("timer_event_conflict") + record.update( + state="fired", + revision=str(int(record["revision"]) + 1), + worker_principal=None, + expires_at=None, + admission_receipt_digest=receipt_digest, + ) + result = _result(request, record) + artifact["operation_receipts"].append( + { + "operation_id": request["operation_id"], + "request_digest": request["request_digest"], + "result": result, + } + ) + self._record_commit(sql, request, result, artifact) + + def check_lease() -> None: + try: + if self._now() >= claim_expiry: + raise TimerError("timer_stale_fence") + except TimerError as error: + raise _TimerReturn(_result(request, claimed_record, error.code)) from error + + sql._before_commit(check_lease) + committed.append(result) + except TimerError as error: + raise _TimerReturn(_result(request, claimed_record, error.code)) from error + except ExecutionHostError as error: + raise _TimerReturn( + _result(request, claimed_record, "timer_admission_rejected") + ) from error + + try: + host.run_shared_transaction(self.root_instance_id, callback) + except _TimerReturn as returned: + return returned.result + return committed[0] + + def execute(self, request: Mapping[str, Any], *, principal: str) -> dict[str, Any]: + if not isinstance(request, Mapping) or request.get("interface") != "determa.timer_helper": + return _result(None, None, "unsupported_timer_protocol") + if type(request.get("interface_version")) is not int or request["interface_version"] != 1: + return _result(None, None, "unsupported_timer_protocol_version") + request = copy.deepcopy(dict(request)) + # Normative invalid-time vectors retain recognized operation identities. + # Check the closed structure with a canonical time placeholder, then check + # the original time domain before any clock read or record write. + structural = copy.deepcopy(request) + if structural.get("operation") == "schedule" and isinstance( + structural.get("arguments"), dict + ): + for field in ("deadline_at", "delay_nanoseconds"): + if field in structural["arguments"]: + structural["arguments"][field] = "0" + try: + valid = _validator("timer-helper-operation-v1.schema.json", "request").is_valid( + structural + ) and request.get("request_digest") == timer_request_digest(request) + except (ArtifactError, TypeError, ValueError, OverflowError): + valid = False + if not valid: + return _result(None, None, "invalid_timer_request") + if principal not in self.principals or ( + request["scope_identity"], + request["root_instance_id"], + request["root_runtime_id"], + ) != (self.scope_identity, self.root_instance_id, self.root_runtime_id): + return _result(request, None, "unauthorized_timer_scope") + if ( + request["operation"] in ("claim_fire", "complete_fire") + and principal not in self.worker_principals + ): + return _result(request, None, "unauthorized_timer_scope") + if request["operation"] == "complete_fire" and self.coordinated_host is not None: + return self._complete(request, principal) + record = None + connection = self._connect() + try: + connection.execute("BEGIN IMMEDIATE") + self._schema(connection) + artifact = self._load(connection) + record = next( + ( + record + for record in artifact["records"] + if record["timer_id"] == request["timer_id"] + and record["root_instance_id"] == self.root_instance_id + and record["root_runtime_id"] == self.root_runtime_id + ), + None, + ) + previous = next( + ( + receipt + for receipt in artifact["operation_receipts"] + if receipt["operation_id"] == request["operation_id"] + ), + None, + ) + if ( + request["operation"] in ("claim_fire", "complete_fire") + and request["arguments"]["worker_principal"] != principal + ): + return _result(request, record, "timer_worker_mismatch") + if previous is not None: + return ( + copy.deepcopy(previous["result"]) + if previous["request_digest"] == request["request_digest"] + else _result(request, record, "timer_operation_conflict") + ) + operation = request["operation"] + if operation == "schedule": + if record is not None: + return _result(request, record, "timer_id_conflict") + self._event(request["arguments"]) + arguments = request["arguments"] + if "deadline_at" in arguments: + deadline = _time(arguments["deadline_at"]) + else: + delay = _time(arguments["delay_nanoseconds"], duration=True) + now = self._now() + deadline = now + delay + if not -(2**63) <= deadline <= 2**63 - 1: + raise TimerError("timer_deadline_overflow") + record = { + key: request[key] + for key in ("scope_identity", "root_instance_id", "root_runtime_id", "timer_id") + } + record.update( + schedule_request_digest=request["request_digest"], + clock_basis="unix_nanoseconds", + deadline_at=str(deadline), + event_name=arguments["event_name"], + payload=arguments["payload"], + correlation_id=arguments["correlation_id"], + event_id=hash_value( + [ + "determa-timer-fire-event-1", + "1", + self.scope_identity, + self.root_instance_id, + self.root_runtime_id, + request["timer_id"], + ] + ), + state="pending", + revision="1", + attempt_fence="0", + worker_principal=None, + expires_at=None, + admission_receipt_digest=None, + ) + artifact["records"].append(record) + elif operation == "read_timer": + return ( + _result(request, record) + if record + else _result(request, None, "timer_not_found") + ) + elif operation == "cancel": + if record is None: + return _result(request, None, "timer_not_found") + if request["arguments"]["expected_revision"] != record["revision"]: + return _result(request, record, "timer_revision_conflict") + if record["state"] == "claimed": + return _result(request, record, "timer_fire_in_progress") + if record["state"] == "fired": + return _result(request, record, "timer_already_fired") + if record["state"] == "pending": + record.update(state="cancelled", revision=str(int(record["revision"]) + 1)) + elif operation == "claim_fire": + if record is None: + return _result(request, None, "timer_not_found") + if request["arguments"]["expected_revision"] != record["revision"]: + return _result(request, record, "timer_revision_conflict") + if record["state"] == "fired": + return _result(request, record, "timer_already_fired") + if record["state"] == "cancelled": + return _result(request, record, "timer_cancelled") + now = self._now() + if record["state"] == "claimed": + if now < _time(record["expires_at"]): + return _result(request, record, "timer_fire_in_progress") + # Dispatch through the retained implementation: an instance or + # class replacement must not bypass the checks inside the proof. + # Function objects are mutable too: attest the retained code + # before entering it, rather than letting it attest itself. + if _FATE_PROOF.__code__ is not _FATE_PROOF_CODE: + raise TimerError("delivery_ambiguous") + _FATE_PROOF(self, connection, artifact, record) + if now < _time(record["deadline_at"]): + return _result(request, record, "timer_not_due") + expires_at = now + self.claim_lease_nanoseconds + if expires_at > 2**63 - 1: + return _result(request, record, "timer_deadline_overflow") + record.update( + state="claimed", + revision=str(int(record["revision"]) + 1), + attempt_fence=str(int(record["attempt_fence"]) + 1), + worker_principal=principal, + expires_at=str(expires_at), + ) + else: + return _result(request, record, "timer_capability_mismatch") + result = _result(request, record) + artifact["operation_receipts"].append( + { + "operation_id": request["operation_id"], + "request_digest": request["request_digest"], + "result": result, + } + ) + self._record_commit(connection, request, result, artifact) + connection.commit() + return result + except TimerError as error: + return _result(request, record, error.code) + finally: + connection.rollback() + connection.close() + + +# Trusted local factory methods remain bound while an instance proves native fate. +# This guard supplements native origin/history; it is not external registration proof. +_FATE_PROOF = SQLiteTimerHelper._prove_local_uncommitted_fire +_FATE_PROOF_CODE = _FATE_PROOF.__code__ +_FATE_METHODS = tuple( + (name, getattr(SQLiteTimerHelper, name), getattr(SQLiteTimerHelper, name).__code__) + for name in ( + "execute", + "_complete", + "_load", + "_history", + "_record_commit", + "_persist", + "_schema", + "_configuration", + "_implementation_digest", + "_prove_local_uncommitted_fire", + ) +) + +_FATE_HOST_METHODS = tuple( + (name, getattr(ExecutionHost, name), getattr(ExecutionHost, name).__code__) + for name in ("run_shared_transaction", "_bound", "_restore", "_stage_replace", "admit_v1") +) +_FATE_STORE_METHODS = tuple( + (name, getattr(SQLiteExecutionStore, name), getattr(SQLiteExecutionStore, name).__code__) + for name in ("shared_transaction", "_connect", "_validate_schema") +) + +_FATE_SHARED_METHODS = tuple( + ( + name, + getattr(SharedExecutionTransaction, name), + getattr(SharedExecutionTransaction, name).__code__, + ) + for name in ("admit_v1", "read_checkpoint", "_stage", "_finish") +) diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index 6b26f25..1367e25 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -13,6 +13,7 @@ seal_journal, validate_journal, ) +from determa.state.native_clock import NativeCommitClock CASE = conformance_root() / "conformance/profiles/committed-native-effects/effect-01-result" @@ -92,8 +93,81 @@ def verify(source, actual): ) handler = VerifiedNativeHandler(registry, configured) return SQLiteCommittedEffectHost( - host.path, host.resolver, host.route, handler, trusted_clock=host.trusted_clock + host.path, + host.resolver, + host.route, + handler, + authority_scope=host.authority_scope, + trusted_clock=host.trusted_clock, + commit_clock=( + host.commit_clock + if host.commit_clock is not None + else NativeCommitClock.controlled(int(host.trusted_clock())) + if host.trusted_clock is not None + else None + ), + ) + + +def installed_retry_handler(host, root): + """Authenticate scoped native receipt bytes independently of worker input.""" + import base64 + import sqlite3 + + saved = host.snapshot(root) + record = saved["journal"]["effect_records"][0] + context = SQLiteCommittedEffectHost._retry_context(saved, record) + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} ) + destination = host.path + ".destination" + receipt = json.dumps( + {key: value for key, value in context.items() if key != "attempt_fence"}, sort_keys=True + ).encode() + with sqlite3.connect(destination) as connection: + connection.execute("CREATE TABLE receipts (scope TEXT, effect TEXT, receipt BLOB)") + connection.execute( + "INSERT INTO receipts VALUES (?, ?, ?)", + (context["scope_identity"], record["effect_id"], receipt), + ) + observations = [] + + def verify_receipts(evidence): + observations.append(copy.deepcopy(evidence)) + with sqlite3.connect(destination) as connection: + row = connection.execute( + "SELECT receipt FROM receipts WHERE scope=? AND effect=?", + (evidence["scope_identity"], evidence["effect_id"]), + ).fetchone() + return ( + row is not None + and all( + base64.b64decode(evidence[name], validate=True) == row[0] + for name in ( + "first_attempt_receipt_bytes_base64", + "repeat_attempt_receipt_bytes_base64", + ) + ) + and json.loads(row[0]) + == { + key: value + for key, value in evidence.items() + if key + not in { + "attempt_fence", + "first_attempt_receipt_bytes_base64", + "repeat_attempt_receipt_bytes_base64", + } + } + ) + + host = installed_test_handler(host, lambda *_: {}, verify_receipts) + proof = { + **context, + "first_attempt_receipt_bytes_base64": base64.b64encode(receipt).decode(), + "repeat_attempt_receipt_bytes_base64": base64.b64encode(receipt).decode(), + } + return host, proof, observations def host_fixture(tmp_path, observer=None): @@ -157,6 +231,183 @@ def test_terminal_outcome_requires_immutable_attempt_evidence(): validate_journal(read("pending-checkpoint.json"), seal_journal(journal)) +def test_other_trusted_root_incarnation_refuses_before_native_seed(tmp_path): + """An independently resolvable origin is not this checkpoint's root identity.""" + from determa.state.wire import hash_value + + checkpoint = read("pending-checkpoint.json") + journal = read("data/leased-journal.json") + original = load_bundle((CASE / "machine.yaml").read_text()) + alternate = load_bundle( + (CASE / "machine.yaml") + .read_text() + .replace( + "namespace: conformance.committed_native_effects", + "namespace: conformance.other_trusted_definition", + ) + ) + resolver = MemoryArtifactResolver( + definitions={original.fingerprint: original, alternate.fingerprint: alternate} + ) + record = journal["effect_records"][0] + origin = record["target"]["runtime_incarnation"] + definition = origin["definition"] + definition["validated_bundle_fingerprint"] = alternate.fingerprint + definition["machine"]["namespace"] = alternate.namespace + machine = definition["machine"] + record["target"]["runtime_id"] = hash_value( + [ + "determa-root-runtime-identity-1", + "1", + alternate.fingerprint, + machine["namespace"], + machine["machine_id"], + machine["machine_version"], + checkpoint["root_instance_id"], + ] + ) + journal = seal_journal(journal) + with pytest.raises(EffectError, match="invalid_effect_journal"): + validate_journal(checkpoint, journal, resolver) + host = SQLiteCommittedEffectHost(tmp_path / "invalid-seed.sqlite", resolver, {}, None) + host.setup_schema() + with pytest.raises(EffectError, match="invalid_effect_journal"): + host.seed(checkpoint, journal) + with host._connect() as connection: + assert ( + connection.execute("SELECT COUNT(*) FROM determa_committed_effects").fetchone()[0] == 0 + ) + + +@pytest.mark.parametrize("state", ["unclaimed", "leased", "ambiguous"]) +def test_prevented_start_without_cancelled_outcome_cannot_be_native_work(tmp_path, state): + checkpoint = read("pending-checkpoint.json") + journal = read("data/unclaimed-journal.json") + record = journal["effect_records"][0] + record["invocation_state"] = state + record["attempt_fence"] = "0" if state == "unclaimed" else "1" + record["cancellation"] = { + "operation_id": "cancel-before-claim", + "reason": "user_requested", + "state": "prevented_start", + } + _assert_invalid_native_seed(tmp_path, checkpoint, seal_journal(journal)) + + +@pytest.mark.parametrize("state", ["unclaimed", "leased", "ambiguous"]) +def test_stripping_terminal_outcome_never_makes_native_work_claimable(tmp_path, state): + checkpoint = read("pending-checkpoint.json") + journal = read("data/outcome-recorded-journal.json") + journal["effect_records"][0].update( + invocation_state=state, outcome=None, result_event_id=None, admission_receipt=None + ) + _assert_invalid_native_seed(tmp_path, checkpoint, seal_journal(journal)) + + +def _assert_invalid_native_seed(tmp_path, checkpoint, journal): + with pytest.raises(EffectError, match="invalid_effect_journal"): + validate_journal(checkpoint, journal) + bundle = load_bundle((CASE / "machine.yaml").read_text()) + resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) + fresh = SQLiteCommittedEffectHost(tmp_path / "bad-seed.sqlite", resolver, {}, None) + fresh.setup_schema() + with pytest.raises(EffectError, match="invalid_effect_journal"): + fresh.seed(checkpoint, journal) + with fresh._connect() as connection: + assert ( + connection.execute("SELECT COUNT(*) FROM determa_committed_effects").fetchone()[0] == 0 + ) + + +@pytest.mark.parametrize( + ("state", "report_kind", "report_fence"), + [ + ("unclaimed", "retryable_failure", "0"), + ("unclaimed", "ambiguous", "1"), + ("ambiguous", "retryable_failure", "1"), + ("leased", "retryable_failure", "1"), + ], +) +def test_completed_attempt_report_cannot_disagree_with_native_invocation_state( + state, report_kind, report_fence +): + from determa.state.wire import hash_value + + checkpoint = read("pending-checkpoint.json") + journal = read("data/leased-journal.json") + record = journal["effect_records"][0] + reason = ( + "no_call_proven" if report_kind == "retryable_failure" else "provider_acceptance_unknown" + ) + record["invocation_state"] = state + record["attempt_records"] = [ + { + "attempt_fence": report_fence, + "report_kind": report_kind, + "report_digest": hash_value( + [ + "determa-effect-attempt-report-1", + record["effect_id"], + record["operation_token"], + report_fence, + report_kind, + ["map", []], + reason, + ] + ), + "reason": reason, + } + ] + with pytest.raises(EffectError, match="invalid_effect_journal"): + validate_journal(checkpoint, seal_journal(journal)) + + +def test_terminal_outcome_cannot_supersede_a_newer_attempt_fence(): + journal = read("data/outcome-recorded-journal.json") + journal["effect_records"][0]["attempt_fence"] = "2" + with pytest.raises(EffectError, match="invalid_effect_journal"): + validate_journal(read("pending-checkpoint.json"), seal_journal(journal)) + + +def test_positive_fence_unclaimed_requires_current_safe_retry_report(tmp_path): + journal = read("data/leased-journal.json") + journal["effect_records"][0]["invocation_state"] = "unclaimed" + _assert_invalid_native_seed(tmp_path, read("pending-checkpoint.json"), seal_journal(journal)) + + +def test_terminal_outcome_cannot_follow_an_earlier_terminal_attempt(tmp_path): + from determa.state.wire import hash_value + + journal = read("data/outcome-recorded-journal.json") + record = journal["effect_records"][0] + outcome = record["outcome"] + report = copy.deepcopy(record["attempt_records"][0]) + record["attempt_fence"] = outcome["attempt_fence"] = report["attempt_fence"] = "2" + report["report_digest"] = hash_value( + [ + "determa-effect-attempt-report-1", + record["effect_id"], + record["operation_token"], + "2", + outcome["kind"], + outcome["payload"], + report["reason"], + ] + ) + outcome["digest"] = hash_value( + [ + "determa-effect-outcome-1", + record["effect_id"], + record["operation_token"], + outcome["kind"], + outcome["payload"], + "2", + ] + ) + record["attempt_records"].append(report) + _assert_invalid_native_seed(tmp_path, read("pending-checkpoint.json"), seal_journal(journal)) + + @pytest.mark.parametrize("field", ["digest", "attempt_fence"]) def test_terminal_outcome_rejects_forged_evidence(field): journal = copy.deepcopy(read("data/outcome-recorded-journal.json")) @@ -230,8 +481,8 @@ def test_worker_result_crossing_expiry_rolls_back_outcome_before_commit(tmp_path submit = host._submit observed = [] - def expires_during_submission(*args): - response = submit(*args) + def expires_during_submission(*args, **kwargs): + response = submit(*args, **kwargs) observed.append(args[0]["journal"]["effect_records"][0]["invocation_state"]) current_time[0] = read("data/active-claim.json")["expires_at"] return response @@ -573,6 +824,11 @@ def test_effect_mutations_refuse_a_torn_authority_journal_pair(tmp_path, changed else: document = host.snapshot(root) document["journal"]["effect_records"][0]["attempt_fence"] = "1" + # Keep the journal internally valid so this exercises the separate + # native authority/journal fence mismatch, not missing retry evidence. + document["journal"]["effect_records"][0]["attempt_records"] = read( + "data/retryable-journal.json" + )["effect_records"][0]["attempt_records"] document["journal"] = seal_journal(document["journal"]) with host._connect() as connection: connection.execute( @@ -621,27 +877,49 @@ def test_authority_claim_refuses_missing_native_participant_evidence(tmp_path, d def test_seed_preserves_historical_claim_without_reviving_it(tmp_path): authority, host, scope, root, record = authority_effect_fixture(tmp_path) - request, invocation = authority_claim_request(authority, scope, root, record) - claim = json.loads(authority.perform(request, invocation))["claim"] - # Model a retained completed attempt whose current journal permits a retry. - journal = read("data/retryable-journal.json") + command, invocation = authority_claim_request(authority, scope, root, record) + claim = json.loads(authority.perform(command, invocation))["claim"] + # Use the actual committed report transition rather than fabricating an older + # portable journal and editing native authority to make it look admissible. + report = { + "effect_id": record["effect_id"], + "operation_token": record["operation_token"], + "attempt_fence": claim["attempt_fence"], + "outcome_kind": "retryable_failure", + "payload": ["map", []], + } + host.trusted_clock = lambda: "0" + host, proof, _ = installed_retry_handler(host, root) + response = host.submit_result( + root, + report, + principal=claim["worker_principal"], + scope=scope, + epoch="0", + trusted_now="0", + deduplication_evidence=proof, + ) + assert response["status"] == "report_recorded" + saved = host.snapshot(root) ledger = authority.inspect(scope) - ledger["active_claims"] = [] + assert ledger["active_claims"] == [] assert claim in ledger["effect_claim_history"] - checkpoint = host.snapshot(root)["checkpoint"] with authority._connect() as connection: - connection.execute("UPDATE determa_scope_authority SET ledger=?", (json.dumps(ledger),)) connection.execute("DELETE FROM determa_committed_effects") connection.commit() - host.seed(checkpoint, journal, claim) + host.seed(saved["checkpoint"], saved["journal"], claim) assert authority.inspect(scope)["active_claims"] == [] assert host.snapshot(root)["claims"][record["effect_id"]] == claim new_claim = host.claim( - root, record["effect_id"], "worker-b", "0", expires_at="30", trusted_now="20" + root, + record["effect_id"], + "worker-b", + "0", + expires_at="30", + trusted_now="20", + deduplication_evidence=proof, ) assert new_claim["attempt_fence"] == "2" - assert authority.inspect(scope)["active_claims"] == [new_claim] - assert claim in authority.inspect(scope)["effect_claim_history"] def test_missing_native_work_cannot_be_rebound_to_another_authorized_root(tmp_path): @@ -733,7 +1011,9 @@ def test_multi_effect_recovery_commits_one_journal_revision_per_effect(tmp_path) journal["checkpoint_revision"] = checkpoint["revision"] journal["checkpoint_digest"] = checkpoint["execution_checkpoint_digest"] journal = seal_journal(journal) - host = SQLiteCommittedEffectHost(tmp_path / "two.sqlite", original.resolver, {}, None) + host = SQLiteCommittedEffectHost( + tmp_path / "two.sqlite", original.resolver, {}, None, trusted_clock=lambda: "0" + ) host.setup_schema() host.seed(checkpoint, journal) for record in journal["effect_records"]: @@ -777,7 +1057,16 @@ def external_acceptance(*args): with pytest.raises(EffectError, match="stale_attempt_fence"): host.dispatch(root, request["effect_id"], credential="test-credential", **context) assert len(calls) == 1 - assert host.snapshot(root) == before + after = host.snapshot(root) + assert after["checkpoint"] == before["checkpoint"] + assert after["journal"]["effect_records"] == before["journal"]["effect_records"] + assert after["journal"] == before["journal"] + assert after["responses"] == before["responses"] + assert len(after["invocation_starts"]) == 1 + current_time[0] = "0" + with pytest.raises(EffectError, match="native_invocation_already_started"): + host.dispatch(root, request["effect_id"], credential="test-credential", **context) + assert len(calls) == 1 assert host.recover(root)["journal"]["effect_records"][0]["invocation_state"] == "ambiguous" @@ -1101,6 +1390,10 @@ def test_historical_result_evidence_survives_processing(tmp_path): "fabricated", "wrong_work", "wrong_destination", + "wrong_root", + "wrong_token", + "wrong_fence", + "wrong_handler", "no_native_receipt", "unavailable", ], @@ -1111,7 +1404,9 @@ def test_ambiguous_retry_requires_verified_native_destination_evidence(tmp_path, original, root, request, _ = host_fixture(tmp_path) journal = read("data/ambiguous-journal.json") - host = SQLiteCommittedEffectHost(tmp_path / "ambiguous.sqlite", original.resolver, {}, None) + host = SQLiteCommittedEffectHost( + tmp_path / "ambiguous.sqlite", original.resolver, {}, None, trusted_clock=lambda: "0" + ) host.setup_schema() host.seed(read("pending-checkpoint.json"), journal) record = journal["effect_records"][0] @@ -1160,6 +1455,11 @@ def verify_receipts(evidence): host = installed_test_handler(host, lambda *_: {}, verify_receipts) encoded = base64.b64encode(receipt_bytes).decode() proof = { + "kind": "destination_deduplication", + "root_instance_id": root, + "operation_token": record["operation_token"], + "attempt_fence": record["attempt_fence"], + "handler_reference": copy.deepcopy(record["handler_reference"]), "scope_identity": journal["scope_identity"], "effect_id": record["effect_id"], "destination_binding_digest": record["destination_binding_digest"], @@ -1176,6 +1476,14 @@ def verify_receipts(evidence): proof["effect_id"] = "sha256:" + "0" * 64 elif damage == "wrong_destination": proof["destination_binding_digest"] = "sha256:" + "0" * 64 + elif damage == "wrong_root": + proof["root_instance_id"] = "other-root" + elif damage == "wrong_token": + proof["operation_token"] = "other-token" + elif damage == "wrong_fence": + proof["attempt_fence"] = "0" + elif damage == "wrong_handler": + proof["handler_reference"]["identifier"] = "other-handler" before = host.snapshot(root) if damage is not None: with pytest.raises(EffectError, match="host_capability_mismatch"): @@ -1225,6 +1533,425 @@ def test_ambiguous_retry_does_not_accept_legacy_boolean_keyword(tmp_path): ) +@pytest.mark.parametrize( + "damage", + [ + None, + "missing", + "boolean", + "fabricated", + "wrong_root", + "wrong_token", + "wrong_fence", + "wrong_handler", + "unavailable", + ], +) +def test_retryable_report_requires_native_original_invocation_proof(tmp_path, damage): + import base64 + + host, root, request, context = host_fixture(tmp_path) + host, proof, checks = installed_retry_handler(host, root) + report = {**request, "outcome_kind": "retryable_failure", "payload": ["map", []]} + if damage == "missing": + proof = None + elif damage == "boolean": + proof = True + elif damage == "fabricated": + proof["first_attempt_receipt_bytes_base64"] = proof[ + "repeat_attempt_receipt_bytes_base64" + ] = base64.b64encode(b"fabricated equal receipts").decode() + elif damage == "wrong_root": + proof["root_instance_id"] = "other-root" + elif damage == "wrong_token": + proof["operation_token"] = "other-token" + elif damage == "wrong_fence": + proof["attempt_fence"] = "0" + elif damage == "wrong_handler": + proof["handler_reference"]["identifier"] = "other-handler" + elif damage == "unavailable": + host.handler._configured._provider.proof_verifier = None + before = host.snapshot(root) + response = host.submit_result(root, report, **context, deduplication_evidence=proof) + if damage is not None: + assert response["error_code"] == "host_capability_mismatch" + assert host.snapshot(root) == before + else: + assert response["status"] == "report_recorded" + assert response["attempt_report"]["reason"] == "destination_deduplication_proven" + assert checks == [proof, proof] + retained = host.snapshot(root)["destination_evidence"][request["effect_id"]][0] + assert retained == { + "decision": "report", + "attempt_fence": request["attempt_fence"], + "evidence": proof, + } + + +@pytest.mark.parametrize("operation", ["report", "claim"]) +@pytest.mark.parametrize("receipt_type", [bytes, bytearray]) +def test_nonstring_native_receipts_refuse_before_checker_or_mutation( + tmp_path, operation, receipt_type +): + host, root, request, context = host_fixture(tmp_path) + if operation == "claim": + host = SQLiteCommittedEffectHost( + tmp_path / "nonstring-claim.sqlite", host.resolver, {}, None, trusted_clock=lambda: "0" + ) + host.setup_schema() + host.seed(read("pending-checkpoint.json"), read("data/ambiguous-journal.json")) + host, proof, checks = installed_retry_handler(host, root) + for field in ("first_attempt_receipt_bytes_base64", "repeat_attempt_receipt_bytes_base64"): + proof[field] = receipt_type(proof[field].encode("ascii")) + before = host.snapshot(root) + if operation == "report": + report = {**request, "outcome_kind": "retryable_failure", "payload": ["map", []]} + assert ( + host.submit_result(root, report, **context, deduplication_evidence=proof)["error_code"] + == "host_capability_mismatch" + ) + else: + with pytest.raises(EffectError, match="host_capability_mismatch"): + host.claim( + root, + request["effect_id"], + "worker", + context["epoch"], + expires_at="10", + trusted_now="0", + deduplication_evidence=proof, + ) + assert checks == [] + assert host.snapshot(root) == before + + +def test_safe_report_replay_and_later_claim_never_reuse_old_verification(tmp_path): + host, root, request, context = host_fixture(tmp_path) + host, proof, checks = installed_retry_handler(host, root) + report = {**request, "outcome_kind": "retryable_failure", "payload": ["map", []]} + first = host.submit_result(root, report, **context, deduplication_evidence=proof) + assert first["status"] == "report_recorded" + before = host.snapshot(root) + with pytest.raises(EffectError, match="host_capability_mismatch"): + host.claim( + root, + request["effect_id"], + "worker-b", + context["epoch"], + expires_at="10", + trusted_now="0", + ) + assert host.snapshot(root) == before + host.claim( + root, + request["effect_id"], + "worker-b", + context["epoch"], + expires_at="10", + trusted_now="0", + deduplication_evidence=proof, + ) + assert len(checks) == 4 + host.handler._configured._provider.proof_verifier = None + host.resolver = MemoryArtifactResolver(definitions={}) + with host._connect() as connection: + before_bytes = connection.execute( + "SELECT document FROM determa_committed_effects" + ).fetchone()[0] + assert host.submit_result(root, report, **context) == first + assert len(checks) == 4 + changed = {**report, "payload": ["map", [["changed", ["boolean", True]]]]} + assert host.submit_result(root, changed, **context)["error_code"] == "effect_result_conflict" + with host._connect() as connection: + assert ( + connection.execute("SELECT document FROM determa_committed_effects").fetchone()[0] + == before_bytes + ) + + +def test_authority_noninitial_native_claim_cannot_bypass_retry_verifier(tmp_path): + authority, host, scope, root, record = authority_effect_fixture(tmp_path) + command, invocation = authority_claim_request(authority, scope, root, record) + claim = json.loads(authority.perform(command, invocation))["claim"] + host.trusted_clock = lambda: "0" + host, proof, checks = installed_retry_handler(host, root) + report = { + "effect_id": record["effect_id"], + "operation_token": record["operation_token"], + "attempt_fence": claim["attempt_fence"], + "outcome_kind": "retryable_failure", + "payload": ["map", []], + } + assert ( + host.submit_result( + root, + report, + principal=claim["worker_principal"], + scope=scope, + epoch="0", + trusted_now="0", + deduplication_evidence=proof, + )["status"] + == "report_recorded" + ) + before = host.snapshot(root), authority.inspect(scope) + command, invocation = authority_claim_request( + authority, + scope, + root, + record, + expected="1", + operation="second-claim-without-native-verification", + ) + response = json.loads(authority.perform(command, invocation)) + assert response["status"] == "rejected" + assert response["error_code"] == "host_capability_mismatch" + assert (host.snapshot(root), authority.inspect(scope)) == before + assert len(checks) == 2 + + +@pytest.mark.parametrize( + "damage", + [ + None, + "unavailable", + "method", + "configuration", + "instance", + "clock_verifier", + "clock_method", + "clock_configuration", + "clock_instance", + "proof_expiry", + "proof_clock_unavailable", + "proof_clock_regression", + "proof_clock_replaced", + "proof_clock_descriptor", + "proof_clock_mode", + ], +) +@pytest.mark.parametrize("operation", ["claim", "report"]) +def test_retry_verifier_is_rechecked_after_native_sql_staging( + tmp_path, monkeypatch, damage, operation +): + import base64 + import sqlite3 + + original, root, request, context = host_fixture(tmp_path) + journal = read( + "data/ambiguous-journal.json" if operation == "claim" else "data/leased-journal.json" + ) + host = SQLiteCommittedEffectHost(tmp_path / "retry-staging.sqlite", original.resolver, {}, None) + host.setup_schema() + host.seed( + read("pending-checkpoint.json"), + journal, + read("data/active-claim.json") if operation == "report" else None, + ) + host.trusted_clock = lambda: "0" + record = journal["effect_records"][0] + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + receipt = b"independently stored destination receipt" + native_path = tmp_path / "retry-destination.sqlite" + with sqlite3.connect(native_path) as connection: + connection.execute("CREATE TABLE receipts (receipt BLOB NOT NULL)") + connection.execute("INSERT INTO receipts VALUES (?)", (receipt,)) + state = {"staged": False, "checks": 0, "late_callbacks": 0} + lease_deadline = ( + 10 if operation == "claim" else int(read("data/active-claim.json")["expires_at"]) + ) + + def verify_receipts(evidence): + state["checks"] += 1 + if state["staged"]: + if damage == "proof_expiry": + host.commit_clock.set_native_sample(lease_deadline) + elif damage == "proof_clock_unavailable": + host.commit_clock.set_native_sample(0, unavailable=True) + elif damage == "proof_clock_regression": + host.commit_clock.set_native_sample(-1) + elif damage == "proof_clock_replaced": + host.commit_clock = NativeCommitClock.controlled(0) + elif damage == "proof_clock_descriptor": + sample = NativeCommitClock._sample.__get__(host.commit_clock, NativeCommitClock) + + def late_sample(_clock): + state["late_callbacks"] += 1 + host.handler._configured._provider.proof_verifier = None + return sample + + monkeypatch.setattr(NativeCommitClock, "_sample", property(late_sample)) + elif damage == "proof_clock_mode": + host.commit_clock._system = True + if state["staged"] and damage == "unavailable": + raise RuntimeError("native verifier unavailable after staging") + with sqlite3.connect(native_path) as connection: + stored = connection.execute("SELECT receipt FROM receipts").fetchone()[0] + return all( + base64.b64decode(evidence[name], validate=True) == stored + for name in ( + "first_attempt_receipt_bytes_base64", + "repeat_attempt_receipt_bytes_base64", + ) + ) + + def trusted_now(): + if state["staged"]: + if damage == "clock_verifier": + host.handler._configured._provider.proof_verifier = None + elif damage == "clock_method": + monkeypatch.setattr( + NativeTestProvider, "verify_deduplication_evidence", lambda *_: True + ) + elif damage == "clock_configuration": + host.handler._configured._configuration["instance_id"] = "clock-substituted" + elif damage == "clock_instance": + object.__setattr__(host.handler._configured, "_instance", {}) + return "0" + + host.trusted_clock = trusted_now + host = installed_test_handler(host, lambda *_: {}, verify_receipts) + encoded = base64.b64encode(receipt).decode() + proof = { + "kind": "destination_deduplication", + "root_instance_id": root, + "operation_token": record["operation_token"], + "attempt_fence": record["attempt_fence"], + "handler_reference": copy.deepcopy(record["handler_reference"]), + "scope_identity": journal["scope_identity"], + "effect_id": record["effect_id"], + "destination_binding_digest": record["destination_binding_digest"], + "first_attempt_receipt_bytes_base64": encoded, + "repeat_attempt_receipt_bytes_base64": encoded, + } + before = host.snapshot(root) + + class StagingConnection(sqlite3.Connection): + def execute(self, sql, parameters=()): + cursor = super().execute(sql, parameters) + if sql.startswith("UPDATE determa_committed_effects SET document"): + # Observe the actual uncommitted row after SQLite has written it. + staged = json.loads( + super() + .execute( + "SELECT document FROM determa_committed_effects WHERE root_instance_id=?", + (root,), + ) + .fetchone()[0] + ) + assert staged["claims"][request["effect_id"]]["attempt_fence"] == ( + "2" if operation == "claim" else "1" + ) + if operation == "report": + assert staged["journal"]["effect_records"][0]["invocation_state"] == "unclaimed" + state["staged"] = True + if damage == "method": + monkeypatch.setattr( + NativeTestProvider, "verify_deduplication_evidence", lambda *_: True + ) + elif damage == "configuration": + host.handler._configured._configuration["instance_id"] = "substituted" + elif damage == "instance": + object.__setattr__(host.handler._configured, "_instance", {}) + return cursor + + monkeypatch.setattr( + host, + "_connect", + lambda: sqlite3.connect(host.path, isolation_level=None, factory=StagingConnection), + ) + arguments = {"expires_at": "10", "trusted_now": "0", "deduplication_evidence": proof} + if damage == "proof_clock_mode": + arguments["expires_at"] = read("data/active-claim.json")["expires_at"] + + def execute(): + if operation == "claim": + return host.claim(root, request["effect_id"], "worker", "0", **arguments) + response = host.submit_result( + root, + {**request, "outcome_kind": "retryable_failure", "payload": ["map", []]}, + **context, + deduplication_evidence=proof, + ) + if response["status"] == "rejected": + raise EffectError(response["error_code"]) + return response + + if damage is None: + response = execute() + assert response["attempt_fence"] == ("2" if operation == "claim" else "1") + assert state["staged"] and state["checks"] >= 2 + else: + code = ( + "stale_attempt_fence" + if damage + in { + "proof_expiry", + "proof_clock_unavailable", + "proof_clock_regression", + "proof_clock_descriptor", + "proof_clock_mode", + } + else "host_capability_mismatch" + ) + with pytest.raises(EffectError, match=code): + execute() + assert state["staged"] + assert host.snapshot(root) == before + assert state["late_callbacks"] == 0 + + +@pytest.mark.parametrize("clock", ["expired", "unavailable", "invalid", "missing"]) +def test_initial_claim_requires_fresh_clock_after_actual_sql_staging(tmp_path, monkeypatch, clock): + import sqlite3 + + original, root, request, _ = host_fixture(tmp_path) + state = {"staged": False} + + def now(): + if not state["staged"]: + return "0" + if clock == "unavailable": + raise RuntimeError("clock unavailable") + return "10" if clock == "expired" else "invalid" + + host = SQLiteCommittedEffectHost( + tmp_path / "claim-clock.sqlite", original.resolver, {}, None, trusted_clock=now + ) + host.setup_schema() + host.seed(read("pending-checkpoint.json"), read("data/unclaimed-journal.json")) + before = host.snapshot(root) + + class StagingConnection(sqlite3.Connection): + def execute(self, sql, parameters=()): + result = super().execute(sql, parameters) + if sql.startswith("UPDATE determa_committed_effects SET document"): + row = ( + super() + .execute( + "SELECT document FROM determa_committed_effects WHERE root_instance_id=?", + (root,), + ) + .fetchone() + ) + assert json.loads(row[0])["claims"][request["effect_id"]]["attempt_fence"] == "1" + state["staged"] = True + if clock == "missing": + host.trusted_clock = None + return result + + monkeypatch.setattr( + host, + "_connect", + lambda: sqlite3.connect(host.path, isolation_level=None, factory=StagingConnection), + ) + with pytest.raises(EffectError, match="stale_attempt_fence"): + host.claim(root, request["effect_id"], "worker", "0", expires_at="10", trusted_now="0") + assert state["staged"] and host.snapshot(root) == before + + def test_historical_admission_survives_actual_root_completion_and_tombstone(tmp_path): import yaml @@ -1537,6 +2264,8 @@ def test_native_producer_replay_retains_exact_kind_and_request_identity(tmp_path def test_portable_producer_identity_is_not_inferred_from_result_destination(tmp_path): + from determa.state.effects import _reconstruct_producer_response + original, root, _, _ = host_fixture(tmp_path) checkpoint = read("pending-checkpoint.json") journal = read("data/unclaimed-journal.json") @@ -1547,15 +2276,372 @@ def test_portable_producer_identity_is_not_inferred_from_result_destination(tmp_ tmp_path / "different-target.sqlite", original.resolver, {}, None ) host.setup_schema() - host.seed(checkpoint, seal_journal(journal)) - arguments = { - "expected_revision": "stale", - "expected_digest": "stale", - "route_generation": "999", + journal = seal_journal(journal) + # This deliberately inconsistent destination must not become native work. + # The pure receipt reconstruction still identifies its producing operation + # from retained checkpoint evidence, rather than the result destination. + with pytest.raises(EffectError, match="invalid_effect_journal"): + host.seed(checkpoint, journal) + identity = { + "operation_kind": "produce", + "root_instance_id": root, + "target_runtime_id": source_runtime, "operation_token": record["operation_token"], } - assert host.produce(root, "produce-1", source_runtime, **arguments) == read( - "data/producer-response.json" - ) + assert _reconstruct_producer_response( + checkpoint, journal, "produce-1", identity, original.resolver + ) == read("data/producer-response.json") + identity["target_runtime_id"] = record["target"]["runtime_id"] with pytest.raises(EffectError, match="replay_evidence_expired"): - host.produce(root, "produce-1", record["target"]["runtime_id"], **arguments) + _reconstruct_producer_response( + checkpoint, journal, "produce-1", identity, original.resolver + ) + + +def test_dispatch_retained_start_denies_repeated_and_restarted_calls(tmp_path): + host, root, request, context = host_fixture(tmp_path) + record = host.snapshot(root)["journal"]["effect_records"][0] + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + calls = [] + host = installed_test_handler(host, lambda *args: calls.append(args) or {"accepted": True}) + assert host.dispatch(root, request["effect_id"], credential="test-credential", **context) == { + "accepted": True + } + after = host.snapshot(root) + assert len(calls) == 1 + with pytest.raises(EffectError, match="native_invocation_already_started"): + host.dispatch(root, request["effect_id"], credential="test-credential", **context) + reopened = SQLiteCommittedEffectHost( + host.path, host.resolver, host.route, host.handler, trusted_clock=lambda: "0" + ) + with pytest.raises(EffectError, match="native_invocation_already_started"): + reopened.dispatch(root, request["effect_id"], credential="test-credential", **context) + assert reopened.snapshot(root) == after + assert len(calls) == 1 + + +def test_dispatch_provider_error_retains_start_and_cannot_be_retried_by_reopening(tmp_path): + host, root, request, context = host_fixture(tmp_path) + before = host.snapshot(root) + record = before["journal"]["effect_records"][0] + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + calls = [] + + def failed_call(*args): + calls.append(args) + raise RuntimeError("external acceptance is unknown") + + host = installed_test_handler(host, failed_call) + with pytest.raises(RuntimeError, match="acceptance is unknown"): + host.dispatch(root, request["effect_id"], credential="test-credential", **context) + after = host.snapshot(root) + assert after["checkpoint"] == before["checkpoint"] + assert after["journal"]["effect_records"] == before["journal"]["effect_records"] + reopened = SQLiteCommittedEffectHost( + host.path, host.resolver, host.route, host.handler, trusted_clock=lambda: "0" + ) + with pytest.raises(EffectError, match="native_invocation_already_started"): + reopened.dispatch(root, request["effect_id"], credential="test-credential", **context) + recovered = reopened.recover(root) + assert recovered["journal"]["effect_records"][0]["invocation_state"] == "ambiguous" + assert len(calls) == 1 + + +def test_dispatch_clock_is_rechecked_after_provider_verification_before_io(tmp_path, monkeypatch): + import sqlite3 + + host, root, request, context = host_fixture(tmp_path) + before = host.snapshot(root) + record = before["journal"]["effect_records"][0] + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + current_time = ["0"] + expiry = read("data/active-claim.json")["expires_at"] + + def health(self, instance): + with sqlite3.connect(host.path) as connection: + document = json.loads( + connection.execute("SELECT document FROM determa_committed_effects").fetchone()[0] + ) + if document["invocation_starts"]: + current_time[0] = expiry + return "healthy" + + monkeypatch.setattr(NativeTestProvider, "health", health) + calls = [] + host.trusted_clock = lambda: current_time[0] + host = installed_test_handler(host, lambda *args: calls.append(args) or {}) + with pytest.raises(EffectError, match="stale_attempt_fence"): + host.dispatch(root, request["effect_id"], credential="test-credential", **context) + after = host.snapshot(root) + assert after["checkpoint"] == before["checkpoint"] + assert after["journal"]["effect_records"] == before["journal"]["effect_records"] + assert len(after["invocation_starts"]) == 1 + assert calls == [] + + +def test_dispatch_postcall_health_loss_retains_start_without_returning_verified_output( + tmp_path, monkeypatch +): + host, root, request, context = host_fixture(tmp_path) + record = host.snapshot(root)["journal"]["effect_records"][0] + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + healthy = [True] + monkeypatch.setattr( + NativeTestProvider, "health", lambda *args: "healthy" if healthy[0] else "unavailable" + ) + calls = [] + + def external_acceptance(*args): + calls.append(args) + healthy[0] = False + return {"accepted": True} + + host = installed_test_handler(host, external_acceptance) + with pytest.raises(EffectError, match="host_capability_mismatch"): + host.dispatch(root, request["effect_id"], credential="test-credential", **context) + after = host.snapshot(root) + healthy[0] = True + with pytest.raises(EffectError, match="native_invocation_already_started"): + host.dispatch(root, request["effect_id"], credential="test-credential", **context) + assert host.snapshot(root) == after + assert len(calls) == 1 + + +def test_identical_dispatch_contenders_issue_one_native_call_under_consuming_lock(tmp_path): + import sqlite3 + import threading + from concurrent.futures import ThreadPoolExecutor + + host, root, request, context = host_fixture(tmp_path) + record = host.snapshot(root)["journal"]["effect_records"][0] + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + entered, release = threading.Event(), threading.Event() + calls = [] + + def native_call(*args): + calls.append(args) + entered.set() + assert release.wait(5) + return {"accepted": True} + + host = installed_test_handler(host, native_call) + rendezvous = threading.Barrier(3) + + def contender(): + rendezvous.wait(timeout=5) + try: + return host.dispatch( + root, request["effect_id"], credential="test-credential", **context + ) + except EffectError as error: + return str(error) + + with ThreadPoolExecutor(max_workers=2) as workers: + futures = [workers.submit(contender) for _ in range(2)] + rendezvous.wait(timeout=5) + try: + assert entered.wait(5) + with sqlite3.connect(host.path, timeout=0, isolation_level=None) as writer: + with pytest.raises(sqlite3.OperationalError, match="locked"): + writer.execute("BEGIN IMMEDIATE") + assert len(calls) == 1 + finally: + release.set() + replies = [future.result(timeout=5) for future in futures] + assert replies.count({"accepted": True}) == 1 + assert replies.count("native_invocation_already_started") == 1 + assert len(calls) == 1 + + +def test_dispatch_native_call_excludes_a_real_competing_result_transaction(tmp_path, monkeypatch): + import sqlite3 + + host, root, request, context = host_fixture(tmp_path) + record = host.snapshot(root)["journal"]["effect_records"][0] + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + other = SQLiteCommittedEffectHost( + host.path, host.resolver, host.route, None, trusted_clock=lambda: "0" + ) + native_connect = other._connect + + def without_waiting(): + connection = native_connect() + connection.execute("PRAGMA busy_timeout=0") + return connection + + monkeypatch.setattr(other, "_connect", without_waiting) + calls = [] + + def native_call(*args): + calls.append(args) + with pytest.raises(sqlite3.OperationalError, match="locked"): + other.submit_result(root, request, **context) + return {"accepted": True} + + host = installed_test_handler(host, native_call) + assert host.dispatch(root, request["effect_id"], credential="test-credential", **context) == { + "accepted": True + } + after = host.snapshot(root) + assert after["journal"]["effect_records"][0]["invocation_state"] == "leased" + assert after["journal"]["effect_records"][0]["outcome"] is None + assert len(calls) == 1 + assert other.submit_result(root, request, **context)["status"] == "committed" + + +@pytest.mark.parametrize("damage", [None, "missing_indexes", "empty_indexes"]) +def test_native_authority_seed_cannot_erase_a_committed_dispatch_start(tmp_path, damage): + import sqlite3 + + authority, host, scope, root, record = authority_effect_fixture(tmp_path) + command, invocation = authority_claim_request(authority, scope, root, record) + assert json.loads(authority.perform(command, invocation))["status"] == "accepted" + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + host.trusted_clock = lambda: "0" + calls = [] + host = installed_test_handler(host, lambda *args: calls.append(args) or {"accepted": True}) + before = host.snapshot(root) + claim = before["claims"][record["effect_id"]] + context = { + "principal": claim["worker_principal"], + "scope": scope, + "epoch": "0", + "trusted_now": "0", + } + host.dispatch(root, record["effect_id"], credential="test-credential", **context) + with sqlite3.connect(host.path) as connection: + if damage is not None: + retained = json.loads( + connection.execute("SELECT ledger FROM determa_scope_authority").fetchone()[0] + ) + for key in ("native_effect_roots", "native_effect_document_bytes"): + if damage == "missing_indexes": + del retained[key] + else: + retained[key] = [] + # Dedicated journal/work/issued-claim evidence still proves prior ownership. + assert retained["native_effect_journal_bytes"] and retained["native_effect_work"] + connection.execute( + "UPDATE determa_scope_authority SET ledger=?", (json.dumps(retained),) + ) + connection.execute( + "DELETE FROM determa_committed_effects WHERE root_instance_id=?", (root,) + ) + ledger = authority.inspect(scope) + with pytest.raises(EffectError, match="unauthorized_scope"): + host.seed(before["checkpoint"], before["journal"], claim) + assert authority.inspect(scope) == ledger + assert len(calls) == 1 + + +@pytest.mark.parametrize( + "damage", ["erase_start", "older_document", "missing_native_history", "missing_native_work"] +) +def test_private_dispatch_start_loss_cannot_be_blessed_by_dispatch_or_authority(tmp_path, damage): + import sqlite3 + + authority, host, scope, root, record = authority_effect_fixture(tmp_path) + command, invocation = authority_claim_request(authority, scope, root, record) + claim = json.loads(authority.perform(command, invocation))["claim"] + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + host.trusted_clock = lambda: "0" + calls = [] + host = installed_test_handler(host, lambda *args: calls.append(args) or {"accepted": True}) + context = { + "principal": claim["worker_principal"], + "scope": scope, + "epoch": "0", + "trusted_now": "0", + } + before = host.snapshot(root) + host.dispatch(root, record["effect_id"], credential="test-credential", **context) + after = host.snapshot(root) + assert after["checkpoint"] == before["checkpoint"] + assert after["journal"] == before["journal"] + assert after["responses"] == before["responses"] + assert len(after["invocation_starts"]) == 1 + fresh_command, fresh_invocation = authority_claim_request( + authority, scope, root, record, operation="after-private-start", expected="1" + ) + with sqlite3.connect(host.path) as connection: + if damage in {"missing_native_history", "missing_native_work"}: + ledger = json.loads( + connection.execute("SELECT ledger FROM determa_scope_authority").fetchone()[0] + ) + del ledger[ + "native_effect_document_bytes" + if damage == "missing_native_history" + else "native_effect_work" + ] + connection.execute("UPDATE determa_scope_authority SET ledger=?", (json.dumps(ledger),)) + else: + corrupted = copy.deepcopy(after if damage == "erase_start" else before) + corrupted["invocation_starts"] = {} + connection.execute( + "UPDATE determa_committed_effects SET document=?", (json.dumps(corrupted).encode(),) + ) + with pytest.raises(EffectError, match="unauthorized_scope"): + host.dispatch(root, record["effect_id"], credential="test-credential", **context) + # The authority's generic mutation validation must not certify the corrupt + # private inventory by mirroring it into a new trusted native history entry. + assert json.loads(authority.perform(command, invocation))["claim"] == claim + assert json.loads(authority.perform(fresh_command, fresh_invocation))["status"] == "rejected" + assert len(calls) == 1 + + +@pytest.mark.parametrize("operation", ["claim", "report"]) +def test_fresh_retry_refuses_a_callback_clock_without_a_sealed_native_reader(tmp_path, operation): + original, root, request, context = host_fixture(tmp_path) + if operation == "claim": + host = SQLiteCommittedEffectHost( + tmp_path / "callback-clock.sqlite", + original.resolver, + {}, + None, + trusted_clock=lambda: "0", + ) + host.setup_schema() + host.seed(read("pending-checkpoint.json"), read("data/ambiguous-journal.json")) + else: + host = original + host, proof, checks = installed_retry_handler(host, root) + host.commit_clock = None + before = host.snapshot(root) + if operation == "claim": + with pytest.raises(EffectError, match="host_capability_mismatch"): + host.claim( + root, + request["effect_id"], + "worker", + "0", + expires_at="10", + trusted_now="0", + deduplication_evidence=proof, + ) + else: + result = host.submit_result( + root, + {**request, "outcome_kind": "retryable_failure", "payload": ["map", []]}, + **context, + deduplication_evidence=proof, + ) + assert result["status"] == "rejected" + assert result["error_code"] == "host_capability_mismatch" + assert checks == [] + assert host.snapshot(root) == before diff --git a/tests/test_effect_cancellation_parity.py b/tests/test_effect_cancellation_parity.py new file mode 100644 index 0000000..d0ddd51 --- /dev/null +++ b/tests/test_effect_cancellation_parity.py @@ -0,0 +1,116 @@ +"""Late cancellation validates pinned payloads and preserves terminal evidence.""" + +import copy +import json +import sqlite3 + +import pytest + +from determa.state.effects import EffectError +from tests.test_committed_effects import ( + authority_claim_request, + authority_effect_fixture, + installed_test_handler, +) + + +def database_snapshot(host): + with sqlite3.connect(host.path) as connection: + return tuple(connection.iterdump()) + + +def invoked_host(tmp_path, state): + authority, host, scope, root, record = authority_effect_fixture(tmp_path) + command, invocation = authority_claim_request(authority, scope, root, record) + result = json.loads(authority.perform(command, invocation)) + assert result["status"] == "accepted" + claim = result["claim"] + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + host.trusted_clock = lambda: "0" + calls = [] + + def sdk_call(*args): + calls.append(args) + return {"provider_reference": record["handler_reference"]["identifier"]} + + host = installed_test_handler(host, sdk_call) + context = { + "principal": claim["worker_principal"], + "scope": scope, + "epoch": "0", + "trusted_now": "0", + } + candidate = host.dispatch( + root, record["effect_id"], credential="native-test-credential", **context + ) + assert len(calls) == 1 + request = { + "effect_id": record["effect_id"], + "operation_token": record["operation_token"], + "attempt_fence": claim["attempt_fence"], + "outcome_kind": "succeeded", + "payload": ["map", [["provider_reference", ["string", candidate["provider_reference"]]]]], + } + if state == "outcome_recorded": + host._record_result(root, request, **context) + elif state == "result_admitted": + assert host.submit_result(root, request, **context)["status"] == "committed" + else: + assert state == "possible" + return host, root, record, calls + + +def cancellation(record, operation="late-cancel"): + return { + "operation_id": operation, + "effect_id": record["effect_id"], + "reason": "owner-stop", + "payload": ["map", []], + } + + +@pytest.mark.parametrize("state", ["outcome_recorded", "result_admitted"]) +def test_terminal_winner_returns_too_late_and_preserves_exact_record(tmp_path, state): + host, root, record, calls = invoked_host(tmp_path, state) + before = host.snapshot(root) + winning = copy.deepcopy(before["journal"]["effect_records"][0]) + request = cancellation(record) + reply = host.cancel(root, request) + assert reply["status"] == "committed" + assert reply["cancellation"]["state"] == "too_late" + assert reply["outcome"] == winning["outcome"] + assert reply["result_event_id"] == winning["result_event_id"] + after = host.snapshot(root) + assert after["checkpoint"] == before["checkpoint"] + assert after["journal"]["effect_records"][0] == winning + retained = database_snapshot(host) + assert host.cancel(root, request) == reply + assert database_snapshot(host) == retained + assert len(calls) == 1 + + +@pytest.mark.parametrize("state", ["possible", "outcome_recorded", "result_admitted"]) +def test_invalid_late_cancellation_payload_preserves_all_native_roles(tmp_path, state): + host, root, record, calls = invoked_host(tmp_path, state) + request = cancellation(record) + request["payload"] = ["map", [["undeclared", ["string", "value"]]]] + before = database_snapshot(host) + with pytest.raises(EffectError, match="invalid_host_request"): + host.cancel(root, request) + assert database_snapshot(host) == before + assert len(calls) == 1 + + +def test_new_late_cancel_preserves_original_prevented_start_evidence(tmp_path): + _, host, _, root, record = authority_effect_fixture(tmp_path) + first_request = cancellation(record, "cancel-first") + first = host.cancel(root, first_request) + assert first["cancellation"]["state"] == "prevented_start" + before = host.snapshot(root) + reply = host.cancel(root, cancellation(record, "cancel-late")) + assert reply["status"] == "committed" + assert reply["cancellation"]["state"] == "too_late" + assert reply["outcome"] == first["outcome"] + assert host.snapshot(root)["journal"]["effect_records"] == before["journal"]["effect_records"] diff --git a/tests/test_effect_dispatch_native_fate.py b/tests/test_effect_dispatch_native_fate.py new file mode 100644 index 0000000..9f03846 --- /dev/null +++ b/tests/test_effect_dispatch_native_fate.py @@ -0,0 +1,215 @@ +"""SIGKILL retains native start/outcome facts across actual independent SQLite I/O.""" + +from __future__ import annotations + +import copy +import json +import multiprocessing +import os +import signal +import sqlite3 +import time +from pathlib import Path + +import pytest + +from determa.state import MemoryArtifactResolver, load_bundle +from determa.state.effects import EffectError, SQLiteCommittedEffectHost + +from .test_committed_effects import ( + CASE, + NativeTestProvider, + authority_claim_request, + authority_effect_fixture, + installed_test_handler, +) + + +def _dispatch_worker(directory, cut, control): + authority, host, scope, root, record = authority_effect_fixture(Path(directory)) + command, invocation = authority_claim_request(authority, scope, root, record) + claim = json.loads(authority.perform(command, invocation))["claim"] + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + host.trusted_clock = lambda: "0" + context = { + "principal": claim["worker_principal"], + "scope": scope, + "epoch": "0", + "trusted_now": "0", + } + information = {"root": root, "scope": scope, "effect_id": record["effect_id"], "claim": claim} + provider_path = Path(directory) / "provider.sqlite" + + def health(self, instance): + if cut == "start": + with sqlite3.connect(host.path) as connection: + document = json.loads( + connection.execute("SELECT document FROM determa_committed_effects").fetchone()[ + 0 + ] + ) + if document["invocation_starts"]: + control.send(information) + time.sleep(60) + return "healthy" + + NativeTestProvider.health = health + + def native_call(payload, metadata, attempt): + # SDK objects stay in this native callback. The independent destination + # transaction commits before acceptance is reported to the host. + class SdkReceipt: + def __init__(self): + self.reference = "provider:" + metadata["effect_id"] + + receipt = SdkReceipt() + evidence = { + "scope_identity": metadata["scope_identity"], + "effect_id": metadata["effect_id"], + "operation_token": metadata["operation_token"], + "attempt_fence": attempt["attempt_fence"], + "request": payload, + "provider_reference": receipt.reference, + } + assert metadata["credential"] == "test-credential" + with sqlite3.connect(provider_path) as destination: + destination.execute("PRAGMA journal_mode=WAL") + destination.execute("PRAGMA synchronous=FULL") + destination.execute( + "CREATE TABLE receipts (scope TEXT NOT NULL, effect TEXT NOT NULL, " + "receipt TEXT NOT NULL, PRIMARY KEY(scope,effect))" + ) + destination.execute( + "INSERT INTO receipts VALUES (?,?,?)", + (scope, record["effect_id"], json.dumps(evidence, sort_keys=True)), + ) + destination.commit() + if cut == "acceptance": + control.send(information) + time.sleep(60) + return { + "report_kind": "succeeded", + "payload": ["map", [["provider_reference", ["string", receipt.reference]]]], + "reason": None, + } + + host = installed_test_handler(host, native_call) + reply = host.dispatch(root, record["effect_id"], credential="test-credential", **context) + if cut == "response": + control.send(information) + time.sleep(60) + + def outcome_committed(*args): + control.send(information) + time.sleep(60) + + if cut == "outcome": + host.core_observer = outcome_committed + request = { + "effect_id": record["effect_id"], + "operation_token": record["operation_token"], + "attempt_fence": claim["attempt_fence"], + "outcome_kind": reply["report_kind"], + "payload": reply["payload"], + } + response = host.submit_result(root, request, **context) + assert cut == "admission" and response["status"] == "committed" + information["response"] = response + control.send(information) + time.sleep(60) + + +@pytest.mark.skipif(os.name != "posix", reason="requires actual SIGKILL") +@pytest.mark.parametrize("cut", ["start", "acceptance", "response", "outcome", "admission"]) +def test_sigkill_native_dispatch_never_repeats_uncertain_provider_call(tmp_path, cut): + context = multiprocessing.get_context("spawn") + parent, child = context.Pipe() + worker = context.Process(target=_dispatch_worker, args=(str(tmp_path), cut, child)) + worker.start() + child.close() + try: + assert parent.poll(20), f"worker did not reach {cut} (exit={worker.exitcode})" + information = parent.recv() + worker.kill() + worker.join(5) + assert worker.exitcode == -signal.SIGKILL + finally: + if worker.is_alive(): + worker.kill() + worker.join(5) + parent.close() + bundle = load_bundle((CASE / "machine.yaml").read_text()) + resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) + route = {"authority_epoch": "0"} + base = SQLiteCommittedEffectHost( + tmp_path / "paired.sqlite", + resolver, + route, + None, + authority_scope=information["scope"], + trusted_clock=lambda: "0", + ) + before = base.snapshot(information["root"]) + record = before["journal"]["effect_records"][0] + base.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + calls = [] + host = installed_test_handler(base, lambda *args: calls.append(args) or {}) + invocation = { + "principal": information["claim"]["worker_principal"], + "scope": information["scope"], + "epoch": "0", + "trusted_now": "0", + } + with pytest.raises(EffectError): + host.dispatch( + information["root"], + information["effect_id"], + credential="test-credential", + **invocation, + ) + assert host.snapshot(information["root"]) == before + assert calls == [] + assert len(before["invocation_starts"]) == 1 + start = next(iter(before["invocation_starts"].values())) + assert start["claim"] == information["claim"] + assert start["effect_record"]["effect_id"] == information["effect_id"] + assert record["invocation_state"] == ( + "result_admitted" + if cut == "admission" + else "outcome_recorded" + if cut == "outcome" + else "leased" + ) + if cut == "start": + assert not (tmp_path / "provider.sqlite").exists() + else: + with sqlite3.connect(tmp_path / "provider.sqlite") as destination: + rows = destination.execute("SELECT scope,effect,receipt FROM receipts").fetchall() + assert len(rows) == 1 + scope, effect, encoded = rows[0] + receipt = json.loads(encoded) + assert (scope, effect) == (information["scope"], information["effect_id"]) + assert receipt["operation_token"] == record["operation_token"] + assert receipt["attempt_fence"] == "1" + assert ( + receipt["request"] + == before["checkpoint"]["pending_outbox_intents"][0]["intent"]["payload"] + ) + host.trusted_clock = lambda: information["claim"]["expires_at"] + recovered = host.recover(information["root"]) + assert calls == [] + assert recovered["journal"]["effect_records"][0]["invocation_state"] == ( + "result_admitted" if cut in {"outcome", "admission"} else "ambiguous" + ) + if cut in {"outcome", "admission"}: + assert int(recovered["checkpoint"]["revision"]) == 3 + assert recovered["journal"]["effect_records"][0]["outcome"] == record["outcome"] + else: + assert recovered["checkpoint"] == before["checkpoint"] + assert recovered["journal"]["effect_records"][0]["outcome"] is None + saved = copy.deepcopy(recovered) + assert host.recover(information["root"]) == saved diff --git a/tests/test_local_authority.py b/tests/test_local_authority.py index 7d49f42..a244dda 100644 --- a/tests/test_local_authority.py +++ b/tests/test_local_authority.py @@ -271,7 +271,146 @@ def test_freeze_refuses_an_untracked_native_checkpoint_without_committing(tmp_pa response = authority.perform(_request("freeze_scope", "freeze-untracked", "0", {}), invocation) assert json.loads(response)["error_code"] == "scope_fence_unproven" assert authority.inspect("scope-1") == ledger - assert bare_host.read_checkpoint("foreign-root").canonical_bytes == before.canonical_bytes + with authority._connect() as connection: + actual = connection.execute( + "SELECT checkpoint FROM determa_execution_checkpoints WHERE root_instance_id=?", + ("foreign-root",), + ).fetchone() + assert actual is not None and bytes(actual[0]) == before.canonical_bytes + with pytest.raises(ExecutionStoreError, match="scope_fence_unproven"): + bare_host.read_checkpoint("foreign-root") + + +@pytest.mark.parametrize( + "damage", + [ + None, + "row", + "table", + "history", + "ownership", + "missing_indexes", + "empty_indexes", + "journal_history", + "missing_journal_history", + "wrong_journal_scope", + "unowned_work", + "malformed_work", + ], +) +def test_freeze_requires_complete_private_native_inventory_even_for_zero_effect_roots( + tmp_path, damage +): + from determa.state import create_checkpoint_v1 + from determa.state.effects import SQLiteCommittedEffectHost, seal_journal + + bundle = load_bundle( + "format: 1\nnamespace: test.private_inventory\nmachines:\n" + " - machine_id: simple\n root: {type: simple}\n" + ) + resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) + authority = SQLiteLocalAuthority(tmp_path / "inventory.sqlite") + authority.setup_schema() + assert authority.allocate("scope-1", "owner-1", roots=("root-a", "root-b")) + host = SQLiteCommittedEffectHost( + authority.path, resolver, {"authority_epoch": "0"}, None, authority_scope="scope-1" + ) + host.setup_schema() + for root in ("root-a", "root-b"): + checkpoint = create_checkpoint_v1(bundle, "simple", root, "create-" + root, {}) + document = checkpoint + journal = seal_journal( + { + "host_effect_journal_format": "determa.host_effect_journal", + "host_effect_journal_schema_version": 1, + "scope_identity": "scope-1", + "root_instance_id": root, + "checkpoint_revision": document["revision"], + "checkpoint_digest": document["execution_checkpoint_digest"], + "journal_revision": "0", + "effect_records": [], + "operation_response_references": [], + } + ) + host.seed(document, journal) + before = authority.inspect("scope-1") + if damage is not None: + with authority._connect() as connection: + if damage == "row": + connection.execute( + "DELETE FROM determa_committed_effects WHERE root_instance_id='root-b'" + ) + elif damage == "table": + connection.execute("DROP TABLE determa_committed_effects") + else: + ledger = json.loads( + connection.execute("SELECT ledger FROM determa_scope_authority").fetchone()[0] + ) + if damage == "history": + ledger["native_effect_document_bytes"] = [ + source + for source in ledger["native_effect_document_bytes"] + if json.loads(source)["checkpoint"]["root_instance_id"] != "root-b" + ] + elif damage == "ownership": + ledger["native_effect_roots"].remove("root-b") + elif damage in {"missing_indexes", "empty_indexes"}: + for key in ("native_effect_roots", "native_effect_document_bytes"): + if damage == "missing_indexes": + del ledger[key] + else: + ledger[key] = [] + connection.execute("DELETE FROM determa_committed_effects") + elif damage == "missing_journal_history": + del ledger["native_effect_journal_bytes"] + elif damage in {"journal_history", "wrong_journal_scope"}: + latest = json.loads(ledger["native_effect_journal_bytes"][-1]) + if damage == "journal_history": + latest["journal_revision"] = "99" + else: + latest["scope_identity"] = "other-scope" + from determa.state.wire import canonical_bytes + + ledger["native_effect_journal_bytes"][-1] = canonical_bytes( + seal_journal(latest) + ).decode() + elif damage == "unowned_work": + ledger["native_effect_work"] = [ + { + "participant": "native_effects", + "work_kind": "effect", + "root_instance_id": "unowned-root", + } + ] + else: + assert damage == "malformed_work" + ledger["native_effect_work"] = None + connection.execute( + "UPDATE determa_scope_authority SET ledger=?", (json.dumps(ledger),) + ) + invocation = { + "authenticated_principal": "owner-1", + "authorized_scopes": ["scope-1"], + "operation_rights": ["freeze_scope"], + } + result = json.loads( + authority.perform( + _request("freeze_scope", "freeze-native-inventory", before["scope_generation"], {}), + invocation, + ) + ) + if damage is None: + assert result["status"] == "accepted" and result["state"] == "frozen" + inventory = authority.inspect("scope-1")["inventory"] + assert {item["identity"] for item in inventory if item["kind"] == "root"} == { + "root-a", + "root-b", + } + assert sum(item["kind"] == "checkpoint" for item in inventory) >= 2 + else: + assert result["error_code"] == "scope_fence_unproven" + assert authority.inspect("scope-1")["state"] == "active" + assert authority.inspect("scope-1")["scope_generation"] == before["scope_generation"] def test_public_checkpoint_commit_serializes_with_freeze(tmp_path) -> None: @@ -380,7 +519,14 @@ def test_guarded_write_cannot_adopt_a_preexisting_untracked_checkpoint(tmp_path) expected_checkpoint_digest=before.document["execution_checkpoint_digest"], ) assert authority.inspect("scope-1") == ledger - assert bare_host.read_checkpoint("root-1").canonical_bytes == before.canonical_bytes + with authority._connect() as connection: + actual = connection.execute( + "SELECT checkpoint FROM determa_execution_checkpoints WHERE root_instance_id=?", + ("root-1",), + ).fetchone() + assert actual is not None and bytes(actual[0]) == before.canonical_bytes + with pytest.raises(ExecutionStoreError, match="scope_fence_unproven"): + bare_host.read_checkpoint("root-1") @pytest.mark.parametrize("terminal", [False, True]) diff --git a/tests/test_native_checkpoint_ownership.py b/tests/test_native_checkpoint_ownership.py new file mode 100644 index 0000000..91731cf --- /dev/null +++ b/tests/test_native_checkpoint_ownership.py @@ -0,0 +1,163 @@ +"""Ordinary public checkpoint writes must not split native effects ownership.""" + +import json +import sqlite3 + +import pytest + +from determa.state import delivery_request_digest, portable_envelope +from determa.state.authority import AuthoritySQLiteExecutionStore +from determa.state.host import ExecutionHost, ExecutionHostError +from determa.state.stores.base import ExecutionStoreError +from determa.state.stores.sqlite import SQLiteExecutionStore +from determa.state.wire import canonical_bytes + +from .test_committed_effects import authority_effect_fixture + + +def database_snapshot(path): + with sqlite3.connect(path) as connection: + return "\n".join(connection.iterdump()) + + +def fixture(tmp_path, *, legacy_checkpoint=False): + authority, effects, scope, root, _record = authority_effect_fixture(tmp_path) + checkpoint = effects.snapshot(root)["checkpoint"] + store = AuthoritySQLiteExecutionStore(authority, scope, "owner", "0") + store.setup_schema() + if legacy_checkpoint: + # Earlier adapters permitted an equal second checkpoint representation. + # Reproduce that extant native state without using the now-refused writer. + with sqlite3.connect(authority.path) as connection: + connection.execute( + "INSERT INTO determa_execution_checkpoints VALUES (?,?,?,?)", + ( + root, + checkpoint["revision"], + checkpoint["execution_checkpoint_digest"], + canonical_bytes(checkpoint), + ), + ) + return authority, effects, root, checkpoint, store + + +@pytest.mark.parametrize("mutation", ["insert", "external_admission"]) +def test_ordinary_checkpoint_mutations_refuse_effects_owned_root(tmp_path, mutation): + authority, effects, root, checkpoint, store = fixture( + tmp_path, legacy_checkpoint=mutation == "external_admission" + ) + before = database_snapshot(authority.path) + initial = effects.snapshot(root) + with pytest.raises((ExecutionStoreError, ExecutionHostError), match="scope_fence_unproven"): + if mutation == "insert": + with store.transaction(root) as transaction: + transaction.insert(canonical_bytes(checkpoint)) + else: + runtime = checkpoint["root_record"]["aggregate_state"]["runtimes"][0] + envelope = portable_envelope( + "native_cancelled", + "external-admission", + {"root": {"root_instance_id": root, "root_runtime_id": runtime["runtime_id"]}}, + {}, + ) + delivery = { + "delivery_mode": "input", + "envelope": envelope, + "envelope_digest": delivery_request_digest(root, "input", envelope), + } + ExecutionHost(store, effects.resolver).admit_v1( + root, + [delivery], + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + assert database_snapshot(authority.path) == before + assert effects.snapshot(root) == initial + + +def test_native_owned_transaction_refuses_before_ordinary_core_or_callback(tmp_path): + authority, effects, root, checkpoint, store = fixture(tmp_path, legacy_checkpoint=True) + before = database_snapshot(authority.path) + entered = False + with pytest.raises(ExecutionStoreError, match="scope_fence_unproven"): + with store.transaction(root): + entered = True + assert not entered + assert effects.snapshot(root)["checkpoint"] == checkpoint + assert database_snapshot(authority.path) == before + + +@pytest.mark.parametrize("shared", [False, True]) +def test_base_sqlite_adapter_cannot_enter_authority_owned_transaction(tmp_path, shared): + authority, effects, root, checkpoint, _store = fixture(tmp_path, legacy_checkpoint=True) + store = SQLiteExecutionStore(authority.path, shared_application_transactions=shared) + before = database_snapshot(authority.path) + entered = False + with pytest.raises(ExecutionStoreError, match="scope_fence_unproven"): + if shared: + with store.shared_transaction(root) as (application, transaction): + entered = True + application.execute("CREATE TABLE application_side_effect (value TEXT)") + assert transaction.load() is not None + else: + with store.transaction(root) as transaction: + entered = True + assert transaction.load() is not None + assert not entered + assert database_snapshot(authority.path) == before + assert effects.snapshot(root)["checkpoint"] == checkpoint + + +@pytest.mark.parametrize( + "marker", + [ + "determa_scope_authority", + "determa_scope_allocations", + "determa_committed_effects", + "DETERMA_SCOPE_AUTHORITY", + "Determa_Scope_Allocations", + "Determa_Committed_Effects", + ], +) +@pytest.mark.parametrize("shared", [False, True]) +def test_partial_native_ownership_markers_fail_closed(tmp_path, marker, shared): + store = SQLiteExecutionStore( + tmp_path / "partial.sqlite", shared_application_transactions=shared + ) + store.setup_schema() + with sqlite3.connect(store.path) as connection: + connection.execute(f"CREATE TABLE {marker} (damaged TEXT)") + before = database_snapshot(store.path) + entered = False + with pytest.raises(ExecutionStoreError, match="scope_fence_unproven"): + context = store.shared_transaction("root") if shared else store.transaction("root") + with context: + entered = True + assert not entered + assert database_snapshot(store.path) == before + + +@pytest.mark.parametrize("damage", ["missing_indexes", "empty_indexes"]) +def test_lost_native_indexes_cannot_reclassify_ordinary_writer_permission(tmp_path, damage): + authority, effects, root, checkpoint, store = fixture(tmp_path) + scope = effects.authority_scope + ledger = authority.inspect(scope) + for key in ("native_effect_roots", "native_effect_document_bytes"): + if damage == "missing_indexes": + del ledger[key] + else: + ledger[key] = [] + with authority._connect() as connection: + connection.execute( + "UPDATE determa_scope_authority SET ledger=? WHERE scope_identity=?", + (json.dumps(ledger), scope), + ) + connection.commit() + before = database_snapshot(authority.path) + entered = False + with pytest.raises(ExecutionStoreError, match="scope_fence_unproven"): + with store.transaction(root) as transaction: + entered = True + transaction.insert(canonical_bytes(checkpoint)) + assert not entered + assert database_snapshot(authority.path) == before diff --git a/tests/test_sqlite_shared_transaction.py b/tests/test_sqlite_shared_transaction.py new file mode 100644 index 0000000..2c96ec5 --- /dev/null +++ b/tests/test_sqlite_shared_transaction.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +import sqlite3 + +import pytest + +from determa.state import ExecutionHost, ExecutionStoreError, SQLiteExecutionStore, load_bundle +from determa.state.stores.base import SHARED_APPLICATION_TRANSACTION +from determa.state.stores.sqlite import sqlite_execution_store_factory + +from .test_execution_stores import MACHINE, _resolver + + +def configured(tmp_path): + store = SQLiteExecutionStore(tmp_path / "shared.sqlite", shared_application_transactions=True) + store.setup_schema() + with sqlite3.connect(store.path) as connection: + connection.execute("CREATE TABLE applications (root TEXT PRIMARY KEY)") + return store, ExecutionHost(store, _resolver()) + + +def test_sqlite_shared_transaction_is_explicit_and_commits_application_and_root(tmp_path): + store, host = configured(tmp_path) + assert SHARED_APPLICATION_TRANSACTION in store.capabilities + assert SHARED_APPLICATION_TRANSACTION not in SQLiteExecutionStore(store.path).capabilities + retained = [] + + def callback(sql, execution): + retained.append(sql) + sql.execute("INSERT INTO applications VALUES (?)", ("root",)) + execution.create_v1(load_bundle(MACHINE), "counter", "create", {}) + # Another reader sees neither uncommitted write. + with sqlite3.connect(store.path) as connection: + assert connection.execute("SELECT root FROM applications").fetchall() == [] + assert ( + connection.execute( + "SELECT root_instance_id FROM determa_execution_checkpoints" + ).fetchall() + == [] + ) + + response = host.run_shared_transaction("root", callback) + assert response["result"] == "committed" + with sqlite3.connect(store.path) as connection: + assert connection.execute("SELECT root FROM applications").fetchall() == [("root",)] + assert ( + ExecutionHost(SQLiteExecutionStore(store.path), _resolver()).read_checkpoint("root") + is not None + ) + with pytest.raises(ExecutionStoreError, match="shared_transaction_closed"): + retained[0].execute("SELECT 1") + + +def test_sqlite_shared_callback_failure_rolls_back_both_writes(tmp_path): + store, host = configured(tmp_path) + + def callback(sql, execution): + sql.execute("INSERT INTO applications VALUES (?)", ("root",)) + execution.create_v1(load_bundle(MACHINE), "counter", "create", {}) + raise RuntimeError("application refused") + + with pytest.raises(RuntimeError, match="application refused"): + host.run_shared_transaction("root", callback) + assert host.read_checkpoint("root") is None + with sqlite3.connect(store.path) as connection: + assert connection.execute("SELECT root FROM applications").fetchall() == [] + + +@pytest.mark.parametrize( + "statement", + [ + "COMMIT", + "ROLLBACK", + "SAVEPOINT early", + "PRAGMA synchronous=OFF", + "ATTACH ':memory:' AS other", + ], +) +def test_sqlite_application_callback_cannot_commit_or_change_durability(tmp_path, statement): + store, host = configured(tmp_path) + + def callback(sql, execution): + sql.execute("INSERT INTO applications VALUES (?)", ("root",)) + execution.create_v1(load_bundle(MACHINE), "counter", "create", {}) + sql.execute(statement) + + with pytest.raises(sqlite3.DatabaseError, match="not authorized"): + host.run_shared_transaction("root", callback) + assert host.read_checkpoint("root") is None + with sqlite3.connect(store.path) as connection: + assert connection.execute("SELECT root FROM applications").fetchall() == [] + + +def test_sqlite_default_store_refuses_shared_transaction(tmp_path): + store = SQLiteExecutionStore(tmp_path / "default.sqlite") + store.setup_schema() + host = ExecutionHost(store, _resolver()) + with pytest.raises(Exception, match="adapter_capability_mismatch"): + host.run_shared_transaction("root", lambda sql, execution: None) + + +def test_bundled_sqlite_uri_requires_an_explicit_shared_transaction_option(tmp_path): + path = tmp_path / "public-uri.sqlite" + ordinary = sqlite_execution_store_factory(f"sqlite://{path}", {}) + ordinary.setup_schema() + selected = sqlite_execution_store_factory( + f"sqlite://{path}?shared_application_transactions=true", {} + ) + assert SHARED_APPLICATION_TRANSACTION in selected.capabilities + assert SHARED_APPLICATION_TRANSACTION not in ordinary.capabilities + for invalid in ("1", "TRUE", "", "yes", "true&shared_application_transactions=false"): + with pytest.raises(ExecutionStoreError, match="invalid_adapter_configuration"): + sqlite_execution_store_factory( + f"sqlite://{path}?shared_application_transactions={invalid}", {} + ) + + +@pytest.mark.parametrize( + "statement", + [ + "DROP TRIGGER determa_execution_checkpoints_forbid_delete", + "DELETE FROM determa_execution_checkpoints WHERE root_instance_id='victim'", + "DROP TABLE determa_execution_checkpoints", + "UPDATE determa_execution_checkpoints SET revision='999' WHERE root_instance_id='victim'", + "CREATE TRIGGER app_delete AFTER INSERT ON applications " + "BEGIN DELETE FROM determa_execution_checkpoints; END", + ], +) +def test_application_sql_cannot_change_protected_engine_storage(tmp_path, statement): + store, host = configured(tmp_path) + host.create_v1(load_bundle(MACHINE), "counter", "victim", "create-victim", {}) + before = host.read_checkpoint("victim").document + + def callback(sql, execution): + execution.create_v1(load_bundle(MACHINE), "counter", "create-other", {}) + sql.execute(statement) + sql.execute("INSERT INTO applications VALUES ('trigger-attack')") + + with pytest.raises(sqlite3.DatabaseError, match="(not authorized|prohibited)"): + host.run_shared_transaction("other-root", callback) + assert host.read_checkpoint("victim").document == before + assert host.read_checkpoint("other-root") is None + assert store.health()["healthy"] diff --git a/tests/test_timer_coordinated_admission.py b/tests/test_timer_coordinated_admission.py new file mode 100644 index 0000000..e601760 --- /dev/null +++ b/tests/test_timer_coordinated_admission.py @@ -0,0 +1,337 @@ +from __future__ import annotations + +import copy +import multiprocessing +import os +import signal + +import pytest + +from determa.state import ExecutionHost, MemoryArtifactResolver, SQLiteExecutionStore +from determa.state.checkpoint_v1 import admit_checkpoint_v1 +from determa.state.host import delivery_request_digest +from determa.state.timers import SQLiteTimerHelper, timer_request_digest +from determa.state.wire import hash_value + +from .test_timers import bundle, request + + +def _killed_timer_worker(path, complete, boundary, control): + definition = bundle() + host = ExecutionHost( + SQLiteExecutionStore( + path, replay_retention="permanent", shared_application_transactions=True + ), + MemoryArtifactResolver(definitions={definition.fingerprint: definition}), + ) + helper = SQLiteTimerHelper( + path, + definition, + scope_identity=complete["scope_identity"], + root_instance_id=complete["root_instance_id"], + root_runtime_id=complete["root_runtime_id"], + principals=frozenset({"operator", "worker-A"}), + worker_principals=frozenset({"worker-A"}), + trusted_clock=lambda: "120", + coordinated_host=host, + ) + + def pause(): + control.send(boundary) + control.recv() + raise AssertionError("the parent must kill this process at the selected boundary") + + if boundary == "both_writes_staged": + persist = helper._persist + + def pause_after_staging(sql, artifact): + persist(sql, artifact) + pause() + + helper._persist = pause_after_staging + else: + + def pause_after_commit(observed): + if observed == "after_commit_before_response": + pause() + + host.fault_injector = pause_after_commit + helper.execute(complete, principal="worker-A") + + +@pytest.mark.skipif(os.name != "posix", reason="requires actual SIGKILL process termination") +@pytest.mark.parametrize("boundary", ["both_writes_staged", "after_commit_before_response"]) +def test_sigkill_restart_resolves_actual_native_timer_and_admission_fate(tmp_path, boundary): + helper, host, complete, expected, now, calls = configured(tmp_path) + before_helper = helper.snapshot() + before_checkpoint = host.read_checkpoint("server-1").document + context = multiprocessing.get_context("spawn") + parent, child = context.Pipe() + worker = context.Process( + target=_killed_timer_worker, args=(helper.path, complete, boundary, child) + ) + worker.start() + child.close() + try: + assert parent.poll(30), "worker never reached the selected native boundary" + assert parent.recv() == boundary + worker.kill() + worker.join(10) + assert worker.exitcode == -signal.SIGKILL + finally: + if worker.is_alive(): + worker.kill() + worker.join(10) + parent.close() + worker.close() + + # Reopen all production objects: no child response or exception is recovery proof. + restarted_host = ExecutionHost( + SQLiteExecutionStore( + helper.path, replay_retention="permanent", shared_application_transactions=True + ), + host.artifact_resolver, + ) + restarted = SQLiteTimerHelper( + helper.path, + helper.bundle, + scope_identity=helper.scope_identity, + root_instance_id=helper.root_instance_id, + root_runtime_id=helper.root_runtime_id, + principals=helper.principals, + worker_principals=helper.worker_principals, + trusted_clock=lambda: "120", + coordinated_host=restarted_host, + ) + if boundary == "both_writes_staged": + assert restarted.snapshot() == before_helper + assert restarted_host.read_checkpoint("server-1").document == before_checkpoint + else: + assert restarted.snapshot()["records"][0]["state"] == "fired" + assert restarted_host.read_checkpoint("server-1").document == expected + + def offline(): + raise AssertionError("committed replay must not read the clock") + + restarted.trusted_clock = offline + result = restarted.execute(complete, principal="worker-A") + assert result["status"] == "accepted" and result["delivery_state"] == "admitted" + committed = restarted.snapshot() + assert restarted_host.read_checkpoint("server-1").document == expected + assert restarted.execute(complete, principal="worker-A") == result + assert restarted.snapshot() == committed + assert restarted_host.read_checkpoint("server-1").document == expected + + +def configured(tmp_path, replay_retention="permanent"): + path = tmp_path / "coordinated.sqlite" + definition = bundle() + resolver = MemoryArtifactResolver(definitions={definition.fingerprint: definition}) + store = SQLiteExecutionStore( + path, replay_retention=replay_retention, shared_application_transactions=True + ) + store.setup_schema() + host = ExecutionHost(store, resolver) + host.create_v1(definition, "timer", "server-1", "create-timer", {}) + checkpoint = host.read_checkpoint("server-1").document + runtime = checkpoint["root_record"]["aggregate_state"]["root_runtime_id"] + now = ["100"] + clock_calls = [] + + def clock(): + clock_calls.append(now[0]) + return now[0] + + helper = SQLiteTimerHelper( + path, + definition, + scope_identity="scope-archive-example", + root_instance_id="server-1", + root_runtime_id=runtime, + principals=frozenset({"operator", "worker-A"}), + worker_principals=frozenset({"worker-A"}), + trusted_clock=clock, + coordinated_host=host, + ) + helper.setup_schema() + + def command(operation, identifier, arguments=None): + value = request(operation, identifier, arguments) + value["root_runtime_id"] = runtime + value["request_digest"] = timer_request_digest(value) + return value + + assert ( + helper.execute(command("schedule", "schedule-A"), principal="operator")["status"] + == "accepted" + ) + now[0] = "110" + assert ( + helper.execute( + command( + "claim_fire", + "claim-A", + { + "expected_revision": "1", + "clock_basis": "unix_nanoseconds", + "worker_principal": "worker-A", + }, + ), + principal="worker-A", + )["status"] + == "accepted" + ) + record = helper.snapshot()["records"][0] + envelope = { + "event": "received", + "event_id": record["event_id"], + "cause_id": record["event_id"], + "source": {"host": True}, + "target": {"root": {"root_instance_id": "server-1", "root_runtime_id": runtime}}, + "payload": ["map", []], + } + delivery = { + "delivery_mode": "input", + "envelope": envelope, + "envelope_digest": delivery_request_digest("server-1", "input", envelope), + } + expected = admit_checkpoint_v1( + checkpoint, + [delivery], + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + receipt = expected["operation_receipts"][-1] + complete = command( + "complete_fire", + "complete-A", + { + "expected_revision": "2", + "attempt_fence": "1", + "worker_principal": "worker-A", + "event_id": record["event_id"], + "admission_receipt_digest": hash_value(["determa-timer-admission-receipt-1", receipt]), + }, + ) + now[0] = "120" + return helper, host, complete, expected, now, clock_calls + + +def test_actual_admission_and_timer_fire_commit_together_and_replay_after_restart(tmp_path): + helper, host, complete, expected, now, calls = configured(tmp_path) + result = helper.execute(complete, principal="worker-A") + assert result["status"] == "accepted" and result["delivery_state"] == "admitted" + assert result["record_revision"] == "3" and result["expires_at"] is None + assert host.read_checkpoint("server-1").document == expected + committed = helper.snapshot() + assert ( + committed["records"][0]["admission_receipt_digest"] + == complete["arguments"]["admission_receipt_digest"] + ) + + def offline(): + raise AssertionError("retained fire replay read the clock") + + restarted_host = ExecutionHost( + SQLiteExecutionStore( + helper.path, replay_retention="permanent", shared_application_transactions=True + ), + host.artifact_resolver, + ) + restarted = SQLiteTimerHelper( + helper.path, + helper.bundle, + scope_identity=helper.scope_identity, + root_instance_id=helper.root_instance_id, + root_runtime_id=helper.root_runtime_id, + principals=helper.principals, + worker_principals=helper.worker_principals, + trusted_clock=offline, + coordinated_host=restarted_host, + ) + assert restarted.execute(complete, principal="worker-A") == result + assert restarted.snapshot() == committed + assert restarted_host.read_checkpoint("server-1").document == expected + assert calls == ["100", "110", "120", "120"] + + +@pytest.mark.parametrize( + "field,value,code", + [ + ("attempt_fence", "0", "timer_stale_fence"), + ("event_id", "sha256:" + "0" * 64, "timer_event_conflict"), + ("admission_receipt_digest", "sha256:" + "0" * 64, "timer_event_conflict"), + ], +) +def test_false_fire_identity_or_receipt_rolls_back_actual_admission(tmp_path, field, value, code): + helper, host, complete, expected, now, calls = configured(tmp_path) + before_helper = helper.snapshot() + before_checkpoint = host.read_checkpoint("server-1").document + changed = copy.deepcopy(complete) + changed["arguments"][field] = value + changed["request_digest"] = timer_request_digest(changed) + assert helper.execute(changed, principal="worker-A")["error_code"] == code + assert helper.snapshot() == before_helper + assert host.read_checkpoint("server-1").document == before_checkpoint + + +def test_failure_after_staging_timer_and_actual_acceptance_rolls_back_both(tmp_path, monkeypatch): + helper, host, complete, expected, now, calls = configured(tmp_path) + before_helper = helper.snapshot() + before_checkpoint = host.read_checkpoint("server-1").document + persist = helper._persist + + def fail_after_write(sql, artifact): + persist(sql, artifact) + raise RuntimeError("native commit interrupted") + + monkeypatch.setattr(helper, "_persist", fail_after_write) + with pytest.raises(RuntimeError, match="native commit interrupted"): + helper.execute(complete, principal="worker-A") + assert helper.snapshot() == before_helper + assert host.read_checkpoint("server-1").document == before_checkpoint + + +def test_commit_before_response_failure_is_reconciled_from_retained_fire(tmp_path): + helper, host, complete, expected, now, calls = configured(tmp_path) + + def interrupted(boundary): + if boundary == "after_commit_before_response": + raise RuntimeError("lost response") + + host.fault_injector = interrupted + with pytest.raises(RuntimeError, match="lost response"): + helper.execute(complete, principal="worker-A") + assert helper.snapshot()["records"][0]["state"] == "fired" + assert host.read_checkpoint("server-1").document == expected + # Retained replay aborts the read-only callback before the response fault hook. + assert helper.execute(complete, principal="worker-A")["status"] == "accepted" + assert calls == ["100", "110", "120", "120"] + + +@pytest.mark.parametrize("fate", ["expired", "offline"]) +def test_lease_is_rechecked_after_staging_and_before_native_commit(tmp_path, fate): + helper, host, complete, expected, now, calls = configured(tmp_path) + before_helper = helper.snapshot() + before_checkpoint = host.read_checkpoint("server-1").document + reads = [] + + def advancing_clock(): + reads.append(now[0]) + if len(reads) == 1: + now[0] = "1000" + return "120" + if fate == "offline": + raise RuntimeError("clock offline") + return now[0] + + helper.trusted_clock = advancing_clock + result = helper.execute(complete, principal="worker-A") + assert result["error_code"] == ( + "timer_stale_fence" if fate == "expired" else "timer_clock_unavailable" + ) + assert result["record_revision"] == "2" and result["attempt_fence"] == "1" + assert helper.snapshot() == before_helper + assert host.read_checkpoint("server-1").document == before_checkpoint + assert reads == ["120", "1000"] diff --git a/tests/test_timer_native_fate.py b/tests/test_timer_native_fate.py new file mode 100644 index 0000000..b03f1bb --- /dev/null +++ b/tests/test_timer_native_fate.py @@ -0,0 +1,297 @@ +"""Recovery uses native atomic history under its writer lock, never caller fate.""" + +from __future__ import annotations + +import copy +import multiprocessing +import os +import signal +import threading +from concurrent.futures import ThreadPoolExecutor + +import pytest + +from determa.state import ExecutionHost, SQLiteExecutionStore +from determa.state.host import delivery_request_digest +from determa.state.timers import SQLiteTimerHelper, timer_request_digest + +from .test_timer_coordinated_admission import _killed_timer_worker, configured + + +def reclaim(complete, revision="2"): + value = copy.deepcopy(complete) + value.update(operation="claim_fire", operation_id="reclaim-B") + value["arguments"] = { + "expected_revision": revision, + "clock_basis": "unix_nanoseconds", + "worker_principal": "worker-A", + } + value["request_digest"] = timer_request_digest(value) + return value + + +def reopened(helper, host, clock): + return SQLiteTimerHelper( + helper.path, + helper.bundle, + scope_identity=helper.scope_identity, + root_instance_id=helper.root_instance_id, + root_runtime_id=helper.root_runtime_id, + principals=helper.principals, + worker_principals=helper.worker_principals, + trusted_clock=clock, + coordinated_host=host, + ) + + +def test_expired_local_claim_reclaims_with_new_fence_then_rejects_old_worker(tmp_path): + helper, host, complete, expected, now, _ = configured(tmp_path) + before_checkpoint = host.read_checkpoint("server-1").document + now[0] = "130" + result = helper.execute(reclaim(complete), principal="worker-A") + assert result["status"] == "accepted" and result["attempt_fence"] == "2" + assert result["record_revision"] == "3" and result["expires_at"] == "150" + assert host.read_checkpoint("server-1").document == before_checkpoint + before = helper.snapshot() + assert helper.execute(complete, principal="worker-A")["error_code"] == "timer_stale_fence" + assert helper.snapshot() == before + now[0] = "140" + replacement = copy.deepcopy(complete) + replacement["operation_id"] = "complete-B" + replacement["arguments"].update(expected_revision="3", attempt_fence="2") + replacement["request_digest"] = timer_request_digest(replacement) + assert helper.execute(replacement, principal="worker-A")["status"] == "accepted" + assert host.read_checkpoint("server-1").document == expected + # Original reclaim's exact receipt remains replayable after fired state and offline clock. + helper.trusted_clock = lambda: (_ for _ in ()).throw(AssertionError("replay read clock")) + assert helper.execute(reclaim(complete), principal="worker-A") == result + + +@pytest.mark.parametrize("replace_class", [False, True]) +def test_replaced_fate_proof_cannot_bypass_its_own_guard(tmp_path, monkeypatch, replace_class): + helper, host, complete, _, now, _ = configured(tmp_path, replay_retention="bounded") + before_helper, before_root = helper.snapshot(), host.read_checkpoint("server-1").document + now[0] = "130" + monkeypatch.setattr( + SQLiteTimerHelper if replace_class else helper, + "_prove_local_uncommitted_fire", + lambda *_: None, + ) + assert ( + helper.execute(reclaim(complete), principal="worker-A")["error_code"] + == "delivery_ambiguous" + ) + assert helper.snapshot() == before_helper + assert host.read_checkpoint("server-1").document == before_root + + +@pytest.mark.parametrize("retention", ["bounded", "permanent"]) +def test_replaced_fate_function_code_is_checked_before_dispatch(tmp_path, monkeypatch, retention): + helper, host, complete, _, now, _ = configured(tmp_path, replay_retention=retention) + before_helper, before_root = helper.snapshot(), host.read_checkpoint("server-1").document + now[0] = "130" + monkeypatch.setattr( + SQLiteTimerHelper._prove_local_uncommitted_fire, + "__code__", + (lambda *_: None).__code__, + ) + assert ( + helper.execute(reclaim(complete), principal="worker-A")["error_code"] + == "delivery_ambiguous" + ) + assert helper.snapshot() == before_helper + assert host.read_checkpoint("server-1").document == before_root + + +@pytest.mark.parametrize("changed_content", [False, True]) +def test_independently_admitted_matching_id_never_becomes_helper_fate_proof( + tmp_path, changed_content +): + helper, host, complete, _, now, _ = configured(tmp_path) + before = host.read_checkpoint("server-1").document + envelope = { + "event": "received", + "event_id": complete["arguments"]["event_id"], + "cause_id": complete["arguments"]["event_id"], + "source": {"host": True}, + "target": { + "root": {"root_instance_id": "server-1", "root_runtime_id": helper.root_runtime_id} + }, + "payload": ["map", []], + } + if changed_content: + envelope["correlation_id"] = "unrelated" + host.admit_v1( + "server-1", + [ + { + "delivery_mode": "input", + "envelope": envelope, + "envelope_digest": delivery_request_digest("server-1", "input", envelope), + } + ], + expected_revision=before["revision"], + expected_checkpoint_digest=before["execution_checkpoint_digest"], + ) + before_helper, before_root = helper.snapshot(), host.read_checkpoint("server-1").document + now[0] = "130" + assert ( + helper.execute(reclaim(complete), principal="worker-A")["error_code"] + == "delivery_ambiguous" + ) + assert helper.snapshot() == before_helper + assert host.read_checkpoint("server-1").document == before_root + + +@pytest.mark.parametrize("changed", ["implementation", "store_policy", "clock"]) +def test_changed_implementation_policy_or_invalid_clock_refuses_without_reclaim(tmp_path, changed): + helper, host, complete, _, now, _ = configured(tmp_path) + before = helper.snapshot() + now[0] = "130" + if changed == "implementation": + host.run_shared_transaction = lambda *_: None + code = "delivery_ambiguous" + elif changed == "store_policy": + host.store.replay_retention = "bounded" + code = "timer_capability_mismatch" + else: + now[0] = "invalid" + code = "timer_clock_unavailable" + assert helper.execute(reclaim(complete), principal="worker-A")["error_code"] == code + if changed == "store_policy": + host.store.replay_retention = "permanent" + assert helper.snapshot() == before + + +def test_completion_staged_before_reclaim_rolls_back_at_expiry_then_new_fence_wins(tmp_path): + helper, host, complete, _, now, _ = configured(tmp_path) + contender = reopened(helper, host, lambda: now[0]) + staged, release = threading.Event(), threading.Event() + persist = helper._persist + + def stop_after_staging(sql, artifact): + persist(sql, artifact) + staged.set() + assert release.wait(20), "test did not release staged transaction" + + helper._persist = stop_after_staging + with ThreadPoolExecutor(max_workers=2) as pool: + previous = pool.submit(helper.execute, complete, principal="worker-A") + try: + assert staged.wait(20), "completion did not stage actual native writes" + now[0] = "130" + replacement = pool.submit(contender.execute, reclaim(complete), principal="worker-A") + finally: + release.set() + assert previous.result(20)["error_code"] == "timer_stale_fence" + assert replacement.result(20)["attempt_fence"] == "2" + assert contender.snapshot()["records"][0]["state"] == "claimed" + assert not any( + item.get("event_id") == complete["arguments"]["event_id"] + for item in host.read_checkpoint("server-1").document["operation_receipts"] + ) + + +@pytest.mark.skipif(os.name != "posix", reason="requires actual SIGKILL") +@pytest.mark.parametrize("boundary", ["both_writes_staged", "after_commit_before_response"]) +def test_sigkill_native_fate_reclaim_or_terminal_replay(tmp_path, boundary): + helper, host, complete, expected, now, _ = configured(tmp_path) + context = multiprocessing.get_context("spawn") + parent, child = context.Pipe() + worker = context.Process( + target=_killed_timer_worker, args=(helper.path, complete, boundary, child) + ) + worker.start() + child.close() + try: + assert parent.poll(30), "worker did not reach native boundary" + assert parent.recv() == boundary + worker.kill() + worker.join(10) + assert worker.exitcode == -signal.SIGKILL + finally: + if worker.is_alive(): + worker.kill() + worker.join(10) + parent.close() + worker.close() + restarted_host = ExecutionHost( + SQLiteExecutionStore( + helper.path, replay_retention="permanent", shared_application_transactions=True + ), + host.artifact_resolver, + ) + now[0] = "130" + restarted = reopened(helper, restarted_host, lambda: now[0]) + if boundary == "both_writes_staged": + result = restarted.execute(reclaim(complete), principal="worker-A") + assert result["status"] == "accepted" and result["attempt_fence"] == "2" + before = restarted.snapshot() + assert ( + restarted.execute(complete, principal="worker-A")["error_code"] == "timer_stale_fence" + ) + assert restarted.snapshot() == before + replacement = copy.deepcopy(complete) + replacement["operation_id"] = "complete-B" + replacement["arguments"].update(expected_revision="3", attempt_fence="2") + replacement["request_digest"] = timer_request_digest(replacement) + now[0] = "140" + assert restarted.execute(replacement, principal="worker-A")["status"] == "accepted" + else: + before = restarted.snapshot() + assert ( + restarted.execute(reclaim(complete, "3"), principal="worker-A")["error_code"] + == "timer_already_fired" + ) + assert restarted.snapshot() == before + assert restarted.execute(complete, principal="worker-A")["status"] == "accepted" + assert restarted_host.read_checkpoint("server-1").document == expected + assert restarted.snapshot()["records"][0]["state"] == "fired" + + +def test_bounded_receipt_history_cannot_prove_uncommitted_fire(tmp_path): + helper, host, complete, _, now, _ = configured(tmp_path, replay_retention="bounded") + before_helper, before_root = helper.snapshot(), host.read_checkpoint("server-1").document + now[0] = "130" + assert ( + helper.execute(reclaim(complete), principal="worker-A")["error_code"] + == "delivery_ambiguous" + ) + assert helper.snapshot() == before_helper + assert host.read_checkpoint("server-1").document == before_root + + +def test_changed_shared_admission_implementation_cannot_establish_native_fate( + tmp_path, monkeypatch +): + from determa.state.host import SharedExecutionTransaction + + helper, _, complete, _, now, _ = configured(tmp_path) + before = helper.snapshot() + now[0] = "130" + monkeypatch.setattr(SharedExecutionTransaction, "admit_v1", lambda *_: None) + assert ( + helper.execute(reclaim(complete), principal="worker-A")["error_code"] + == "delivery_ambiguous" + ) + assert helper.snapshot() == before + + +@pytest.mark.parametrize("missing", ["determa_timer_origin", "determa_timer_commits"]) +def test_missing_origin_or_native_history_cannot_be_rebuilt_for_reclaim(tmp_path, missing): + import sqlite3 + + helper, _, complete, _, now, _ = configured(tmp_path) + now[0] = "130" + with sqlite3.connect(helper.path) as connection: + connection.execute(f"DROP TABLE {missing}") + before = connection.execute("SELECT document FROM determa_timer_helpers").fetchall() + assert ( + helper.execute(reclaim(complete), principal="worker-A")["error_code"] + == "timer_capability_mismatch" + ) + with sqlite3.connect(helper.path) as connection: + assert connection.execute("SELECT document FROM determa_timer_helpers").fetchall() == before + assert connection.execute( + "SELECT COUNT(*) FROM sqlite_master WHERE name=?", (missing,) + ).fetchone() == (0,) diff --git a/tests/test_timers.py b/tests/test_timers.py new file mode 100644 index 0000000..e2325a8 --- /dev/null +++ b/tests/test_timers.py @@ -0,0 +1,352 @@ +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest + +from determa.state import load_bundle +from determa.state.timers import ( + SQLiteTimerHelper, + TimerError, + seal_timer_records, + timer_request_digest, +) + +_ROOT_RUNTIME = "sha256:9b0e3238e782cb7d912febc1acc4e4c03a121d201764d0bc9658b969feac4dc8" + + +@pytest.mark.parametrize("damage", ["record", "receipt", "unretained_record"]) +def test_resealed_portable_records_do_not_replace_retained_native_evidence(tmp_path, damage): + import sqlite3 + + from determa.state.wire import canonical_bytes + + installed = helper(tmp_path / "native-evidence.sqlite", lambda: "100") + installed.setup_schema() + assert installed.execute(request(), principal="operator")["status"] == "accepted" + artifact = installed.snapshot() + if damage == "record": + artifact["records"][0]["deadline_at"] = "999" + elif damage == "receipt": + artifact["operation_receipts"][0]["result"]["record_revision"] = "999" + else: + artifact["operation_receipts"] = [] + with sqlite3.connect(installed.path) as connection: + connection.execute( + "UPDATE determa_timer_helpers SET document=?", + (canonical_bytes(seal_timer_records(artifact)),), + ) + result = installed.execute(request("read_timer", "read-A", {}), principal="operator") + assert result["error_code"] == "timer_capability_mismatch" + + +def test_native_origin_is_storage_bound_and_setup_cannot_repair_missing_evidence(tmp_path): + import sqlite3 + + original = helper(tmp_path / "original.sqlite", lambda: "100") + original.setup_schema() + original.execute(request(), principal="operator") + before = original.snapshot() + copied_path = tmp_path / "copied.sqlite" + with sqlite3.connect(original.path) as source, sqlite3.connect(copied_path) as copied: + source.backup(copied) + inert = helper(copied_path, lambda: "100") + with pytest.raises(TimerError, match="timer_capability_mismatch"): + inert.snapshot() + with sqlite3.connect(copied_path) as connection: + connection.execute("DROP TABLE determa_timer_origin") + recorded = connection.execute("SELECT document FROM determa_timer_helpers").fetchall() + with pytest.raises(TimerError, match="timer_capability_mismatch"): + inert.setup_schema() + with sqlite3.connect(copied_path) as connection: + assert connection.execute( + "SELECT COUNT(*) FROM sqlite_master WHERE name='determa_timer_origin'" + ).fetchone() == (0,) + assert ( + connection.execute("SELECT document FROM determa_timer_helpers").fetchall() == recorded + ) + assert original.snapshot() == before + + +def test_native_operation_history_is_immutable_and_configuration_changes_fail_closed(tmp_path): + import sqlite3 + + installed = helper(tmp_path / "configuration.sqlite", lambda: "100") + installed.setup_schema() + installed.execute(request(), principal="operator") + with sqlite3.connect(installed.path) as connection: + for statement in ( + "DELETE FROM determa_timer_commits", + "UPDATE determa_timer_commits SET operation_id='changed'", + "DELETE FROM determa_timer_origin", + "UPDATE determa_timer_origin SET configuration=X'00'", + ): + with pytest.raises(sqlite3.IntegrityError, match="timer_native_evidence_immutable"): + connection.execute(statement) + installed.claim_lease_nanoseconds += 1 + result = installed.execute(request("read_timer", "read-A", {}), principal="operator") + assert result["error_code"] == "timer_capability_mismatch" + + +def bundle(): + return load_bundle( + { + "format": 1, + "namespace": "timer.tests", + "events": {"received": {"direction": "input"}}, + "machines": [ + { + "machine_id": "timer", + "root": { + "type": "composite", + "initial": {"transition_to": "waiting"}, + "states": {"waiting": {"on_events": {"received": {}}}}, + }, + } + ], + } + ) + + +def request(operation="schedule", operation_id="schedule-A", arguments=None): + value = { + "interface": "determa.timer_helper", + "interface_version": 1, + "scope_identity": "scope-archive-example", + "root_instance_id": "server-1", + "root_runtime_id": _ROOT_RUNTIME, + "timer_id": "timer-A", + "operation": operation, + "operation_id": operation_id, + "arguments": arguments + if arguments is not None + else { + "clock_basis": "unix_nanoseconds", + "delay_nanoseconds": "10", + "event_name": "received", + "payload": ["map", []], + "correlation_id": None, + }, + } + value["request_digest"] = timer_request_digest(value) + return value + + +def helper(path: Path, clock): + value = request() + return SQLiteTimerHelper( + path, + bundle(), + scope_identity=value["scope_identity"], + root_instance_id=value["root_instance_id"], + root_runtime_id=value["root_runtime_id"], + principals=frozenset({"operator"}), + trusted_clock=clock, + ) + + +def test_committed_schedule_survives_restart_and_replays_original_result_after_cancel(tmp_path): + calls = [] + initial = helper(tmp_path / "timer.sqlite", lambda: calls.append("clock") or "100") + initial.setup_schema() + scheduled = initial.execute(request(), principal="operator") + assert scheduled["status"] == "accepted" + assert initial.snapshot()["records"][0]["deadline_at"] == "110" + assert calls == ["clock"] + + def offline(): + raise AssertionError("retained replay/cancel/read must not access the clock") + + restarted = helper(tmp_path / "timer.sqlite", offline) + cancelled = restarted.execute( + request("cancel", "cancel-A", {"expected_revision": "1"}), principal="operator" + ) + assert cancelled["status"] == "accepted" and cancelled["record_revision"] == "2" + before = restarted.snapshot() + assert restarted.execute(request(), principal="operator") == scheduled + assert restarted.snapshot() == before + read = restarted.execute(request("read_timer", "read-A", {}), principal="operator") + assert read["record_revision"] == "2" and read["event_id"] == scheduled["event_id"] + assert restarted.snapshot() == before + + +@pytest.mark.parametrize( + "time", ["-0", "+1", "01", "1.5", "1e2", "9223372036854775808", "-1", 1, True] +) +def test_invalid_duration_changes_no_records_and_never_reads_clock(tmp_path, time): + def forbidden_clock(): + raise AssertionError("invalid time reached the clock") + + installed = helper(tmp_path / "timer.sqlite", forbidden_clock) + installed.setup_schema() + before = installed.snapshot() + value = request() + value["arguments"]["delay_nanoseconds"] = time + value["request_digest"] = timer_request_digest(value) + result = installed.execute(value, principal="operator") + assert result["error_code"] == "invalid_timer_time" + assert installed.snapshot() == before + + +def test_target_incarnation_and_principal_are_authorized_before_replay(tmp_path): + installed = helper(tmp_path / "timer.sqlite", lambda: "100") + installed.setup_schema() + assert installed.execute(request(), principal="operator")["status"] == "accepted" + before = installed.snapshot() + for value, principal in [(request(), "stranger"), (request(), "operator")]: + if principal == "operator": + value["root_runtime_id"] = "sha256:" + "0" * 64 + value["request_digest"] = timer_request_digest(value) + result = installed.execute(value, principal=principal) + assert result["error_code"] == "unauthorized_timer_scope" + assert result["record_revision"] is None and result["event_id"] is None + assert installed.snapshot() == before + + +def test_operation_conflict_preserves_exact_durable_schedule(tmp_path): + installed = helper(tmp_path / "timer.sqlite", lambda: "100") + installed.setup_schema() + value = request() + assert installed.execute(value, principal="operator")["status"] == "accepted" + before = installed.snapshot() + changed = copy.deepcopy(value) + changed["arguments"]["delay_nanoseconds"] = "20" + changed["request_digest"] = timer_request_digest(changed) + assert ( + installed.execute(changed, principal="operator")["error_code"] == "timer_operation_conflict" + ) + assert installed.snapshot() == before + + +def test_invalid_record_digest_refuses_read_after_restart(tmp_path): + import sqlite3 + + path = tmp_path / "timer.sqlite" + installed = helper(path, lambda: "100") + installed.setup_schema() + assert installed.execute(request(), principal="operator")["status"] == "accepted" + damaged = installed.snapshot() + damaged["records"][0]["deadline_at"] = "999" + with sqlite3.connect(path) as connection: + connection.execute( + "UPDATE determa_timer_helpers SET document=?", (json.dumps(damaged).encode(),) + ) + restarted = helper(path, lambda: "100") + result = restarted.execute(request("read_timer", "read-A", {}), principal="operator") + assert result["error_code"] == "timer_capability_mismatch" + + +def worker_helper(path, clock): + return SQLiteTimerHelper( + path, + bundle(), + scope_identity="scope-archive-example", + root_instance_id="server-1", + root_runtime_id=_ROOT_RUNTIME, + principals=frozenset({"operator", "worker-A", "worker-B"}), + worker_principals=frozenset({"worker-A", "worker-B"}), + trusted_clock=clock, + ) + + +def test_due_claim_is_durable_and_replay_does_not_recheck_clock_or_revision(tmp_path): + path = tmp_path / "timers.sqlite" + now = ["100"] + installed = worker_helper(path, lambda: now[0]) + installed.setup_schema() + installed.execute(request(), principal="operator") + claim = request( + "claim_fire", + "claim-A", + { + "expected_revision": "1", + "clock_basis": "unix_nanoseconds", + "worker_principal": "worker-A", + }, + ) + before = installed.snapshot() + now[0] = "109" + assert installed.execute(claim, principal="worker-A")["error_code"] == "timer_not_due" + assert installed.snapshot() == before + now[0] = "110" + result = installed.execute(claim, principal="worker-A") + assert result["status"] == "accepted" and result["attempt_fence"] == "1" + assert result["record_revision"] == "2" and result["expires_at"] == "130" + committed = installed.snapshot() + + def offline(): + raise AssertionError("retained claim replay read the clock") + + restarted = worker_helper(path, offline) + assert restarted.execute(claim, principal="worker-A") == result + assert restarted.snapshot() == committed + assert ( + restarted.execute( + request("cancel", "cancel-A", {"expected_revision": "2"}), principal="operator" + )["error_code"] + == "timer_fire_in_progress" + ) + assert restarted.snapshot() == committed + + +def test_expired_claim_without_commit_fate_proof_never_grants_a_new_fence(tmp_path): + path = tmp_path / "timers.sqlite" + now = ["100"] + installed = worker_helper(path, lambda: now[0]) + installed.setup_schema() + installed.execute(request(), principal="operator") + now[0] = "110" + claim = request( + "claim_fire", + "claim-A", + { + "expected_revision": "1", + "clock_basis": "unix_nanoseconds", + "worker_principal": "worker-A", + }, + ) + assert installed.execute(claim, principal="worker-A")["status"] == "accepted" + before = installed.snapshot() + retry = request( + "claim_fire", + "retry-B", + { + "expected_revision": "2", + "clock_basis": "unix_nanoseconds", + "worker_principal": "worker-B", + }, + ) + now[0] = "131" + refused = worker_helper(path, lambda: now[0]).execute(retry, principal="worker-B") + assert refused["error_code"] == "delivery_ambiguous" and refused["attempt_fence"] == "1" + assert installed.snapshot() == before + assert installed.execute(retry, principal="worker-A")["error_code"] == "timer_worker_mismatch" + assert installed.snapshot() == before + + +@pytest.mark.parametrize("encoding", ["duplicate", "noncanonical", "invalid"]) +def test_stored_artifact_requires_strict_canonical_json(tmp_path, encoding): + import sqlite3 + + from determa.state.wire import canonical_bytes + + path = tmp_path / "timers.sqlite" + installed = helper(path, lambda: "100") + installed.setup_schema() + installed.execute(request(), principal="operator") + document = installed.snapshot() + raw = canonical_bytes(document) + if encoding == "duplicate": + raw = b'{"records":[],' + raw[1:] + elif encoding == "noncanonical": + raw = json.dumps(document, indent=2).encode() + else: + raw = b"invalid" + with sqlite3.connect(path) as connection: + connection.execute("UPDATE determa_timer_helpers SET document=?", (raw,)) + assert ( + installed.execute(request("read_timer", "read-A", {}), principal="operator")["error_code"] + == "timer_capability_mismatch" + )