From 097f53d71f7e7638c6392629a49491eecd6e0e43 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 12:52:34 +0000 Subject: [PATCH 01/12] Bound JSON encoder setup in Cell proof regression cases Exercise each default-field refusal and JSON numeric distinction before consolidating repeated default comparisons. Signed-off-by: zlareb1 (cherry picked from commit 73ef63b0b7b5e65254dc2618d19c267017fed904) Signed-off-by: zlareb1 --- .../test_cell_trigger_default_comparison.py | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 tests/unit/test_cell_trigger_default_comparison.py diff --git a/tests/unit/test_cell_trigger_default_comparison.py b/tests/unit/test_cell_trigger_default_comparison.py new file mode 100644 index 0000000000..4f712913a7 --- /dev/null +++ b/tests/unit/test_cell_trigger_default_comparison.py @@ -0,0 +1,66 @@ +"""Default-field proof checks keep their refusals without per-field JSON setup.""" + +from copy import deepcopy + +import pytest + +from tests.unit.test_cell_trigger_ci_creator import keeper_config_case as keeper_config_case +from tests.unit.test_cell_trigger_ci_creator import protocol_reader as protocol_reader +from tests.unit.test_cell_trigger_provenance_reuse import _case + + +def _expected(native, specification, context, row, audit, api): + native.Codec(specification).expected( + row, + audit, + context, + api=api, + metadata={"summary": {"jti": "b" * 32}}, + driver={"host": "proof-host", "pid": 701}, + ) + + +@pytest.mark.parametrize("api", [False, True], ids=["delivery", "api"]) +def test_default_proof_bounds_json_encoder_setup( + keeper_config_case, protocol_reader, monkeypatch, api +): + native, *_ = keeper_config_case + specification, context, _, _, memories, audits, _ = _case(native, 1, api) + calls = 0 + original = native.json.dumps + + def counted(*args, **kwargs): + nonlocal calls + calls += 1 + return original(*args, **kwargs) + + monkeypatch.setattr(native.json, "dumps", counted) + _expected(native, specification, context, memories[0], audits[0], api) + # One row has 43 defaults; encoder setup must not repeat for each field. + assert calls <= 16 + + +@pytest.mark.parametrize("index", range(43)) +def test_default_proof_refuses_each_changed_field(keeper_config_case, protocol_reader, index): + native, *_ = keeper_config_case + specification, context, _, _, memories, audits, _ = _case(native, 1, False) + field, default = list(specification["sparse_defaults"].items())[index] + assert default != {"unexpected": True} + row = deepcopy(memories[0]) + row[field] = {"unexpected": True} + with pytest.raises(native.Refused): + _expected(native, specification, context, row, audits[0], False) + + +@pytest.mark.parametrize("replacement", [False, 0.0]) +def test_default_proof_keeps_json_numeric_type_refusals( + keeper_config_case, protocol_reader, replacement +): + native, *_ = keeper_config_case + specification, context, _, _, memories, audits, _ = _case(native, 1, False) + assert specification["sparse_defaults"]["pinned"] == 0 + row = deepcopy(memories[0]) + row["pinned"] = replacement + assert row["pinned"] == 0 + with pytest.raises(native.Refused): + _expected(native, specification, context, row, audits[0], False) From 4443caa2972180c331ffe75c540da92a8073fb27 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 13:14:00 +0000 Subject: [PATCH 02/12] Compare Cell proof defaults in one canonical mapping Keep every default-field and JSON type check while avoiding a separate encoder setup for each field. Signed-off-by: zlareb1 (cherry picked from commit 0bfc712dcca9684e362af47a7e924432e87e53dd) Signed-off-by: zlareb1 --- scripts/gates/testsupport/cell_trigger/capture.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/gates/testsupport/cell_trigger/capture.py b/scripts/gates/testsupport/cell_trigger/capture.py index 635df62581..0360fbdb8c 100644 --- a/scripts/gates/testsupport/cell_trigger/capture.py +++ b/scripts/gates/testsupport/cell_trigger/capture.py @@ -255,8 +255,10 @@ def expected(self, row, audit, context, *, api, metadata, driver): "workload semantic/active", ) need(row["confidence"] == 1.0, "workload confidence") - for field, default in self.p["sparse_defaults"].items(): - equal(row[field], default) + equal( + {field: row[field] for field in self.p["sparse_defaults"]}, + self.p["sparse_defaults"], + ) need( row["value"] == json.dumps(payload(key)) and row["content_sha256"] == content_hash(key), "exact stored payload/hash", From b74fc6e279e387f802f55b7da54d0a4f5f8fbaa2 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 21:09:53 +0000 Subject: [PATCH 03/12] Scope person-query observations to the decision task Shared client schema reads run with an independent context. Record only decision-tagged statements while retaining unbounded decision reads as failures. Signed-off-by: zlareb1 (cherry picked from commit 57327d5ea5778042f860cadf6cebec9eebb25821) Signed-off-by: zlareb1 --- tests/integration/test_scim_person_live_ch.py | 8 +- .../test_scim_person_query_observation.py | 116 ++++++++++++++++++ 2 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_scim_person_query_observation.py diff --git a/tests/integration/test_scim_person_live_ch.py b/tests/integration/test_scim_person_live_ch.py index 34818825b7..a3eeafaa1b 100644 --- a/tests/integration/test_scim_person_live_ch.py +++ b/tests/integration/test_scim_person_live_ch.py @@ -125,13 +125,17 @@ async def people(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[People]: query_all, query_one = client.query_all, client.query_one async def all_rows(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) -> Any: + observing_person = ch.current_query_tag() == "auth.scim_people" answer = await query_all(sql, params=params, **kwargs) - made.results.append((sql, ch.statement_settings(sql), answer)) + if observing_person: + made.results.append((sql, ch.statement_settings(sql), answer)) return answer async def one_row(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) -> Any: + observing_person = ch.current_query_tag() == "auth.scim_people" answer = await query_one(sql, params=params, **kwargs) - made.results.append((sql, ch.statement_settings(sql), answer)) + if observing_person: + made.results.append((sql, ch.statement_settings(sql), answer)) return answer monkeypatch.setattr(client, "query_all", all_rows) diff --git a/tests/unit/test_scim_person_query_observation.py b/tests/unit/test_scim_person_query_observation.py new file mode 100644 index 0000000000..a0f9ef7f06 --- /dev/null +++ b/tests/unit/test_scim_person_query_observation.py @@ -0,0 +1,116 @@ +"""Person-query evidence excludes reads from independent background tasks.""" + +from __future__ import annotations + +import asyncio +from contextvars import Context +from typing import Any + +import pytest + +from server.auth import scim_people +from server.core import ch, rmt_carry +from server.settings import settings +from tests.integration.test_scim_person_live_ch import people as person_fixture +from tests.unit.test_ch_client_hygiene import _client, _FakeAsync + + +class Queries: + def __init__(self) -> None: + self.started = asyncio.Event() + self.background_done = asyncio.Event() + self.block_person = False + self.transport: list[tuple[str, dict[str, Any]]] = [] + + async def connect(self) -> None: + pass + + async def disconnect(self) -> None: + pass + + async def _read(self, sql: str) -> dict[str, Any]: + self.transport.append((sql, ch.statement_settings(sql, cls="read"))) + if sql == "SELECT 1 AS person" and self.block_person: + self.started.set() + await self.background_done.wait() + return {"n": 0, "at": None, "altered": None} + + async def query_one(self, sql: str, params: Any = None) -> dict[str, Any]: + return await self._read(sql) + + async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: + return [await self._read(sql)] + + +@pytest.fixture +async def recording(monkeypatch): + client = Queries() + monkeypatch.setattr(ch, "get_client", lambda: client) + stream = person_fixture.__wrapped__(monkeypatch) + observed = await anext(stream) + try: + yield observed, client + finally: + await stream.aclose() + + +@pytest.mark.parametrize("read_timeout", [0, 2.5]) +@pytest.mark.parametrize("many", [False, True]) +async def test_background_schema_reads_do_not_enter_person_evidence( + recording, monkeypatch, read_timeout, many +): + observed, client = recording + monkeypatch.setattr(settings, "ch_timeout_read_s", read_timeout) + client.block_person = True + + async def background(): + await client.started.wait() + try: + with ch.query_tag("rmt_carry.columns"): + await rmt_carry._schema_stamp(client) + finally: + client.background_done.set() + + watcher = asyncio.create_task(background(), context=Context()) + try: + with ch.statement_timeout(1.75), ch.query_tag("auth.scim_people"): + query = client.query_all if many else client.query_one + await query("SELECT 1 AS person") + await watcher + finally: + if not watcher.done(): + watcher.cancel() + await asyncio.gather(watcher, return_exceptions=True) + + assert len(client.transport) == 2 + assert any("system.tables" in sql for sql, _ in client.transport) + assert [sql for sql, _, _ in observed.results] == ["SELECT 1 AS person"] + assert 0 < observed.results[0][1]["max_execution_time"] <= 1.75 + + +@pytest.mark.parametrize("many", [False, True]) +async def test_unbounded_person_reads_remain_visible(recording, monkeypatch, many): + observed, client = recording + monkeypatch.setattr(settings, "ch_timeout_read_s", 0) + with ch.query_tag("auth.scim_people"): + query = client.query_all if many else client.query_one + await query("SELECT 1 AS person") + assert len(observed.results) == 1 + with pytest.raises(KeyError, match="max_execution_time"): + assert 0 < observed.results[0][1]["max_execution_time"] <= 1.75 + + +@pytest.mark.parametrize("read_timeout", [0, 2.5]) +@pytest.mark.parametrize("many", [False, True]) +async def test_actual_person_reader_applies_its_finite_budget_to_driver( + monkeypatch, read_timeout, many +): + driver = _FakeAsync() + client = _client(monkeypatch, driver, ch_timeout_read_s=read_timeout) + monkeypatch.setattr(ch, "get_client", lambda: client) + decision = scim_people._Decision("org", "owner", "owner@example.test", scim_people._now() + 10) + with ch.statement_timeout(1.75), ch.query_tag("auth.scim_people"): + await decision.query("SELECT 1 AS person", {}, many=many) + calls = [applied for kind, sql, applied in driver.calls if sql == "SELECT 1 AS person"] + assert len(calls) == 1 + assert 0 < calls[0]["max_execution_time"] <= 1.75 From 9d57ebb32e730533322404e7b6ce4c4e139a8c45 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 22:25:38 +0000 Subject: [PATCH 04/12] Cleanup tests measure their shared idle deadline Keep the blocked close, deadline observers and loop timers on one private clock. A controlled independent clock advance reproduces the elapsed-time dependency while the original cleanup and refusal assertions stay intact. Signed-off-by: zlareb1 (cherry picked from commit 4b03db4d8e5ac44d69af2146c371066907a5ec06) Signed-off-by: zlareb1 --- tests/unit/test_cell_attach_cleanup.py | 72 +++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_cell_attach_cleanup.py b/tests/unit/test_cell_attach_cleanup.py index ec359f3d40..88df8ee92b 100644 --- a/tests/unit/test_cell_attach_cleanup.py +++ b/tests/unit/test_cell_attach_cleanup.py @@ -6,6 +6,7 @@ from unittest.mock import AsyncMock import pytest +import pytest_asyncio from server.cli import cell_attach, cell_control from tests.unit.cell_control_owner import owner @@ -96,7 +97,8 @@ async def work(): await asyncio.gather(*owned_tasks, return_exceptions=True) -@pytest.mark.asyncio +@pytest.mark.asyncio(loop_scope="function") +@pytest.mark.usefixtures("_cleanup_clock") async def test_lock_owner_bounds_close_inside_the_same_cleanup_allowance(): closing = asyncio.Event() conn = AsyncMock() @@ -152,3 +154,71 @@ async def open(owned, what): if failed: assert caught.value is original conn.close.assert_awaited_once() + + +@pytest.mark.parametrize("scheduler_s", [0.0, 0.16]) +@pytest.mark.asyncio(loop_scope="function") +@pytest.mark.usefixtures("_cleanup_clock") +async def test_cleanup_allowance_ignores_independent_scheduler_time(monkeypatch, scheduler_s): + import time as wall_time + + real_monotonic = wall_time.monotonic + elapsed = 0.0 + + def monotonic(): + return real_monotonic() + elapsed + + actual_close = cell_control.Owner.close + + async def close(owned): + nonlocal elapsed + try: + return await actual_close(owned) + finally: + elapsed += scheduler_s + + monkeypatch.setattr(wall_time, "monotonic", monotonic) + monkeypatch.setattr(cell_control.Owner, "close", close) + await test_lock_owner_bounds_close_inside_the_same_cleanup_allowance() + + +@pytest_asyncio.fixture(loop_scope="function") +async def _cleanup_clock(monkeypatch): + from server.core import pg + from tests.unit.clock_fake import ManualClock + + before = set(asyncio.all_tasks()) + loop = asyncio.get_running_loop() + clock = ManualClock(t=loop.time()) + selector = loop._selector + real_select = selector.select + + def select(timeout=None): + if timeout is not None: + clock.advance(max(0.0, timeout)) + timeout = 0 + return real_select(timeout) + + # The cleanup protocol spends idle time, independently of CPU scheduling. + monkeypatch.setattr(loop, "time", clock.monotonic) + monkeypatch.setattr(selector, "select", select) + monkeypatch.setattr(cell_control, "time", clock) + monkeypatch.setattr(pg, "time", clock) + monkeypatch.setattr(f"{__name__}.time", clock) + yield clock + leaked = asyncio.all_tasks() - before - {asyncio.current_task()} + for task in leaked: + task.cancel() + await asyncio.gather(*leaked, return_exceptions=True) + assert not leaked, "cell cleanup leaves no task on its private loop" + + +@pytest.mark.asyncio(loop_scope="function") +async def test_cleanup_clock_preserves_the_close_deadline(_cleanup_clock): + from server.core import pg + + loop = asyncio.get_running_loop() + assert loop.time == time.monotonic == cell_control.time.monotonic == pg.time.monotonic + began = _cleanup_clock.monotonic() + await test_lock_owner_bounds_close_inside_the_same_cleanup_allowance() + assert _cleanup_clock.monotonic() - began == pytest.approx(0.03) From d48c7fdf3aeb3e75ddae0a66935e884ea05bb92a Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Thu, 19 Nov 2026 21:00:41 +0000 Subject: [PATCH 05/12] Clear key-lock health after successful lease acquisition Signed-off-by: zlareb1 --- server/memory/store/key_write_lock.py | 3 +- tests/unit/test_memory_key_write_lock.py | 42 ++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/server/memory/store/key_write_lock.py b/server/memory/store/key_write_lock.py index e0fb405404..bda6d2fdfd 100644 --- a/server/memory/store/key_write_lock.py +++ b/server/memory/store/key_write_lock.py @@ -20,7 +20,7 @@ from server.core.errors import PreconditionFailedError from server.core.leases import LeaseBusy, LeaseLost from server.core.repo_base import _CHClient, _default_client -from server.observability import degradations +from server.observability import degradations, provider_health def lock_name( @@ -109,6 +109,7 @@ async def _take(self, name: str, until: float) -> leases.Lease: ) from e if lease is None: raise _busy() + provider_health.record_success("memory_key_lock") return lease @asynccontextmanager diff --git a/tests/unit/test_memory_key_write_lock.py b/tests/unit/test_memory_key_write_lock.py index d09592ea7b..e491b6aeaf 100644 --- a/tests/unit/test_memory_key_write_lock.py +++ b/tests/unit/test_memory_key_write_lock.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +from contextlib import nullcontext from types import SimpleNamespace from typing import Any @@ -13,6 +14,7 @@ from server.memory.app import update as update_mod from server.memory.app import write as write_mod from server.memory.store import key_write_lock as kl +from server.observability import provider_health from tests.unit.claims_fake import FakeLockTable _IDENT = ("org", "ns", "agent", "-", "-", "proc/deploy/steps") @@ -92,6 +94,46 @@ async def test_keeper_failure_refuses_the_write( assert notes == [("memory_key_lock", "keeper_unavailable")] +@pytest.mark.parametrize("body_fails", [False, True]) +async def test_key_lock_health_recovers_when_the_lease_is_acquired( + monkeypatch: pytest.MonkeyPatch, body_fails: bool +) -> None: + monkeypatch.setattr(provider_health, "_state", {}) + ch = FakeKeeper(fail=True) + locks = _locks(ch, monkeypatch) + with pytest.raises(BackendUnavailableError): + async with locks.hold(*_IDENT): + pytest.fail("entered without a lease") + assert provider_health.status()["providers"]["memory_key_lock"]["failing"] is True + + provider_health.record_failure("other_provider", "still unavailable") + ch.down = False + with pytest.raises(ValueError, match="body failed") if body_fails else nullcontext(): + async with locks.hold(*_IDENT): + assert provider_health.status()["providers"]["memory_key_lock"]["failing"] is False + if body_fails: + raise ValueError("body failed") + assert provider_health.status()["providers"]["other_provider"]["failing"] is True + assert ch.keys == {} and len(locks) == 0 + + +async def test_a_busy_key_does_not_clear_the_last_acquisition_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(provider_health, "_state", {}) + ch = FakeKeeper(fail=True) + locks = _locks(ch, monkeypatch) + with pytest.raises(BackendUnavailableError): + async with locks.hold(*_IDENT): + pytest.fail("entered without a lease") + ch.down = False + ch.seed(kl.lock_name(*_IDENT), "other") + with pytest.raises(PreconditionFailedError): + async with locks.hold(*_IDENT): + pytest.fail("entered a held lock") + assert provider_health.status()["providers"]["memory_key_lock"]["failing"] is True + + async def test_racing_conditional_sets_let_one_through( monkeypatch: pytest.MonkeyPatch, ) -> None: From c103988bf747874067d08b2bc12f5aca1a8f4d09 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 21:09:53 +0000 Subject: [PATCH 06/12] Scope person-query observations to the decision task Shared client schema reads run with an independent context. Record only decision-tagged statements while retaining unbounded decision reads as failures. Signed-off-by: zlareb1 (cherry picked from commit 57327d5ea5778042f860cadf6cebec9eebb25821) Signed-off-by: zlareb1 --- tests/integration/test_scim_person_live_ch.py | 8 +- .../test_scim_person_query_observation.py | 116 ++++++++++++++++++ 2 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_scim_person_query_observation.py diff --git a/tests/integration/test_scim_person_live_ch.py b/tests/integration/test_scim_person_live_ch.py index 34818825b7..a3eeafaa1b 100644 --- a/tests/integration/test_scim_person_live_ch.py +++ b/tests/integration/test_scim_person_live_ch.py @@ -125,13 +125,17 @@ async def people(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[People]: query_all, query_one = client.query_all, client.query_one async def all_rows(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) -> Any: + observing_person = ch.current_query_tag() == "auth.scim_people" answer = await query_all(sql, params=params, **kwargs) - made.results.append((sql, ch.statement_settings(sql), answer)) + if observing_person: + made.results.append((sql, ch.statement_settings(sql), answer)) return answer async def one_row(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) -> Any: + observing_person = ch.current_query_tag() == "auth.scim_people" answer = await query_one(sql, params=params, **kwargs) - made.results.append((sql, ch.statement_settings(sql), answer)) + if observing_person: + made.results.append((sql, ch.statement_settings(sql), answer)) return answer monkeypatch.setattr(client, "query_all", all_rows) diff --git a/tests/unit/test_scim_person_query_observation.py b/tests/unit/test_scim_person_query_observation.py new file mode 100644 index 0000000000..a0f9ef7f06 --- /dev/null +++ b/tests/unit/test_scim_person_query_observation.py @@ -0,0 +1,116 @@ +"""Person-query evidence excludes reads from independent background tasks.""" + +from __future__ import annotations + +import asyncio +from contextvars import Context +from typing import Any + +import pytest + +from server.auth import scim_people +from server.core import ch, rmt_carry +from server.settings import settings +from tests.integration.test_scim_person_live_ch import people as person_fixture +from tests.unit.test_ch_client_hygiene import _client, _FakeAsync + + +class Queries: + def __init__(self) -> None: + self.started = asyncio.Event() + self.background_done = asyncio.Event() + self.block_person = False + self.transport: list[tuple[str, dict[str, Any]]] = [] + + async def connect(self) -> None: + pass + + async def disconnect(self) -> None: + pass + + async def _read(self, sql: str) -> dict[str, Any]: + self.transport.append((sql, ch.statement_settings(sql, cls="read"))) + if sql == "SELECT 1 AS person" and self.block_person: + self.started.set() + await self.background_done.wait() + return {"n": 0, "at": None, "altered": None} + + async def query_one(self, sql: str, params: Any = None) -> dict[str, Any]: + return await self._read(sql) + + async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: + return [await self._read(sql)] + + +@pytest.fixture +async def recording(monkeypatch): + client = Queries() + monkeypatch.setattr(ch, "get_client", lambda: client) + stream = person_fixture.__wrapped__(monkeypatch) + observed = await anext(stream) + try: + yield observed, client + finally: + await stream.aclose() + + +@pytest.mark.parametrize("read_timeout", [0, 2.5]) +@pytest.mark.parametrize("many", [False, True]) +async def test_background_schema_reads_do_not_enter_person_evidence( + recording, monkeypatch, read_timeout, many +): + observed, client = recording + monkeypatch.setattr(settings, "ch_timeout_read_s", read_timeout) + client.block_person = True + + async def background(): + await client.started.wait() + try: + with ch.query_tag("rmt_carry.columns"): + await rmt_carry._schema_stamp(client) + finally: + client.background_done.set() + + watcher = asyncio.create_task(background(), context=Context()) + try: + with ch.statement_timeout(1.75), ch.query_tag("auth.scim_people"): + query = client.query_all if many else client.query_one + await query("SELECT 1 AS person") + await watcher + finally: + if not watcher.done(): + watcher.cancel() + await asyncio.gather(watcher, return_exceptions=True) + + assert len(client.transport) == 2 + assert any("system.tables" in sql for sql, _ in client.transport) + assert [sql for sql, _, _ in observed.results] == ["SELECT 1 AS person"] + assert 0 < observed.results[0][1]["max_execution_time"] <= 1.75 + + +@pytest.mark.parametrize("many", [False, True]) +async def test_unbounded_person_reads_remain_visible(recording, monkeypatch, many): + observed, client = recording + monkeypatch.setattr(settings, "ch_timeout_read_s", 0) + with ch.query_tag("auth.scim_people"): + query = client.query_all if many else client.query_one + await query("SELECT 1 AS person") + assert len(observed.results) == 1 + with pytest.raises(KeyError, match="max_execution_time"): + assert 0 < observed.results[0][1]["max_execution_time"] <= 1.75 + + +@pytest.mark.parametrize("read_timeout", [0, 2.5]) +@pytest.mark.parametrize("many", [False, True]) +async def test_actual_person_reader_applies_its_finite_budget_to_driver( + monkeypatch, read_timeout, many +): + driver = _FakeAsync() + client = _client(monkeypatch, driver, ch_timeout_read_s=read_timeout) + monkeypatch.setattr(ch, "get_client", lambda: client) + decision = scim_people._Decision("org", "owner", "owner@example.test", scim_people._now() + 10) + with ch.statement_timeout(1.75), ch.query_tag("auth.scim_people"): + await decision.query("SELECT 1 AS person", {}, many=many) + calls = [applied for kind, sql, applied in driver.calls if sql == "SELECT 1 AS person"] + assert len(calls) == 1 + assert 0 < calls[0]["max_execution_time"] <= 1.75 From 24acbce546f99a55139f9b5d882cbc36aa828eb8 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 12:52:34 +0000 Subject: [PATCH 07/12] Bound JSON encoder setup in Cell proof regression cases Exercise each default-field refusal and JSON numeric distinction before consolidating repeated default comparisons. Signed-off-by: zlareb1 (cherry picked from commit 73ef63b0b7b5e65254dc2618d19c267017fed904) Signed-off-by: zlareb1 (cherry picked from commit cb594d01ade7d2b883849670db9f314a46f4abc4) Signed-off-by: zlareb1 --- .../test_cell_trigger_default_comparison.py | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 tests/unit/test_cell_trigger_default_comparison.py diff --git a/tests/unit/test_cell_trigger_default_comparison.py b/tests/unit/test_cell_trigger_default_comparison.py new file mode 100644 index 0000000000..4f712913a7 --- /dev/null +++ b/tests/unit/test_cell_trigger_default_comparison.py @@ -0,0 +1,66 @@ +"""Default-field proof checks keep their refusals without per-field JSON setup.""" + +from copy import deepcopy + +import pytest + +from tests.unit.test_cell_trigger_ci_creator import keeper_config_case as keeper_config_case +from tests.unit.test_cell_trigger_ci_creator import protocol_reader as protocol_reader +from tests.unit.test_cell_trigger_provenance_reuse import _case + + +def _expected(native, specification, context, row, audit, api): + native.Codec(specification).expected( + row, + audit, + context, + api=api, + metadata={"summary": {"jti": "b" * 32}}, + driver={"host": "proof-host", "pid": 701}, + ) + + +@pytest.mark.parametrize("api", [False, True], ids=["delivery", "api"]) +def test_default_proof_bounds_json_encoder_setup( + keeper_config_case, protocol_reader, monkeypatch, api +): + native, *_ = keeper_config_case + specification, context, _, _, memories, audits, _ = _case(native, 1, api) + calls = 0 + original = native.json.dumps + + def counted(*args, **kwargs): + nonlocal calls + calls += 1 + return original(*args, **kwargs) + + monkeypatch.setattr(native.json, "dumps", counted) + _expected(native, specification, context, memories[0], audits[0], api) + # One row has 43 defaults; encoder setup must not repeat for each field. + assert calls <= 16 + + +@pytest.mark.parametrize("index", range(43)) +def test_default_proof_refuses_each_changed_field(keeper_config_case, protocol_reader, index): + native, *_ = keeper_config_case + specification, context, _, _, memories, audits, _ = _case(native, 1, False) + field, default = list(specification["sparse_defaults"].items())[index] + assert default != {"unexpected": True} + row = deepcopy(memories[0]) + row[field] = {"unexpected": True} + with pytest.raises(native.Refused): + _expected(native, specification, context, row, audits[0], False) + + +@pytest.mark.parametrize("replacement", [False, 0.0]) +def test_default_proof_keeps_json_numeric_type_refusals( + keeper_config_case, protocol_reader, replacement +): + native, *_ = keeper_config_case + specification, context, _, _, memories, audits, _ = _case(native, 1, False) + assert specification["sparse_defaults"]["pinned"] == 0 + row = deepcopy(memories[0]) + row["pinned"] = replacement + assert row["pinned"] == 0 + with pytest.raises(native.Refused): + _expected(native, specification, context, row, audits[0], False) From bc30ceea8f7193108db8fa584a234595d8a28a6b Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 13:14:00 +0000 Subject: [PATCH 08/12] Compare Cell proof defaults in one canonical mapping Keep every default-field and JSON type check while avoiding a separate encoder setup for each field. Signed-off-by: zlareb1 (cherry picked from commit 0bfc712dcca9684e362af47a7e924432e87e53dd) Signed-off-by: zlareb1 (cherry picked from commit a94d2f909a47d61532d0f6ac24aad79b09ac42b9) Signed-off-by: zlareb1 --- scripts/gates/testsupport/cell_trigger/capture.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/gates/testsupport/cell_trigger/capture.py b/scripts/gates/testsupport/cell_trigger/capture.py index 635df62581..0360fbdb8c 100644 --- a/scripts/gates/testsupport/cell_trigger/capture.py +++ b/scripts/gates/testsupport/cell_trigger/capture.py @@ -255,8 +255,10 @@ def expected(self, row, audit, context, *, api, metadata, driver): "workload semantic/active", ) need(row["confidence"] == 1.0, "workload confidence") - for field, default in self.p["sparse_defaults"].items(): - equal(row[field], default) + equal( + {field: row[field] for field in self.p["sparse_defaults"]}, + self.p["sparse_defaults"], + ) need( row["value"] == json.dumps(payload(key)) and row["content_sha256"] == content_hash(key), "exact stored payload/hash", From d66d60b642a8198e617ac8b30d1fd2f9a2f76ba0 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 22:25:38 +0000 Subject: [PATCH 09/12] Cleanup tests measure their shared idle deadline Keep the blocked close, deadline observers and loop timers on one private clock. A controlled independent clock advance reproduces the elapsed-time dependency while the original cleanup and refusal assertions stay intact. Signed-off-by: zlareb1 (cherry picked from commit 4b03db4d8e5ac44d69af2146c371066907a5ec06) Signed-off-by: zlareb1 --- tests/unit/test_cell_attach_cleanup.py | 72 +++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_cell_attach_cleanup.py b/tests/unit/test_cell_attach_cleanup.py index ec359f3d40..88df8ee92b 100644 --- a/tests/unit/test_cell_attach_cleanup.py +++ b/tests/unit/test_cell_attach_cleanup.py @@ -6,6 +6,7 @@ from unittest.mock import AsyncMock import pytest +import pytest_asyncio from server.cli import cell_attach, cell_control from tests.unit.cell_control_owner import owner @@ -96,7 +97,8 @@ async def work(): await asyncio.gather(*owned_tasks, return_exceptions=True) -@pytest.mark.asyncio +@pytest.mark.asyncio(loop_scope="function") +@pytest.mark.usefixtures("_cleanup_clock") async def test_lock_owner_bounds_close_inside_the_same_cleanup_allowance(): closing = asyncio.Event() conn = AsyncMock() @@ -152,3 +154,71 @@ async def open(owned, what): if failed: assert caught.value is original conn.close.assert_awaited_once() + + +@pytest.mark.parametrize("scheduler_s", [0.0, 0.16]) +@pytest.mark.asyncio(loop_scope="function") +@pytest.mark.usefixtures("_cleanup_clock") +async def test_cleanup_allowance_ignores_independent_scheduler_time(monkeypatch, scheduler_s): + import time as wall_time + + real_monotonic = wall_time.monotonic + elapsed = 0.0 + + def monotonic(): + return real_monotonic() + elapsed + + actual_close = cell_control.Owner.close + + async def close(owned): + nonlocal elapsed + try: + return await actual_close(owned) + finally: + elapsed += scheduler_s + + monkeypatch.setattr(wall_time, "monotonic", monotonic) + monkeypatch.setattr(cell_control.Owner, "close", close) + await test_lock_owner_bounds_close_inside_the_same_cleanup_allowance() + + +@pytest_asyncio.fixture(loop_scope="function") +async def _cleanup_clock(monkeypatch): + from server.core import pg + from tests.unit.clock_fake import ManualClock + + before = set(asyncio.all_tasks()) + loop = asyncio.get_running_loop() + clock = ManualClock(t=loop.time()) + selector = loop._selector + real_select = selector.select + + def select(timeout=None): + if timeout is not None: + clock.advance(max(0.0, timeout)) + timeout = 0 + return real_select(timeout) + + # The cleanup protocol spends idle time, independently of CPU scheduling. + monkeypatch.setattr(loop, "time", clock.monotonic) + monkeypatch.setattr(selector, "select", select) + monkeypatch.setattr(cell_control, "time", clock) + monkeypatch.setattr(pg, "time", clock) + monkeypatch.setattr(f"{__name__}.time", clock) + yield clock + leaked = asyncio.all_tasks() - before - {asyncio.current_task()} + for task in leaked: + task.cancel() + await asyncio.gather(*leaked, return_exceptions=True) + assert not leaked, "cell cleanup leaves no task on its private loop" + + +@pytest.mark.asyncio(loop_scope="function") +async def test_cleanup_clock_preserves_the_close_deadline(_cleanup_clock): + from server.core import pg + + loop = asyncio.get_running_loop() + assert loop.time == time.monotonic == cell_control.time.monotonic == pg.time.monotonic + began = _cleanup_clock.monotonic() + await test_lock_owner_bounds_close_inside_the_same_cleanup_allowance() + assert _cleanup_clock.monotonic() - began == pytest.approx(0.03) From f78fe82a2a618cb39f69da7c65818de4262a93f6 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Sat, 21 Nov 2026 00:31:45 +0000 Subject: [PATCH 10/12] Keep first-content deadline fixture on an idle clock The fake leg delays and request budget share a private event loop clock. Preserve the original timing inputs and assertions, and require blocked legs to time out and cancel. Signed-off-by: zlareb1 (cherry picked from commit bfb9d4070e9f03d13ef3d7db71ffe30715823daf) Signed-off-by: zlareb1 --- tests/unit/test_code_search_deadline.py | 138 ++++++++++++++++++++++++ 1 file changed, 138 insertions(+) diff --git a/tests/unit/test_code_search_deadline.py b/tests/unit/test_code_search_deadline.py index 738495b27e..c7866d5243 100644 --- a/tests/unit/test_code_search_deadline.py +++ b/tests/unit/test_code_search_deadline.py @@ -15,7 +15,10 @@ from __future__ import annotations import asyncio +import sys import time +from contextlib import contextmanager +from types import SimpleNamespace from typing import Any from unittest.mock import patch @@ -630,3 +633,138 @@ async def _put(*a: Any, **k: Any) -> None: assert data["epistemic"] == "partial" assert data["retrieval_work"]["vector_leg_failed"] == 1 assert puts == [] + + +@contextmanager +def _first_content_clock(): + from server.code.domain import deadline as code_deadline + from tests.unit.clock_fake import ManualClock + + clock = ManualClock(t=1000.0) + real_asyncio = asyncio + + def loop_factory(): + loop = real_asyncio.new_event_loop() + selector = loop._selector + real_select = selector.select + + def select(timeout=None): + events = real_select(0) + if not events and timeout is not None: + clock.advance(max(0.0, timeout)) + return events + + loop.time = clock.monotonic + selector.select = select + return loop + + async def checked(coro): + before = real_asyncio.all_tasks() + try: + return await coro + finally: + leaked = real_asyncio.all_tasks() - before - {real_asyncio.current_task()} + for task in leaked: + task.cancel() + await real_asyncio.gather(*leaked, return_exceptions=True) + assert not leaked, "search deadline leaves no task on its private loop" + + def run(coro): + with real_asyncio.Runner(loop_factory=loop_factory) as runner: + return runner.run(checked(coro)) + + # Fake leg sleeps and request budgets share idle time, independently of CPU scheduling. + timers = SimpleNamespace( + **{**vars(time), "monotonic": clock.monotonic, "perf_counter": clock.monotonic} + ) + with ( + patch.object( + sys.modules[__name__], "asyncio", SimpleNamespace(**{**vars(real_asyncio), "run": run}) + ), + patch.object(sys.modules[__name__], "time", timers), + patch.object(code_deadline, "time", timers), + patch.object(legs, "time", timers), + ): + yield clock + + +@pytest.fixture(autouse=True) +def _first_content_deadline_clock(request): + if request.node.name != "test_the_deadline_waits_for_the_first_content_leg": + yield + return + with _first_content_clock(): + yield + + +@pytest.mark.parametrize("jump", [0.0, 0.12]) +def test_first_content_clock_ignores_unrelated_pre_leg_time(jump): + from server.code.domain import deadline as code_deadline + + offset = 0.0 + calls = [] + real_monotonic = time.monotonic + actual_budget = cs_gather._leg_budget + + def leg_budget(*args, **kwargs): + nonlocal offset + offset += jump + calls.append(offset) + return actual_budget(*args, **kwargs) + + with ( + patch.object( + code_deadline, "time", SimpleNamespace(monotonic=lambda: real_monotonic() + offset) + ), + patch.object(cs_gather, "_leg_budget", leg_budget), + _first_content_clock(), + ): + test_the_deadline_waits_for_the_first_content_leg() + assert calls == [jump] + + +def test_first_content_clock_still_drops_and_cancels_genuinely_blocked_legs(): + cancelled = [] + + class BlockedRepo(_Repo): + async def lexical_search_chunks(self, *args, **kwargs): + try: + await asyncio.Event().wait() + finally: + cancelled.append("lexical") + + async def vector_search_chunks(self, *args, **kwargs): + try: + await asyncio.Event().wait() + finally: + cancelled.append("vector") + + with _first_content_clock() as clock: + _, res, wall = _run(BlockedRepo(), llm_legs="none", deadline_ms=400) + assert clock.t == pytest.approx(1000.4, abs=0.001) + assert 0.399 <= wall <= 0.401 + assert res.units == [] and res.abstain and res.partial and res.epistemic == "partial" + assert set(res.legs_timed_out) == {"lexical", "vector"} + assert sorted(cancelled) == ["lexical", "vector"] + + +def test_first_content_clock_drains_and_rejects_an_orphan_task(): + cancelled = [] + + async def orphan(): + try: + await asyncio.Event().wait() + finally: + cancelled.append(True) + + async def leak(): + asyncio.create_task(orphan()) + await asyncio.sleep(0) + + real_asyncio, real_time = asyncio, time + real_run, real_monotonic = asyncio.run, time.monotonic + with _first_content_clock(), pytest.raises(AssertionError, match="leaves no task"): + asyncio.run(leak()) + assert cancelled == [True] + assert asyncio is real_asyncio and asyncio.run is real_run + assert time is real_time and time.monotonic is real_monotonic From 85a332343229b685ed1931f0ae770bd32540df32 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Sat, 21 Nov 2026 02:29:31 +0000 Subject: [PATCH 11/12] Keep trigger deadline fixtures on their private loop Signed-off-by: zlareb1 --- tests/unit/test_cell_attach.py | 145 ++++++++++++++++++++++++++++++++- 1 file changed, 144 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 6862f8ea0d..44cd116a09 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -8,12 +8,15 @@ import argparse import asyncio import json +import sys from collections.abc import Callable +from contextlib import asynccontextmanager from dataclasses import dataclass, field from typing import Any import httpx import pytest +import pytest_asyncio from server.cli import cell_attach, cell_control, cell_scan from server.cli import cell_attach_beats as beats @@ -1104,7 +1107,7 @@ async def ensure_table(*_a: Any, lock_timeout: str, **_kw: Any) -> Any: assert asked == [f"{apg.TRY_FLOOR_MS}ms"] -@pytest.mark.asyncio +@pytest.mark.asyncio(loop_scope="function") @pytest.mark.parametrize("stall", ["migration", "reset", "ddl", "commit", "rollback", "unlock"]) async def test_a_trigger_try_deadline_covers_transaction_and_lock_cleanup( monkeypatch: pytest.MonkeyPatch, clock: _Clock, stall: str @@ -1479,3 +1482,143 @@ async def pg_orgs(table: str, _budget: Any) -> list[str]: "not read: postgres loom.a_unindexed: has no plain btree index led by org", ) assert "cell_stray_orgs_unchecked" not in lines + + +@asynccontextmanager +async def _trigger_try_idle_clock(): + loop = asyncio.get_running_loop() + clock = ManualClock(t=loop.time()) + before = asyncio.all_tasks() + selector = loop._selector + real_select = selector.select + actual_short_phases = _short_phases + + def select(timeout=None): + events = real_select(0) + if not events and timeout is not None: + clock.advance(max(0.0, timeout)) + return events + + with pytest.MonkeyPatch.context() as scoped: + + def short_phases(_monkeypatch, bounds=None): + assert asyncio.get_running_loop() is loop, ( + "trigger test and clock fixture must use the same loop" + ) + actual_short_phases(scoped, bounds) + scoped.setattr(cell_control, "time", clock) + scoped.setattr(cell_control.pg, "time", clock) + + # Work, close and the watchdog spend the same private loop's idle time. + scoped.setattr(loop, "time", clock.monotonic) + scoped.setattr(selector, "select", select) + scoped.setattr(sys.modules[__name__], "_short_phases", short_phases) + try: + yield clock + finally: + leaked = asyncio.all_tasks() - before - {asyncio.current_task()} + for task in leaked: + task.cancel() + await asyncio.gather(*leaked, return_exceptions=True) + assert not leaked, "trigger deadline leaves no task on its private loop" + + +@pytest.fixture(autouse=True) +def _trigger_try_deadline_clock(request): + if ( + getattr(request.node, "originalname", "") + == "test_a_trigger_try_deadline_covers_transaction_and_lock_cleanup" + ): + request.getfixturevalue("_trigger_try_clock") + + +@pytest_asyncio.fixture(loop_scope="function") +async def _trigger_try_clock(): + async with _trigger_try_idle_clock(): + yield + + +@pytest.mark.asyncio(loop_scope="function") +@pytest.mark.parametrize("jump", [0.0, 0.25]) +async def test_trigger_try_clock_ignores_unrelated_scheduling(monkeypatch, clock, jump): + loop = asyncio.get_running_loop() + external = ManualClock(t=loop.time()) + real_select = loop._selector.select + actual_start = cell_control.Owner._start + calls = [] + + def select(timeout=None): + events = real_select(0) + if not events and timeout is not None: + external.advance(max(0.0, timeout)) + return events + + def start(self, work): + task = actual_start(self, work) + external.advance(jump) + calls.append(jump) + return task + + with pytest.MonkeyPatch.context() as outer: + outer.setattr(loop, "time", external.monotonic) + outer.setattr(loop._selector, "select", select) + outer.setattr(cell_control.Owner, "_start", start) + async with _trigger_try_idle_clock(): + await test_a_trigger_try_deadline_covers_transaction_and_lock_cleanup( + monkeypatch, clock, "reset" + ) + assert calls == [jump] + + +@pytest.mark.asyncio(loop_scope="function") +async def test_trigger_try_clock_still_cancels_genuinely_blocked_cleanup(monkeypatch): + cancelled = asyncio.Event() + + class BlockedClose(FakeConn): + async def close(self, **kwargs): + assert kwargs["timeout"] > 0 + try: + await asyncio.Future() + finally: + cancelled.set() + + async def work(): + await asyncio.Future() + + conn = BlockedClose() + owner = _owner(conn) + async with _trigger_try_idle_clock() as clock: + _short_phases(monkeypatch) + started = clock.t + task = asyncio.create_task(owner.call(work, total_s=1.0)) + done, _ = await asyncio.wait([task], timeout=0.2) + assert task in done + assert isinstance(task.exception(), TimeoutError) + await asyncio.wait_for(cancelled.wait(), timeout=0.01) + assert clock.t - started == pytest.approx(0.01) + assert "TERMINATE" in conn.executed and "CLOSE" not in conn.executed + assert owner._close_task is not None and owner._close_task.done() + + +@pytest.mark.asyncio(loop_scope="function") +async def test_trigger_try_clock_drains_and_rejects_an_orphan_task(): + cancelled = [] + + async def orphan(): + try: + await asyncio.Future() + finally: + cancelled.append(True) + + loop = asyncio.get_running_loop() + real_loop_clock = loop.time + real_owner_clock, real_pg_clock = cell_control.time, cell_control.pg.time + real_short_phases = _short_phases + with pytest.raises(AssertionError, match="leaves no task"): + async with _trigger_try_idle_clock(): + asyncio.create_task(orphan()) + await asyncio.sleep(0) + assert cancelled == [True] + assert loop.time == real_loop_clock + assert cell_control.time is real_owner_clock and cell_control.pg.time is real_pg_clock + assert _short_phases is real_short_phases From a191e0d25aca3500c0976d6497504ba10cf53dad Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Sat, 21 Nov 2026 09:32:34 +0000 Subject: [PATCH 12/12] Cover the review provenance default in Cell proof refusals Signed-off-by: zlareb1 --- tests/unit/test_cell_trigger_default_comparison.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_cell_trigger_default_comparison.py b/tests/unit/test_cell_trigger_default_comparison.py index 4f712913a7..f090ed9cf7 100644 --- a/tests/unit/test_cell_trigger_default_comparison.py +++ b/tests/unit/test_cell_trigger_default_comparison.py @@ -36,11 +36,11 @@ def counted(*args, **kwargs): monkeypatch.setattr(native.json, "dumps", counted) _expected(native, specification, context, memories[0], audits[0], api) - # One row has 43 defaults; encoder setup must not repeat for each field. + # Encoder setup stays bounded as the protocol adds default fields. assert calls <= 16 -@pytest.mark.parametrize("index", range(43)) +@pytest.mark.parametrize("index", range(44)) def test_default_proof_refuses_each_changed_field(keeper_config_case, protocol_reader, index): native, *_ = keeper_config_case specification, context, _, _, memories, audits, _ = _case(native, 1, False)