Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions scripts/gates/testsupport/cell_trigger/capture.py
Original file line number Diff line number Diff line change
Expand Up @@ -256,8 +256,10 @@ def expected(self, row, audit, context, *, api, metadata, driver):
"workload semantic/active",
)
need(row["confidence"] == 1.0, "workload confidence")
for field, default in self.p["sparse_defaults"].items():
equal(row[field], default)
equal(
{field: row[field] for field in self.p["sparse_defaults"]},
self.p["sparse_defaults"],
)
need(
row["value"] == json.dumps(payload(key)) and row["content_sha256"] == content_hash(key),
"exact stored payload/hash",
Expand Down
3 changes: 3 additions & 0 deletions server/workers/leader.py
Original file line number Diff line number Diff line change
Expand Up @@ -334,6 +334,9 @@ async def _until_done(
done, _ = await asyncio.wait({fut}, timeout=timeout)
if done:
return running
# A renewal can land while the previous deadline's wait resumes.
if lease.fresh():
continue
log.warning(
"supervisor lease: no renewal within %.0f s; stopping the workers",
float(settings.supervisor_lease_ttl_s) / 2,
Expand Down
72 changes: 71 additions & 1 deletion tests/unit/test_cell_attach_cleanup.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from unittest.mock import AsyncMock

import pytest
import pytest_asyncio

from server.cli import cell_attach, cell_control
from tests.unit.cell_control_owner import owner
Expand Down Expand Up @@ -96,7 +97,8 @@ async def work():
await asyncio.gather(*owned_tasks, return_exceptions=True)


@pytest.mark.asyncio
@pytest.mark.asyncio(loop_scope="function")
@pytest.mark.usefixtures("_cleanup_clock")
async def test_lock_owner_bounds_close_inside_the_same_cleanup_allowance():
closing = asyncio.Event()
conn = AsyncMock()
Expand Down Expand Up @@ -152,3 +154,71 @@ async def open(owned, what):
if failed:
assert caught.value is original
conn.close.assert_awaited_once()


@pytest.mark.parametrize("scheduler_s", [0.0, 0.16])
@pytest.mark.asyncio(loop_scope="function")
@pytest.mark.usefixtures("_cleanup_clock")
async def test_cleanup_allowance_ignores_independent_scheduler_time(monkeypatch, scheduler_s):
import time as wall_time

real_monotonic = wall_time.monotonic
elapsed = 0.0

def monotonic():
return real_monotonic() + elapsed

actual_close = cell_control.Owner.close

async def close(owned):
nonlocal elapsed
try:
return await actual_close(owned)
finally:
elapsed += scheduler_s

monkeypatch.setattr(wall_time, "monotonic", monotonic)
monkeypatch.setattr(cell_control.Owner, "close", close)
await test_lock_owner_bounds_close_inside_the_same_cleanup_allowance()


@pytest_asyncio.fixture(loop_scope="function")
async def _cleanup_clock(monkeypatch):
from server.core import pg
from tests.unit.clock_fake import ManualClock

before = set(asyncio.all_tasks())
loop = asyncio.get_running_loop()
clock = ManualClock(t=loop.time())
selector = loop._selector
real_select = selector.select

def select(timeout=None):
if timeout is not None:
clock.advance(max(0.0, timeout))
timeout = 0
return real_select(timeout)

# The cleanup protocol spends idle time, independently of CPU scheduling.
monkeypatch.setattr(loop, "time", clock.monotonic)
monkeypatch.setattr(selector, "select", select)
monkeypatch.setattr(cell_control, "time", clock)
monkeypatch.setattr(pg, "time", clock)
monkeypatch.setattr(f"{__name__}.time", clock)
yield clock
leaked = asyncio.all_tasks() - before - {asyncio.current_task()}
for task in leaked:
task.cancel()
await asyncio.gather(*leaked, return_exceptions=True)
assert not leaked, "cell cleanup leaves no task on its private loop"


@pytest.mark.asyncio(loop_scope="function")
async def test_cleanup_clock_preserves_the_close_deadline(_cleanup_clock):
from server.core import pg

loop = asyncio.get_running_loop()
assert loop.time == time.monotonic == cell_control.time.monotonic == pg.time.monotonic
began = _cleanup_clock.monotonic()
await test_lock_owner_bounds_close_inside_the_same_cleanup_allowance()
assert _cleanup_clock.monotonic() - began == pytest.approx(0.03)
66 changes: 66 additions & 0 deletions tests/unit/test_cell_trigger_default_comparison.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
"""Default-field proof checks keep their refusals without per-field JSON setup."""

from copy import deepcopy

import pytest

from tests.unit.test_cell_trigger_ci_creator import keeper_config_case as keeper_config_case
from tests.unit.test_cell_trigger_ci_creator import protocol_reader as protocol_reader
from tests.unit.test_cell_trigger_provenance_reuse import _case


def _expected(native, specification, context, row, audit, api):
native.Codec(specification).expected(
row,
audit,
context,
api=api,
metadata={"summary": {"jti": "b" * 32}},
driver={"host": "proof-host", "pid": 701},
)


@pytest.mark.parametrize("api", [False, True], ids=["delivery", "api"])
def test_default_proof_bounds_json_encoder_setup(
keeper_config_case, protocol_reader, monkeypatch, api
):
native, *_ = keeper_config_case
specification, context, _, _, memories, audits, _ = _case(native, 1, api)
calls = 0
original = native.json.dumps

def counted(*args, **kwargs):
nonlocal calls
calls += 1
return original(*args, **kwargs)

monkeypatch.setattr(native.json, "dumps", counted)
_expected(native, specification, context, memories[0], audits[0], api)
# Encoder setup stays bounded as the protocol adds default fields.
assert calls <= 16


@pytest.mark.parametrize("index", range(44))
def test_default_proof_refuses_each_changed_field(keeper_config_case, protocol_reader, index):
native, *_ = keeper_config_case
specification, context, _, _, memories, audits, _ = _case(native, 1, False)
field, default = list(specification["sparse_defaults"].items())[index]
assert default != {"unexpected": True}
row = deepcopy(memories[0])
row[field] = {"unexpected": True}
with pytest.raises(native.Refused):
_expected(native, specification, context, row, audits[0], False)


@pytest.mark.parametrize("replacement", [False, 0.0])
def test_default_proof_keeps_json_numeric_type_refusals(
keeper_config_case, protocol_reader, replacement
):
native, *_ = keeper_config_case
specification, context, _, _, memories, audits, _ = _case(native, 1, False)
assert specification["sparse_defaults"]["pinned"] == 0
row = deepcopy(memories[0])
row["pinned"] = replacement
assert row["pinned"] == 0
with pytest.raises(native.Refused):
_expected(native, specification, context, row, audits[0], False)
48 changes: 48 additions & 0 deletions tests/unit/test_supervisor_lease.py
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,54 @@ def test_a_holder_that_gave_up_takes_its_own_key_back_on_the_next_round() -> Non
assert "holds workers_supervisor" in lease.reason


@pytest.mark.parametrize(
("renew_at", "resume_at", "should_run"),
[(10.0, 16.0, True), (14.0, 20.0, True), (None, 16.0, False), (10.0, 26.0, False)],
)
async def test_old_wait_timeout_rechecks_confirmed_renewal(
monkeypatch: pytest.MonkeyPatch,
renew_at: float | None,
resume_at: float,
should_run: bool,
) -> None:
now = [0.0]
lock = _Lock([True], cfg=leader.lease_config(30))
lock.clock = lambda: now[0]
monkeypatch.setattr(leader, "_stand_aside_reason", lambda: "")
monkeypatch.setattr(leader, "newer_build", lambda _name: 0)
lease = _auto(lock)
lease.step()
pending = asyncio.get_running_loop().create_future()
stops: list[bool] = []
waits: list[float | None] = []

async def on_lost() -> None:
stops.append(True)

async def stale_wait(
fs: set[Any],
*,
timeout: float | None = None, # noqa: ASYNC109 — mirrors asyncio.wait's keyword
) -> Any:
assert fs == {pending}
waits.append(timeout)
if len(waits) == 1:
assert timeout == 15.0
if renew_at is not None:
now[0] = renew_at
lock.renew_if_due()
now[0] = resume_at
return set(), {pending}
pending.set_result(None)
return {pending}, set()

monkeypatch.setattr(leader.asyncio, "wait", stale_wait)
running = await leader._until_done(pending, lease, True, on_lost)
assert running is should_run
assert stops == ([] if should_run else [True])
assert waits == [15.0, (renew_at + 15.0 - resume_at) if should_run else None]


async def test_the_workers_stop_at_the_deadline_while_a_renewal_blocks(
monkeypatch: pytest.MonkeyPatch,
) -> None:
Expand Down
Loading