diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 6862f8ea0d..44cd116a09 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -8,12 +8,15 @@ import argparse import asyncio import json +import sys from collections.abc import Callable +from contextlib import asynccontextmanager from dataclasses import dataclass, field from typing import Any import httpx import pytest +import pytest_asyncio from server.cli import cell_attach, cell_control, cell_scan from server.cli import cell_attach_beats as beats @@ -1104,7 +1107,7 @@ async def ensure_table(*_a: Any, lock_timeout: str, **_kw: Any) -> Any: assert asked == [f"{apg.TRY_FLOOR_MS}ms"] -@pytest.mark.asyncio +@pytest.mark.asyncio(loop_scope="function") @pytest.mark.parametrize("stall", ["migration", "reset", "ddl", "commit", "rollback", "unlock"]) async def test_a_trigger_try_deadline_covers_transaction_and_lock_cleanup( monkeypatch: pytest.MonkeyPatch, clock: _Clock, stall: str @@ -1479,3 +1482,143 @@ async def pg_orgs(table: str, _budget: Any) -> list[str]: "not read: postgres loom.a_unindexed: has no plain btree index led by org", ) assert "cell_stray_orgs_unchecked" not in lines + + +@asynccontextmanager +async def _trigger_try_idle_clock(): + loop = asyncio.get_running_loop() + clock = ManualClock(t=loop.time()) + before = asyncio.all_tasks() + selector = loop._selector + real_select = selector.select + actual_short_phases = _short_phases + + def select(timeout=None): + events = real_select(0) + if not events and timeout is not None: + clock.advance(max(0.0, timeout)) + return events + + with pytest.MonkeyPatch.context() as scoped: + + def short_phases(_monkeypatch, bounds=None): + assert asyncio.get_running_loop() is loop, ( + "trigger test and clock fixture must use the same loop" + ) + actual_short_phases(scoped, bounds) + scoped.setattr(cell_control, "time", clock) + scoped.setattr(cell_control.pg, "time", clock) + + # Work, close and the watchdog spend the same private loop's idle time. + scoped.setattr(loop, "time", clock.monotonic) + scoped.setattr(selector, "select", select) + scoped.setattr(sys.modules[__name__], "_short_phases", short_phases) + try: + yield clock + finally: + leaked = asyncio.all_tasks() - before - {asyncio.current_task()} + for task in leaked: + task.cancel() + await asyncio.gather(*leaked, return_exceptions=True) + assert not leaked, "trigger deadline leaves no task on its private loop" + + +@pytest.fixture(autouse=True) +def _trigger_try_deadline_clock(request): + if ( + getattr(request.node, "originalname", "") + == "test_a_trigger_try_deadline_covers_transaction_and_lock_cleanup" + ): + request.getfixturevalue("_trigger_try_clock") + + +@pytest_asyncio.fixture(loop_scope="function") +async def _trigger_try_clock(): + async with _trigger_try_idle_clock(): + yield + + +@pytest.mark.asyncio(loop_scope="function") +@pytest.mark.parametrize("jump", [0.0, 0.25]) +async def test_trigger_try_clock_ignores_unrelated_scheduling(monkeypatch, clock, jump): + loop = asyncio.get_running_loop() + external = ManualClock(t=loop.time()) + real_select = loop._selector.select + actual_start = cell_control.Owner._start + calls = [] + + def select(timeout=None): + events = real_select(0) + if not events and timeout is not None: + external.advance(max(0.0, timeout)) + return events + + def start(self, work): + task = actual_start(self, work) + external.advance(jump) + calls.append(jump) + return task + + with pytest.MonkeyPatch.context() as outer: + outer.setattr(loop, "time", external.monotonic) + outer.setattr(loop._selector, "select", select) + outer.setattr(cell_control.Owner, "_start", start) + async with _trigger_try_idle_clock(): + await test_a_trigger_try_deadline_covers_transaction_and_lock_cleanup( + monkeypatch, clock, "reset" + ) + assert calls == [jump] + + +@pytest.mark.asyncio(loop_scope="function") +async def test_trigger_try_clock_still_cancels_genuinely_blocked_cleanup(monkeypatch): + cancelled = asyncio.Event() + + class BlockedClose(FakeConn): + async def close(self, **kwargs): + assert kwargs["timeout"] > 0 + try: + await asyncio.Future() + finally: + cancelled.set() + + async def work(): + await asyncio.Future() + + conn = BlockedClose() + owner = _owner(conn) + async with _trigger_try_idle_clock() as clock: + _short_phases(monkeypatch) + started = clock.t + task = asyncio.create_task(owner.call(work, total_s=1.0)) + done, _ = await asyncio.wait([task], timeout=0.2) + assert task in done + assert isinstance(task.exception(), TimeoutError) + await asyncio.wait_for(cancelled.wait(), timeout=0.01) + assert clock.t - started == pytest.approx(0.01) + assert "TERMINATE" in conn.executed and "CLOSE" not in conn.executed + assert owner._close_task is not None and owner._close_task.done() + + +@pytest.mark.asyncio(loop_scope="function") +async def test_trigger_try_clock_drains_and_rejects_an_orphan_task(): + cancelled = [] + + async def orphan(): + try: + await asyncio.Future() + finally: + cancelled.append(True) + + loop = asyncio.get_running_loop() + real_loop_clock = loop.time + real_owner_clock, real_pg_clock = cell_control.time, cell_control.pg.time + real_short_phases = _short_phases + with pytest.raises(AssertionError, match="leaves no task"): + async with _trigger_try_idle_clock(): + asyncio.create_task(orphan()) + await asyncio.sleep(0) + assert cancelled == [True] + assert loop.time == real_loop_clock + assert cell_control.time is real_owner_clock and cell_control.pg.time is real_pg_clock + assert _short_phases is real_short_phases diff --git a/tests/unit/test_code_search_deadline.py b/tests/unit/test_code_search_deadline.py index 738495b27e..c7866d5243 100644 --- a/tests/unit/test_code_search_deadline.py +++ b/tests/unit/test_code_search_deadline.py @@ -15,7 +15,10 @@ from __future__ import annotations import asyncio +import sys import time +from contextlib import contextmanager +from types import SimpleNamespace from typing import Any from unittest.mock import patch @@ -630,3 +633,138 @@ async def _put(*a: Any, **k: Any) -> None: assert data["epistemic"] == "partial" assert data["retrieval_work"]["vector_leg_failed"] == 1 assert puts == [] + + +@contextmanager +def _first_content_clock(): + from server.code.domain import deadline as code_deadline + from tests.unit.clock_fake import ManualClock + + clock = ManualClock(t=1000.0) + real_asyncio = asyncio + + def loop_factory(): + loop = real_asyncio.new_event_loop() + selector = loop._selector + real_select = selector.select + + def select(timeout=None): + events = real_select(0) + if not events and timeout is not None: + clock.advance(max(0.0, timeout)) + return events + + loop.time = clock.monotonic + selector.select = select + return loop + + async def checked(coro): + before = real_asyncio.all_tasks() + try: + return await coro + finally: + leaked = real_asyncio.all_tasks() - before - {real_asyncio.current_task()} + for task in leaked: + task.cancel() + await real_asyncio.gather(*leaked, return_exceptions=True) + assert not leaked, "search deadline leaves no task on its private loop" + + def run(coro): + with real_asyncio.Runner(loop_factory=loop_factory) as runner: + return runner.run(checked(coro)) + + # Fake leg sleeps and request budgets share idle time, independently of CPU scheduling. + timers = SimpleNamespace( + **{**vars(time), "monotonic": clock.monotonic, "perf_counter": clock.monotonic} + ) + with ( + patch.object( + sys.modules[__name__], "asyncio", SimpleNamespace(**{**vars(real_asyncio), "run": run}) + ), + patch.object(sys.modules[__name__], "time", timers), + patch.object(code_deadline, "time", timers), + patch.object(legs, "time", timers), + ): + yield clock + + +@pytest.fixture(autouse=True) +def _first_content_deadline_clock(request): + if request.node.name != "test_the_deadline_waits_for_the_first_content_leg": + yield + return + with _first_content_clock(): + yield + + +@pytest.mark.parametrize("jump", [0.0, 0.12]) +def test_first_content_clock_ignores_unrelated_pre_leg_time(jump): + from server.code.domain import deadline as code_deadline + + offset = 0.0 + calls = [] + real_monotonic = time.monotonic + actual_budget = cs_gather._leg_budget + + def leg_budget(*args, **kwargs): + nonlocal offset + offset += jump + calls.append(offset) + return actual_budget(*args, **kwargs) + + with ( + patch.object( + code_deadline, "time", SimpleNamespace(monotonic=lambda: real_monotonic() + offset) + ), + patch.object(cs_gather, "_leg_budget", leg_budget), + _first_content_clock(), + ): + test_the_deadline_waits_for_the_first_content_leg() + assert calls == [jump] + + +def test_first_content_clock_still_drops_and_cancels_genuinely_blocked_legs(): + cancelled = [] + + class BlockedRepo(_Repo): + async def lexical_search_chunks(self, *args, **kwargs): + try: + await asyncio.Event().wait() + finally: + cancelled.append("lexical") + + async def vector_search_chunks(self, *args, **kwargs): + try: + await asyncio.Event().wait() + finally: + cancelled.append("vector") + + with _first_content_clock() as clock: + _, res, wall = _run(BlockedRepo(), llm_legs="none", deadline_ms=400) + assert clock.t == pytest.approx(1000.4, abs=0.001) + assert 0.399 <= wall <= 0.401 + assert res.units == [] and res.abstain and res.partial and res.epistemic == "partial" + assert set(res.legs_timed_out) == {"lexical", "vector"} + assert sorted(cancelled) == ["lexical", "vector"] + + +def test_first_content_clock_drains_and_rejects_an_orphan_task(): + cancelled = [] + + async def orphan(): + try: + await asyncio.Event().wait() + finally: + cancelled.append(True) + + async def leak(): + asyncio.create_task(orphan()) + await asyncio.sleep(0) + + real_asyncio, real_time = asyncio, time + real_run, real_monotonic = asyncio.run, time.monotonic + with _first_content_clock(), pytest.raises(AssertionError, match="leaves no task"): + asyncio.run(leak()) + assert cancelled == [True] + assert asyncio is real_asyncio and asyncio.run is real_run + assert time is real_time and time.monotonic is real_monotonic