Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 144 additions & 1 deletion tests/unit/test_cell_attach.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,15 @@
import argparse
import asyncio
import json
import sys
from collections.abc import Callable
from contextlib import asynccontextmanager
from dataclasses import dataclass, field
from typing import Any

import httpx
import pytest
import pytest_asyncio

from server.cli import cell_attach, cell_control, cell_scan
from server.cli import cell_attach_beats as beats
Expand Down Expand Up @@ -1104,7 +1107,7 @@ async def ensure_table(*_a: Any, lock_timeout: str, **_kw: Any) -> Any:
assert asked == [f"{apg.TRY_FLOOR_MS}ms"]


@pytest.mark.asyncio
@pytest.mark.asyncio(loop_scope="function")
@pytest.mark.parametrize("stall", ["migration", "reset", "ddl", "commit", "rollback", "unlock"])
async def test_a_trigger_try_deadline_covers_transaction_and_lock_cleanup(
monkeypatch: pytest.MonkeyPatch, clock: _Clock, stall: str
Expand Down Expand Up @@ -1479,3 +1482,143 @@ async def pg_orgs(table: str, _budget: Any) -> list[str]:
"not read: postgres loom.a_unindexed: has no plain btree index led by org",
)
assert "cell_stray_orgs_unchecked" not in lines


@asynccontextmanager
async def _trigger_try_idle_clock():
loop = asyncio.get_running_loop()
clock = ManualClock(t=loop.time())
before = asyncio.all_tasks()
selector = loop._selector
real_select = selector.select
actual_short_phases = _short_phases

def select(timeout=None):
events = real_select(0)
if not events and timeout is not None:
clock.advance(max(0.0, timeout))
return events

with pytest.MonkeyPatch.context() as scoped:

def short_phases(_monkeypatch, bounds=None):
assert asyncio.get_running_loop() is loop, (
"trigger test and clock fixture must use the same loop"
)
actual_short_phases(scoped, bounds)
scoped.setattr(cell_control, "time", clock)
scoped.setattr(cell_control.pg, "time", clock)

# Work, close and the watchdog spend the same private loop's idle time.
scoped.setattr(loop, "time", clock.monotonic)
scoped.setattr(selector, "select", select)
scoped.setattr(sys.modules[__name__], "_short_phases", short_phases)
try:
yield clock
finally:
leaked = asyncio.all_tasks() - before - {asyncio.current_task()}
for task in leaked:
task.cancel()
await asyncio.gather(*leaked, return_exceptions=True)
assert not leaked, "trigger deadline leaves no task on its private loop"


@pytest.fixture(autouse=True)
def _trigger_try_deadline_clock(request):
if (
getattr(request.node, "originalname", "")
== "test_a_trigger_try_deadline_covers_transaction_and_lock_cleanup"
):
request.getfixturevalue("_trigger_try_clock")


@pytest_asyncio.fixture(loop_scope="function")
async def _trigger_try_clock():
async with _trigger_try_idle_clock():
yield


@pytest.mark.asyncio(loop_scope="function")
@pytest.mark.parametrize("jump", [0.0, 0.25])
async def test_trigger_try_clock_ignores_unrelated_scheduling(monkeypatch, clock, jump):
loop = asyncio.get_running_loop()
external = ManualClock(t=loop.time())
real_select = loop._selector.select
actual_start = cell_control.Owner._start
calls = []

def select(timeout=None):
events = real_select(0)
if not events and timeout is not None:
external.advance(max(0.0, timeout))
return events

def start(self, work):
task = actual_start(self, work)
external.advance(jump)
calls.append(jump)
return task

with pytest.MonkeyPatch.context() as outer:
outer.setattr(loop, "time", external.monotonic)
outer.setattr(loop._selector, "select", select)
outer.setattr(cell_control.Owner, "_start", start)
async with _trigger_try_idle_clock():
await test_a_trigger_try_deadline_covers_transaction_and_lock_cleanup(
monkeypatch, clock, "reset"
)
assert calls == [jump]


@pytest.mark.asyncio(loop_scope="function")
async def test_trigger_try_clock_still_cancels_genuinely_blocked_cleanup(monkeypatch):
cancelled = asyncio.Event()

class BlockedClose(FakeConn):
async def close(self, **kwargs):
assert kwargs["timeout"] > 0
try:
await asyncio.Future()
finally:
cancelled.set()

async def work():
await asyncio.Future()

conn = BlockedClose()
owner = _owner(conn)
async with _trigger_try_idle_clock() as clock:
_short_phases(monkeypatch)
started = clock.t
task = asyncio.create_task(owner.call(work, total_s=1.0))
done, _ = await asyncio.wait([task], timeout=0.2)
assert task in done
assert isinstance(task.exception(), TimeoutError)
await asyncio.wait_for(cancelled.wait(), timeout=0.01)
assert clock.t - started == pytest.approx(0.01)
assert "TERMINATE" in conn.executed and "CLOSE" not in conn.executed
assert owner._close_task is not None and owner._close_task.done()


@pytest.mark.asyncio(loop_scope="function")
async def test_trigger_try_clock_drains_and_rejects_an_orphan_task():
cancelled = []

async def orphan():
try:
await asyncio.Future()
finally:
cancelled.append(True)

loop = asyncio.get_running_loop()
real_loop_clock = loop.time
real_owner_clock, real_pg_clock = cell_control.time, cell_control.pg.time
real_short_phases = _short_phases
with pytest.raises(AssertionError, match="leaves no task"):
async with _trigger_try_idle_clock():
asyncio.create_task(orphan())
await asyncio.sleep(0)
assert cancelled == [True]
assert loop.time == real_loop_clock
assert cell_control.time is real_owner_clock and cell_control.pg.time is real_pg_clock
assert _short_phases is real_short_phases
138 changes: 138 additions & 0 deletions tests/unit/test_code_search_deadline.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,10 @@
from __future__ import annotations

import asyncio
import sys
import time
from contextlib import contextmanager
from types import SimpleNamespace
from typing import Any
from unittest.mock import patch

Expand Down Expand Up @@ -630,3 +633,138 @@ async def _put(*a: Any, **k: Any) -> None:
assert data["epistemic"] == "partial"
assert data["retrieval_work"]["vector_leg_failed"] == 1
assert puts == []


@contextmanager
def _first_content_clock():
from server.code.domain import deadline as code_deadline
from tests.unit.clock_fake import ManualClock

clock = ManualClock(t=1000.0)
real_asyncio = asyncio

def loop_factory():
loop = real_asyncio.new_event_loop()
selector = loop._selector
real_select = selector.select

def select(timeout=None):
events = real_select(0)
if not events and timeout is not None:
clock.advance(max(0.0, timeout))
return events

loop.time = clock.monotonic
selector.select = select
return loop

async def checked(coro):
before = real_asyncio.all_tasks()
try:
return await coro
finally:
leaked = real_asyncio.all_tasks() - before - {real_asyncio.current_task()}
for task in leaked:
task.cancel()
await real_asyncio.gather(*leaked, return_exceptions=True)
assert not leaked, "search deadline leaves no task on its private loop"

def run(coro):
with real_asyncio.Runner(loop_factory=loop_factory) as runner:
return runner.run(checked(coro))

# Fake leg sleeps and request budgets share idle time, independently of CPU scheduling.
timers = SimpleNamespace(
**{**vars(time), "monotonic": clock.monotonic, "perf_counter": clock.monotonic}
)
with (
patch.object(
sys.modules[__name__], "asyncio", SimpleNamespace(**{**vars(real_asyncio), "run": run})
),
patch.object(sys.modules[__name__], "time", timers),
patch.object(code_deadline, "time", timers),
patch.object(legs, "time", timers),
):
yield clock


@pytest.fixture(autouse=True)
def _first_content_deadline_clock(request):
if request.node.name != "test_the_deadline_waits_for_the_first_content_leg":
yield
return
with _first_content_clock():
yield


@pytest.mark.parametrize("jump", [0.0, 0.12])
def test_first_content_clock_ignores_unrelated_pre_leg_time(jump):
from server.code.domain import deadline as code_deadline

offset = 0.0
calls = []
real_monotonic = time.monotonic
actual_budget = cs_gather._leg_budget

def leg_budget(*args, **kwargs):
nonlocal offset
offset += jump
calls.append(offset)
return actual_budget(*args, **kwargs)

with (
patch.object(
code_deadline, "time", SimpleNamespace(monotonic=lambda: real_monotonic() + offset)
),
patch.object(cs_gather, "_leg_budget", leg_budget),
_first_content_clock(),
):
test_the_deadline_waits_for_the_first_content_leg()
assert calls == [jump]


def test_first_content_clock_still_drops_and_cancels_genuinely_blocked_legs():
cancelled = []

class BlockedRepo(_Repo):
async def lexical_search_chunks(self, *args, **kwargs):
try:
await asyncio.Event().wait()
finally:
cancelled.append("lexical")

async def vector_search_chunks(self, *args, **kwargs):
try:
await asyncio.Event().wait()
finally:
cancelled.append("vector")

with _first_content_clock() as clock:
_, res, wall = _run(BlockedRepo(), llm_legs="none", deadline_ms=400)
assert clock.t == pytest.approx(1000.4, abs=0.001)
assert 0.399 <= wall <= 0.401
assert res.units == [] and res.abstain and res.partial and res.epistemic == "partial"
assert set(res.legs_timed_out) == {"lexical", "vector"}
assert sorted(cancelled) == ["lexical", "vector"]


def test_first_content_clock_drains_and_rejects_an_orphan_task():
cancelled = []

async def orphan():
try:
await asyncio.Event().wait()
finally:
cancelled.append(True)

async def leak():
asyncio.create_task(orphan())
await asyncio.sleep(0)

real_asyncio, real_time = asyncio, time
real_run, real_monotonic = asyncio.run, time.monotonic
with _first_content_clock(), pytest.raises(AssertionError, match="leaves no task"):
asyncio.run(leak())
assert cancelled == [True]
assert asyncio is real_asyncio and asyncio.run is real_run
assert time is real_time and time.monotonic is real_monotonic