From 60bf01d974f51c9de5561481206b757bbfd371be Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 22:31:25 +0000 Subject: [PATCH 01/10] Record boundary context for linear cost samples Signed-off-by: zlareb1 --- tests/unit/code_fixtures.py | 32 ++++++++- tests/unit/lane_walks.py | 6 +- tests/unit/test_code_fixtures.py | 119 ++++++++++++++++++++++++++++++- 3 files changed, 152 insertions(+), 5 deletions(-) diff --git a/tests/unit/code_fixtures.py b/tests/unit/code_fixtures.py index cafbec4830..f7643b2774 100644 --- a/tests/unit/code_fixtures.py +++ b/tests/unit/code_fixtures.py @@ -54,6 +54,23 @@ def _linear_failure_context() -> dict[str, Any]: } +def linear_sample_context() -> dict[str, Any]: + """State outside a cost call; clock deltas include the observer's own work.""" + try: + return { + "process_cpu_ns": time.process_time_ns(), + "monotonic_ns": time.monotonic_ns(), + "thread_cpu_ns": time.thread_time_ns(), + "allocated_blocks": sys.getallocatedblocks(), + "gc_enabled": gc.isenabled(), + "gc_count": gc.get_count(), + "trace_active": sys.gettrace() is not None, + "profile_active": sys.getprofile() is not None, + } + except Exception as error: # noqa: BLE001 — observation must not change the cost judge + return {"context_error": type(error).__name__} + + def assert_linear( cost: Callable[[Any], float], small: Any, @@ -66,6 +83,16 @@ def assert_linear( quadratic one takes 16) or under `floor` seconds. On a loaded host the best of three runs of each counts. No absolute bound: a linear cost at `big` takes several times as long on a slow runner as on a fast one.""" + samples: list[dict[str, Any]] = [] + original = cost + + def observed(size: Any) -> float: + before = linear_sample_context() + value = original(size) + samples.append({"cost_s": value, "before": before, "after": linear_sample_context()}) + return value + + cost = observed a, b = cost(small), cost(big) if b > ratio * a: a, b = min(a, cost(small), cost(small)), min(b, cost(big), cost(big)) @@ -73,10 +100,11 @@ def assert_linear( assert b < ratio * a or b < floor, (a, b) except AssertionError as error: try: - # Diagnostic imports and allocations must follow the failed judge. + # JSON encoding and cumulative context follow the failed judge. import json - note = "linear_cost_context=" + json.dumps(_linear_failure_context(), sort_keys=True) + context = {**_linear_failure_context(), "samples": samples} + note = "linear_cost_context=" + json.dumps(context, sort_keys=True) except Exception as context_error: # Preserve the failed judge. error.add_note("linear_cost_context_error=" + type(context_error).__name__) raise error from context_error diff --git a/tests/unit/lane_walks.py b/tests/unit/lane_walks.py index c0d1b67fa9..3b85ac8053 100644 --- a/tests/unit/lane_walks.py +++ b/tests/unit/lane_walks.py @@ -10,7 +10,7 @@ import time from collections.abc import Callable -from tests.unit.code_fixtures import untraced +from tests.unit.code_fixtures import linear_sample_context, untraced def _markdown(text: str) -> object: @@ -779,6 +779,8 @@ def cost( source = (head + text(n)).encode() parse_s, parse_bytes, parse_raised = 0.0, 0, False errors = {} + # These boundaries include untraced setup and restoration, outside the timed walk. + before = linear_sample_context() if samples is not None else None with untraced(): t0 = time.process_time() # CPU time, as in `walk_cost` if samples is None: @@ -794,6 +796,8 @@ def cost( "n": n, "source_bytes": len(source), "cpu_s": took, + "before": before, + "after": linear_sample_context(), "parsed_bytes": parse_bytes, "lane": lane, "finished": finished, diff --git a/tests/unit/test_code_fixtures.py b/tests/unit/test_code_fixtures.py index a3711d8d9f..29827c4d96 100644 --- a/tests/unit/test_code_fixtures.py +++ b/tests/unit/test_code_fixtures.py @@ -100,7 +100,10 @@ def cost(size: str) -> float: assert len(notes) == 1 assert notes[0].startswith("linear_cost_context=") assert "private-small" not in notes[0] and "private-big" not in notes[0] - assert json.loads(notes[0].removeprefix("linear_cost_context=")) == context + received = json.loads(notes[0].removeprefix("linear_cost_context=")) + samples = received.pop("samples") + assert [sample["cost_s"] for sample in samples] == values + assert received == context @pytest.mark.parametrize("failure", [AssertionError("cost failed"), RuntimeError("cost failed")]) @@ -167,6 +170,118 @@ def numeric(value: object) -> bool: return all(numeric(item) for item in value.values()) if isinstance(value, list): return all(numeric(item) for item in value) - return type(value) in (int, bool) + return type(value) in (int, float, bool) assert numeric(context) + + +@pytest.mark.parametrize("values", [[1.0, 8.0], [1.0, 16.0, 2.0, 3.0, 17.0, 18.0]]) +def test_failed_cost_samples_keep_values_and_boundary_context(values: list[float]) -> None: + calls: list[str] = [] + measurements = iter(values) + + def cost(size: str) -> float: + calls.append(size) + return next(measurements) + + with pytest.raises(AssertionError) as error: + code_fixtures.assert_linear(cost, "private-small", "private-big") + context = json.loads(error.value.__notes__[0].removeprefix("linear_cost_context=")) + samples = context["samples"] + assert [sample["cost_s"] for sample in samples] == values + expected = ["private-small", "private-big"] + if len(values) == 6: + expected += ["private-small", "private-small", "private-big", "private-big"] + assert calls == expected + assert error.value.args == ((values[0], values[1]),) + assert "private-small" not in error.value.__notes__[0] + for sample in samples: + before, after = sample["before"], sample["after"] + for clock in ("process_cpu_ns", "monotonic_ns", "thread_cpu_ns"): + assert after[clock] >= before[clock] >= 0 + assert before["trace_active"] == after["trace_active"] == (sys.gettrace() is not None) + assert before["profile_active"] == after["profile_active"] == (sys.getprofile() is not None) + assert isinstance(before["gc_enabled"], bool) + assert len(before["gc_count"]) == len(after["gc_count"]) == 3 + + +def test_registry_sample_context_surrounds_original_cost_and_excludes_warmup( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from types import SimpleNamespace + + from server.code.domain.extract import bounded_parse + from tests.unit import lane_walks + + # The registry normally runs in a child; this check must restore its wrapper. + monkeypatch.setattr(bounded_parse, "parse", bounded_parse.parse) + clocks = iter([0.0, 1.0, 10.0, 11.0, 20.0, 24.0]) + monkeypatch.setattr(lane_walks, "time", SimpleNamespace(process_time=lambda: next(clocks))) + monkeypatch.setattr(lane_walks, "SHAPES", [("observed", lambda n: "a\\n" * n)]) + calls: list[int] = [] + + def extract(path: str, source: bytes, **kwargs: object) -> tuple[object, bool]: + calls.append(len(source)) + return SimpleNamespace(lang="python", tree=object()), True + + monkeypatch.setattr(lane_walks, "_extract", extract) + diagnostics: dict[str, list[dict[str, object]]] = {} + got = lane_walks.registry_cost("python", diagnostics=diagnostics) + assert got == [("observed", "N", 1.0, 4.0, 0, 0)] + assert len(calls) == 3 + samples = diagnostics["observed"] + assert [sample["cpu_s"] for sample in samples] == [1.0, 4.0] + for sample in samples: + assert sample["after"]["thread_cpu_ns"] >= sample["before"]["thread_cpu_ns"] + + +@pytest.mark.parametrize("large", [4.0, 8.0]) +def test_sample_context_error_preserves_the_original_judge( + monkeypatch: pytest.MonkeyPatch, large: float +) -> None: + def unavailable() -> int: + raise RuntimeError("private context error") + + monkeypatch.setattr(code_fixtures.time, "thread_time_ns", unavailable) + + def cost(n: int) -> float: + return 1.0 if n == 1 else large + + if large == 4.0: + code_fixtures.assert_linear(cost, 1, 4) + else: + with pytest.raises(AssertionError) as error: + code_fixtures.assert_linear(cost, 1, 4) + assert error.value.args == ((1.0, 8.0),) + context = json.loads(error.value.__notes__[0].removeprefix("linear_cost_context=")) + assert [sample["cost_s"] for sample in context["samples"]] == [1.0, 8.0] + for sample in context["samples"]: + assert sample["before"] == sample["after"] == {"context_error": "RuntimeError"} + assert "private context error" not in error.value.__notes__[0] + + +def test_sample_context_is_collected_outside_the_cost_call( + monkeypatch: pytest.MonkeyPatch, +) -> None: + inside = False + snapshots = 0 + real = code_fixtures.linear_sample_context + + def snapshot() -> dict[str, object]: + nonlocal snapshots + assert not inside + snapshots += 1 + return real() + + monkeypatch.setattr(code_fixtures, "linear_sample_context", snapshot) + + def cost(n: int) -> float: + nonlocal inside + inside = True + try: + return float(n) + finally: + inside = False + + code_fixtures.assert_linear(cost, 1, 4) + assert snapshots == 4 From 036e42492b05b6a12a017e7d4e89b6044b84ebd9 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 22:47:46 +0000 Subject: [PATCH 02/10] Record boundary context for SQL ratio samples Signed-off-by: zlareb1 --- tests/unit/test_code_fixtures.py | 90 ++++++++++++++++++++++++++ tests/unit/test_document_extraction.py | 26 +++++--- 2 files changed, 107 insertions(+), 9 deletions(-) diff --git a/tests/unit/test_code_fixtures.py b/tests/unit/test_code_fixtures.py index 29827c4d96..d404e79e6e 100644 --- a/tests/unit/test_code_fixtures.py +++ b/tests/unit/test_code_fixtures.py @@ -285,3 +285,93 @@ def cost(n: int) -> float: code_fixtures.assert_linear(cost, 1, 4) assert snapshots == 4 + + +@pytest.mark.parametrize("raises", [False, True]) +def test_document_cpu_samples_surround_the_original_timed_call( + monkeypatch: pytest.MonkeyPatch, raises: bool +) -> None: + from types import SimpleNamespace + + from tests.unit import test_document_extraction as document + + clocks = iter([3.0, 5.0]) + monkeypatch.setattr(document, "time", SimpleNamespace(process_time=lambda: next(clocks))) + snapshots: list[str] = [] + inside = False + real = code_fixtures.linear_sample_context + + def snapshot() -> dict[str, object]: + assert not inside + snapshots.append("boundary") + return real() + + monkeypatch.setattr(document, "linear_sample_context", snapshot, raising=False) + failure = RuntimeError("private cost failure") + + def cost() -> None: + nonlocal inside + inside = True + try: + assert sys.gettrace() is None + if raises: + raise failure + finally: + inside = False + + samples: list[dict[str, object]] = [] + if raises: + with pytest.raises(RuntimeError) as error: + document._cpu(cost, samples=samples) + assert error.value is failure + else: + assert document._cpu(cost, samples=samples) == 2.0 + assert snapshots == ["boundary", "boundary"] + assert len(samples) == 1 + assert samples[0]["after"]["thread_cpu_ns"] >= samples[0]["before"]["thread_cpu_ns"] + assert "private cost failure" not in json.dumps(samples) + + +def test_document_cpu_without_samples_does_not_collect_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from types import SimpleNamespace + + from tests.unit import test_document_extraction as document + + clocks = iter([3.0, 5.0]) + monkeypatch.setattr(document, "time", SimpleNamespace(process_time=lambda: next(clocks))) + + def forbidden() -> dict[str, object]: + pytest.fail("A caller without diagnostics must not collect context") + + monkeypatch.setattr(document, "linear_sample_context", forbidden, raising=False) + assert document._cpu(lambda: None) == 2.0 + + +def test_sql_failure_keeps_the_complete_sample_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from tests.unit import test_document_extraction as document + + values = iter([1.0, 25.0]) + recorded: list[dict[str, object]] = [] + + def cpu(fn: object, *, samples: list[dict[str, object]]) -> float: + context = { + "before": {"thread_cpu_ns": len(recorded)}, + "after": {"thread_cpu_ns": len(recorded) + 1}, + } + samples.append(context) + recorded.append(context) + return next(values) + + monkeypatch.setattr(document, "_cpu", cpu) + with pytest.raises(AssertionError) as error: + document.test_one_line_sql_with_a_comment_after_every_semicolon_splits_in_linear_time() + assert json.loads(str(error.value.args[0]).splitlines()[0]) == { + "small_s": 1.0, + "large_s": 25.0, + "samples": recorded, + } + assert len(recorded) == 2 diff --git a/tests/unit/test_document_extraction.py b/tests/unit/test_document_extraction.py index 879b5796a1..c403655e18 100644 --- a/tests/unit/test_document_extraction.py +++ b/tests/unit/test_document_extraction.py @@ -25,7 +25,7 @@ from loom_extractor import ExtractionError, Options, decode, detect_format, extract from loom_extractor.artifact import Artifact, Block, Coverage, Locator from server.knowledge.domain.extract import InvalidArtifact, parse_artifact, structural -from tests.unit.code_fixtures import untraced +from tests.unit.code_fixtures import linear_sample_context, untraced FIXTURES = Path(__file__).resolve().parent / "fixtures" / "knowledge" @@ -1388,14 +1388,19 @@ def test_a_heading_line_with_a_long_run_of_blanks_is_read_whole() -> None: assert (heading.type, heading.text) == ("heading", "a b") -def _cpu(fn: Any) -> float: +def _cpu(fn: Any, *, samples: list[dict[str, Any]] | None = None) -> float: """This process's CPU seconds `fn` takes, timed `untraced`, so neither a test running beside it nor the tracer counts. Every caller runs extractor code, which the unit job's coverage does not measure, so the pause loses no coverage.""" - with untraced(): - start = time.process_time() - fn() - return time.process_time() - start + before = linear_sample_context() if samples is not None else None + try: + with untraced(): + start = time.process_time() + fn() + return time.process_time() - start + finally: + if samples is not None: + samples.append({"before": before, "after": linear_sample_context()}) def _within(seconds: float, fn: Any) -> None: @@ -1583,13 +1588,16 @@ def test_one_line_sql_with_a_comment_after_every_semicolon_splits_in_linear_time sql._spans("SELECT 1;", "clickhouse") # imports sqlglot and the dialect before timing small_text = "SELECT 1; /* c */ " * 10_000 - small = _cpu(lambda: sql._spans(small_text, "clickhouse")) + samples: list[dict[str, Any]] = [] + small = _cpu(lambda: sql._spans(small_text, "clickhouse"), samples=samples) text = "SELECT 1; /* c */ " * 160_000 out: list[list[Any]] = [] - large = _cpu(lambda: out.append(sql._spans(text, "clickhouse"))) + large = _cpu(lambda: out.append(sql._spans(text, "clickhouse")), samples=samples) # A linear split costs about 17x for 16x the statements; a newline search from each `;` # not stopped at the next token scans to the end of this one-line text (about 50x). - assert large < 24 * small + 0.05, (small, large) + assert large < 24 * small + 0.05, json.dumps( + {"small_s": small, "large_s": large, "samples": samples}, sort_keys=True + ) assert len(out[0]) == 160_000 # No span cuts a comment: each holds every `*/` its `/*` opens. assert all( From 01a65909af783bd0b4ac7f2bb3bcb31165f6b877 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 23:41:03 +0000 Subject: [PATCH 03/10] Lane-walk failures retain all six sample contexts Signed-off-by: zlareb1 --- tests/unit/lane_walks.py | 19 ++- tests/unit/test_code_fixtures.py | 245 +++++++++++++++++++++++++++++ tests/unit/test_code_regex_cost.py | 10 +- 3 files changed, 269 insertions(+), 5 deletions(-) diff --git a/tests/unit/lane_walks.py b/tests/unit/lane_walks.py index 3b85ac8053..1c298155ab 100644 --- a/tests/unit/lane_walks.py +++ b/tests/unit/lane_walks.py @@ -506,17 +506,32 @@ def _inline_templates(n: int) -> str: _WALK_N = 100_000 -def walk_cost(name: str) -> tuple[float, float]: +def walk_cost( + name: str, *, diagnostics: list[dict[str, object]] | None = None +) -> tuple[float, float]: """The best of three times of the walk at N and 4N characters.""" walk, text = next((w[1], w[2]) for w in WALKS if w[0] == name) - def cost(n: int) -> float: + def timed_cost(n: int) -> float: t = text(n) with untraced(): # a full collection costs in proportion to the heap, not to the text t0 = time.process_time() # CPU time: a loaded host stretches wall time, not this walk(t) return time.process_time() - t0 + def cost(n: int) -> float: + if diagnostics is None: + return timed_cost(n) + before = linear_sample_context() + cpu_s = None + try: + cpu_s = timed_cost(n) + return cpu_s + finally: + diagnostics.append( + {"n": n, "cpu_s": cpu_s, "before": before, "after": linear_sample_context()} + ) + return min(cost(_WALK_N) for _ in range(3)), min(cost(4 * _WALK_N) for _ in range(3)) diff --git a/tests/unit/test_code_fixtures.py b/tests/unit/test_code_fixtures.py index d404e79e6e..3dfa1d5469 100644 --- a/tests/unit/test_code_fixtures.py +++ b/tests/unit/test_code_fixtures.py @@ -375,3 +375,248 @@ def cpu(fn: object, *, samples: list[dict[str, object]]) -> float: "samples": recorded, } assert len(recorded) == 2 + + +@pytest.mark.parametrize( + "values", [[3.0, 1.0, 2.0, 6.0, 4.0, 5.0], [3.0, 1.0, 2.0, 10.0, 8.0, 9.0]] +) +def test_walk_samples_preserve_six_costs_and_boundary_context( + monkeypatch: pytest.MonkeyPatch, values: list[float] +) -> None: + from contextlib import contextmanager + from types import SimpleNamespace + + from tests.unit import lane_walks + + clocks = iter( + value + for index, value in enumerate(values) + for value in (index * 100.0, index * 100.0 + value) + ) + events: list[tuple[str, object]] = [] + inside = False + + @contextmanager + def region(): + nonlocal inside + inside = True + try: + yield + finally: + inside = False + + def source(n: int) -> str: + events.append(("source", n)) + return str(n) + + def walk(text: str) -> None: + assert inside + events.append(("walk", text)) + + def snapshot() -> dict[str, object]: + assert not inside + events.append(("snapshot", len(events))) + return {"boundary": len(events)} + + monkeypatch.setattr(lane_walks, "WALKS", [("observed", walk, source)]) + monkeypatch.setattr(lane_walks, "untraced", region) + monkeypatch.setattr(lane_walks, "time", SimpleNamespace(process_time=lambda: next(clocks))) + monkeypatch.setattr(lane_walks, "linear_sample_context", snapshot) + samples: list[dict[str, object]] = [] + small, big = lane_walks.walk_cost("observed", diagnostics=samples) + assert (small, big) == (min(values[:3]), min(values[3:])) + assert (big < 0.1 or big < 8.0 * small) == (values[-2] == 4.0) + sizes = [100_000] * 3 + [400_000] * 3 + assert [sample["n"] for sample in samples] == sizes + assert [sample["cpu_s"] for sample in samples] == values + assert [value for event, value in events if event == "source"] == sizes + assert [value for event, value in events if event == "walk"] == [str(n) for n in sizes] + assert len([event for event, _ in events if event == "snapshot"]) == 12 + for sample in samples: + assert sample["after"]["boundary"] > sample["before"]["boundary"] + + +def test_walk_sample_keeps_the_original_walker_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from types import SimpleNamespace + + from tests.unit import lane_walks + + failure = RuntimeError("private walker error") + seen: list[str] = [] + + def walk(text: str) -> None: + seen.append(text) + raise failure + + monkeypatch.setattr(lane_walks, "WALKS", [("observed", walk, lambda n: "private source")]) + monkeypatch.setattr(lane_walks, "time", SimpleNamespace(process_time=lambda: 0.0)) + samples: list[dict[str, object]] = [] + with pytest.raises(RuntimeError) as error: + lane_walks.walk_cost("observed", diagnostics=samples) + assert error.value is failure + assert seen == ["private source"] + assert len(samples) == 1 + assert samples[0]["n"] == 100_000 + assert samples[0]["cpu_s"] is None + assert "private source" not in json.dumps(samples) + assert "private walker error" not in json.dumps(samples) + + +@pytest.mark.parametrize("large", [4.0, 8.0]) +def test_walk_context_error_keeps_selected_costs( + monkeypatch: pytest.MonkeyPatch, large: float +) -> None: + from types import SimpleNamespace + + from tests.unit import lane_walks + + def unavailable() -> int: + raise RuntimeError("private observer error") + + clocks = iter( + [ + 0.0, + 1.0, + 10.0, + 11.0, + 20.0, + 21.0, + 30.0, + 30.0 + large, + 40.0, + 40.0 + large, + 50.0, + 50.0 + large, + ] + ) + monkeypatch.setattr(code_fixtures.time, "thread_time_ns", unavailable) + monkeypatch.setattr(lane_walks, "WALKS", [("observed", lambda text: None, lambda n: "")]) + monkeypatch.setattr(lane_walks, "time", SimpleNamespace(process_time=lambda: next(clocks))) + samples: list[dict[str, object]] = [] + small, big = lane_walks.walk_cost("observed", diagnostics=samples) + assert (small, big) == (1.0, large) + assert (big < 0.1 or big < 8.0 * small) == (large == 4.0) + assert len(samples) == 6 + for sample in samples: + assert sample["before"] == sample["after"] == {"context_error": "RuntimeError"} + assert "private observer error" not in json.dumps(samples) + + +def test_walk_without_diagnostics_does_not_collect_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from types import SimpleNamespace + + from tests.unit import lane_walks + + def forbidden() -> dict[str, object]: + pytest.fail("A caller without diagnostics must not collect context") + + monkeypatch.setattr(lane_walks, "linear_sample_context", forbidden) + monkeypatch.setattr(lane_walks, "WALKS", [("observed", lambda text: None, lambda n: "")]) + clocks = iter(range(12)) + monkeypatch.setattr(lane_walks, "time", SimpleNamespace(process_time=lambda: next(clocks))) + assert lane_walks.walk_cost("observed") == (1, 1) + + +def test_walk_child_publishes_the_complete_sample_packet( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + import faulthandler + from types import SimpleNamespace + + from tests.unit import lane_walks + from tests.unit import test_code_regex_cost as costs + + samples = [ + {"n": n, "cpu_s": cpu, "before": {"clock": i}, "after": {"clock": i + 1}} + for i, (n, cpu) in enumerate( + zip([100_000] * 3 + [400_000] * 3, [3.0, 1.0, 2.0, 6.0, 4.0, 5.0], strict=True) + ) + ] + + def walk_cost(name: str, *, diagnostics: list[dict[str, object]]) -> tuple[float, float]: + assert name == "observed" + diagnostics.extend(samples) + return 1.0, 4.0 + + monkeypatch.setattr(lane_walks, "walk_cost", walk_cost) + monkeypatch.setattr(sys, "argv", ["child", "observed"]) + + child_faults = {name: getattr(faulthandler, name) for name in dir(faulthandler)} + child_faults["dump_traceback_later"] = lambda *a, **kw: None + monkeypatch.setitem(sys.modules, "faulthandler", SimpleNamespace(**child_faults)) + exec(costs._WALK_CHILD, {}) + assert json.loads(capsys.readouterr().out) == {"costs": [1.0, 4.0], "samples": samples} + + +@pytest.mark.parametrize("large", [4.0, 8.0]) +def test_walk_parent_keeps_judge_and_failure_packet( + monkeypatch: pytest.MonkeyPatch, large: float +) -> None: + from types import SimpleNamespace + + from tests.unit import test_code_regex_cost as costs + + samples = [ + {"n": n, "cpu_s": cpu, "before": {"clock": i}, "after": {"clock": i + 1}} + for i, (n, cpu) in enumerate( + zip( + [100_000] * 3 + [400_000] * 3, + [3.0, 1.0, 2.0, large + 2, large, large + 1], + strict=True, + ) + ) + ] + packet = {"costs": [1.0, large], "samples": samples} + + def run(argv: list[str], **kwargs: object) -> object: + assert argv[:2] == [sys.executable, "-c"] + assert argv[-1] == "observed" + assert kwargs["capture_output"] is True and kwargs["text"] is True + assert kwargs["timeout"] == 120 + return SimpleNamespace(returncode=0, stdout=json.dumps(packet), stderr="") + + monkeypatch.setattr(costs.subprocess, "run", run) + if large == 4.0: + costs.test_a_lane_walk_costs_linear_time("observed") + else: + with pytest.raises(AssertionError) as error: + costs.test_a_lane_walk_costs_linear_time("observed") + assert json.loads(str(error.value.args[0]).splitlines()[0]) == { + "name": "observed", + **packet, + } + + +@pytest.mark.parametrize("failure", ["child", "invalid", "timeout"]) +def test_walk_child_errors_remain_failures(monkeypatch: pytest.MonkeyPatch, failure: str) -> None: + from types import SimpleNamespace + + from tests.unit import test_code_regex_cost as costs + + timeout = costs.subprocess.TimeoutExpired("walk-child", 120) + + def run(*args: object, **kwargs: object) -> object: + if failure == "timeout": + raise timeout + return SimpleNamespace( + returncode=1 if failure == "child" else 0, + stdout="invalid packet", + stderr="child failed", + ) + + monkeypatch.setattr(costs.subprocess, "run", run) + expected = { + "child": AssertionError, + "invalid": json.JSONDecodeError, + "timeout": costs.subprocess.TimeoutExpired, + }[failure] + with pytest.raises(expected) as error: + costs.test_a_lane_walk_costs_linear_time("observed") + if failure == "timeout": + assert error.value is timeout + elif failure == "child": + assert str(error.value.args[0]).startswith("child failed") diff --git a/tests/unit/test_code_regex_cost.py b/tests/unit/test_code_regex_cost.py index da46a0698b..800bc9e248 100644 --- a/tests/unit/test_code_regex_cost.py +++ b/tests/unit/test_code_regex_cost.py @@ -293,7 +293,8 @@ def cost(n: int, pre: str = pre, unit: str = unit, end: str = end) -> float: import faulthandler, json, sys faulthandler.dump_traceback_later({_WALK_DEADLINE_S}, exit=True) from tests.unit.lane_walks import walk_cost -print(json.dumps(walk_cost(sys.argv[1]))) +samples = [] +print(json.dumps({{"costs": walk_cost(sys.argv[1], diagnostics=samples), "samples": samples}})) """ @@ -307,8 +308,11 @@ def test_a_lane_walk_costs_linear_time(name: str) -> None: cwd=pathlib.Path(__file__).resolve().parents[2], ) assert done.returncode == 0, done.stderr[-4000:] - small, big = json.loads(done.stdout) - assert big < _FLOOR_S or big < RATIO * small, (name, small, big) + packet = json.loads(done.stdout) + small, big = packet["costs"] + assert big < _FLOOR_S or big < RATIO * small, json.dumps( + {"name": name, **packet}, sort_keys=True + ) # The class guard: every lane parser in the ingest's registry, over generated shapes (runs From d2fd00f921dba812cd694a0cc661627040946531 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 23:36:42 +0000 Subject: [PATCH 04/10] Hook contract cleanup waits for detached jobs Keep each test sandbox until its detached flusher removes its final job marker. Preserve hook deadlines and verify unfinished jobs refuse cleanup. Signed-off-by: zlareb1 (cherry picked from commit 2e637644d2abe404e48ac541967164b85a735036) Signed-off-by: zlareb1 --- .../tests/hook-contract.test.mjs | 13 +- .../claude-code-plugin/tests/hook-sandbox.mjs | 12 ++ .../tests/hook-sandbox.test.mjs | 176 ++++++++++++++++++ 3 files changed, 195 insertions(+), 6 deletions(-) create mode 100644 integrations/claude-code-plugin/tests/hook-sandbox.mjs create mode 100644 integrations/claude-code-plugin/tests/hook-sandbox.test.mjs diff --git a/integrations/claude-code-plugin/tests/hook-contract.test.mjs b/integrations/claude-code-plugin/tests/hook-contract.test.mjs index 7150c7da1b..bb3a7e4b6c 100644 --- a/integrations/claude-code-plugin/tests/hook-contract.test.mjs +++ b/integrations/claude-code-plugin/tests/hook-contract.test.mjs @@ -13,6 +13,7 @@ import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { hostEnv } from "./host-env.mjs"; +import { removeSandbox } from "./hook-sandbox.mjs"; const here = path.dirname(fileURLToPath(import.meta.url)); const root = path.join(here, ".."); @@ -285,7 +286,7 @@ test("malformed, empty and non-object payloads: every hook exits 0 silently", as assertContract(await runAll(cases), 8000); } finally { await loom.close(); - fs.rmSync(box.dir, { recursive: true, force: true }); + await removeSandbox(box); } }); @@ -305,7 +306,7 @@ test("no token anywhere: every hook exits 0 silently and sends no request", asyn assert.deepEqual(loom.ops, [], "a hook without a token reached the server"); } finally { await loom.close(); - fs.rmSync(box.dir, { recursive: true, force: true }); + await removeSandbox(box); } }); @@ -330,7 +331,7 @@ test("server unreachable, answering 401, or hanging: every hook stays within its } finally { await unauthorized.close(); await hang.close(); - fs.rmSync(box.dir, { recursive: true, force: true }); + await removeSandbox(box); } }); @@ -348,7 +349,7 @@ test("a host that never closes stdin does not hold a hook open", async () => { assertContract(await runAll(cases), 8000); } finally { await loom.close(); - fs.rmSync(box.dir, { recursive: true, force: true }); + await removeSandbox(box); } }); @@ -366,7 +367,7 @@ test("a well-formed session against a healthy server: every hook exits 0 silentl assert.ok(loom.ops.length > 0, "no hook reached a healthy server"); } finally { await loom.close(); - fs.rmSync(box.dir, { recursive: true, force: true }); + await removeSandbox(box); } }); @@ -385,6 +386,6 @@ test("a payload over the stdin cap is dropped, not parsed: every hook exits 0 si assertContract(await runAll(cases, 4), 15000); } finally { await loom.close(); - fs.rmSync(box.dir, { recursive: true, force: true }); + await removeSandbox(box); } }); diff --git a/integrations/claude-code-plugin/tests/hook-sandbox.mjs b/integrations/claude-code-plugin/tests/hook-sandbox.mjs new file mode 100644 index 0000000000..127c0ff544 --- /dev/null +++ b/integrations/claude-code-plugin/tests/hook-sandbox.mjs @@ -0,0 +1,12 @@ +import fs from "node:fs"; +import { setTimeout as delay } from "node:timers/promises"; + +export async function removeSandbox(box, { timeoutMs = 45000 } = {}) { + const deadline = performance.now() + timeoutMs; + // The flusher removes its job only after its last cursor/recording write. + while (fs.readdirSync(box.tmp).some((name) => /^loom-flush-\d+-\d+\.json$/.test(name))) { + if (performance.now() >= deadline) throw new Error("unfinished detached jobs in hook sandbox"); + await delay(10); + } + fs.rmSync(box.dir, { recursive: true, force: true }); +} diff --git a/integrations/claude-code-plugin/tests/hook-sandbox.test.mjs b/integrations/claude-code-plugin/tests/hook-sandbox.test.mjs new file mode 100644 index 0000000000..2584c21e18 --- /dev/null +++ b/integrations/claude-code-plugin/tests/hook-sandbox.test.mjs @@ -0,0 +1,176 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { fork, spawn } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import http from "node:http"; +import path from "node:path"; +import { once } from "node:events"; +import { setImmediate as immediate } from "node:timers/promises"; +import { removeSandbox } from "./hook-sandbox.mjs"; +import { dispatchFlush } from "../lib/flush-job.mjs"; + +function sandbox() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "loom-hook-drain-")); + const tmp = path.join(dir, "tmp"); + fs.mkdirSync(tmp); + return { dir, tmp }; +} + +test("sandbox removal waits for its detached job's final write", async () => { + const box = sandbox(); + const script = path.join(box.dir, "writer.cjs"); + fs.writeFileSync( + script, + ` +const fs = require("node:fs"); +const path = require("node:path"); +const root = process.argv[2]; +const job = path.join(root, "tmp", "loom-flush-1-" + process.pid + ".json"); +fs.writeFileSync(job, "{}"); +process.send("ready"); +process.once("message", () => { + fs.writeFileSync(path.join(root, "cursor"), "done"); + fs.unlinkSync(job); + process.send("written", () => process.disconnect()); +}); +`, + ); + const child = fork(script, [box.dir], { env: {}, stdio: ["ignore", "ignore", "pipe", "ipc"], detached: true }); + const closed = once(child, "close"); + let removing; + let released = false; + let stderr = ""; + child.stderr.on("data", (part) => { + stderr += part; + }); + try { + const [ready] = await once(child, "message"); + assert.equal(ready, "ready"); + let removed = false; + removing = removeSandbox(box).then(() => { + removed = true; + }); + await immediate(); + assert.equal(removed, false, "the owned writer still has a pending job"); + assert.equal(fs.existsSync(box.dir), true); + const written = once(child, "message"); + released = true; + child.send("release"); + assert.equal((await written)[0], "written"); + await removing; + assert.deepEqual(await closed, [0, null], stderr); + assert.equal(fs.existsSync(box.dir), false); + } finally { + if (!released && child.connected) child.send("release"); + await closed; + if (removing) await removing; + fs.rmSync(box.dir, { recursive: true, force: true }); + } +}); + +test("an unfinished job refuses removal at the cleanup deadline", async () => { + const box = sandbox(); + const job = path.join(box.tmp, "loom-flush-2-3.json"); + fs.writeFileSync(job, "{}"); + try { + await assert.rejects(removeSandbox(box, { timeoutMs: 0 }), /unfinished detached jobs/); + assert.equal(fs.existsSync(job), true); + } finally { + fs.rmSync(box.dir, { recursive: true, force: true }); + } +}); + +test("an empty job census permits removal with other sandbox files", async () => { + const box = sandbox(); + fs.writeFileSync(path.join(box.tmp, "cursor"), "42"); + await removeSandbox(box); + assert.equal(fs.existsSync(box.dir), false); +}); + +test("sandbox removal waits for the real flusher's completion marker", async () => { + const box = sandbox(); + let arrived; + const request = new Promise((resolve) => { + arrived = resolve; + }); + let respond; + const server = http.createServer((req, res) => { + req.resume(); + req.on("end", () => { + respond = () => { + if (!res.writableEnded) { + res.writeHead(200, { "content-type": "application/json" }); + res.end("{}"); + } + }; + arrived(); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + let child; + let closed; + let removing; + const token = [ + "e30", + Buffer.from(JSON.stringify({ org: "test", namespaces: ["scope"] })).toString("base64url"), + "test", + ].join("."); + try { + assert.equal( + dispatchFlush( + { + appends: [{ org: "test", namespace: "scope", agent: "test", session_id: "test" }], + cursorPath: path.join(box.tmp, "cursor"), + advanceTo: 42, + }, + { + tmp: box.tmp, + timeoutMs: 1000, + spawnImpl: (command, args, options) => { + child = spawn(command, args, { + ...options, + env: { + HOME: box.dir, + TMPDIR: box.tmp, + LOOM_BASE_URL: `http://127.0.0.1:${server.address().port}`, + LOOM_API_KEY: token, + LOOM_HOOK_TIMEOUT_MS: "1000", + LOOM_AUTO_SIGNUP: "off", + }, + }); + closed = once(child, "close"); + return child; + }, + }, + ), + true, + ); + await Promise.race([ + request, + closed.then(() => { + throw new Error("flusher exited before its request"); + }), + ]); + let removed = false; + removing = removeSandbox(box).then(() => { + removed = true; + }); + await immediate(); + assert.equal(removed, false); + assert.equal(fs.existsSync(box.dir), true); + respond(); + assert.deepEqual(await closed, [0, null]); + await removing; + assert.equal(fs.existsSync(box.dir), false); + } finally { + respond?.(); + try { + if (closed) await closed; + if (removing) await removing; + } finally { + await new Promise((resolve) => server.close(resolve)); + fs.rmSync(box.dir, { recursive: true, force: true }); + } + } +}); From 01e42b08876c5fd0b13b98365c27e6d7975b686a Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 23:41:55 +0000 Subject: [PATCH 05/10] Hook sandbox regressions use isolated homes on every platform Signed-off-by: zlareb1 (cherry picked from commit 334a924acbf4e7b561a37d2bdcc7ca632f36a373) Signed-off-by: zlareb1 --- .../claude-code-plugin/tests/hook-sandbox.test.mjs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/integrations/claude-code-plugin/tests/hook-sandbox.test.mjs b/integrations/claude-code-plugin/tests/hook-sandbox.test.mjs index 2584c21e18..d9be5727eb 100644 --- a/integrations/claude-code-plugin/tests/hook-sandbox.test.mjs +++ b/integrations/claude-code-plugin/tests/hook-sandbox.test.mjs @@ -1,3 +1,4 @@ +import "./temp-home.mjs"; import { test } from "node:test"; import assert from "node:assert/strict"; import { fork, spawn } from "node:child_process"; @@ -8,6 +9,7 @@ import path from "node:path"; import { once } from "node:events"; import { setImmediate as immediate } from "node:timers/promises"; import { removeSandbox } from "./hook-sandbox.mjs"; +import { hostEnv } from "./host-env.mjs"; import { dispatchFlush } from "../lib/flush-job.mjs"; function sandbox() { @@ -36,7 +38,11 @@ process.once("message", () => { }); `, ); - const child = fork(script, [box.dir], { env: {}, stdio: ["ignore", "ignore", "pipe", "ipc"], detached: true }); + const child = fork(script, [box.dir], { + env: hostEnv({}), + stdio: ["ignore", "ignore", "pipe", "ipc"], + detached: true, + }); const closed = once(child, "close"); let removing; let released = false; @@ -130,14 +136,14 @@ test("sandbox removal waits for the real flusher's completion marker", async () spawnImpl: (command, args, options) => { child = spawn(command, args, { ...options, - env: { + env: hostEnv({ HOME: box.dir, TMPDIR: box.tmp, LOOM_BASE_URL: `http://127.0.0.1:${server.address().port}`, LOOM_API_KEY: token, LOOM_HOOK_TIMEOUT_MS: "1000", LOOM_AUTO_SIGNUP: "off", - }, + }), }); closed = once(child, "close"); return child; From 557f8eefa51b1d1d73a5b5cbf492645f975996ee Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Sat, 21 Nov 2026 04:28:40 +0000 Subject: [PATCH 06/10] Record ingestion outcomes when fixture setup fails Signed-off-by: zlareb1 --- .../integration/test_knowledge_ingest_live.py | 22 +- .../test_knowledge_retrieval_floors.py | 37 +- ...est_knowledge_retrieval_failure_context.py | 351 ++++++++++++++++++ 3 files changed, 405 insertions(+), 5 deletions(-) create mode 100644 tests/unit/test_knowledge_retrieval_failure_context.py diff --git a/tests/integration/test_knowledge_ingest_live.py b/tests/integration/test_knowledge_ingest_live.py index 2383c512d0..03af05ef8d 100644 --- a/tests/integration/test_knowledge_ingest_live.py +++ b/tests/integration/test_knowledge_ingest_live.py @@ -78,7 +78,7 @@ async def _op(c: httpx.AsyncClient, op: str, **body: Any) -> dict[str, Any]: return dict(r.json()) -async def _work(org: str, ns: str, ingestion_id: str) -> None: +async def _work(org: str, ns: str, ingestion_id: str) -> str: """Run the job in this process, on the same ClickHouse, as a worker pass that took it does. A pass takes one job of the whole queue, which other tests' jobs share, so it may not be this one.""" @@ -91,7 +91,9 @@ async def _work(org: str, ns: str, ingestion_id: str) -> None: try: job = await ingestions.get_ingestion(org, ns, ingestion_id) assert job is not None, ingestion_id - assert await ingest_run.take_and_run(job) is not None, f"job {ingestion_id} did not run" + state = await ingest_run.take_and_run(job) + assert state is not None, f"job {ingestion_id} did not run" + return state finally: await ch.disconnect() @@ -648,10 +650,22 @@ async def test_a_delete_bounded_before_a_metadata_edit_keeps_the_carried_snapsho c, "knowledge.ingest", org=org, namespace=ns, upload_id=up.json()["upload_id"], source_uri=uri, ) # fmt: skip - await _work(org, ns, job["ingestion_id"]) - (d,) = (await _op(c, "knowledge.dataset_get", org=org, namespace=ns, source_uri=uri))[ + worker_state = await _work(org, ns, job["ingestion_id"]) + datasets = (await _op(c, "knowledge.dataset_get", org=org, namespace=ns, source_uri=uri))[ "datasets" ] + dataset_count = len(datasets) + assert dataset_count == 1, { + "dataset_count": dataset_count, + "worker_state": ( + worker_state + if type(worker_state) is str + and worker_state + in {"pending", "processing", "ready", "partial", "failed", "cancelled"} + else "unknown" + ), + } + (d,) = datasets ch = get_client() await ch.connect() try: diff --git a/tests/integration/test_knowledge_retrieval_floors.py b/tests/integration/test_knowledge_retrieval_floors.py index 10bd835305..3ff0b402d8 100644 --- a/tests/integration/test_knowledge_retrieval_floors.py +++ b/tests/integration/test_knowledge_retrieval_floors.py @@ -235,6 +235,37 @@ async def _op(c: httpx.AsyncClient, op: str, **body: Any) -> dict[str, Any]: return dict(r.json()) +async def _ingest_failure_context(org: str, ns: str, ingestion_id: str) -> dict[str, Any]: + """Read only fixed status fields; a retry message can contain source content.""" + from server.knowledge.store import ingestions + + try: + async with asyncio.timeout(1): + row = await ingestions.get_ingestion(org, ns, ingestion_id) + except Exception: # noqa: BLE001 — diagnostic reads must not replace the failed assertion + return {"observation": "unavailable"} + if row is None: + return {"observation": "missing"} + choices = { + "state": {"pending", "processing", "ready", "partial", "failed", "cancelled"}, + "stage": {"", "parse", "embed", "publish"}, + "error_code": {"", "retrying", "retries_exhausted", "revision_conflict"}, + } + out: dict[str, Any] = {} + for key, allowed in choices.items(): + value = row.get(key) + out[key] = ( + value + if type(value) is str and value in allowed + else "other" + if key == "error_code" + else "unknown" + ) + attempts = row.get("attempts") + out["attempts"] = attempts if type(attempts) is int and 0 <= attempts <= 1_000_000 else None + return out + + async def _ingest( c: httpx.AsyncClient, org: str, ns: str, name: str, data: bytes, tags: list[str] ) -> None: @@ -268,7 +299,11 @@ async def _ingest( try: row = await ingestions.get_ingestion(org, ns, job["ingestion_id"]) assert row is not None - assert await ingest_run.take_and_run(row) == "ready", name + state = await ingest_run.take_and_run(row) + assert state == "ready", ( + name, + await _ingest_failure_context(org, ns, job["ingestion_id"]), + ) finally: await ch.disconnect() diff --git a/tests/unit/test_knowledge_retrieval_failure_context.py b/tests/unit/test_knowledge_retrieval_failure_context.py new file mode 100644 index 0000000000..f34391422d --- /dev/null +++ b/tests/unit/test_knowledge_retrieval_failure_context.py @@ -0,0 +1,351 @@ +"""Fixture failures expose ingestion outcomes without printing stored content.""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import httpx +import pytest + +from server.core import ch +from server.knowledge.app import ingest_run +from server.knowledge.store import ingestions +from tests.integration import test_knowledge_ingest_live as ingest_native +from tests.integration import test_knowledge_retrieval_floors as native + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("observed", "expected"), + [ + ( + {"state": "pending", "stage": "embed", "error_code": "retrying", "attempts": 1}, + {"state": "pending", "stage": "embed", "error_code": "retrying", "attempts": 1}, + ), + ( + {"state": "pending", "stage": "publish", "error_code": "", "attempts": 0}, + {"state": "pending", "stage": "publish", "error_code": "", "attempts": 0}, + ), + (None, {"observation": "missing"}), + ( + {"state": "secret", "stage": ["secret"], "error_code": "secret", "attempts": True}, + {"state": "unknown", "stage": "unknown", "error_code": "other", "attempts": None}, + ), + ( + {"state": "failed", "stage": "", "error_code": "retries_exhausted", "attempts": -1}, + {"state": "failed", "stage": "", "error_code": "retries_exhausted", "attempts": None}, + ), + (RuntimeError("secret"), {"observation": "unavailable"}), + ], +) +async def test_failed_ingestion_reads_bounded_status_once( + monkeypatch: pytest.MonkeyPatch, observed: Any, expected: dict[str, Any] +) -> None: + calls = await _install(monkeypatch, "pending", observed) + with pytest.raises(AssertionError) as caught: + await native._ingest(None, "fixture-org", "fixture-ns", "regions.csv", b"csv", []) + assert str(caught.value).split("\n", 1)[0] == repr(("regions.csv", expected)) + assert "secret" not in str(caught.value) + assert calls == ["connect", "read", "run", "read", "disconnect"] + + +@pytest.mark.asyncio +async def test_ready_ingestion_does_not_read_diagnostic_status( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls = await _install(monkeypatch, "ready", RuntimeError("must not read")) + await native._ingest(None, "fixture-org", "fixture-ns", "regions.csv", b"csv", []) + assert calls == ["connect", "read", "run", "disconnect"] + + +async def _install(monkeypatch: pytest.MonkeyPatch, result: str, observed: Any) -> list[str]: + calls: list[str] = [] + + class Client: + async def connect(self) -> None: + calls.append("connect") + + async def disconnect(self) -> None: + calls.append("disconnect") + + async def post(*_: Any, **__: Any) -> httpx.Response: + return httpx.Response(201, json={"upload_id": "fixture-upload"}) + + async def op(*_: Any, **__: Any) -> dict[str, str]: + return {"ingestion_id": "fixture-ingestion"} + + async def get(org: str, namespace: str, ingestion_id: str) -> Any: + assert (org, namespace, ingestion_id) == ("fixture-org", "fixture-ns", "fixture-ingestion") + calls.append("read") + if calls.count("read") == 1: + return {"ingestion_id": "fixture-ingestion"} + if isinstance(observed, Exception): + raise observed + if isinstance(observed, asyncio.Event): + try: + await observed.wait() + finally: + calls.append("observation-cancelled") + if observed is None: + return None + return {**observed, "error_message": "secret", "content": "secret", "metadata": "secret"} + + async def run(row: dict[str, str]) -> str: + assert row == {"ingestion_id": "fixture-ingestion"} + calls.append("run") + return result + + monkeypatch.setattr(ch, "get_client", Client) + monkeypatch.setattr(native, "_post", post) + monkeypatch.setattr(native, "_op", op) + monkeypatch.setattr(ingestions, "get_ingestion", get) + monkeypatch.setattr(ingest_run, "take_and_run", run) + return calls + + +@pytest.mark.asyncio +async def test_blocked_status_read_does_not_hide_the_ingestion_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls = await _install(monkeypatch, "pending", asyncio.Event()) + with pytest.raises(AssertionError) as caught: + await asyncio.wait_for( + native._ingest(None, "fixture-org", "fixture-ns", "regions.csv", b"csv", []), + timeout=2, + ) + assert "unavailable" in str(caught.value) + assert calls == ["connect", "read", "run", "read", "observation-cancelled", "disconnect"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("worker_state", "count", "expected"), + [ + ("pending", 0, "pending"), + ("processing", 0, "processing"), + ("ready", 0, "ready"), + ("partial", 0, "partial"), + ("failed", 0, "failed"), + ("cancelled", 0, "cancelled"), + ("ready", 2, "ready"), + ("future-state", 0, "unknown"), + ("secret", 0, "unknown"), + ], +) +async def test_dataset_failure_reports_worker_outcome_once( + monkeypatch: pytest.MonkeyPatch, worker_state: str, count: int, expected: str +) -> None: + calls = _install_dataset(monkeypatch, worker_state, count) + with pytest.raises(AssertionError) as caught: + await ingest_native.test_a_delete_bounded_before_a_metadata_edit_keeps_the_carried_snapshots_rows( + "http://fixture.test" + ) + assert str(caught.value).split("\n", 1)[0] == repr( + {"dataset_count": count, "worker_state": expected} + ) + assert "secret" not in str(caught.value) + assert calls == [ + "formats", + "http-enter", + "upload", + "ingest", + "work", + "dataset_get", + "http-exit", + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("worker_state", ["ready", "pending"]) +async def test_one_dataset_keeps_the_metadata_and_delete_checks_without_status_reads( + monkeypatch: pytest.MonkeyPatch, worker_state: str +) -> None: + calls = _install_dataset(monkeypatch, worker_state, 1) + await ( + ingest_native.test_a_delete_bounded_before_a_metadata_edit_keeps_the_carried_snapshots_rows( + "http://fixture.test" + ) + ) + assert calls == [ + "formats", + "http-enter", + "upload", + "ingest", + "work", + "dataset_get", + "connect", + "live_version", + "update_metadata", + "clock", + "forget_published", + "disconnect", + "catalog", + "deletes", + "catalog", + "deletes", + "dataset_get_sample", + "http-exit", + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field", ["row_count", "rows"]) +async def test_one_dataset_still_checks_the_carried_snapshot_rows( + monkeypatch: pytest.MonkeyPatch, field: str +) -> None: + calls = _install_dataset(monkeypatch, "ready", 1, corrupt=field) + with pytest.raises(AssertionError): + await ingest_native.test_a_delete_bounded_before_a_metadata_edit_keeps_the_carried_snapshots_rows( + "http://fixture.test" + ) + assert calls[-2:] == ["dataset_get_sample", "http-exit"] + assert "read" not in calls + + +def _install_dataset( + monkeypatch: pytest.MonkeyPatch, worker_state: str, count: int, *, corrupt: str = "" +) -> list[str]: + from server.knowledge.app import extractors + from server.knowledge.store import datasets + + calls: list[str] = [] + org, ns, uri = "secret-org", "docs", "repo://edit.csv" + initial = {"snapshot_id": "secret-snapshot", "row_count": 2, "content": "secret"} + + class Client: + async def connect(self) -> None: + calls.append("connect") + + async def disconnect(self) -> None: + calls.append("disconnect") + + async def query_one(self, sql: str) -> dict[str, str]: + assert sql == "SELECT now() AS t" + calls.append("clock") + return {"t": "fixture-clock"} + + class HTTPClient: + def __init__(self, *, base_url: str, timeout: int) -> None: + assert (base_url, timeout) == ("http://fixture.test", 60) + + async def __aenter__(self) -> HTTPClient: + calls.append("http-enter") + return self + + async def __aexit__(self, *_: Any) -> None: + calls.append("http-exit") + + async def formats() -> set[str]: + calls.append("formats") + return {"csv"} + + async def upload(client: Any, *args: Any) -> httpx.Response: + assert isinstance(client, HTTPClient) + assert args == (org, ns, b"k,v\na,1\nb,2\n", "edit.csv") + calls.append("upload") + return httpx.Response(201, json={"upload_id": "secret-upload"}) + + async def op(client: Any, operation: str, **body: Any) -> dict[str, Any]: + assert isinstance(client, HTTPClient) + assert (body["org"], body["namespace"], body["source_uri"]) == (org, ns, uri) + if operation == "knowledge.ingest": + assert body["upload_id"] == "secret-upload" + calls.append("ingest") + return {"ingestion_id": "secret-ingestion"} + if operation == "knowledge.dataset_get": + if "sample_rows" not in body: + calls.append("dataset_get") + return {"datasets": [dict(initial) for _ in range(count)]} + assert body["sample_rows"] == 5 + calls.append("dataset_get_sample") + served = {**initial, "rows": [{"k": "a"}, {"k": "b"}]} + if corrupt: + served[corrupt] = 3 if corrupt == "row_count" else [{"k": "wrong"}] + return {"datasets": [served]} + assert operation == "knowledge.update_metadata" + assert body["title"] == "Edited" + calls.append("update_metadata") + return {} + + async def work(*args: Any) -> str: + assert args == (org, ns, "secret-ingestion") + calls.append("work") + return worker_state + + async def read(*_: Any, **__: Any) -> None: + calls.append("read") + raise AssertionError("no diagnostic status lookup") + + async def live_version(client: Any, *args: Any) -> int: + assert isinstance(client, Client) and args == (org, ns, uri) + calls.append("live_version") + return 7 + + async def forget(*args: Any, **kwargs: Any) -> None: + assert args == (org, ns, {uri: 7}) and isinstance(kwargs["ch"], Client) + calls.append("forget_published") + + async def catalog(*args: Any) -> bool: + assert args == (org, ns, uri, 7) + calls.append("catalog") + return False + + async def deletes(since: str) -> bool: + assert since == "fixture-clock" + calls.append("deletes") + return False + + monkeypatch.setattr(ch, "get_client", Client) + monkeypatch.setattr(extractors, "available_formats", formats) + monkeypatch.setattr(ingest_native.httpx, "AsyncClient", HTTPClient) + monkeypatch.setattr(ingest_native, "unique_org", lambda _: org) + monkeypatch.setattr(ingest_native, "_upload", upload) + monkeypatch.setattr(ingest_native, "_op", op) + monkeypatch.setattr(ingest_native, "_work", work) + monkeypatch.setattr(ingest_native, "_live_version", live_version) + monkeypatch.setattr(ingest_native, "_catalog_at", catalog) + monkeypatch.setattr(ingest_native, "_deletes_running", deletes) + monkeypatch.setattr(datasets, "forget_published", forget) + monkeypatch.setattr(ingestions, "get_ingestion", read) + return calls + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "outcome", + ["ready", "pending", "processing", "partial", "failed", "cancelled", "future-state", None], +) +async def test_worker_helper_returns_its_single_attempt_outcome( + monkeypatch: pytest.MonkeyPatch, outcome: str | None +) -> None: + calls: list[str] = [] + + class Client: + async def connect(self) -> None: + calls.append("connect") + + async def disconnect(self) -> None: + calls.append("disconnect") + + async def read(*args: Any) -> dict[str, str]: + assert args == ("fixture-org", "fixture-ns", "fixture-ingestion") + calls.append("read") + return {"ingestion_id": "fixture-ingestion"} + + async def run(row: dict[str, str]) -> str | None: + assert row == {"ingestion_id": "fixture-ingestion"} + calls.append("run") + return outcome + + monkeypatch.setattr(ch, "get_client", Client) + monkeypatch.setattr(ingestions, "get_ingestion", read) + monkeypatch.setattr(ingest_run, "take_and_run", run) + if outcome is None: + with pytest.raises(AssertionError, match="job fixture-ingestion did not run"): + await ingest_native._work("fixture-org", "fixture-ns", "fixture-ingestion") + else: + assert ( + await ingest_native._work("fixture-org", "fixture-ns", "fixture-ingestion") == outcome + ) + assert calls == ["connect", "read", "run", "disconnect"] From e4838a653ffee8d7aaa612a2d95785843db348cb Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Sat, 21 Nov 2026 04:34:14 +0000 Subject: [PATCH 07/10] Report worker outcomes when workbook dataset counts differ Signed-off-by: zlareb1 --- tests/integration/test_knowledge_ingest_live.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_knowledge_ingest_live.py b/tests/integration/test_knowledge_ingest_live.py index 03af05ef8d..feb0bea5cd 100644 --- a/tests/integration/test_knowledge_ingest_live.py +++ b/tests/integration/test_knowledge_ingest_live.py @@ -724,9 +724,19 @@ async def test_a_workbooks_sheets_are_datasets_each_sampled_past_its_rejected_ro c, "knowledge.ingest", org=org, namespace=ns, upload_id=up.json()["upload_id"], source_uri=uri, ) # fmt: skip - await _work(org, ns, job["ingestion_id"]) + worker_state = await _work(org, ns, job["ingestion_id"]) got = await _op(c, "knowledge.dataset_get", org=org, namespace=ns, source_uri=uri, sample_rows=2) # fmt: skip + dataset_count = len(got["datasets"]) + assert dataset_count == 2, { + "dataset_count": dataset_count, + "worker_state": ( + worker_state + if type(worker_state) is str + and worker_state in {"pending", "processing", "ready", "partial", "failed", "cancelled"} + else "unknown" + ), + } one, two = got["datasets"] assert (one["sheet"], one["row_count"], one["rejected_rows"]) == ("One", 3, 1) assert one["rows"] == [{"k": "a", "v": 1}, {"k": "c", "v": 3}] From 3ddc2924d9977490127aa1ea4faf0fefc36fcd3a Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Sat, 21 Nov 2026 04:41:17 +0000 Subject: [PATCH 08/10] Fixture failures report workbook dataset counts Signed-off-by: zlareb1 --- .../integration/test_knowledge_ingest_live.py | 34 +- .../test_knowledge_retrieval_floors.py | 37 +- ...est_knowledge_retrieval_failure_context.py | 351 ++++++++++++++++++ 3 files changed, 416 insertions(+), 6 deletions(-) create mode 100644 tests/unit/test_knowledge_retrieval_failure_context.py diff --git a/tests/integration/test_knowledge_ingest_live.py b/tests/integration/test_knowledge_ingest_live.py index 2383c512d0..feb0bea5cd 100644 --- a/tests/integration/test_knowledge_ingest_live.py +++ b/tests/integration/test_knowledge_ingest_live.py @@ -78,7 +78,7 @@ async def _op(c: httpx.AsyncClient, op: str, **body: Any) -> dict[str, Any]: return dict(r.json()) -async def _work(org: str, ns: str, ingestion_id: str) -> None: +async def _work(org: str, ns: str, ingestion_id: str) -> str: """Run the job in this process, on the same ClickHouse, as a worker pass that took it does. A pass takes one job of the whole queue, which other tests' jobs share, so it may not be this one.""" @@ -91,7 +91,9 @@ async def _work(org: str, ns: str, ingestion_id: str) -> None: try: job = await ingestions.get_ingestion(org, ns, ingestion_id) assert job is not None, ingestion_id - assert await ingest_run.take_and_run(job) is not None, f"job {ingestion_id} did not run" + state = await ingest_run.take_and_run(job) + assert state is not None, f"job {ingestion_id} did not run" + return state finally: await ch.disconnect() @@ -648,10 +650,22 @@ async def test_a_delete_bounded_before_a_metadata_edit_keeps_the_carried_snapsho c, "knowledge.ingest", org=org, namespace=ns, upload_id=up.json()["upload_id"], source_uri=uri, ) # fmt: skip - await _work(org, ns, job["ingestion_id"]) - (d,) = (await _op(c, "knowledge.dataset_get", org=org, namespace=ns, source_uri=uri))[ + worker_state = await _work(org, ns, job["ingestion_id"]) + datasets = (await _op(c, "knowledge.dataset_get", org=org, namespace=ns, source_uri=uri))[ "datasets" ] + dataset_count = len(datasets) + assert dataset_count == 1, { + "dataset_count": dataset_count, + "worker_state": ( + worker_state + if type(worker_state) is str + and worker_state + in {"pending", "processing", "ready", "partial", "failed", "cancelled"} + else "unknown" + ), + } + (d,) = datasets ch = get_client() await ch.connect() try: @@ -710,9 +724,19 @@ async def test_a_workbooks_sheets_are_datasets_each_sampled_past_its_rejected_ro c, "knowledge.ingest", org=org, namespace=ns, upload_id=up.json()["upload_id"], source_uri=uri, ) # fmt: skip - await _work(org, ns, job["ingestion_id"]) + worker_state = await _work(org, ns, job["ingestion_id"]) got = await _op(c, "knowledge.dataset_get", org=org, namespace=ns, source_uri=uri, sample_rows=2) # fmt: skip + dataset_count = len(got["datasets"]) + assert dataset_count == 2, { + "dataset_count": dataset_count, + "worker_state": ( + worker_state + if type(worker_state) is str + and worker_state in {"pending", "processing", "ready", "partial", "failed", "cancelled"} + else "unknown" + ), + } one, two = got["datasets"] assert (one["sheet"], one["row_count"], one["rejected_rows"]) == ("One", 3, 1) assert one["rows"] == [{"k": "a", "v": 1}, {"k": "c", "v": 3}] diff --git a/tests/integration/test_knowledge_retrieval_floors.py b/tests/integration/test_knowledge_retrieval_floors.py index 10bd835305..3ff0b402d8 100644 --- a/tests/integration/test_knowledge_retrieval_floors.py +++ b/tests/integration/test_knowledge_retrieval_floors.py @@ -235,6 +235,37 @@ async def _op(c: httpx.AsyncClient, op: str, **body: Any) -> dict[str, Any]: return dict(r.json()) +async def _ingest_failure_context(org: str, ns: str, ingestion_id: str) -> dict[str, Any]: + """Read only fixed status fields; a retry message can contain source content.""" + from server.knowledge.store import ingestions + + try: + async with asyncio.timeout(1): + row = await ingestions.get_ingestion(org, ns, ingestion_id) + except Exception: # noqa: BLE001 — diagnostic reads must not replace the failed assertion + return {"observation": "unavailable"} + if row is None: + return {"observation": "missing"} + choices = { + "state": {"pending", "processing", "ready", "partial", "failed", "cancelled"}, + "stage": {"", "parse", "embed", "publish"}, + "error_code": {"", "retrying", "retries_exhausted", "revision_conflict"}, + } + out: dict[str, Any] = {} + for key, allowed in choices.items(): + value = row.get(key) + out[key] = ( + value + if type(value) is str and value in allowed + else "other" + if key == "error_code" + else "unknown" + ) + attempts = row.get("attempts") + out["attempts"] = attempts if type(attempts) is int and 0 <= attempts <= 1_000_000 else None + return out + + async def _ingest( c: httpx.AsyncClient, org: str, ns: str, name: str, data: bytes, tags: list[str] ) -> None: @@ -268,7 +299,11 @@ async def _ingest( try: row = await ingestions.get_ingestion(org, ns, job["ingestion_id"]) assert row is not None - assert await ingest_run.take_and_run(row) == "ready", name + state = await ingest_run.take_and_run(row) + assert state == "ready", ( + name, + await _ingest_failure_context(org, ns, job["ingestion_id"]), + ) finally: await ch.disconnect() diff --git a/tests/unit/test_knowledge_retrieval_failure_context.py b/tests/unit/test_knowledge_retrieval_failure_context.py new file mode 100644 index 0000000000..f34391422d --- /dev/null +++ b/tests/unit/test_knowledge_retrieval_failure_context.py @@ -0,0 +1,351 @@ +"""Fixture failures expose ingestion outcomes without printing stored content.""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import httpx +import pytest + +from server.core import ch +from server.knowledge.app import ingest_run +from server.knowledge.store import ingestions +from tests.integration import test_knowledge_ingest_live as ingest_native +from tests.integration import test_knowledge_retrieval_floors as native + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("observed", "expected"), + [ + ( + {"state": "pending", "stage": "embed", "error_code": "retrying", "attempts": 1}, + {"state": "pending", "stage": "embed", "error_code": "retrying", "attempts": 1}, + ), + ( + {"state": "pending", "stage": "publish", "error_code": "", "attempts": 0}, + {"state": "pending", "stage": "publish", "error_code": "", "attempts": 0}, + ), + (None, {"observation": "missing"}), + ( + {"state": "secret", "stage": ["secret"], "error_code": "secret", "attempts": True}, + {"state": "unknown", "stage": "unknown", "error_code": "other", "attempts": None}, + ), + ( + {"state": "failed", "stage": "", "error_code": "retries_exhausted", "attempts": -1}, + {"state": "failed", "stage": "", "error_code": "retries_exhausted", "attempts": None}, + ), + (RuntimeError("secret"), {"observation": "unavailable"}), + ], +) +async def test_failed_ingestion_reads_bounded_status_once( + monkeypatch: pytest.MonkeyPatch, observed: Any, expected: dict[str, Any] +) -> None: + calls = await _install(monkeypatch, "pending", observed) + with pytest.raises(AssertionError) as caught: + await native._ingest(None, "fixture-org", "fixture-ns", "regions.csv", b"csv", []) + assert str(caught.value).split("\n", 1)[0] == repr(("regions.csv", expected)) + assert "secret" not in str(caught.value) + assert calls == ["connect", "read", "run", "read", "disconnect"] + + +@pytest.mark.asyncio +async def test_ready_ingestion_does_not_read_diagnostic_status( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls = await _install(monkeypatch, "ready", RuntimeError("must not read")) + await native._ingest(None, "fixture-org", "fixture-ns", "regions.csv", b"csv", []) + assert calls == ["connect", "read", "run", "disconnect"] + + +async def _install(monkeypatch: pytest.MonkeyPatch, result: str, observed: Any) -> list[str]: + calls: list[str] = [] + + class Client: + async def connect(self) -> None: + calls.append("connect") + + async def disconnect(self) -> None: + calls.append("disconnect") + + async def post(*_: Any, **__: Any) -> httpx.Response: + return httpx.Response(201, json={"upload_id": "fixture-upload"}) + + async def op(*_: Any, **__: Any) -> dict[str, str]: + return {"ingestion_id": "fixture-ingestion"} + + async def get(org: str, namespace: str, ingestion_id: str) -> Any: + assert (org, namespace, ingestion_id) == ("fixture-org", "fixture-ns", "fixture-ingestion") + calls.append("read") + if calls.count("read") == 1: + return {"ingestion_id": "fixture-ingestion"} + if isinstance(observed, Exception): + raise observed + if isinstance(observed, asyncio.Event): + try: + await observed.wait() + finally: + calls.append("observation-cancelled") + if observed is None: + return None + return {**observed, "error_message": "secret", "content": "secret", "metadata": "secret"} + + async def run(row: dict[str, str]) -> str: + assert row == {"ingestion_id": "fixture-ingestion"} + calls.append("run") + return result + + monkeypatch.setattr(ch, "get_client", Client) + monkeypatch.setattr(native, "_post", post) + monkeypatch.setattr(native, "_op", op) + monkeypatch.setattr(ingestions, "get_ingestion", get) + monkeypatch.setattr(ingest_run, "take_and_run", run) + return calls + + +@pytest.mark.asyncio +async def test_blocked_status_read_does_not_hide_the_ingestion_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls = await _install(monkeypatch, "pending", asyncio.Event()) + with pytest.raises(AssertionError) as caught: + await asyncio.wait_for( + native._ingest(None, "fixture-org", "fixture-ns", "regions.csv", b"csv", []), + timeout=2, + ) + assert "unavailable" in str(caught.value) + assert calls == ["connect", "read", "run", "read", "observation-cancelled", "disconnect"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("worker_state", "count", "expected"), + [ + ("pending", 0, "pending"), + ("processing", 0, "processing"), + ("ready", 0, "ready"), + ("partial", 0, "partial"), + ("failed", 0, "failed"), + ("cancelled", 0, "cancelled"), + ("ready", 2, "ready"), + ("future-state", 0, "unknown"), + ("secret", 0, "unknown"), + ], +) +async def test_dataset_failure_reports_worker_outcome_once( + monkeypatch: pytest.MonkeyPatch, worker_state: str, count: int, expected: str +) -> None: + calls = _install_dataset(monkeypatch, worker_state, count) + with pytest.raises(AssertionError) as caught: + await ingest_native.test_a_delete_bounded_before_a_metadata_edit_keeps_the_carried_snapshots_rows( + "http://fixture.test" + ) + assert str(caught.value).split("\n", 1)[0] == repr( + {"dataset_count": count, "worker_state": expected} + ) + assert "secret" not in str(caught.value) + assert calls == [ + "formats", + "http-enter", + "upload", + "ingest", + "work", + "dataset_get", + "http-exit", + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("worker_state", ["ready", "pending"]) +async def test_one_dataset_keeps_the_metadata_and_delete_checks_without_status_reads( + monkeypatch: pytest.MonkeyPatch, worker_state: str +) -> None: + calls = _install_dataset(monkeypatch, worker_state, 1) + await ( + ingest_native.test_a_delete_bounded_before_a_metadata_edit_keeps_the_carried_snapshots_rows( + "http://fixture.test" + ) + ) + assert calls == [ + "formats", + "http-enter", + "upload", + "ingest", + "work", + "dataset_get", + "connect", + "live_version", + "update_metadata", + "clock", + "forget_published", + "disconnect", + "catalog", + "deletes", + "catalog", + "deletes", + "dataset_get_sample", + "http-exit", + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field", ["row_count", "rows"]) +async def test_one_dataset_still_checks_the_carried_snapshot_rows( + monkeypatch: pytest.MonkeyPatch, field: str +) -> None: + calls = _install_dataset(monkeypatch, "ready", 1, corrupt=field) + with pytest.raises(AssertionError): + await ingest_native.test_a_delete_bounded_before_a_metadata_edit_keeps_the_carried_snapshots_rows( + "http://fixture.test" + ) + assert calls[-2:] == ["dataset_get_sample", "http-exit"] + assert "read" not in calls + + +def _install_dataset( + monkeypatch: pytest.MonkeyPatch, worker_state: str, count: int, *, corrupt: str = "" +) -> list[str]: + from server.knowledge.app import extractors + from server.knowledge.store import datasets + + calls: list[str] = [] + org, ns, uri = "secret-org", "docs", "repo://edit.csv" + initial = {"snapshot_id": "secret-snapshot", "row_count": 2, "content": "secret"} + + class Client: + async def connect(self) -> None: + calls.append("connect") + + async def disconnect(self) -> None: + calls.append("disconnect") + + async def query_one(self, sql: str) -> dict[str, str]: + assert sql == "SELECT now() AS t" + calls.append("clock") + return {"t": "fixture-clock"} + + class HTTPClient: + def __init__(self, *, base_url: str, timeout: int) -> None: + assert (base_url, timeout) == ("http://fixture.test", 60) + + async def __aenter__(self) -> HTTPClient: + calls.append("http-enter") + return self + + async def __aexit__(self, *_: Any) -> None: + calls.append("http-exit") + + async def formats() -> set[str]: + calls.append("formats") + return {"csv"} + + async def upload(client: Any, *args: Any) -> httpx.Response: + assert isinstance(client, HTTPClient) + assert args == (org, ns, b"k,v\na,1\nb,2\n", "edit.csv") + calls.append("upload") + return httpx.Response(201, json={"upload_id": "secret-upload"}) + + async def op(client: Any, operation: str, **body: Any) -> dict[str, Any]: + assert isinstance(client, HTTPClient) + assert (body["org"], body["namespace"], body["source_uri"]) == (org, ns, uri) + if operation == "knowledge.ingest": + assert body["upload_id"] == "secret-upload" + calls.append("ingest") + return {"ingestion_id": "secret-ingestion"} + if operation == "knowledge.dataset_get": + if "sample_rows" not in body: + calls.append("dataset_get") + return {"datasets": [dict(initial) for _ in range(count)]} + assert body["sample_rows"] == 5 + calls.append("dataset_get_sample") + served = {**initial, "rows": [{"k": "a"}, {"k": "b"}]} + if corrupt: + served[corrupt] = 3 if corrupt == "row_count" else [{"k": "wrong"}] + return {"datasets": [served]} + assert operation == "knowledge.update_metadata" + assert body["title"] == "Edited" + calls.append("update_metadata") + return {} + + async def work(*args: Any) -> str: + assert args == (org, ns, "secret-ingestion") + calls.append("work") + return worker_state + + async def read(*_: Any, **__: Any) -> None: + calls.append("read") + raise AssertionError("no diagnostic status lookup") + + async def live_version(client: Any, *args: Any) -> int: + assert isinstance(client, Client) and args == (org, ns, uri) + calls.append("live_version") + return 7 + + async def forget(*args: Any, **kwargs: Any) -> None: + assert args == (org, ns, {uri: 7}) and isinstance(kwargs["ch"], Client) + calls.append("forget_published") + + async def catalog(*args: Any) -> bool: + assert args == (org, ns, uri, 7) + calls.append("catalog") + return False + + async def deletes(since: str) -> bool: + assert since == "fixture-clock" + calls.append("deletes") + return False + + monkeypatch.setattr(ch, "get_client", Client) + monkeypatch.setattr(extractors, "available_formats", formats) + monkeypatch.setattr(ingest_native.httpx, "AsyncClient", HTTPClient) + monkeypatch.setattr(ingest_native, "unique_org", lambda _: org) + monkeypatch.setattr(ingest_native, "_upload", upload) + monkeypatch.setattr(ingest_native, "_op", op) + monkeypatch.setattr(ingest_native, "_work", work) + monkeypatch.setattr(ingest_native, "_live_version", live_version) + monkeypatch.setattr(ingest_native, "_catalog_at", catalog) + monkeypatch.setattr(ingest_native, "_deletes_running", deletes) + monkeypatch.setattr(datasets, "forget_published", forget) + monkeypatch.setattr(ingestions, "get_ingestion", read) + return calls + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "outcome", + ["ready", "pending", "processing", "partial", "failed", "cancelled", "future-state", None], +) +async def test_worker_helper_returns_its_single_attempt_outcome( + monkeypatch: pytest.MonkeyPatch, outcome: str | None +) -> None: + calls: list[str] = [] + + class Client: + async def connect(self) -> None: + calls.append("connect") + + async def disconnect(self) -> None: + calls.append("disconnect") + + async def read(*args: Any) -> dict[str, str]: + assert args == ("fixture-org", "fixture-ns", "fixture-ingestion") + calls.append("read") + return {"ingestion_id": "fixture-ingestion"} + + async def run(row: dict[str, str]) -> str | None: + assert row == {"ingestion_id": "fixture-ingestion"} + calls.append("run") + return outcome + + monkeypatch.setattr(ch, "get_client", Client) + monkeypatch.setattr(ingestions, "get_ingestion", read) + monkeypatch.setattr(ingest_run, "take_and_run", run) + if outcome is None: + with pytest.raises(AssertionError, match="job fixture-ingestion did not run"): + await ingest_native._work("fixture-org", "fixture-ns", "fixture-ingestion") + else: + assert ( + await ingest_native._work("fixture-org", "fixture-ns", "fixture-ingestion") == outcome + ) + assert calls == ["connect", "read", "run", "disconnect"] From 3e6a4bdaf6fd3eb9715ba7318b080a2acdd51cfe Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Sat, 21 Nov 2026 05:05:48 +0000 Subject: [PATCH 09/10] Ingestion fixture failures expose bounded retry causes Signed-off-by: zlareb1 --- .../test_knowledge_retrieval_floors.py | 99 +++- ...est_knowledge_retrieval_failure_context.py | 445 +++++++++++++++++- 2 files changed, 538 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_knowledge_retrieval_floors.py b/tests/integration/test_knowledge_retrieval_floors.py index 3ff0b402d8..439de634cb 100644 --- a/tests/integration/test_knowledge_retrieval_floors.py +++ b/tests/integration/test_knowledge_retrieval_floors.py @@ -23,6 +23,8 @@ import asyncio import hashlib import uuid +from collections.abc import Awaitable +from contextvars import ContextVar from dataclasses import dataclass, field from pathlib import Path from typing import Any @@ -235,6 +237,88 @@ async def _op(c: httpx.AsyncClient, op: str, **body: Any) -> dict[str, Any]: return dict(r.json()) +@dataclass +class _RetryAttempt: + scope: tuple[str, str, str] + task: asyncio.Task[Any] | None + details: dict[str, Any] = field(default_factory=dict) + + +_retry_attempt: ContextVar[_RetryAttempt | None] = ContextVar("ingest_retry_attempt", default=None) + + +def _retry_failure(error: Exception) -> dict[str, Any]: + from clickhouse_connect.driver.exceptions import DatabaseError, OperationalError + + from server.core.ch_errors import error_code + from server.core.errors import ( + BackendTimeoutError, + BackendUnavailableError, + EmbedTimeoutError, + OverloadedError, + ) + from server.core.pg import PostgresUnavailableError + from server.knowledge.app.extractors import ExtractorUnavailable + + labels = ( + (BackendUnavailableError, "BackendUnavailableError"), + (BackendTimeoutError, "BackendTimeoutError"), + (OverloadedError, "OverloadedError"), + (EmbedTimeoutError, "EmbedTimeoutError"), + (ExtractorUnavailable, "ExtractorUnavailable"), + (PostgresUnavailableError, "PostgresUnavailableError"), + ) + label = next((name for cls, name in labels if type(error) is cls), "other") + out: dict[str, Any] = { + "retry_exception": label, + "native_exception": "none", + "native_code": None, + } + current: BaseException | None = error + seen: set[int] = set() + try: + for depth in range(5): # the caught exception and at most four causes + if current is None or id(current) in seen: + break + seen.add(id(current)) + if type(current) in (DatabaseError, OperationalError): + out["native_exception"] = ( + "DatabaseError" if type(current) is DatabaseError else "OperationalError" + ) + attrs = vars(current) + args = current.args + code = attrs.get("code") + if code is None and args and type(args[0]) is str: + code = error_code(args[0][:4096]) + out["native_code"] = code if type(code) is int and 0 <= code <= 1_000_000 else None + break + if depth: + out["native_exception"] = "other" + if not any(type(current) is cls for cls, _ in labels): + break + current = BaseException.__getattribute__(current, "__cause__") + except Exception: # noqa: BLE001 — diagnostics must not replace the worker outcome + out["native_exception"], out["native_code"] = "unavailable", None + return out + + +@pytest.fixture +def _observe_ingest_retries(monkeypatch: pytest.MonkeyPatch) -> None: + from server.knowledge.app import ingest_run + + original = ingest_run._retry_or_fail + + def observe(job: dict[str, Any], error: Exception) -> Awaitable[str]: + attempt = _retry_attempt.get() + if attempt is not None and attempt.task is asyncio.current_task() and type(job) is dict: + scope = tuple(job.get(key) for key in ("org", "namespace", "ingestion_id")) + if all(type(value) is str for value in scope) and scope == attempt.scope: + attempt.details.update(_retry_failure(error)) + return original(job, error) + + monkeypatch.setattr(ingest_run, "_retry_or_fail", observe) + + async def _ingest_failure_context(org: str, ns: str, ingestion_id: str) -> dict[str, Any]: """Read only fixed status fields; a retry message can contain source content.""" from server.knowledge.store import ingestions @@ -299,10 +383,15 @@ async def _ingest( try: row = await ingestions.get_ingestion(org, ns, job["ingestion_id"]) assert row is not None - state = await ingest_run.take_and_run(row) - assert state == "ready", ( - name, - await _ingest_failure_context(org, ns, job["ingestion_id"]), + attempt = _RetryAttempt((org, ns, job["ingestion_id"]), asyncio.current_task()) + token = _retry_attempt.set(attempt) + try: + state = await ingest_run.take_and_run(row) + finally: + _retry_attempt.reset(token) + # pytest truncates non-string messages before all diagnostic fields are shown. + assert state == "ready", repr( + (name, await _ingest_failure_context(org, ns, job["ingestion_id"]), attempt.details) ) finally: await ch.disconnect() @@ -342,7 +431,7 @@ def _measure( } -@pytest.mark.usefixtures("reference_extractor") +@pytest.mark.usefixtures("reference_extractor", "_observe_ingest_retries") @pytest.mark.asyncio async def test_each_retrieval_slice_meets_its_floor(control_plane_url: str) -> None: from server.embedding.embedder import embedder_is_fake diff --git a/tests/unit/test_knowledge_retrieval_failure_context.py b/tests/unit/test_knowledge_retrieval_failure_context.py index f34391422d..f52ed5fccb 100644 --- a/tests/unit/test_knowledge_retrieval_failure_context.py +++ b/tests/unit/test_knowledge_retrieval_failure_context.py @@ -2,6 +2,7 @@ from __future__ import annotations +import ast import asyncio from typing import Any @@ -14,6 +15,8 @@ from tests.integration import test_knowledge_ingest_live as ingest_native from tests.integration import test_knowledge_retrieval_floors as native +_observe_ingest_retries = native._observe_ingest_retries + @pytest.mark.asyncio @pytest.mark.parametrize( @@ -45,7 +48,7 @@ async def test_failed_ingestion_reads_bounded_status_once( calls = await _install(monkeypatch, "pending", observed) with pytest.raises(AssertionError) as caught: await native._ingest(None, "fixture-org", "fixture-ns", "regions.csv", b"csv", []) - assert str(caught.value).split("\n", 1)[0] == repr(("regions.csv", expected)) + assert str(caught.value).split("\n", 1)[0] == repr(("regions.csv", expected, {})) assert "secret" not in str(caught.value) assert calls == ["connect", "read", "run", "read", "disconnect"] @@ -349,3 +352,443 @@ async def run(row: dict[str, str]) -> str | None: await ingest_native._work("fixture-org", "fixture-ns", "fixture-ingestion") == outcome ) assert calls == ["connect", "read", "run", "disconnect"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("kind", "expected", "native_type", "code"), + [ + ("overload", "OverloadedError", "DatabaseError", 241), + ("embed", "EmbedTimeoutError", "none", None), + ], +) +async def test_retry_assertion_identifies_caught_class_and_numeric_native_code( + monkeypatch: pytest.MonkeyPatch, + request: pytest.FixtureRequest, + kind: str, + expected: str, + native_type: str, + code: int | None, +) -> None: + from clickhouse_connect.driver.exceptions import DatabaseError + + from server.core.errors import EmbedTimeoutError, OverloadedError + + error = OverloadedError("private-wrapper") if kind == "overload" else EmbedTimeoutError() + if kind == "overload": + error.__cause__ = DatabaseError("Code: 241. private-SQL-URL-token") + calls, delegated, job = await _install_retry(monkeypatch, error) + request.getfixturevalue("_observe_ingest_retries") + with pytest.raises(AssertionError) as caught: + await native._ingest(None, "fixture-org", "fixture-ns", "regions.csv", b"csv", []) + observed = ast.literal_eval(str(caught.value).split("\n", 1)[0])[2] + assert observed.get("retry_exception") == expected + assert observed.get("native_exception") == native_type + assert observed.get("native_code") == code + assert "private-" not in str(caught.value) + assert len(delegated) == 1 and delegated[0][0] is job and delegated[0][1] is error + assert calls == ["connect", "read", "run", "read", "disconnect"] + + +async def _install_retry( + monkeypatch: pytest.MonkeyPatch, + error: Exception, + *, + result: str = "pending", + raised: BaseException | None = None, +) -> tuple[list[str], list[tuple[dict[str, Any], Exception]], dict[str, Any]]: + calls = await _install( + monkeypatch, + result, + {"state": "pending", "stage": "embed", "error_code": "retrying", "attempts": 1}, + ) + delegated: list[tuple[dict[str, Any], Exception]] = [] + job = {"org": "fixture-org", "namespace": "fixture-ns", "ingestion_id": "fixture-ingestion"} + + async def retry(actual_job: dict[str, Any], actual_error: Exception) -> str: + delegated.append((actual_job, actual_error)) + if raised is not None: + raise raised + return result + + async def work(row: dict[str, Any]) -> str: + assert row == {"ingestion_id": "fixture-ingestion"} + calls.append("run") + return await ingest_run._retry_or_fail(job, error) + + monkeypatch.setattr(ingest_run, "_retry_or_fail", retry) + monkeypatch.setattr(ingest_run, "take_and_run", work) + return calls, delegated, job + + +@pytest.mark.parametrize( + "kind", + [ + "BackendUnavailableError", + "BackendTimeoutError", + "OverloadedError", + "EmbedTimeoutError", + "ExtractorUnavailable", + "PostgresUnavailableError", + ], +) +def test_retry_class_labels_are_closed(kind: str) -> None: + from server.core import errors, pg + from server.knowledge.app import extractors + + cls = getattr(errors, kind, None) or getattr(pg, kind, None) or getattr(extractors, kind) + assert native._retry_failure(cls()) == { + "retry_exception": kind, + "native_exception": "none", + "native_code": None, + } + + +@pytest.mark.parametrize( + ("code", "expected"), + [ + (-1, None), + (0, 0), + (159, 159), + (1_000_000, 1_000_000), + (1_000_001, None), + (True, None), + ("private-code", None), + (None, 241), + ], +) +def test_native_code_requires_a_bounded_strict_integer(code: Any, expected: int | None) -> None: + from clickhouse_connect.driver.exceptions import DatabaseError + + from server.core.errors import OverloadedError + + error = OverloadedError("private-wrapper") + cause = DatabaseError("Code: 241. private-fallback") + cause.code = code + error.__cause__ = cause + assert native._retry_failure(error) == { + "retry_exception": "OverloadedError", + "native_exception": "DatabaseError", + "native_code": expected, + } + + +@pytest.mark.parametrize(("late", "expected"), [(False, 159), (True, None)]) +def test_native_code_parser_reads_only_a_bounded_existing_string( + late: bool, expected: int | None +) -> None: + from clickhouse_connect.driver.exceptions import OperationalError + + from server.core.errors import BackendUnavailableError + + error = BackendUnavailableError() + error.__cause__ = OperationalError( + ("private-" * 600 if late else "") + "Code: 159. private-SQL" + ) + out = native._retry_failure(error) + assert out == { + "retry_exception": "BackendUnavailableError", + "native_exception": "OperationalError", + "native_code": expected, + } + assert "private-" not in repr(out) + + +@pytest.mark.parametrize("depth", [4, 5]) +def test_native_cause_walk_stops_after_four_links(depth: int) -> None: + from clickhouse_connect.driver.exceptions import DatabaseError + + from server.core.errors import BackendUnavailableError + + error = BackendUnavailableError() + previous = error + for _ in range(depth - 1): + cause = BackendUnavailableError() + previous.__cause__ = cause + previous = cause + previous.__cause__ = DatabaseError("Code: 241. private-query") + out = native._retry_failure(error) + assert out["native_code"] == (241 if depth == 4 else None) + assert "private-" not in repr(out) + + +def test_cyclic_native_causes_and_unknown_classes_emit_no_free_text() -> None: + from server.core.errors import BackendUnavailableError, OverloadedError + + class PrivateError(Exception): + def __str__(self) -> str: + raise AssertionError("must not format exception") + + unknown = native._retry_failure(PrivateError("private-value")) + assert unknown == { + "retry_exception": "other", + "native_exception": "none", + "native_code": None, + } + error, cause = OverloadedError(), BackendUnavailableError() + error.__cause__, cause.__cause__ = cause, error + assert native._retry_failure(error) == { + "retry_exception": "OverloadedError", + "native_exception": "other", + "native_code": None, + } + + +@pytest.mark.parametrize("shape", ["argument", "subclass"]) +def test_native_diagnostic_does_not_format_unknown_objects(shape: str) -> None: + from clickhouse_connect.driver.exceptions import DatabaseError + + from server.core.errors import OverloadedError + + class PrivateValue: + def __str__(self) -> str: + raise AssertionError("must not format argument") + + def __repr__(self) -> str: + raise AssertionError("must not represent argument") + + class PrivateDatabaseError(DatabaseError): + def __init__(self) -> None: + Exception.__init__(self, "private-subclass") + + def __str__(self) -> str: + raise AssertionError("must not format subclass") + + @property + def code(self) -> int: + raise AssertionError("must not read subclass property") + + error = OverloadedError() + error.__cause__ = ( + DatabaseError(PrivateValue()) if shape == "argument" else PrivateDatabaseError() + ) + out = native._retry_failure(error) + assert out["retry_exception"] == "OverloadedError" and out["native_code"] is None + assert out["native_exception"] == ("DatabaseError" if shape == "argument" else "other") + + +@pytest.mark.asyncio +async def test_failed_numeric_observation_cannot_replace_worker_outcome( + monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest +) -> None: + from clickhouse_connect.driver.exceptions import DatabaseError + + from server.core import ch_errors + from server.core.errors import OverloadedError + + def unavailable(_: str) -> int: + raise RuntimeError("private-parser") + + error = OverloadedError() + error.__cause__ = DatabaseError("Code: 241. private-query") + calls, delegated, _ = await _install_retry(monkeypatch, error) + monkeypatch.setattr(ch_errors, "error_code", unavailable) + request.getfixturevalue("_observe_ingest_retries") + with pytest.raises(AssertionError) as caught: + await native._ingest(None, "fixture-org", "fixture-ns", "regions.csv", b"csv", []) + observed = ast.literal_eval(str(caught.value).split("\n", 1)[0])[2] + assert observed["native_exception"] == "unavailable" and observed["native_code"] is None + assert len(delegated) == 1 and calls == ["connect", "read", "run", "read", "disconnect"] + assert "private-" not in str(caught.value) and native._retry_attempt.get() is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field", ["org", "namespace", "ingestion_id"]) +async def test_retry_observer_excludes_every_other_scope( + monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest, field: str +) -> None: + from server.core.errors import OverloadedError + + calls, delegated, job = await _install_retry(monkeypatch, OverloadedError("private-error")) + job[field] = "private-other-scope" + request.getfixturevalue("_observe_ingest_retries") + with pytest.raises(AssertionError) as caught: + await native._ingest(None, "fixture-org", "fixture-ns", "regions.csv", b"csv", []) + observed = ast.literal_eval(str(caught.value).split("\n", 1)[0])[2] + assert "retry_exception" not in observed and "private-" not in str(caught.value) + assert len(delegated) == 1 and calls == ["connect", "read", "run", "read", "disconnect"] + + +@pytest.mark.asyncio +async def test_retry_observer_excludes_an_inheriting_sibling_task( + monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest +) -> None: + from server.core.errors import OverloadedError + + error = OverloadedError("private-error") + calls, delegated, job = await _install_retry(monkeypatch, error) + + async def child() -> str: + return await ingest_run._retry_or_fail(job, error) + + async def work(_: Any) -> str: + calls.append("run") + return await asyncio.create_task(child()) + + monkeypatch.setattr(ingest_run, "take_and_run", work) + request.getfixturevalue("_observe_ingest_retries") + with pytest.raises(AssertionError) as caught: + await native._ingest(None, "fixture-org", "fixture-ns", "regions.csv", b"csv", []) + observed = ast.literal_eval(str(caught.value).split("\n", 1)[0])[2] + assert "retry_exception" not in observed and native._retry_attempt.get() is None + assert len(delegated) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("same_scope", [False, True]) +async def test_overlapping_helpers_keep_observations_separate_without_repatching( + monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest, same_scope: bool +) -> None: + from clickhouse_connect.driver.exceptions import DatabaseError + + from server.core.errors import BackendUnavailableError, OverloadedError + + calls: list[str] = [] + reads: dict[Any, int] = {} + entered: list[dict[str, Any]] = [] + delegated: list[tuple[dict[str, Any], Exception]] = [] + both_entered = asyncio.Event() + errors = [OverloadedError("private-first"), BackendUnavailableError("private-second")] + for error, code in zip(errors, [241, 210], strict=True): + error.__cause__ = DatabaseError(f"Code: {code}. private-SQL") + + class Client: + async def connect(self) -> None: + calls.append("connect") + + async def disconnect(self) -> None: + assert native._retry_attempt.get() is None + calls.append("disconnect") + + async def post(*_: Any, **__: Any) -> httpx.Response: + return httpx.Response(201, json={"upload_id": "private-upload"}) + + async def op(*_: Any, **body: Any) -> dict[str, str]: + return {"ingestion_id": body["namespace"]} + + async def get(org: str, ns: str, ingestion_id: str) -> dict[str, Any]: + assert native._retry_attempt.get() is None + assert ingest_run._retry_or_fail is installed + task = asyncio.current_task() + reads[task] = reads.get(task, 0) + 1 + calls.append("read") + if reads[task] == 1: + return {"org": org, "namespace": ns, "ingestion_id": ingestion_id} + return { + "state": "pending", + "stage": "embed", + "error_code": "retrying", + "attempts": 1, + "error_message": "private-status", + } + + async def retry(job: dict[str, Any], error: Exception) -> str: + delegated.append((job, error)) + return "pending" + + async def work(row: dict[str, Any]) -> str: + index = len(entered) + entered.append(row) + calls.append("run") + if len(entered) == 2: + both_entered.set() + await both_entered.wait() + return await ingest_run._retry_or_fail(row, errors[index]) + + monkeypatch.setattr(ch, "get_client", Client) + monkeypatch.setattr(native, "_post", post) + monkeypatch.setattr(native, "_op", op) + monkeypatch.setattr(ingestions, "get_ingestion", get) + monkeypatch.setattr(ingest_run, "_retry_or_fail", retry) + monkeypatch.setattr(ingest_run, "take_and_run", work) + request.getfixturevalue("_observe_ingest_retries") + installed = ingest_run._retry_or_fail + request.getfixturevalue("_observe_ingest_retries") + assert ingest_run._retry_or_fail is installed + namespaces = ["private-first", "private-first" if same_scope else "private-second"] + results = await asyncio.wait_for( + asyncio.gather( + *( + native._ingest(None, "private-org", ns, "regions.csv", b"csv", []) + for ns in namespaces + ), + return_exceptions=True, + ), + timeout=2, + ) + observed = [] + for result in results: + assert isinstance(result, AssertionError) + assert "private-" not in str(result) + observed.append(ast.literal_eval(str(result).split("\n", 1)[0])[2]) + assert [(out["retry_exception"], out["native_code"]) for out in observed] == [ + ("OverloadedError", 241), + ("BackendUnavailableError", 210), + ] + assert len(delegated) == 2 + assert {(id(job), id(error)) for job, error in delegated} == { + (id(entered[i]), id(errors[i])) for i in range(2) + } + assert calls.count("connect") == calls.count("disconnect") == calls.count("run") == 2 + assert calls.count("read") == 4 and native._retry_attempt.get() is None + assert ingest_run._retry_or_fail is installed + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cancelled", [False, True]) +async def test_delegate_exception_and_cancellation_keep_identity_and_cleanup( + monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest, cancelled: bool +) -> None: + from server.core.errors import OverloadedError + + raised = ( + asyncio.CancelledError("private-cancel") if cancelled else RuntimeError("private-error") + ) + calls, delegated, _ = await _install_retry(monkeypatch, OverloadedError(), raised=raised) + request.getfixturevalue("_observe_ingest_retries") + with pytest.raises(type(raised)) as caught: + await native._ingest(None, "fixture-org", "fixture-ns", "regions.csv", b"csv", []) + assert caught.value is raised and native._retry_attempt.get() is None + assert len(delegated) == 1 and calls == ["connect", "read", "run", "disconnect"] + + +@pytest.mark.asyncio +async def test_ready_delegate_adds_no_status_read_or_failure_message( + monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest +) -> None: + from server.core.errors import OverloadedError + + calls, delegated, _ = await _install_retry( + monkeypatch, OverloadedError("private"), result="ready" + ) + request.getfixturevalue("_observe_ingest_retries") + await native._ingest(None, "fixture-org", "fixture-ns", "regions.csv", b"csv", []) + assert len(delegated) == 1 and calls == ["connect", "read", "run", "disconnect"] + assert native._retry_attempt.get() is None + + +@pytest.mark.asyncio +async def test_dispatcher_returns_the_same_awaitable_and_fixture_restores_once() -> None: + original = ingest_run._retry_or_fail + calls: list[tuple[Any, Any]] = [] + job, error, result = {}, RuntimeError("private"), object() + + class Answer: + def __await__(self): + if False: + yield + return result + + answer = Answer() + + def delegate(actual_job: Any, actual_error: Any) -> Any: + calls.append((actual_job, actual_error)) + return answer + + with pytest.MonkeyPatch.context() as boundary: + boundary.setattr(ingest_run, "_retry_or_fail", delegate) + native._observe_ingest_retries.__wrapped__(boundary) + returned = ingest_run._retry_or_fail(job, error) + assert returned is answer and await returned is result + assert len(calls) == 1 and calls[0][0] is job and calls[0][1] is error + assert ingest_run._retry_or_fail is original and native._retry_attempt.get() is None From c0f1d985c61ffbdbf0e5502d3693afd10427f294 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Sat, 21 Nov 2026 05:11:37 +0000 Subject: [PATCH 10/10] Retry diagnostics read at most four cause links Signed-off-by: zlareb1 --- .../test_knowledge_retrieval_floors.py | 2 +- ...est_knowledge_retrieval_failure_context.py | 26 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_knowledge_retrieval_floors.py b/tests/integration/test_knowledge_retrieval_floors.py index 439de634cb..e86f7084f8 100644 --- a/tests/integration/test_knowledge_retrieval_floors.py +++ b/tests/integration/test_knowledge_retrieval_floors.py @@ -294,7 +294,7 @@ def _retry_failure(error: Exception) -> dict[str, Any]: break if depth: out["native_exception"] = "other" - if not any(type(current) is cls for cls, _ in labels): + if depth == 4 or not any(type(current) is cls for cls, _ in labels): break current = BaseException.__getattribute__(current, "__cause__") except Exception: # noqa: BLE001 — diagnostics must not replace the worker outcome diff --git a/tests/unit/test_knowledge_retrieval_failure_context.py b/tests/unit/test_knowledge_retrieval_failure_context.py index f52ed5fccb..8b104676b4 100644 --- a/tests/unit/test_knowledge_retrieval_failure_context.py +++ b/tests/unit/test_knowledge_retrieval_failure_context.py @@ -792,3 +792,29 @@ def delegate(actual_job: Any, actual_error: Any) -> Any: assert returned is answer and await returned is result assert len(calls) == 1 and calls[0][0] is job and calls[0][1] is error assert ingest_run._retry_or_fail is original and native._retry_attempt.get() is None + + +def test_native_cause_bound_does_not_even_read_a_fifth_link( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from clickhouse_connect.driver.exceptions import DatabaseError + + from server.core.errors import BackendUnavailableError + + error = BackendUnavailableError() + previous = error + for _ in range(4): + cause = BackendUnavailableError() + previous.__cause__ = cause + previous = cause + previous.__cause__ = DatabaseError("Code: 241. private-fifth-link") + reads: list[BaseException] = [] + descriptor = BaseException.__dict__["__cause__"] + + def cause_of(exc: BaseException) -> BaseException | None: + reads.append(exc) + return descriptor.__get__(exc) + + monkeypatch.setattr(BackendUnavailableError, "__cause__", property(cause_of)) + assert native._retry_failure(error)["native_code"] is None + assert len(reads) == 4