Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 29 additions & 5 deletions tests/integration/test_knowledge_ingest_live.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ async def _op(c: httpx.AsyncClient, op: str, **body: Any) -> dict[str, Any]:
return dict(r.json())


async def _work(org: str, ns: str, ingestion_id: str) -> None:
async def _work(org: str, ns: str, ingestion_id: str) -> str:
"""Run the job in this process, on the same ClickHouse, as a worker pass that
took it does. A pass takes one job of the whole queue, which other tests'
jobs share, so it may not be this one."""
Expand All @@ -91,7 +91,9 @@ async def _work(org: str, ns: str, ingestion_id: str) -> None:
try:
job = await ingestions.get_ingestion(org, ns, ingestion_id)
assert job is not None, ingestion_id
assert await ingest_run.take_and_run(job) is not None, f"job {ingestion_id} did not run"
state = await ingest_run.take_and_run(job)
assert state is not None, f"job {ingestion_id} did not run"
return state
finally:
await ch.disconnect()

Expand Down Expand Up @@ -648,10 +650,22 @@ async def test_a_delete_bounded_before_a_metadata_edit_keeps_the_carried_snapsho
c, "knowledge.ingest", org=org, namespace=ns, upload_id=up.json()["upload_id"],
source_uri=uri,
) # fmt: skip
await _work(org, ns, job["ingestion_id"])
(d,) = (await _op(c, "knowledge.dataset_get", org=org, namespace=ns, source_uri=uri))[
worker_state = await _work(org, ns, job["ingestion_id"])
datasets = (await _op(c, "knowledge.dataset_get", org=org, namespace=ns, source_uri=uri))[
"datasets"
]
dataset_count = len(datasets)
assert dataset_count == 1, {
"dataset_count": dataset_count,
"worker_state": (
worker_state
if type(worker_state) is str
and worker_state
in {"pending", "processing", "ready", "partial", "failed", "cancelled"}
else "unknown"
),
}
(d,) = datasets
ch = get_client()
await ch.connect()
try:
Expand Down Expand Up @@ -710,9 +724,19 @@ async def test_a_workbooks_sheets_are_datasets_each_sampled_past_its_rejected_ro
c, "knowledge.ingest", org=org, namespace=ns, upload_id=up.json()["upload_id"],
source_uri=uri,
) # fmt: skip
await _work(org, ns, job["ingestion_id"])
worker_state = await _work(org, ns, job["ingestion_id"])
got = await _op(c, "knowledge.dataset_get", org=org, namespace=ns, source_uri=uri,
sample_rows=2) # fmt: skip
dataset_count = len(got["datasets"])
assert dataset_count == 2, {
"dataset_count": dataset_count,
"worker_state": (
worker_state
if type(worker_state) is str
and worker_state in {"pending", "processing", "ready", "partial", "failed", "cancelled"}
else "unknown"
),
}
one, two = got["datasets"]
assert (one["sheet"], one["row_count"], one["rejected_rows"]) == ("One", 3, 1)
assert one["rows"] == [{"k": "a", "v": 1}, {"k": "c", "v": 3}]
Expand Down
128 changes: 126 additions & 2 deletions tests/integration/test_knowledge_retrieval_floors.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
import asyncio
import hashlib
import uuid
from collections.abc import Awaitable
from contextvars import ContextVar
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
Expand Down Expand Up @@ -235,6 +237,119 @@ async def _op(c: httpx.AsyncClient, op: str, **body: Any) -> dict[str, Any]:
return dict(r.json())


@dataclass
class _RetryAttempt:
scope: tuple[str, str, str]
task: asyncio.Task[Any] | None
details: dict[str, Any] = field(default_factory=dict)


_retry_attempt: ContextVar[_RetryAttempt | None] = ContextVar("ingest_retry_attempt", default=None)


def _retry_failure(error: Exception) -> dict[str, Any]:
from clickhouse_connect.driver.exceptions import DatabaseError, OperationalError

from server.core.ch_errors import error_code
from server.core.errors import (
BackendTimeoutError,
BackendUnavailableError,
EmbedTimeoutError,
OverloadedError,
)
from server.core.pg import PostgresUnavailableError
from server.knowledge.app.extractors import ExtractorUnavailable

labels = (
(BackendUnavailableError, "BackendUnavailableError"),
(BackendTimeoutError, "BackendTimeoutError"),
(OverloadedError, "OverloadedError"),
(EmbedTimeoutError, "EmbedTimeoutError"),
(ExtractorUnavailable, "ExtractorUnavailable"),
(PostgresUnavailableError, "PostgresUnavailableError"),
)
label = next((name for cls, name in labels if type(error) is cls), "other")
out: dict[str, Any] = {
"retry_exception": label,
"native_exception": "none",
"native_code": None,
}
current: BaseException | None = error
seen: set[int] = set()
try:
for depth in range(5): # the caught exception and at most four causes
if current is None or id(current) in seen:
break
seen.add(id(current))
if type(current) in (DatabaseError, OperationalError):
out["native_exception"] = (
"DatabaseError" if type(current) is DatabaseError else "OperationalError"
)
attrs = vars(current)
args = current.args
code = attrs.get("code")
if code is None and args and type(args[0]) is str:
code = error_code(args[0][:4096])
out["native_code"] = code if type(code) is int and 0 <= code <= 1_000_000 else None
break
if depth:
out["native_exception"] = "other"
if depth == 4 or not any(type(current) is cls for cls, _ in labels):
break
current = BaseException.__getattribute__(current, "__cause__")
except Exception: # noqa: BLE001 — diagnostics must not replace the worker outcome
out["native_exception"], out["native_code"] = "unavailable", None
return out


@pytest.fixture
def _observe_ingest_retries(monkeypatch: pytest.MonkeyPatch) -> None:
from server.knowledge.app import ingest_run

original = ingest_run._retry_or_fail

def observe(job: dict[str, Any], error: Exception) -> Awaitable[str]:
attempt = _retry_attempt.get()
if attempt is not None and attempt.task is asyncio.current_task() and type(job) is dict:
scope = tuple(job.get(key) for key in ("org", "namespace", "ingestion_id"))
if all(type(value) is str for value in scope) and scope == attempt.scope:
attempt.details.update(_retry_failure(error))
return original(job, error)

monkeypatch.setattr(ingest_run, "_retry_or_fail", observe)


async def _ingest_failure_context(org: str, ns: str, ingestion_id: str) -> dict[str, Any]:
"""Read only fixed status fields; a retry message can contain source content."""
from server.knowledge.store import ingestions

try:
async with asyncio.timeout(1):
row = await ingestions.get_ingestion(org, ns, ingestion_id)
except Exception: # noqa: BLE001 — diagnostic reads must not replace the failed assertion
return {"observation": "unavailable"}
if row is None:
return {"observation": "missing"}
choices = {
"state": {"pending", "processing", "ready", "partial", "failed", "cancelled"},
"stage": {"", "parse", "embed", "publish"},
"error_code": {"", "retrying", "retries_exhausted", "revision_conflict"},
}
out: dict[str, Any] = {}
for key, allowed in choices.items():
value = row.get(key)
out[key] = (
value
if type(value) is str and value in allowed
else "other"
if key == "error_code"
else "unknown"
)
attempts = row.get("attempts")
out["attempts"] = attempts if type(attempts) is int and 0 <= attempts <= 1_000_000 else None
return out


async def _ingest(
c: httpx.AsyncClient, org: str, ns: str, name: str, data: bytes, tags: list[str]
) -> None:
Expand Down Expand Up @@ -268,7 +383,16 @@ async def _ingest(
try:
row = await ingestions.get_ingestion(org, ns, job["ingestion_id"])
assert row is not None
assert await ingest_run.take_and_run(row) == "ready", name
attempt = _RetryAttempt((org, ns, job["ingestion_id"]), asyncio.current_task())
token = _retry_attempt.set(attempt)
try:
state = await ingest_run.take_and_run(row)
finally:
_retry_attempt.reset(token)
# pytest truncates non-string messages before all diagnostic fields are shown.
assert state == "ready", repr(
(name, await _ingest_failure_context(org, ns, job["ingestion_id"]), attempt.details)
)
finally:
await ch.disconnect()

Expand Down Expand Up @@ -307,7 +431,7 @@ def _measure(
}


@pytest.mark.usefixtures("reference_extractor")
@pytest.mark.usefixtures("reference_extractor", "_observe_ingest_retries")
@pytest.mark.asyncio
async def test_each_retrieval_slice_meets_its_floor(control_plane_url: str) -> None:
from server.embedding.embedder import embedder_is_fake
Expand Down
Loading
Loading