diff --git a/.lint_baselines/falsey_clobber.json b/.lint_baselines/falsey_clobber.json index 1246857..1015322 100644 --- a/.lint_baselines/falsey_clobber.json +++ b/.lint_baselines/falsey_clobber.json @@ -22,24 +22,24 @@ "axonflow/adapters/tool_wrapper.py:190:20", "axonflow/adapters/tool_wrapper.py:208:20", "axonflow/adapters/tool_wrapper.py:220:20", - "axonflow/client.py:1111:16", - "axonflow/client.py:1188:16", - "axonflow/client.py:1686:37", - "axonflow/client.py:1727:18", - "axonflow/client.py:1785:37", - "axonflow/client.py:2309:24", - "axonflow/client.py:2330:33", - "axonflow/client.py:2331:31", - "axonflow/client.py:2343:25", - "axonflow/client.py:2404:28", - "axonflow/client.py:2445:69", + "axonflow/client.py:1116:16", + "axonflow/client.py:1193:16", + "axonflow/client.py:1691:37", + "axonflow/client.py:1732:18", + "axonflow/client.py:1790:37", + "axonflow/client.py:2314:24", + "axonflow/client.py:2335:33", + "axonflow/client.py:2336:31", + "axonflow/client.py:2348:25", + "axonflow/client.py:2409:28", + "axonflow/client.py:2450:69", "axonflow/client.py:300:14", "axonflow/client.py:305:24", "axonflow/client.py:306:20", - "axonflow/client.py:529:44", - "axonflow/client.py:6486:25", - "axonflow/client.py:845:20", - "axonflow/client.py:931:20", + "axonflow/client.py:534:44", + "axonflow/client.py:6491:25", + "axonflow/client.py:850:20", + "axonflow/client.py:936:20", "axonflow/execution.py:205:19", "axonflow/masfeat.py:296:23", "axonflow/masfeat.py:297:24", diff --git a/CHANGELOG.md b/CHANGELOG.md index 25f2eac..e5b3228 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Real wire fields `policy_decision`, `policy_details`, `response_time_ms` on + the audit read model (`AuditLogEntry`), and `action` on audit search + (`AuditSearchRequest`). + +### Deprecated + +- `query_summary`/`success`/`blocked`/`risk_score`/`latency_ms`/ + `policy_violations`/`metadata` (read model) and `request_type` (search + request) - never served/read on the 9.x line (#3254). Removal rides the + next major. + ## [9.0.0] - 2026-07-18 ### Changed (BREAKING) diff --git a/axonflow/client.py b/axonflow/client.py index b72c65e..cdeb9da 100644 --- a/axonflow/client.py +++ b/axonflow/client.py @@ -444,6 +444,11 @@ def _build_audit_search_body(request: AuditSearchRequest) -> dict[str, Any]: body["start_time"] = request.start_time.isoformat() if request.end_time: body["end_time"] = request.end_time.isoformat() + if request.action: + body["action"] = request.action + # Deprecated (#3254): the 9.x server does not read request_type as a + # search filter. Still sent when set (harmless, ignored) until the next + # major removes the field. if request.request_type: body["request_type"] = request.request_type if request.decision_id: diff --git a/axonflow/types.py b/axonflow/types.py index f4b9f35..26ddaeb 100644 --- a/axonflow/types.py +++ b/axonflow/types.py @@ -891,7 +891,11 @@ class AuditSearchRequest(BaseModel): client_id: Filter by client/application ID start_time: Start of time range to search end_time: End of time range to search - request_type: Filter by request type (e.g., "llm_chat", "policy_check") + action: Filters by action/request type with verdict normalization on + the server side. + request_type: Deprecated: the 9.x server does not read this filter; a + search filtered only by it returns unfiltered results. Use + ``action``. Scheduled for removal in the next major (#3254). limit: Maximum results to return (default: 100, max: 1000) offset: Pagination offset (default: 0) """ @@ -900,7 +904,21 @@ class AuditSearchRequest(BaseModel): client_id: str | None = Field(default=None, description="Filter by client ID") start_time: datetime | None = Field(default=None, description="Start of time range") end_time: datetime | None = Field(default=None, description="End of time range") - request_type: str | None = Field(default=None, description="Filter by request type") + action: str | None = Field( + default=None, + description=( + "Filters by action/request type with verdict normalization on the server side." + ), + ) + request_type: str | None = Field( + default=None, + description=( + "Deprecated: the 9.x server does not read this filter; a search " + "filtered only by it returns unfiltered results. Use `action`. " + "Scheduled for removal in the next major (#3254). Still sent when " + "set (harmless, ignored)." + ), + ) # ADR-043: explainability + audit cross-reference filters. decision_id: str | None = Field(default=None, description="Filter by decision ID") policy_name: str | None = Field(default=None, description="Filter by matched policy name") @@ -957,16 +975,63 @@ class AuditLogEntry(BaseModel): client_id: Client/application that made the request tenant_id: Tenant identifier request_type: Type of request (e.g., "llm_chat", "sql", "mcp-query") - query_summary: Summary of the query/request - success: Whether the request succeeded - blocked: Whether the request was blocked by policy - risk_score: Calculated risk score (0.0-1.0) + policy_decision: Policy verdict for the request. Open string set, not + an enum: "allowed", "blocked", "redacted" observed in code and + "error" observed live; newer platforms may add values. + policy_details: Policy evaluation context (object with arbitrary + keys, e.g. tool_name, success, error_message). + response_time_ms: Server-measured response time in milliseconds. + query_summary: Deprecated: never populated on the 9.x line - the + server has never sent this field + (getaxonflow/axonflow-enterprise#3254); the wire carries + `query`/`query_hash`, not modeled in this interim. Read + `policy_decision` for the verdict ("blocked" replaces + `blocked=true`; "allowed" replaces `success=true`), + `policy_details` for violation context, and `response_time_ms` + for latency. Scheduled for removal in the next major. + success: Deprecated: never populated on the 9.x line - the server has + never sent this field (getaxonflow/axonflow-enterprise#3254). + Read `policy_decision` for the verdict ("blocked" replaces + `blocked=true`; "allowed" replaces `success=true`), + `policy_details` for violation context, and `response_time_ms` + for latency. Scheduled for removal in the next major. + blocked: Deprecated: never populated on the 9.x line - the server has + never sent this field (getaxonflow/axonflow-enterprise#3254). + Read `policy_decision` for the verdict ("blocked" replaces + `blocked=true`; "allowed" replaces `success=true`), + `policy_details` for violation context, and `response_time_ms` + for latency. Scheduled for removal in the next major. + risk_score: Deprecated: never populated on the 9.x line - the server + has never sent this field + (getaxonflow/axonflow-enterprise#3254); no wire equivalent. Read + `policy_decision` for the verdict ("blocked" replaces + `blocked=true`; "allowed" replaces `success=true`), + `policy_details` for violation context, and `response_time_ms` + for latency. Scheduled for removal in the next major. provider: LLM provider used (if applicable) model: Model used (if applicable) tokens_used: Total tokens consumed - latency_ms: Request latency in milliseconds - policy_violations: List of violated policy IDs (if any) - metadata: Additional context + latency_ms: Deprecated: never populated on the 9.x line - the server + has never sent this field (getaxonflow/axonflow-enterprise#3254). + Read `policy_decision` for the verdict ("blocked" replaces + `blocked=true`; "allowed" replaces `success=true`), + `policy_details` for violation context, and `response_time_ms` + for latency. Scheduled for removal in the next major. + policy_violations: Deprecated: never populated on the 9.x line - the + server has never sent this field + (getaxonflow/axonflow-enterprise#3254). Read `policy_decision` + for the verdict ("blocked" replaces `blocked=true`; "allowed" + replaces `success=true`), `policy_details` for violation context, + and `response_time_ms` for latency. Scheduled for removal in the + next major. + metadata: Deprecated: never populated on the 9.x line - the server + has never sent this field + (getaxonflow/axonflow-enterprise#3254); the wire carries + `policy_details`/`security_metrics` instead. Read + `policy_decision` for the verdict ("blocked" replaces + `blocked=true`; "allowed" replaces `success=true`), + `policy_details` for violation context, and `response_time_ms` + for latency. Scheduled for removal in the next major. """ id: str = Field(..., description="Unique audit log ID") @@ -976,16 +1041,95 @@ class AuditLogEntry(BaseModel): client_id: str = Field(default="", description="Client ID") tenant_id: str = Field(default="", description="Tenant ID") request_type: str = Field(default="", description="Request type") - query_summary: str = Field(default="", description="Query summary") - success: bool = Field(default=True, description="Request succeeded") - blocked: bool = Field(default=False, description="Request was blocked") - risk_score: float = Field(default=0.0, ge=0.0, le=1.0, description="Risk score") + policy_decision: str = Field( + default="", + description=( + "Policy verdict. Open string set, not an enum: 'allowed', " + "'blocked', 'redacted' observed in code and 'error' observed " + "live; newer platforms may add values." + ), + ) + policy_details: dict[str, Any] = Field( + default_factory=dict, + description="Policy evaluation context (arbitrary keys)", + ) + response_time_ms: int = Field( + default=0, ge=0, description="Server-measured response time in ms" + ) + query_summary: str = Field( + default="", + description=( + "Deprecated: never populated on the 9.x line (#3254); the wire " + "carries query/query_hash. Removal rides the next major." + ), + ) + success: bool = Field( + default=True, + description=( + "Deprecated: never populated on the 9.x line (#3254); read " + "policy_decision ('allowed' replaces success=true). Removal " + "rides the next major." + ), + ) + blocked: bool = Field( + default=False, + description=( + "Deprecated: never populated on the 9.x line (#3254); read " + "policy_decision ('blocked' replaces blocked=true). Removal " + "rides the next major." + ), + ) + risk_score: float = Field( + default=0.0, + ge=0.0, + le=1.0, + description=( + "Deprecated: never populated on the 9.x line (#3254); no wire " + "equivalent. Removal rides the next major." + ), + ) provider: str = Field(default="", description="LLM provider") model: str = Field(default="", description="Model used") tokens_used: int = Field(default=0, ge=0, description="Tokens consumed") - latency_ms: int = Field(default=0, ge=0, description="Latency in ms") - policy_violations: list[str] = Field(default_factory=list, description="Violated policies") - metadata: dict[str, Any] = Field(default_factory=dict, description="Additional metadata") + latency_ms: int = Field( + default=0, + ge=0, + description=( + "Deprecated: never populated on the 9.x line (#3254); read " + "response_time_ms. Removal rides the next major." + ), + ) + policy_violations: list[str] = Field( + default_factory=list, + description=( + "Deprecated: never populated on the 9.x line (#3254); read " + "policy_details for violation context. Removal rides the next " + "major." + ), + ) + metadata: dict[str, Any] = Field( + default_factory=dict, + description=( + "Deprecated: never populated on the 9.x line (#3254); the wire " + "carries policy_details/security_metrics instead. Removal rides " + "the next major." + ), + ) + + @field_validator("policy_details", "metadata", mode="before") + @classmethod + def _coerce_none_to_dict(cls, v: object) -> object: + # The orchestrator marshals a nil Go map as JSON null (observed + # live on real /api/v1/audit/search rows, #3254): null-tolerant, + # not merely absence-tolerant. + return v if v is not None else {} + + @field_validator("policy_violations", mode="before") + @classmethod + def _coerce_none_to_list(cls, v: object) -> object: + # Same class as above for a nil Go slice, defensively. + return v if v is not None else [] + data_residency: str | None = Field( default=None, description="ISO 3166-1 alpha-2 data residency code" ) diff --git a/runtime-e2e/audit_real_wire_fields/README.md b/runtime-e2e/audit_real_wire_fields/README.md new file mode 100644 index 0000000..73e322a --- /dev/null +++ b/runtime-e2e/audit_real_wire_fields/README.md @@ -0,0 +1,34 @@ +# audit_real_wire_fields (#3254) + +Real-stack proof for the audit read model's real wire fields +(getaxonflow/axonflow-enterprise#3254 additive interim). + +The orchestrator has never served `query_summary`/`success`/`blocked`/ +`risk_score`/`latency_ms`/`policy_violations`/`metadata` on the 9.x +line; the real wire carries `policy_decision`, `policy_details` and +`response_time_ms`, and the search filter the server reads is `action` +(not `request_type`). + +## What this proves + +Drives the real SDK (`client.audit_tool_call` + `client.search_audit_logs`) +against a real running agent: + +1. Freshly written success/failure rows come back with `policy_decision` + populated (`allowed` / `error`) and `policy_details` carrying the tool + name and error message, on the TYPED `AuditLogEntry`. +2. The seven deprecated fiction fields stay at their defaults on real rows. +3. `AuditSearchRequest(action=...)` filters server-side (returns only the + matching verdict, including this run's row). +4. A search filtered only by a nonsense `request_type` returns unfiltered + results - the 9.x server does not read that filter (the deprecation + claim). + +## Run + +``` +export AXONFLOW_AGENT_URL=http://localhost:8080 +export AXONFLOW_TENANT_ID= +export AXONFLOW_TENANT_SECRET= +python runtime-e2e/audit_real_wire_fields/test.py +``` diff --git a/runtime-e2e/audit_real_wire_fields/test.py b/runtime-e2e/audit_real_wire_fields/test.py new file mode 100644 index 0000000..4e62d20 --- /dev/null +++ b/runtime-e2e/audit_real_wire_fields/test.py @@ -0,0 +1,191 @@ +"""Real-stack assertion: the audit read model parses the REAL 9.x wire +shape (getaxonflow/axonflow-enterprise#3254 additive interim). + +The orchestrator's audit_logger.go AuditEntry has never served +query_summary/success/blocked/risk_score/latency_ms/policy_violations/ +metadata on the 9.x line - the SDK's pre-#3254 model was built to spec +fiction and silently parsed real audit rows into zero-values. This test +drives the real SDK end to end against a real running agent: + + 1. Write two fresh audit rows via `client.audit_tool_call` (one + success-shaped, one failure-shaped) with a per-run tool-name nonce. + 2. Poll `client.search_audit_logs` (real POST /api/v1/audit/search + through the agent proxy) until both rows land (the orchestrator's + AuditLogger batches writes, flush every 10s). + 3. Assert on the TYPED AuditLogEntry: `policy_decision` is populated + ("allowed" on the success row, "error" on the failure row - the + verdict set is open), `policy_details` carries the tool name and + error message, `response_time_ms` parses; while the seven + deprecated fiction fields stay at their defaults on every row. + 4. Prove the new `action` search filter is READ server-side: a search + for the failure row's verdict returns only entries with that + verdict, including our row. + 5. Prove the `request_type` deprecation claim: a search filtered ONLY + by a nonsense request_type returns unfiltered results (the 9.x + server does not read the filter - silent no-op). + +Usage:: + + export AXONFLOW_AGENT_URL=http://localhost:8080 + export AXONFLOW_TENANT_ID= # e.g. demo-client + export AXONFLOW_TENANT_SECRET= # e.g. demo-secret + python runtime-e2e/audit_real_wire_fields/test.py + +Community-mode note: audit reads are tenant-scoped to "community" while +tool-call writes through the agent proxy land under the same scope, so +write and read agree with any registered credential. See ../README.md. +""" + +from __future__ import annotations + +import asyncio +import os +import sys +import time +import uuid + +from axonflow import AxonFlow +from axonflow.types import AuditLogEntry, AuditSearchRequest, AuditToolCallRequest + +AGENT_URL = os.environ.get("AXONFLOW_AGENT_URL", "http://localhost:8080") +CLIENT_ID = os.environ.get("AXONFLOW_TENANT_ID", "demo-client") +SECRET = os.environ.get("AXONFLOW_TENANT_SECRET", "demo-secret") + +_RUN_ID = uuid.uuid4().hex[:12] +OK_TOOL = f"e2e-real-wire-ok-{_RUN_ID}" +FAIL_TOOL = f"e2e-real-wire-fail-{_RUN_ID}" +FAIL_ERROR = f"e2e-real-wire-error-{_RUN_ID}" + +# The orchestrator's AuditLogger batches writes (flush every 10s); give +# generous headroom under CI/local load. +POLL_DEADLINE_SECONDS = 45.0 +POLL_INTERVAL_SECONDS = 2.0 +SEARCH_LIMIT = 100 + + +def _fail(msg: str) -> None: + sys.stderr.write(f"FAIL: {msg}\n") + sys.exit(1) + + +def _tool_name(entry: AuditLogEntry) -> str: + return str(entry.policy_details.get("tool_name", "")) + + +async def _poll_for_rows(client: AxonFlow) -> tuple[AuditLogEntry, AuditLogEntry]: + deadline = time.monotonic() + POLL_DEADLINE_SECONDS + ok_row = fail_row = None + while time.monotonic() < deadline: + result = await client.search_audit_logs(AuditSearchRequest(limit=SEARCH_LIMIT)) + for entry in result.entries: + if _tool_name(entry) == OK_TOOL: + ok_row = entry + elif _tool_name(entry) == FAIL_TOOL: + fail_row = entry + if ok_row is not None and fail_row is not None: + return ok_row, fail_row + await asyncio.sleep(POLL_INTERVAL_SECONDS) + _fail( + f"rows did not land within {POLL_DEADLINE_SECONDS}s " + f"(ok={ok_row is not None} fail={fail_row is not None}); " + "is the orchestrator's audit batch writer running?" + ) + raise AssertionError # unreachable; _fail exits + + +def _assert_fiction_fields_default(entry: AuditLogEntry, label: str) -> None: + """The seven deprecated fields are never served on 9.x - they must + stay at their model defaults on a REAL row.""" + checks = [ + ("query_summary", entry.query_summary, ""), + ("success", entry.success, True), + ("blocked", entry.blocked, False), + ("risk_score", entry.risk_score, 0.0), + ("latency_ms", entry.latency_ms, 0), + ("policy_violations", entry.policy_violations, []), + ("metadata", entry.metadata, {}), + ] + for name, got, want in checks: + if got != want: + _fail(f"{label}: fiction field {name} = {got!r}, expected default {want!r}") + + +async def main() -> int: + async with AxonFlow( + endpoint=AGENT_URL, + client_id=CLIENT_ID, + client_secret=SECRET, + ) as client: + # 1. Write one success-shaped and one failure-shaped row. + ok_write = await client.audit_tool_call( + AuditToolCallRequest( + tool_name=OK_TOOL, + caller_name="sdk-python-runtime-e2e", + success=True, + duration_ms=12, + ) + ) + fail_write = await client.audit_tool_call( + AuditToolCallRequest( + tool_name=FAIL_TOOL, + caller_name="sdk-python-runtime-e2e", + success=False, + error_message=FAIL_ERROR, + ) + ) + print(f"wrote rows: ok={ok_write.audit_id} fail={fail_write.audit_id}") + + # 2. Poll the real search endpoint until both rows land. + ok_row, fail_row = await _poll_for_rows(client) + + # 3. Typed assertions on the real wire shape. + if ok_row.policy_decision != "allowed": + _fail(f"success row policy_decision = {ok_row.policy_decision!r}, want 'allowed'") + if fail_row.policy_decision != "error": + _fail(f"failure row policy_decision = {fail_row.policy_decision!r}, want 'error'") + if fail_row.policy_details.get("error_message") != FAIL_ERROR: + _fail( + "failure row policy_details.error_message = " + f"{fail_row.policy_details.get('error_message')!r}, want {FAIL_ERROR!r}" + ) + if not isinstance(ok_row.response_time_ms, int) or ok_row.response_time_ms < 0: + _fail(f"response_time_ms did not parse: {ok_row.response_time_ms!r}") + _assert_fiction_fields_default(ok_row, "success row") + _assert_fiction_fields_default(fail_row, "failure row") + print( + f"typed parse OK: ok.policy_decision={ok_row.policy_decision!r} " + f"fail.policy_decision={fail_row.policy_decision!r} " + "fiction fields at defaults on both rows" + ) + + # 4. The action filter is read server-side. + filtered = await client.search_audit_logs( + AuditSearchRequest(action="error", limit=SEARCH_LIMIT) + ) + wrong = [e.policy_decision for e in filtered.entries if e.policy_decision != "error"] + if wrong: + _fail(f"action='error' returned non-error verdicts: {wrong}") + if not any(_tool_name(e) == FAIL_TOOL for e in filtered.entries): + _fail("action='error' did not return this run's failure row") + print(f"action filter OK: {len(filtered.entries)} entries, all verdict 'error'") + + # 5. request_type is a server-side no-op (#3254 deprecation claim). + noop = await client.search_audit_logs( + AuditSearchRequest(request_type=f"nonexistent-type-{_RUN_ID}", limit=SEARCH_LIMIT) + ) + if not any(_tool_name(e) == OK_TOOL for e in noop.entries): + _fail( + "request_type= filtered rows out - the server appears to " + "read request_type after all; re-check the #3254 deprecation" + ) + print( + f"request_type no-op confirmed: nonsense filter still returned " + f"{len(noop.entries)} rows including this run's" + ) + + print("PASS: audit_real_wire_fields") + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) diff --git a/tests/fixtures/audit_search_live_v9130.json b/tests/fixtures/audit_search_live_v9130.json new file mode 100644 index 0000000..c2fd68a --- /dev/null +++ b/tests/fixtures/audit_search_live_v9130.json @@ -0,0 +1 @@ +{"entries":[{"id":"audit_1785794706_23m371y7","request_id":"","timestamp":"2026-08-03T22:05:06.947296Z","user_id":0,"user_email":"","user_role":"","client_id":"community","tenant_id":"community","org_id":"","request_type":"tool_call_audit","query":"Tool: s3254_blocked_probe","query_hash":"","policy_decision":"error","policy_details":{"caller_name":"unknown","error_message":"blocked by policy sys_sqli_or_true","success":false,"tool_name":"s3254_blocked_probe"},"provider":"","model":"","response_time_ms":0,"tokens_used":0,"cost":0,"redacted_fields":null,"error_message":"blocked by policy sys_sqli_or_true","response_sample":"","compliance_flags":null,"security_metrics":null},{"id":"audit_1785794693_wiccqrjt","request_id":"","timestamp":"2026-08-03T22:04:53.408794Z","user_id":0,"user_email":"","user_role":"","client_id":"community","tenant_id":"community","org_id":"","request_type":"tool_call_audit","query":"Tool: s3254_capture_probe","query_hash":"","policy_decision":"allowed","policy_details":{"caller_name":"unknown","success":true,"tool_name":"s3254_capture_probe"},"provider":"","model":"","response_time_ms":0,"tokens_used":0,"cost":0,"redacted_fields":null,"response_sample":"","compliance_flags":null,"security_metrics":null}],"total":2,"limit":10,"offset":0} diff --git a/tests/fixtures/wire_shape_baseline.json b/tests/fixtures/wire_shape_baseline.json index 527aba5..31e5e3f 100644 --- a/tests/fixtures/wire_shape_baseline.json +++ b/tests/fixtures/wire_shape_baseline.json @@ -180,19 +180,23 @@ "openapi_specs_sha": "0bd9256237ebbffb9c0101126da71f2c940a1695", "per_model_drift": { "AuditLogEntry": { - "note": "spec-bug-pending: #1745 \u2014 agent-api.yaml AuditLogEntry omits metadata/model/policy_violations the agent emits on every audit-log read.", + "note": "spec-bug-pending: #1745 \u2014 agent-api.yaml AuditLogEntry omits metadata/model/policy_violations the agent emits on every audit-log read. Plus getaxonflow/axonflow-enterprise#3254 additive interim: policy_decision/policy_details/response_time_ms are the REAL 9.x wire fields (platform/orchestrator/audit_logger.go AuditEntry serves them at v9.6.1 and v9.13.0); this pre-v9 spec pin predates them, so they read as sdk_only until the pin moves to v9.13.0 (PR #214).", "sdk_only": [ "data_residency", "metadata", "model", + "policy_decision", + "policy_details", "policy_violations", + "response_time_ms", "transfer_basis" ], "spec_only": [] }, "AuditSearchRequest": { - "note": "acknowledged-sdk-superset: tracked in #1745. SDK accepts decision_id/offset/override_id/policy_name as query params; agent-api.yaml AuditSearchRequest doesn't yet declare them.", + "note": "acknowledged-sdk-superset: tracked in #1745. SDK accepts decision_id/offset/override_id/policy_name as query params; agent-api.yaml AuditSearchRequest doesn't yet declare them. Plus getaxonflow/axonflow-enterprise#3254: `action` is the search filter the 9.x server actually reads (audit_read_handlers.go); this pre-v9 spec pin predates it, so it reads as sdk_only until the pin moves to v9.13.0 (PR #214).", "sdk_only": [ + "action", "decision_id", "offset", "override_id", diff --git a/tests/test_audit.py b/tests/test_audit.py index e3cead8..37bae63 100644 --- a/tests/test_audit.py +++ b/tests/test_audit.py @@ -2,7 +2,9 @@ from __future__ import annotations +import json from datetime import datetime, timezone +from pathlib import Path from typing import Any import pytest @@ -442,3 +444,169 @@ def test_audit_search_response_structure(self) -> None: assert len(response.entries) == 2 assert response.total == 100 assert response.limit == 10 + + +# Real capture: captured 2026-08-03 from an isolated community v9.13.0 stack +# (getaxonflow/axonflow tag v9.13.0 = df027c788), session 3254. Raw +# POST /api/v1/audit/search response through the agent proxy, verbatim. +REAL_CAPTURE_PATH = Path(__file__).parent / "fixtures" / "audit_search_live_v9130.json" + + +class TestAuditRealWireShape: + """#3254: the audit read model against the REAL 9.x wire shape. + + The seven fiction fields (query_summary, success, blocked, risk_score, + latency_ms, policy_violations, metadata) have never been served on the + 9.x line; the real wire carries policy_decision, policy_details and + response_time_ms. These tests pin the additive interim: real fields + populate, fiction fields stay at defaults, nothing throws. + """ + + def test_real_capture_parses_with_new_fields_populated(self) -> None: + """Deserialize the REAL captured payload, unmodified. + + Fixture provenance: captured 2026-08-03 from an isolated community + v9.13.0 stack, session 3254 (see REAL_CAPTURE_PATH comment). + """ + payload = json.loads(REAL_CAPTURE_PATH.read_text()) + response = AuditSearchResponse.model_validate(payload) + + assert response.total == 2 + assert len(response.entries) == 2 + + error_entry = response.entries[0] + allowed_entry = response.entries[1] + + # New real-wire fields are populated from the capture. + assert error_entry.policy_decision == "error" + assert error_entry.policy_details["tool_name"] == "s3254_blocked_probe" + expected_error = "blocked by policy sys_sqli_or_true" + assert error_entry.policy_details["error_message"] == expected_error + assert allowed_entry.policy_decision == "allowed" + assert allowed_entry.policy_details["success"] is True + assert allowed_entry.response_time_ms == 0 + + # The verdict set is OPEN: "error" is not in the code-documented + # allowed/blocked/redacted set and must still parse as a plain string. + assert isinstance(error_entry.policy_decision, str) + + # The seven fiction fields are ABSENT from the real wire and must + # stay at their defaults, silently. + for entry in response.entries: + assert entry.query_summary == "" + assert entry.success is True + assert entry.blocked is False + assert entry.risk_score == 0.0 + assert entry.latency_ms == 0 + assert entry.policy_violations == [] + assert entry.metadata == {} + + # Real fields that were already modeled keep parsing. + assert error_entry.request_type == "tool_call_audit" + assert error_entry.tenant_id == "community" + + def test_old_server_payload_without_new_fields_defaults(self) -> None: + """Old-server tolerance: a payload WITHOUT the three new fields + parses and the new fields default (absence-tolerant contract). + + Hand-modified capture: the real 2026-08-03 session-3254 capture with + policy_decision, policy_details and response_time_ms removed. + """ + payload = json.loads(REAL_CAPTURE_PATH.read_text()) + for raw in payload["entries"]: + del raw["policy_decision"] + del raw["policy_details"] + del raw["response_time_ms"] + + response = AuditSearchResponse.model_validate(payload) + + assert len(response.entries) == 2 + for entry in response.entries: + assert entry.policy_decision == "" + assert entry.policy_details == {} + assert entry.response_time_ms == 0 + + def test_both_fiction_and_real_fields_in_one_payload(self) -> None: + """Fictional AND real fields together parse with no collision. + + Hand-modified capture: the real 2026-08-03 session-3254 capture with + the seven fiction fields injected alongside the real ones. + """ + payload = json.loads(REAL_CAPTURE_PATH.read_text()) + fiction = { + "query_summary": "legacy summary", + "success": False, + "blocked": True, + "risk_score": 0.75, + "latency_ms": 1234, + "policy_violations": ["legacy-policy-1"], + "metadata": {"legacy": True}, + } + for raw in payload["entries"]: + raw.update(fiction) + + response = AuditSearchResponse.model_validate(payload) + + for entry in response.entries: + # Fiction fields parse when present (kept for compatibility). + assert entry.query_summary == "legacy summary" + assert entry.success is False + assert entry.blocked is True + assert entry.risk_score == 0.75 + assert entry.latency_ms == 1234 + assert entry.policy_violations == ["legacy-policy-1"] + assert entry.metadata == {"legacy": True} + # Real fields are untouched by the fiction fields' presence. + assert response.entries[0].policy_decision == "error" + assert response.entries[1].policy_decision == "allowed" + assert response.entries[0].policy_details["tool_name"] == "s3254_blocked_probe" + + def test_null_policy_details_parses(self) -> None: + """The orchestrator marshals a nil Go map/slice as JSON null - + observed live on real /api/v1/audit/search rows (session 3254, + runtime-e2e/audit_real_wire_fields). Null-tolerant, not merely + absence-tolerant. + + Hand-modified capture: the real 2026-08-03 session-3254 capture + with policy_details/metadata/policy_violations set to null. + """ + payload = json.loads(REAL_CAPTURE_PATH.read_text()) + for raw in payload["entries"]: + raw["policy_details"] = None + raw["metadata"] = None + raw["policy_violations"] = None + + response = AuditSearchResponse.model_validate(payload) + + for entry in response.entries: + assert entry.policy_details == {} + assert entry.metadata == {} + assert entry.policy_violations == [] + assert response.entries[0].policy_decision == "error" + + def test_search_request_action_field(self) -> None: + """AuditSearchRequest.action is optional and defaults to None.""" + request = AuditSearchRequest() + assert request.action is None + + request = AuditSearchRequest(action="blocked") + assert request.action == "blocked" + + @pytest.mark.asyncio + async def test_search_sends_action_filter( + self, + client: AxonFlow, + httpx_mock: HTTPXMock, + ) -> None: + """The action filter is sent on the wire; request_type is still + sent when set (deprecated but harmless, #3254).""" + httpx_mock.add_response(json={"entries": [], "total": 0, "limit": 100, "offset": 0}) + + await client.search_audit_logs( + AuditSearchRequest(action="error", request_type="legacy_filter") + ) + + sent = httpx_mock.get_requests()[-1] + body = json.loads(sent.content) + assert body["action"] == "error" + assert body["request_type"] == "legacy_filter" diff --git a/tests/test_wire_shape.py b/tests/test_wire_shape.py index 53b3f78..e26d98d 100644 --- a/tests/test_wire_shape.py +++ b/tests/test_wire_shape.py @@ -72,12 +72,38 @@ def _specs_dir() -> Path | None: - """Return the OpenAPI specs directory, or None if not set / missing.""" + """Return the OpenAPI specs directory, or None when the env var is unset. + + Unset AXONFLOW_OPENAPI_SPECS_DIR is the designed local-dev skip. A + variable that IS set - but empty, or pointing at a missing directory + or a non-directory - is a misconfiguration and fails loudly instead: + previously those cases produced 7 silent skips and exit 0, so a + broken CI checkout read as green. Set-but-empty fails (rather than + reading as unset) because a CI consumer wiring the var from an + expression that evaluates empty is exactly the broken-checkout class. + """ env = os.environ.get("AXONFLOW_OPENAPI_SPECS_DIR") - if not env: + if env is None: return None + if not env.strip(): + pytest.fail( + "AXONFLOW_OPENAPI_SPECS_DIR is set but empty. Refusing to treat " + "it as unset: an empty value usually means the CI expression " + "that was supposed to produce the specs path evaluated to " + "nothing, and a silent skip would make that read as a green " + "wire-shape gate. Fix the expression, or unset the variable to " + "skip locally." + ) p = Path(env) - return p if p.is_dir() else None + if not p.is_dir(): + pytest.fail( + f"AXONFLOW_OPENAPI_SPECS_DIR is set to {env!r} but that is not an " + "existing directory. Refusing to skip: with the variable set, a " + "silent skip would make a broken specs checkout read as a green " + "wire-shape gate. Fix the path, or unset the variable to skip " + "locally." + ) + return p def _wire_fields(model: type[BaseModel]) -> list[str]: @@ -230,8 +256,8 @@ def loaded_specs() -> tuple[dict[str, list[str]], dict[str, dict[str, list[str]] spec_dir = _specs_dir() if spec_dir is None: pytest.skip( - "AXONFLOW_OPENAPI_SPECS_DIR not set to an existing directory; " - "wire-shape contract tests skipped. The dedicated CI job clones " + "AXONFLOW_OPENAPI_SPECS_DIR not set; wire-shape contract tests " + "skipped. The dedicated CI job clones " "https://github.com/getaxonflow/axonflow and exports the specs " "dir before running this file." ) @@ -527,3 +553,71 @@ def test_unmapped_spec_schemas_are_tracked( print(f"\n{len(unmapped)} OpenAPI schema(s) have no matching Python SDK model:") for name in unmapped: print(f" - {name}") + + +# --------------------------------------------------------------------------- +# _specs_dir misconfiguration guard (#3254 review follow-up) +# --------------------------------------------------------------------------- +# These do not use the loaded_specs fixture, so they run in the regular +# suite too (the module's other tests skip there via the fixture). + + +def test_specs_dir_unset_keeps_designed_skip(monkeypatch: pytest.MonkeyPatch) -> None: + """Env unset -> None, which the loaded_specs fixture turns into the + designed local-dev skip.""" + monkeypatch.delenv("AXONFLOW_OPENAPI_SPECS_DIR", raising=False) + assert _specs_dir() is None + + +def test_specs_dir_set_but_missing_fails_loudly( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Env SET but pointing at a missing directory must FAIL, not skip. + + Before this guard, that misconfiguration produced 7 silent skips and + exit 0 - a broken CI specs checkout read as a green gate. + """ + missing = tmp_path / "no-such-specs-dir" + monkeypatch.setenv("AXONFLOW_OPENAPI_SPECS_DIR", str(missing)) + with pytest.raises(pytest.fail.Exception, match="AXONFLOW_OPENAPI_SPECS_DIR"): + _specs_dir() + + +def test_specs_dir_set_but_empty_fails_loudly(monkeypatch: pytest.MonkeyPatch) -> None: + """Env SET but EMPTY must FAIL, not read as unset. + + Choice (of fail-on-empty vs empty-as-unset): fail. A CI consumer + wiring the variable from an expression that evaluates empty is the + same broken-checkout class as a missing directory - treating it as + the designed local-dev skip would green-light a gate that checked + nothing. Whitespace-only counts as empty. + """ + monkeypatch.setenv("AXONFLOW_OPENAPI_SPECS_DIR", "") + with pytest.raises(pytest.fail.Exception, match="set but empty"): + _specs_dir() + monkeypatch.setenv("AXONFLOW_OPENAPI_SPECS_DIR", " ") + with pytest.raises(pytest.fail.Exception, match="set but empty"): + _specs_dir() + + +def test_specs_dir_set_to_a_file_fails_loudly( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Env SET but pointing at a FILE (not a directory) must FAIL. + + Behaviorally covered by the is_dir() check; pinned here so a future + refactor to exists() cannot silently reopen the class. + """ + a_file = tmp_path / "specs.yaml" + a_file.write_text("openapi: 3.0.0\n") + monkeypatch.setenv("AXONFLOW_OPENAPI_SPECS_DIR", str(a_file)) + with pytest.raises(pytest.fail.Exception, match="not an existing directory"): + _specs_dir() + + +def test_specs_dir_set_and_present_resolves( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Env set to an existing directory resolves to that path.""" + monkeypatch.setenv("AXONFLOW_OPENAPI_SPECS_DIR", str(tmp_path)) + assert _specs_dir() == tmp_path