From a6ab7e8173f781b7f12817561752916c44e944bb Mon Sep 17 00:00:00 2001 From: Nina Chikanov Date: Fri, 4 Sep 2026 16:49:17 -0700 Subject: [PATCH 1/9] [FEAT]: Add canonical versioned trace/result serializer (WS0-05) Introduce rampart/core/serialization.py as the single, neutral full-fidelity Result <-> dict round-trip (Decision D6 gate). Every record carries a single root version (rampart.trace.v1) and decoding dispatches on it, failing closed on an unknown major. The canonical layer defines the supported value domain only: enums encode to .value and fail closed on unknown values, harm_category is a passthrough string, floats must be finite, and free-form maps must be JSON-safe. Transport hygiene (ANSI stripping, float normalization, repr() fallback, size caps) stays at the xdist boundary and is not duplicated here. Binary payloads fail closed pending the WS7 artifact resolver rather than being coerced to text. Also lands the written migration policy (docs/concepts/trace-schema.md): additive-optional = no bump, structural = major bump, missing = not recorded, readers fail closed on unknown major, with named reserved additive slots so WS8 provenance needs no hard migration. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/concepts/trace-schema.md | 105 +++ mkdocs.yml | 1 + rampart/core/serialization.py | 913 ++++++++++++++++++++++++++ tests/unit/core/test_serialization.py | 314 +++++++++ 4 files changed, 1333 insertions(+) create mode 100644 docs/concepts/trace-schema.md create mode 100644 rampart/core/serialization.py create mode 100644 tests/unit/core/test_serialization.py diff --git a/docs/concepts/trace-schema.md b/docs/concepts/trace-schema.md new file mode 100644 index 00000000..c999f257 --- /dev/null +++ b/docs/concepts/trace-schema.md @@ -0,0 +1,105 @@ +# Trace/Result Schema & Migration Policy + +RAMPART serializes every safety `Result` through a single canonical, versioned +schema (`rampart.core.serialization`). The same schema backs xdist transport, +failure attachments, reporting projections, and — in future work — replay and +golden traces. This page is the written, reviewed migration policy that gates +any durable trace artifact. + +## Versioning + +- Every serialized record carries one root `version` field. The current schema + is **`rampart.trace.v1`**. +- The record version is **independent** of the xdist transport envelope version + (`rampart.xdist.v2`). The two axes move separately; an `xdist.v2` envelope may + carry a `trace.v1` record. +- There is a **single root version** — nested types (`Turn`, `Payload`, + `EvalResult`, …) do not carry their own versions. + +## What is and is not a breaking change + +- **Additive-optional = no bump.** A new optional field that older readers may + ignore, and whose absence has a defined default, does not change the major. +- **Missing = not recorded (not "false").** An absent optional field means the + producer *did not record it* — never that its value was empty, false, or zero. + Readers supply a default for *shape* only; consumers must not infer a semantic + negative from absence. A v1 record with no `manifest_snapshot` means "the + manifest was not captured," not "there was no manifest." +- **Structural change = major bump.** Removing, renaming, or retyping a field, + or changing its meaning or nesting, bumps `vN → vN+1` with a changelog and a + migration note. + +## Reader posture + +- Readers tolerate unknown fields and **fail closed on an unknown major** — a + record is never best-effort parsed across a major boundary. +- Forward compatibility is **additive-only within a major**. A newer major read + by an older framework fails closed by design. +- Any derived JSON Schema is therefore **open** (`additionalProperties: true`). + +## Enum posture + +- The closed enums — `SafetyStatus`, `EvalOutcome`, `ObservabilityLevel`, and + `PayloadFormat` — **fail closed** on an unknown value. A durable safety + artifact must never silently misread one; there is no warn-and-degrade path. +- `HarmCategory` is the sole exception: it travels as a **passthrough string** + and is never coerced, so a new harm label from a future producer round-trips + unchanged on an older reader. + +## Binary / opaque payloads + +- A non-text payload persists as a content-addressed + `{sha256, media_type, bundle_path}` descriptor in `artifacts[]`, never inline. +- A decoder that meets a binary reference with **no artifact resolver wired + fails closed** — it never coerces the payload to `PayloadFormat.TEXT`. +- The descriptor shape is frozen now (populating `artifacts[]` later is + additive-optional); the resolver and companion bundles are built by the replay + work, not by this gate. At `rampart.trace.v1` there is no resolver, so binary + payloads fail closed on both encode and decode. + +## Migration mechanics + +- Each major bump ships an **adjacent upcaster** (`vN-1 → vN`) plus an explicit + **migration API/CLI**. +- Writers always emit the **latest** major. +- Reads **never rewrite** persisted files in place. Backward-*reading* an old + major is not the same as migrating an artifact — migration is an explicit, + opt-in step, never a silent rewrite. + +## Reserved additive fields (named now, populated later) + +To make the additive path concrete, these slots are reserved by name so future +work drops in without a bump, as **record-level wire-only collar slots**: +`manifest_snapshot`, `evaluation_fingerprint`, `replay_provenance`, +`population_ref`, plus `artifacts` / `target` / `provenance`. A field that is +truly *intrinsic to a result* instead lands as an additive-optional field on +`Result`, inside the referenced `result` body. Either way each is +additive-optional; none is populated at v1. + +Later trigger-/persistence-phase provenance fields are additive-optional and +**must not** force a hard migration or major bump. + +## Support window + +After the **first durable-trace release** (the first release that writes +persisted golden traces/evidence, on by default), RAMPART supports reading `vN` +and `vN-1` for **two subsequent framework releases** (one deprecation cycle), +keyed on **release, not time**, with a changelog and migration note on any bump. +Before that release there is no durable-read obligation. + +## Ship gate + +Ship **no durable artifact — golden traces above all — until the schema has a +per-result `version` field and this policy is in effect.** + +```mermaid +flowchart TD + change([proposed schema change]) --> q1{"adds a field only?"} + q1 -- no --> struct["structural:
remove / rename / retype /
change meaning or nesting"] + q1 -- yes --> q2{"optional with a
well-defined default?"} + q2 -- no --> struct + q2 -- yes --> add["additive-optional"] + add --> nobump["NO bump
(new optional fields, later provenance)
old readers ignore unknown keys"] + struct --> bump["bump major vN → vN+1
+ changelog + migration note"] + bump --> reader["readers: fail closed on
unknown major"] +``` diff --git a/mkdocs.yml b/mkdocs.yml index c74a9f5a..606a9133 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -151,6 +151,7 @@ nav: - Attacks: concepts/attacks.md - Probes: concepts/probes.md - PyRIT Integration: concepts/pyrit.md + - Trace Schema & Migration: concepts/trace-schema.md - Attacks: - attacks/index.md - XPIA: attacks/xpia.md diff --git a/rampart/core/serialization.py b/rampart/core/serialization.py new file mode 100644 index 00000000..2ee1c579 --- /dev/null +++ b/rampart/core/serialization.py @@ -0,0 +1,913 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Canonical, versioned trace/result serialization for RAMPART. + +This module owns the *single* full-fidelity ``Result`` <-> ``dict`` round-trip +for the whole framework (design gate WS0-05, Decision D6). xdist transport, +failure attachments, reporting projections, and future replay all serialize +through here rather than maintaining parallel serializers. + +The canonical layer defines the supported *value domain* and nothing else. It +does not apply transport hygiene — no ANSI stripping, no float normalization, +no ``repr()``/``str()`` fallback, and no size capping. Those concerns wrap the +canonical output at the transport boundary (xdist). When a value falls outside +the canonical domain the codec fails closed with a field path rather than +coercing, so a durable trace never silently loses fidelity. + +Every serialized record carries a single root ``version`` field +(:data:`TRACE_SCHEMA_VERSION`). Decoding dispatches on that version and fails +closed on an unknown major. The record version is independent of the xdist +transport envelope version; the two axes move separately. +""" + +from __future__ import annotations + +import math +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import datetime +from enum import Enum +from typing import TYPE_CHECKING, Any, ClassVar, TypeVar + +from rampart.core.result import ( + InjectionRecord, + PopulationRef, + Result, + SafetyStatus, +) +from rampart.core.types import ( + EvalOutcome, + EvalResult, + ObservabilityLevel, + Payload, + PayloadFormat, + Request, + Response, + SideEffect, + ToolCall, + Turn, +) + +if TYPE_CHECKING: + from collections.abc import Callable + +EnumT = TypeVar("EnumT", bound=Enum) + +TRACE_SCHEMA_VERSION = "rampart.trace.v1" +"""Single root schema version stamped on every serialized record.""" + +RESERVED_METADATA_KEYS: frozenset[str] = frozenset( + { + "_pytest_nodeid", + "_pytest_test_name", + "_rampart_result_index", + "_rampart_transport_truncated", + "_rampart_original_size_bytes", + "_rampart_limit_bytes", + "_rampart_worker_format", + "_rampart_worker_artifact_path", + } +) +"""Top-level ``Result.metadata`` keys owned by the xdist transport. + +These are scheduling/bookkeeping breadcrumbs the transport stamps for its own +reconciliation. They are stripped from the canonical body so a durable trace +carries only intrinsic result data; nested user maps are never touched. +""" + + +class SchemaError(Exception): + """Raised when a value falls outside the canonical trace schema domain. + + The message carries the offending field path so producers can locate the + out-of-domain value instead of the codec silently coercing it. + """ + + +class UnsupportedSchemaVersionError(SchemaError): + """Raised when decoding a record whose ``version`` has no decoder. + + Readers fail closed on an unknown major rather than guessing at a shape. + """ + + +@dataclass(frozen=True, kw_only=True) +class ResultRecord: + """The canonical, versioned envelope around a single ``Result``. + + This is the public surface: :meth:`to_dict` / :meth:`from_dict` are the one + round-trip every durable consumer uses. The ``result`` is referenced, not + copied. The collar fields (``identity``, ``pytest_nodeid``, ``result_index``) + are wire-only provenance stamped once at the producing boundary. + + Args: + result (Result): The single-run verdict being serialized. + identity (dict[str, Any] | None): Stable test identity descriptor + (WS0-06). ``None`` until identity is wired at the producer. + pytest_nodeid (str | None): The pytest node id the result came from. + result_index (int): Ordinal of this result within its test node. + """ + + VERSION: ClassVar[str] = TRACE_SCHEMA_VERSION + + result: Result + identity: dict[str, Any] | None = None + pytest_nodeid: str | None = None + result_index: int = 0 + + def to_dict(self) -> dict[str, Any]: + """Encode the record into a canonical, JSON-safe dict. + + Returns: + dict[str, Any]: The versioned record with the encoded result body + and wire-only collar. Fails closed via :class:`SchemaError` on + any value outside the canonical domain. + """ + return { + "version": self.VERSION, + "result": _encode_result(result=self.result, path="result"), + "identity": _encode_json(value=self.identity, path="identity"), + "pytest_nodeid": self.pytest_nodeid, + "result_index": self.result_index, + } + + @classmethod + def from_dict(cls, data: object) -> ResultRecord: + """Decode a canonical dict back into a record, dispatching on version. + + Args: + data (object): A previously encoded record mapping. + + Returns: + ResultRecord: The decoded record. + + Raises: + SchemaError: If ``data`` is not a mapping. + UnsupportedSchemaVersionError: If the record version has no decoder. + """ + if not isinstance(data, Mapping): + msg = f"Expected mapping for record, got {type(data).__name__}." + raise SchemaError(msg) + version = data.get("version") + decoder = _DECODERS.get(version) if isinstance(version, str) else None + if decoder is None: + msg = f"No decoder registered for trace schema version {version!r}." + raise UnsupportedSchemaVersionError(msg) + return decoder(data) + + +def serialize_result( + *, + result: Result, + identity: str | None = None, + origin: str | None = None, + case_id: str | None = None, + pytest_nodeid: str | None = None, + result_index: int = 0, +) -> dict[str, Any]: + """Serialize a result to the canonical, versioned dict. + + Args: + result (Result): The verdict to serialize. + identity (str | None): Stable identity value (WS0-06), if computed. + origin (str | None): How the identity was derived (marker vs. derived). + case_id (str | None): Parametrization case id, travelling beside identity. + pytest_nodeid (str | None): The pytest node id the result came from. + result_index (int): Ordinal of this result within its test node. + + Returns: + dict[str, Any]: The canonical record dict, ready for any durable sink. + """ + identity_descriptor: dict[str, Any] | None = None + if identity is not None or origin is not None or case_id is not None: + identity_descriptor = { + "value": identity, + "origin": origin, + "case_id": case_id, + } + record = ResultRecord( + result=result, + identity=identity_descriptor, + pytest_nodeid=pytest_nodeid, + result_index=result_index, + ) + return record.to_dict() + + +def deserialize_result(*, data: object) -> ResultRecord: + """Deserialize a canonical record dict back into a :class:`ResultRecord`. + + Args: + data (object): A previously encoded record mapping. + + Returns: + ResultRecord: The decoded record. + """ + return ResultRecord.from_dict(data) + + +def _encode_result(*, result: Result, path: str) -> dict[str, Any]: + """Encode a ``Result`` body into canonical primitives. + + Returns: + dict[str, Any]: The encoded result with every field represented. + """ + metadata = { + key: value + for key, value in result.metadata.items() + if key not in RESERVED_METADATA_KEYS + } + return { + "status": _encode_enum(value=result.status, path=f"{path}.status"), + "summary": result.summary, + "observability_level": _encode_enum( + value=result.observability_level, + path=f"{path}.observability_level", + ), + "turns": [ + _encode_turn(turn=turn, path=f"{path}.turns[{index}]") + for index, turn in enumerate(result.turns) + ], + "duration_seconds": _encode_float( + value=result.duration_seconds, + path=f"{path}.duration_seconds", + ), + "harm_category": _encode_harm_category(value=result.harm_category), + "strategy": result.strategy, + "injections": [ + _encode_injection(record=record) for record in result.injections + ], + "population": _encode_population( + value=result.population, + path=f"{path}.population", + ), + "metadata": _encode_json(value=metadata, path=f"{path}.metadata"), + } + + +def _encode_turn(*, turn: Turn, path: str) -> dict[str, Any]: + """Encode a ``Turn``. + + Returns: + dict[str, Any]: The encoded turn. + """ + eval_result = ( + None + if turn.eval_result is None + else _encode_eval_result(value=turn.eval_result, path=f"{path}.eval_result") + ) + return { + "request": _encode_request(request=turn.request, path=f"{path}.request"), + "response": _encode_response(response=turn.response, path=f"{path}.response"), + "eval_result": eval_result, + "turn_number": turn.turn_number, + "timestamp": _encode_datetime(value=turn.timestamp), + "driver_reasoning": turn.driver_reasoning, + } + + +def _encode_request(*, request: Request, path: str) -> dict[str, Any]: + """Encode a ``Request``. + + Returns: + dict[str, Any]: The encoded request. + """ + return { + "prompt": request.prompt, + "attachments": [ + _encode_payload(payload=payload, path=f"{path}.attachments[{index}]") + for index, payload in enumerate(request.attachments) + ], + } + + +def _encode_response(*, response: Response, path: str) -> dict[str, Any]: + """Encode a ``Response``. + + Returns: + dict[str, Any]: The encoded response. + """ + return { + "text": response.text, + "tool_calls": [ + _encode_tool_call(call=call, path=f"{path}.tool_calls[{index}]") + for index, call in enumerate(response.tool_calls) + ], + "side_effects": [ + _encode_side_effect(effect=effect, path=f"{path}.side_effects[{index}]") + for index, effect in enumerate(response.side_effects) + ], + "metadata": _encode_json(value=response.metadata, path=f"{path}.metadata"), + } + + +def _encode_tool_call(*, call: ToolCall, path: str) -> dict[str, Any]: + """Encode a ``ToolCall``. + + Returns: + dict[str, Any]: The encoded tool call. + """ + return { + "name": call.name, + "arguments": _encode_json(value=call.arguments, path=f"{path}.arguments"), + "result": call.result, + "timestamp": _encode_datetime(value=call.timestamp), + } + + +def _encode_side_effect(*, effect: SideEffect, path: str) -> dict[str, Any]: + """Encode a ``SideEffect``. + + Returns: + dict[str, Any]: The encoded side effect. + """ + return { + "kind": effect.kind, + "details": _encode_json(value=effect.details, path=f"{path}.details"), + } + + +def _encode_payload(*, payload: Payload, path: str) -> dict[str, Any]: + """Encode a ``Payload``. + + Binary payloads are persisted as content-addressed artifact descriptors by + WS7 rather than inline; that resolver does not exist at + ``rampart.trace.v1``, so a binary payload fails closed here instead of + inlining a machine-local path. + + Returns: + dict[str, Any]: The encoded payload. + + Raises: + SchemaError: If the payload uses a binary format. + """ + if payload.format.is_binary: + msg = ( + f"{path}: binary payload format {payload.format.value!r} requires the " + f"WS7 artifact resolver, unsupported in {TRACE_SCHEMA_VERSION}." + ) + raise SchemaError(msg) + return { + "content": payload.content, + "id": payload.id, + "format": _encode_enum(value=payload.format, path=f"{path}.format"), + "artifact": None, + "metadata": _encode_json(value=payload.metadata, path=f"{path}.metadata"), + } + + +def _encode_eval_result(*, value: EvalResult, path: str) -> dict[str, Any]: + """Encode an ``EvalResult``. + + Returns: + dict[str, Any]: The encoded evaluation result. + """ + return { + "outcome": _encode_enum(value=value.outcome, path=f"{path}.outcome"), + "confidence": _encode_float( + value=value.confidence, + path=f"{path}.confidence", + ), + "evidence": list(value.evidence), + "rationale": value.rationale, + "undetermined_operands": list(value.undetermined_operands), + } + + +def _encode_injection(*, record: InjectionRecord) -> dict[str, Any]: + """Encode an ``InjectionRecord``. + + Returns: + dict[str, Any]: The encoded injection record. + """ + return { + "payload_id": record.payload_id, + "surface_name": record.surface_name, + } + + +def _encode_population( + *, value: PopulationRef | None, path: str +) -> dict[str, Any] | None: + """Encode an optional ``PopulationRef``. + + Returns: + dict[str, Any] | None: The encoded reference, or ``None``. + """ + if value is None: + return None + return { + "id": value.id, + "index": value.index, + "size": value.size, + "threshold": _encode_float(value=value.threshold, path=f"{path}.threshold"), + } + + +def _encode_enum(*, value: Enum, path: str) -> str: + """Encode an enum member to its wire value. + + Returns: + str: The enum ``.value``. + + Raises: + SchemaError: If ``value`` is not an enum member. + """ + if not isinstance(value, Enum): + msg = f"{path}: expected enum, got {type(value).__name__}." + raise SchemaError(msg) + return str(value.value) + + +def _encode_harm_category(*, value: object) -> str | None: + """Encode a harm category as a passthrough string. + + Returns: + str | None: The category string, or ``None`` when unset. + """ + if value is None: + return None + return str(value) + + +def _encode_datetime(*, value: datetime | None) -> str | None: + """Encode a datetime to ISO 8601. + + Returns: + str | None: The ISO timestamp, or ``None``. + """ + if value is None: + return None + return value.isoformat() + + +def _encode_float(*, value: float, path: str) -> float: + """Validate and pass through a float within the canonical domain. + + Returns: + float: The finite float value. + + Raises: + SchemaError: If ``value`` is not a finite real number. Normalizing + non-finite floats is transport hygiene, not a canonical concern. + """ + if isinstance(value, bool) or not isinstance(value, int | float): + msg = f"{path}: expected a real number, got {type(value).__name__}." + raise SchemaError(msg) + if not math.isfinite(value): + msg = f"{path}: expected a finite number, got {value!r}." + raise SchemaError(msg) + return float(value) + + +def _encode_json(*, value: object, path: str) -> object: + """Validate that ``value`` is JSON-safe, failing closed otherwise. + + Recurses through lists and string-keyed maps of primitives. Anything + outside the domain (bytes, ``Path``, arbitrary objects, non-finite floats, + non-string map keys) raises rather than being coerced via ``repr()``. + + Returns: + Any: A JSON-safe copy of ``value``. + + Raises: + SchemaError: If ``value`` contains anything outside the JSON domain. + """ + if value is None or isinstance(value, str | bool): + return value + if isinstance(value, int): + return value + if isinstance(value, float): + return _encode_float(value=value, path=path) + if isinstance(value, Mapping): + return _encode_json_map(value=value, path=path) + if isinstance(value, list | tuple): + return [ + _encode_json(value=item, path=f"{path}[{index}]") + for index, item in enumerate(value) + ] + msg = f"{path}: value of type {type(value).__name__} is outside the JSON domain." + raise SchemaError(msg) + + +def _encode_json_map(*, value: Mapping[Any, Any], path: str) -> dict[str, Any]: + """Validate and copy a JSON-safe string-keyed map. + + Returns: + dict[str, Any]: A JSON-safe copy of the map. + + Raises: + SchemaError: If any key is not a string. + """ + encoded: dict[str, Any] = {} + for key, item in value.items(): + if not isinstance(key, str): + msg = f"{path}: map key {key!r} is not a string." + raise SchemaError(msg) + encoded[key] = _encode_json(value=item, path=f"{path}.{key}") + return encoded + + +def _decode_v1(data: Mapping[str, Any]) -> ResultRecord: + """Decode a ``rampart.trace.v1`` record. + + Returns: + ResultRecord: The decoded record. + + Raises: + SchemaError: If the record body is not a mapping. + """ + body = data.get("result") + if not isinstance(body, Mapping): + msg = f"record 'result' body must be a mapping, got {type(body).__name__}." + raise SchemaError(msg) + identity = data.get("identity") + result_index = data.get("result_index", 0) + pytest_nodeid = data.get("pytest_nodeid") + return ResultRecord( + result=_decode_result(data=body, path="result"), + identity=identity if isinstance(identity, Mapping) else None, + pytest_nodeid=pytest_nodeid if isinstance(pytest_nodeid, str) else None, + result_index=result_index if isinstance(result_index, int) else 0, + ) + + +def _decode_result(*, data: Mapping[str, Any], path: str) -> Result: + """Decode a ``Result`` body. + + Returns: + Result: The reconstructed result. + """ + return Result( + status=_decode_enum( + enum=SafetyStatus, + value=data.get("status"), + path=f"{path}.status", + ), + summary=_decode_str(value=data.get("summary"), path=f"{path}.summary"), + observability_level=_decode_enum( + enum=ObservabilityLevel, + value=data.get("observability_level"), + path=f"{path}.observability_level", + ), + turns=[ + _decode_turn(data=item, path=f"{path}.turns[{index}]") + for index, item in enumerate(_decode_list(value=data.get("turns"))) + ], + duration_seconds=_encode_float( + value=data.get("duration_seconds", 0.0), + path=f"{path}.duration_seconds", + ), + harm_category=_decode_harm_category(value=data.get("harm_category")), + strategy=_decode_str(value=data.get("strategy", ""), path=f"{path}.strategy"), + injections=[ + _decode_injection(data=item, path=f"{path}.injections[{index}]") + for index, item in enumerate(_decode_list(value=data.get("injections"))) + ], + population=_decode_population( + value=data.get("population"), + path=f"{path}.population", + ), + metadata=dict( + _decode_optional_map(value=data.get("metadata"), path=f"{path}.metadata") + ), + ) + + +def _decode_turn(*, data: object, path: str) -> Turn: + """Decode a ``Turn``. + + Returns: + Turn: The reconstructed turn. + """ + typed = _decode_map(value=data, path=path) + raw_eval = typed.get("eval_result") + eval_result = ( + None + if raw_eval is None + else _decode_eval_result(data=raw_eval, path=f"{path}.eval_result") + ) + return Turn( + request=_decode_request(data=typed.get("request"), path=f"{path}.request"), + response=_decode_response(data=typed.get("response"), path=f"{path}.response"), + eval_result=eval_result, + turn_number=_decode_int( + value=typed.get("turn_number", 0), path=f"{path}.turn_number" + ), + timestamp=_decode_datetime( + value=typed.get("timestamp"), path=f"{path}.timestamp" + ), + driver_reasoning=_decode_str( + value=typed.get("driver_reasoning", ""), + path=f"{path}.driver_reasoning", + ), + ) + + +def _decode_request(*, data: object, path: str) -> Request: + """Decode a ``Request``. + + Returns: + Request: The reconstructed request. + """ + typed = _decode_map(value=data, path=path) + raw_prompt = typed.get("prompt") + prompt = raw_prompt if isinstance(raw_prompt, str) else None + return Request( + prompt=prompt, + attachments=[ + _decode_payload(data=item, path=f"{path}.attachments[{index}]") + for index, item in enumerate(_decode_list(value=typed.get("attachments"))) + ], + ) + + +def _decode_response(*, data: object, path: str) -> Response: + """Decode a ``Response``. + + Returns: + Response: The reconstructed response. + """ + typed = _decode_map(value=data, path=path) + return Response( + text=_decode_str(value=typed.get("text", ""), path=f"{path}.text"), + tool_calls=[ + _decode_tool_call(data=item, path=f"{path}.tool_calls[{index}]") + for index, item in enumerate(_decode_list(value=typed.get("tool_calls"))) + ], + side_effects=[ + _decode_side_effect(data=item, path=f"{path}.side_effects[{index}]") + for index, item in enumerate(_decode_list(value=typed.get("side_effects"))) + ], + metadata=dict( + _decode_optional_map(value=typed.get("metadata"), path=f"{path}.metadata") + ), + ) + + +def _decode_tool_call(*, data: object, path: str) -> ToolCall: + """Decode a ``ToolCall``. + + Returns: + ToolCall: The reconstructed tool call. + """ + typed = _decode_map(value=data, path=path) + raw_result = typed.get("result") + return ToolCall( + name=_decode_str(value=typed.get("name", ""), path=f"{path}.name"), + arguments=dict( + _decode_optional_map(value=typed.get("arguments"), path=f"{path}.arguments") + ), + result=raw_result if isinstance(raw_result, str) else None, + timestamp=_decode_datetime( + value=typed.get("timestamp"), path=f"{path}.timestamp" + ), + ) + + +def _decode_side_effect(*, data: object, path: str) -> SideEffect: + """Decode a ``SideEffect``. + + Returns: + SideEffect: The reconstructed side effect. + """ + typed = _decode_map(value=data, path=path) + return SideEffect( + kind=_decode_str(value=typed.get("kind", ""), path=f"{path}.kind"), + details=dict( + _decode_optional_map(value=typed.get("details"), path=f"{path}.details") + ), + ) + + +def _decode_payload(*, data: object, path: str) -> Payload: + """Decode a ``Payload``. + + A binary payload has no artifact resolver at ``rampart.trace.v1`` and fails + closed rather than being coerced to a text payload. + + Returns: + Payload: The reconstructed payload. + + Raises: + SchemaError: If the payload declares a binary format. + """ + typed = _decode_map(value=data, path=path) + payload_format = _decode_enum( + enum=PayloadFormat, + value=typed.get("format", PayloadFormat.TEXT.value), + path=f"{path}.format", + ) + if payload_format.is_binary: + msg = ( + f"{path}: binary payload format {payload_format.value!r} requires the " + f"WS7 artifact resolver, unsupported in {TRACE_SCHEMA_VERSION}." + ) + raise SchemaError(msg) + return Payload( + content=_decode_str(value=typed.get("content", ""), path=f"{path}.content"), + id=_decode_str(value=typed.get("id", ""), path=f"{path}.id"), + format=payload_format, + artifact=None, + metadata=dict( + _decode_optional_map(value=typed.get("metadata"), path=f"{path}.metadata") + ), + ) + + +def _decode_eval_result(*, data: object, path: str) -> EvalResult: + """Decode an ``EvalResult``. + + Returns: + EvalResult: The reconstructed evaluation result. + """ + typed = _decode_map(value=data, path=path) + return EvalResult( + outcome=_decode_enum( + enum=EvalOutcome, + value=typed.get("outcome"), + path=f"{path}.outcome", + ), + confidence=_encode_float( + value=typed.get("confidence", 1.0), + path=f"{path}.confidence", + ), + evidence=_decode_str_list(value=typed.get("evidence"), path=f"{path}.evidence"), + rationale=_decode_str( + value=typed.get("rationale", ""), path=f"{path}.rationale" + ), + undetermined_operands=_decode_str_list( + value=typed.get("undetermined_operands"), + path=f"{path}.undetermined_operands", + ), + ) + + +def _decode_injection(*, data: object, path: str) -> InjectionRecord: + """Decode an ``InjectionRecord``. + + Returns: + InjectionRecord: The reconstructed injection record. + """ + typed = _decode_map(value=data, path=path) + raw_payload_id = typed.get("payload_id") + return InjectionRecord( + payload_id=raw_payload_id if isinstance(raw_payload_id, str) else None, + surface_name=_decode_str( + value=typed.get("surface_name", ""), + path=f"{path}.surface_name", + ), + ) + + +def _decode_population(*, value: object, path: str) -> PopulationRef | None: + """Decode an optional ``PopulationRef``. + + Returns: + PopulationRef | None: The reconstructed reference, or ``None``. + """ + if value is None: + return None + typed = _decode_map(value=value, path=path) + return PopulationRef( + id=_decode_str(value=typed.get("id", ""), path=f"{path}.id"), + index=_decode_int(value=typed.get("index", 0), path=f"{path}.index"), + size=_decode_int(value=typed.get("size", 0), path=f"{path}.size"), + threshold=_encode_float( + value=typed.get("threshold", 0.0), path=f"{path}.threshold" + ), + ) + + +def _decode_enum(*, enum: type[EnumT], value: object, path: str) -> EnumT: + """Decode an enum member from its wire value, failing closed on unknown. + + Returns: + EnumT: The enum member. + + Raises: + SchemaError: If ``value`` is not a member of ``enum``. + """ + try: + return enum(value) + except ValueError as exc: + msg = f"{path}: {value!r} is not a valid {enum.__name__}." + raise SchemaError(msg) from exc + + +def _decode_harm_category(*, value: object) -> str | None: + """Decode a harm category as a passthrough string. + + Returns: + str | None: The category string, or ``None``. + """ + if value is None: + return None + return str(value) + + +def _decode_datetime(*, value: object, path: str) -> datetime | None: + """Decode an ISO 8601 timestamp. + + Returns: + datetime | None: The parsed datetime, or ``None``. + + Raises: + SchemaError: If ``value`` is neither ``None`` nor a valid ISO string. + """ + if value is None: + return None + if not isinstance(value, str): + msg = f"{path}: expected an ISO timestamp string, got {type(value).__name__}." + raise SchemaError(msg) + try: + return datetime.fromisoformat(value) + except ValueError as exc: + msg = f"{path}: {value!r} is not a valid ISO 8601 timestamp." + raise SchemaError(msg) from exc + + +def _decode_str(*, value: object, path: str) -> str: + """Decode a required string field. + + Returns: + str: The string value. + + Raises: + SchemaError: If ``value`` is not a string. + """ + if not isinstance(value, str): + msg = f"{path}: expected a string, got {type(value).__name__}." + raise SchemaError(msg) + return value + + +def _decode_int(*, value: object, path: str) -> int: + """Decode a required integer field. + + Returns: + int: The integer value. + + Raises: + SchemaError: If ``value`` is not an integer. + """ + if isinstance(value, bool) or not isinstance(value, int): + msg = f"{path}: expected an integer, got {type(value).__name__}." + raise SchemaError(msg) + return value + + +def _decode_list(*, value: object) -> list[Any]: + """Coerce an optional wire list to a list. + + Returns: + list[Any]: The list, or an empty list when absent. + """ + return list(value) if isinstance(value, list) else [] + + +def _decode_str_list(*, value: object, path: str) -> list[str]: + """Decode a list of strings. + + Returns: + list[str]: The decoded strings. + """ + return [ + _decode_str(value=item, path=f"{path}[{index}]") + for index, item in enumerate(_decode_list(value=value)) + ] + + +def _decode_map(*, value: object, path: str) -> Mapping[str, Any]: + """Decode a required mapping field. + + Returns: + Mapping[str, Any]: The mapping value. + + Raises: + SchemaError: If ``value`` is not a mapping. + """ + if not isinstance(value, Mapping): + msg = f"{path}: expected a mapping, got {type(value).__name__}." + raise SchemaError(msg) + return value + + +def _decode_optional_map(*, value: object, path: str) -> Mapping[str, Any]: + """Decode an optional mapping field, defaulting to empty when absent. + + Returns: + Mapping[str, Any]: The mapping value, or an empty mapping when ``None``. + + Raises: + SchemaError: If ``value`` is present but not a mapping. + """ + if value is None: + return {} + return _decode_map(value=value, path=path) + + +_DECODERS: dict[str, Callable[[Mapping[str, Any]], ResultRecord]] = { + TRACE_SCHEMA_VERSION: _decode_v1, +} diff --git a/tests/unit/core/test_serialization.py b/tests/unit/core/test_serialization.py new file mode 100644 index 00000000..33c20f6a --- /dev/null +++ b/tests/unit/core/test_serialization.py @@ -0,0 +1,314 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Unit tests for the canonical trace/result serializer.""" + +from __future__ import annotations + +import math +from dataclasses import fields +from datetime import UTC, datetime + +import pytest + +from rampart.core.result import ( + InjectionRecord, + PopulationRef, + Result, + SafetyStatus, +) +from rampart.core.serialization import ( + TRACE_SCHEMA_VERSION, + ResultRecord, + SchemaError, + UnsupportedSchemaVersionError, + deserialize_result, + serialize_result, +) +from rampart.core.types import ( + EvalOutcome, + EvalResult, + ObservabilityLevel, + Payload, + PayloadFormat, + Request, + Response, + SideEffect, + ToolCall, + Turn, +) + +_TIMESTAMP = datetime(2026, 1, 2, 3, 4, 5, tzinfo=UTC) + + +def _make_eval_result() -> EvalResult: + return EvalResult( + outcome=EvalOutcome.DETECTED, + confidence=0.75, + evidence=["saw the thing", "and another"], + rationale="because reasons", + undetermined_operands=["left operand undetermined"], + ) + + +def _make_turn() -> Turn: + request = Request( + prompt="do the thing", + attachments=[ + Payload( + content="poisoned doc text", + id="payload-1", + format=PayloadFormat.MARKDOWN, + metadata={"persona": "attacker"}, + ), + ], + ) + response = Response( + text="agent said this", + tool_calls=[ + ToolCall( + name="send_email", + arguments={"to": "a@b.com", "nested": {"count": 2}}, + result="ok", + timestamp=_TIMESTAMP, + ), + ], + side_effects=[SideEffect(kind="http_request", details={"url": "http://x"})], + metadata={"latency_ms": 12}, + ) + return Turn( + request=request, + response=response, + eval_result=_make_eval_result(), + turn_number=3, + timestamp=_TIMESTAMP, + driver_reasoning="escalate", + ) + + +def _make_full_result(*, metadata: dict | None = None) -> Result: + return Result( + status=SafetyStatus.UNSAFE, + summary="a violation was detected", + observability_level=ObservabilityLevel.TOOL_AND_SIDE_EFFECTS, + turns=[_make_turn()], + duration_seconds=1.5, + harm_category="prompt_injection", + strategy="xpia", + injections=[InjectionRecord(payload_id="payload-1", surface_name="SharePoint")], + population=PopulationRef(id="pop-1", index=0, size=5, threshold=0.8), + metadata={"note": "user data", "nested": {"k": [1, 2]}} + if metadata is None + else metadata, + ) + + +def _minimal_record_dict() -> dict: + return { + "version": TRACE_SCHEMA_VERSION, + "result": { + "status": "safe", + "summary": "clean", + "observability_level": "response_only", + }, + } + + +class TestRoundTrip: + def test_full_result_round_trips_to_equal_value(self) -> None: + original = _make_full_result() + encoded = ResultRecord(result=original).to_dict() + + decoded = deserialize_result(data=encoded).result + + assert decoded == original + + def test_version_is_stamped_on_the_record(self) -> None: + encoded = ResultRecord(result=_make_full_result()).to_dict() + + assert encoded["version"] == TRACE_SCHEMA_VERSION + assert ResultRecord.VERSION == "rampart.trace.v1" + + def test_serialize_result_builds_identity_collar(self) -> None: + encoded = serialize_result( + result=_make_full_result(), + identity="auto:mod::test", + origin="derived", + case_id="case-0", + pytest_nodeid="tests/test_x.py::test_x", + result_index=2, + ) + + assert encoded["identity"] == { + "value": "auto:mod::test", + "origin": "derived", + "case_id": "case-0", + } + assert encoded["pytest_nodeid"] == "tests/test_x.py::test_x" + assert encoded["result_index"] == 2 + + def test_serialize_result_omits_identity_when_unset(self) -> None: + encoded = serialize_result(result=_make_full_result()) + + assert encoded["identity"] is None + + def test_nested_values_survive_the_round_trip(self) -> None: + decoded = deserialize_result( + data=ResultRecord(result=_make_full_result()).to_dict() + ).result + + turn = decoded.turns[0] + assert turn.request.attachments[0].format is PayloadFormat.MARKDOWN + assert turn.response.tool_calls[0].arguments == { + "to": "a@b.com", + "nested": {"count": 2}, + } + assert turn.response.tool_calls[0].timestamp == _TIMESTAMP + assert turn.response.side_effects[0].kind == "http_request" + assert turn.eval_result is not None + assert turn.eval_result.outcome is EvalOutcome.DETECTED + assert decoded.injections[0].surface_name == "SharePoint" + assert decoded.population == PopulationRef( + id="pop-1", index=0, size=5, threshold=0.8 + ) + + +class TestFieldExhaustiveness: + def test_every_field_of_every_type_is_serialized(self) -> None: + body = ResultRecord(result=_make_full_result()).to_dict()["result"] + turn = body["turns"][0] + + cases = [ + (Result, body), + (Turn, turn), + (Request, turn["request"]), + (Payload, turn["request"]["attachments"][0]), + (Response, turn["response"]), + (ToolCall, turn["response"]["tool_calls"][0]), + (SideEffect, turn["response"]["side_effects"][0]), + (EvalResult, turn["eval_result"]), + (InjectionRecord, body["injections"][0]), + (PopulationRef, body["population"]), + ] + + for dataclass_type, encoded in cases: + expected = {field.name for field in fields(dataclass_type)} + assert expected == set(encoded), dataclass_type.__name__ + + +class TestVersionDispatch: + def test_unknown_major_fails_closed(self) -> None: + data = {"version": "rampart.trace.v2", "result": {}} + + with pytest.raises(UnsupportedSchemaVersionError, match="v2"): + deserialize_result(data=data) + + def test_missing_version_fails_closed(self) -> None: + with pytest.raises(UnsupportedSchemaVersionError): + deserialize_result(data={"result": {}}) + + def test_non_mapping_record_fails_closed(self) -> None: + with pytest.raises(SchemaError, match="mapping"): + deserialize_result(data=[1, 2, 3]) + + +class TestMigrationTolerance: + def test_unknown_extra_fields_decode(self) -> None: + encoded = ResultRecord(result=_make_full_result()).to_dict() + encoded["future_collar"] = {"anything": True} + encoded["result"]["future_intrinsic"] = 42 + + decoded = deserialize_result(data=encoded).result + + assert decoded.status is SafetyStatus.UNSAFE + + def test_missing_optional_fields_use_defaults(self) -> None: + decoded = deserialize_result(data=_minimal_record_dict()).result + + assert decoded.status is SafetyStatus.SAFE + assert decoded.turns == [] + assert decoded.duration_seconds == pytest.approx(0.0) + assert decoded.harm_category is None + assert decoded.injections == [] + assert decoded.population is None + assert decoded.metadata == {} + + +class TestValueDomain: + def test_reserved_metadata_keys_are_stripped(self) -> None: + result = _make_full_result( + metadata={"_pytest_nodeid": "x::y", "note": "keep me"}, + ) + + encoded = ResultRecord(result=result).to_dict() + + assert encoded["result"]["metadata"] == {"note": "keep me"} + + def test_harm_category_is_passed_through_as_string(self) -> None: + result = _make_full_result() + result.harm_category = "custom_product_risk" + + encoded = ResultRecord(result=result).to_dict() + decoded = deserialize_result(data=encoded).result + + assert encoded["result"]["harm_category"] == "custom_product_risk" + assert decoded.harm_category == "custom_product_risk" + + def test_non_finite_float_fails_closed(self) -> None: + result = _make_full_result() + result.duration_seconds = math.inf + + with pytest.raises(SchemaError, match="duration_seconds"): + ResultRecord(result=result).to_dict() + + def test_non_json_metadata_fails_closed(self) -> None: + result = _make_full_result(metadata={"blob": object()}) + + with pytest.raises(SchemaError, match="metadata"): + ResultRecord(result=result).to_dict() + + def test_bad_enum_value_fails_closed_on_decode(self) -> None: + data = _minimal_record_dict() + data["result"]["status"] = "not_a_status" + + with pytest.raises(SchemaError, match="status"): + deserialize_result(data=data) + + +class TestBinaryPayloadFailsClosed: + def test_encoding_a_binary_payload_fails_closed(self, tmp_path) -> None: + artifact = tmp_path / "doc.pdf" + artifact.write_bytes(b"%PDF-1.4 fake") + result = _make_full_result() + result.turns = [ + Turn( + request=Request( + attachments=[ + Payload( + content="binary doc", + format=PayloadFormat.PDF, + artifact=artifact, + ), + ], + ), + response=Response(text="ok"), + ), + ] + + with pytest.raises(SchemaError, match="binary payload"): + ResultRecord(result=result).to_dict() + + def test_decoding_a_binary_payload_fails_closed(self) -> None: + data = _minimal_record_dict() + data["result"]["turns"] = [ + { + "request": { + "prompt": None, + "attachments": [{"content": "x", "id": "p", "format": "pdf"}], + }, + "response": {"text": "ok"}, + }, + ] + + with pytest.raises(SchemaError, match="binary payload"): + deserialize_result(data=data) From e7da361690db5b1718adc0cf0cc35c6911be5c15 Mon Sep 17 00:00:00 2001 From: Nina Chikanov Date: Tue, 8 Sep 2026 11:38:28 -0700 Subject: [PATCH 2/9] [DOCS]: Align trace schema policy with current implementation Describe only the canonical serializer behavior present on this branch, express later migration and consumer work as policy constraints, generalize future additions outside the reserved collar fields, and remove the ship gate. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/concepts/trace-schema.md | 83 ++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 41 deletions(-) diff --git a/docs/concepts/trace-schema.md b/docs/concepts/trace-schema.md index c999f257..5005b9d5 100644 --- a/docs/concepts/trace-schema.md +++ b/docs/concepts/trace-schema.md @@ -1,18 +1,20 @@ # Trace/Result Schema & Migration Policy -RAMPART serializes every safety `Result` through a single canonical, versioned -schema (`rampart.core.serialization`). The same schema backs xdist transport, -failure attachments, reporting projections, and — in future work — replay and -golden traces. This page is the written, reviewed migration policy that gates -any durable trace artifact. +`rampart.core.serialization` defines RAMPART's canonical, versioned +`Result`-record format. The current implementation provides the neutral +`ResultRecord.to_dict()` / `ResultRecord.from_dict()` round-trip and the +`serialize_result()` / `deserialize_result()` convenience functions. Existing +xdist and reporting consumers are not yet wired to this module. + +This page defines how the schema may evolve as consumers adopt it. ## Versioning - Every serialized record carries one root `version` field. The current schema is **`rampart.trace.v1`**. -- The record version is **independent** of the xdist transport envelope version - (`rampart.xdist.v2`). The two axes move separately; an `xdist.v2` envelope may - carry a `trace.v1` record. +- The record version is **independent** of transport or projection versions, + including the existing xdist envelope version (`rampart.xdist.v2`). Each + version describes its own layer and may evolve separately. - There is a **single root version** — nested types (`Turn`, `Payload`, `EvalResult`, …) do not carry their own versions. @@ -35,62 +37,61 @@ any durable trace artifact. record is never best-effort parsed across a major boundary. - Forward compatibility is **additive-only within a major**. A newer major read by an older framework fails closed by design. -- Any derived JSON Schema is therefore **open** (`additionalProperties: true`). +- Schema descriptions and validators derived from this format must remain open + to unknown properties within a major version. ## Enum posture - The closed enums — `SafetyStatus`, `EvalOutcome`, `ObservabilityLevel`, and - `PayloadFormat` — **fail closed** on an unknown value. A durable safety - artifact must never silently misread one; there is no warn-and-degrade path. + `PayloadFormat` — **fail closed** on an unknown value. A serialized safety + result must never silently misread one; there is no warn-and-degrade path. - `HarmCategory` is the sole exception: it travels as a **passthrough string** and is never coerced, so a new harm label from a future producer round-trips unchanged on an older reader. -## Binary / opaque payloads +## Value domain -- A non-text payload persists as a content-addressed - `{sha256, media_type, bundle_path}` descriptor in `artifacts[]`, never inline. -- A decoder that meets a binary reference with **no artifact resolver wired - fails closed** — it never coerces the payload to `PayloadFormat.TEXT`. -- The descriptor shape is frozen now (populating `artifacts[]` later is - additive-optional); the resolver and companion bundles are built by the replay - work, not by this gate. At `rampart.trace.v1` there is no resolver, so binary - payloads fail closed on both encode and decode. +- Free-form mappings must already contain JSON-safe values. The canonical codec + does not coerce unsupported objects with `str()` or `repr()`. +- Numeric values must be finite. Transport-specific normalization is outside + the canonical schema. +- `rampart.trace.v1` does not define a durable representation for binary or + opaque payload artifacts. Encoding or decoding one fails closed rather than + coercing it to text. +- Transport bookkeeping keys are removed from top-level `Result.metadata`; + nested user mappings are preserved. ## Migration mechanics -- Each major bump ships an **adjacent upcaster** (`vN-1 → vN`) plus an explicit - **migration API/CLI**. -- Writers always emit the **latest** major. -- Reads **never rewrite** persisted files in place. Backward-*reading* an old - major is not the same as migrating an artifact — migration is an explicit, - opt-in step, never a silent rewrite. +Only `rampart.trace.v1` exists today. If a later structural change introduces a +new major: + +- writers emit the latest supported major; +- support for an older major uses an explicit adjacent upcaster + (`vN-1 → vN`); +- migrating persisted data is an explicit operation; reading never rewrites an + artifact in place; and +- encountering an unsupported major fails closed. ## Reserved additive fields (named now, populated later) -To make the additive path concrete, these slots are reserved by name so future -work drops in without a bump, as **record-level wire-only collar slots**: +These record-level wire-only collar slots are reserved by name so they can be +added without a major bump: `manifest_snapshot`, `evaluation_fingerprint`, `replay_provenance`, `population_ref`, plus `artifacts` / `target` / `provenance`. A field that is truly *intrinsic to a result* instead lands as an additive-optional field on `Result`, inside the referenced `result` body. Either way each is additive-optional; none is populated at v1. -Later trigger-/persistence-phase provenance fields are additive-optional and -**must not** force a hard migration or major bump. +Other future fields follow the same general rule: optional additions with a +defined absence behavior do not require a major bump; structural changes do. ## Support window -After the **first durable-trace release** (the first release that writes -persisted golden traces/evidence, on by default), RAMPART supports reading `vN` -and `vN-1` for **two subsequent framework releases** (one deprecation cycle), -keyed on **release, not time**, with a changelog and migration note on any bump. -Before that release there is no durable-read obligation. - -## Ship gate - -Ship **no durable artifact — golden traces above all — until the schema has a -per-result `version` field and this policy is in effect.** +Starting with the first release that writes durable trace records by default, +RAMPART supports reading `vN` and `vN-1` for **two subsequent framework +releases** (one deprecation cycle). The window is keyed on releases, not time. +Any major bump includes a changelog entry and migration note. ```mermaid flowchart TD @@ -99,7 +100,7 @@ flowchart TD q1 -- yes --> q2{"optional with a
well-defined default?"} q2 -- no --> struct q2 -- yes --> add["additive-optional"] - add --> nobump["NO bump
(new optional fields, later provenance)
old readers ignore unknown keys"] + add --> nobump["NO bump
(new optional fields)
old readers ignore unknown keys"] struct --> bump["bump major vN → vN+1
+ changelog + migration note"] bump --> reader["readers: fail closed on
unknown major"] ``` From 1d8a596b8163501cd55e545098b315bfe902a9a6 Mon Sep 17 00:00:00 2001 From: Nina Chikanov Date: Tue, 8 Sep 2026 12:05:46 -0700 Subject: [PATCH 3/9] [MAINT]: Address canonical serializer review feedback Remove planning references, place constant comments before declarations, drop the unused identity origin field, and make unsupported binary payload messages independent of planned work. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- rampart/core/serialization.py | 52 +++++++++------------------ tests/unit/core/test_serialization.py | 2 -- 2 files changed, 16 insertions(+), 38 deletions(-) diff --git a/rampart/core/serialization.py b/rampart/core/serialization.py index 2ee1c579..cf809512 100644 --- a/rampart/core/serialization.py +++ b/rampart/core/serialization.py @@ -4,21 +4,15 @@ """Canonical, versioned trace/result serialization for RAMPART. This module owns the *single* full-fidelity ``Result`` <-> ``dict`` round-trip -for the whole framework (design gate WS0-05, Decision D6). xdist transport, -failure attachments, reporting projections, and future replay all serialize -through here rather than maintaining parallel serializers. +for the whole framework. The canonical layer defines the supported *value domain* and nothing else. It -does not apply transport hygiene — no ANSI stripping, no float normalization, -no ``repr()``/``str()`` fallback, and no size capping. Those concerns wrap the -canonical output at the transport boundary (xdist). When a value falls outside -the canonical domain the codec fails closed with a field path rather than -coercing, so a durable trace never silently loses fidelity. +does not apply transport hygiene. When a value falls outside the canonical +domain the codec fails closed with a field path rather than coercing it. Every serialized record carries a single root ``version`` field (:data:`TRACE_SCHEMA_VERSION`). Decoding dispatches on that version and fails -closed on an unknown major. The record version is independent of the xdist -transport envelope version; the two axes move separately. +closed on an unknown major. """ from __future__ import annotations @@ -54,9 +48,12 @@ EnumT = TypeVar("EnumT", bound=Enum) +# Single root schema version stamped on every serialized record. TRACE_SCHEMA_VERSION = "rampart.trace.v1" -"""Single root schema version stamped on every serialized record.""" +# Top-level ``Result.metadata`` keys owned by the xdist transport. These +# scheduling and bookkeeping values are stripped from the canonical body; +# nested user maps are never touched. RESERVED_METADATA_KEYS: frozenset[str] = frozenset( { "_pytest_nodeid", @@ -69,12 +66,6 @@ "_rampart_worker_artifact_path", } ) -"""Top-level ``Result.metadata`` keys owned by the xdist transport. - -These are scheduling/bookkeeping breadcrumbs the transport stamps for its own -reconciliation. They are stripped from the canonical body so a durable trace -carries only intrinsic result data; nested user maps are never touched. -""" class SchemaError(Exception): @@ -103,8 +94,8 @@ class ResultRecord: Args: result (Result): The single-run verdict being serialized. - identity (dict[str, Any] | None): Stable test identity descriptor - (WS0-06). ``None`` until identity is wired at the producer. + identity (dict[str, Any] | None): Stable test identity descriptor. + ``None`` until identity is wired at the producer. pytest_nodeid (str | None): The pytest node id the result came from. result_index (int): Ordinal of this result within its test node. """ @@ -161,7 +152,6 @@ def serialize_result( *, result: Result, identity: str | None = None, - origin: str | None = None, case_id: str | None = None, pytest_nodeid: str | None = None, result_index: int = 0, @@ -170,8 +160,7 @@ def serialize_result( Args: result (Result): The verdict to serialize. - identity (str | None): Stable identity value (WS0-06), if computed. - origin (str | None): How the identity was derived (marker vs. derived). + identity (str | None): Stable identity value, if computed. case_id (str | None): Parametrization case id, travelling beside identity. pytest_nodeid (str | None): The pytest node id the result came from. result_index (int): Ordinal of this result within its test node. @@ -180,10 +169,9 @@ def serialize_result( dict[str, Any]: The canonical record dict, ready for any durable sink. """ identity_descriptor: dict[str, Any] | None = None - if identity is not None or origin is not None or case_id is not None: + if identity is not None or case_id is not None: identity_descriptor = { "value": identity, - "origin": origin, "case_id": case_id, } record = ResultRecord( @@ -331,11 +319,6 @@ def _encode_side_effect(*, effect: SideEffect, path: str) -> dict[str, Any]: def _encode_payload(*, payload: Payload, path: str) -> dict[str, Any]: """Encode a ``Payload``. - Binary payloads are persisted as content-addressed artifact descriptors by - WS7 rather than inline; that resolver does not exist at - ``rampart.trace.v1``, so a binary payload fails closed here instead of - inlining a machine-local path. - Returns: dict[str, Any]: The encoded payload. @@ -344,8 +327,8 @@ def _encode_payload(*, payload: Payload, path: str) -> dict[str, Any]: """ if payload.format.is_binary: msg = ( - f"{path}: binary payload format {payload.format.value!r} requires the " - f"WS7 artifact resolver, unsupported in {TRACE_SCHEMA_VERSION}." + f"{path}: binary payload format {payload.format.value!r} is unsupported " + f"in {TRACE_SCHEMA_VERSION}." ) raise SchemaError(msg) return { @@ -684,9 +667,6 @@ def _decode_side_effect(*, data: object, path: str) -> SideEffect: def _decode_payload(*, data: object, path: str) -> Payload: """Decode a ``Payload``. - A binary payload has no artifact resolver at ``rampart.trace.v1`` and fails - closed rather than being coerced to a text payload. - Returns: Payload: The reconstructed payload. @@ -701,8 +681,8 @@ def _decode_payload(*, data: object, path: str) -> Payload: ) if payload_format.is_binary: msg = ( - f"{path}: binary payload format {payload_format.value!r} requires the " - f"WS7 artifact resolver, unsupported in {TRACE_SCHEMA_VERSION}." + f"{path}: binary payload format {payload_format.value!r} is unsupported " + f"in {TRACE_SCHEMA_VERSION}." ) raise SchemaError(msg) return Payload( diff --git a/tests/unit/core/test_serialization.py b/tests/unit/core/test_serialization.py index 33c20f6a..92bd493e 100644 --- a/tests/unit/core/test_serialization.py +++ b/tests/unit/core/test_serialization.py @@ -133,7 +133,6 @@ def test_serialize_result_builds_identity_collar(self) -> None: encoded = serialize_result( result=_make_full_result(), identity="auto:mod::test", - origin="derived", case_id="case-0", pytest_nodeid="tests/test_x.py::test_x", result_index=2, @@ -141,7 +140,6 @@ def test_serialize_result_builds_identity_collar(self) -> None: assert encoded["identity"] == { "value": "auto:mod::test", - "origin": "derived", "case_id": "case-0", } assert encoded["pytest_nodeid"] == "tests/test_x.py::test_x" From 7000184f256b7bb1200745cac81316d434647632 Mon Sep 17 00:00:00 2001 From: Nina Chikanov Date: Tue, 8 Sep 2026 12:21:55 -0700 Subject: [PATCH 4/9] [MAINT]: Defer stable identity schema fields Remove identity and case_id until their producer is implemented, keep the existing pytest attribution fields optional, and make the reserved metadata constant private. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- rampart/core/serialization.py | 62 +++++++++++++-------------- tests/unit/core/test_serialization.py | 25 +++++++---- 2 files changed, 47 insertions(+), 40 deletions(-) diff --git a/rampart/core/serialization.py b/rampart/core/serialization.py index cf809512..3778450b 100644 --- a/rampart/core/serialization.py +++ b/rampart/core/serialization.py @@ -54,7 +54,7 @@ # Top-level ``Result.metadata`` keys owned by the xdist transport. These # scheduling and bookkeeping values are stripped from the canonical body; # nested user maps are never touched. -RESERVED_METADATA_KEYS: frozenset[str] = frozenset( +_RESERVED_METADATA_KEYS: frozenset[str] = frozenset( { "_pytest_nodeid", "_pytest_test_name", @@ -89,23 +89,20 @@ class ResultRecord: This is the public surface: :meth:`to_dict` / :meth:`from_dict` are the one round-trip every durable consumer uses. The ``result`` is referenced, not - copied. The collar fields (``identity``, ``pytest_nodeid``, ``result_index``) - are wire-only provenance stamped once at the producing boundary. + copied. The ``pytest_nodeid`` and ``result_index`` collar fields are + wire-only attribution stamped once at the producing boundary. Args: result (Result): The single-run verdict being serialized. - identity (dict[str, Any] | None): Stable test identity descriptor. - ``None`` until identity is wired at the producer. pytest_nodeid (str | None): The pytest node id the result came from. - result_index (int): Ordinal of this result within its test node. + result_index (int | None): Ordinal of this result within its test node. """ VERSION: ClassVar[str] = TRACE_SCHEMA_VERSION result: Result - identity: dict[str, Any] | None = None pytest_nodeid: str | None = None - result_index: int = 0 + result_index: int | None = None def to_dict(self) -> dict[str, Any]: """Encode the record into a canonical, JSON-safe dict. @@ -115,13 +112,15 @@ def to_dict(self) -> dict[str, Any]: and wire-only collar. Fails closed via :class:`SchemaError` on any value outside the canonical domain. """ - return { + encoded = { "version": self.VERSION, "result": _encode_result(result=self.result, path="result"), - "identity": _encode_json(value=self.identity, path="identity"), - "pytest_nodeid": self.pytest_nodeid, - "result_index": self.result_index, } + if self.pytest_nodeid is not None: + encoded["pytest_nodeid"] = self.pytest_nodeid + if self.result_index is not None: + encoded["result_index"] = self.result_index + return encoded @classmethod def from_dict(cls, data: object) -> ResultRecord: @@ -151,32 +150,21 @@ def from_dict(cls, data: object) -> ResultRecord: def serialize_result( *, result: Result, - identity: str | None = None, - case_id: str | None = None, pytest_nodeid: str | None = None, - result_index: int = 0, + result_index: int | None = None, ) -> dict[str, Any]: """Serialize a result to the canonical, versioned dict. Args: result (Result): The verdict to serialize. - identity (str | None): Stable identity value, if computed. - case_id (str | None): Parametrization case id, travelling beside identity. pytest_nodeid (str | None): The pytest node id the result came from. - result_index (int): Ordinal of this result within its test node. + result_index (int | None): Ordinal of this result within its test node. Returns: dict[str, Any]: The canonical record dict, ready for any durable sink. """ - identity_descriptor: dict[str, Any] | None = None - if identity is not None or case_id is not None: - identity_descriptor = { - "value": identity, - "case_id": case_id, - } record = ResultRecord( result=result, - identity=identity_descriptor, pytest_nodeid=pytest_nodeid, result_index=result_index, ) @@ -204,7 +192,7 @@ def _encode_result(*, result: Result, path: str) -> dict[str, Any]: metadata = { key: value for key, value in result.metadata.items() - if key not in RESERVED_METADATA_KEYS + if key not in _RESERVED_METADATA_KEYS } return { "status": _encode_enum(value=result.status, path=f"{path}.status"), @@ -505,14 +493,26 @@ def _decode_v1(data: Mapping[str, Any]) -> ResultRecord: if not isinstance(body, Mapping): msg = f"record 'result' body must be a mapping, got {type(body).__name__}." raise SchemaError(msg) - identity = data.get("identity") - result_index = data.get("result_index", 0) + result_index = data.get("result_index") pytest_nodeid = data.get("pytest_nodeid") + if pytest_nodeid is not None and not isinstance(pytest_nodeid, str): + msg = ( + "record 'pytest_nodeid' must be a string or null, " + f"got {type(pytest_nodeid).__name__}." + ) + raise SchemaError(msg) + if result_index is not None and ( + isinstance(result_index, bool) or not isinstance(result_index, int) + ): + msg = ( + "record 'result_index' must be an integer or null, " + f"got {type(result_index).__name__}." + ) + raise SchemaError(msg) return ResultRecord( result=_decode_result(data=body, path="result"), - identity=identity if isinstance(identity, Mapping) else None, - pytest_nodeid=pytest_nodeid if isinstance(pytest_nodeid, str) else None, - result_index=result_index if isinstance(result_index, int) else 0, + pytest_nodeid=pytest_nodeid, + result_index=result_index, ) diff --git a/tests/unit/core/test_serialization.py b/tests/unit/core/test_serialization.py index 92bd493e..ce142c12 100644 --- a/tests/unit/core/test_serialization.py +++ b/tests/unit/core/test_serialization.py @@ -129,26 +129,33 @@ def test_version_is_stamped_on_the_record(self) -> None: assert encoded["version"] == TRACE_SCHEMA_VERSION assert ResultRecord.VERSION == "rampart.trace.v1" - def test_serialize_result_builds_identity_collar(self) -> None: + def test_serialize_result_builds_attribution_collar(self) -> None: encoded = serialize_result( result=_make_full_result(), - identity="auto:mod::test", - case_id="case-0", pytest_nodeid="tests/test_x.py::test_x", result_index=2, ) - assert encoded["identity"] == { - "value": "auto:mod::test", - "case_id": "case-0", - } assert encoded["pytest_nodeid"] == "tests/test_x.py::test_x" assert encoded["result_index"] == 2 - def test_serialize_result_omits_identity_when_unset(self) -> None: + def test_serialize_result_omits_attribution_when_unset(self) -> None: encoded = serialize_result(result=_make_full_result()) - assert encoded["identity"] is None + assert "pytest_nodeid" not in encoded + assert "result_index" not in encoded + + def test_attribution_collar_round_trips(self) -> None: + encoded = serialize_result( + result=_make_full_result(), + pytest_nodeid="tests/test_x.py::test_x", + result_index=2, + ) + + decoded = deserialize_result(data=encoded) + + assert decoded.pytest_nodeid == "tests/test_x.py::test_x" + assert decoded.result_index == 2 def test_nested_values_survive_the_round_trip(self) -> None: decoded = deserialize_result( From 3f108a0b2c22c2371d136b28fef930adfb764d30 Mon Sep 17 00:00:00 2001 From: Nina Chikanov Date: Tue, 8 Sep 2026 13:05:02 -0700 Subject: [PATCH 5/9] [FIX]: Reject malformed canonical records Fail closed on malformed collection fields, incomplete population references, and non-string harm categories. Validate result indices before serialization so the encoder cannot emit boolean indices rejected by the decoder. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- rampart/core/serialization.py | 123 +++++++++++++++++++------- tests/unit/core/test_serialization.py | 32 +++++++ 2 files changed, 124 insertions(+), 31 deletions(-) diff --git a/rampart/core/serialization.py b/rampart/core/serialization.py index 3778450b..46414ce3 100644 --- a/rampart/core/serialization.py +++ b/rampart/core/serialization.py @@ -104,6 +104,14 @@ class ResultRecord: pytest_nodeid: str | None = None result_index: int | None = None + def __post_init__(self) -> None: + """Validate record attribution fields. + + Raises: + SchemaError: If ``result_index`` is not an integer or ``None``. + """ + _validate_result_index(value=self.result_index) + def to_dict(self) -> dict[str, Any]: """Encode the record into a canonical, JSON-safe dict. @@ -209,7 +217,10 @@ def _encode_result(*, result: Result, path: str) -> dict[str, Any]: value=result.duration_seconds, path=f"{path}.duration_seconds", ), - "harm_category": _encode_harm_category(value=result.harm_category), + "harm_category": _encode_harm_category( + value=result.harm_category, + path=f"{path}.harm_category", + ), "strategy": result.strategy, "injections": [ _encode_injection(record=record) for record in result.injections @@ -391,15 +402,21 @@ def _encode_enum(*, value: Enum, path: str) -> str: return str(value.value) -def _encode_harm_category(*, value: object) -> str | None: +def _encode_harm_category(*, value: object, path: str) -> str | None: """Encode a harm category as a passthrough string. Returns: str | None: The category string, or ``None`` when unset. + + Raises: + SchemaError: If ``value`` is not a string or ``None``. """ if value is None: return None - return str(value) + if not isinstance(value, str): + msg = f"{path}: expected a string or null, got {type(value).__name__}." + raise SchemaError(msg) + return value def _encode_datetime(*, value: datetime | None) -> str | None: @@ -413,7 +430,7 @@ def _encode_datetime(*, value: datetime | None) -> str | None: return value.isoformat() -def _encode_float(*, value: float, path: str) -> float: +def _encode_float(*, value: object, path: str) -> float: """Validate and pass through a float within the canonical domain. Returns: @@ -501,14 +518,6 @@ def _decode_v1(data: Mapping[str, Any]) -> ResultRecord: f"got {type(pytest_nodeid).__name__}." ) raise SchemaError(msg) - if result_index is not None and ( - isinstance(result_index, bool) or not isinstance(result_index, int) - ): - msg = ( - "record 'result_index' must be an integer or null, " - f"got {type(result_index).__name__}." - ) - raise SchemaError(msg) return ResultRecord( result=_decode_result(data=body, path="result"), pytest_nodeid=pytest_nodeid, @@ -536,17 +545,24 @@ def _decode_result(*, data: Mapping[str, Any], path: str) -> Result: ), turns=[ _decode_turn(data=item, path=f"{path}.turns[{index}]") - for index, item in enumerate(_decode_list(value=data.get("turns"))) + for index, item in enumerate( + _decode_list(value=data.get("turns"), path=f"{path}.turns") + ) ], duration_seconds=_encode_float( value=data.get("duration_seconds", 0.0), path=f"{path}.duration_seconds", ), - harm_category=_decode_harm_category(value=data.get("harm_category")), + harm_category=_decode_harm_category( + value=data.get("harm_category"), + path=f"{path}.harm_category", + ), strategy=_decode_str(value=data.get("strategy", ""), path=f"{path}.strategy"), injections=[ _decode_injection(data=item, path=f"{path}.injections[{index}]") - for index, item in enumerate(_decode_list(value=data.get("injections"))) + for index, item in enumerate( + _decode_list(value=data.get("injections"), path=f"{path}.injections") + ) ], population=_decode_population( value=data.get("population"), @@ -601,7 +617,12 @@ def _decode_request(*, data: object, path: str) -> Request: prompt=prompt, attachments=[ _decode_payload(data=item, path=f"{path}.attachments[{index}]") - for index, item in enumerate(_decode_list(value=typed.get("attachments"))) + for index, item in enumerate( + _decode_list( + value=typed.get("attachments"), + path=f"{path}.attachments", + ) + ) ], ) @@ -617,11 +638,21 @@ def _decode_response(*, data: object, path: str) -> Response: text=_decode_str(value=typed.get("text", ""), path=f"{path}.text"), tool_calls=[ _decode_tool_call(data=item, path=f"{path}.tool_calls[{index}]") - for index, item in enumerate(_decode_list(value=typed.get("tool_calls"))) + for index, item in enumerate( + _decode_list( + value=typed.get("tool_calls"), + path=f"{path}.tool_calls", + ) + ) ], side_effects=[ _decode_side_effect(data=item, path=f"{path}.side_effects[{index}]") - for index, item in enumerate(_decode_list(value=typed.get("side_effects"))) + for index, item in enumerate( + _decode_list( + value=typed.get("side_effects"), + path=f"{path}.side_effects", + ) + ) ], metadata=dict( _decode_optional_map(value=typed.get("metadata"), path=f"{path}.metadata") @@ -751,12 +782,10 @@ def _decode_population(*, value: object, path: str) -> PopulationRef | None: return None typed = _decode_map(value=value, path=path) return PopulationRef( - id=_decode_str(value=typed.get("id", ""), path=f"{path}.id"), - index=_decode_int(value=typed.get("index", 0), path=f"{path}.index"), - size=_decode_int(value=typed.get("size", 0), path=f"{path}.size"), - threshold=_encode_float( - value=typed.get("threshold", 0.0), path=f"{path}.threshold" - ), + id=_decode_str(value=typed.get("id"), path=f"{path}.id"), + index=_decode_int(value=typed.get("index"), path=f"{path}.index"), + size=_decode_int(value=typed.get("size"), path=f"{path}.size"), + threshold=_encode_float(value=typed.get("threshold"), path=f"{path}.threshold"), ) @@ -776,15 +805,21 @@ def _decode_enum(*, enum: type[EnumT], value: object, path: str) -> EnumT: raise SchemaError(msg) from exc -def _decode_harm_category(*, value: object) -> str | None: +def _decode_harm_category(*, value: object, path: str) -> str | None: """Decode a harm category as a passthrough string. Returns: str | None: The category string, or ``None``. + + Raises: + SchemaError: If ``value`` is not a string or ``None``. """ if value is None: return None - return str(value) + if not isinstance(value, str): + msg = f"{path}: expected a string or null, got {type(value).__name__}." + raise SchemaError(msg) + return value def _decode_datetime(*, value: object, path: str) -> datetime | None: @@ -838,13 +873,21 @@ def _decode_int(*, value: object, path: str) -> int: return value -def _decode_list(*, value: object) -> list[Any]: - """Coerce an optional wire list to a list. +def _decode_list(*, value: object, path: str) -> list[Any]: + """Decode an optional wire list. Returns: - list[Any]: The list, or an empty list when absent. + list[Any]: The list, or an empty list when absent or ``None``. + + Raises: + SchemaError: If ``value`` is present but not a list. """ - return list(value) if isinstance(value, list) else [] + if value is None: + return [] + if not isinstance(value, list): + msg = f"{path}: expected a list or null, got {type(value).__name__}." + raise SchemaError(msg) + return value def _decode_str_list(*, value: object, path: str) -> list[str]: @@ -855,10 +898,28 @@ def _decode_str_list(*, value: object, path: str) -> list[str]: """ return [ _decode_str(value=item, path=f"{path}[{index}]") - for index, item in enumerate(_decode_list(value=value)) + for index, item in enumerate(_decode_list(value=value, path=path)) ] +def _validate_result_index(*, value: object) -> int | None: + """Validate an optional result index. + + Returns: + int | None: The validated index. + + Raises: + SchemaError: If ``value`` is not an integer or ``None``. + """ + if value is not None and (isinstance(value, bool) or not isinstance(value, int)): + msg = ( + "record 'result_index' must be an integer or null, " + f"got {type(value).__name__}." + ) + raise SchemaError(msg) + return value + + def _decode_map(*, value: object, path: str) -> Mapping[str, Any]: """Decode a required mapping field. diff --git a/tests/unit/core/test_serialization.py b/tests/unit/core/test_serialization.py index ce142c12..09cad12b 100644 --- a/tests/unit/core/test_serialization.py +++ b/tests/unit/core/test_serialization.py @@ -238,6 +238,20 @@ def test_missing_optional_fields_use_defaults(self) -> None: assert decoded.population is None assert decoded.metadata == {} + def test_malformed_present_list_fails_closed(self) -> None: + data = _minimal_record_dict() + data["result"]["turns"] = "not-a-list" + + with pytest.raises(SchemaError, match=r"result\.turns"): + deserialize_result(data=data) + + def test_incomplete_population_reference_fails_closed(self) -> None: + data = _minimal_record_dict() + data["result"]["population"] = {} + + with pytest.raises(SchemaError, match=r"result\.population\.id"): + deserialize_result(data=data) + class TestValueDomain: def test_reserved_metadata_keys_are_stripped(self) -> None: @@ -279,6 +293,24 @@ def test_bad_enum_value_fails_closed_on_decode(self) -> None: with pytest.raises(SchemaError, match="status"): deserialize_result(data=data) + def test_non_string_harm_category_fails_closed_on_encode(self) -> None: + result = _make_full_result() + result.__dict__["harm_category"] = 42 + + with pytest.raises(SchemaError, match="harm_category"): + ResultRecord(result=result).to_dict() + + def test_non_string_harm_category_fails_closed_on_decode(self) -> None: + data = _minimal_record_dict() + data["result"]["harm_category"] = {"category": "custom"} + + with pytest.raises(SchemaError, match="harm_category"): + deserialize_result(data=data) + + def test_boolean_result_index_fails_before_encoding(self) -> None: + with pytest.raises(SchemaError, match="result_index"): + serialize_result(result=_make_full_result(), result_index=True) + class TestBinaryPayloadFailsClosed: def test_encoding_a_binary_payload_fails_closed(self, tmp_path) -> None: From 02c6b6e802b3f75c10fb9306126f240cde8fbcb7 Mon Sep 17 00:00:00 2001 From: Nina Chikanov Date: Thu, 10 Sep 2026 16:22:19 -0700 Subject: [PATCH 6/9] [FEAT]: Add adapter-backed Result serialization and schema Replace handwritten nested codecs with a cached TypeAdapter on Result, keep ResultRecord as the versioned envelope, and generate the open schema with a CI drift check. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 3 + docs/concepts/trace-schema.md | 45 +- pyproject.toml | 4 + rampart/core/_schema.py | 123 ++++ rampart/core/errors.py | 8 + rampart/core/result.py | 116 +++- rampart/core/serialization.py | 929 +++----------------------- rampart/core/types.py | 86 ++- schemas/trace.v1.schema.json | 490 ++++++++++++++ scripts/generate_trace_schema.py | 40 ++ tests/unit/core/test_serialization.py | 369 +++++++++- uv.lock | 166 +++++ 12 files changed, 1513 insertions(+), 866 deletions(-) create mode 100644 rampart/core/_schema.py create mode 100644 schemas/trace.v1.schema.json create mode 100644 scripts/generate_trace_schema.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ed1ddd9..f4f733b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,6 +50,9 @@ jobs: - name: ty run: uv run ty check + - name: Trace schema drift + run: uv run python scripts/generate_trace_schema.py --check + test: name: Test (Python ${{ matrix.python-version }}) needs: lint diff --git a/docs/concepts/trace-schema.md b/docs/concepts/trace-schema.md index 5005b9d5..d9dfc16c 100644 --- a/docs/concepts/trace-schema.md +++ b/docs/concepts/trace-schema.md @@ -1,13 +1,39 @@ # Trace/Result Schema & Migration Policy `rampart.core.serialization` defines RAMPART's canonical, versioned -`Result`-record format. The current implementation provides the neutral -`ResultRecord.to_dict()` / `ResultRecord.from_dict()` round-trip and the -`serialize_result()` / `deserialize_result()` convenience functions. Existing +`Result`-record format. `ResultRecord.to_dict()` / `ResultRecord.from_dict()` own +the versioned envelope and optional `pytest_nodeid` / `result_index` attribution. +`serialize_result()` / `deserialize_result()` are convenience functions. Existing xdist and reporting consumers are not yet wired to this module. This page defines how the schema may evolve as consumers adopt it. +## Serialization and schema generation + +`Result.to_dict()` / `Result.from_dict()` own the **unversioned body**, using one +cached Pydantic `TypeAdapter` over the existing standard dataclasses. Body dicts +are fragments, not standalone durable records: persist a `ResultRecord` to +include the version. `ResultRecord` references the live result; serialization +does not mutate it. + +The adapter validates nested fields without string, boolean, or integer +coercion. Dictionary input is checked for JSON-only values before strict +JSON-mode validation reconstructs the dataclasses. Missing fields use their +declared defaults; explicit `null` is accepted only on nullable fields. Payload +IDs must be recorded, not generated during deserialization. These boundary +rules do not replace the normal dataclass constructors used during execution. + +`ResultRecord.json_schema()` returns the adapter-derived body schema plus the +versioned envelope. Small schema customizations describe the trace-only payload +restrictions and the request invariant (a prompt or at least one attachment). +`JsonSchemaValue` is the return type, not a separate model or validator. +The open Draft 2020-12 contract is committed at `schemas/trace.v1.schema.json`. + +Regenerate it with `uv run python scripts/generate_trace_schema.py`. +CI runs the same command with `--check` to detect drift. Changes to generated +output still require a compatibility review; generation does not decide whether +a version bump is needed. + ## Versioning - Every serialized record carries one root `version` field. The current schema @@ -51,15 +77,20 @@ This page defines how the schema may evolve as consumers adopt it. ## Value domain -- Free-form mappings must already contain JSON-safe values. The canonical codec - does not coerce unsupported objects with `str()` or `repr()`. +- Free-form mappings must already contain JSON-safe values: null, strings, + booleans, finite numbers, lists, and string-keyed mappings. Tuples, bytes, + cycles, and opaque objects are rejected rather than coerced. - Numeric values must be finite. Transport-specific normalization is outside the canonical schema. +- Timestamps retain Python's ISO 8601 representation, including naive datetimes + and UTC offsets. The schema describes strings rather than RFC 3339 + `date-time`, which would exclude some supported Python datetimes. - `rampart.trace.v1` does not define a durable representation for binary or opaque payload artifacts. Encoding or decoding one fails closed rather than coercing it to text. -- Transport bookkeeping keys are removed from top-level `Result.metadata`; - nested user mappings are preserved. +- `ResultRecord` removes transport bookkeeping keys, including + `_rampart_source_worker`, from top-level `Result.metadata`; body serialization + does not. Nested user mappings are preserved. ## Migration mechanics diff --git a/pyproject.toml b/pyproject.toml index c8faf744..b563e629 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,7 @@ dev = [ "flake8>=7.3.0", "hatch-vcs>=0.5.0", "hatchling>=1.30.1", + "jsonschema>=4.26.0", "pre-commit>=4.5.1", "pytest-cov>=6.1.0", "pytest-xdist[psutil]>=3.8.0", @@ -128,6 +129,9 @@ external = ["RMP001", "RMP002"] "scripts/hatch_build.py" = [ "implicit-namespace-package", # Top-level build hook ] +"scripts/generate_trace_schema.py" = [ + "implicit-namespace-package", # Standalone schema generation command +] "tests/integration/conftest.py" = [ "unused-noqa", # Ruff 0.16.4 does not recognize pytest-fixture-autouse. ] diff --git a/rampart/core/_schema.py b/rampart/core/_schema.py new file mode 100644 index 00000000..b0105459 --- /dev/null +++ b/rampart/core/_schema.py @@ -0,0 +1,123 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Shared value-domain rules for dataclass trace adapters.""" + +from __future__ import annotations + +import math +from collections.abc import Mapping +from datetime import datetime +from typing import ( + TYPE_CHECKING, + Annotated, + Any, + TypeAlias, +) + +from pydantic import ( + BeforeValidator, + PlainSerializer, + WithJsonSchema, +) + +if TYPE_CHECKING: + from pydantic import ValidationError, ValidationInfo + + +def json_value(value: object) -> object: + """Check and copy JSON values without lossy coercion. + + Returns: + object: JSON primitives, lists, and string-keyed dictionaries. + + Raises: + ValueError: If a value is non-finite, cyclic, or outside the JSON domain. + """ + return _json_value(value=value, path="$", active=set()) + + +def _json_value(*, value: object, path: str, active: set[int]) -> object: + """Recursively validate the JSON domain, retaining the offending path. + + Returns: + object: A JSON-safe copy. + + Raises: + ValueError: If the value cannot be represented faithfully in JSON. + """ + if value is None or isinstance(value, str | bool | int): + return value + if isinstance(value, float) and math.isfinite(value): + return value + if not isinstance(value, Mapping | list): + msg = f"{path}: {type(value).__name__} is outside the finite JSON domain" + raise ValueError(msg) # ruff: ignore[type-check-without-type-error] Pydantic wraps ValueError. + if id(value) in active: + msg = f"{path}: cyclic JSON value" + raise ValueError(msg) + active.add(id(value)) + try: + if isinstance(value, list): + return [ + _json_value(value=item, path=f"{path}[{index}]", active=active) + for index, item in enumerate(value) + ] + result: dict[str, object] = {} + for key, item in value.items(): + if not isinstance(key, str): + msg = f"{path}: JSON object keys must be strings" + raise ValueError(msg) # ruff: ignore[type-check-without-type-error] Pydantic wraps ValueError. + result[key] = _json_value(value=item, path=f"{path}.{key}", active=active) + return result + finally: + active.remove(id(value)) + + +# Standard dataclass construction stays permissive; adapters validate these maps. +JsonMapping: TypeAlias = Annotated[dict[str, Any], BeforeValidator(json_value)] + + +# Pydantic supplies value/info positionally to BeforeValidator callbacks. +def _iso_datetime(value: object, info: ValidationInfo) -> object: + """Retain Python ISO datetime support, including naive and subminute offsets. + + Returns: + object: Parsed datetime strings, or the unchanged value for validation. + + Raises: + ValueError: If the string is not an ISO datetime. + """ + if info.mode == "json" and isinstance(value, str): + try: + return datetime.fromisoformat(value) + except ValueError as exc: + msg = "expected an ISO 8601 datetime string" + raise ValueError(msg) from exc + return value + + +IsoDatetime: TypeAlias = Annotated[ + datetime, + BeforeValidator(_iso_datetime), + PlainSerializer(datetime.isoformat), + # Python ISO datetimes include values outside RFC 3339's date-time format. + WithJsonSchema( + {"type": "string", "description": "ISO 8601 datetime; UTC offset is optional."} + ), +] + + +def validation_message(*, error: ValidationError, path: str) -> str: + """Render Pydantic errors without including producer data in the message. + + Returns: + str: Field paths and validation reasons. + """ + messages: list[str] = [] + for detail in error.errors(include_url=False, include_input=False): + location = path + for part in detail["loc"]: + location += f"[{part}]" if isinstance(part, int) else f".{part}" + messages.append(f"{location}: {detail['msg']}") + return "; ".join(messages) diff --git a/rampart/core/errors.py b/rampart/core/errors.py index a5d8655a..f585a917 100644 --- a/rampart/core/errors.py +++ b/rampart/core/errors.py @@ -47,3 +47,11 @@ class EvaluatorError(InfrastructureError): ``InfrastructureError`` base class) and produces a Result with SafetyStatus.ERROR. """ + + +class SchemaError(Exception): + """A value cannot be represented by the canonical trace schema.""" + + +class UnsupportedSchemaVersionError(SchemaError): + """A record's version has no registered decoder.""" diff --git a/rampart/core/result.py b/rampart/core/result.py index 2683a2e3..48fffe83 100644 --- a/rampart/core/result.py +++ b/rampart/core/result.py @@ -11,15 +11,35 @@ from __future__ import annotations +import json from dataclasses import dataclass, field from enum import Enum, StrEnum +from functools import cache from typing import TYPE_CHECKING, Any +from pydantic import ( + ConfigDict, + TypeAdapter, + ValidationError, + with_config, +) +from pydantic.json_schema import GenerateJsonSchema, JsonSchemaValue +from pydantic_core import PydanticSerializationError, core_schema + from rampart.common.text import safe_str, safe_str_list +from rampart.core._schema import ( + JsonMapping, + json_value, + validation_message, +) +from rampart.core.errors import SchemaError from rampart.core.types import ( EvalOutcome, EvalResult, ObservabilityLevel, + Payload, + PayloadFormat, + Request, Turn, ) @@ -110,6 +130,9 @@ class PopulationRef: threshold: float +@with_config( + ConfigDict(strict=True, revalidate_instances="always", allow_inf_nan=False) +) @dataclass(kw_only=True) class Result: """The outcome of a safety test. @@ -157,7 +180,7 @@ class Result: default_factory=list[InjectionRecord], ) population: PopulationRef | None = None - metadata: dict[str, Any] = field(default_factory=dict[str, Any]) + metadata: JsonMapping = field(default_factory=dict[str, Any]) @property def safe(self) -> bool: @@ -194,6 +217,97 @@ def __repr__(self) -> str: f"summary={self.summary!r})" ) + def to_dict(self) -> dict[str, Any]: + """Serialize the unversioned body, not a standalone durable record. + + Returns: + dict[str, Any]: A JSON-safe body for a ResultRecord envelope. + + Raises: + SchemaError: If the result is outside the trace value domain. + """ + adapter = _result_adapter() + try: + validated = adapter.validate_python(self, context={"trace": True}) + return adapter.dump_python(validated, mode="json", warnings="error") + except ValidationError as exc: + raise SchemaError(validation_message(error=exc, path="result")) from exc + except (PydanticSerializationError, RecursionError) as exc: + msg = f"result: cannot serialize canonical body ({type(exc).__name__})" + raise SchemaError(msg) from exc + + @classmethod + def from_dict(cls, data: object) -> Result: + """Validate and reconstruct an unversioned canonical body. + + Args: + data (object): A JSON-compatible body from a versioned record. + + Returns: + Result: The reconstructed result. + + Raises: + SchemaError: If the body is malformed or outside the trace domain. + """ + try: + # JSON-mode strict validation accepts wire enums/dates, not coercions. + encoded = json.dumps(json_value(data), allow_nan=False) + return _result_adapter().validate_json(encoded, context={"trace": True}) + except ValidationError as exc: + raise SchemaError(validation_message(error=exc, path="result")) from exc + except (ValueError, RecursionError) as exc: + msg = f"result: {exc}" + raise SchemaError(msg) from exc + + @classmethod + def json_schema(cls) -> JsonSchemaValue: + """Generate the body contract from the configured dataclass adapter. + + Returns: + JsonSchemaValue: The JSON Schema for the unversioned body. + """ + return _result_adapter().json_schema(schema_generator=_ResultJsonSchema) + + +@cache +def _result_adapter() -> TypeAdapter[Result]: + """Build the recursive adapter once, on first serialization use. + + Returns: + TypeAdapter[Result]: The cached adapter. + """ + return TypeAdapter(Result) + + +class _ResultJsonSchema(GenerateJsonSchema): + """Describe trace-only restrictions alongside the dataclass field schemas.""" + + def dataclass_schema(self, schema: core_schema.DataclassSchema) -> JsonSchemaValue: + """Add trace policies that do not restrict live dataclass construction. + + Returns: + JsonSchemaValue: An open object schema matching the trace validators. + """ + result = super().dataclass_schema(schema) + result["additionalProperties"] = True + if schema["cls"] is Payload: + result["properties"]["format"] = { + "type": "string", + "enum": [value.value for value in PayloadFormat if value.is_text], + "default": PayloadFormat.TEXT.value, + } + result["properties"]["artifact"] = {"type": "null", "default": None} + result["required"] = [*result["required"], "id"] + elif schema["cls"] is Request: + result["anyOf"] = [ + {"required": ["prompt"], "properties": {"prompt": {"type": "string"}}}, + { + "required": ["attachments"], + "properties": {"attachments": {"type": "array", "minItems": 1}}, + }, + ] + return result + @dataclass(kw_only=True) class PopulationResult: diff --git a/rampart/core/serialization.py b/rampart/core/serialization.py index 46414ce3..b9c8a521 100644 --- a/rampart/core/serialization.py +++ b/rampart/core/serialization.py @@ -1,64 +1,37 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Canonical, versioned trace/result serialization for RAMPART. - -This module owns the *single* full-fidelity ``Result`` <-> ``dict`` round-trip -for the whole framework. - -The canonical layer defines the supported *value domain* and nothing else. It -does not apply transport hygiene. When a value falls outside the canonical -domain the codec fails closed with a field path rather than coercing it. - -Every serialized record carries a single root ``version`` field -(:data:`TRACE_SCHEMA_VERSION`). Decoding dispatches on that version and fails -closed on an unknown major. -""" +"""Versioned ResultRecord envelopes around the canonical Result body codec.""" from __future__ import annotations -import math from collections.abc import Mapping -from dataclasses import dataclass -from datetime import datetime -from enum import Enum -from typing import TYPE_CHECKING, Any, ClassVar, TypeVar - -from rampart.core.result import ( - InjectionRecord, - PopulationRef, - Result, - SafetyStatus, -) -from rampart.core.types import ( - EvalOutcome, - EvalResult, - ObservabilityLevel, - Payload, - PayloadFormat, - Request, - Response, - SideEffect, - ToolCall, - Turn, +from dataclasses import dataclass, replace +from typing import ( + TYPE_CHECKING, + Any, + ClassVar, ) +from rampart.core.errors import SchemaError, UnsupportedSchemaVersionError +from rampart.core.result import Result + if TYPE_CHECKING: from collections.abc import Callable -EnumT = TypeVar("EnumT", bound=Enum) + from pydantic.json_schema import JsonSchemaValue + # Single root schema version stamped on every serialized record. TRACE_SCHEMA_VERSION = "rampart.trace.v1" -# Top-level ``Result.metadata`` keys owned by the xdist transport. These -# scheduling and bookkeeping values are stripped from the canonical body; -# nested user maps are never touched. -_RESERVED_METADATA_KEYS: frozenset[str] = frozenset( +# Strip only top-level transport bookkeeping, never matching nested user keys. +_RESERVED_METADATA_KEYS = frozenset( { "_pytest_nodeid", "_pytest_test_name", "_rampart_result_index", + "_rampart_source_worker", "_rampart_transport_truncated", "_rampart_original_size_bytes", "_rampart_limit_bytes", @@ -68,34 +41,14 @@ ) -class SchemaError(Exception): - """Raised when a value falls outside the canonical trace schema domain. - - The message carries the offending field path so producers can locate the - out-of-domain value instead of the codec silently coercing it. - """ - - -class UnsupportedSchemaVersionError(SchemaError): - """Raised when decoding a record whose ``version`` has no decoder. - - Readers fail closed on an unknown major rather than guessing at a shape. - """ - - @dataclass(frozen=True, kw_only=True) class ResultRecord: - """The canonical, versioned envelope around a single ``Result``. - - This is the public surface: :meth:`to_dict` / :meth:`from_dict` are the one - round-trip every durable consumer uses. The ``result`` is referenced, not - copied. The ``pytest_nodeid`` and ``result_index`` collar fields are - wire-only attribution stamped once at the producing boundary. + """A versioned envelope referencing one Result and optional attribution. Args: - result (Result): The single-run verdict being serialized. - pytest_nodeid (str | None): The pytest node id the result came from. - result_index (int | None): Ordinal of this result within its test node. + result (Result): The referenced result; its fields are not copied. + pytest_nodeid (str | None): Producing test location, when recorded. + result_index (int | None): Within-node ordinal, when recorded. """ VERSION: ClassVar[str] = TRACE_SCHEMA_VERSION @@ -105,25 +58,37 @@ class ResultRecord: result_index: int | None = None def __post_init__(self) -> None: - """Validate record attribution fields. + """Validate attribution without copying or revalidating the live result. Raises: - SchemaError: If ``result_index`` is not an integer or ``None``. + SchemaError: If attribution has invalid types. """ - _validate_result_index(value=self.result_index) + if self.pytest_nodeid is not None and not isinstance(self.pytest_nodeid, str): + msg = "record.pytest_nodeid: expected a string or null" + raise SchemaError(msg) + if self.result_index is not None and type(self.result_index) is not int: + msg = "record.result_index: expected an integer or null" + raise SchemaError(msg) def to_dict(self) -> dict[str, Any]: - """Encode the record into a canonical, JSON-safe dict. + """Serialize the envelope using the single Result body codec. Returns: - dict[str, Any]: The versioned record with the encoded result body - and wire-only collar. Fails closed via :class:`SchemaError` on - any value outside the canonical domain. + dict[str, Any]: A versioned, JSON-safe record. + + Raises: + SchemaError: If the referenced result is outside the trace domain. """ - encoded = { - "version": self.VERSION, - "result": _encode_result(result=self.result, path="result"), + if not isinstance(self.result.metadata, Mapping): + msg = "result.metadata: expected a mapping" + raise SchemaError(msg) + metadata = { + key: value + for key, value in self.result.metadata.items() + if key not in _RESERVED_METADATA_KEYS } + body = replace(self.result, metadata=metadata).to_dict() + encoded: dict[str, Any] = {"version": self.VERSION, "result": body} if self.pytest_nodeid is not None: encoded["pytest_nodeid"] = self.pytest_nodeid if self.result_index is not None: @@ -132,20 +97,17 @@ def to_dict(self) -> dict[str, Any]: @classmethod def from_dict(cls, data: object) -> ResultRecord: - """Decode a canonical dict back into a record, dispatching on version. - - Args: - data (object): A previously encoded record mapping. + """Dispatch a record to its version-specific decoder. Returns: - ResultRecord: The decoded record. + ResultRecord: The reconstructed body and attribution. Raises: - SchemaError: If ``data`` is not a mapping. - UnsupportedSchemaVersionError: If the record version has no decoder. + SchemaError: If the record is not a mapping. + UnsupportedSchemaVersionError: If the version is unsupported. """ if not isinstance(data, Mapping): - msg = f"Expected mapping for record, got {type(data).__name__}." + msg = "record: expected a mapping" raise SchemaError(msg) version = data.get("version") decoder = _DECODERS.get(version) if isinstance(version, str) else None @@ -154,6 +116,31 @@ def from_dict(cls, data: object) -> ResultRecord: raise UnsupportedSchemaVersionError(msg) return decoder(data) + @classmethod + def json_schema(cls) -> JsonSchemaValue: + """Compose the versioned contract with the adapter-generated body schema. + + Returns: + JsonSchemaValue: An open Draft 2020-12 schema. + """ + body = Result.json_schema() + definitions = body.pop("$defs", {}) + return { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": f"urn:rampart:trace:{TRACE_SCHEMA_VERSION.rsplit('.', 1)[-1]}", + "$defs": definitions, + "title": "ResultRecord", + "type": "object", + "additionalProperties": True, + "required": ["version", "result"], + "properties": { + "version": {"type": "string", "const": TRACE_SCHEMA_VERSION}, + "result": body, + "pytest_nodeid": {"type": ["string", "null"]}, + "result_index": {"type": ["integer", "null"]}, + }, + } + def serialize_result( *, @@ -161,794 +148,40 @@ def serialize_result( pytest_nodeid: str | None = None, result_index: int | None = None, ) -> dict[str, Any]: - """Serialize a result to the canonical, versioned dict. - - Args: - result (Result): The verdict to serialize. - pytest_nodeid (str | None): The pytest node id the result came from. - result_index (int | None): Ordinal of this result within its test node. + """Serialize a result with its optional attribution. Returns: - dict[str, Any]: The canonical record dict, ready for any durable sink. + dict[str, Any]: The canonical versioned record. """ - record = ResultRecord( + return ResultRecord( result=result, pytest_nodeid=pytest_nodeid, result_index=result_index, - ) - return record.to_dict() + ).to_dict() def deserialize_result(*, data: object) -> ResultRecord: - """Deserialize a canonical record dict back into a :class:`ResultRecord`. - - Args: - data (object): A previously encoded record mapping. + """Deserialize a canonical record. Returns: - ResultRecord: The decoded record. + ResultRecord: The result and its attribution. """ return ResultRecord.from_dict(data) -def _encode_result(*, result: Result, path: str) -> dict[str, Any]: - """Encode a ``Result`` body into canonical primitives. - - Returns: - dict[str, Any]: The encoded result with every field represented. - """ - metadata = { - key: value - for key, value in result.metadata.items() - if key not in _RESERVED_METADATA_KEYS - } - return { - "status": _encode_enum(value=result.status, path=f"{path}.status"), - "summary": result.summary, - "observability_level": _encode_enum( - value=result.observability_level, - path=f"{path}.observability_level", - ), - "turns": [ - _encode_turn(turn=turn, path=f"{path}.turns[{index}]") - for index, turn in enumerate(result.turns) - ], - "duration_seconds": _encode_float( - value=result.duration_seconds, - path=f"{path}.duration_seconds", - ), - "harm_category": _encode_harm_category( - value=result.harm_category, - path=f"{path}.harm_category", - ), - "strategy": result.strategy, - "injections": [ - _encode_injection(record=record) for record in result.injections - ], - "population": _encode_population( - value=result.population, - path=f"{path}.population", - ), - "metadata": _encode_json(value=metadata, path=f"{path}.metadata"), - } - - -def _encode_turn(*, turn: Turn, path: str) -> dict[str, Any]: - """Encode a ``Turn``. - - Returns: - dict[str, Any]: The encoded turn. - """ - eval_result = ( - None - if turn.eval_result is None - else _encode_eval_result(value=turn.eval_result, path=f"{path}.eval_result") - ) - return { - "request": _encode_request(request=turn.request, path=f"{path}.request"), - "response": _encode_response(response=turn.response, path=f"{path}.response"), - "eval_result": eval_result, - "turn_number": turn.turn_number, - "timestamp": _encode_datetime(value=turn.timestamp), - "driver_reasoning": turn.driver_reasoning, - } - - -def _encode_request(*, request: Request, path: str) -> dict[str, Any]: - """Encode a ``Request``. - - Returns: - dict[str, Any]: The encoded request. - """ - return { - "prompt": request.prompt, - "attachments": [ - _encode_payload(payload=payload, path=f"{path}.attachments[{index}]") - for index, payload in enumerate(request.attachments) - ], - } - - -def _encode_response(*, response: Response, path: str) -> dict[str, Any]: - """Encode a ``Response``. - - Returns: - dict[str, Any]: The encoded response. - """ - return { - "text": response.text, - "tool_calls": [ - _encode_tool_call(call=call, path=f"{path}.tool_calls[{index}]") - for index, call in enumerate(response.tool_calls) - ], - "side_effects": [ - _encode_side_effect(effect=effect, path=f"{path}.side_effects[{index}]") - for index, effect in enumerate(response.side_effects) - ], - "metadata": _encode_json(value=response.metadata, path=f"{path}.metadata"), - } - - -def _encode_tool_call(*, call: ToolCall, path: str) -> dict[str, Any]: - """Encode a ``ToolCall``. - - Returns: - dict[str, Any]: The encoded tool call. - """ - return { - "name": call.name, - "arguments": _encode_json(value=call.arguments, path=f"{path}.arguments"), - "result": call.result, - "timestamp": _encode_datetime(value=call.timestamp), - } - - -def _encode_side_effect(*, effect: SideEffect, path: str) -> dict[str, Any]: - """Encode a ``SideEffect``. - - Returns: - dict[str, Any]: The encoded side effect. - """ - return { - "kind": effect.kind, - "details": _encode_json(value=effect.details, path=f"{path}.details"), - } - - -def _encode_payload(*, payload: Payload, path: str) -> dict[str, Any]: - """Encode a ``Payload``. - - Returns: - dict[str, Any]: The encoded payload. - - Raises: - SchemaError: If the payload uses a binary format. - """ - if payload.format.is_binary: - msg = ( - f"{path}: binary payload format {payload.format.value!r} is unsupported " - f"in {TRACE_SCHEMA_VERSION}." - ) - raise SchemaError(msg) - return { - "content": payload.content, - "id": payload.id, - "format": _encode_enum(value=payload.format, path=f"{path}.format"), - "artifact": None, - "metadata": _encode_json(value=payload.metadata, path=f"{path}.metadata"), - } - - -def _encode_eval_result(*, value: EvalResult, path: str) -> dict[str, Any]: - """Encode an ``EvalResult``. - - Returns: - dict[str, Any]: The encoded evaluation result. - """ - return { - "outcome": _encode_enum(value=value.outcome, path=f"{path}.outcome"), - "confidence": _encode_float( - value=value.confidence, - path=f"{path}.confidence", - ), - "evidence": list(value.evidence), - "rationale": value.rationale, - "undetermined_operands": list(value.undetermined_operands), - } - - -def _encode_injection(*, record: InjectionRecord) -> dict[str, Any]: - """Encode an ``InjectionRecord``. - - Returns: - dict[str, Any]: The encoded injection record. - """ - return { - "payload_id": record.payload_id, - "surface_name": record.surface_name, - } - - -def _encode_population( - *, value: PopulationRef | None, path: str -) -> dict[str, Any] | None: - """Encode an optional ``PopulationRef``. - - Returns: - dict[str, Any] | None: The encoded reference, or ``None``. - """ - if value is None: - return None - return { - "id": value.id, - "index": value.index, - "size": value.size, - "threshold": _encode_float(value=value.threshold, path=f"{path}.threshold"), - } - - -def _encode_enum(*, value: Enum, path: str) -> str: - """Encode an enum member to its wire value. - - Returns: - str: The enum ``.value``. - - Raises: - SchemaError: If ``value`` is not an enum member. - """ - if not isinstance(value, Enum): - msg = f"{path}: expected enum, got {type(value).__name__}." - raise SchemaError(msg) - return str(value.value) - - -def _encode_harm_category(*, value: object, path: str) -> str | None: - """Encode a harm category as a passthrough string. - - Returns: - str | None: The category string, or ``None`` when unset. - - Raises: - SchemaError: If ``value`` is not a string or ``None``. - """ - if value is None: - return None - if not isinstance(value, str): - msg = f"{path}: expected a string or null, got {type(value).__name__}." - raise SchemaError(msg) - return value - - -def _encode_datetime(*, value: datetime | None) -> str | None: - """Encode a datetime to ISO 8601. - - Returns: - str | None: The ISO timestamp, or ``None``. - """ - if value is None: - return None - return value.isoformat() - - -def _encode_float(*, value: object, path: str) -> float: - """Validate and pass through a float within the canonical domain. - - Returns: - float: The finite float value. - - Raises: - SchemaError: If ``value`` is not a finite real number. Normalizing - non-finite floats is transport hygiene, not a canonical concern. - """ - if isinstance(value, bool) or not isinstance(value, int | float): - msg = f"{path}: expected a real number, got {type(value).__name__}." - raise SchemaError(msg) - if not math.isfinite(value): - msg = f"{path}: expected a finite number, got {value!r}." - raise SchemaError(msg) - return float(value) - - -def _encode_json(*, value: object, path: str) -> object: - """Validate that ``value`` is JSON-safe, failing closed otherwise. - - Recurses through lists and string-keyed maps of primitives. Anything - outside the domain (bytes, ``Path``, arbitrary objects, non-finite floats, - non-string map keys) raises rather than being coerced via ``repr()``. - - Returns: - Any: A JSON-safe copy of ``value``. - - Raises: - SchemaError: If ``value`` contains anything outside the JSON domain. - """ - if value is None or isinstance(value, str | bool): - return value - if isinstance(value, int): - return value - if isinstance(value, float): - return _encode_float(value=value, path=path) - if isinstance(value, Mapping): - return _encode_json_map(value=value, path=path) - if isinstance(value, list | tuple): - return [ - _encode_json(value=item, path=f"{path}[{index}]") - for index, item in enumerate(value) - ] - msg = f"{path}: value of type {type(value).__name__} is outside the JSON domain." - raise SchemaError(msg) - - -def _encode_json_map(*, value: Mapping[Any, Any], path: str) -> dict[str, Any]: - """Validate and copy a JSON-safe string-keyed map. - - Returns: - dict[str, Any]: A JSON-safe copy of the map. - - Raises: - SchemaError: If any key is not a string. - """ - encoded: dict[str, Any] = {} - for key, item in value.items(): - if not isinstance(key, str): - msg = f"{path}: map key {key!r} is not a string." - raise SchemaError(msg) - encoded[key] = _encode_json(value=item, path=f"{path}.{key}") - return encoded - - def _decode_v1(data: Mapping[str, Any]) -> ResultRecord: - """Decode a ``rampart.trace.v1`` record. + """Reconstruct a v1 envelope through the current body codec. Returns: - ResultRecord: The decoded record. - - Raises: - SchemaError: If the record body is not a mapping. + ResultRecord: The reconstructed record. """ - body = data.get("result") - if not isinstance(body, Mapping): - msg = f"record 'result' body must be a mapping, got {type(body).__name__}." - raise SchemaError(msg) - result_index = data.get("result_index") - pytest_nodeid = data.get("pytest_nodeid") - if pytest_nodeid is not None and not isinstance(pytest_nodeid, str): - msg = ( - "record 'pytest_nodeid' must be a string or null, " - f"got {type(pytest_nodeid).__name__}." - ) - raise SchemaError(msg) return ResultRecord( - result=_decode_result(data=body, path="result"), - pytest_nodeid=pytest_nodeid, - result_index=result_index, - ) - - -def _decode_result(*, data: Mapping[str, Any], path: str) -> Result: - """Decode a ``Result`` body. - - Returns: - Result: The reconstructed result. - """ - return Result( - status=_decode_enum( - enum=SafetyStatus, - value=data.get("status"), - path=f"{path}.status", - ), - summary=_decode_str(value=data.get("summary"), path=f"{path}.summary"), - observability_level=_decode_enum( - enum=ObservabilityLevel, - value=data.get("observability_level"), - path=f"{path}.observability_level", - ), - turns=[ - _decode_turn(data=item, path=f"{path}.turns[{index}]") - for index, item in enumerate( - _decode_list(value=data.get("turns"), path=f"{path}.turns") - ) - ], - duration_seconds=_encode_float( - value=data.get("duration_seconds", 0.0), - path=f"{path}.duration_seconds", - ), - harm_category=_decode_harm_category( - value=data.get("harm_category"), - path=f"{path}.harm_category", - ), - strategy=_decode_str(value=data.get("strategy", ""), path=f"{path}.strategy"), - injections=[ - _decode_injection(data=item, path=f"{path}.injections[{index}]") - for index, item in enumerate( - _decode_list(value=data.get("injections"), path=f"{path}.injections") - ) - ], - population=_decode_population( - value=data.get("population"), - path=f"{path}.population", - ), - metadata=dict( - _decode_optional_map(value=data.get("metadata"), path=f"{path}.metadata") - ), - ) - - -def _decode_turn(*, data: object, path: str) -> Turn: - """Decode a ``Turn``. - - Returns: - Turn: The reconstructed turn. - """ - typed = _decode_map(value=data, path=path) - raw_eval = typed.get("eval_result") - eval_result = ( - None - if raw_eval is None - else _decode_eval_result(data=raw_eval, path=f"{path}.eval_result") - ) - return Turn( - request=_decode_request(data=typed.get("request"), path=f"{path}.request"), - response=_decode_response(data=typed.get("response"), path=f"{path}.response"), - eval_result=eval_result, - turn_number=_decode_int( - value=typed.get("turn_number", 0), path=f"{path}.turn_number" - ), - timestamp=_decode_datetime( - value=typed.get("timestamp"), path=f"{path}.timestamp" - ), - driver_reasoning=_decode_str( - value=typed.get("driver_reasoning", ""), - path=f"{path}.driver_reasoning", - ), - ) - - -def _decode_request(*, data: object, path: str) -> Request: - """Decode a ``Request``. - - Returns: - Request: The reconstructed request. - """ - typed = _decode_map(value=data, path=path) - raw_prompt = typed.get("prompt") - prompt = raw_prompt if isinstance(raw_prompt, str) else None - return Request( - prompt=prompt, - attachments=[ - _decode_payload(data=item, path=f"{path}.attachments[{index}]") - for index, item in enumerate( - _decode_list( - value=typed.get("attachments"), - path=f"{path}.attachments", - ) - ) - ], - ) - - -def _decode_response(*, data: object, path: str) -> Response: - """Decode a ``Response``. - - Returns: - Response: The reconstructed response. - """ - typed = _decode_map(value=data, path=path) - return Response( - text=_decode_str(value=typed.get("text", ""), path=f"{path}.text"), - tool_calls=[ - _decode_tool_call(data=item, path=f"{path}.tool_calls[{index}]") - for index, item in enumerate( - _decode_list( - value=typed.get("tool_calls"), - path=f"{path}.tool_calls", - ) - ) - ], - side_effects=[ - _decode_side_effect(data=item, path=f"{path}.side_effects[{index}]") - for index, item in enumerate( - _decode_list( - value=typed.get("side_effects"), - path=f"{path}.side_effects", - ) - ) - ], - metadata=dict( - _decode_optional_map(value=typed.get("metadata"), path=f"{path}.metadata") - ), - ) - - -def _decode_tool_call(*, data: object, path: str) -> ToolCall: - """Decode a ``ToolCall``. - - Returns: - ToolCall: The reconstructed tool call. - """ - typed = _decode_map(value=data, path=path) - raw_result = typed.get("result") - return ToolCall( - name=_decode_str(value=typed.get("name", ""), path=f"{path}.name"), - arguments=dict( - _decode_optional_map(value=typed.get("arguments"), path=f"{path}.arguments") - ), - result=raw_result if isinstance(raw_result, str) else None, - timestamp=_decode_datetime( - value=typed.get("timestamp"), path=f"{path}.timestamp" - ), - ) - - -def _decode_side_effect(*, data: object, path: str) -> SideEffect: - """Decode a ``SideEffect``. - - Returns: - SideEffect: The reconstructed side effect. - """ - typed = _decode_map(value=data, path=path) - return SideEffect( - kind=_decode_str(value=typed.get("kind", ""), path=f"{path}.kind"), - details=dict( - _decode_optional_map(value=typed.get("details"), path=f"{path}.details") - ), - ) - - -def _decode_payload(*, data: object, path: str) -> Payload: - """Decode a ``Payload``. - - Returns: - Payload: The reconstructed payload. - - Raises: - SchemaError: If the payload declares a binary format. - """ - typed = _decode_map(value=data, path=path) - payload_format = _decode_enum( - enum=PayloadFormat, - value=typed.get("format", PayloadFormat.TEXT.value), - path=f"{path}.format", - ) - if payload_format.is_binary: - msg = ( - f"{path}: binary payload format {payload_format.value!r} is unsupported " - f"in {TRACE_SCHEMA_VERSION}." - ) - raise SchemaError(msg) - return Payload( - content=_decode_str(value=typed.get("content", ""), path=f"{path}.content"), - id=_decode_str(value=typed.get("id", ""), path=f"{path}.id"), - format=payload_format, - artifact=None, - metadata=dict( - _decode_optional_map(value=typed.get("metadata"), path=f"{path}.metadata") - ), - ) - - -def _decode_eval_result(*, data: object, path: str) -> EvalResult: - """Decode an ``EvalResult``. - - Returns: - EvalResult: The reconstructed evaluation result. - """ - typed = _decode_map(value=data, path=path) - return EvalResult( - outcome=_decode_enum( - enum=EvalOutcome, - value=typed.get("outcome"), - path=f"{path}.outcome", - ), - confidence=_encode_float( - value=typed.get("confidence", 1.0), - path=f"{path}.confidence", - ), - evidence=_decode_str_list(value=typed.get("evidence"), path=f"{path}.evidence"), - rationale=_decode_str( - value=typed.get("rationale", ""), path=f"{path}.rationale" - ), - undetermined_operands=_decode_str_list( - value=typed.get("undetermined_operands"), - path=f"{path}.undetermined_operands", - ), - ) - - -def _decode_injection(*, data: object, path: str) -> InjectionRecord: - """Decode an ``InjectionRecord``. - - Returns: - InjectionRecord: The reconstructed injection record. - """ - typed = _decode_map(value=data, path=path) - raw_payload_id = typed.get("payload_id") - return InjectionRecord( - payload_id=raw_payload_id if isinstance(raw_payload_id, str) else None, - surface_name=_decode_str( - value=typed.get("surface_name", ""), - path=f"{path}.surface_name", - ), - ) - - -def _decode_population(*, value: object, path: str) -> PopulationRef | None: - """Decode an optional ``PopulationRef``. - - Returns: - PopulationRef | None: The reconstructed reference, or ``None``. - """ - if value is None: - return None - typed = _decode_map(value=value, path=path) - return PopulationRef( - id=_decode_str(value=typed.get("id"), path=f"{path}.id"), - index=_decode_int(value=typed.get("index"), path=f"{path}.index"), - size=_decode_int(value=typed.get("size"), path=f"{path}.size"), - threshold=_encode_float(value=typed.get("threshold"), path=f"{path}.threshold"), + result=Result.from_dict(data.get("result")), + pytest_nodeid=data.get("pytest_nodeid"), + result_index=data.get("result_index"), ) -def _decode_enum(*, enum: type[EnumT], value: object, path: str) -> EnumT: - """Decode an enum member from its wire value, failing closed on unknown. - - Returns: - EnumT: The enum member. - - Raises: - SchemaError: If ``value`` is not a member of ``enum``. - """ - try: - return enum(value) - except ValueError as exc: - msg = f"{path}: {value!r} is not a valid {enum.__name__}." - raise SchemaError(msg) from exc - - -def _decode_harm_category(*, value: object, path: str) -> str | None: - """Decode a harm category as a passthrough string. - - Returns: - str | None: The category string, or ``None``. - - Raises: - SchemaError: If ``value`` is not a string or ``None``. - """ - if value is None: - return None - if not isinstance(value, str): - msg = f"{path}: expected a string or null, got {type(value).__name__}." - raise SchemaError(msg) - return value - - -def _decode_datetime(*, value: object, path: str) -> datetime | None: - """Decode an ISO 8601 timestamp. - - Returns: - datetime | None: The parsed datetime, or ``None``. - - Raises: - SchemaError: If ``value`` is neither ``None`` nor a valid ISO string. - """ - if value is None: - return None - if not isinstance(value, str): - msg = f"{path}: expected an ISO timestamp string, got {type(value).__name__}." - raise SchemaError(msg) - try: - return datetime.fromisoformat(value) - except ValueError as exc: - msg = f"{path}: {value!r} is not a valid ISO 8601 timestamp." - raise SchemaError(msg) from exc - - -def _decode_str(*, value: object, path: str) -> str: - """Decode a required string field. - - Returns: - str: The string value. - - Raises: - SchemaError: If ``value`` is not a string. - """ - if not isinstance(value, str): - msg = f"{path}: expected a string, got {type(value).__name__}." - raise SchemaError(msg) - return value - - -def _decode_int(*, value: object, path: str) -> int: - """Decode a required integer field. - - Returns: - int: The integer value. - - Raises: - SchemaError: If ``value`` is not an integer. - """ - if isinstance(value, bool) or not isinstance(value, int): - msg = f"{path}: expected an integer, got {type(value).__name__}." - raise SchemaError(msg) - return value - - -def _decode_list(*, value: object, path: str) -> list[Any]: - """Decode an optional wire list. - - Returns: - list[Any]: The list, or an empty list when absent or ``None``. - - Raises: - SchemaError: If ``value`` is present but not a list. - """ - if value is None: - return [] - if not isinstance(value, list): - msg = f"{path}: expected a list or null, got {type(value).__name__}." - raise SchemaError(msg) - return value - - -def _decode_str_list(*, value: object, path: str) -> list[str]: - """Decode a list of strings. - - Returns: - list[str]: The decoded strings. - """ - return [ - _decode_str(value=item, path=f"{path}[{index}]") - for index, item in enumerate(_decode_list(value=value, path=path)) - ] - - -def _validate_result_index(*, value: object) -> int | None: - """Validate an optional result index. - - Returns: - int | None: The validated index. - - Raises: - SchemaError: If ``value`` is not an integer or ``None``. - """ - if value is not None and (isinstance(value, bool) or not isinstance(value, int)): - msg = ( - "record 'result_index' must be an integer or null, " - f"got {type(value).__name__}." - ) - raise SchemaError(msg) - return value - - -def _decode_map(*, value: object, path: str) -> Mapping[str, Any]: - """Decode a required mapping field. - - Returns: - Mapping[str, Any]: The mapping value. - - Raises: - SchemaError: If ``value`` is not a mapping. - """ - if not isinstance(value, Mapping): - msg = f"{path}: expected a mapping, got {type(value).__name__}." - raise SchemaError(msg) - return value - - -def _decode_optional_map(*, value: object, path: str) -> Mapping[str, Any]: - """Decode an optional mapping field, defaulting to empty when absent. - - Returns: - Mapping[str, Any]: The mapping value, or an empty mapping when ``None``. - - Raises: - SchemaError: If ``value`` is present but not a mapping. - """ - if value is None: - return {} - return _decode_map(value=value, path=path) - - _DECODERS: dict[str, Callable[[Mapping[str, Any]], ResultRecord]] = { TRACE_SCHEMA_VERSION: _decode_v1, } diff --git a/rampart/core/types.py b/rampart/core/types.py index 96da246a..bb8aa77c 100644 --- a/rampart/core/types.py +++ b/rampart/core/types.py @@ -10,14 +10,26 @@ from __future__ import annotations import uuid +from collections.abc import Mapping from dataclasses import dataclass, field from enum import Enum +from pathlib import ( + Path, # ruff: ignore[typing-only-standard-library-import] Resolved by TypeAdapter. +) from typing import TYPE_CHECKING, Any -if TYPE_CHECKING: - from datetime import datetime - from pathlib import Path +from pydantic import ( + ValidationInfo, + field_validator, + model_validator, +) + +from rampart.core._schema import ( + IsoDatetime, # ruff: ignore[typing-only-first-party-import] Resolved by TypeAdapter. + JsonMapping, # ruff: ignore[typing-only-first-party-import] Resolved by TypeAdapter. +) +if TYPE_CHECKING: from rampart.core.manifest import AppManifest @@ -139,7 +151,63 @@ class Payload: id: str = field(default_factory=lambda: uuid.uuid4().hex[:12]) format: PayloadFormat = PayloadFormat.TEXT artifact: Path | None = None - metadata: dict[str, Any] = field(default_factory=dict[str, Any]) + metadata: JsonMapping = field(default_factory=dict[str, Any]) + + # Pydantic invokes validator callbacks with positional value/info arguments. + @model_validator(mode="before") + @classmethod + def _validate_trace_id(cls, value: object, info: ValidationInfo) -> object: + """Require recorded payload identity instead of generating one on read. + + Returns: + object: The unchanged input. + + Raises: + ValueError: If a trace payload has no recorded id. + """ + if ( + info.context + and info.context.get("trace") + and isinstance(value, Mapping) + and "id" not in value + ): + msg = "id: a trace payload must record its id" + raise ValueError(msg) + return value + + @field_validator("format") + @classmethod + def _validate_trace_format( + cls, value: PayloadFormat, info: ValidationInfo + ) -> PayloadFormat: + """Reject binary formats at the trace boundary, not during live use. + + Returns: + PayloadFormat: The validated format. + + Raises: + ValueError: If a trace contains a binary payload. + """ + if info.context and info.context.get("trace") and value.is_binary: + msg = f"binary payload format {value.value!r} is unsupported in traces" + raise ValueError(msg) + return value + + @field_validator("artifact", mode="before") + @classmethod + def _validate_trace_artifact(cls, value: object, info: ValidationInfo) -> object: + """Reject trace artifacts before path construction or filesystem access. + + Returns: + object: The unchanged artifact for live use, or None for a trace. + + Raises: + ValueError: If a trace contains a non-null artifact. + """ + if info.context and info.context.get("trace") and value is not None: + msg = "artifact: only null is supported in traces" + raise ValueError(msg) + return value def __post_init__(self) -> None: """Validate content-format-artifact consistency. @@ -195,9 +263,9 @@ class ToolCall: """ name: str - arguments: dict[str, Any] = field(default_factory=dict[str, Any]) + arguments: JsonMapping = field(default_factory=dict[str, Any]) result: str | None = None - timestamp: datetime | None = None + timestamp: IsoDatetime | None = None @dataclass(kw_only=True) @@ -214,7 +282,7 @@ class SideEffect: """ kind: str - details: dict[str, Any] = field(default_factory=dict[str, Any]) + details: JsonMapping = field(default_factory=dict[str, Any]) @dataclass(kw_only=True) @@ -233,7 +301,7 @@ class Response: text: str tool_calls: list[ToolCall] = field(default_factory=list[ToolCall]) side_effects: list[SideEffect] = field(default_factory=list[SideEffect]) - metadata: dict[str, Any] = field(default_factory=dict[str, Any]) + metadata: JsonMapping = field(default_factory=dict[str, Any]) @dataclass(kw_only=True) @@ -285,7 +353,7 @@ class Turn: response: Response eval_result: EvalResult | None = None turn_number: int = 0 - timestamp: datetime | None = None + timestamp: IsoDatetime | None = None driver_reasoning: str = "" diff --git a/schemas/trace.v1.schema.json b/schemas/trace.v1.schema.json new file mode 100644 index 00000000..554434a8 --- /dev/null +++ b/schemas/trace.v1.schema.json @@ -0,0 +1,490 @@ +{ + "$defs": { + "EvalOutcome": { + "description": "What the evaluator determined.\n\nDETECTED: The condition was found.\nNOT_DETECTED: The condition was not found.\nUNDETERMINED: The evaluator could not make a determination.", + "enum": [ + "detected", + "not_detected", + "undetermined" + ], + "title": "EvalOutcome", + "type": "string" + }, + "EvalResult": { + "additionalProperties": true, + "description": "What an evaluator returns \u2014 a raw condition detection signal.\n\nThis is NOT a safety judgment. Whether DETECTED means \"safe\" or\n\"unsafe\" depends on context.\n\nArgs:\n outcome: What the evaluator determined.\n confidence: How confident the evaluator is (0.0 to 1.0).\n evidence: Specific observations supporting the outcome.\n rationale: Human-readable explanation.\n undetermined_operands: Why parts of the evaluation stayed\n undetermined, one distinct reason per entry. ``&`` and ``|``\n record every operand they ran that came back UNDETERMINED,\n taking the reasons that operand already carries or, for a\n leaf, its rationale, or a fixed phrase when it gave none;\n repeats are collapsed. ``~`` carries its inner result's\n entries through. An evaluator that is not a composite\n records nothing. It says nothing about ``outcome``: a\n DETECTED or NOT_DETECTED result with entries here reached a\n definitive answer while part of the evaluation did not, and\n an UNDETERMINED result can carry entries recorded further\n down the expression.", + "properties": { + "confidence": { + "default": 1.0, + "title": "Confidence", + "type": "number" + }, + "evidence": { + "items": { + "type": "string" + }, + "title": "Evidence", + "type": "array" + }, + "outcome": { + "$ref": "#/$defs/EvalOutcome" + }, + "rationale": { + "default": "", + "title": "Rationale", + "type": "string" + }, + "undetermined_operands": { + "items": { + "type": "string" + }, + "title": "Undetermined Operands", + "type": "array" + } + }, + "required": [ + "outcome" + ], + "title": "EvalResult", + "type": "object" + }, + "HarmCategory": { + "description": "Classification of the safety concern being tested.\n\nUsed by the pytest @harm marker for categorization, by reporting\nsinks for grouping, and by safety gates for threshold configuration.\n\nHarmCategory is a StrEnum so that its values are native strings. This\nenables teams to use custom string categories alongside the built-in\nvalues: @pytest.mark.harm(\"custom_product_risk\") is valid, and the\nstring flows through Result.harm_category, reporting sinks, and\ndashboard grouping without requiring enum membership. Built-in values\nprovide IDE completion and typo protection for common categories;\nplain strings provide extensibility for team-specific risks.\n\nPhase availability:\n Phase 1: All values are defined and usable with MockAdapter.\n Phase 2: PROMPT_INJECTION, JAILBREAK, and remaining categories\n gain execution strategy support via PyRIT integration.", + "enum": [ + "memory_poisoning", + "prompt_injection", + "jailbreak", + "data_exfiltration", + "over_permissive_action", + "data_leakage", + "content_safety", + "hallucination", + "behavioral_regression" + ], + "title": "HarmCategory", + "type": "string" + }, + "InjectionRecord": { + "additionalProperties": true, + "description": "Records what was injected and where, for reproduction and reporting.\n\nPopulated by XPIAExecution after handles are activated and stored\non Result. Provides the complete injection context needed to\nreproduce a test run: which payload was placed in which surface.\n\nArgs:\n payload_id: The injected payload's identifier. None if\n the surface implementation does not track payload IDs.\n surface_name: The surface this payload was injected into\n (e.g., \"SharePoint\", \"Exchange\").", + "properties": { + "payload_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Payload Id" + }, + "surface_name": { + "title": "Surface Name", + "type": "string" + } + }, + "required": [ + "payload_id", + "surface_name" + ], + "title": "InjectionRecord", + "type": "object" + }, + "ObservabilityLevel": { + "description": "What the adapter can reliably observe during agent execution.\n\nDeclared by the adapter to inform evaluators and reporting. An\nevaluator that needs an evidence channel the adapter does not report\nreturns UNDETERMINED rather than a false NOT_DETECTED. That covers\ntool call data under RESPONSE_ONLY, and side effect data under\neither TOOL_ONLY or RESPONSE_ONLY.\n\nThe guarantee is per channel, not per field. A level that reports a\nchannel is taken at its word for what it puts in it, so a tool call\nreported with redacted or partial arguments still counts as observed\nand a predicate over those arguments can return NOT_DETECTED.\n\nThe ``observes_tool_calls`` and ``observes_side_effects`` properties\nlet evaluators ask what evidence is available without listing every\nenum member.", + "enum": [ + "tool_and_side_effects", + "tool_only", + "response_only" + ], + "title": "ObservabilityLevel", + "type": "string" + }, + "Payload": { + "additionalProperties": true, + "description": "Content to inject into a surface or send alongside a prompt.\n\n``content`` is always the semantic text \u2014 the attack instruction,\nthe adversarial prompt, or a description of the payload's purpose.\nIt is what reports display and what makes a payload reproducible.\n\nFor text formats (TEXT, HTML, MARKDOWN), ``content`` is delivered\ndirectly \u2014 no artifact is needed. For binary formats (IMAGE, PDF,\nDOCX), ``artifact`` points to the file that surfaces and adapters\ndeliver. The ``content`` field still holds the human-readable text\nfor reporting and debugging.\n\nArgs:\n content (str): The semantic text content. Always human-readable.\n id (str): Stable identifier for reproduction and cache keys.\n format (PayloadFormat): Delivery format. TEXT by default.\n artifact (Path | None): Path to a rendered binary file.\n Required for binary formats, must be None for text formats.\n metadata (dict[str, Any]): Provenance tracking (persona,\n template, variant index, generation params).", + "properties": { + "artifact": { + "default": null, + "type": "null" + }, + "content": { + "title": "Content", + "type": "string" + }, + "format": { + "default": "text", + "enum": [ + "text", + "html", + "markdown" + ], + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" + } + }, + "required": [ + "content", + "id" + ], + "title": "Payload", + "type": "object" + }, + "PopulationRef": { + "additionalProperties": true, + "description": "Identifies the trial population that a Result belongs to.\n\nArgs:\n id: Unique identifier shared by every result in the population.\n index: Zero-based position of the result within the population.\n size: Number of results requested for the population.\n threshold: Required safe-result rate for the population.", + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "index": { + "title": "Index", + "type": "integer" + }, + "size": { + "title": "Size", + "type": "integer" + }, + "threshold": { + "title": "Threshold", + "type": "number" + } + }, + "required": [ + "id", + "index", + "size", + "threshold" + ], + "title": "PopulationRef", + "type": "object" + }, + "Request": { + "additionalProperties": true, + "anyOf": [ + { + "properties": { + "prompt": { + "type": "string" + } + }, + "required": [ + "prompt" + ] + }, + { + "properties": { + "attachments": { + "minItems": 1, + "type": "array" + } + }, + "required": [ + "attachments" + ] + } + ], + "description": "What is sent to the agent in a single turn.\n\nCombines prompt text and inline payloads into a single object.\nAt least one of ``prompt`` or ``attachments`` must be provided.\n\nArgs:\n prompt: The text prompt to send. None when only\n attachments are sent (e.g., inline XPIA).\n attachments: Payloads sent alongside the prompt for\n inline delivery (e.g., poisoned documents).", + "properties": { + "attachments": { + "items": { + "$ref": "#/$defs/Payload" + }, + "title": "Attachments", + "type": "array" + }, + "prompt": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Prompt" + } + }, + "title": "Request", + "type": "object" + }, + "Response": { + "additionalProperties": true, + "description": "What the agent returned for a single prompt.\n\nThe adapter populates every field it can observe.\n\nArgs:\n text: The agent's text response.\n tool_calls: Tool invocations observed during this interaction.\n side_effects: Other observable effects.\n metadata: Adapter-specific diagnostic data.", + "properties": { + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" + }, + "side_effects": { + "items": { + "$ref": "#/$defs/SideEffect" + }, + "title": "Side Effects", + "type": "array" + }, + "text": { + "title": "Text", + "type": "string" + }, + "tool_calls": { + "items": { + "$ref": "#/$defs/ToolCall" + }, + "title": "Tool Calls", + "type": "array" + } + }, + "required": [ + "text" + ], + "title": "Response", + "type": "object" + }, + "SafetyStatus": { + "description": "Categorical safety status for structured reporting.\n\nSAFE: The agent behaved correctly.\nUNSAFE: A safety violation was detected.\nUNDETERMINED: The framework could not determine safety\n (typically an observability gap).\nERROR: The test encountered an infrastructure error.", + "enum": [ + "safe", + "unsafe", + "undetermined", + "error" + ], + "title": "SafetyStatus", + "type": "string" + }, + "SideEffect": { + "additionalProperties": true, + "description": "An observable side effect beyond tool invocations.\n\nCovers effects like HTTP requests, file system changes, or\ndatabase writes that the adapter can observe but that are not\nmodeled as tool calls in the agent's API.\n\nArgs:\n kind: Effect category (e.g., \"http_request\", \"file_write\").\n details: Structured data about the effect.", + "properties": { + "details": { + "additionalProperties": true, + "title": "Details", + "type": "object" + }, + "kind": { + "title": "Kind", + "type": "string" + } + }, + "required": [ + "kind" + ], + "title": "SideEffect", + "type": "object" + }, + "ToolCall": { + "additionalProperties": true, + "description": "A tool invocation observed during agent execution.\n\nAdapters populate this from whatever observability they have \u2014 API\nresponse fields, telemetry streams, log parsing.\n\nArgs:\n name: Tool name as the agent reported it (e.g., \"send_email\").\n arguments: Parameters passed to the tool.\n result: Tool return value, if the adapter can observe it.\n timestamp: When the invocation occurred, if available.", + "properties": { + "arguments": { + "additionalProperties": true, + "title": "Arguments", + "type": "object" + }, + "name": { + "title": "Name", + "type": "string" + }, + "result": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Result" + }, + "timestamp": { + "anyOf": [ + { + "description": "ISO 8601 datetime; UTC offset is optional.", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Timestamp" + } + }, + "required": [ + "name" + ], + "title": "ToolCall", + "type": "object" + }, + "Turn": { + "additionalProperties": true, + "description": "One prompt-response exchange.\n\nTurn is immutable. The execution loop constructs a provisional Turn\n(without eval_result) for the evaluator call, then produces the\nfinal Turn via dataclasses.replace before appending to history.\n\nArgs:\n request: What was sent to the agent.\n response: What the agent returned.\n eval_result: Evaluator outcome for this turn.\n turn_number: Position in the conversation (0-indexed).\n timestamp: When this exchange occurred.\n driver_reasoning: Why the driver chose this request.", + "properties": { + "driver_reasoning": { + "default": "", + "title": "Driver Reasoning", + "type": "string" + }, + "eval_result": { + "anyOf": [ + { + "$ref": "#/$defs/EvalResult" + }, + { + "type": "null" + } + ], + "default": null + }, + "request": { + "$ref": "#/$defs/Request" + }, + "response": { + "$ref": "#/$defs/Response" + }, + "timestamp": { + "anyOf": [ + { + "description": "ISO 8601 datetime; UTC offset is optional.", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Timestamp" + }, + "turn_number": { + "default": 0, + "title": "Turn Number", + "type": "integer" + } + }, + "required": [ + "request", + "response" + ], + "title": "Turn", + "type": "object" + } + }, + "$id": "urn:rampart:trace:v1", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": true, + "properties": { + "pytest_nodeid": { + "type": [ + "string", + "null" + ] + }, + "result": { + "additionalProperties": true, + "description": "The outcome of a safety test.\n\nThis is the single result type for the entire framework. Attacks\nand probes both produce Result objects. The reporting infrastructure,\npytest plugin, and dashboards all consume them.\n\nThe critical invariant: bool(result) returns result.safe. This\nmeans ``assert result, result.summary`` always means \"assert the\nagent behaved safely\" \u2014 and failures include the summary explaining\nwhat was detected.\n\n``safe`` is a derived property (``status is SafetyStatus.SAFE``),\nnot a stored field, so it can never drift out of sync with ``status``.\n\nArgs:\n status: Categorical status for structured reporting.\n summary: Human-readable one-line summary.\n observability_level: What the adapter could observe. Required, so\n that a report states a level someone chose rather than one the\n framework assumed. Built-in strategies pass\n ``adapter.observability_profile``.\n turns: The full conversation for evidence and debugging.\n duration_seconds: How long the test execution took.\n harm_category: Which harm category this test covers.\n Accepts HarmCategory enum values for built-in categories or plain strings\n for team-defined categories (e.g., \"custom_product_risk\"). Both are strings\n at runtime since HarmCategory is a StrEnum.\n strategy: Name of the execution strategy (e.g., \"xpia\", \"crescendo\").\n injections: What was injected and into which surfaces,\n for full reproduction of multi-surface attacks. Empty for non-XPIA tests.\n population: Trial population provenance. None for single executions.\n metadata: Additional structured data for reporting.", + "properties": { + "duration_seconds": { + "default": 0.0, + "title": "Duration Seconds", + "type": "number" + }, + "harm_category": { + "anyOf": [ + { + "$ref": "#/$defs/HarmCategory" + }, + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Harm Category" + }, + "injections": { + "items": { + "$ref": "#/$defs/InjectionRecord" + }, + "title": "Injections", + "type": "array" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" + }, + "observability_level": { + "$ref": "#/$defs/ObservabilityLevel" + }, + "population": { + "anyOf": [ + { + "$ref": "#/$defs/PopulationRef" + }, + { + "type": "null" + } + ], + "default": null + }, + "status": { + "$ref": "#/$defs/SafetyStatus" + }, + "strategy": { + "default": "", + "title": "Strategy", + "type": "string" + }, + "summary": { + "title": "Summary", + "type": "string" + }, + "turns": { + "items": { + "$ref": "#/$defs/Turn" + }, + "title": "Turns", + "type": "array" + } + }, + "required": [ + "status", + "summary", + "observability_level" + ], + "title": "Result", + "type": "object" + }, + "result_index": { + "type": [ + "integer", + "null" + ] + }, + "version": { + "const": "rampart.trace.v1", + "type": "string" + } + }, + "required": [ + "version", + "result" + ], + "title": "ResultRecord", + "type": "object" +} diff --git a/scripts/generate_trace_schema.py b/scripts/generate_trace_schema.py new file mode 100644 index 00000000..123e175c --- /dev/null +++ b/scripts/generate_trace_schema.py @@ -0,0 +1,40 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Regenerate the checked-in canonical trace schema from the Result adapter.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from rampart.core.serialization import ResultRecord + + +def main() -> None: + """Write the schema or fail when the committed contract has drifted. + + Raises: + SystemExit: If --check finds a missing or outdated schema. + """ + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + path = Path(__file__).resolve().parents[1] / "schemas" / "trace.v1.schema.json" + content = json.dumps(ResultRecord.json_schema(), indent=2, sort_keys=True) + "\n" + if args.check: + if not path.exists() or path.read_text(encoding="utf-8") != content: + parser.exit( + status=1, + message=( + "Trace schema is outdated; run scripts/generate_trace_schema.py\n" + ), + ) + else: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8", newline="\n") + + +if __name__ == "__main__": + main() diff --git a/tests/unit/core/test_serialization.py b/tests/unit/core/test_serialization.py index 09cad12b..0c2df486 100644 --- a/tests/unit/core/test_serialization.py +++ b/tests/unit/core/test_serialization.py @@ -5,11 +5,23 @@ from __future__ import annotations +import json import math +import re from dataclasses import fields -from datetime import UTC, datetime +from datetime import ( + UTC, + datetime, + timedelta, + timezone, +) +from pathlib import Path +from typing import TYPE_CHECKING, Any +from unittest.mock import patch import pytest +from jsonschema import Draft202012Validator +from pydantic import TypeAdapter from rampart.core.result import ( InjectionRecord, @@ -38,6 +50,9 @@ Turn, ) +if TYPE_CHECKING: + from collections.abc import MutableMapping + _TIMESTAMP = datetime(2026, 1, 2, 3, 4, 5, tzinfo=UTC) @@ -114,6 +129,16 @@ def _minimal_record_dict() -> dict: } +def _freeform_maps(result: Result) -> list[MutableMapping[str, Any]]: + return [ + result.metadata, + result.turns[0].request.attachments[0].metadata, + result.turns[0].response.metadata, + result.turns[0].response.tool_calls[0].arguments, + result.turns[0].response.side_effects[0].details, + ] + + class TestRoundTrip: def test_full_result_round_trips_to_equal_value(self) -> None: original = _make_full_result() @@ -349,3 +374,345 @@ def test_decoding_a_binary_payload_fails_closed(self) -> None: with pytest.raises(SchemaError, match="binary payload"): deserialize_result(data=data) + + @pytest.mark.parametrize("payload_format", ["pdf", "docx", "text"]) + def test_artifact_is_rejected_before_filesystem_access( + self, payload_format: str + ) -> None: + data = serialize_result(result=_make_full_result()) + payload = data["result"]["turns"][0]["request"]["attachments"][0] + payload.update(format=payload_format, artifact="untrusted-artifact") + + with ( + patch.object( + Path, "exists", side_effect=AssertionError("filesystem access") + ), + pytest.raises(SchemaError, match="artifact"), + ): + deserialize_result(data=data) + + def test_live_binary_payload_is_still_supported(self, tmp_path: Path) -> None: + artifact = tmp_path / "doc.pdf" + artifact.write_bytes(b"%PDF-1.4 fake") + payload = Payload(content="doc", format=PayloadFormat.PDF, artifact=artifact) + + assert TypeAdapter(Payload).validate_python(payload) is payload + assert payload.artifact == artifact + + +class TestResultAdapter: + def test_body_methods_round_trip_through_json(self) -> None: + original = _make_full_result() + + body = original.to_dict() + restored = Result.from_dict(json.loads(json.dumps(body, allow_nan=False))) + + assert restored == original + assert isinstance(restored.turns[0], Turn) + assert "version" not in body + assert body["turns"][0]["timestamp"] == _TIMESTAMP.isoformat() + assert body["turns"][0]["response"]["tool_calls"][0]["timestamp"] == ( + _TIMESTAMP.isoformat() + ) + + @pytest.mark.parametrize( + "timestamp", + [ + _TIMESTAMP, + _TIMESTAMP.replace(tzinfo=None), + _TIMESTAMP.replace(tzinfo=timezone(timedelta(seconds=30))), + _TIMESTAMP.replace(tzinfo=timezone(timedelta(hours=-5))), + ], + ) + def test_python_iso_datetimes_preserve_their_wire_text( + self, timestamp: datetime + ) -> None: + result = _make_full_result() + result.turns[0].__dict__["timestamp"] = timestamp + result.turns[0].response.tool_calls[0].timestamp = timestamp + + encoded = serialize_result(result=result) + + assert encoded["result"]["turns"][0]["timestamp"] == timestamp.isoformat() + assert ResultRecord.from_dict(encoded).result == result + Draft202012Validator( + ResultRecord.json_schema(), + format_checker=Draft202012Validator.FORMAT_CHECKER, + ).validate(encoded) + + def test_record_filters_only_top_level_metadata_without_mutation(self) -> None: + original = _make_full_result( + metadata={ + "_rampart_source_worker": "gw0", + "_pytest_nodeid": "test", + "_rampart_worker_artifact_path": object(), + "user": {"_rampart_source_worker": "keep"}, + } + ) + record = ResultRecord(result=original) + original.summary = "updated after wrapping" + + body = record.to_dict()["result"] + body["metadata"]["user"]["extra"] = True + + assert record.result is original + assert body["summary"] == original.summary + assert body["metadata"] == { + "user": {"_rampart_source_worker": "keep", "extra": True} + } + assert original.metadata["user"] == {"_rampart_source_worker": "keep"} + assert "_rampart_worker_artifact_path" in original.metadata + + def test_body_does_not_own_transport_filtering(self) -> None: + result = _make_full_result(metadata={"_rampart_source_worker": "gw0"}) + + assert result.to_dict()["metadata"] == result.metadata + assert ResultRecord(result=result).to_dict()["result"]["metadata"] == {} + + @pytest.mark.parametrize("index", [None, 0, 2]) + def test_optional_attribution_is_not_inferred(self, index: int | None) -> None: + record = ResultRecord(result=_make_full_result(), result_index=index) + + encoded = record.to_dict() + + assert ResultRecord.from_dict(encoded).result_index == index + assert ("result_index" in encoded) is (index is not None) + + @pytest.mark.parametrize("nodeid", [False, 1, [], {}]) + def test_invalid_nodeid_is_rejected(self, nodeid: object) -> None: + data = _minimal_record_dict() + data["pytest_nodeid"] = nodeid + + with pytest.raises(SchemaError, match="pytest_nodeid"): + ResultRecord.from_dict(data) + + def test_nested_mutations_are_revalidated(self) -> None: + result = _make_full_result() + result.turns[0].response.__dict__["text"] = 42 + + with pytest.raises(SchemaError, match=r"result\.turns\[0\]\.response\.text"): + result.to_dict() + + @pytest.mark.parametrize("invalid", [True, 1.5, "1"]) + def test_integer_fields_are_not_coerced(self, invalid: object) -> None: + result = _make_full_result() + assert result.population is not None + result.population.__dict__["index"] = invalid + + with pytest.raises(SchemaError, match=r"result\.population\.index"): + result.to_dict() + + def test_missing_payload_identity_is_not_generated(self) -> None: + body = _make_full_result().to_dict() + del body["turns"][0]["request"]["attachments"][0]["id"] + + with pytest.raises(SchemaError, match="id"): + Result.from_dict(body) + + def test_invalid_timestamp_is_rejected(self) -> None: + body = _make_full_result().to_dict() + body["turns"][0]["timestamp"] = "not a date" + + with pytest.raises(SchemaError, match=r"result\.turns\[0\]\.timestamp"): + Result.from_dict(body) + + @pytest.mark.parametrize( + "field", ["turns", "injections", "metadata", "duration_seconds"] + ) + def test_null_is_not_a_default_for_nonnullable_fields(self, field: str) -> None: + body = _make_full_result().to_dict() + body[field] = None + + with pytest.raises(SchemaError, match=field): + Result.from_dict(body) + + +class TestJsonValueDomain: + @pytest.mark.parametrize("map_index", range(5)) + @pytest.mark.parametrize( + "invalid", + [ + pytest.param((1, 2), id="tuple"), + pytest.param(b"bytes", id="bytes"), + pytest.param(Path("file"), id="path"), + pytest.param(object(), id="opaque"), + pytest.param(math.inf, id="infinity"), + pytest.param(-math.inf, id="negative-infinity"), + pytest.param(math.nan, id="nan"), + pytest.param({1: "non-string key"}, id="non-string-key"), + ], + ) + def test_freeform_values_are_not_lossily_encoded( + self, *, map_index: int, invalid: object + ) -> None: + result = _make_full_result() + _freeform_maps(result)[map_index]["nested"] = {"bad": invalid} + + with pytest.raises(SchemaError, match="nested"): + result.to_dict() + + @pytest.mark.parametrize( + "invalid", + [(1, 2), b"bytes", Path("file"), object(), math.inf, math.nan, {1: "x"}], + ) + def test_dictionary_input_is_checked_before_json_encoding( + self, invalid: object + ) -> None: + body = _make_full_result().to_dict() + body["metadata"]["bad"] = invalid + + with pytest.raises(SchemaError, match="metadata"): + Result.from_dict(body) + + def test_cyclic_values_fail_with_a_field_path(self) -> None: + result = _make_full_result() + result.metadata["cycle"] = result.metadata + + with pytest.raises(SchemaError, match=r"metadata.*cycle"): + result.to_dict() + body = _minimal_record_dict()["result"] + body["metadata"] = result.metadata + with pytest.raises(SchemaError, match=r"metadata.*cycle"): + Result.from_dict(body) + + def test_supported_values_round_trip_without_mutation(self) -> None: + metadata = { + "values": [None, True, False, 0, -(2**80), 2**80, 1.25, "text"], + "nested": {"list": [{"text": "hello"}]}, + } + result = _make_full_result(metadata=metadata) + + restored = Result.from_dict(result.to_dict()) + restored.metadata["nested"]["list"][0]["text"] = "changed" + + assert result.metadata == metadata + assert metadata["nested"]["list"][0]["text"] == "hello" + assert restored.metadata["values"] == metadata["values"] + + +class TestGeneratedSchema: + def test_generated_schema_is_valid_and_matches_checked_in_contract(self) -> None: + schema = ResultRecord.json_schema() + path = Path(__file__).resolve().parents[3] / "schemas" / "trace.v1.schema.json" + + Draft202012Validator.check_schema(schema) + + assert json.loads(path.read_text(encoding="utf-8")) == schema + assert schema["properties"]["version"]["const"] == TRACE_SCHEMA_VERSION + + @pytest.mark.parametrize("payload_format", list(PayloadFormat)) + def test_schema_and_decoder_agree_on_payload_formats( + self, payload_format: PayloadFormat + ) -> None: + data = serialize_result(result=_make_full_result()) + data["result"]["turns"][0]["request"]["attachments"][0]["format"] = ( + payload_format.value + ) + validator = Draft202012Validator(ResultRecord.json_schema()) + + assert validator.is_valid(data) is payload_format.is_text + if payload_format.is_text: + assert ResultRecord.from_dict(data).result.turns[0].request.attachments + else: + with pytest.raises(SchemaError, match="binary payload"): + ResultRecord.from_dict(data) + + @pytest.mark.parametrize("full", [False, True]) + def test_full_and_minimal_records_conform(self, *, full: bool) -> None: + data = ( + serialize_result(result=_make_full_result(), result_index=0) + if full + else _minimal_record_dict() + ) + validator = Draft202012Validator( + ResultRecord.json_schema(), + format_checker=Draft202012Validator.FORMAT_CHECKER, + ) + + validator.validate(data) + validator.validate(ResultRecord.from_dict(data).to_dict()) + + def test_unknown_additive_fields_are_allowed_at_every_level(self) -> None: + data = serialize_result(result=_make_full_result()) + body = data["result"] + turn = body["turns"][0] + objects = [ + data, + body, + turn, + turn["request"], + turn["request"]["attachments"][0], + turn["response"], + turn["response"]["tool_calls"][0], + turn["response"]["side_effects"][0], + turn["eval_result"], + body["injections"][0], + body["population"], + ] + for item in objects: + item["future"] = {"recorded": True} + + Draft202012Validator(ResultRecord.json_schema()).validate(data) + assert ResultRecord.from_dict(data).result == _make_full_result() + + @pytest.mark.parametrize( + ("path", "invalid"), + [ + (("result", "summary"), 123), + (("result", "population", "index"), True), + (("result", "population", "threshold"), "0.8"), + (("result", "turns", 0, "request", "prompt"), 123), + (("result", "turns", 0, "timestamp"), False), + (("result", "turns", 0, "response", "text"), None), + (("result", "turns", 0, "response", "tool_calls", 0, "result"), 123), + (("result", "turns", 0, "request", "attachments", 0, "artifact"), "file"), + (("result", "turns", 0, "request", "attachments", 0, "format"), "unknown"), + (("result", "turns", 0, "eval_result", "outcome"), "unknown"), + (("result", "injections", 0, "payload_id"), 123), + (("pytest_nodeid",), 123), + (("result_index",), True), + ], + ) + def test_schema_and_decoder_reject_malformed_fields( + self, *, path: tuple[str | int, ...], invalid: object + ) -> None: + data = serialize_result(result=_make_full_result()) + parent: Any = data + for key in path[:-1]: + parent = parent[key] + parent[path[-1]] = invalid + + assert not Draft202012Validator(ResultRecord.json_schema()).is_valid(data) + with pytest.raises(SchemaError, match=re.escape(str(path[-1]))): + ResultRecord.from_dict(data) + + @pytest.mark.parametrize( + ("prompt", "attachments", "valid"), + [ + (None, False, False), + ("", False, True), + ("text", False, True), + (None, True, True), + ], + ) + def test_request_invariant_is_in_schema( + self, *, prompt: str | None, attachments: bool, valid: bool + ) -> None: + data = serialize_result(result=_make_full_result()) + request = data["result"]["turns"][0]["request"] + request["prompt"] = prompt + if not attachments: + request["attachments"] = [] + + assert Draft202012Validator(ResultRecord.json_schema()).is_valid(data) is valid + if valid: + ResultRecord.from_dict(data) + else: + with pytest.raises(SchemaError, match="request"): + ResultRecord.from_dict(data) + + def test_schema_requires_recorded_payload_id(self) -> None: + data = serialize_result(result=_make_full_result()) + del data["result"]["turns"][0]["request"]["attachments"][0]["id"] + + assert not Draft202012Validator(ResultRecord.json_schema()).is_valid(data) diff --git a/uv.lock b/uv.lock index 1d9bf2dd..0474aa4b 100644 --- a/uv.lock +++ b/uv.lock @@ -1394,6 +1394,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c8/8d/302cb2057b7513327b4d575cff6b1d066ee6431a5357fc3f8867cd684406/jiter-0.15.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54d5d6090cdc1b7c9e780dfb04949a990adb1e301a2fc0bbcee7de4638d33f9a", size = 344469, upload-time = "2026-05-19T10:09:46.864Z" }, ] +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + [[package]] name = "lxml" version = "6.1.1" @@ -3087,6 +3114,7 @@ dev = [ { name = "flake8" }, { name = "hatch-vcs" }, { name = "hatchling" }, + { name = "jsonschema" }, { name = "pre-commit" }, { name = "pytest-cov" }, { name = "pytest-xdist", extra = ["psutil"] }, @@ -3119,6 +3147,7 @@ dev = [ { name = "flake8", specifier = ">=7.3.0" }, { name = "hatch-vcs", specifier = ">=0.5.0" }, { name = "hatchling", specifier = ">=1.30.1" }, + { name = "jsonschema", specifier = ">=4.26.0" }, { name = "pre-commit", specifier = ">=4.5.1" }, { name = "pytest-cov", specifier = ">=6.1.0" }, { name = "pytest-xdist", extras = ["psutil"], specifier = ">=3.8.0" }, @@ -3132,6 +3161,20 @@ docs = [ { name = "mkdocstrings", extras = ["python"], specifier = ">=1.0.4" }, ] +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + [[package]] name = "regex" version = "2026.5.9" @@ -3277,6 +3320,129 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, ] +[[package]] +name = "rpds-py" +version = "2026.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/1f/a2dca5ffdbf1d475ffc4e80e4d5d720ff3a00f691795910116960ee12511/rpds_py-2026.6.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7b689145a1485c335569bd056464f3243a29af7ed3871c7be31ad624ba239bc7", size = 342174, upload-time = "2026-06-30T07:14:54.821Z" }, + { url = "https://files.pythonhosted.org/packages/4d/dc/323d08583c0832911768663d1944f0107fcd4088704858d84b5e06d105a0/rpds_py-2026.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:db08f45aecde626498fb3df07bcf6d2ec040af42e859a4f5040d79c200342911", size = 345513, upload-time = "2026-06-30T07:14:56.515Z" }, + { url = "https://files.pythonhosted.org/packages/0b/2a/e31989834d18d2f26ec1d2774c5b1eb3331df4ea8ada525175294c94b48a/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:acc992ab27b15f852c76755eb2ab7dce86585ddadba6fa5946e58556088845b4", size = 373783, upload-time = "2026-06-30T07:14:57.736Z" }, + { url = "https://files.pythonhosted.org/packages/87/fe/e80107ee3639585c9941c17d6a42cd65325022f656c023191fce78c324c8/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f88d653e7b3b779d71ae7454e20dcc9b6bae903f33c269db9f2be41bda3f261", size = 378316, upload-time = "2026-06-30T07:14:59.077Z" }, + { url = "https://files.pythonhosted.org/packages/22/6f/81e3adf81acfb6fa694de2a6e4e7d8863121e3e0799e0a7725e6cf5679c4/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e52655eaf81e32593abedaa4bfe33170c8cfedf3365ed9be6e11e07f148f0278", size = 499423, upload-time = "2026-06-30T07:15:00.488Z" }, + { url = "https://files.pythonhosted.org/packages/2d/9a/41263969df0ce3d9af2a96d5005a288200af1989aed3354bfceb5fc0b21f/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dfcc8b909769d19db55c7cc9541eb64b9b774b1057ffffb4f1048070475bb9f9", size = 386077, upload-time = "2026-06-30T07:15:01.911Z" }, + { url = "https://files.pythonhosted.org/packages/5e/19/7e98f468bd50346faff5b10e5297374b443bfdddacc8e9fbc65984539597/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c1255b302953c86a486b81d330d5ee1d5bd937691ce271b6be0ef0e299eaab7", size = 371315, upload-time = "2026-06-30T07:15:03.317Z" }, + { url = "https://files.pythonhosted.org/packages/99/3c/2b973b4d371906a134b03decfea7f5d9835a2c6d263454392e15b64b5b18/rpds_py-2026.6.3-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:8d2294a31386bfa251d8c8a39472beee17db67d4f1a6eabea665d35c9a4461c3", size = 383502, upload-time = "2026-06-30T07:15:04.627Z" }, + { url = "https://files.pythonhosted.org/packages/98/2a/12e2799500af0a307bca76b63361c51f9fe479223561489c29eea1f2ee41/rpds_py-2026.6.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f8f23ead891a3b762f35ab3b04623da7056545b48aa60d59957e6789914545da", size = 402673, upload-time = "2026-06-30T07:15:05.856Z" }, + { url = "https://files.pythonhosted.org/packages/2d/e3/21e5872d165fe08be4f229e3d5ee9d90019c0bf0e5538de60dbd54009450/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:421aba32367055614287a4292b6a17f1939c9452299f7a0209c117e990b646d4", size = 549964, upload-time = "2026-06-30T07:15:07.159Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d0/5ee0fe36844297de8123bee27bc12078c1a7416ad9f1b8a8ca18d6b0c0ac/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1e5822dfc2f0d4ab7e745eaa6d85945069329beeccef965af3f3bb26058fcab6", size = 615446, upload-time = "2026-06-30T07:15:08.531Z" }, + { url = "https://files.pythonhosted.org/packages/b1/80/1ea5873cb683f2fbe5f21b23ea1f6d179ead19f3c5b249b7eb5dca568ef2/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:83e35b57523816c8613fd0776b40cd8bb9f596b37ddd2692eb4a6bb5ab2f8c93", size = 576975, upload-time = "2026-06-30T07:15:09.97Z" }, + { url = "https://files.pythonhosted.org/packages/c9/e1/90ef639217a5ddb15b7f4f61b1c33911fd044ad03c311bafdd2bcab85582/rpds_py-2026.6.3-cp311-cp311-win32.whl", hash = "sha256:de3eceba0b683bcbb1ab93da016d0270df1f9ae7be716b40214c5dafac6ea45a", size = 204453, upload-time = "2026-06-30T07:15:11.324Z" }, + { url = "https://files.pythonhosted.org/packages/f2/b7/b7a1695d7af36f521fb11e80d6d3adbd744f73b921859bd3c2a2c0dc706f/rpds_py-2026.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:2c54a076ca4d370980ab57bc0e31df57bbe8d41340436a90ef8b1219a3cbb127", size = 223219, upload-time = "2026-06-30T07:15:12.476Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a2/145afacf796e4506062825941176ad9445c2dcf2b3b6a1f13d3030a15e19/rpds_py-2026.6.3-cp311-cp311-win_arm64.whl", hash = "sha256:168c733a7112e071bb7a66460e667edfcff06c017a3c523f7a8a8e08d0140804", size = 219137, upload-time = "2026-06-30T07:15:13.631Z" }, + { url = "https://files.pythonhosted.org/packages/5c/be/2e8974163072e7bab7df1a5acd54c4498e75e35d6d18b864d3a9d5dadc92/rpds_py-2026.6.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a0811d33247c3d6128a3001d763f2aa056bb3425204335400ac54f89eec3a0d0", size = 343691, upload-time = "2026-06-30T07:15:14.96Z" }, + { url = "https://files.pythonhosted.org/packages/a4/73/319dfa745dd668efe89309141ded489126461fcecd2b8f3a3cda185129b6/rpds_py-2026.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:538949e262e46caa31ac01bdb3c1e8f642622922cacbabbae6a8445d9dc33eaf", size = 338542, upload-time = "2026-06-30T07:15:16.267Z" }, + { url = "https://files.pythonhosted.org/packages/21/63/4239893be1c4d09b709b1a8f6be4188f0870084ff547f46606b8a75f1b03/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55927d532399c2c646100ff7feb48eaa940ad70f42cd68e1328f3ded9f81ca24", size = 368180, upload-time = "2026-06-30T07:15:17.62Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ca/9c5de382225234ceb37b1844ebdb140db12b2a278bb9efe2fcd19f6c82ce/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f56f1695bc5c0871cbc33dc0130fcf503aab0c57dcc5a6700a4f49eba4f2652e", size = 375067, upload-time = "2026-06-30T07:15:18.952Z" }, + { url = "https://files.pythonhosted.org/packages/87/dc/863f69d1bf04ade34b7fe0d59b9fdf6f0135fe2d7cbca74f1d665589559d/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:270b293dae9058fc9fcedab50f13cebf46fb8ed1d1d54e0521a9da5d6b211975", size = 490509, upload-time = "2026-06-30T07:15:20.434Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ef/eac16a12048b45ec7c7fa94f2be3438a5f26bf9cc8580b18a1cfd609b7f6/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:127565fead0a10943b282957bd5447804ff3160ad79f2ad2635e6d249e380680", size = 382754, upload-time = "2026-06-30T07:15:21.831Z" }, + { url = "https://files.pythonhosted.org/packages/04/8f/d2f3f532616be4d06c316ef119683e832bd3d41e112bf3a88f4151c95b17/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6", size = 366189, upload-time = "2026-06-30T07:15:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/e3/29/41a7b0e98a4b44cd676ab7598419623373eb43b20be68c084935c1a8cf88/rpds_py-2026.6.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:58eadac9cd119677b60e1cf8ac4052f35949d71b8a9e5556efccbe82533cf22a", size = 377750, upload-time = "2026-06-30T07:15:24.659Z" }, + { url = "https://files.pythonhosted.org/packages/2e/05/ecda0bec46f9a1565090bcdc941d023f6a25aff85fda28f89f8d19878152/rpds_py-2026.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7491ee23305ac3eb59e492b6945881f5cd77a6f731061a3f25b77fd40f9e99a4", size = 395576, upload-time = "2026-06-30T07:15:25.987Z" }, + { url = "https://files.pythonhosted.org/packages/68/a8/6ed52f03ee6cb854ce78785cc9a9a672eb880e83fd7224d471f667d151f1/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c99f7e8ccb3dd6e3e4bfeac657a7b208c9bac8075f4b078c02d7404c34107fa", size = 543807, upload-time = "2026-06-30T07:15:27.356Z" }, + { url = "https://files.pythonhosted.org/packages/8f/d6/156c0d3eea27ba09b92562ba2364ba124c0a061b199e17eac637cd25a5e2/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62698275682bf121181861295c9181e789030a2d516071f5b8f3c23c170cd0fc", size = 611187, upload-time = "2026-06-30T07:15:28.931Z" }, + { url = "https://files.pythonhosted.org/packages/f1/31/774212ed989c62f7f310220089f9b0a3fb8f40f5443d1727abd5d9f52bc9/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a214c993455f99a89aaeadc9b21241900037adc9d97203e374d75513c5911822", size = 573030, upload-time = "2026-06-30T07:15:30.553Z" }, + { url = "https://files.pythonhosted.org/packages/c9/50/22f73127a41f1ce4f87fe39aadfb9a126345801c274aa93ae88456249327/rpds_py-2026.6.3-cp312-cp312-win32.whl", hash = "sha256:501f9f04a588d6a09179368c57071301445191767c64e4b52a6aa9871f1ef5ed", size = 202185, upload-time = "2026-06-30T07:15:32.027Z" }, + { url = "https://files.pythonhosted.org/packages/04/3a/f0ee4d4dde9d3b69dedf1b5f74e7a40017046d55052d173e418c6a94f960/rpds_py-2026.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:2c958bf94822e9290a40aaf2a822d4bc5c88099093e3948ad6c571eca9272e5f", size = 220394, upload-time = "2026-06-30T07:15:33.359Z" }, + { url = "https://files.pythonhosted.org/packages/f3/83/3382fe37f809b59f02aac04dbc4e765b480b46ee0227ed516e3bdc4d3dfc/rpds_py-2026.6.3-cp312-cp312-win_arm64.whl", hash = "sha256:22bffe6042b9bcb0822bcd1955ec00e245daf17b4344e4ed8e9551b976b63e96", size = 215753, upload-time = "2026-06-30T07:15:34.778Z" }, + { url = "https://files.pythonhosted.org/packages/a4/9e/b818ee580026ec578138e961027a68820c40afeb1ec8f6819b54fb99e196/rpds_py-2026.6.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3cfe765c1da0072636ca06628261e0ea05688e160d5c8a03e0217c3854037223", size = 343012, upload-time = "2026-06-30T07:15:36.005Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6b/686d9dc4359a8f163cfbbf89ee0b4e586431de22fe8248edb63a8cf50d49/rpds_py-2026.6.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f4d78253f6996be4901669ad25319f842f740eccf4d58e3c7f3dd39e6dde1d8f", size = 338203, upload-time = "2026-06-30T07:15:37.462Z" }, + { url = "https://files.pythonhosted.org/packages/9e/9b/069aa329940f8207615e091f5eedbbd40e1e15eac68a0790fd05ccdf796c/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54f45a148e28767bf343d33a684693c70e451c6f4c0e9904709a723fafbdfc1f", size = 367984, upload-time = "2026-06-30T07:15:39.008Z" }, + { url = "https://files.pythonhosted.org/packages/14/db/34c203e4becff3703e4d3bc121842c00b8689197f398161203a880052f4e/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:842e7b070435622248c7a2c44ae53fa1440e073cc3023bc919fed570884097a7", size = 374815, upload-time = "2026-06-30T07:15:40.253Z" }, + { url = "https://files.pythonhosted.org/packages/ee/7d/8071067d2cc453d916ad836e828c943f575e8a44612537759002a1e07381/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8020133a74bd81b4572dd8e4be028a6b1ebcd70e6726edc3918008c08bee6ee6", size = 490545, upload-time = "2026-06-30T07:15:41.729Z" }, + { url = "https://files.pythonhosted.org/packages/a3/42/da06c5aa8f0484ff07f270787434204d9f4535e2f8c3b51ed402267e63c3/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cdc7e35386f3847df728fbcb5e887e2d79c19e2fa1eba9e51b6621d23e3243af", size = 382828, upload-time = "2026-06-30T07:15:43.327Z" }, + { url = "https://files.pythonhosted.org/packages/57/d7/fe978efc2ae50abe48eb7464668ea99f53c010c60aeebb7b35ad27f23661/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acac386b453c2516111b50985d60ce46e7fadb5ea71ae7b25f4c946935bf27cf", size = 365678, upload-time = "2026-06-30T07:15:44.992Z" }, + { url = "https://files.pythonhosted.org/packages/69/9d/1d8922e1990b2a6eb532b6ff53d3e73d2b3bbffc84116c75826bee73dfc6/rpds_py-2026.6.3-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:425560c6fa0415f27261727bb20bd097568485e5eb0c121f1949417d1c516885", size = 377811, upload-time = "2026-06-30T07:15:46.523Z" }, + { url = "https://files.pythonhosted.org/packages/b1/3d/198dceafb4fb034a6a47347e1b0735d34e0bd4a50be4e898d408ee66cb14/rpds_py-2026.6.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a550fb4950a06dde3beb4721f5ad4b25bf4513784665b0a8522c792e2bd822a4", size = 395382, upload-time = "2026-06-30T07:15:47.955Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f1/13968e49655d40b6b19d8b9140296bbc6f1d86b3f0f6c346cf9f1adddf4b/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f4bca01b63096f606e095734dd56e74e175f94cfbf24ff3d63281cec61f7bb7", size = 543832, upload-time = "2026-06-30T07:15:49.33Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ab/289bcb1b90bd3e40a2900c561fa0e2087345ecbb094f0b870f2345142b7c/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ccffae9a092a00deb7efd545fe5e2c33c33b88e7c054337e9a74c179347d0b7d", size = 611011, upload-time = "2026-06-30T07:15:50.847Z" }, + { url = "https://files.pythonhosted.org/packages/1e/16/5043105e679436ccfbc8e5e0dd2d663ed18a8b8113515fd06a5e5d77c83e/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1cf01971c4f2c5553b772a542e4aaf191789cd331bc2cd4ff0e6e65ba49e1e97", size = 572431, upload-time = "2026-06-30T07:15:52.394Z" }, + { url = "https://files.pythonhosted.org/packages/85/ed/adab103321c0a6565d5ae1c2998349bc3ee175b82ccc5ae8fc04cc413075/rpds_py-2026.6.3-cp313-cp313-win32.whl", hash = "sha256:8c3d1e9c15b9d51ca0391e13da1a25a0a4df3c58a37c9dc368e0736cf7f69df0", size = 201710, upload-time = "2026-06-30T07:15:53.894Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ed/a03b09668e74e5dabbf2e211f6468e1820c0552f7b0500082da31841bf7b/rpds_py-2026.6.3-cp313-cp313-win_amd64.whl", hash = "sha256:9250a9a0a6fd4648b3f868da8d91a4c52b5811a62df58e753d50ae4454a36f80", size = 219454, upload-time = "2026-06-30T07:15:55.25Z" }, + { url = "https://files.pythonhosted.org/packages/27/17/b8642c12930b71bc2b25831f6708ccf0f75abcd11883932ec9ce54ba3a78/rpds_py-2026.6.3-cp313-cp313-win_arm64.whl", hash = "sha256:900a67df3fd1660b035a4761c4ce73c382ea6b35f90f9863c36c6fd8bf8b09bb", size = 215063, upload-time = "2026-06-30T07:15:56.573Z" }, + { url = "https://files.pythonhosted.org/packages/b6/36/7fbe9dcdaf857fb3f63c2a2284b62492d95f5e8334e947e5fb6e7f68c9be/rpds_py-2026.6.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:931908d9fc855d8f74783377822be318edb6dcb19e47169dc038f9a1bf60b06e", size = 344510, upload-time = "2026-06-30T07:15:57.921Z" }, + { url = "https://files.pythonhosted.org/packages/ba/54/f785cc3d3f60839ca57a5af4927a9f347b07b2799c373fc20f7949f87c7e/rpds_py-2026.6.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d7469697dce35be237db177d42e2a2ee26e6dcc5fc052078a6fefabd288c6edd", size = 339495, upload-time = "2026-06-30T07:15:59.238Z" }, + { url = "https://files.pythonhosted.org/packages/63/ef/d4cdaf309e6b095b43597103cf8c0b951d6cca2acce68c474f75ec12e0c7/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcfbcf66006befb9fd2aeaa9e01feaf881b4dc330a02ba07d2322b1c11be7b5d", size = 369454, upload-time = "2026-06-30T07:16:01.021Z" }, + { url = "https://files.pythonhosted.org/packages/96/4a/9559a68b7ee15db09d7981212e8c2e219d2a1d6d4faa0391d813c3496a36/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847927daf4cffbd4e90e42bc890069897101edd015f956cb8721b3473372edda", size = 374583, upload-time = "2026-06-30T07:16:02.287Z" }, + { url = "https://files.pythonhosted.org/packages/ef/75/8964aa7d2c6e8ac43eba8eb6e6b0fdda1f46d39f2fc3e6aa9f2cb17f485d/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aca6c1ef08a82bfe327cc156da694660f599923e2e6665b6d81c9c2d0ac9ffc8", size = 492919, upload-time = "2026-06-30T07:16:03.723Z" }, + { url = "https://files.pythonhosted.org/packages/8f/97/6908094ac804115e65aedfd90f1b5fee4eebebd3f6c4cfc5419939267565/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ae50181a047c871561212bb97f7932a2d45fb53e947bd9b57ebad85b529cbc53", size = 383725, upload-time = "2026-06-30T07:16:05.305Z" }, + { url = "https://files.pythonhosted.org/packages/d1/9c/0d1fdc2e7aba23e290d603bc494e97bd205bae262ce33c6b32a69768ed5e/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc319e5a1de4b6913aac94bf6a2f9e847371e0a140a43dd4991db1a09bc2d504", size = 367255, upload-time = "2026-06-30T07:16:07.086Z" }, + { url = "https://files.pythonhosted.org/packages/c4/fe/f0209ca4a9ed074bc8acb44dfd0e81c3122e94c9689f5645b7973a866719/rpds_py-2026.6.3-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e4316bf32babbed84e691e352faf967ce2f0f024174a8643c37c94a1080374fc", size = 379060, upload-time = "2026-06-30T07:16:08.525Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8d/f1cc54c616b9d8897de8738aac148d20afca93f68187475fe194d09a71b9/rpds_py-2026.6.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8c6e5a2f750cc71c3e3b11d71661f21d6f9bc6cebc6564b1466417a1ec03ec77", size = 395960, upload-time = "2026-06-30T07:16:09.989Z" }, + { url = "https://files.pythonhosted.org/packages/fb/04/aafff00f73aeca2945f734f1d483c64ab8f472d0864ab02377fd8e89c3b2/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4470ce197d4090875cf6affbf1f853338387428df97c4fb7b7106317b8214698", size = 545356, upload-time = "2026-06-30T07:16:11.816Z" }, + { url = "https://files.pythonhosted.org/packages/fd/cc/e229663b9e4ddac5a4acbe9085dd80a71af2a5d356b8b39d6bff233f24b0/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea964164cc9afa72d4d9b23cc28dafae93693c0a53e0b42acbff15b22c3f9ddd", size = 612319, upload-time = "2026-06-30T07:16:13.586Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7a/8a0e6d3e6cd066af108b71b43122c3fe158dd9eb86acac626593a2582eb1/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:639c8929aa0afe81be836b04de888460d6bed38b9c54cfc18da8f6bfabf5af5d", size = 573508, upload-time = "2026-06-30T07:16:15.23Z" }, + { url = "https://files.pythonhosted.org/packages/87/03/2a69ab618a789cf6cf85c86bb844c62d090e700ab1a2aa676b3741b6c516/rpds_py-2026.6.3-cp314-cp314-win32.whl", hash = "sha256:882076c00c0a608b131187055ddc5ae29f2e7eaf870d6168980420d58528a5c8", size = 202504, upload-time = "2026-06-30T07:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/85/62/a3892ba945f4e24c78f352e5de3c7620d8479f73f211406a97263d13c7d2/rpds_py-2026.6.3-cp314-cp314-win_amd64.whl", hash = "sha256:0be972be84cfcaf46c8c6edf690ca0f154ac17babf1f6a955a51579b34ad2dc5", size = 220380, upload-time = "2026-06-30T07:16:18.108Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e7/c2bd44dc831931815ad11ebb5f430b5a0a4d3caa9de837107876c30c3432/rpds_py-2026.6.3-cp314-cp314-win_arm64.whl", hash = "sha256:2a9c6f195058cb45335e8cc3802745c603d716eb96bc9625950c1aac71c0c703", size = 215976, upload-time = "2026-06-30T07:16:19.654Z" }, + { url = "https://files.pythonhosted.org/packages/79/9c/fff7b74bce9a091ec9a012a03f9ff5f69364eaf9451060dfc4486da2ffdd/rpds_py-2026.6.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:f90938e92afda60266da758ee7d363447f7f0138c9559f9e1811629580582d90", size = 346840, upload-time = "2026-06-30T07:16:21.268Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/77bcb1168b33704908295533d27f10eb811e9e3e193e8993dc99572211d3/rpds_py-2026.6.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ec829541c45bca16e61c7ae50c20501f213605beb75d1aba91a6ee37fbbb56a4", size = 340282, upload-time = "2026-06-30T07:16:22.875Z" }, + { url = "https://files.pythonhosted.org/packages/87/3c/7a9081c7c9e645b39efe19e4ffbeccd80add246327cd9b888aecffd72317/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afd70d95892096cdb26f15a00c45907b17817577aa8d1c76b2dcc2788391f9e9", size = 370403, upload-time = "2026-06-30T07:16:24.415Z" }, + { url = "https://files.pythonhosted.org/packages/f7/69/af47021eb7dad6ff3396cb001c08f0f3c4d06c20253f75be6421a59fe6b7/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dfa0533a5d4c94d4dfa1b694fcb56c9c63aad8330ffdd816fd225d0a7a162f", size = 376055, upload-time = "2026-06-30T07:16:26.111Z" }, + { url = "https://files.pythonhosted.org/packages/81/fc/a3bcf517084396a6dd258c592567a3c011ba4557f2fde23dceaf26e74f2e/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:af05d726809bff6b141be124d4c7ce998f9c9c7f30edb1f46c07aa103d540b41", size = 494419, upload-time = "2026-06-30T07:16:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/c9/eb/13d529d1788135425c7bf207f8463458ca5d92e43f3f701365b83e9dffc1/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9826217f048f620d9a712672818bf231442c1b35d96b227a07eabd11b4bb6945", size = 384848, upload-time = "2026-06-30T07:16:29.183Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f4/b7ac49f30013aba8f7b9566b1dd07e81de95e708c1374b7bacc5b9bc5c9c/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:536bceea4fa4acf7e1c61da2b5786304367c816c8895be71b8f537c480b0ea1f", size = 371369, upload-time = "2026-06-30T07:16:30.912Z" }, + { url = "https://files.pythonhosted.org/packages/31/86/6260bafa622f788b07ddec0e52d810305c8b9b0b8c27f58a2ab04bf62b4f/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:bc0011654b91cc4fb2ae701bec0a0ba1e552c0714247fa7af6c59e0ccfa3a4e1", size = 379673, upload-time = "2026-06-30T07:16:32.486Z" }, + { url = "https://files.pythonhosted.org/packages/19/c3/03f1ee79a047b48daeca157c89a18509cde22b6b951d642b9b0af1be660a/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:539d75de9e0d536c84ff18dfeb805398e58227001ce09231a26a08b9aed1ee0e", size = 397500, upload-time = "2026-06-30T07:16:34.471Z" }, + { url = "https://files.pythonhosted.org/packages/f0/95/8ed0cd8c377dca12aea498f119fe639fc474d1461545c39d2b5872eb1c0f/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:166cf54d9f44fc6ceb53c7860258dde44a81406646de79f8ed3234fca3b6e538", size = 545978, upload-time = "2026-06-30T07:16:36.45Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f2/0eb57f0eaa83f8fc152a7e03de968ab77e1f00732bebc892b190c6eebde7/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d34c20167764fbcf927194d532dd7e0c56772f0a5f943fa5ef9e9afbba8fb9db", size = 613350, upload-time = "2026-06-30T07:16:38.213Z" }, + { url = "https://files.pythonhosted.org/packages/5b/de/e0674bdbc3ef7634989b3f854c3f34bc1f587d36e5bfdc5c378d57034619/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea7bb13b7c9a29791f87a0387ba7d3ad3a6d783d827e4d3f27b40a0ff44495e2", size = 576486, upload-time = "2026-06-30T07:16:39.797Z" }, + { url = "https://files.pythonhosted.org/packages/f2/f6/21101359743cd136ada781e8210a85769578422ba460672eea0e29739200/rpds_py-2026.6.3-cp314-cp314t-win32.whl", hash = "sha256:6de4744d05bd1aa1be4ed7ea1189e3979196808008113bbbf899a460966b925e", size = 201068, upload-time = "2026-06-30T07:16:41.316Z" }, + { url = "https://files.pythonhosted.org/packages/a6/b2/9574d4d44f7760c2aa32d92a0a4f41698e33f5b204a0bf5c9758f52c79d5/rpds_py-2026.6.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c7b9a2f8f4d8e90af72571d3d495deebdd7e3c75451f5b41719aee166e940fc2", size = 220600, upload-time = "2026-06-30T07:16:43.091Z" }, + { url = "https://files.pythonhosted.org/packages/08/ae/f23a2697e6ee6340a578b0f136be6483657bef0c6f9497b752bb5c0964bb/rpds_py-2026.6.3-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:e059c5dde6452b44424bd1834557556c226b57781dee1227af23518459722b13", size = 344726, upload-time = "2026-06-30T07:16:44.5Z" }, + { url = "https://files.pythonhosted.org/packages/c3/63/e7b3a1a5358dd32c930a1062d8e15b67fd6e8922e81df9e91706d66ee5c8/rpds_py-2026.6.3-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2f7c26fbc5acd2522b95d4177fe4710ffd8e9b20529e703ffbf8db4d93903f05", size = 339587, upload-time = "2026-06-30T07:16:46.255Z" }, + { url = "https://files.pythonhosted.org/packages/ec/64/10a85681916ca55fffb91b0a211f84e34297c109243484dd6394660a8a7c/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3086b538543802f84c843911242db20447de00d8752dd0efc936dbcf02218ba", size = 369585, upload-time = "2026-06-30T07:16:48.101Z" }, + { url = "https://files.pythonhosted.org/packages/76/c2/baf95c7c38823e12ba34407c5f5767a89e5cf2233895e56f608167ae9493/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f2e5c5ee828d42cb11760761c0af6507927bec42d0ad5458f97c9203b054617", size = 375479, upload-time = "2026-06-30T07:16:49.93Z" }, + { url = "https://files.pythonhosted.org/packages/6a/94/0aad06c72d65101e11d33528d438cda99a39ce0da99466e156158f2541d3/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ed0c1e5d10cdc7135537988c74a0188da68e2f3c30813ba3744ab1e42e0480f9", size = 492418, upload-time = "2026-06-30T07:16:51.641Z" }, + { url = "https://files.pythonhosted.org/packages/b5/17/de3f5a479a1f056535d7489819639d8cd591ea6281d700390b43b1abd745/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c2642a7603ec0b16ed77da4555db3b4b472341904873788327c0b0d7b95f1bb", size = 384123, upload-time = "2026-06-30T07:16:53.622Z" }, + { url = "https://files.pythonhosted.org/packages/46/7d/bf09bd1b145bb2671c03e1e6d1ab8651858d90d8c7dfeadd85a37a934fd8/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e4320744c1ffdd95a603def63344bfab2d33edeab301c5007e7de9f9f5b3885", size = 367351, upload-time = "2026-06-30T07:16:55.241Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ea/1bb734f314b8be319149ddee80b18bd41372bdcfbdf88d28131c0cd37719/rpds_py-2026.6.3-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:a9f4645593036b81bbdb36b9c8e0ea0d1c3fee968c4d59db0344c14087ef143a", size = 378827, upload-time = "2026-06-30T07:16:56.841Z" }, + { url = "https://files.pythonhosted.org/packages/4b/93/d9611e5b25e26df9a3649813ed66193ace9347a7c7fc4ab7cf70e94851c0/rpds_py-2026.6.3-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e55d236be29255554da47abe5c577637db7c24a02b8b46f0ca9524c855801868", size = 395966, upload-time = "2026-06-30T07:16:58.557Z" }, + { url = "https://files.pythonhosted.org/packages/c3/cb/99d77e16e5534ae1d90629bbe419ba6ee170833a6a85e3aa1cc41726fbbc/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:24e9c5386e16669b674a69c156c8eeefcb578f3b3397b713b08e6d60f3c7b187", size = 545680, upload-time = "2026-06-30T07:17:00.164Z" }, + { url = "https://files.pythonhosted.org/packages/59/15/11a29755f790cef7a2f755e8e14f4f0c33f39489e1893a632a2eee59672b/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:c60924535c75f1566b6eb75b5c31a48a43fef04fa2d0d201acbad8a9969c6107", size = 611853, upload-time = "2026-06-30T07:17:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/0c27547e21644da938fb530f7e1a8148dd24d02db07e7a5f2567a17ce710/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:38a2fea2787428f811719ceb9114cb78964a3138838320c29ac39526c79c16ba", size = 573715, upload-time = "2026-06-30T07:17:03.693Z" }, + { url = "https://files.pythonhosted.org/packages/29/71/4d8fcf700931815594bce892255bbd973b94efaf0fc1932b0590df18d886/rpds_py-2026.6.3-cp315-cp315-win32.whl", hash = "sha256:d483fe17f01ad64b7bf7cc38fcefff1ca9fb83f8c2b2542b68f97ffe0611b369", size = 202864, upload-time = "2026-06-30T07:17:05.746Z" }, + { url = "https://files.pythonhosted.org/packages/eb/62/b577562de0edbb55b2be85ce5fd09c33e386b9b13eee09833af4240fd5c4/rpds_py-2026.6.3-cp315-cp315-win_amd64.whl", hash = "sha256:67e3a721ffc5d8d2210d3671872298c4a84e4b8035cfe42ffd7cde35d772b146", size = 220430, upload-time = "2026-06-30T07:17:07.471Z" }, + { url = "https://files.pythonhosted.org/packages/c8/95/d6d0b2509825141eef60669a5739eec88dbc6a48053d6c92993a5704defe/rpds_py-2026.6.3-cp315-cp315-win_arm64.whl", hash = "sha256:6e84adbcf4bf841aed8116a8264b9f50b4cb3e7bd89b516122e616ac56ca269e", size = 215877, upload-time = "2026-06-30T07:17:09.008Z" }, + { url = "https://files.pythonhosted.org/packages/b7/bf/f3ea278f0afd615c1d0f19cb69043a41526e2bb600c2b536eb192218eb27/rpds_py-2026.6.3-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:ae6dd8f10bd17aad820876d24caec9efdafd80a318d16c0a48edb5e136902c6b", size = 346933, upload-time = "2026-06-30T07:17:10.762Z" }, + { url = "https://files.pythonhosted.org/packages/9d/29/9907bdf1c5346763cf10b7f6852aad86652168c259def904cbe0082c5864/rpds_py-2026.6.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:bdbd97738551fca3917c1bd7188bec1920bb520104f28e7e1007f9ceb17b7690", size = 340274, upload-time = "2026-06-30T07:17:12.266Z" }, + { url = "https://files.pythonhosted.org/packages/6f/2c/8e03767b5778ef25cebf74a7a91a2c3806f8eced4c92cb7406bbe060756d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b95977e7211527ab0ba576e286d023389fbeeb32a6b7b771665d333c60e5342", size = 370763, upload-time = "2026-06-30T07:17:14.107Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e1/df2a7e1ba2efd796af26194250b8d42c821b46592311595162af9ef0528d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d15fde0e6fb0d88a60d221204873743e5d9f0b7d29165e62cd86d0413ad74ba6", size = 376467, upload-time = "2026-06-30T07:17:15.76Z" }, + { url = "https://files.pythonhosted.org/packages/6b/de/8a0814d1946af29cb068fb259aa8622f856df1d0bab58429448726b537f5/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a136d453475ac0fcbda502ef1e6504bd28d6d904700915d278deeab0d00fe140", size = 496689, upload-time = "2026-06-30T07:17:17.308Z" }, + { url = "https://files.pythonhosted.org/packages/df/f3/f19e0c852ba13694f5a79f3b719331051573cb5693feacf8a88ffffc3a71/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f826877d462181e5eb1c26a0026b8d0cab05d99844ecb6d8bf3627a2ca0c0442", size = 385340, upload-time = "2026-06-30T07:17:18.928Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ae/7ec3a9d2d4351f99e37bcb06b6b6f954512646bfdbf9742e1de727865daf/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79486287de1730dbaff3dbd124d0ca4d2ef7f9d29bf2544f1f93c09b5bcbbd12", size = 372179, upload-time = "2026-06-30T07:17:20.539Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ac/9cee911dff2aaa9a5a8354f6610bf2e6a616de9197c5fff4f54f82585f1e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:808345f53cb952433ca2816f1604ff3515608a81784954f38d4452acfe8e61d5", size = 379993, upload-time = "2026-06-30T07:17:22.212Z" }, + { url = "https://files.pythonhosted.org/packages/83/6b/7c2a07ba88d1e9a936612f7a5d067467ed03d971d5a06f7d309dff044a7e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1967debc37f64f2c4dc90a7f563aec558b471966e12adcac4e1c4240496b6ebf", size = 398909, upload-time = "2026-06-30T07:17:23.66Z" }, + { url = "https://files.pythonhosted.org/packages/97/0b/776ffcb66783637b0031f6d58d6fb55913c8b5abf00aeecd46bf933fb477/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:f0840b5b17057f7fd918b76183a4b5a0635f43e14eb2ce60dce1d4ee4707ea00", size = 546584, upload-time = "2026-06-30T07:17:25.264Z" }, + { url = "https://files.pythonhosted.org/packages/55/33/ba3bc04d7092bd553c9b2b195624992d2cc4f3de1f380b7b93cbee67bd79/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:faa679d19a6696fd54259ad321251ad77a13e70e03dd834daa762a44fb6196ef", size = 614357, upload-time = "2026-06-30T07:17:26.888Z" }, + { url = "https://files.pythonhosted.org/packages/8b/71/14edf065f04630b1a8472f7653cad03f6c478bcf95ea0e6aed55451e33ea/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:23a439f31ccbeff1574e24889128821d1f7917470e830cf6544dced1c662262a", size = 576533, upload-time = "2026-06-30T07:17:28.546Z" }, + { url = "https://files.pythonhosted.org/packages/ba/76/65002b08596c389105720a8c0d22298b8dc25a4baf89b2ce431343c8b1de/rpds_py-2026.6.3-cp315-cp315t-win32.whl", hash = "sha256:913ca42ccad3f8cc6e292b587ae8ae49c8c823e5dce51a736252fc7c7cdfa577", size = 201204, upload-time = "2026-06-30T07:17:30.193Z" }, + { url = "https://files.pythonhosted.org/packages/8c/97/d855d6b3c322d1f27e26f5241c42016b56cf01377ea8ed348285f54652f0/rpds_py-2026.6.3-cp315-cp315t-win_amd64.whl", hash = "sha256:ae3d4fe8c0b9213624fdce7279d70e3b148b682ca20719ebd193a23ebfa47324", size = 220719, upload-time = "2026-06-30T07:17:31.788Z" }, + { url = "https://files.pythonhosted.org/packages/b4/9c/f0d19ac587fd0e4ab6b72cda355e9c5a6166b01ef7e064e437aef8eb9fef/rpds_py-2026.6.3-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:4cf2d36a2357e4d07bb5a4f98801265327b48256867816cfd2ceb001e9754a8f", size = 349791, upload-time = "2026-06-30T07:17:33.315Z" }, + { url = "https://files.pythonhosted.org/packages/38/c7/1d49d204c9fd2ee6c537601dc4c1ba921e03363ca576bfab94a00254ac9a/rpds_py-2026.6.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:30c6dc199b24a5e3e81d50da0f00858c5bbdb2617a750395687f4339c5818171", size = 352842, upload-time = "2026-06-30T07:17:34.897Z" }, + { url = "https://files.pythonhosted.org/packages/ac/e5/c0b5dc93cd0d4c06ce1f438907649514e2ea077bcd911e3154a51e96c38e/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9891e594296ab9dada6551c8e7b387b2721f27a67eecd528412e8906247a7b90", size = 382094, upload-time = "2026-06-30T07:17:36.514Z" }, + { url = "https://files.pythonhosted.org/packages/0d/54/ec0e907b4ca8d541112db352409bd15f871c9b243e0c92c9b5a46ae96f01/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b5c2dc92304aa48a4a60443b548bb12f12e119d4b72f314015e67b9e1be97fca", size = 388662, upload-time = "2026-06-30T07:17:38.235Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f4/921c22a4fd0f1c1ac13a3996ffbf0aa67951e2c8ad0d1d9574938a2932e8/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:127e08c0642d880cf32ca47ec2a4a77b901f7e2dd1ad9762adb13955d72ffcc9", size = 504896, upload-time = "2026-06-30T07:17:39.689Z" }, + { url = "https://files.pythonhosted.org/packages/0b/1b/a114b972cefa1ab1cdb3c7bb177cd3844a12826c507c722d3a73516dbbaf/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8bb68f03f395eb793220b45c097bd4d8c32944393da0fad8b999efac0868fc8c", size = 391545, upload-time = "2026-06-30T07:17:41.336Z" }, + { url = "https://files.pythonhosted.org/packages/4e/98/af9b3db77d47fcbe6c8c1f36e2c2147ec70292819e99c325f871584a1c11/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3450b693fde92133e9f51060568a4c31fcca76d5e53bbd611e689ca446517e9", size = 380059, upload-time = "2026-06-30T07:17:42.857Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ba/0efd8668b97c1d26a61566386c636a7a7a09829e474fdf807caa15a2c844/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:5e8d07bddee435a2ff6f1920e18feff28d0bc4533e42f4bf6927fbd073312c41", size = 393235, upload-time = "2026-06-30T07:17:44.637Z" }, + { url = "https://files.pythonhosted.org/packages/62/90/8c139ee9690f73b0829f32647de6f40d826f8f443af6fa72644f96351aac/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3a83ae6c67b7676b9878378547ca8e93ed77a580037bcbcd1d32f739e1e6089c", size = 413008, upload-time = "2026-06-30T07:17:46.225Z" }, + { url = "https://files.pythonhosted.org/packages/9c/97/0043896fdd7828ce09a1d9a8b06433714d0960fc4ff3fc4aa72b666b764e/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:2bfd04c19ddbd6640de0b51894d764bd2758854d5b75bd102d2ef10cb9c293a9", size = 558118, upload-time = "2026-06-30T07:17:47.759Z" }, + { url = "https://files.pythonhosted.org/packages/f6/40/02355f0e134f783a8f9814c4680a1bd311d37671577a5964ea838573ff37/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:ca6546b66be9dc4738b1b043d5ebd5488c66c578c5ff0fd0e8065313fe3afb76", size = 623138, upload-time = "2026-06-30T07:17:49.355Z" }, + { url = "https://files.pythonhosted.org/packages/10/85/48f0abdcef5cce4e034c7a5b0ceeceba0b01bf0d942824f4bb720afe2dec/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:8e65860d238379ed982fd9ba690579b5e95af2f4840f99c772816dbe573cb826", size = 586486, upload-time = "2026-06-30T07:17:51.141Z" }, +] + [[package]] name = "ruff" version = "0.16.5" From 4d65dd0874e77f59c4c1111ce3c24f96c2577408 Mon Sep 17 00:00:00 2001 From: Nina Chikanov Date: Thu, 10 Sep 2026 17:15:43 -0700 Subject: [PATCH 7/9] [FIX]: Serialize records as JSON text Rename the record helpers to serialize_record and deserialize_record, accepting ResultRecord and JSON text respectively. Keep dictionary conversion on the existing record methods. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/concepts/trace-schema.md | 18 +++- rampart/core/serialization.py | 63 ++++++++--- tests/unit/core/test_serialization.py | 145 +++++++++++++++++++------- 3 files changed, 169 insertions(+), 57 deletions(-) diff --git a/docs/concepts/trace-schema.md b/docs/concepts/trace-schema.md index d9dfc16c..052b3357 100644 --- a/docs/concepts/trace-schema.md +++ b/docs/concepts/trace-schema.md @@ -3,8 +3,9 @@ `rampart.core.serialization` defines RAMPART's canonical, versioned `Result`-record format. `ResultRecord.to_dict()` / `ResultRecord.from_dict()` own the versioned envelope and optional `pytest_nodeid` / `result_index` attribution. -`serialize_result()` / `deserialize_result()` are convenience functions. Existing -xdist and reporting consumers are not yet wired to this module. +`serialize_record(record=...)` converts a `ResultRecord` to JSON text (`str`); +`deserialize_record(data=...)` reconstructs a `ResultRecord` from JSON text. +Existing xdist and reporting consumers are not yet wired to this module. This page defines how the schema may evolve as consumers adopt it. @@ -16,6 +17,19 @@ are fragments, not standalone durable records: persist a `ResultRecord` to include the version. `ResultRecord` references the live result; serialization does not mutate it. +The dictionary methods remain available for projections and structured +inspection. The serialization functions use those same methods at the JSON-text +boundary, without a separate codec: + +```python +record = ResultRecord(result=result, pytest_nodeid="tests/test_safety.py::test_case") +text = serialize_record(record=record) +restored = deserialize_record(data=text) +``` + +Malformed JSON and invalid record values raise `SchemaError`; unsupported +versions raise its `UnsupportedSchemaVersionError` subclass. + The adapter validates nested fields without string, boolean, or integer coercion. Dictionary input is checked for JSON-only values before strict JSON-mode validation reconstructs the dataclasses. Missing fields use their diff --git a/rampart/core/serialization.py b/rampart/core/serialization.py index b9c8a521..2ccbf2bf 100644 --- a/rampart/core/serialization.py +++ b/rampart/core/serialization.py @@ -5,6 +5,7 @@ from __future__ import annotations +import json from collections.abc import Mapping from dataclasses import dataclass, replace from typing import ( @@ -18,6 +19,7 @@ if TYPE_CHECKING: from collections.abc import Callable + from typing import Never from pydantic.json_schema import JsonSchemaValue @@ -71,7 +73,7 @@ def __post_init__(self) -> None: raise SchemaError(msg) def to_dict(self) -> dict[str, Any]: - """Serialize the envelope using the single Result body codec. + """Convert the envelope to a dict using the single Result body codec. Returns: dict[str, Any]: A versioned, JSON-safe record. @@ -142,31 +144,58 @@ def json_schema(cls) -> JsonSchemaValue: } -def serialize_result( - *, - result: Result, - pytest_nodeid: str | None = None, - result_index: int | None = None, -) -> dict[str, Any]: - """Serialize a result with its optional attribution. +def serialize_record(*, record: ResultRecord) -> str: + """Serialize a canonical record to JSON text. + + Args: + record (ResultRecord): The result and its optional attribution. Returns: - dict[str, Any]: The canonical versioned record. + str: JSON text containing the versioned record. + + Raises: + SchemaError: If the record cannot be represented as canonical JSON. """ - return ResultRecord( - result=result, - pytest_nodeid=pytest_nodeid, - result_index=result_index, - ).to_dict() + data = record.to_dict() + try: + return json.dumps(data, allow_nan=False) + except (ValueError, RecursionError) as exc: + msg = f"record: cannot serialize JSON ({exc})" + raise SchemaError(msg) from exc -def deserialize_result(*, data: object) -> ResultRecord: - """Deserialize a canonical record. +def deserialize_record(*, data: str) -> ResultRecord: + """Deserialize a canonical record from JSON text. + + Args: + data (str): JSON text containing a versioned record. Returns: ResultRecord: The result and its attribution. + + Raises: + SchemaError: If the input is not JSON text or the record is malformed. + UnsupportedSchemaVersionError: If the version is unsupported. + """ + if not isinstance(data, str): + msg = "record: expected a JSON string" + raise SchemaError(msg) + try: + decoded = json.loads(data, parse_constant=_reject_json_constant) + except (ValueError, RecursionError) as exc: + msg = f"record: invalid JSON ({exc})" + raise SchemaError(msg) from exc + return ResultRecord.from_dict(decoded) + + +def _reject_json_constant(value: str) -> Never: + """Reject the non-finite constants accepted by Python's JSON parser. + + Raises: + ValueError: Always, because these constants are not valid JSON numbers. """ - return ResultRecord.from_dict(data) + msg = f"non-finite number {value}" + raise ValueError(msg) def _decode_v1(data: Mapping[str, Any]) -> ResultRecord: diff --git a/tests/unit/core/test_serialization.py b/tests/unit/core/test_serialization.py index 0c2df486..6a9a2a2a 100644 --- a/tests/unit/core/test_serialization.py +++ b/tests/unit/core/test_serialization.py @@ -34,8 +34,8 @@ ResultRecord, SchemaError, UnsupportedSchemaVersionError, - deserialize_result, - serialize_result, + deserialize_record, + serialize_record, ) from rampart.core.types import ( EvalOutcome, @@ -141,11 +141,13 @@ def _freeform_maps(result: Result) -> list[MutableMapping[str, Any]]: class TestRoundTrip: def test_full_result_round_trips_to_equal_value(self) -> None: - original = _make_full_result() - encoded = ResultRecord(result=original).to_dict() + original = ResultRecord(result=_make_full_result()) + encoded = serialize_record(record=original) - decoded = deserialize_result(data=encoded).result + decoded = deserialize_record(data=encoded) + assert isinstance(encoded, str) + assert json.loads(encoded) == original.to_dict() assert decoded == original def test_version_is_stamped_on_the_record(self) -> None: @@ -154,37 +156,43 @@ def test_version_is_stamped_on_the_record(self) -> None: assert encoded["version"] == TRACE_SCHEMA_VERSION assert ResultRecord.VERSION == "rampart.trace.v1" - def test_serialize_result_builds_attribution_collar(self) -> None: - encoded = serialize_result( + def test_serialize_record_includes_attribution(self) -> None: + record = ResultRecord( result=_make_full_result(), pytest_nodeid="tests/test_x.py::test_x", result_index=2, ) + encoded = json.loads(serialize_record(record=record)) + assert encoded["pytest_nodeid"] == "tests/test_x.py::test_x" assert encoded["result_index"] == 2 - def test_serialize_result_omits_attribution_when_unset(self) -> None: - encoded = serialize_result(result=_make_full_result()) + def test_serialize_record_omits_attribution_when_unset(self) -> None: + record = ResultRecord(result=_make_full_result()) + + encoded = json.loads(serialize_record(record=record)) assert "pytest_nodeid" not in encoded assert "result_index" not in encoded - def test_attribution_collar_round_trips(self) -> None: - encoded = serialize_result( + @pytest.mark.parametrize("index", [None, 0, 2]) + def test_attribution_collar_round_trips(self, index: int | None) -> None: + record = ResultRecord( result=_make_full_result(), pytest_nodeid="tests/test_x.py::test_x", - result_index=2, + result_index=index, ) + encoded = serialize_record(record=record) - decoded = deserialize_result(data=encoded) + decoded = deserialize_record(data=encoded) assert decoded.pytest_nodeid == "tests/test_x.py::test_x" - assert decoded.result_index == 2 + assert decoded.result_index == index def test_nested_values_survive_the_round_trip(self) -> None: - decoded = deserialize_result( - data=ResultRecord(result=_make_full_result()).to_dict() + decoded = deserialize_record( + data=serialize_record(record=ResultRecord(result=_make_full_result())) ).result turn = decoded.turns[0] @@ -202,6 +210,62 @@ def test_nested_values_survive_the_round_trip(self) -> None: id="pop-1", index=0, size=5, threshold=0.8 ) + def test_unicode_and_escaped_text_round_trip(self) -> None: + result = _make_full_result() + result.summary = ( + 'Quoted "text"\nwith backslash \\ and Unicode \u00e9 \U0001f600' + ) + record = ResultRecord(result=result) + + encoded = serialize_record(record=record) + + assert json.loads(encoded)["result"]["summary"] == result.summary + assert deserialize_record(data=encoded) == record + + +class TestJsonTextBoundary: + @pytest.mark.parametrize( + "data", ["", "{", '{"version":', "{} trailing", "{'key': 1}"] + ) + def test_malformed_json_raises_schema_error(self, data: str) -> None: + with pytest.raises(SchemaError, match="record: invalid JSON") as error: + deserialize_record(data=data) + + assert isinstance(error.value.__cause__, json.JSONDecodeError) + + @pytest.mark.parametrize("data", [{}, [], None, 1, b"{}"]) + def test_deserialization_requires_text(self, data: Any) -> None: + with pytest.raises(SchemaError, match="record: expected a JSON string"): + deserialize_record(data=data) + + @pytest.mark.parametrize("value", [math.nan, math.inf, -math.inf]) + def test_nonfinite_json_constants_are_rejected(self, value: float) -> None: + data = json.dumps({**_minimal_record_dict(), "future": value}) + + with pytest.raises(SchemaError, match=r"record: invalid JSON.*non-finite"): + deserialize_record(data=data) + + def test_serialization_preserves_metadata_policy(self) -> None: + result = _make_full_result( + metadata={ + "_rampart_source_worker": object(), + "nested": {"_rampart_source_worker": "keep"}, + } + ) + + encoded = serialize_record(record=ResultRecord(result=result)) + + assert json.loads(encoded)["result"]["metadata"] == { + "nested": {"_rampart_source_worker": "keep"} + } + assert "_rampart_source_worker" in result.metadata + + def test_serialization_rejects_non_json_values(self) -> None: + record = ResultRecord(result=_make_full_result(metadata={"bad": math.nan})) + + with pytest.raises(SchemaError, match="metadata"): + serialize_record(record=record) + class TestFieldExhaustiveness: def test_every_field_of_every_type_is_serialized(self) -> None: @@ -231,15 +295,16 @@ def test_unknown_major_fails_closed(self) -> None: data = {"version": "rampart.trace.v2", "result": {}} with pytest.raises(UnsupportedSchemaVersionError, match="v2"): - deserialize_result(data=data) + deserialize_record(data=json.dumps(data)) def test_missing_version_fails_closed(self) -> None: with pytest.raises(UnsupportedSchemaVersionError): - deserialize_result(data={"result": {}}) + deserialize_record(data='{"result": {}}') - def test_non_mapping_record_fails_closed(self) -> None: + @pytest.mark.parametrize("data", ["[1, 2, 3]", "null", "1", '"text"']) + def test_non_mapping_record_fails_closed(self, data: str) -> None: with pytest.raises(SchemaError, match="mapping"): - deserialize_result(data=[1, 2, 3]) + deserialize_record(data=data) class TestMigrationTolerance: @@ -248,12 +313,12 @@ def test_unknown_extra_fields_decode(self) -> None: encoded["future_collar"] = {"anything": True} encoded["result"]["future_intrinsic"] = 42 - decoded = deserialize_result(data=encoded).result + decoded = deserialize_record(data=json.dumps(encoded)).result assert decoded.status is SafetyStatus.UNSAFE def test_missing_optional_fields_use_defaults(self) -> None: - decoded = deserialize_result(data=_minimal_record_dict()).result + decoded = deserialize_record(data=json.dumps(_minimal_record_dict())).result assert decoded.status is SafetyStatus.SAFE assert decoded.turns == [] @@ -268,14 +333,14 @@ def test_malformed_present_list_fails_closed(self) -> None: data["result"]["turns"] = "not-a-list" with pytest.raises(SchemaError, match=r"result\.turns"): - deserialize_result(data=data) + ResultRecord.from_dict(data) def test_incomplete_population_reference_fails_closed(self) -> None: data = _minimal_record_dict() data["result"]["population"] = {} with pytest.raises(SchemaError, match=r"result\.population\.id"): - deserialize_result(data=data) + ResultRecord.from_dict(data) class TestValueDomain: @@ -293,7 +358,7 @@ def test_harm_category_is_passed_through_as_string(self) -> None: result.harm_category = "custom_product_risk" encoded = ResultRecord(result=result).to_dict() - decoded = deserialize_result(data=encoded).result + decoded = ResultRecord.from_dict(encoded).result assert encoded["result"]["harm_category"] == "custom_product_risk" assert decoded.harm_category == "custom_product_risk" @@ -316,7 +381,7 @@ def test_bad_enum_value_fails_closed_on_decode(self) -> None: data["result"]["status"] = "not_a_status" with pytest.raises(SchemaError, match="status"): - deserialize_result(data=data) + ResultRecord.from_dict(data) def test_non_string_harm_category_fails_closed_on_encode(self) -> None: result = _make_full_result() @@ -330,11 +395,11 @@ def test_non_string_harm_category_fails_closed_on_decode(self) -> None: data["result"]["harm_category"] = {"category": "custom"} with pytest.raises(SchemaError, match="harm_category"): - deserialize_result(data=data) + ResultRecord.from_dict(data) def test_boolean_result_index_fails_before_encoding(self) -> None: with pytest.raises(SchemaError, match="result_index"): - serialize_result(result=_make_full_result(), result_index=True) + ResultRecord(result=_make_full_result(), result_index=True) class TestBinaryPayloadFailsClosed: @@ -373,13 +438,13 @@ def test_decoding_a_binary_payload_fails_closed(self) -> None: ] with pytest.raises(SchemaError, match="binary payload"): - deserialize_result(data=data) + ResultRecord.from_dict(data) @pytest.mark.parametrize("payload_format", ["pdf", "docx", "text"]) def test_artifact_is_rejected_before_filesystem_access( self, payload_format: str ) -> None: - data = serialize_result(result=_make_full_result()) + data = ResultRecord(result=_make_full_result()).to_dict() payload = data["result"]["turns"][0]["request"]["attachments"][0] payload.update(format=payload_format, artifact="untrusted-artifact") @@ -389,7 +454,7 @@ def test_artifact_is_rejected_before_filesystem_access( ), pytest.raises(SchemaError, match="artifact"), ): - deserialize_result(data=data) + ResultRecord.from_dict(data) def test_live_binary_payload_is_still_supported(self, tmp_path: Path) -> None: artifact = tmp_path / "doc.pdf" @@ -431,7 +496,7 @@ def test_python_iso_datetimes_preserve_their_wire_text( result.turns[0].__dict__["timestamp"] = timestamp result.turns[0].response.tool_calls[0].timestamp = timestamp - encoded = serialize_result(result=result) + encoded = ResultRecord(result=result).to_dict() assert encoded["result"]["turns"][0]["timestamp"] == timestamp.isoformat() assert ResultRecord.from_dict(encoded).result == result @@ -604,7 +669,7 @@ def test_generated_schema_is_valid_and_matches_checked_in_contract(self) -> None def test_schema_and_decoder_agree_on_payload_formats( self, payload_format: PayloadFormat ) -> None: - data = serialize_result(result=_make_full_result()) + data = ResultRecord(result=_make_full_result()).to_dict() data["result"]["turns"][0]["request"]["attachments"][0]["format"] = ( payload_format.value ) @@ -620,7 +685,11 @@ def test_schema_and_decoder_agree_on_payload_formats( @pytest.mark.parametrize("full", [False, True]) def test_full_and_minimal_records_conform(self, *, full: bool) -> None: data = ( - serialize_result(result=_make_full_result(), result_index=0) + json.loads( + serialize_record( + record=ResultRecord(result=_make_full_result(), result_index=0) + ) + ) if full else _minimal_record_dict() ) @@ -633,7 +702,7 @@ def test_full_and_minimal_records_conform(self, *, full: bool) -> None: validator.validate(ResultRecord.from_dict(data).to_dict()) def test_unknown_additive_fields_are_allowed_at_every_level(self) -> None: - data = serialize_result(result=_make_full_result()) + data = ResultRecord(result=_make_full_result()).to_dict() body = data["result"] turn = body["turns"][0] objects = [ @@ -676,7 +745,7 @@ def test_unknown_additive_fields_are_allowed_at_every_level(self) -> None: def test_schema_and_decoder_reject_malformed_fields( self, *, path: tuple[str | int, ...], invalid: object ) -> None: - data = serialize_result(result=_make_full_result()) + data = ResultRecord(result=_make_full_result()).to_dict() parent: Any = data for key in path[:-1]: parent = parent[key] @@ -698,7 +767,7 @@ def test_schema_and_decoder_reject_malformed_fields( def test_request_invariant_is_in_schema( self, *, prompt: str | None, attachments: bool, valid: bool ) -> None: - data = serialize_result(result=_make_full_result()) + data = ResultRecord(result=_make_full_result()).to_dict() request = data["result"]["turns"][0]["request"] request["prompt"] = prompt if not attachments: @@ -712,7 +781,7 @@ def test_request_invariant_is_in_schema( ResultRecord.from_dict(data) def test_schema_requires_recorded_payload_id(self) -> None: - data = serialize_result(result=_make_full_result()) + data = ResultRecord(result=_make_full_result()).to_dict() del data["result"]["turns"][0]["request"]["attachments"][0]["id"] assert not Draft202012Validator(ResultRecord.json_schema()).is_valid(data) From a3eb05e18a95906ec1ae190e4e3b71b42c354dbc Mon Sep 17 00:00:00 2001 From: Nina Chikanov Date: Thu, 10 Sep 2026 17:56:03 -0700 Subject: [PATCH 8/9] [FIX]: Isolate canonical adapter policies and clarify compatibility Restore public dataclass annotations, apply canonical policies through adapter-local schema hooks, document structural schema limits and transport preparation, and add generated round-trip coverage. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/concepts/trace-schema.md | 49 +++++ pyproject.toml | 1 + rampart/core/_schema.py | 180 +++++++++++++--- rampart/core/result.py | 58 +++++- rampart/core/serialization.py | 5 + rampart/core/types.py | 83 +------- schemas/trace.v1.schema.json | 20 +- tests/unit/core/test_serialization.py | 176 +++++++++++++++- .../core/test_serialization_properties.py | 194 ++++++++++++++++++ uv.lock | 95 +++++++++ 10 files changed, 726 insertions(+), 135 deletions(-) create mode 100644 tests/unit/core/test_serialization_properties.py diff --git a/docs/concepts/trace-schema.md b/docs/concepts/trace-schema.md index 052b3357..4e7473d7 100644 --- a/docs/concepts/trace-schema.md +++ b/docs/concepts/trace-schema.md @@ -37,6 +37,11 @@ declared defaults; explicit `null` is accepted only on nullable fields. Payload IDs must be recorded, not generated during deserialization. These boundary rules do not replace the normal dataclass constructors used during execution. +These policies belong to the cached canonical adapter, not to the public +dataclass annotations or configuration. Fields remain `dict[str, Any]` and +`datetime | None`. Independently constructed Pydantic adapters retain their +normal behavior, including live binary payload support. + `ResultRecord.json_schema()` returns the adapter-derived body schema plus the versioned envelope. Small schema customizations describe the trace-only payload restrictions and the request invariant (a prompt or at least one attachment). @@ -48,6 +53,50 @@ CI runs the same command with `--check` to detect drift. Changes to generated output still require a compatibility review; generation does not decide whether a version bump is needed. +### Structural schema and decoder semantics + +The JSON Schema checks structure; passing it is necessary but **not sufficient** +for successful record decoding. Use `deserialize_record()` (or +`ResultRecord.from_dict()` for dictionaries) for the complete contract. + +The decoder additionally enforces these representation rules: + +- Integer fields use integer notation, not floating-point notation. JSON Schema + accepts `0.0` as an integer mathematically; the strict decoder rejects it for + fields such as `result_index`, `turn_number`, and population `index` / `size`. +- Timestamp strings must parse with Python's `datetime.fromisoformat()`. The + schema intentionally does not claim RFC 3339 validation, since Python supports + naive datetimes and subminute UTC offsets. +- Numbers must be finite and representable by the corresponding Python field. + For example, an overflowing JSON exponent cannot become an infinite float. + +External producers should emit integer notation for integer fields, supported +ISO datetime strings, and finite numbers, then exercise the canonical reader +as well as structural schema validation. These are decoder requirements, not +additional serializers. + +## Transport compatibility boundary + +The canonical codec preserves supported values; it does not provide a lenient +transport mode. Existing transports and flat reports can accept data outside +that domain, so adopting the codec is not a direct replacement of their current +serialization calls. + +Transport normalization must happen **before** canonical encoding when the live +result contains unsupported values. Prepare a separate result without mutating +the original, then use the same record codec. Caps, rendering sanitization, +worker bookkeeping, and explicit loss/truncation markers remain transport +responsibilities. None belongs in a second field-by-field result serializer. + +A text placeholder prepared from a binary payload is a lossy transport view, +not a durable copy of that payload. Original format/path information must remain +available for transport diagnostics, and the containing transport must identify +the loss. Do not persist that view as a full-fidelity replay artifact. The +canonical reader itself never performs this conversion or opens a worker path. +Supporting durable binary artifacts requires a separately designed +representation and compatibility review; reserving an `artifacts` field alone +does not make currently rejected formats readable by older readers. + ## Versioning - Every serialized record carries one root `version` field. The current schema diff --git a/pyproject.toml b/pyproject.toml index b563e629..05759e6f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,7 @@ dev = [ "flake8>=7.3.0", "hatch-vcs>=0.5.0", "hatchling>=1.30.1", + "hypothesis>=6.168.0", "jsonschema>=4.26.0", "pre-commit>=4.5.1", "pytest-cov>=6.1.0", diff --git a/rampart/core/_schema.py b/rampart/core/_schema.py index b0105459..aa69215f 100644 --- a/rampart/core/_schema.py +++ b/rampart/core/_schema.py @@ -1,28 +1,165 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Shared value-domain rules for dataclass trace adapters.""" +"""Adapter-local policies for the canonical dataclass trace schema.""" from __future__ import annotations import math from collections.abc import Mapping from datetime import datetime -from typing import ( - TYPE_CHECKING, - Annotated, - Any, - TypeAlias, -) - -from pydantic import ( - BeforeValidator, - PlainSerializer, - WithJsonSchema, -) +from typing import TYPE_CHECKING + +from pydantic_core import core_schema + +from rampart.core.types import Payload, PayloadFormat if TYPE_CHECKING: - from pydantic import ValidationError, ValidationInfo + from pydantic import ( + GetCoreSchemaHandler, + ValidationError, + ValidationInfo, + ) + + +# Pydantic supplies source/handler positionally to schema hooks. +def trace_schema( + source: object, handler: GetCoreSchemaHandler +) -> core_schema.CoreSchema: + """Apply canonical policies to an adapter's schema, not its live classes. + + Returns: + CoreSchema: A configured copy of the generated dataclass schema. + """ + return _trace_schema(schema=handler(source), handler=handler, references=set()) + + +def _trace_schema( + *, + schema: core_schema.CoreSchema, + handler: GetCoreSchemaHandler, + references: set[str], +) -> core_schema.CoreSchema: + """Copy generated schema nodes while applying shared trace rules. + + Returns: + CoreSchema: The adapter-local schema, preserving definition references. + """ + if schema["type"] == "definition-ref": + reference = schema["schema_ref"] + if reference in references: + return schema + references.add(reference) + return _trace_schema( + schema=handler.resolve_ref_schema(schema), + handler=handler, + references=references, + ) + + schema = schema.copy() + if schema["type"] == "dataclass": + return _trace_dataclass(schema=schema, handler=handler, references=references) + if schema["type"] == "default" or schema["type"] == "nullable": + schema["schema"] = _trace_schema( + schema=schema["schema"], handler=handler, references=references + ) + elif schema["type"] == "dataclass-args": + schema["fields"] = [ + { + **field, + "schema": _trace_schema( + schema=field["schema"], handler=handler, references=references + ), + } + for field in schema["fields"] + ] + elif schema["type"] == "list": + schema["items_schema"] = _trace_schema( + schema=schema["items_schema"], handler=handler, references=references + ) + elif schema["type"] == "union": + schema["choices"] = [ + ( + _trace_schema(schema=choice[0], handler=handler, references=references), + choice[1], + ) + if isinstance(choice, tuple) + else _trace_schema(schema=choice, handler=handler, references=references) + for choice in schema["choices"] + ] + elif schema["type"] == "dict": + return core_schema.no_info_before_validator_function(json_value, schema) + elif schema["type"] == "datetime": + return core_schema.with_info_before_validator_function( + _iso_datetime, + schema, + serialization=core_schema.plain_serializer_function_ser_schema( + datetime.isoformat, return_schema=core_schema.str_schema() + ), + ) + + return schema + + +def _trace_dataclass( + *, + schema: core_schema.DataclassSchema, + handler: GetCoreSchemaHandler, + references: set[str], +) -> core_schema.CoreSchema: + """Configure a copied dataclass schema without changing its class. + + Returns: + CoreSchema: A revalidating schema with trace-only payload guards. + """ + schema["schema"] = _trace_schema( + schema=schema["schema"], handler=handler, references=references + ) + schema["config"] = { + **schema.get("config", {}), + "strict": True, + "revalidate_instances": "always", + "allow_inf_nan": False, + } + if schema["cls"] is Payload: + reference = schema.pop("ref", None) + return core_schema.no_info_before_validator_function( + _trace_payload, schema, ref=reference + ) + return schema + + +def _trace_payload(value: object) -> object: + """Reject unsupported artifacts before dataclass construction touches them. + + Returns: + object: The unchanged payload for normal field validation. + + Raises: + ValueError: If identity is absent or a binary artifact is encountered. + """ + if isinstance(value, Mapping): + if "id" not in value: + msg = "id: a trace payload must record its id" + raise ValueError(msg) + payload_format = value.get("format", PayloadFormat.TEXT) + artifact = value.get("artifact") + elif isinstance(value, Payload): + payload_format = value.format + artifact = value.artifact + else: + return value + if isinstance(payload_format, PayloadFormat): + payload_format = payload_format.value + if isinstance(payload_format, str) and payload_format in { + member.value for member in PayloadFormat if member.is_binary + }: + msg = "binary payload format and artifact are unsupported in traces" + raise ValueError(msg) + if artifact is not None: + msg = "artifact: only null is supported in traces" + raise ValueError(msg) + return value def json_value(value: object) -> object: @@ -74,10 +211,6 @@ def _json_value(*, value: object, path: str, active: set[int]) -> object: active.remove(id(value)) -# Standard dataclass construction stays permissive; adapters validate these maps. -JsonMapping: TypeAlias = Annotated[dict[str, Any], BeforeValidator(json_value)] - - # Pydantic supplies value/info positionally to BeforeValidator callbacks. def _iso_datetime(value: object, info: ValidationInfo) -> object: """Retain Python ISO datetime support, including naive and subminute offsets. @@ -97,17 +230,6 @@ def _iso_datetime(value: object, info: ValidationInfo) -> object: return value -IsoDatetime: TypeAlias = Annotated[ - datetime, - BeforeValidator(_iso_datetime), - PlainSerializer(datetime.isoformat), - # Python ISO datetimes include values outside RFC 3339's date-time format. - WithJsonSchema( - {"type": "string", "description": "ISO 8601 datetime; UTC offset is optional."} - ), -] - - def validation_message(*, error: ValidationError, path: str) -> str: """Render Pydantic errors without including producer data in the message. diff --git a/rampart/core/result.py b/rampart/core/result.py index 48fffe83..cd800630 100644 --- a/rampart/core/result.py +++ b/rampart/core/result.py @@ -15,21 +15,24 @@ from dataclasses import dataclass, field from enum import Enum, StrEnum from functools import cache -from typing import TYPE_CHECKING, Any +from typing import ( + TYPE_CHECKING, + Annotated, + Any, +) from pydantic import ( - ConfigDict, + GetPydanticSchema, TypeAdapter, ValidationError, - with_config, ) from pydantic.json_schema import GenerateJsonSchema, JsonSchemaValue from pydantic_core import PydanticSerializationError, core_schema from rampart.common.text import safe_str, safe_str_list from rampart.core._schema import ( - JsonMapping, json_value, + trace_schema, validation_message, ) from rampart.core.errors import SchemaError @@ -46,6 +49,8 @@ if TYPE_CHECKING: from collections.abc import Iterable + from pydantic.json_schema import JsonSchemaMode + class SafetyStatus(Enum): """Categorical safety status for structured reporting. @@ -130,9 +135,6 @@ class PopulationRef: threshold: float -@with_config( - ConfigDict(strict=True, revalidate_instances="always", allow_inf_nan=False) -) @dataclass(kw_only=True) class Result: """The outcome of a safety test. @@ -180,7 +182,7 @@ class Result: default_factory=list[InjectionRecord], ) population: PopulationRef | None = None - metadata: JsonMapping = field(default_factory=dict[str, Any]) + metadata: dict[str, Any] = field(default_factory=dict[str, Any]) @property def safe(self) -> bool: @@ -228,7 +230,7 @@ def to_dict(self) -> dict[str, Any]: """ adapter = _result_adapter() try: - validated = adapter.validate_python(self, context={"trace": True}) + validated = adapter.validate_python(self, strict=True) return adapter.dump_python(validated, mode="json", warnings="error") except ValidationError as exc: raise SchemaError(validation_message(error=exc, path="result")) from exc @@ -252,7 +254,7 @@ def from_dict(cls, data: object) -> Result: try: # JSON-mode strict validation accepts wire enums/dates, not coercions. encoded = json.dumps(json_value(data), allow_nan=False) - return _result_adapter().validate_json(encoded, context={"trace": True}) + return _result_adapter().validate_json(encoded, strict=True) except ValidationError as exc: raise SchemaError(validation_message(error=exc, path="result")) from exc except (ValueError, RecursionError) as exc: @@ -276,12 +278,32 @@ def _result_adapter() -> TypeAdapter[Result]: Returns: TypeAdapter[Result]: The cached adapter. """ - return TypeAdapter(Result) + return TypeAdapter(Annotated[Result, GetPydanticSchema(trace_schema)]) class _ResultJsonSchema(GenerateJsonSchema): """Describe trace-only restrictions alongside the dataclass field schemas.""" + def generate( + self, schema: core_schema.CoreSchema, mode: JsonSchemaMode = "validation" + ) -> JsonSchemaValue: + """Omit runtime class documentation from the published wire contract. + + Returns: + JsonSchemaValue: A schema with only trace-specific descriptions. + """ + result = super().generate(schema, mode=mode) + result.pop("description", None) + definitions = result.get("$defs", {}) + for definition in definitions.values(): + definition.pop("description", None) + if "Payload" in definitions: + definitions["Payload"]["description"] = ( + "Recorded text payload. Binary formats and file artifacts " + "are not supported by this trace schema." + ) + return result + def dataclass_schema(self, schema: core_schema.DataclassSchema) -> JsonSchemaValue: """Add trace policies that do not restrict live dataclass construction. @@ -308,6 +330,20 @@ def dataclass_schema(self, schema: core_schema.DataclassSchema) -> JsonSchemaVal ] return result + def datetime_schema(self, schema: core_schema.DatetimeSchema) -> JsonSchemaValue: + """Describe Python datetimes without claiming RFC 3339 validation. + + Returns: + JsonSchemaValue: A string with decoder-enforced datetime semantics. + """ + result = super().datetime_schema(schema) + result.pop("format", None) + result["description"] = ( + "Python ISO 8601 datetime; UTC offset is optional. " + "Parseability is enforced by the record decoder, not this schema." + ) + return result + @dataclass(kw_only=True) class PopulationResult: diff --git a/rampart/core/serialization.py b/rampart/core/serialization.py index 2ccbf2bf..b112bc9d 100644 --- a/rampart/core/serialization.py +++ b/rampart/core/serialization.py @@ -132,6 +132,11 @@ def json_schema(cls) -> JsonSchemaValue: "$id": f"urn:rampart:trace:{TRACE_SCHEMA_VERSION.rsplit('.', 1)[-1]}", "$defs": definitions, "title": "ResultRecord", + "description": ( + "Structural trace contract. The record decoder additionally " + "requires parseable Python ISO datetimes, finite numbers, " + "and integer fields without floating-point notation." + ), "type": "object", "additionalProperties": True, "required": ["version", "result"], diff --git a/rampart/core/types.py b/rampart/core/types.py index bb8aa77c..174a7605 100644 --- a/rampart/core/types.py +++ b/rampart/core/types.py @@ -10,25 +10,16 @@ from __future__ import annotations import uuid -from collections.abc import Mapping from dataclasses import dataclass, field +from datetime import ( + datetime, # ruff: ignore[typing-only-standard-library-import] Resolved by TypeAdapter. +) from enum import Enum from pathlib import ( Path, # ruff: ignore[typing-only-standard-library-import] Resolved by TypeAdapter. ) from typing import TYPE_CHECKING, Any -from pydantic import ( - ValidationInfo, - field_validator, - model_validator, -) - -from rampart.core._schema import ( - IsoDatetime, # ruff: ignore[typing-only-first-party-import] Resolved by TypeAdapter. - JsonMapping, # ruff: ignore[typing-only-first-party-import] Resolved by TypeAdapter. -) - if TYPE_CHECKING: from rampart.core.manifest import AppManifest @@ -151,63 +142,7 @@ class Payload: id: str = field(default_factory=lambda: uuid.uuid4().hex[:12]) format: PayloadFormat = PayloadFormat.TEXT artifact: Path | None = None - metadata: JsonMapping = field(default_factory=dict[str, Any]) - - # Pydantic invokes validator callbacks with positional value/info arguments. - @model_validator(mode="before") - @classmethod - def _validate_trace_id(cls, value: object, info: ValidationInfo) -> object: - """Require recorded payload identity instead of generating one on read. - - Returns: - object: The unchanged input. - - Raises: - ValueError: If a trace payload has no recorded id. - """ - if ( - info.context - and info.context.get("trace") - and isinstance(value, Mapping) - and "id" not in value - ): - msg = "id: a trace payload must record its id" - raise ValueError(msg) - return value - - @field_validator("format") - @classmethod - def _validate_trace_format( - cls, value: PayloadFormat, info: ValidationInfo - ) -> PayloadFormat: - """Reject binary formats at the trace boundary, not during live use. - - Returns: - PayloadFormat: The validated format. - - Raises: - ValueError: If a trace contains a binary payload. - """ - if info.context and info.context.get("trace") and value.is_binary: - msg = f"binary payload format {value.value!r} is unsupported in traces" - raise ValueError(msg) - return value - - @field_validator("artifact", mode="before") - @classmethod - def _validate_trace_artifact(cls, value: object, info: ValidationInfo) -> object: - """Reject trace artifacts before path construction or filesystem access. - - Returns: - object: The unchanged artifact for live use, or None for a trace. - - Raises: - ValueError: If a trace contains a non-null artifact. - """ - if info.context and info.context.get("trace") and value is not None: - msg = "artifact: only null is supported in traces" - raise ValueError(msg) - return value + metadata: dict[str, Any] = field(default_factory=dict[str, Any]) def __post_init__(self) -> None: """Validate content-format-artifact consistency. @@ -263,9 +198,9 @@ class ToolCall: """ name: str - arguments: JsonMapping = field(default_factory=dict[str, Any]) + arguments: dict[str, Any] = field(default_factory=dict[str, Any]) result: str | None = None - timestamp: IsoDatetime | None = None + timestamp: datetime | None = None @dataclass(kw_only=True) @@ -282,7 +217,7 @@ class SideEffect: """ kind: str - details: JsonMapping = field(default_factory=dict[str, Any]) + details: dict[str, Any] = field(default_factory=dict[str, Any]) @dataclass(kw_only=True) @@ -301,7 +236,7 @@ class Response: text: str tool_calls: list[ToolCall] = field(default_factory=list[ToolCall]) side_effects: list[SideEffect] = field(default_factory=list[SideEffect]) - metadata: JsonMapping = field(default_factory=dict[str, Any]) + metadata: dict[str, Any] = field(default_factory=dict[str, Any]) @dataclass(kw_only=True) @@ -353,7 +288,7 @@ class Turn: response: Response eval_result: EvalResult | None = None turn_number: int = 0 - timestamp: IsoDatetime | None = None + timestamp: datetime | None = None driver_reasoning: str = "" diff --git a/schemas/trace.v1.schema.json b/schemas/trace.v1.schema.json index 554434a8..1e818587 100644 --- a/schemas/trace.v1.schema.json +++ b/schemas/trace.v1.schema.json @@ -1,7 +1,6 @@ { "$defs": { "EvalOutcome": { - "description": "What the evaluator determined.\n\nDETECTED: The condition was found.\nNOT_DETECTED: The condition was not found.\nUNDETERMINED: The evaluator could not make a determination.", "enum": [ "detected", "not_detected", @@ -12,7 +11,6 @@ }, "EvalResult": { "additionalProperties": true, - "description": "What an evaluator returns \u2014 a raw condition detection signal.\n\nThis is NOT a safety judgment. Whether DETECTED means \"safe\" or\n\"unsafe\" depends on context.\n\nArgs:\n outcome: What the evaluator determined.\n confidence: How confident the evaluator is (0.0 to 1.0).\n evidence: Specific observations supporting the outcome.\n rationale: Human-readable explanation.\n undetermined_operands: Why parts of the evaluation stayed\n undetermined, one distinct reason per entry. ``&`` and ``|``\n record every operand they ran that came back UNDETERMINED,\n taking the reasons that operand already carries or, for a\n leaf, its rationale, or a fixed phrase when it gave none;\n repeats are collapsed. ``~`` carries its inner result's\n entries through. An evaluator that is not a composite\n records nothing. It says nothing about ``outcome``: a\n DETECTED or NOT_DETECTED result with entries here reached a\n definitive answer while part of the evaluation did not, and\n an UNDETERMINED result can carry entries recorded further\n down the expression.", "properties": { "confidence": { "default": 1.0, @@ -49,7 +47,6 @@ "type": "object" }, "HarmCategory": { - "description": "Classification of the safety concern being tested.\n\nUsed by the pytest @harm marker for categorization, by reporting\nsinks for grouping, and by safety gates for threshold configuration.\n\nHarmCategory is a StrEnum so that its values are native strings. This\nenables teams to use custom string categories alongside the built-in\nvalues: @pytest.mark.harm(\"custom_product_risk\") is valid, and the\nstring flows through Result.harm_category, reporting sinks, and\ndashboard grouping without requiring enum membership. Built-in values\nprovide IDE completion and typo protection for common categories;\nplain strings provide extensibility for team-specific risks.\n\nPhase availability:\n Phase 1: All values are defined and usable with MockAdapter.\n Phase 2: PROMPT_INJECTION, JAILBREAK, and remaining categories\n gain execution strategy support via PyRIT integration.", "enum": [ "memory_poisoning", "prompt_injection", @@ -66,7 +63,6 @@ }, "InjectionRecord": { "additionalProperties": true, - "description": "Records what was injected and where, for reproduction and reporting.\n\nPopulated by XPIAExecution after handles are activated and stored\non Result. Provides the complete injection context needed to\nreproduce a test run: which payload was placed in which surface.\n\nArgs:\n payload_id: The injected payload's identifier. None if\n the surface implementation does not track payload IDs.\n surface_name: The surface this payload was injected into\n (e.g., \"SharePoint\", \"Exchange\").", "properties": { "payload_id": { "anyOf": [ @@ -92,7 +88,6 @@ "type": "object" }, "ObservabilityLevel": { - "description": "What the adapter can reliably observe during agent execution.\n\nDeclared by the adapter to inform evaluators and reporting. An\nevaluator that needs an evidence channel the adapter does not report\nreturns UNDETERMINED rather than a false NOT_DETECTED. That covers\ntool call data under RESPONSE_ONLY, and side effect data under\neither TOOL_ONLY or RESPONSE_ONLY.\n\nThe guarantee is per channel, not per field. A level that reports a\nchannel is taken at its word for what it puts in it, so a tool call\nreported with redacted or partial arguments still counts as observed\nand a predicate over those arguments can return NOT_DETECTED.\n\nThe ``observes_tool_calls`` and ``observes_side_effects`` properties\nlet evaluators ask what evidence is available without listing every\nenum member.", "enum": [ "tool_and_side_effects", "tool_only", @@ -103,7 +98,7 @@ }, "Payload": { "additionalProperties": true, - "description": "Content to inject into a surface or send alongside a prompt.\n\n``content`` is always the semantic text \u2014 the attack instruction,\nthe adversarial prompt, or a description of the payload's purpose.\nIt is what reports display and what makes a payload reproducible.\n\nFor text formats (TEXT, HTML, MARKDOWN), ``content`` is delivered\ndirectly \u2014 no artifact is needed. For binary formats (IMAGE, PDF,\nDOCX), ``artifact`` points to the file that surfaces and adapters\ndeliver. The ``content`` field still holds the human-readable text\nfor reporting and debugging.\n\nArgs:\n content (str): The semantic text content. Always human-readable.\n id (str): Stable identifier for reproduction and cache keys.\n format (PayloadFormat): Delivery format. TEXT by default.\n artifact (Path | None): Path to a rendered binary file.\n Required for binary formats, must be None for text formats.\n metadata (dict[str, Any]): Provenance tracking (persona,\n template, variant index, generation params).", + "description": "Recorded text payload. Binary formats and file artifacts are not supported by this trace schema.", "properties": { "artifact": { "default": null, @@ -141,7 +136,6 @@ }, "PopulationRef": { "additionalProperties": true, - "description": "Identifies the trial population that a Result belongs to.\n\nArgs:\n id: Unique identifier shared by every result in the population.\n index: Zero-based position of the result within the population.\n size: Number of results requested for the population.\n threshold: Required safe-result rate for the population.", "properties": { "id": { "title": "Id", @@ -194,7 +188,6 @@ ] } ], - "description": "What is sent to the agent in a single turn.\n\nCombines prompt text and inline payloads into a single object.\nAt least one of ``prompt`` or ``attachments`` must be provided.\n\nArgs:\n prompt: The text prompt to send. None when only\n attachments are sent (e.g., inline XPIA).\n attachments: Payloads sent alongside the prompt for\n inline delivery (e.g., poisoned documents).", "properties": { "attachments": { "items": { @@ -221,7 +214,6 @@ }, "Response": { "additionalProperties": true, - "description": "What the agent returned for a single prompt.\n\nThe adapter populates every field it can observe.\n\nArgs:\n text: The agent's text response.\n tool_calls: Tool invocations observed during this interaction.\n side_effects: Other observable effects.\n metadata: Adapter-specific diagnostic data.", "properties": { "metadata": { "additionalProperties": true, @@ -254,7 +246,6 @@ "type": "object" }, "SafetyStatus": { - "description": "Categorical safety status for structured reporting.\n\nSAFE: The agent behaved correctly.\nUNSAFE: A safety violation was detected.\nUNDETERMINED: The framework could not determine safety\n (typically an observability gap).\nERROR: The test encountered an infrastructure error.", "enum": [ "safe", "unsafe", @@ -266,7 +257,6 @@ }, "SideEffect": { "additionalProperties": true, - "description": "An observable side effect beyond tool invocations.\n\nCovers effects like HTTP requests, file system changes, or\ndatabase writes that the adapter can observe but that are not\nmodeled as tool calls in the agent's API.\n\nArgs:\n kind: Effect category (e.g., \"http_request\", \"file_write\").\n details: Structured data about the effect.", "properties": { "details": { "additionalProperties": true, @@ -286,7 +276,6 @@ }, "ToolCall": { "additionalProperties": true, - "description": "A tool invocation observed during agent execution.\n\nAdapters populate this from whatever observability they have \u2014 API\nresponse fields, telemetry streams, log parsing.\n\nArgs:\n name: Tool name as the agent reported it (e.g., \"send_email\").\n arguments: Parameters passed to the tool.\n result: Tool return value, if the adapter can observe it.\n timestamp: When the invocation occurred, if available.", "properties": { "arguments": { "additionalProperties": true, @@ -312,7 +301,7 @@ "timestamp": { "anyOf": [ { - "description": "ISO 8601 datetime; UTC offset is optional.", + "description": "Python ISO 8601 datetime; UTC offset is optional. Parseability is enforced by the record decoder, not this schema.", "type": "string" }, { @@ -331,7 +320,6 @@ }, "Turn": { "additionalProperties": true, - "description": "One prompt-response exchange.\n\nTurn is immutable. The execution loop constructs a provisional Turn\n(without eval_result) for the evaluator call, then produces the\nfinal Turn via dataclasses.replace before appending to history.\n\nArgs:\n request: What was sent to the agent.\n response: What the agent returned.\n eval_result: Evaluator outcome for this turn.\n turn_number: Position in the conversation (0-indexed).\n timestamp: When this exchange occurred.\n driver_reasoning: Why the driver chose this request.", "properties": { "driver_reasoning": { "default": "", @@ -358,7 +346,7 @@ "timestamp": { "anyOf": [ { - "description": "ISO 8601 datetime; UTC offset is optional.", + "description": "Python ISO 8601 datetime; UTC offset is optional. Parseability is enforced by the record decoder, not this schema.", "type": "string" }, { @@ -385,6 +373,7 @@ "$id": "urn:rampart:trace:v1", "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": true, + "description": "Structural trace contract. The record decoder additionally requires parseable Python ISO datetimes, finite numbers, and integer fields without floating-point notation.", "properties": { "pytest_nodeid": { "type": [ @@ -394,7 +383,6 @@ }, "result": { "additionalProperties": true, - "description": "The outcome of a safety test.\n\nThis is the single result type for the entire framework. Attacks\nand probes both produce Result objects. The reporting infrastructure,\npytest plugin, and dashboards all consume them.\n\nThe critical invariant: bool(result) returns result.safe. This\nmeans ``assert result, result.summary`` always means \"assert the\nagent behaved safely\" \u2014 and failures include the summary explaining\nwhat was detected.\n\n``safe`` is a derived property (``status is SafetyStatus.SAFE``),\nnot a stored field, so it can never drift out of sync with ``status``.\n\nArgs:\n status: Categorical status for structured reporting.\n summary: Human-readable one-line summary.\n observability_level: What the adapter could observe. Required, so\n that a report states a level someone chose rather than one the\n framework assumed. Built-in strategies pass\n ``adapter.observability_profile``.\n turns: The full conversation for evidence and debugging.\n duration_seconds: How long the test execution took.\n harm_category: Which harm category this test covers.\n Accepts HarmCategory enum values for built-in categories or plain strings\n for team-defined categories (e.g., \"custom_product_risk\"). Both are strings\n at runtime since HarmCategory is a StrEnum.\n strategy: Name of the execution strategy (e.g., \"xpia\", \"crescendo\").\n injections: What was injected and into which surfaces,\n for full reproduction of multi-surface attacks. Empty for non-XPIA tests.\n population: Trial population provenance. None for single executions.\n metadata: Additional structured data for reporting.", "properties": { "duration_seconds": { "default": 0.0, diff --git a/tests/unit/core/test_serialization.py b/tests/unit/core/test_serialization.py index 6a9a2a2a..2e63f35f 100644 --- a/tests/unit/core/test_serialization.py +++ b/tests/unit/core/test_serialization.py @@ -8,7 +8,7 @@ import json import math import re -from dataclasses import fields +from dataclasses import fields, replace from datetime import ( UTC, datetime, @@ -16,7 +16,11 @@ timezone, ) from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import ( + TYPE_CHECKING, + Any, + get_type_hints, +) from unittest.mock import patch import pytest @@ -592,6 +596,137 @@ def test_null_is_not_a_default_for_nonnullable_fields(self, field: str) -> None: Result.from_dict(body) +class TestAdapterIsolation: + def test_public_annotations_remain_standard_types(self) -> None: + for cls, name in [ + (Result, "metadata"), + (Payload, "metadata"), + (Response, "metadata"), + (ToolCall, "arguments"), + (SideEffect, "details"), + ]: + assert get_type_hints(cls, include_extras=True)[name] == dict[str, Any] + for cls in [ToolCall, Turn]: + assert get_type_hints(cls, include_extras=True)["timestamp"] == ( + datetime | None + ) + assert not hasattr(Result, "__pydantic_config__") + + def test_regular_adapters_are_unchanged_before_and_after_canonical_use( + self, + ) -> None: + adapter = TypeAdapter(Result) + original_schema = adapter.json_schema() + result = _make_full_result(metadata={"tuple": (1, 2), "opaque": object()}) + + with pytest.raises(SchemaError, match="metadata"): + result.to_dict() + Result.json_schema() + + assert adapter.validate_python(result) is result + assert TypeAdapter(Result).validate_python(result) is result + assert adapter.json_schema() == original_schema + assert TypeAdapter(Result).json_schema() == original_schema + + def test_regular_adapter_can_still_generate_payload_ids(self) -> None: + _make_full_result().to_dict() + + payload = TypeAdapter(Payload).validate_python( + {"content": "live", "metadata": {"tuple": (1, 2)}} + ) + + assert payload.id + assert payload.metadata["tuple"] == (1, 2) + + def test_regular_adapter_retains_pydantic_datetime_behavior(self) -> None: + result = _make_full_result() + + canonical = result.to_dict() + regular = TypeAdapter(Result).dump_python(result, mode="json") + + assert canonical["turns"][0]["timestamp"].endswith("+00:00") + assert regular["turns"][0]["timestamp"].endswith("Z") + + def test_regular_adapter_can_decode_live_binary_payloads( + self, tmp_path: Path + ) -> None: + artifact = tmp_path / "document.pdf" + artifact.write_bytes(b"%PDF-1.4 fake") + _make_full_result().to_dict() + + payload = TypeAdapter(Payload).validate_python( + {"content": "doc", "format": "pdf", "artifact": str(artifact)} + ) + + assert payload.format is PayloadFormat.PDF + assert payload.artifact == artifact + + def test_reused_nested_schemas_still_validate_all_instances(self) -> None: + result = _make_full_result() + result.turns.append(_make_turn()) + result.turns[1].request.attachments[0].metadata["bad"] = (1, 2) + + with pytest.raises(SchemaError, match=r"turns\[1\].*metadata"): + result.to_dict() + + def test_nested_numeric_fields_are_still_finite(self) -> None: + result = _make_full_result() + assert result.turns[0].eval_result is not None + result.turns[0].eval_result.confidence = math.inf + + with pytest.raises(SchemaError, match="confidence"): + result.to_dict() + + +class TestTransportPreparationBoundary: + def test_prepared_copy_does_not_relax_the_original_record( + self, tmp_path: Path + ) -> None: + artifact = tmp_path / "worker.pdf" + artifact.write_bytes(b"%PDF-1.4 fake") + original = _make_full_result(metadata={"tuple": (1, 2)}) + binary = Payload( + content="document text", format=PayloadFormat.PDF, artifact=artifact + ) + original.turns[0].request.attachments = [binary] + with pytest.raises(SchemaError): + serialize_record(record=ResultRecord(result=original)) + + display_payload = replace( + binary, + format=PayloadFormat.TEXT, + artifact=None, + metadata={ + "_rampart_worker_format": "pdf", + "_rampart_worker_artifact_path": str(artifact), + }, + ) + prepared = replace( + original, + metadata={"tuple": [1, 2]}, + turns=[ + replace( + original.turns[0], + request=replace( + original.turns[0].request, attachments=[display_payload] + ), + ) + ], + ) + + restored = deserialize_record( + data=serialize_record(record=ResultRecord(result=prepared)) + ).result + + assert restored == prepared + assert original.metadata["tuple"] == (1, 2) + assert original.turns[0].request.attachments[0] is binary + assert binary.format is PayloadFormat.PDF + assert binary.artifact == artifact + with pytest.raises(SchemaError): + serialize_record(record=ResultRecord(result=original)) + + class TestJsonValueDomain: @pytest.mark.parametrize("map_index", range(5)) @pytest.mark.parametrize( @@ -656,15 +791,46 @@ def test_supported_values_round_trip_without_mutation(self) -> None: class TestGeneratedSchema: - def test_generated_schema_is_valid_and_matches_checked_in_contract(self) -> None: + def test_generated_schema_is_valid(self) -> None: schema = ResultRecord.json_schema() - path = Path(__file__).resolve().parents[3] / "schemas" / "trace.v1.schema.json" Draft202012Validator.check_schema(schema) - assert json.loads(path.read_text(encoding="utf-8")) == schema assert schema["properties"]["version"]["const"] == TRACE_SCHEMA_VERSION + def test_schema_omits_runtime_class_documentation(self) -> None: + schema = ResultRecord.json_schema() + + assert "description" not in schema["properties"]["result"] + assert "description" not in schema["$defs"]["SafetyStatus"] + assert "not supported" in schema["$defs"]["Payload"]["description"] + assert "Args:" not in json.dumps(schema) + + @pytest.mark.parametrize( + ("path", "value"), + [ + (("result_index",), 0.0), + (("result", "population", "index"), 0.0), + (("result", "turns", 0, "timestamp"), "not-a-date"), + ], + ) + def test_structural_validation_does_not_replace_decoder_semantics( + self, *, path: tuple[str | int, ...], value: object + ) -> None: + data = ResultRecord(result=_make_full_result()).to_dict() + parent: Any = data + for key in path[:-1]: + parent = parent[key] + parent[path[-1]] = value + validator = Draft202012Validator( + ResultRecord.json_schema(), + format_checker=Draft202012Validator.FORMAT_CHECKER, + ) + + validator.validate(data) + with pytest.raises(SchemaError, match=re.escape(str(path[-1]))): + deserialize_record(data=json.dumps(data)) + @pytest.mark.parametrize("payload_format", list(PayloadFormat)) def test_schema_and_decoder_agree_on_payload_formats( self, payload_format: PayloadFormat diff --git a/tests/unit/core/test_serialization_properties.py b/tests/unit/core/test_serialization_properties.py new file mode 100644 index 00000000..7ba94f0c --- /dev/null +++ b/tests/unit/core/test_serialization_properties.py @@ -0,0 +1,194 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Generated round-trips over the canonical trace value domain.""" + +from __future__ import annotations + +import json +from dataclasses import replace +from datetime import timedelta, timezone +from typing import TYPE_CHECKING, Any + +from hypothesis import given +from hypothesis import strategies as st +from jsonschema import Draft202012Validator + +from rampart.core.result import ( + HarmCategory, + InjectionRecord, + PopulationRef, + Result, + SafetyStatus, +) +from rampart.core.serialization import ( + ResultRecord, + deserialize_record, + serialize_record, +) +from rampart.core.types import ( + EvalOutcome, + EvalResult, + ObservabilityLevel, + Payload, + PayloadFormat, + Request, + Response, + SideEffect, + ToolCall, + Turn, +) + +if TYPE_CHECKING: + from datetime import datetime + + from hypothesis.strategies import SearchStrategy + + +def _json_maps() -> SearchStrategy[dict[str, Any]]: + values = st.recursive( + st.none() + | st.booleans() + | st.integers(min_value=-(2**128), max_value=2**128) + | st.floats(allow_nan=False, allow_infinity=False) + | st.text(max_size=40), + lambda children: ( + st.lists(children, max_size=4) + | st.dictionaries(st.text(max_size=20), children, max_size=4) + ), + max_leaves=10, + ) + return st.dictionaries(st.text(max_size=20), values, max_size=4) + + +def _timestamps() -> SearchStrategy[datetime | None]: + zones = st.none() | st.integers(-86399, 86399).map( + lambda seconds: timezone(timedelta(seconds=seconds)) + ) + return st.none() | st.datetimes(timezones=zones) + + +def _payloads() -> SearchStrategy[Payload]: + return st.builds( + Payload, + content=st.text(max_size=100), + id=st.text(max_size=30), + format=st.sampled_from([value for value in PayloadFormat if value.is_text]), + metadata=_json_maps(), + ) + + +def _requests() -> SearchStrategy[Request]: + return st.one_of( + st.builds( + Request, + prompt=st.text(max_size=100), + attachments=st.lists(_payloads(), max_size=2), + ), + st.builds( + Request, + prompt=st.none(), + attachments=st.lists(_payloads(), min_size=1, max_size=2), + ), + ) + + +def _responses() -> SearchStrategy[Response]: + calls = st.builds( + ToolCall, + name=st.text(max_size=30), + arguments=_json_maps(), + result=st.none() | st.text(max_size=100), + timestamp=_timestamps(), + ) + effects = st.builds(SideEffect, kind=st.text(max_size=30), details=_json_maps()) + return st.builds( + Response, + text=st.text(max_size=100), + tool_calls=st.lists(calls, max_size=2), + side_effects=st.lists(effects, max_size=2), + metadata=_json_maps(), + ) + + +def _turns() -> SearchStrategy[Turn]: + evaluations = st.builds( + EvalResult, + outcome=st.sampled_from(EvalOutcome), + confidence=st.floats(min_value=0, max_value=1), + evidence=st.lists(st.text(max_size=30), max_size=3), + rationale=st.text(max_size=50), + undetermined_operands=st.lists(st.text(max_size=30), max_size=3), + ) + return st.builds( + Turn, + request=_requests(), + response=_responses(), + eval_result=st.none() | evaluations, + turn_number=st.integers(min_value=0, max_value=100), + timestamp=_timestamps(), + driver_reasoning=st.text(max_size=50), + ) + + +def _results() -> SearchStrategy[Result]: + injections = st.builds( + InjectionRecord, + payload_id=st.none() | st.text(max_size=30), + surface_name=st.text(max_size=30), + ) + populations = st.builds( + PopulationRef, + id=st.text(max_size=30), + index=st.integers(min_value=0, max_value=9), + size=st.just(10), + threshold=st.floats(min_value=0, max_value=1), + ) + return st.builds( + Result, + status=st.sampled_from(SafetyStatus), + summary=st.text(max_size=100), + observability_level=st.sampled_from(ObservabilityLevel), + turns=st.lists(_turns(), max_size=3), + duration_seconds=st.floats(min_value=0, allow_infinity=False), + harm_category=st.none() | st.text(max_size=30) | st.sampled_from(HarmCategory), + strategy=st.text(max_size=30), + injections=st.lists(injections, max_size=2), + population=st.none() | populations, + metadata=_json_maps(), + ) + + +class TestGeneratedRoundTrips: + @given(result=_results()) + def test_body_preserves_supported_values(self, result: Result) -> None: + body = result.to_dict() + + restored = Result.from_dict(json.loads(json.dumps(body, allow_nan=False))) + + assert restored == result + assert restored.to_dict() == body + assert result.to_dict() == body + + @given( + result=_results(), + nodeid=st.none() | st.text(max_size=40), + index=st.none() | st.integers(min_value=0, max_value=100), + ) + def test_record_round_trip_matches_the_structural_schema( + self, *, result: Result, nodeid: str | None, index: int | None + ) -> None: + record = ResultRecord(result=result, pytest_nodeid=nodeid, result_index=index) + original_body = result.to_dict() + encoded = serialize_record(record=record) + body = json.loads(encoded) + + restored = deserialize_record(data=encoded) + + assert restored.result == replace(result, metadata=body["result"]["metadata"]) + assert restored.pytest_nodeid == nodeid + assert restored.result_index == index + assert restored.to_dict() == body + assert record.to_dict() == body + assert result.to_dict() == original_body + Draft202012Validator(ResultRecord.json_schema()).validate(body) diff --git a/uv.lock b/uv.lock index 0474aa4b..821526f7 100644 --- a/uv.lock +++ b/uv.lock @@ -1256,6 +1256,90 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, ] +[[package]] +name = "hypothesis" +version = "6.168.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/ce/c0946bebffb99b62426a6a7643d4272cc6c5cf777a488b3b4d0ee724e960/hypothesis-6.168.0.tar.gz", hash = "sha256:72af51087b7b5ab21c49f0d502f803c20897678652835596bd2a8b169a39135e", size = 510805, upload-time = "2026-09-08T18:48:36.072Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/f8/8b2cc9ae7b439538f6f2d32a92892340b6343a6b04d171c516666901dedc/hypothesis-6.168.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:47b89491ff02e3ae9b302c440457938e87b47a45b9a1d98ff5575b6910d779e2", size = 791358, upload-time = "2026-09-08T18:47:37.076Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f4/4d7d897310cde5085779fb96feadb8529d98cb8e51ed7b24f7da9b6c6bdc/hypothesis-6.168.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:1f4cd0ff11bd470a1a846296ed5fe55e84214194850370994fd1370fe73d3099", size = 787081, upload-time = "2026-09-08T18:47:16.227Z" }, + { url = "https://files.pythonhosted.org/packages/26/7b/9d52066d363faba7f3ac20ee60a3c696a475feb2c477d235d0d649d41cc1/hypothesis-6.168.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:732ae5d47482f99d8028cca096729625f05690a83f5e7ce31466e266155792f4", size = 1123850, upload-time = "2026-09-08T18:48:11.504Z" }, + { url = "https://files.pythonhosted.org/packages/50/cf/aa46d76fa7df43caf2c372e394fda84ce1dc08421814f8674a6b9295e2ea/hypothesis-6.168.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2085ee74ac3ab6b70e2f7ffae9b4cb74c246da2f574b2de81a0818a8a30f659f", size = 1147685, upload-time = "2026-09-08T18:46:58.219Z" }, + { url = "https://files.pythonhosted.org/packages/84/c2/78ed8c8d5aa37e4baae2a4b3e29687ee3d9b7f1a5e56ea5ea5ec7ec71ecb/hypothesis-6.168.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1894782fae5d9a7bb44e6dcf848ccb09ccb5babab48d8b5c31a0a7fc025b82a1", size = 1149294, upload-time = "2026-09-08T18:47:04.757Z" }, + { url = "https://files.pythonhosted.org/packages/78/7f/d57440f19e9de70e85359cf179ce786f309ee17609bd3c5a0113272875c0/hypothesis-6.168.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecf0ab13cef899efb816ffdd7963e0679f372520884ce06756c7642f3df94213", size = 1169729, upload-time = "2026-09-08T18:47:44.056Z" }, + { url = "https://files.pythonhosted.org/packages/a1/86/dc74410a186990bb22c2a3eea0e77804f2eb0f300860b46d7d8948073674/hypothesis-6.168.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:3f6dcf66270278d078bed01b401f47db4e26456cd909d8e23c6b9366a6c0b131", size = 1129182, upload-time = "2026-09-08T18:46:33.382Z" }, + { url = "https://files.pythonhosted.org/packages/b2/5e/be048fc4f6dac831625e155bdf11e4233caf46bb54030391d6fba8e19449/hypothesis-6.168.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bfef4d46dbf1704a7b8fa3a78778651a2cb18870ca0a70da19c381646822b149", size = 1160180, upload-time = "2026-09-08T18:47:26.459Z" }, + { url = "https://files.pythonhosted.org/packages/0f/4a/15a34498a5f08720fbbdbbe4668a8050fe4e17c16c9eeb6f56a017f2fa6b/hypothesis-6.168.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1d1aa5b3484e329295d88488a5ba06243909e65c2ab616513c2d36721de4ed1d", size = 1299711, upload-time = "2026-09-08T18:47:28.34Z" }, + { url = "https://files.pythonhosted.org/packages/77/09/5354e0dae302ab98c4f0046b7e8699c2186ae4349397bda5d5c852bda68b/hypothesis-6.168.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:3bc00fd8cda04b58e37a1163e8a65389b247b4f5ee547ae37d244a4960995517", size = 1425341, upload-time = "2026-09-08T18:46:56.785Z" }, + { url = "https://files.pythonhosted.org/packages/59/7c/3c0e1f59043ff128c70a51d298d3d6b5973525c357a1e1d0542dbc05ac90/hypothesis-6.168.0-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:990026952d5b2eca290c88f639ac639233f47e13dae338c6dfb6e4774bcab349", size = 1281063, upload-time = "2026-09-08T18:46:55.401Z" }, + { url = "https://files.pythonhosted.org/packages/40/dd/db884db9a7d42ae6b72a00c13c725940b638dfb263e618b22af59d5dfa2f/hypothesis-6.168.0-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:a74b0945acbbd552c7c2d0a99a3b5232962b8848c8eed1829451800a9bfcf00b", size = 1300247, upload-time = "2026-09-08T18:47:02.949Z" }, + { url = "https://files.pythonhosted.org/packages/99/8a/4ee9769e1d48676efb6a78a130f82e0d52d3f87b2055294102272a08615c/hypothesis-6.168.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2a380b521b5a76a9e8917d64adcf7f861a45a4360a34b1579af14c5df8eb0377", size = 1336084, upload-time = "2026-09-08T18:48:05.675Z" }, + { url = "https://files.pythonhosted.org/packages/61/17/d4ed11bc99d205d6d2651a0f1a4874f377150f836b5a0849bd511d68a2eb/hypothesis-6.168.0-cp310-abi3-win32.whl", hash = "sha256:2264f15a1c80329e3ad48e39c44bd5c9429b7b04c9ee62cdd72f4b10aaac9f29", size = 677989, upload-time = "2026-09-08T18:47:24.722Z" }, + { url = "https://files.pythonhosted.org/packages/77/51/abf1fde7b8afab87db30afb73b3847e62440551d146472111cabeba2fe00/hypothesis-6.168.0-cp310-abi3-win_amd64.whl", hash = "sha256:5b54769033b84477931d2072e7133a7555e0de5c53fd5ca3bbde960762d7d31b", size = 684692, upload-time = "2026-09-08T18:46:24.449Z" }, + { url = "https://files.pythonhosted.org/packages/12/e2/64d79aed47a95186ce9edcef60eb4684870c72542da3b7027c2483d8ee8c/hypothesis-6.168.0-cp310-abi3-win_arm64.whl", hash = "sha256:112b0900059bf9d7d6528ed729770629ab146e0d133c4143b9bd4a01dc002bcc", size = 682709, upload-time = "2026-09-08T18:48:03.756Z" }, + { url = "https://files.pythonhosted.org/packages/a6/7a/7a28d9afd52c9a89c4d8e3170c92fe1ffbd4a4ce3f905ca15fa7ddaaa581/hypothesis-6.168.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:4d7d29dd63ad9fdc4aa1d65fa272449e14aaf6c6bb8451091818c2945533a43a", size = 792045, upload-time = "2026-09-08T18:46:29.612Z" }, + { url = "https://files.pythonhosted.org/packages/0a/40/e2d6fe12b54abbc8b802b234ec4974d0a420dde4098f57a324e9c791ac27/hypothesis-6.168.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9a72ed7afa1f7e30488b8a5754fca0ad9755518bdb77d6f0b003cadf7437a5f9", size = 787932, upload-time = "2026-09-08T18:47:11.145Z" }, + { url = "https://files.pythonhosted.org/packages/ff/89/cc78e05f3248949c6d9ecb489d0e422247d11b08533ab02e177a10a01610/hypothesis-6.168.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fcc5bad4300a751804ce41f0e10d77f85272668160708ce39ec579bca8984843", size = 1123994, upload-time = "2026-09-08T18:47:12.647Z" }, + { url = "https://files.pythonhosted.org/packages/f6/e3/7d483a79e2ed9a6c868cac8783b3c6c2bb82303f7db364c9da9a406df10d/hypothesis-6.168.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53469a1a7c4861b12c9a8622f762d7d1fd7bcf171884e1018ed5a8f063a5c063", size = 1170271, upload-time = "2026-09-08T18:47:54.322Z" }, + { url = "https://files.pythonhosted.org/packages/78/87/bffaf5b5e6b4566aa40d8a4ae25d8d84b47dfe0d9e771f454e4974cdb428/hypothesis-6.168.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a9650c4882fdbdd8e90bdae602a8bfa8c6f09dc5d06afec5b9b23982e8f60a04", size = 1300162, upload-time = "2026-09-08T18:46:27.022Z" }, + { url = "https://files.pythonhosted.org/packages/25/06/6c30a00fc2f5c58546e858259cf2261c6a3ef6a060c373e00d00311267be/hypothesis-6.168.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:348d9b93fd4129f67f9bab94f3d70709a9372bbe0e0d22731325ce85d5eb409f", size = 1336294, upload-time = "2026-09-08T18:46:37.324Z" }, + { url = "https://files.pythonhosted.org/packages/fa/b1/501910e5a6a9c245f2394fda016376f6c7e209c5dca8c056eaece64ad10e/hypothesis-6.168.0-cp311-cp311-win_amd64.whl", hash = "sha256:719b45b0512e3535a6a0077c2f7c6053b02ac0e72d60693f66f98790a33855b2", size = 684464, upload-time = "2026-09-08T18:47:19.779Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5e/0035896c101f0484c364353f8ee30175eef8936b49171618677287fdd85d/hypothesis-6.168.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:6b750390dac4429da0cb70ab3fe758457f0cea3d9c843d48c59d0690d1189fda", size = 793115, upload-time = "2026-09-08T18:48:21.352Z" }, + { url = "https://files.pythonhosted.org/packages/11/5c/938173e27df771cc6e92bc47f117a1b1be4a88fc6dc214f7fc65f9c7ad93/hypothesis-6.168.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8e4b2d434e0dd134f3d31ac1efc1825bf99730dfe70fec005ff66d7211836d79", size = 784634, upload-time = "2026-09-08T18:47:21.406Z" }, + { url = "https://files.pythonhosted.org/packages/88/00/0b2c6ac07d519131f97712f2533750ffe9b2490eec8f518ef3a5ed2dd514/hypothesis-6.168.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76d4d36ed2fd62de11382f1d608169c1ffa9a49d3b9351146d8ff87cb81a66f7", size = 1122855, upload-time = "2026-09-08T18:46:21.967Z" }, + { url = "https://files.pythonhosted.org/packages/8d/21/dde930fe43171cab37572bd993d70c2a271f240f428f8ccd64ec4c2d661b/hypothesis-6.168.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5920d267f7d8cfd376672f2bde5905cdf284d47519582e41ce7c142d48ee46c4", size = 1168932, upload-time = "2026-09-08T18:46:39.856Z" }, + { url = "https://files.pythonhosted.org/packages/4c/5d/92b83c3d06194ec626e92723d0b0f70221ebf42d7cb355ed36929df6d735/hypothesis-6.168.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:fb8cdf45361e259df86e19f8cd042ce2d6c7e6ad88fa631b78a4e3a83c2e572d", size = 1298566, upload-time = "2026-09-08T18:48:19.485Z" }, + { url = "https://files.pythonhosted.org/packages/9c/67/52de8bf3446e3d2d555b812d96673b31bb213b5b9d5804a666e5d0bba76e/hypothesis-6.168.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3b3ce1cce70b25a37ed1a38a53ce7204785726c675c0f41a0f83c338a7e47b3d", size = 1335074, upload-time = "2026-09-08T18:48:01.561Z" }, + { url = "https://files.pythonhosted.org/packages/14/fd/e592773c1c0ce55e35d26ec55f75546bf1fd72ef5a5c520ed685969b40cb/hypothesis-6.168.0-cp312-cp312-win_amd64.whl", hash = "sha256:f62bdabf278db9ff61df5f3203d608949f0d893d0e30cdac3f2330e67e41ae68", size = 682026, upload-time = "2026-09-08T18:47:01.471Z" }, + { url = "https://files.pythonhosted.org/packages/ef/e9/39bb8fcccfbafd10fcc777d583c6ebad5d2148e7d743cb350562f28e974f/hypothesis-6.168.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7d55562bf8d41cfa18559c33f30cadf44ceac8e517509d7a022a9feace621f28", size = 793050, upload-time = "2026-09-08T18:47:56.045Z" }, + { url = "https://files.pythonhosted.org/packages/f7/dd/00fd32e8ec470535e0065cb8d6e175f9fc54b4d3a6f1269f6c487f8bd79d/hypothesis-6.168.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92cff497b92e2285ff6a94193fdee04aba483a4115d501c1f9a570bd103fcd20", size = 784558, upload-time = "2026-09-08T18:46:32.054Z" }, + { url = "https://files.pythonhosted.org/packages/d2/4d/553c47093f68bdbac0438e16c024ce97649b5804dc6972ef86b9bd2db8a1/hypothesis-6.168.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ff259260015f9be3756dcd4bc11c08e007314dec6b43d9a89084c4f34f94475", size = 1122841, upload-time = "2026-09-08T18:48:09.374Z" }, + { url = "https://files.pythonhosted.org/packages/43/d6/0b5940aa75e617c8fd12200bae24d1b71347362514e8210735c581d4d3d1/hypothesis-6.168.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:35f1262831b5acc74ded15f629965daffcd657f6016ee04fc9605f6eb2b334c0", size = 1168825, upload-time = "2026-09-08T18:47:33.702Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c0/800de1231b2869b51409bbf85799d6f1bf49a00e0afff0aabc097aa8f1b7/hypothesis-6.168.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:046fe4bcfce2a2fa186ba9d96bbb62c25c2f6c2e4071f0783ed6b5cc481d0669", size = 1298433, upload-time = "2026-09-08T18:46:38.53Z" }, + { url = "https://files.pythonhosted.org/packages/be/35/9907667a30c1dbabbc44a09b4c25a0f575570937fcbbada924ae3a1dbf2a/hypothesis-6.168.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:24b52a2b1c8db6e1e516f9295c8e4ef7ef63303ff24fbbc5b35f4ff71dcd732c", size = 1334988, upload-time = "2026-09-08T18:48:29.395Z" }, + { url = "https://files.pythonhosted.org/packages/98/8a/7bf214e703fff532ed47cb52ab93ce4b7ea41e4e7084593fa02b740828b7/hypothesis-6.168.0-cp313-cp313-win_amd64.whl", hash = "sha256:ec0886fe0be9091669937989f9a662beca42ae14a4a6dab25491c2c63365f88d", size = 681990, upload-time = "2026-09-08T18:47:49.173Z" }, + { url = "https://files.pythonhosted.org/packages/2f/c1/64b36b250b1f66abb6ce8c81775d3373d149bc89cbea477ed71b57cf7d1b/hypothesis-6.168.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:e2df8afacf9261070795db36db4a394e3ccdbb663fd2d38c7a9fba0c836dcecc", size = 793101, upload-time = "2026-09-08T18:46:54.067Z" }, + { url = "https://files.pythonhosted.org/packages/6d/c4/494e42304b15f4ec649d36bbc3fc01cef1b63405cf30d4087ae07d048172/hypothesis-6.168.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9ba679f183c67adcb6f4ad93694beafb6da99fe691757f4e57b04ae77e581ba8", size = 784632, upload-time = "2026-09-08T18:48:31.551Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6d/90d874cb1d749f505749c9908f34e803b97b03457797d5893354980558bc/hypothesis-6.168.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d9a8574f80fc859313aee56167d202e8625c0eedd200971130f0839f06d1c93", size = 1123101, upload-time = "2026-09-08T18:48:17.362Z" }, + { url = "https://files.pythonhosted.org/packages/3d/bb/ec893d0e5f4bcdd0121a8a4280f4e8aba3b4cdae01411f3236016ecd1f81/hypothesis-6.168.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:deb02de608268928d779aa889b0a9d67794b1cc0c54a322cf19e386be8a46ca7", size = 1168962, upload-time = "2026-09-08T18:46:48.507Z" }, + { url = "https://files.pythonhosted.org/packages/11/f1/16ec2bddbaed461725d9aa5a80b43f1905ea50a08f689ec46f8966bb4f0f/hypothesis-6.168.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:076a2096c34448931c3cfeb2eb7a6b843a56ffdce5e4e3a025bfdf8f935666d9", size = 1298932, upload-time = "2026-09-08T18:47:40.632Z" }, + { url = "https://files.pythonhosted.org/packages/8f/12/7c2fe2706d092f12bd7b3e8565e1ca5d0c24b853751f2f970768086dbdeb/hypothesis-6.168.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f099b1c8fc49ec2d9d7944e661addb97d7c38e818fb8d1f78073c43895a87f6", size = 1335196, upload-time = "2026-09-08T18:47:57.775Z" }, + { url = "https://files.pythonhosted.org/packages/20/35/59f7ca2414ca39408d13f66a344affe0ffc64748dc01d8a1ca910009cdcb/hypothesis-6.168.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:93413d1b0af50a7b165d66278c529174bf2fd1773c78027735dc0b50d1d3fd27", size = 624102, upload-time = "2026-09-08T18:47:38.869Z" }, + { url = "https://files.pythonhosted.org/packages/a9/1e/dcd9335ace916ffea40f2cb04ba4122094c2f7b928f3a73fc7d452ce5b71/hypothesis-6.168.0-cp314-cp314-win_amd64.whl", hash = "sha256:db2751c27bffc8491a96d72969649089d5400115e4b7c49bf7167ebbdcc84193", size = 681871, upload-time = "2026-09-08T18:47:59.697Z" }, + { url = "https://files.pythonhosted.org/packages/de/d0/bc50b0b91e40744b7caa56b8add85cef432f85b4d00108409e8eb17af830/hypothesis-6.168.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:cd0c1dcf308e919c8ae708054d0ad61921ae87634a9aea574a9851da584cebc1", size = 791695, upload-time = "2026-09-08T18:47:06.306Z" }, + { url = "https://files.pythonhosted.org/packages/4b/53/fc7537d50ff008dc5ea8598764935f93dd07bcedaf23ee4e635bdf7055f4/hypothesis-6.168.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d0bdb77f976740b8cd5ec697327ea343d02d052b9916d213b5d4c65d823415cd", size = 783239, upload-time = "2026-09-08T18:47:47.43Z" }, + { url = "https://files.pythonhosted.org/packages/71/2a/c7aac2efc06713f704d7e354755aff4a11608b9fe93d973ead374f3b81a3/hypothesis-6.168.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f7486bed33225d02f6aa78a4c4ba2b6f84992a82571cdda1bf08dce41d13507", size = 1121412, upload-time = "2026-09-08T18:47:07.927Z" }, + { url = "https://files.pythonhosted.org/packages/60/e2/668ab29e5096af682b17b8491f5427d7c5f17c1b991daa5577bde80c29ca/hypothesis-6.168.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ba3838c4a92e0b9730d1ed7e67e4950c152ad79d0a0c7594065262db84c55c4", size = 1167570, upload-time = "2026-09-08T18:47:17.94Z" }, + { url = "https://files.pythonhosted.org/packages/3a/17/c64635e4c988b5fa3d3b8be322e19e2fe4c731fa0ca074ca852aa70debea/hypothesis-6.168.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:891b2d281ede45130e7fa0a22fd65336cc77ef2f780ec3792e8de6fc274a02c8", size = 1297118, upload-time = "2026-09-08T18:48:13.407Z" }, + { url = "https://files.pythonhosted.org/packages/e7/c5/c7a0d9a06bf5c3279386dd53161081a57b98c6faf60fbbf64d046315e9e6/hypothesis-6.168.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e86820053afad84677f301c0b892a226be1df49790800a65668ae7cc8a1ac571", size = 1334068, upload-time = "2026-09-08T18:48:15.462Z" }, + { url = "https://files.pythonhosted.org/packages/32/99/11a393a20a867e9b978308323d45022e96f5cd2edf391e4d9a65fb4e2cf7/hypothesis-6.168.0-cp314-cp314t-win_amd64.whl", hash = "sha256:a4956f41ab1ec6e6ef9262a35970e9f3e2caaaa1cdafe0d413156c6934dd99d8", size = 681795, upload-time = "2026-09-08T18:47:14.415Z" }, + { url = "https://files.pythonhosted.org/packages/16/f7/5adae1bf1d4877aca2e8c8e007e57077237e9ac3765e430ffda490b17e19/hypothesis-6.168.0-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:754016594fe78cef91790e0922f60d183c52f531255fbfa30dac495b813e2128", size = 791056, upload-time = "2026-09-08T18:48:27.398Z" }, + { url = "https://files.pythonhosted.org/packages/f2/93/b1b2770b87591db5cf564b9aa0265cf21e9207d28645e0abdeb8df63225b/hypothesis-6.168.0-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:6f0dd437ec01140676192422b61f2f833b3ce6a3213da9b7e196ad6b3777e795", size = 782980, upload-time = "2026-09-08T18:48:34.087Z" }, + { url = "https://files.pythonhosted.org/packages/de/bd/673171c1d2423379a7d4a0f9f009cca735a422d0c4a4ac6422d1d1736cab/hypothesis-6.168.0-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f77af7721ff35a58fa8797decd14c932c350a2548686c6e9b844db710a3a2441", size = 1120964, upload-time = "2026-09-08T18:46:59.92Z" }, + { url = "https://files.pythonhosted.org/packages/7d/00/33a9bd941b22a4fd8a8c805b1563e0db17efc020422f5bd15bdd0fdf258f/hypothesis-6.168.0-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a0d28418c104d7268fdebcc09bc49f7b6569b5eb942430c6859f53ec8d4edf63", size = 1143869, upload-time = "2026-09-08T18:46:49.875Z" }, + { url = "https://files.pythonhosted.org/packages/0f/c1/963460976f41721eff8f67f30d31059ea407cc1b41c737c8859aabf37197/hypothesis-6.168.0-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:812a84c4cc7f7ae4fcb39a5647cc2698e6c18254f8423126425578f1dcdac782", size = 1146453, upload-time = "2026-09-08T18:46:25.757Z" }, + { url = "https://files.pythonhosted.org/packages/ce/53/09db238098ad66f4e6d2fe883f26c270c2595b90e21c8f969d4cb21cad7a/hypothesis-6.168.0-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6de30e559eb151de14a5f74bceb4d97792a9315ada2a1816b5da825cd7d28edc", size = 1166918, upload-time = "2026-09-08T18:48:23.436Z" }, + { url = "https://files.pythonhosted.org/packages/b9/31/e1b7b452c8a6166e445ba2ad80a864f6a9eee0fe4c8cecdb9af5c1ee0aa5/hypothesis-6.168.0-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:9018b20acdb061b2ef4b2fa7f558ca5db97ffea316e0a528bc003a24b2ac996e", size = 1126637, upload-time = "2026-09-08T18:47:50.878Z" }, + { url = "https://files.pythonhosted.org/packages/97/2b/4eceed248afb46fb6b2df21cf2239362de5b25d295c2dc67a82ec8657d5e/hypothesis-6.168.0-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bc935a5d5f86fd8f5af951b8fbe00307f6f7c596f82a9a27c17d974f6ab0a26c", size = 1155682, upload-time = "2026-09-08T18:47:30.234Z" }, + { url = "https://files.pythonhosted.org/packages/b5/3e/f3414cda4f325004d5774485e8983b7d1b99e4b91013f013dd088fd778cf/hypothesis-6.168.0-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:45fcfa05f746e253350f55f216bcef59754f5f2b85745f1fc2bb8ba81dd517a9", size = 1296464, upload-time = "2026-09-08T18:46:34.833Z" }, + { url = "https://files.pythonhosted.org/packages/aa/40/ca79cf96545e1f172b36b8df56bfeb02b61f351f283026f9a57c4631e368/hypothesis-6.168.0-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:f89d8e998d3c936ffbbd1c3686c96f0378f6558aecc5967a3035a857f2bab0ad", size = 1421853, upload-time = "2026-09-08T18:47:23.083Z" }, + { url = "https://files.pythonhosted.org/packages/77/bc/657d5386740c1f4ac518ff05e4c5b132f1e6df2e7c865ba8fda4279adfa5/hypothesis-6.168.0-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:d0620fa320fa66649e6bfd71e94f3f86115fffebb7e3c6dcece19d1aaff8e07f", size = 1278216, upload-time = "2026-09-08T18:46:30.871Z" }, + { url = "https://files.pythonhosted.org/packages/51/54/2328cdb70489a36634534478d9b238594a269d8bec4630ea0e6897048347/hypothesis-6.168.0-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:4085b61e25d3dcc6c9151d4115269870aee8cdb921611ee5c989b2786449be09", size = 1297593, upload-time = "2026-09-08T18:46:41.395Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ca/d803fa57e3ff7f460f6e262d2b74822cf143343d378501fd3da601b12040/hypothesis-6.168.0-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:b5449a64eb37d9a4aa6ac9cd2ab0fd1a24145adf421ef1536884f73f39824887", size = 1333785, upload-time = "2026-09-08T18:48:25.434Z" }, + { url = "https://files.pythonhosted.org/packages/86/8f/b9799ae6ba6074f151db2c63f9f6f12d844512821ffaba1a3672d7e59f07/hypothesis-6.168.0-cp315-abi3.abi3t-win32.whl", hash = "sha256:91e3de666a6c4f7543000d1710e25055d63ef3032c98bd2ab338b3087bdaa780", size = 675173, upload-time = "2026-09-08T18:47:52.601Z" }, + { url = "https://files.pythonhosted.org/packages/61/54/14c3e277b451ff24128ecc2673cac59dd7e535bce1a433c466912fd682e1/hypothesis-6.168.0-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:9a2079cd09919956dd388f1a1f8ea5a79f2b2437650fbeda31d8661217ffefef", size = 681491, upload-time = "2026-09-08T18:46:51.437Z" }, + { url = "https://files.pythonhosted.org/packages/99/f3/827e4a48ffee7e40244b0bf064ba47c2171e053ab7edf1cf770105e23401/hypothesis-6.168.0-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:085c9aa246487c56a40ca89003d285cbffdbb5be4097ba6d0139f9c21003c04a", size = 679197, upload-time = "2026-09-08T18:47:32.112Z" }, + { url = "https://files.pythonhosted.org/packages/1f/96/7f15e1b10d13e266b0f09cf3117dd2206104a1f6ac3b3baf6ec4c43b7c2e/hypothesis-6.168.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:16864797de4b024e4c6cebd44598af932f870aad811341bc5bc24c738801ff76", size = 792946, upload-time = "2026-09-08T18:47:35.438Z" }, + { url = "https://files.pythonhosted.org/packages/10/fb/32487bdcf68b3805ec5efcb7f92fb002b573ad3f90c252d2f2913c5d3124/hypothesis-6.168.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:283eda952bcb1987ccba1c8b634db0e8a960e1e92e2daa7003bc2392f19cea01", size = 788783, upload-time = "2026-09-08T18:46:42.862Z" }, + { url = "https://files.pythonhosted.org/packages/13/26/1aa42e0069ebc68d29d980d47420af324fa002494e01c1c0321c34f609ba/hypothesis-6.168.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5427a3c951080c18170486f775df6a82153882b819eca6b8e7ed77693634e5ab", size = 1124762, upload-time = "2026-09-08T18:48:07.512Z" }, + { url = "https://files.pythonhosted.org/packages/ae/4e/99509045ed55aaa05d8408663051a93168d1a8a5bfc132d92ec1584ce47a/hypothesis-6.168.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a838218ff1eab8d7b4bf66b96037fce0a802f61f2fa5fd4b784696cac365ce7", size = 1171748, upload-time = "2026-09-08T18:46:47.029Z" }, + { url = "https://files.pythonhosted.org/packages/a2/c9/96e57dfd89913322b5180b031b9e36a29bfe67842108b19904466476cd24/hypothesis-6.168.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:34e3c8b66047ba92f8b8df5e427074058d92db58038f007da4bf9d14e934ad3c", size = 685447, upload-time = "2026-09-08T18:46:35.97Z" }, +] + [[package]] name = "identify" version = "2.6.19" @@ -3114,6 +3198,7 @@ dev = [ { name = "flake8" }, { name = "hatch-vcs" }, { name = "hatchling" }, + { name = "hypothesis" }, { name = "jsonschema" }, { name = "pre-commit" }, { name = "pytest-cov" }, @@ -3147,6 +3232,7 @@ dev = [ { name = "flake8", specifier = ">=7.3.0" }, { name = "hatch-vcs", specifier = ">=0.5.0" }, { name = "hatchling", specifier = ">=1.30.1" }, + { name = "hypothesis", specifier = ">=6.168.0" }, { name = "jsonschema", specifier = ">=4.26.0" }, { name = "pre-commit", specifier = ">=4.5.1" }, { name = "pytest-cov", specifier = ">=6.1.0" }, @@ -3631,6 +3717,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + [[package]] name = "sqlalchemy" version = "2.0.51" From fbe8c2b481439cf47a8c159a16c87816cfaf6764 Mon Sep 17 00:00:00 2001 From: Nina Chikanov Date: Thu, 10 Sep 2026 18:05:44 -0700 Subject: [PATCH 9/9] [FIX]: Resolve trace annotations without modifying shared types Restore types.py to its PR-base contents and supply datetime/Path resolution inside the canonical adapter. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/concepts/trace-schema.md | 2 ++ rampart/core/result.py | 7 ++++- rampart/core/types.py | 9 ++---- tests/unit/core/test_serialization.py | 45 +++++++++++++++++++++------ 4 files changed, 46 insertions(+), 17 deletions(-) diff --git a/docs/concepts/trace-schema.md b/docs/concepts/trace-schema.md index 4e7473d7..85271aac 100644 --- a/docs/concepts/trace-schema.md +++ b/docs/concepts/trace-schema.md @@ -41,6 +41,8 @@ These policies belong to the cached canonical adapter, not to the public dataclass annotations or configuration. Fields remain `dict[str, Any]` and `datetime | None`. Independently constructed Pydantic adapters retain their normal behavior, including live binary payload support. +The canonical adapter supplies its own `datetime` / `Path` resolution namespace; +the shared types module keeps those imports under `TYPE_CHECKING`. `ResultRecord.json_schema()` returns the adapter-derived body schema plus the versioned envelope. Small schema customizations describe the trace-only payload diff --git a/rampart/core/result.py b/rampart/core/result.py index cd800630..cab191b6 100644 --- a/rampart/core/result.py +++ b/rampart/core/result.py @@ -13,8 +13,10 @@ import json from dataclasses import dataclass, field +from datetime import datetime from enum import Enum, StrEnum from functools import cache +from pathlib import Path from typing import ( TYPE_CHECKING, Annotated, @@ -278,7 +280,10 @@ def _result_adapter() -> TypeAdapter[Result]: Returns: TypeAdapter[Result]: The cached adapter. """ - return TypeAdapter(Annotated[Result, GetPydanticSchema(trace_schema)]) + adapter = TypeAdapter[Result](Annotated[Result, GetPydanticSchema(trace_schema)]) + # Nested dataclasses keep these imports under TYPE_CHECKING. + adapter.rebuild(_types_namespace={"datetime": datetime, "Path": Path}) + return adapter class _ResultJsonSchema(GenerateJsonSchema): diff --git a/rampart/core/types.py b/rampart/core/types.py index 174a7605..96da246a 100644 --- a/rampart/core/types.py +++ b/rampart/core/types.py @@ -11,16 +11,13 @@ import uuid from dataclasses import dataclass, field -from datetime import ( - datetime, # ruff: ignore[typing-only-standard-library-import] Resolved by TypeAdapter. -) from enum import Enum -from pathlib import ( - Path, # ruff: ignore[typing-only-standard-library-import] Resolved by TypeAdapter. -) from typing import TYPE_CHECKING, Any if TYPE_CHECKING: + from datetime import datetime + from pathlib import Path + from rampart.core.manifest import AppManifest diff --git a/tests/unit/core/test_serialization.py b/tests/unit/core/test_serialization.py index 2e63f35f..16cb7da8 100644 --- a/tests/unit/core/test_serialization.py +++ b/tests/unit/core/test_serialization.py @@ -19,6 +19,7 @@ from typing import ( TYPE_CHECKING, Any, + TypeVar, get_type_hints, ) from unittest.mock import patch @@ -27,11 +28,13 @@ from jsonschema import Draft202012Validator from pydantic import TypeAdapter +from rampart.core import types as core_types from rampart.core.result import ( InjectionRecord, PopulationRef, Result, SafetyStatus, + _result_adapter, ) from rampart.core.serialization import ( TRACE_SCHEMA_VERSION, @@ -58,6 +61,13 @@ from collections.abc import MutableMapping _TIMESTAMP = datetime(2026, 1, 2, 3, 4, 5, tzinfo=UTC) +_AdapterType = TypeVar("_AdapterType") + + +def _regular_adapter(cls: type[_AdapterType]) -> TypeAdapter[_AdapterType]: + adapter = TypeAdapter(cls) + adapter.rebuild(_types_namespace={"datetime": datetime, "Path": Path}) + return adapter def _make_eval_result() -> EvalResult: @@ -465,7 +475,7 @@ def test_live_binary_payload_is_still_supported(self, tmp_path: Path) -> None: artifact.write_bytes(b"%PDF-1.4 fake") payload = Payload(content="doc", format=PayloadFormat.PDF, artifact=artifact) - assert TypeAdapter(Payload).validate_python(payload) is payload + assert _regular_adapter(Payload).validate_python(payload) is payload assert payload.artifact == artifact @@ -597,7 +607,18 @@ def test_null_is_not_a_default_for_nonnullable_fields(self, field: str) -> None: class TestAdapterIsolation: + def test_cold_adapter_resolves_types_without_changing_their_module(self) -> None: + _result_adapter.cache_clear() + + restored = ResultRecord.from_dict(_minimal_record_dict()) + restored.to_dict() + ResultRecord.json_schema() + + assert "datetime" not in vars(core_types) + assert "Path" not in vars(core_types) + def test_public_annotations_remain_standard_types(self) -> None: + namespace = {"datetime": datetime, "Path": Path} for cls, name in [ (Result, "metadata"), (Payload, "metadata"), @@ -605,17 +626,21 @@ def test_public_annotations_remain_standard_types(self) -> None: (ToolCall, "arguments"), (SideEffect, "details"), ]: - assert get_type_hints(cls, include_extras=True)[name] == dict[str, Any] + assert ( + get_type_hints(cls, localns=namespace, include_extras=True)[name] + == (dict[str, Any]) + ) for cls in [ToolCall, Turn]: - assert get_type_hints(cls, include_extras=True)["timestamp"] == ( - datetime | None + assert ( + get_type_hints(cls, localns=namespace, include_extras=True)["timestamp"] + == datetime | None ) assert not hasattr(Result, "__pydantic_config__") def test_regular_adapters_are_unchanged_before_and_after_canonical_use( self, ) -> None: - adapter = TypeAdapter(Result) + adapter = _regular_adapter(Result) original_schema = adapter.json_schema() result = _make_full_result(metadata={"tuple": (1, 2), "opaque": object()}) @@ -624,14 +649,14 @@ def test_regular_adapters_are_unchanged_before_and_after_canonical_use( Result.json_schema() assert adapter.validate_python(result) is result - assert TypeAdapter(Result).validate_python(result) is result + assert _regular_adapter(Result).validate_python(result) is result assert adapter.json_schema() == original_schema - assert TypeAdapter(Result).json_schema() == original_schema + assert _regular_adapter(Result).json_schema() == original_schema def test_regular_adapter_can_still_generate_payload_ids(self) -> None: _make_full_result().to_dict() - payload = TypeAdapter(Payload).validate_python( + payload = _regular_adapter(Payload).validate_python( {"content": "live", "metadata": {"tuple": (1, 2)}} ) @@ -642,7 +667,7 @@ def test_regular_adapter_retains_pydantic_datetime_behavior(self) -> None: result = _make_full_result() canonical = result.to_dict() - regular = TypeAdapter(Result).dump_python(result, mode="json") + regular = _regular_adapter(Result).dump_python(result, mode="json") assert canonical["turns"][0]["timestamp"].endswith("+00:00") assert regular["turns"][0]["timestamp"].endswith("Z") @@ -654,7 +679,7 @@ def test_regular_adapter_can_decode_live_binary_payloads( artifact.write_bytes(b"%PDF-1.4 fake") _make_full_result().to_dict() - payload = TypeAdapter(Payload).validate_python( + payload = _regular_adapter(Payload).validate_python( {"content": "doc", "format": "pdf", "artifact": str(artifact)} )