From 90e819086d4a9dda4b8af5d2c3716fbe7f41ad4a Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 4 Sep 2026 15:13:05 -0400 Subject: [PATCH 1/6] feat(pricing): dated product price snapshot type with rate-only digest (DSE-1514) Claude-Session: https://claude.ai/code/session_01K1dHPjZ1bZcE2GnX3KMMSH --- src/conclave/pricing.py | 131 ++++++++++++++++++++++++++++++++- tests/test_pricing_snapshot.py | 87 ++++++++++++++++++++++ 2 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 tests/test_pricing_snapshot.py diff --git a/src/conclave/pricing.py b/src/conclave/pricing.py index b6542d4..659af7f 100644 --- a/src/conclave/pricing.py +++ b/src/conclave/pricing.py @@ -22,11 +22,15 @@ from __future__ import annotations +import hashlib +import json from collections.abc import Sequence from dataclasses import dataclass +from datetime import date from decimal import ROUND_CEILING, Decimal, localcontext +from typing import Literal -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator # The quantum every emitted amount is rounded up to: one USD micro-cent. USD_MICROCENT = Decimal("0.000001") @@ -263,3 +267,128 @@ def reported_usage_cost( + Decimal(unattributed) * higher ) / TOKENS_PER_MILLION return cost.quantize(USD_MICROCENT, rounding=ROUND_CEILING) + + +# A snapshot older than this many days emits a bounded warning on the manifest. +# It is a STALENESS SIGNAL ONLY: an old snapshot still prices at exactly the +# rates it records. Substituting a rate because one looks old would replace a +# falsifiable claim with a guess, which is the failure this module exists to +# prevent. +PRICE_SNAPSHOT_MAX_AGE_DAYS = 90 + +# Digest namespace for the PRODUCT snapshot. Deliberately distinct from the eval +# harness's ``conclave_model_prices_v1`` so the two hash spaces can never collide +# and an eval price book can never be mistaken for a product snapshot. +_PRODUCT_PRICE_HASH_NAMESPACE = "conclave_product_prices_v1" + + +def _canonical_decimal(value: Decimal) -> str: + """Render a Decimal in a normalized, representation-independent form. + + ``Decimal("3.00")`` and ``Decimal("3")`` are the same rate and must produce + the same digest, so trailing fractional zeros are stripped and exponent + notation is expanded. + """ + sign, digits, exponent = value.as_tuple() + digit_text = "".join(str(digit) for digit in digits) + if exponent >= 0: + integer = digit_text + ("0" * exponent) + fraction = "" + else: + split_at = len(digit_text) + exponent + if split_at > 0: + integer = digit_text[:split_at] + fraction = digit_text[split_at:] + else: + integer = "0" + fraction = ("0" * -split_at) + digit_text + fraction = fraction.rstrip("0") + canonical = integer if not fraction else f"{integer}.{fraction}" + return f"-{canonical}" if sign else canonical + + +class PriceSnapshot(BaseModel): + """One dated, frozen, hand-verified set of product price ceilings. + + Snapshots are checked-in artifacts under ``src/conclave/data/``, never + fetched at runtime (explicitly out of scope). Every entry cites the vendor + page its rate was read from; a model whose published list price could not be + verified is OMITTED, which makes it *unpriced* rather than guessed. + + Attributes: + snapshot_id: Stable identifier, conventionally + ``conclave-default-prices-``. + captured_at: The date the rates were read from the vendor pages. + currency: Always ``"USD"``. + entries: The priced models, keyed by full provider-prefixed model id. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + snapshot_id: str = Field(min_length=1) + captured_at: date + currency: Literal["USD"] + entries: tuple[PriceRates, ...] = Field(min_length=1) + + @model_validator(mode="after") + def validate_entries(self) -> PriceSnapshot: + """Require unique model ids and a citation on every entry.""" + model_ids = [entry.model_id for entry in self.entries] + if len(set(model_ids)) != len(model_ids): + raise ValueError("price snapshot model ids must be unique") + uncited = sorted(entry.model_id for entry in self.entries if not entry.source_url) + if uncited: + raise ValueError(f"price snapshot entries must carry a source_url: {uncited}") + return self + + def rates_for(self, model_id: str) -> PriceRates | None: + """Return the EXACT entry for ``model_id``, or ``None``. + + Matching is exact and total: there is no prefix match, no provider + fallback, and no nearest-neighbour rate. An absent model is unpriced, + full stop. + """ + for entry in self.entries: + if entry.model_id == model_id: + return entry + return None + + def is_stale(self, *, as_of: date | None = None) -> bool: + """Return whether this snapshot is older than the staleness threshold.""" + reference = as_of or date.today() + return (reference - self.captured_at).days > PRICE_SNAPSHOT_MAX_AGE_DAYS + + def digest(self) -> str: + """Return an order- and representation-independent digest of the RATES. + + Covers the namespace plus, per entry, ``provider_id``, ``model_id``, both + ceiling rates in canonical decimal form, and + ``max_output_bytes_per_token``. It deliberately EXCLUDES ``snapshot_id``, + ``captured_at``, and ``source_url``: this digest joins cache identity, and + re-dating or re-citing a snapshot whose rates are byte-identical must not + invalidate entries whose ceiling would come out exactly the same. + """ + ordered = sorted(self.entries, key=lambda entry: (entry.provider_id, entry.model_id)) + canonical = json.dumps( + { + "namespace": _PRODUCT_PRICE_HASH_NAMESPACE, + "entries": [ + { + "provider_id": entry.provider_id, + "model_id": entry.model_id, + "input_ceiling_usd_per_million_tokens": _canonical_decimal( + entry.input_ceiling_usd_per_million_tokens + ), + "output_ceiling_usd_per_million_tokens": _canonical_decimal( + entry.output_ceiling_usd_per_million_tokens + ), + "max_output_bytes_per_token": str(entry.max_output_bytes_per_token), + } + for entry in ordered + ], + }, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return f"sha256:{hashlib.sha256(canonical).hexdigest()}" diff --git a/tests/test_pricing_snapshot.py b/tests/test_pricing_snapshot.py new file mode 100644 index 0000000..8806625 --- /dev/null +++ b/tests/test_pricing_snapshot.py @@ -0,0 +1,87 @@ +"""The dated product price snapshot: shape, digest, lookup, staleness (DSE-1514).""" + +from __future__ import annotations + +from datetime import date +from decimal import Decimal + +import pytest +from pydantic import ValidationError + +from conclave.pricing import PRICE_SNAPSHOT_MAX_AGE_DAYS, PriceRates, PriceSnapshot + + +def _rates(model_id: str, *, source_url: str | None = "https://example.test/pricing") -> PriceRates: + return PriceRates( + provider_id=model_id.split("/", 1)[0], + model_id=model_id, + input_ceiling_usd_per_million_tokens=Decimal("3.00"), + output_ceiling_usd_per_million_tokens=Decimal("15.00"), + max_output_bytes_per_token=8, + source_url=source_url, + ) + + +def _snapshot(**overrides) -> PriceSnapshot: + payload = { + "snapshot_id": "test-prices-2026-09-03", + "captured_at": date(2026, 9, 3), + "currency": "USD", + "entries": (_rates("anthropic/claude-sonnet-4-6"), _rates("openai/gpt-4.1")), + } + payload.update(overrides) + return PriceSnapshot(**payload) + + +def test_every_entry_must_cite_a_source_url(): + with pytest.raises(ValidationError, match="source_url"): + _snapshot(entries=(_rates("openai/gpt-4.1", source_url=None),)) + + +def test_duplicate_model_ids_are_rejected(): + with pytest.raises(ValidationError, match="unique"): + _snapshot(entries=(_rates("openai/gpt-4.1"), _rates("openai/gpt-4.1"))) + + +def test_rates_for_is_exact_and_never_fuzzy(): + snapshot = _snapshot() + assert snapshot.rates_for("openai/gpt-4.1").model_id == "openai/gpt-4.1" + assert snapshot.rates_for("openai/gpt-4.1-mini") is None + assert snapshot.rates_for("anthropic/claude-opus-4") is None + + +def test_digest_is_order_independent_and_ignores_citations(): + forward = _snapshot() + reversed_entries = _snapshot(entries=tuple(reversed(forward.entries))) + recited = _snapshot( + entries=tuple( + entry.model_copy(update={"source_url": "https://elsewhere.test/prices"}) + for entry in forward.entries + ) + ) + assert forward.digest() == reversed_entries.digest() + assert forward.digest() == recited.digest() + assert forward.digest().startswith("sha256:") + + +def test_digest_changes_when_any_rate_changes(): + forward = _snapshot() + bumped = _snapshot( + entries=( + forward.entries[0].model_copy( + update={"output_ceiling_usd_per_million_tokens": Decimal("15.000001")} + ), + forward.entries[1], + ) + ) + assert forward.digest() != bumped.digest() + + +def test_staleness_is_reported_but_never_changes_a_rate(): + fresh = _snapshot(captured_at=date(2026, 9, 1)) + stale = _snapshot(captured_at=date(2026, 1, 1)) + assert PRICE_SNAPSHOT_MAX_AGE_DAYS == 90 + assert fresh.is_stale(as_of=date(2026, 9, 3)) is False + assert stale.is_stale(as_of=date(2026, 9, 3)) is True + entry = stale.rates_for("openai/gpt-4.1") + assert entry.output_ceiling_usd_per_million_tokens == Decimal("15.00") From 066716327ee799e7bec806fb5293b46af377fda9 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 4 Sep 2026 15:18:21 -0400 Subject: [PATCH 2/6] feat(pricing): dated vendor-cited default price snapshot + packaged loader (DSE-1514) Claude-Session: https://claude.ai/code/session_01K1dHPjZ1bZcE2GnX3KMMSH --- src/conclave/data/__init__.py | 1 + src/conclave/data/prices-2026-09-04.json | 64 ++++++++++++++++++++++++ src/conclave/pricing.py | 48 +++++++++++++++++- tests/test_pricing_snapshot.py | 52 +++++++++++++++++++ 4 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 src/conclave/data/__init__.py create mode 100644 src/conclave/data/prices-2026-09-04.json diff --git a/src/conclave/data/__init__.py b/src/conclave/data/__init__.py new file mode 100644 index 0000000..4c94643 --- /dev/null +++ b/src/conclave/data/__init__.py @@ -0,0 +1 @@ +"""Packaged, dated price snapshots. Data only -- no code, no runtime fetching.""" diff --git a/src/conclave/data/prices-2026-09-04.json b/src/conclave/data/prices-2026-09-04.json new file mode 100644 index 0000000..728c67f --- /dev/null +++ b/src/conclave/data/prices-2026-09-04.json @@ -0,0 +1,64 @@ +{ + "_note": "Hand-verified vendor list prices, rounded UP, read 2026-09-04 from each vendor's own official pricing page (see per-entry source_url). A model absent from this file is UNPRICED, never estimated -- never carried over from a sibling model. Two of conclave.registry.DEFAULT_MODELS are OMITTED for exactly that reason: groq/llama-3.3-70b-versatile was moved off every published on-demand rate to an Enterprise-only 'Contact Sales' tier on 2026-08-16 (console.groq.com/docs/deprecations); deepseek/deepseek-chat was fully retired on 2026-07-24 (api-docs.deepseek.com/updates) with no successor sharing that exact model id -- its replacement id, deepseek-v4-flash, is a DIFFERENT model id conclave does not resolve, so it is not a substitute rate. Every priced entry takes the HIGHEST standard on-demand rate that could apply to a plain, unconfigured chat request: for xai/grok-4.3 and gemini/gemini-2.5-pro that is the >=200K/>200K long-context tier (both step ALL of input+output up together, never just one side); for openai/gpt-4.1 that is the standard tier -- the more expensive 'fast mode' priority tier ($3.50/$14.00) requires an explicit opt-in service-tier parameter, so it is not what a plain request is billed at, the same reasoning that excludes Anthropic's opt-in US-only-inference tier ($3.30/$16.50) for anthropic/claude-sonnet-4-6. Cached-input discounts and batch-API discounts are excluded everywhere as strictly cheaper, never-default lanes. perplexity/sonar-pro also carries a separate per-request search fee (up to $0.014/request at the highest context-size tier) that is NOT a per-token rate and has no field in this schema; it is deliberately not folded into either ceiling here and is called out so a caller relying solely on this snapshot for perplexity/sonar-pro undercounts by that per-request amount. max_output_bytes_per_token is an attestation of 8 bytes/token (2x the eval harness fixture value of 4): it only ever converts an upstream output-token cap into a downstream input BYTE bound, so a larger value is strictly more pessimistic. Regenerate deliberately; never fetch at runtime.", + "snapshot_id": "conclave-default-prices-2026-09-04", + "captured_at": "2026-09-04", + "currency": "USD", + "entries": [ + { + "provider_id": "openai", + "model_id": "openai/gpt-4.1", + "input_ceiling_usd_per_million_tokens": "2.00", + "output_ceiling_usd_per_million_tokens": "8.00", + "max_output_bytes_per_token": 8, + "source_url": "https://openai.com/api/pricing/" + }, + { + "provider_id": "anthropic", + "model_id": "anthropic/claude-sonnet-4-6", + "input_ceiling_usd_per_million_tokens": "3.00", + "output_ceiling_usd_per_million_tokens": "15.00", + "max_output_bytes_per_token": 8, + "source_url": "https://docs.anthropic.com/en/docs/about-claude/pricing" + }, + { + "provider_id": "xai", + "model_id": "xai/grok-4.3", + "input_ceiling_usd_per_million_tokens": "2.50", + "output_ceiling_usd_per_million_tokens": "5.00", + "max_output_bytes_per_token": 8, + "source_url": "https://docs.x.ai/developers/pricing" + }, + { + "provider_id": "gemini", + "model_id": "gemini/gemini-2.5-pro", + "input_ceiling_usd_per_million_tokens": "2.50", + "output_ceiling_usd_per_million_tokens": "15.00", + "max_output_bytes_per_token": 8, + "source_url": "https://ai.google.dev/gemini-api/docs/pricing" + }, + { + "provider_id": "perplexity", + "model_id": "perplexity/sonar-pro", + "input_ceiling_usd_per_million_tokens": "3.00", + "output_ceiling_usd_per_million_tokens": "15.00", + "max_output_bytes_per_token": 8, + "source_url": "https://docs.perplexity.ai/docs/getting-started/pricing" + }, + { + "provider_id": "mistral", + "model_id": "mistral/mistral-large-latest", + "input_ceiling_usd_per_million_tokens": "0.50", + "output_ceiling_usd_per_million_tokens": "1.50", + "max_output_bytes_per_token": 8, + "source_url": "https://mistral.ai/pricing/api/" + }, + { + "provider_id": "together", + "model_id": "together/meta-llama/Llama-3.3-70B-Instruct-Turbo", + "input_ceiling_usd_per_million_tokens": "1.04", + "output_ceiling_usd_per_million_tokens": "1.04", + "max_output_bytes_per_token": 8, + "source_url": "https://www.together.ai/pricing" + } + ] +} diff --git a/src/conclave/pricing.py b/src/conclave/pricing.py index 659af7f..363b11b 100644 --- a/src/conclave/pricing.py +++ b/src/conclave/pricing.py @@ -28,9 +28,15 @@ from dataclasses import dataclass from datetime import date from decimal import ROUND_CEILING, Decimal, localcontext +from functools import lru_cache +from pathlib import Path from typing import Literal -from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator + +from .logging import get_logger + +logger = get_logger("pricing") # The quantum every emitted amount is rounded up to: one USD micro-cent. USD_MICROCENT = Decimal("0.000001") @@ -392,3 +398,43 @@ def digest(self) -> str: sort_keys=True, ).encode("utf-8") return f"sha256:{hashlib.sha256(canonical).hexdigest()}" + + +def _price_data_dir() -> Path: + """Return the packaged directory holding the dated price snapshots.""" + return Path(__file__).parent / "data" + + +@lru_cache(maxsize=1) +def load_default_price_snapshot() -> PriceSnapshot | None: + """Load the newest packaged price snapshot, or ``None`` when unavailable. + + Picks the lexicographically-last ``prices-YYYY-MM-DD.json`` in + ``conclave/data`` (ISO dates sort chronologically), parses it with + ``parse_float=Decimal`` so a JSON number can never become a float rate, and + validates it. Memoized for the life of the process: a snapshot is a frozen + artifact, so re-reading it per run would be pure overhead. + + Returns ``None`` -- never raises -- when the directory is absent, empty, + unreadable, or holds a file that fails validation, and logs a warning. A + missing snapshot means a run carries NO ceiling; it must never mean a failed + run. (The pre-flight spend gate treats the same condition as a refusal + instead; that asymmetry is deliberate.) + """ + try: + candidates = sorted(_price_data_dir().glob("prices-*.json")) + except OSError as exc: + logger.warning("price snapshot directory unreadable: %s; pricing disabled", exc) + return None + if not candidates: + logger.warning("no packaged price snapshot found; pricing disabled") + return None + path = candidates[-1] + try: + with path.open(encoding="utf-8") as handle: + payload = json.load(handle, parse_float=Decimal) + payload.pop("_note", None) + return PriceSnapshot.model_validate(payload) + except (OSError, json.JSONDecodeError, ValidationError, TypeError) as exc: + logger.warning("price snapshot %s is unusable: %s; pricing disabled", path.name, exc) + return None diff --git a/tests/test_pricing_snapshot.py b/tests/test_pricing_snapshot.py index 8806625..ecac031 100644 --- a/tests/test_pricing_snapshot.py +++ b/tests/test_pricing_snapshot.py @@ -85,3 +85,55 @@ def test_staleness_is_reported_but_never_changes_a_rate(): assert stale.is_stale(as_of=date(2026, 9, 3)) is True entry = stale.rates_for("openai/gpt-4.1") assert entry.output_ceiling_usd_per_million_tokens == Decimal("15.00") + + +def test_default_snapshot_loads_and_prices_the_verified_default_models(): + from conclave.pricing import load_default_price_snapshot + from conclave.registry import DEFAULT_MODELS + + snapshot = load_default_price_snapshot() + assert snapshot is not None + assert snapshot.currency == "USD" + assert snapshot.snapshot_id.startswith("conclave-default-prices-") + + priced = {entry.model_id for entry in snapshot.entries} + # Every priced entry must be one of the shipped defaults -- the snapshot is + # not a place to accumulate models the product does not resolve. + assert priced <= set(DEFAULT_MODELS.values()) + # The four frontier defaults must be priced; anything unverifiable is omitted + # deliberately and shows up as an unpriced model at runtime. + assert { + "openai/gpt-4.1", + "anthropic/claude-sonnet-4-6", + "xai/grok-4.3", + "gemini/gemini-2.5-pro", + } <= priced + + for entry in snapshot.entries: + assert isinstance(entry.input_ceiling_usd_per_million_tokens, Decimal) + assert isinstance(entry.output_ceiling_usd_per_million_tokens, Decimal) + assert entry.source_url and entry.source_url.startswith("https://") + assert entry.max_output_bytes_per_token == 8 + assert entry.provider_id == entry.model_id.split("/", 1)[0] + + +def test_default_snapshot_is_memoized_and_ships_inside_the_package(): + from pathlib import Path + + import conclave + from conclave.pricing import load_default_price_snapshot + + assert load_default_price_snapshot() is load_default_price_snapshot() + data_dir = Path(conclave.__file__).parent / "data" + assert sorted(path.name for path in data_dir.glob("prices-*.json")) + + +def test_a_missing_snapshot_directory_degrades_to_none(monkeypatch, tmp_path): + from conclave import pricing + + pricing.load_default_price_snapshot.cache_clear() + monkeypatch.setattr(pricing, "_price_data_dir", lambda: tmp_path) + try: + assert pricing.load_default_price_snapshot() is None + finally: + pricing.load_default_price_snapshot.cache_clear() From 908b3895dc166d996c2857400d92f88adff200fd Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 4 Sep 2026 15:23:24 -0400 Subject: [PATCH 3/6] feat(manifest): additive cost-ceiling fields on receipts and the run manifest (DSE-1514) Claude-Session: https://claude.ai/code/session_01K1dHPjZ1bZcE2GnX3KMMSH --- src/conclave/manifest.py | 84 ++++++++++++++++++++++++++++++-- tests/test_pricing_receipts.py | 88 ++++++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+), 4 deletions(-) create mode 100644 tests/test_pricing_receipts.py diff --git a/src/conclave/manifest.py b/src/conclave/manifest.py index 833e6f4..5f6c852 100644 --- a/src/conclave/manifest.py +++ b/src/conclave/manifest.py @@ -9,9 +9,15 @@ resolved model ids, the generation settings used, per-call execution receipts, total latency, and total token usage; * cost (carefully — Scope Plan §8) — token ``total_usage`` is always present; - ``estimated_cost`` is left ``None`` (a wrong number inside an audit receipt is - worse than none) and ``pricing_snapshot_date`` is the dated-estimate slot a - later pricing table would stamp; + ``estimated_cost`` is left ``None`` FOREVER (a wrong number inside an audit + receipt is worse than none) and ``pricing_snapshot_date`` is the + dated-estimate slot a pricing table would stamp. A DIFFERENT, additive slot + carries a real answer instead (DSE-1514): ``cost_ceiling_usd`` (with its + per-receipt sibling ``cost_ceiling_usd``/``cost_basis``) is a falsifiable + UPPER BOUND priced against a dated ``price_snapshot_digest``/``priced_as_of`` + snapshot, all-or-nothing — any ``unpriced_models``/``unpriced_receipts`` + nulls the run-level ceiling rather than emit a partial sum, with + ``pricing_warnings`` carrying only bounded fixed identifiers; * HOW the verdict was made — ``verdict_extraction`` provenance (which model + prompt version produced the disagreement analysis), ``verdict_type``, ``consensus_method``, and the ``verdict_absent_reason`` (DD-2 ripple). These @@ -36,9 +42,10 @@ from __future__ import annotations +from decimal import Decimal from typing import Literal -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, field_validator from .models import FailureCategory, TokenUsage @@ -140,6 +147,31 @@ class AdjudicationAttempt(BaseModel): http_status: int | None = None +# How a receipt's ``cost_ceiling_usd`` was derived. ``reported_usage`` means the +# provider told us its token counts and they were charged at ceiling rates; +# ``reservation`` means the call produced no usage (it failed, or the provider +# reported none) and the ceiling is the pessimistic pre-call reservation instead. +# There is no third basis: a call with neither usage nor a reservable output cap +# is UNPRICED (``None``/``None``), never estimated. +CostBasis = Literal["reported_usage", "reservation"] + + +def _reject_float_cost_ceiling(value: object) -> object: + """Reject a float/bool ``cost_ceiling_usd`` outright rather than coercing it. + + Plain ``Decimal | None`` fields accept ``float`` in pydantic's default (lax) + mode, silently reproducing the exact drift ``conclave.pricing.PriceRates`` + was built to prevent (``Decimal(0.4)`` is not ``Decimal("0.4")``). Mirrors + ``PriceRates.require_exact_decimal_rate`` byte for byte; ``str`` and + ``Decimal`` still pass through untouched, so the JSON round trip through + ``model_dump(mode="json")`` (which renders a ``Decimal`` as a string) keeps + working. + """ + if isinstance(value, (bool, float)): + raise ValueError("cost_ceiling_usd must be an exact decimal value, not a float") + return value + + class ProviderExecutionReceipt(BaseModel): """A per-call execution record for one council member that was CALLED. @@ -165,6 +197,13 @@ class ProviderExecutionReceipt(BaseModel): estimated_cost: Trustworthy per-call cost when a dated pricing source is available. Conclave currently has no pricing table, so this stays ``None`` rather than inventing a number. + cost_ceiling_usd: A falsifiable UPPER BOUND on this call's cost in USD, + or ``None`` when the call could not be bounded. Exact ``Decimal``, + ROUND_CEILING, priced against the run's dated snapshot. Distinct + from ``estimated_cost`` on purpose: an estimate is a guess and stays + ``None`` forever; a ceiling is a checkable claim. + cost_basis: Which rule produced ``cost_ceiling_usd`` -- see + :data:`CostBasis`. ``None`` exactly when ``cost_ceiling_usd`` is. error: Compatibility field containing only the bounded error category, never raw provider text, URLs, bodies, prompts, or exception chains. error_category: Secret-free bounded failure category. @@ -182,6 +221,8 @@ class ProviderExecutionReceipt(BaseModel): latency_ms: float = 0.0 usage: TokenUsage | None = None estimated_cost: float | None = None + cost_ceiling_usd: Decimal | None = None + cost_basis: CostBasis | None = None error: str | None = None error_category: ReceiptErrorCategory | None = None schema_valid: bool | None = None @@ -189,6 +230,10 @@ class ProviderExecutionReceipt(BaseModel): prompt_version: str | None = None schema_version: str | None = None + _validate_cost_ceiling_usd = field_validator("cost_ceiling_usd", mode="before")( + _reject_float_cost_ceiling + ) + class ModelHarnessManifest(BaseModel): """The auditable execution + provenance receipt for a council run (§3). @@ -238,6 +283,24 @@ class ModelHarnessManifest(BaseModel): candidate tried across every adjudication role in this run, including skipped-unkeyed candidates. Empty for a run that never called :meth:`conclave.council.Council.adjudicate`. + cost_ceiling_usd: The run's total cost CEILING in USD -- the sum of every + receipt's ceiling -- populated only when the whole run is priceable. + All-or-nothing: any unpriced model or unpriced receipt leaves this + ``None`` with ``unpriced_models``/``unpriced_receipts`` naming why. A + partial sum reads exactly like a complete one and is never emitted. + price_snapshot_digest: Rate digest of the snapshot the ceiling was priced + against. Accompanies every non-``None`` ceiling. + priced_as_of: ISO date the snapshot's rates were captured. + unpriced_models: Sorted model ids that ran (or were called) in this run + and have no snapshot entry. Non-empty forces the run ceiling to + ``None``. Drawn from the same vocabulary as ``model_ids``, so it adds + no new secret-scan surface. + unpriced_receipts: How many receipts could not be priced at all. + pricing_warnings: Bounded, fixed identifiers only -- one of + ``price_snapshot_stale``, ``price_snapshot_unavailable``, + ``unpriced_models_present``, ``unpriced_receipts_present``, + ``no_output_cap_configured``. NEVER provider text, never + interpolated, so ``scan_for_secret_material`` stays provable. """ # REQUIRED identity. @@ -262,6 +325,15 @@ class ModelHarnessManifest(BaseModel): estimated_cost: float | None = None pricing_snapshot_date: str | None = None + # Bounded cost ceilings (DSE-1514). Additive; ``estimated_cost`` above is + # untouched and stays ``None`` forever. + cost_ceiling_usd: Decimal | None = None + price_snapshot_digest: str | None = None + priced_as_of: str | None = None + unpriced_models: list[str] = Field(default_factory=list) + unpriced_receipts: int = Field(default=0, ge=0) + pricing_warnings: list[str] = Field(default_factory=list) + # Structured-output validity (CAC-02) + redacted member errors. schema_valid: bool | None = None redacted_errors: list[str] = Field(default_factory=list) @@ -278,6 +350,10 @@ class ModelHarnessManifest(BaseModel): # Synthesizer/judge/verdict-extractor succession ledger (DSE-1512). adjudication_succession: list[AdjudicationAttempt] = Field(default_factory=list) + _validate_cost_ceiling_usd = field_validator("cost_ceiling_usd", mode="before")( + _reject_float_cost_ceiling + ) + def scan_for_secret_material(manifest: ModelHarnessManifest) -> bool: """Return True when the serialized manifest is CLEAN of key material. diff --git a/tests/test_pricing_receipts.py b/tests/test_pricing_receipts.py new file mode 100644 index 0000000..9e8cc87 --- /dev/null +++ b/tests/test_pricing_receipts.py @@ -0,0 +1,88 @@ +"""Ceiling fields on the manifest: additive, Decimal-only, all-or-nothing (DSE-1514).""" + +from __future__ import annotations + +from decimal import Decimal + +import pytest +from pydantic import ValidationError + +from conclave.manifest import ( + ModelHarnessManifest, + ProviderExecutionReceipt, + scan_for_secret_material, +) + + +def _receipt(**overrides) -> ProviderExecutionReceipt: + payload = { + "name": "claude", + "provider": "anthropic", + "model_id": "anthropic/claude-sonnet-4-6", + } + payload.update(overrides) + return ProviderExecutionReceipt(**payload) + + +def test_new_receipt_fields_default_to_none_and_estimated_cost_is_untouched(): + receipt = _receipt() + assert receipt.cost_ceiling_usd is None + assert receipt.cost_basis is None + assert receipt.estimated_cost is None + + +def test_receipt_ceiling_is_decimal_and_rejects_a_float(): + receipt = _receipt(cost_ceiling_usd=Decimal("0.001234"), cost_basis="reported_usage") + assert isinstance(receipt.cost_ceiling_usd, Decimal) + assert receipt.cost_ceiling_usd == Decimal("0.001234") + with pytest.raises(ValidationError): + _receipt(cost_ceiling_usd=0.001234) + + +def test_receipt_cost_basis_is_a_closed_vocabulary(): + assert _receipt(cost_basis="reservation").cost_basis == "reservation" + with pytest.raises(ValidationError): + _receipt(cost_basis="guess") + + +def test_manifest_ceiling_fields_default_empty_and_estimated_cost_stays_none(): + manifest = ModelHarnessManifest(request_id="r", conclave_version="1.3.0", mode="synthesize") + assert manifest.cost_ceiling_usd is None + assert manifest.price_snapshot_digest is None + assert manifest.priced_as_of is None + assert manifest.unpriced_models == [] + assert manifest.unpriced_receipts == 0 + assert manifest.pricing_warnings == [] + assert manifest.estimated_cost is None + assert manifest.pricing_snapshot_date is None + + +def test_populated_pricing_fields_keep_the_secret_scan_clean(): + manifest = ModelHarnessManifest( + request_id="r", + conclave_version="1.3.0", + mode="elite", + model_ids=["anthropic/claude-sonnet-4-6", "deepseek/deepseek-chat"], + receipts=[_receipt(cost_ceiling_usd=Decimal("0.5"), cost_basis="reported_usage")], + cost_ceiling_usd=Decimal("0.5"), + price_snapshot_digest="sha256:" + "b" * 64, + priced_as_of="2026-09-03", + unpriced_models=["deepseek/deepseek-chat"], + unpriced_receipts=0, + pricing_warnings=["unpriced_models_present", "price_snapshot_stale"], + ) + assert scan_for_secret_material(manifest) is True + + +def test_ceilings_round_trip_through_json_as_exact_decimals(): + manifest = ModelHarnessManifest( + request_id="r", + conclave_version="1.3.0", + mode="synthesize", + receipts=[_receipt(cost_ceiling_usd=Decimal("0.000001"), cost_basis="reservation")], + cost_ceiling_usd=Decimal("0.000001"), + ) + restored = ModelHarnessManifest.model_validate(manifest.model_dump(mode="json")) + assert restored.cost_ceiling_usd == Decimal("0.000001") + assert restored.receipts[0].cost_ceiling_usd == Decimal("0.000001") + assert isinstance(restored.cost_ceiling_usd, Decimal) From e12cf007ec415ca546b9e9dba37dc3f910275d1c Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 4 Sep 2026 15:27:47 -0400 Subject: [PATCH 4/6] feat(council): price the manifest last, all-or-nothing, never estimating (DSE-1514) Claude-Session: https://claude.ai/code/session_01K1dHPjZ1bZcE2GnX3KMMSH --- src/conclave/council.py | 144 +++++++++++++++++++++++++++++++++ src/conclave/streaming.py | 6 ++ tests/test_pricing_receipts.py | 141 ++++++++++++++++++++++++++++++++ 3 files changed, 291 insertions(+) diff --git a/src/conclave/council.py b/src/conclave/council.py index 05a1416..7109eea 100644 --- a/src/conclave/council.py +++ b/src/conclave/council.py @@ -54,6 +54,7 @@ import asyncio from collections.abc import AsyncIterator, Awaitable, Callable, Sequence from dataclasses import dataclass +from decimal import Decimal from typing import TYPE_CHECKING from uuid import uuid4 @@ -79,6 +80,12 @@ StreamEvent, TokenUsage, ) +from .pricing import ( + PriceSnapshot, + load_default_price_snapshot, + reported_usage_cost, + reserve_cost, +) from .prompts import ELITE_PROMPT_VERSION, SYNTHESIS_PROMPT_VERSION from .providers import call_model, receipt_from_answer from .registry import key_present @@ -109,6 +116,15 @@ "Do not invent a model's position; rely only on the answers provided." ) +# Fixed allowances used when a FAILED call must be priced from a reservation +# rather than from reported usage. A failed call carries no usage and no +# recorded message list, so its input is bounded by the raw prompt bytes plus +# these two constants: a template allowance covering any system/instruction +# wording the mode wrapped around the prompt, and the same per-request framing +# allowance the eval runner attests (64 + 16 per message, taken at 4 messages). +_PRICING_TEMPLATE_ALLOWANCE = 4096 +_PRICING_FRAMING_ALLOWANCE = 64 + (16 * 4) + # Re-exported for callers that want the version without importing prompts. __all__ = ["Council", "SYNTHESIS_PROMPT_VERSION"] @@ -237,6 +253,10 @@ def __init__( # that DEBUG band and accept the responsibility. if not allow_transport_debug_logging: transport.guard_transport_logging() + # Replaced by the real config/argument resolution in the output-cap task + # (DSE-1514 Task 9). Needed already by `_price_manifest` (Task 7), which + # reads it to decide whether a failed call's reservation can be priced. + self.max_output_tokens: int | None = None @staticmethod def _resolve_chain(spec: str | Sequence[str] | None, config: ConclaveConfig) -> list[str]: @@ -385,6 +405,7 @@ async def _cached_run( if not self.cache_enabled: result = await run() self._ensure_manifest(result, mode) + self._price_manifest(result) return result key = self._cache_key( @@ -399,10 +420,12 @@ async def _cached_run( if hit is not None: logger.info("cache hit for %s run (%s)", mode, key[:12]) self._ensure_manifest(hit, mode) + self._price_manifest(hit) return hit result = await run() self._ensure_manifest(result, mode) + self._price_manifest(result) if result.primary_failed_over: logger.info( "not caching %s run (%s): primary adjudicator failed for an infrastructure reason", @@ -673,6 +696,127 @@ def _recompute_manifest_accounting(manifest: ModelHarnessManifest) -> None: ] manifest.secret_safety = verified_secret_safety(manifest) + def _price_manifest(self, result: CouncilResult) -> None: + """Stamp cost ceilings on the manifest -- the LAST step of a run (DSE-1514). + + Runs after ``_ensure_manifest`` and after every receipt is appended, so + it sees the complete ledger. It is idempotent: it recomputes every field + from the receipts each time, so re-pricing a cache hit is harmless. + + The rule, per receipt: + + * the model has no snapshot entry -> unpriced (``None``/``None``), and + the model id joins ``unpriced_models``; + * the provider reported usage -> ``reported_usage_cost`` at ceiling + rates, basis ``"reported_usage"``; + * no usage (the call failed, or the provider reported none) AND an + output cap is configured -> the call's own pessimistic reservation, + basis ``"reservation"``; + * no usage and no output cap -> unpriced. Nothing is estimated. + + And at run level, ALL-OR-NOTHING: ``cost_ceiling_usd`` is the sum of + every receipt ceiling only when ``unpriced_models`` is empty AND + ``unpriced_receipts`` is zero. A run with no receipts at all (the + memberless path) has a provable ceiling of ``Decimal("0")`` -- no call + was made, so nothing was spent -- deliberately distinguished from + ``None`` ("could not be bounded"). + + Never raises: an unusable snapshot, an unreadable file, or a provider + that reports an impossible token total degrades to "unpriced" with a + bounded warning. A missing ceiling is an acceptable outcome; a failed + council run because of pricing is not. + """ + manifest = result.manifest + if manifest is None: + return + + snapshot: PriceSnapshot | None = load_default_price_snapshot() + if snapshot is None: + for receipt in manifest.receipts: + receipt.cost_ceiling_usd = None + receipt.cost_basis = None + manifest.cost_ceiling_usd = None + manifest.price_snapshot_digest = None + manifest.priced_as_of = None + manifest.unpriced_models = [] + manifest.unpriced_receipts = len(manifest.receipts) + manifest.pricing_warnings = ["price_snapshot_unavailable"] + manifest.secret_safety = verified_secret_safety(manifest) + return + + cap = self.max_output_tokens + unpriced_models: set[str] = { + model_id for model_id in manifest.model_ids if snapshot.rates_for(model_id) is None + } + unpriced_receipts = 0 + for receipt in manifest.receipts: + rates = snapshot.rates_for(receipt.model_id) + if rates is None: + unpriced_models.add(receipt.model_id) + receipt.cost_ceiling_usd = None + receipt.cost_basis = None + unpriced_receipts += 1 + continue + if receipt.usage is not None: + try: + receipt.cost_ceiling_usd = reported_usage_cost( + rates, + prompt_tokens=receipt.usage.prompt_tokens, + completion_tokens=receipt.usage.completion_tokens, + total_tokens=receipt.usage.total_tokens, + ) + except ValueError: + # A provider reported a total below its own attributed usage. + # Bounding it would require inventing the missing tokens. + logger.warning( + "unusable reported usage for %s; leaving the call unpriced", + receipt.model_id, + ) + receipt.cost_ceiling_usd = None + receipt.cost_basis = None + unpriced_receipts += 1 + continue + receipt.cost_basis = "reported_usage" + continue + if cap is None: + receipt.cost_ceiling_usd = None + receipt.cost_basis = None + unpriced_receipts += 1 + continue + receipt.cost_ceiling_usd = reserve_cost( + rates, + prompt_token_upper_bound=len(result.prompt.encode("utf-8")), + prompt_template_token_allowance=_PRICING_TEMPLATE_ALLOWANCE, + provider_framing_token_allowance=_PRICING_FRAMING_ALLOWANCE, + upstream_output_token_ceilings=(), + upstream_output_bytes_per_token=rates.max_output_bytes_per_token, + max_output_tokens=cap, + ).reserved_cost_usd + receipt.cost_basis = "reservation" + + warnings: list[str] = [] + if unpriced_models: + warnings.append("unpriced_models_present") + if unpriced_receipts: + warnings.append("unpriced_receipts_present") + if cap is None: + warnings.append("no_output_cap_configured") + if snapshot.is_stale(): + warnings.append("price_snapshot_stale") + + manifest.price_snapshot_digest = snapshot.digest() + manifest.priced_as_of = snapshot.captured_at.isoformat() + manifest.unpriced_models = sorted(unpriced_models) + manifest.unpriced_receipts = unpriced_receipts + manifest.pricing_warnings = warnings + manifest.cost_ceiling_usd = ( + sum((receipt.cost_ceiling_usd for receipt in manifest.receipts), Decimal("0")) + if not unpriced_models and not unpriced_receipts + else None + ) + # Re-stamp: the ceiling fields were written after _build_manifest's scan. + manifest.secret_safety = verified_secret_safety(manifest) + def _append_manifest_receipts( self, result: CouncilResult, diff --git a/src/conclave/streaming.py b/src/conclave/streaming.py index e28dee3..e068297 100644 --- a/src/conclave/streaming.py +++ b/src/conclave/streaming.py @@ -194,6 +194,9 @@ async def stream_ask( result.manifest = council._build_manifest( mode=result.mode, members=[], skipped=skipped, answers=[] ) + # Pricing is the LAST step: it must see every receipt, including the verdict + # extraction receipts _apply_verdict just appended. Mirrors Council._cached_run. + council._price_manifest(result) yield StreamEvent(type="done", result=result) return @@ -267,6 +270,9 @@ async def stream_ask( # receipt capture is currently the buffered/Elite contract. await council._apply_verdict(result, record_receipts=False) + # Pricing is the LAST step: it must see every receipt, including the verdict + # extraction receipts _apply_verdict just appended. Mirrors Council._cached_run. + council._price_manifest(result) yield StreamEvent(type="done", result=result) diff --git a/tests/test_pricing_receipts.py b/tests/test_pricing_receipts.py index 9e8cc87..94d5c09 100644 --- a/tests/test_pricing_receipts.py +++ b/tests/test_pricing_receipts.py @@ -2,16 +2,20 @@ from __future__ import annotations +from datetime import date from decimal import Decimal import pytest from pydantic import ValidationError +from conclave.council import Council from conclave.manifest import ( ModelHarnessManifest, ProviderExecutionReceipt, scan_for_secret_material, ) +from conclave.pricing import PriceRates, PriceSnapshot +from tests.conftest import make_response def _receipt(**overrides) -> ProviderExecutionReceipt: @@ -86,3 +90,140 @@ def test_ceilings_round_trip_through_json_as_exact_decimals(): assert restored.cost_ceiling_usd == Decimal("0.000001") assert restored.receipts[0].cost_ceiling_usd == Decimal("0.000001") assert isinstance(restored.cost_ceiling_usd, Decimal) + + +def _snapshot(*model_ids: str, captured_at: date = date(2026, 9, 3)) -> PriceSnapshot: + return PriceSnapshot( + snapshot_id="test-prices", + captured_at=captured_at, + currency="USD", + entries=tuple( + PriceRates( + provider_id=model_id.split("/", 1)[0], + model_id=model_id, + input_ceiling_usd_per_million_tokens=Decimal("3.00"), + output_ceiling_usd_per_million_tokens=Decimal("15.00"), + max_output_bytes_per_token=8, + source_url="https://example.test/pricing", + ) + for model_id in model_ids + ), + ) + + +def _install_snapshot(monkeypatch, snapshot): + import conclave.council as council_mod + + monkeypatch.setattr(council_mod, "load_default_price_snapshot", lambda: snapshot) + + +async def test_a_fully_priced_run_carries_a_ceiling_digest_and_date( + monkeypatch, patch_call_model, keys +): + _install_snapshot(monkeypatch, _snapshot("xai/grok-4.3", "anthropic/claude-sonnet-4-6")) + patch_call_model(lambda model_id, messages: make_response("ok")) + council = Council(models=["grok"], synthesizer="claude", extract_verdict=False) + result = await council.ask("q") + + manifest = result.manifest + assert manifest.unpriced_models == [] + assert manifest.unpriced_receipts == 0 + assert isinstance(manifest.cost_ceiling_usd, Decimal) + assert manifest.cost_ceiling_usd == sum( + (receipt.cost_ceiling_usd for receipt in manifest.receipts), Decimal("0") + ) + assert manifest.price_snapshot_digest.startswith("sha256:") + assert manifest.priced_as_of == "2026-09-03" + assert all(receipt.cost_basis == "reported_usage" for receipt in manifest.receipts) + # The estimate slot is untouched, forever. + assert manifest.estimated_cost is None + assert all(receipt.estimated_cost is None for receipt in manifest.receipts) + + +async def test_one_unpriced_model_nulls_the_whole_run_ceiling(monkeypatch, patch_call_model, keys): + # grok is priced, claude is NOT -> all-or-nothing. + _install_snapshot(monkeypatch, _snapshot("xai/grok-4.3")) + patch_call_model(lambda model_id, messages: make_response("ok")) + council = Council(models=["grok"], synthesizer="claude", extract_verdict=False) + result = await council.ask("q") + + manifest = result.manifest + assert manifest.cost_ceiling_usd is None + assert manifest.unpriced_models == ["anthropic/claude-sonnet-4-6"] + assert manifest.unpriced_receipts == 1 + assert "unpriced_models_present" in manifest.pricing_warnings + # The PRICED receipts still carry their own ceilings -- only the SUM is withheld. + priced = [r for r in manifest.receipts if r.model_id == "xai/grok-4.3"] + assert priced and all(r.cost_ceiling_usd is not None for r in priced) + + +async def test_a_stale_snapshot_warns_but_never_changes_a_rate(monkeypatch, patch_call_model, keys): + patch_call_model(lambda model_id, messages: make_response("ok")) + council = Council(models=["grok"], synthesizer="grok", extract_verdict=False) + + _install_snapshot(monkeypatch, _snapshot("xai/grok-4.3", captured_at=date(2026, 9, 3))) + fresh = (await council.ask("q", synthesize=False)).manifest + + _install_snapshot(monkeypatch, _snapshot("xai/grok-4.3", captured_at=date(2026, 1, 1))) + stale = (await council.ask("q", synthesize=False)).manifest + + assert fresh.pricing_warnings == [] + assert "price_snapshot_stale" in stale.pricing_warnings + assert stale.cost_ceiling_usd == fresh.cost_ceiling_usd + + +async def test_a_missing_snapshot_leaves_no_ceiling_and_never_raises( + monkeypatch, patch_call_model, keys +): + _install_snapshot(monkeypatch, None) + patch_call_model(lambda model_id, messages: make_response("ok")) + council = Council(models=["grok"], synthesizer="grok", extract_verdict=False) + manifest = (await council.ask("q", synthesize=False)).manifest + + assert manifest.cost_ceiling_usd is None + assert manifest.price_snapshot_digest is None + assert manifest.pricing_warnings == ["price_snapshot_unavailable"] + assert manifest.secret_safety == "verified_no_secrets" + + +async def test_a_failed_call_with_no_usage_is_unpriced_without_an_output_cap( + monkeypatch, patch_call_model, keys +): + _install_snapshot(monkeypatch, _snapshot("xai/grok-4.3")) + + def handler(model_id, messages): + raise RuntimeError("boom") + + patch_call_model(handler) + council = Council(models=["grok"], synthesizer="grok", extract_verdict=False) + manifest = (await council.ask("q", synthesize=False)).manifest + + assert manifest.receipts[0].cost_ceiling_usd is None + assert manifest.receipts[0].cost_basis is None + assert manifest.unpriced_receipts == 1 + assert manifest.cost_ceiling_usd is None + assert "unpriced_receipts_present" in manifest.pricing_warnings + assert "no_output_cap_configured" in manifest.pricing_warnings + + +async def test_an_impossible_usage_total_degrades_to_unpriced(monkeypatch, keys): + import conclave.council as council_mod + from conclave.models import ModelAnswer, TokenUsage + + _install_snapshot(monkeypatch, _snapshot("xai/grok-4.3")) + + async def bad_usage(name, model_id, messages, **kwargs): + return ModelAnswer( + name=name, + model_id=model_id, + answer="ok", + usage=TokenUsage(prompt_tokens=10, completion_tokens=10, total_tokens=5), + ) + + monkeypatch.setattr(council_mod, "call_model", bad_usage) + council = Council(models=["grok"], synthesizer="grok", extract_verdict=False) + manifest = (await council.ask("q", synthesize=False)).manifest + + assert manifest.receipts[0].cost_ceiling_usd is None + assert manifest.cost_ceiling_usd is None + assert manifest.unpriced_receipts == 1 From 0b06ad29de2d44c0dc44af4aeea841017bf24b5b Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 4 Sep 2026 15:31:52 -0400 Subject: [PATCH 5/6] feat(cache): price snapshot fingerprint + output cap in identity, format v5 (DSE-1514) Claude-Session: https://claude.ai/code/session_01K1dHPjZ1bZcE2GnX3KMMSH --- src/conclave/cache.py | 36 ++++++++++++++++- src/conclave/council.py | 3 ++ tests/test_cache.py | 87 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 123 insertions(+), 3 deletions(-) diff --git a/src/conclave/cache.py b/src/conclave/cache.py index ebcf106..159d0cc 100644 --- a/src/conclave/cache.py +++ b/src/conclave/cache.py @@ -61,7 +61,10 @@ # old entries simply miss instead of being mis-served against new code. # v4 (DSE-1512): identity now carries the full ordered synthesizer/judge chain, # not just the primary candidate. -CACHE_FORMAT_VERSION = "4" +# v5 (DSE-1514): identity now carries the price-snapshot rate fingerprint and the +# max_output_tokens cap, so a re-priced or differently-capped run can never be +# served a stale ceiling (or a longer/shorter answer) from a prior entry. +CACHE_FORMAT_VERSION = "5" _SECRET_QUERY_PARTS = ( "authorization", "auth", @@ -167,6 +170,8 @@ def build_identity( endpoint_urls: Mapping[str, str] | None = None, source_bundle_digest: str | None = None, synthesizer_chain: Sequence[tuple[str, str]] | None = None, + price_snapshot_digest: str | None = None, + max_output_tokens: int | None = None, cache_format_version: str = CACHE_FORMAT_VERSION, protocol_version: str = ELITE_PROTOCOL_VERSION, synthesis_prompt_version: str = SYNTHESIS_PROMPT_VERSION, @@ -186,6 +191,13 @@ def build_identity( chain of one built from ``synthesizer``/``synthesizer_model_id``, so a direct caller that never passes it still gets a stable, meaningful value and existing single-candidate callers are unaffected. + + ``price_snapshot_digest`` (DSE-1514) is the rate digest of the price + snapshot a run's ceilings are computed against; two runs priced under + different rates must not collide, because a hit would serve a ceiling that + was never true of those rates. ``max_output_tokens`` (DSE-1514) is the hard + output cap; it changes the answers themselves, so it is part of generation + identity. """ chain = ( list(synthesizer_chain) @@ -209,7 +221,11 @@ def build_identity( # The full ordered failover ladder (DSE-1512); a chain of one is # byte-for-byte equivalent to the legacy "synthesizer" pair above. "synthesizer_chain": [[name, model_id] for name, model_id in chain], - "generation": {"temperature": temperature, "timeout": timeout}, + "generation": { + "temperature": temperature, + "timeout": timeout, + "max_output_tokens": max_output_tokens, + }, "extract_verdict": extract_verdict, "endpoint_fingerprints": { prefix: _endpoint_fingerprint(url) @@ -220,6 +236,12 @@ def build_identity( "source_bundle_fingerprint": ( _digest(source_bundle_digest) if source_bundle_digest is not None else None ), + # Re-hash the snapshot digest for the same reason the source bundle + # digest is re-hashed: a malformed caller value must never appear in an + # inspectable identity document, while still invalidating prior entries. + "price_snapshot_fingerprint": ( + _digest(price_snapshot_digest) if price_snapshot_digest is not None else None + ), "mode_params": {}, } mode_params = payload["mode_params"] @@ -251,6 +273,8 @@ def make_key( endpoint_urls: Mapping[str, str] | None = None, source_bundle_digest: str | None = None, synthesizer_chain: Sequence[tuple[str, str]] | None = None, + price_snapshot_digest: str | None = None, + max_output_tokens: int | None = None, cache_format_version: str = CACHE_FORMAT_VERSION, protocol_version: str = ELITE_PROTOCOL_VERSION, synthesis_prompt_version: str = SYNTHESIS_PROMPT_VERSION, @@ -292,6 +316,12 @@ def make_key( collide, since a later run over the same prompt could fail over differently. Defaults to a chain of one built from ``synthesizer``/``synthesizer_model_id`` when omitted. + price_snapshot_digest: The rate digest of the price snapshot a run's + ceilings are computed against (DSE-1514); two runs priced under + different rates must not collide, because a hit would serve a + ceiling that was never true of those rates. + max_output_tokens: The hard output cap; it changes the answers + themselves, so it is part of generation identity. Returns: A 64-char lowercase hex SHA-256 digest. Contains zero key material. @@ -312,6 +342,8 @@ def make_key( endpoint_urls=endpoint_urls, source_bundle_digest=source_bundle_digest, synthesizer_chain=synthesizer_chain, + price_snapshot_digest=price_snapshot_digest, + max_output_tokens=max_output_tokens, cache_format_version=cache_format_version, protocol_version=protocol_version, synthesis_prompt_version=synthesis_prompt_version, diff --git a/src/conclave/council.py b/src/conclave/council.py index 7109eea..50c1431 100644 --- a/src/conclave/council.py +++ b/src/conclave/council.py @@ -331,6 +331,7 @@ def _cache_key( for _name, model_id in [*members, *chain_pairs] if "/" in model_id } + snapshot = load_default_price_snapshot() return cache_mod.make_key( prompt=prompt, mode=mode, @@ -351,6 +352,8 @@ def _cache_key( if prefix in used_prefixes }, source_bundle_digest=self.source_bundle_digest, + price_snapshot_digest=None if snapshot is None else snapshot.digest(), + max_output_tokens=self.max_output_tokens, protocol_version=ELITE_PROTOCOL_VERSION, synthesis_prompt_version=SYNTHESIS_PROMPT_VERSION, elite_prompt_version=ELITE_PROMPT_VERSION, diff --git a/tests/test_cache.py b/tests/test_cache.py index 082f553..af8412c 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -724,7 +724,12 @@ def test_identity_chain_defaults_to_primary_when_omitted(): def test_cache_format_version_bumped(): - assert cache_mod.CACHE_FORMAT_VERSION == "4" + # Tracks the CURRENT format version, not a frozen historical one -- DSE-1514 + # bumped this from "4" (DSE-1512's synthesizer-chain identity) to "5" (the + # price-snapshot fingerprint + max_output_tokens cap); see + # test_cache_format_version_is_five_for_price_identity for the DSE-1514-named + # assertion of the same fact. + assert cache_mod.CACHE_FORMAT_VERSION == "5" async def test_result_adjudicated_by_successor_is_not_stored(monkeypatch, keys, cache_home): @@ -1088,3 +1093,83 @@ async def test_stream_primary_run_is_stored(monkeypatch, keys, cache_home): r2 = await c.ask("q") assert r2.cached is True + + +def test_cache_format_version_is_five_for_price_identity(): + from conclave import cache as cache_mod + + assert cache_mod.CACHE_FORMAT_VERSION == "5" + + +def test_identity_carries_the_price_snapshot_fingerprint_and_output_cap(): + from conclave.cache import build_identity + + base = { + "prompt": "q", + "mode": "synthesize", + "members": [("grok", "xai/grok-4.3")], + "synthesizer": "claude", + "synthesizer_model_id": "anthropic/claude-sonnet-4-6", + "temperature": 0.7, + } + plain = build_identity(**base) + assert plain["price_snapshot_fingerprint"] is None + assert plain["generation"]["max_output_tokens"] is None + + priced = build_identity(**base, price_snapshot_digest="sha256:" + "c" * 64) + other = build_identity(**base, price_snapshot_digest="sha256:" + "d" * 64) + assert priced["price_snapshot_fingerprint"] != plain["price_snapshot_fingerprint"] + assert priced["price_snapshot_fingerprint"] != other["price_snapshot_fingerprint"] + # The raw digest never enters the inspectable identity document -- it is + # re-hashed, matching how source_bundle_digest is handled. + assert "c" * 64 not in str(priced) + + capped = build_identity(**base, max_output_tokens=512) + assert capped["generation"]["max_output_tokens"] == 512 + + +def test_two_snapshots_never_share_a_cache_key(): + from conclave.cache import make_key + + base = { + "prompt": "q", + "mode": "synthesize", + "members": [("grok", "xai/grok-4.3")], + "synthesizer": "claude", + "synthesizer_model_id": "anthropic/claude-sonnet-4-6", + "temperature": 0.7, + } + assert make_key(**base, price_snapshot_digest="sha256:" + "c" * 64) != make_key( + **base, price_snapshot_digest="sha256:" + "d" * 64 + ) + assert make_key(**base, max_output_tokens=512) != make_key(**base, max_output_tokens=1024) + + +async def test_a_cached_result_round_trips_its_decimal_ceiling(tmp_path, monkeypatch, keys): + """A cache hit must return an exact Decimal ceiling, not a float or a string.""" + from decimal import Decimal + + import conclave.council as council_mod + from conclave.council import Council + from conclave.models import ModelAnswer, TokenUsage + from tests.test_pricing_receipts import _install_snapshot, _snapshot + + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path)) + _install_snapshot(monkeypatch, _snapshot("xai/grok-4.3")) + + async def ok(name, model_id, messages, **kwargs): + return ModelAnswer( + name=name, + model_id=model_id, + answer="ok", + usage=TokenUsage(prompt_tokens=10, completion_tokens=20, total_tokens=30), + ) + + monkeypatch.setattr(council_mod, "call_model", ok) + council = Council(models=["grok"], synthesizer="grok", cache=True, extract_verdict=False) + live = await council.ask("q", synthesize=False) + hit = await council.ask("q", synthesize=False) + + assert hit.cached is True + assert isinstance(hit.manifest.cost_ceiling_usd, Decimal) + assert hit.manifest.cost_ceiling_usd == live.manifest.cost_ceiling_usd From d4b43e3a95e5abe0cd433c810d647f550be76f3f Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 4 Sep 2026 18:07:07 -0400 Subject: [PATCH 6/6] fix(council): usage-less receipts are unpriced, not under-reserved; debate receipts cover every round; zero usage is not a cost (DSE-1514 QA C1/C2/I1) QA C1 (Round 3 half): _price_manifest priced a usage-less receipt via a flat reservation (prompt_template_token_allowance=4096, upstream_output_token_ceilings=()) that ignores which phase the call belongs to, so synthesis/judge/verdict/elite-revision receipts (which embed upstream output) came out 3-9x low while unpriced_receipts stayed 0 and the run ceiling looked complete. usage is None on a SUCCESSFUL call too (openai_compat.py returns None when a provider omits usage, including some streams). This round has no phase-aware call plan to reserve from, so the honest rule is: a receipt without reported usage is unpriced (cost_ceiling_usd=None, cost_basis=None, counted in unpriced_receipts, "unpriced_receipts_present" warning), and the run ceiling is None (all-or-nothing). Removed the dead reservation branch and its two flat constants from _price_manifest; a phase-aware reservation returns in Round 4. QA I1: an all-zero reported TokenUsage (usage: {} coerced to TokenUsage(0, 0, 0)) is now treated as "no usage reported" via a new _usage_is_reported() helper, never priced as $0.000000 -- that would be a false floor, not a bound. QA C2 (Critical): debate's manifest receipts were built from result.answers (the final round only), so rounds 1..R-1 and any dropped-out member had no receipt and the run ceiling covered a strict subset of the calls actually made (4 of 7 at N=3/R=2) while reporting complete. Added Council._build_debate_manifest, which builds one receipt per answer per round, phase-stamped "round-{n}" (matching _plan_table's existing debate-round phase naming so a Round 4 phase-aware reservation can map a receipt onto its plan row by plain equality), in round order, before the debate_final consolidation receipt is appended. providers_called/model_ids stay full resolved membership, unchanged. Claude-Session: https://claude.ai/code/session_01K1dHPjZ1bZcE2GnX3KMMSH --- src/conclave/council.py | 175 +++++++++++++++++++++++++------ tests/test_manifest_all_modes.py | 18 ++++ tests/test_pricing_receipts.py | 85 +++++++++++++++ 3 files changed, 244 insertions(+), 34 deletions(-) diff --git a/src/conclave/council.py b/src/conclave/council.py index 50c1431..0377414 100644 --- a/src/conclave/council.py +++ b/src/conclave/council.py @@ -84,7 +84,6 @@ PriceSnapshot, load_default_price_snapshot, reported_usage_cost, - reserve_cost, ) from .prompts import ELITE_PROMPT_VERSION, SYNTHESIS_PROMPT_VERSION from .providers import call_model, receipt_from_answer @@ -116,19 +115,37 @@ "Do not invent a model's position; rely only on the answers provided." ) -# Fixed allowances used when a FAILED call must be priced from a reservation -# rather than from reported usage. A failed call carries no usage and no -# recorded message list, so its input is bounded by the raw prompt bytes plus -# these two constants: a template allowance covering any system/instruction -# wording the mode wrapped around the prompt, and the same per-request framing -# allowance the eval runner attests (64 + 16 per message, taken at 4 messages). -_PRICING_TEMPLATE_ALLOWANCE = 4096 -_PRICING_FRAMING_ALLOWANCE = 64 + (16 * 4) - # Re-exported for callers that want the version without importing prompts. __all__ = ["Council", "SYNTHESIS_PROMPT_VERSION"] +def _usage_is_reported(usage: TokenUsage | None) -> bool: + """Whether ``usage`` is a trustworthy, non-zero signal worth pricing (DSE-1514 QA I1). + + ``usage is None`` is not proof a call cost nothing: it is the shape of both + a FAILED call (no usage was ever produced) and a SUCCESSFUL one whose + provider simply omitted the usage field -- + :func:`conclave.adapters.openai_compat`'s chat-completion path returns + ``None`` when a provider does this, including for some streamed responses. + And a provider can report a technically-present ``TokenUsage`` that is all + zeros, which is the same "nothing to price" signal wearing a non-``None`` + shape. Treating either as ``$0.000000`` would assert a false floor rather + than an honest bound, so :meth:`Council._price_manifest` calls this helper + to decide "does this receipt carry a number worth pricing" before ever + looking at the ceiling math. + + Args: + usage: The receipt's :class:`~conclave.models.TokenUsage`, or ``None``. + + Returns: + ``True`` only when ``usage`` is present AND at least one of its three + counters is non-zero. + """ + return usage is not None and bool( + usage.prompt_tokens or usage.completion_tokens or usage.total_tokens + ) + + @dataclass class AdjudicationOutcome: """Return value of :meth:`Council.adjudicate`. @@ -462,9 +479,11 @@ def _ensure_manifest(self, result: CouncilResult, mode: str) -> None: resolution :meth:`_cache_key` already performs) rather than threaded back through every mode's return value; this keeps ``providers_called`` / ``model_ids`` reflecting the full resolved membership even for debate rounds - where a member later dropped out, while the per-answer receipts are built - from ``result.answers``. :meth:`_build_manifest` stamps ``secret_safety`` - VERIFIED when the assembled manifest is provably clean. + where a member later dropped out. For ``debate`` with ``result.rounds`` + populated the per-answer receipts are built from EVERY round (see + :meth:`_build_debate_manifest`, DSE-1514 QA C2); every other mode builds + them from ``result.answers`` alone. :meth:`_build_manifest` stamps + ``secret_safety`` VERIFIED when the assembled manifest is provably clean. Args: result: The result to attach a manifest to. Mutated in place. @@ -479,6 +498,12 @@ def _ensure_manifest(self, result: CouncilResult, mode: str) -> None: skipped=skipped, result=result, ) + elif mode == "debate" and result.rounds: + result.manifest = self._build_debate_manifest( + members=members, + skipped=skipped, + result=result, + ) else: result.manifest = self._build_manifest( mode=mode, members=members, skipped=skipped, answers=result.answers @@ -616,6 +641,82 @@ def _build_manifest( self._recompute_manifest_accounting(manifest) return manifest + def _build_debate_manifest( + self, + *, + members: list[tuple[str, str]], + skipped: list[str], + result: CouncilResult, + ) -> ModelHarnessManifest: + """Assemble a debate manifest with one receipt per answer per ROUND (DSE-1514 QA C2). + + :meth:`_build_manifest` builds receipts only from ``result.answers``, + which :func:`conclave.modes.run_debate` mirrors from the FINAL round + only. For a multi-round debate that silently under-counts every call: + rounds ``1..R-1`` and any member that dropped out mid-debate never get + a receipt, so the manifest -- and therefore + :meth:`_price_manifest`'s run-level cost ceiling, which sums exactly + the receipts present -- covers a strict subset of the calls a real + debate makes (4 of 7 at 3 members / 2 rounds, 4 of 10 at 3 members / 3 + rounds) while still reporting a complete, non-``None`` ceiling. + + This builds one receipt per answer for EVERY round in + ``result.rounds``, in round order, phase-stamped + ``f"round-{round_number}"`` -- the exact same ``"round-N"`` string + :meth:`_plan_table` gives its debate-round plan rows (DSE-1514 Round + 4), so a usage-less receipt's phase maps onto its plan row by plain + equality, with no pattern-parsing needed. The ``debate_final`` + adjudication receipt is appended AFTERWARDS by + :func:`conclave.modes._debate_synthesize` via + :meth:`_adjudicate_and_record`, so it is deliberately not built here + -- :func:`conclave.modes.run_debate` calls this (via + :meth:`_ensure_manifest`) BEFORE that consolidation step runs. + + ``providers_called``/``model_ids`` still reflect the full resolved + membership (unchanged from :meth:`_build_manifest`), not just the + per-round callees, so a member that drops out mid-debate stays + visible as "called" even though a later round has no receipt for it. + + Args: + members: ``(friendly_name, model_id)`` pairs resolvable (keyed) + for this run. + skipped: Friendly names skipped for a missing key. + result: The in-flight debate result. ``result.rounds`` is already + fully populated -- :func:`conclave.modes.run_debate` calls + this only after its round loop completes. + + Returns: + A fully-assembled, secret-safety-stamped manifest whose receipts + cover every round's calls, in round order. + """ + from . import __version__ + + receipts = [ + receipt_from_answer( + answer, + temperature=self.temperature, + timeout=self.timeout, + phase=f"round-{debate_round.round_number}", + ) + for debate_round in result.rounds + for answer in debate_round.answers + ] + manifest = ModelHarnessManifest( + request_id=uuid4().hex, + conclave_version=__version__, + mode="debate", + providers_considered=list(self.requested_models), + providers_called=[name for name, _model_id in members], + providers_skipped=[ + ProviderSkip(name=name, reason="no API key in environment") for name in skipped + ], + model_ids=[model_id for _name, model_id in members], + generation_settings={"temperature": self.temperature, "timeout": self.timeout}, + receipts=receipts, + ) + self._recompute_manifest_accounting(manifest) + return manifest + def _build_elite_manifest( self, *, @@ -710,12 +811,28 @@ def _price_manifest(self, result: CouncilResult) -> None: * the model has no snapshot entry -> unpriced (``None``/``None``), and the model id joins ``unpriced_models``; - * the provider reported usage -> ``reported_usage_cost`` at ceiling + * the provider reported a trustworthy, non-zero usage figure + (:func:`_usage_is_reported`) -> ``reported_usage_cost`` at ceiling rates, basis ``"reported_usage"``; - * no usage (the call failed, or the provider reported none) AND an - output cap is configured -> the call's own pessimistic reservation, - basis ``"reservation"``; - * no usage and no output cap -> unpriced. Nothing is estimated. + * usage is not reported -> unpriced (``None``/``None``), counted in + ``unpriced_receipts``. This covers three shapes deliberately treated + alike: the call FAILED (no usage was ever produced); the call + SUCCEEDED but the provider omitted the usage field + (:mod:`conclave.adapters.openai_compat` returns ``usage=None`` in + that case, including for some streamed responses -- this is a normal + outcome on a clean call, not a failure signal); or the provider + reported an all-zero ``TokenUsage`` (QA I1 -- a technically-present + but empty usage is the same "nothing to price" shape as ``None``, + and pricing it at ``$0.000000`` would assert a false floor rather + than an honest bound). A receipt without reported usage is NEVER + estimated from a flat reservation in this round: the input a + synthesis/judge/verdict call embeds is bounded by which PHASE it + belongs to (member calls carry only the prompt; synthesis/judge/ + verdict calls additionally embed one or more upstream answers), and + this round has no phase-aware call plan to reserve from -- pricing a + synthesis receipt from a flat member-sized allowance silently + under-counts it. See DSE-1514 Round 4 for the phase-aware + reservation that replaces this blanket "unpriced" rule. And at run level, ALL-OR-NOTHING: ``cost_ceiling_usd`` is the sum of every receipt ceiling only when ``unpriced_models`` is empty AND @@ -760,7 +877,7 @@ def _price_manifest(self, result: CouncilResult) -> None: receipt.cost_basis = None unpriced_receipts += 1 continue - if receipt.usage is not None: + if _usage_is_reported(receipt.usage): try: receipt.cost_ceiling_usd = reported_usage_cost( rates, @@ -781,21 +898,11 @@ def _price_manifest(self, result: CouncilResult) -> None: continue receipt.cost_basis = "reported_usage" continue - if cap is None: - receipt.cost_ceiling_usd = None - receipt.cost_basis = None - unpriced_receipts += 1 - continue - receipt.cost_ceiling_usd = reserve_cost( - rates, - prompt_token_upper_bound=len(result.prompt.encode("utf-8")), - prompt_template_token_allowance=_PRICING_TEMPLATE_ALLOWANCE, - provider_framing_token_allowance=_PRICING_FRAMING_ALLOWANCE, - upstream_output_token_ceilings=(), - upstream_output_bytes_per_token=rates.max_output_bytes_per_token, - max_output_tokens=cap, - ).reserved_cost_usd - receipt.cost_basis = "reservation" + # No trustworthy usage: failed, successful-but-unreported, or + # all-zero. Unpriced -- see the docstring; never estimated. + receipt.cost_ceiling_usd = None + receipt.cost_basis = None + unpriced_receipts += 1 warnings: list[str] = [] if unpriced_models: diff --git a/tests/test_manifest_all_modes.py b/tests/test_manifest_all_modes.py index 23b567d..262a352 100644 --- a/tests/test_manifest_all_modes.py +++ b/tests/test_manifest_all_modes.py @@ -409,6 +409,12 @@ def handler(model, messages, **kwargs): # Full resolved membership is recorded even though only survivors answer. assert result.manifest.providers_considered == ["grok", "gemini", "perplexity"] assert result.manifest.providers_called == ["grok", "gemini", "perplexity"] + # DSE-1514 QA C2 (old: 3 receipts, final round only): every round gets a + # receipt per member, in round order, THEN the debate_final consolidation. + assert [r.phase for r in result.manifest.receipts] == ( + ["round-1"] * 3 + ["round-2"] * 3 + ["debate_final"] + ) + assert len(result.manifest.receipts) == 7 # 3 + 3 + 1, matching every real call made async def test_debate_dropped_member_still_in_manifest_membership(monkeypatch, patch_call_model): @@ -432,6 +438,18 @@ def handler(model, messages, **kwargs): _assert_verified_manifest(result, "debate") # Membership reflects everyone that was called, not just final-round survivors. assert set(result.manifest.providers_called) == {"grok", "gemini", "perplexity"} + # DSE-1514 QA C2 (old: 2 receipts, final round only): round 1 called all 3 + # members (gemini fails and drops out); round 2 called only the 2 + # survivors; debate_final consolidates -- 3 + 2 + 1 = 6 receipts total, + # matching every real council-seam call this run made. + assert [r.phase for r in result.manifest.receipts] == ( + ["round-1"] * 3 + ["round-2"] * 2 + ["debate_final"] + ) + assert len(result.manifest.receipts) == 6 + round_1_names = {r.name for r in result.manifest.receipts if r.phase == "round-1"} + round_2_names = {r.name for r in result.manifest.receipts if r.phase == "round-2"} + assert round_1_names == {"grok", "gemini", "perplexity"} + assert round_2_names == {"grok", "perplexity"} # gemini dropped out after round 1 async def test_debate_no_members_still_carries_manifest(monkeypatch, patch_call_model, clear_keys): diff --git a/tests/test_pricing_receipts.py b/tests/test_pricing_receipts.py index 94d5c09..9763faf 100644 --- a/tests/test_pricing_receipts.py +++ b/tests/test_pricing_receipts.py @@ -18,6 +18,20 @@ from tests.conftest import make_response +def test_usage_is_reported_rejects_none_and_all_zero_accepts_any_nonzero_counter(): + """DSE-1514 QA I1: the shared "is this usage worth pricing" predicate.""" + from conclave.council import _usage_is_reported + from conclave.models import TokenUsage + + assert _usage_is_reported(None) is False + assert _usage_is_reported(TokenUsage(prompt_tokens=0, completion_tokens=0, total_tokens=0)) is ( + False + ) + assert _usage_is_reported(TokenUsage(prompt_tokens=1, completion_tokens=0, total_tokens=0)) + assert _usage_is_reported(TokenUsage(prompt_tokens=0, completion_tokens=1, total_tokens=0)) + assert _usage_is_reported(TokenUsage(prompt_tokens=0, completion_tokens=0, total_tokens=1)) + + def _receipt(**overrides) -> ProviderExecutionReceipt: payload = { "name": "claude", @@ -227,3 +241,74 @@ async def bad_usage(name, model_id, messages, **kwargs): assert manifest.receipts[0].cost_ceiling_usd is None assert manifest.cost_ceiling_usd is None assert manifest.unpriced_receipts == 1 + + +async def test_a_successful_call_with_no_reported_usage_is_unpriced_not_a_reservation( + monkeypatch, keys +): + """QA C1 (Round 3 half): usage=None on a SUCCESSFUL call is unpriced, never estimated. + + :mod:`conclave.adapters.openai_compat` returns ``usage=None`` when a + provider omits the usage field on an otherwise-clean response -- this is a + normal outcome on a clean call, not a failure. Pricing it via a flat + reservation that ignores which phase the call belongs to would silently + mis-price synthesis/judge/verdict calls (which embed upstream output) by + 3-9x. This round has no phase-aware plan to reserve from, so the honest + rule is: no reported usage -> unpriced, exactly like a failed call. + """ + import conclave.council as council_mod + from conclave.models import ModelAnswer + + _install_snapshot(monkeypatch, _snapshot("xai/grok-4.3")) + + async def no_usage(name, model_id, messages, **kwargs): + return ModelAnswer(name=name, model_id=model_id, answer="ok", usage=None) + + monkeypatch.setattr(council_mod, "call_model", no_usage) + council = Council(models=["grok"], synthesizer="grok", extract_verdict=False) + manifest = (await council.ask("q", synthesize=False)).manifest + + assert manifest.receipts[0].outcome == "success" + assert manifest.receipts[0].usage is None + assert manifest.receipts[0].cost_ceiling_usd is None + assert manifest.receipts[0].cost_basis is None + assert manifest.unpriced_receipts == 1 + assert manifest.cost_ceiling_usd is None + assert "unpriced_receipts_present" in manifest.pricing_warnings + + +async def test_a_zero_usage_success_is_unpriced_not_free(monkeypatch, keys): + """QA I1: an all-zero reported ``TokenUsage`` is "no usage", never ``$0``. + + ``usage: {}`` on the wire coerces to ``TokenUsage(0, 0, 0)`` -- a + technically-present but empty usage figure. Pricing that as + ``$0.000000`` would assert a false floor (the call definitely cost + something; a 5,000-byte prompt was sent) rather than an honest bound, so + it must be treated exactly like "no usage reported" -- unpriced. + """ + import conclave.council as council_mod + from conclave.models import ModelAnswer, TokenUsage + + _install_snapshot(monkeypatch, _snapshot("xai/grok-4.3")) + + async def zero_usage(name, model_id, messages, **kwargs): + return ModelAnswer( + name=name, + model_id=model_id, + answer="ok", + usage=TokenUsage(prompt_tokens=0, completion_tokens=0, total_tokens=0), + ) + + monkeypatch.setattr(council_mod, "call_model", zero_usage) + council = Council(models=["grok"], synthesizer="grok", extract_verdict=False) + manifest = (await council.ask("q" * 5000, synthesize=False)).manifest + + assert manifest.receipts[0].usage == TokenUsage( + prompt_tokens=0, completion_tokens=0, total_tokens=0 + ) + assert manifest.receipts[0].cost_ceiling_usd is None + assert manifest.receipts[0].cost_basis is None + assert manifest.unpriced_receipts == 1 + assert manifest.cost_ceiling_usd is None + assert manifest.cost_ceiling_usd != Decimal("0") + assert "unpriced_receipts_present" in manifest.pricing_warnings