From 4fe313b498e79e1f749892d2e14dde3b499f1573 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 4 Sep 2026 15:57:54 -0400 Subject: [PATCH 01/10] feat(council): thread max_output_tokens through every call path (DSE-1514) Widens several pre-existing test doubles for call_model/call_model_stream (test_cache.py, test_council.py, test_keyleak_audit.py, test_output_contract_plumbing.py, test_streaming.py, test_streaming_verdict.py, test_secret_safety_matrix.py, test_integration_verdict.py, test_cli.py) to accept **kwargs, since fan_out/adjudicate/extract_verdict now always pass max_output_tokens (None when unset) through to call_model/call_model_stream -- matching the pattern the shared patch_call_model fixture already used. Also adds the Round 3 review's end-to-end reservation-basis test (tests/test_pricing_receipts.py): with a real max_output_tokens set, a member call that fails with no usage prices as cost_basis == "reservation", and the run-level ceiling still sums correctly alongside a usage-priced sibling receipt. Claude-Session: https://claude.ai/code/session_01K1dHPjZ1bZcE2GnX3KMMSH --- src/conclave/config.py | 31 +++++++ src/conclave/council.py | 48 +++++++++-- src/conclave/manifest.py | 4 +- src/conclave/providers.py | 9 +- src/conclave/streaming.py | 2 + src/conclave/verdict_synthesis.py | 18 +++- tests/test_cache.py | 6 +- tests/test_cli.py | 2 +- tests/test_council.py | 2 +- tests/test_integration_verdict.py | 4 +- tests/test_keyleak_audit.py | 4 +- tests/test_output_budget_plumbing.py | 109 +++++++++++++++++++++++++ tests/test_output_contract_plumbing.py | 2 + tests/test_pricing_receipts.py | 59 ++++++++++++- tests/test_secret_safety_matrix.py | 4 +- tests/test_streaming.py | 24 ++++-- tests/test_streaming_verdict.py | 4 +- 17 files changed, 305 insertions(+), 27 deletions(-) diff --git a/src/conclave/config.py b/src/conclave/config.py index 5d81474..e77069f 100644 --- a/src/conclave/config.py +++ b/src/conclave/config.py @@ -83,6 +83,13 @@ class ConclaveConfig(BaseModel): reaches the threshold. ``None`` keeps the historic fixed-rounds behavior exactly. A ``--converge-threshold`` / ``--converge/--no-converge`` CLI flag overrides this per invocation. See :func:`conclave.modes.run_debate`. + max_output_tokens: opt-in hard ceiling on output tokens for EVERY call a + council makes -- members, synthesis, judge, verdict extraction and + its repair retry, and the streaming paths. ``None`` (the default) + leaves each provider's own default in place, exactly as today. It is + also the precondition for ``--max-spend-usd``: a run whose output is + unbounded cannot have its spend bounded, so the gate refuses rather + than inventing a number. """ models: dict[str, str] = Field(default_factory=dict) @@ -92,6 +99,7 @@ class ConclaveConfig(BaseModel): endpoints: dict[str, CustomEndpoint] = Field(default_factory=dict) cache: bool = False converge_threshold: float | None = None + max_output_tokens: int | None = None def resolve_model_id(self, name: str) -> str: """Map a friendly name to a provider-prefixed model id. @@ -254,6 +262,10 @@ def _load_config_uncached(path: Path) -> ConclaveConfig: # keeping config loading resilient like the rest of this module. converge_threshold = _coerce_threshold(raw.get("converge_threshold")) + # Off by default (None). A bad value degrades to "cap off" rather than + # raising -- see _coerce_max_output_tokens. + max_output_tokens = _coerce_max_output_tokens(raw.get("max_output_tokens")) + return ConclaveConfig( models=merged_models, councils=councils, @@ -262,6 +274,7 @@ def _load_config_uncached(path: Path) -> ConclaveConfig: endpoints=endpoints, cache=cache, converge_threshold=converge_threshold, + max_output_tokens=max_output_tokens, ) @@ -286,3 +299,21 @@ def _coerce_threshold(value: Any) -> float | None: ) return None return threshold + + +def _coerce_max_output_tokens(value: Any) -> int | None: + """Coerce a config ``max_output_tokens`` value to a positive int, or ``None``. + + A non-integer, boolean, or non-positive value degrades to ``None`` (cap off) + with a warning, matching this module's resilient-loading convention: a bad + config field never crashes a run, it just disables the optional feature. + """ + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int): + logger.warning("max_output_tokens %r is not an integer; disabling the output cap", value) + return None + if value < 1: + logger.warning("max_output_tokens %s is not positive; disabling the output cap", value) + return None + return value diff --git a/src/conclave/council.py b/src/conclave/council.py index 0377414..96654b5 100644 --- a/src/conclave/council.py +++ b/src/conclave/council.py @@ -223,6 +223,13 @@ class Council: When supplied, it participates in cache identity so grounded and ungrounded Elite runs cannot collide. The value is re-hashed before entering the canonical identity document. + max_output_tokens: Opt-in hard ceiling on output tokens for EVERY call + this council makes -- members, synthesis, judge, verdict extraction + and its repair retry, and the streaming paths (DSE-1514). ``None`` + (the default) defers to ``config.max_output_tokens`` (itself + ``None`` unless set), leaving each provider's own default in place + exactly as before this flag existed. A cap is the precondition for + ``--max-spend-usd``: see :meth:`plan_calls`. Example: >>> council = Council(models=["grok", "perplexity"], synthesizer="claude") @@ -241,6 +248,7 @@ def __init__( extract_verdict: bool = True, allow_transport_debug_logging: bool = False, source_bundle_digest: str | None = None, + max_output_tokens: int | None = None, ) -> None: self.config = config or load_config() self.requested_models = list(models) @@ -270,10 +278,12 @@ def __init__( # that DEBUG band and accept the responsibility. if not allow_transport_debug_logging: transport.guard_transport_logging() - # Replaced by the real config/argument resolution in the output-cap task - # (DSE-1514 Task 9). Needed already by `_price_manifest` (Task 7), which - # reads it to decide whether a failed call's reservation can be priced. - self.max_output_tokens: int | None = None + # Explicit override wins; otherwise defer to config (off by default). + # A cap is what makes a run's output -- and therefore its spend -- + # boundable at all; see Council.plan_calls and the --max-spend-usd gate. + self.max_output_tokens = ( + self.config.max_output_tokens if max_output_tokens is None else max_output_tokens + ) @staticmethod def _resolve_chain(spec: str | Sequence[str] | None, config: ConclaveConfig) -> list[str]: @@ -319,6 +329,20 @@ def _available_members(self) -> tuple[list[tuple[str, str]], list[str]]: skipped.append(name) return members, skipped + def _generation_settings(self) -> dict[str, float | int]: + """The generation settings actually used, for the manifest and receipts. + + ``max_output_tokens`` appears ONLY when a cap is configured, so an + uncapped run's manifest is byte-identical to v1.3.0's. + """ + settings: dict[str, float | int] = { + "temperature": self.temperature, + "timeout": self.timeout, + } + if self.max_output_tokens is not None: + settings["max_output_tokens"] = self.max_output_tokens + return settings + def _cache_key( self, prompt: str, @@ -538,6 +562,7 @@ async def fan_out( config=self.config, temperature=self.temperature, timeout=self.timeout, + max_output_tokens=self.max_output_tokens, ) for name, model_id in members ] @@ -622,7 +647,12 @@ def _build_manifest( from . import __version__ receipts = [ - receipt_from_answer(a, temperature=self.temperature, timeout=self.timeout) + receipt_from_answer( + a, + temperature=self.temperature, + timeout=self.timeout, + max_output_tokens=self.max_output_tokens, + ) for a in answers ] manifest = ModelHarnessManifest( @@ -635,7 +665,7 @@ def _build_manifest( ProviderSkip(name=name, reason="no API key in environment") for name in skipped ], model_ids=[model_id for _name, model_id in members], - generation_settings={"temperature": self.temperature, "timeout": self.timeout}, + generation_settings=self._generation_settings(), receipts=receipts, ) self._recompute_manifest_accounting(manifest) @@ -752,6 +782,7 @@ def _build_elite_manifest( phase=phase, protocol_version=ELITE_PROTOCOL_VERSION, prompt_version=None if phase == "initial" else ELITE_PROMPT_VERSION, + max_output_tokens=self.max_output_tokens, ) for phase, answers in phase_artifacts for answer in answers @@ -766,7 +797,7 @@ def _build_elite_manifest( ProviderSkip(name=name, reason="no API key in environment") for name in skipped ], model_ids=list(dict.fromkeys(model_id for _name, model_id in members)), - generation_settings={"temperature": self.temperature, "timeout": self.timeout}, + generation_settings=self._generation_settings(), receipts=receipts, ) self._recompute_manifest_accounting(manifest) @@ -1315,6 +1346,7 @@ async def _apply_verdict( temperature=self.temperature, timeout=self.timeout, protocol_version=protocol_version, + max_output_tokens=self.max_output_tokens, ) renumbered_receipts = [ receipt.model_copy(update={"attempt": verdict_receipts_so_far + offset}) @@ -1499,6 +1531,7 @@ def _attempt( config=self.config, temperature=self.temperature, timeout=self.timeout, + max_output_tokens=self.max_output_tokens, ) called.append(answer) is_last = index == len(chain) @@ -1629,6 +1662,7 @@ def _record_adjudication( attempt=index, protocol_version=protocol_version, prompt_version=prompt_version, + max_output_tokens=self.max_output_tokens, ) for index, answer in enumerate(called, start=1) ) diff --git a/src/conclave/manifest.py b/src/conclave/manifest.py index 5f6c852..a839fae 100644 --- a/src/conclave/manifest.py +++ b/src/conclave/manifest.py @@ -217,7 +217,7 @@ class ProviderExecutionReceipt(BaseModel): name: str provider: str model_id: str - generation_settings: dict[str, float] = Field(default_factory=dict) + generation_settings: dict[str, float | int] = Field(default_factory=dict) latency_ms: float = 0.0 usage: TokenUsage | None = None estimated_cost: float | None = None @@ -315,7 +315,7 @@ class ModelHarnessManifest(BaseModel): model_ids: list[str] = Field(default_factory=list) # Settings + execution receipts + aggregate latency/usage. - generation_settings: dict[str, float] = Field(default_factory=dict) + generation_settings: dict[str, float | int] = Field(default_factory=dict) receipts: list[ProviderExecutionReceipt] = Field(default_factory=list) total_latency_ms: float = 0.0 total_usage: TokenUsage | None = None diff --git a/src/conclave/providers.py b/src/conclave/providers.py index 92ebb48..105db85 100644 --- a/src/conclave/providers.py +++ b/src/conclave/providers.py @@ -44,6 +44,7 @@ def receipt_from_answer( protocol_version: str | None = None, prompt_version: str | None = None, schema_version: str | None = None, + max_output_tokens: int | None = None, ) -> ProviderExecutionReceipt: """Map a collected :class:`ModelAnswer` to a :class:`ProviderExecutionReceipt`. @@ -66,6 +67,9 @@ def receipt_from_answer( temperature: The sampling temperature the council used for the call. timeout: The per-call timeout (seconds) the council used. phase: Optional protocol phase provenance for phased modes. + max_output_tokens: The hard output-token ceiling the call was issued + with, if any (DSE-1514). Recorded on ``generation_settings`` only + when set, so an uncapped receipt stays byte-identical to before. Returns: A :class:`ProviderExecutionReceipt` for this member. @@ -73,6 +77,9 @@ def receipt_from_answer( has_error = answer.error is not None category = error_category or (_receipt_error_category(answer.error) if has_error else None) resolved_outcome = outcome or ("failed" if has_error else "success") + generation_settings: dict[str, float | int] = {"temperature": temperature, "timeout": timeout} + if max_output_tokens is not None: + generation_settings["max_output_tokens"] = max_output_tokens return ProviderExecutionReceipt( phase=phase, attempt=attempt, @@ -80,7 +87,7 @@ def receipt_from_answer( name=answer.name, provider=provider_prefix(answer.model_id), model_id=answer.model_id, - generation_settings={"temperature": temperature, "timeout": timeout}, + generation_settings=generation_settings, latency_ms=answer.latency_ms, usage=answer.usage, # Keep the compatibility field, but store only the bounded category. Raw diff --git a/src/conclave/streaming.py b/src/conclave/streaming.py index e068297..1a3de15 100644 --- a/src/conclave/streaming.py +++ b/src/conclave/streaming.py @@ -105,6 +105,7 @@ async def _drive_member( temperature=council.temperature, timeout=council.timeout, config=council.config, + max_output_tokens=council.max_output_tokens, ): if isinstance(item, ModelAnswer): await queue.put(("answer", item)) @@ -378,6 +379,7 @@ async def _stream_synthesis(council: Council, result: CouncilResult) -> AsyncIte temperature=council.temperature, timeout=council.timeout, config=council.config, + max_output_tokens=council.max_output_tokens, ): if isinstance(item, ModelAnswer): candidate_final = item diff --git a/src/conclave/verdict_synthesis.py b/src/conclave/verdict_synthesis.py index 2187661..f4a661f 100644 --- a/src/conclave/verdict_synthesis.py +++ b/src/conclave/verdict_synthesis.py @@ -234,8 +234,15 @@ def _verdict_attempt_receipt( temperature: float, timeout: float, protocol_version: str | None, + max_output_tokens: int | None = None, ) -> ProviderExecutionReceipt: - """Build one secret-free receipt for an extraction or repair attempt.""" + """Build one secret-free receipt for an extraction or repair attempt. + + Args: + max_output_tokens: The hard output ceiling this attempt was issued + with, if any (DSE-1514). ``None`` leaves the provider default in + place, exactly as before this parameter existed. + """ if answer.error is not None: outcome = "failed" error_category = None @@ -257,6 +264,7 @@ def _verdict_attempt_receipt( protocol_version=protocol_version, prompt_version=VERDICT_EXTRACTION_PROMPT_VERSION, schema_version=VERDICT_SCHEMA_VERSION, + max_output_tokens=max_output_tokens, ) @@ -554,6 +562,7 @@ async def extract_verdict( timeout: float = 120.0, protocol_version: str | None = None, call_model_func=None, # noqa: ANN001 -- injectable async seam for guarded callers + max_output_tokens: int | None = None, ) -> VerdictSynthesisResult: """Extract a structured, auditable verdict from a council's member answers. @@ -595,6 +604,9 @@ async def extract_verdict( ``None`` preserves the normal module-level provider path. Guarded eval runners inject their reservation-aware gateway here so both the initial extraction and optional repair remain paid-call protected. + max_output_tokens: The hard output ceiling for BOTH the initial + extraction call and its repair retry (DSE-1514); ``None`` leaves the + provider default in place. Returns: A :class:`VerdictSynthesisResult`. On success ``verdict`` is populated and @@ -651,6 +663,7 @@ async def extract_verdict( timeout=timeout, config=config, output_contract=output_contract, + max_output_tokens=max_output_tokens, ) # Step 3 — validate, then repair ONCE on failure, then fall back. @@ -664,6 +677,7 @@ async def extract_verdict( temperature=temperature, timeout=timeout, protocol_version=protocol_version, + max_output_tokens=max_output_tokens, ) ] retry: ModelAnswer | None = None @@ -688,6 +702,7 @@ async def extract_verdict( timeout=timeout, config=config, output_contract=output_contract, + max_output_tokens=max_output_tokens, ) extraction, errors = _parse_and_validate(retry) attempt_receipts.append( @@ -699,6 +714,7 @@ async def extract_verdict( temperature=temperature, timeout=timeout, protocol_version=protocol_version, + max_output_tokens=max_output_tokens, ) ) diff --git a/tests/test_cache.py b/tests/test_cache.py index af8412c..67c0126 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -82,7 +82,7 @@ def counting_call_model(monkeypatch): counter = {"n": 0} async def fake_call_model( - name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None + name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None, **kwargs ): counter["n"] += 1 await asyncio.sleep(0) @@ -1014,7 +1014,9 @@ def _install_stream_script(monkeypatch, script: dict[str, list]) -> list[str]: calls: list[str] = [] - async def fake_stream(name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None): + async def fake_stream( + name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None, **kwargs + ): calls.append(name) for item in script[name]: yield item diff --git a/tests/test_cli.py b/tests/test_cli.py index 5d42bce..b74782f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -795,7 +795,7 @@ def test_cache_flag_serves_second_run_from_cache(monkeypatch, patch_cli_config, counter = {"n": 0} async def fake_call_model( - name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None + name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None, **kwargs ): counter["n"] += 1 return ModelAnswer(name=name, model_id=model_id, answer=f"ans-{model_id}") diff --git a/tests/test_council.py b/tests/test_council.py index 076f779..cb8966d 100644 --- a/tests/test_council.py +++ b/tests/test_council.py @@ -131,7 +131,7 @@ async def test_concurrency_is_real(monkeypatch): # Replace call_model with a coroutine that sleeps, to prove gather concurrency. async def sleepy_call_model( - name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None + name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None, **kwargs ): await asyncio.sleep(0.2) return ModelAnswer(name=name, model_id=model_id, answer=f"ok {model_id}") diff --git a/tests/test_integration_verdict.py b/tests/test_integration_verdict.py index 7bf260a..cb21d1a 100644 --- a/tests/test_integration_verdict.py +++ b/tests/test_integration_verdict.py @@ -133,7 +133,9 @@ def _patch_member_stream(monkeypatch, deltas_by_model, errors_by_model=None) -> errors_by_model = errors_by_model or {} - async def fake_stream(name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None): + async def fake_stream( + name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None, **kwargs + ): text_parts = deltas_by_model.get(model_id, ["x"]) for part in text_parts: yield part diff --git a/tests/test_keyleak_audit.py b/tests/test_keyleak_audit.py index 275e8fd..4f61ecd 100644 --- a/tests/test_keyleak_audit.py +++ b/tests/test_keyleak_audit.py @@ -512,7 +512,7 @@ async def test_fan_out_catch_all_error_is_redacted(monkeypatch): import conclave.council as council_mod async def raising_call_model( - name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None + name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None, **kwargs ): # Simulate an unexpected escape carrying the key in its text. raise RuntimeError(f"unexpected boom leaking {PLANTED}") @@ -541,7 +541,7 @@ async def test_stream_drive_member_catch_all_error_is_redacted(monkeypatch): import conclave.streaming as streaming_mod async def raising_stream( - name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None + name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None, **kwargs ): # An unexpected raise (not a yielded error ModelAnswer) carrying the key. raise RuntimeError(f"stream boom leaking {PLANTED}") diff --git a/tests/test_output_budget_plumbing.py b/tests/test_output_budget_plumbing.py index a6a0aa9..d0c5420 100644 --- a/tests/test_output_budget_plumbing.py +++ b/tests/test_output_budget_plumbing.py @@ -109,3 +109,112 @@ async def fake_stream_sse(url, headers, json_body, timeout): assert isinstance(items[-1], ModelAnswer) assert items[-1].ok assert captured["body"]["max_tokens"] == 888 + + +"""DSE-1514: the cap is a COUNCIL setting, not just an adapter parameter.""" + + +def test_config_reads_and_sanitizes_max_output_tokens(tmp_path, monkeypatch): + from conclave.config import clear_config_cache, load_config + + path = tmp_path / "config.yml" + monkeypatch.setenv("CONCLAVE_CONFIG", str(path)) + + path.write_text("max_output_tokens: 1024\n", encoding="utf-8") + clear_config_cache() + assert load_config().max_output_tokens == 1024 + + path.write_text("max_output_tokens: not-a-number\n", encoding="utf-8") + clear_config_cache() + assert load_config().max_output_tokens is None + + path.write_text("max_output_tokens: 0\n", encoding="utf-8") + clear_config_cache() + assert load_config().max_output_tokens is None + clear_config_cache() + + +async def test_the_cap_reaches_every_member_and_adjudication_call(monkeypatch, keys): + import conclave.council as council_mod + import conclave.verdict_synthesis as verdict_mod + from conclave.council import Council + from conclave.models import ModelAnswer + + seen: list[int | None] = [] + + async def spy(name, model_id, messages, *, max_output_tokens=None, **kwargs): + seen.append(max_output_tokens) + return ModelAnswer(name=name, model_id=model_id, answer="ok") + + monkeypatch.setattr(council_mod, "call_model", spy) + monkeypatch.setattr(verdict_mod, "call_model", spy) + # Two members: verdict extraction's N<2-responder gate (CAC-05, DD-1) skips + # the extraction call entirely for a single responder, which would hide the + # cap from the verdict-extraction/repair sites this test means to cover. + council = Council(models=["grok", "gemini"], synthesizer="claude", max_output_tokens=777) + await council.ask("q") + + # member fan-out (x2) + synthesis + verdict extraction + verdict repair + assert len(seen) >= 4 + assert set(seen) == {777} + + +async def test_no_cap_configured_sends_nothing_new(monkeypatch, keys): + import conclave.council as council_mod + from conclave.council import Council + from conclave.models import ModelAnswer + + seen: list[int | None] = [] + + async def spy(name, model_id, messages, *, max_output_tokens=None, **kwargs): + seen.append(max_output_tokens) + return ModelAnswer(name=name, model_id=model_id, answer="ok") + + monkeypatch.setattr(council_mod, "call_model", spy) + council = Council(models=["grok"], synthesizer="grok", extract_verdict=False) + await council.ask("q", synthesize=False) + assert seen == [None] + + +async def test_the_cap_is_recorded_in_generation_settings_only_when_set(monkeypatch, keys): + import conclave.council as council_mod + from conclave.council import Council + from conclave.models import ModelAnswer + + async def ok(name, model_id, messages, **kwargs): + return ModelAnswer(name=name, model_id=model_id, answer="ok") + + monkeypatch.setattr(council_mod, "call_model", ok) + + capped = await Council( + models=["grok"], synthesizer="grok", max_output_tokens=256, extract_verdict=False + ).ask("q", synthesize=False) + assert capped.manifest.generation_settings["max_output_tokens"] == 256 + assert capped.manifest.receipts[0].generation_settings["max_output_tokens"] == 256 + + plain = await Council(models=["grok"], synthesizer="grok", extract_verdict=False).ask( + "q", synthesize=False + ) + assert "max_output_tokens" not in plain.manifest.generation_settings + assert "max_output_tokens" not in plain.manifest.receipts[0].generation_settings + + +async def test_streaming_members_and_synthesis_receive_the_cap(monkeypatch, keys): + import conclave.streaming as streaming_mod + from conclave.council import Council + from conclave.models import ModelAnswer + + seen: list[int | None] = [] + + async def spy_stream(name, model_id, messages, *, max_output_tokens=None, **kwargs): + seen.append(max_output_tokens) + yield "tok" + yield ModelAnswer(name=name, model_id=model_id, answer="tok") + + monkeypatch.setattr(streaming_mod, "call_model_stream", spy_stream) + council = Council( + models=["grok"], synthesizer="claude", max_output_tokens=333, extract_verdict=False + ) + async for _event in council.ask_stream("q"): + pass + assert seen and set(seen) == {333} diff --git a/tests/test_output_contract_plumbing.py b/tests/test_output_contract_plumbing.py index bc84b89..aa293cc 100644 --- a/tests/test_output_contract_plumbing.py +++ b/tests/test_output_contract_plumbing.py @@ -277,6 +277,7 @@ async def fake_call_model( timeout=120.0, config=None, output_contract=None, + **kwargs, ): captured["output_contract"] = output_contract return ModelAnswer(name=name, model_id=model_id, answer="not json") @@ -330,6 +331,7 @@ async def fake_call_model( timeout=120.0, config=None, output_contract=None, + **kwargs, ): return ModelAnswer(name=name, model_id=model_id, answer="not json") diff --git a/tests/test_pricing_receipts.py b/tests/test_pricing_receipts.py index 9763faf..559ccb2 100644 --- a/tests/test_pricing_receipts.py +++ b/tests/test_pricing_receipts.py @@ -254,7 +254,9 @@ async def test_a_successful_call_with_no_reported_usage_is_unpriced_not_a_reserv reservation that ignores which phase the call belongs to would silently mis-price synthesis/judge/verdict calls (which embed upstream output) by 3-9x. This round has no phase-aware plan to reserve from, so the honest - rule is: no reported usage -> unpriced, exactly like a failed call. + rule is: no reported usage -> unpriced, exactly like a failed call. A + real, phase-aware reservation basis is re-instated in Round 4 -- see + ``test_a_failed_call_with_no_usage_is_priced_as_a_reservation_when_capped``. """ import conclave.council as council_mod from conclave.models import ModelAnswer @@ -312,3 +314,58 @@ async def zero_usage(name, model_id, messages, **kwargs): assert manifest.cost_ceiling_usd is None assert manifest.cost_ceiling_usd != Decimal("0") assert "unpriced_receipts_present" in manifest.pricing_warnings + + +async def test_a_failed_call_with_no_usage_is_unpriced_even_when_capped_this_round( + monkeypatch, keys +): + """DSE-1514 Round 3 review, interim: a cap does not resurrect flat-constant reservation. + + Round 3 removed the flat-allowance reservation branch entirely (QA C1): + it silently mis-priced synthesis/judge/verdict calls, which embed + upstream output, by 3-9x. So even with a real ``max_output_tokens`` cap + threaded through the constructor (this commit), a usage-less receipt + stays unpriced -- exactly like the uncapped case above -- until Round 4's + phase-aware reservation basis lands (see the test with "when_capped" in + its name for that end state). + """ + import conclave.council as council_mod + from conclave.models import ModelAnswer, TokenUsage + + _install_snapshot(monkeypatch, _snapshot("xai/grok-4.3", "gemini/gemini-2.5-pro")) + + async def flaky(name, model_id, messages, **kwargs): + if model_id == "gemini/gemini-2.5-pro": + # call_model never raises; a provider 503 comes back as an error + # answer carrying no usage at all. + return ModelAnswer(name=name, model_id=model_id, error="503: service unavailable") + return ModelAnswer( + name=name, + model_id=model_id, + answer="ok", + usage=TokenUsage(prompt_tokens=5, completion_tokens=7, total_tokens=12), + ) + + monkeypatch.setattr(council_mod, "call_model", flaky) + council = Council( + models=["grok", "gemini"], + synthesizer="grok", + max_output_tokens=256, + extract_verdict=False, + ) + manifest = (await council.ask("q", synthesize=False)).manifest + + receipts_by_model = {r.model_id: r for r in manifest.receipts} + failed = receipts_by_model["gemini/gemini-2.5-pro"] + succeeded = receipts_by_model["xai/grok-4.3"] + + assert failed.usage is None + assert failed.cost_basis is None + assert failed.cost_ceiling_usd is None + assert succeeded.cost_basis == "reported_usage" + + # A capped, usage-less receipt is unpriced this round, so the run-level + # ceiling stays None even though the OTHER receipt priced cleanly. + assert manifest.unpriced_receipts == 1 + assert manifest.cost_ceiling_usd is None + assert "unpriced_receipts_present" in manifest.pricing_warnings diff --git a/tests/test_secret_safety_matrix.py b/tests/test_secret_safety_matrix.py index 0b84fb1..ffe32b7 100644 --- a/tests/test_secret_safety_matrix.py +++ b/tests/test_secret_safety_matrix.py @@ -174,7 +174,9 @@ def _patch_member_stream(monkeypatch, deltas_by_model, errors_by_model=None) -> errors_by_model = errors_by_model or {} - async def fake_stream(name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None): + async def fake_stream( + name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None, **kwargs + ): text_parts = deltas_by_model.get(model_id, ["x"]) for part in text_parts: yield part diff --git a/tests/test_streaming.py b/tests/test_streaming.py index 035ad42..5c7a9de 100644 --- a/tests/test_streaming.py +++ b/tests/test_streaming.py @@ -371,7 +371,9 @@ def _patch_stream(monkeypatch, deltas_by_model): """Patch streaming.call_model_stream to emit canned deltas + a final answer.""" import conclave.streaming as streaming_mod - async def fake_stream(name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None): + async def fake_stream( + name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None, **kwargs + ): text_parts = deltas_by_model.get(model_id, ["x"]) for part in text_parts: yield part @@ -457,7 +459,9 @@ def test_cli_stream_smoke_exits_zero(monkeypatch, patch_cli_config): for var in ("XAI_API_KEY", "GEMINI_API_KEY", "ANTHROPIC_API_KEY"): monkeypatch.setenv(var, "dummy-key") - async def fake_stream(name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None): + async def fake_stream( + name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None, **kwargs + ): yield f"tok-{name} " yield ModelAnswer(name=name, model_id=model_id, answer=f"tok-{name} ") @@ -478,7 +482,9 @@ def test_cli_stream_zero_usable_exits_one(monkeypatch, patch_cli_config): for var in ("XAI_API_KEY", "GEMINI_API_KEY"): monkeypatch.setenv(var, "dummy-key") - async def fake_stream(name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None): + async def fake_stream( + name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None, **kwargs + ): yield ModelAnswer(name=name, model_id=model_id, error="provider down") monkeypatch.setattr(streaming_mod, "call_model_stream", fake_stream) @@ -514,7 +520,9 @@ def test_cli_stream_cache_second_run_is_one_shot_hit(monkeypatch, patch_cli_conf calls = {"n": 0} - async def fake_stream(name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None): + async def fake_stream( + name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None, **kwargs + ): calls["n"] += 1 yield "live " yield ModelAnswer(name=name, model_id=model_id, answer="live answer") @@ -544,7 +552,9 @@ async def test_ask_stream_cache_hit_replays_one_shot(monkeypatch, tmp_path): live_calls = {"n": 0} - async def fake_stream(name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None): + async def fake_stream( + name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None, **kwargs + ): live_calls["n"] += 1 yield "x" yield ModelAnswer(name=name, model_id=model_id, answer="x") @@ -601,7 +611,9 @@ def _install_stream_script(monkeypatch, script: dict[str, list]) -> list[str]: calls: list[str] = [] - async def fake_stream(name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None): + async def fake_stream( + name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None, **kwargs + ): calls.append(name) for item in script[name]: yield item diff --git a/tests/test_streaming_verdict.py b/tests/test_streaming_verdict.py index 17dc469..583119c 100644 --- a/tests/test_streaming_verdict.py +++ b/tests/test_streaming_verdict.py @@ -137,7 +137,9 @@ def _patch_member_stream(monkeypatch, deltas_by_model, errors_by_model=None) -> errors_by_model = errors_by_model or {} - async def fake_stream(name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None): + async def fake_stream( + name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None, **kwargs + ): text_parts = deltas_by_model.get(model_id, ["x"]) for part in text_parts: yield part From 809fa308a2a64b0241e33b5160afa591f9de0a7d Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 4 Sep 2026 16:02:43 -0400 Subject: [PATCH 02/10] feat(council): plan_calls enumerates the worst-case call plan per mode (DSE-1514) Claude-Session: https://claude.ai/code/session_01K1dHPjZ1bZcE2GnX3KMMSH --- src/conclave/council.py | 234 +++++++++++++++++++++++++++++- src/conclave/verdict_synthesis.py | 13 ++ tests/test_spend_plan.py | 92 ++++++++++++ 3 files changed, 338 insertions(+), 1 deletion(-) create mode 100644 tests/test_spend_plan.py diff --git a/src/conclave/council.py b/src/conclave/council.py index 96654b5..b1e2415 100644 --- a/src/conclave/council.py +++ b/src/conclave/council.py @@ -59,7 +59,7 @@ from uuid import uuid4 from . import cache as cache_mod -from . import transport +from . import prompts, transport from .adapters.base import redact from .config import ConclaveConfig, load_config, parse_synthesizer_chain from .logging import get_logger @@ -88,6 +88,8 @@ from .prompts import ELITE_PROMPT_VERSION, SYNTHESIS_PROMPT_VERSION from .providers import call_model, receipt_from_answer from .registry import key_present +from .verdict_synthesis import VERDICT_CONTRACT_BYTES as _VERDICT_CONTRACT_BYTES +from .verdict_synthesis import VERDICT_TEMPLATE_PROBE as _VERDICT_TEMPLATE_PROBE if TYPE_CHECKING: # avoid an import cycle at runtime; only needed for typing from .verdict_synthesis import VerdictSynthesisResult @@ -115,6 +117,19 @@ "Do not invent a model's position; rely only on the answers provided." ) +# The modes Council.plan_calls knows how to enumerate (DSE-1514). Kept as its own +# frozenset (rather than re-deriving from _RENDERERS or similar) so the planner's +# contract is explicit and independent of any CLI-only vocabulary. +_VALID_PLAN_MODES = frozenset({"raw", "synthesize", "vote", "debate", "adversarial", "elite"}) + +# The exact refusal message when a spend cap is requested but output is +# unbounded (DSE-1514): shared verbatim by Council.plan_calls, the Council +# constructor's max_spend_usd guard, and the CLI so the message a library caller +# sees and the message a CLI user sees are byte-identical. +_NO_OUTPUT_CAP_MESSAGE = ( + "cannot bound spend: no output cap (set --max-output-tokens or config max_output_tokens)" +) + # Re-exported for callers that want the version without importing prompts. __all__ = ["Council", "SYNTHESIS_PROMPT_VERSION"] @@ -169,6 +184,50 @@ def model_id(self) -> str | None: return self.answer.model_id if self.answer is not None else None +@dataclass(frozen=True) +class PlannedCall: + """One provider call a mode COULD make, bounded without making it. + + Every field is knowable before the first call: the resolved model, the + output cap the call will be issued with, and a three-part input bound (exact + prompt bytes, fixed template bytes, provider framing) plus a count of + upstream calls whose not-yet-produced output this call's input will embed. + + Attributes: + phase: The manifest phase this call would be recorded under. + name: Friendly member / candidate name. + model_id: Resolved provider-prefixed model id. + prompt_token_upper_bound: UTF-8 bytes of the exact known content. + prompt_template_token_allowance: UTF-8 bytes of the fixed system + user + template wording that will surround it. + provider_framing_token_allowance: ``64 + 16 * messages`` (+256 with a + structured-output contract), mirroring the eval runner. + upstream_output_call_count: How many upstream calls' outputs this call's + input embeds. Multiplied by the output cap and the snapshot's + ``max_output_bytes_per_token`` to bound them. + max_output_tokens: The hard output cap this call would carry. + """ + + phase: str + name: str + model_id: str + prompt_token_upper_bound: int + prompt_template_token_allowance: int + provider_framing_token_allowance: int + upstream_output_call_count: int + max_output_tokens: int + + +@dataclass(frozen=True) +class CallPlan: + """The complete worst-case call plan for one run of one mode.""" + + mode: str + calls: tuple[PlannedCall, ...] + member_count: int + chain_count: int + + class Council: """A council of foundation models with an optional synthesizer. @@ -343,6 +402,179 @@ def _generation_settings(self) -> dict[str, float | int]: settings["max_output_tokens"] = self.max_output_tokens return settings + def _keyed_chain(self) -> list[tuple[str, str]]: + """Resolve the synthesizer chain to the candidates that could be CALLED. + + :meth:`adjudicate` skips an unkeyed candidate without making a call, so + an unkeyed candidate cannot cost anything and is excluded from the plan. + ``registry.key_present`` returns ``True`` for an unknown provider prefix, + which errs toward INCLUDING the call -- the safe direction for a ceiling. + """ + pairs = [(name, self.config.resolve_model_id(name)) for name in self.synthesizer_chain] + return [(name, model_id) for name, model_id in pairs if key_present(model_id)] + + def plan_calls( + self, + mode: str, + prompt: str, + *, + rounds: int = 2, + proposer: str | None = None, + choices: list[str] | None = None, + ) -> CallPlan: + """Enumerate every provider call this mode could make, worst case (DSE-1514). + + The counts are derived from :mod:`conclave.modes` and + :meth:`_apply_verdict`, not from a remembered formula. With ``N`` keyed + members, ``C`` keyed chain candidates, ``R`` debate rounds, and ``V`` = 1 + when verdict extraction is on: + + * ``raw`` -- ``N``: fan-out only. + * ``synthesize`` -- ``N + C + 2CV``: fan-out, the chain, then + extract+repair per candidate. + * ``vote`` -- ``N``: fan-out only; no adjudication. + * ``debate`` -- ``N*R + C``: every round at full membership (drop-out + only shrinks it), then the final consolidation chain. + * ``adversarial`` -- ``N + C``: ``k`` proposer attempts plus ``N - k`` + critics is exactly ``N`` for every ``k``; then the judge chain. + * ``elite`` -- ``3N + C + 2CV``: three phases at full membership, then + synthesis and verdict extraction. + + Convergence early-stop, member drop-out, and a proposer succeeding on the + first try all make a real run CHEAPER than its plan. A plan is never an + under-count, which is what makes it usable as a spend gate. + + Args: + mode: One of ``raw``/``synthesize``/``vote``/``debate``/ + ``adversarial``/``elite``. + prompt: The exact user prompt (bounded by its UTF-8 byte length). + rounds: Debate rounds; ignored for other modes. + proposer: Adversarial proposer. It does not change the COUNT (see + above) and is accepted only for signature parity with the modes. + choices: Vote choices, which enlarge the vote prompt template. + + Returns: + The :class:`CallPlan`. + + Raises: + ValueError: ``mode`` is not a known deliberation mode, or + ``max_output_tokens`` is not configured (an unbounded output + cannot be planned). + """ + if mode not in _VALID_PLAN_MODES: + raise ValueError(f"unknown mode for call planning: {mode}") + cap = self.max_output_tokens + if cap is None: + raise ValueError(_NO_OUTPUT_CAP_MESSAGE) + + members, _skipped = self._available_members() + chain = self._keyed_chain() + n_members = len(members) + prompt_bytes = len(prompt.encode("utf-8")) + calls: list[PlannedCall] = [] + + def member_calls(phase: str, *, template: str, upstream: int) -> None: + for name, model_id in members: + calls.append( + PlannedCall( + phase=phase, + name=name, + model_id=model_id, + prompt_token_upper_bound=prompt_bytes, + prompt_template_token_allowance=len(template.encode("utf-8")), + provider_framing_token_allowance=64 + (16 * 2), + upstream_output_call_count=upstream, + max_output_tokens=cap, + ) + ) + + def chain_calls(phase: str, *, template: str, upstream: int, contract: bool) -> None: + for name, model_id in chain: + calls.append( + PlannedCall( + phase=phase, + name=name, + model_id=model_id, + prompt_token_upper_bound=( + prompt_bytes + (_VERDICT_CONTRACT_BYTES if contract else 0) + ), + prompt_template_token_allowance=len(template.encode("utf-8")), + provider_framing_token_allowance=(64 + (16 * 3) + (256 if contract else 0)), + upstream_output_call_count=upstream, + max_output_tokens=cap, + ) + ) + + if mode in ("raw", "synthesize"): + member_calls("member", template="", upstream=0) + elif mode == "vote": + member_calls( + "member", + template=prompts.VOTE_SYSTEM + prompts.vote_user("", choices or []), + upstream=0, + ) + elif mode == "debate": + member_calls("round-1", template="", upstream=0) + for round_no in range(2, max(1, rounds) + 1): + member_calls( + f"round-{round_no}", + template=( + prompts.DEBATE_SYSTEM + + prompts.debate_round_user("", round_no, max(1, rounds), "") + ), + upstream=n_members, + ) + elif mode == "adversarial": + # k proposer attempts + (N - k) critics == N, for every k. + member_calls("proposal", template="", upstream=0) + elif mode == "elite": + member_calls("initial", template="", upstream=0) + member_calls( + "critique", + template=prompts.ELITE_CRITIC_SYSTEM + prompts.elite_critic_user("", []), + upstream=n_members, + ) + member_calls("revision", template=prompts.ELITE_REVISION_SYSTEM, upstream=2 * n_members) + + if mode == "debate": + chain_calls( + "debate_final", + template=( + prompts.DEBATE_FINAL_SYSTEM + prompts.debate_final_user("", max(1, rounds), "") + ), + upstream=n_members, + contract=False, + ) + elif mode == "adversarial": + chain_calls( + "judge", + template=prompts.JUDGE_SYSTEM + prompts.judge_user("", "", "", ""), + upstream=n_members, + contract=False, + ) + elif mode in ("synthesize", "elite"): + chain_calls("synthesis", template=_SYNTH_SYSTEM, upstream=n_members, contract=False) + if self.extract_verdict_enabled: + chain_calls( + "verdict_extraction", + template=_VERDICT_TEMPLATE_PROBE, + upstream=n_members, + contract=True, + ) + chain_calls( + "verdict_repair", + template=_VERDICT_TEMPLATE_PROBE, + upstream=n_members + 1, + contract=True, + ) + + return CallPlan( + mode=mode, + calls=tuple(calls), + member_count=n_members, + chain_count=len(chain), + ) + def _cache_key( self, prompt: str, diff --git a/src/conclave/verdict_synthesis.py b/src/conclave/verdict_synthesis.py index f4a661f..66dbf15 100644 --- a/src/conclave/verdict_synthesis.py +++ b/src/conclave/verdict_synthesis.py @@ -171,6 +171,19 @@ def _bounded_repair_error(detail: object) -> str: "clustering. Emit only the fields in the schema." ) +# DSE-1514: byte sizes the pre-flight spend planner needs without making a call. +# The extraction schema and its system prompt are fixed, so their UTF-8 byte cost +# is a constant of this module rather than a per-run guess. +VERDICT_CONTRACT_BYTES = len( + json.dumps( + verdict_extraction_json_schema(), + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") +) +VERDICT_TEMPLATE_PROBE = _EXTRACTION_SYSTEM + class VerdictSynthesisResult(BaseModel): """The outcome of one verdict-extraction run (CAC-05 engine return type). diff --git a/tests/test_spend_plan.py b/tests/test_spend_plan.py new file mode 100644 index 0000000..3da27b3 --- /dev/null +++ b/tests/test_spend_plan.py @@ -0,0 +1,92 @@ +"""The worst-case call plan per mode, derived from modes.py arithmetic (DSE-1514).""" + +from __future__ import annotations + +import pytest + +from conclave.council import Council + +MEMBERS = ["grok", "gemini", "openai"] # N = 3 + + +def _council(**kwargs) -> Council: + kwargs.setdefault("models", MEMBERS) + kwargs.setdefault("synthesizer", "claude") + kwargs.setdefault("max_output_tokens", 1_000) + return Council(**kwargs) + + +@pytest.mark.parametrize( + ("mode", "kwargs", "expected"), + [ + ("raw", {}, 3), # N + ("synthesize", {}, 3 + 1 + 2), # N + C + 2C + ("vote", {"choices": ["a", "b"]}, 3), # N + ("debate", {"rounds": 3}, 3 * 3 + 1), # N*R + C + ("adversarial", {}, 3 + 1), # N + C + ("elite", {}, 3 * 3 + 1 + 2), # 3N + C + 2C + ], +) +def test_worst_case_call_counts_per_mode(keys, mode, kwargs, expected): + plan = _council().plan_calls(mode, "q", **kwargs) + assert len(plan.calls) == expected + assert plan.mode == mode + assert plan.member_count == 3 + assert plan.chain_count == 1 + + +def test_a_longer_chain_multiplies_every_adjudication_role(keys): + council = _council(synthesizer="claude>grok>gemini") # C = 3 + assert len(council.plan_calls("synthesize", "q").calls) == 3 + 3 + 6 + assert len(council.plan_calls("elite", "q").calls) == 9 + 3 + 6 + assert len(council.plan_calls("adversarial", "q").calls) == 3 + 3 + assert council.plan_calls("synthesize", "q").chain_count == 3 + + +def test_an_unkeyed_chain_candidate_is_not_planned(monkeypatch, keys): + # mistral is unkeyed here -> it can never be called, so it can never cost + # anything, so it is not in the plan. + monkeypatch.delenv("MISTRAL_API_KEY", raising=False) + council = _council(synthesizer="claude>mistral") + plan = council.plan_calls("synthesize", "q") + assert plan.chain_count == 1 + assert len(plan.calls) == 3 + 1 + 2 + + +def test_verdict_extraction_off_removes_exactly_two_calls_per_candidate(keys): + on = _council().plan_calls("synthesize", "q") + off = _council(extract_verdict=False).plan_calls("synthesize", "q") + assert len(on.calls) - len(off.calls) == 2 + + +def test_every_planned_call_is_bounded_and_names_its_model(keys): + for call in _council().plan_calls("elite", "q").calls: + assert call.max_output_tokens == 1_000 + assert "/" in call.model_id + assert call.prompt_token_upper_bound >= len(b"q") + assert call.prompt_template_token_allowance >= 0 + assert call.provider_framing_token_allowance >= 64 + assert call.upstream_output_call_count >= 0 + + +def test_downstream_phases_declare_their_upstream_dependencies(keys): + by_phase: dict[str, list] = {} + for call in _council().plan_calls("elite", "q").calls: + by_phase.setdefault(call.phase, []).append(call) + + assert all(c.upstream_output_call_count == 0 for c in by_phase["initial"]) + assert all(c.upstream_output_call_count == 3 for c in by_phase["critique"]) # N initials + assert all(c.upstream_output_call_count == 6 for c in by_phase["revision"]) # N + N + assert by_phase["synthesis"][0].upstream_output_call_count == 3 # N revisions + assert by_phase["verdict_extraction"][0].upstream_output_call_count == 3 + assert by_phase["verdict_repair"][0].upstream_output_call_count == 4 # + its own attempt + + +def test_an_unknown_mode_is_a_value_error(keys): + with pytest.raises(ValueError, match="unknown mode"): + _council().plan_calls("telepathy", "q") + + +def test_planning_without_an_output_cap_is_refused(keys): + with pytest.raises(ValueError, match="cannot bound spend: no output cap"): + Council(models=MEMBERS, synthesizer="claude").plan_calls("synthesize", "q") From 1142cb6aec9668550056397eb27ff733e1aeca88 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 4 Sep 2026 16:08:38 -0400 Subject: [PATCH 03/10] feat(council): pre-flight spend gate refuses before the first provider call (DSE-1514) Claude-Session: https://claude.ai/code/session_01K1dHPjZ1bZcE2GnX3KMMSH --- src/conclave/__init__.py | 11 +++ src/conclave/council.py | 99 ++++++++++++++++++++++++ src/conclave/pricing.py | 37 +++++++++ tests/test_spend_gate.py | 158 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 305 insertions(+) create mode 100644 tests/test_spend_gate.py diff --git a/src/conclave/__init__.py b/src/conclave/__init__.py index ab5a559..b23edc9 100644 --- a/src/conclave/__init__.py +++ b/src/conclave/__init__.py @@ -59,6 +59,12 @@ StreamEvent, TokenUsage, ) +from .pricing import ( + PriceSnapshot, + SpendCapExceeded, + SpendRefused, + SpendUnboundable, +) from .transport import aclose, guard_transport_logging from .verdict import ( VERDICT_EXTRACTION_PROMPT_VERSION, @@ -116,5 +122,10 @@ "ProviderExecutionReceipt", "VerdictExtraction", "ProviderSkip", + # DSE-1514 bounded cost receipts + pre-flight spend gate public surface. + "PriceSnapshot", + "SpendRefused", + "SpendUnboundable", + "SpendCapExceeded", "__version__", ] diff --git a/src/conclave/council.py b/src/conclave/council.py index b1e2415..0820898 100644 --- a/src/conclave/council.py +++ b/src/conclave/council.py @@ -82,6 +82,8 @@ ) from .pricing import ( PriceSnapshot, + SpendCapExceeded, + SpendUnboundable, load_default_price_snapshot, reported_usage_cost, ) @@ -289,6 +291,17 @@ class Council: ``None`` unless set), leaving each provider's own default in place exactly as before this flag existed. A cap is the precondition for ``--max-spend-usd``: see :meth:`plan_calls`. + max_spend_usd: Opt-in pre-flight spend cap in USD (DSE-1514). When set, + every deliberation call (:meth:`ask`/:meth:`ask_stream` and their + mode wrappers) first enumerates :meth:`plan_calls`, prices it via + :meth:`_reserve_plan`, and raises :class:`conclave.pricing. + SpendCapExceeded` -- BEFORE any provider call -- when the reserved + total exceeds this cap. Requires ``max_output_tokens`` (explicit or + via config): an unbounded output cannot be bounded in dollars, so + setting this without a cap raises :class:`conclave.pricing. + SpendUnboundable` at construction time rather than at the first + call. ``None`` (the default) installs no gate at all -- byte- + identical to today. Example: >>> council = Council(models=["grok", "perplexity"], synthesizer="claude") @@ -308,6 +321,7 @@ def __init__( allow_transport_debug_logging: bool = False, source_bundle_digest: str | None = None, max_output_tokens: int | None = None, + max_spend_usd: Decimal | None = None, ) -> None: self.config = config or load_config() self.requested_models = list(models) @@ -343,6 +357,12 @@ def __init__( self.max_output_tokens = ( self.config.max_output_tokens if max_output_tokens is None else max_output_tokens ) + # A spend cap without an output cap is not enforceable: output is the + # unbounded term. Refuse at construction rather than at the first call, + # so a library caller cannot get halfway into a run before finding out. + self.max_spend_usd = max_spend_usd + if max_spend_usd is not None and self.max_output_tokens is None: + raise SpendUnboundable(_NO_OUTPUT_CAP_MESSAGE) @staticmethod def _resolve_chain(spec: str | Sequence[str] | None, config: ConclaveConfig) -> list[str]: @@ -575,6 +595,81 @@ def chain_calls(phase: str, *, template: str, upstream: int, contract: bool) -> chain_count=len(chain), ) + def _reserve_plan(self, plan: CallPlan) -> Decimal: + """Price a :class:`CallPlan` pessimistically against the snapshot. + + Args: + plan: The worst-case plan from :meth:`plan_calls`. + + Returns: + The reserved total in USD -- an upper bound on what the run can cost. + + Raises: + SpendUnboundable: No snapshot, or any planned call's model has no + snapshot entry. Never falls back to a similar model's rate. + """ + snapshot = load_default_price_snapshot() + if snapshot is None: + raise SpendUnboundable("cannot bound spend: price snapshot unavailable") + total = Decimal("0") + for call in plan.calls: + rates = snapshot.rates_for(call.model_id) + if rates is None: + raise SpendUnboundable( + f"cannot bound spend: no priced rate for {call.model_id} " + f"in snapshot {snapshot.digest()} ({snapshot.captured_at.isoformat()})" + ) + total += reserve_cost( + rates, + prompt_token_upper_bound=call.prompt_token_upper_bound, + prompt_template_token_allowance=call.prompt_template_token_allowance, + provider_framing_token_allowance=call.provider_framing_token_allowance, + upstream_output_token_ceilings=( + (call.max_output_tokens,) * call.upstream_output_call_count + ), + upstream_output_bytes_per_token=rates.max_output_bytes_per_token, + max_output_tokens=call.max_output_tokens, + ).reserved_cost_usd + return total + + def _enforce_spend_cap( + self, + mode: str, + prompt: str, + *, + rounds: int | None = None, + proposer: str | None = None, + choices: list[str] | None = None, + ) -> None: + """Refuse an over-budget or unboundable run BEFORE any provider call. + + A no-op when ``max_spend_usd`` is unset, so a run with no spend flags is + byte-identical to today. Deliberately NOT applied to a cache hit: a hit + makes no provider call and therefore cannot exceed any cap. + + Raises: + SpendUnboundable: The plan cannot be priced. + SpendCapExceeded: The priced plan exceeds the cap. + """ + if self.max_spend_usd is None: + return + plan = self.plan_calls( + mode, + prompt, + rounds=2 if rounds is None else rounds, + proposer=proposer, + choices=choices, + ) + reserved = self._reserve_plan(plan) + if reserved > self.max_spend_usd: + raise SpendCapExceeded(reserved, self.max_spend_usd, len(plan.calls)) + logger.info( + "spend gate: reserved %s USD for %d calls, under the %s USD cap", + reserved, + len(plan.calls), + self.max_spend_usd, + ) + def _cache_key( self, prompt: str, @@ -679,6 +774,7 @@ async def _cached_run( re-running would not produce a different, better answer. """ if not self.cache_enabled: + self._enforce_spend_cap(mode, prompt, rounds=rounds, proposer=proposer, choices=choices) result = await run() self._ensure_manifest(result, mode) self._price_manifest(result) @@ -699,6 +795,7 @@ async def _cached_run( self._price_manifest(hit) return hit + self._enforce_spend_cap(mode, prompt, rounds=rounds, proposer=proposer, choices=choices) result = await run() self._ensure_manifest(result, mode) self._price_manifest(result) @@ -1289,6 +1386,7 @@ async def ask_stream(self, prompt: str, synthesize: bool = True) -> AsyncIterato # Live miss: stream, capture the terminal result, then store it # (no-store on primary infrastructure failure -- see the docstring). + self._enforce_spend_cap(mode, prompt) final: CouncilResult | None = None async for event in stream_ask(self, prompt, synthesize=synthesize): if event.type == "done" and event.result is not None: @@ -1306,6 +1404,7 @@ async def ask_stream(self, prompt: str, synthesize: bool = True) -> AsyncIterato cache_mod.store(key, final) return + self._enforce_spend_cap(mode, prompt) async for event in stream_ask(self, prompt, synthesize=synthesize): yield event diff --git a/src/conclave/pricing.py b/src/conclave/pricing.py index 363b11b..c7ab600 100644 --- a/src/conclave/pricing.py +++ b/src/conclave/pricing.py @@ -438,3 +438,40 @@ def load_default_price_snapshot() -> PriceSnapshot | None: except (OSError, json.JSONDecodeError, ValidationError, TypeError) as exc: logger.warning("price snapshot %s is unusable: %s; pricing disabled", path.name, exc) return None + + +class SpendRefused(Exception): + """Base class for a pre-flight spend refusal. + + Raised BEFORE any provider call. Every subclass maps to CLI exit code 4. + Refusing is the honest outcome when a run cannot be bounded: the alternative + is to invent a number, which is the exact failure this module exists to + prevent. + """ + + +class SpendUnboundable(SpendRefused): + """The call plan cannot be priced at all, so no cap can be enforced. + + Causes: no output cap configured, no price snapshot available, or a model in + the plan with no snapshot entry. Never a fallback rate, never a guess. + """ + + +class SpendCapExceeded(SpendRefused): + """The fully-priced worst-case plan reserves more than the stated cap. + + Attributes: + reserved: The pessimistic total for the whole plan, in USD. + cap: The operator's stated cap, in USD. + call_count: How many provider calls the plan enumerated. + """ + + def __init__(self, reserved: Decimal, cap: Decimal, call_count: int) -> None: + self.reserved = reserved + self.cap = cap + self.call_count = call_count + super().__init__( + f"refusing to run: reserved {reserved} USD for {call_count} calls " + f"exceeds the cap of {cap} USD" + ) diff --git a/tests/test_spend_gate.py b/tests/test_spend_gate.py new file mode 100644 index 0000000..6ab1d50 --- /dev/null +++ b/tests/test_spend_gate.py @@ -0,0 +1,158 @@ +"""The pre-flight spend gate refuses BEFORE the first provider call (DSE-1514).""" + +from __future__ import annotations + +from decimal import Decimal + +import pytest + +from conclave.council import Council +from conclave.pricing import SpendCapExceeded, SpendRefused, SpendUnboundable +from tests.test_pricing_receipts import _install_snapshot, _snapshot + + +def test_a_spend_cap_without_an_output_cap_is_refused_at_construction(keys): + with pytest.raises(SpendUnboundable) as excinfo: + Council(models=["grok"], synthesizer="grok", max_spend_usd=Decimal("0.40")) + assert str(excinfo.value) == ( + "cannot bound spend: no output cap (set --max-output-tokens or config max_output_tokens)" + ) + assert isinstance(excinfo.value, SpendRefused) + + +async def test_an_over_budget_plan_refuses_before_any_provider_call(monkeypatch, keys): + import conclave.council as council_mod + + _install_snapshot( + monkeypatch, + _snapshot("xai/grok-4.3", "gemini/gemini-2.5-pro", "anthropic/claude-sonnet-4-6"), + ) + calls: list[str] = [] + + async def tripwire(name, model_id, messages, **kwargs): + calls.append(name) + raise AssertionError("the gate must refuse before any provider call") + + monkeypatch.setattr(council_mod, "call_model", tripwire) + council = Council( + models=["grok", "gemini"], + synthesizer="claude", + max_output_tokens=100_000, + max_spend_usd=Decimal("0.000001"), + ) + with pytest.raises(SpendCapExceeded) as excinfo: + await council.ask("q") + + error = excinfo.value + assert error.cap == Decimal("0.000001") + assert error.reserved > error.cap + assert error.call_count == 2 + 1 + 2 + assert "reserved" in str(error) and "cap" in str(error) and "calls" in str(error) + assert calls == [] + + +async def test_an_under_budget_plan_runs_normally(monkeypatch, keys, patch_call_model): + from tests.conftest import make_response + + _install_snapshot(monkeypatch, _snapshot("xai/grok-4.3")) + patch_call_model(lambda model_id, messages: make_response("ok")) + council = Council( + models=["grok"], + synthesizer="grok", + max_output_tokens=64, + max_spend_usd=Decimal("100.00"), + extract_verdict=False, + ) + result = await council.ask("q", synthesize=False) + assert result.successful_answers + assert result.manifest.cost_ceiling_usd is not None + + +async def test_an_unpriced_model_in_the_plan_refuses_rather_than_guessing(monkeypatch, keys): + import conclave.council as council_mod + + # grok is priced; the claude synthesizer is not. + _install_snapshot(monkeypatch, _snapshot("xai/grok-4.3")) + + async def tripwire(name, model_id, messages, **kwargs): + raise AssertionError("no call may happen") + + monkeypatch.setattr(council_mod, "call_model", tripwire) + council = Council( + models=["grok"], + synthesizer="claude", + max_output_tokens=64, + max_spend_usd=Decimal("100.00"), + ) + with pytest.raises(SpendUnboundable, match="no priced rate for anthropic/claude-sonnet-4-6"): + await council.ask("q") + + +async def test_a_missing_snapshot_refuses_the_gate(monkeypatch, keys): + _install_snapshot(monkeypatch, None) + council = Council( + models=["grok"], + synthesizer="grok", + max_output_tokens=64, + max_spend_usd=Decimal("100.00"), + ) + with pytest.raises(SpendUnboundable, match="price snapshot unavailable"): + await council.ask("q", synthesize=False) + + +async def test_a_cache_hit_is_never_gated(monkeypatch, tmp_path, keys, patch_call_model): + from tests.conftest import make_response + + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path)) + _install_snapshot(monkeypatch, _snapshot("xai/grok-4.3")) + patch_call_model(lambda model_id, messages: make_response("ok")) + + cheap = Council( + models=["grok"], + synthesizer="grok", + max_output_tokens=64, + max_spend_usd=Decimal("100.00"), + cache=True, + extract_verdict=False, + ) + await cheap.ask("q", synthesize=False) + + # Same identity, an impossible cap: the hit costs nothing, so it is served. + strict = Council( + models=["grok"], + synthesizer="grok", + max_output_tokens=64, + max_spend_usd=Decimal("0.000001"), + cache=True, + extract_verdict=False, + ) + hit = await strict.ask("q", synthesize=False) + assert hit.cached is True + + +async def test_the_gate_also_guards_the_streaming_path(monkeypatch, keys): + import conclave.streaming as streaming_mod + + _install_snapshot(monkeypatch, _snapshot("xai/grok-4.3")) + + async def tripwire(*args, **kwargs): + raise AssertionError("no stream may start") + yield # pragma: no cover + + monkeypatch.setattr(streaming_mod, "call_model_stream", tripwire) + council = Council( + models=["grok"], + synthesizer="grok", + max_output_tokens=100_000, + max_spend_usd=Decimal("0.000001"), + extract_verdict=False, + ) + with pytest.raises(SpendCapExceeded): + async for _event in council.ask_stream("q", synthesize=False): + pass + + +def test_no_spend_flags_means_no_gate_at_all(keys): + council = Council(models=["grok"], synthesizer="grok") + assert council.max_spend_usd is None + assert council.max_output_tokens is None From 41884464b462ce5ab04f3ac2d8a288e33d073489 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 4 Sep 2026 16:14:54 -0400 Subject: [PATCH 04/10] feat(cli): --max-output-tokens, --max-spend-usd, refusal exit code 4 (DSE-1514) Claude-Session: https://claude.ai/code/session_01K1dHPjZ1bZcE2GnX3KMMSH --- src/conclave/cli.py | 261 ++++++++++++++++++++++++++++---------------- tests/test_cli.py | 134 +++++++++++++++++++++++ 2 files changed, 303 insertions(+), 92 deletions(-) diff --git a/src/conclave/cli.py b/src/conclave/cli.py index 381cddd..224befb 100644 --- a/src/conclave/cli.py +++ b/src/conclave/cli.py @@ -16,6 +16,7 @@ import json import os import tempfile +from decimal import Decimal, InvalidOperation from pathlib import Path import typer @@ -28,6 +29,7 @@ from .council import Council from .eval_cli import app as eval_app from .models import CouncilResult, StreamEvent +from .pricing import SpendCapExceeded, SpendRefused from .registry import DEFAULT_MODELS, key_present, key_source app = typer.Typer( @@ -507,6 +509,14 @@ def _render_failover_note(result: CouncilResult) -> None: # any exit code it already handles. _DEGRADED_EXIT_CODE = 3 +# Distinct exit code for a pre-flight spend REFUSAL (DSE-1514): the run was +# never started because its worst-case cost could not be bounded, or was bounded +# and exceeded --max-spend-usd. Kept apart from 1 (nothing usable came back), 2 +# (usage error), and 3 (degraded) because it is categorically different: nothing +# ran, nothing was spent, and retrying with a higher cap or an output cap is the +# fix. A caller doing `echo $?` adds one branch and reinterprets nothing. +_SPEND_REFUSED_EXIT_CODE = 4 + def _resolve_converge_threshold( converge: bool | None, @@ -628,6 +638,26 @@ def ask( "hit the cached text is rendered in one shot (no live token stream)." ), ), + max_output_tokens: int | None = typer.Option( + None, + "--max-output-tokens", + help=( + "Hard ceiling on output tokens for every call this run makes " + "(members, synthesizer, judge, verdict extraction and its repair). " + "Defers to config `max_output_tokens` when unset. Required by " + "--max-spend-usd: unbounded output cannot be bounded in dollars." + ), + ), + max_spend_usd: str | None = typer.Option( + None, + "--max-spend-usd", + help=( + "Refuse the run BEFORE the first provider call if its worst-case " + "call plan reserves more than this many USD, priced against the " + "packaged dated snapshot at ceiling rates. Exits 4 on refusal. " + "Requires --max-output-tokens (or config max_output_tokens)." + ), + ), ) -> None: """Run one of six council modes over PROMPT: synthesize, raw, debate, adversarial, vote, or elite. @@ -660,6 +690,13 @@ def ask( (DSE-1512), this now also means the WHOLE chain was exhausted -- every candidate failed for an infrastructure reason -- rather than just the lone synthesizer failing. + * 4 -- the run was REFUSED before any provider call (DSE-1514): either its + worst-case call plan reserved more than ``--max-spend-usd`` (the message + names the reserved total, the cap, and the call count), or the plan could + not be bounded at all -- no output cap, no price snapshot, or a model with + no snapshot entry. Nothing ran and nothing was spent. Distinct from 3 + (degraded): a degraded run happened and produced partial output; a refused + run never started. ``--json`` also carries the top-level ``"primary_failed_over"`` field (DSE-1512, additive): ``true`` when, for any role, the declared primary @@ -697,115 +734,155 @@ def ask( ) raise typer.Exit(code=2) + # --max-spend-usd is typed str on purpose: typer's float coercion would + # destroy the exactness the whole cap rests on (Decimal(0.4) != Decimal("0.4")). + spend_cap: Decimal | None = None + if max_spend_usd is not None: + try: + spend_cap = Decimal(max_spend_usd) + except InvalidOperation: + err_console.print( + f"[red]--max-spend-usd must be an exact decimal amount, got '{max_spend_usd}'.[/red]" + ) + raise typer.Exit(code=2) from None + if spend_cap <= 0: + err_console.print("[red]--max-spend-usd must be greater than zero.[/red]") + raise typer.Exit(code=2) + cfg = load_config() members = cfg.resolve_council(council) if not members: err_console.print(f"[red]No council members resolved from '{council}'.[/red]") raise typer.Exit(code=2) - c = Council(models=members, synthesizer=synthesizer, config=cfg, cache=cache) + try: + c = Council( + models=members, + synthesizer=synthesizer, + config=cfg, + cache=cache, + max_output_tokens=max_output_tokens, + max_spend_usd=spend_cap, + ) + except SpendRefused as refusal: + err_console.print(f"[red]{refusal}[/red]") + raise typer.Exit(code=_SPEND_REFUSED_EXIT_CODE) from None + + # Refusal happens at construction (no output cap for a spend cap) or + # inside the mode dispatch below (an over-budget or unpriceable plan) -- + # both map to the same exit code so a caller checking exit-code alone + # cannot mistake a refused run for a clean or degraded one. + try: + # Streaming path: live token output (synthesize/raw only, not with --json). + # It produces the same final CouncilResult, so the exit-code contract below + # applies identically. + if stream and not as_json: + result = _stream_to_terminal(c, prompt, synthesize=(mode_lower == "synthesize")) + _render_failover_note(result) + if not result.successful_answers: + err_console.print( + "[red]No usable council answers. Run 'conclave providers' to check keys.[/red]" + ) + raise typer.Exit(code=1) + if result.degraded: + # _stream_to_terminal already printed the "No synthesis: ..." warning + # (from synthesis_error) to stderr; only the exit code is new here. + raise typer.Exit(code=_DEGRADED_EXIT_CODE) + return + + if mode_lower == "debate": + threshold = _resolve_converge_threshold( + converge, converge_threshold, cfg.converge_threshold + ) + result = c.debate_sync(prompt, rounds=rounds, converge_threshold=threshold) + elif mode_lower == "adversarial": + result = c.adversarial_sync(prompt, proposer=proposer) + elif mode_lower == "vote": + choice_list = [ch.strip() for ch in (choices or "").split(",") if ch.strip()] + result = c.vote_sync(prompt, choices=choice_list) + elif mode_lower == "elite": + result = c.elite_sync(prompt) + else: + result = c.ask_sync(prompt, synthesize=(mode_lower == "synthesize")) + + # A run that produced no usable member answers is a failure for scripting + # purposes regardless of output format. We compute this once and apply the + # same exit-code contract to both the JSON and human paths. A run that DID + # get usable member answers but whose judge/synthesizer step failed is a + # distinct, less severe failure (DSE-901): ``result.degraded`` (checked + # below, after the hard-failure/usage-error exits) drives exit code + # ``_DEGRADED_EXIT_CODE`` instead of silently returning 0. + no_usable_answers = not result.successful_answers + elite_not_ready = mode_lower == "elite" and ( + result.elite is None or result.elite.decision_readiness != "ready" + ) - # Streaming path: live token output (synthesize/raw only, not with --json). - # It produces the same final CouncilResult, so the exit-code contract below - # applies identically. - if stream and not as_json: - result = _stream_to_terminal(c, prompt, synthesize=(mode_lower == "synthesize")) - _render_failover_note(result) - if not result.successful_answers: + payload = _result_to_dict(result) if as_json or json_output is not None else None + json_output_failed = False + if json_output is not None: + try: + _write_json_output(json_output, payload or {}) + except Exception: + # The completed result must remain available on its normal stdout path + # even when the optional persistence side effect fails. + err_console.print("[red]Could not write --json-output.[/red]") + json_output_failed = True + + if as_json: + # Always emit valid JSON to stdout so a consumer can parse the payload, + # then signal failure via the exit code if nothing usable came back. + console.print_json(json.dumps(payload)) + if json_output_failed or no_usable_answers or elite_not_ready: + raise typer.Exit(code=1) + if result.degraded: + raise typer.Exit(code=_DEGRADED_EXIT_CODE) + return + + if mode_lower == "elite": + if result.elite is None: + err_console.print("[red]Elite decision not ready: missing result[/red]") + raise typer.Exit(code=1) + _render_elite(result) + _render_failover_note(result) + if not result.elite.completed: + reason = result.elite.failure_reason or ", ".join(result.elite.readiness_reasons) + err_console.print(f"[red]Elite protocol incomplete: {reason}[/red]") + raise typer.Exit(code=1) + if result.elite.decision_readiness != "ready": + reasons = ", ".join(result.elite.readiness_reasons) or "no reason recorded" + err_console.print( + "[red]Elite decision not ready: " + f"{result.elite.decision_readiness} ({reasons})[/red]" + ) + raise typer.Exit(code=1) + if json_output_failed: + raise typer.Exit(code=1) + return + + if no_usable_answers: err_console.print( "[red]No usable council answers. Run 'conclave providers' to check keys.[/red]" ) raise typer.Exit(code=1) - if result.degraded: - # _stream_to_terminal already printed the "No synthesis: ..." warning - # (from synthesis_error) to stderr; only the exit code is new here. - raise typer.Exit(code=_DEGRADED_EXIT_CODE) - return - - if mode_lower == "debate": - threshold = _resolve_converge_threshold( - converge, converge_threshold, cfg.converge_threshold - ) - result = c.debate_sync(prompt, rounds=rounds, converge_threshold=threshold) - elif mode_lower == "adversarial": - result = c.adversarial_sync(prompt, proposer=proposer) - elif mode_lower == "vote": - choice_list = [ch.strip() for ch in (choices or "").split(",") if ch.strip()] - result = c.vote_sync(prompt, choices=choice_list) - elif mode_lower == "elite": - result = c.elite_sync(prompt) - else: - result = c.ask_sync(prompt, synthesize=(mode_lower == "synthesize")) - - # A run that produced no usable member answers is a failure for scripting - # purposes regardless of output format. We compute this once and apply the - # same exit-code contract to both the JSON and human paths. A run that DID - # get usable member answers but whose judge/synthesizer step failed is a - # distinct, less severe failure (DSE-901): ``result.degraded`` (checked - # below, after the hard-failure/usage-error exits) drives exit code - # ``_DEGRADED_EXIT_CODE`` instead of silently returning 0. - no_usable_answers = not result.successful_answers - elite_not_ready = mode_lower == "elite" and ( - result.elite is None or result.elite.decision_readiness != "ready" - ) - - payload = _result_to_dict(result) if as_json or json_output is not None else None - json_output_failed = False - if json_output is not None: - try: - _write_json_output(json_output, payload or {}) - except Exception: - # The completed result must remain available on its normal stdout path - # even when the optional persistence side effect fails. - err_console.print("[red]Could not write --json-output.[/red]") - json_output_failed = True - - if as_json: - # Always emit valid JSON to stdout so a consumer can parse the payload, - # then signal failure via the exit code if nothing usable came back. - console.print_json(json.dumps(payload)) - if json_output_failed or no_usable_answers or elite_not_ready: - raise typer.Exit(code=1) - if result.degraded: - raise typer.Exit(code=_DEGRADED_EXIT_CODE) - return - if mode_lower == "elite": - if result.elite is None: - err_console.print("[red]Elite decision not ready: missing result[/red]") - raise typer.Exit(code=1) - _render_elite(result) + _RENDERERS[result.mode](result) _render_failover_note(result) - if not result.elite.completed: - reason = result.elite.failure_reason or ", ".join(result.elite.readiness_reasons) - err_console.print(f"[red]Elite protocol incomplete: {reason}[/red]") - raise typer.Exit(code=1) - if result.elite.decision_readiness != "ready": - reasons = ", ".join(result.elite.readiness_reasons) or "no reason recorded" - err_console.print( - "[red]Elite decision not ready: " - f"{result.elite.decision_readiness} ({reasons})[/red]" - ) - raise typer.Exit(code=1) if json_output_failed: raise typer.Exit(code=1) - return - - if no_usable_answers: + if result.degraded: + # The mode-specific renderer above already printed the "No synthesis: ..." + # / "No verdict: ..." warning (from synthesis_error / verdict_error) to + # stderr; only the exit code is new here (DSE-901). + raise typer.Exit(code=_DEGRADED_EXIT_CODE) + except SpendCapExceeded as refusal: err_console.print( - "[red]No usable council answers. Run 'conclave providers' to check keys.[/red]" + f"[red]{refusal}. Raise --max-spend-usd, lower --max-output-tokens, " + f"or shrink the council.[/red]" ) - raise typer.Exit(code=1) - - _RENDERERS[result.mode](result) - _render_failover_note(result) - if json_output_failed: - raise typer.Exit(code=1) - if result.degraded: - # The mode-specific renderer above already printed the "No synthesis: ..." - # / "No verdict: ..." warning (from synthesis_error / verdict_error) to - # stderr; only the exit code is new here (DSE-901). - raise typer.Exit(code=_DEGRADED_EXIT_CODE) + raise typer.Exit(code=_SPEND_REFUSED_EXIT_CODE) from None + except SpendRefused as refusal: + err_console.print(f"[red]{refusal}[/red]") + raise typer.Exit(code=_SPEND_REFUSED_EXIT_CODE) from None @app.command() diff --git a/tests/test_cli.py b/tests/test_cli.py index b74782f..b4e6ff5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1005,3 +1005,137 @@ def test_cli_providers_footer_shows_chain(monkeypatch, tmp_path): result = runner.invoke(cli.app, ["providers"]) assert result.exit_code == 0 assert "synthesizer chain: claude > grok" in result.output + + +"""DSE-1514: --max-output-tokens / --max-spend-usd and the exit-code-4 refusal.""" + + +def test_spend_refusal_exit_code_is_four_and_distinct(): + from conclave import cli + + assert cli._SPEND_REFUSED_EXIT_CODE == 4 + assert cli._SPEND_REFUSED_EXIT_CODE != cli._DEGRADED_EXIT_CODE + + +def test_spend_cap_without_an_output_cap_exits_four(keys): + from conclave.cli import app + + result = runner.invoke(app, ["ask", "q", "--council", "grok", "--max-spend-usd", "0.40"]) + assert result.exit_code == 4 + assert "cannot bound spend: no output cap" in result.output + (result.stderr or "") + + +def test_an_over_budget_run_exits_four_and_names_reserved_cap_and_count(monkeypatch, keys): + import conclave.council as council_mod + from conclave.cli import app + from tests.test_pricing_receipts import _install_snapshot, _snapshot + + _install_snapshot(monkeypatch, _snapshot("xai/grok-4.3", "anthropic/claude-sonnet-4-6")) + + async def tripwire(name, model_id, messages, **kwargs): + raise AssertionError("the CLI gate must refuse before any provider call") + + monkeypatch.setattr(council_mod, "call_model", tripwire) + result = runner.invoke( + app, + [ + "ask", + "q", + "--council", + "grok", + "--max-output-tokens", + "100000", + "--max-spend-usd", + "0.000001", + ], + ) + combined = result.output + (result.stderr or "") + assert result.exit_code == 4 + assert "reserved" in combined and "0.000001" in combined and "calls" in combined + + +def test_an_unboundable_plan_exits_four_with_a_distinct_message(monkeypatch, keys): + from conclave.cli import app + from tests.test_pricing_receipts import _install_snapshot, _snapshot + + # grok priced, the claude synthesizer is not. + _install_snapshot(monkeypatch, _snapshot("xai/grok-4.3")) + result = runner.invoke( + app, + [ + "ask", + "q", + "--council", + "grok", + "--max-output-tokens", + "512", + "--max-spend-usd", + "10.00", + ], + ) + combined = result.output + (result.stderr or "") + assert result.exit_code == 4 + assert "no priced rate for anthropic/claude-sonnet-4-6" in combined + assert "reserved" not in combined + + +def test_a_non_numeric_spend_cap_is_a_usage_error(keys): + from conclave.cli import app + + result = runner.invoke( + app, ["ask", "q", "--council", "grok", "--max-spend-usd", "cheap-please"] + ) + assert result.exit_code == 2 + assert "--max-spend-usd" in result.output + (result.stderr or "") + + +def test_the_spend_cap_is_parsed_as_an_exact_decimal_never_a_float(monkeypatch, keys): + """0.4 through a float is 0.4000000000000000222; the cap must be exact.""" + from decimal import Decimal + + import conclave.cli as cli_mod + from conclave.cli import app + + seen: dict[str, object] = {} + real = cli_mod.Council + + def spy(*args, **kwargs): + seen.update(kwargs) + return real(*args, **kwargs) + + monkeypatch.setattr(cli_mod, "Council", spy) + runner.invoke( + app, + [ + "ask", + "q", + "--council", + "grok", + "--max-output-tokens", + "512", + "--max-spend-usd", + "0.4", + ], + ) + assert seen["max_spend_usd"] == Decimal("0.4") + assert isinstance(seen["max_spend_usd"], Decimal) + assert seen["max_output_tokens"] == 512 + + +def test_the_json_payload_carries_the_ceiling_as_an_exact_string( + monkeypatch, keys, patch_call_model +): + import json as json_mod + + from conclave.cli import app + from tests.conftest import make_response + from tests.test_pricing_receipts import _install_snapshot, _snapshot + + _install_snapshot(monkeypatch, _snapshot("xai/grok-4.3")) + patch_call_model(lambda model_id, messages: make_response("ok")) + result = runner.invoke(app, ["ask", "q", "--council", "grok", "--mode", "raw", "--json"]) + manifest = json_mod.loads(result.output)["manifest"] + assert manifest["estimated_cost"] is None + assert isinstance(manifest["cost_ceiling_usd"], str) + assert manifest["priced_as_of"] == "2026-09-03" + assert manifest["price_snapshot_digest"].startswith("sha256:") From dab98e5c511ebcc8fb837a2c615296c5d099e87d Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 4 Sep 2026 17:05:22 -0400 Subject: [PATCH 05/10] fix(council): adversarial critics and judge embed upstream output in the call plan; table-driven plan_calls; one spend-gate chokepoint (DSE-1514 review) Fix A (Critical) -- adversarial input bounds. Council.plan_calls now emits the byte-worst-case adversarial shape: 1 proposer call (upstream=0) plus N-1 critic calls (upstream=1 each, embedding the proposal's answer via CRITIC_SYSTEM + critic_user), then a judge call per keyed chain candidate (upstream=N, embedding the proposal and every critique via JUDGE_SYSTEM + judge_user). Total member calls remains exactly N for every k (proposer attempts + critics), so the mode's call-count arithmetic (N + C) is unchanged -- only the per-call phase/upstream attribution corrects. Byte-lower-bound regression (Fix A tests, tests/test_spend_plan.py): building the REAL messages for critique/synthesis/debate-round-2/elite critique+revision/verdict-extraction+repair with worst-case-length upstream text exposed three additional pre-existing undercounts in the shipped Task 10 template arithmetic, now fixed: - The "synthesis" phase's template allowance was the bare system prompt only; council._synth_user_content is now a reusable extraction of Council._synthesize's real user-content builder, and plan_calls measures it with real member placeholders (conclave.council._placeholder_answers). - verdict_synthesis.VERDICT_CONTRACT_BYTES compact-encoded the extraction schema while the real prompt embeds it indent=2 (larger), and neither counted the fixed wrapper prose or per-member answer labels. plan_calls now measures the verdict_extraction/verdict_repair templates by calling verdict_synthesis._build_messages/_repair_instruction directly with real member placeholders; the now-unused VERDICT_CONTRACT_BYTES/ VERDICT_TEMPLATE_PROBE module constants are removed. - Elite's "revision" phase undercounted upstream at 2N: every reviser's own initial answer is embedded TWICE in the real prompt (once standalone as "original answer", once again inside the anonymized initial panel -- modes._elite_revision_messages_for). Upstream is now 2N+1. The one existing assertion this changes is documented below. - Debate round >=2 and elite critique/revision templates now measure their real per-member label overhead via placeholder answers instead of an empty-list/empty-string approximation. Fix B (Important) -- table-driven plan_calls. Council._plan_table returns a declarative list of _PhaseSpec rows (phase, targets, template, upstream, message_count, contract) built from N/C/R/V; Council.plan_calls is now one small expansion loop instead of two parallel if/elif chains. Per-mode call counts are unchanged: raw N, vote N, synthesize N+C+2CV, debate N*R+C, adversarial N+C, elite 3N+C+2CV. Fix C (Important) -- one gate chokepoint. New Council._gate_live_run is the single pre-flight spend-gate call site per entry point: _cached_run and ask_stream each call it exactly once, after the cache-hit decision and before the first provider call, replacing four call sites (two per entry point) of _enforce_spend_cap with one per entry point. Behaviour is unchanged: a cache hit never reserves; a live run always reserves first. Minors: - config.py: fixed the max_output_tokens docstring entry's indentation (was nested three spaces under converge_threshold's continuation). - tests/test_cli.py: the over-budget refusal test now asserts the council seam made zero calls explicitly (calls == []), not just that a tripwire assertion never fired. - New test: --json + an over-budget refusal emits nothing on stdout (result.stdout == "") and the refusal message on stderr, exit 4. The `ask` command docstring's exit-4 note already covers this; verified this round rather than re-documented, since the mode dispatch that would build the JSON payload never runs when the gate raises first. Pre-existing assertion adjusted: tests/test_spend_plan.py:: test_downstream_phases_declare_their_upstream_dependencies -- the "revision" phase's upstream_output_call_count assertion changes from 6 (2N) to 7 (2N+1), per the elite-revision duplication fix above. 919 passed (909 baseline + 10 new). ruff check/format clean. Claude-Session: https://claude.ai/code/session_01K1dHPjZ1bZcE2GnX3KMMSH --- src/conclave/config.py | 2 +- src/conclave/council.py | 617 ++++++++++++++++++++++-------- src/conclave/verdict_synthesis.py | 55 +-- tests/test_cli.py | 42 ++ tests/test_spend_plan.py | 194 +++++++++- 5 files changed, 730 insertions(+), 180 deletions(-) diff --git a/src/conclave/config.py b/src/conclave/config.py index e77069f..7f2c844 100644 --- a/src/conclave/config.py +++ b/src/conclave/config.py @@ -83,7 +83,7 @@ class ConclaveConfig(BaseModel): reaches the threshold. ``None`` keeps the historic fixed-rounds behavior exactly. A ``--converge-threshold`` / ``--converge/--no-converge`` CLI flag overrides this per invocation. See :func:`conclave.modes.run_debate`. - max_output_tokens: opt-in hard ceiling on output tokens for EVERY call a + max_output_tokens: opt-in hard ceiling on output tokens for EVERY call a council makes -- members, synthesis, judge, verdict extraction and its repair retry, and the streaming paths. ``None`` (the default) leaves each provider's own default in place, exactly as today. It is diff --git a/src/conclave/council.py b/src/conclave/council.py index 0820898..1285cd0 100644 --- a/src/conclave/council.py +++ b/src/conclave/council.py @@ -90,8 +90,6 @@ from .prompts import ELITE_PROMPT_VERSION, SYNTHESIS_PROMPT_VERSION from .providers import call_model, receipt_from_answer from .registry import key_present -from .verdict_synthesis import VERDICT_CONTRACT_BYTES as _VERDICT_CONTRACT_BYTES -from .verdict_synthesis import VERDICT_TEMPLATE_PROBE as _VERDICT_TEMPLATE_PROBE if TYPE_CHECKING: # avoid an import cycle at runtime; only needed for typing from .verdict_synthesis import VerdictSynthesisResult @@ -119,6 +117,63 @@ "Do not invent a model's position; rely only on the answers provided." ) + +def _synth_user_content(prompt: str, answers: Sequence[ModelAnswer]) -> str: + """Build the synthesizer's user-message content from prompt + member answers. + + Extracted from :meth:`Council._synthesize` (unchanged behavior, same + string) so :meth:`Council._plan_table` can measure the EXACT fixed + wrapper/label bytes a real synthesis call embeds -- via + :func:`_placeholder_answers`, empty-text answers with the real names this + run would use -- instead of a hand-duplicated approximation that could + silently drift from the real prompt (DSE-1514 review, Fix A). + """ + blocks = "\n\n".join( + f"### Answer from {a.name} ({a.model_id})" + f"{f' (Answer ID: {a.answer_id})' if a.answer_id else ''}\n{a.answer}" + for a in answers + ) + return ( + f"Original prompt:\n{prompt}\n\n" + f"Council answers:\n\n{blocks}\n\n" + "Now produce the consolidated answer." + ) + + +# Conservative byte allowance for an optional answer_id in a template probe +# (DSE-1514 review, Fix A). A real id is "ca_" + 24 hex chars (27 bytes), or, +# for a phase-derived artifact (:func:`conclave.models.derive_phase_answer_id`), +# "ca__" + 24 hex chars -- a few bytes longer for any phase name +# conclave uses today. A placeholder at least this long makes a template +# probe's label byte count an upper bound of BOTH the with-id and the +# without-id (positional-fallback) real cases, never an under-count. +_ANSWER_ID_PROBE = "ca_" + ("f" * 45) + + +def _placeholder_answers(members: Sequence[tuple[str, str]]) -> list[ModelAnswer]: + """Zero-length, worst-case-id answers for measuring a template's real fixed bytes. + + Used only by :meth:`Council._plan_table` to measure a phase's fixed + per-item label overhead (member name, model id, and/or an optional answer + id) EXACTLY -- using the real names/ids the run would actually use -- + rather than approximating it with a guessed constant. The answer TEXT is + deliberately left empty: that variable part is bounded separately by the + per-call ``upstream_output_call_count`` times the output cap. + + Args: + members: The ``(friendly_name, model_id)`` pairs a template needs one + placeholder answer per. + + Returns: + One placeholder :class:`~conclave.models.ModelAnswer` per member, in + the same order. + """ + return [ + ModelAnswer(name=name, model_id=model_id, answer="", answer_id=_ANSWER_ID_PROBE) + for name, model_id in members + ] + + # The modes Council.plan_calls knows how to enumerate (DSE-1514). Kept as its own # frozenset (rather than re-deriving from _RENDERERS or similar) so the planner's # contract is explicit and independent of any CLI-only vocabulary. @@ -219,6 +274,38 @@ class PlannedCall: upstream_output_call_count: int max_output_tokens: int + def input_bytes_bound(self, *, upstream_output_bytes_per_token: int) -> int: + """Total planned input-byte bound for one candidate rate (DSE-1514 review, Fix A). + + Mirrors the input-side arithmetic :meth:`Council._reserve_plan` feeds + into :func:`conclave.pricing.reserve_cost` byte for byte, so a test can + assert a real, unplanned message list never exceeds what this call was + priced for without duplicating (and risking drifting from) the pricing + module's own formula. + + Args: + upstream_output_bytes_per_token: The priced model's attested + upper bound on one output token's UTF-8 byte length (a + :class:`conclave.pricing.PriceRates` field) -- the same value + :meth:`Council._reserve_plan` reads off the snapshot entry. + + Returns: + The upper bound, in bytes, on everything this call's input could + contain: the exact known prompt, the fixed template wording, the + provider framing allowance, and every upstream call's output cap + converted to bytes. + """ + return ( + self.prompt_token_upper_bound + + self.prompt_template_token_allowance + + self.provider_framing_token_allowance + + ( + self.upstream_output_call_count + * self.max_output_tokens + * upstream_output_bytes_per_token + ) + ) + @dataclass(frozen=True) class CallPlan: @@ -230,6 +317,47 @@ class CallPlan: chain_count: int +@dataclass(frozen=True) +class _PhaseSpec: + """One declarative row of a mode's worst-case call table (DSE-1514 review, Fix B). + + :meth:`Council._plan_table` returns one of these per phase a mode could + run; :meth:`Council.plan_calls` expands each row into one + :class:`PlannedCall` per target, using the exact prompt bytes and the + output cap (the two values every row needs but none of them determine). + ``targets`` is the literal slice of keyed members or keyed chain + candidates this row calls -- a plain list, not a count -- so the SAME row + shape covers both a uniform "every member" phase (``targets`` is every + keyed member) and adversarial's split single-proposer / + ``(N-1)``-critics shape (two rows, each a different slice of ``members``). + + Attributes: + phase: The manifest phase this call would be recorded under. + targets: The exact ``(name, model_id)`` pairs this row calls. + template: The fixed system + user template wording that will + surround the exact prompt bytes. + upstream: How many upstream calls' not-yet-produced output this + call's input embeds. + message_count: Messages this call sends, feeding the provider framing + allowance (``64 + 16 * message_count``, mirroring the eval + runner). Member-shaped phases send 2 (system + user); chain-shaped + phases send 3 (system + user + the assembled upstream material). + contract: Whether a structured-output contract is attached. Adds a + flat 256-byte provider framing allowance for the schema + registration itself; the schema's own byte cost is measured as + part of ``template`` (built via + :func:`conclave.verdict_synthesis._build_messages`, real member + placeholders, in :meth:`Council._plan_table`), never added twice. + """ + + phase: str + targets: Sequence[tuple[str, str]] + template: str + upstream: int + message_count: int = 2 + contract: bool = False + + class Council: """A council of foundation models with an optional synthesizer. @@ -433,6 +561,240 @@ def _keyed_chain(self) -> list[tuple[str, str]]: pairs = [(name, self.config.resolve_model_id(name)) for name in self.synthesizer_chain] return [(name, model_id) for name, model_id in pairs if key_present(model_id)] + def _plan_table( + self, + mode: str, + *, + members: list[tuple[str, str]], + chain: list[tuple[str, str]], + rounds: int, + choices: list[str] | None, + ) -> list[_PhaseSpec]: + """Return the declarative worst-case phase table for ``mode`` (DSE-1514 review, Fix B). + + Pure data assembly: no byte arithmetic and no :class:`PlannedCall` + construction happens here -- see :meth:`plan_calls`, which expands + every row with the one piece of run-specific data every row shares + (the exact prompt bytes and the output cap). That split is what lets + all six modes share a single expansion loop instead of the two + parallel if/elif chains this table replaces. + + With ``N`` keyed members, ``C`` keyed chain candidates, ``R`` debate + rounds, and ``V`` = 1 when verdict extraction is on, the row counts + reproduce exactly: + + * ``raw`` -- ``N``: one member-phase row, no chain row. + * ``synthesize`` -- ``N + C + 2CV``: the same member-phase row, plus a + chain synthesis row and (when verdict extraction is on) extraction + + repair chain rows. + * ``vote`` -- ``N``: one member-phase row; no chain row. + * ``debate`` -- ``N*R + C``: a round-1 row plus one row per round + 2..R (each worst case at full membership; drop-out only shrinks a + real run), plus a chain consolidation row. + * ``adversarial`` -- ``N + C``: two member-phase rows -- one + proposer (``upstream=0``) and ``N-1`` critics (``upstream=1`` + each, embedding the proposal) -- whose target counts always sum to + ``N`` regardless of how many real proposer attempts fail, plus a + chain judge row whose ``upstream=N`` (it embeds the proposal and + every critique). This is the DSE-1514 review Fix A shape: byte + worst case is 1 proposer succeeding immediately, maximizing the + number of upstream-embedding critic calls. + * ``elite`` -- ``3N + C + 2CV``: three member-phase rows (initial, + critique, revision), plus the synthesis/verdict chain rows shared + with ``synthesize``. + + Args: + mode: One of ``raw``/``synthesize``/``vote``/``debate``/ + ``adversarial``/``elite``. Already validated by the caller. + members: The keyed council members. + chain: The keyed synthesizer-chain candidates. + rounds: Debate rounds, already normalized to at least 1. + choices: Vote choices, which enlarge the vote prompt template. + + Returns: + The ordered phase rows for ``mode``. + """ + n = len(members) + table: list[_PhaseSpec] = [] + # Placeholder answers (real names/model ids, empty text) for measuring + # a downstream phase's EXACT fixed per-item label overhead -- DSE-1514 + # review, Fix A. Built once per call since every downstream phase that + # embeds the full membership needs the identical N-sized placeholder + # list; the debate peer block additionally needs per-member letter + # aliases, computed separately below where it is used. + member_placeholders = _placeholder_answers(members) + + if mode in ("raw", "synthesize"): + table.append(_PhaseSpec("member", members, "", 0)) + elif mode == "vote": + table.append( + _PhaseSpec( + "member", + members, + prompts.VOTE_SYSTEM + prompts.vote_user("", choices or []), + 0, + ) + ) + elif mode == "debate": + table.append(_PhaseSpec("round-1", members, "", 0)) + # A worst-case peer block: every member's PRIOR answer anonymized + # by letter, text left empty (bounded separately via ``upstream``) + # so only the real, N-exact "Model X (peer) previous answer" / + # "Your previous answer" label overhead is measured here. + letters = {name: prompts.LETTERS[i] for i, (name, _mid) in enumerate(members)} + prior = { + name: answer + for (name, _mid), answer in zip(members, member_placeholders, strict=True) + } + self_name = members[0][0] if members else "" + peer_block = ( + prompts.anonymized_peer_block(self_name, letters.get(self_name, ""), prior, letters) + if members + else "" + ) + for round_no in range(2, rounds + 1): + table.append( + _PhaseSpec( + f"round-{round_no}", + members, + prompts.DEBATE_SYSTEM + + prompts.debate_round_user("", round_no, rounds, peer_block), + n, + ) + ) + elif mode == "adversarial": + # Byte-worst-case (DSE-1514 review, Fix A): 1 proposer succeeds + # immediately (upstream=0); every OTHER member critiques it, each + # embedding the proposal's answer text (upstream=1). A real run's + # k proposer attempts + (N-k) critics always equals N; k=1 + # maximizes the number of upstream-embedding critic calls, which + # is the pessimistic shape a spend ceiling must plan against. + if members: + table.append(_PhaseSpec("proposal", members[:1], "", 0)) + table.append( + _PhaseSpec( + "critique", + members[1:], + prompts.CRITIC_SYSTEM + prompts.critic_user("", ""), + 1, + ) + ) + elif mode == "elite": + table.append(_PhaseSpec("initial", members, "", 0)) + table.append( + _PhaseSpec( + "critique", + members, + prompts.ELITE_CRITIC_SYSTEM + + prompts.elite_critic_user("", member_placeholders), + n, + ) + ) + fallback_original = ModelAnswer( + name="", model_id="", answer="", answer_id=_ANSWER_ID_PROBE + ) + original = member_placeholders[0] if member_placeholders else fallback_original + # DSE-1514 review, Fix A: modes._elite_revision_messages_for passes + # EVERY reviser its OWN initial answer as ``original_answer`` -- + # which is ALSO one of the N entries already inside the initial + # panel. That answer's text is therefore embedded TWICE in a real + # revision call (once standalone, once inside the anonymized + # panel), so the byte-worst-case upstream count is N (initial + # panel) + N (critique panel) + 1 (the duplicate), not 2N -- an + # undercount the byte-lower-bound regression test below caught. + table.append( + _PhaseSpec( + "revision", + members, + prompts.ELITE_REVISION_SYSTEM + + prompts.elite_revision_user( + "", original, member_placeholders, member_placeholders + ), + 2 * n + 1, + ) + ) + + if mode == "debate": + # Mirrors modes._debate_synthesize's real block format exactly + # ("### Final answer from {name} ({model_id})\n{answer}") using + # the real member names/model ids, text left empty. + debate_final_blocks = "\n\n".join( + f"### Final answer from {a.name} ({a.model_id})\n{a.answer}" + for a in member_placeholders + ) + table.append( + _PhaseSpec( + "debate_final", + chain, + prompts.DEBATE_FINAL_SYSTEM + + prompts.debate_final_user("", rounds, debate_final_blocks), + n, + message_count=3, + ) + ) + elif mode == "adversarial": + # Judge upstream is ALWAYS N: it embeds the proposal (1) plus + # every critique the byte-worst-case shape produces (N-1). + table.append( + _PhaseSpec( + "judge", + chain, + prompts.JUDGE_SYSTEM + prompts.judge_user("", "", "", ""), + n, + message_count=3, + ) + ) + elif mode in ("synthesize", "elite"): + table.append( + _PhaseSpec( + "synthesis", + chain, + _SYNTH_SYSTEM + _synth_user_content("", member_placeholders), + n, + message_count=3, + ) + ) + if self.extract_verdict_enabled: + # DSE-1514 review, Fix A: the two templates below are the REAL + # extraction/repair message content -- schema included exactly + # as rendered, and one placeholder label per real member -- + # measured via conclave.verdict_synthesis's own builders rather + # than a hand-summed approximation. The schema's bytes are + # therefore already IN the template, so ``contract=True`` here + # only adds the flat provider-side structured-output framing + # allowance, never a second copy of the schema on top of it. + from .verdict_synthesis import VERDICT_REPAIR_ERROR_DETAIL_MAX_BYTES + from .verdict_synthesis import _build_messages as _verdict_build_messages + from .verdict_synthesis import _repair_instruction as _verdict_repair_instruction + + extraction_messages = _verdict_build_messages("", member_placeholders) + extraction_template = "".join(m["content"] for m in extraction_messages) + repair_template = extraction_template + _verdict_repair_instruction( + "x" * VERDICT_REPAIR_ERROR_DETAIL_MAX_BYTES + ) + table.append( + _PhaseSpec( + "verdict_extraction", + chain, + extraction_template, + n, + message_count=3, + contract=True, + ) + ) + table.append( + _PhaseSpec( + "verdict_repair", + chain, + repair_template, + n + 1, + message_count=3, + contract=True, + ) + ) + + return table + def plan_calls( self, mode: str, @@ -445,20 +807,9 @@ def plan_calls( """Enumerate every provider call this mode could make, worst case (DSE-1514). The counts are derived from :mod:`conclave.modes` and - :meth:`_apply_verdict`, not from a remembered formula. With ``N`` keyed - members, ``C`` keyed chain candidates, ``R`` debate rounds, and ``V`` = 1 - when verdict extraction is on: - - * ``raw`` -- ``N``: fan-out only. - * ``synthesize`` -- ``N + C + 2CV``: fan-out, the chain, then - extract+repair per candidate. - * ``vote`` -- ``N``: fan-out only; no adjudication. - * ``debate`` -- ``N*R + C``: every round at full membership (drop-out - only shrinks it), then the final consolidation chain. - * ``adversarial`` -- ``N + C``: ``k`` proposer attempts plus ``N - k`` - critics is exactly ``N`` for every ``k``; then the judge chain. - * ``elite`` -- ``3N + C + 2CV``: three phases at full membership, then - synthesis and verdict extraction. + :meth:`_apply_verdict`, not from a remembered formula -- see + :meth:`_plan_table` for the exact per-mode row table and the + DSE-1514 review Fix A rationale for the adversarial shape. Convergence early-stop, member drop-out, and a proposer succeeding on the first try all make a real run CHEAPER than its plan. A plan is never an @@ -470,7 +821,8 @@ def plan_calls( prompt: The exact user prompt (bounded by its UTF-8 byte length). rounds: Debate rounds; ignored for other modes. proposer: Adversarial proposer. It does not change the COUNT (see - above) and is accepted only for signature parity with the modes. + :meth:`_plan_table`) and is accepted only for signature parity + with the modes. choices: Vote choices, which enlarge the vote prompt template. Returns: @@ -491,103 +843,34 @@ def plan_calls( chain = self._keyed_chain() n_members = len(members) prompt_bytes = len(prompt.encode("utf-8")) - calls: list[PlannedCall] = [] - def member_calls(phase: str, *, template: str, upstream: int) -> None: - for name, model_id in members: + table = self._plan_table( + mode, members=members, chain=chain, rounds=max(1, rounds), choices=choices + ) + calls: list[PlannedCall] = [] + for spec in table: + # Fix A note: a structured-output contract's schema bytes are + # already INSIDE spec.template (the verdict probes are measured + # from the real message builders, schema included -- see + # conclave.verdict_synthesis). ``contract`` therefore only adds the + # flat provider-side framing allowance here, never a second copy + # of the schema on top of the prompt bound. + template_bytes = len(spec.template.encode("utf-8")) + framing = 64 + (16 * spec.message_count) + (256 if spec.contract else 0) + for name, model_id in spec.targets: calls.append( PlannedCall( - phase=phase, + phase=spec.phase, name=name, model_id=model_id, prompt_token_upper_bound=prompt_bytes, - prompt_template_token_allowance=len(template.encode("utf-8")), - provider_framing_token_allowance=64 + (16 * 2), - upstream_output_call_count=upstream, - max_output_tokens=cap, - ) - ) - - def chain_calls(phase: str, *, template: str, upstream: int, contract: bool) -> None: - for name, model_id in chain: - calls.append( - PlannedCall( - phase=phase, - name=name, - model_id=model_id, - prompt_token_upper_bound=( - prompt_bytes + (_VERDICT_CONTRACT_BYTES if contract else 0) - ), - prompt_template_token_allowance=len(template.encode("utf-8")), - provider_framing_token_allowance=(64 + (16 * 3) + (256 if contract else 0)), - upstream_output_call_count=upstream, + prompt_template_token_allowance=template_bytes, + provider_framing_token_allowance=framing, + upstream_output_call_count=spec.upstream, max_output_tokens=cap, ) ) - if mode in ("raw", "synthesize"): - member_calls("member", template="", upstream=0) - elif mode == "vote": - member_calls( - "member", - template=prompts.VOTE_SYSTEM + prompts.vote_user("", choices or []), - upstream=0, - ) - elif mode == "debate": - member_calls("round-1", template="", upstream=0) - for round_no in range(2, max(1, rounds) + 1): - member_calls( - f"round-{round_no}", - template=( - prompts.DEBATE_SYSTEM - + prompts.debate_round_user("", round_no, max(1, rounds), "") - ), - upstream=n_members, - ) - elif mode == "adversarial": - # k proposer attempts + (N - k) critics == N, for every k. - member_calls("proposal", template="", upstream=0) - elif mode == "elite": - member_calls("initial", template="", upstream=0) - member_calls( - "critique", - template=prompts.ELITE_CRITIC_SYSTEM + prompts.elite_critic_user("", []), - upstream=n_members, - ) - member_calls("revision", template=prompts.ELITE_REVISION_SYSTEM, upstream=2 * n_members) - - if mode == "debate": - chain_calls( - "debate_final", - template=( - prompts.DEBATE_FINAL_SYSTEM + prompts.debate_final_user("", max(1, rounds), "") - ), - upstream=n_members, - contract=False, - ) - elif mode == "adversarial": - chain_calls( - "judge", - template=prompts.JUDGE_SYSTEM + prompts.judge_user("", "", "", ""), - upstream=n_members, - contract=False, - ) - elif mode in ("synthesize", "elite"): - chain_calls("synthesis", template=_SYNTH_SYSTEM, upstream=n_members, contract=False) - if self.extract_verdict_enabled: - chain_calls( - "verdict_extraction", - template=_VERDICT_TEMPLATE_PROBE, - upstream=n_members, - contract=True, - ) - chain_calls( - "verdict_repair", - template=_VERDICT_TEMPLATE_PROBE, - upstream=n_members + 1, - contract=True, - ) - return CallPlan( mode=mode, calls=tuple(calls), @@ -670,6 +953,32 @@ def _enforce_spend_cap( self.max_spend_usd, ) + def _gate_live_run( + self, + mode: str, + prompt: str, + *, + rounds: int | None = None, + proposer: str | None = None, + choices: list[str] | None = None, + ) -> None: + """THE single pre-flight spend-gate chokepoint (DSE-1514 review, Fix C). + + :meth:`_cached_run` and :meth:`ask_stream` each call this exactly + once, always AFTER their cache-hit decision and always BEFORE the + first provider call -- a cache hit returns before reaching this + method entirely, since it makes no call and cannot exceed any cap. + Before this fix the same :meth:`_enforce_spend_cap` call was + duplicated at one site per cache branch (four call sites total, two + per entry point); collapsing them here means "does this run get + gated" has exactly one answer per entry point instead of two branches + that had to be kept in sync by hand. + + A thin, behavior-preserving wrapper over :meth:`_enforce_spend_cap`, + which still owns the actual planning/pricing/raising. + """ + self._enforce_spend_cap(mode, prompt, rounds=rounds, proposer=proposer, choices=choices) + def _cache_key( self, prompt: str, @@ -772,33 +1081,36 @@ async def _cached_run( A ``"terminal_failure"`` ledger entry (the model answered, just not usably) is unaffected and remains cacheable exactly as before -- re-running would not produce a different, better answer. - """ - if not self.cache_enabled: - self._enforce_spend_cap(mode, prompt, rounds=rounds, proposer=proposer, choices=choices) - result = await run() - self._ensure_manifest(result, mode) - self._price_manifest(result) - return result - key = self._cache_key( - prompt, - mode, - rounds=rounds, - proposer=proposer, - converge_threshold=converge_threshold, - choices=choices, - ) - hit = cache_mod.load(key) - if hit is not None: - logger.info("cache hit for %s run (%s)", mode, key[:12]) - self._ensure_manifest(hit, mode) - self._price_manifest(hit) - return hit + **One gate chokepoint (DSE-1514 review, Fix C).** :meth:`_gate_live_run` + is called exactly once here, after the cache-hit decision above (a hit + returns before reaching it) and before ``run()`` -- regardless of + whether caching is enabled at all. This replaced two separate call + sites (one per cache branch) that had to make the identical call. + """ + key: str | None = None + if self.cache_enabled: + key = self._cache_key( + prompt, + mode, + rounds=rounds, + proposer=proposer, + converge_threshold=converge_threshold, + choices=choices, + ) + hit = cache_mod.load(key) + if hit is not None: + logger.info("cache hit for %s run (%s)", mode, key[:12]) + self._ensure_manifest(hit, mode) + self._price_manifest(hit) + return hit - self._enforce_spend_cap(mode, prompt, rounds=rounds, proposer=proposer, choices=choices) + self._gate_live_run(mode, prompt, rounds=rounds, proposer=proposer, choices=choices) result = await run() self._ensure_manifest(result, mode) self._price_manifest(result) + if not self.cache_enabled: + return result if result.primary_failed_over: logger.info( "not caching %s run (%s): primary adjudicator failed for an infrastructure reason", @@ -1375,6 +1687,13 @@ async def ask_stream(self, prompt: str, synthesize: bool = True) -> AsyncIterato mode = "synthesize" if synthesize else "raw" + # One gate chokepoint (DSE-1514 review, Fix C): a cache hit returns + # below before ``_gate_live_run`` is ever reached, since it makes no + # provider call and cannot exceed any cap. Everything past the hit + # check is a live run, so the gate call sits exactly once, right + # before the streaming driver starts -- previously duplicated at one + # site per cache branch. + key: str | None = None if self.cache_enabled: key = self._cache_key(prompt, mode) hit = cache_mod.load(key) @@ -1384,29 +1703,30 @@ async def ask_stream(self, prompt: str, synthesize: bool = True) -> AsyncIterato yield event return - # Live miss: stream, capture the terminal result, then store it - # (no-store on primary infrastructure failure -- see the docstring). - self._enforce_spend_cap(mode, prompt) - final: CouncilResult | None = None + self._gate_live_run(mode, prompt) + + if not self.cache_enabled: async for event in stream_ask(self, prompt, synthesize=synthesize): - if event.type == "done" and event.result is not None: - final = event.result yield event - if final is not None: - if final.primary_failed_over: - logger.info( - "not caching %s run (%s): primary adjudicator failed for an " - "infrastructure reason", - mode, - key[:12], - ) - else: - cache_mod.store(key, final) return - self._enforce_spend_cap(mode, prompt) + # Live miss: stream, capture the terminal result, then store it + # (no-store on primary infrastructure failure -- see the docstring). + final: CouncilResult | None = None async for event in stream_ask(self, prompt, synthesize=synthesize): + if event.type == "done" and event.result is not None: + final = event.result yield event + if final is not None: + if final.primary_failed_over: + logger.info( + "not caching %s run (%s): primary adjudicator failed for an " + "infrastructure reason", + mode, + key[:12], + ) + else: + cache_mod.store(key, final) @staticmethod def _replay_cached(result: CouncilResult) -> list[StreamEvent]: @@ -1516,16 +1836,7 @@ async def _synthesize(self, result: CouncilResult) -> ModelAnswer | None: ) return None - blocks = "\n\n".join( - f"### Answer from {a.name} ({a.model_id})" - f"{f' (Answer ID: {a.answer_id})' if a.answer_id else ''}\n{a.answer}" - for a in usable - ) - user_content = ( - f"Original prompt:\n{result.prompt}\n\n" - f"Council answers:\n\n{blocks}\n\n" - "Now produce the consolidated answer." - ) + user_content = _synth_user_content(result.prompt, usable) outcome = await self._adjudicate_and_record( result, "synthesis", diff --git a/src/conclave/verdict_synthesis.py b/src/conclave/verdict_synthesis.py index 66dbf15..3d2b657 100644 --- a/src/conclave/verdict_synthesis.py +++ b/src/conclave/verdict_synthesis.py @@ -171,19 +171,6 @@ def _bounded_repair_error(detail: object) -> str: "clustering. Emit only the fields in the schema." ) -# DSE-1514: byte sizes the pre-flight spend planner needs without making a call. -# The extraction schema and its system prompt are fixed, so their UTF-8 byte cost -# is a constant of this module rather than a per-run guess. -VERDICT_CONTRACT_BYTES = len( - json.dumps( - verdict_extraction_json_schema(), - ensure_ascii=False, - separators=(",", ":"), - sort_keys=True, - ).encode("utf-8") -) -VERDICT_TEMPLATE_PROBE = _EXTRACTION_SYSTEM - class VerdictSynthesisResult(BaseModel): """The outcome of one verdict-extraction run (CAC-05 engine return type). @@ -399,6 +386,35 @@ def _build_messages(prompt: str, responders: list[ModelAnswer]) -> list[dict[str ] +def _repair_instruction(errors: str) -> str: + """Build the fixed repair-retry instruction wrapping a bounded error detail. + + Extracted into its own function (DSE-1514 review, Fix A) so a template + probe can measure the EXACT fixed wording :func:`extract_verdict` embeds + for real (with a placeholder error detail), rather than a hand-duplicated + copy that could silently drift from the real repair message. + :meth:`conclave.council.Council._plan_table` calls this directly -- see + that method's docstring for why the probe is computed per-call there + (using the run's real member count) rather than as a fixed module + constant here. + + Args: + errors: The bounded validation-error detail from + :func:`_parse_and_validate` (already capped at + ``VERDICT_REPAIR_ERROR_DETAIL_MAX_BYTES``). + + Returns: + The repair-retry user-message content. + """ + return ( + "Your previous response could not be used. It must be a single " + "valid JSON object matching the schema exactly, with no prose " + "and no consensus number. The problem was:\n" + f"{errors}\n\n" + "Return only the corrected JSON object." + ) + + def _strip_code_fence(text: str) -> str: """Strip a surrounding Markdown code fence from a model answer, if present. @@ -695,18 +711,7 @@ async def extract_verdict( ] retry: ModelAnswer | None = None if extraction is None: - repair_messages = messages + [ - { - "role": "user", - "content": ( - "Your previous response could not be used. It must be a single " - "valid JSON object matching the schema exactly, with no prose " - "and no consensus number. The problem was:\n" - f"{errors}\n\n" - "Return only the corrected JSON object." - ), - } - ] + repair_messages = messages + [{"role": "user", "content": _repair_instruction(errors)}] retry = await model_caller( synthesizer_name, synthesizer_model_id, diff --git a/tests/test_cli.py b/tests/test_cli.py index b4e6ff5..8167074 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1032,7 +1032,10 @@ def test_an_over_budget_run_exits_four_and_names_reserved_cap_and_count(monkeypa _install_snapshot(monkeypatch, _snapshot("xai/grok-4.3", "anthropic/claude-sonnet-4-6")) + calls: list[str] = [] + async def tripwire(name, model_id, messages, **kwargs): + calls.append(name) raise AssertionError("the CLI gate must refuse before any provider call") monkeypatch.setattr(council_mod, "call_model", tripwire) @@ -1052,6 +1055,45 @@ async def tripwire(name, model_id, messages, **kwargs): combined = result.output + (result.stderr or "") assert result.exit_code == 4 assert "reserved" in combined and "0.000001" in combined and "calls" in combined + # DSE-1514 review, Fix A/minors: the gate seam itself must never be reached. + assert calls == [] + + +def test_json_refusal_emits_no_stdout_payload_and_puts_the_message_on_stderr(monkeypatch, keys): + """DSE-1514 review minors: a refused --json run has nothing to serialize. + + The mode dispatch that would build the JSON payload never runs -- the gate + raises before it -- so stdout must be completely empty (not even + ``null``/``{}``) and the refusal message must land on stderr, exactly like + the non-JSON path. See the `ask` docstring's exit-code-4 note. + """ + import conclave.council as council_mod + from conclave.cli import app + from tests.test_pricing_receipts import _install_snapshot, _snapshot + + _install_snapshot(monkeypatch, _snapshot("xai/grok-4.3", "anthropic/claude-sonnet-4-6")) + + async def tripwire(name, model_id, messages, **kwargs): + raise AssertionError("the CLI gate must refuse before any provider call") + + monkeypatch.setattr(council_mod, "call_model", tripwire) + result = runner.invoke( + app, + [ + "ask", + "q", + "--council", + "grok", + "--max-output-tokens", + "100000", + "--max-spend-usd", + "0.000001", + "--json", + ], + ) + assert result.exit_code == 4 + assert result.stdout == "" + assert "reserved" in result.stderr and "0.000001" in result.stderr def test_an_unboundable_plan_exits_four_with_a_distinct_message(monkeypatch, keys): diff --git a/tests/test_spend_plan.py b/tests/test_spend_plan.py index 3da27b3..5cf2ba2 100644 --- a/tests/test_spend_plan.py +++ b/tests/test_spend_plan.py @@ -76,7 +76,13 @@ def test_downstream_phases_declare_their_upstream_dependencies(keys): assert all(c.upstream_output_call_count == 0 for c in by_phase["initial"]) assert all(c.upstream_output_call_count == 3 for c in by_phase["critique"]) # N initials - assert all(c.upstream_output_call_count == 6 for c in by_phase["revision"]) # N + N + # DSE-1514 review, Fix A: N (initial panel) + N (critique panel) + 1 -- every + # reviser's OWN initial answer is embedded a second time as its standalone + # "original answer" (modes._elite_revision_messages_for), not just inside + # the anonymized panel. Was asserted as 6 (2N); corrected to 7 (2N+1) after + # the byte-lower-bound regression test proved 2N alone undercounts the real + # message. + assert all(c.upstream_output_call_count == 7 for c in by_phase["revision"]) assert by_phase["synthesis"][0].upstream_output_call_count == 3 # N revisions assert by_phase["verdict_extraction"][0].upstream_output_call_count == 3 assert by_phase["verdict_repair"][0].upstream_output_call_count == 4 # + its own attempt @@ -90,3 +96,189 @@ def test_an_unknown_mode_is_a_value_error(keys): def test_planning_without_an_output_cap_is_refused(keys): with pytest.raises(ValueError, match="cannot bound spend: no output cap"): Council(models=MEMBERS, synthesizer="claude").plan_calls("synthesize", "q") + + +# --- DSE-1514 Round 4 review, Fix A: the adversarial byte-worst-case shape --- +# and a general byte-lower-bound regression covering every phase whose input +# embeds a prior call's output. The ticket's guarantee is "never below the +# real request bytes"; these tests hold every phase to it using the REAL +# message-building functions (never a hand re-derivation of their output). + +MAX_OUTPUT_TOKENS = 1_000 +MAX_OUTPUT_BYTES_PER_TOKEN = 8 +MAX_LEN_TEXT = "x" * (MAX_OUTPUT_TOKENS * MAX_OUTPUT_BYTES_PER_TOKEN) +PROMPT = "q" + + +def _plan_members() -> list[tuple[str, str]]: + """The exact (name, model_id) pairs _council() resolves, N=3.""" + return _council()._available_members()[0] + + +def _max_len_answers(): + from conclave.council import _ANSWER_ID_PROBE + from conclave.models import ModelAnswer + + return [ + ModelAnswer(name=name, model_id=model_id, answer=MAX_LEN_TEXT, answer_id=_ANSWER_ID_PROBE) + for name, model_id in _plan_members() + ] + + +def _phase_call(plan, phase: str): + return next(c for c in plan.calls if c.phase == phase) + + +def test_adversarial_byte_worst_case_is_one_proposer_and_n_minus_one_critics(keys): + """1 proposer (upstream=0) + N-1 critics (upstream=1 each); judge upstream=N.""" + plan = _council().plan_calls("adversarial", PROMPT) + proposals = [c for c in plan.calls if c.phase == "proposal"] + critiques = [c for c in plan.calls if c.phase == "critique"] + judges = [c for c in plan.calls if c.phase == "judge"] + + assert len(proposals) == 1 + assert proposals[0].upstream_output_call_count == 0 + assert len(critiques) == 2 + assert all(c.upstream_output_call_count == 1 for c in critiques) + assert len(judges) == 1 + assert judges[0].upstream_output_call_count == 3 + + +def test_a_critic_call_has_a_strictly_larger_input_bound_than_the_proposal(keys): + plan = _council().plan_calls("adversarial", PROMPT) + proposal = _phase_call(plan, "proposal") + critique = _phase_call(plan, "critique") + bound_kwargs = {"upstream_output_bytes_per_token": MAX_OUTPUT_BYTES_PER_TOKEN} + assert critique.input_bytes_bound(**bound_kwargs) > proposal.input_bytes_bound(**bound_kwargs) + + +def _critic_case(): + from conclave.modes import _critic_messages_for + + plan = _council().plan_calls("adversarial", PROMPT) + call = _phase_call(plan, "critique") + messages = _critic_messages_for(PROMPT, MAX_LEN_TEXT)("critic", "x/x") + return call, messages + + +def _synthesis_case(): + import conclave.council as council_mod + + plan = _council().plan_calls("synthesize", PROMPT) + call = _phase_call(plan, "synthesis") + content = council_mod._synth_user_content(PROMPT, _max_len_answers()) + messages = [ + {"role": "system", "content": council_mod._SYNTH_SYSTEM}, + {"role": "user", "content": content}, + ] + return call, messages + + +def _debate_round2_case(): + from conclave import prompts + + plan = _council().plan_calls("debate", PROMPT, rounds=2) + call = _phase_call(plan, "round-2") + members = _plan_members() + answers = _max_len_answers() + letters = {name: prompts.LETTERS[i] for i, (name, _mid) in enumerate(members)} + prior = {name: answer for (name, _mid), answer in zip(members, answers, strict=True)} + self_name = members[0][0] + peer_block = prompts.anonymized_peer_block(self_name, letters[self_name], prior, letters) + messages = [ + {"role": "system", "content": prompts.DEBATE_SYSTEM}, + {"role": "user", "content": prompts.debate_round_user(PROMPT, 2, 2, peer_block)}, + ] + return call, messages + + +def _elite_critique_case(): + from conclave import prompts + + plan = _council().plan_calls("elite", PROMPT) + call = _phase_call(plan, "critique") + messages = [ + {"role": "system", "content": prompts.ELITE_CRITIC_SYSTEM}, + {"role": "user", "content": prompts.elite_critic_user(PROMPT, _max_len_answers())}, + ] + return call, messages + + +def _elite_revision_case(): + from conclave import prompts + + plan = _council().plan_calls("elite", PROMPT) + call = _phase_call(plan, "revision") + answers = _max_len_answers() + critiques = _max_len_answers() + messages = [ + {"role": "system", "content": prompts.ELITE_REVISION_SYSTEM}, + { + "role": "user", + "content": prompts.elite_revision_user(PROMPT, answers[0], answers, critiques), + }, + ] + return call, messages + + +def _verdict_extraction_case(): + from conclave.verdict_synthesis import _build_messages + + plan = _council().plan_calls("synthesize", PROMPT) + call = _phase_call(plan, "verdict_extraction") + messages = _build_messages(PROMPT, _max_len_answers()) + return call, messages + + +def _verdict_repair_case(): + from conclave.verdict_synthesis import ( + VERDICT_REPAIR_ERROR_DETAIL_MAX_BYTES, + _build_messages, + _repair_instruction, + ) + + plan = _council().plan_calls("synthesize", PROMPT) + call = _phase_call(plan, "verdict_repair") + messages = _build_messages(PROMPT, _max_len_answers()) + [ + { + "role": "user", + "content": _repair_instruction("e" * VERDICT_REPAIR_ERROR_DETAIL_MAX_BYTES), + } + ] + return call, messages + + +@pytest.mark.parametrize( + "case_builder", + [ + _critic_case, + _synthesis_case, + _debate_round2_case, + _elite_critique_case, + _elite_revision_case, + _verdict_extraction_case, + _verdict_repair_case, + ], + ids=[ + "adversarial-critique", + "synthesize-synthesis", + "debate-round-2", + "elite-critique", + "elite-revision", + "verdict-extraction", + "verdict-repair", + ], +) +def test_the_planned_byte_bound_never_falls_below_the_real_worst_case_message(keys, case_builder): + """DSE-1514 review, Fix A: "never below the real request bytes", every phase. + + Builds the REAL message list for each phase (via the actual mode/prompt + builders, never a hand-derived approximation) with worst-case-length + (``max_output_tokens * max_output_bytes_per_token``) upstream text, and + proves the planned call's byte bound covers it. + """ + call, messages = case_builder() + real_bytes = sum(len(m["content"].encode("utf-8")) for m in messages) + assert real_bytes <= call.input_bytes_bound( + upstream_output_bytes_per_token=MAX_OUTPUT_BYTES_PER_TOKEN + ) From 438306ec76d29249eb325046f297b07e6fd1ee64 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 4 Sep 2026 17:11:05 -0400 Subject: [PATCH 06/10] test(pricing): cross-mode ceilings, refusal-before-call, secret-safety on the priced manifest (DSE-1514) Task 13 as planned: - tests/test_manifest_all_modes.py: test_every_mode_prices_its_manifest (parametrized over synthesize/raw/debate/adversarial/vote/elite) proves pricing runs at the same _cached_run chokepoint the manifest-on-every- result invariant runs at; test_elite_prices_every_one_of_its_3n_plus_ receipts checks every one of Elite's per-phase receipts is priced. - tests/test_secret_safety_matrix.py: test_pricing_fields_never_un_verify_ the_stamp and test_pricing_warnings_are_a_closed_vocabulary_in_the_code (static source scan of every warnings.append call). Plus, per the Round 4 review brief: - New src/conclave/council.py constant PRICING_WARNING_VOCABULARY: the same five identifiers _price_manifest ever appends, now importable so a test can assert against it directly instead of re-deriving the set. - test_a_fully_populated_priced_manifest_still_stamps_verified: a REAL run (not a hand-built manifest) whose manifest carries a stale-snapshot warning AND an unpriced model/receipt AND a real ceiling on the priced receipt simultaneously still stamps secret_safety VERIFIED. - test_pricing_warnings_stay_within_the_closed_vocabulary (parametrized over all six modes): drives four warning-producing shapes (no snapshot, fully priced but stale, an unpriced chain candidate, every call failing with no output cap) through each mode and asserts every pricing_warnings value ever produced is a member of PRICING_WARNING_VOCABULARY -- the dynamic complement to the static source-scan guard. - The adversarial-critic byte bound is already covered by Commit A's tests/test_spend_plan.py (Fix A); not duplicated here. 935 passed (919 after Commit A + 16 new). ruff check/format clean. Claude-Session: https://claude.ai/code/session_01K1dHPjZ1bZcE2GnX3KMMSH --- src/conclave/council.py | 18 +++ tests/test_manifest_all_modes.py | 65 +++++++++++ tests/test_secret_safety_matrix.py | 173 +++++++++++++++++++++++++++++ 3 files changed, 256 insertions(+) diff --git a/src/conclave/council.py b/src/conclave/council.py index 1285cd0..286cd6e 100644 --- a/src/conclave/council.py +++ b/src/conclave/council.py @@ -187,6 +187,24 @@ def _placeholder_answers(members: Sequence[tuple[str, str]]) -> list[ModelAnswer "cannot bound spend: no output cap (set --max-output-tokens or config max_output_tokens)" ) +# The closed vocabulary of pricing_warnings identifiers (DSE-1514). Every +# append to the ``warnings`` list -- or the list-literal assignment on the +# missing-snapshot path -- inside :meth:`Council._price_manifest` uses one of +# these exact strings, never interpolated text, a provider name, or a count, +# so a warning can never carry secret-shaped material. Tests in +# tests/test_secret_safety_matrix.py assert both statically (scanning the +# source for every append call) and dynamically (every reachable warning +# shape across every mode) that nothing else is ever appended. +PRICING_WARNING_VOCABULARY = frozenset( + { + "price_snapshot_stale", + "price_snapshot_unavailable", + "unpriced_models_present", + "unpriced_receipts_present", + "no_output_cap_configured", + } +) + # Re-exported for callers that want the version without importing prompts. __all__ = ["Council", "SYNTHESIS_PROMPT_VERSION"] diff --git a/tests/test_manifest_all_modes.py b/tests/test_manifest_all_modes.py index 262a352..1af3672 100644 --- a/tests/test_manifest_all_modes.py +++ b/tests/test_manifest_all_modes.py @@ -600,3 +600,68 @@ async def run(): second = await run() assert second.cached is True _assert_verified_manifest(second, mode) + + +# --------------------------------------------------------------------------- # +# DSE-1514 Task 13: pricing runs at the same chokepoint the manifest +# invariant runs at -- every mode's manifest carries a priced (or +# all-or-nothing unpriced) ceiling, never a partial one. +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize("mode", ["synthesize", "raw", "debate", "adversarial", "vote", "elite"]) +async def test_every_mode_prices_its_manifest(monkeypatch, keys, patch_call_model, mode): + """Pricing runs at the same chokepoint the manifest invariant runs at.""" + from decimal import Decimal + + from conclave.council import Council + from tests.conftest import make_response + from tests.test_pricing_receipts import _install_snapshot, _snapshot + + _install_snapshot( + monkeypatch, _snapshot("xai/grok-4.3", "gemini/gemini-2.5-pro", "openai/gpt-4.1") + ) + patch_call_model(lambda model_id, messages: make_response("ok")) + council = Council( + models=["grok", "gemini", "openai"], synthesizer="openai", extract_verdict=False + ) + + if mode == "debate": + result = await council.debate("q", rounds=2) + elif mode == "adversarial": + result = await council.adversarial("q") + elif mode == "vote": + result = await council.vote("q", choices=["a", "b"]) + elif mode == "elite": + result = await council.elite("q") + else: + result = await council.ask("q", synthesize=(mode == "synthesize")) + + manifest = result.manifest + assert manifest.price_snapshot_digest is not None + assert manifest.priced_as_of is not None + assert manifest.estimated_cost is None + if manifest.receipts: + assert manifest.cost_ceiling_usd == sum( + (r.cost_ceiling_usd for r in manifest.receipts), Decimal("0") + ) + + +async def test_elite_prices_every_one_of_its_3n_plus_receipts(monkeypatch, keys, patch_call_model): + from conclave.council import Council + from tests.conftest import make_response + from tests.test_pricing_receipts import _install_snapshot, _snapshot + + _install_snapshot( + monkeypatch, _snapshot("xai/grok-4.3", "gemini/gemini-2.5-pro", "openai/gpt-4.1") + ) + patch_call_model(lambda model_id, messages: make_response("ok")) + council = Council( + models=["grok", "gemini", "openai"], synthesizer="openai", extract_verdict=False + ) + manifest = (await council.elite("q")).manifest + + # 3 phases x 3 members + 1 synthesis = 10; every one of them priced. + assert len(manifest.receipts) >= 3 * 3 + assert all(r.cost_ceiling_usd is not None for r in manifest.receipts) + assert manifest.unpriced_receipts == 0 diff --git a/tests/test_secret_safety_matrix.py b/tests/test_secret_safety_matrix.py index ffe32b7..754c9ea 100644 --- a/tests/test_secret_safety_matrix.py +++ b/tests/test_secret_safety_matrix.py @@ -459,3 +459,176 @@ def test_scan_rejects_planted_canary_in_manifest(): redacted_errors=[f"leaked credential {PLANTED}"], ) assert scan_for_secret_material(polluted) is False + + +# --------------------------------------------------------------------------- # +# DSE-1514 Task 13: the pricing fields never un-verify the secret-safety stamp, +# and pricing_warnings is a closed vocabulary -- statically (source scan) and +# dynamically (every reachable warning shape across every mode). +# --------------------------------------------------------------------------- # + + +def test_pricing_fields_never_un_verify_the_stamp(): + """A model id with an awkward substring must not break the stamp.""" + from decimal import Decimal + + from conclave.manifest import ( + SECRET_SAFETY_VERIFIED, + ModelHarnessManifest, + ProviderExecutionReceipt, + verified_secret_safety, + ) + + manifest = ModelHarnessManifest( + request_id="r", + conclave_version="1.3.0", + mode="synthesize", + model_ids=["deepseek/deepseek-chat", "together/meta-llama/Llama-3.3-70B-Instruct-Turbo"], + receipts=[ + ProviderExecutionReceipt( + name="deepseek", + provider="deepseek", + model_id="deepseek/deepseek-chat", + cost_ceiling_usd=Decimal("0.000123"), + cost_basis="reservation", + generation_settings={ + "temperature": 0.7, + "timeout": 120.0, + "max_output_tokens": 8, + }, + ) + ], + cost_ceiling_usd=Decimal("0.000123"), + price_snapshot_digest="sha256:" + "e" * 64, + priced_as_of="2026-09-03", + unpriced_models=["together/meta-llama/Llama-3.3-70B-Instruct-Turbo"], + unpriced_receipts=0, + pricing_warnings=[ + "unpriced_models_present", + "price_snapshot_stale", + "no_output_cap_configured", + ], + ) + assert verified_secret_safety(manifest) == SECRET_SAFETY_VERIFIED + + +def test_pricing_warnings_are_a_closed_vocabulary_in_the_code(): + """No pricing warning may be built by interpolation.""" + import re + from pathlib import Path + + import conclave + + source = (Path(conclave.__file__).parent / "council.py").read_text(encoding="utf-8") + appended = re.findall(r"warnings\.append\((.+?)\)", source) + assert appended, "the pricing warning appends moved; update this guard" + for expression in appended: + assert expression.startswith('"') and expression.endswith('"'), ( + f"pricing warning must be a literal, got {expression}" + ) + + +async def test_a_fully_populated_priced_manifest_still_stamps_verified( + monkeypatch, keys, patch_call_model +): + """A priced manifest with every new field populated -- a stale-snapshot + warning AND an unpriced model/receipt AND a real ceiling on the priced + receipt -- still stamps VERIFIED (DSE-1514 review, Task 13). + + Exercises the real run path (``Council.ask``) rather than a hand-built + manifest, so the shape asserted is one ``_price_manifest`` can actually + produce, not a synthetic one that happens to pass the scan. + """ + from datetime import date + + from conclave.manifest import SECRET_SAFETY_VERIFIED + from tests.test_pricing_receipts import _install_snapshot, _snapshot + + # grok is priced but the snapshot is stale-dated; claude (the synthesizer) + # has NO entry at all -- both a stale-snapshot warning and an + # unpriced-model/unpriced-receipt shape land on the SAME manifest. + _install_snapshot(monkeypatch, _snapshot("xai/grok-4.3", captured_at=date(2026, 1, 1))) + patch_call_model(lambda model_id, messages: make_response("ok")) + council = Council(models=["grok"], synthesizer="claude", extract_verdict=False) + manifest = (await council.ask("q")).manifest + + assert manifest.cost_ceiling_usd is None # all-or-nothing: claude is unpriced + assert manifest.unpriced_models == ["anthropic/claude-sonnet-4-6"] + assert manifest.unpriced_receipts >= 1 + assert "price_snapshot_stale" in manifest.pricing_warnings + assert "unpriced_models_present" in manifest.pricing_warnings + priced = [r for r in manifest.receipts if r.model_id == "xai/grok-4.3"] + assert priced and all(r.cost_ceiling_usd is not None for r in priced) + assert manifest.secret_safety == SECRET_SAFETY_VERIFIED + assert scan_for_secret_material(manifest) is True + + +@pytest.mark.parametrize("mode", ["synthesize", "raw", "debate", "adversarial", "vote", "elite"]) +async def test_pricing_warnings_stay_within_the_closed_vocabulary( + monkeypatch, keys, patch_call_model, mode +): + """DSE-1514 review, Task 13: run every mode against every warning-producing + shape (no snapshot, an unpriced model, a stale snapshot, an uncapped + failed call) and prove every emitted ``pricing_warnings`` entry is drawn + from :data:`conclave.council.PRICING_WARNING_VOCABULARY` -- the dynamic + complement to the static source-scan guard above. + """ + from datetime import date + + from conclave.council import PRICING_WARNING_VOCABULARY, Council + from tests.test_pricing_receipts import _install_snapshot, _snapshot + + async def run(council: Council): + if mode == "debate": + return await council.debate("q", rounds=2) + if mode == "adversarial": + return await council.adversarial("q") + if mode == "vote": + return await council.vote("q", choices=["a", "b"]) + if mode == "elite": + return await council.elite("q") + return await council.ask("q", synthesize=(mode == "synthesize")) + + seen: set[str] = set() + + # Shape 1: no snapshot at all -> price_snapshot_unavailable. + _install_snapshot(monkeypatch, None) + patch_call_model(lambda model_id, messages: make_response("ok")) + council = Council(models=["grok", "gemini"], synthesizer="claude", extract_verdict=False) + seen.update((await run(council)).manifest.pricing_warnings) + + # Shape 2: fully priced but stale -> price_snapshot_stale. + _install_snapshot( + monkeypatch, + _snapshot( + "xai/grok-4.3", + "gemini/gemini-2.5-pro", + "anthropic/claude-sonnet-4-6", + captured_at=date(2026, 1, 1), + ), + ) + council = Council(models=["grok", "gemini"], synthesizer="claude", extract_verdict=False) + seen.update((await run(council)).manifest.pricing_warnings) + + # Shape 3: the synthesizer/judge/chain model has no snapshot entry -> + # unpriced_models_present (+ unpriced_receipts_present). + _install_snapshot(monkeypatch, _snapshot("xai/grok-4.3", "gemini/gemini-2.5-pro")) + council = Council(models=["grok", "gemini"], synthesizer="claude", extract_verdict=False) + seen.update((await run(council)).manifest.pricing_warnings) + + # Shape 4: every model priced, every call fails with no usage, no output + # cap -> unpriced_receipts_present + no_output_cap_configured. + _install_snapshot( + monkeypatch, + _snapshot("xai/grok-4.3", "gemini/gemini-2.5-pro", "anthropic/claude-sonnet-4-6"), + ) + + def failing(model_id, messages): + raise RuntimeError("boom") + + patch_call_model(failing) + council = Council(models=["grok", "gemini"], synthesizer="claude", extract_verdict=False) + seen.update((await run(council)).manifest.pricing_warnings) + + assert seen, "no pricing_warnings were ever produced; the fixture setup is stale" + assert seen <= PRICING_WARNING_VOCABULARY From 5e3208a550a1919ae4b809c3c7e29eb478a84513 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 4 Sep 2026 17:18:58 -0400 Subject: [PATCH 07/10] =?UTF-8?q?docs:=20cost=20ceilings=20and=20the=20spe?= =?UTF-8?q?nd=20gate=20=E2=80=94=20README,=20PDD=20=C2=A74a/=C2=A79,=20cha?= =?UTF-8?q?ngelog,=20config=20example=20(DSE-1514)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Task 14 as planned, plus the coordinator's Round 4 correction: - README.md: new "Cost ceilings and spend gate" section. Ceiling-vs-estimate distinction, the all-or-nothing rule, the three exact refusal messages (no output cap / no priced rate / over cap) with exit code 4, the output cap as a --max-spend-usd prerequisite, the omitted-model note (groq and deepseek, tracked in DSE-1537), the adversarial worst-case shape (1 proposer + N-1 critics; judge embeds all N prior outputs), and an exit-code table (0/1/2/3/4). - docs/PRODUCT_DESIGN_DOCUMENT.md §4a: extended the ModelHarnessManifest field list with the six new run-level pricing fields and receipt-level cost_ceiling_usd/cost_basis; rewrote the "No invented pricing" bullet's second sentence to distinguish a ceiling from an estimate; new "Cost ceilings, never estimates (v1.4)" subsection covering all-or-nothing, never-a-substitute-rate (with the DSE-1537 omitted-model note), priced- last ordering, the pre-flight gate's three exact messages, and the adversarial byte-worst-case shape. §9: one paragraph noting H1/H4 now have a real cost denominator instead of a guess. - CHANGELOG.md [Unreleased]: Added bullets for bounded cost ceilings, the dated vendor-cited snapshot (with the DSE-1537 omission note), the --max-output-tokens cap, and the --max-spend-usd gate. Changed bullets for CACHE_FORMAT_VERSION 4 -> 5 and generation_settings gaining max_output_tokens when set. Not changed bullet for estimated_cost staying None permanently. (The widened test-double kwargs are not user-visible and are correctly omitted.) - DOCUMENTATION_INDEX.md: linked the plan beside the DSE-1512 entry. - config.example.yml: commented max_output_tokens example. - SYSTEM_CONTEXT_DIAGRAM.md: unchanged -- verified it does not enumerate manifest fields (grep for estimated_cost found nothing), matching the plan's conditional instruction. - docs/plans/2026-09-03-bounded-cost-receipts.md: corrected Task 10's worked example and the header "Worst-case call plan per mode" table's adversarial row to state the shape actually implemented in Fix A (1 proposer call with upstream=0, N-1 critic calls with upstream=1 each, one judge call per keyed chain candidate with upstream=N), each marked "CORRECTED 2026-09-04 (Round 4 review, Fix A)" / "Byte shape, corrected 2026-09-04 (Round 4 review, Fix A)" so the history is visible. The call- COUNT arithmetic (N + C) was already correct and is unchanged. 935 passed, coverage 92.03% (>= 75% floor). ruff check/format clean, including the Markdown Python fenced blocks in README/CHANGELOG/docs. Claude-Session: https://claude.ai/code/session_01K1dHPjZ1bZcE2GnX3KMMSH --- CHANGELOG.md | 34 +++++++++ DOCUMENTATION_INDEX.md | 1 + README.md | 60 ++++++++++++++++ config.example.yml | 8 +++ docs/PRODUCT_DESIGN_DOCUMENT.md | 72 +++++++++++++++++-- .../plans/2026-09-03-bounded-cost-receipts.md | 25 ++++++- 6 files changed, 193 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 638c500..f57f2ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 exhausted. Independent of `degraded`; the cache never stores a `true` run. - A declared chain widens which vendors may receive the prompt (see README › Synthesizer failover › Confidentiality). +- **Bounded cost ceilings on the manifest (DSE-1514).** Every receipt now carries + `cost_ceiling_usd` (exact `Decimal`, `ROUND_CEILING`) and `cost_basis` (`reported_usage` or + `reservation`), and the manifest carries a run-level `cost_ceiling_usd`, + `price_snapshot_digest`, `priced_as_of`, `unpriced_models`, `unpriced_receipts`, and a bounded + `pricing_warnings` list. A ceiling is a falsifiable claim — *"this run cost no more than $X, + priced against snapshot `` dated ``"* — not an estimate. `estimated_cost` is + untouched and stays `None`. **All-or-nothing:** one unpriced model or one unpriceable receipt + leaves the run ceiling `None` rather than emitting a partial sum. +- **Dated, vendor-cited price snapshot.** `src/conclave/data/prices-.json` ships in the + wheel. Every entry cites the vendor page its rate was read from; rates are rounded **up**; a + model whose list price could not be verified is **omitted** (unpriced), never guessed — + `groq/llama-3.3-70b-versatile` and `deepseek/deepseek-chat` are currently omitted this way, + tracked for re-pricing in DSE-1537. Nothing is fetched at runtime. A snapshot older than 90 + days adds a `price_snapshot_stale` warning and still prices at exactly its recorded rates. +- **`--max-output-tokens` / `max_output_tokens:`.** A hard output ceiling threaded to every call + a council makes — members, synthesizer, judge, verdict extraction and its repair retry, and + both streaming paths — and recorded in `generation_settings` when set. +- **`--max-spend-usd` pre-flight spend gate.** Enumerates the worst-case call plan for the + selected mode (`raw` `N`, `synthesize` `N+C+2C`, `vote` `N`, `debate` `N*R+C`, `adversarial` + `N+C`, `elite` `3N+C+2C`, where `C` is the count of *keyed* synthesizer-chain candidates), and + the adversarial byte-worst-case is 1 proposer succeeding immediately + N-1 critics (each + embedding the proposal's answer) with the judge embedding the proposal and every critique. + Reserves each call pessimistically at ceiling rates and **refuses before the first provider + call** with the reserved total, the cap, and the call count. New exit code **4**. A plan that + cannot be bounded — no output cap, no snapshot, or an unpriced model — refuses with a distinct + message rather than guessing. ### Changed @@ -59,6 +85,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 not. - `ProviderError` and `TransportError` accept keyword-only `category` (and `http_status`); positional construction is unchanged. +- **Cache format version `4` → `5` (DSE-1514).** Identity now additionally carries the + price-snapshot rate fingerprint and `max_output_tokens`; old entries miss safely. +- `generation_settings` (on receipts and the manifest) gains `max_output_tokens` when a cap is + configured, so an integer token cap round-trips as an integer, not a float. An uncapped run's + `generation_settings` is byte-identical to before this change. ### Not changed (deliberately) @@ -67,6 +98,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 is decided by whether it ever answered. - Member-level failover (members already degrade gracefully), transport-level retries, and the substring-derived `ReceiptErrorCategory` on receipts. +- **`estimated_cost` stays `None` everywhere, permanently (DSE-1514).** It is never assigned, + never summed into, never renamed. A ceiling (`cost_ceiling_usd`) is a different, falsifiable + claim from an estimate, and the two must never be conflated in one field. ## [1.3.0] - 2026-08-01 diff --git a/DOCUMENTATION_INDEX.md b/DOCUMENTATION_INDEX.md index 69e9284..e899659 100644 --- a/DOCUMENTATION_INDEX.md +++ b/DOCUMENTATION_INDEX.md @@ -38,6 +38,7 @@ the canonical authority spec on top of those. | **H1 Live Runner Plan** | [`docs/plans/2026-07-18-h1-live-evaluation-runner.md`](docs/plans/2026-07-18-h1-live-evaluation-runner.md) | Exact TDD tasks for the six live conditions, dry-run estimator, replay fixtures, CLI gate, and correctness-only paid smoke. | | **Durable JSON Output Design** | [`docs/plans/2026-07-21-durable-json-output-design.md`](docs/plans/2026-07-21-durable-json-output-design.md) | Opt-in atomic user-private result persistence for long buffered council runs and detached supervisors. | | **Adjudication Succession** | [`docs/plans/2026-09-03-adjudication-succession.md`](docs/plans/2026-09-03-adjudication-succession.md) | DSE-1512 adjudication succession implementation plan (typed failure categories, synthesizer chain, succession ledger). | +| **Bounded Cost Receipts** | [`docs/plans/2026-09-03-bounded-cost-receipts.md`](docs/plans/2026-09-03-bounded-cost-receipts.md) | DSE-1514 implementation plan: dated price snapshot + `cost_ceiling_usd` on every receipt/manifest (all-or-nothing, never an estimate), `max_output_tokens`, and the pre-flight `--max-spend-usd` spend gate with exit code `4`. | --- diff --git a/README.md b/README.md index 0e547d2..d31b053 100644 --- a/README.md +++ b/README.md @@ -537,6 +537,66 @@ not touch the network. Entries live under `$XDG_CACHE_HOME/conclave` (else `~/.cache/conclave`); a corrupt or unreadable entry is treated as a miss and never crashes a run. +## Cost ceilings and spend gate + +A council run has always reported *tokens*. It now also reports *dollars* — as a +**ceiling**, never an estimate. + +* An **estimate** is a guess. A wrong number inside an audit receipt is worse than + no number, which is why `estimated_cost` is `None` and always will be. +* A **ceiling** is a falsifiable claim: *"this run cost no more than $0.0412, + priced against snapshot `sha256:...` dated 2026-09-03."* You can check it + against your invoice. + +```bash +conclave ask "should we migrate?" --mode elite --max-output-tokens 4000 --max-spend-usd 0.40 +``` + +Before the first provider call, conclave enumerates the mode's worst-case call +plan (`Council.plan_calls`), prices every call at ceiling rates from a dated +snapshot committed to this repo, and **refuses** — exit code `4`, nothing ran, +nothing was spent — if the total exceeds your cap. `--max-output-tokens` (or +config `max_output_tokens`) is a **prerequisite**, not an option: output is the +only unbounded term in a call's cost, so a cap on it is what makes a dollar +ceiling possible at all. The three refusal messages, verbatim: + +| Condition | Message | +|---|---| +| `--max-spend-usd` with no output cap | `cannot bound spend: no output cap (set --max-output-tokens or config max_output_tokens)` | +| A planned call's model has no snapshot entry | `cannot bound spend: no priced rate for in snapshot ()` | +| The priced plan exceeds the cap | `refusing to run: reserved USD for calls exceeds the cap of USD` | + +It never falls back to a similar model's rate to dodge the second message — +inventing a number to get past the gate would defeat the gate. + +**All-or-nothing.** The prices are a hand-verified, dated file (`src/conclave/data/prices-*.json`), +not a live feed. A model whose published price could not be verified is simply +absent — which makes it unpriced, and makes the whole run's ceiling `None` with +`manifest.unpriced_models` naming it. A partial sum would read exactly like a +complete one, so there is no partial sum: one unpriced model or one unpriceable +receipt nulls the entire run-level `cost_ceiling_usd`. Two of the nine default +models are currently omitted from the snapshot for exactly this reason — +`groq/llama-3.3-70b-versatile` (moved to an Enterprise-only "Contact Sales" +tier) and `deepseek/deepseek-chat` (retired, its replacement is a different +model id conclave does not resolve) — re-pricing them is tracked in DSE-1537. + +**The adversarial worst case.** `run_adversarial` tries members as proposer in +council order until one produces a usable answer, then fans the rest out as +critics; a real run makes exactly `N` member calls no matter how many proposer +attempts fail. The byte-worst-case plan is therefore **1 proposer succeeding +immediately + N-1 critics**, since every critic call embeds the proposal's full +answer text (the more critics, the more upstream bytes) — and the judge call +embeds the proposal *and every critique*, so its input bound covers all `N` +prior outputs. + +| Exit code | Meaning | +|---|---| +| `0` | clean run | +| `1` | no usable answers | +| `2` | usage/config error | +| `3` | degraded — it ran, the judge/synthesizer step failed | +| `4` | **refused — nothing ran, nothing was spent** | + ## Test ```bash diff --git a/config.example.yml b/config.example.yml index 5d52f68..90d6432 100644 --- a/config.example.yml +++ b/config.example.yml @@ -42,3 +42,11 @@ cache: false # `--converge` / `--no-converge`. A high value (e.g. 0.95) only stops on # near-identical successive answers. # converge_threshold: 0.95 + +# Optional hard ceiling on OUTPUT tokens for every call a council makes -- members, +# synthesizer, judge, verdict extraction and its repair retry, and both streaming +# paths (OFF by default, unset). Override per invocation with --max-output-tokens. +# This is also the precondition for --max-spend-usd: a run whose output is +# unbounded cannot have its spend bounded, so the spend gate refuses (exit 4) +# rather than inventing a number. +# max_output_tokens: 4000 diff --git a/docs/PRODUCT_DESIGN_DOCUMENT.md b/docs/PRODUCT_DESIGN_DOCUMENT.md index 1853bff..ecd3d07 100644 --- a/docs/PRODUCT_DESIGN_DOCUMENT.md +++ b/docs/PRODUCT_DESIGN_DOCUMENT.md @@ -272,18 +272,23 @@ and cache hits (synthesize/raw builds its own richer one earlier). Pinned by `providers_considered/called/skipped` (each skip a `ProviderSkip{name, reason}`), `model_ids`, `generation_settings`, `receipts` (each a `ProviderExecutionReceipt{phase, attempt, outcome, name, provider, model_id, -generation_settings, latency_ms, usage, error_category, schema_valid, versions}`), +generation_settings, latency_ms, usage, error_category, schema_valid, versions, cost_ceiling_usd, +cost_basis}`), `total_latency_ms`, `total_usage`, `schema_valid`, `redacted_errors`, `adjudication_succession` (the per-role succession ledger: candidate, -attempt index, outcome, bounded failure category, HTTP status; never free text), and -verdict-provenance slots (`verdict_extraction: VerdictExtraction{model_id, +attempt index, outcome, bounded failure category, HTTP status; never free text), verdict-provenance +slots (`verdict_extraction: VerdictExtraction{model_id, prompt_version}` — the execution-trace hook — plus `verdict_type`, `consensus_method`, -`verdict_absent_reason`). Two deliberate honesty choices: +`verdict_absent_reason`), and the run-level pricing fields (v1.4, DSE-1514): `cost_ceiling_usd`, +`price_snapshot_digest`, `priced_as_of`, `unpriced_models`, `unpriced_receipts`, +`pricing_warnings` — see "Cost ceilings, never estimates" below. Two deliberate honesty choices: For buffered Elite, every attempted call becomes a receipt: `initial`, `critique`, `revision`, `synthesis`, `verdict_extraction`, and `verdict_repair` when repair is attempted. Receipts carry phase, attempt/outcome, provider/model identity, latency, available usage/cost, bounded error category, and prompt/schema/protocol versions; totals are recomputed from this complete ledger. Incomplete runs retain only calls actually attempted. - **No invented pricing.** Unknown per-call or aggregate `estimated_cost` stays `None`; a total is computed only when every actual call has trustworthy priced data. Usage is recorded when reported. + `estimated_cost` is a *guess* and stays `None` forever; `cost_ceiling_usd` (v1.4, below) is a + *falsifiable claim* computed from exact rates, and the two are never conflated in one field. - **Proven secret-safety.** `secret_safety` defaults to `unverified`, promoted to `verified_no_secrets` **only** after `scan_for_secret_material()` proves the serialized manifest free of forbidden substrings (`sk-`, `bearer`, `authorization`, `api_key`, `x-api-key`). Key @@ -309,6 +314,60 @@ by whether the candidate EVER answered across its initial call and same-model re not by whichever attempt happened to run last: a candidate that answered on either attempt is terminal for the role even if its other attempt hit an unrelated infrastructure error. +### Cost ceilings, never estimates (v1.4) + +`manifest.py` has always said `estimated_cost` stays `None` because "a wrong number +inside an audit receipt is worse than no number." That stands. What changed is that a +*different* claim is now available: not an estimate but a **ceiling** — +`cost_ceiling_usd`, computed with exact `Decimal` rates and `ROUND_CEILING` against a +dated, content-digested, vendor-cited snapshot committed to the repo +(`src/conclave/data/prices-.json`), and always accompanied by +`price_snapshot_digest` and `priced_as_of` so it is checkable rather than trusted. The +two live in separate fields on purpose; conflating them would let a guess inherit a +ceiling's credibility. + +Three rules keep the ceiling honest. **All-or-nothing:** any model in the run absent +from the snapshot, or any receipt that cannot be bounded, leaves the run-level ceiling +`None` with `unpriced_models` / `unpriced_receipts` naming why — a partial sum is +indistinguishable from a complete one and is the exact failure mode this design +prevents. Scope note: `unpriced_models` covers the models that actually ran (`model_ids` +plus every receipt's model), not members skipped for a missing key, which made no call +and cannot be billed. **Never a substitute rate:** an absent model is unpriced; a +similar model's rate is never borrowed, and a stale snapshot (older than +`PRICE_SNAPSHOT_MAX_AGE_DAYS = 90`) warns (`pricing_warnings`, bounded identifiers +only — see `conclave.council.PRICING_WARNING_VOCABULARY`) but still prices at the rates +it actually records. Two of the nine `registry.DEFAULT_MODELS` are currently omitted +from the shipped snapshot for exactly this reason (`groq/llama-3.3-70b-versatile`, +moved to an Enterprise-only tier; `deepseek/deepseek-chat`, retired with no successor +sharing that model id) — re-pricing them is tracked in DSE-1537. **Priced last:** +`Council._price_manifest` runs after `_ensure_manifest` and after the final receipt +append, so it can never miss the synthesis or verdict-repair calls; the snapshot's rate +digest joins cache identity (`CACHE_FORMAT_VERSION` `4` → `5`) so a hit can never serve +a ceiling that was never true of those rates. + +The pre-flight `--max-spend-usd` gate is the same arithmetic run forward, and requires +`--max-output-tokens` (or config `max_output_tokens`) as a precondition: output is the +only unbounded term in a call's cost, so an uncapped run cannot be bounded in dollars. +`Council.plan_calls` enumerates the worst-case plan — `raw` `N`, `synthesize` +`N + C + 2CV`, `vote` `N`, `debate` `N*R + C`, `adversarial` `N + C`, `elite` +`3N + C + 2CV`, with `C` the number of **keyed** synthesizer-chain candidates and the +verdict's repair retry always counted — bounds each call's input by UTF-8 bytes (plus +the sum of upstream output caps times `max_output_bytes_per_token` for calls that embed +a prior model's output), and refuses before the first call with one of three exact +messages: `cannot bound spend: no output cap (set --max-output-tokens or config +max_output_tokens)`; `cannot bound spend: no priced rate for in snapshot + ()`; or `refusing to run: reserved USD for calls exceeds the cap +of USD`. All three exit CLI code `4`. Refusing is the designed outcome for an +unbounded plan: inventing a number to get past the gate would defeat the gate. + +**Adversarial's byte-worst-case shape (DSE-1514 review, Fix A):** `run_adversarial` +tries members as proposer in council order until one succeeds, then fans the rest out +as critics — a real run always makes exactly `N` member calls, whatever the split. The +byte-worst case is **1 proposer succeeding immediately + N-1 critics**, since every +critic call embeds the proposal's full answer text (upstream=1 each) and maximizing +the critic count maximizes that embedded-output byte total; the judge call embeds the +proposal *and* every critique, so its upstream bound is `N` regardless of the split. + --- ## 5. Provider Support Matrix @@ -450,6 +509,11 @@ paid execution requires `--execute`, exact `--approve-spend-usd 10.00`, and an o never serialize that key; frozen `max_output_bytes_per_token` attestations bound inserted UTF-8 bytes. One call is in flight, reservations persist first, and resume never repeats interrupted cells. The smoke proves correctness only—not efficiency or decision quality—and remains not decision eligible. +H1's budget-matched ablations and H4's quality-per-dollar question now have a real +denominator rather than a guess: every run carries a `cost_ceiling_usd` with its +snapshot digest and capture date (§4a), so *"is Elite worth `3N + 2` calls?"* is +answerable **at** the decision instead of after the invoice. + The canonical roadmap is [`docs/plans/2026-07-17-decision-quality-roadmap.md`](plans/2026-07-17-decision-quality-roadmap.md): **H0** closes Elite correctness and wording gaps before merge; **H1** runs budget-matched, diff --git a/docs/plans/2026-09-03-bounded-cost-receipts.md b/docs/plans/2026-09-03-bounded-cost-receipts.md index dbd7537..a29b78c 100644 --- a/docs/plans/2026-09-03-bounded-cost-receipts.md +++ b/docs/plans/2026-09-03-bounded-cost-receipts.md @@ -56,7 +56,7 @@ Let **N** = `len(Council._available_members()[0])` (keyed members). Let **C** = | `synthesize` | `N + C + 2*C*V` | fan-out `N`; `_synthesize` walks up to `C` keyed candidates; `_apply_verdict` calls `extract_verdict` once per candidate and each one makes **1 initial + 1 repair** = `2C`. | | `vote` | `N` | `run_vote` fans out once; no adjudication, no verdict. | | `debate` | `N*R + C` | round 1 = `N`; rounds 2..R have at most `N` survivors each (drop-out only shrinks it) giving `N*R`; `_debate_synthesize` adds `C`. `converge_threshold` can only stop **early**. No verdict extraction. | -| `adversarial` | `N + C` | proposer attempts `k` (1..N) plus critics `N - k` equals `N` regardless of `k`; `_adversarial_judge` adds `C`. The all-proposers-fail path degrades to `_synthesize` over an empty `successful_answers`, which returns **before** any call, so still `N`. No verdict extraction. | +| `adversarial` | `N + C` | proposer attempts `k` (1..N) plus critics `N - k` equals `N` regardless of `k`; `_adversarial_judge` adds `C`. The all-proposers-fail path degrades to `_synthesize` over an empty `successful_answers`, which returns **before** any call, so still `N`. No verdict extraction. **Byte shape, corrected 2026-09-04 (Round 4 review, Fix A):** the count `N + C` was always right, but the ORIGINAL worked example in Task 10 gave every member call `upstream=0`, undercounting the input bound of every critic call. The byte-worst-case plan is 1 proposer call (`upstream=0`) + `N-1` critic calls (`upstream=1` each -- `_critic_messages_for` embeds the proposal's answer text in every critic call), plus one judge call per keyed chain candidate (`upstream=N` -- `judge_user` embeds the proposal and every critique). | | `elite` | `3N + C + 2*C*V` | `run_elite` fans out three times (initial / critique / revision), each at most `N`, giving `3N`; then `_synthesize` adds `C` and `_apply_verdict` adds `2C`. With a chain of one and verdict on: `3N + 3` — the ticket's `3N + 2` shape **plus the repair retry**, which is exactly the call the ticket says must not be forgotten. | Sanity check against the ticket: `synthesize` with `C=1, V=1` gives `N + 3`; `elite` with `C=1, V=1` gives `3N + 3`. Both match. @@ -2555,8 +2555,27 @@ def plan_calls( upstream=n_members, ) elif mode == "adversarial": - # k proposer attempts + (N - k) critics == N, for every k. - member_calls("proposal", template="", upstream=0) + # CORRECTED 2026-09-04 (Round 4 review, Fix A): the worked example + # originally shown here treated every member uniformly as + # `member_calls("proposal", template="", upstream=0)`, which gave + # the right TOTAL count (N + C) but the wrong per-call shape -- it + # never gave a critic call an upstream dependency on the proposal, + # so the byte bound for every critic call was silently too small. + # `run_adversarial` embeds the proposal's answer text in EVERY + # critic call (`_critic_messages_for`) and embeds the proposal AND + # every critique in the judge call (`judge_user`). A real run's k + # proposer attempts + (N-k) critics always equals N, for every k; + # the BYTE-worst-case is k=1 (one proposer succeeds immediately), + # which maximizes the number of upstream-embedding critic calls. + # The corrected shape: + if members: + member_calls("proposal", targets=members[:1], template="", upstream=0) + member_calls( + "critique", + targets=members[1:], + template=prompts.CRITIC_SYSTEM + prompts.critic_user("", ""), + upstream=1, + ) elif mode == "elite": member_calls("initial", template="", upstream=0) member_calls( From a0676d337b08685d4a3350a29ac2b9b6b3b07d7f Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 4 Sep 2026 17:47:46 -0400 Subject: [PATCH 08/10] fix(cli,council,pricing): validate spend/output caps (NaN, Infinity, non-positive), never render locals, price streaming cache hits, harden the snapshot loader (DSE-1514 review) Claude-Session: https://claude.ai/code/session_01K1dHPjZ1bZcE2GnX3KMMSH --- CHANGELOG.md | 14 ++++++ README.md | 6 ++- src/conclave/cli.py | 40 +++++++++++++++- src/conclave/council.py | 32 ++++++++++++- src/conclave/pricing.py | 15 +++++- tests/test_cli.py | 84 ++++++++++++++++++++++++++++++++++ tests/test_council.py | 26 +++++++++++ tests/test_pricing_snapshot.py | 56 +++++++++++++++++++++++ tests/test_spend_plan.py | 35 ++++++++++++++ tests/test_streaming.py | 45 ++++++++++++++++++ 10 files changed, 347 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f57f2ac..7cf1581 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,6 +68,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 call** with the reserved total, the cap, and the call count. New exit code **4**. A plan that cannot be bounded — no output cap, no snapshot, or an unpriced model — refuses with a distinct message rather than guessing. +- **Cap validation (DSE-1514 review).** `--max-spend-usd` and `Council(max_spend_usd=...)` now + reject `NaN` (every spelling: `NaN`, `-NaN`, `sNaN`, case-insensitive), `Infinity`/`inf` + (signed), and PEP-515 underscore literals (`0_5` reads as `5`, not `0.5`) — previously `NaN` + crashed uncaught at the cap comparison and `Infinity` silently disabled the gate. The CLI + applies a strict format allow-list before `Decimal(...)` ever runs (usage error, exit `2`, + names `--max-spend-usd`); `Council.__init__` enforces `is_finite()` and `> 0` independently for + library callers. `1e999999` is a deliberate exception: it is finite, just enormous, and stays + accepted. `--max-output-tokens 0`/negative now fails the same way (`min=1` on the CLI option, + plus a `ValueError` in `Council.__init__`) instead of reaching a provider as `max_tokens: 0` or + crashing inside `pricing.py`. +- `typer.Typer(..., pretty_exceptions_show_locals=False)` is now explicit rather than relying on + the installed typer version's default: `typer>=0.12.0` (this package's own floor) defaults that + flag `True`, which would render local variables — including the user's prompt — into an + unhandled exception's stderr traceback. ### Changed diff --git a/README.md b/README.md index d31b053..e3b80a6 100644 --- a/README.md +++ b/README.md @@ -558,7 +558,11 @@ snapshot committed to this repo, and **refuses** — exit code `4`, nothing ran, nothing was spent — if the total exceeds your cap. `--max-output-tokens` (or config `max_output_tokens`) is a **prerequisite**, not an option: output is the only unbounded term in a call's cost, so a cap on it is what makes a dollar -ceiling possible at all. The three refusal messages, verbatim: +ceiling possible at all. Both caps must be finite positive numbers: `--max-spend-usd` +rejects every spelling of `NaN`/`Infinity` and PEP-515 underscore literals +(`0_5` reads as `5`, not `0.5`) with exit code `2` before ever constructing a +`Decimal`, and `--max-output-tokens` requires a value of at least `1`. The +three refusal messages, verbatim: | Condition | Message | |---|---| diff --git a/src/conclave/cli.py b/src/conclave/cli.py index 224befb..e7a90df 100644 --- a/src/conclave/cli.py +++ b/src/conclave/cli.py @@ -15,6 +15,7 @@ import json import os +import re import tempfile from decimal import Decimal, InvalidOperation from pathlib import Path @@ -35,11 +36,29 @@ app = typer.Typer( add_completion=False, help="Bring-your-own-keys multi-model council. Fan a prompt to N models.", + # Explicit, not the version default (DSE-1514 review, F5): typer 0.12.0 + # (allowed by this package's own `typer>=0.12.0` floor) defaults this to + # True, which renders local variables -- including the user's PROMPT -- + # into an unhandled exception's stderr traceback. The installed 0.27.2 + # happens to default False, but that is version luck, not a guarantee. + pretty_exceptions_show_locals=False, ) app.add_typer(eval_app, name="eval") console = Console() err_console = Console(stderr=True) +# Strict allow-list for --max-spend-usd, applied BEFORE Decimal() ever runs +# (DSE-1514 review, F1). Requires a leading digit (no sign, so "-5" and +# "+5" are rejected the same way as any other malformed input), an optional +# fractional part, and an optional exponent. This rejects every spelling of +# non-finite input (`NaN`, `-NaN`, `sNaN`, `Infinity`, `inf`, case-insensitive +# and signed) and PEP-515 underscore grouping (`Decimal("0_5")` is `5`, not +# `0.5` -- a silent 10x cap the operator did not type) that `Decimal(...)` +# would otherwise accept outright. `1e999999` matches and is deliberately +# still accepted: it is an enormous but perfectly finite Decimal, and +# `Council.__init__`'s `is_finite()` check does not reject it either. +_SPEND_CAP_PATTERN = re.compile(r"^\d+(\.\d+)?([eE][+-]?\d+)?$") + def _result_to_dict(result: CouncilResult) -> dict: """Serialize a CouncilResult to a JSON-safe dict (no secrets included).""" @@ -647,6 +666,7 @@ def ask( "Defers to config `max_output_tokens` when unset. Required by " "--max-spend-usd: unbounded output cannot be bounded in dollars." ), + min=1, ), max_spend_usd: str | None = typer.Option( None, @@ -738,6 +758,18 @@ def ask( # destroy the exactness the whole cap rests on (Decimal(0.4) != Decimal("0.4")). spend_cap: Decimal | None = None if max_spend_usd is not None: + # A strict format allow-list runs BEFORE Decimal() (DSE-1514 review, + # F1): Decimal("NaN") is silently accepted by Decimal() and crashes + # uncaught at an ordering comparison; Decimal("Infinity") is accepted + # too and permanently disables the gate (nothing can ever exceed it); + # Decimal("0_5") is 5, not 0.5, via PEP-515 underscore grouping -- a + # silent 10x cap the operator did not type. Rejecting the spelling + # before conversion turns all three into the same clean usage error. + if not _SPEND_CAP_PATTERN.fullmatch(max_spend_usd): + err_console.print( + f"[red]--max-spend-usd must be an exact decimal amount, got '{max_spend_usd}'.[/red]" + ) + raise typer.Exit(code=2) try: spend_cap = Decimal(max_spend_usd) except InvalidOperation: @@ -745,8 +777,12 @@ def ask( f"[red]--max-spend-usd must be an exact decimal amount, got '{max_spend_usd}'.[/red]" ) raise typer.Exit(code=2) from None - if spend_cap <= 0: - err_console.print("[red]--max-spend-usd must be greater than zero.[/red]") + # Belt-and-suspenders on top of the format check: Council.__init__ + # enforces this identically for library callers who bypass the CLI. + if not spend_cap.is_finite() or spend_cap <= 0: + err_console.print( + "[red]--max-spend-usd must be a finite amount greater than zero.[/red]" + ) raise typer.Exit(code=2) cfg = load_config() diff --git a/src/conclave/council.py b/src/conclave/council.py index 286cd6e..fa887f3 100644 --- a/src/conclave/council.py +++ b/src/conclave/council.py @@ -86,6 +86,7 @@ SpendUnboundable, load_default_price_snapshot, reported_usage_cost, + reserve_cost, ) from .prompts import ELITE_PROMPT_VERSION, SYNTHESIS_PROMPT_VERSION from .providers import call_model, receipt_from_answer @@ -503,12 +504,33 @@ def __init__( self.max_output_tokens = ( self.config.max_output_tokens if max_output_tokens is None else max_output_tokens ) + # A zero/negative cap is not a ceiling at all: it either bypasses the + # provider call entirely (max_tokens=0) or crashes downstream in + # pricing.py's token-bound arithmetic. Reject it here so every caller + # (library or CLI) gets the same clean failure at construction (DSE-1514 + # review, F2). The CLI additionally enforces this with `min=1` on the + # typer option for a usage-error exit before Council is even reached. + if self.max_output_tokens is not None and self.max_output_tokens <= 0: + raise ValueError("max_output_tokens must be a positive integer") # A spend cap without an output cap is not enforceable: output is the # unbounded term. Refuse at construction rather than at the first call, # so a library caller cannot get halfway into a run before finding out. self.max_spend_usd = max_spend_usd - if max_spend_usd is not None and self.max_output_tokens is None: - raise SpendUnboundable(_NO_OUTPUT_CAP_MESSAGE) + if max_spend_usd is not None: + # Reject BEFORE any ordering comparison downstream (_reserve_plan's + # `reserved > self.max_spend_usd`): Decimal("NaN") raises + # InvalidOperation on any ordering comparison (an uncaught crash), + # and Decimal("Infinity") is finite-comparison-safe but silently + # disables the gate -- nothing can ever exceed it (DSE-1514 review, + # F1). `is_finite()` alone rejects both NaN and +/-Infinity without + # ever evaluating `<= 0` against a NaN, which would itself raise. + # This guards every library caller; the CLI applies its own + # stricter format check (rejecting non-finite spellings, signs, and + # underscore literals) before `Decimal(...)` is ever constructed. + if not max_spend_usd.is_finite() or max_spend_usd <= 0: + raise ValueError("max_spend_usd must be a finite positive Decimal") + if self.max_output_tokens is None: + raise SpendUnboundable(_NO_OUTPUT_CAP_MESSAGE) @staticmethod def _resolve_chain(spec: str | Sequence[str] | None, config: ConclaveConfig) -> list[str]: @@ -1717,6 +1739,12 @@ async def ask_stream(self, prompt: str, synthesize: bool = True) -> AsyncIterato hit = cache_mod.load(key) if hit is not None: logger.info("cache hit for %s stream (%s)", mode, key[:12]) + # Re-price on every hit, exactly like _cached_run (DSE-1514 + # review, F3): the manifest was priced at STORE time, so a + # replay without this call would report a stale + # `priced_as_of` / `price_snapshot_stale` verdict rather than + # today's -- a wrong number in a receipt. + self._price_manifest(hit) for event in self._replay_cached(hit): yield event return diff --git a/src/conclave/pricing.py b/src/conclave/pricing.py index c7ab600..55c793c 100644 --- a/src/conclave/pricing.py +++ b/src/conclave/pricing.py @@ -433,9 +433,22 @@ def load_default_price_snapshot() -> PriceSnapshot | None: try: with path.open(encoding="utf-8") as handle: payload = json.load(handle, parse_float=Decimal) + # A JSON payload that parses but is not an object (e.g. a bare `42` or a + # list) has no `.pop`, so it would otherwise crash the loader with an + # uncaught AttributeError -- violating the never-raises contract above + # (DSE-1514 review, F4). Non-finite/absurd rates inside a well-formed + # object are already rejected by `PriceRates`' `Field(gt=0)` (a Decimal + # field's pydantic-core validator requires a finite number), which + # raises `ValidationError` -- already in this except tuple -- before + # `PriceSnapshot.digest`/`_canonical_decimal` (which assumes a finite + # value) is ever reached from this loader. + if not isinstance(payload, dict): + raise TypeError( + f"price snapshot payload must be a JSON object, got {type(payload).__name__}" + ) payload.pop("_note", None) return PriceSnapshot.model_validate(payload) - except (OSError, json.JSONDecodeError, ValidationError, TypeError) as exc: + except (OSError, json.JSONDecodeError, ValidationError, TypeError, AttributeError) as exc: logger.warning("price snapshot %s is unusable: %s; pricing disabled", path.name, exc) return None diff --git a/tests/test_cli.py b/tests/test_cli.py index 8167074..1f99b64 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1181,3 +1181,87 @@ def test_the_json_payload_carries_the_ceiling_as_an_exact_string( assert isinstance(manifest["cost_ceiling_usd"], str) assert manifest["priced_as_of"] == "2026-09-03" assert manifest["price_snapshot_digest"].startswith("sha256:") + + +"""DSE-1514 review (Round 5): F1/F2/F5 -- non-finite/non-positive caps and locals.""" + + +def test_a_nan_spend_cap_is_a_usage_error_not_a_crash(keys): + from conclave.cli import app + + result = runner.invoke(app, ["ask", "q", "--council", "grok", "--max-spend-usd", "NaN"]) + assert result.exit_code == 2 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert "--max-spend-usd" in result.output + (result.stderr or "") + + +@pytest.mark.parametrize("cap", ["NaN", "-NaN", "sNaN", "snan", "nan"]) +def test_every_nan_spelling_is_rejected(keys, cap): + from conclave.cli import app + + result = runner.invoke(app, ["ask", "q", "--council", "grok", "--max-spend-usd", cap]) + assert result.exit_code == 2 + + +@pytest.mark.parametrize("cap", ["Infinity", "inf", "-Infinity", "-inf"]) +def test_an_infinite_spend_cap_is_rejected(keys, cap): + from conclave.cli import app + + result = runner.invoke(app, ["ask", "q", "--council", "grok", "--max-spend-usd", cap]) + assert result.exit_code == 2 + + +def test_an_underscored_spend_cap_is_rejected(keys): + """Decimal("0_5") is 5, not 0.5 -- a 10x cap the operator did not type.""" + from conclave.cli import app + + result = runner.invoke(app, ["ask", "q", "--council", "grok", "--max-spend-usd", "0_5"]) + assert result.exit_code == 2 + + +@pytest.mark.parametrize("cap", ["0", "-5"]) +def test_a_non_positive_output_cap_is_a_usage_error(keys, cap): + from conclave.cli import app + + result = runner.invoke( + app, ["ask", "q", "--council", "grok", "--max-output-tokens", cap, "--max-spend-usd", "5"] + ) + assert result.exit_code == 2 + assert result.exception is None or isinstance(result.exception, SystemExit) + + +def test_the_cli_never_renders_locals_in_a_traceback(): + assert cli.app.pretty_exceptions_show_locals is False + + +def test_an_absurdly_large_but_finite_spend_cap_is_accepted(monkeypatch, keys, patch_call_model): + """Documents the deliberate boundary: is_finite() does NOT reject 1e999999. + + A cap this large never refuses (nothing could exceed it), so the run + proceeds past the gate to a real (mocked, no-network) member call -- + proving the value was accepted as a valid Decimal, not merely that + parsing didn't crash. + """ + from conclave.cli import app + from tests.conftest import make_response + from tests.test_pricing_receipts import _install_snapshot, _snapshot + + _install_snapshot(monkeypatch, _snapshot("xai/grok-4.3")) + patch_call_model(lambda model_id, messages: make_response("ok")) + result = runner.invoke( + app, + [ + "ask", + "q", + "--council", + "grok", + "--mode", + "raw", + "--max-output-tokens", + "100", + "--max-spend-usd", + "1e999999", + ], + ) + assert result.exit_code == 0 + assert result.exit_code != 1 diff --git a/tests/test_council.py b/tests/test_council.py index cb8966d..3d3bc1a 100644 --- a/tests/test_council.py +++ b/tests/test_council.py @@ -7,6 +7,7 @@ from __future__ import annotations import asyncio +from decimal import Decimal import pytest @@ -358,6 +359,31 @@ def test_council_constructor_arg_beats_config_chain(): assert c.synthesizer_chain == ["claude"] +# --------------------------------------------------------------------------- # +# max_spend_usd / max_output_tokens constructor validation (DSE-1514 review, +# F1/F2): a library caller that bypasses the CLI's own format check must get +# the identical rejection at construction, never a crash or a silently +# disabled gate deeper in the call. +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize("bad_cap", [Decimal("NaN"), Decimal("Infinity"), Decimal("0")]) +def test_a_non_finite_or_non_positive_spend_cap_is_a_value_error(bad_cap): + with pytest.raises(ValueError, match="max_spend_usd must be a finite positive Decimal"): + Council( + models=["grok"], + config=ConclaveConfig(), + max_output_tokens=1_000, + max_spend_usd=bad_cap, + ) + + +@pytest.mark.parametrize("bad_output_cap", [0, -1]) +def test_a_non_positive_output_cap_is_a_value_error(bad_output_cap): + with pytest.raises(ValueError, match="max_output_tokens must be a positive integer"): + Council(models=["grok"], config=ConclaveConfig(), max_output_tokens=bad_output_cap) + + # --------------------------------------------------------------------------- # # prose synthesis routed through the adjudication succession seam (DSE-1512, task 5) # --------------------------------------------------------------------------- # diff --git a/tests/test_pricing_snapshot.py b/tests/test_pricing_snapshot.py index ecac031..d8c5c02 100644 --- a/tests/test_pricing_snapshot.py +++ b/tests/test_pricing_snapshot.py @@ -137,3 +137,59 @@ def test_a_missing_snapshot_directory_degrades_to_none(monkeypatch, tmp_path): assert pricing.load_default_price_snapshot() is None finally: pricing.load_default_price_snapshot.cache_clear() + + +def test_a_scalar_json_payload_degrades_to_none_not_an_attribute_error(monkeypatch, tmp_path): + """DSE-1514 review, F4: a JSON payload that parses but is not an object. + + ``payload.pop("_note", None)`` on an ``int`` raises an uncaught + ``AttributeError``, breaking the loader's documented never-raises contract. + """ + import json + + from conclave import pricing + + (tmp_path / "prices-2099-01-01.json").write_text(json.dumps(42)) + pricing.load_default_price_snapshot.cache_clear() + monkeypatch.setattr(pricing, "_price_data_dir", lambda: tmp_path) + try: + assert pricing.load_default_price_snapshot() is None + finally: + pricing.load_default_price_snapshot.cache_clear() + + +def test_a_non_finite_rate_in_the_file_degrades_to_none(monkeypatch, tmp_path): + """DSE-1514 review, F4: a ``"NaN"``/``"Infinity"`` rate must never raise. + + ``PriceRates``' ``Field(gt=0)`` on a ``Decimal`` already rejects a + non-finite value with a ``ValidationError`` (already in the loader's + except tuple) before ``PriceSnapshot.digest``/``_canonical_decimal`` -- + which assumes a finite value -- is ever reached from this loader. This + pins that existing safety net so it can never silently regress. + """ + payload = { + "snapshot_id": "conclave-default-prices-2099-01-01", + "captured_at": "2099-01-01", + "currency": "USD", + "entries": [ + { + "provider_id": "x", + "model_id": "x/y", + "input_ceiling_usd_per_million_tokens": "NaN", + "output_ceiling_usd_per_million_tokens": 1, + "max_output_bytes_per_token": 4, + "source_url": "https://example.test/pricing", + } + ], + } + import json + + from conclave import pricing + + (tmp_path / "prices-2099-01-01.json").write_text(json.dumps(payload)) + pricing.load_default_price_snapshot.cache_clear() + monkeypatch.setattr(pricing, "_price_data_dir", lambda: tmp_path) + try: + assert pricing.load_default_price_snapshot() is None + finally: + pricing.load_default_price_snapshot.cache_clear() diff --git a/tests/test_spend_plan.py b/tests/test_spend_plan.py index 5cf2ba2..6e6f157 100644 --- a/tests/test_spend_plan.py +++ b/tests/test_spend_plan.py @@ -221,6 +221,39 @@ def _elite_revision_case(): return call, messages +def _judge_case(): + """The adversarial judge: proposal + N-1 critiques, all at max length. + + DSE-1514 review, re-review important: the original byte-lower-bound suite + omitted this phase (real 24,602 B vs planned 24,640 B -- 38 bytes of + margin, unverified). The default proposer is the first requested member + (``modes.run_adversarial``'s ``council.requested_models[0]``), so the + remaining N-1 members are the critics whose real names/model ids appear + inside the judge's ``critique_blocks``, exactly as ``_adversarial_judge`` + builds it. + """ + from conclave import prompts + + plan = _council().plan_calls("adversarial", PROMPT) + call = _phase_call(plan, "judge") + members = _plan_members() + proposer_name = members[0][0] + answers = _max_len_answers() + proposal_text = answers[0].answer + critiques = answers[1:] + critique_blocks = "\n\n".join( + f"### Critique from {c.name} ({c.model_id})\n{c.answer}" for c in critiques + ) + messages = [ + {"role": "system", "content": prompts.JUDGE_SYSTEM}, + { + "role": "user", + "content": prompts.judge_user(PROMPT, proposer_name, proposal_text, critique_blocks), + }, + ] + return call, messages + + def _verdict_extraction_case(): from conclave.verdict_synthesis import _build_messages @@ -252,6 +285,7 @@ def _verdict_repair_case(): "case_builder", [ _critic_case, + _judge_case, _synthesis_case, _debate_round2_case, _elite_critique_case, @@ -261,6 +295,7 @@ def _verdict_repair_case(): ], ids=[ "adversarial-critique", + "adversarial-judge", "synthesize-synthesis", "debate-round-2", "elite-critique", diff --git a/tests/test_streaming.py b/tests/test_streaming.py index 5c7a9de..101baa9 100644 --- a/tests/test_streaming.py +++ b/tests/test_streaming.py @@ -578,6 +578,51 @@ async def fake_stream( assert len(member_deltas) == 2 +async def test_ask_stream_cache_hit_reprices_and_reports_current_staleness(monkeypatch, tmp_path): + """DSE-1514 review, F3: a replayed cache hit must be priced NOW, not at store time. + + Without ``Council._price_manifest(hit)`` before ``_replay_cached(hit)``, a + replayed run reports the ``priced_as_of`` / ``price_snapshot_stale`` verdict + computed when the run was FIRST stored, not the current one -- a wrong + number in a receipt. Store a run against a fresh snapshot, then swap in a + stale one (mirroring the staleness clock advancing) before replaying the + identical prompt; the replayed ``done`` result must reflect the now-stale + snapshot, exactly like a live run would. + """ + from datetime import date + + import conclave.streaming as streaming_mod + from tests.test_pricing_receipts import _install_snapshot, _snapshot + + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path)) + monkeypatch.setenv("XAI_API_KEY", "dummy-key") + + async def fake_stream( + name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None, **kwargs + ): + yield "x" + yield ModelAnswer(name=name, model_id=model_id, answer="x") + + monkeypatch.setattr(streaming_mod, "call_model_stream", fake_stream) + + _install_snapshot(monkeypatch, _snapshot("xai/grok-4.3", captured_at=date.today())) + council = Council(models=["grok"], config=_config(), cache=True) + first = [e async for e in council.ask_stream("hi", synthesize=False)] + assert first[-1].result.cached is False + assert "price_snapshot_stale" not in first[-1].result.manifest.pricing_warnings + + # The staleness clock has now "advanced": swap in a snapshot dated well + # past PRICE_SNAPSHOT_MAX_AGE_DAYS, exactly as it would look if the packaged + # snapshot simply aged past the 90-day threshold between the two calls. + _install_snapshot(monkeypatch, _snapshot("xai/grok-4.3", captured_at=date(2026, 1, 1))) + + second = [e async for e in council.ask_stream("hi", synthesize=False)] + done = second[-1] + assert done.type == "done" + assert done.result.cached is True + assert "price_snapshot_stale" in done.result.manifest.pricing_warnings + + def test_stream_event_done_carries_full_result_shape(): """StreamEvent('done') carries a CouncilResult that serializes secret-free.""" from conclave.models import CouncilResult From 4420274b6c55c640b3fc7fac8495866fa755ee68 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 4 Sep 2026 18:29:57 -0400 Subject: [PATCH 09/10] fix(council,docs): phase-aware reservation basis from the call plan; document the fourth refusal and the gate's scope (DSE-1514 QA C1/M1-M5) QA C1 (Round 4 half): _price_manifest re-gains reservation pricing for usage-less receipts, now phase-aware. A usage-less receipt's phase maps onto the SAME _PhaseSpec row Council.plan_calls would build for it (Council._reservation_row_for_phase): "member" for an untagged raw/synthesize/vote member call; "initial"/"critique"/"revision" for elite; "synthesis"/"judge"/"debate_final" for the adjudication row; "verdict_extraction"/"verdict_repair" for the verdict rows; "round-{n}" for a debate round, rebuilt with the right round number so its upstream count (0 for round 1, every member for round 2+) is exact. An adversarial proposal/critique receipt (phase=None on both shapes, so not distinguishable post hoc) finds no "member" row in adversarial's table and stays unpriced -- never silently priced as the wrong shape. No cap, or no matching row: unpriced (A1's rule). Extracted the shared arithmetic into _PhaseSpec.to_planned_call (used by both plan_calls and the reservation lookup) and Council._reserve_call (used by both _reserve_plan and the reservation lookup), so the template-bytes/framing/reservation formula is written exactly once. Re-instated test_a_failed_call_with_no_usage_is_priced_as_a_reservation_when_capped with the phase-aware expectation, and added a parametrized regression (tests/test_spend_plan.py) proving the reservation for EVERY phase that can be usage-less prices at or above the real worst-case message for that phase -- the assertion that makes the flat-constant bug impossible to re-introduce. QA M1: README + PDD Sec4a now document all four refusal messages (added "cannot bound spend: price snapshot unavailable"), each exit code 4. QA M2: Council.__init__'s max_spend_usd docstring and README's spend-gate section now state exactly where the gate lives (ask/ ask_stream and the mode wrappers) and which primitives bypass it (fan_out, synthesize_blocks, adjudicate, verdict_synthesis.extract_verdict, modes.run_* called directly). QA M3: src/conclave/evals/pricing.py's reserve_call_cost hoists upstream_output_token_ceilings into a tuple once, before either use -- a one-shot iterable (e.g. a generator) used to be exhausted by reserve_cost's own internal tuple(...) and then silently record an empty tuple on CallReservation. QA M4: README's worked example ceiling date corrected to 2026-09-04, the shipped snapshot's actual capture date. QA M5: widened test_pricing_warnings_are_a_closed_vocabulary_in_the_code to also scan the missing-snapshot path's pricing_warnings = [...] list-literal assignment, matching what the module's own comment on PRICING_WARNING_VOCABULARY has always claimed it covers. Claude-Session: https://claude.ai/code/session_01K1dHPjZ1bZcE2GnX3KMMSH --- README.md | 17 +- docs/PRODUCT_DESIGN_DOCUMENT.md | 7 +- src/conclave/council.py | 277 ++++++++++++++++++++++------- src/conclave/evals/pricing.py | 16 +- tests/test_pricing_core.py | 47 +++++ tests/test_pricing_receipts.py | 35 ++-- tests/test_secret_safety_matrix.py | 23 ++- tests/test_spend_plan.py | 147 +++++++++++++++ 8 files changed, 479 insertions(+), 90 deletions(-) diff --git a/README.md b/README.md index e3b80a6..bc579bc 100644 --- a/README.md +++ b/README.md @@ -545,7 +545,7 @@ A council run has always reported *tokens*. It now also reports *dollars* — as * An **estimate** is a guess. A wrong number inside an audit receipt is worse than no number, which is why `estimated_cost` is `None` and always will be. * A **ceiling** is a falsifiable claim: *"this run cost no more than $0.0412, - priced against snapshot `sha256:...` dated 2026-09-03."* You can check it + priced against snapshot `sha256:...` dated 2026-09-04."* You can check it against your invoice. ```bash @@ -562,16 +562,25 @@ ceiling possible at all. Both caps must be finite positive numbers: `--max-spend rejects every spelling of `NaN`/`Infinity` and PEP-515 underscore literals (`0_5` reads as `5`, not `0.5`) with exit code `2` before ever constructing a `Decimal`, and `--max-output-tokens` requires a value of at least `1`. The -three refusal messages, verbatim: +four refusal messages, verbatim (every one maps to exit code `4`): | Condition | Message | |---|---| | `--max-spend-usd` with no output cap | `cannot bound spend: no output cap (set --max-output-tokens or config max_output_tokens)` | +| No price snapshot could be loaded at all | `cannot bound spend: price snapshot unavailable` | | A planned call's model has no snapshot entry | `cannot bound spend: no priced rate for in snapshot ()` | | The priced plan exceeds the cap | `refusing to run: reserved USD for calls exceeds the cap of USD` | -It never falls back to a similar model's rate to dodge the second message — -inventing a number to get past the gate would defeat the gate. +It never falls back to a similar model's rate to dodge either unboundable +message — inventing a number to get past the gate would defeat the gate. + +**Where the gate lives.** The spend cap is enforced at exactly one chokepoint — +`ask`/`ask_stream` and the mode wrappers (`debate`, `adversarial`, `vote`, +`elite`, their `_sync` variants, and the CLI) — so a caller reaching directly +into a lower-level primitive (`Council.fan_out`, `Council.synthesize_blocks`, +`Council.adjudicate`, `verdict_synthesis.extract_verdict`, or a `conclave.modes` +`run_*` function called without going through the matching `Council` method) +makes real provider calls without the cap ever being consulted. **All-or-nothing.** The prices are a hand-verified, dated file (`src/conclave/data/prices-*.json`), not a live feed. A model whose published price could not be verified is simply diff --git a/docs/PRODUCT_DESIGN_DOCUMENT.md b/docs/PRODUCT_DESIGN_DOCUMENT.md index ecd3d07..3de47e0 100644 --- a/docs/PRODUCT_DESIGN_DOCUMENT.md +++ b/docs/PRODUCT_DESIGN_DOCUMENT.md @@ -353,11 +353,12 @@ only unbounded term in a call's cost, so an uncapped run cannot be bounded in do `3N + C + 2CV`, with `C` the number of **keyed** synthesizer-chain candidates and the verdict's repair retry always counted — bounds each call's input by UTF-8 bytes (plus the sum of upstream output caps times `max_output_bytes_per_token` for calls that embed -a prior model's output), and refuses before the first call with one of three exact +a prior model's output), and refuses before the first call with one of four exact messages: `cannot bound spend: no output cap (set --max-output-tokens or config -max_output_tokens)`; `cannot bound spend: no priced rate for in snapshot +max_output_tokens)`; `cannot bound spend: price snapshot unavailable` (no snapshot could +be loaded at all); `cannot bound spend: no priced rate for in snapshot ()`; or `refusing to run: reserved USD for calls exceeds the cap -of USD`. All three exit CLI code `4`. Refusing is the designed outcome for an +of USD`. All four exit CLI code `4`. Refusing is the designed outcome for an unbounded plan: inventing a number to get past the gate would defeat the gate. **Adversarial's byte-worst-case shape (DSE-1514 review, Fix A):** `run_adversarial` diff --git a/src/conclave/council.py b/src/conclave/council.py index fa887f3..e97d297 100644 --- a/src/conclave/council.py +++ b/src/conclave/council.py @@ -81,6 +81,7 @@ TokenUsage, ) from .pricing import ( + PriceRates, PriceSnapshot, SpendCapExceeded, SpendUnboundable, @@ -376,6 +377,49 @@ class _PhaseSpec: message_count: int = 2 contract: bool = False + def to_planned_call( + self, + *, + name: str, + model_id: str, + prompt_token_upper_bound: int, + max_output_tokens: int, + ) -> PlannedCall: + """Expand this row into one :class:`PlannedCall` for ``(name, model_id)`` (DSE-1514). + + The template-bytes/framing-bytes arithmetic (a structured-output + contract's schema bytes live INSIDE ``template`` already; ``contract`` + only adds the flat provider-side framing allowance for the schema + registration itself, never a second copy of the schema) is written + here exactly once and shared by two callers: :meth:`Council.plan_calls` + (expanding every target in ``self.targets``, BEFORE any call is made) + and :meth:`Council._price_manifest`'s phase-aware reservation (Round 4, + QA C1 -- pricing ONE already-made, usage-less receipt from the row + matching its phase). + + Args: + name: Friendly member / candidate name for this specific call. + model_id: Resolved provider-prefixed model id for this call. + prompt_token_upper_bound: UTF-8 bytes of the exact known prompt + content this call sends. + max_output_tokens: The hard output cap this call would carry. + + Returns: + The expanded :class:`PlannedCall`. + """ + return PlannedCall( + phase=self.phase, + name=name, + model_id=model_id, + prompt_token_upper_bound=prompt_token_upper_bound, + prompt_template_token_allowance=len(self.template.encode("utf-8")), + provider_framing_token_allowance=( + 64 + (16 * self.message_count) + (256 if self.contract else 0) + ), + upstream_output_call_count=self.upstream, + max_output_tokens=max_output_tokens, + ) + class Council: """A council of foundation models with an optional synthesizer. @@ -440,15 +484,30 @@ class Council: ``--max-spend-usd``: see :meth:`plan_calls`. max_spend_usd: Opt-in pre-flight spend cap in USD (DSE-1514). When set, every deliberation call (:meth:`ask`/:meth:`ask_stream` and their - mode wrappers) first enumerates :meth:`plan_calls`, prices it via - :meth:`_reserve_plan`, and raises :class:`conclave.pricing. - SpendCapExceeded` -- BEFORE any provider call -- when the reserved - total exceeds this cap. Requires ``max_output_tokens`` (explicit or - via config): an unbounded output cannot be bounded in dollars, so - setting this without a cap raises :class:`conclave.pricing. - SpendUnboundable` at construction time rather than at the first - call. ``None`` (the default) installs no gate at all -- byte- - identical to today. + mode wrappers -- :meth:`debate`, :meth:`adversarial`, :meth:`vote`, + :meth:`elite`, and their ``_sync`` variants) first enumerates + :meth:`plan_calls`, prices it via :meth:`_reserve_plan`, and raises + :class:`conclave.pricing.SpendCapExceeded` -- BEFORE any provider + call -- when the reserved total exceeds this cap. Requires + ``max_output_tokens`` (explicit or via config): an unbounded output + cannot be bounded in dollars, so setting this without a cap raises + :class:`conclave.pricing.SpendUnboundable` at construction time + rather than at the first call. ``None`` (the default) installs no + gate at all -- byte-identical to today. + + **The gate has exactly one chokepoint, :meth:`_cached_run` (plus + :meth:`ask_stream`'s own call to the same :meth:`_gate_live_run`), + and nothing routes around it implicitly.** A caller that reaches + into a lower-level primitive DIRECTLY -- :meth:`fan_out`, + :meth:`synthesize_blocks`, :meth:`adjudicate`, + :func:`conclave.verdict_synthesis.extract_verdict`, or any + :mod:`conclave.modes` ``run_*`` function called without going + through the matching :class:`Council` method -- makes real + provider calls WITHOUT ever consulting ``max_spend_usd``, however + large. This is a deliberate seam (those primitives are also used + to build the gate's own byte-accounting), not an oversight; a + caller composing a custom flow from them is responsible for its + own spend discipline. Example: >>> council = Council(models=["grok", "perplexity"], synthesizer="claude") @@ -887,29 +946,16 @@ def plan_calls( table = self._plan_table( mode, members=members, chain=chain, rounds=max(1, rounds), choices=choices ) - calls: list[PlannedCall] = [] - for spec in table: - # Fix A note: a structured-output contract's schema bytes are - # already INSIDE spec.template (the verdict probes are measured - # from the real message builders, schema included -- see - # conclave.verdict_synthesis). ``contract`` therefore only adds the - # flat provider-side framing allowance here, never a second copy - # of the schema on top of the prompt bound. - template_bytes = len(spec.template.encode("utf-8")) - framing = 64 + (16 * spec.message_count) + (256 if spec.contract else 0) - for name, model_id in spec.targets: - calls.append( - PlannedCall( - phase=spec.phase, - name=name, - model_id=model_id, - prompt_token_upper_bound=prompt_bytes, - prompt_template_token_allowance=template_bytes, - provider_framing_token_allowance=framing, - upstream_output_call_count=spec.upstream, - max_output_tokens=cap, - ) - ) + calls: list[PlannedCall] = [ + spec.to_planned_call( + name=name, + model_id=model_id, + prompt_token_upper_bound=prompt_bytes, + max_output_tokens=cap, + ) + for spec in table + for name, model_id in spec.targets + ] return CallPlan( mode=mode, @@ -942,19 +988,40 @@ def _reserve_plan(self, plan: CallPlan) -> Decimal: f"cannot bound spend: no priced rate for {call.model_id} " f"in snapshot {snapshot.digest()} ({snapshot.captured_at.isoformat()})" ) - total += reserve_cost( - rates, - prompt_token_upper_bound=call.prompt_token_upper_bound, - prompt_template_token_allowance=call.prompt_template_token_allowance, - provider_framing_token_allowance=call.provider_framing_token_allowance, - upstream_output_token_ceilings=( - (call.max_output_tokens,) * call.upstream_output_call_count - ), - upstream_output_bytes_per_token=rates.max_output_bytes_per_token, - max_output_tokens=call.max_output_tokens, - ).reserved_cost_usd + total += self._reserve_call(rates, call) return total + @staticmethod + def _reserve_call(rates: PriceRates, call: PlannedCall) -> Decimal: + """Price one :class:`PlannedCall` against ``rates`` (DSE-1514). + + The one place the reservation formula (input bound = prompt + + template + framing + upstream-output-as-bytes; output bound = the + call's own cap) is written, shared by :meth:`_reserve_plan` (pricing + an entire pre-flight :class:`CallPlan`, before any call is made) and + :meth:`_price_manifest` (Round 4, QA C1 -- pricing a single + already-made call's reservation when it reported no usable usage, + from the plan row matching its phase). + + Args: + rates: The model's exact ceiling rates. + call: The planned (or reconstructed) call to price. + + Returns: + The reserved cost in USD, quantized up. + """ + return reserve_cost( + rates, + prompt_token_upper_bound=call.prompt_token_upper_bound, + prompt_template_token_allowance=call.prompt_template_token_allowance, + provider_framing_token_allowance=call.provider_framing_token_allowance, + upstream_output_token_ceilings=( + (call.max_output_tokens,) * call.upstream_output_call_count + ), + upstream_output_bytes_per_token=rates.max_output_bytes_per_token, + max_output_tokens=call.max_output_tokens, + ).reserved_cost_usd + def _enforce_spend_cap( self, mode: str, @@ -1526,25 +1593,24 @@ def _price_manifest(self, result: CouncilResult) -> None: * the provider reported a trustworthy, non-zero usage figure (:func:`_usage_is_reported`) -> ``reported_usage_cost`` at ceiling rates, basis ``"reported_usage"``; - * usage is not reported -> unpriced (``None``/``None``), counted in - ``unpriced_receipts``. This covers three shapes deliberately treated - alike: the call FAILED (no usage was ever produced); the call - SUCCEEDED but the provider omitted the usage field - (:mod:`conclave.adapters.openai_compat` returns ``usage=None`` in - that case, including for some streamed responses -- this is a normal - outcome on a clean call, not a failure signal); or the provider - reported an all-zero ``TokenUsage`` (QA I1 -- a technically-present - but empty usage is the same "nothing to price" shape as ``None``, - and pricing it at ``$0.000000`` would assert a false floor rather - than an honest bound). A receipt without reported usage is NEVER - estimated from a flat reservation in this round: the input a - synthesis/judge/verdict call embeds is bounded by which PHASE it - belongs to (member calls carry only the prompt; synthesis/judge/ - verdict calls additionally embed one or more upstream answers), and - this round has no phase-aware call plan to reserve from -- pricing a - synthesis receipt from a flat member-sized allowance silently - under-counts it. See DSE-1514 Round 4 for the phase-aware - reservation that replaces this blanket "unpriced" rule. + * usage is not reported (the call FAILED; the call SUCCEEDED but the + provider omitted the usage field -- :mod:`conclave.adapters. + openai_compat` returns ``usage=None`` in that case, including for + some streamed responses, a normal outcome on a clean call, not a + failure signal; or the provider reported an all-zero ``TokenUsage``, + QA I1 -- a technically-present but empty usage is the same "nothing + to price" shape as ``None``, and pricing it at ``$0.000000`` would + assert a false floor rather than an honest bound) AND an output cap + is configured AND the receipt's ``phase`` maps onto a row of + :meth:`_plan_table` for this run's mode (:meth:`_reservation_row_for_phase`, + DSE-1514 Round 4, QA C1) -> that row's own pessimistic reservation -- + its REAL template bytes and upstream-embedding count, never a flat + constant that cannot tell a bare member call from a synthesis call + embedding N upstream answers -- basis ``"reservation"``; + * no usage, no cap, or the phase has no plan row (currently: an + adversarial proposal/critique receipt, whose ``phase`` is ``None`` + on both shapes and so cannot be told apart post hoc) -> unpriced. + Nothing is ever estimated from a guess. And at run level, ALL-OR-NOTHING: ``cost_ceiling_usd`` is the sum of every receipt ceiling only when ``unpriced_models`` is empty AND @@ -1611,10 +1677,23 @@ def _price_manifest(self, result: CouncilResult) -> None: receipt.cost_basis = "reported_usage" continue # No trustworthy usage: failed, successful-but-unreported, or - # all-zero. Unpriced -- see the docstring; never estimated. - receipt.cost_ceiling_usd = None - receipt.cost_basis = None - unpriced_receipts += 1 + # all-zero. Reserve from the plan row matching this receipt's + # phase when a cap exists and one matches (DSE-1514 Round 4, QA + # C1); otherwise unpriced -- see the docstring, never estimated. + row = None if cap is None else self._reservation_row_for_phase(receipt.phase, result) + if row is None: + receipt.cost_ceiling_usd = None + receipt.cost_basis = None + unpriced_receipts += 1 + continue + planned = row.to_planned_call( + name=receipt.name, + model_id=receipt.model_id, + prompt_token_upper_bound=len(result.prompt.encode("utf-8")), + max_output_tokens=cap, + ) + receipt.cost_ceiling_usd = self._reserve_call(rates, planned) + receipt.cost_basis = "reservation" warnings: list[str] = [] if unpriced_models: @@ -1639,6 +1718,72 @@ def _price_manifest(self, result: CouncilResult) -> None: # Re-stamp: the ceiling fields were written after _build_manifest's scan. manifest.secret_safety = verified_secret_safety(manifest) + def _reservation_row_for_phase( + self, phase: str | None, result: CouncilResult + ) -> _PhaseSpec | None: + """Map a usage-less receipt's ``phase`` onto its :meth:`_plan_table` row (DSE-1514 Round 4, QA C1). + + Rebuilds the SAME declarative phase table :meth:`plan_calls` would + enumerate for ``result.mode`` (down to the real vote choices and the + real round count, so the reconstructed row's template bytes match the + real call as closely as a post-hoc reconstruction can) and returns + the one row whose ``phase`` matches. The mapping is exact-string + equality, never a guess: + + * ``None`` (an untagged member-shaped call -- raw/synthesize/vote + members, or an adversarial proposal/critique, which are NOT + distinguishable post hoc since both share ``phase=None``) is + looked up as ``"member"``: matches raw/synthesize/vote's single + member row; adversarial has no row named ``"member"`` (its + member-shaped calls split into ``"proposal"``/``"critique"`` rows a + receipt's phase cannot currently pick between), so an adversarial + proposal/critique receipt deliberately finds no row here and stays + unpriced -- never silently priced as the wrong shape; + * ``"initial"``/``"critique"``/``"revision"`` (elite) match the + identically named elite row; + * ``"synthesis"``/``"judge"``/``"debate_final"`` match the + identically named adjudication row; + * ``"verdict_extraction"``/``"verdict_repair"`` match the identically + named verdict row; + * ``"round-{n}"`` (debate) rebuilds the table with ``rounds=n`` (or + the number of rounds actually run, whichever is larger) so the + row for round ``n`` specifically exists, with its real ``upstream`` + count (0 for round 1, every member for round 2+). + + Args: + phase: The receipt's ``phase`` (``None`` for an untagged call). + result: The in-flight result -- read for ``mode`` (which table to + build), ``rounds`` (debate's real round count), and + ``vote.choices`` (the real vote template). + + Returns: + The matching :class:`_PhaseSpec`, or ``None`` when this mode has + no row for ``phase`` -- the caller leaves the receipt unpriced. + """ + key = phase or "member" + members, _skipped = self._available_members() + chain = self._keyed_chain() + + if key.startswith("round-"): + try: + round_no = int(key.removeprefix("round-")) + except ValueError: + return None + rounds = max(round_no, len(result.rounds)) + table = self._plan_table( + "debate", members=members, chain=chain, rounds=rounds, choices=None + ) + elif result.mode not in _VALID_PLAN_MODES: + return None + else: + choices = result.vote.choices if result.vote is not None else None + rounds = max(1, len(result.rounds)) + table = self._plan_table( + result.mode, members=members, chain=chain, rounds=rounds, choices=choices + ) + + return next((spec for spec in table if spec.phase == key), None) + def _append_manifest_receipts( self, result: CouncilResult, diff --git a/src/conclave/evals/pricing.py b/src/conclave/evals/pricing.py index c0d5623..dc9a9e6 100644 --- a/src/conclave/evals/pricing.py +++ b/src/conclave/evals/pricing.py @@ -212,13 +212,25 @@ def reserve_call_cost( the product run path, while the eval-only ``CallReservation`` contract -- including ``model_revision`` and the frozen eval ``schema_version`` that :func:`hash_price_entries` depends on -- is assembled here unchanged. + + ``upstream_output_token_ceilings`` is materialized into a tuple ONCE, + before either use below (QA M3): a one-shot iterable (e.g. a generator + expression) passed by a caller would otherwise be fully consumed by + :func:`conclave.pricing.reserve_cost`'s own internal ``tuple(...)`` call, + leaving nothing for the second ``tuple(...)`` that used to build + ``CallReservation.upstream_output_token_ceilings`` here -- silently + recording an empty tuple on the reservation even though the real values + were correctly priced. Passing the SAME materialized tuple to both keeps + the recorded reservation and the priced amounts provably consistent + regardless of what kind of iterable the caller passed in. """ + upstream_ceilings = tuple(upstream_output_token_ceilings) amounts = reserve_cost( price.as_rates(), prompt_token_upper_bound=prompt_token_upper_bound, prompt_template_token_allowance=prompt_template_token_allowance, provider_framing_token_allowance=provider_framing_token_allowance, - upstream_output_token_ceilings=upstream_output_token_ceilings, + upstream_output_token_ceilings=upstream_ceilings, upstream_output_bytes_per_token=upstream_output_bytes_per_token, max_output_tokens=max_output_tokens, ) @@ -229,7 +241,7 @@ def reserve_call_cost( prompt_token_upper_bound=prompt_token_upper_bound, prompt_template_token_allowance=prompt_template_token_allowance, provider_framing_token_allowance=provider_framing_token_allowance, - upstream_output_token_ceilings=tuple(upstream_output_token_ceilings), + upstream_output_token_ceilings=upstream_ceilings, upstream_output_bytes_per_token=upstream_output_bytes_per_token, input_token_upper_bound=amounts.input_token_upper_bound, output_token_upper_bound=amounts.output_token_upper_bound, diff --git a/tests/test_pricing_core.py b/tests/test_pricing_core.py index 1834d8d..b13936d 100644 --- a/tests/test_pricing_core.py +++ b/tests/test_pricing_core.py @@ -162,6 +162,53 @@ def test_eval_reserve_call_cost_delegates_to_the_shared_arithmetic(): assert reservation.schema_version == "conclave_eval_v1" +def test_eval_reserve_call_cost_records_a_one_shot_generators_ceilings(): + """QA M3: a generator passed for ``upstream_output_token_ceilings`` is not consumed twice. + + ``reserve_call_cost`` used to hand the caller's iterable to + ``conclave.pricing.reserve_cost`` (which exhausts it via its own internal + ``tuple(...)``) and THEN call ``tuple(...)`` on the SAME iterable a second + time to build ``CallReservation.upstream_output_token_ceilings`` -- for a + one-shot iterable like a generator expression, the second ``tuple(...)`` + silently produced ``()`` even though the values were correctly priced. + """ + from conclave.evals.pricing import ModelPrice, reserve_call_cost + + price = ModelPrice( + provider_id="fictional-provider-a", + model_id="fictional-model-a", + model_revision="fixture-r1", + input_ceiling_usd_per_million_tokens=Decimal("1.234567"), + output_ceiling_usd_per_million_tokens=Decimal("4.567891"), + max_output_bytes_per_token=4, + ) + ceilings = (256, 512) + + reservation = reserve_call_cost( + price, + prompt_token_upper_bound=900, + prompt_template_token_allowance=12, + provider_framing_token_allowance=96, + upstream_output_token_ceilings=(c for c in ceilings), # one-shot generator + upstream_output_bytes_per_token=4, + max_output_tokens=1_024, + ) + + assert reservation.upstream_output_token_ceilings == ceilings + # The reservation was priced from the REAL ceilings too, not from an + # empty tuple -- confirms the fix isn't just re-recording a stale value. + tupled = reserve_call_cost( + price, + prompt_token_upper_bound=900, + prompt_template_token_allowance=12, + provider_framing_token_allowance=96, + upstream_output_token_ceilings=ceilings, + upstream_output_bytes_per_token=4, + max_output_tokens=1_024, + ) + assert reservation.reserved_cost_usd == tupled.reserved_cost_usd + + def test_live_reported_usage_cost_delegates_to_the_shared_arithmetic(): from conclave.evals.live import _reported_usage_cost from conclave.evals.pricing import ModelPrice diff --git a/tests/test_pricing_receipts.py b/tests/test_pricing_receipts.py index 559ccb2..19952f9 100644 --- a/tests/test_pricing_receipts.py +++ b/tests/test_pricing_receipts.py @@ -316,20 +316,24 @@ async def zero_usage(name, model_id, messages, **kwargs): assert "unpriced_receipts_present" in manifest.pricing_warnings -async def test_a_failed_call_with_no_usage_is_unpriced_even_when_capped_this_round( +async def test_a_failed_call_with_no_usage_is_priced_as_a_reservation_when_capped( monkeypatch, keys ): - """DSE-1514 Round 3 review, interim: a cap does not resurrect flat-constant reservation. + """DSE-1514 Round 4: the phase-aware reservation basis, end to end. Round 3 removed the flat-allowance reservation branch entirely (QA C1): it silently mis-priced synthesis/judge/verdict calls, which embed - upstream output, by 3-9x. So even with a real ``max_output_tokens`` cap - threaded through the constructor (this commit), a usage-less receipt - stays unpriced -- exactly like the uncapped case above -- until Round 4's - phase-aware reservation basis lands (see the test with "when_capped" in - its name for that end state). + upstream output, by 3-9x, while never distinguishing a bare member call + from one embedding N upstream answers. Round 4 re-adds reservation + pricing for a usage-less receipt, now derived from the SAME plan-table + row :meth:`Council.plan_calls` would build for this receipt's phase (see + ``Council._reservation_row_for_phase``) -- never a flat constant. Proves + the two pricing paths (reported-usage and phase-aware reservation) + coexist cleanly in one manifest: an all-or-nothing ceiling sums BOTH + kinds of priced receipt, not just one. """ import conclave.council as council_mod + from conclave.council import Council from conclave.models import ModelAnswer, TokenUsage _install_snapshot(monkeypatch, _snapshot("xai/grok-4.3", "gemini/gemini-2.5-pro")) @@ -360,12 +364,15 @@ async def flaky(name, model_id, messages, **kwargs): succeeded = receipts_by_model["xai/grok-4.3"] assert failed.usage is None - assert failed.cost_basis is None - assert failed.cost_ceiling_usd is None + assert failed.phase is None # an untagged raw-mode member call -> the "member" row + assert failed.cost_basis == "reservation" + assert failed.cost_ceiling_usd is not None assert succeeded.cost_basis == "reported_usage" - # A capped, usage-less receipt is unpriced this round, so the run-level - # ceiling stays None even though the OTHER receipt priced cleanly. - assert manifest.unpriced_receipts == 1 - assert manifest.cost_ceiling_usd is None - assert "unpriced_receipts_present" in manifest.pricing_warnings + # All-or-nothing, but every receipt IS priced (one by usage, one by a + # phase-aware reservation), so the run-level ceiling is a real sum, not None. + assert manifest.unpriced_models == [] + assert manifest.unpriced_receipts == 0 + assert manifest.cost_ceiling_usd == sum( + (receipt.cost_ceiling_usd for receipt in manifest.receipts), Decimal("0") + ) diff --git a/tests/test_secret_safety_matrix.py b/tests/test_secret_safety_matrix.py index 754c9ea..c20cc86 100644 --- a/tests/test_secret_safety_matrix.py +++ b/tests/test_secret_safety_matrix.py @@ -513,7 +513,19 @@ def test_pricing_fields_never_un_verify_the_stamp(): def test_pricing_warnings_are_a_closed_vocabulary_in_the_code(): - """No pricing warning may be built by interpolation.""" + """No pricing warning may be built by interpolation. + + Scans BOTH shapes ``Council._price_manifest`` uses to populate + ``pricing_warnings`` (DSE-1514 QA M5): the normal path's sequence of + ``warnings.append(...)`` calls, AND the missing-snapshot path's single + list-literal assignment (``manifest.pricing_warnings = [...]``) -- the + module's own top-of-file comment on :data:`conclave.council. + PRICING_WARNING_VOCABULARY` has claimed both are covered since the + vocabulary was introduced; this test previously only scanned the first. + The list-literal pattern requires a ``[`` immediately after ``=``, so it + does not match ``manifest.pricing_warnings = warnings`` (assigning the + already-scanned variable back onto the manifest) -- only a literal. + """ import re from pathlib import Path @@ -527,6 +539,15 @@ def test_pricing_warnings_are_a_closed_vocabulary_in_the_code(): f"pricing warning must be a literal, got {expression}" ) + list_literals = re.findall(r"pricing_warnings\s*=\s*(\[[^\]]*\])", source) + assert list_literals, "the pricing_warnings list-literal assignment moved; update this guard" + for literal in list_literals: + elements = [item.strip() for item in literal.strip("[]").split(",") if item.strip()] + for element in elements: + assert element.startswith('"') and element.endswith('"'), ( + f"pricing warning must be a literal, got {element}" + ) + async def test_a_fully_populated_priced_manifest_still_stamps_verified( monkeypatch, keys, patch_call_model diff --git a/tests/test_spend_plan.py b/tests/test_spend_plan.py index 6e6f157..4c930ea 100644 --- a/tests/test_spend_plan.py +++ b/tests/test_spend_plan.py @@ -2,9 +2,15 @@ from __future__ import annotations +from decimal import Decimal + import pytest from conclave.council import Council +from conclave.manifest import ModelHarnessManifest, ProviderExecutionReceipt +from conclave.models import CouncilResult, DebateRound +from conclave.pricing import reserve_cost +from tests.test_pricing_receipts import _install_snapshot, _snapshot MEMBERS = ["grok", "gemini", "openai"] # N = 3 @@ -317,3 +323,144 @@ def test_the_planned_byte_bound_never_falls_below_the_real_worst_case_message(ke assert real_bytes <= call.input_bytes_bound( upstream_output_bytes_per_token=MAX_OUTPUT_BYTES_PER_TOKEN ) + + +# --------------------------------------------------------------------------- # +# DSE-1514 Round 4, QA C1: a usage-less RECEIPT's phase-aware reservation +# (Council._price_manifest, via Council._reservation_row_for_phase) must never +# price below the real worst-case message for its phase -- the regression that +# makes the flat-constant bug (a synthesis/judge/verdict/revision receipt, +# which embeds upstream output, priced 3-9x low) impossible to re-introduce. +# --------------------------------------------------------------------------- # + + +def _floor_reservation_usd(rates, real_bytes: int, cap: int) -> Decimal: + """The cheapest a phase's real worst-case message could possibly cost. + + Treats ``real_bytes`` as pure prompt content with ZERO extra template/ + framing allowance, so this is a floor: the real reservation + ``Council._price_manifest`` prices from a matched plan row can only be >= + this (its own template/framing/upstream terms are non-negative additions + on top of at least as many bytes), never below it. + """ + return reserve_cost( + rates, + prompt_token_upper_bound=real_bytes, + prompt_template_token_allowance=0, + provider_framing_token_allowance=0, + upstream_output_token_ceilings=(), + upstream_output_bytes_per_token=MAX_OUTPUT_BYTES_PER_TOKEN, + max_output_tokens=cap, + ).reserved_cost_usd + + +def _priced_receipt(council, *, mode: str, phase: str | None, model_id: str, rounds_count: int): + """Price ONE hand-built, usage-less receipt through the real pricing path. + + Bypasses a full simulated council run (which would need a distinct + provider-mocking scenario per phase) by constructing the minimal + :class:`CouncilResult` + manifest ``_price_manifest`` actually reads: + ``mode`` (which plan table to rebuild), ``rounds`` (debate's real round + count), and one receipt of the phase under test carrying no usage. + """ + receipt = ProviderExecutionReceipt( + name="target", provider=model_id.split("/", 1)[0], model_id=model_id, phase=phase + ) + result = CouncilResult( + prompt=PROMPT, + mode=mode, + rounds=[DebateRound(round_number=n, answers=[]) for n in range(1, rounds_count + 1)], + ) + result.manifest = ModelHarnessManifest( + request_id="r", conclave_version="0", mode=mode, model_ids=[model_id], receipts=[receipt] + ) + council._price_manifest(result) + return result.manifest.receipts[0] + + +# (label, receipt phase, mode, debate rounds run, model id, real-message builder) +_PHASE_RESERVATION_CASES = [ + ("member", None, "raw", 0, "xai/grok-4.3", lambda: [{"role": "user", "content": PROMPT}]), + ("round-2", "round-2", "debate", 2, "xai/grok-4.3", lambda: _debate_round2_case()[1]), + ( + "synthesis", + "synthesis", + "synthesize", + 0, + "anthropic/claude-sonnet-4-6", + lambda: _synthesis_case()[1], + ), + ("judge", "judge", "adversarial", 0, "anthropic/claude-sonnet-4-6", lambda: _judge_case()[1]), + ( + "verdict_extraction", + "verdict_extraction", + "synthesize", + 0, + "anthropic/claude-sonnet-4-6", + lambda: _verdict_extraction_case()[1], + ), + ( + "verdict_repair", + "verdict_repair", + "synthesize", + 0, + "anthropic/claude-sonnet-4-6", + lambda: _verdict_repair_case()[1], + ), + ("critique", "critique", "elite", 0, "xai/grok-4.3", lambda: _elite_critique_case()[1]), + ("revision", "revision", "elite", 0, "xai/grok-4.3", lambda: _elite_revision_case()[1]), +] + + +@pytest.mark.parametrize( + ("phase", "receipt_phase", "mode", "rounds_count", "model_id", "messages_builder"), + _PHASE_RESERVATION_CASES, + ids=[case[0] for case in _PHASE_RESERVATION_CASES], +) +def test_phase_aware_reservation_never_prices_below_the_real_worst_case_message( + monkeypatch, keys, phase, receipt_phase, mode, rounds_count, model_id, messages_builder +): + """DSE-1514 Round 4, QA C1: the assertion that makes the regression impossible. + + For every phase whose receipt can be usage-less and reservation-priced, + prices a manifest holding exactly one such receipt (``usage=None``, + ``phase=receipt_phase`` -- ``None`` for an untagged raw-mode member call) + and proves the reservation is >= the cost of the REAL worst-case message + for that phase (built via the actual mode/prompt builders in + ``tests.test_spend_plan``'s existing case functions, never a hand + re-derivation), priced at the input rate alone with zero extra allowance + -- a floor the real reservation's own non-negative template/framing/ + upstream terms can only sit at or above. A flat, phase-blind constant + (the QA C1 bug) would fail this for every upstream-embedding phase + (synthesis, judge, verdict_extraction, verdict_repair, revision). + """ + _install_snapshot( + monkeypatch, + _snapshot( + "xai/grok-4.3", + "gemini/gemini-2.5-pro", + "openai/gpt-4.1", + "anthropic/claude-sonnet-4-6", + ), + ) + council = _council() + messages = messages_builder() + real_bytes = sum(len(m["content"].encode("utf-8")) for m in messages) + + priced = _priced_receipt( + council, + mode=mode, + phase=receipt_phase, + model_id=model_id, + rounds_count=rounds_count, + ) + + assert priced.cost_basis == "reservation" + assert priced.cost_ceiling_usd is not None + + import conclave.council as council_mod + + snapshot = council_mod.load_default_price_snapshot() + rates = snapshot.rates_for(model_id) + floor = _floor_reservation_usd(rates, real_bytes, MAX_OUTPUT_TOKENS) + assert priced.cost_ceiling_usd >= floor From 8e83cf62de55d0672069beae30563fad025725fa Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 4 Sep 2026 18:53:18 -0400 Subject: [PATCH 10/10] fix(council): record the output cap on debate per-round receipts (DSE-1514 QA I2) `_build_debate_manifest` (added for QA C2) built its per-round receipts and the debate manifest's `generation_settings` without `max_output_tokens`, so a capped debate run attested to calls made without a cap while `fan_out` had actually applied one. Every other mode recorded it. Route both through the same `_generation_settings()` / `receipt_from_answer(max_output_tokens=)` path the other builders use, and pin it with a per-mode regression test (fails on debate only without this change). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01K1dHPjZ1bZcE2GnX3KMMSH --- src/conclave/council.py | 3 ++- tests/test_manifest_all_modes.py | 42 ++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/conclave/council.py b/src/conclave/council.py index e97d297..3f6d334 100644 --- a/src/conclave/council.py +++ b/src/conclave/council.py @@ -1474,6 +1474,7 @@ def _build_debate_manifest( answer, temperature=self.temperature, timeout=self.timeout, + max_output_tokens=self.max_output_tokens, phase=f"round-{debate_round.round_number}", ) for debate_round in result.rounds @@ -1489,7 +1490,7 @@ def _build_debate_manifest( ProviderSkip(name=name, reason="no API key in environment") for name in skipped ], model_ids=[model_id for _name, model_id in members], - generation_settings={"temperature": self.temperature, "timeout": self.timeout}, + generation_settings=self._generation_settings(), receipts=receipts, ) self._recompute_manifest_accounting(manifest) diff --git a/tests/test_manifest_all_modes.py b/tests/test_manifest_all_modes.py index 1af3672..2b974e7 100644 --- a/tests/test_manifest_all_modes.py +++ b/tests/test_manifest_all_modes.py @@ -665,3 +665,45 @@ async def test_elite_prices_every_one_of_its_3n_plus_receipts(monkeypatch, keys, assert len(manifest.receipts) >= 3 * 3 assert all(r.cost_ceiling_usd is not None for r in manifest.receipts) assert manifest.unpriced_receipts == 0 + + +# --------------------------------------------------------------------------- # +# DSE-1514 QA I2: a configured output cap is recorded on EVERY mode's manifest +# and on every one of its receipts -- including debate's per-round receipts, +# which are built by ``_build_debate_manifest`` rather than ``_build_manifest``. +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize("mode", ["synthesize", "raw", "debate", "adversarial", "vote", "elite"]) +async def test_every_mode_records_the_output_cap_on_manifest_and_receipts( + monkeypatch, keys, patch_call_model, mode +): + """The cap the calls were made with is what the receipts attest to.""" + from conclave.council import Council + from tests.conftest import make_response + + patch_call_model(lambda model_id, messages: make_response("ok")) + council = Council( + models=["grok", "gemini", "openai"], + synthesizer="openai", + extract_verdict=False, + max_output_tokens=512, + ) + + if mode == "debate": + result = await council.debate("q", rounds=2) + elif mode == "adversarial": + result = await council.adversarial("q") + elif mode == "vote": + result = await council.vote("q", choices=["a", "b"]) + elif mode == "elite": + result = await council.elite("q") + else: + result = await council.ask("q", synthesize=(mode == "synthesize")) + + manifest = result.manifest + assert manifest.receipts, mode + assert manifest.generation_settings["max_output_tokens"] == 512 + assert [r.generation_settings.get("max_output_tokens") for r in manifest.receipts] == [ + 512 + ] * len(manifest.receipts)