diff --git a/.ci/behavior-ownership/partition.v1.json b/.ci/behavior-ownership/partition.v1.json index d8cd48b7e..f9192a2d8 100644 --- a/.ci/behavior-ownership/partition.v1.json +++ b/.ci/behavior-ownership/partition.v1.json @@ -76,6 +76,10 @@ "group": "artifacts", "target": "backend/app/adapters/checkers/model_runtime.py" }, + { + "group": "artifacts", + "target": "backend/app/adapters/checkers/openrouter_model_provider.py" + }, { "group": "shared", "target": "backend/app/adapters/github/app_facts.py" @@ -104,6 +108,10 @@ "group": "shared", "target": "backend/app/adapters/guide_document_tools/service.py" }, + { + "group": "shared", + "target": "backend/app/adapters/model_usage/openrouter.py" + }, { "group": "shared", "target": "backend/app/adapters/observability.py" @@ -136,10 +144,6 @@ "group": "lifecycle", "target": "backend/app/adapters/project_agents/openai_workspace.py" }, - { - "group": "lifecycle", - "target": "backend/app/adapters/project_agents/openrouter_policy.py" - }, { "group": "lifecycle", "target": "backend/app/adapters/project_agents/provider_resilience.py" @@ -2533,7 +2537,7 @@ "target": "backend/scripts/validate_test_lane_evidence.py" } ], - "authority_digest": "75e9af95d5cd0ca50daa68772cfce8a5e166c2147a1ce2611d4d2e1ac40b469a", + "authority_digest": "6822cec6baeab801bdfdc9a21bb901d68e84d3e2de91736e6d2786591ed4a3e1", "protected_base_commit": "7676ce4347db0c9694962a9b587a20765e16eac6", "schema": "workstream.behavior-ownership-partition.v1" } diff --git a/.commitrail/changes/post-model-provider-selection.md b/.commitrail/changes/post-model-provider-selection.md new file mode 100644 index 000000000..c0dd4e970 --- /dev/null +++ b/.commitrail/changes/post-model-provider-selection.md @@ -0,0 +1,171 @@ +# POST model provider selection + +- Initiative: None +- Durable disposition: Complete +- Intended merge outcome: Model-backed POST checks select their supported provider from the locked project profile; OpenRouter-only deployments need no OpenAI credential. + +## Intent + +The approved project checks determine whether inference is needed. Ordinary POST +checks must remain independent of models. For inference, the provider/model is a +profile choice, not an OpenAI requirement imposed by the proxy. Preserve shared +project budgets, exact paid-call custody, immutable evidence and no blind retry. + +## Gap addressed + +Before this repair, `configured_model_provider` accepted only `openai_responses/responses.v1` and its +factory registered only OpenAI. Setup already supported OpenRouter. Hosted proxy +composition and entrypoint required both provider secrets regardless of selection. +The optional model execution wrapper already passed ordinary checks through. + +## Bounded change + +### Allowed + +- `backend/app/adapters/checkers/{__init__,model_runtime,model_provider,openrouter_model_provider}.py`: profile-driven typed provider composition and bounded OpenRouter POST adapter. +- `backend/app/adapters/model_usage/openrouter.py`, affected imports in `backend/app/adapters/project_agents/{setup_model_handler,openai_agent_sdk,omnicore_agent}.py` and removal of `backend/app/adapters/project_agents/openrouter_policy.py`: share existing provider HTTP/metadata/pricing/request controls without duplicating setup behavior. +- `backend/app/interfaces/checker_model_provider.py`: distinguish bounded response IDs from slash-qualified model identities. +- `backend/tests/checkers/model_provider/{test_openai_responses,test_openrouter}.py`, `backend/tests/checkers_model_calls/{test_model_runtime,test_proxy_orchestration,test_composed_execution,test_setup_model_handler,test_setup_model_handler_postgresql}.py`, `backend/tests/test_omnicore_agent_sdk.py`: discriminating provider, custody and non-model regressions. +- `deploy/hosted/{compose.inference.yml,compose.model-openai.yml,compose.model-openrouter.yml,preflight.py,release.env.example,README.md}`, `docker/backend/release-entrypoint.sh`, `scripts/test_hosted_release_config.py`, affected `scripts/test_hosted_release_images.py`: mount/load only configured provider credentials, preserving role separation and cached-image proof. +- `backend/.env.example`, `README.md`, `docs/operations_checker_models.md`, `docs/roadmap_status.md`: reconcile supported configured providers and actual proof limits. +- Exact registrations in `.ci/behavior-ownership/partition.v1.json`, `backend/scripts/{behavior_ownership,test_lane_catalogue}.py`, and `backend/tests/{test_behavior_ownership,test_ci_lane_catalogue}.py` where required for changed/new files; regenerate canonical digests without weakening gates. +- `backend/tests/test_artifact_architecture.py`: reconcile the exact concrete-provider composition inventory and protect both provider constructors against out-of-root construction. +- This change record. + +### Not allowed + +- No new budget/account/authorization subsystem, routing/acceptance changes, provider fallback, hidden paid retry or deletion of retained evidence. +- No POST model requirement for ordinary checkers. No checker image redesign or takeover of active PR #677. +- No new CI workflows/lanes, weaker assertions/skips/timeouts, dependency additions or compatibility aliases. +- No paid live calls until the existing account allowance is reconciled; controlled HTTP/provider tests establish this repair first. + +## Design and decisions + +Use the existing typed `CheckerModelProvider` factory. OpenAI retains its current +stored Responses behavior; OpenRouter uses one bounded non-streaming Chat +Completions request, exact model identity and locked parameters. Reuse the setup +provider's current metadata/pricing and request controls through a shared adapter +module. Metadata refusal happens before dispatch. Transport/response ambiguity +retains an uncertain effect and cannot create again; OpenRouter recovery never +pretends a response ID alone is retrievable. Successful synchronous completion +uses existing ART custody, settlement and replay. + +Provider secret file paths remain the deployment inputs. Enable one or both +provider-specific secret overlays from those explicit paths, require the selected +setup provider's secret, and reject missing/invalid configured secrets. No extra +provider selector is needed at deployment: POST uses its immutable profile. +Only `openai_responses/responses.v1` and `openrouter/chat_completions` are +supported POST pairs. Validate the pair before reading only its credential. +Unknown pairs fail closed with no fallback. Shared code contains only concrete +OpenRouter HTTP/read, direct-model/routing, metadata/pricing and usage primitives; +setup retains its DTOs, tools, canonical ART and replay. The checker owns strict +single-assistant-text choice/status parsing. Recovery returns UNAVAILABLE with +zero network I/O. Remove the superseded policy module and update all callers; +retain no import alias. + +Credential matrix: disabled proxy admits no provider file paths or overlays; +enabled inference setup requires its configured provider file, with the second +optional; enabled manager-authored setup requires at least one provider file. +Only nonempty allowlisted file paths select overlays. Every configured file is +metadata-validated, mounted and conditionally loaded only in model-proxy; absent +providers are never mounted/exported. Clear ambient provider keys first. Cover +OpenRouter-only, OpenAI-only and dual-provider configurations. + +## Acceptance criteria + +- [x] An OpenRouter POST profile dispatches through the real adapter/HTTP parser with no OpenAI credential; an OpenAI profile still selects its adapter; unsupported profiles deny. +- [x] Exact model/usage/output bounds, pricing metadata and no routing fallback or automatic paid retry are enforced; malformed/nonterminal/foreign-model output cannot settle as success. +- [x] Persisted success settles/replays once with exact ART evidence; lost/ambiguous output stays held without a second paid invocation; provider recovery never creates. +- [x] Ordinary POST checks construct no model provider and spend no model budget. +- [x] OpenRouter-only, OpenAI-only and dual-provider hosted composition/entrypoint load only configured secrets; missing selected credentials fail closed; other roles receive none. +- [x] Existing OpenAI/setup behavior, current AWS proof policy, authorization and immutable custody remain protected. +- [x] Current docs/default examples describe optional model checks and supported profile-based dispatch accurately. + +## Risk and review routing + +- Risk class: L1 (paid provider effects, secrets and external-adapter composition). +- Required reviewers: security, architecture, reuse_dedup, qa, test_delta, documentation, ci_integrity. +- Human review focus: provider selection vs optional model use, secret boundaries, and conservative uncertain-effect handling without automatic retries. + +## Evidence + +Plan review preceded implementation. Named regressions in `test_openrouter.py` +cover metadata refusal before POST, one bounded nonstreaming dispatch with fixed +routing/plugins, exact single-choice/status parsing, identity/usage/output +substitution, uncertain no-second-POST and recovery with zero I/O. +`test_model_runtime.py` covers both exact profile pairs, selected-key isolation, +unknown-pair denial, and client close. Extend existing `test_composed_execution.py` +with a controlled real OpenRouter HTTP adapter to prove PostgreSQL/ART success +settlement/replay and held ambiguous response with no redispatch. The ordinary +POST regression rejects any model-budget database access and requires no credentials. Setup contract tests protect extraction. +Hosted tests cover disabled/single/dual credentials in rendered Compose and the +real entrypoint, rejection of missing selected credentials and role isolation. + +Commands (backend commands run from `backend/` with the existing test environment): + +```sh +python -m pytest tests/checkers/model_provider tests/checkers_model_calls/test_model_runtime.py tests/checkers_model_calls/test_proxy_orchestration.py tests/checkers_model_calls/test_setup_model_handler.py -q +python scripts/run_isolated_tests.py --metadata-json /tmp/post-provider-pg.json --timeout-seconds 900 -- python -m pytest tests/checkers_model_calls/test_composed_execution.py tests/checkers_model_calls/test_setup_model_handler_postgresql.py -q +ruff check app/adapters/checkers app/adapters/model_usage app/adapters/project_agents app/interfaces/checker_model_provider.py tests/checkers/model_provider tests/checkers_model_calls +python scripts/module_boundaries.py +python scripts/authorization_boundary.py +python scripts/behavior_ownership.py +python -m pytest tests/test_ci_lane_catalogue.py tests/test_behavior_ownership.py -q +``` + +Repository-root commands: + +```sh +python -m unittest scripts.test_hosted_release_config scripts.test_hosted_release_images +python scripts/check_commitrail_records.py --base-ref origin/main +python scripts/check_markdown_links.py +python scripts/check_stale_workstream_wording.py +git diff --check +``` + +Required proof: controlled HTTP tests through +the actual parser, profile/factory composition, real PostgreSQL/ART settlement and +replay, credential-free hosted rendering/entrypoint tests, existing affected setup +and OpenAI regressions, static boundaries/catalogues, links, stale wording and full +hosted CI. Live provider and full project journey remain separately qualified. + +## Review corrections + +The response parser accepts an absent, null or empty tool-call list as no tool +calls, while rejecting nonempty or malformed values. Every completion choice +must carry integer index zero; missing, null and boolean indices are invalid. +Focused HTTP regressions cover both boundaries, and the retained-ART replay +case uses a valid empty tool-call list through real PostgreSQL custody. +Authenticated failed/incomplete responses may omit text and still settle their +known usage; completed responses must contain text. The terminal-status matrix +covers text, null and absent content without turning malformed values into success. + +The external prompt-count concern does not demonstrate an accepted token-budget +bypass under the adopted provider contract: metadata context `C` must fit profile +cap `M`, and accepted input honoring that context is at most `C <= M`. OpenRouter +reserves the full window rather than counting this prompt. The misleading unused +count/comparison is removed. The regression now varies root and top-provider +context together so only the profile ceiling rejects it, with a separate context +agreement case. Removing the ceiling guard must make the named regression fail. +Over-context requests may still be sent and enter an uncertain hold; no exact +local tokenizer or blanket no-charge claim is introduced. + +The complete architecture suite exposed a stale expectation that model runtime +still imports a concrete provider. Both provider constructors now belong only to +the existing CHECKERS factory. Update the exact import/call inventory, track the +OpenRouter constructor, and retain all existing negative cases while adding +aliased and module-qualified construction probes for both providers. No production +code or CI selection changes are required for this repair. + +## Reconciliation + +Base: current main `ae788e6a8af71075059b784a9be5ec512fde91de`, including merged #675, +#676 and AWS proof fix. Preserve #675's separate Smol trusted-intake factory, +hosted socket/image checks and public-drill scope together with profile-selected +model providers and optional credential overlays. The combined architecture +inventory has two ART factory/construction sites and three CHECKERS concrete +constructors; ownership and lane fingerprints cover both sets of additions. +Coordinate shared hosted files with active #677 without +editing its worktree. No database migration: existing profile/SQL model identities already allow slash-qualified names through 160 characters. Response IDs stay slash-free through 256 characters. +Next usable boundary: deploy the configured provider repair with project POST +implementations and run the approved real-project drill. diff --git a/README.md b/README.md index e56a65557..2059dd0cc 100644 --- a/README.md +++ b/README.md @@ -302,10 +302,19 @@ no shell or model-selected storage paths. Rebuild the local image after changing parser code. Committed original-document readiness runs one compilation and stops at findings and proposals; manager review and approval remain separate. +Model-backed POST checkers select OpenAI Responses or OpenRouter Chat Completions +from their locked project model profile through the shared budgeted proxy. +Ordinary POST checkers need no model. An OpenRouter-only deployment needs only +its OpenRouter credential; setup runtime selection does not force POST to use +OpenAI. See [model runtime operations](docs/operations_checker_models.md) for +provider bounds, evidence custody and recovery limits. + To compile guides without any model provider, select the manager-authored runtime instead. It makes no inference request and allocates no provider -resources. It uses the same isolated document tools. The hosted release omits -the provider credential for this exact `none/none/none` configuration: +resources. It uses the same isolated document tools. With the model proxy +disabled, the hosted release omits provider credentials for this exact +`none/none/none` configuration; model-backed POST can independently enable the +proxy with its selected provider credential: ```bash WORKSTREAM_PROJECT_AGENT_RUNTIME=manager_authored diff --git a/backend/app/adapters/checkers/__init__.py b/backend/app/adapters/checkers/__init__.py index b5c828915..44b1464ab 100644 --- a/backend/app/adapters/checkers/__init__.py +++ b/backend/app/adapters/checkers/__init__.py @@ -22,6 +22,7 @@ CheckerModelProvider, ) from app.modules.tasks.api import TaskSubmissionContextFacts +from app.modules.model_usage.api import ProjectModelProfile from app.modules.checkers.api.external_policy import LockedExternalPostSubmitPolicy from app.modules.checkers.api.selected_post_submit import ( @@ -68,28 +69,75 @@ def external_checker_execution_factory( return factory -def create_checker_model_provider(settings): - """Construct the sole provider registered for one locked model profile.""" - from openai import AsyncOpenAI +def create_checker_model_provider( + profile: ProjectModelProfile, *, api_key: str +) -> CheckerModelProvider: + """Select one supported adapter from the exact immutable project profile.""" + from app.modules.model_usage.api import ModelCallUnavailable + + identity = (profile.spec.provider, profile.spec.api) + provider_key = { + ("openai_responses", "responses.v1"): "openai_responses", + ("openrouter", "chat_completions"): "openrouter", + }.get(identity) + if provider_key is None: + raise ModelCallUnavailable("model_profile_provider_unavailable") + if not isinstance(api_key, str) or not api_key or len(api_key) > 8_192: + raise ModelCallUnavailable("model_provider_credentials_unavailable") + + def openai_provider(): + from openai import AsyncOpenAI + + from app.adapters.checkers.model_provider import ( + OpenAIResponsesCheckerModelProvider, + OpenAIResponsesSettings, + ) - from app.adapters.checkers.model_provider import ( - OPENAI_RESPONSES_PROVIDER_KEY, - OpenAIResponsesCheckerModelProvider, - ) + spec = profile.spec + settings = OpenAIResponsesSettings( + api_key=api_key, + model=spec.model_snapshot, + maximum_request_bytes=spec.maximum_request_bytes, + maximum_input_tokens=spec.maximum_input_tokens, + maximum_output_tokens=spec.maximum_output_tokens, + maximum_output_bytes=spec.maximum_output_bytes, + request_timeout_seconds=float(spec.deadline_ms / 1000), + reasoning_effort=spec.settings.reasoning_effort, + text_verbosity=spec.settings.text_verbosity, + ) + return OpenAIResponsesCheckerModelProvider( + settings, + AsyncOpenAI(api_key=api_key, max_retries=0, timeout=settings.request_timeout_seconds), + ) + + def openrouter_provider(): + import httpx + + from app.adapters.checkers.openrouter_model_provider import OpenRouterCheckerModelProvider + + from app.adapters.model_usage.openrouter import require_direct_openrouter_model + + try: + require_direct_openrouter_model(profile.spec.model_snapshot) + except ValueError: + raise ModelCallUnavailable("model_profile_provider_unavailable") from None + client = httpx.AsyncClient( + transport=httpx.AsyncHTTPTransport(retries=0, trust_env=False), + trust_env=False, + follow_redirects=False, + timeout=profile.spec.deadline_ms / 1000.0, + headers={ + "Authorization": "Bearer " + api_key, + "Content-Type": "application/json", + "Accept": "application/json", + }, + ) + return OpenRouterCheckerModelProvider(profile, client) factory = ExternalServiceAdapterFactory[CheckerModelProvider](CHECKER_MODEL_CAPABILITY_KEY) - factory.register( - OPENAI_RESPONSES_PROVIDER_KEY, - lambda: OpenAIResponsesCheckerModelProvider( - settings, - AsyncOpenAI( - api_key=settings.api_key, - max_retries=0, - timeout=settings.request_timeout_seconds, - ), - ), - ) - return factory.create(OPENAI_RESPONSES_PROVIDER_KEY) + factory.register("openai_responses", openai_provider) + factory.register("openrouter", openrouter_provider) + return factory.create(provider_key) def configured_external_checker_execution_factory(settings): diff --git a/backend/app/adapters/checkers/model_runtime.py b/backend/app/adapters/checkers/model_runtime.py index 84c9e9790..b9057b5b4 100644 --- a/backend/app/adapters/checkers/model_runtime.py +++ b/backend/app/adapters/checkers/model_runtime.py @@ -27,9 +27,6 @@ model_invocation_grants, model_profile_reader, ) -from app.adapters.checkers.model_provider import ( - OpenAIResponsesSettings, -) from app.adapters.checkers.model_proxy_transport import ModelProxyTransport from app.adapters.model_usage import setup_model_usage_service from app.adapters.project_agents.setup_model_handler import ( @@ -44,24 +41,16 @@ async def configured_model_provider(profile: ProjectModelProfile): """Construct exactly the provider selected by a trusted immutable profile.""" spec = profile.spec - if (spec.provider, spec.api) != ("openai_responses", "responses.v1"): + credential = { + ("openai_responses", "responses.v1"): "OPENAI_API_KEY", + ("openrouter", "chat_completions"): "OPENROUTER_API_KEY", + }.get((spec.provider, spec.api)) + if credential is None: raise ModelCallUnavailable("model_profile_provider_unavailable") - key = os.environ.get("OPENAI_API_KEY", "") + key = os.environ.get(credential, "") if not key: raise ModelCallUnavailable("model_provider_credentials_unavailable") - provider = create_checker_model_provider( - OpenAIResponsesSettings( - api_key=key, - model=spec.model_snapshot, - maximum_request_bytes=spec.maximum_request_bytes, - maximum_input_tokens=spec.maximum_input_tokens, - maximum_output_tokens=spec.maximum_output_tokens, - maximum_output_bytes=spec.maximum_output_bytes, - request_timeout_seconds=float(spec.deadline_ms / 1000), - reasoning_effort=spec.settings.reasoning_effort, - text_verbosity=spec.settings.text_verbosity, - ) - ) + provider = create_checker_model_provider(profile, api_key=key) try: yield provider finally: diff --git a/backend/app/adapters/checkers/openrouter_model_provider.py b/backend/app/adapters/checkers/openrouter_model_provider.py new file mode 100644 index 000000000..6e69a3bca --- /dev/null +++ b/backend/app/adapters/checkers/openrouter_model_provider.py @@ -0,0 +1,271 @@ +"""OpenRouter Chat Completions transport for policy-bound checker model calls.""" + +from __future__ import annotations + +import json + +import httpx + +from app.adapters.model_usage.openrouter import ( + OPENROUTER_CHAT_COMPLETIONS_URL, + bounded_response_bytes, + openrouter_request_options, + openrouter_usage, + preflight_openrouter_profile, + require_direct_openrouter_model, +) +from app.interfaces.checker_model_provider import ( + CHECKER_MODEL_CAPABILITY_KEY, + CheckerModelCreateFailure, + CheckerModelCreateOutcome, + CheckerModelCreateResult, + CheckerModelObservation, + CheckerModelProviderStatus, + CheckerModelRecoveryOutcome, + CheckerModelRecoveryResult, + CheckerModelRequest, + CheckerModelResponseReference, + CheckerModelUsage, +) +from app.interfaces.external_services import ExternalServiceAdapterIdentity +from app.modules.model_usage.api import ModelCallUnavailable, ProjectModelProfile + + +OPENROUTER_PROVIDER_KEY = "openrouter" +_IDENTITY = ExternalServiceAdapterIdentity( + CHECKER_MODEL_CAPABILITY_KEY, + OPENROUTER_PROVIDER_KEY, +) +_FAILED_FINISH_REASONS = frozenset({"content_filter", "error"}) + + +class OpenRouterCheckerModelProvider: + """Dispatch one bounded direct-model Chat Completions request without retry.""" + + __slots__ = ("_client", "_profile") + + def __init__(self, profile: ProjectModelProfile, client: httpx.AsyncClient) -> None: + """Bind one immutable OpenRouter profile to an injected no-retry client.""" + if type(profile) is not ProjectModelProfile or ( + profile.spec.provider, + profile.spec.api, + ) != ("openrouter", "chat_completions"): + raise ValueError("OpenRouter checker model profile is invalid") + require_direct_openrouter_model(profile.spec.model_snapshot) + if not isinstance(client, httpx.AsyncClient): + raise ValueError("OpenRouter checker model client is invalid") + self._profile = profile + self._client = client + + @property + def identity(self) -> ExternalServiceAdapterIdentity: + """Return the fixed identity checked by the shared typed factory.""" + return _IDENTITY + + async def create(self, request: CheckerModelRequest) -> CheckerModelCreateResult: + """Preflight current metadata, then dispatch exactly one paid request.""" + if type(request) is not CheckerModelRequest: + raise ValueError("checker model request is invalid") + body_json = self._request_json(request.prompt) + if len(body_json.encode("utf-8")) > self._profile.spec.maximum_request_bytes: + return CheckerModelCreateResult( + CheckerModelCreateOutcome.NO_EFFECT, + failure=CheckerModelCreateFailure.REQUEST_BYTES_EXCEEDED, + ) + + try: + # Qualify the full provider window; this is not a prompt tokenizer. + await preflight_openrouter_profile(self._client, self._profile) + except ModelCallUnavailable: + return CheckerModelCreateResult( + CheckerModelCreateOutcome.NO_EFFECT, + failure=CheckerModelCreateFailure.INPUT_COUNT_UNAVAILABLE, + ) + try: + async with self._client.stream( + "POST", + OPENROUTER_CHAT_COMPLETIONS_URL, + content=body_json, + ) as response: + status_code = response.status_code + try: + raw = await bounded_response_bytes( + response, + self._profile.spec.maximum_response_bytes, + ) + except (TypeError, ValueError, UnicodeError): + return CheckerModelCreateResult( + CheckerModelCreateOutcome.UNCERTAIN, + failure=CheckerModelCreateFailure.RESPONSE_INVALID, + ) + except Exception: + return CheckerModelCreateResult( + CheckerModelCreateOutcome.UNCERTAIN, + failure=CheckerModelCreateFailure.DELIVERY_UNCERTAIN, + ) + + value = _response_object(raw) + known_response = _known_response(value) + if not 200 <= status_code < 300: + return CheckerModelCreateResult( + CheckerModelCreateOutcome.UNCERTAIN, + known_response=known_response, + failure=CheckerModelCreateFailure.DELIVERY_UNCERTAIN, + ) + try: + observation = self._observation(value) + except (TypeError, ValueError, UnicodeError): + return CheckerModelCreateResult( + CheckerModelCreateOutcome.UNCERTAIN, + known_response=known_response, + failure=CheckerModelCreateFailure.RESPONSE_INVALID, + ) + return CheckerModelCreateResult( + CheckerModelCreateOutcome.ACCEPTED, + observation=observation, + known_response=observation.reference, + ) + + async def recover( + self, reference: CheckerModelResponseReference + ) -> CheckerModelRecoveryResult: + """Report retrieval unavailable without issuing any network request.""" + if type(reference) is not CheckerModelResponseReference: + raise ValueError("checker model response identity is invalid") + return CheckerModelRecoveryResult( + CheckerModelRecoveryOutcome.UNAVAILABLE, + reference, + ) + + async def aclose(self) -> None: + """Close the exact HTTP client created for this profile adapter.""" + await self._client.aclose() + + def _request_json(self, prompt: str) -> str: + body = { + "model": self._profile.spec.model_snapshot, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": self._profile.spec.maximum_output_tokens, + "stream": False, + "tools": [], + "tool_choice": "none", + "usage": {"include": True}, + "reasoning_effort": self._profile.spec.settings.reasoning_effort, + "verbosity": self._profile.spec.settings.text_verbosity, + "temperature": 0, + "top_p": 1, + "store": False, + "parallel_tool_calls": False, + **openrouter_request_options(), + } + return json.dumps( + body, + sort_keys=True, + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + ) + + def _observation(self, value: object) -> CheckerModelObservation: + if not isinstance(value, dict): + raise ValueError("provider response is invalid") + reference = CheckerModelResponseReference(value.get("id")) + provider_model = value.get("model") + if provider_model != self._profile.spec.model_snapshot: + raise ValueError("provider model identity changed") + choices = value.get("choices") + if not isinstance(choices, list) or len(choices) != 1: + raise ValueError("provider output is invalid") + choice = choices[0] + if not isinstance(choice, dict): + raise ValueError("provider output is invalid") + index = choice.get("index") + if type(index) is not int or index != 0: + raise ValueError("provider output is invalid") + message = choice.get("message") + if not isinstance(message, dict) or message.get("role") != "assistant": + raise ValueError("provider output is invalid") + if ( + message.get("tool_calls") not in (None, []) + or message.get("function_call") is not None + or message.get("audio") is not None + ): + raise ValueError("provider output is not text-only") + refusal = message.get("refusal") + output_text = message.get("content") + finish_reason = choice.get("finish_reason") + if isinstance(refusal, str) and refusal: + if output_text is not None and not isinstance(output_text, str): + raise ValueError("provider output is invalid") + if finish_reason not in {"stop", "content_filter"}: + raise ValueError("provider refusal status is invalid") + if len(refusal.encode("utf-8")) > self._profile.spec.maximum_output_bytes: + raise ValueError("provider output exceeds profile bound") + output_text = None + status = CheckerModelProviderStatus.FAILED + elif refusal not in (None, ""): + raise ValueError("provider output is invalid") + else: + status = _provider_status(finish_reason) + if output_text is None: + if status is CheckerModelProviderStatus.COMPLETED: + raise ValueError("completed provider output is missing") + elif not isinstance(output_text, str): + raise ValueError("provider output is invalid") + elif len(output_text.encode("utf-8")) > self._profile.spec.maximum_output_bytes: + raise ValueError("provider output exceeds profile bound") + normalized = openrouter_usage(value.get("usage")) + if normalized.total_tokens is None: + raise ValueError("provider usage is incomplete") + usage = CheckerModelUsage( + input_tokens=normalized.input_tokens, + cached_input_tokens=normalized.cached_input_tokens, + output_tokens=normalized.output_tokens, + reasoning_tokens=normalized.reasoning_tokens, + total_tokens=normalized.total_tokens, + ) + if ( + usage.input_tokens > self._profile.spec.maximum_input_tokens + or usage.output_tokens > self._profile.spec.maximum_output_tokens + ): + raise ValueError("provider usage exceeds profile bound") + return CheckerModelObservation( + reference=reference, + provider_model=provider_model, + status=status, + output_text=output_text, + usage=usage, + ) + + +def _response_object(raw: bytes) -> object: + try: + return json.loads(raw, parse_constant=_reject_constant) + except (TypeError, ValueError, UnicodeError, RecursionError): + return None + + +def _known_response(value: object) -> CheckerModelResponseReference | None: + try: + if not isinstance(value, dict): + return None + return CheckerModelResponseReference(value.get("id")) + except (TypeError, ValueError): + return None + + +def _provider_status(value: object) -> CheckerModelProviderStatus: + if value == "stop": + return CheckerModelProviderStatus.COMPLETED + if value == "length": + return CheckerModelProviderStatus.INCOMPLETE + if value in _FAILED_FINISH_REASONS: + return CheckerModelProviderStatus.FAILED + raise ValueError("provider finish status is invalid") + + +def _reject_constant(_: str) -> None: + raise ValueError("non-finite provider number") + + +__all__ = ("OPENROUTER_PROVIDER_KEY", "OpenRouterCheckerModelProvider") diff --git a/backend/app/adapters/model_usage/openrouter.py b/backend/app/adapters/model_usage/openrouter.py new file mode 100644 index 000000000..67300be1f --- /dev/null +++ b/backend/app/adapters/model_usage/openrouter.py @@ -0,0 +1,275 @@ +"""Shared concrete OpenRouter transport and profile-validation primitives.""" + +from __future__ import annotations + +from dataclasses import dataclass +from decimal import ROUND_CEILING, Decimal, InvalidOperation +import json + +from app.modules.model_usage.api import ModelCallUnavailable, ProjectModelProfile + + +OPENROUTER_CHAT_COMPLETIONS_URL = "https://openrouter.ai/api/v1/chat/completions" +_OPENROUTER_MODEL_URL = "https://openrouter.ai/api/v1/model/{author}/{slug}" +_OPENROUTER_REQUIRED_PARAMETERS = frozenset( + {"max_tokens", "reasoning_effort", "tool_choice", "tools", "verbosity"} +) +_MAX_METADATA_BYTES = 1_048_576 +_MAX_TOKEN_COUNT = 2**63 - 1 + + +@dataclass(frozen=True, slots=True) +class OpenRouterUsage: + """Complete normalized Chat Completions token counters.""" + + input_tokens: int + cached_input_tokens: int + output_tokens: int + reasoning_tokens: int + total_tokens: int | None + + def __post_init__(self) -> None: + values = ( + self.input_tokens, + self.cached_input_tokens, + self.output_tokens, + self.reasoning_tokens, + ) + if any( + type(value) is not int or not 0 <= value <= _MAX_TOKEN_COUNT + for value in values + ): + raise ValueError("OpenRouter usage is invalid") + if self.total_tokens is not None and ( + type(self.total_tokens) is not int + or not 0 <= self.total_tokens <= _MAX_TOKEN_COUNT + or self.total_tokens != self.input_tokens + self.output_tokens + ): + raise ValueError("OpenRouter usage is invalid") + if ( + self.cached_input_tokens > self.input_tokens + or self.reasoning_tokens > self.output_tokens + ): + raise ValueError("OpenRouter usage is invalid") + + +def require_direct_openrouter_model(model: str) -> str: + """Reject provider-selected routing variants while preserving the exact model.""" + if not isinstance(model, str): + raise ValueError("OpenRouter model routing is unsupported") + normalized = model.lower() + parts = model.split("/") + if ( + ":" in normalized + or normalized.startswith("openrouter/") + or len(parts) != 2 + or not all(parts) + ): + raise ValueError("OpenRouter model routing is unsupported") + return model + + +def openrouter_model_metadata_url(model: str) -> str: + """Return the exact metadata endpoint for one direct author/model identity.""" + direct = require_direct_openrouter_model(model) + model_parts = direct.split("/") + return _OPENROUTER_MODEL_URL.format(author=model_parts[0], slug=model_parts[1]) + + +def openrouter_request_options() -> dict[str, object]: + """Return fixed routing and plugin controls for a direct OpenRouter request. + + Alignment currently rejects an ``enabled`` key, even when false. All account + plugins must therefore be disabled operationally before private model use; + enforced account settings can also override these per-request controls. + ``data_collection=deny`` is not a zero-data-retention guarantee. + """ + return { + "provider": {"allow_fallbacks": False, "data_collection": "deny"}, + "plugins": [ + {"id": plugin, "enabled": False} + for plugin in ( + "web", + "file-parser", + "response-healing", + "armor", + "pareto-router", + "context-compression", + ) + ], + } + + +async def preflight_openrouter_profile(client, profile: ProjectModelProfile) -> int: + """Refresh bounded metadata and reserve the profile's full input context.""" + try: + url = openrouter_model_metadata_url(profile.spec.model_snapshot) + async with client.stream("GET", url) as response: + raw = await bounded_response_bytes(response, _MAX_METADATA_BYTES) + if not 200 <= response.status_code < 300 or not raw: + raise ValueError("provider metadata unavailable") + value = json.loads(raw, parse_constant=_reject_constant) + validate_openrouter_metadata(profile, value) + except ModelCallUnavailable: + raise + except BaseException as exc: + raise ModelCallUnavailable("model_profile_provider_unavailable") from exc + return profile.spec.maximum_input_tokens + + +def validate_openrouter_metadata(profile: ProjectModelProfile, value: object) -> None: + """Fail before spend unless current metadata fits the pinned profile.""" + if not isinstance(value, dict) or set(value) != {"data"}: + raise ModelCallUnavailable("model_profile_provider_unavailable") + data = value["data"] + if not isinstance(data, dict): + raise ModelCallUnavailable("model_profile_provider_unavailable") + top_provider = data.get("top_provider") + supported = data.get("supported_parameters") + context_length = data.get("context_length") + if ( + data.get("id") != profile.spec.model_snapshot + or not isinstance(data.get("canonical_slug"), str) + or not data["canonical_slug"] + or type(context_length) is not int + or context_length < 1 + or context_length > profile.spec.maximum_input_tokens + or not isinstance(top_provider, dict) + or type(top_provider.get("context_length")) is not int + or top_provider["context_length"] != context_length + or type(top_provider.get("max_completion_tokens")) is not int + or not 1 + <= profile.spec.maximum_output_tokens + <= top_provider["max_completion_tokens"] + or not isinstance(supported, list) + or any(type(item) is not str for item in supported) + or not _OPENROUTER_REQUIRED_PARAMETERS.issubset(supported) + ): + raise ModelCallUnavailable("model_profile_provider_unavailable") + _validate_openrouter_pricing(profile, data.get("pricing")) + + +def openrouter_usage(value: object) -> OpenRouterUsage: + """Decode nested Chat Completions token counters without owner-specific DTOs.""" + if not isinstance(value, dict): + raise ValueError("OpenRouter usage is unavailable") + input_details = value.get("prompt_tokens_details") or {} + output_details = value.get("completion_tokens_details") or {} + if not isinstance(input_details, dict) or not isinstance(output_details, dict): + raise ValueError("OpenRouter usage is invalid") + return OpenRouterUsage( + input_tokens=value.get("prompt_tokens"), + cached_input_tokens=input_details.get("cached_tokens", 0), + output_tokens=value.get("completion_tokens"), + reasoning_tokens=output_details.get("reasoning_tokens", 0), + total_tokens=value.get("total_tokens"), + ) + + +def openrouter_generated_output_bytes(root: object) -> int: + """Count bounded content, refusal and tool-argument strings recursively.""" + total = 0 + stack = [root] + while stack: + current = stack.pop() + if isinstance(current, list): + stack.extend(current) + elif isinstance(current, dict): + for key, item in current.items(): + if key in {"content", "refusal", "arguments"} and isinstance(item, str): + total += len(item.encode("utf-8")) + elif isinstance(item, (dict, list)): + stack.append(item) + return total + + +async def bounded_response_bytes(response, maximum_bytes: int) -> bytes: + """Read a provider body without accumulating beyond its locked cap.""" + body = bytearray() + async for chunk in response.aiter_bytes(): + if not chunk: + continue + if len(chunk) > maximum_bytes - len(body): + raise ValueError("provider response exceeds profile") + body.extend(chunk) + return bytes(body) + + +def _validate_openrouter_pricing(profile: ProjectModelProfile, value: object) -> None: + """Require pinned rates to cover every advertised tier and cache class.""" + if not isinstance(value, dict): + raise ModelCallUnavailable("model_profile_provider_unavailable") + pricing_rows = [value] + overrides = value.get("overrides", []) + if not isinstance(overrides, list) or any(not isinstance(row, dict) for row in overrides): + raise ModelCallUnavailable("model_profile_provider_unavailable") + pricing_rows.extend(overrides) + input_ceiling = 0 + cached_ceiling = 0 + output_ceiling = 0 + for row in pricing_rows: + _require_zero_unsupported_fees(row) + input_ceiling = max( + input_ceiling, + _nanodollars(row.get("prompt")), + _nanodollars(row.get("input_cache_write", row.get("prompt"))), + ) + cached_ceiling = max( + cached_ceiling, + _nanodollars(row.get("input_cache_read", row.get("prompt"))), + ) + output_ceiling = max(output_ceiling, _nanodollars(row.get("completion"))) + pinned = profile.spec.pricing + if ( + input_ceiling > pinned.uncached_input_nanodollars_per_token + or cached_ceiling > pinned.cached_input_nanodollars_per_token + or output_ceiling > pinned.output_nanodollars_per_token + ): + raise ModelCallUnavailable("model_profile_provider_unavailable") + + +def _require_zero_unsupported_fees(row: dict) -> None: + supported = { + "prompt", + "completion", + "input_cache_read", + "input_cache_write", + "overrides", + "min_prompt_tokens", + } + for key, value in row.items(): + if key not in supported and _nanodollars(value) != 0: + raise ModelCallUnavailable("model_profile_provider_unavailable") + + +def _nanodollars(value: object) -> int: + if type(value) is not str: + raise ModelCallUnavailable("model_profile_provider_unavailable") + try: + amount = Decimal(value) + except InvalidOperation: + raise ModelCallUnavailable("model_profile_provider_unavailable") from None + if not amount.is_finite() or amount < 0: + raise ModelCallUnavailable("model_profile_provider_unavailable") + converted = (amount * Decimal(1_000_000_000)).to_integral_value(rounding=ROUND_CEILING) + if converted > 9_000_000_000_000_000_000: + raise ModelCallUnavailable("model_profile_provider_unavailable") + return int(converted) + + +def _reject_constant(_: str) -> None: + raise ValueError("non-finite provider number") + + +__all__ = ( + "OPENROUTER_CHAT_COMPLETIONS_URL", + "OpenRouterUsage", + "bounded_response_bytes", + "openrouter_generated_output_bytes", + "openrouter_model_metadata_url", + "openrouter_request_options", + "openrouter_usage", + "preflight_openrouter_profile", + "require_direct_openrouter_model", + "validate_openrouter_metadata", +) diff --git a/backend/app/adapters/project_agents/omnicore_agent.py b/backend/app/adapters/project_agents/omnicore_agent.py index a1eb455f0..e89913154 100644 --- a/backend/app/adapters/project_agents/omnicore_agent.py +++ b/backend/app/adapters/project_agents/omnicore_agent.py @@ -28,7 +28,8 @@ OmniCoreAgentProfile, require_omnicore_agent_adapter_profile, ) -from app.adapters.project_agents.openrouter_policy import ( +from app.adapters.model_usage.openrouter import ( + OPENROUTER_CHAT_COMPLETIONS_URL as _OPENROUTER_CHAT_COMPLETIONS_URL, openrouter_request_options, require_direct_openrouter_model, ) @@ -48,7 +49,6 @@ from app.interfaces.project_guide_runtime import ProjectGuideRuntimeConfiguration from app.modules.projects.api.guide_documents import GuideRuntimeCapabilities -_OPENROUTER_CHAT_COMPLETIONS_URL = "https://openrouter.ai/api/v1/chat/completions" _READ_DESCRIPTION = "Read bounded text from an assigned original using its exact opaque handle." _SEARCH_DESCRIPTION = "Search bounded text in an assigned original using its exact opaque handle." _READ_SCHEMA = { @@ -180,7 +180,12 @@ def __init__( or not configuration.uses_inference ): raise ProjectAgentRuntimeConfigurationError("project guide runtime is unsupported") - require_direct_openrouter_model(configuration.model) + try: + require_direct_openrouter_model(configuration.model) + except ValueError: + raise ProjectAgentRuntimeConfigurationError( + "project guide OpenRouter model routing is unsupported" + ) from None if configuration.maximum_turns < 2: raise ProjectAgentRuntimeConfigurationError( "project guide runtime requires draft and review turns" diff --git a/backend/app/adapters/project_agents/openai_agent_sdk.py b/backend/app/adapters/project_agents/openai_agent_sdk.py index d98f5672d..ae1180636 100644 --- a/backend/app/adapters/project_agents/openai_agent_sdk.py +++ b/backend/app/adapters/project_agents/openai_agent_sdk.py @@ -23,6 +23,10 @@ validate_project_guide_runtime_result, ) from app.adapters.project_agents.setup_model_proxy import create_setup_httpx_client +from app.adapters.model_usage.openrouter import ( + openrouter_request_options, + require_direct_openrouter_model, +) from app.interfaces.external_services import ExternalServiceAdapterIdentity from app.interfaces.project_agents import ( ProjectAgentRuntimeConfigurationError, @@ -36,8 +40,6 @@ from app.interfaces.project_guide_runtime import ProjectGuideRuntimeConfiguration from app.modules.projects.api.guide_documents import GuideRuntimeCapabilities -from .openrouter_policy import openrouter_request_options, require_direct_openrouter_model - _INVALID_SCHEMA_OUTPUT = object() _PROXY_BASE_URL = "https://model-proxy.invalid/v1" @@ -427,7 +429,12 @@ def _require_supported_provider( configuration.model_provider == "openrouter" and configuration.model_api == "chat_completions" ): - require_direct_openrouter_model(configuration.model) + try: + require_direct_openrouter_model(configuration.model) + except ValueError: + raise ProjectAgentRuntimeConfigurationError( + "project guide OpenRouter model routing is unsupported" + ) from None return elif configuration.model_provider == "openai" and configuration.model_api in { "responses", diff --git a/backend/app/adapters/project_agents/openrouter_policy.py b/backend/app/adapters/project_agents/openrouter_policy.py deleted file mode 100644 index 1a6724eac..000000000 --- a/backend/app/adapters/project_agents/openrouter_policy.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Qualified OpenRouter request controls shared by guide runtime adapters.""" - -from app.interfaces.project_agents import ProjectAgentRuntimeConfigurationError - - -def require_direct_openrouter_model(model: str) -> str: - """Reject provider-selected variants while preserving the configured model.""" - normalized = model.lower() - if ":" in normalized or normalized.startswith("openrouter/"): - raise ProjectAgentRuntimeConfigurationError( - "project guide OpenRouter model routing is unsupported" - ) - return model - - -def openrouter_request_options() -> dict[str, object]: - """Request fixed routing and disable plugins that accept request overrides. - - Alignment currently rejects an ``enabled`` key, even when false. All account - plugins must therefore be disabled operationally before private guide use; - enforced account settings can also override these per-request controls. - ``data_collection=deny`` is not a zero-data-retention guarantee. - """ - return { - "provider": {"allow_fallbacks": False, "data_collection": "deny"}, - "plugins": [ - {"id": plugin, "enabled": False} - for plugin in ( - "web", "file-parser", "response-healing", "armor", - "pareto-router", "context-compression", - ) - ], - } diff --git a/backend/app/adapters/project_agents/setup_model_handler.py b/backend/app/adapters/project_agents/setup_model_handler.py index d279e1318..bee33cbf3 100644 --- a/backend/app/adapters/project_agents/setup_model_handler.py +++ b/backend/app/adapters/project_agents/setup_model_handler.py @@ -8,10 +8,17 @@ from collections.abc import Callable from contextlib import AbstractAsyncContextManager, asynccontextmanager from dataclasses import dataclass -from decimal import ROUND_CEILING, Decimal, InvalidOperation from typing import Protocol -from app.adapters.project_agents.openrouter_policy import openrouter_request_options +from app.adapters.model_usage.openrouter import ( + OPENROUTER_CHAT_COMPLETIONS_URL, + bounded_response_bytes, + openrouter_generated_output_bytes, + openrouter_model_metadata_url, + openrouter_request_options, + openrouter_usage, + preflight_openrouter_profile, +) from app.core.hashing import canonical_json_hash from app.interfaces.setup_model_proxy import ( SetupModelProxyReply, @@ -34,11 +41,6 @@ ) _OPENAI_RESPONSES_URL = "https://api.openai.com/v1/responses" -_OPENROUTER_CHAT_URL = "https://openrouter.ai/api/v1/chat/completions" -_OPENROUTER_MODEL_URL = "https://openrouter.ai/api/v1/model/{author}/{slug}" -_OPENROUTER_REQUIRED_PARAMETERS = frozenset( - {"max_tokens", "reasoning_effort", "tool_choice", "tools", "verbosity"} -) _OPENAI_RESPONSE_FIELDS = frozenset( { "background", @@ -168,7 +170,7 @@ async def dispatch(self, body_json: str) -> SetupProviderResponse: async with self._client.stream( "POST", _OPENAI_RESPONSES_URL, content=body_json ) as response: - raw = await _bounded_response_bytes( + raw = await bounded_response_bytes( response, self._profile.spec.maximum_response_bytes ) except BaseException as exc: @@ -220,11 +222,11 @@ def __init__(self, *, api_key: str, profile: ProjectModelProfile) -> None: """Bind one no-retry client to an exact direct OpenRouter model.""" import httpx - model_parts = profile.spec.model_snapshot.split("/") - if len(model_parts) != 2 or not all(model_parts): + try: + openrouter_model_metadata_url(profile.spec.model_snapshot) + except ValueError: raise ModelCallUnavailable("model_profile_provider_unavailable") self._profile = profile - self._model_url = _OPENROUTER_MODEL_URL.format(author=model_parts[0], slug=model_parts[1]) self._client = httpx.AsyncClient( trust_env=False, follow_redirects=False, @@ -238,27 +240,16 @@ def __init__(self, *, api_key: str, profile: ProjectModelProfile) -> None: async def count_input(self, _body: dict[str, object]) -> int: """Refresh exact model metadata, then reserve the full approved context.""" - try: - async with self._client.stream("GET", self._model_url) as response: - raw = await _bounded_response_bytes(response, 1_048_576) - if not 200 <= response.status_code < 300 or not raw: - raise ValueError("provider metadata unavailable") - value = json.loads(raw, parse_constant=_reject_constant) - _validate_openrouter_metadata(self._profile, value) - except ModelCallUnavailable: - raise - except BaseException as exc: - raise ModelCallUnavailable("model_profile_provider_unavailable") from exc - return self._profile.spec.maximum_input_tokens + return await preflight_openrouter_profile(self._client, self._profile) async def dispatch(self, body_json: str) -> SetupProviderResponse: """Send one bounded Chat Completions request and classify uncertainty.""" raw = b"" try: async with self._client.stream( - "POST", _OPENROUTER_CHAT_URL, content=body_json + "POST", OPENROUTER_CHAT_COMPLETIONS_URL, content=body_json ) as response: - raw = await _bounded_response_bytes( + raw = await bounded_response_bytes( response, self._profile.spec.maximum_response_bytes ) except BaseException as exc: @@ -686,100 +677,6 @@ def _safe_chat_response_format(value: object) -> bool: ) -def _validate_openrouter_metadata(profile: ProjectModelProfile, value: object) -> None: - """Fail before spend unless current provider metadata fits the pinned profile.""" - if not isinstance(value, dict) or set(value) != {"data"}: - raise ModelCallUnavailable("model_profile_provider_unavailable") - data = value["data"] - if not isinstance(data, dict): - raise ModelCallUnavailable("model_profile_provider_unavailable") - top_provider = data.get("top_provider") - supported = data.get("supported_parameters") - context_length = data.get("context_length") - if ( - data.get("id") != profile.spec.model_snapshot - or not isinstance(data.get("canonical_slug"), str) - or not data["canonical_slug"] - or type(context_length) is not int - or context_length < 1 - or context_length > profile.spec.maximum_input_tokens - or not isinstance(top_provider, dict) - or type(top_provider.get("context_length")) is not int - or top_provider["context_length"] != context_length - or type(top_provider.get("max_completion_tokens")) is not int - or not 1 <= profile.spec.maximum_output_tokens <= top_provider["max_completion_tokens"] - or not isinstance(supported, list) - or any(type(item) is not str for item in supported) - or not _OPENROUTER_REQUIRED_PARAMETERS.issubset(supported) - ): - raise ModelCallUnavailable("model_profile_provider_unavailable") - _validate_openrouter_pricing(profile, data.get("pricing")) - - -def _validate_openrouter_pricing(profile: ProjectModelProfile, value: object) -> None: - """Require pinned rates to cover every advertised tier and cache class.""" - if not isinstance(value, dict): - raise ModelCallUnavailable("model_profile_provider_unavailable") - pricing_rows = [value] - overrides = value.get("overrides", []) - if not isinstance(overrides, list) or any(not isinstance(row, dict) for row in overrides): - raise ModelCallUnavailable("model_profile_provider_unavailable") - pricing_rows.extend(overrides) - input_ceiling = 0 - cached_ceiling = 0 - output_ceiling = 0 - for row in pricing_rows: - _require_zero_unsupported_fees(row) - input_ceiling = max( - input_ceiling, - _nanodollars(row.get("prompt")), - _nanodollars(row.get("input_cache_write", row.get("prompt"))), - ) - cached_ceiling = max( - cached_ceiling, - _nanodollars(row.get("input_cache_read", row.get("prompt"))), - ) - output_ceiling = max(output_ceiling, _nanodollars(row.get("completion"))) - pinned = profile.spec.pricing - if ( - input_ceiling > pinned.uncached_input_nanodollars_per_token - or cached_ceiling > pinned.cached_input_nanodollars_per_token - or output_ceiling > pinned.output_nanodollars_per_token - ): - raise ModelCallUnavailable("model_profile_provider_unavailable") - - -def _require_zero_unsupported_fees(row: dict) -> None: - """Reject advertised charges the token-only ledger cannot reserve.""" - supported = { - "prompt", - "completion", - "input_cache_read", - "input_cache_write", - "overrides", - "min_prompt_tokens", - } - for key, value in row.items(): - if key not in supported and _nanodollars(value) != 0: - raise ModelCallUnavailable("model_profile_provider_unavailable") - - -def _nanodollars(value: object) -> int: - """Round one nonnegative per-token USD string up to nanodollars.""" - if type(value) is not str: - raise ModelCallUnavailable("model_profile_provider_unavailable") - try: - amount = Decimal(value) - except InvalidOperation: - raise ModelCallUnavailable("model_profile_provider_unavailable") from None - if not amount.is_finite() or amount < 0: - raise ModelCallUnavailable("model_profile_provider_unavailable") - converted = (amount * Decimal(1_000_000_000)).to_integral_value(rounding=ROUND_CEILING) - if converted > 9_000_000_000_000_000_000: - raise ModelCallUnavailable("model_profile_provider_unavailable") - return int(converted) - - def _openai_response( profile: ProjectModelProfile, status_code: int, value: dict[str, object] ) -> SetupProviderResponse: @@ -814,13 +711,25 @@ def _openrouter_response( or provider_model != profile.spec.model_snapshot ): raise ValueError("provider response identity is invalid") + usage = openrouter_usage(value.get("usage")) + choices = value.get("choices") + if not isinstance(choices, list): + raise TypeError("provider output is invalid") return SetupProviderResponse( status_code=status_code, response_json=_canonical_bytes(value).decode("utf-8"), provider_response_id=provider_id, provider_model=provider_model, - usage=_chat_usage(value.get("usage")), - usable=_chat_output_bytes(value) <= profile.spec.maximum_output_bytes, + usage=ModelCallUsage( + input_tokens=usage.input_tokens, + cached_input_tokens=usage.cached_input_tokens, + output_tokens=usage.output_tokens, + reasoning_tokens=usage.reasoning_tokens, + ), + usable=( + openrouter_generated_output_bytes(choices) + <= profile.spec.maximum_output_bytes + ), ) @@ -854,20 +763,6 @@ def _usage(value: object) -> ModelCallUsage: ) -def _chat_usage(value: object) -> ModelCallUsage: - """Decode complete Chat Completions usage into the shared ledger shape.""" - if not isinstance(value, dict): - raise TypeError("provider usage is unavailable") - input_details = value.get("prompt_tokens_details") or {} - output_details = value.get("completion_tokens_details") or {} - return ModelCallUsage( - input_tokens=value.get("prompt_tokens"), - cached_input_tokens=input_details.get("cached_tokens", 0), - output_tokens=value.get("completion_tokens"), - reasoning_tokens=output_details.get("reasoning_tokens", 0), - ) - - def _model_output_bytes(value: dict[str, object]) -> int: """Count UTF-8 model-authored text and tool arguments without response metadata.""" output = value.get("output") @@ -888,31 +783,6 @@ def _model_output_bytes(value: dict[str, object]) -> int: return total -def _chat_output_bytes(value: dict[str, object]) -> int: - """Count model-authored UTF-8 bytes across all completion choices.""" - choices = value.get("choices") - if not isinstance(choices, list): - raise TypeError("provider output is invalid") - return _generated_string_bytes(choices) - - -def _generated_string_bytes(root: object) -> int: - """Count bounded content, refusal and tool-argument strings recursively.""" - total = 0 - stack = [root] - while stack: - current = stack.pop() - if isinstance(current, list): - stack.extend(current) - elif isinstance(current, dict): - for key, item in current.items(): - if key in {"content", "refusal", "arguments"} and isinstance(item, str): - total += len(item.encode("utf-8")) - elif isinstance(item, (dict, list)): - stack.append(item) - return total - - def _canonical_bytes(value: object) -> bytes: """Serialize provider evidence to the canonical retained JSON form.""" return json.dumps( @@ -924,18 +794,6 @@ def _canonical_bytes(value: object) -> bytes: ).encode("utf-8") -async def _bounded_response_bytes(response, maximum_bytes: int) -> bytes: - """Read a provider body without ever accumulating beyond its locked cap.""" - body = bytearray() - async for chunk in response.aiter_bytes(): - if not chunk: - continue - if len(chunk) > maximum_bytes - len(body): - raise ValueError("provider response exceeds profile") - body.extend(chunk) - return bytes(body) - - def _sha(data: bytes) -> str: """Return the canonical prefixed digest for retained bytes.""" return "sha256:" + hashlib.sha256(data).hexdigest() diff --git a/backend/app/interfaces/checker_model_provider.py b/backend/app/interfaces/checker_model_provider.py index e6a442d46..3fef0bd2e 100644 --- a/backend/app/interfaces/checker_model_provider.py +++ b/backend/app/interfaces/checker_model_provider.py @@ -14,10 +14,11 @@ MAX_CHECKER_MODEL_PROMPT_BYTES = 1 * 1024 * 1024 MAX_CHECKER_MODEL_OUTPUT_BYTES = 1 * 1024 * 1024 MAX_CHECKER_MODEL_PROVIDER_ID_LENGTH = 256 -MAX_CHECKER_MODEL_NAME_LENGTH = 128 +MAX_CHECKER_MODEL_NAME_LENGTH = 160 MAX_CHECKER_MODEL_TOKEN_COUNT = 2**63 - 1 -_PROVIDER_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]*$") +_PROVIDER_RESPONSE_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]*$") +_PROVIDER_MODEL_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/-]*$") class CheckerModelProviderStatus(StrEnum): @@ -85,6 +86,7 @@ def __post_init__(self) -> None: self.provider_response_id, maximum_length=MAX_CHECKER_MODEL_PROVIDER_ID_LENGTH, label="checker model response identity", + pattern=_PROVIDER_RESPONSE_IDENTIFIER, ) @@ -136,6 +138,7 @@ def __post_init__(self) -> None: self.provider_model, maximum_length=MAX_CHECKER_MODEL_NAME_LENGTH, label="checker model provider identity", + pattern=_PROVIDER_MODEL_IDENTIFIER, ) if type(self.status) is not CheckerModelProviderStatus: raise ValueError("checker model observation is invalid") @@ -234,10 +237,16 @@ async def aclose(self) -> None: """Close provider transport resources owned by this adapter.""" -def _require_provider_identifier(value: object, *, maximum_length: int, label: str) -> None: +def _require_provider_identifier( + value: object, + *, + maximum_length: int, + label: str, + pattern: re.Pattern[str], +) -> None: if ( not isinstance(value, str) or len(value) > maximum_length - or _PROVIDER_IDENTIFIER.fullmatch(value) is None + or pattern.fullmatch(value) is None ): raise ValueError(f"{label} is invalid") diff --git a/backend/scripts/behavior_ownership.py b/backend/scripts/behavior_ownership.py index 34cebc95f..04d5d884b 100644 --- a/backend/scripts/behavior_ownership.py +++ b/backend/scripts/behavior_ownership.py @@ -129,6 +129,8 @@ "backend/app/adapters/checkers/model_proxy.py", "backend/app/adapters/checkers/model_proxy_transport.py", "backend/app/adapters/checkers/model_runtime.py", + "backend/app/adapters/checkers/openrouter_model_provider.py", + "backend/app/adapters/model_usage/openrouter.py", "backend/app/adapters/project_agents/setup_model_handler.py", "backend/app/adapters/project_agents/setup_model_proxy.py", "backend/app/api/deps/checker_models.py", @@ -454,11 +456,11 @@ "backend/app/workers/project_setup_topology.py", }) NEUTRAL_GUIDE_RUNTIME_REMOVED_TARGETS = frozenset({ + "backend/app/adapters/project_agents/openrouter_policy.py", "backend/app/modules/artifacts/zip_safety.py", }) NEUTRAL_GUIDE_RUNTIME_TARGETS = frozenset({ - "backend/app/adapters/project_agents/openrouter_policy.py", "backend/app/core/zip_safety.py", "backend/app/adapters/guide_document_tools/child.py", "backend/app/adapters/guide_document_tools/client.py", @@ -1075,6 +1077,14 @@ def _validate_additive_partition_transition( current_by_target = {item["target"]: item for item in current_assignments} _validate_policy_lineage_relocation(set(trusted_targets), set(current_by_target)) removed = set(trusted_targets) - set(current_by_target) + if ( + "backend/app/adapters/project_agents/openrouter_policy.py" in removed + and not { + "backend/app/adapters/checkers/openrouter_model_provider.py", + "backend/app/adapters/model_usage/openrouter.py", + }.issubset(current_by_target) + ): + raise BehaviorOwnershipError("untrusted_partition_change") if removed & REV_04C_REMOVED_TARGETS and ( "backend/app/modules/reviews/api/acceptance.py" not in current_by_target ): diff --git a/backend/scripts/test_lane_catalogue.py b/backend/scripts/test_lane_catalogue.py index 373c5f738..6a806b661 100644 --- a/backend/scripts/test_lane_catalogue.py +++ b/backend/scripts/test_lane_catalogue.py @@ -508,6 +508,7 @@ class TestLane: "tests/checkers_model_calls/test_setup_model_proxy_transport.py", "tests/checkers_model_calls/test_setup_output_custody.py", "tests/checkers/model_provider/test_openai_responses.py", + "tests/checkers/model_provider/test_openrouter.py", "tests/checkers/external/test_model_transport.py", "tests/checkers/external/test_model_proxy_transport.py", "tests/authorization/checker_models/test_contracts.py", diff --git a/backend/tests/checkers/model_provider/test_openai_responses.py b/backend/tests/checkers/model_provider/test_openai_responses.py index c2b0e7b48..f205dcec6 100644 --- a/backend/tests/checkers/model_provider/test_openai_responses.py +++ b/backend/tests/checkers/model_provider/test_openai_responses.py @@ -449,7 +449,13 @@ def client(**kwargs): return FakeClient() monkeypatch.setattr("openai.AsyncOpenAI", client) - provider = create_checker_model_provider(settings()) + from tests.checkers_model_calls.test_proxy_orchestration import Harness + + profile = Harness().profile + profile = profile.model_copy(update={ + "spec": profile.spec.model_copy(update={"deadline_ms": 3000}) + }) + provider = create_checker_model_provider(profile, api_key="secret-test-key") assert provider.identity.capability_key == CHECKER_MODEL_CAPABILITY_KEY assert provider.identity.provider_key == "openai_responses" diff --git a/backend/tests/checkers/model_provider/test_openrouter.py b/backend/tests/checkers/model_provider/test_openrouter.py new file mode 100644 index 000000000..ebba6b8e0 --- /dev/null +++ b/backend/tests/checkers/model_provider/test_openrouter.py @@ -0,0 +1,551 @@ +"""Offline OpenRouter transport tests for checker model custody.""" + +from __future__ import annotations + +from datetime import UTC, datetime +import json + +import httpx +import pytest + +from app.adapters.checkers.openrouter_model_provider import ( + OpenRouterCheckerModelProvider, +) +from app.adapters.model_usage.openrouter import ( + bounded_response_bytes, + openrouter_request_options, + openrouter_usage, + require_direct_openrouter_model, +) +from app.core.hashing import canonical_json_hash +from app.core.identifiers import new_record_id +from app.interfaces.checker_model_provider import ( + CHECKER_MODEL_CAPABILITY_KEY, + MAX_CHECKER_MODEL_NAME_LENGTH, + CheckerModelCreateFailure, + CheckerModelCreateOutcome, + CheckerModelObservation, + CheckerModelProviderStatus, + CheckerModelRecoveryOutcome, + CheckerModelRequest, + CheckerModelResponseReference, + CheckerModelUsage, +) +from app.modules.model_usage.api import ProjectModelProfile, ProjectModelProfileSpec +from tests.checkers_model_calls.test_model_call_contract import profile_spec + + +def profile(**overrides) -> ProjectModelProfile: + values = profile_spec().model_dump(mode="python") + values.update( + provider="openrouter", + api="chat_completions", + model_snapshot="openai/model", + **overrides, + ) + values.update( + maximum_total_request_bytes=values["maximum_request_bytes"], + maximum_total_response_bytes=values["maximum_response_bytes"], + maximum_total_input_tokens=values["maximum_input_tokens"], + maximum_total_output_tokens=values["maximum_output_tokens"], + ) + spec = ProjectModelProfileSpec.model_validate(values) + return ProjectModelProfile( + operation_id=new_record_id(), + actor_profile_id=new_record_id(), + spec=spec, + profile_digest=canonical_json_hash(spec.model_dump(mode="json")), + profile_id=new_record_id(), + authorization_evidence_id=new_record_id(), + created_at=datetime.now(UTC), + ) + + +def metadata(**changes) -> dict: + data = { + "id": "openai/model", + "canonical_slug": "openai/model-20261026", + "context_length": 100, + "top_provider": { + "context_length": 100, + "max_completion_tokens": 50, + }, + "supported_parameters": [ + "max_tokens", + "reasoning_effort", + "tool_choice", + "tools", + "verbosity", + ], + "pricing": { + "prompt": "0.000000011", + "completion": "0.000000029", + "input_cache_read": "0.000000002", + "input_cache_write": "0.000000011", + }, + } + data.update(changes) + return {"data": data} + + +def completion(**changes) -> dict: + value = { + "id": "generation-known", + "model": "openai/model", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "verbatim output"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 8, + "prompt_tokens_details": {"cached_tokens": 3}, + "completion_tokens": 5, + "completion_tokens_details": {"reasoning_tokens": 2}, + "total_tokens": 13, + }, + } + value.update(changes) + return value + + +def adapter(handler, *, configured_profile=None) -> OpenRouterCheckerModelProvider: + client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + return OpenRouterCheckerModelProvider(configured_profile or profile(), client) + + +def test_shared_identity_and_request_controls_are_exact_and_fresh() -> None: + model = "a" * 79 + "/" + "b" * 80 + assert len(model) == MAX_CHECKER_MODEL_NAME_LENGTH + observed = CheckerModelObservation( + CheckerModelResponseReference("generation-known"), + model, + CheckerModelProviderStatus.COMPLETED, + "bounded", + CheckerModelUsage(1, 0, 1, 0, 2), + ) + assert observed.provider_model == model + with pytest.raises(ValueError, match="response identity"): + CheckerModelResponseReference("provider/response") + with pytest.raises(ValueError, match="provider identity"): + CheckerModelObservation( + CheckerModelResponseReference("generation-known"), + "x" * (MAX_CHECKER_MODEL_NAME_LENGTH + 1), + CheckerModelProviderStatus.FAILED, + None, + None, + ) + assert require_direct_openrouter_model("openai/model") == "openai/model" + for invalid in ("model", "openai/model/variant", "openrouter/model", "openai/model:free"): + with pytest.raises(ValueError, match="routing"): + require_direct_openrouter_model(invalid) + first = openrouter_request_options() + second = openrouter_request_options() + assert first == second + assert first is not second + assert first["provider"] == {"allow_fallbacks": False, "data_collection": "deny"} + assert all(plugin["enabled"] is False for plugin in first["plugins"]) + + +async def test_shared_usage_and_body_reader_enforce_complete_bounded_values() -> None: + normalized = openrouter_usage(completion()["usage"]) + assert normalized.input_tokens == 8 + assert normalized.cached_input_tokens == 3 + assert normalized.output_tokens == 5 + assert normalized.reasoning_tokens == 2 + assert normalized.total_tokens == 13 + with pytest.raises(ValueError, match="usage"): + openrouter_usage(completion()["usage"] | {"total_tokens": 12}) + + class Response: + async def aiter_bytes(self): + yield b"a" * 8 + yield b"b" + + with pytest.raises(ValueError, match="exceeds profile"): + await bounded_response_bytes(Response(), 8) + + +@pytest.mark.parametrize("empty_tool_calls", [False, True]) +async def test_create_preflights_then_dispatches_one_fixed_nonstreaming_request( + empty_tool_calls: bool, +) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + if request.method == "GET": + return httpx.Response(200, json=metadata()) + response = completion() + if empty_tool_calls: + response["choices"][0]["message"]["tool_calls"] = [] + return httpx.Response(200, json=response) + + provider = adapter(handler) + result = await provider.create(CheckerModelRequest("inspect this package")) + + assert result.outcome is CheckerModelCreateOutcome.ACCEPTED + assert result.failure is None + assert result.known_response == CheckerModelResponseReference("generation-known") + assert result.observation is not None + assert result.observation.provider_model == "openai/model" + assert result.observation.status is CheckerModelProviderStatus.COMPLETED + assert result.observation.output_text == "verbatim output" + assert result.observation.usage == CheckerModelUsage(8, 3, 5, 2, 13) + assert provider.identity.capability_key == CHECKER_MODEL_CAPABILITY_KEY + assert provider.identity.provider_key == "openrouter" + assert [(item.method, item.url.path) for item in requests] == [ + ("GET", "/api/v1/model/openai/model"), + ("POST", "/api/v1/chat/completions"), + ] + body = json.loads(requests[1].content) + assert body == { + "max_tokens": 50, + "messages": [{"content": "inspect this package", "role": "user"}], + "model": "openai/model", + "parallel_tool_calls": False, + "plugins": openrouter_request_options()["plugins"], + "provider": openrouter_request_options()["provider"], + "reasoning_effort": "medium", + "store": False, + "stream": False, + "temperature": 0, + "tool_choice": "none", + "tools": [], + "top_p": 1, + "usage": {"include": True}, + "verbosity": "low", + } + await provider.aclose() + + +async def test_request_bound_refuses_before_metadata_or_paid_dispatch() -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + raise AssertionError("request bound must prevent network I/O") + + provider = adapter(handler, configured_profile=profile(maximum_request_bytes=400)) + result = await provider.create(CheckerModelRequest("x" * 100)) + + assert result.outcome is CheckerModelCreateOutcome.NO_EFFECT + assert result.failure is CheckerModelCreateFailure.REQUEST_BYTES_EXCEEDED + assert requests == [] + await provider.aclose() + + +@pytest.mark.parametrize( + "drift", + [ + {"id": "openai/other"}, + { + "context_length": 101, + "top_provider": {"context_length": 101, "max_completion_tokens": 50}, + }, + {"top_provider": {"context_length": 101, "max_completion_tokens": 50}}, + {"supported_parameters": ["max_tokens"]}, + {"pricing": {"prompt": "0.000000012", "completion": "0.000000029"}}, + ], + ids=["model", "context-ceiling", "context-disagreement", "parameters", "pricing"], +) +async def test_metadata_refusal_prevents_paid_dispatch(drift: dict) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + if request.method == "GET": + return httpx.Response(200, json=metadata(**drift)) + return httpx.Response(200, json=completion()) + + provider = adapter(handler) + result = await provider.create(CheckerModelRequest("one effect")) + + assert result.outcome is CheckerModelCreateOutcome.NO_EFFECT + assert result.failure is CheckerModelCreateFailure.INPUT_COUNT_UNAVAILABLE + assert [request.method for request in requests] == ["GET"] + await provider.aclose() + + +@pytest.mark.parametrize( + ("post_result", "failure", "known"), + [ + (httpx.Response(503, json={"id": "generation-known"}), + CheckerModelCreateFailure.DELIVERY_UNCERTAIN, True), + (httpx.Response(200, json=completion(choices=[])), + CheckerModelCreateFailure.RESPONSE_INVALID, True), + (httpx.Response(200, content=b"not-json"), + CheckerModelCreateFailure.RESPONSE_INVALID, False), + ], + ids=["non-success", "invalid-output", "invalid-json"], +) +async def test_paid_response_failures_are_uncertain_without_second_post( + post_result: httpx.Response, + failure: CheckerModelCreateFailure, + known: bool, +) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + if request.method == "GET": + return httpx.Response(200, json=metadata()) + return post_result + + provider = adapter(handler) + result = await provider.create(CheckerModelRequest("one paid effect")) + + assert result.outcome is CheckerModelCreateOutcome.UNCERTAIN + assert result.failure is failure + assert (result.known_response is not None) is known + assert [request.method for request in requests].count("POST") == 1 + await provider.aclose() + + +async def test_lost_dispatch_is_uncertain_and_never_retried() -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + if request.method == "GET": + return httpx.Response(200, json=metadata()) + raise httpx.ReadTimeout("lost response", request=request) + + provider = adapter(handler) + result = await provider.create(CheckerModelRequest("one paid effect")) + + assert result.outcome is CheckerModelCreateOutcome.UNCERTAIN + assert result.failure is CheckerModelCreateFailure.DELIVERY_UNCERTAIN + assert [request.method for request in requests].count("POST") == 1 + await provider.aclose() + + +@pytest.mark.parametrize( + ("finish_reason", "status"), + [ + ("length", CheckerModelProviderStatus.INCOMPLETE), + ("content_filter", CheckerModelProviderStatus.FAILED), + ("error", CheckerModelProviderStatus.FAILED), + ], +) +@pytest.mark.parametrize("content_mode", ["text", "null", "absent"]) +async def test_known_terminal_failures_settle_with_complete_usage( + finish_reason: str, + status: CheckerModelProviderStatus, + content_mode: str, +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "GET": + return httpx.Response(200, json=metadata()) + body = completion() + body["choices"][0]["finish_reason"] = finish_reason + if content_mode == "null": + body["choices"][0]["message"]["content"] = None + elif content_mode == "absent": + del body["choices"][0]["message"]["content"] + return httpx.Response(200, json=body) + + provider = adapter(handler) + result = await provider.create(CheckerModelRequest("one paid effect")) + + assert result.outcome is CheckerModelCreateOutcome.ACCEPTED + assert result.observation is not None + assert result.observation.status is status + assert result.observation.output_text == ("verbatim output" if content_mode == "text" else None) + assert result.observation.usage == CheckerModelUsage(8, 3, 5, 2, 13) + await provider.aclose() + + +@pytest.mark.parametrize( + "body", + [ + completion(model="openai/foreign"), + completion(choices=[]), + completion(choices=[completion()["choices"][0], completion()["choices"][0]]), + completion(usage=completion()["usage"] | {"total_tokens": 12}), + completion(usage=completion()["usage"] | {"prompt_tokens": 101, "total_tokens": 106}), + completion(usage=completion()["usage"] | {"completion_tokens": 51, "total_tokens": 59}), + ], + ids=[ + "foreign-model", + "missing-choice", + "multiple-choices", + "usage-total", + "input-bound", + "output-token-bound", + ], +) +async def test_identity_choice_and_usage_substitution_cannot_settle(body: dict) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "GET": + return httpx.Response(200, json=metadata()) + return httpx.Response(200, json=body) + + provider = adapter(handler) + result = await provider.create(CheckerModelRequest("one paid effect")) + + assert result.outcome is CheckerModelCreateOutcome.UNCERTAIN + assert result.failure is CheckerModelCreateFailure.RESPONSE_INVALID + assert result.known_response == CheckerModelResponseReference("generation-known") + await provider.aclose() + + +@pytest.mark.parametrize( + "choice", + [ + { + "index": False, + "message": {"role": "assistant", "content": "bounded"}, + "finish_reason": "stop", + }, + { + "index": 0, + "message": {"role": "assistant", "content": None}, + "finish_reason": "stop", + }, + { + "index": 0, + "message": {"role": "assistant"}, + "finish_reason": "stop", + }, + { + "index": 0, + "message": {"role": "assistant", "content": 0}, + "finish_reason": "error", + }, + + { + "message": {"role": "assistant", "content": "bounded"}, + "finish_reason": "stop", + }, + { + "index": None, + "message": {"role": "assistant", "content": "bounded"}, + "finish_reason": "stop", + }, + { + "index": 0, + "message": {"role": "assistant", "content": "bounded", "tool_calls": {}}, + "finish_reason": "stop", + }, + + { + "index": 0, + "message": { + "role": "assistant", + "content": "bounded", + "tool_calls": [{"id": "call-hidden"}], + }, + "finish_reason": "stop", + }, + { + "index": 0, + "message": { + "role": "assistant", + "content": "bounded", + "function_call": {"name": "hidden"}, + }, + "finish_reason": "stop", + }, + ], + ids=[ + "boolean-index", "null-completed-content", "missing-completed-content", + "malformed-failed-content", "missing-index", "null-index", + "malformed-tool-calls", "tool-call", "function-call", + ], +) +async def test_non_text_choice_payload_cannot_settle(choice: dict) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "GET": + return httpx.Response(200, json=metadata()) + return httpx.Response(200, json=completion(choices=[choice])) + + provider = adapter(handler) + result = await provider.create(CheckerModelRequest("one paid effect")) + + assert result.outcome is CheckerModelCreateOutcome.UNCERTAIN + assert result.failure is CheckerModelCreateFailure.RESPONSE_INVALID + assert result.known_response == CheckerModelResponseReference("generation-known") + await provider.aclose() + + +@pytest.mark.parametrize("finish_reason", ["stop", "content_filter"]) +async def test_structurally_complete_refusal_settles_as_known_failure( + finish_reason: str, +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "GET": + return httpx.Response(200, json=metadata()) + choice = { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "refusal": "request refused", + }, + "finish_reason": finish_reason, + } + return httpx.Response(200, json=completion(choices=[choice])) + + provider = adapter(handler) + result = await provider.create(CheckerModelRequest("one paid effect")) + + assert result.outcome is CheckerModelCreateOutcome.ACCEPTED + assert result.observation is not None + assert result.observation.status is CheckerModelProviderStatus.FAILED + assert result.observation.output_text is None + assert result.observation.usage == CheckerModelUsage(8, 3, 5, 2, 13) + await provider.aclose() + + +async def test_response_and_output_byte_bounds_remain_uncertain_after_post() -> None: + async def exercise(configured_profile, body): + requests = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + if request.method == "GET": + return httpx.Response(200, json=metadata()) + return httpx.Response(200, json=body) + + provider = adapter(handler, configured_profile=configured_profile) + result = await provider.create(CheckerModelRequest("one paid effect")) + await provider.aclose() + assert [request.method for request in requests].count("POST") == 1 + return result + + output_result = await exercise( + profile(maximum_output_bytes=10, maximum_response_bytes=5_000), + completion(choices=[{ + "index": 0, + "message": {"role": "assistant", "content": "x" * 11}, + "finish_reason": "stop", + }]), + ) + response_result = await exercise( + profile(maximum_output_bytes=10, maximum_response_bytes=4_156), + completion() | {"padding": "x" * 5_000}, + ) + for result in (output_result, response_result): + assert result.outcome is CheckerModelCreateOutcome.UNCERTAIN + assert result.failure is CheckerModelCreateFailure.RESPONSE_INVALID + + +async def test_recover_is_unavailable_with_zero_network_io() -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + raise AssertionError("OpenRouter recovery must not use network I/O") + + provider = adapter(handler) + reference = CheckerModelResponseReference("generation-known") + result = await provider.recover(reference) + + assert result.outcome is CheckerModelRecoveryOutcome.UNAVAILABLE + assert result.reference == reference + assert result.observation is None + assert requests == [] + await provider.aclose() diff --git a/backend/tests/checkers_model_calls/test_composed_execution.py b/backend/tests/checkers_model_calls/test_composed_execution.py index 983b59730..54c1e22b8 100644 --- a/backend/tests/checkers_model_calls/test_composed_execution.py +++ b/backend/tests/checkers_model_calls/test_composed_execution.py @@ -2,9 +2,11 @@ from contextlib import asynccontextmanager import asyncio +import json import os from uuid import UUID, uuid4 +import httpx import pytest from sqlalchemy import select, func @@ -17,6 +19,7 @@ model_profile_reader, ) from app.adapters.checkers.model_execution import ModelAwareExternalExecution +from app.adapters.checkers.openrouter_model_provider import OpenRouterCheckerModelProvider from app.interfaces.checker_model_provider import ( CheckerModelCreateFailure, CheckerModelCreateOutcome, @@ -36,6 +39,7 @@ from app.interfaces.project_agents import CapabilityParameter, CheckSpecification from app.modules.actors.api import ServiceIdentity from app.modules.artifacts.preparation import HARD_MAXIMUM_ARTIFACT_BYTES +from app.modules.artifacts.models import ArtifactBinding from app.modules.checkers.api.external import ExternalCheckerSchema, external_checker_json_hash from app.modules.model_usage.models import ( ModelUsageEffect, @@ -108,15 +112,120 @@ def model_registry_spec(): ) +def _openrouter_metadata(): + return { + "data": { + "id": "openai/model", + "canonical_slug": "openai/model-20261026", + "context_length": 100, + "top_provider": {"context_length": 100, "max_completion_tokens": 50}, + "supported_parameters": [ + "max_tokens", + "reasoning_effort", + "tool_choice", + "tools", + "verbosity", + ], + "pricing": { + "prompt": "0.000000011", + "completion": "0.000000029", + "input_cache_read": "0.000000002", + "input_cache_write": "0.000000011", + }, + } + } + + +def _assert_single_openrouter_dispatch(provider_requests): + assert provider_requests == [ + ("GET", "https://openrouter.ai/api/v1/model/openai/model"), + ("POST", "https://openrouter.ai/api/v1/chat/completions"), + ] + + +async def _assert_effect_art_identities(session, effect, project_id, roles): + bindings = ( + await session.scalars( + select(ArtifactBinding) + .where( + ArtifactBinding.resource_type == "checker_model_call", + ArtifactBinding.resource_id == str(effect.id), + ) + .order_by(ArtifactBinding.logical_role) + ) + ).all() + expected_roles = sorted(f"model_{role}" for role in roles) + assert [binding.logical_role for binding in bindings] == expected_roles + by_role = {binding.logical_role: binding for binding in bindings} + for role in roles: + binding = by_role[f"model_{role}"] + assert binding.project_id == str(project_id) + assert binding.resource_id == str(effect.id) + assert binding.scope_version == 1 + assert binding.id == getattr(effect, f"{role}_binding_id") + assert binding.content_id == getattr(effect, f"{role}_content_id") + assert binding.put_attempt_id == getattr(effect, f"{role}_put_attempt_id") + assert binding.verification_receipt_id == getattr( + effect, f"{role}_verification_receipt_id" + ) + + @pytest.mark.parametrize( - ("storage_provider", "lose_first_image_reply", "concurrent_callback", "terminal_mode"), + ( + "storage_provider", + "lose_first_image_reply", + "concurrent_callback", + "terminal_mode", + "provider_case", + ), [ - pytest.param("local", False, False, None, id="completed"), - pytest.param("local", True, False, None, id="retry-retains-paid-effect"), - pytest.param("minio", True, False, None, id="minio-retry-retains-paid-effect"), - pytest.param("local", False, True, None, id="concurrent-callback-single-charge"), - pytest.param("local", False, False, "no_effect", id="operator-recovers-no-effect"), - pytest.param("minio", False, False, "failed", id="operator-recovers-settled-failure"), + pytest.param("local", False, False, None, "controlled", id="completed"), + pytest.param( + "local", True, False, None, "controlled", id="retry-retains-paid-effect" + ), + pytest.param( + "minio", True, False, None, "controlled", id="minio-retry-retains-paid-effect" + ), + pytest.param( + "local", + False, + True, + None, + "controlled", + id="concurrent-callback-single-charge", + ), + pytest.param( + "local", + False, + False, + "no_effect", + "controlled", + id="operator-recovers-no-effect", + ), + pytest.param( + "minio", + False, + False, + "failed", + "controlled", + id="operator-recovers-settled-failure", + ), + pytest.param( + "local", + True, + False, + None, + "openrouter_completed", + id="openrouter-lost-image-reply-replays-retained-art", + ), + pytest.param( + "local", + False, + False, + None, + "openrouter_ambiguous", + id="openrouter-ambiguous-response-is-held-without-redispatch", + ), ], ) async def test_public_profile_to_private_model_call_and_completed_checker( @@ -128,6 +237,7 @@ async def test_public_profile_to_private_model_call_and_completed_checker( lose_first_image_reply, concurrent_callback, terminal_mode, + provider_case, ): if storage_provider == "minio": from tests import test_s3_artifact_store @@ -145,6 +255,14 @@ async def publish(_factory, project_id, registry): "registry_entry_digest": registry.entry_digest, } ) + if provider_case.startswith("openrouter"): + spec = spec.model_copy( + update={ + "provider": "openrouter", + "api": "chat_completions", + "model_snapshot": "openai/model", + } + ) if terminal_mode is not None: # An authorized child is a second provider dispatch in the same # immutable envelope, even though it reuses logical call slot zero. @@ -205,6 +323,7 @@ async def publish(_factory, project_id, registry): ) as h: await provision_post_submit_service(h.factory, ServiceIdentity.ARTIFACT_CHECKER_OUTPUT) creates = [] + provider_requests = [] lost_replies = [] recovered_references = [] first_response_waiting = asyncio.Event() @@ -255,9 +374,53 @@ async def recover(self, reference): ), ) + async def openrouter_transport(request): + provider_requests.append((request.method, str(request.url))) + if request.method == "GET": + return httpx.Response(200, json=_openrouter_metadata()) + assert request.method == "POST" + body = json.loads(request.content) + assert body["model"] == "openai/model" + assert body["messages"] == [ + {"role": "user", "content": "Evaluate the verified work"} + ] + response = { + "id": "gen-composed-openrouter-1", + "model": "openai/model", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": { + "role": "assistant", + "tool_calls": [], + "content": '{"verdict":"passed"}', + }, + } + ], + "usage": { + "prompt_tokens": 20, + "completion_tokens": 10, + "total_tokens": 30, + "prompt_tokens_details": {"cached_tokens": 5}, + "completion_tokens_details": {"reasoning_tokens": 2}, + }, + } + if provider_case == "openrouter_ambiguous": + response["choices"] = [] + return httpx.Response(200, json=response) + @asynccontextmanager - async def provider(_): - yield Provider() + async def provider(profile): + if provider_case.startswith("openrouter"): + client = httpx.AsyncClient(transport=httpx.MockTransport(openrouter_transport)) + selected = OpenRouterCheckerModelProvider(profile, client=client) + try: + yield selected + finally: + await selected.aclose() + else: + yield Provider() outputs = checker_output_storage( sessions=h.factory, @@ -328,6 +491,17 @@ async def execute(self, request, grant, *, model_grant=None): reply = replies[0] else: reply = await proxy.handle(callback) + if provider_case == "openrouter_ambiguous": + assert reply.outcome == "unavailable" + assert reply.failure_code == ( + "model_provider_delivery_uncertain" + if self.invocations == 1 + else "model_provider_recovery_unavailable" + ) + repeated = await proxy.handle(callback) + assert repeated.outcome == "unavailable" + assert repeated.failure_code == "model_provider_recovery_unavailable" + raise ExternalServiceUnavailableError(self.identity) if terminal_mode is not None and self.invocations == 1: assert reply.outcome == "unavailable", reply.outcome assert reply.failure_code == ( @@ -353,7 +527,12 @@ async def execute(self, request, grant, *, model_grant=None): ) current_request = h.request result = await executor.evaluate_post_submission(current_request) - if lose_first_image_reply or terminal_mode is not None: + needs_retry = ( + lose_first_image_reply + or terminal_mode is not None + or provider_case == "openrouter_ambiguous" + ) + if needs_retry: assert result.outcome == "infrastructure_failed" if lose_first_image_reply: assert len(lost_replies) == 1, result.infrastructure_failure_code @@ -405,10 +584,32 @@ async def execute(self, request, grant, *, model_grant=None): current_request = reserved.request assert current_request.evaluation_generation == 2 result = await executor.evaluate_post_submission(current_request) + if provider_case == "openrouter_ambiguous": + assert result.outcome == "infrastructure_failed" + _assert_single_openrouter_dispatch(provider_requests) + async with h.factory() as session: + assert await session.scalar( + select(func.count()).select_from(ModelUsageEffect) + ) == 1 + attempt = await session.scalar(select(ModelUsageEffect)) + account = await session.get(ModelUsageBudgetAccount, str(h.request.project_id)) + assert attempt.state == "uncertain" + assert attempt.provider_response_id == "gen-composed-openrouter-1" + assert attempt.response_binding_id is None + assert attempt.settled_nanodollars is None + assert account.lifetime_reserved_nanodollars > 0 + assert account.lifetime_settled_nanodollars == 0 + await _assert_effect_art_identities( + session, attempt, h.request.project_id, {"request"} + ) + return assert result.outcome == "completed" assert len(result.execution.external_members) == 1 assert await executor.evaluate_post_submission(current_request) == result - assert len(creates) == (2 if terminal_mode else 1) + if provider_case == "openrouter_completed": + _assert_single_openrouter_dispatch(provider_requests) + else: + assert len(creates) == (2 if terminal_mode else 1) async with h.factory() as session: assert await session.scalar(select(func.count()).select_from(ModelUsageEffect)) == ( 2 if terminal_mode else 1 @@ -424,3 +625,6 @@ async def execute(self, request, grant, *, model_grant=None): ) assert UUID(attempt.response_binding_id).version == 7 assert UUID(attempt.request_binding_id).version == 7 + await _assert_effect_art_identities( + session, attempt, h.request.project_id, {"request", "response"} + ) diff --git a/backend/tests/checkers_model_calls/test_model_runtime.py b/backend/tests/checkers_model_calls/test_model_runtime.py index bdd1badd1..2a212aade 100644 --- a/backend/tests/checkers_model_calls/test_model_runtime.py +++ b/backend/tests/checkers_model_calls/test_model_runtime.py @@ -49,31 +49,96 @@ def test_proxy_requires_absolute_socket_and_enabled_storage(tmp_path): assert settings.checker_model_proxy_socket == tmp_path / "sockets/proxy.sock" -async def test_provider_uses_profile_bounds_and_closes_owned_client(monkeypatch): +@pytest.mark.parametrize( + ("provider_name", "api", "model", "credential", "other_credential"), + [ + ("openai_responses", "responses.v1", "gpt-5.6-terra", "OPENAI_API_KEY", "OPENROUTER_API_KEY"), + ("openrouter", "chat_completions", "openai/gpt-5.6-terra", "OPENROUTER_API_KEY", "OPENAI_API_KEY"), + ], +) +async def test_provider_uses_exact_profile_and_only_selected_key( + monkeypatch, provider_name, api, model, credential, other_credential +): profile = Harness().profile + profile = profile.model_copy(update={"spec": profile.spec.model_copy(update={ + "provider": provider_name, "api": api, "model_snapshot": model, + })}) captured = [] closed = [] async def close(): closed.append(True) - def construct(settings): - captured.append(settings) + def construct(received_profile, *, api_key): + captured.append((received_profile, api_key)) return SimpleNamespace(aclose=close) - monkeypatch.setenv("OPENAI_API_KEY", "private-test-key") + monkeypatch.delenv(other_credential, raising=False) + monkeypatch.setenv(credential, "private-test-key") monkeypatch.setattr(model_runtime, "create_checker_model_provider", construct) async with model_runtime.configured_model_provider(profile): - assert len(captured) == 1 - settings = captured[0] - assert settings.model == profile.spec.model_snapshot - assert settings.maximum_input_tokens == profile.spec.maximum_input_tokens - assert settings.maximum_output_tokens == profile.spec.maximum_output_tokens - assert settings.reasoning_effort == profile.spec.settings.reasoning_effort - assert "private-test-key" not in repr(settings) + assert captured == [(profile, "private-test-key")] assert closed == [True] +@pytest.mark.parametrize("provider_name,api", [ + ("unknown", "chat_completions"), ("openrouter", "responses.v1"), + ("openai_responses", "chat_completions"), +]) +async def test_unsupported_pair_denies_before_credential_or_client_access( + monkeypatch, provider_name, api +): + profile = Harness().profile + profile = profile.model_copy(update={"spec": profile.spec.model_copy(update={ + "provider": provider_name, "api": api, + })}) + + class NoCredentials: + def get(self, *_args): + raise AssertionError("unsupported profile accessed credentials") + + monkeypatch.setattr(model_runtime, "os", SimpleNamespace(environ=NoCredentials())) + with pytest.raises(ModelCallUnavailable, match="model_profile_provider_unavailable"): + async with model_runtime.configured_model_provider(profile): + pass + + +async def test_openrouter_factory_needs_no_openai_client(monkeypatch): + import httpx + import openai + from app.adapters.checkers import create_checker_model_provider + + def forbidden(*_args, **_kwargs): + raise AssertionError("OpenRouter constructed an OpenAI client") + + profile = Harness().profile + profile = profile.model_copy(update={"spec": profile.spec.model_copy(update={ + "provider": "openrouter", "api": "chat_completions", + "model_snapshot": "openai/gpt-5.6-terra", + })}) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.setattr(openai, "AsyncOpenAI", forbidden) + captured = [] + original = httpx.AsyncClient + + class CapturingClient(original): + def __init__(self, **kwargs): + captured.append(kwargs) + super().__init__(**kwargs) + + monkeypatch.setattr(httpx, "AsyncClient", CapturingClient) + provider = create_checker_model_provider(profile, api_key="private-openrouter-key") + try: + assert provider.identity.provider_key == "openrouter" + assert len(captured) == 1 + assert captured[0]["trust_env"] is False + assert captured[0]["follow_redirects"] is False + assert captured[0]["timeout"] == profile.spec.deadline_ms / 1000 + assert captured[0]["headers"]["Authorization"] == "Bearer private-openrouter-key" + finally: + await provider.aclose() + + async def test_missing_provider_key_stops_before_reserving_budget(monkeypatch): monkeypatch.delenv("OPENAI_API_KEY", raising=False) h = Harness() @@ -86,6 +151,43 @@ async def test_missing_provider_key_stops_before_reserving_budget(monkeypatch): pass +async def test_missing_openrouter_key_cannot_use_openai_key(monkeypatch): + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + monkeypatch.setenv("OPENAI_API_KEY", "other-provider-key") + h = Harness() + h.profile = h.profile.model_copy(update={"spec": h.profile.spec.model_copy(update={ + "provider": "openrouter", "api": "chat_completions", + "model_snapshot": "openai/model", + })}) + h.provider = model_runtime.configured_model_provider + assert (await h.invoke()).outcome == "unavailable" + assert h.state == "prepared" + assert "dispatch_committed" not in h.events + + +async def test_ordinary_post_needs_no_model_credentials_or_budget(monkeypatch): + from app.adapters.checkers.model_execution import ModelAwareExternalExecution + + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + request = SimpleNamespace(configuration={}, identity=SimpleNamespace(phase="post_submit")) + grant, result = object(), object() + calls = [] + + async def execute(received_request, received_grant): + calls.append((received_request, received_grant)) + return result + + def forbidden(): + raise AssertionError("ordinary POST opened model authorization/budget storage") + + adapter = ModelAwareExternalExecution( + execution=SimpleNamespace(execute=execute), sessions=forbidden, + ) + assert await adapter.execute(request, grant) is result + assert calls == [(request, grant)] + + async def test_singleton_worker_drains_before_engine_disposal(monkeypatch): """SIGTERM stops admission and awaits the owned runtime before DB disposal.""" calls = [] diff --git a/backend/tests/checkers_model_calls/test_setup_model_handler.py b/backend/tests/checkers_model_calls/test_setup_model_handler.py index b4e68485a..e04252f36 100644 --- a/backend/tests/checkers_model_calls/test_setup_model_handler.py +++ b/backend/tests/checkers_model_calls/test_setup_model_handler.py @@ -7,13 +7,15 @@ import pytest -from app.adapters.project_agents.openrouter_policy import openrouter_request_options +from app.adapters.model_usage.openrouter import ( + bounded_response_bytes, + openrouter_request_options, +) from app.adapters.project_agents.setup_model_handler import ( OpenAISetupProvider, OpenRouterSetupProvider, SetupModelProxyHandler, SetupProviderResponse, - _bounded_response_bytes, _completed_reply, _validate_request, ) @@ -140,7 +142,7 @@ async def aiter_bytes(self): yield b"b" with pytest.raises(ValueError, match="exceeds profile"): - await _bounded_response_bytes(Response(), 8) + await bounded_response_bytes(Response(), 8) async def test_openai_count_includes_every_semantic_input_field(): diff --git a/backend/tests/checkers_model_calls/test_setup_model_handler_postgresql.py b/backend/tests/checkers_model_calls/test_setup_model_handler_postgresql.py index 51eeb1334..383eb2810 100644 --- a/backend/tests/checkers_model_calls/test_setup_model_handler_postgresql.py +++ b/backend/tests/checkers_model_calls/test_setup_model_handler_postgresql.py @@ -36,7 +36,7 @@ from app.adapters.auth import setup_model_service_authorization from app.adapters.checkers import model_profile_reader from app.adapters.model_usage import setup_model_usage_service -from app.adapters.project_agents.openrouter_policy import openrouter_request_options +from app.adapters.model_usage.openrouter import openrouter_request_options from app.adapters.project_agents.setup_model_handler import ( SetupModelProxyHandler, SetupProviderResponse, diff --git a/backend/tests/test_artifact_architecture.py b/backend/tests/test_artifact_architecture.py index 7726561c6..e2e150aa7 100644 --- a/backend/tests/test_artifact_architecture.py +++ b/backend/tests/test_artifact_architecture.py @@ -29,7 +29,6 @@ ) COMPOSITION_ROOT = APP_ROOT / "adapters" / "artifacts" / "__init__.py" CHECKER_COMPOSITION_ROOT = APP_ROOT / "adapters" / "checkers" / "__init__.py" -MODEL_RUNTIME_COMPOSITION = APP_ROOT / "adapters" / "checkers" / "model_runtime.py" AGENT_COMPOSITION_ROOT = APP_ROOT / "adapters/project_agents/__init__.py" GUIDE_PARSER_COMPOSITION_ROOT = APP_ROOT / "adapters/guide_document_tools/__init__.py" OBSERVABILITY_COMPOSITION_ROOT = APP_ROOT / "adapters" / "observability.py" @@ -99,6 +98,7 @@ "app.adapters.artifacts.trusted_intake", "app.adapters.checkers.external_service", "app.adapters.checkers.model_provider", + "app.adapters.checkers.openrouter_model_provider", } CONCRETE_OBSERVABILITY_MODULES = { "opentelemetry.exporter.otlp.proto.http.metric_exporter", @@ -372,6 +372,7 @@ def _assert_concrete_adapter_composition( "UnixSocketExternalCheckerAdapter", "UnixSocketTrustedIntakeAdapter", "OpenAIResponsesCheckerModelProvider", + "OpenRouterCheckerModelProvider", } ) agent_calls.extend(path for name in calls if name in AGENT_RUNTIME_CONSTRUCTORS) @@ -391,13 +392,12 @@ def _assert_concrete_adapter_composition( GITHUB_COMPOSITION_ROOT: 1, } assert Counter(adapter_calls) == { - COMPOSITION_ROOT: 2, S3_ADAPTER_MODULE: 1, CHECKER_COMPOSITION_ROOT: 2, + COMPOSITION_ROOT: 2, S3_ADAPTER_MODULE: 1, CHECKER_COMPOSITION_ROOT: 3, } assert set(concrete_imports) == { COMPOSITION_ROOT, CHECKER_COMPOSITION_ROOT, AWS_LIVE_PROOF_MODULE, - MODEL_RUNTIME_COMPOSITION, } assert agent_imports == [AGENT_COMPOSITION_ROOT] # Constructors are values in the one explicit registration map, invoked only @@ -427,6 +427,10 @@ def test_concrete_adapter_construction_has_one_composition_path( "import app.adapters.project_agents.omnicore_agent as sdk\nsdk.OmniCoreAgentProjectGuideRuntime(configuration)\n", "from app.adapters.project_agents.manager_authored import ManagerAuthoredProjectGuideRuntime as runtime\nruntime(configuration)\n", "import app.adapters.project_agents.manager_authored as sdk\nsdk.ManagerAuthoredProjectGuideRuntime(configuration)\n", + "from app.adapters.checkers.model_provider import OpenAIResponsesCheckerModelProvider as provider\nprovider(settings, client)\n", + "import app.adapters.checkers.model_provider as sdk\nsdk.OpenAIResponsesCheckerModelProvider(settings, client)\n", + "from app.adapters.checkers.openrouter_model_provider import OpenRouterCheckerModelProvider as provider\nprovider(profile, client)\n", + "import app.adapters.checkers.openrouter_model_provider as sdk\nsdk.OpenRouterCheckerModelProvider(profile, client)\n", ], ) def test_composition_proof_rejects_aliased_factories_and_concrete_runtimes( diff --git a/backend/tests/test_behavior_ownership.py b/backend/tests/test_behavior_ownership.py index e86674ea3..ec5bbe429 100644 --- a/backend/tests/test_behavior_ownership.py +++ b/backend/tests/test_behavior_ownership.py @@ -2644,6 +2644,38 @@ def test_partition_allows_only_exact_guide_compilation_draft_target(): ) +def test_openrouter_provider_ownership_replaces_the_superseded_policy_module(): + removed = "backend/app/adapters/project_agents/openrouter_policy.py" + additions = { + "backend/app/adapters/checkers/openrouter_model_provider.py", + "backend/app/adapters/model_usage/openrouter.py", + } + retained = "backend/app/core/config.py" + assert removed in ownership.NEUTRAL_GUIDE_RUNTIME_REMOVED_TARGETS + assert removed not in ownership.NEUTRAL_GUIDE_RUNTIME_TARGETS + assert additions <= ownership.CHECKER_MODEL_RUNTIME_TARGETS + assert {ownership.group_for_target(target) for target in additions} == { + "artifacts", + "shared", + } + trusted = _partition(sorted({removed, retained})) + current = _partition(sorted({retained, *additions})) + ownership._validate_additive_partition_transition(current, trusted) + + with pytest.raises(ownership.BehaviorOwnershipError, match="untrusted_partition_change"): + ownership._validate_additive_partition_transition(_partition([retained]), trusted) + for missing in additions: + incomplete = additions - {missing} + with pytest.raises( + ownership.BehaviorOwnershipError, + match="untrusted_partition_change", + ): + ownership._validate_additive_partition_transition( + _partition(sorted({retained, *incomplete})), + trusted, + ) + + def test_revision_ownership_does_not_allow_a_parallel_exhaustion_writer(): retained = "backend/app/core/config.py" trusted = _partition([retained]) diff --git a/backend/tests/test_ci_lane_catalogue.py b/backend/tests/test_ci_lane_catalogue.py index 3ea8a73bf..67525802d 100644 --- a/backend/tests/test_ci_lane_catalogue.py +++ b/backend/tests/test_ci_lane_catalogue.py @@ -80,6 +80,7 @@ def test_checker_delivery_and_routing_preparation_partition_every_node_once() -> "tests/checkers_model_calls/test_setup_model_proxy_transport.py", "tests/checkers_model_calls/test_setup_output_custody.py", "tests/checkers/model_provider/test_openai_responses.py", + "tests/checkers/model_provider/test_openrouter.py", "tests/checkers/external/test_model_transport.py", "tests/checkers/external/test_model_proxy_transport.py", "tests/authorization/checker_models/test_contracts.py", @@ -291,8 +292,8 @@ def test_semantic_family_membership_has_an_independent_fingerprint() -> None: "76bee594b1d740d89d42fa4bf055eb6057458ca066c97cc793f9896bac67b281", ), "project": ( - 232, - "45ed34925d7cae23ee0b5a4f62b33a0492659f7152eb111ed774346406958843", + 233, + "31df0d9eabcef7dba1db36eaf4e3fecad45f7b9766125ed728b2647991fa632e", ), "task": ( 95, @@ -364,9 +365,9 @@ def test_project_lane_ownership_has_an_independent_fingerprint() -> None: ) + "\n" ) - assert len(catalogue.PROJECT_PARTITIONED_MODULES) == 232 + assert len(catalogue.PROJECT_PARTITIONED_MODULES) == 233 assert hashlib.sha256(canonical.encode()).hexdigest() == ( - "cd5f60726209bc59d8464cf4ef7c5490f819daabaf980635562cd422d25611d5" + "f1364ffce62471dd20ce38c372b756d18b85fb954cb2f25ede59d8cb2c933ca3" ) diff --git a/backend/tests/test_omnicore_agent_sdk.py b/backend/tests/test_omnicore_agent_sdk.py index dd3610fbb..53f5a107f 100644 --- a/backend/tests/test_omnicore_agent_sdk.py +++ b/backend/tests/test_omnicore_agent_sdk.py @@ -26,7 +26,7 @@ _validated_test_upstream, _validate_request_body, ) -from app.adapters.project_agents.openrouter_policy import openrouter_request_options +from app.adapters.model_usage.openrouter import openrouter_request_options from app.adapters.project_agents.provider_resilience import ( ProviderCircuit, ProviderCircuitOpen, diff --git a/deploy/hosted/README.md b/deploy/hosted/README.md index ad2e1961c..de552cdce 100644 --- a/deploy/hosted/README.md +++ b/deploy/hosted/README.md @@ -158,12 +158,16 @@ not secret values. Select real Flow issuer, audience, JWKS and algorithm values. The release profile uses `WORKSTREAM_AUTH_PROVIDER=flow`; local Flow-HMAC and development authentication are absent. -Create each secret selected by the configured runtime as a one-value regular file at the configured path. Make it -owned by the runtime UID/GID with mode `0400`. The explicit role loader gives the +Create each secret named by a nonempty configured path as a one-value regular +file. Make it owned by the runtime UID/GID with mode `0400`. The explicit role loader gives the API only the database, broker, rate-limit and pagination secrets; the worker gets the database and broker. `WORKSTREAM_MODEL_PROXY_ENABLED=true` starts one trusted model-proxy process independently of the setup runtime. Only that process -gets the database, the two closed provider credentials and the runtime ART identity. +gets the database, each configured provider credential and the runtime ART identity. +For non-manager setup, the selected setup provider path is required and the other +provider is optional. Manager-authored setup requires at least one provider path +when the proxy is enabled. Nonempty provider paths select provider-specific Compose +overlays; the common inference overlay contains only the socket topology. The setup worker receives a read-only mount of `/var/lib/workstream/model-runtime` and the fixed socket selector; it receives no provider credential or endpoint override. A proxy-disabled @@ -183,10 +187,11 @@ The service waits for the proxy to become healthy and mounts only the current invocation's relay into a project image. Project images receive neither the global socket nor provider credentials. API, Beat and proof jobs receive neither model directory. -The two provider selectors are trusted release configuration. The proxy loads -only its dedicated files as `OPENROUTER_API_KEY` and `OPENAI_API_KEY`; this is +The two provider file paths are trusted release configuration. The proxy loads +only configured dedicated files as `OPENROUTER_API_KEY` or `OPENAI_API_KEY`; this is independent of the setup provider because approved POST and setup profiles may -use different providers. It clears both native variables before loading them +use different providers. An absent provider is not mounted or exported. The +proxy clears both native variables before loading configured files and never exports a neutral key variable. The endpoints are fixed by the registered adapter and cannot be supplied in the release file. The parser receives no database, broker, S3, Flow, model or provider credential. diff --git a/deploy/hosted/compose.inference.yml b/deploy/hosted/compose.inference.yml index 0a7561002..01c7ac5a9 100644 --- a/deploy/hosted/compose.inference.yml +++ b/deploy/hosted/compose.inference.yml @@ -23,25 +23,3 @@ services: depends_on: model-proxy: condition: service_healthy - - model-proxy: - environment: - WORKSTREAM_MODEL_PROXY_OPENAI_API_KEY_FILE: /run/secrets/model_proxy_openai_api_key - WORKSTREAM_MODEL_PROXY_OPENROUTER_API_KEY_FILE: /run/secrets/model_proxy_openrouter_api_key - secrets: - - source: model_proxy_openai_api_key - target: model_proxy_openai_api_key - uid: "${WORKSTREAM_RUNTIME_UID}" - gid: "${WORKSTREAM_RUNTIME_GID}" - mode: 0400 - - source: model_proxy_openrouter_api_key - target: model_proxy_openrouter_api_key - uid: "${WORKSTREAM_RUNTIME_UID}" - gid: "${WORKSTREAM_RUNTIME_GID}" - mode: 0400 - -secrets: - model_proxy_openai_api_key: - file: ${WORKSTREAM_MODEL_PROXY_OPENAI_API_KEY_FILE:?set the private model-proxy OpenAI key file} - model_proxy_openrouter_api_key: - file: ${WORKSTREAM_MODEL_PROXY_OPENROUTER_API_KEY_FILE:?set the private model-proxy OpenRouter key file} diff --git a/deploy/hosted/compose.model-openai.yml b/deploy/hosted/compose.model-openai.yml new file mode 100644 index 000000000..eb0ccdd59 --- /dev/null +++ b/deploy/hosted/compose.model-openai.yml @@ -0,0 +1,14 @@ +services: + model-proxy: + environment: + WORKSTREAM_MODEL_PROXY_OPENAI_API_KEY_FILE: /run/secrets/model_proxy_openai_api_key + secrets: + - source: model_proxy_openai_api_key + target: model_proxy_openai_api_key + uid: "${WORKSTREAM_RUNTIME_UID}" + gid: "${WORKSTREAM_RUNTIME_GID}" + mode: 0400 + +secrets: + model_proxy_openai_api_key: + file: ${WORKSTREAM_MODEL_PROXY_OPENAI_API_KEY_FILE:?set the private model-proxy OpenAI key file} diff --git a/deploy/hosted/compose.model-openrouter.yml b/deploy/hosted/compose.model-openrouter.yml new file mode 100644 index 000000000..be25df85b --- /dev/null +++ b/deploy/hosted/compose.model-openrouter.yml @@ -0,0 +1,14 @@ +services: + model-proxy: + environment: + WORKSTREAM_MODEL_PROXY_OPENROUTER_API_KEY_FILE: /run/secrets/model_proxy_openrouter_api_key + secrets: + - source: model_proxy_openrouter_api_key + target: model_proxy_openrouter_api_key + uid: "${WORKSTREAM_RUNTIME_UID}" + gid: "${WORKSTREAM_RUNTIME_GID}" + mode: 0400 + +secrets: + model_proxy_openrouter_api_key: + file: ${WORKSTREAM_MODEL_PROXY_OPENROUTER_API_KEY_FILE:?set the private model-proxy OpenRouter key file} diff --git a/deploy/hosted/preflight.py b/deploy/hosted/preflight.py index 7abe41442..8a713a6fd 100644 --- a/deploy/hosted/preflight.py +++ b/deploy/hosted/preflight.py @@ -20,6 +20,8 @@ COMPOSE_PATH = Path(__file__).with_name("compose.yml") INFERENCE_COMPOSE_PATH = Path(__file__).with_name("compose.inference.yml") +MODEL_OPENAI_COMPOSE_PATH = Path(__file__).with_name("compose.model-openai.yml") +MODEL_OPENROUTER_COMPOSE_PATH = Path(__file__).with_name("compose.model-openrouter.yml") SCRATCH_ROOT = Path("/var/lib/workstream/scratch") SOCKET_ROOT = Path("/var/lib/workstream/checker-runtime") SOCKET_PATH = SOCKET_ROOT / "checker.sock" @@ -54,6 +56,14 @@ "WORKSTREAM_MODEL_PROXY_OPENAI_API_KEY_FILE", "WORKSTREAM_MODEL_PROXY_OPENROUTER_API_KEY_FILE", ) +MODEL_PROVIDER_SECRET_FILE_KEYS = { + "openai": "WORKSTREAM_MODEL_PROXY_OPENAI_API_KEY_FILE", + "openrouter": "WORKSTREAM_MODEL_PROXY_OPENROUTER_API_KEY_FILE", +} +MODEL_PROVIDER_COMPOSE_PATHS = { + "openai": MODEL_OPENAI_COMPOSE_PATH, + "openrouter": MODEL_OPENROUTER_COMPOSE_PATH, +} SECRET_FILE_KEYS = (*BASE_SECRET_FILE_KEYS, *MODEL_PROXY_SECRET_FILE_KEYS) SECRET_FILE_TARGETS = { "WORKSTREAM_DATABASE_URL_FILE": ("database_url", "/run/secrets/database_url"), @@ -372,8 +382,20 @@ def validate_environment(values: dict[str, str]) -> None: if runtime != "manager_authored" and not model_proxy_enabled: raise PreflightError("inference setup requires the trusted model proxy") if model_proxy_enabled: - for key in MODEL_PROXY_SECRET_FILE_KEYS: - _required(values, key) + configured_provider_keys = { + key for key in MODEL_PROXY_SECRET_FILE_KEYS if values.get(key) + } + if runtime == "manager_authored": + if not configured_provider_keys: + raise PreflightError( + "enabled model proxy requires at least one provider credential" + ) + else: + selected_provider_key = MODEL_PROVIDER_SECRET_FILE_KEYS[provider] + if selected_provider_key not in configured_provider_keys: + raise PreflightError( + "inference setup requires its selected provider credential" + ) elif any(values.get(key) for key in MODEL_PROXY_SECRET_FILE_KEYS): raise PreflightError("disabled model proxy cannot select provider credentials") try: @@ -512,14 +534,20 @@ def _private_metadata(path: Path, *, kind: str, uid: int, gid: int, mode: int) - def _secret_file_keys(values: dict[str, str]) -> tuple[str, ...]: if values["WORKSTREAM_MODEL_PROXY_ENABLED"] == "true": - return SECRET_FILE_KEYS + return ( + *BASE_SECRET_FILE_KEYS, + *(key for key in MODEL_PROXY_SECRET_FILE_KEYS if values.get(key)), + ) return BASE_SECRET_FILE_KEYS def _role_secret_file_keys(role: str, values: dict[str, str]) -> tuple[str, ...]: keys = ROLE_SECRET_FILE_KEYS[role] if role == "model-proxy" and values["WORKSTREAM_MODEL_PROXY_ENABLED"] == "true": - return (*keys, *MODEL_PROXY_SECRET_FILE_KEYS) + return ( + *keys, + *(key for key in MODEL_PROXY_SECRET_FILE_KEYS if values.get(key)), + ) return keys @@ -658,6 +686,11 @@ def _compose_file_arguments(values: dict[str, str]) -> list[str]: arguments = ["--file", str(COMPOSE_PATH)] if values["WORKSTREAM_MODEL_PROXY_ENABLED"] == "true": arguments.extend(("--file", str(INFERENCE_COMPOSE_PATH))) + for provider, key in MODEL_PROVIDER_SECRET_FILE_KEYS.items(): + if values.get(key): + arguments.extend( + ("--file", str(MODEL_PROVIDER_COMPOSE_PATHS[provider])) + ) arguments.extend(("--profile", "model-proxy")) return arguments diff --git a/deploy/hosted/release.env.example b/deploy/hosted/release.env.example index 2a2a2831e..b52cf47a4 100644 --- a/deploy/hosted/release.env.example +++ b/deploy/hosted/release.env.example @@ -18,15 +18,16 @@ WORKSTREAM_DATABASE_URL_FILE=/etc/workstream/secrets/database_url WORKSTREAM_CELERY_BROKER_URL_FILE=/etc/workstream/secrets/celery_broker_url WORKSTREAM_API_RATE_LIMIT_KEY_SECRET_FILE=/etc/workstream/secrets/api_rate_limit_key WORKSTREAM_PAGINATION_CURSOR_HMAC_SECRET_FILE=/etc/workstream/secrets/pagination_cursor_hmac -# Required only when WORKSTREAM_MODEL_PROXY_ENABLED=true. They are independent -# of the setup SDK/provider because approved POST profiles may select OpenAI -# while setup selects OpenRouter. Omit both when the proxy is disabled. -WORKSTREAM_MODEL_PROXY_OPENAI_API_KEY_FILE=/etc/workstream/secrets/model_proxy_openai_api_key +# Nonempty paths select the matching provider overlay. The selected setup +# provider is required; the other provider is optional for approved POST +# profiles. Manager-authored setup requires at least one. Leave both blank when +# the proxy is disabled. This example selects OpenRouter only. +WORKSTREAM_MODEL_PROXY_OPENAI_API_KEY_FILE= WORKSTREAM_MODEL_PROXY_OPENROUTER_API_KEY_FILE=/etc/workstream/secrets/model_proxy_openrouter_api_key # The runtime, provider, model and bounded effective profile are public trusted -# configuration. Only the dedicated trusted model-proxy entrypoint maps the two -# closed provider credential files to native variables. The proxy +# configuration. Only the dedicated trusted model-proxy entrypoint maps each +# configured provider credential file to its native variable. The proxy # image is always built with the pinned `agents` dependency set; workers may use # either mutually exclusive project-agent SDK image and never receive the key. WORKSTREAM_PROJECT_AGENT_RUNTIME=openai_agents_sdk diff --git a/docker/backend/release-entrypoint.sh b/docker/backend/release-entrypoint.sh index e657004d5..9bf832102 100644 --- a/docker/backend/release-entrypoint.sh +++ b/docker/backend/release-entrypoint.sh @@ -104,8 +104,40 @@ case "$role" in ;; model-proxy) load_required_secret WORKSTREAM_DATABASE_URL "${WORKSTREAM_DATABASE_URL_FILE:-}" - load_required_secret OPENAI_API_KEY "${WORKSTREAM_MODEL_PROXY_OPENAI_API_KEY_FILE:-}" - load_required_secret OPENROUTER_API_KEY "${WORKSTREAM_MODEL_PROXY_OPENROUTER_API_KEY_FILE:-}" + openai_key_file="${WORKSTREAM_MODEL_PROXY_OPENAI_API_KEY_FILE:-}" + openrouter_key_file="${WORKSTREAM_MODEL_PROXY_OPENROUTER_API_KEY_FILE:-}" + if [ "$runtime" = "manager_authored" ]; then + if [ -z "$openai_key_file" ] && [ -z "$openrouter_key_file" ]; then + echo "enabled model proxy requires at least one provider credential" >&2 + exit 1 + fi + else + case "$provider" in + openai) + if [ -z "$openai_key_file" ]; then + echo "inference setup requires its selected provider credential" >&2 + exit 1 + fi + ;; + openrouter) + if [ -z "$openrouter_key_file" ]; then + echo "inference setup requires its selected provider credential" >&2 + exit 1 + fi + ;; + *) + echo "project agent model provider is unsupported" >&2 + exit 1 + ;; + esac + fi + if [ -n "$openai_key_file" ]; then + load_required_secret OPENAI_API_KEY "$openai_key_file" + fi + if [ -n "$openrouter_key_file" ]; then + load_required_secret OPENROUTER_API_KEY "$openrouter_key_file" + fi + unset openai_key_file openrouter_key_file ;; beat) load_required_secret WORKSTREAM_DATABASE_URL "${WORKSTREAM_DATABASE_URL_FILE:-}" diff --git a/docs/operations_checker_models.md b/docs/operations_checker_models.md index ebbdf9b66..1fe398452 100644 --- a/docs/operations_checker_models.md +++ b/docs/operations_checker_models.md @@ -65,9 +65,11 @@ unavailable before spend. ## Supported provider boundaries -POST model sessions currently use the closed `openai_responses` / `responses.v1` -adapter. Setup preserves both native agent runtimes through a credential-free -local transport: +POST model sessions select either `openai_responses` / `responses.v1` or +`openrouter` / `chat_completions` from the exact locked project profile. Ordinary +POST images require neither a model profile nor provider credentials. Unsupported +provider/API pairs deny without fallback. Setup preserves both native agent +runtimes through a credential-free local transport: - OpenAI Agents SDK uses the approved `openai` / `responses` request shape; - OmniCoreAgent uses the approved `openrouter` / `chat_completions` shape. @@ -81,7 +83,11 @@ the exact non-alias model metadata endpoint before dispatch and requires current context, completion, parameter and pricing metadata to fit the immutable profile; unknown or unsupported non-token fees fail before spend. Metadata-to-dispatch consistency remains an external provider assumption that must be requalified for -the selected account and model. +the selected account and model. OpenRouter reserves the full qualified context +window; it does not count each prompt locally. An over-context request can still +reach the provider and be rejected, entering the conservative uncertain/held +path. This availability limitation must not be described as exact pre-dispatch +prompt-token validation or proof that every failed request is unbilled. Setup's fixed `read_document` and `search_document` capability retains its own owner-side actual call/output counters. A model session opens only when those @@ -100,9 +106,12 @@ Hosted inference uses a dedicated digest-pinned `agents` backend image for the proxy even when the setup worker uses the mutually exclusive `omnicore` image. The proxy image contains the pinned OpenAI input-count/Responses dependencies and the OpenRouter HTTP adapter. Its entrypoint validates the root-owned SDK marker -before reading the two closed proxy-only provider secrets. Proxy enablement is -independent of setup runtime selection, so `manager_authored` setup can coexist -with model-backed POST checks and OpenRouter setup can coexist with OpenAI POST. +before reading only the configured proxy-only provider secrets. An OpenRouter-only +deployment needs no OpenAI key. Nonempty provider secret-file paths select the +corresponding hosted overlays; inference setup requires its selected provider, +while enabled `manager_authored` composition requires at least one provider for +POST. A missing credential for the locked POST profile fails before paid dispatch. +Proxy enablement is independent of setup runtime selection. See [`deploy/hosted/README.md`](../deploy/hosted/README.md). @@ -164,6 +173,11 @@ held until authenticated evidence can establish a safe terminal outcome. The exa Operator read provides visibility but cannot assert no effect, release money or create a replacement. +OpenRouter POST likewise cannot retrieve completion text from a response ID: +recovery performs no provider I/O. Retained, verified response ART can replay; +otherwise an ambiguous response keeps its reservation without another paid call. +OpenAI POST retains its existing stored-Responses retrieval contract. + For POST, budget exhaustion, provider outage, unavailable runtime and invalid provider output remain infrastructure failure on the original Submission. They do not prepare a revision, start a revision deadline or spend a revision round. diff --git a/docs/roadmap_status.md b/docs/roadmap_status.md index a76c400e3..7f16d0842 100644 --- a/docs/roadmap_status.md +++ b/docs/roadmap_status.md @@ -105,14 +105,17 @@ one finite lifetime project budget, private ART call evidence and recovery of the same paid effect across setup-worker renewal and checker retries. Existing OpenAI Agents SDK and OmniCoreAgent setup loops use the same account as approved POST image sessions while retaining distinct setup-attempt and checker-execution -lineage. Images keep their own rubrics and receive only an invocation-local -socket through the generic Rust service; hosted setup workers connect read-only +lineage. Model-backed POST selects OpenAI Responses or OpenRouter Chat Completions +from the locked profile; ordinary POST remains model-independent. Hosted +OpenRouter-only operation needs no OpenAI credential. Images keep their own +rubrics and receive only an invocation-local socket through the generic Rust service; hosted setup workers connect read-only to the trusted proxy and receive no provider key. This provides the model capability needed by the real Terminal Systems POST proposal; it does not establish that project's activation, model quality, current provider account, gVisor qualification or live contribution journey. See [model runtime operations](operations_checker_models.md) and its -[bounded change](../.commitrail/changes/checker-model-runtime.md). +[bounded change](../.commitrail/changes/checker-model-runtime.md) and +[POST provider selection](../.commitrail/changes/post-model-provider-selection.md). The public automated-acceptance journey passed its bounded connected local drill from public guide inputs with recorded inference, local Flow-compatible auth and @@ -561,7 +564,7 @@ cannot be reused as post-submission review-gate evidence. See the | Lifecycle stage | Intended merged status | What is already proven | What remains before v0.1 | | --- | --- | --- | --- | -| Shared setup and project-image model use | **Budgeted runtime and public Operator controls** | One project-lifetime funded account serves exact setup-attempt and approved POST image lineage; immutable bounded profiles; finite multi-call/run envelopes and setup document-tool caps; private ART request/response evidence; stable paid-effect replay/recovery; once-only settlement; exact redacted Operator effect inspection; hosted composition gives provider credentials only to the trusted proxy | Complete full CI and independent reviews; qualify the external account/current pricing and each project's substantive rubric under intended gVisor/AWS isolation; then perform only the approved aggregate-capped live rehearsal. Repository proof is controlled/offline and not deployment or model-quality evidence | +| Shared setup and project-image model use | **Budgeted runtime and public Operator controls** | One project-lifetime funded account serves exact setup-attempt and approved POST image lineage; immutable bounded profiles; finite multi-call/run envelopes and setup document-tool caps; private ART request/response evidence; stable paid-effect replay/recovery; once-only settlement; exact redacted Operator effect inspection; hosted composition gives only configured provider credentials to the trusted proxy; locked POST profiles select OpenAI Responses or OpenRouter Chat Completions, while ordinary POST needs neither | Qualify the external account/current pricing and each project's substantive rubric under intended gVisor/AWS isolation; then perform only the approved aggregate-capped live rehearsal. Repository proof is controlled/offline and not deployment or model-quality evidence | | Identity and actor resolution | **Live foundation** | Flow-token verification; canonical ActorProfile and ActorIdentityLink; human/service separation; lifecycle controls; canonical `/actors/me` self-read with duplicate `/auth/me` removed; suspension denies both self-read and self-update; CLI privileged exact-actor/profile and identity-link reads and explicit separate profile/link lifecycle writes remain distinct from self-service and backend guards | Final end-to-end operational and conformance proof | | Authorization kernel | **Live foundation** | Closed action/permission catalogues; deny-by-default evaluation; grants; exact-project, privacy-safe candidate read now consumed by the CLI; fixed services; rate controls; opaque transaction-bound PREP; atomic decision evidence; CLI scope-filtered administrative grant/catalogue reads and explicit keyed grant issue/revoke consume existing public APIs without local authority decisions | Activate only the remaining owner-proven TASK, checker, REV, and CON boundaries; remove obsolete authority after replacement paths are live | | System project catalogue | **Bounded public API and CLI read** | WS-CLI-001-46 adds `GET /api/v1/projects` and `workstream project list` for current system Project Manager, Operator, Finance or Audit `project.read` grants; each signed page exposes only ID, name and status. `me projects` retains exact-grant membership semantics. | System-catalogue TUI picker integration, deployed CLI and hosted conformance remain separate | diff --git a/scripts/test_hosted_release_config.py b/scripts/test_hosted_release_config.py index b39c073d4..4f1d3b1d8 100644 --- a/scripts/test_hosted_release_config.py +++ b/scripts/test_hosted_release_config.py @@ -222,10 +222,12 @@ def test_valid_configuration_and_rendered_topology(self) -> None: self.assertEqual(services["guide-document-parser"]["stop_grace_period"], "30s") self.assertEqual(services["api"]["ports"][0]["host_ip"], "127.0.0.1") expected_secrets = { - target[0] for target in preflight.SECRET_FILE_TARGETS.values() + preflight.SECRET_FILE_TARGETS[key][0] + for key in preflight._secret_file_keys(self.environment) } self.assertEqual(set(rendered["secrets"]), expected_secrets) - for key, (secret_name, _target) in preflight.SECRET_FILE_TARGETS.items(): + for key in preflight._secret_file_keys(self.environment): + secret_name, _target = preflight.SECRET_FILE_TARGETS[key] self.assertEqual( rendered["secrets"][secret_name]["file"], self.environment[key], @@ -1052,6 +1054,11 @@ def test_manager_setup_can_enable_post_proxy_with_both_provider_credentials( ), }.items(): manager_text = manager_text.replace(old, new) + manager_text = manager_text.replace( + "WORKSTREAM_MODEL_PROXY_OPENAI_API_KEY_FILE=\n", + "WORKSTREAM_MODEL_PROXY_OPENAI_API_KEY_FILE=" + "/etc/workstream/secrets/model_proxy_openai_api_key\n", + ) manager_path = self.root / "manager-with-post-proxy.env" manager_path.write_text(manager_text, encoding="utf-8") manager = preflight.parse_environment(manager_path) @@ -1073,7 +1080,7 @@ def test_manager_setup_can_enable_post_proxy_with_both_provider_credentials( for key in preflight.MODEL_PROXY_SECRET_FILE_KEYS: self.assertNotIn(key, worker_environment) - def test_openrouter_setup_keeps_independent_openai_post_credential(self) -> None: + def test_openrouter_only_render_omits_openai_credential(self) -> None: self.assertEqual( self.environment["WORKSTREAM_PROJECT_AGENT_MODEL_PROVIDER"], "openrouter" ) @@ -1087,14 +1094,68 @@ def test_openrouter_setup_keeps_independent_openai_post_credential(self) -> None for item in proxy["secrets"] if item["source"] != "database_url" }, - { - "model_proxy_openai_api_key", - "model_proxy_openrouter_api_key", - }, + {"model_proxy_openrouter_api_key"}, + ) + self.assertNotIn( + "WORKSTREAM_MODEL_PROXY_OPENAI_API_KEY_FILE", proxy["environment"] ) for key in preflight.MODEL_PROXY_SECRET_FILE_KEYS: self.assertNotIn(key, rendered["services"]["worker"]["environment"]) + def test_openai_only_render_omits_openrouter_credential(self) -> None: + environment_text = self.environment_path.read_text(encoding="utf-8") + environment_text = environment_text.replace( + "WORKSTREAM_PROJECT_AGENT_MODEL_PROVIDER=openrouter", + "WORKSTREAM_PROJECT_AGENT_MODEL_PROVIDER=openai", + ).replace( + "WORKSTREAM_PROJECT_AGENT_MODEL_API=chat_completions", + "WORKSTREAM_PROJECT_AGENT_MODEL_API=responses", + ) + environment_text = environment_text.replace( + "WORKSTREAM_MODEL_PROXY_OPENAI_API_KEY_FILE=\n", + "WORKSTREAM_MODEL_PROXY_OPENAI_API_KEY_FILE=" + "/etc/workstream/secrets/model_proxy_openai_api_key\n", + ).replace( + "WORKSTREAM_MODEL_PROXY_OPENROUTER_API_KEY_FILE=" + "/etc/workstream/secrets/model_proxy_openrouter_api_key", + "WORKSTREAM_MODEL_PROXY_OPENROUTER_API_KEY_FILE=", + ) + environment_path = self.root / "openai-only.env" + environment_path.write_text(environment_text, encoding="utf-8") + environment = preflight.parse_environment(environment_path) + + preflight.validate_environment(environment) + rendered = preflight.render_compose(environment_path, environment) + preflight.validate_compose(rendered, environment) + + proxy = rendered["services"]["model-proxy"] + self.assertEqual( + { + item["source"] + for item in proxy["secrets"] + if item["source"] != "database_url" + }, + {"model_proxy_openai_api_key"}, + ) + self.assertNotIn( + "WORKSTREAM_MODEL_PROXY_OPENROUTER_API_KEY_FILE", proxy["environment"] + ) + + def test_enabled_setup_rejects_missing_selected_provider_credential(self) -> None: + missing_openrouter = dict(self.environment) + missing_openrouter["WORKSTREAM_MODEL_PROXY_OPENROUTER_API_KEY_FILE"] = "" + with self.assertRaisesRegex(preflight.PreflightError, "selected provider"): + preflight.validate_environment(missing_openrouter) + + manager = missing_openrouter | { + "WORKSTREAM_PROJECT_AGENT_RUNTIME": "manager_authored", + "WORKSTREAM_PROJECT_AGENT_MODEL_PROVIDER": "none", + "WORKSTREAM_PROJECT_AGENT_MODEL_API": "none", + "WORKSTREAM_PROJECT_AGENT_MODEL": "none", + } + with self.assertRaisesRegex(preflight.PreflightError, "at least one"): + preflight.validate_environment(manager) + def test_checker_config_rejects_non_hosted_or_substituted_identity(self) -> None: cases = { "runc": ("runtime", "runc"), @@ -1760,11 +1821,7 @@ class ReleaseSecretLoaderTests(unittest.TestCase): "WORKSTREAM_DATABASE_URL_FILE", "WORKSTREAM_CELERY_BROKER_URL_FILE", ), - "model-proxy": ( - "WORKSTREAM_DATABASE_URL_FILE", - "WORKSTREAM_MODEL_PROXY_OPENAI_API_KEY_FILE", - "WORKSTREAM_MODEL_PROXY_OPENROUTER_API_KEY_FILE", - ), + "model-proxy": ("WORKSTREAM_DATABASE_URL_FILE",), "beat": ( "WORKSTREAM_DATABASE_URL_FILE", "WORKSTREAM_CELERY_BROKER_URL_FILE", @@ -1785,6 +1842,7 @@ def _run_loader( runtime: str = "openai_agents_sdk", sdk_environment: str = "agents", include_runtime_configuration: bool = True, + provider_files: tuple[str, ...] = ("openrouter",), ) -> subprocess.CompletedProcess[str]: environment = os.environ.copy() file_names = { @@ -1797,7 +1855,17 @@ def _run_loader( "model-proxy-openrouter" ), } + configured_provider_variables = { + f"WORKSTREAM_MODEL_PROXY_{name.upper()}_API_KEY_FILE" + for name in provider_files + } for variable, name in file_names.items(): + if ( + variable in preflight.MODEL_PROXY_SECRET_FILE_KEYS + and variable not in configured_provider_variables + ): + environment.pop(variable, None) + continue path = root / name if path.exists(): path.chmod(0o600) @@ -1844,7 +1912,10 @@ def _run_loader( "/run/private/private-key.pem" ) environment["EXPECT_PARSER_SELECTOR_SCRUB"] = "1" if role == "parser" else "0" - expected = {self.FILE_TO_SECRET[key] for key in self.ROLE_FILES.get(role, ())} + role_files = self.ROLE_FILES.get(role, ()) + if role == "model-proxy": + role_files = (*role_files, *sorted(configured_provider_variables)) + expected = {self.FILE_TO_SECRET[key] for key in role_files} environment["EXPECTED_SECRET_NAMES"] = ",".join(sorted(expected)) environment["EXPECTED_SECRET_SHA256"] = hashlib.sha256( content.encode() @@ -1922,26 +1993,47 @@ def test_loader_loads_only_role_secrets_and_rejects_missing_or_unknown_role( self.assertEqual(completed.returncode, 0, completed.stderr) self.assertEqual(completed.stdout, "loader-ok\n") self.assertEqual(completed.stderr, "") - if self.ROLE_FILES[role]: + required_files = self.ROLE_FILES[role] + if role == "model-proxy": + required_files = ( + *required_files, + "WORKSTREAM_MODEL_PROXY_OPENROUTER_API_KEY_FILE", + ) + if required_files: missing = self._run_loader( root, role=role, - unset_selector=self.ROLE_FILES[role][0], + unset_selector=required_files[0], ) self.assertNotEqual(missing.returncode, 0) self.assertIn( "required secret file is unavailable", missing.stderr ) - openai = self._run_loader(root, role="model-proxy", provider="openai") + openai = self._run_loader( + root, + role="model-proxy", + provider="openai", + provider_files=("openai",), + ) self.assertEqual(openai.returncode, 0, openai.stderr) - setup_provider_independent = self._run_loader( - root, role="model-proxy", provider="unknown" + both = self._run_loader( + root, + role="model-proxy", + provider="openrouter", + provider_files=("openai", "openrouter"), ) - self.assertEqual( - setup_provider_independent.returncode, - 0, - setup_provider_independent.stderr, + self.assertEqual(both.returncode, 0, both.stderr) + missing_selected = self._run_loader( + root, + role="model-proxy", + provider="openrouter", + provider_files=("openai",), ) + self.assertNotEqual(missing_selected.returncode, 0) + self.assertIn("selected provider credential", missing_selected.stderr) + unsupported = self._run_loader(root, role="model-proxy", provider="unknown") + self.assertNotEqual(unsupported.returncode, 0) + self.assertIn("provider is unsupported", unsupported.stderr) unknown = self._run_loader(root, role="unknown") self.assertNotEqual(unknown.returncode, 0) self.assertEqual(unknown.stdout, "") @@ -1971,6 +2063,18 @@ def test_loader_binds_runtime_to_sdk_marker_before_provider_secret(self) -> None sdk_environment="agents", ) self.assertEqual(manager_proxy.returncode, 0, manager_proxy.stderr) + manager_without_provider = self._run_loader( + root, + role="model-proxy", + runtime="manager_authored", + provider="none", + sdk_environment="agents", + provider_files=(), + ) + self.assertNotEqual(manager_without_provider.returncode, 0) + self.assertIn( + "at least one provider credential", manager_without_provider.stderr + ) for sdk_environment in ("agents", "omnicore"): with self.subTest( runtime="manager_authored", sdk_environment=sdk_environment