From a08b73db7217f89819a61473544a81aa228d3dd6 Mon Sep 17 00:00:00 2001 From: Eric Law <39393654+acn-ericlaw@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:59:14 -0700 Subject: [PATCH] feat(llm-helper): AWS Bedrock as the second backend - the SDK's Bedrock client, IAM credentials, the anthropic. model prefix, no server-side fallbacks llm.backend: 'bedrock' builds the SDK's AsyncAnthropicBedrockMantle over the AWS credential chain (or a Bedrock API key in AWS_BEARER_TOKEN_BEDROCK), with the region from llm.aws.region or AWS_REGION - without one the helper answers 503 AWS region missing instead of starting a call. A bare model id is sent with the anthropic. prefix, an id already spelled for Bedrock as written; server-side refusal fallbacks are never sent there; a call whose credential chain is empty is a 503 AWS credential missing by name; llm.health reports the region and resolves the chain the way a call would, without spending a token. The new bedrock extra adds boto3. The Backend seam carries it (backend_for), so the routes and the contract are unchanged; the shared vector file gains three Bedrock cases (66, byte-identical with the Node pack) and the client, region and credential checks are pinned by unit tests that run with no AWS environment at all. --- CHANGELOG.md | 15 +- examples/llm-helper/README.md | 63 +++++-- examples/llm-helper/llm_helper.py | 120 +++++++++--- examples/llm-helper/resources/application.yml | 9 +- pyproject.toml | 5 +- tests/test_llm_helper.py | 130 ++++++++++++- tests/vectors/llm-helper-vectors.json | 176 +++++++++++++++++- 7 files changed, 465 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 802ea3f..0e329d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,20 @@ ## Unreleased -Nothing yet. +### Added + +1. **AWS Bedrock as the LLM helper's second backend (`llm.backend: 'bedrock'`).** The helper reaches the same Claude models + through the SDK's Bedrock client, signed with the AWS credential chain through IAM (`AWS_ACCESS_KEY_ID` and + `AWS_SECRET_ACCESS_KEY`, a profile, a role) or a Bedrock API key in `AWS_BEARER_TOKEN_BEDROCK` - no vendor API key, the + platform-mediated access an enterprise security policy asks for. The new `bedrock` extra adds boto3 + (`pip install 'mercury-composable[llm,bedrock]'`); the region comes from `llm.aws.region` or `AWS_REGION`, and without one + the helper answers `503 AWS region missing` instead of starting a call. A bare model id is sent with the `anthropic.` + prefix and an id already spelled for Bedrock as written; server-side refusal fallbacks are never sent there (Bedrock offers + none); a call whose credential chain is empty is a `503 AWS credential missing` by name. `llm.health` reports `region` + beside `backend` and `model`, and its check resolves the credential chain the way a call would, without spending a token. + The routes and the contract are unchanged: the shared vector file gains three Bedrock cases (66), byte-identical with the + Node pack, and the Bedrock client, its region and credential checks are pinned by unit tests. **Upgrade note:** none - the + Claude API backend is the default and behaves as before; `llm.backend` now also accepts `bedrock`. ## Version 4.12.21, 10/7/2026 diff --git a/examples/llm-helper/README.md b/examples/llm-helper/README.md index b5c32de..7bf2f35 100644 --- a/examples/llm-helper/README.md +++ b/examples/llm-helper/README.md @@ -23,6 +23,15 @@ export ANTHROPIC_API_KEY=... # from the environment, never from a mercury-serve examples/llm-helper/llm_helper.py ``` +On AWS Bedrock instead (see Backends), the `bedrock` extra adds boto3, which signs each call with the +AWS credential chain, and a region is required: + +```bash +pip install 'mercury-composable[llm,bedrock]' +export AWS_REGION=us-east-1 # or llm.aws.region; the AWS credential chain signs each call +mercury-serve examples/llm-helper/llm_helper.py -Dllm.backend=bedrock +``` + The app listens on port 8086 (`rest.server.port` in `resources/application.yml`). Map its routes from the engine's `event-over-http.yaml`, as the Java and Rust MiniGraph Playgrounds do: @@ -45,7 +54,7 @@ token batches as they are produced. | `prompt` or `messages` | A single user turn, or conversation turns `[{role: user\|assistant, content}]`. `messages` wins when both are given. Content is plain text. | | `system` | Optional system prompt (text). | | `schema` | `llm.chat` only. A JSON schema; the reply is then structured output, returned parsed as `data`. `additionalProperties` defaults to `false`. A schema on `llm.stream` is a 400. | -| `params.model` | The model. Default `claude-opus-5-5` (`llm.model`). | +| `params.model` | The model. Default `claude-opus-5-5` (`llm.model`). On Bedrock a bare id is sent with the `anthropic.` prefix; an id already spelled for Bedrock is sent as written. | | `params.max_tokens` | Output cap. Default 16000 (`llm.max.tokens`). | | `params.timeout_ms` | `llm.chat`: a deadline for the whole call, SDK retries included. `llm.stream`: the idle allowance between events. Default 60000 (`llm.timeout.ms`). | | `params.effort` | `low`, `medium`, `high`, `xhigh` or `max`. Sent only when set; otherwise the model's own default applies. A model that takes no effort parameter answers 400. | @@ -93,7 +102,7 @@ the envelope into a graph's error context or an HTTP edge. | 408 | The deadline or the SDK's timeout | `LLM request timed out after {n} ms` | | 422 | The reply carries nothing usable | `LLM refused the request …`, `LLM reply is empty …`, `LLM reply is not valid JSON for the requested schema …` | | 429 | Rate limited | `LLM provider rate limit - 429 rate_limit_error: …` | -| 503 | No credential, or the API is unreachable | `LLM provider credential missing - set ANTHROPIC_API_KEY in the environment`, `LLM provider unreachable - …` | +| 503 | No credential, no region, or the API is unreachable | `LLM provider credential missing - set ANTHROPIC_API_KEY in the environment`, `AWS credential missing - …`, `AWS region missing - …`, `LLM provider unreachable - …` | **A reply that carries nothing usable is an error, never an empty success.** A refusal with no text, an empty reply, or a structured reply cut off before it is valid JSON is a 422 that says why @@ -110,20 +119,22 @@ the 422 above, not an empty stream. Raise `params.max_tokens` or lower `params.e | Key | Default | Meaning | |---|---|---| -| `llm.backend` | `anthropic` | The route to the models; see Backends. | +| `llm.backend` | `anthropic` | The route to the models: `anthropic` (the Claude API) or `bedrock` (AWS Bedrock); see Backends. | +| `llm.aws.region` | unset | The AWS region a Bedrock call is signed for. Unset, the SDK reads `AWS_REGION` / `AWS_DEFAULT_REGION`. | | `llm.model` | `claude-opus-5-5` | The default model. | | `llm.max.tokens` | `16000` | The default output cap. | | `llm.timeout.ms` | `60000` | The default deadline (chat) or idle allowance (stream). | | `llm.max.retries` | `2` | SDK retries for connection errors, 408, 409, 429 and 5xx. | -| `llm.fallbacks` | `default` | `default` or `off`. Server-side refusal fallbacks (`fallbacks: "default"`) on the models that support them (the Opus 5, Fable 5 and Sonnet 5.5 families). The reply's `model` names the model that answered. | +| `llm.fallbacks` | `default` | `default` or `off`. Server-side refusal fallbacks (`fallbacks: "default"`) on the models that support them (the Opus 5, Fable 5 and Sonnet 5.5 families). The reply's `model` names the model that answered. The Claude API only: Bedrock offers none, so the switch has no effect there. | | `llm.effort` | unset | The default effort. | | `llm.log.batches` | `false` | Log each streamed batch's number, size and arrival time (never its text). | | `llm.provider` | `anthropic` | Only `anthropic`; any other value is a 400. | The credential is never configured here. `ANTHROPIC_API_KEY` (or whatever the SDK resolves: an -auth token, or a profile) comes from the environment. `llm.health` reports whether a call could -be sent (a credential is present) with no network traffic, so a health probe never spends a token. -`/health` includes it through `mandatory.health.dependencies`. +auth token, or a profile) comes from the environment for the Claude API; the AWS credential chain, +or a Bedrock API key in `AWS_BEARER_TOKEN_BEDROCK`, for Bedrock. `llm.health` reports whether a call +could be sent (a credential is present) without reaching the model, so a health probe never spends a +token. `/health` includes it through `mandatory.health.dependencies`. ### The default model and the token budget @@ -148,15 +159,26 @@ annotations (`llm_model`, `llm_stop_reason`, `llm_input_tokens`, `llm_output_tok ## Backends `get_backend()` builds the one client the app uses, and a `Backend` says what that route to the -models supports. Today there is one, `anthropic` (the Claude API). - -**Planned: AWS Bedrock through IAM.** It is additive. Build the SDK's Bedrock client in -`get_backend()` (it authenticates with the AWS credentials of the default chain instead of an API -key, and takes a region), give the `Backend` a `provider_model` that prefixes the model id (`anthropic.claude-opus-5-5`), -report `supports_fallbacks=False` (server-side fallbacks are not offered on Bedrock), and describe a -missing AWS credential in `credential_problem()`. Select it with `llm.backend: 'bedrock'`. The routes, -the contract and the vector file stay as they are, because nothing outside `Backend` knows which -route answered. +models supports: how it spells a model id, whether server-side fallbacks exist, and what a missing +credential looks like. Nothing else in the module knows which route answered, so the routes, the +contract and the vector file are the same on every backend. + +| `llm.backend` | Client | Credential | Model id | Server-side fallbacks | +|---|---|---|---|---| +| `anthropic` (default) | the Claude API, `AsyncAnthropic` | `ANTHROPIC_API_KEY`, or whatever the SDK resolves | as written | on the models that support them | +| `bedrock` | AWS Bedrock, the SDK's `AsyncAnthropicBedrockMantle` | the AWS credential chain through IAM (`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`, a profile, a role), or a Bedrock API key in `AWS_BEARER_TOKEN_BEDROCK` | a bare id gets the `anthropic.` prefix; an id already spelled for Bedrock is sent as written | none - the request never carries them | + +**AWS Bedrock.** Install the extra (`pip install 'mercury-composable[llm,bedrock]'` - it adds boto3, +which signs each call), set `llm.backend: 'bedrock'` and a region (`llm.aws.region`, or `AWS_REGION` +in the environment; without one the helper answers `503 AWS region missing` instead of starting a +call), and give the process AWS credentials the way any AWS client gets them. The model must be +enabled for the account in that region. `llm.health` reports `region` beside `backend` and `model`, +and its health check resolves the credential chain the way a call would - on an EC2 or ECS host that +includes the instance metadata service - so a probe tells a missing credential from a present one +without spending a token; a call whose chain turns out empty is a `503 AWS credential missing`. +Effort, structured output and streaming work as on the Claude API. The SDK's client-side refusal +fallback middleware is not wired in; a refusal on Bedrock is the 422 or the `stop_details` of the +contract, like a refusal on the Claude API with `llm.fallbacks: off`. ## What it does not do @@ -167,10 +189,11 @@ deliberate: the helper is the bounded AI node of a graph, not an agent runtime. ## Tests `tests/test_llm_helper.py` runs the shared `tests/vectors/llm-helper-vectors.json` against a fake of -the SDK (63 cases: request validation, the exact SDK call, replies, the error contract, streaming), -plus the tests a vector cannot express (token batches are never held back, no prompt text in a -log, the deadline cancels the call, trace annotations, health, the backend seam). No token is -spent and no credential is needed. +the SDK (66 cases: request validation, the exact SDK call, replies, the error contract, streaming, +the shape of a Bedrock call), plus the tests a vector cannot express (token batches are never held +back, no prompt text in a log, the deadline cancels the call, trace annotations, health, the backend +seam, the Bedrock client with its region and credential checks). No token is spent and no credential +is needed. ```bash pip install -e '.[dev]' diff --git a/examples/llm-helper/llm_helper.py b/examples/llm-helper/llm_helper.py index de33708..d7dee99 100644 --- a/examples/llm-helper/llm_helper.py +++ b/examples/llm-helper/llm_helper.py @@ -8,12 +8,13 @@ Run: pip install 'mercury-composable[llm]' mercury-serve examples/llm-helper/llm_helper.py -The credential comes from the environment (ANTHROPIC_API_KEY), never from a config file. -Settings live in resources/application.yml (or -Dkey=value): llm.backend, llm.model, -llm.max.tokens, llm.timeout.ms, llm.max.retries, llm.fallbacks, llm.effort. The contract, -the keys and the backend seam are documented in README.md next to this file. The Node.js -twin (mercury-nodejs examples/llm-helper) speaks the same contract, and both are pinned by -one shared vector file. +The credential comes from the environment, never from a config file: ANTHROPIC_API_KEY for +the Claude API, the AWS credential chain (or AWS_BEARER_TOKEN_BEDROCK) for AWS Bedrock. +Settings live in resources/application.yml (or -Dkey=value): llm.backend (anthropic or +bedrock), llm.aws.region, llm.model, llm.max.tokens, llm.timeout.ms, llm.max.retries, +llm.fallbacks, llm.effort. The contract, the keys and the backends are documented in +README.md next to this file. The Node.js twin (mercury-nodejs examples/llm-helper) speaks +the same contract, and both are pinned by one shared vector file. Then map the routes from a Mercury engine application (event-over-http.yaml): @@ -28,6 +29,7 @@ from __future__ import annotations import asyncio +import importlib.util import json import time from collections.abc import Callable @@ -60,12 +62,15 @@ DEFAULT_PROVIDER = "anthropic" DEFAULT_BACKEND = "anthropic" +BEDROCK_BACKEND = "bedrock" DEFAULT_MODEL = "claude-opus-5-5" DEFAULT_MAX_TOKENS = 16000 DEFAULT_TIMEOUT_MS = 60000 DEFAULT_MAX_RETRIES = 2 PROVIDERS = (DEFAULT_PROVIDER,) -BACKENDS = (DEFAULT_BACKEND,) +BACKENDS = (DEFAULT_BACKEND, BEDROCK_BACKEND) +# Bedrock spells a model id with the vendor prefix; a bare id gets it, a prefixed one is kept +BEDROCK_MODEL_PREFIX = "anthropic." EFFORTS = ("low", "medium", "high", "xhigh", "max") ROLES = ("user", "assistant") # per-call params. Anything else is rejected: the current models take no sampling @@ -76,8 +81,16 @@ FALLBACKS_BETA = "server-side-fallback-2026-07-01" FALLBACK_MODEL_PREFIXES = ("claude-opus-5", "claude-fable-5", "claude-sonnet-5-5") MISSING_CREDENTIAL = "LLM provider credential missing - set ANTHROPIC_API_KEY in the environment" +MISSING_AWS_CREDENTIAL = ( + "AWS credential missing - configure the AWS credential chain (AWS_ACCESS_KEY_ID and " + "AWS_SECRET_ACCESS_KEY, a profile, or a role) or set AWS_BEARER_TOKEN_BEDROCK" +) +MISSING_AWS_REGION = "AWS region missing - set llm.aws.region or AWS_REGION in the environment" +MISSING_BEDROCK_SDK = "The Bedrock backend needs boto3 - pip install 'mercury-composable[bedrock]'" # the SDK raises a bare TypeError, before sending anything, when no credential resolves CREDENTIAL_TYPE_ERROR = "Could not resolve authentication method" +# the SDK's Bedrock client raises a bare RuntimeError, while signing, when the AWS chain is empty +AWS_CREDENTIAL_RUNTIME_ERROR = "could not resolve credentials from session" def _prop(key: str, default: str | None = None) -> str | None: @@ -251,32 +264,88 @@ def _same_model(model: str) -> str: return model +def _bedrock_model(model: str) -> str: + """A bare model id gets the prefix Bedrock expects (``anthropic.claude-opus-5-5``); an id + already spelled for Bedrock - prefixed, or an inference profile - is sent as written.""" + return BEDROCK_MODEL_PREFIX + model if model.startswith("claude-") else model + + +def _anthropic_credentials(client: Any) -> str | None: + """The Claude API client: a key, a token, an OAuth profile or a custom auth, all local.""" + names = ("api_key", "auth_token", "credentials", "custom_auth") + if any(getattr(client, name, None) for name in names): + return None + return MISSING_CREDENTIAL + + +def _bedrock_credentials(client: Any) -> str | None: + """The Bedrock client: a Bedrock API key, else what the AWS credential chain resolves - the + environment, a profile, a role; on an EC2 or ECS host that includes the instance metadata + service, the one lookup the chain makes outside the process. No token is spent.""" + if getattr(client, "api_key", None): + return None + import boto3 # the [bedrock] extra; get_backend() checked that it is installed + + session = boto3.Session( + profile_name=getattr(client, "aws_profile", None), + region_name=getattr(client, "aws_region", None), + aws_access_key_id=getattr(client, "aws_access_key", None), + aws_secret_access_key=getattr(client, "aws_secret_key", None), + aws_session_token=getattr(client, "aws_session_token", None), + ) + return None if session.get_credentials() else MISSING_AWS_CREDENTIAL + + @dataclass(frozen=True) class Backend: - """One way to reach Claude: the client, and what this route to the model supports. - - This is the seam for a second route to the same models - AWS Bedrock through IAM is the - planned one (see README.md, "Backends"): build its client in get_backend(), give it a - provider_model that prefixes the model id, report no server-side fallbacks, and describe - a missing credential. Nothing else in this module knows which route answered. + """One way to reach Claude: the client, and what this route to the model supports - how it + spells a model id, whether server-side fallbacks exist, and what a missing credential looks + like. Nothing else in this module knows which route answered, so the routes, the contract + and the vector file are the same on every backend (see README.md, "Backends"). """ name: str client: Any supports_fallbacks: bool provider_model: Callable[[str], str] = _same_model + credentials: Callable[[Any], str | None] = _anthropic_credentials def credential_problem(self) -> str | None: - """What is missing for a call to be sent, or None. No network traffic.""" - names = ("api_key", "auth_token", "credentials", "custom_auth") - if any(getattr(self.client, name, None) for name in names): - return None - return MISSING_CREDENTIAL + """What is missing for a call to be sent, or None. No token is spent.""" + return self.credentials(self.client) + + +def backend_for(name: str, client: Any) -> Backend: + """The Backend of a route over a client: get_backend() builds the client, tests inject one.""" + if name == BEDROCK_BACKEND: + return Backend( + BEDROCK_BACKEND, + client, + supports_fallbacks=False, # server-side refusal fallbacks are not offered on Bedrock + provider_model=_bedrock_model, + credentials=_bedrock_credentials, + ) + return Backend(DEFAULT_BACKEND, client, supports_fallbacks=True) _backend: Backend | None = None # built on first use - module-level so tests can inject +def _bedrock_client() -> Any: + """AWS Bedrock through the SDK's Mantle client: the AWS credential chain (or a Bedrock API + key) signs each call, and the region comes from llm.aws.region or the AWS environment.""" + if importlib.util.find_spec("boto3") is None: + raise AppException(503, MISSING_BEDROCK_SDK) + # the SDK reaches its Bedrock client lazily at the top level; the package path is the typed one + from anthropic.lib.bedrock import AsyncAnthropicBedrockMantle + + region = _prop("llm.aws.region") + try: + return AsyncAnthropicBedrockMantle(aws_region=str(region) if region else None) + except anthropic.AnthropicError as exc: # no region, and no base URL to stand in for one + raise AppException(503, MISSING_AWS_REGION) from exc + + def get_backend() -> Backend: global _backend backend = _backend @@ -287,7 +356,8 @@ def get_backend() -> Backend: 501, f"unknown LLM backend '{name}' - this helper serves: {', '.join(BACKENDS)}" ) # credentials resolve per call, so the app starts (and reports its health) without one - backend = Backend(name, anthropic.AsyncAnthropic(), supports_fallbacks=True) + client = _bedrock_client() if name == BEDROCK_BACKEND else anthropic.AsyncAnthropic() + backend = backend_for(name, client) _backend = backend return backend @@ -457,6 +527,10 @@ def _failure(exc: BaseException, timeout_ms: int) -> AppException | None: return AppException(503, f"LLM provider unreachable - {exc}") if isinstance(exc, TypeError) and CREDENTIAL_TYPE_ERROR in str(exc): return AppException(503, MISSING_CREDENTIAL) + if (isinstance(exc, RuntimeError) and AWS_CREDENTIAL_RUNTIME_ERROR in str(exc)) or type( + exc + ).__name__ == "NoCredentialsError": + return AppException(503, MISSING_AWS_CREDENTIAL) return None @@ -606,16 +680,20 @@ async def llm_health(headers: dict[str, str], _body: Body) -> Any: Activated for the /health actuator endpoint by mandatory.health.dependencies in resources/application.yml. It reports whether a call could be sent (a credential is - present) without any network traffic, so a probe never spends a token. + present) without reaching the model, so a probe never spends a token. """ backend = get_backend() if headers.get("type") == "info": - return { + info: dict[str, Any] = { "service": "llm.helper", "href": "http://127.0.0.1", "backend": backend.name, "model": _prop("llm.model", DEFAULT_MODEL), } + region = getattr(backend.client, "aws_region", None) + if region: + info["region"] = region # which region a Bedrock call is signed for + return info problem = backend.credential_problem() if problem: raise AppException(503, problem) diff --git a/examples/llm-helper/resources/application.yml b/examples/llm-helper/resources/application.yml index 94cfe03..f4ffbd4 100644 --- a/examples/llm-helper/resources/application.yml +++ b/examples/llm-helper/resources/application.yml @@ -23,14 +23,19 @@ show.env.variables: 'LOG_LEVEL' show.application.properties: 'application.name, rest.server.port, llm.backend, llm.model' # --- the helper's settings (a call's params.* win over these) ----------------------------- -# The credential is never configured here: ANTHROPIC_API_KEY comes from the environment. +# The credential is never configured here: ANTHROPIC_API_KEY (the Claude API) or the AWS +# credential chain (AWS Bedrock) comes from the environment. +# anthropic (the Claude API, default) | bedrock (AWS Bedrock through IAM - the [bedrock] extra) llm.backend: 'anthropic' +# the region a Bedrock call is signed for; unset, the SDK reads AWS_REGION / AWS_DEFAULT_REGION +# llm.aws.region: 'us-east-1' llm.model: 'claude-opus-5-5' llm.max.tokens: 16000 # llm.chat: the whole call, SDK retries included. llm.stream: the idle allowance between events. llm.timeout.ms: 60000 llm.max.retries: 2 -# server-side refusal fallbacks on the models that support them: default | off +# server-side refusal fallbacks on the models that support them: default | off (the Claude +# API only - Bedrock offers none, so the switch has no effect there) llm.fallbacks: 'default' # low | medium | high | xhigh | max - left unset, the model's own default applies (a model # that takes no effort parameter rejects one, so set this only for a model that does) diff --git a/pyproject.toml b/pyproject.toml index eb72c0b..74f170d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,10 @@ Changelog = "https://github.com/Accenture/mercury-python/blob/main/CHANGELOG.md" [project.optional-dependencies] # the LLM helper app (examples/llm-helper) - the package itself stays SDK-free llm = ["anthropic>=1,<2"] -dev = ["pytest>=8", "pytest-asyncio>=0.23", "ruff>=0.9", "anthropic>=1,<2"] +# AWS Bedrock as the helper's backend (llm.backend: bedrock) - boto3 signs each call with the +# AWS credential chain; the Claude API install stays lean without it +bedrock = ["anthropic[bedrock]>=1,<2"] +dev = ["pytest>=8", "pytest-asyncio>=0.23", "ruff>=0.9", "anthropic[bedrock]>=1,<2"] [project.scripts] mercury-serve = "mercury_composable.cli:main" diff --git a/tests/test_llm_helper.py b/tests/test_llm_helper.py index 4b69982..2559ec4 100644 --- a/tests/test_llm_helper.py +++ b/tests/test_llm_helper.py @@ -24,6 +24,7 @@ import anthropic import httpx2 import pytest +from anthropic.lib.bedrock import AsyncAnthropicBedrockMantle from mercury_composable import AppException @@ -31,7 +32,7 @@ _HELPER = _TESTS.parent / "examples" / "llm-helper" / "llm_helper.py" _VECTORS = _TESTS / "vectors" / "llm-helper-vectors.json" # the Node.js twin pins the same digest: change the file in both packs or in neither -_VECTORS_SHA256 = "1f4823d9259ed6ff26e96d044b5f6b3c7bc9b342cc8ad76047434c2d3a2b8fa0" +_VECTORS_SHA256 = "1b758e73605b2b088594faa8e573483778d7b85f29073d4779c3bba20ac024c1" _spec = importlib.util.spec_from_file_location("llm_helper_under_test", _HELPER) assert _spec is not None @@ -198,9 +199,9 @@ def _install_backend( ) -> None: settings = dict(config or {}) monkeypatch.setattr(helper, "_prop", lambda key, default=None: settings.get(key, default)) - monkeypatch.setattr( - helper, "_backend", helper.Backend("anthropic", client, supports_fallbacks=True) - ) + # the fake client under the shape of the backend the case names (the Claude API by default) + backend = helper.backend_for(settings.get("llm.backend", helper.DEFAULT_BACKEND), client) + monkeypatch.setattr(helper, "_backend", backend) def _install_writer(monkeypatch: pytest.MonkeyPatch) -> _FakeWriter: @@ -512,13 +513,13 @@ def test_an_unknown_backend_is_a_501_that_names_what_is_served( ) -> None: monkeypatch.setattr(helper, "_backend", None) monkeypatch.setattr( - helper, "_prop", lambda key, default=None: "bedrock" if key == "llm.backend" else default + helper, "_prop", lambda key, default=None: "vertex" if key == "llm.backend" else default ) with pytest.raises(AppException) as raised: helper.get_backend() assert raised.value.status == 501 - assert "unknown LLM backend 'bedrock'" in raised.value.message - assert "this helper serves: anthropic" in raised.value.message + assert "unknown LLM backend 'vertex'" in raised.value.message + assert "this helper serves: anthropic, bedrock" in raised.value.message def test_the_default_backend_is_the_anthropic_client(monkeypatch: pytest.MonkeyPatch) -> None: @@ -529,3 +530,118 @@ def test_the_default_backend_is_the_anthropic_client(monkeypatch: pytest.MonkeyP assert backend.supports_fallbacks is True assert isinstance(backend.client, anthropic.AsyncAnthropic) assert helper.get_backend() is backend, "the client is built once" + + +# --- the Bedrock backend: the SDK's Mantle client, built without a credential or a token ---- + +_AWS_ENVIRONMENT = ( + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_PROFILE", + "AWS_BEARER_TOKEN_BEDROCK", + "ANTHROPIC_AWS_API_KEY", + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", + "AWS_CONTAINER_CREDENTIALS_FULL_URI", + "AWS_WEB_IDENTITY_TOKEN_FILE", + "AWS_REGION", + "AWS_DEFAULT_REGION", + "ANTHROPIC_BEDROCK_MANTLE_BASE_URL", +) + + +def _bedrock_settings(monkeypatch: pytest.MonkeyPatch, **settings: str) -> None: + """llm.backend: bedrock in a process with no AWS environment at all: the shared credential + and config files pointed nowhere, the instance metadata lookup switched off, so every + check below stays local and deterministic.""" + config = {"llm.backend": "bedrock", **settings} + monkeypatch.setattr(helper, "_backend", None) + monkeypatch.setattr(helper, "_prop", lambda key, default=None: config.get(key, default)) + for name in _AWS_ENVIRONMENT: + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("AWS_SHARED_CREDENTIALS_FILE", "/nonexistent/aws/credentials") + monkeypatch.setenv("AWS_CONFIG_FILE", "/nonexistent/aws/config") + monkeypatch.setenv("AWS_EC2_METADATA_DISABLED", "true") + + +def test_the_bedrock_backend_is_the_sdk_mantle_client(monkeypatch: pytest.MonkeyPatch) -> None: + """The SDK's Bedrock client over the region from llm.aws.region, the model prefix for a bare + id (an id already spelled for Bedrock is kept), no server-side fallbacks; built once.""" + _bedrock_settings(monkeypatch, **{"llm.aws.region": "us-east-1"}) + backend = helper.get_backend() + assert backend.name == "bedrock" + assert isinstance(backend.client, AsyncAnthropicBedrockMantle) + assert backend.client.aws_region == "us-east-1" + assert backend.supports_fallbacks is False + assert backend.provider_model("claude-opus-5-5") == "anthropic.claude-opus-5-5" + assert backend.provider_model("us.anthropic.claude-opus-5-5") == "us.anthropic.claude-opus-5-5" + assert helper.get_backend() is backend + + +def test_the_bedrock_backend_reads_the_region_from_the_aws_environment( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _bedrock_settings(monkeypatch) + monkeypatch.setenv("AWS_REGION", "eu-west-1") + assert helper.get_backend().client.aws_region == "eu-west-1" + + +def test_the_bedrock_backend_without_a_region_is_a_503_that_names_the_setting( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _bedrock_settings(monkeypatch) + with pytest.raises(AppException) as raised: + helper.get_backend() + assert raised.value.status == 503 + assert "AWS region missing" in raised.value.message + assert "llm.aws.region" in raised.value.message + + +def test_the_bedrock_backend_without_boto3_is_a_503_that_names_the_extra( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _bedrock_settings(monkeypatch, **{"llm.aws.region": "us-east-1"}) + monkeypatch.setattr(helper.importlib.util, "find_spec", lambda _name: None) + with pytest.raises(AppException) as raised: + helper.get_backend() + assert raised.value.status == 503 + assert "mercury-composable[bedrock]" in raised.value.message + + +async def test_bedrock_health_reports_the_region_and_resolves_the_aws_chain( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The health route names the region a call is signed for, and its check resolves the AWS + credential chain the way a call would: nothing in the chain is a 503 by name, the AWS + environment is a credential, and so is a Bedrock API key.""" + _bedrock_settings(monkeypatch, **{"llm.aws.region": "us-east-1"}) + info = await helper.llm_health({"type": "info"}, None) + assert info["backend"] == "bedrock" + assert info["region"] == "us-east-1" + with pytest.raises(AppException) as raised: + await helper.llm_health({"type": "health"}, None) + assert raised.value.status == 503 + assert "AWS credential missing" in raised.value.message + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIATESTONLY") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-only-secret") + assert await helper.llm_health({"type": "health"}, None) == "llm.helper is running fine" + monkeypatch.delenv("AWS_ACCESS_KEY_ID") + monkeypatch.delenv("AWS_SECRET_ACCESS_KEY") + monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "test-only-token") + monkeypatch.setattr(helper, "_backend", None) # the key is read when the client is built + assert await helper.llm_health({"type": "health"}, None) == "llm.helper is running fine" + + +def test_a_missing_aws_credential_at_call_time_is_a_503_by_name() -> None: + """The SDK signs each Bedrock call with the AWS chain and raises a bare RuntimeError when + the chain is empty (botocore its NoCredentialsError); both map to the portable 503.""" + from botocore.exceptions import NoCredentialsError + + failure = helper._failure(RuntimeError("could not resolve credentials from session"), 25) + assert failure is not None + assert failure.status == 503 + assert "AWS credential missing" in failure.message + failure = helper._failure(NoCredentialsError(), 25) + assert failure is not None + assert failure.status == 503 + assert helper._failure(RuntimeError("something else entirely"), 25) is None diff --git a/tests/vectors/llm-helper-vectors.json b/tests/vectors/llm-helper-vectors.json index e409679..49bfba6 100644 --- a/tests/vectors/llm-helper-vectors.json +++ b/tests/vectors/llm-helper-vectors.json @@ -1,7 +1,7 @@ { "contract": "llm-helper", "version": 1, - "description": "One contract, two function hosts. mercury-python and mercury-nodejs ship this file byte-identical and run every case against their own fake of the Anthropic SDK, so the two helpers cannot drift apart. A case names the route, the config keys in force, the request body and what the fake provider does; the expectation is the exact SDK call (namespace, parameters, options), then either the reply map or the error (status and message fragments) for llm.chat, or the head, the segments in order, the terminal metadata (a subset) or the in-band error for llm.stream.", + "description": "One contract, two function hosts. mercury-python and mercury-nodejs ship this file byte-identical and run every case against their own fake of the Anthropic SDK, so the two helpers cannot drift apart. A case's config may name llm.backend, and the harness then installs its fake under that backend's shape (bedrock: the anthropic. model prefix for a bare id, no server-side fallbacks, the messages namespace). A case names the route, the config keys in force, the request body and what the fake provider does; the expectation is the exact SDK call (namespace, parameters, options), then either the reply map or the error (status and message fragments) for llm.chat, or the head, the segments in order, the terminal metadata (a subset) or the in-band error for llm.stream.", "provider_kinds": { "message": "succeeds: deltas are the token batches; text is the final message text (default: the deltas joined); usage, model, stop_reason, request_id (the request-id response header), stop_details", "status": "an HTTP error from the API: status, type, message, request_id; after = the number of deltas delivered before it is raised (0: at open), deltas = those batches", @@ -2181,6 +2181,180 @@ ] } } + }, + { + "name": "chat/bedrock-prefixes-a-bare-model-and-sends-no-fallbacks", + "route": "llm.chat", + "config": { + "llm.backend": "bedrock" + }, + "body": { + "prompt": "x" + }, + "provider": { + "kind": "message", + "model": "anthropic.claude-opus-5-5", + "stop_reason": "end_turn", + "deltas": [ + "ok" + ], + "usage": { + "input_tokens": 5, + "output_tokens": 1 + }, + "request_id": "req_ok_001" + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "anthropic.claude-opus-5-5", + "max_tokens": 16000, + "messages": [ + { + "role": "user", + "content": "x" + } + ] + }, + "options": { + "timeout_ms": 60000, + "max_retries": 2 + } + }, + "result": { + "text": "ok", + "model": "anthropic.claude-opus-5-5", + "stop_reason": "end_turn", + "usage": { + "input_tokens": 5, + "output_tokens": 1 + }, + "request_id": "req_ok_001" + } + } + }, + { + "name": "chat/bedrock-keeps-a-model-already-spelled-for-bedrock", + "route": "llm.chat", + "config": { + "llm.backend": "bedrock" + }, + "body": { + "prompt": "x", + "params": { + "model": "us.anthropic.claude-opus-5-5" + } + }, + "provider": { + "kind": "message", + "model": "us.anthropic.claude-opus-5-5", + "stop_reason": "end_turn", + "deltas": [ + "ok" + ], + "usage": { + "input_tokens": 5, + "output_tokens": 1 + }, + "request_id": "req_ok_001" + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "us.anthropic.claude-opus-5-5", + "max_tokens": 16000, + "messages": [ + { + "role": "user", + "content": "x" + } + ] + }, + "options": { + "timeout_ms": 60000, + "max_retries": 2 + } + }, + "result": { + "text": "ok", + "model": "us.anthropic.claude-opus-5-5", + "stop_reason": "end_turn", + "usage": { + "input_tokens": 5, + "output_tokens": 1 + }, + "request_id": "req_ok_001" + } + } + }, + { + "name": "stream/bedrock-batches-forwarded-without-fallbacks", + "route": "llm.stream", + "config": { + "llm.backend": "bedrock" + }, + "headers": { + "my_correlation_id": "biz-777" + }, + "body": { + "prompt": "greet me" + }, + "provider": { + "kind": "message", + "model": "anthropic.claude-opus-5-5", + "stop_reason": "end_turn", + "deltas": [ + "Hello ", + "wor", + "ld" + ], + "usage": { + "input_tokens": 12, + "output_tokens": 34 + }, + "request_id": "req_ok_001" + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "anthropic.claude-opus-5-5", + "max_tokens": 16000, + "messages": [ + { + "role": "user", + "content": "greet me" + } + ] + }, + "options": { + "timeout_ms": 60000, + "max_retries": 2 + } + }, + "head": [ + 200, + "text/event-stream" + ], + "frames": [ + "Hello ", + "wor", + "ld" + ], + "terminal": { + "model": "anthropic.claude-opus-5-5", + "stop_reason": "end_turn", + "usage": { + "input_tokens": 12, + "output_tokens": 34 + }, + "request_id": "req_ok_001", + "my_correlation_id": "biz-777", + "trace_id": null + } + } } ] }