-
Notifications
You must be signed in to change notification settings - Fork 22
feat(models): openai-compatible vendor presets (zai, deepseek, moonshot, groq, xai, openrouter) #17
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
b8ce0de
219db2f
bdba0d9
c681310
4863fcd
0a9d92e
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -16,6 +16,7 @@ | |
| from langchain_core.language_models import BaseChatModel | ||
|
|
||
| from agent_engine.logging_config import log | ||
| from agent_engine.models.presets import OPENAI_COMPAT_PRESETS | ||
|
|
||
| logger = logging.getLogger(__name__) | ||
|
|
||
|
|
@@ -34,6 +35,8 @@ def build_chat_model( | |
| region: str | None = None, | ||
| max_tokens: int | None = None, | ||
| top_p: float | None = None, | ||
| base_url: str | None = None, | ||
| api_key_env: str | None = None, | ||
| ) -> BaseChatModel: | ||
| """Construct a chat model from flat config fields. | ||
|
|
||
|
|
@@ -44,11 +47,25 @@ def build_chat_model( | |
| if not model_name: | ||
| raise ModelConfigurationError("Model name must not be empty.") | ||
|
|
||
| # A preset id (zai, deepseek, moonshot, groq, xai, openrouter, ...) is | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. what about checking if provider is gemini, but user declare base url or api key, or just one of them? we should error him on validate yaml that its not acceptable |
||
| # shorthand for provider: openai with that vendor's base_url/api_key_env | ||
| # filled in. YAML-supplied base_url/api_key_env still win if both are | ||
| # set explicitly alongside a preset id, so a preset can be overridden | ||
| # (a proxy in front of it, a different key variable name) without | ||
| # switching away from the shorthand. | ||
| preset_id = normalized_provider | ||
| preset = OPENAI_COMPAT_PRESETS.get(preset_id) | ||
| if preset is not None: | ||
| base_url = base_url or preset.base_url | ||
| api_key_env = api_key_env or preset.api_key_env | ||
| normalized_provider = "openai" | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. if we remove preset always provider will be openai
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yes, right, drop the registry and this branch goes with it, proider is just openai plus whatever base_url/key the user passed. they live or die together |
||
|
|
||
| log( | ||
| logger, | ||
| logging.INFO, | ||
| "llm configured", | ||
| provider=normalized_provider, | ||
| provider_preset=preset_id if preset is not None else None, | ||
| model=model_name, | ||
| temperature=temperature, | ||
| region=region, | ||
|
|
@@ -81,8 +98,10 @@ def build_chat_model( | |
| temperature=temperature, | ||
| max_tokens=max_tokens, | ||
| top_p=top_p, | ||
| base_url=base_url, | ||
| api_key_env=api_key_env, | ||
| ) | ||
| supported = ", ".join(_SUPPORTED_PROVIDERS) | ||
| supported = ", ".join((*_SUPPORTED_PROVIDERS, *OPENAI_COMPAT_PRESETS)) | ||
| raise ModelConfigurationError( | ||
| f"Unsupported model provider '{provider}'. Supported providers: {supported}." | ||
| ) | ||
|
|
@@ -205,11 +224,19 @@ def _build_openai_model( | |
| temperature: float | None, | ||
| max_tokens: int | None, | ||
| top_p: float | None, | ||
| base_url: str | None = None, | ||
| api_key_env: str | None = None, | ||
| ) -> BaseChatModel: | ||
| api_key = _resolve_openai_api_key() | ||
| # `api_key_env` lets this provider point at ANY OpenAI-compatible chat | ||
| # completions endpoint (Z.AI, DeepSeek, Moonshot, Groq, xAI, OpenRouter, | ||
| # a local Ollama/vLLM server, ...) by naming the env var that holds that | ||
| # vendor's key. Defaults to OPENAI_API_KEY to keep existing YAML working | ||
| # unchanged. | ||
| key_env_var = api_key_env.strip() if api_key_env and api_key_env.strip() else "OPENAI_API_KEY" | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If api_key_env is "" we not should error so user meant to override and not use OPEN_API_KEY?
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. empty string should error, yeah. the OPENAI_API_KEY default is meant for when the field is omitted, not when someone explicitly blanks it. An explicit "" means they were mid-override and got it wrong, and falling back silently just turns that into a confusing auth failure further down. I'll split the two: omitted stays defaulted, empty or whitespace raises. what u think?
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I still wondering if user will declare on yaml api_key_env = " " |
||
| api_key = _resolve_openai_api_key(key_env_var) | ||
| if not api_key: | ||
| raise ModelConfigurationError( | ||
| "OpenAI provider requires OPENAI_API_KEY. " | ||
| f"OpenAI-compatible provider requires {key_env_var}. " | ||
| "Set it in your environment before running agentctl." | ||
| ) | ||
|
|
||
|
|
@@ -223,14 +250,17 @@ def _build_openai_model( | |
| ) from exc | ||
|
|
||
| # OpenAI's chat model uses the same `max_tokens` name as extra's config, so | ||
| # no remapping is needed. The API key is passed explicitly so OPENAI_API_KEY | ||
| # is honored regardless of how the installed SDK resolves credentials. | ||
| # no remapping is needed. The API key is passed explicitly so the resolved | ||
| # env var is honored regardless of how the installed SDK resolves | ||
| # credentials. `base_url` repoints the client at any OpenAI-compatible | ||
| # endpoint; when unset, ChatOpenAI's own default (api.openai.com) applies. | ||
| try: | ||
| return ChatOpenAI( | ||
| **_without_none( | ||
| { | ||
| "model": name, | ||
| "api_key": api_key, | ||
| "base_url": base_url, | ||
| "temperature": temperature, | ||
| "max_tokens": max_tokens, | ||
| "top_p": top_p, | ||
|
|
@@ -239,8 +269,8 @@ def _build_openai_model( | |
| ) | ||
| except Exception as exc: | ||
| raise ModelConfigurationError( | ||
| "Could not initialize OpenAI chat model. Verify OPENAI_API_KEY is valid " | ||
| "and the model name is an OpenAI model your key can access." | ||
| f"Could not initialize OpenAI-compatible chat model. Verify {key_env_var} is " | ||
| "valid and the model name is one your key/endpoint can access." | ||
| ) from exc | ||
|
|
||
|
|
||
|
|
@@ -257,8 +287,8 @@ def _resolve_gemini_api_key() -> str | None: | |
| return key.strip() if key and key.strip() else None | ||
|
|
||
|
|
||
| def _resolve_openai_api_key() -> str | None: | ||
| key = os.getenv("OPENAI_API_KEY") | ||
| def _resolve_openai_api_key(env_var: str = "OPENAI_API_KEY") -> str | None: | ||
| key = os.getenv(env_var) | ||
| return key.strip() if key and key.strip() else None | ||
|
|
||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| """Known OpenAI-compatible vendor presets. | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'm not sure this file is needed. what you have built, its generic infra for custom models.
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. the maintenance worry is fair but smaller than it looks I think these base URLs don't move, and the table is six entries. The generic base_url/api_key_env path already covers everything the registry doesn't, including overriding a preset, so the registry adds no capability, it just saves every user re-typing the same two known values for a vendor the library could already name, or at least that was my initial tought. But if the team would rather ship zero vendor knowledge though, killing presets.py is one commit since the generic path stands on its own, lmk
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @Asaf-prog wdyt? we need more opinion
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Secondly i feel good with that. But we have to doc it also on official docs mdx files. focused and clear |
||
|
|
||
| Each preset supplies the ``base_url`` and ``api_key_env`` for a vendor whose | ||
| API speaks the OpenAI chat completions protocol, so ``provider: <preset id>`` | ||
| in YAML resolves those two fields automatically instead of requiring them to | ||
| be typed out by hand every time. A YAML ``model`` block can still override | ||
| ``base_url`` and/or ``api_key_env`` for a listed vendor (a self-hosted proxy | ||
| in front of it, a non-default key variable name, ...), and ``provider: | ||
| openai`` with explicit ``base_url``/``api_key_env`` remains the escape hatch | ||
| for any vendor or self-hosted server not listed here. | ||
|
|
||
| Presets deliberately do not carry a default model id: no provider in this | ||
| factory picks a model on the caller's behalf, and vendor model catalogs | ||
| change independently of this codebase, so ``name`` stays required in YAML | ||
| exactly as it is for every other provider. | ||
| """ | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| from dataclasses import dataclass | ||
|
|
||
|
|
||
| @dataclass(frozen=True) | ||
| class OpenAICompatPreset: | ||
| id: str | ||
| name: str | ||
| base_url: str | ||
| api_key_env: str | ||
|
|
||
|
|
||
| _PRESET_LIST: tuple[OpenAICompatPreset, ...] = ( | ||
| OpenAICompatPreset( | ||
| id="zai", | ||
| name="Z.AI", | ||
| base_url="https://api.z.ai/api/coding/paas/v4", | ||
| api_key_env="ZAI_API_KEY", | ||
| ), | ||
| OpenAICompatPreset( | ||
| id="deepseek", | ||
| name="DeepSeek", | ||
| base_url="https://api.deepseek.com/v1", | ||
| api_key_env="DEEPSEEK_API_KEY", | ||
| ), | ||
| OpenAICompatPreset( | ||
| id="moonshot", | ||
| name="Moonshot", | ||
| base_url="https://api.moonshot.ai/v1", | ||
| api_key_env="MOONSHOT_API_KEY", | ||
| ), | ||
| OpenAICompatPreset( | ||
| id="groq", | ||
| name="Groq", | ||
| base_url="https://api.groq.com/openai/v1", | ||
| api_key_env="GROQ_API_KEY", | ||
| ), | ||
| OpenAICompatPreset( | ||
| id="xai", | ||
| name="xAI", | ||
| base_url="https://api.x.ai/v1", | ||
| api_key_env="XAI_API_KEY", | ||
| ), | ||
| OpenAICompatPreset( | ||
| id="openrouter", | ||
| name="OpenRouter", | ||
| base_url="https://openrouter.ai/api/v1", | ||
| api_key_env="OPENROUTER_API_KEY", | ||
| ), | ||
| ) | ||
|
|
||
| OPENAI_COMPAT_PRESETS: dict[str, OpenAICompatPreset] = {p.id: p for p in _PRESET_LIST} | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -32,7 +32,7 @@ | |
| from agent_engine.runtime.hooks.models import HOOK_POINTS | ||
|
|
||
| _SECRET_MARKERS = ("api_key", "apikey", "secret", "token", "password", "private_key") | ||
| _SECRET_KEY_EXEMPTIONS = {"max_tokens"} | ||
| _SECRET_KEY_EXEMPTIONS = {"max_tokens", "api_key_env"} | ||
|
|
||
| # For *values*, a marker word alone is not evidence — ordinary prose like | ||
| # "Handles password reset requests" must pass. Flag a string value only when it | ||
|
|
@@ -50,6 +50,7 @@ | |
| r"|-----BEGIN [A-Z ]*PRIVATE KEY-----" # PEM private keys | ||
| ) | ||
| _SUPPORTED_MODEL_PROVIDERS = ("anthropic", "bedrock", "gemini", "openai") | ||
| _FIXED_ENDPOINT_PROVIDERS = ("anthropic", "bedrock", "gemini") | ||
|
|
||
|
|
||
| def _validate_plugins(plugins: Any, errors: list[ValidationError]) -> None: | ||
|
|
@@ -349,6 +350,17 @@ def _validate_model(path: str, raw: Any, errors: list[ValidationError]) -> None: | |
| ) | ||
| ) | ||
|
|
||
| if provider in _FIXED_ENDPOINT_PROVIDERS: | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. instead of nested ifs lets try make it better like (pseudo) : |
||
| for field_name in ("base_url", "api_key_env"): | ||
| if raw.get(field_name) is not None: | ||
| errors.append( | ||
| ValidationError( | ||
| f"{path}.{field_name}", | ||
| f"Not supported for provider '{provider}'; only OpenAI-compatible " | ||
| "providers accept a custom endpoint or key variable.", | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. on error message: base url or api key naming variable? |
||
| ) | ||
| ) | ||
|
|
||
| name = raw.get("name") | ||
| if not isinstance(name, str) or not name.strip(): | ||
| errors.append(ValidationError(f"{path}.name", "Required non-empty string")) | ||
|
|
@@ -604,6 +616,8 @@ def _build_model(self, raw: dict[str, Any]) -> ModelConfig: | |
| region=raw.get("region"), | ||
| max_tokens=raw.get("max_tokens"), | ||
| top_p=raw.get("top_p"), | ||
| base_url=raw.get("base_url"), | ||
| api_key_env=raw.get("api_key_env"), | ||
| ) | ||
|
|
||
| def _build_resolvers( | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.