From 7cb24aba3b28e2dc2a6d854abafcfc79e996d0e3 Mon Sep 17 00:00:00 2001 From: Tamaghna Sarkar Date: Mon, 20 Jul 2026 15:37:44 +0000 Subject: [PATCH 1/3] test(fixtures): add self-contained openai-default test system YAML Add a minimal, validating agent system under tests/fixtures/ for exercising the platform without the flagship example. - test_system.yaml defaults to the openai provider so it runs without an Anthropic key; any of anthropic/openai/gemini/bedrock is selectable via provider:. A comment makes explicit that the key is supplied via the environment, never in YAML. - Prompts, resolver stubs, and the echo_tool stub are implemented so agentctl validate passes fully offline (no LLM key required). --- tests/fixtures/plugins/__init__.py | 0 tests/fixtures/plugins/plugins.toml | 12 ++++ .../plugins/resolvers/greeting_agent.py | 8 +++ tests/fixtures/plugins/resolvers/shared.py | 14 ++++ tests/fixtures/plugins/tools/echo_tool.py | 5 ++ tests/fixtures/prompts/echo_agent/system.md | 3 + .../fixtures/prompts/greeting_agent/system.md | 4 ++ .../prompts/root_router/orchestrator.md | 7 ++ tests/fixtures/prompts/root_router/system.md | 4 ++ tests/fixtures/test_system.yaml | 68 +++++++++++++++++++ 10 files changed, 125 insertions(+) create mode 100644 tests/fixtures/plugins/__init__.py create mode 100644 tests/fixtures/plugins/plugins.toml create mode 100644 tests/fixtures/plugins/resolvers/greeting_agent.py create mode 100644 tests/fixtures/plugins/resolvers/shared.py create mode 100644 tests/fixtures/plugins/tools/echo_tool.py create mode 100644 tests/fixtures/prompts/echo_agent/system.md create mode 100644 tests/fixtures/prompts/greeting_agent/system.md create mode 100644 tests/fixtures/prompts/root_router/orchestrator.md create mode 100644 tests/fixtures/prompts/root_router/system.md create mode 100644 tests/fixtures/test_system.yaml diff --git a/tests/fixtures/plugins/__init__.py b/tests/fixtures/plugins/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/plugins/plugins.toml b/tests/fixtures/plugins/plugins.toml new file mode 100644 index 0000000..f637684 --- /dev/null +++ b/tests/fixtures/plugins/plugins.toml @@ -0,0 +1,12 @@ +# Auto-generated plugin manifest. Maps plugin refs to importable modules. +[tools.echo_tool] +module = "tools.echo_tool" +callable = "echo_tool" + +[resolvers.shared] +module = "resolvers.shared" +class = "SharedResolver" + +[resolvers.greeting_agent] +module = "resolvers.greeting_agent" +class = "Resolver" diff --git a/tests/fixtures/plugins/resolvers/greeting_agent.py b/tests/fixtures/plugins/resolvers/greeting_agent.py new file mode 100644 index 0000000..2c5ac9e --- /dev/null +++ b/tests/fixtures/plugins/resolvers/greeting_agent.py @@ -0,0 +1,8 @@ +from __future__ import annotations + +from shared import SharedResolver + + +class Resolver(SharedResolver): + def __init__(self) -> None: + super().__init__() diff --git a/tests/fixtures/plugins/resolvers/shared.py b/tests/fixtures/plugins/resolvers/shared.py new file mode 100644 index 0000000..bf56301 --- /dev/null +++ b/tests/fixtures/plugins/resolvers/shared.py @@ -0,0 +1,14 @@ +from __future__ import annotations + + +class SharedResolver: + def __init__(self) -> None: + pass + + def current_date(self, ctx: dict) -> str: + """Returns the value for {{current_date}}""" + return "2026-07-19" + + def user_name(self, ctx: dict) -> str: + """Returns the value for {{user_name}}""" + return "Tester" diff --git a/tests/fixtures/plugins/tools/echo_tool.py b/tests/fixtures/plugins/tools/echo_tool.py new file mode 100644 index 0000000..c0b1082 --- /dev/null +++ b/tests/fixtures/plugins/tools/echo_tool.py @@ -0,0 +1,5 @@ +def echo_tool(input: dict) -> str: + """Echo back the input text. Used to verify tool wiring end to end. +""" + text = input.get("text", "") + return f"echo: {text}" diff --git a/tests/fixtures/prompts/echo_agent/system.md b/tests/fixtures/prompts/echo_agent/system.md new file mode 100644 index 0000000..b93735d --- /dev/null +++ b/tests/fixtures/prompts/echo_agent/system.md @@ -0,0 +1,3 @@ +# Echo Agent — system + +You echo the user's input back to them using the `echo_tool` tool. diff --git a/tests/fixtures/prompts/greeting_agent/system.md b/tests/fixtures/prompts/greeting_agent/system.md new file mode 100644 index 0000000..1efbb64 --- /dev/null +++ b/tests/fixtures/prompts/greeting_agent/system.md @@ -0,0 +1,4 @@ +# Greeting Agent — system + +You handle greetings and small talk. Be friendly, brief, and use the +`current_date` and `user_name` context when natural. diff --git a/tests/fixtures/prompts/root_router/orchestrator.md b/tests/fixtures/prompts/root_router/orchestrator.md new file mode 100644 index 0000000..9a4c3d7 --- /dev/null +++ b/tests/fixtures/prompts/root_router/orchestrator.md @@ -0,0 +1,7 @@ +# Root Router — orchestrator + +You are the entry point of the test system. Read the user's request and route it +to the most appropriate child specialist: + +- `greeting_agent` for greetings and small talk. +- `echo_agent` for requests that ask to repeat or echo text. diff --git a/tests/fixtures/prompts/root_router/system.md b/tests/fixtures/prompts/root_router/system.md new file mode 100644 index 0000000..aae826e --- /dev/null +++ b/tests/fixtures/prompts/root_router/system.md @@ -0,0 +1,4 @@ +# Root Router — system + +You route each incoming request to a single child specialist. Do not answer +directly; delegate to `greeting_agent` or `echo_agent`. diff --git a/tests/fixtures/test_system.yaml b/tests/fixtures/test_system.yaml new file mode 100644 index 0000000..cc69875 --- /dev/null +++ b/tests/fixtures/test_system.yaml @@ -0,0 +1,68 @@ +system: + name: "Test Agent System" + +# Provider is declared here in YAML. The matching API key is NOT set in YAML — +# it comes from the environment (e.g. OPENAI_API_KEY). `agentctl validate` +# needs no key; `agentctl run`/`serve` need a key for the chosen provider. +defaults: + model: + provider: openai + name: gpt-4o-mini + temperature: 0.0 + +execution: + max_iterations: 10 + max_tool_calls: 5 + max_tool_calls_per_agent: 3 + max_child_agent_calls: 4 + allow_duplicate_tool_calls: false + +tools: + echo_tool: + description: > + Echo back the input text. Used to verify tool wiring end to end. + +resolvers: + current_date: + scope: shared + + user_name: + scope: shared + +orchestrators: + root_router: + name: "Root Router" + description: > + Entry point. Routes the user request to the appropriate specialist. + model: + provider: openai + name: gpt-4o + temperature: 0.0 + prompts: + orchestrator: prompts/root_router/orchestrator.md + system: prompts/root_router/system.md + +agents: + greeting_agent: + name: "Greeting Agent" + description: > + Handles simple greetings and small talk. + prompts: + system: prompts/greeting_agent/system.md + resolvers: + - current_date + - user_name + + echo_agent: + name: "Echo Agent" + description: > + Demonstrates tool usage by echoing user input. + prompts: + system: prompts/echo_agent/system.md + tools: + - echo_tool + +graph: + root_router: + greeting_agent: + echo_agent: From 1931c1ad12b591c3fa8241726f1f17736bd5a38b Mon Sep 17 00:00:00 2001 From: Tamaghna Sarkar Date: Sun, 26 Jul 2026 18:04:24 +0000 Subject: [PATCH 2/3] test(fixtures): avoid sys.modules conflict by removing shared.py --- tests/engine/test_engine_flow.py | 54 +-------- tests/fixtures/plugins/plugins.toml | 4 - .../plugins/resolvers/greeting_agent.py | 9 +- tests/fixtures/plugins/resolvers/shared.py | 14 --- tests/fixtures/support.py | 111 ++++++++++++++++++ tests/fixtures/test_infra.py | 38 ++++++ tests/fixtures/test_system.yaml | 8 +- 7 files changed, 161 insertions(+), 77 deletions(-) delete mode 100644 tests/fixtures/plugins/resolvers/shared.py create mode 100644 tests/fixtures/support.py create mode 100644 tests/fixtures/test_infra.py diff --git a/tests/engine/test_engine_flow.py b/tests/engine/test_engine_flow.py index ba8856c..d7f6d6a 100644 --- a/tests/engine/test_engine_flow.py +++ b/tests/engine/test_engine_flow.py @@ -14,15 +14,13 @@ from __future__ import annotations -from collections.abc import AsyncIterator, Callable +from collections.abc import Callable from dataclasses import dataclass from pathlib import Path -from typing import Any, cast +from typing import Any import pytest from langchain_core.language_models import BaseChatModel -from langchain_core.messages import AIMessage, HumanMessage, ToolMessage -from langchain_core.messages.tool import ToolCall from agent_engine.core.spec import ( AgentSpec, @@ -40,60 +38,16 @@ from agent_engine.engine.types import RunResult from agent_engine.runtime.hooks import AuthContext, RunContext from agent_engine.runtime.state import GraphState +from tests.fixtures.support import fake_model_factory # --------------------------------------------------------------------------- # Fake chat model # --------------------------------------------------------------------------- -class FakeChatModel: - """A scriptless stand-in: route through one tool, then answer.""" - - def __init__(self, answer: str = "ok", tool_names: list[str] | None = None) -> None: - self._answer = answer - self._tool_names = tool_names or [] - - def bind_tools(self, tools: list[Any]) -> FakeChatModel: - return FakeChatModel(self._answer, [t.name for t in tools]) - - async def ainvoke(self, messages: list[Any]) -> AIMessage: - return self._respond(messages) - - async def astream(self, messages: list[Any]) -> AsyncIterator[AIMessage]: - yield self._respond(messages) - - def _respond(self, messages: list[Any]) -> AIMessage: - already_called = any(isinstance(m, ToolMessage) for m in messages) - if self._tool_names and not already_called: - call = ToolCall( - name=self._select(messages), - args={"message": self._user_text(messages)}, - id="call_1", - ) - return AIMessage(content="", tool_calls=[call]) - return AIMessage(content=self._answer) - - def _select(self, messages: list[Any]) -> str: - text = self._user_text(messages).lower() - for name in self._tool_names: - if name.lower() in text: - return name - return self._tool_names[0] - - @staticmethod - def _user_text(messages: list[Any]) -> str: - for m in reversed(messages): - if isinstance(m, HumanMessage): - return str(m.content) - return "" - - @pytest.fixture def model_factory() -> Callable[[str, str, float | None], BaseChatModel]: - def factory(provider: str, name: str, temperature: float | None) -> BaseChatModel: - return cast(BaseChatModel, FakeChatModel()) - - return factory + return fake_model_factory # --------------------------------------------------------------------------- diff --git a/tests/fixtures/plugins/plugins.toml b/tests/fixtures/plugins/plugins.toml index f637684..b580637 100644 --- a/tests/fixtures/plugins/plugins.toml +++ b/tests/fixtures/plugins/plugins.toml @@ -3,10 +3,6 @@ module = "tools.echo_tool" callable = "echo_tool" -[resolvers.shared] -module = "resolvers.shared" -class = "SharedResolver" - [resolvers.greeting_agent] module = "resolvers.greeting_agent" class = "Resolver" diff --git a/tests/fixtures/plugins/resolvers/greeting_agent.py b/tests/fixtures/plugins/resolvers/greeting_agent.py index 2c5ac9e..0759d44 100644 --- a/tests/fixtures/plugins/resolvers/greeting_agent.py +++ b/tests/fixtures/plugins/resolvers/greeting_agent.py @@ -1,8 +1,9 @@ from __future__ import annotations -from shared import SharedResolver +class Resolver: + def current_date(self, ctx: dict) -> str: + return "2026-07-19" -class Resolver(SharedResolver): - def __init__(self) -> None: - super().__init__() + def user_name(self, ctx: dict) -> str: + return "Tester" diff --git a/tests/fixtures/plugins/resolvers/shared.py b/tests/fixtures/plugins/resolvers/shared.py deleted file mode 100644 index bf56301..0000000 --- a/tests/fixtures/plugins/resolvers/shared.py +++ /dev/null @@ -1,14 +0,0 @@ -from __future__ import annotations - - -class SharedResolver: - def __init__(self) -> None: - pass - - def current_date(self, ctx: dict) -> str: - """Returns the value for {{current_date}}""" - return "2026-07-19" - - def user_name(self, ctx: dict) -> str: - """Returns the value for {{user_name}}""" - return "Tester" diff --git a/tests/fixtures/support.py b/tests/fixtures/support.py new file mode 100644 index 0000000..fc8e6df --- /dev/null +++ b/tests/fixtures/support.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from langchain_core.messages import AIMessage +from langchain_core.messages.tool import ToolCall + +from agent_engine.core.spec import SystemSpec +from agent_engine.engine.langgraph.engine import LangGraphEngine +from agent_engine.engine.types import RunResult + + +class FakeChatModel: + def __init__( + self, + answer: str = "ok", + tool_names: list[str] | None = None, + ) -> None: + self._answer = answer + self._tool_names = tool_names or [] + + def bind_tools(self, tools: list[Any]) -> FakeChatModel: + return FakeChatModel( + self._answer, + [t.name for t in tools], + ) + + async def ainvoke(self, messages: list[Any]) -> AIMessage: + return self._respond(messages) + + async def astream(self, messages: list[Any]) -> Any: + yield self._respond(messages) + + def _respond(self, messages: list[Any]) -> AIMessage: + already_called = any( + m.__class__.__name__ == "ToolMessage" for m in messages + ) + if self._tool_names and not already_called: + return AIMessage( + content="", + tool_calls=[ + ToolCall( + name=self._select(messages), + args={"message": self._user_text(messages)}, + id="call_1", + ) + ], + ) + return AIMessage(content=self._answer) + + def _select(self, messages: list[Any]) -> str: + text = self._user_text(messages).lower() + for name in self._tool_names: + if name.lower() in text: + return name + return self._tool_names[0] + + @staticmethod + def _user_text(messages: list[Any]) -> str: + for m in reversed(messages): + if m.__class__.__name__ == "HumanMessage": + return str(m.content) + return "" + + +def fake_model_factory( + provider: str = "fake", + name: str = "fake", + temperature: float | None = None, + *, + answer: str = "ok", +) -> Any: + return FakeChatModel(answer=answer) + + +FIXTURE_DIR = Path(__file__).resolve().parent +FIXTURE_SPEC = FIXTURE_DIR / "test_system.yaml" + + +def load_test_system() -> tuple[SystemSpec, Path]: + from agent_engine.core.validator import SystemSpecValidator + from agent_engine.parsers.yaml.parser import YAMLParser + + spec = YAMLParser().parse(str(FIXTURE_SPEC)) + errors = SystemSpecValidator().validate(spec, FIXTURE_DIR) + if errors: + raise AssertionError( + "Fixture validation failed:\n" + "\n".join(str(e) for e in errors) + ) + return spec, FIXTURE_DIR + + +class FakeEngine: + def __init__(self, answer: str = "ok") -> None: + self._answer = answer + self._engine: LangGraphEngine | None = None + + async def __aenter__(self) -> FakeEngine: + spec, base_dir = load_test_system() + self._engine = LangGraphEngine(base_dir, model_factory=fake_model_factory) + await self._engine.build(spec) + return self + + async def __aexit__(self, *args: Any) -> None: + if self._engine is not None: + await self._engine.__aexit__(*args) + + async def run(self, message: str) -> RunResult: + assert self._engine is not None + return await self._engine.run(message) diff --git a/tests/fixtures/test_infra.py b/tests/fixtures/test_infra.py new file mode 100644 index 0000000..6a59d55 --- /dev/null +++ b/tests/fixtures/test_infra.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import pytest +from tests.fixtures.support import FakeEngine, load_test_system + + +def test_spec_validates_offline() -> None: + load_test_system() + + +@pytest.mark.asyncio +async def test_engine_builds_from_fixture() -> None: + async with FakeEngine(): + pass + + +@pytest.mark.asyncio +async def test_orchestrator_routes_to_greeting() -> None: + async with FakeEngine() as engine: + result = await engine.run("greeting_agent hello") + assert "root_router/greeting_agent" in result.visited + assert result.answer + + +@pytest.mark.asyncio +async def test_orchestrator_routes_to_echo_and_calls_tool() -> None: + async with FakeEngine() as engine: + result = await engine.run("echo_agent echo test") + assert "root_router/echo_agent" in result.visited + assert any(record.name == "echo_tool" for record in result.used_tools) + assert result.answer + + +@pytest.mark.asyncio +async def test_resolvers_resolve_at_runtime() -> None: + async with FakeEngine() as engine: + result = await engine.run("hello") + assert result.status == "completed" diff --git a/tests/fixtures/test_system.yaml b/tests/fixtures/test_system.yaml index cc69875..85b6b98 100644 --- a/tests/fixtures/test_system.yaml +++ b/tests/fixtures/test_system.yaml @@ -24,10 +24,7 @@ tools: resolvers: current_date: - scope: shared - user_name: - scope: shared orchestrators: root_router: @@ -44,7 +41,7 @@ orchestrators: agents: greeting_agent: - name: "Greeting Agent" + name: "greeting" description: > Handles simple greetings and small talk. prompts: @@ -54,13 +51,14 @@ agents: - user_name echo_agent: - name: "Echo Agent" + name: "echo" description: > Demonstrates tool usage by echoing user input. prompts: system: prompts/echo_agent/system.md tools: - echo_tool + auto: true graph: root_router: From 75e892f5306128e330941ddf356662c5cf7dda93 Mon Sep 17 00:00:00 2001 From: Tamaghna Sarkar Date: Mon, 27 Jul 2026 19:58:53 +0000 Subject: [PATCH 3/3] test(fixtures): add issue #24 infra foundation - test_system.yaml: small openai-default fixture with auto:true on tool-using agent - prompts/ + plugins/: implemented stubs so agentctl validate passes offline - utils.py: shared FakeChatModel, fake_model_factory, load_test_system, FakeEngine - test_infra.py: 5 core tests covering validation, build, routing, tools, resolvers - test_engine_flow.py: dedup FakeChatModel into tests.fixtures.utils Closes the shared-module import collision with an autouse cleanup fixture. --- tests/engine/test_engine_flow.py | 2 +- tests/fixtures/plugins/plugins.toml | 6 +++++- tests/fixtures/plugins/resolvers/greeting_agent.py | 9 ++++----- tests/fixtures/plugins/resolvers/shared.py | 14 ++++++++++++++ tests/fixtures/test_infra.py | 10 +++++++++- tests/fixtures/test_system.yaml | 4 +--- tests/fixtures/{support.py => utils.py} | 0 7 files changed, 34 insertions(+), 11 deletions(-) create mode 100644 tests/fixtures/plugins/resolvers/shared.py rename tests/fixtures/{support.py => utils.py} (100%) diff --git a/tests/engine/test_engine_flow.py b/tests/engine/test_engine_flow.py index d7f6d6a..5c25088 100644 --- a/tests/engine/test_engine_flow.py +++ b/tests/engine/test_engine_flow.py @@ -38,7 +38,7 @@ from agent_engine.engine.types import RunResult from agent_engine.runtime.hooks import AuthContext, RunContext from agent_engine.runtime.state import GraphState -from tests.fixtures.support import fake_model_factory +from tests.fixtures.utils import fake_model_factory # --------------------------------------------------------------------------- # Fake chat model diff --git a/tests/fixtures/plugins/plugins.toml b/tests/fixtures/plugins/plugins.toml index b580637..a731f13 100644 --- a/tests/fixtures/plugins/plugins.toml +++ b/tests/fixtures/plugins/plugins.toml @@ -1,8 +1,12 @@ -# Auto-generated plugin manifest. Maps plugin refs to importable modules. + [tools.echo_tool] module = "tools.echo_tool" callable = "echo_tool" +[resolvers.shared] +module = "resolvers.shared" +class = "SharedResolver" + [resolvers.greeting_agent] module = "resolvers.greeting_agent" class = "Resolver" diff --git a/tests/fixtures/plugins/resolvers/greeting_agent.py b/tests/fixtures/plugins/resolvers/greeting_agent.py index 0759d44..2c5ac9e 100644 --- a/tests/fixtures/plugins/resolvers/greeting_agent.py +++ b/tests/fixtures/plugins/resolvers/greeting_agent.py @@ -1,9 +1,8 @@ from __future__ import annotations +from shared import SharedResolver -class Resolver: - def current_date(self, ctx: dict) -> str: - return "2026-07-19" - def user_name(self, ctx: dict) -> str: - return "Tester" +class Resolver(SharedResolver): + def __init__(self) -> None: + super().__init__() diff --git a/tests/fixtures/plugins/resolvers/shared.py b/tests/fixtures/plugins/resolvers/shared.py new file mode 100644 index 0000000..bf56301 --- /dev/null +++ b/tests/fixtures/plugins/resolvers/shared.py @@ -0,0 +1,14 @@ +from __future__ import annotations + + +class SharedResolver: + def __init__(self) -> None: + pass + + def current_date(self, ctx: dict) -> str: + """Returns the value for {{current_date}}""" + return "2026-07-19" + + def user_name(self, ctx: dict) -> str: + """Returns the value for {{user_name}}""" + return "Tester" diff --git a/tests/fixtures/test_infra.py b/tests/fixtures/test_infra.py index 6a59d55..efa7f49 100644 --- a/tests/fixtures/test_infra.py +++ b/tests/fixtures/test_infra.py @@ -1,7 +1,15 @@ from __future__ import annotations +import sys + import pytest -from tests.fixtures.support import FakeEngine, load_test_system +from tests.fixtures.utils import FakeEngine, load_test_system + + +@pytest.fixture(autouse=True) +def _cleanup_shared() -> None: + yield + sys.modules.pop("shared", None) def test_spec_validates_offline() -> None: diff --git a/tests/fixtures/test_system.yaml b/tests/fixtures/test_system.yaml index 85b6b98..eb9d773 100644 --- a/tests/fixtures/test_system.yaml +++ b/tests/fixtures/test_system.yaml @@ -1,9 +1,7 @@ system: name: "Test Agent System" -# Provider is declared here in YAML. The matching API key is NOT set in YAML — -# it comes from the environment (e.g. OPENAI_API_KEY). `agentctl validate` -# needs no key; `agentctl run`/`serve` need a key for the chosen provider. + defaults: model: provider: openai diff --git a/tests/fixtures/support.py b/tests/fixtures/utils.py similarity index 100% rename from tests/fixtures/support.py rename to tests/fixtures/utils.py