From 01be3bcf9a5882143ae948b63028be7e09821868 Mon Sep 17 00:00:00 2001 From: LukasParke <5702154+LukasParke@users.noreply.github.com> Date: Fri, 24 Jul 2026 16:29:48 -0500 Subject: [PATCH 1/7] feat: automated Upstreamer port pipeline for @openrouter/agent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This repo is a port of @openrouter/agent but had no automation keeping it in sync: one commit (the manual 0.7.2 port), no CI, no drift signal. Upstream is now at 0.8.0, so the port is a minor version behind and missing HooksManager, versioned state serialization, and the #61-#68 fixes. Wires up Upstreamer (github.com/mountgram/upstreamer, MIT) so the port tracks upstream automatically, with gates strong enough to run unattended. - .upstreamer/upstreamer.md — the rewrite contract: scope (packages/agent only, MCP explicitly out), required public API as the parity floor, the TS->Python naming map, permanent idiomatic divergences (Pydantic/Zod, async-first, generator convention, looser typing), and the `openrouter` substrate pin. - .upstreamer/scripts/verify.sh — mechanical gate: ruff/mypy/pytest, required API presence, version consistency vs the ported upstream release. - .upstreamer/eval.md — parity gate, run in a fresh context against the TS reference. This is what catches a port that compiles but sits a version behind. - scripts/upstream — wrapper adapted for in-repo downstream: this repo IS the output, so no publish leg is needed. - .github/workflows/upstreamer-port.yaml — triggered by repository_dispatch from typescript-agent on npm publish (ports track releases, not every commit), weekly cron as backstop. Opens a PR; never pushes to main. - .github/workflows/ci.yaml — this repo had no CI at all, so the verifier's test calls only ever ran inside the sync job. State is seeded at 0.7.2 (adc7939) so the first run is an incremental 0.7.2 -> 0.8.0 delta rather than an unreviewable full regeneration. Failed gates deliberately leave state.yaml stale: the next run retries the same delta instead of skipping the gap, and the PR is labelled eval-failed. Credentials are OPENROUTER_API_KEY + OPENCODE_MODEL, read from .upstreamer/port.env locally (gitignored) or repo secret/variable in CI. See PORTING.md. --- .github/actions/port-toolchain/action.yml | 17 ++ .github/workflows/ci.yaml | 54 +++++ .github/workflows/upstreamer-port.yaml | 133 ++++++++++++ .gitignore | 9 + .upstreamer/eval.md | 96 +++++++++ .upstreamer/port.env.example | 24 +++ .upstreamer/scripts/verify.sh | 101 +++++++++ .../skills/upstreamer-converter/SKILL.md | 151 ++++++++++++++ .upstreamer/state.yaml | 6 + .upstreamer/upstreamer.md | 193 ++++++++++++++++++ PORTING.md | 117 +++++++++++ README.md | 3 + scripts/upstream | 173 ++++++++++++++++ 13 files changed, 1077 insertions(+) create mode 100644 .github/actions/port-toolchain/action.yml create mode 100644 .github/workflows/ci.yaml create mode 100644 .github/workflows/upstreamer-port.yaml create mode 100644 .upstreamer/eval.md create mode 100644 .upstreamer/port.env.example create mode 100755 .upstreamer/scripts/verify.sh create mode 100644 .upstreamer/skills/upstreamer-converter/SKILL.md create mode 100644 .upstreamer/state.yaml create mode 100644 .upstreamer/upstreamer.md create mode 100644 PORTING.md create mode 100755 scripts/upstream diff --git a/.github/actions/port-toolchain/action.yml b/.github/actions/port-toolchain/action.yml new file mode 100644 index 0000000..31424e6 --- /dev/null +++ b/.github/actions/port-toolchain/action.yml @@ -0,0 +1,17 @@ +name: Port toolchain (Python) +description: Install the toolchain the port verifier needs (uv + Python). + +runs: + using: composite + steps: + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + + - name: Sync dependencies + shell: bash + run: uv sync --all-extras diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..6b31bd8 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,54 @@ +name: CI + +# This repo had no CI. Without it the port verifier's pytest/ruff/mypy calls only +# ever ran inside the sync job, so a generated PR reached review with no +# independent signal. This runs the same checks on every PR and push. + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +jobs: + check: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + + - run: uv sync --all-extras + + - name: Lint + run: uv run ruff check . + + - name: Format + run: uv run ruff format --check . + + - name: Type check + run: uv run mypy src + + # Deterministic tests only. tests/e2e needs OPENROUTER_API_KEY and skips + # cleanly without it. + - name: Tests + run: uv run pytest tests/unit -q + + # Runs the port's own mechanical gate on PRs that touch the port machinery, so + # a broken verifier or contract is caught here rather than mid-sync. + verify-port: + runs-on: ubuntu-latest + timeout-minutes: 15 + if: github.event_name == 'pull_request' + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/port-toolchain + - name: Port verifier + run: ./.upstreamer/scripts/verify.sh diff --git a/.github/workflows/upstreamer-port.yaml b/.github/workflows/upstreamer-port.yaml new file mode 100644 index 0000000..2c8df16 --- /dev/null +++ b/.github/workflows/upstreamer-port.yaml @@ -0,0 +1,133 @@ +name: Upstreamer Port + +# Ports @openrouter/agent into this repo. Two triggers: +# 1. repository_dispatch from typescript-agent's publish.yaml on a new npm release +# (event type: openrouter-agent-published) — the intended path. Ports track +# published releases, not every commit to upstream main. +# 2. Weekly cron as a safety net for missed dispatches, plus manual dispatch. +# +# Opens a PR. Never pushes to main. A failed parity eval leaves +# .upstreamer/state.yaml unchanged, so the next run retries the same delta. + +on: + repository_dispatch: + types: [openrouter-agent-published] + schedule: + - cron: "23 6 * * 1" + workflow_dispatch: + inputs: + ref: + description: "Upstream ref to port (blank = upstream default branch HEAD)" + required: false + type: string + force: + description: "Re-run even if the upstream commit is unchanged" + required: false + default: false + type: boolean + +permissions: + contents: write + pull-requests: write + +concurrency: + group: upstreamer-port + cancel-in-progress: false + +jobs: + port: + runs-on: ubuntu-latest + timeout-minutes: 300 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: oven-sh/setup-bun@v2 + + - name: Install opencode + run: bun install -g opencode-ai + + - name: Set up language toolchain + uses: ./.github/actions/port-toolchain + + - name: Run port + env: + # Provide these in repo settings: + # Secret OPENROUTER_API_KEY — sk-or-... key opencode uses for inference + # Variable OPENCODE_MODEL — e.g. openrouter/anthropic/claude-sonnet-latest + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + OPENCODE_MODEL: ${{ vars.OPENCODE_MODEL }} + UPSTREAMER_TIMEOUT_SECONDS: 14400 + run: | + set -euo pipefail + if [ -z "${OPENROUTER_API_KEY:-}" ]; then + echo "::error::OPENROUTER_API_KEY secret is not set. See .upstreamer/port.env.example." + exit 1 + fi + args=() + [ "${{ inputs.force }}" = "true" ] && args+=(--force) + REF="${{ inputs.ref || github.event.client_payload.ref }}" + [ -n "$REF" ] && args+=(--ref "$REF") + ./scripts/upstream "${args[@]}" + + - name: Check for changes + id: diff + run: | + if [ -n "$(git status --porcelain -- . ':!tmp')" ]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + else + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "No changes — upstream unchanged or port was a no-op." + fi + + # State only advances when the verifier AND the parity eval passed, so an + # unchanged state file next to a changed tree means the port did not pass. + # Label the PR accordingly instead of letting it look green. + - name: Detect eval failure + if: steps.diff.outputs.changed == 'true' + id: gate + run: | + if git diff --quiet -- .upstreamer/state.yaml; then + echo "passed=false" >> "$GITHUB_OUTPUT" + echo "::warning::state.yaml did not advance — parity eval did not pass. See .upstreamer/eval-report.md." + else + echo "passed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Open PR + if: steps.diff.outputs.changed == 'true' + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ secrets.GITHUB_TOKEN }} + branch: upstreamer/sync + delete-branch: true + title: >- + ${{ steps.gate.outputs.passed == 'true' + && 'port: sync with @openrouter/agent upstream' + || 'port: sync with @openrouter/agent upstream (EVAL FAILED — do not merge)' }} + commit-message: "port: sync with @openrouter/agent upstream" + labels: >- + ${{ steps.gate.outputs.passed == 'true' + && 'upstreamer, automated' + || 'upstreamer, automated, eval-failed' }} + body: | + Automated Upstreamer port of `@openrouter/agent` into this repo. + + - Contract: `.upstreamer/upstreamer.md` + - Run log: `.upstreamer/logs/` + - Parity eval: `.upstreamer/eval-report.md` + - Parity eval passed: **${{ steps.gate.outputs.passed }}** + + Review the diff as a port, not as a normal PR: check behavioral parity + against the TypeScript reference, not just that it compiles. If + `.upstreamer/state.yaml` did not advance, the eval did not pass and this + PR must not be merged as-is. + + - name: Upload logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: upstreamer-logs + path: .upstreamer/logs/ + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index e87499d..0c2a0f4 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,12 @@ build/ *.egg-info/ .env .env.* + +# Upstreamer port machinery +# Upstream checkout + scratch work +tmp/upstreamer/ +# Run logs +.upstreamer/logs/ +# LOCAL SECRETS - never commit +.upstreamer/port.env +!.upstreamer/port.env.example diff --git a/.upstreamer/eval.md b/.upstreamer/eval.md new file mode 100644 index 0000000..f859116 --- /dev/null +++ b/.upstreamer/eval.md @@ -0,0 +1,96 @@ +# Port Parity Eval — Python + +Run after mechanical verification passes, in a **fresh review context**. You are +grading a port you did not write. Do not rely on the converter's reasoning, +summary, or changelog — read the code. + +## Goal + +Decide whether this repository is a faithful, usable Python port of +`@openrouter/agent` at the target upstream commit. The question is not "does it +build" — the verifier already answered that. The question is **"would a user of +this package get the same behavior a TypeScript user gets?"** + +This eval exists because a port can compile, pass its own tests, and still be a +version behind on real behavior. That failure mode is the one to catch. + +## Inputs + +- Contract: `.upstreamer/upstreamer.md` +- Upstream reference: `tmp/upstreamer/upstream/packages/agent/` +- This repo: `src/openrouter_agent/`, `tests/` +- Last ported commit: `.upstreamer/state.yaml` +- Converter's run log: `.upstreamer/logs/` (read last, and skeptically) + +## Method + +1. Read the contract's Required Public API list. +2. For each entry, find it in this repo **and** find its upstream counterpart. + Compare behavior, not names. +3. Read the upstream delta yourself: + ```bash + git -C tmp/upstreamer/upstream diff .. -- packages/agent/src + ``` + For each behavioral change, verify the port reflects it. A missing edge case + is a real finding. +4. Run the test suite and read what it actually asserts. Tests that assert the + port's own shape rather than upstream behavior are not parity coverage. +5. Prefer reading upstream tests: they encode the behavior contract most + precisely. Check the port covers the same cases. + +## Required Qualities + +**Public API completeness.** Every symbol in the contract's Required Public API +list is present, exported, and reachable from the package's public entry point. + +**Version honesty.** The port's declared/recorded upstream version matches what +was actually ported. A port claiming 0.8.0 while missing `HooksManager` is a FAIL, +not a warning. + +**The load-bearing loop.** Tool execution, multi-turn continuation, and stop +conditions behave as upstream: correct request sequence, accumulated input and +history preserved across turns, `previous_response_id` carried forward. + +**Streaming.** Event order and turn boundaries match. Every consumer sees stream +and transport errors — no consumer hangs or ends silently. Multi-consumer fan-out +works. + +**State.** Serialization round-trips. The version constant exists and a version +mismatch raises/returns an error rather than silently accepting a foreign blob. +State survives a pause/resume cycle mid-approval. + +**Approval / HITL ordering.** Mixed turns are the classic bug: when one turn has +both auto-executable and approval-required calls, auto-executable outputs are +recorded *before* the pause and replayed with the decisions. Regular-tool output +produced before a HITL pause is not lost. Resume order is `function_call` then +`function_call_output`. + +**Hooks.** Lifecycle hooks fire at the right points. Session id is threaded +per-emit so a shared manager is concurrency-safe. `SessionEnd` and drain happen +even on no-tools stream error paths. + +**Compatibility helpers.** Claude/Chat conversion round-trips preserve metadata, +reasoning, tool use, and unsupported content. + +**Divergences are the documented ones.** Every difference from upstream is either +in the contract's Idiomatic Divergences section or recorded as a compatibility +note. An undocumented divergence is a finding. + +**Repo-owned files intact.** CI, license, release config, and package identity +were not rewritten by the port. + +## Verdict + +Return `PASS`, `PASS WITH WARNINGS`, or `FAIL` with concrete findings — file, +symbol, and what specifically differs from upstream. + +- `FAIL` — a required API symbol is missing, a behavioral parity gap exists in the + load-bearing loop / state / approval ordering / hooks, or the declared version + overstates what was ported. +- `PASS WITH WARNINGS` — parity holds on behavior; gaps are cosmetic, type-level, + or already documented as divergences. +- `PASS` — no findings. + +Be specific and be willing to fail. A false PASS is worse than no eval: it +advances `.upstreamer/state.yaml` and the next run skips past the gap, which is +exactly how a port silently falls a version behind. diff --git a/.upstreamer/port.env.example b/.upstreamer/port.env.example new file mode 100644 index 0000000..61bf379 --- /dev/null +++ b/.upstreamer/port.env.example @@ -0,0 +1,24 @@ +# Upstreamer port credentials + model choice. +# +# Copy to .upstreamer/port.env and fill in. That path is gitignored — never commit it. +# cp .upstreamer/port.env.example .upstreamer/port.env +# +# In CI these same two names come from repo secrets/variables instead of this file +# (see .github/workflows/upstreamer-port.yaml). + +# OpenRouter API key that opencode uses for inference. Starts with sk-or-. +# Create at https://openrouter.ai/settings/keys +# The wrapper writes this into ~/.local/share/opencode/auth.json so headless runs +# work without the interactive `opencode /connect` flow. +OPENROUTER_API_KEY= + +# Model opencode drives the port with, as an OpenRouter model id prefixed with the +# provider key. Overrides the `model:` field in .upstreamer/upstreamer.md. +# +# This is a load-bearing SDK port with a strict parity eval — use a strong coding +# model. Suggested starting point: +OPENCODE_MODEL=openrouter/anthropic/claude-sonnet-latest + +# Optional: hard wall-clock cap for one run, in seconds. A full-surface +# reconciliation is much slower than an incremental delta port. +UPSTREAMER_TIMEOUT_SECONDS=7200 diff --git a/.upstreamer/scripts/verify.sh b/.upstreamer/scripts/verify.sh new file mode 100755 index 0000000..97e28f7 --- /dev/null +++ b/.upstreamer/scripts/verify.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# Mechanical verification for the Python port. Objective checks only — +# judgment-heavy parity review lives in .upstreamer/eval.md. +set -uo pipefail +cd "$(dirname "$0")/../.." + +FAILURES=0 +pass() { echo " PASS: $1"; } +fail() { echo " FAIL: $1"; FAILURES=$((FAILURES + 1)); } + +run() { + local label="$1"; shift + if "$@" >/tmp/verify-out 2>&1; then + pass "$label" + else + fail "$label" + sed 's/^/ /' /tmp/verify-out | tail -40 + fi +} + +echo "=== Verification: python-agent ===" +echo + +echo "-- Toolchain" +if command -v uv >/dev/null 2>&1; then + run "uv sync" uv sync --all-extras + run "ruff check" uv run ruff check . + run "ruff format" uv run ruff format --check . + run "mypy" uv run mypy src + run "pytest" uv run pytest tests/unit -q +else + fail "uv not installed (required to build and test this package)" +fi +echo + +# Required public API. This is the parity floor from .upstreamer/upstreamer.md. +# Presence only — the eval judges behavior. +echo "-- Required public API importable from openrouter_agent" +REQUIRED_SYMBOLS=( + call_model OpenRouter tool server_tool ModelResult + create_initial_state append_to_messages update_state partition_tool_calls + serialize_conversation_state deserialize_conversation_state CONVERSATION_STATE_VERSION + step_count_is has_tool_call max_tokens_used max_cost finish_reason_is + HooksManager HookName + DEFAULT_FINAL_RESPONSE_DIRECTIVE + from_claude_messages to_claude_message from_chat_messages to_chat_message + extract_unsupported_content has_unsupported_content get_unsupported_content_summary + is_claude_style_messages + ToolContextStore ToolEventBroadcaster SDKHooks +) +if command -v uv >/dev/null 2>&1; then + missing=$(uv run python - "${REQUIRED_SYMBOLS[@]}" <<'PY' 2>/dev/null +import importlib, sys +mod = importlib.import_module("openrouter_agent") +print(" ".join(n for n in sys.argv[1:] if not hasattr(mod, n))) +PY +) + status=$? + if [ $status -ne 0 ]; then + fail "could not import openrouter_agent to check public API" + elif [ -n "${missing// /}" ]; then + fail "missing public API symbols: $missing" + else + pass "all ${#REQUIRED_SYMBOLS[@]} required symbols exported" + fi +fi +echo + +echo "-- Version consistency" +declared=$(grep -m1 '^version' pyproject.toml | sed 's/.*"\(.*\)".*/\1/') +upstream_pkg="tmp/upstreamer/upstream/packages/agent/package.json" +if [ -f "$upstream_pkg" ]; then + target=$(grep -m1 '"version"' "$upstream_pkg" | sed 's/.*"version": *"\([^"]*\)".*/\1/') + if [ "$declared" = "$target" ]; then + pass "version $declared matches ported @openrouter/agent $target" + else + fail "version drift: pyproject.toml=$declared, upstream @openrouter/agent=$target" + fi +else + echo " SKIP: no upstream checkout to compare version against" +fi +echo + +echo "-- No leaked TypeScript artifacts" +leaked=$(find src tests -type f \( -name '*.ts' -o -name '*.js' -o -name 'package.json' \ + -o -name 'tsconfig*.json' -o -name 'pnpm-lock.yaml' \) 2>/dev/null) +[ -z "$leaked" ] && pass "no TS/JS artifacts in src or tests" \ + || fail "leaked upstream artifacts: $leaked" + +echo "-- Repo-owned files present" +for f in LICENSE README.md pyproject.toml scripts/upstream; do + [ -e "$f" ] && pass "$f present" || fail "$f missing (port must not delete repo-owned files)" +done +echo + +if [ "$FAILURES" -eq 0 ]; then + echo "=== PASS: 0 failures ===" + exit 0 +fi +echo "=== FAIL: $FAILURES failure(s) ===" +exit 1 diff --git a/.upstreamer/skills/upstreamer-converter/SKILL.md b/.upstreamer/skills/upstreamer-converter/SKILL.md new file mode 100644 index 0000000..6274a8f --- /dev/null +++ b/.upstreamer/skills/upstreamer-converter/SKILL.md @@ -0,0 +1,151 @@ +--- +name: upstreamer-converter +description: Port an upstream source repository into this repository following an upstreamer.md contract. Use when running scripts/upstream, syncing this port with upstream, or reconciling the ported public API surface against the upstream reference. +--- + +# Upstreamer Converter (source port) + +Port an upstream repository into this repository following the contract at +`.upstreamer/upstreamer.md`. The contract is the source of truth; this skill +supplies execution discipline. + +Adapted from `mountgram/upstreamer` (MIT). Key difference: upstream's converter +generates a fresh downstream tree from scratch each time. This one maintains a +**living port of a versioned SDK** — the repo already exists, has consumers, and +publishes to a package registry. Incremental correctness matters more than +regeneration. + +## Step 0: Read the contract + +Read `.upstreamer/upstreamer.md` completely before touching anything. Parse the +frontmatter (`upstream`, `model`). Treat every section as binding: scope, +required public API, naming maps, idiomatic divergences, substrate pins, output +shape, verification. If this skill conflicts with the contract, follow the +contract and note the conflict in the final report. + +## Step 1: Establish the delta + +1. The upstream checkout is already at the target commit. Do not re-clone or + change its checkout. +2. Read `upstream_commit` from `.upstreamer/state.yaml`. This is the last commit + successfully ported *and* verified *and* eval-passed. +3. If a last commit exists and force is 0: + ```bash + git -C tmp/upstreamer/upstream diff --name-status .. + git -C tmp/upstreamer/upstream log --oneline .. + ``` + Scope work to changed files plus their consequences in this repo. Do not + refactor unrelated ported code — a large unreviewable diff is a failed port + even if it is correct. +4. If no last commit exists, or force is 1, reconcile the **entire** required + public API list in the contract against this repo. Report every gap found. +5. Read the changed upstream files in full. Diffs alone hide behavioral intent; + the surrounding code and its tests carry it. + +## Step 2: Port, don't transliterate + +For each upstream change: + +1. Apply the contract's naming map. Never invent a mapping that the contract + does not specify — if a new upstream symbol has no mapping, derive one that + follows the documented convention and **list it in the final report** so it + can be added to the contract. +2. Honor the contract's idiomatic divergences. These are deliberate and + permanent. Do not "fix" them toward the TypeScript shape. +3. Preserve *observable behavior*: call ordering, error surfaces, stream event + sequence and boundaries, state-shape compatibility, pause/resume semantics. + These are the port's actual contract with users. Type-level convenience is + secondary and the contract says where it is allowed to be looser. +4. Where the upstream behavior cannot be reproduced faithfully in this language, + do not silently approximate. Implement the closest honest equivalent and + record it as a compatibility note in `upstreamer-changelog.md`. +5. Never port build tooling, package manifests from upstream, CI, changesets, or + generated output. The contract's drop list is authoritative. + +## Step 3: Never clobber repo-owned files + +This repo is not disposable generated output. These are owned by the repo and +must not be rewritten by a port run unless the contract explicitly says to: + +- `.github/` — CI and release workflows +- `LICENSE` +- `.upstreamer/` — except `state.yaml`, `eval-report.md`, and `logs/` +- `scripts/upstream` +- Release/publish configuration and package identity (name, module path) + +Package **version** and dependency pins change only where the contract's +substrate-pin section directs it. + +## Step 4: Tests + +Ported behavior without a test proves nothing. For every behavioral change: + +1. Add or update deterministic tests in this repo's existing test layout and style. +2. Cover the specific upstream behavior that changed, not just the happy path. + Upstream fixes are usually edge cases — that edge case is the test. +3. Tests must pass without network access or paid credentials. Live/e2e tests + must skip cleanly when credentials are absent. + +## Step 5: Mechanical verification + +Run the verifier and fix what it reports: + +```bash +.upstreamer/scripts/verify.sh +``` + +It checks objective facts: toolchain build, lint, type check, tests, required +public API symbols present, no upstream-language artifacts leaked, declared +version consistent with the ported upstream version. A verifier failure is never +acceptable to hand off. + +## Step 6: Qualitative parity eval + +After mechanical verification passes, run `.upstreamer/eval.md` in a **fresh +subagent or separate review context**. This matters: the converter cannot +usefully grade its own port. The evaluator reads the contract, the upstream +reference, and this repo directly, and returns `PASS`, `PASS WITH WARNINGS`, or +`FAIL` with concrete findings. Write the result to `.upstreamer/eval-report.md`. + +`FAIL` is a blocker. Fix, re-run mechanical verification, re-run the eval. Up to +three focused attempts. + +## Step 7: State, or bankruptcy + +If the verifier passed and the eval returned `PASS` or `PASS WITH WARNINGS`, +write the target commit to `.upstreamer/state.yaml`: + +```yaml +upstream_commit: +``` + +Otherwise **declare bankruptcy**: + +1. Do not touch `.upstreamer/state.yaml`. +2. Write the failed eval result, what you attempted, the remaining blocker, and + the recommended human next action to `.upstreamer/eval-report.md`. +3. Say clearly in the final report that the eval failed. + +A stale state file is the correct outcome for a failed port. It is what makes the +next run retry the same delta instead of skipping past it. Never advance state to +make a run look successful. + +## Final report + +Begin with `Run summary`: + +1. **Upstream changes since last run** — meaningful commits/files/behavior + inspected. For a full reconciliation, say so and summarize the snapshot. +2. **Changes made here** — modules, public API, tests touched. +3. **Why these changes** — tie back to the contract, especially judgment calls. +4. **Verification** — commands run and results. +5. **Parity eval** — result and `eval-report.md` path. + +Then: previous and target upstream commit; incremental vs full; any new naming +mappings you had to derive (so they can be added to the contract); any parity gap +left open and why; any place the contract was ambiguous or wrong. + +Also update `upstreamer-changelog.md` at the repo root with user-facing +release-note bullets. That file is for users of this package: behavior changes, +new API, compatibility notes. Keep commit hashes, `state.yaml`, and verifier +internals out of it. diff --git a/.upstreamer/state.yaml b/.upstreamer/state.yaml new file mode 100644 index 0000000..793b4a6 --- /dev/null +++ b/.upstreamer/state.yaml @@ -0,0 +1,6 @@ +# Last upstream commit successfully ported AND verified AND eval-passed. +# Seeded at @openrouter/agent@0.7.2 (commit adc7939), the release these ports +# were generated from, so the first run is an incremental 0.7.2 -> 0.8.0 delta +# rather than a full from-scratch regeneration. +# Only scripts/upstream runs should change this. +upstream_commit: adc7939f4b7ed85b1a060d13433b8be6063cff73 diff --git a/.upstreamer/upstreamer.md b/.upstreamer/upstreamer.md new file mode 100644 index 0000000..7d609ab --- /dev/null +++ b/.upstreamer/upstreamer.md @@ -0,0 +1,193 @@ +--- +upstream: OpenRouterTeam/typescript-agent +downstream: OpenRouterTeam/python-agent +model: openrouter/anthropic/claude-sonnet-latest +--- + +# Python Port Contract — `@openrouter/agent` → `openrouter-agent` + +This repository is an async-first Python port of the OpenRouter TypeScript Agent +SDK (`@openrouter/agent`, in `packages/agent/` upstream). TypeScript is the +**reference spec**. Behavioral divergence is a bug unless it appears in the +Idiomatic Divergences section below. + +## Scope + +Port **only** `packages/agent/` from upstream. + +Explicitly out of scope: + +- `packages/mcp/` (`@openrouter/mcp`). Not ported. Do not begin porting it as a + side effect of a sync. Adding it is a deliberate contract change. +- Upstream JS/TS infrastructure: `package.json`, `pnpm-lock.yaml`, `tsconfig*`, + `turbo.json`, `biome.json`, vitest config, `.changeset/`, `.github/`, `esm/` + build output, `node_modules/`. +- Upstream README/docs prose. This repo's `README.md` is its own product surface; + update it when the public API changes, do not translate upstream's. + +## Substrate Pin + +The port sits on the generated `openrouter` Python SDK and must not reimplement +HTTP, auth, retries, or model schemas. + +- Target: `openrouter>=0.10.2` (current `pyproject.toml` pin). +- `call_model` sends through `client.beta.responses.send_async` — the Responses + API, matching upstream's Responses path. Do not switch to Chat Completions. + +Do **not** bump the `openrouter` dependency on your own initiative. Upstream +tracks `@openrouter/sdk`; the Python generated SDK moves independently and is +currently at a much newer major. Crossing that boundary is a breaking change +that needs its own PR. If an upstream change *requires* a newer `openrouter`, +stop and report it as a blocker rather than bumping. + +## Package Version + +`pyproject.toml` `version` tracks the ported `@openrouter/agent` version. Read it +from the upstream `packages/agent/package.json` at the target commit and set it +to match. If the target commit is between releases, keep the last released +version and note the drift in the final report. + +## Required Public API + +Every symbol below must be importable from `openrouter_agent` and behaviorally +faithful. This list is the parity floor — the verifier enforces presence, the +eval enforces behavior. + +Entry point and client: +- `call_model`, `OpenRouter` (with `call_model` convenience method) + +Tools: +- `tool`, `server_tool` — regular, generator, manual, HITL, and server tool + shapes; Pydantic v2 schema validation and JSON Schema generation; schema + sanitization before the request + +Result consumption (`ModelResult`): +- `get_text`, `get_response`, `get_text_stream`, `get_reasoning_stream`, + `get_tool_stream`, `get_tool_calls_stream`, `get_tool_calls`, + `get_full_responses_stream`, `get_new_messages_stream` +- Turn boundary events (`turn.start` / `turn.end`), context updates, pending + approvals, state inspection + +State: +- `create_initial_state`, `append_to_messages`, `update_state`, + `partition_tool_calls` +- **Versioned serialization contract** (upstream #66): `serialize_conversation_state`, + `deserialize_conversation_state`, `CONVERSATION_STATE_VERSION`, and the + invalid/mismatched-state error types. Round-trip must be stable and version + mismatch must raise rather than silently accept. + +Stop conditions: +- `step_count_is`, `has_tool_call`, `max_tokens_used`, `max_cost`, + `finish_reason_is` + +Lifecycle hooks (upstream #7, #67 — the 0.8.0 headline): +- `HooksManager` and its options, hook name enum / `BUILT_IN_HOOKS`, hook + definition/entry/handler/registry types, tool matchers, `PostModelCall` + telemetry, `SessionEnd` usage totals +- Session-id threading must be per-emit so a shared manager is concurrency-safe +- `SessionEnd` / drain must be guaranteed on no-tools stream error paths + +Final-turn control: +- `allow_final_response`, `DEFAULT_FINAL_RESPONSE_DIRECTIVE` (upstream #68 — + bare `True` uses the default directive; a string appends that string) + +Manual/HITL state: +- `awaiting_client_tools` persistence for unresolved manual tool calls + (upstream #64) + +Compatibility: +- `from_claude_messages`, `to_claude_message`, `from_chat_messages`, + `to_chat_message` +- `extract_unsupported_content`, `has_unsupported_content`, + `get_unsupported_content_summary` +- Claude content block / role enums, `is_claude_style_messages` + +Support: +- Tool context (`ToolContextStore`, context builder), `ToolEventBroadcaster`, + reusable streams, next-turn params, async param resolution, request options + passthrough, turn context, item/stream type guards, `SDKHooks` + +## Naming Map + +TypeScript `camelCase` → Python `snake_case` for functions and methods. Classes +and enums keep their names (`ModelResult`, `ToolContextStore`, +`ToolEventBroadcaster`, `ToolType`, `HooksManager`, `HookName`). + +Established mappings — do not re-derive: + +| TypeScript | Python | +| --- | --- | +| `callModel` | `call_model` | +| `serverTool` | `server_tool` | +| `getText` / `getResponse` | `get_text` / `get_response` | +| `getTextStream` / `getReasoningStream` | `get_text_stream` / `get_reasoning_stream` | +| `getToolStream` / `getToolCallsStream` / `getToolCalls` | `get_tool_stream` / `get_tool_calls_stream` / `get_tool_calls` | +| `getFullResponsesStream` / `getNewMessagesStream` | `get_full_responses_stream` / `get_new_messages_stream` | +| `stepCountIs`, `hasToolCall`, `maxTokensUsed`, `maxCost`, `finishReasonIs` | `step_count_is`, `has_tool_call`, `max_tokens_used`, `max_cost`, `finish_reason_is` | +| `fromClaudeMessages` / `toClaudeMessage` | `from_claude_messages` / `to_claude_message` | +| `fromChatMessages` / `toChatMessage` | `from_chat_messages` / `to_chat_message` | +| `createInitialState` / `appendToMessages` / `updateState` | `create_initial_state` / `append_to_messages` / `update_state` | +| `partitionToolCalls` | `partition_tool_calls` | +| `serializeConversationState` / `deserializeConversationState` | `serialize_conversation_state` / `deserialize_conversation_state` | +| `CONVERSATION_STATE_VERSION` | `CONVERSATION_STATE_VERSION` | +| `DEFAULT_FINAL_RESPONSE_DIRECTIVE` | `DEFAULT_FINAL_RESPONSE_DIRECTIVE` | +| `allowFinalResponse` | `allow_final_response` | +| `onToolCalled` / `onResponseReceived` | `on_tool_called` / `on_response_received` | +| `approveToolCalls` / `rejectToolCalls` | `approve_tool_calls` / `reject_tool_calls` | +| `extractUnsupportedContent` | `extract_unsupported_content` | + +If upstream adds a symbol with no mapping here, follow the convention and +**report the new mapping** so it can be added to this table. + +## Idiomatic Divergences + +Deliberate and permanent. Do not converge these toward TypeScript. + +1. **Pydantic v2 replaces Zod.** Tool inputs/outputs validated at execution + boundaries; schemas sanitized before hitting the Responses API. +2. **Async-first.** `async`/`await` throughout; `send_async` not `send`. +3. **Generator tools.** Python async generators cannot return a value the way JS + async generators can. Generator tools validate yielded values against + event/output schemas, emit preliminary events live, and treat the + output-shaped final yield as the final result. +4. **Typing is looser.** Python cannot reproduce TypeScript's tuple/conditional + inference for per-tool input/output/context narrowing. Runtime parity is + prioritized; exported aliases plus `py.typed` give best-effort static support. + Do not contort the runtime to chase a type-level feature. +5. **Cancellation** uses Python-native mechanisms, not `AbortSignal`. +6. **Structural Protocols** stand in where the generated Python SDK has no + matching symbol (e.g. `BeforeCreateRequestContext` / `BeforeCreateRequestHook` + until the generated SDK exposes them natively). Bind to real generated symbols + as soon as they exist. + +## Output Shape + +This repo IS the downstream. Write in place: + +```text +src/openrouter_agent/ # port lives here, one module per upstream lib module +tests/unit/ # deterministic tests +tests/e2e/ # live tests, must skip without OPENROUTER_API_KEY +pyproject.toml # version tracks ported @openrouter/agent version +README.md # this package's own docs +upstreamer-changelog.md # user-facing port notes +``` + +Module naming follows the existing layout: upstream `lib/tool-executor.ts` → +`src/openrouter_agent/tool_executor.py`. Keep that correspondence for new +modules so the port stays navigable against the reference. + +## Verification + +`.upstreamer/scripts/verify.sh` must pass: `ruff`, `mypy`, `pytest`, plus the +required-public-API presence check and no-TS-artifact check. + +Then `.upstreamer/eval.md` must return PASS or PASS WITH WARNINGS from a fresh +review context before state advances. + +## Final Report + +Include: upstream delta, modules and public API touched, tests added, any new +naming mappings derived, any parity gap deliberately left open with reasoning, +verifier result, eval result and report path, and whether the `openrouter` +substrate pin blocked anything. diff --git a/PORTING.md b/PORTING.md new file mode 100644 index 0000000..66296c0 --- /dev/null +++ b/PORTING.md @@ -0,0 +1,117 @@ +# Porting + +This package is a **port** of the OpenRouter TypeScript Agent SDK +(`@openrouter/agent`). TypeScript is the reference spec; this repo tracks it +automatically using [Upstreamer](https://github.com/mountgram/upstreamer) (MIT). + +Behavioral divergence from the TypeScript reference is a **bug**, unless it is +listed in the Idiomatic Divergences section of `.upstreamer/upstreamer.md`. + +## How it works + +``` +typescript-agent publishes @openrouter/agent to npm + │ + │ repository_dispatch: openrouter-agent-published + ▼ +.github/workflows/upstreamer-port.yaml + │ + ▼ +scripts/upstream + │ 1. fetch upstream, resolve target commit + │ 2. compare against .upstreamer/state.yaml — skip if unchanged + │ 3. opencode runs the port against .upstreamer/upstreamer.md + │ 4. .upstreamer/scripts/verify.sh (mechanical gate) + │ 5. .upstreamer/eval.md (parity gate, fresh context) + │ 6. advance state.yaml — ONLY if both gates pass + ▼ + Pull request (never a direct push to main) +``` + +A weekly cron backs up the dispatch in case one is missed, and +`workflow_dispatch` allows a manual run against any ref. + +## The contract is the product + +`.upstreamer/upstreamer.md` is the durable artifact — it defines scope, the +required public API, naming maps, permanent idiomatic divergences, and the +substrate pin. The ported source is an *output* of that contract. + +So when the port gets something wrong, **fix the contract**, not just the +generated code. A code-only fix gets re-broken on the next sync; a contract fix +holds. + +## Files + +| Path | What | +|------|------| +| `.upstreamer/upstreamer.md` | The rewrite contract. Binding. | +| `.upstreamer/state.yaml` | Last commit ported *and* verified *and* eval-passed. | +| `.upstreamer/scripts/verify.sh` | Mechanical gate: build, lint, types, tests, required API. | +| `.upstreamer/eval.md` | Parity gate: fresh-context behavioral review. | +| `.upstreamer/eval-report.md` | Latest eval result, or a bankruptcy report. | +| `.upstreamer/skills/upstreamer-converter/` | Execution discipline for the porting agent. | +| `.upstreamer/port.env` | Local secrets. **Gitignored.** | +| `scripts/upstream` | The wrapper. | + +## Two gates, and why state matters + +**Mechanical** (`verify.sh`) — objective: does it build, lint, type-check, pass +tests, and export every symbol the contract requires. + +**Parity** (`eval.md`) — judgment, run in a fresh context that reads the upstream +reference directly: does it actually *behave* like upstream. This is the gate that +catches a port which compiles cleanly while sitting a version behind on real +behavior. + +If either gate fails the run **declares bankruptcy**: `state.yaml` is left +untouched and `.upstreamer/eval-report.md` explains why. A stale state file is the +correct outcome for a failed port — it makes the next run retry the same delta +instead of skipping past the gap. The workflow labels such a PR `eval-failed` and +marks the title `do not merge`. + +Never hand-edit `state.yaml` to make a run look successful. + +## Running it locally + +```bash +# one-time +bun install -g opencode-ai # or: npm install -g opencode-ai +cp .upstreamer/port.env.example .upstreamer/port.env +# then edit .upstreamer/port.env and fill in OPENROUTER_API_KEY + OPENCODE_MODEL + +./scripts/upstream # sync if upstream changed +./scripts/upstream --force # re-run after editing the contract +./scripts/upstream --ref v0.8.0 # port a specific upstream ref +./scripts/upstream -- --print-logs # pass args through to opencode +``` + +`--force` is the escape hatch for a changed contract with unchanged upstream. +Expect to use it often while the contract is still settling. + +## Credentials + +Two values, same names locally and in CI: + +| Name | Where | What | +|------|-------|------| +| `OPENROUTER_API_KEY` | local: `.upstreamer/port.env` · CI: repo **secret** | `sk-or-…` key opencode uses for inference | +| `OPENCODE_MODEL` | local: `.upstreamer/port.env` · CI: repo **variable** | e.g. `openrouter/anthropic/claude-sonnet-latest` | + +The wrapper writes the key into `~/.local/share/opencode/auth.json` so headless +runs work without the interactive `opencode /connect` flow. + +This is a load-bearing SDK port behind a strict parity eval — use a strong coding +model. `OPENCODE_MODEL` overrides the `model:` field in the contract, so you can +change models without a code change. + +## Reviewing a port PR + +Review it as a *port*, not a normal diff: + +1. Check `.upstreamer/eval-report.md` first. If state did not advance, stop. +2. Read the upstream delta yourself for anything load-bearing — the tool loop, + state serialization, approval/HITL ordering, hooks, streaming. +3. Confirm new tests assert *upstream behavior*, not merely the port's own shape. +4. Any new naming mapping the run derived should be promoted into the contract's + naming table. diff --git a/README.md b/README.md index e60da53..8ef44ac 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,9 @@ This package builds on the official `openrouter` Python SDK. It does not reimplement HTTP, auth, retries, or model schemas; `call_model` sends requests through `client.beta.responses.send_async`, the same Responses API surface used by the TypeScript package. +> **This package is a port.** `@openrouter/agent` (TypeScript) is the reference +> spec; this repo is kept in sync automatically. See [PORTING.md](PORTING.md). + ## Install ```bash diff --git a/scripts/upstream b/scripts/upstream new file mode 100755 index 0000000..cb4f9a9 --- /dev/null +++ b/scripts/upstream @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +# Upstreamer wrapper, adapted from mountgram/upstreamer (MIT) for in-repo downstream. +# +# Difference from upstream tool: mountgram/upstreamer writes generated output to +# codebases//downstream/ inside the upstreamer repo. Here THIS repo is the +# downstream, so output is written in place at the repo root and reviewed as a PR. +# +# Usage: +# scripts/upstream # sync if upstream changed +# scripts/upstream --force # re-run even if unchanged (after editing the contract) +# scripts/upstream --ref v1.2.3 # port a specific upstream ref instead of origin/HEAD +# scripts/upstream -- --print-logs # pass extra args through to opencode +set -euo pipefail + +cd "$(dirname "$0")/.." +repo_root="$PWD" + +contract=".upstreamer/upstreamer.md" +state_file=".upstreamer/state.yaml" +eval_file=".upstreamer/eval.md" +eval_report_file=".upstreamer/eval-report.md" +verifier=".upstreamer/scripts/verify.sh" +env_file=".upstreamer/port.env" +work_dir="tmp/upstreamer" +upstream_dir="$work_dir/upstream" +log_dir=".upstreamer/logs" + +force=0 +ref="" +while [ "$#" -gt 0 ]; do + case "${1:-}" in + --force) force=1; shift ;; + --ref) ref="${2:-}"; shift 2 ;; + -h|--help) sed -n '4,12p' "$0"; exit 0 ;; + --) shift; break ;; + *) break ;; + esac +done + +[ -f "$contract" ] || { echo "ERROR: missing contract: $contract" >&2; exit 2; } + +# Local secrets/model choice. Never committed; CI supplies the same names as env. +# shellcheck disable=SC1090 +if [ -f "$env_file" ]; then set -a; . "./$env_file"; set +a; fi + +read_frontmatter_field() { + local field="$1" line line_number=0 value + while IFS= read -r line; do + line_number=$((line_number + 1)) + if [ "$line_number" -eq 1 ]; then [ "$line" = "---" ] || return 0; continue; fi + [ "$line" = "---" ] && return 0 + case "$line" in + "$field":*) + value="${line#"$field":}" + value="${value#"${value%%[![:space:]]*}"}" + value="${value%"${value##*[![:space:]]}"}" + value="${value%\"}"; value="${value#\"}" + value="${value%\'}"; value="${value#\'}" + printf '%s\n' "$value"; return 0 ;; + esac + done < "$contract" +} + +upstream_repo="$(read_frontmatter_field upstream)" +frontmatter_model="$(read_frontmatter_field model)" +[ -n "$upstream_repo" ] || { echo "ERROR: contract has no 'upstream:' field" >&2; exit 2; } + +# Model precedence: OPENCODE_MODEL env (or port.env) > contract frontmatter. +model="${OPENCODE_MODEL:-$frontmatter_model}" +model_args=() +[ -n "$model" ] && model_args=(--model "$model") + +# opencode reads credentials from auth.json. Seed it from OPENROUTER_API_KEY so +# headless CI and local runs work without the interactive /connect flow. +if [ -n "${OPENROUTER_API_KEY:-}" ]; then + auth_dir="${XDG_DATA_HOME:-$HOME/.local/share}/opencode" + mkdir -p "$auth_dir" + if [ ! -f "$auth_dir/auth.json" ] || ! grep -q '"openrouter"' "$auth_dir/auth.json" 2>/dev/null; then + printf '{"openrouter":{"type":"api","key":"%s"}}\n' "$OPENROUTER_API_KEY" > "$auth_dir/auth.json" + chmod 600 "$auth_dir/auth.json" + fi +fi + +mkdir -p "$work_dir" "$log_dir" +export TMPDIR="$repo_root/$work_dir" +log_file="$log_dir/$(date -u +%Y%m%dT%H%M%SZ).log" + +last_upstream_commit="" +if [ -f "$state_file" ]; then + last_upstream_commit="$(awk -F: '$1 == "upstream_commit" { gsub(/^[[:space:]]+|[[:space:]]+$/, "", $2); print $2; exit }' "$state_file")" +fi + +upstream_url="$upstream_repo" +case "$upstream_url" in + http://*|https://*|git@*) ;; + *) upstream_url="https://github.com/$upstream_url.git" ;; +esac + +if [ -d "$upstream_dir/.git" ]; then + git -C "$upstream_dir" remote set-url origin "$upstream_url" + git -C "$upstream_dir" fetch --tags --force origin +else + rm -rf "$upstream_dir" + git clone "$upstream_url" "$upstream_dir" +fi + +if [ -n "$ref" ]; then + target_commit="$(git -C "$upstream_dir" rev-parse "$ref^{commit}")" +else + target_commit="$(git -C "$upstream_dir" rev-parse origin/HEAD 2>/dev/null || git -C "$upstream_dir" rev-parse origin/main)" +fi +git -C "$upstream_dir" checkout -q --detach "$target_commit" + +echo "upstream: $upstream_repo" >&2 +echo "target: $target_commit${ref:+ ($ref)}" >&2 +echo "last port: ${last_upstream_commit:-none}" >&2 +echo "model: ${model:-}" >&2 + +if [ "$force" -eq 0 ] && [ "$target_commit" = "$last_upstream_commit" ]; then + { + echo "Run summary" + echo "===========" + echo "Upstream changes since last run: none." + echo "Downstream changes made: none." + echo "Upstream commit: $target_commit" + if [ -x "$verifier" ]; then + echo; echo "Running verifier: $verifier" + "$verifier" + fi + } 2>&1 | tee "$log_file" + exit "${PIPESTATUS[0]}" +fi + +command -v opencode >/dev/null 2>&1 || { echo "ERROR: opencode not installed" >&2; exit 127; } + +prompt_file="$work_dir/prompt.txt" +cat > "$prompt_file" <&2 + +if [ -n "${UPSTREAMER_TIMEOUT_SECONDS:-}" ]; then + perl -e 'alarm shift; exec @ARGV' "$UPSTREAMER_TIMEOUT_SECONDS" \ + opencode "${model_args[@]}" "$@" run "$prompt" 2>&1 | tee "$log_file" +else + opencode "${model_args[@]}" "$@" run "$prompt" 2>&1 | tee "$log_file" +fi From 0ca906aca59c82a620aad795e048976753d6027b Mon Sep 17 00:00:00 2001 From: LukasParke <5702154+LukasParke@users.noreply.github.com> Date: Fri, 24 Jul 2026 16:32:06 -0500 Subject: [PATCH 2/7] ci: make the port verifier advisory on PRs, blocking in sync The required-API check fails by design until the port catches up to upstream 0.8.0. A permanently-red required check on every unrelated PR just teaches people to ignore CI, so report it in the job summary instead. It stays blocking where it matters: inside scripts/upstream, where it gates whether .upstreamer/state.yaml advances. Also make the verifier's version check say when it skipped for lack of an upstream checkout, rather than passing silently. --- .github/workflows/ci.yaml | 35 +++++++++++++++++++++++++++++++---- .upstreamer/scripts/verify.sh | 4 +++- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6b31bd8..afaceca 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -41,8 +41,13 @@ jobs: - name: Tests run: uv run pytest tests/unit -q - # Runs the port's own mechanical gate on PRs that touch the port machinery, so - # a broken verifier or contract is caught here rather than mid-sync. + # Reports the port's own mechanical gate. Advisory here, BLOCKING inside the + # sync job (scripts/upstream) where it gates whether state.yaml advances. + # + # Advisory on purpose: the port is currently a minor version behind upstream, so + # the required-API check fails by design until the first sync lands. Making that + # a red required check on every unrelated PR just teaches people to ignore CI. + # The signal still shows up in the job summary. verify-port: runs-on: ubuntu-latest timeout-minutes: 15 @@ -50,5 +55,27 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/port-toolchain - - name: Port verifier - run: ./.upstreamer/scripts/verify.sh + - name: Port verifier (advisory) + id: verify + continue-on-error: true + run: | + set -o pipefail + ./.upstreamer/scripts/verify.sh 2>&1 | tee /tmp/verify.log + + - name: Summarize + if: always() + run: | + { + echo "## Port verifier" + echo + if [ "${{ steps.verify.outcome }}" = "success" ]; then + echo "Port is in sync with its parity floor." + else + echo "Parity gaps below. Expected until the port catches up to upstream —" + echo "advisory here, blocking inside the sync job." + fi + echo + echo '```' + cat /tmp/verify.log 2>/dev/null || echo "(no output)" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.upstreamer/scripts/verify.sh b/.upstreamer/scripts/verify.sh index 97e28f7..84a0302 100755 --- a/.upstreamer/scripts/verify.sh +++ b/.upstreamer/scripts/verify.sh @@ -77,7 +77,9 @@ if [ -f "$upstream_pkg" ]; then fail "version drift: pyproject.toml=$declared, upstream @openrouter/agent=$target" fi else - echo " SKIP: no upstream checkout to compare version against" + # Only present during a sync run. Standalone/CI invocations legitimately have no + # upstream checkout; not a failure, but say so rather than passing silently. + echo " SKIP: no upstream checkout — version parity unchecked (declared $declared)" fi echo From 4322e97a8f685df209526aaf39ca94c242b5d5e6 Mon Sep 17 00:00:00 2001 From: LukasParke <5702154+LukasParke@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:21:55 -0500 Subject: [PATCH 3/7] fix(upstream): let an explicit OPENROUTER_API_KEY override stored opencode auth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wrapper only seeded auth.json when no openrouter credential existed. Anyone who had previously run 'opencode /connect' would have their port.env key silently ignored and the run billed to the old key — close to undebuggable. An explicitly provided key now always wins. Other providers in auth.json are preserved rather than clobbered, and a corrupt auth.json recovers instead of failing the run. --- scripts/upstream | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/scripts/upstream b/scripts/upstream index cb4f9a9..ce561de 100755 --- a/scripts/upstream +++ b/scripts/upstream @@ -72,12 +72,35 @@ model_args=() # opencode reads credentials from auth.json. Seed it from OPENROUTER_API_KEY so # headless CI and local runs work without the interactive /connect flow. +# +# An explicitly-provided key always wins: if you put a key in port.env, that is +# the key that gets used, even when an openrouter credential already exists from +# a previous `opencode /connect`. Silently preferring the stored one makes +# "why is it still using the old key" nearly impossible to debug. +# +# Other providers in auth.json are preserved — only the openrouter entry is +# replaced. if [ -n "${OPENROUTER_API_KEY:-}" ]; then auth_dir="${XDG_DATA_HOME:-$HOME/.local/share}/opencode" + auth_file="$auth_dir/auth.json" mkdir -p "$auth_dir" - if [ ! -f "$auth_dir/auth.json" ] || ! grep -q '"openrouter"' "$auth_dir/auth.json" 2>/dev/null; then - printf '{"openrouter":{"type":"api","key":"%s"}}\n' "$OPENROUTER_API_KEY" > "$auth_dir/auth.json" - chmod 600 "$auth_dir/auth.json" + if OPENROUTER_API_KEY="$OPENROUTER_API_KEY" AUTH_FILE="$auth_file" python3 - <<'PYAUTH' +import json, os, pathlib +path = pathlib.Path(os.environ["AUTH_FILE"]) +try: + data = json.loads(path.read_text()) + if not isinstance(data, dict): + data = {} +except Exception: + data = {} +data["openrouter"] = {"type": "api", "key": os.environ["OPENROUTER_API_KEY"]} +path.write_text(json.dumps(data, indent=2) + "\n") +path.chmod(0o600) +PYAUTH + then + echo "auth: openrouter credential set from OPENROUTER_API_KEY" >&2 + else + echo "WARNING: could not write $auth_file; falling back to whatever opencode has stored" >&2 fi fi From 66b5c3256b4c9df3bbc7092794886898205d6959 Mon Sep 17 00:00:00 2001 From: Luke Parke <5702154+LukasParke@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:49:33 -0500 Subject: [PATCH 4/7] fix: three CI-breaking issues found in the first local port run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Model id: openrouter/anthropic/claude-sonnet-latest does not exist on OpenRouter; opencode resolves the alias as ~anthropic/claude-sonnet-latest. Fixed in the contract frontmatter, port.env.example, PORTING.md, and the workflow comment. Set the OPENCODE_MODEL repo variable with the ~ form. - Committed opencode.json denying external_directory: without it, the model writing any scratch file outside the workspace (observed: /tmp/hookspec) trips an interactive permission ask that a headless `opencode run` can never answer — the run wedges silently until the job timeout. Deny makes opencode refuse the write so the agent course-corrects. - Timeouts: job timeout 300→150 min and inner wrapper alarm 14400→7200 s, so a wedged run dies with logs intact (inner alarm first) and stops blocking the upstreamer-port concurrency group for 5 hours. The full 0.7.2→0.8.0 jumbo delta completed locally in ~110 min; routine single-release deltas are far smaller. Verified locally: with these fixes the 0.7.2→0.8.0 python port ran end to end, flipped the verifier to PASS, and passed the parity eval on its second attempt (PASS WITH WARNINGS). Co-Authored-By: Claude Fable 5 --- .github/workflows/upstreamer-port.yaml | 6 +++--- .upstreamer/port.env.example | 2 +- .upstreamer/upstreamer.md | 2 +- PORTING.md | 2 +- opencode.json | 6 ++++++ 5 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 opencode.json diff --git a/.github/workflows/upstreamer-port.yaml b/.github/workflows/upstreamer-port.yaml index 2c8df16..0527bac 100644 --- a/.github/workflows/upstreamer-port.yaml +++ b/.github/workflows/upstreamer-port.yaml @@ -37,7 +37,7 @@ concurrency: jobs: port: runs-on: ubuntu-latest - timeout-minutes: 300 + timeout-minutes: 150 steps: - uses: actions/checkout@v4 with: @@ -55,10 +55,10 @@ jobs: env: # Provide these in repo settings: # Secret OPENROUTER_API_KEY — sk-or-... key opencode uses for inference - # Variable OPENCODE_MODEL — e.g. openrouter/anthropic/claude-sonnet-latest + # Variable OPENCODE_MODEL — e.g. openrouter/~anthropic/claude-sonnet-latest OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} OPENCODE_MODEL: ${{ vars.OPENCODE_MODEL }} - UPSTREAMER_TIMEOUT_SECONDS: 14400 + UPSTREAMER_TIMEOUT_SECONDS: 7200 run: | set -euo pipefail if [ -z "${OPENROUTER_API_KEY:-}" ]; then diff --git a/.upstreamer/port.env.example b/.upstreamer/port.env.example index 61bf379..01d8578 100644 --- a/.upstreamer/port.env.example +++ b/.upstreamer/port.env.example @@ -17,7 +17,7 @@ OPENROUTER_API_KEY= # # This is a load-bearing SDK port with a strict parity eval — use a strong coding # model. Suggested starting point: -OPENCODE_MODEL=openrouter/anthropic/claude-sonnet-latest +OPENCODE_MODEL=openrouter/~anthropic/claude-sonnet-latest # Optional: hard wall-clock cap for one run, in seconds. A full-surface # reconciliation is much slower than an incremental delta port. diff --git a/.upstreamer/upstreamer.md b/.upstreamer/upstreamer.md index 7d609ab..af87d1f 100644 --- a/.upstreamer/upstreamer.md +++ b/.upstreamer/upstreamer.md @@ -1,7 +1,7 @@ --- upstream: OpenRouterTeam/typescript-agent downstream: OpenRouterTeam/python-agent -model: openrouter/anthropic/claude-sonnet-latest +model: openrouter/~anthropic/claude-sonnet-latest --- # Python Port Contract — `@openrouter/agent` → `openrouter-agent` diff --git a/PORTING.md b/PORTING.md index 66296c0..6b22c92 100644 --- a/PORTING.md +++ b/PORTING.md @@ -96,7 +96,7 @@ Two values, same names locally and in CI: | Name | Where | What | |------|-------|------| | `OPENROUTER_API_KEY` | local: `.upstreamer/port.env` · CI: repo **secret** | `sk-or-…` key opencode uses for inference | -| `OPENCODE_MODEL` | local: `.upstreamer/port.env` · CI: repo **variable** | e.g. `openrouter/anthropic/claude-sonnet-latest` | +| `OPENCODE_MODEL` | local: `.upstreamer/port.env` · CI: repo **variable** | e.g. `openrouter/~anthropic/claude-sonnet-latest` | The wrapper writes the key into `~/.local/share/opencode/auth.json` so headless runs work without the interactive `opencode /connect` flow. diff --git a/opencode.json b/opencode.json new file mode 100644 index 0000000..469ee5e --- /dev/null +++ b/opencode.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://opencode.ai/config.json", + "permission": { + "external_directory": "deny" + } +} From 8467f1bac282b101667906055ed2df5ef55f754f Mon Sep 17 00:00:00 2001 From: Luke Parke <5702154+LukasParke@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:15:20 -0500 Subject: [PATCH 5/7] port: sync with @openrouter/agent 0.8.0 (680bceb) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generated by the Upstreamer pipeline in this PR, run locally end to end (scripts/upstream --ref @openrouter/agent@0.8.0). Both gates passed: mechanical verifier === PASS: 0 failures ===, parity eval PASS WITH WARNINGS on the second pass (.upstreamer/eval-report.md). Ports the 0.7.2 -> 0.8.0 delta: lifecycle hooks system (HooksManager + nine built-in hooks), versioned conversation-state serialization, awaiting_client_tools for unresolved manual tool calls, default-on allow_final_response with DEFAULT_FINAL_RESPONSE_DIRECTIVE, strict_final_response / empty-final-retry tolerance, and MCP tool-result source discrimination. state.yaml advances to 680bceb, so the first CI run after merge is a no-op until the next upstream release. First-pass eval FAILed on three real findings (Stop-hook force_resume was not a zero-cost retry, MCP branding surface missing, thin hooks test coverage); all three fixed and re-verified in the second pass — see the eval report for the traced evidence. Co-Authored-By: Claude Fable 5 --- .upstreamer/eval-report.md | 267 +++++ .upstreamer/state.yaml | 12 +- README.md | 22 +- pyproject.toml | 2 +- src/openrouter_agent/__init__.py | 84 +- src/openrouter_agent/async_params.py | 4 + src/openrouter_agent/call_model.py | 5 + src/openrouter_agent/conversation_state.py | 154 ++- src/openrouter_agent/hooks_emit.py | 195 ++++ src/openrouter_agent/hooks_manager.py | 198 ++++ src/openrouter_agent/hooks_matchers.py | 25 + src/openrouter_agent/hooks_resolve.py | 46 + src/openrouter_agent/hooks_schemas.py | 184 ++++ src/openrouter_agent/hooks_types.py | 128 +++ src/openrouter_agent/model_result.py | 917 ++++++++++++++---- src/openrouter_agent/reusable_stream.py | 7 + src/openrouter_agent/tool.py | 10 + src/openrouter_agent/tool_executor.py | 26 +- src/openrouter_agent/tool_types.py | 25 + tests/unit/test_allow_final_response.py | 142 +++ tests/unit/test_call_model.py | 5 +- .../test_conversation_state_serialization.py | 121 +++ tests/unit/test_hooks_manager.py | 200 ++++ tests/unit/test_hooks_matchers.py | 37 + tests/unit/test_manual_tool_pending_state.py | 250 +++++ tests/unit/test_mcp_tool_branding.py | 91 ++ tests/unit/test_model_result_hooks.py | 439 +++++++++ tests/unit/test_resume_string_input.py | 74 ++ tests/unit/test_tool_terminal_empty_final.py | 209 ++++ upstreamer-changelog.md | 14 +- uv.lock | 16 +- 31 files changed, 3700 insertions(+), 209 deletions(-) create mode 100644 .upstreamer/eval-report.md create mode 100644 src/openrouter_agent/hooks_emit.py create mode 100644 src/openrouter_agent/hooks_manager.py create mode 100644 src/openrouter_agent/hooks_matchers.py create mode 100644 src/openrouter_agent/hooks_resolve.py create mode 100644 src/openrouter_agent/hooks_schemas.py create mode 100644 src/openrouter_agent/hooks_types.py create mode 100644 tests/unit/test_allow_final_response.py create mode 100644 tests/unit/test_conversation_state_serialization.py create mode 100644 tests/unit/test_hooks_manager.py create mode 100644 tests/unit/test_hooks_matchers.py create mode 100644 tests/unit/test_manual_tool_pending_state.py create mode 100644 tests/unit/test_mcp_tool_branding.py create mode 100644 tests/unit/test_model_result_hooks.py create mode 100644 tests/unit/test_resume_string_input.py create mode 100644 tests/unit/test_tool_terminal_empty_final.py diff --git a/.upstreamer/eval-report.md b/.upstreamer/eval-report.md new file mode 100644 index 0000000..95e04b6 --- /dev/null +++ b/.upstreamer/eval-report.md @@ -0,0 +1,267 @@ +PASS WITH WARNINGS + +# Port Parity Eval — Python (`@openrouter/agent` 0.7.2 → 0.8.0), second pass + +Fresh re-review against upstream commit `680bceb4598f228d3e2ec58e2416e4335cdff059`, +delta baseline `adc7939f4b7ed85b1a060d13433b8be6063cff73`, scoped to `packages/agent/`. +This is a second-pass review after a prior FAIL with three findings. All three +are verified fixed below by reading the code directly (not by trusting the +converter's summary), and a broader sweep of the rest of the delta and the +eval's Required Qualities turned up no new material gap — only pre-existing, +cosmetic items. + +## Verdict rationale + +All three prior findings are genuinely fixed, confirmed by tracing execution +and by breaking the fix and watching the test fail. No required public API +symbol is missing, no load-bearing loop / state / approval-ordering / hooks +behavior regressed, and the declared version (`0.8.0`) matches what's +actually ported. Remaining items are documentation/coverage gaps that don't +change behavior — hence WARNINGS, not FAIL. + +## Finding 1 (prior FAIL #1) — Stop hook `force_resume` zero-cost retry: FIXED, verified + +`src/openrouter_agent/model_result.py:677-694`, inside `_run()`: + +```python +while stop_conditions and await is_stop_condition_met(stop_conditions, self._steps): + stop_decision = await self._run_stop_hook(force_resume_count, current_request) + if stop_decision == "resume": + force_resume_count += 1 + continue + session_end_reason = "max_turns" + stopped_by_stop_when = True + break +``` + +- `is_stop_condition_met` (`stop_conditions.py`) is a pure function over + `self._steps` (already-collected `StepResult`s) — it does not call `_send` + or `_send_and_track`. The `continue` re-enters the `while` and re-evaluates + this pure check against the *same* steps; no request is sent on the + "resume" path. This is a faithful match of upstream's + `model-result.ts:3050-3068` (`shouldStopExecution()` / `runStopHook()` / + `continue`), confirmed by reading both side by side. +- `tests/unit/test_model_result_hooks.py::test_stop_hook_force_resume_is_a_zero_cost_retry_no_extra_model_request` + genuinely pins this: the mock `QueuedResponses` has exactly one queued + response, and asserts `len(client.beta.responses.requests) == 1` after a + run in which the Stop hook is invoked twice (once forcing resume, once + not). If the "resume" branch sent a real request, the mock's second + `pop(0)` would raise `IndexError` on the empty list. +- I verified this is a real regression guard, not an accidental pass: I + temporarily replaced the `continue` branch with a version that calls + `await self._send_and_track(...)` (simulating the old bug), reran the test, + and both `test_stop_hook_force_resume_is_a_zero_cost_retry_no_extra_model_request` + and `test_stop_hook_force_resume_then_falls_through_to_normal_tool_round` + failed immediately with `IndexError: pop from empty list`. Reverted + afterward; `git diff --stat` confirms the file is back to its pre-sabotage + state and the tests pass again. +- Minor cosmetic nit: the second test's docstring says the halted round's + tool calls "execute via the normal round path (not the final-directive + coercion)" — tracing the code, they actually go through the *same* + final-directive-coercion branch (`stopped_by_stop_when` → `resolvable_pending` + execute → `tool_choice: "none"` follow-up), which is also what upstream does + (`model-result.ts:3205+`). The assertions themselves (text output, request + count) are correct; only the prose comment is a little misleading. Not a + behavioral gap. + +## Finding 2 (prior FAIL #2) — MCP tool-branding surface: FIXED, verified + +- `tool.py:57-64` — `mark_mcp()` returns `{**tool_to_mark, "_mcp": True}`, a + new dict; the underlying `"function"` value is not copied (shallow), same + as upstream's `{...toolToMark, _mcp: true}`. Confirmed non-mutating via + `tests/unit/test_mcp_tool_branding.py::test_mark_mcp_is_non_mutating_and_is_mcp_tool_detects_the_brand` + (`base is not branded`, `branded["function"] is base["function"]`). +- `tool_types.py:264-270` — `McpBranded` alias and `is_mcp_tool()` structural + check (`tool.get("_mcp") is True`), matching upstream's `isMcpTool`. +- `tool_executor.py` — `source = "mcp" if is_mcp_tool(tool) else "client"` is + computed and threaded through all three execute paths + (`execute_regular_tool`, `execute_generator_tool`, `execute_hitl_tool`, + lines 62/86/151) and returned in every result dict (success and error + branches). +- `model_result.py:825, 852` — the `tool.result` / `tool_result` stream event + carries `"source"`, preferring the executor-computed `result.get("source", ...)` + and falling back to `is_mcp_tool(tool)` for the parse-error path where no + executor result exists. +- `test_mcp_tool_branding.py::test_mcp_branded_tool_result_carries_source_mcp_in_tool_result_event` + and `test_regular_client_tool_result_carries_source_client` both drive + `call_model(...)` end-to-end through a mocked Responses API and assert + `source` on the actual `tool_result` event from `get_tool_stream()` — this + is genuine parity coverage, not a unit-level stub check. + +## Finding 3 (prior FAIL #3) — Over-exported hook internals: FIXED, verified + +- `src/openrouter_agent/__init__.py` — grepped the import block and `__all__`: + `BUILT_IN_HOOKS`, `BUILT_IN_HOOK_NAMES`, `matches_tool`, `resolve_hooks`, + `execute_handler_chain` appear nowhere in either. +- No wildcard (`import *`) re-export exists anywhere in `__init__.py` that + could smuggle these back in. `resolve_hooks` is used internally by + `call_model.py` via `from .hooks_resolve import resolve_hooks` but is never + re-exported from the package `__init__`. +- `.upstreamer/scripts/verify.sh`'s `REQUIRED_SYMBOLS` list was inspected — + it does not reference any of the five removed names, so the mechanical + verifier's required-symbol check is unaffected by the removal. +- This matches upstream's own `index.ts`, which explicitly documents (in a + code comment added in this exact delta) that `matchesTool`, `resolveHooks`, + `BUILT_IN_HOOKS`, and raw schema objects are deliberately not exported. + +## Finding 4 (prior FAIL #4) — Thin hook test coverage: FIXED, verified + +Read all four new tests in `tests/unit/test_model_result_hooks.py` in full: + +- `test_user_prompt_submit_can_reject_a_string_prompt` — asserts the raised + `ValueError` message and, critically, `len(client.beta.responses.requests) == 0`, + proving the model was never called after rejection (not just that an + exception was raised somewhere). +- `test_user_prompt_submit_can_mutate_a_string_prompt` — asserts the + *actually-sent* request's `input[0]["content"]` contains the mutated text, + reading `client.beta.responses.requests[0]`, i.e., proving the mutation + reached the wire request, not just the hook's return value. +- `test_user_prompt_submit_mutates_last_user_message_in_array_input` — same, + for array-shaped `input`, asserting `sent_input[-1]["content"]`. +- `test_pre_tool_use_mutated_input_actually_reaches_tool_execute` — the tool's + `execute` callback records its received `params` into a list; the test + asserts the *tool implementation actually observed* the hook's + `mutated_input`. I traced the plumbing: `HookEntry` mutation config + (`hooks_types.py:117`, `mutations={"mutated_input": "tool_input"}`) maps the + hook's `mutated_input` return key onto `final_payload["tool_input"]`, which + `model_result.py:492-493` reads via `pre.final_payload.get("tool_input")` + and substitutes into `effective_call.arguments` before `execute_tool()` is + called. This is a genuine end-to-end proof, not an API-surface check. + +All four exercise actual behavior with concrete, falsifiable assertions +(request contents, request counts, executed-tool arguments) — not "call and +check nothing crashed." + +## Full battery re-run + +- `bash .upstreamer/scripts/verify.sh` → `PASS: 0 failures` (uv sync, ruff + check, ruff format, mypy, pytest tests/unit, all 31 required public-API + symbols present, version `0.8.0` matches upstream `package.json`, no + leaked TS artifacts, all repo-owned files present). +- `uv run pytest tests/unit tests/e2e -q` → `102 passed` (e2e tests skip + cleanly without `OPENROUTER_API_KEY`, per contract). + +## Broader sweep beyond the four claimed fixes + +Re-read the full upstream delta file list +(`git -C tmp/upstreamer/upstream diff --name-status adc7939..680bceb -- packages/agent/`) +and spot-checked every source file not already covered above: + +- **`conversation-state.ts` (versioned serialization, upstream #66)** — fully + present and correct in `conversation_state.py`: `CONVERSATION_STATE_VERSION`, + `InvalidStateError`, `UnsupportedStateVersionError`, + `serialize_conversation_state` / `deserialize_conversation_state` with the + same version-check-before-structural-validation ordering, same required + fields (`id`, `messages`, `status`), same "absence of `version` means v1" + policy. Covered by `tests/unit/test_conversation_state_serialization.py`. +- **`call-model.ts` / `async-params.ts`** — `strict_final_response` and + `hooks` (via `resolve_hooks`) are both threaded through `call_model.py` and + `async_params.py`'s reserved-key list, matching upstream's added fields. +- **`tool.ts` / `tool-types.ts` (the other ~85% of their diffs)** — almost + entirely TypeScript generic-variance engineering (`TContext` → `TCtx` / + `ContextFromSchema`, `bivarianceHack` method-syntax tricks so concretely + typed tools stay assignable to the wide `Tool` union). Zero runtime + behavior attached; correctly not chased in Python per Idiomatic Divergence + #4 ("Typing is looser... do not contort the runtime to chase a type-level + feature"). +- **`reusable-stream.ts`** — adds a `get isComplete()` getter. Not part of + the Required Public API list and not load-bearing (it's a cache-hit + optimization signal); not found ported 1:1 in `reusable_stream.py`, but + this is pre-existing from before this delta's scope and cosmetic (no + behavioral test depends on it upstream either). +- **`tool-orchestrator.ts` (`executeToolLoop`)** — upstream added a `source` + field here too, but this function is dead code in upstream itself: it is + not exported from `index.ts` and not imported by any other upstream + source file (only mentioned in a code *comment* in one test). The port's + `tool_orchestrator.py` has always been (since the original 0.7.2 port, not + this delta) a thin re-export shim (`ModelResult`, `execute_tool`, + `partition_tool_calls`) rather than a translation of `executeToolLoop`, so + there's no `source`-field gap to fix — the shim never had the field to + begin with, and the real tool-execution path (`model_result.py` + + `tool_executor.py`) does have `source` correctly threaded (see Finding 2). + Not a new gap; pre-existing and inconsequential since the dead code isn't + reachable either upstream or downstream. +- **Hooks system files** (`hooks-emit.ts`, `hooks-manager.ts`, + `hooks-matchers.ts`, `hooks-resolve.ts`, `hooks-schemas.ts`, + `hooks-types.ts`) — all six have Python counterparts + (`hooks_emit.py`, `hooks_manager.py`, `hooks_matchers.py`, + `hooks_resolve.py`, `hooks_schemas.py`, `hooks_types.py`). Spot-checked + `hooks_matchers.py` against `hooks-matchers.ts` line by line: `None`/wildcard, + exact string, compiled regex, and predicate-callable branches all match + (Python's `re.Pattern` has no `lastIndex`-style statefulness, so the + upstream comment about resetting `RegExp.lastIndex` has no Python + equivalent bug to guard against — correctly omitted, not a gap). + Session-id-per-emit threading (`hooks_manager.py:51-60`) is present and + tested (`test_hooks_manager.py::test_session_id_threads_per_emit_for_shared_manager`). +- **SessionEnd/drain on no-tools error paths** — behaviorally verified via a + throwaway script: a transport that raises on `send_async` for a *no-tools* + `call_model` still fires `SessionStart` then `SessionEnd` with + `reason: "error"` before the exception propagates (the `try/except/finally` + wrapping the entire `_run()` body guarantees this uniformly for tool and + no-tool paths alike). This is correct but has no dedicated committed test + pinning it — a pre-existing coverage gap, not a regression from this delta. +- **"Tool executes exactly once per round" regression** (upstream's + `tool-execution-once.test.ts`, guarding against a historical bug where + `handleApprovalCheck` pre-executed auto-approve tools and the main loop + re-executed them) — verified behaviorally via a throwaway script with a + mixed auto-tool + approval-gated-tool round and a `PermissionRequest` + `"allow"` decision: each tool's `execute` fired exactly once. Correct, but + there's no dedicated Python test file pinning this specific historical + regression by name — again a coverage gap, not a behavioral gap. +- **Approval/HITL resume ordering** (`function_call` before + `function_call_output`) — already covered by pre-existing + `test_parity_requirements.py::test_approval_pause_persists_tool_call_turn_and_resume_orders_output_after_call` + and `test_hitl_pause_persists_tool_call_turn_and_resume_orders_output_after_call`, + and by the new `test_manual_tool_pending_state.py::test_mixed_auto_and_manual_round_persists_auto_output_and_pauses_manual` + for the specific "auto output recorded before the pause" mixed-round case + this delta's contract calls out. No regression found. + +## Documentation gap (new finding, cosmetic) + +`upstreamer-changelog.md` was not updated for this 0.7.2 → 0.8.0 sync. It +still reads "Latest Sync ... against the current `@openrouter/agent` 0.7.2 +surface" and has no entry for the hooks system (upstream #7/#67, explicitly +called "the 0.8.0 headline" in the contract), MCP tool branding, versioned +conversation-state serialization (upstream #66), the `allow_final_response` +default-on `tool_choice: "none"` behavior change (upstream #68), or +`awaiting_client_tools` (upstream #64) — all of which *are* correctly +reflected in `README.md` (diff confirms new "Lifecycle Hooks" section, +updated Stop Conditions section, and a new manual-tools paragraph). The +contract's Output Shape section lists `upstreamer-changelog.md` alongside +`README.md` as this repo's own product surface / user-facing port notes; +leaving it stale after a headline feature release is a real (if purely +documentation-level) gap. Does not affect behavior, the public API, or any +Required Quality in eval.md — hence a warning, not a blocker. + +## Idiomatic Divergences section check + +The Stop-hook zero-cost-retry semantics are not a divergence — the port now +matches upstream's exact `runStopHook`/`shouldStopExecution` loop structure +(see Finding 1), so nothing needs to be added to +`.upstreamer/upstreamer.md`'s Idiomatic Divergences section for it. No other +undocumented divergence was found in this sweep; the existing six divergences +(Pydantic v2, async-first, generator tools, looser typing, Python-native +cancellation, structural protocols for `BeforeCreateRequestContext`) still +accurately describe the only permanent deltas from upstream. + +## Summary + +- Public API completeness: PASS (31/31 required symbols, plus the full + hooks/MCP-branding/versioned-state surface added in this delta). +- Version honesty: PASS (`0.8.0` == upstream `package.json` at target commit). +- Load-bearing loop: PASS, including the fixed Stop-hook zero-cost retry. +- Streaming / State / Approval-HITL ordering / Hooks / Compatibility helpers: + PASS, no regressions found beyond the four already-fixed items. +- Repo-owned files: PASS (LICENSE, README.md, pyproject.toml, scripts/upstream + all present and not clobbered). +- Warnings: `upstreamer-changelog.md` stale (documentation only); a few + historical-regression behaviors (SessionEnd-on-error, exactly-once tool + execution) are behaviorally correct but lack dedicated pinning tests; one + test docstring (`test_stop_hook_force_resume_then_falls_through_to_normal_tool_round`) + slightly mischaracterizes which code path runs, though its assertions are + correct; `reusable_stream.py` lacks upstream's new `is_complete` property + (non-load-bearing). + +**Verdict: PASS WITH WARNINGS.** All three prior FAIL findings are +substantively and verifiably fixed — not just superficially patched. The +remaining items are documentation or coverage nits with no behavioral impact. diff --git a/.upstreamer/state.yaml b/.upstreamer/state.yaml index 793b4a6..af2336a 100644 --- a/.upstreamer/state.yaml +++ b/.upstreamer/state.yaml @@ -1,6 +1,10 @@ # Last upstream commit successfully ported AND verified AND eval-passed. -# Seeded at @openrouter/agent@0.7.2 (commit adc7939), the release these ports -# were generated from, so the first run is an incremental 0.7.2 -> 0.8.0 delta -# rather than a full from-scratch regeneration. +# 680bceb ported the 0.7.2 -> 0.8.0 delta: the lifecycle hooks system +# (HooksManager, PreToolUse/PostToolUse/PostToolUseFailure/UserPromptSubmit/ +# Stop/PermissionRequest/SessionStart/SessionEnd/PostModelCall), versioned +# ConversationState serialization, awaiting_client_tools for unresolved +# manual tool calls, default-on allow_final_response with +# DEFAULT_FINAL_RESPONSE_DIRECTIVE, strict_final_response / empty-final-retry +# tolerance, and MCP tool-result source discrimination. # Only scripts/upstream runs should change this. -upstream_commit: adc7939f4b7ed85b1a060d13433b8be6063cff73 +upstream_commit: 680bceb4598f228d3e2ec58e2416e4335cdff059 diff --git a/README.md b/README.md index 8ef44ac..7d9e1b9 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,10 @@ web = server_tool({"type": "web_search_2025_08_26", "max_results": 5}) Use `require_approval` on a tool or `require_approval` on the request to pause sensitive calls before execution. Approval resume requires a state accessor with async `load()` and `save()` methods. +Manual tools (`execute=False`, no `on_tool_called`) pause the loop with status `"awaiting_client_tools"` when the model calls them, instead of silently dropping the call. Read the unresolved calls via `get_pending_tool_calls()` / `get_state()`, execute them yourself, and continue by calling `call_model` again with `function_call_output` items in `input`. + +For durable cross-process storage, serialize state with `serialize_conversation_state` / `deserialize_conversation_state` rather than storing raw dataclass fields. The wire format is versioned (`CONVERSATION_STATE_VERSION`); a version mismatch raises `UnsupportedStateVersionError` and malformed JSON raises `InvalidStateError`, so a store can never silently misinterpret a future shape. + Tool context is kept outside the model transcript. Provide a context mapping with per-tool keys and optional `shared` state. Tool execution receives `ctx["local"]`, `ctx["shared"]`, `ctx["set_context"]`, and `ctx["set_shared_context"]`. ```python @@ -151,6 +155,22 @@ result = call_model( ) ``` +## Lifecycle Hooks + +Pass a `HooksManager` (or an inline `{hook_name: [HookEntry(...)]}` dict of built-in hooks) via `hooks=` to observe or intervene in a run: `PreToolUse`, `PostToolUse`, `PostToolUseFailure`, `UserPromptSubmit`, `Stop`, `PermissionRequest`, `SessionStart`, `SessionEnd`, and `PostModelCall`. Handlers receive a validated payload dict and a `LifecycleHookContext` (`session_id`, `hook_name`, `cancel_event`). + +```python +from openrouter_agent import HookEntry, HookName, HooksManager + +hooks = HooksManager() +hooks.on(HookName.PreToolUse.value, HookEntry(handler=lambda payload, ctx: None, matcher="delete_file")) +hooks.on(HookName.SessionEnd.value, HookEntry(handler=lambda payload, ctx: print(payload["total_usage"]))) + +result = call_model(client, {"model": model, "input": prompt, "tools": tools, "hooks": hooks}) +``` + +`SessionStart` fires once per run with a config summary; `SessionEnd` fires once with aggregated `total_usage` (summed across every `PostModelCall`) and is guaranteed to fire — and any pending async hook work drained — even when a no-tools stream raises. `PreToolUse` can block a call (`{"block": "reason"}`) or mutate its input (`{"mutated_input": {...}}`); `PermissionRequest` can pre-empt the approval gate with `{"decision": "allow" | "deny" | "ask_user"}`; `Stop` can force the loop to keep going past a `stop_when` hit with `{"force_resume": True, "append_prompt": "..."}`. A `HooksManager` instance is safe to share across concurrent `call_model` runs — session identity is threaded per emit, not stored as manager-level mutable state. + ## Stop Conditions The built-ins mirror the TypeScript package and OR together when provided as a list: @@ -161,7 +181,7 @@ The built-ins mirror the TypeScript package and OR together when provided as a l - `max_cost(dollars)` - `finish_reason_is(reason)` -Set `allow_final_response=True` or a string to ask for one final no-tools turn when a stop condition fires on a tool-call turn. +When a stop condition fires while the model is still emitting tool calls, `call_model` makes one more turn with `tool_choice="none"` by default (tools stay in the request so the prompt-cache prefix survives) so the run ends with a natural-language answer. `allow_final_response` tunes this: `True` or omitted appends `DEFAULT_FINAL_RESPONSE_DIRECTIVE` as a user message, a non-empty string replaces the wording, `""` appends nothing, and `False` disables the extra turn entirely. ## Format Compatibility diff --git a/pyproject.toml b/pyproject.toml index 10e1615..babd814 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "openrouter-agent" -version = "0.7.2" +version = "0.8.0" description = "Python port of @openrouter/agent: OpenRouter tool orchestration, streaming, state, and format compatibility." readme = "README.md" requires-python = ">=3.9.2" diff --git a/src/openrouter_agent/__init__.py b/src/openrouter_agent/__init__.py index b7e075f..5df8a08 100644 --- a/src/openrouter_agent/__init__.py +++ b/src/openrouter_agent/__init__.py @@ -65,15 +65,54 @@ from .claude_constants import ClaudeContentBlockType, NonClaudeMessageRole from .claude_type_guards import is_claude_style_messages from .conversation_state import ( + CONVERSATION_STATE_VERSION, + InvalidStateError, + UnsupportedStateVersionError, append_to_messages, create_initial_state, create_rejected_result, create_unsent_result, + deserialize_conversation_state, generate_conversation_id, partition_tool_calls, + serialize_conversation_state, tool_requires_approval, update_state, ) +from .hooks_manager import HooksManager +from .hooks_schemas import ( + HookDefinition, + HookName, + HookRegistry, + ModelCallUsage, + PermissionRequestPayload, + PermissionRequestResult, + PostModelCallPayload, + PostToolUseFailurePayload, + PostToolUsePayload, + PreToolUsePayload, + PreToolUseResult, + SessionEndPayload, + SessionStartPayload, + SessionUsageTotals, + StopPayload, + StopResult, + UserPromptSubmitPayload, + UserPromptSubmitResult, +) +from .hooks_types import ( + DEFAULT_ASYNC_TIMEOUT_MS, + HOOK_BEHAVIOR, + AsyncOutput, + EmitResult, + HookBehavior, + HookEntry, + HookHandler, + InlineHookConfig, + LifecycleHookContext, + ToolMatcher, + is_async_output, +) from .item_types import ( AssistantMessageItem, CallFileSearchItem, @@ -90,7 +129,7 @@ SystemMessageItem, UserMessageItem, ) -from .model_result import GetResponseOptions, ModelResult +from .model_result import DEFAULT_FINAL_RESPONSE_DIRECTIVE, GetResponseOptions, ModelResult from .next_turn_params import ( apply_next_turn_params_to_request, build_next_turn_params_context, @@ -111,7 +150,7 @@ get_unsupported_content_summary, has_unsupported_content, ) -from .tool import server_tool, tool +from .tool import mark_mcp, server_tool, tool from .tool_context import ContextInput, ToolContextStore, build_tool_execute_context from .tool_event_broadcaster import ToolEventBroadcaster from .tool_types import ( @@ -129,6 +168,7 @@ InferToolOutput, InferToolOutputsUnion, ManualTool, + McpBranded, NextTurnParamsContext, NextTurnParamsFunctions, ParsedToolCall, @@ -172,6 +212,7 @@ is_generator_tool, is_hitl_tool, is_manual_tool, + is_mcp_tool, is_regular_execute_tool, is_server_tool, is_tool_call_output_event, @@ -202,11 +243,13 @@ def before_create_request(self, hook_ctx: BeforeCreateRequestContext, request: A "AfterSuccessContext", "AfterSuccessHook", "AssistantMessageItem", + "AsyncOutput", "BaseInputsUnion", "BeforeCreateRequestContext", "BeforeCreateRequestHook", "BeforeRequestContext", "BeforeRequestHook", + "CONVERSATION_STATE_VERSION", "CallFileSearchItem", "CallFunctionToolItem", "CallImageGenerationItem", @@ -220,11 +263,14 @@ def before_create_request(self, hook_ctx: BeforeCreateRequestContext, request: A "ClientTool", "ConversationState", "ConversationStatus", + "DEFAULT_ASYNC_TIMEOUT_MS", + "DEFAULT_FINAL_RESPONSE_DIRECTIVE", "DeveloperMessageItem", "EasyInputMessage", "EasyInputMessageContentInputImage", "EasyInputMessageContentUnion1", "EasyInputMessageRoleUnion", + "EmitResult", "EnhancedResponseStreamEvent", "ErrorEvent", "ErrorItem", @@ -235,14 +281,23 @@ def before_create_request(self, hook_ctx: BeforeCreateRequestContext, request: A "GetResponseOptions", "HITLTool", "HITLToolFunction", + "HOOK_BEHAVIOR", "HasApprovalTools", "Hook", + "HookBehavior", "HookContext", + "HookDefinition", + "HookEntry", + "HookHandler", + "HookName", + "HookRegistry", + "HooksManager", "InferToolEvent", "InferToolEventsUnion", "InferToolInput", "InferToolOutput", "InferToolOutputsUnion", + "InlineHookConfig", "InputAudio", "InputFile", "InputImage", @@ -250,8 +305,12 @@ def before_create_request(self, hook_ctx: BeforeCreateRequestContext, request: A "InputText", "InputVideo", "InputsUnion", + "InvalidStateError", "Item", + "LifecycleHookContext", "ManualTool", + "McpBranded", + "ModelCallUsage", "ModelResult", "NewUserMessageItem", "NextTurnParamsContext", @@ -272,6 +331,13 @@ def before_create_request(self, hook_ctx: BeforeCreateRequestContext, request: A "OutputWebSearchCallItem", "ParsedToolCall", "PartialResponse", + "PermissionRequestPayload", + "PermissionRequestResult", + "PostModelCallPayload", + "PostToolUseFailurePayload", + "PostToolUsePayload", + "PreToolUsePayload", + "PreToolUseResult", "ReasoningItem", "RequestOptions", "ResolvedCallModelInput", @@ -286,9 +352,14 @@ def before_create_request(self, hook_ctx: BeforeCreateRequestContext, request: A "ServerToolConfig", "ServerToolResultItem", "ServerToolType", + "SessionEndPayload", + "SessionStartPayload", + "SessionUsageTotals", "StateAccessor", "StepResult", "StopCondition", + "StopPayload", + "StopResult", "StopWhen", "StreamEvents", "StreamableOutputItem", @@ -303,6 +374,7 @@ def before_create_request(self, hook_ctx: BeforeCreateRequestContext, request: A "ToolExecutionResult", "ToolExecutionResultUnion", "ToolHasApproval", + "ToolMatcher", "ToolOutputContentItem", "ToolPreliminaryResultEvent", "ToolResultEvent", @@ -317,8 +389,11 @@ def before_create_request(self, hook_ctx: BeforeCreateRequestContext, request: A "TypedToolCall", "TypedToolCallUnion", "UnsentToolResult", + "UnsupportedStateVersionError", "Usage", "UserMessageItem", + "UserPromptSubmitPayload", + "UserPromptSubmitResult", "Warning", "append_to_messages", "apply_next_turn_params_to_request", @@ -329,6 +404,7 @@ def before_create_request(self, hook_ctx: BeforeCreateRequestContext, request: A "create_initial_state", "create_rejected_result", "create_unsent_result", + "deserialize_conversation_state", "execute_next_turn_params_functions", "extract_unsupported_content", "finish_reason_is", @@ -341,12 +417,14 @@ def before_create_request(self, hook_ctx: BeforeCreateRequestContext, request: A "has_execute_function", "has_tool_call", "has_unsupported_content", + "is_async_output", "is_auto_resolvable_tool", "is_client_tool", "is_claude_style_messages", "is_generator_tool", "is_hitl_tool", "is_manual_tool", + "is_mcp_tool", "is_regular_execute_tool", "is_server_tool", "is_stop_condition_met", @@ -355,11 +433,13 @@ def before_create_request(self, hook_ctx: BeforeCreateRequestContext, request: A "is_tool_result_event", "is_turn_end_event", "is_turn_start_event", + "mark_mcp", "max_cost", "max_tokens_used", "normalize_input_to_array", "partition_tool_calls", "resolve_async_functions", + "serialize_conversation_state", "server_tool", "step_count_is", "to_chat_message", diff --git a/src/openrouter_agent/async_params.py b/src/openrouter_agent/async_params.py index 685c6b5..49141b8 100644 --- a/src/openrouter_agent/async_params.py +++ b/src/openrouter_agent/async_params.py @@ -19,6 +19,8 @@ class CallModelInput(TypedDict, total=False): context: Mapping[str, Any] shared_context_schema: Any allow_final_response: Any + strict_final_response: bool + hooks: Any CallModelInputWithState = CallModelInput @@ -42,6 +44,8 @@ class ResolvedCallModelInput(TypedDict, total=False): "on_turn_start", "on_turn_end", "allow_final_response", + "strict_final_response", + "hooks", } diff --git a/src/openrouter_agent/call_model.py b/src/openrouter_agent/call_model.py index 30dceaa..daad651 100644 --- a/src/openrouter_agent/call_model.py +++ b/src/openrouter_agent/call_model.py @@ -2,6 +2,7 @@ from typing import Any, Mapping, Optional +from .hooks_resolve import resolve_hooks from .model_result import ModelResult from .tool_executor import convert_tools_to_api_format @@ -21,6 +22,8 @@ def call_model(client: Any, request: Mapping[str, Any], options: Optional[Mappin "on_turn_start", "on_turn_end", "allow_final_response", + "strict_final_response", + "hooks", ): final_request.pop(key, None) if tools is not None: @@ -45,5 +48,7 @@ def call_model(client: Any, request: Mapping[str, Any], options: Optional[Mappin "on_turn_start": request.get("on_turn_start"), "on_turn_end": request.get("on_turn_end"), "allow_final_response": request.get("allow_final_response"), + "strict_final_response": request.get("strict_final_response"), + "hooks": resolve_hooks(request.get("hooks")), } ) diff --git a/src/openrouter_agent/conversation_state.py b/src/openrouter_agent/conversation_state.py index 531da27..72b7b03 100644 --- a/src/openrouter_agent/conversation_state.py +++ b/src/openrouter_agent/conversation_state.py @@ -1,14 +1,52 @@ from __future__ import annotations +import dataclasses +import json import time import uuid from dataclasses import replace from typing import Any, Dict, List, Mapping, Optional, Sequence from ._utils import json_dumps, maybe_await -from .tool_types import ConversationState, ParsedToolCall, Tool, UnsentToolResult, get_tool_function, is_client_tool +from .tool_types import ( + ConversationState, + ParsedToolCall, + PartialResponse, + Tool, + UnsentToolResult, + get_tool_function, + is_client_tool, +) from .turn_context import normalize_input_to_array +#: Currently supported ConversationState serialization version. +CONVERSATION_STATE_VERSION = 1 + + +class UnsupportedStateVersionError(Exception): + """Raised by `deserialize_conversation_state` when a state blob's + `version` is not supported by this SDK build.""" + + def __init__(self, found: int, supported: Sequence[int] = (CONVERSATION_STATE_VERSION,)) -> None: + supported_list = list(supported) + super().__init__( + f"Unsupported ConversationState version {found}; supported version(s): " + f"{', '.join(str(v) for v in supported_list)}" + ) + self.name = "UnsupportedStateVersionError" + self.found = found + self.supported = supported_list + + +class InvalidStateError(Exception): + """Raised by `deserialize_conversation_state` when given JSON that is not + a well-formed ConversationState (missing/wrong required fields, or + invalid JSON).""" + + def __init__(self, message: str) -> None: + super().__init__(message) + self.name = "InvalidStateError" + def _now_ms() -> int: return int(time.time() * 1000) @@ -21,17 +59,129 @@ def generate_conversation_id() -> str: def create_initial_state(id: Optional[str] = None) -> ConversationState: now = _now_ms() return ConversationState( - id=id or generate_conversation_id(), messages=[], status="in_progress", created_at=now, updated_at=now + id=id or generate_conversation_id(), + messages=[], + status="in_progress", + created_at=now, + updated_at=now, + version=CONVERSATION_STATE_VERSION, ) def update_state(state: ConversationState, updates: Mapping[str, Any]) -> ConversationState: normalized: Dict[str, Any] = {} for key, value in updates.items(): + if key in ("id", "created_at", "version"): + continue normalized[key] = value return replace(state, **normalized, updated_at=_now_ms()) +def _describe_type(value: Any) -> str: + if value is None: + return "null" + if isinstance(value, list): + return "array" + return type(value).__name__ + + +def serialize_conversation_state(state: ConversationState) -> str: + """Serialize a `ConversationState` to a stable JSON string for durable + storage. + + Guarantees the `version` field is present (injects `CONVERSATION_STATE_VERSION` + when the input state lacks it). Treat the returned JSON as **opaque**: + consumers should round-trip via these helpers rather than introspecting + item shapes. + + Note: the StateAccessor load/save contract is unchanged -- these helpers + are opt-in for callers that need a durable, versioned wire format. + """ + payload = dataclasses.asdict(state) + payload["version"] = state.version if state.version is not None else CONVERSATION_STATE_VERSION + return json.dumps(payload, separators=(",", ":"), ensure_ascii=False) + + +def deserialize_conversation_state(raw_json: str) -> ConversationState: + """Parse and validate a previously serialized `ConversationState`. + + Accepts version-less legacy blobs and states with `version: 1`, + normalizing both to `version: 1`. Raises `UnsupportedStateVersionError` + for any other version. Raises `InvalidStateError` for malformed JSON or + missing required fields (`id`, `messages`, `status`, `created_at`, + `updated_at`). + """ + try: + parsed = json.loads(raw_json) + except json.JSONDecodeError as error: + raise InvalidStateError(f"Invalid ConversationState JSON: {error}") from error + + if not isinstance(parsed, dict): + raise InvalidStateError("ConversationState must be a JSON object") + + # Version check runs before structural validation: a future-version blob + # may have a different shape, and it must fail with + # UnsupportedStateVersionError rather than a misleading InvalidStateError. + version = parsed.get("version") + if version is not None and version != CONVERSATION_STATE_VERSION: + if not isinstance(version, int) or isinstance(version, bool): + raise InvalidStateError( + f'ConversationState field "version" must be a number when present (got {_describe_type(version)})' + ) + raise UnsupportedStateVersionError(version, [CONVERSATION_STATE_VERSION]) + + if not isinstance(parsed.get("id"), str): + raise InvalidStateError( + f'ConversationState missing or invalid field "id" (expected string, got {_describe_type(parsed.get("id"))})' + ) + if not isinstance(parsed.get("messages"), list): + raise InvalidStateError( + 'ConversationState missing or invalid field "messages" ' + f"(expected array, got {_describe_type(parsed.get('messages'))})" + ) + if not isinstance(parsed.get("status"), str): + raise InvalidStateError( + f'ConversationState missing or invalid field "status" ' + f"(expected string, got {_describe_type(parsed.get('status'))})" + ) + if not isinstance(parsed.get("created_at"), (int, float)) or isinstance(parsed.get("created_at"), bool): + raise InvalidStateError( + 'ConversationState missing or invalid field "created_at" ' + f"(expected number, got {_describe_type(parsed.get('created_at'))})" + ) + if not isinstance(parsed.get("updated_at"), (int, float)) or isinstance(parsed.get("updated_at"), bool): + raise InvalidStateError( + 'ConversationState missing or invalid field "updated_at" ' + f"(expected number, got {_describe_type(parsed.get('updated_at'))})" + ) + + pending_tool_calls = None + if parsed.get("pending_tool_calls") is not None: + pending_tool_calls = [ParsedToolCall(**item) for item in parsed["pending_tool_calls"]] + + unsent_tool_results = None + if parsed.get("unsent_tool_results") is not None: + unsent_tool_results = [UnsentToolResult(**item) for item in parsed["unsent_tool_results"]] + + partial_response = None + if parsed.get("partial_response") is not None: + partial_response = PartialResponse(**parsed["partial_response"]) + + return ConversationState( + id=parsed["id"], + messages=parsed["messages"], + status=parsed["status"], + created_at=parsed["created_at"], + updated_at=parsed["updated_at"], + previous_response_id=parsed.get("previous_response_id"), + pending_tool_calls=pending_tool_calls, + unsent_tool_results=unsent_tool_results, + partial_response=partial_response, + interrupted_by=parsed.get("interrupted_by"), + version=CONVERSATION_STATE_VERSION, + ) + + def append_to_messages(current: Any, new_items: Sequence[Any]) -> List[Any]: return [*normalize_input_to_array(current), *list(new_items)] diff --git a/src/openrouter_agent/hooks_emit.py b/src/openrouter_agent/hooks_emit.py new file mode 100644 index 0000000..4391677 --- /dev/null +++ b/src/openrouter_agent/hooks_emit.py @@ -0,0 +1,195 @@ +"""Sequential hook handler-chain execution. Mirrors upstream `hooks-emit.ts`. + +Supports: +- ToolMatcher and filter-based skipping (matcher fails closed: a handler with + a matcher and no `tool_name` for this emit is skipped) +- Sync results validated against the hook's result schema and collected +- Async fire-and-forget via a returned `AsyncOutput` -- its `work` is tracked + without being awaited; the manager drains/times it out +- Per-hook mutation piping (driven by `HOOK_BEHAVIOR`) +- Short-circuit on block/reject fields (non-empty string or `True`) +- Cooperative cancellation via `context.cancel_event` +""" + +from __future__ import annotations + +import asyncio +import warnings +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Sequence, Type + +from pydantic import BaseModel, ValidationError + +from ._utils import maybe_await +from .hooks_matchers import matches_tool +from .hooks_types import ( + DEFAULT_ASYNC_TIMEOUT_MS, + EmitResult, + HOOK_BEHAVIOR, + HookEntry, + LifecycleHookContext, + is_async_output, +) + + +@dataclass(frozen=True) +class _ChainOptions: + hook_name: str + throw_on_handler_error: bool + tool_name: Optional[str] = None + result_schema: Optional[Type[BaseModel]] = None + on_async_timeout: Optional[Callable[[str], None]] = None + + +async def execute_handler_chain( + entries: Sequence[HookEntry], + initial_payload: Dict[str, Any], + context: LifecycleHookContext, + *, + hook_name: str, + throw_on_handler_error: bool, + tool_name: Optional[str] = None, + result_schema: Optional[Type[BaseModel]] = None, + on_async_timeout: Optional[Callable[[str], None]] = None, +) -> EmitResult: + options = _ChainOptions( + hook_name=hook_name, + throw_on_handler_error=throw_on_handler_error, + tool_name=tool_name, + result_schema=result_schema, + on_async_timeout=on_async_timeout, + ) + results: List[Any] = [] + pending: List["asyncio.Task[None]"] = [] + current_payload: Dict[str, Any] = dict(initial_payload) if isinstance(initial_payload, dict) else initial_payload + blocked = False + mutated = False + + behavior = HOOK_BEHAVIOR.get(hook_name) + + for index, entry in enumerate(entries): + if context.cancel_event.is_set(): + break + + gate = _evaluate_entry_gate(entry, current_payload, index, options) + if gate == "skip": + continue + + try: + return_value = await maybe_await(entry.handler(current_payload, context)) + outcome = _classify_handler_return(return_value, index, options) + + if outcome[0] == "async": + tracked = outcome[1] + if tracked is not None: + pending.append(tracked) + continue + if outcome[0] == "skip": + continue + + result = outcome[1] + results.append(result) + + if behavior and behavior.mutations: + applied = _apply_mutations(current_payload, result, behavior.mutations) + if applied is not current_payload: + current_payload = applied + mutated = True + + if behavior and behavior.block_field and _is_block_triggered(result, behavior.block_field): + blocked = True + break + except Exception as error: # noqa: BLE001 - policy decides re-raise vs. warn + if throw_on_handler_error: + raise + warnings.warn(f'[HooksManager] Handler {index} for hook "{hook_name}" threw: {error}', stacklevel=2) + + return EmitResult( + results=results, + pending=pending, + final_payload=current_payload, + blocked=blocked, + mutated=mutated, + ) + + +def _evaluate_entry_gate(entry: HookEntry, payload: Dict[str, Any], index: int, options: _ChainOptions) -> str: + try: + matcher_passes = entry.matcher is None or ( + options.tool_name is not None and matches_tool(entry.matcher, options.tool_name) + ) + if not matcher_passes: + return "skip" + return "run" if not entry.filter or bool(entry.filter(payload)) else "skip" + except Exception as error: # noqa: BLE001 + if options.throw_on_handler_error: + raise + warnings.warn( + f'[HooksManager] Matcher/filter for handler {index} of hook "{options.hook_name}" threw: {error}', + stacklevel=2, + ) + return "skip" + + +def _classify_handler_return(return_value: Any, index: int, options: _ChainOptions) -> Any: + if is_async_output(return_value): + return ("async", _track_async_work(return_value, options.hook_name, options.on_async_timeout)) + if return_value is None: + return ("skip", None) + if options.result_schema is None: + return ("result", return_value) + try: + validated = options.result_schema.model_validate(return_value) + except ValidationError as error: + message = f'[HooksManager] Handler {index} for hook "{options.hook_name}" returned an invalid result: {error}' + if options.throw_on_handler_error: + raise RuntimeError(message) from error + warnings.warn(message, stacklevel=2) + return ("skip", None) + return ("result", validated.model_dump()) + + +def _track_async_work( + output: Any, + hook_name: str, + on_timeout: Optional[Callable[[str], None]], +) -> Optional["asyncio.Task[None]"]: + if output.work is None: + return None + timeout_s = (output.async_timeout_ms or DEFAULT_ASYNC_TIMEOUT_MS) / 1000 + + async def _wait() -> None: + try: + await asyncio.wait_for(output.work, timeout=timeout_s) + except asyncio.TimeoutError: + warnings.warn( + f'[HooksManager] Async work for hook "{hook_name}" exceeded its timeout; abandoning wait.', + stacklevel=2, + ) + if on_timeout is not None: + on_timeout(hook_name) + except Exception as error: # noqa: BLE001 + warnings.warn(f'[HooksManager] Async work for hook "{hook_name}" rejected: {error}', stacklevel=2) + + return asyncio.ensure_future(_wait()) + + +def _apply_mutations(payload: Dict[str, Any], result: Any, mutation_map: Dict[str, str]) -> Dict[str, Any]: + if not isinstance(result, dict): + return payload + mutated = payload + for result_field, payload_field in mutation_map.items(): + if result_field in result: + value = result[result_field] + if value is not None: + mutated = {**mutated, payload_field: value} + return mutated + + +def _is_block_triggered(result: Any, block_field: str) -> bool: + if not isinstance(result, dict): + return False + value = result.get(block_field) + if value is True: + return True + return isinstance(value, str) and len(value) > 0 diff --git a/src/openrouter_agent/hooks_manager.py b/src/openrouter_agent/hooks_manager.py new file mode 100644 index 0000000..aec6068 --- /dev/null +++ b/src/openrouter_agent/hooks_manager.py @@ -0,0 +1,198 @@ +"""Typed, extensible hook system for agent lifecycle events. Mirrors upstream +`hooks-manager.ts`. + +Supports both built-in hooks (PreToolUse, PostToolUse, ...) and user-defined +custom hooks. Unlike the TypeScript reference, there is no internal-registrar +symbol trick: Python's `on()` is not statically constrained to known hook +names, so `resolve_hooks` can register inline-config entries directly. +""" + +from __future__ import annotations + +import asyncio +import warnings +from typing import Any, Callable, Dict, List, Optional, Set + +from pydantic import ValidationError + +from .hooks_emit import execute_handler_chain +from .hooks_schemas import BUILT_IN_HOOK_NAMES, BUILT_IN_HOOKS, HookDefinition, HookRegistry +from .hooks_types import EmitResult, HookEntry, LifecycleHookContext + + +class HooksManager: + """See module docstring. Register handlers with `on()`, fire them with + `emit()`, and await outstanding fire-and-forget work with `drain()`.""" + + def __init__( + self, + custom_hooks: Optional[HookRegistry] = None, + *, + throw_on_handler_error: bool = False, + ) -> None: + self._entries: Dict[str, List[HookEntry]] = {} + self._pending_async: Set[Any] = set() + self._inflight: Set[asyncio.Event] = set() + self._throw_on_handler_error = throw_on_handler_error + self._session_id = "" + + if custom_hooks: + for name in custom_hooks: + if name == "": + raise ValueError("Custom hook names must be non-empty strings.") + if name in BUILT_IN_HOOK_NAMES: + raise ValueError( + f'Custom hook name "{name}" collides with a built-in hook. Choose a different name.' + ) + self._custom_hooks: HookRegistry = dict(custom_hooks) + else: + self._custom_hooks = {} + + def set_session_id(self, session_id: str) -> None: + """Set the manager-level default session ID exposed as + `context.session_id` to handler invocations. + + This is a single mutable default on the manager instance: when one + manager is shared by concurrent runs, callers MUST pass `session_id` + to `emit()` instead (as `ModelResult` does), otherwise the last + `set_session_id()` call wins and concurrent emits observe the wrong id. + """ + self._session_id = session_id + + def on(self, hook_name: str, entry: HookEntry) -> Callable[[], None]: + """Register a handler for a hook. Returns an unsubscribe function.""" + return self._register(hook_name, entry) + + def off(self, hook_name: str, handler: Callable[..., Any]) -> bool: + """Remove a specific handler function from a hook.""" + entries = self._entries.get(hook_name) + if not entries: + return False + for index, entry in enumerate(entries): + if entry.handler == handler: + entries.pop(index) + if not entries: + del self._entries[hook_name] + return True + return False + + def remove_all(self, hook_name: Optional[str] = None) -> None: + """Remove all handlers for a specific hook, or all handlers if omitted.""" + if hook_name is not None: + self._entries.pop(hook_name, None) + else: + self._entries.clear() + + async def emit( + self, + hook_name: str, + payload: Dict[str, Any], + *, + tool_name: Optional[str] = None, + session_id: Optional[str] = None, + ) -> EmitResult: + """Validate the payload (and each handler's result) against the + registered schemas, invoke matching handlers, and return results. + + Payload validation failure is handled per `throw_on_handler_error`: + strict mode re-raises, default mode warns and returns an empty result + without invoking any handlers. + """ + entries = list(self._entries.get(hook_name, [])) + definition = self._definition_for(hook_name) + + chain_payload: Dict[str, Any] = payload + if definition is not None: + try: + parsed = definition.payload.model_validate(payload) + except ValidationError as error: + message = f'[HooksManager] Invalid payload for hook "{hook_name}": {error}' + if self._throw_on_handler_error: + raise RuntimeError(message) from error + warnings.warn(message, stacklevel=2) + return EmitResult(results=[], pending=[], final_payload=chain_payload, blocked=False, mutated=False) + chain_payload = parsed.model_dump() + + cancel_event = asyncio.Event() + self._inflight.add(cancel_event) + + context = LifecycleHookContext( + cancel_event=cancel_event, + hook_name=hook_name, + session_id=session_id if session_id is not None else self._session_id, + ) + + has_detached_work = False + try: + result_schema = definition.result if definition is not None else None + result = await execute_handler_chain( + entries, + chain_payload, + context, + hook_name=hook_name, + throw_on_handler_error=self._throw_on_handler_error, + tool_name=tool_name, + result_schema=result_schema, + on_async_timeout=lambda _name: cancel_event.set(), + ) + + has_detached_work = len(result.pending) > 0 + if has_detached_work: + remaining = len(result.pending) + + def _make_cleanup(task: Any) -> Callable[[Any], None]: + def _cleanup(_done: Any) -> None: + nonlocal remaining + self._pending_async.discard(task) + remaining -= 1 + if remaining == 0: + self._inflight.discard(cancel_event) + + return _cleanup + + for task in result.pending: + self._pending_async.add(task) + task.add_done_callback(_make_cleanup(task)) + + return result + finally: + if not has_detached_work: + self._inflight.discard(cancel_event) + + async def drain(self) -> None: + """Await all in-flight async handler work. Used for graceful shutdown.""" + while self._pending_async: + snapshot = list(self._pending_async) + await asyncio.gather(*snapshot, return_exceptions=True) + + def abort_inflight(self) -> None: + """Signal cancellation to every in-flight `emit()`. Does not remove + pending async work -- call `drain()` afterward to wait it out.""" + for event in self._inflight: + event.set() + + def has_handlers(self, hook_name: str) -> bool: + """Check if any handlers are registered for a given hook.""" + entries = self._entries.get(hook_name) + return entries is not None and len(entries) > 0 + + def _register(self, hook_name: str, entry: HookEntry) -> Callable[[], None]: + entries = self._entries.setdefault(hook_name, []) + entries.append(entry) + + def _unsubscribe() -> None: + current = self._entries.get(hook_name) + if not current: + return + try: + current.remove(entry) + except ValueError: + pass + + return _unsubscribe + + def _definition_for(self, hook_name: str) -> Optional[HookDefinition]: + built_in = BUILT_IN_HOOKS.get(hook_name) + if built_in is not None: + return built_in + return self._custom_hooks.get(hook_name) diff --git a/src/openrouter_agent/hooks_matchers.py b/src/openrouter_agent/hooks_matchers.py new file mode 100644 index 0000000..801c89b --- /dev/null +++ b/src/openrouter_agent/hooks_matchers.py @@ -0,0 +1,25 @@ +"""Tool-name matching for tool-scoped hook entries. Mirrors `hooks-matchers.ts`.""" + +from __future__ import annotations + +import re +from typing import Optional + +from .hooks_types import ToolMatcher + + +def matches_tool(matcher: Optional[ToolMatcher], tool_name: str) -> bool: + """Evaluate a ToolMatcher against a tool name. + + - `None` -> wildcard, matches all tools + - `str` -> exact match + - compiled regex pattern -> `.search(tool_name)` is truthy + - callable -> arbitrary predicate (coerced to bool) + """ + if matcher is None: + return True + if isinstance(matcher, str): + return matcher == tool_name + if isinstance(matcher, re.Pattern): + return matcher.search(tool_name) is not None + return bool(matcher(tool_name)) diff --git a/src/openrouter_agent/hooks_resolve.py b/src/openrouter_agent/hooks_resolve.py new file mode 100644 index 0000000..83f7a5d --- /dev/null +++ b/src/openrouter_agent/hooks_resolve.py @@ -0,0 +1,46 @@ +"""Normalize a `hooks` option into a `HooksManager` instance. Mirrors +upstream `hooks-resolve.ts`.""" + +from __future__ import annotations + +from typing import Optional, Union + +from .hooks_manager import HooksManager +from .hooks_schemas import BUILT_IN_HOOK_NAMES +from .hooks_types import HookEntry, InlineHookConfig + + +def resolve_hooks(hooks: Optional[Union[InlineHookConfig, HooksManager]]) -> Optional[HooksManager]: + """ + - `None` -> `None` (no hooks) + - `HooksManager` -> passthrough + - plain dict (`InlineHookConfig`) -> construct a `HooksManager` and + register every entry + + Any non-built-in key in the inline config is ignored with a warning: + inline config only supports built-in hooks, custom hooks must be + registered through a `HooksManager` instance via `on()`. + """ + if not hooks: + return None + + if isinstance(hooks, HooksManager): + return hooks + + manager = HooksManager() + for hook_name, entries in hooks.items(): + if not entries: + continue + if hook_name not in BUILT_IN_HOOK_NAMES: + import warnings + + warnings.warn( + f'[resolve_hooks] Ignoring inline hook entry for unknown hook name "{hook_name}". ' + "Inline config only supports built-in hooks; register custom hooks via a HooksManager instance.", + stacklevel=2, + ) + continue + for entry in entries: + manager.on(hook_name, entry if isinstance(entry, HookEntry) else HookEntry(**entry)) + + return manager diff --git a/src/openrouter_agent/hooks_schemas.py b/src/openrouter_agent/hooks_schemas.py new file mode 100644 index 0000000..3bccb34 --- /dev/null +++ b/src/openrouter_agent/hooks_schemas.py @@ -0,0 +1,184 @@ +"""Pydantic v2 payload/result schemas for the lifecycle hooks system. + +Mirrors upstream `hooks-schemas.ts`: HookName, HookDefinition, and the +built-in hook registry live here (next to the schemas they describe) so the +runtime validation and the static shape can never drift apart -- the same +"single source of truth" discipline the TypeScript reference applies with Zod. + +Field names are idiomatic Python snake_case (e.g. ``tool_name``, +``duration_ms``) rather than the upstream camelCase -- these payloads are an +internal Python-facing API for hook handlers, not wire types serialized to +the OpenRouter API, so there is no interop reason to keep camelCase. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Any, Dict, Optional, Type, Union + +from pydantic import BaseModel +from typing_extensions import Literal + + +class HookName(str, Enum): + PreToolUse = "PreToolUse" + PostToolUse = "PostToolUse" + PostToolUseFailure = "PostToolUseFailure" + UserPromptSubmit = "UserPromptSubmit" + Stop = "Stop" + PermissionRequest = "PermissionRequest" + SessionStart = "SessionStart" + SessionEnd = "SessionEnd" + PostModelCall = "PostModelCall" + + +@dataclass(frozen=True) +class HookDefinition: + """A hook definition pairs a payload schema with an optional result schema. + + ``result=None`` marks an observation-only hook: handlers may return + anything and it is collected as an opaque result without validation + (mirrors upstream's `z.void()` result schema, minus the schema-shape + introspection -- Python just uses ``None`` directly). + """ + + payload: Type[BaseModel] + result: Optional[Type[BaseModel]] + + +#: A registry maps hook names to their definitions. Used for both the +#: built-in hooks (below) and a HooksManager's custom hook registry. +HookRegistry = Dict[str, HookDefinition] + + +# --------------------------------------------------------------------------- +# Payload schemas +# --------------------------------------------------------------------------- + + +class PreToolUsePayload(BaseModel): + tool_name: str + tool_input: Dict[str, Any] + + +class PostToolUsePayload(BaseModel): + tool_name: str + tool_input: Dict[str, Any] + tool_output: Any = None + duration_ms: float + + +class PostToolUseFailurePayload(BaseModel): + """Fired when a tool EXECUTION throws or returns an error. + + Deliberately NOT fired when a tool never ran: a PermissionRequest 'deny', + a user rejection on approval resume, or a PreToolUse block all synthesize + a rejected result without execution, so no failure event is emitted. + Observe those outcomes via the PermissionRequest / PreToolUse hooks + themselves. + """ + + tool_name: str + tool_input: Dict[str, Any] + error: Any = None + + +class StopPayload(BaseModel): + reason: Literal["max_turns"] + + +class PermissionRequestPayload(BaseModel): + tool_name: str + tool_input: Dict[str, Any] + risk_level: Literal["low", "medium", "high"] + + +class UserPromptSubmitPayload(BaseModel): + prompt: str + + +class SessionStartPayload(BaseModel): + config: Optional[Dict[str, Any]] = None + + +class ModelCallUsage(BaseModel): + input_tokens: int + output_tokens: int + total_tokens: int + cached_tokens: int + reasoning_tokens: int + cost: Optional[float] = None + + +class SessionUsageTotals(ModelCallUsage): + model_calls: int + + +class SessionEndPayload(BaseModel): + reason: Literal["user", "error", "max_turns", "complete"] + total_usage: Optional[SessionUsageTotals] = None + + +class PostModelCallPayload(BaseModel): + session_id: str + response_id: str + model: str + duration_ms: float + turn_type: Literal["initial", "resume", "tool_round", "final", "retry"] + turn_number: int + usage: Optional[ModelCallUsage] = None + + +# --------------------------------------------------------------------------- +# Result schemas +# --------------------------------------------------------------------------- + + +class PreToolUseResult(BaseModel): + mutated_input: Optional[Dict[str, Any]] = None + block: Optional[Union[bool, str]] = None + + +class StopResult(BaseModel): + """Result of a Stop hook handler. + + ``force_resume=True`` alone does NOT change any state: the stop + condition (e.g. ``step_count_is``) will typically fire again immediately + on the next iteration, so a bare force_resume burns through the + consecutive-override cap in rapid succession and then stops. Pair it + with ``append_prompt`` (injects a user message, advancing the + conversation) to make resumption useful. + """ + + force_resume: Optional[bool] = None + append_prompt: Optional[str] = None + + +class PermissionRequestResult(BaseModel): + decision: Literal["allow", "deny", "ask_user"] + reason: Optional[str] = None + + +class UserPromptSubmitResult(BaseModel): + mutated_prompt: Optional[str] = None + reject: Optional[Union[bool, str]] = None + + +# --------------------------------------------------------------------------- +# Built-in hook registry +# --------------------------------------------------------------------------- + +BUILT_IN_HOOKS: HookRegistry = { + HookName.PreToolUse.value: HookDefinition(payload=PreToolUsePayload, result=PreToolUseResult), + HookName.PostToolUse.value: HookDefinition(payload=PostToolUsePayload, result=None), + HookName.PostToolUseFailure.value: HookDefinition(payload=PostToolUseFailurePayload, result=None), + HookName.UserPromptSubmit.value: HookDefinition(payload=UserPromptSubmitPayload, result=UserPromptSubmitResult), + HookName.Stop.value: HookDefinition(payload=StopPayload, result=StopResult), + HookName.PermissionRequest.value: HookDefinition(payload=PermissionRequestPayload, result=PermissionRequestResult), + HookName.SessionStart.value: HookDefinition(payload=SessionStartPayload, result=None), + HookName.SessionEnd.value: HookDefinition(payload=SessionEndPayload, result=None), + HookName.PostModelCall.value: HookDefinition(payload=PostModelCallPayload, result=None), +} + +BUILT_IN_HOOK_NAMES = frozenset(BUILT_IN_HOOKS.keys()) diff --git a/src/openrouter_agent/hooks_types.py b/src/openrouter_agent/hooks_types.py new file mode 100644 index 0000000..953af4c --- /dev/null +++ b/src/openrouter_agent/hooks_types.py @@ -0,0 +1,128 @@ +"""Core types for the lifecycle hooks system. + +Mirrors upstream `hooks-types.ts`, adapted to Python: + +- Cancellation uses an `asyncio.Event` rather than `AbortSignal` (idiomatic + divergence: this repo's cancellation is Python-native). +- The `AsyncOutput` fire-and-forget signal is detected via `isinstance` + rather than schema validation -- Python has real classes, so there is no + need for zod's structural-shape check. +- Payload/result validation failures raise `pydantic.ValidationError` + instead of zod's `ZodError`; handled the same way (throw in strict mode, + warn otherwise). +""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, field +from typing import Any, Awaitable, Callable, Dict, List, Optional, Pattern, Sequence, Union + +from .hooks_schemas import HookName # noqa: F401 (re-exported for import-surface parity) + +#: Matcher for tool-scoped hooks. Filters handler invocation by tool name. +ToolMatcher = Union[str, Pattern[str], Callable[[str], bool]] + +#: A hook handler receives the validated payload (dict) and context. May be +#: sync or async -- callers should invoke it through `maybe_await`. +HookHandler = Callable[[Dict[str, Any], "LifecycleHookContext"], Any] + + +@dataclass(frozen=True) +class LifecycleHookContext: + """Context provided to every lifecycle-hook handler invocation. + + `cancel_event` is set if the manager's `abort_inflight()` is called + while the emit is still running. Handlers that kick off background work + via `AsyncOutput` should observe it for cancellation. + + `session_id` is the single source for session identity in handlers -- + payloads deliberately do not repeat it. The engine threads it per emit + (safe for a manager shared across concurrent runs); direct `emit()` + callers get the manager-level default from `set_session_id()` unless + they pass a per-emit override. + """ + + cancel_event: asyncio.Event + hook_name: str + session_id: str + + +#: Default milliseconds before an async fire-and-forget handler is aborted. +DEFAULT_ASYNC_TIMEOUT_MS = 30_000 + + +@dataclass(frozen=True) +class AsyncOutput: + """Returned by a handler to signal fire-and-forget mode. + + The chain proceeds immediately without waiting for completion. Any + background work the handler kicked off should be attached as `work` so + the manager can track it for `drain()` and enforce `async_timeout_ms`. + """ + + work: Optional[Awaitable[Any]] = None + async_timeout_ms: float = DEFAULT_ASYNC_TIMEOUT_MS + + +def is_async_output(value: Any) -> bool: + """Type guard for an `AsyncOutput` fire-and-forget signal.""" + return isinstance(value, AsyncOutput) + + +@dataclass(frozen=True) +class HookEntry: + """An entry registered for a specific hook.""" + + handler: HookHandler + matcher: Optional[ToolMatcher] = None + filter: Optional[Callable[[Dict[str, Any]], bool]] = None + + +@dataclass(frozen=True) +class EmitResult: + """Result of emitting a hook through the handler chain. + + INVARIANT: every entry in `results` passed the hook's result schema + (invalid results are skipped or raised per the error policy, never + collected), so consumers can rely on the shape without re-validating. + For void-result hooks (no schema) entries are opaque dicts/values. + """ + + results: List[Any] = field(default_factory=list) + #: Handles to detached async handler work (asyncio Tasks). + pending: List["asyncio.Task[None]"] = field(default_factory=list) + #: The payload after all mutation piping has been applied. + final_payload: Dict[str, Any] = field(default_factory=dict) + #: True if any handler triggered a block/reject short-circuit. + blocked: bool = False + #: True if any handler's result actually piped a mutation into the payload. + mutated: bool = False + + +@dataclass(frozen=True) +class HookBehavior: + """Per-hook chain behavior: which result fields pipe mutations back into + the payload, and which result field short-circuits the chain.""" + + mutations: Optional[Dict[str, str]] = None + block_field: Optional[str] = None + + +#: Keyed by HookName value. Hooks absent from this table (all +#: observation-only hooks and every custom hook) collect results without +#: altering the payload or short-circuiting the chain. +HOOK_BEHAVIOR: Dict[str, HookBehavior] = { + HookName.PreToolUse.value: HookBehavior( + mutations={"mutated_input": "tool_input"}, + block_field="block", + ), + HookName.UserPromptSubmit.value: HookBehavior( + mutations={"mutated_prompt": "prompt"}, + block_field="reject", + ), +} + +#: Inline hook config passed directly to call_model: `{hook_name: [entries]}`. +#: Only supports built-in hook names. For custom hooks, use a HooksManager. +InlineHookConfig = Dict[str, Sequence[HookEntry]] diff --git a/src/openrouter_agent/model_result.py b/src/openrouter_agent/model_result.py index ecab0a4..445c53e 100644 --- a/src/openrouter_agent/model_result.py +++ b/src/openrouter_agent/model_result.py @@ -1,16 +1,22 @@ from __future__ import annotations import asyncio +import dataclasses import json import time -from typing import Any, AsyncIterator, Dict, List, Mapping, Optional, Sequence +import warnings +from typing import Any, AsyncIterator, Dict, List, Mapping, Optional, Sequence, Tuple from ._utils import get_field, is_async_iterable, json_dumps, maybe_await, sdk_request_kwargs from .async_params import resolve_async_functions from .conversation_state import ( append_to_messages, create_initial_state, + create_rejected_result, + create_unsent_result, + generate_conversation_id, partition_tool_calls, + unsent_results_to_api_format, update_state, ) from .next_turn_params import apply_next_turn_params_to_request, execute_next_turn_params_functions @@ -22,15 +28,75 @@ ParsedToolCall, StepResult, Tool, + UnsentToolResult, get_tool_function, is_auto_resolvable_tool, is_client_tool, is_manual_tool, + is_mcp_tool, + tool_has_approval_configured, ) from .turn_context import normalize_input_to_array GetResponseOptions = Dict[str, Any] +#: Default directive appended as a final user message on the forced final +#: turn (`allow_final_response` defaulting to on, or explicitly `True`). +#: Forbidding tools via `tool_choice: "none"` alone is not enough: models +#: that emit tool-call syntax as text will attempt another call and leak it +#: into `content` as unparsed text unless they are told this is the final +#: turn. Pass a non-empty string to `allow_final_response` to override the +#: wording, or `""` to append no message at all (legacy behavior). +DEFAULT_FINAL_RESPONSE_DIRECTIVE = ( + "You have reached the tool-use limit, and tools are no longer available. " + "Do not attempt to call any more tools. Using the information you already have, " + "write your final answer now." +) + +#: Safety cap on turns so a misbehaving loop cannot spin forever. +_MAX_TURNS = 20 + +#: Cap consecutive Stop-hook force_resume overrides so a misbehaving handler +#: cannot spin the loop forever. +_MAX_FORCE_RESUME_OVERRIDES = 3 + + +def _now_ms() -> int: + return int(time.time() * 1000) + + +def _extract_model_call_usage(usage: Any) -> Optional[Dict[str, Any]]: + """Map the response's usage block onto the hook-facing ModelCallUsage + shape. Returns None when the response carried no usage accounting.""" + if usage is None: + return None + input_details = get_field(usage, "input_tokens_details") + output_details = get_field(usage, "output_tokens_details") + result: Dict[str, Any] = { + "input_tokens": int(get_field(usage, "input_tokens", 0) or 0), + "output_tokens": int(get_field(usage, "output_tokens", 0) or 0), + "total_tokens": int(get_field(usage, "total_tokens", 0) or 0), + "cached_tokens": int(get_field(input_details, "cached_tokens", 0) or 0) if input_details is not None else 0, + "reasoning_tokens": int(get_field(output_details, "reasoning_tokens", 0) or 0) + if output_details is not None + else 0, + } + cost = get_field(usage, "cost", None) + if cost is not None: + result["cost"] = cost + return result + + +def _is_user_string_message(value: Any) -> bool: + return isinstance(value, Mapping) and value.get("role") == "user" and isinstance(value.get("content"), str) + + +def _find_latest_user_string_index(items: Sequence[Any]) -> int: + for index in range(len(items) - 1, -1, -1): + if _is_user_string_message(items[index]): + return index + return -1 + class ModelResult: def __init__(self, options: Mapping[str, Any]) -> None: @@ -46,11 +112,29 @@ def __init__(self, options: Mapping[str, Any]) -> None: self._context_store: Optional[ToolContextStore] = None self._condition = asyncio.Condition() self._run_done = False + self._hooks = self.options.get("hooks") + self._session_id = "" + self._resuming_from_client_tools = False + self._session_start_emitted = False + self._session_end_emitted = False + self._all_tool_rounds = 0 + self._session_usage: Dict[str, Any] = { + "model_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "cached_tokens": 0, + "reasoning_tokens": 0, + "cost": 0.0, + "has_cost": False, + } if (self.options.get("approve_tool_calls") or self.options.get("reject_tool_calls")) and not self.options.get( "state" ): raise ValueError("approve_tool_calls and reject_tool_calls require a state accessor") + # -- transport ----------------------------------------------------------- + async def _send(self, request: Mapping[str, Any]) -> Any: client = self.options["client"] kwargs = sdk_request_kwargs(request) @@ -96,6 +180,13 @@ async def _coerce_response(self, value: Any) -> Any: return completed or {"id": "response_from_events", "output": []} return value + async def _send_and_track(self, request: Mapping[str, Any], turn_type: str, turn_number: int) -> Any: + """Send a request, coerce its response, and emit PostModelCall for it.""" + started = time.monotonic() + response = await self._coerce_response(await self._send(request)) + await self._emit_post_model_call(response, started, turn_type, turn_number) + return response + async def _append_event(self, event: Any) -> None: async with self._condition: self._events.append(event) @@ -106,14 +197,18 @@ def _ensure_task(self) -> asyncio.Task[Any]: self._run_task = asyncio.create_task(self._run()) return self._run_task + # -- state ----------------------------------------------------------- + async def _load_state(self) -> None: accessor = self.options.get("state") if accessor is None: return loaded = await maybe_await(accessor.load()) + self._resuming_from_client_tools = bool(loaded is not None and loaded.status == "awaiting_client_tools") self._state = loaded or create_initial_state() - self._state = update_state(self._state, {"status": "in_progress"}) - await maybe_await(accessor.save(self._state)) + if not self._resuming_from_client_tools: + self._state = update_state(self._state, {"status": "in_progress"}) + await maybe_await(accessor.save(self._state)) async def _save_state(self, **updates: Any) -> None: accessor = self.options.get("state") @@ -134,58 +229,404 @@ async def _save_response_to_state(self, response: Any) -> None: messages = append_to_messages(messages, self._fresh_items_for_state) self._fresh_items_for_state = [] messages = append_to_messages(messages, self._response_output_items(response)) - await self._save_state(messages=messages, previous_response_id=get_field(response, "id")) + updates: Dict[str, Any] = {"messages": messages, "previous_response_id": get_field(response, "id")} + if self._resuming_from_client_tools: + # Manual calls stay durable until a resume request actually + # produces a response -- clearing before the request would lose + # the only copy if that request failed. + updates["pending_tool_calls"] = None + updates["status"] = "in_progress" + self._resuming_from_client_tools = False + await self._save_state(**updates) async def _save_tool_outputs_to_state(self, outputs: Sequence[Any]) -> None: if self._state is None or not outputs: return await self._save_state(messages=append_to_messages(self._state.messages, list(outputs))) - async def _run(self) -> Any: - await self._load_state() - tools: Sequence[Tool] = self.options.get("tools") or [] - request = await resolve_async_functions(self.options["request"], {"number_of_turns": 0}) - request["stream"] = True - base_input = request.get("input") - historical_input = normalize_input_to_array(self._state.messages) if self._state is not None else [] - fresh_input = normalize_input_to_array(base_input) - if self.options.get("context") is not None: - resolved_context = await resolve_context(self.options.get("context"), {"number_of_turns": 0}) - self._context_store = ToolContextStore(resolved_context) - if tools and fresh_input: - historical_function_calls = [ - item for item in historical_input if get_field(item, "type") == "function_call" - ] - synthetic_input = [*historical_function_calls, *fresh_input] - hooked_input = await apply_on_response_received_hooks( - synthetic_input, - tools, - {"number_of_turns": 0}, - self._context_store, - self.options.get("shared_context_schema"), - ) - hooked_array = normalize_input_to_array(hooked_input) - fresh_input = hooked_array[len(historical_function_calls) :] + async def _persist_client_tools_pause(self, response: Any, unresolved_calls: Sequence[ParsedToolCall]) -> None: + """Persist state when the loop stops because of unresolved manual + (client-executed) tool calls -- tools with neither `execute` nor + `on_tool_called`. Mirrors the HITL pause path but uses the distinct + `awaiting_client_tools` status so callers can tell the two apart. + + Without a StateAccessor nothing is persisted: `get_pending_tool_calls()` + returns `[]` and the caller must read the unresolved calls off the + response output directly. + """ + self._final_response = response + if self.options.get("state") is None or not unresolved_calls: + return + await self._save_state(pending_tool_calls=list(unresolved_calls), status="awaiting_client_tools") + + def _validate_final_response(self, response: Any, allow_empty_output: bool = False) -> None: + response_id = get_field(response, "id") + output = get_field(response, "output") + if not response_id or output is None: + raise ValueError("Invalid final response: missing required fields") + if not isinstance(output, list) or len(output) == 0: + if allow_empty_output: + return + raise ValueError("Invalid final response: empty or invalid output") + + # -- hooks ----------------------------------------------------------- + + def _has_approval_configured(self, tools: Sequence[Tool]) -> bool: + if self.options.get("require_approval"): + return True + return any(tool_has_approval_configured(t) for t in tools) + + async def _emit_session_start(self, tools: Sequence[Tool]) -> None: + if not self._hooks: + return + self._hooks.set_session_id(self._session_id) + await self._hooks.emit( + "SessionStart", + { + "config": { + "has_tools": bool(tools), + "has_approval": self._has_approval_configured(tools), + "has_state": self.options.get("state") is not None, + } + }, + session_id=self._session_id, + ) + self._session_start_emitted = True + + async def _emit_session_end(self, reason: str) -> None: + if not self._hooks or not self._session_start_emitted or self._session_end_emitted: + return + self._session_end_emitted = True + payload: Dict[str, Any] = {"reason": reason} + if self._session_usage["model_calls"] > 0: + total_usage = { + "model_calls": self._session_usage["model_calls"], + "input_tokens": self._session_usage["input_tokens"], + "output_tokens": self._session_usage["output_tokens"], + "total_tokens": self._session_usage["total_tokens"], + "cached_tokens": self._session_usage["cached_tokens"], + "reasoning_tokens": self._session_usage["reasoning_tokens"], + } + if self._session_usage["has_cost"]: + total_usage["cost"] = self._session_usage["cost"] + payload["total_usage"] = total_usage + try: + await self._hooks.emit("SessionEnd", payload, session_id=self._session_id) + except Exception as error: # noqa: BLE001 - teardown must never mask the real error + warnings.warn(f"[SessionEnd] error during session teardown: {error}", stacklevel=2) + + async def _emit_post_model_call(self, response: Any, started_at: float, turn_type: str, turn_number: int) -> None: + if not self._hooks: + return + usage = _extract_model_call_usage(get_field(response, "usage")) + self._session_usage["model_calls"] += 1 + if usage: + self._session_usage["input_tokens"] += usage["input_tokens"] + self._session_usage["output_tokens"] += usage["output_tokens"] + self._session_usage["total_tokens"] += usage["total_tokens"] + self._session_usage["cached_tokens"] += usage["cached_tokens"] + self._session_usage["reasoning_tokens"] += usage["reasoning_tokens"] + if usage.get("cost") is not None: + self._session_usage["cost"] += usage["cost"] + self._session_usage["has_cost"] = True + payload = { + "session_id": self._session_id, + "response_id": get_field(response, "id", ""), + "model": get_field(response, "model", "") or "", + "duration_ms": (time.monotonic() - started_at) * 1000, + "turn_type": turn_type, + "turn_number": turn_number, + } + if usage: + payload["usage"] = usage + await self._hooks.emit("PostModelCall", payload, session_id=self._session_id) + + async def _emit_permission_request(self, call: ParsedToolCall, tools: Sequence[Tool]) -> Tuple[str, Optional[str]]: + if not self._hooks: + return "ask_user", None + if isinstance(call.arguments, str): + # Raw-string arguments mean the model produced invalid JSON. Fail + # closed (fall through to the human approval flow). + return "ask_user", None + + tool = next( + (t for t in tools if is_client_tool(t) and get_tool_function(t).get("name") == call.name), + None, + ) + require_approval = get_tool_function(tool).get("require_approval") if tool else None + if callable(require_approval) or self.options.get("require_approval"): + risk_level = "high" + elif require_approval is True: + risk_level = "medium" + else: + risk_level = "low" + + emitted = await self._hooks.emit( + "PermissionRequest", + { + "tool_name": call.name, + "tool_input": call.arguments if isinstance(call.arguments, dict) else {}, + "risk_level": risk_level, + }, + tool_name=call.name, + session_id=self._session_id, + ) + if not emitted.results: + return "ask_user", None + last = emitted.results[-1] + return last.get("decision", "ask_user"), last.get("reason") + + async def _maybe_run_user_prompt_submit(self, input_value: Any) -> Any: + if not self._hooks or input_value is None: + return input_value + + if isinstance(input_value, str): + prompt = input_value + emitted = await self._hooks.emit("UserPromptSubmit", {"prompt": prompt}, session_id=self._session_id) + if emitted.blocked: + reject = next((r.get("reject") for r in emitted.results if r.get("reject")), None) + raise ValueError(reject if isinstance(reject, str) else "Prompt rejected by hook") + if emitted.mutated: + return emitted.final_payload.get("prompt", prompt) + return input_value + + if isinstance(input_value, list): + target_index = _find_latest_user_string_index(input_value) + if target_index == -1: + return input_value + prompt = input_value[target_index]["content"] + emitted = await self._hooks.emit("UserPromptSubmit", {"prompt": prompt}, session_id=self._session_id) + if emitted.blocked: + reject = next((r.get("reject") for r in emitted.results if r.get("reject")), None) + raise ValueError(reject if isinstance(reject, str) else "Prompt rejected by hook") + if not emitted.mutated: + return input_value + mutated_prompt = emitted.final_payload.get("prompt", prompt) + new_items = list(input_value) + new_items[target_index] = {**new_items[target_index], "content": mutated_prompt} + return new_items + + return input_value + + async def _inject_append_prompt_message(self, prompt: str, current_request: Dict[str, Any]) -> None: + injected = {"role": "user", "content": prompt} if self._state is not None: - request["input"] = append_to_messages(historical_input, fresh_input) - if not ( - self._state.pending_tool_calls + next_messages = append_to_messages(self._state.messages, [injected]) + self._state = update_state(self._state, {"messages": next_messages}) + if self.options.get("state") is not None: + await self._save_state() + current_input = current_request.get("input") + if isinstance(current_input, list): + current_input.append(injected) + elif current_input: + current_request["input"] = [{"role": "user", "content": current_input}, injected] + else: + current_request["input"] = [injected] + + async def _run_stop_hook(self, force_resume_count: int, current_request: Dict[str, Any]) -> str: + """Emit the Stop hook when a stop_when condition halts the loop. + Returns "resume" when the loop should continue, "stop" otherwise.""" + if not self._hooks: + return "stop" + emitted = await self._hooks.emit("Stop", {"reason": "max_turns"}, session_id=self._session_id) + should_force_resume = any(r.get("force_resume") is True for r in emitted.results) + append_prompt = "\n".join( + r.get("append_prompt") + for r in emitted.results + if isinstance(r.get("append_prompt"), str) and r.get("append_prompt") + ) + if append_prompt: + await self._inject_append_prompt_message(append_prompt, current_request) + if not should_force_resume: + return "stop" + if force_resume_count >= _MAX_FORCE_RESUME_OVERRIDES: + warnings.warn( + f"[Stop hook] force_resume honored {_MAX_FORCE_RESUME_OVERRIDES} times without new " + "progress; stopping to prevent an infinite loop.", + stacklevel=2, + ) + return "stop" + return "resume" + + # -- tool execution ---------------------------------------------------- + + def _find_tool(self, name: str, tools: Sequence[Tool]) -> Optional[Tool]: + return next((t for t in tools if is_client_tool(t) and get_tool_function(t).get("name") == name), None) + + async def _run_tool_with_hooks( + self, tool: Tool, call: ParsedToolCall, context: Mapping[str, Any] + ) -> Dict[str, Any]: + """Execute a single tool and emit the PreToolUse/PostToolUse/ + PostToolUseFailure lifecycle hooks around it. + + Returns a tagged outcome: + - `{"type": "parse_error", "call", "error_message"}` -- the model + produced invalid JSON for this call's arguments. No hooks fire. + - `{"type": "hook_blocked", "call", "reason"}` -- PreToolUse blocked + the call. + - `{"type": "execution", "call", "result"}` -- the tool ran (or + paused, if `result` is `None` for a HITL tool). `call` reflects any + `mutated_input` piped by PreToolUse. + """ + if isinstance(call.arguments, str): + error_message = ( + f'Failed to parse tool call arguments for "{call.name}": The model provided invalid JSON. ' + f'Raw arguments received: "{call.arguments}". ' + "Please provide valid JSON arguments for this tool call." + ) + return {"type": "parse_error", "call": call, "error_message": error_message} + + effective_call = call + if self._hooks: + original_input = call.arguments if isinstance(call.arguments, dict) else {} + pre = await self._hooks.emit( + "PreToolUse", + {"tool_name": call.name, "tool_input": original_input}, + tool_name=call.name, + session_id=self._session_id, + ) + if pre.blocked: + block = next((r.get("block") for r in pre.results if r.get("block")), None) + reason = block if isinstance(block, str) else "Blocked by PreToolUse hook" + return {"type": "hook_blocked", "call": call, "reason": reason} + if pre.mutated: + effective_call = dataclasses.replace(call, arguments=pre.final_payload.get("tool_input")) + + started = time.monotonic() + result = await execute_tool( + tool, + effective_call, + context, + self._record_preliminary, + self._context_store, + self.options.get("shared_context_schema"), + ) + duration_ms = (time.monotonic() - started) * 1000 + + if self._hooks and result is not None: + tool_input = effective_call.arguments if isinstance(effective_call.arguments, dict) else {} + if result.get("error") is not None: + await self._hooks.emit( + "PostToolUseFailure", + {"tool_name": effective_call.name, "tool_input": tool_input, "error": str(result["error"])}, + tool_name=effective_call.name, + session_id=self._session_id, + ) + else: + await self._hooks.emit( + "PostToolUse", + { + "tool_name": effective_call.name, + "tool_input": tool_input, + "tool_output": result.get("result"), + "duration_ms": duration_ms, + }, + tool_name=effective_call.name, + session_id=self._session_id, + ) + + return {"type": "execution", "call": effective_call, "result": result} + + async def _record_preliminary(self, call_id: str, value: Any) -> None: + await self._append_event( + { + "type": "tool.preliminary_result", + "toolCallId": call_id, + "tool_call_id": call_id, + "result": value, + "timestamp": _now_ms(), + } + ) + + async def _tool_result_to_output( + self, call: ParsedToolCall, tool: Tool, result: Mapping[str, Any] + ) -> Dict[str, Any]: + if result.get("error") is not None: + output: Any = json_dumps({"error": str(result["error"])}) + else: + converter = get_tool_function(tool).get("to_model_output") + if converter: + converted = await maybe_await(converter({"output": result.get("result"), "input": call.arguments})) + if isinstance(converted, Mapping) and converted.get("type") == "content": + output = converted.get("value", []) + else: + output = json_dumps(result.get("result")) + else: + output = json_dumps(result.get("result")) + return {"type": "function_call_output", "id": f"output_{call.id}", "callId": call.id, "output": output} + + def _rejected_output(self, call_id: str, reason: str) -> Dict[str, Any]: + return { + "type": "function_call_output", + "id": f"output_{call_id}", + "callId": call_id, + "output": json_dumps({"error": reason}), + } + + # -- the main loop ----------------------------------------------------- + + async def _run(self) -> Any: + session_end_reason = "complete" + hooks = self._hooks + try: + await self._load_state() + self._session_id = self._state.id if self._state is not None else generate_conversation_id() + + tools: Sequence[Tool] = self.options.get("tools") or [] + await self._emit_session_start(tools) + + request = await resolve_async_functions(self.options["request"], {"number_of_turns": 0}) + request["stream"] = True + base_input = request.get("input") + + if hooks and base_input is not None: + base_input = await self._maybe_run_user_prompt_submit(base_input) + request["input"] = base_input + + historical_input = normalize_input_to_array(self._state.messages) if self._state is not None else [] + fresh_input = normalize_input_to_array(base_input) + if self.options.get("context") is not None: + resolved_context = await resolve_context(self.options.get("context"), {"number_of_turns": 0}) + self._context_store = ToolContextStore(resolved_context) + if tools and fresh_input: + historical_function_calls = [ + item for item in historical_input if get_field(item, "type") == "function_call" + ] + synthetic_input = [*historical_function_calls, *fresh_input] + hooked_input = await apply_on_response_received_hooks( + synthetic_input, + tools, + {"number_of_turns": 0}, + self._context_store, + self.options.get("shared_context_schema"), + ) + hooked_array = normalize_input_to_array(hooked_input) + fresh_input = hooked_array[len(historical_function_calls) :] + if self._state is not None: + request["input"] = append_to_messages(historical_input, fresh_input) + if not ( + self._state.pending_tool_calls + and (self.options.get("approve_tool_calls") or self.options.get("reject_tool_calls")) + ): + self._fresh_items_for_state = list(fresh_input) + elif fresh_input: + request["input"] = fresh_input + + current_request = request + is_resume_turn = self._resuming_from_client_tools + if ( + self._state is not None + and self._state.pending_tool_calls and (self.options.get("approve_tool_calls") or self.options.get("reject_tool_calls")) ): - self._fresh_items_for_state = list(fresh_input) - elif fresh_input: - request["input"] = fresh_input - - current_request = request - if ( - self._state is not None - and self._state.pending_tool_calls - and (self.options.get("approve_tool_calls") or self.options.get("reject_tool_calls")) - ): - current_request = await self._build_resume_request(request) - final_response = None - try: - for turn_number in range(20): + current_request = await self._build_resume_request(request) + is_resume_turn = True + + final_response = None + force_resume_count = 0 + pending_final_directive = False + + for turn_number in range(_MAX_TURNS): if self.options.get("on_turn_start"): await maybe_await(self.options["on_turn_start"]({"number_of_turns": turn_number})) await self._append_event( @@ -193,16 +634,24 @@ async def _run(self) -> Any: "type": "turn.start", "turnNumber": turn_number, "turn_number": turn_number, - "timestamp": int(time.time() * 1000), + "timestamp": _now_ms(), } ) - response = await self._coerce_response(await self._send(current_request)) + if turn_number == 0: + turn_type = "resume" if is_resume_turn else "initial" + elif pending_final_directive: + turn_type = "final" + else: + turn_type = "tool_round" + pending_final_directive = False + + response = await self._send_and_track(current_request, turn_type, turn_number) await self._append_event( { "type": "turn.end", "turnNumber": turn_number, "turn_number": turn_number, - "timestamp": int(time.time() * 1000), + "timestamp": _now_ms(), } ) if self.options.get("on_turn_end"): @@ -221,48 +670,59 @@ async def _run(self) -> Any: usage=get_field(response, "usage"), ) self._steps.append(step) - if not calls: + if not calls or not tools: await self._save_state(status="complete") break + stop_when = self.options.get("stop_when") stop_conditions = list(stop_when) if isinstance(stop_when, list) else ([stop_when] if stop_when else []) - if stop_conditions and await is_stop_condition_met(stop_conditions, self._steps): - if self.options.get("allow_final_response"): - final_outputs = [] + stopped_by_stop_when = False + while stop_conditions and await is_stop_condition_met(stop_conditions, self._steps): + stop_decision = await self._run_stop_hook(force_resume_count, current_request) + if stop_decision == "resume": + # Zero-cost retry: re-check the stop condition against + # the SAME already-fetched response/steps -- no new + # model request. Bare force_resume alone typically + # doesn't change anything the condition inspects, so + # this burns through the consecutive-override cap + # quickly unless append_prompt (injected above) or + # external state changes what the condition sees. + force_resume_count += 1 + continue + session_end_reason = "max_turns" + stopped_by_stop_when = True + break + + if stopped_by_stop_when: + allow_final_response = self.options.get("allow_final_response") + final_response_enabled = allow_final_response is not False + resolvable_pending = [c for c in calls if self._call_is_auto_resolvable(c, tools)] + if final_response_enabled and resolvable_pending: + final_outputs: List[Any] = [] + turn_context = {"number_of_turns": turn_number + 1, "turn_request": current_request} for call in calls: - matching_tool = next( - ( - candidate - for candidate in tools - if is_client_tool(candidate) - and get_tool_function(candidate).get("name") == call.name - ), - None, - ) + matching_tool = self._find_tool(call.name, tools) if matching_tool and is_auto_resolvable_tool(matching_tool): - result = await execute_tool( - matching_tool, - call, - { - "number_of_turns": turn_number + 1, - "tool_call": call, - "turn_request": current_request, - }, - self._record_preliminary, - self._context_store, - self.options.get("shared_context_schema"), - ) - if result is not None: - output = await self._tool_result_to_output(call, matching_tool, result) - final_outputs.append(output) - self._tool_outputs.append(output) - await self._append_event( - { - "type": "tool.call_output", - "output": output, - "timestamp": int(time.time() * 1000), - } - ) + outcome = await self._run_tool_with_hooks(matching_tool, call, turn_context) + if outcome["type"] == "parse_error": + final_outputs.append(self._rejected_output(call.id, outcome["error_message"])) + elif outcome["type"] == "hook_blocked": + final_outputs.append(self._rejected_output(call.id, outcome["reason"])) + else: + result = outcome["result"] + if result is not None: + output = await self._tool_result_to_output( + outcome["call"], matching_tool, result + ) + final_outputs.append(output) + self._tool_outputs.append(output) + await self._append_event( + { + "type": "tool.call_output", + "output": output, + "timestamp": _now_ms(), + } + ) else: final_outputs.append( { @@ -274,43 +734,113 @@ async def _run(self) -> Any: ) await self._save_tool_outputs_to_state(final_outputs) current_request = self._build_final_request( - current_request, response, final_outputs, self.options.get("allow_final_response") + current_request, response, final_outputs, allow_final_response ) + pending_final_directive = True continue break + partition = await partition_tool_calls( calls, tools, {"number_of_turns": turn_number + 1}, self.options.get("require_approval") ) - if partition["requires_approval"]: + requires_approval = list(partition["requires_approval"]) + hook_resolved_unsent: List[UnsentToolResult] = [] + if requires_approval and hooks: + still_pending: List[ParsedToolCall] = [] + for call in requires_approval: + decision, reason = await self._emit_permission_request(call, tools) + if decision == "allow": + promo_tool = self._find_tool(call.name, tools) + if promo_tool and is_auto_resolvable_tool(promo_tool): + outcome = await self._run_tool_with_hooks( + promo_tool, + call, + { + "number_of_turns": turn_number + 1, + "tool_call": call, + "turn_request": current_request, + }, + ) + if outcome["type"] == "parse_error": + hook_resolved_unsent.append( + create_rejected_result(call.id, call.name, outcome["error_message"]) + ) + elif outcome["type"] == "hook_blocked": + hook_resolved_unsent.append( + create_rejected_result(call.id, call.name, outcome["reason"]) + ) + elif outcome["result"] is None: + still_pending.append(call) + elif outcome["result"].get("error") is not None: + hook_resolved_unsent.append( + create_rejected_result(call.id, call.name, str(outcome["result"]["error"])) + ) + else: + hook_resolved_unsent.append( + create_unsent_result(call.id, call.name, outcome["result"].get("result")) + ) + else: + still_pending.append(call) + elif decision == "deny": + hook_resolved_unsent.append( + create_rejected_result(call.id, call.name, reason or "Denied by PermissionRequest hook") + ) + else: + still_pending.append(call) + requires_approval = still_pending + + if requires_approval: if self.options.get("state") is None: - names = ", ".join(call.name for call in partition["requires_approval"]) + names = ", ".join(call.name for call in requires_approval) raise ValueError(f"Tool(s) require approval but no state accessor is configured: {names}") - await self._save_state( - pending_tool_calls=partition["requires_approval"], status="awaiting_approval" - ) - break - outputs = [] + save_kwargs: Dict[str, Any] = { + "pending_tool_calls": requires_approval, + "status": "awaiting_approval", + } + if hook_resolved_unsent: + save_kwargs["unsent_tool_results"] = hook_resolved_unsent + await self._save_state(**save_kwargs) + self._final_response = final_response + return final_response + + outputs: List[Any] = unsent_results_to_api_format(hook_resolved_unsent) if hook_resolved_unsent else [] paused: List[ParsedToolCall] = [] executed_calls: List[ParsedToolCall] = [] for call in partition["auto_execute"]: - tool = next( - ( - candidate - for candidate in tools - if is_client_tool(candidate) and get_tool_function(candidate).get("name") == call.name - ), - None, - ) + tool = self._find_tool(call.name, tools) if not tool or not is_auto_resolvable_tool(tool): continue - result = await execute_tool( - tool, - call, - {"number_of_turns": turn_number + 1, "tool_call": call, "turn_request": current_request}, - self._record_preliminary, - self._context_store, - self.options.get("shared_context_schema"), - ) + turn_context = { + "number_of_turns": turn_number + 1, + "tool_call": call, + "turn_request": current_request, + } + outcome = await self._run_tool_with_hooks(tool, call, turn_context) + if outcome["type"] == "parse_error": + await self._append_event( + { + "type": "tool.result", + "toolCallId": call.id, + "tool_call_id": call.id, + "source": "mcp" if is_mcp_tool(tool) else "client", + "result": {"error": outcome["error_message"]}, + "timestamp": _now_ms(), + } + ) + output = self._rejected_output(call.id, outcome["error_message"]) + outputs.append(output) + self._tool_outputs.append(output) + await self._append_event({"type": "tool.call_output", "output": output, "timestamp": _now_ms()}) + continue + if outcome["type"] == "hook_blocked": + output = self._rejected_output(call.id, outcome["reason"]) + outputs.append(output) + self._tool_outputs.append(output) + await self._append_event({"type": "tool.call_output", "output": output, "timestamp": _now_ms()}) + continue + + effective_call = outcome["call"] + result = outcome["result"] if result is None: paused.append(call) continue @@ -319,52 +849,84 @@ async def _run(self) -> Any: "type": "tool.result", "toolCallId": call.id, "tool_call_id": call.id, + "source": result.get("source", "mcp" if is_mcp_tool(tool) else "client"), "result": {"error": str(result["error"])} if result.get("error") is not None else result.get("result"), "preliminaryResults": result.get("preliminary_results"), "preliminary_results": result.get("preliminary_results"), - "timestamp": int(time.time() * 1000), + "timestamp": _now_ms(), } ) - output = await self._tool_result_to_output(call, tool, result) + output = await self._tool_result_to_output(effective_call, tool, result) outputs.append(output) - executed_calls.append(call) + executed_calls.append(effective_call) self._tool_outputs.append(output) - await self._append_event( - {"type": "tool.call_output", "output": output, "timestamp": int(time.time() * 1000)} - ) + await self._append_event({"type": "tool.call_output", "output": output, "timestamp": _now_ms()}) step.tool_results.append(result) + if paused: await self._save_tool_outputs_to_state(outputs) await self._save_state(pending_tool_calls=paused, status="awaiting_hitl") - break + self._final_response = final_response + return final_response + + await self._save_tool_outputs_to_state(outputs) + + resolved_ids = {get_field(o, "callId") for o in outputs} + unresolved_calls = [c for c in calls if c.id not in resolved_ids] + if unresolved_calls: + await self._persist_client_tools_pause(response, unresolved_calls) + return final_response + if not outputs: break - await self._save_tool_outputs_to_state(outputs) + self._all_tool_rounds += 1 + force_resume_count = 0 next_params = await execute_next_turn_params_functions(executed_calls, tools, current_request) if next_params: current_request = apply_next_turn_params_to_request(current_request, next_params) current_request = self._build_followup_request(current_request, response, outputs) else: raise RuntimeError("call_model exceeded the 20-turn safety limit") + + # Tolerate an empty final response after at least one completed + # tool round (mini-class models intermittently return an empty + # final turn after the tool call was the answer): retry once, + # then accept the empty output rather than reporting failure -- + # unless strict_final_response opts back into the legacy throw. + can_tolerate_empty = self._all_tool_rounds > 0 and self.options.get("strict_final_response") is not True + output = get_field(final_response, "output") + is_empty_output = isinstance(output, list) and len(output) == 0 + if can_tolerate_empty and is_empty_output: + retry_turn_number = self._all_tool_rounds + 1 + final_response = await self._retry_current_request(current_request, retry_turn_number) + await self._save_response_to_state(final_response) + output = get_field(final_response, "output") + is_empty_output = isinstance(output, list) and len(output) == 0 + + allow_empty_output = can_tolerate_empty and is_empty_output + self._validate_final_response(final_response, allow_empty_output) self._final_response = final_response + await self._save_state(status="complete") return final_response + except Exception: + session_end_reason = "error" + raise finally: + try: + await self._emit_session_end(session_end_reason) + if hooks: + await hooks.drain() + except Exception as error: # noqa: BLE001 - teardown must never mask the real error + warnings.warn(f"[SessionEnd] error during session teardown: {error}", stacklevel=2) async with self._condition: self._run_done = True self._condition.notify_all() - async def _record_preliminary(self, call_id: str, value: Any) -> None: - await self._append_event( - { - "type": "tool.preliminary_result", - "toolCallId": call_id, - "tool_call_id": call_id, - "result": value, - "timestamp": int(time.time() * 1000), - } - ) + def _call_is_auto_resolvable(self, call: ParsedToolCall, tools: Sequence[Tool]) -> bool: + tool = self._find_tool(call.name, tools) + return bool(tool and is_auto_resolvable_tool(tool)) async def _build_resume_request(self, request: Mapping[str, Any]) -> Dict[str, Any]: self._fresh_items_for_state = [] @@ -372,44 +934,21 @@ async def _build_resume_request(self, request: Mapping[str, Any]) -> Dict[str, A rejected = set(self.options.get("reject_tool_calls") or []) outputs: List[Any] = [] if self._state is not None and self._state.unsent_tool_results: - for item in self._state.unsent_tool_results: - if item.error: - output = json_dumps({"error": item.error}) - else: - output = json_dumps(item.output) - outputs.append( - { - "type": "function_call_output", - "id": f"output_{item.call_id}", - "callId": item.call_id, - "output": output, - } - ) + outputs.extend(unsent_results_to_api_format(self._state.unsent_tool_results)) pending: List[ParsedToolCall] = self._state.pending_tool_calls if self._state is not None else [] for call in pending or []: - tool = next( - ( - candidate - for candidate in self.options.get("tools") or [] - if is_client_tool(candidate) and get_tool_function(candidate).get("name") == call.name - ), - None, - ) + tool = self._find_tool(call.name, self.options.get("tools") or []) if call.id in rejected: - outputs.append( - { - "type": "function_call_output", - "id": f"output_{call.id}", - "callId": call.id, - "output": json_dumps({"error": "Tool call rejected by user"}), - } - ) + outputs.append(self._rejected_output(call.id, "Tool call rejected by user")) elif call.id in approved and tool and is_auto_resolvable_tool(tool): - result = await execute_tool( - tool, call, {"number_of_turns": 0}, self._record_preliminary, self._context_store - ) - if result is not None: - outputs.append(await self._tool_result_to_output(call, tool, result)) + turn_context = {"number_of_turns": 0} + outcome = await self._run_tool_with_hooks(tool, call, turn_context) + if outcome["type"] == "parse_error": + outputs.append(self._rejected_output(call.id, outcome["error_message"])) + elif outcome["type"] == "hook_blocked": + outputs.append(self._rejected_output(call.id, outcome["reason"])) + elif outcome["result"] is not None: + outputs.append(await self._tool_result_to_output(outcome["call"], tool, outcome["result"])) updated_messages = append_to_messages(self._state.messages, outputs) if self._state is not None else outputs await self._save_state( messages=updated_messages, @@ -436,23 +975,6 @@ async def _iter_events_live(self) -> AsyncIterator[Any]: yield event await self.get_response() - async def _tool_result_to_output( - self, call: ParsedToolCall, tool: Tool, result: Mapping[str, Any] - ) -> Dict[str, Any]: - if result.get("error") is not None: - output: Any = json_dumps({"error": str(result["error"])}) - else: - converter = get_tool_function(tool).get("to_model_output") - if converter: - converted = await maybe_await(converter({"output": result.get("result"), "input": call.arguments})) - if isinstance(converted, Mapping) and converted.get("type") == "content": - output = converted.get("value", []) - else: - output = json_dumps(result.get("result")) - else: - output = json_dumps(result.get("result")) - return {"type": "function_call_output", "id": f"output_{call.id}", "callId": call.id, "output": output} - def _build_followup_request( self, request: Mapping[str, Any], response: Any, outputs: Sequence[Any] ) -> Dict[str, Any]: @@ -466,19 +988,37 @@ def _build_followup_request( } def _build_final_request( - self, request: Mapping[str, Any], response: Any, outputs: Sequence[Any], final: Any + self, request: Mapping[str, Any], response: Any, outputs: Sequence[Any], allow_final_response: Any ) -> Dict[str, Any]: new_request = self._build_followup_request(request, response, outputs) - new_request.pop("tools", None) - new_request.pop("tool_choice", None) - new_request.pop("parallel_tool_calls", None) - if isinstance(final, str) and final: + # Forbid tool calls without dropping the `tools` block: removing it + # would invalidate the prompt-cache prefix. + if new_request.get("tools") is not None: + new_request["tool_choice"] = "none" + directive = ( + DEFAULT_FINAL_RESPONSE_DIRECTIVE + if allow_final_response is True or allow_final_response is None + else allow_final_response + ) + if isinstance(directive, str) and directive: new_request["input"] = [ *normalize_input_to_array(new_request.get("input")), - {"role": "user", "content": final}, + {"role": "user", "content": directive}, ] return new_request + async def _retry_current_request(self, current_request: Mapping[str, Any], turn_number: int) -> Any: + """Re-send the current resolved request once, forcing `tool_choice: + "none"` when tools are present so the retry coerces a text turn + instead of a fresh (silently dropped) function call.""" + new_request = dict(current_request) + if new_request.get("tools") is not None: + new_request["tool_choice"] = "none" + new_request["stream"] = True + return await self._send_and_track(new_request, "retry", turn_number) + + # -- public streaming/result surface ------------------------------------ + async def get_response(self) -> Any: return await self._ensure_task() @@ -532,6 +1072,7 @@ async def get_tool_stream(self) -> AsyncIterator[Dict[str, Any]]: "type": "tool_result", "toolCallId": get_field(event, "toolCallId"), "tool_call_id": get_field(event, "tool_call_id", get_field(event, "toolCallId")), + "source": get_field(event, "source", "client"), "result": get_field(event, "result"), "preliminaryResults": get_field(event, "preliminaryResults"), "preliminary_results": get_field(event, "preliminary_results"), @@ -653,7 +1194,13 @@ async def get_full_chat_stream(self) -> AsyncIterator[Dict[str, Any]]: async def requires_approval(self) -> bool: await self.get_response() state = self._state - return bool(state and (state.status in {"awaiting_approval", "awaiting_hitl"} or state.pending_tool_calls)) + return bool( + state + and ( + state.status in {"awaiting_approval", "awaiting_hitl", "awaiting_client_tools"} + or state.pending_tool_calls + ) + ) async def get_pending_tool_calls(self) -> List[ParsedToolCall]: await self.get_response() diff --git a/src/openrouter_agent/reusable_stream.py b/src/openrouter_agent/reusable_stream.py index 797a5c7..5fb982a 100644 --- a/src/openrouter_agent/reusable_stream.py +++ b/src/openrouter_agent/reusable_stream.py @@ -18,6 +18,13 @@ def _ensure_started(self) -> None: self._started = True asyncio.create_task(self._pump()) + @property + def is_complete(self) -> bool: + """True once the source stream has been fully read into the buffer. + A fresh consumer created after this point replays the retained + buffer without waiting on the source.""" + return self._complete + async def _pump(self) -> None: try: async for item in self._source: diff --git a/src/openrouter_agent/tool.py b/src/openrouter_agent/tool.py index cb19221..4bd894b 100644 --- a/src/openrouter_agent/tool.py +++ b/src/openrouter_agent/tool.py @@ -52,3 +52,13 @@ def tool( def server_tool(config: Dict[str, Any]) -> Dict[str, Any]: return {"_brand": "server-tool", "config": dict(config)} + + +def mark_mcp(tool_to_mark: Dict[str, Any]) -> Dict[str, Any]: + """Add the additive MCP brand to an already-built client tool (see + `is_mcp_tool`). Non-mutating: returns a shallow copy carrying `_mcp`, so + the tool's runtime behavior and wire shape are unchanged -- only the + `is_mcp_tool` check now identifies it as MCP-originated. Used by + `@openrouter/mcp`-equivalent integrations to mark wrapped remote tools. + """ + return {**tool_to_mark, "_mcp": True} diff --git a/src/openrouter_agent/tool_executor.py b/src/openrouter_agent/tool_executor.py index 9a86004..a768529 100644 --- a/src/openrouter_agent/tool_executor.py +++ b/src/openrouter_agent/tool_executor.py @@ -11,6 +11,7 @@ is_client_tool, is_generator_tool, is_hitl_tool, + is_mcp_tool, is_server_tool, ) @@ -58,14 +59,21 @@ async def execute_regular_tool( tool: Tool, tool_call: ParsedToolCall, context: Optional[Mapping[str, Any]] = None ) -> Dict[str, Any]: fn = get_tool_function(tool) + source = "mcp" if is_mcp_tool(tool) else "client" try: args = validate_schema(fn.get("input_schema"), tool_call.arguments) result = await maybe_await(fn["execute"](args, context)) if fn.get("output_schema") is not None: result = validate_schema(fn.get("output_schema"), result) - return {"tool_call_id": tool_call.id, "tool_name": tool_call.name, "result": result} + return {"tool_call_id": tool_call.id, "tool_name": tool_call.name, "source": source, "result": result} except Exception as exc: - return {"tool_call_id": tool_call.id, "tool_name": tool_call.name, "result": None, "error": exc} + return { + "tool_call_id": tool_call.id, + "tool_name": tool_call.name, + "source": source, + "result": None, + "error": exc, + } async def execute_generator_tool( @@ -75,6 +83,7 @@ async def execute_generator_tool( on_preliminary_result: Optional[Callable[[str, Any], Any]] = None, ) -> Dict[str, Any]: fn = get_tool_function(tool) + source = "mcp" if is_mcp_tool(tool) else "client" preliminary: List[Any] = [] try: args = validate_schema(fn.get("input_schema"), tool_call.arguments) @@ -120,6 +129,7 @@ async def execute_generator_tool( return { "tool_call_id": tool_call.id, "tool_name": tool_call.name, + "source": source, "result": final, "preliminary_results": preliminary, } @@ -127,6 +137,7 @@ async def execute_generator_tool( return { "tool_call_id": tool_call.id, "tool_name": tool_call.name, + "source": source, "result": None, "preliminary_results": preliminary, "error": exc, @@ -137,15 +148,22 @@ async def execute_hitl_tool( tool: Tool, tool_call: ParsedToolCall, context: Optional[Mapping[str, Any]] = None ) -> Optional[Dict[str, Any]]: fn = get_tool_function(tool) + source = "mcp" if is_mcp_tool(tool) else "client" try: args = validate_schema(fn.get("input_schema"), tool_call.arguments) result = await maybe_await(fn["on_tool_called"](args, context)) if result is None: return None result = validate_schema(fn.get("output_schema"), result) - return {"tool_call_id": tool_call.id, "tool_name": tool_call.name, "result": result} + return {"tool_call_id": tool_call.id, "tool_name": tool_call.name, "source": source, "result": result} except Exception as exc: - return {"tool_call_id": tool_call.id, "tool_name": tool_call.name, "result": None, "error": exc} + return { + "tool_call_id": tool_call.id, + "tool_name": tool_call.name, + "source": source, + "result": None, + "error": exc, + } async def execute_tool( diff --git a/src/openrouter_agent/tool_types.py b/src/openrouter_agent/tool_types.py index 22c90f5..f6db4b9 100644 --- a/src/openrouter_agent/tool_types.py +++ b/src/openrouter_agent/tool_types.py @@ -52,6 +52,11 @@ class ConversationState: unsent_tool_results: Optional[List[UnsentToolResult]] = None partial_response: Optional[PartialResponse] = None interrupted_by: Optional[str] = None + #: Serialization-contract version for this state blob (see + #: `serialize_conversation_state` / `deserialize_conversation_state`). + #: Optional so legacy (pre-version-field) states remain constructible; + #: absence is treated as version 1 by `deserialize_conversation_state`. + version: Optional[int] = None @dataclass(frozen=True) @@ -121,6 +126,10 @@ class ServerTool(TypedDict): class ToolExecutionResult(TypedDict, total=False): tool_call_id: str tool_name: str + #: Origin of the tool: "mcp" for tools branded via `mark_mcp` (wrapped + #: from a remote MCP server, whose result is untyped), "client" for + #: locally-defined tools. See `is_mcp_tool` / `mark_mcp`. + source: Literal["client", "mcp"] result: object preliminary_results: List[object] error: BaseException @@ -148,6 +157,9 @@ class ToolResultEvent(TypedDict, total=False): type: Literal["tool.result", "tool_result"] toolCallId: str tool_call_id: str + #: Origin of the tool result: "mcp" for MCP-branded tools, "client" + #: otherwise. See `is_mcp_tool` / `mark_mcp`. + source: Literal["client", "mcp"] result: object preliminaryResults: List[object] preliminary_results: List[object] @@ -245,6 +257,19 @@ def is_client_tool(tool: Mapping[str, Any]) -> bool: return not is_server_tool(tool) +#: A client tool additionally branded as originating from an MCP server (see +#: `mark_mcp`). Purely a structural marker: it does not change execution or +#: wire serialization, only how `is_mcp_tool` and the `source` field on tool +#: results discriminate it from a precisely-typed client tool. +McpBranded = Mapping[str, Any] + + +def is_mcp_tool(tool: Mapping[str, Any]) -> bool: + """Type guard: true if the tool carries the additive MCP brand (see + `mark_mcp`).""" + return bool(isinstance(tool, Mapping) and tool.get("_mcp") is True) + + def get_tool_function(tool: Mapping[str, Any]) -> Dict[str, Any]: value = tool.get("function", {}) return value if isinstance(value, dict) else {} diff --git a/tests/unit/test_allow_final_response.py b/tests/unit/test_allow_final_response.py new file mode 100644 index 0000000..3eec26c --- /dev/null +++ b/tests/unit/test_allow_final_response.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +from typing import Any, Dict, List + +from openrouter_agent import call_model, step_count_is, tool +from openrouter_agent.model_result import DEFAULT_FINAL_RESPONSE_DIRECTIVE + + +class QueuedResponses: + def __init__(self, responses: List[Dict[str, Any]]) -> None: + self._responses = list(responses) + self.requests: List[Dict[str, Any]] = [] + + async def send_async(self, **kwargs: Any) -> Any: + self.requests.append(kwargs) + return self._responses.pop(0) + + +class QueuedClient: + def __init__(self, responses: List[Dict[str, Any]]) -> None: + self.beta = type("Beta", (), {"responses": QueuedResponses(responses)})() + + +def function_call_item(call_id: str, name: str, arguments: str) -> Dict[str, Any]: + return {"type": "function_call", "id": f"fc_{call_id}", "callId": call_id, "name": name, "arguments": arguments} + + +def text_response(response_id: str, text: str) -> Dict[str, Any]: + return { + "id": response_id, + "output": [{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": text}]}], + } + + +def tool_call_response(response_id: str) -> Dict[str, Any]: + return {"id": response_id, "output": [function_call_item("call_weather", "get_weather", '{"city":"nyc"}')]} + + +weather_tool = tool( + name="get_weather", + input_schema=dict, + output_schema=dict, + execute=lambda params, ctx: {"temperature": 22}, +) + + +async def test_bare_true_appends_default_directive() -> None: + client = QueuedClient([tool_call_response("resp_1"), text_response("resp_2", "Final summary.")]) + + text = await call_model( + client, + { + "model": "test-model", + "input": "weather?", + "tools": [weather_tool], + "stop_when": step_count_is(0), + "allow_final_response": True, + }, + ).get_text() + + assert text == "Final summary." + second_request = client.beta.responses.requests[1] + assert "tools" in second_request + assert second_request["tool_choice"] == "none" + last_item = second_request["input"][-1] + assert last_item == {"role": "user", "content": DEFAULT_FINAL_RESPONSE_DIRECTIVE} + + +async def test_omitted_allow_final_response_defaults_to_enabled_with_directive() -> None: + client = QueuedClient([tool_call_response("resp_1"), text_response("resp_2", "Final summary.")]) + + text = await call_model( + client, + { + "model": "test-model", + "input": "weather?", + "tools": [weather_tool], + "stop_when": step_count_is(0), + # allow_final_response deliberately omitted + }, + ).get_text() + + assert text == "Final summary." + second_request = client.beta.responses.requests[1] + assert second_request["tool_choice"] == "none" + assert second_request["input"][-1] == {"role": "user", "content": DEFAULT_FINAL_RESPONSE_DIRECTIVE} + + +async def test_non_empty_string_overrides_directive() -> None: + client = QueuedClient([tool_call_response("resp_1"), text_response("resp_2", "Final summary.")]) + + await call_model( + client, + { + "model": "test-model", + "input": "weather?", + "tools": [weather_tool], + "stop_when": step_count_is(0), + "allow_final_response": "Summarize now.", + }, + ).get_text() + + second_request = client.beta.responses.requests[1] + assert second_request["input"][-1] == {"role": "user", "content": "Summarize now."} + + +async def test_empty_string_appends_no_message() -> None: + client = QueuedClient([tool_call_response("resp_1"), text_response("resp_2", "Final summary.")]) + + await call_model( + client, + { + "model": "test-model", + "input": "weather?", + "tools": [weather_tool], + "stop_when": step_count_is(0), + "allow_final_response": "", + }, + ).get_text() + + second_request = client.beta.responses.requests[1] + last_item = second_request["input"][-1] + assert last_item.get("type") == "function_call_output" + + +async def test_false_disables_the_final_turn_entirely() -> None: + client = QueuedClient([tool_call_response("resp_1")]) + + result = call_model( + client, + { + "model": "test-model", + "input": "weather?", + "tools": [weather_tool], + "stop_when": step_count_is(0), + "allow_final_response": False, + }, + ) + response = await result.get_response() + + assert response["id"] == "resp_1" + assert len(client.beta.responses.requests) == 1 diff --git a/tests/unit/test_call_model.py b/tests/unit/test_call_model.py index bc60968..aedb529 100644 --- a/tests/unit/test_call_model.py +++ b/tests/unit/test_call_model.py @@ -96,7 +96,10 @@ async def test_allow_final_response_executes_pending_tool_before_no_tools_turn() assert await result.get_text() == "4" assert len(client.beta.responses.requests) == 2 - assert "tools" not in client.beta.responses.requests[1] + # Tools stay in the request (prompt-cache prefix preserved); calling is + # forbidden via tool_choice instead of stripping the tools block. + assert "tools" in client.beta.responses.requests[1] + assert client.beta.responses.requests[1]["tool_choice"] == "none" second_input = client.beta.responses.requests[1]["input"] types = [item.get("type") for item in second_input] assert types.index("function_call") < types.index("function_call_output") diff --git a/tests/unit/test_conversation_state_serialization.py b/tests/unit/test_conversation_state_serialization.py new file mode 100644 index 0000000..69fd3f6 --- /dev/null +++ b/tests/unit/test_conversation_state_serialization.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +import dataclasses + +import pytest + +from openrouter_agent import ( + CONVERSATION_STATE_VERSION, + InvalidStateError, + UnsupportedStateVersionError, + create_initial_state, + deserialize_conversation_state, + serialize_conversation_state, +) +from openrouter_agent.tool_types import ParsedToolCall, UnsentToolResult + + +def test_round_trips_a_fresh_initial_state_with_version_1() -> None: + state = create_initial_state("conv_fresh") + assert state.version == 1 + + restored = deserialize_conversation_state(serialize_conversation_state(state)) + + assert restored == state + assert restored.version == 1 + + +def test_round_trips_a_rich_awaiting_client_tools_state() -> None: + base = create_initial_state("conv_manual_pause") + rich = dataclasses.replace( + base, + status="awaiting_client_tools", + previous_response_id="resp_manual", + messages=[ + {"type": "message", "role": "user", "content": "run ls"}, + { + "type": "function_call", + "id": "fc_call_manual_1", + "callId": "call_manual_1", + "name": "exec_command", + "arguments": '{"command":"ls"}', + "status": "completed", + }, + ], + pending_tool_calls=[ParsedToolCall(id="call_manual_1", name="exec_command", arguments={"command": "ls"})], + unsent_tool_results=[ + UnsentToolResult(call_id="call_auto_1", name="auto_search", output={"result": "found it"}) + ], + ) + + restored = deserialize_conversation_state(serialize_conversation_state(rich)) + + assert restored == rich + assert restored.status == "awaiting_client_tools" + assert restored.pending_tool_calls == [ + ParsedToolCall(id="call_manual_1", name="exec_command", arguments={"command": "ls"}) + ] + assert restored.unsent_tool_results[0].call_id == "call_auto_1" + + +def test_deserializes_version_less_legacy_json_and_normalizes_to_version_1() -> None: + legacy_json = ( + '{"id": "conv_legacy", "messages": [{"type": "message", "role": "user", "content": "hi"}], ' + '"status": "complete", "created_at": 1600000000000, "updated_at": 1600000000100, ' + '"previous_response_id": "resp_legacy"}' + ) + + restored = deserialize_conversation_state(legacy_json) + + assert restored.version == 1 + assert restored.id == "conv_legacy" + assert restored.status == "complete" + assert len(restored.messages) == 1 + assert restored.previous_response_id == "resp_legacy" + + +def test_raises_unsupported_state_version_error_for_a_future_version() -> None: + future_json = ( + '{"version": 2, "id": "conv_future", "messages": [], "status": "in_progress", "created_at": 1, "updated_at": 1}' + ) + + with pytest.raises(UnsupportedStateVersionError) as exc_info: + deserialize_conversation_state(future_json) + + error = exc_info.value + assert error.found == 2 + assert error.supported == [1] + assert error.name == "UnsupportedStateVersionError" + + +def test_raises_invalid_state_error_for_malformed_json() -> None: + with pytest.raises(InvalidStateError): + deserialize_conversation_state("not json") + + +@pytest.mark.parametrize( + "raw_json", + [ + '{"messages": [], "status": "in_progress", "created_at": 1, "updated_at": 1}', # missing id + '{"id": "x", "status": "in_progress", "created_at": 1, "updated_at": 1}', # missing messages + '{"id": "x", "messages": "not-an-array", "status": "in_progress", "created_at": 1, "updated_at": 1}', + '{"id": "x", "messages": [], "created_at": 1, "updated_at": 1}', # missing status + '{"id": "x", "messages": [], "status": "in_progress", "updated_at": 1}', # missing created_at + ], +) +def test_raises_invalid_state_error_for_missing_required_fields(raw_json: str) -> None: + with pytest.raises(InvalidStateError): + deserialize_conversation_state(raw_json) + + +def test_serialize_injects_version_when_absent() -> None: + state = create_initial_state("conv_no_version") + stateless = dataclasses.replace(state, version=None) + + json_str = serialize_conversation_state(stateless) + + assert '"version":1' in json_str.replace(" ", "") + + +def test_conversation_state_version_constant() -> None: + assert CONVERSATION_STATE_VERSION == 1 diff --git a/tests/unit/test_hooks_manager.py b/tests/unit/test_hooks_manager.py new file mode 100644 index 0000000..f25397d --- /dev/null +++ b/tests/unit/test_hooks_manager.py @@ -0,0 +1,200 @@ +from __future__ import annotations + +import asyncio + +import pytest +from pydantic import ValidationError + +from openrouter_agent import HookEntry, HookName, HooksManager +from openrouter_agent.hooks_types import AsyncOutput + + +async def test_emit_validates_payload_and_invokes_matching_handlers() -> None: + manager = HooksManager() + seen = [] + + def handler(payload, ctx): + seen.append((payload, ctx)) + return {"block": False} + + manager.on(HookName.PreToolUse.value, HookEntry(handler=handler)) + + result = await manager.emit(HookName.PreToolUse.value, {"tool_name": "search", "tool_input": {"q": "x"}}) + + assert result.results == [{"mutated_input": None, "block": False}] + assert seen[0][0] == {"tool_name": "search", "tool_input": {"q": "x"}} + assert seen[0][1].session_id == "" + assert seen[0][1].hook_name == HookName.PreToolUse.value + + +async def test_emit_invalid_payload_warns_and_skips_handlers_by_default() -> None: + manager = HooksManager() + calls = [] + manager.on(HookName.PreToolUse.value, HookEntry(handler=lambda payload, ctx: calls.append(payload))) + + with pytest.warns(UserWarning): + result = await manager.emit(HookName.PreToolUse.value, {"tool_name": "search"}) # missing tool_input + + assert calls == [] + assert result.results == [] + + +async def test_emit_invalid_payload_raises_when_throw_on_handler_error() -> None: + manager = HooksManager(throw_on_handler_error=True) + manager.on(HookName.PreToolUse.value, HookEntry(handler=lambda payload, ctx: None)) + + with pytest.raises(RuntimeError): + await manager.emit(HookName.PreToolUse.value, {"tool_name": "search"}) + + +async def test_pre_tool_use_block_short_circuits_and_mutation_pipes_into_payload() -> None: + manager = HooksManager() + + def mutate(payload, ctx): + return {"mutated_input": {"q": "mutated"}} + + def block(payload, ctx): + assert payload["tool_input"] == {"q": "mutated"} + return {"block": "not allowed"} + + never_called = [] + manager.on(HookName.PreToolUse.value, HookEntry(handler=mutate)) + manager.on(HookName.PreToolUse.value, HookEntry(handler=block)) + manager.on(HookName.PreToolUse.value, HookEntry(handler=lambda p, c: never_called.append(p))) + + result = await manager.emit(HookName.PreToolUse.value, {"tool_name": "search", "tool_input": {"q": "orig"}}) + + assert result.blocked is True + assert result.mutated is True + assert result.final_payload["tool_input"] == {"q": "mutated"} + assert never_called == [] + + +async def test_tool_matcher_filters_by_tool_name() -> None: + manager = HooksManager() + calls = [] + manager.on( + HookName.PreToolUse.value, + HookEntry(handler=lambda p, c: calls.append(p["tool_name"]), matcher="search"), + ) + + await manager.emit(HookName.PreToolUse.value, {"tool_name": "other", "tool_input": {}}, tool_name="other") + await manager.emit(HookName.PreToolUse.value, {"tool_name": "search", "tool_input": {}}, tool_name="search") + + assert calls == ["search"] + + +async def test_regex_matcher_and_predicate_matcher() -> None: + import re + + manager = HooksManager() + seen = [] + manager.on( + HookName.PreToolUse.value, + HookEntry(handler=lambda p, c: seen.append("regex"), matcher=re.compile(r"^search_.*")), + ) + manager.on( + HookName.PreToolUse.value, + HookEntry(handler=lambda p, c: seen.append("predicate"), matcher=lambda name: name == "search_docs"), + ) + + await manager.emit(HookName.PreToolUse.value, {"tool_name": "x", "tool_input": {}}, tool_name="search_docs") + + assert seen == ["regex", "predicate"] + + +async def test_off_removes_handler_and_remove_all_clears_hook() -> None: + manager = HooksManager() + calls = [] + + def handler(payload, ctx): + calls.append(payload) + + manager.on(HookName.PreToolUse.value, HookEntry(handler=handler)) + assert manager.off(HookName.PreToolUse.value, handler) is True + await manager.emit(HookName.PreToolUse.value, {"tool_name": "x", "tool_input": {}}) + assert calls == [] + + manager.on(HookName.PreToolUse.value, HookEntry(handler=handler)) + manager.remove_all(HookName.PreToolUse.value) + assert manager.has_handlers(HookName.PreToolUse.value) is False + + +async def test_async_output_is_tracked_and_drained() -> None: + manager = HooksManager() + done = [] + + async def background() -> None: + await asyncio.sleep(0.01) + done.append("bg") + + def handler(payload, ctx): + return AsyncOutput(work=background()) + + manager.on(HookName.PostToolUse.value, HookEntry(handler=handler)) + result = await manager.emit( + HookName.PostToolUse.value, + {"tool_name": "x", "tool_input": {}, "tool_output": 1, "duration_ms": 1.0}, + ) + + assert len(result.pending) == 1 + assert done == [] + await manager.drain() + assert done == ["bg"] + + +async def test_session_id_threads_per_emit_for_shared_manager() -> None: + manager = HooksManager() + seen = [] + manager.on(HookName.SessionStart.value, HookEntry(handler=lambda p, c: seen.append(c.session_id))) + + manager.set_session_id("default") + await manager.emit(HookName.SessionStart.value, {}, session_id="run-a") + await manager.emit(HookName.SessionStart.value, {}, session_id="run-b") + await manager.emit(HookName.SessionStart.value, {}) + + assert seen == ["run-a", "run-b", "default"] + + +async def test_custom_hook_registration_collides_with_built_in_name() -> None: + from openrouter_agent.hooks_schemas import HookDefinition, PreToolUsePayload + + with pytest.raises(ValueError): + HooksManager({HookName.PreToolUse.value: HookDefinition(payload=PreToolUsePayload, result=None)}) + + +async def test_void_result_hook_accepts_arbitrary_handler_return_values() -> None: + manager = HooksManager() + manager.on(HookName.PostToolUse.value, HookEntry(handler=lambda p, c: "anything goes")) + + result = await manager.emit( + HookName.PostToolUse.value, + {"tool_name": "x", "tool_input": {}, "tool_output": 1, "duration_ms": 1.0}, + ) + + assert result.results == ["anything goes"] + + +async def test_custom_void_result_hook_skips_validation_like_built_ins() -> None: + from openrouter_agent.hooks_schemas import HookDefinition + from pydantic import BaseModel + + class AuditPayload(BaseModel): + pass + + manager = HooksManager( + {"Audit": HookDefinition(payload=AuditPayload, result=None)}, + throw_on_handler_error=True, + ) + manager.on("Audit", HookEntry(handler=lambda p, c: {"logged": True})) + + result = await manager.emit("Audit", {}) + + assert result.results == [{"logged": True}] + + +def test_stop_result_pydantic_schema_validates_fields() -> None: + from openrouter_agent.hooks_schemas import StopResult + + with pytest.raises(ValidationError): + StopResult.model_validate({"force_resume": "not-a-bool"}) diff --git a/tests/unit/test_hooks_matchers.py b/tests/unit/test_hooks_matchers.py new file mode 100644 index 0000000..6e07e21 --- /dev/null +++ b/tests/unit/test_hooks_matchers.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import re + +from openrouter_agent.hooks_matchers import matches_tool + + +def test_matches_all_tools_when_matcher_is_none() -> None: + assert matches_tool(None, "Bash") is True + assert matches_tool(None, "ReadFile") is True + + +def test_matches_exact_string() -> None: + assert matches_tool("Bash", "Bash") is True + assert matches_tool("Bash", "ReadFile") is False + assert matches_tool("Bash", "bash") is False + + +def test_matches_regex_pattern() -> None: + pattern = re.compile(r"^(Read|Write)File$") + assert matches_tool(pattern, "ReadFile") is True + assert matches_tool(pattern, "WriteFile") is True + assert matches_tool(pattern, "DeleteFile") is False + + +def test_matches_function_predicate() -> None: + matcher = lambda name: name.startswith("File") # noqa: E731 + assert matches_tool(matcher, "FileRead") is True + assert matches_tool(matcher, "Bash") is False + + +def test_regex_matcher_is_stateless_across_repeated_calls() -> None: + # Python compiled patterns have no JS-style lastIndex statefulness, but + # pin the repeated-call behavior anyway since it mirrors an upstream fix. + pattern = re.compile(r"^tool_\w+$") + for _ in range(3): + assert matches_tool(pattern, "tool_abc") is True diff --git a/tests/unit/test_manual_tool_pending_state.py b/tests/unit/test_manual_tool_pending_state.py new file mode 100644 index 0000000..d5dd25b --- /dev/null +++ b/tests/unit/test_manual_tool_pending_state.py @@ -0,0 +1,250 @@ +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from openrouter_agent import call_model, tool +from openrouter_agent.tool_types import ConversationState + + +class QueuedResponses: + def __init__(self, responses: List[Dict[str, Any]]) -> None: + self._responses = list(responses) + self.requests: List[Dict[str, Any]] = [] + + async def send_async(self, **kwargs: Any) -> Any: + self.requests.append(kwargs) + return self._responses.pop(0) + + +class QueuedClient: + def __init__(self, responses: List[Dict[str, Any]]) -> None: + self.beta = type("Beta", (), {"responses": QueuedResponses(responses)})() + + +class MemoryStateAccessor: + def __init__(self) -> None: + self.stored: Optional[ConversationState] = None + + async def load(self) -> Optional[ConversationState]: + return self.stored + + async def save(self, state: ConversationState) -> None: + self.stored = state + + +def function_call_item(call_id: str, name: str, arguments: str) -> Dict[str, Any]: + return { + "type": "function_call", + "id": f"fc_{call_id}", + "callId": call_id, + "name": name, + "arguments": arguments, + } + + +def text_response(response_id: str, text: str) -> Dict[str, Any]: + return { + "id": response_id, + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": text}], + } + ], + } + + +def make_response(response_id: str, output: List[Dict[str, Any]]) -> Dict[str, Any]: + return {"id": response_id, "output": output} + + +auto_tool = tool( + name="auto_search", + input_schema=dict, + output_schema=dict, + execute=lambda params, ctx: {"result": "found it"}, +) + +# No `execute` -- the client is responsible for running this tool. +manual_tool = tool(name="exec_command", input_schema=dict, execute=False) + + +async def test_all_manual_round_stops_loop_with_awaiting_client_tools() -> None: + client = QueuedClient( + [make_response("resp_manual", [function_call_item("call_manual_1", "exec_command", '{"command":"ls"}')])] + ) + accessor = MemoryStateAccessor() + + result = call_model( + client, + {"model": "test-model", "input": "run ls", "tools": [manual_tool], "state": accessor}, + ) + + pending = await result.get_pending_tool_calls() + assert len(pending) == 1 + assert pending[0].id == "call_manual_1" + assert pending[0].name == "exec_command" + + state = await result.get_state() + assert state.status == "awaiting_client_tools" + assert len(state.pending_tool_calls) == 1 + + # No follow-up request -- the loop stopped after the unresolved manual call. + assert len(client.beta.responses.requests) == 1 + assert accessor.stored.status == "awaiting_client_tools" + + +async def test_mixed_auto_and_manual_round_persists_auto_output_and_pauses_manual() -> None: + client = QueuedClient( + [ + make_response( + "resp_mixed", + [ + function_call_item("call_auto_1", "auto_search", '{"query":"docs"}'), + function_call_item("call_manual_1", "exec_command", '{"command":"ls"}'), + ], + ) + ] + ) + accessor = MemoryStateAccessor() + + result = call_model( + client, + { + "model": "test-model", + "input": "do both", + "tools": [auto_tool, manual_tool], + "state": accessor, + }, + ) + + pending = await result.get_pending_tool_calls() + assert len(pending) == 1 + assert pending[0].id == "call_manual_1" + + # No follow-up request: it would contain exec_command's function_call with + # no matching function_call_output, which providers reject. + assert len(client.beta.responses.requests) == 1 + + state = await result.get_state() + assert state.status == "awaiting_client_tools" + + auto_output = next( + (m for m in state.messages if m.get("type") == "function_call_output" and m.get("callId") == "call_auto_1"), + None, + ) + assert auto_output is not None + assert "found it" in auto_output["output"] + manual_output = next( + (m for m in state.messages if m.get("type") == "function_call_output" and m.get("callId") == "call_manual_1"), + None, + ) + assert manual_output is None + + +async def test_no_state_accessor_nothing_persisted_but_response_readable() -> None: + client = QueuedClient( + [make_response("resp_manual", [function_call_item("call_manual_1", "exec_command", '{"command":"ls"}')])] + ) + + result = call_model(client, {"model": "test-model", "input": "run ls", "tools": [manual_tool]}) + + response = await result.get_response() + assert response["id"] == "resp_manual" + assert len(client.beta.responses.requests) == 1 + + pending = await result.get_pending_tool_calls() + assert pending == [] + + +async def test_clears_pending_manual_calls_only_after_a_resume_succeeds() -> None: + accessor = MemoryStateAccessor() + client = QueuedClient( + [ + make_response("resp_manual", [function_call_item("call_manual_1", "exec_command", '{"command":"pwd"}')]), + text_response("resp_done", "done"), + ] + ) + + await call_model( + client, {"model": "test-model", "input": "run pwd", "tools": [manual_tool], "state": accessor} + ).get_pending_tool_calls() + + await call_model( + client, + { + "model": "test-model", + "input": [{"type": "function_call_output", "callId": "call_manual_1", "output": '{"stdout":"/tmp"}'}], + "tools": [manual_tool], + "state": accessor, + }, + ).get_response() + + assert accessor.stored.status == "complete" + assert not accessor.stored.pending_tool_calls + + +async def test_keeps_pending_manual_calls_when_a_resume_request_fails() -> None: + accessor = MemoryStateAccessor() + + class FlakyResponses: + def __init__(self) -> None: + self.calls = 0 + self.requests: List[Dict[str, Any]] = [] + + async def send_async(self, **kwargs: Any) -> Any: + self.requests.append(kwargs) + self.calls += 1 + if self.calls == 1: + return make_response( + "resp_manual", [function_call_item("call_manual_1", "exec_command", '{"command":"pwd"}')] + ) + raise RuntimeError("temporary failure") + + client = type("Client", (), {"beta": type("Beta", (), {"responses": FlakyResponses()})()})() + + await call_model( + client, {"model": "test-model", "input": "run pwd", "tools": [manual_tool], "state": accessor} + ).get_pending_tool_calls() + + try: + await call_model( + client, + { + "model": "test-model", + "input": [{"type": "function_call_output", "callId": "call_manual_1", "output": '{"stdout":"/tmp"}'}], + "tools": [manual_tool], + "state": accessor, + }, + ).get_response() + raised = False + except RuntimeError: + raised = True + + assert raised + assert accessor.stored.status == "awaiting_client_tools" + assert accessor.stored.pending_tool_calls[0].id == "call_manual_1" + + +async def test_hitl_pause_still_yields_awaiting_hitl_no_regression() -> None: + client = QueuedClient([make_response("resp_hitl", [function_call_item("call_hitl_1", "approve", '{"amount":5}')])]) + accessor = MemoryStateAccessor() + + approve = tool( + name="approve", + input_schema=dict, + output_schema=dict, + on_tool_called=lambda params, ctx: None, + ) + + result = call_model(client, {"model": "test-model", "input": "approve 5", "tools": [approve], "state": accessor}) + + await result.get_response() + pending = await result.get_pending_tool_calls() + assert len(pending) == 1 + assert pending[0].id == "call_hitl_1" + + state = await result.get_state() + assert state.status == "awaiting_hitl" + assert state.status != "awaiting_client_tools" diff --git a/tests/unit/test_mcp_tool_branding.py b/tests/unit/test_mcp_tool_branding.py new file mode 100644 index 0000000..a35e8cc --- /dev/null +++ b/tests/unit/test_mcp_tool_branding.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +from typing import Any, Dict, List + +from openrouter_agent import call_model, is_mcp_tool, mark_mcp, tool + + +class QueuedResponses: + def __init__(self, responses: List[Dict[str, Any]]) -> None: + self._responses = list(responses) + self.requests: List[Dict[str, Any]] = [] + + async def send_async(self, **kwargs: Any) -> Any: + self.requests.append(kwargs) + return self._responses.pop(0) + + +class QueuedClient: + def __init__(self, responses: List[Dict[str, Any]]) -> None: + self.beta = type("Beta", (), {"responses": QueuedResponses(responses)})() + + +def function_call_item(call_id: str, name: str, arguments: str) -> Dict[str, Any]: + return {"type": "function_call", "id": f"fc_{call_id}", "callId": call_id, "name": name, "arguments": arguments} + + +def text_response(response_id: str, text: str) -> Dict[str, Any]: + return { + "id": response_id, + "output": [{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": text}]}], + } + + +def make_response(response_id: str, output: List[Dict[str, Any]]) -> Dict[str, Any]: + return {"id": response_id, "output": output} + + +def test_mark_mcp_is_non_mutating_and_is_mcp_tool_detects_the_brand() -> None: + base = tool(name="remote_search", input_schema=dict, output_schema=dict, execute=lambda p, c: {"ok": True}) + + branded = mark_mcp(base) + + assert base is not branded + assert is_mcp_tool(base) is False + assert is_mcp_tool(branded) is True + # Runtime shape (type/function) unaffected -- only the additive brand. + assert branded["type"] == base["type"] + assert branded["function"] is base["function"] + + +def test_is_mcp_tool_false_for_plain_mappings_and_server_tools() -> None: + from openrouter_agent import server_tool + + assert is_mcp_tool({"type": "function", "function": {"name": "x"}}) is False + assert is_mcp_tool(server_tool({"type": "web_search"})) is False + + +async def test_mcp_branded_tool_result_carries_source_mcp_in_tool_result_event() -> None: + remote_tool = mark_mcp( + tool(name="remote_echo", input_schema=dict, output_schema=dict, execute=lambda p, c: {"ok": True}) + ) + client = QueuedClient( + [make_response("r1", [function_call_item("call_1", "remote_echo", "{}")]), text_response("r2", "done")] + ) + + result = call_model(client, {"model": "test-model", "input": "hi", "tools": [remote_tool]}) + + events = [] + async for event in result.get_tool_stream(): + if event.get("type") == "tool_result": + events.append(event) + await result.get_response() + + assert events[0]["source"] == "mcp" + + +async def test_regular_client_tool_result_carries_source_client() -> None: + local_tool = tool(name="local_echo", input_schema=dict, output_schema=dict, execute=lambda p, c: {"ok": True}) + client = QueuedClient( + [make_response("r1", [function_call_item("call_1", "local_echo", "{}")]), text_response("r2", "done")] + ) + + result = call_model(client, {"model": "test-model", "input": "hi", "tools": [local_tool]}) + + events = [] + async for event in result.get_tool_stream(): + if event.get("type") == "tool_result": + events.append(event) + await result.get_response() + + assert events[0]["source"] == "client" diff --git a/tests/unit/test_model_result_hooks.py b/tests/unit/test_model_result_hooks.py new file mode 100644 index 0000000..9e50f2f --- /dev/null +++ b/tests/unit/test_model_result_hooks.py @@ -0,0 +1,439 @@ +from __future__ import annotations + +from typing import Any, Dict, List + +from openrouter_agent import HookEntry, HookName, HooksManager, call_model, step_count_is, tool + + +class QueuedResponses: + def __init__(self, responses: List[Dict[str, Any]]) -> None: + self._responses = list(responses) + self.requests: List[Dict[str, Any]] = [] + + async def send_async(self, **kwargs: Any) -> Any: + self.requests.append(kwargs) + return self._responses.pop(0) + + +class QueuedClient: + def __init__(self, responses: List[Dict[str, Any]]) -> None: + self.beta = type("Beta", (), {"responses": QueuedResponses(responses)})() + + +def function_call_item(call_id: str, name: str, arguments: str) -> Dict[str, Any]: + return {"type": "function_call", "id": f"fc_{call_id}", "callId": call_id, "name": name, "arguments": arguments} + + +def text_response(response_id: str, text: str, usage: Any = None) -> Dict[str, Any]: + resp = { + "id": response_id, + "model": "test-model-v1", + "output": [{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": text}]}], + } + if usage is not None: + resp["usage"] = usage + return resp + + +def tool_call_response(response_id: str, usage: Any = None) -> Dict[str, Any]: + resp = {"id": response_id, "output": [function_call_item(f"call_{response_id}", "echo", "{}")]} + if usage is not None: + resp["usage"] = usage + return resp + + +def usage_block(**overrides: Any) -> Dict[str, Any]: + base = {"input_tokens": 100, "output_tokens": 50, "total_tokens": 150, "cost": 0.002} + base.update(overrides) + return base + + +echo_tool = tool(name="echo", input_schema=dict, output_schema=dict, execute=lambda params, ctx: {"ok": True}) + + +async def test_session_start_and_end_pair_on_no_tools_get_text() -> None: + client = QueuedClient([text_response("resp_1", "hello back")]) + hooks = HooksManager() + events: List[str] = [] + hooks.on(HookName.SessionStart.value, HookEntry(handler=lambda p, c: events.append("start"))) + hooks.on(HookName.SessionEnd.value, HookEntry(handler=lambda p, c: events.append("end"))) + + text = await call_model(client, {"model": "test-model", "input": "hi", "hooks": hooks}).get_text() + + assert text == "hello back" + assert events == ["start", "end"] + + +async def test_session_start_config_reflects_tools_and_state() -> None: + client = QueuedClient([text_response("resp_1", "hello back")]) + hooks = HooksManager() + starts: List[Dict[str, Any]] = [] + hooks.on(HookName.SessionStart.value, HookEntry(handler=lambda p, c: starts.append(p))) + + await call_model(client, {"model": "test-model", "input": "hi", "tools": [echo_tool], "hooks": hooks}).get_text() + + assert starts[0]["config"] == {"has_tools": True, "has_approval": False, "has_state": False} + + +async def test_session_end_reason_max_turns_when_stop_condition_halts() -> None: + client = QueuedClient([tool_call_response("r1"), tool_call_response("r2")]) + hooks = HooksManager() + ends: List[Dict[str, Any]] = [] + hooks.on(HookName.SessionEnd.value, HookEntry(handler=lambda p, c: ends.append(p))) + + await call_model( + client, + { + "model": "test-model", + "input": "hi", + "tools": [echo_tool], + "stop_when": step_count_is(1), + "allow_final_response": False, + "hooks": hooks, + }, + ).get_response() + + assert ends[0]["reason"] == "max_turns" + + +async def test_post_model_call_emits_once_per_turn_with_turn_type_labels() -> None: + client = QueuedClient([tool_call_response("r1", usage_block()), text_response("r2", "done", usage_block())]) + hooks = HooksManager() + calls: List[Dict[str, Any]] = [] + hooks.on(HookName.PostModelCall.value, HookEntry(handler=lambda p, c: calls.append(p))) + + await call_model(client, {"model": "test-model", "input": "hi", "tools": [echo_tool], "hooks": hooks}).get_text() + + assert [c["turn_type"] for c in calls] == ["initial", "tool_round"] + assert calls[0]["usage"]["total_tokens"] == 150 + assert calls[0]["response_id"] == "r1" + + +async def test_session_end_aggregates_usage_totals_across_calls() -> None: + client = QueuedClient( + [ + tool_call_response("r1", usage_block(input_tokens=10, output_tokens=5, total_tokens=15, cost=0.01)), + text_response("r2", "done", usage_block(input_tokens=20, output_tokens=8, total_tokens=28, cost=0.02)), + ] + ) + hooks = HooksManager() + ends: List[Dict[str, Any]] = [] + hooks.on(HookName.SessionEnd.value, HookEntry(handler=lambda p, c: ends.append(p))) + + await call_model(client, {"model": "test-model", "input": "hi", "tools": [echo_tool], "hooks": hooks}).get_text() + + totals = ends[0]["total_usage"] + assert totals["model_calls"] == 2 + assert totals["input_tokens"] == 30 + assert totals["output_tokens"] == 13 + assert round(totals["cost"], 4) == 0.03 + + +async def test_pre_and_post_tool_use_fire_around_tool_execution() -> None: + client = QueuedClient([tool_call_response("r1"), text_response("r2", "done")]) + hooks = HooksManager() + events: List[str] = [] + hooks.on(HookName.PreToolUse.value, HookEntry(handler=lambda p, c: events.append(f"pre:{p['tool_name']}"))) + hooks.on( + HookName.PostToolUse.value, + HookEntry(handler=lambda p, c: events.append(f"post:{p['tool_name']}:{p['tool_output']}")), + ) + + await call_model(client, {"model": "test-model", "input": "hi", "tools": [echo_tool], "hooks": hooks}).get_text() + + assert events == ["pre:echo", "post:echo:{'ok': True}"] + + +async def test_pre_tool_use_block_prevents_execution_and_synthesizes_rejection() -> None: + client = QueuedClient([tool_call_response("r1"), text_response("r2", "done")]) + hooks = HooksManager() + executed: List[str] = [] + blocking_tool = tool( + name="echo", + input_schema=dict, + output_schema=dict, + execute=lambda params, ctx: executed.append("ran") or {"ok": True}, + ) + hooks.on(HookName.PreToolUse.value, HookEntry(handler=lambda p, c: {"block": "not allowed"})) + + await call_model( + client, {"model": "test-model", "input": "hi", "tools": [blocking_tool], "hooks": hooks} + ).get_text() + + assert executed == [] + followup_input = client.beta.responses.requests[1]["input"] + output_item = next(i for i in followup_input if i.get("type") == "function_call_output") + assert "not allowed" in output_item["output"] + + +async def test_post_tool_use_failure_fires_on_tool_error() -> None: + client = QueuedClient([tool_call_response("r1"), text_response("r2", "done")]) + hooks = HooksManager() + failures: List[Dict[str, Any]] = [] + hooks.on(HookName.PostToolUseFailure.value, HookEntry(handler=lambda p, c: failures.append(p))) + + def boom(params: Any, ctx: Any) -> Any: + raise RuntimeError("kaboom") + + failing_tool = tool(name="echo", input_schema=dict, output_schema=dict, execute=boom) + + await call_model(client, {"model": "test-model", "input": "hi", "tools": [failing_tool], "hooks": hooks}).get_text() + + assert len(failures) == 1 + assert failures[0]["tool_name"] == "echo" + assert "kaboom" in failures[0]["error"] + + +async def test_stop_hook_force_resume_is_a_zero_cost_retry_no_extra_model_request() -> None: + """force_resume re-checks the stop condition against the SAME + already-fetched response, without sending a new model request. Pinned by + isolating allow_final_response=False so the only way a second request + could appear is via an (incorrect) extra send on the "resume" branch + itself.""" + client = QueuedClient([tool_call_response("r1")]) + hooks = HooksManager() + stop_calls = {"n": 0} + + def stop_handler(payload: Any, ctx: Any) -> Any: + stop_calls["n"] += 1 + if stop_calls["n"] == 1: + return {"force_resume": True} + return None + + hooks.on(HookName.Stop.value, HookEntry(handler=stop_handler)) + + result = call_model( + client, + { + "model": "test-model", + "input": "hi", + "tools": [echo_tool], + "stop_when": step_count_is(1), + "allow_final_response": False, + "hooks": hooks, + }, + ) + response = await result.get_response() + + # The Stop hook fired twice (resume, then stop)... + assert stop_calls["n"] == 2 + # ...but exactly one model request was ever sent -- the resume itself + # cost nothing. + assert len(client.beta.responses.requests) == 1 + assert response["id"] == "r1" + + +async def test_stop_hook_force_resume_then_falls_through_to_normal_tool_round() -> None: + """Once the Stop hook stops forcing a resume, the halted round's pending + tool calls execute via the default-enabled final-directive coercion path + (tool_choice="none", tools retained) and the loop ends with exactly one + real follow-up request -- not one request per Stop-hook invocation.""" + client = QueuedClient([tool_call_response("r1"), text_response("r2", "All done.")]) + hooks = HooksManager() + stop_calls = {"n": 0} + + def stop_handler(payload: Any, ctx: Any) -> Any: + stop_calls["n"] += 1 + if stop_calls["n"] == 1: + return {"force_resume": True, "append_prompt": "please wrap up"} + return None + + hooks.on(HookName.Stop.value, HookEntry(handler=stop_handler)) + + result = call_model( + client, + {"model": "test-model", "input": "hi", "tools": [echo_tool], "stop_when": step_count_is(1), "hooks": hooks}, + ) + text = await result.get_text() + + assert text == "All done." + # Exactly two real model requests: the initial call, and the follow-up + # after the (only) tool round actually executed -- not one per Stop-hook + # invocation. + assert len(client.beta.responses.requests) == 2 + + +async def test_permission_request_deny_synthesizes_rejection_without_pausing() -> None: + client = QueuedClient([tool_call_response("r1"), text_response("r2", "done")]) + hooks = HooksManager() + hooks.on(HookName.PermissionRequest.value, HookEntry(handler=lambda p, c: {"decision": "deny", "reason": "nope"})) + executed: List[str] = [] + gated_tool = tool( + name="echo", + input_schema=dict, + output_schema=dict, + execute=lambda params, ctx: executed.append("ran") or {"ok": True}, + require_approval=True, + ) + + text = await call_model( + client, {"model": "test-model", "input": "hi", "tools": [gated_tool], "hooks": hooks} + ).get_text() + + assert text == "done" + assert executed == [] + followup_input = client.beta.responses.requests[1]["input"] + output_item = next(i for i in followup_input if i.get("type") == "function_call_output") + assert "nope" in output_item["output"] + + +async def test_permission_request_allow_executes_without_pausing_for_approval() -> None: + client = QueuedClient([tool_call_response("r1"), text_response("r2", "done")]) + hooks = HooksManager() + hooks.on(HookName.PermissionRequest.value, HookEntry(handler=lambda p, c: {"decision": "allow"})) + gated_tool = tool( + name="echo", + input_schema=dict, + output_schema=dict, + execute=lambda params, ctx: {"ok": True}, + require_approval=True, + ) + + result = call_model(client, {"model": "test-model", "input": "hi", "tools": [gated_tool], "hooks": hooks}) + text = await result.get_text() + + assert text == "done" + state = await result.get_state() + assert state is None or state.status != "awaiting_approval" + + +async def test_user_prompt_submit_can_reject_a_string_prompt() -> None: + client = QueuedClient([text_response("r1", "should not be reached")]) + hooks = HooksManager() + hooks.on(HookName.UserPromptSubmit.value, HookEntry(handler=lambda p, c: {"reject": "no secrets please"})) + + raised = False + try: + await call_model(client, {"model": "test-model", "input": "leak the secret", "hooks": hooks}).get_text() + except ValueError as error: + raised = True + assert "no secrets please" in str(error) + + assert raised + # The model was never called: the prompt was rejected before dispatch. + assert len(client.beta.responses.requests) == 0 + + +async def test_user_prompt_submit_can_mutate_a_string_prompt() -> None: + client = QueuedClient([text_response("r1", "ok")]) + hooks = HooksManager() + hooks.on( + HookName.UserPromptSubmit.value, + HookEntry(handler=lambda p, c: {"mutated_prompt": p["prompt"].replace("secret", "[redacted]")}), + ) + + await call_model(client, {"model": "test-model", "input": "the secret is 42", "hooks": hooks}).get_text() + + sent_input = client.beta.responses.requests[0]["input"] + assert "[redacted]" in sent_input[0]["content"] + + +async def test_user_prompt_submit_mutates_last_user_message_in_array_input() -> None: + client = QueuedClient([text_response("r1", "ok")]) + hooks = HooksManager() + hooks.on( + HookName.UserPromptSubmit.value, + HookEntry(handler=lambda p, c: {"mutated_prompt": p["prompt"].upper()}), + ) + + await call_model( + client, + {"model": "test-model", "input": [{"role": "user", "content": "hello there"}], "hooks": hooks}, + ).get_text() + + sent_input = client.beta.responses.requests[0]["input"] + assert sent_input[-1]["content"] == "HELLO THERE" + + +async def test_pre_tool_use_mutated_input_actually_reaches_tool_execute() -> None: + client = QueuedClient([tool_call_response("r1"), text_response("r2", "done")]) + hooks = HooksManager() + received_args: List[Any] = [] + hooks.on( + HookName.PreToolUse.value, + HookEntry(handler=lambda p, c: {"mutated_input": {"mutated": True, **p["tool_input"]}}), + ) + recording_tool = tool( + name="echo", + input_schema=dict, + output_schema=dict, + execute=lambda params, ctx: received_args.append(params) or {"ok": True}, + ) + + await call_model( + client, {"model": "test-model", "input": "hi", "tools": [recording_tool], "hooks": hooks} + ).get_text() + + assert received_args == [{"mutated": True}] + + +async def test_session_end_fires_with_reason_error_on_no_tools_transport_failure() -> None: + """SessionStart/SessionEnd(reason='error') must fire even when the + no-tools path's transport raises, and the drain must not mask the + original error.""" + + class FailingResponses: + async def send_async(self, **kwargs: Any) -> Any: + raise RuntimeError("transport exploded") + + client = type("Client", (), {"beta": type("Beta", (), {"responses": FailingResponses()})()})() + hooks = HooksManager() + events: List[str] = [] + hooks.on(HookName.SessionStart.value, HookEntry(handler=lambda p, c: events.append("start"))) + hooks.on(HookName.SessionEnd.value, HookEntry(handler=lambda p, c: events.append(f"end:{p['reason']}"))) + + raised = False + try: + await call_model(client, {"model": "test-model", "input": "hi", "hooks": hooks}).get_text() + except RuntimeError as error: + raised = True + assert "transport exploded" in str(error) + + assert raised + assert events == ["start", "end:error"] + + +async def test_permission_request_allow_executes_promoted_tool_exactly_once() -> None: + """A tool call promoted from requires_approval to executed-now by a + PermissionRequest 'allow' decision must run exactly once -- not once in + the promotion branch and again in the normal auto-execute round.""" + client = QueuedClient( + [ + { + "id": "r1", + "output": [ + function_call_item("call_auto", "auto_run", "{}"), + function_call_item("call_gated", "gated_run", "{}"), + ], + }, + text_response("r2", "done"), + ] + ) + hooks = HooksManager() + hooks.on(HookName.PermissionRequest.value, HookEntry(handler=lambda p, c: {"decision": "allow"})) + + executions: Dict[str, int] = {"auto_run": 0, "gated_run": 0} + + def make_counting_tool(name: str, requires_approval: bool = False) -> Any: + def execute(params: Any, ctx: Any) -> Any: + executions[name] += 1 + return {"ok": True} + + return tool( + name=name, + input_schema=dict, + output_schema=dict, + execute=execute, + require_approval=requires_approval, + ) + + auto_tool_ = make_counting_tool("auto_run") + gated_tool = make_counting_tool("gated_run", requires_approval=True) + + text = await call_model( + client, + {"model": "test-model", "input": "hi", "tools": [auto_tool_, gated_tool], "hooks": hooks}, + ).get_text() + + assert text == "done" + assert executions == {"auto_run": 1, "gated_run": 1} diff --git a/tests/unit/test_resume_string_input.py b/tests/unit/test_resume_string_input.py new file mode 100644 index 0000000..1515023 --- /dev/null +++ b/tests/unit/test_resume_string_input.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from openrouter_agent import call_model +from openrouter_agent.tool_types import ConversationState + + +class QueuedResponses: + def __init__(self, responses: List[Dict[str, Any]]) -> None: + self._responses = list(responses) + self.requests: List[Dict[str, Any]] = [] + + async def send_async(self, **kwargs: Any) -> Any: + self.requests.append(kwargs) + return self._responses.pop(0) + + +class QueuedClient: + def __init__(self, responses: List[Dict[str, Any]]) -> None: + self.beta = type("Beta", (), {"responses": QueuedResponses(responses)})() + + +class MemoryStateAccessor: + def __init__(self) -> None: + self.stored: Optional[ConversationState] = None + + async def load(self) -> Optional[ConversationState]: + return self.stored + + async def save(self, state: ConversationState) -> None: + self.stored = state + + +def text_response(response_id: str, text: str) -> Dict[str, Any]: + return { + "id": response_id, + "output": [{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": text}]}], + } + + +async def test_normalizes_a_bare_string_input_when_resuming_loaded_history() -> None: + accessor = MemoryStateAccessor() + client = QueuedClient([text_response("resp_1", "First answer."), text_response("resp_2", "Second answer.")]) + + await call_model(client, {"model": "test-model", "input": "First question", "state": accessor}).get_text() + assert accessor.stored is not None + assert len(accessor.stored.messages) > 0 + + await call_model(client, {"model": "test-model", "input": "Follow-up question", "state": accessor}).get_text() + + request = client.beta.responses.requests[1] + assert isinstance(request["input"], list) + for item in request["input"]: + assert not isinstance(item, str) + last = request["input"][-1] + assert last["role"] == "user" + assert last["content"] == "Follow-up question" + + +async def test_still_accepts_array_input_when_resuming_loaded_history() -> None: + accessor = MemoryStateAccessor() + client = QueuedClient([text_response("resp_1", "First answer."), text_response("resp_2", "Second answer.")]) + + await call_model(client, {"model": "test-model", "input": "First question", "state": accessor}).get_text() + await call_model( + client, + {"model": "test-model", "input": [{"role": "user", "content": "Follow-up question"}], "state": accessor}, + ).get_text() + + request = client.beta.responses.requests[1] + last = request["input"][-1] + assert last["role"] == "user" + assert last["content"] == "Follow-up question" diff --git a/tests/unit/test_tool_terminal_empty_final.py b/tests/unit/test_tool_terminal_empty_final.py new file mode 100644 index 0000000..5b23480 --- /dev/null +++ b/tests/unit/test_tool_terminal_empty_final.py @@ -0,0 +1,209 @@ +from __future__ import annotations + +from typing import Any, Dict, List + +from openrouter_agent import call_model, tool + + +class QueuedResponses: + def __init__(self, responses: List[Dict[str, Any]]) -> None: + self._responses = list(responses) + self.requests: List[Dict[str, Any]] = [] + + async def send_async(self, **kwargs: Any) -> Any: + self.requests.append(kwargs) + return self._responses.pop(0) + + +class QueuedClient: + def __init__(self, responses: List[Dict[str, Any]]) -> None: + self.beta = type("Beta", (), {"responses": QueuedResponses(responses)})() + + +def function_call_item(call_id: str, name: str, arguments: str) -> Dict[str, Any]: + return {"type": "function_call", "id": f"fc_{call_id}", "callId": call_id, "name": name, "arguments": arguments} + + +def text_response(response_id: str, text: str) -> Dict[str, Any]: + return { + "id": response_id, + "output": [{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": text}]}], + } + + +def make_response(response_id: str, output: List[Dict[str, Any]]) -> Dict[str, Any]: + return {"id": response_id, "output": output} + + +def empty_response(response_id: str = "resp_empty") -> Dict[str, Any]: + return {"id": response_id, "output": []} + + +auto_tool = tool( + name="auto_search", input_schema=dict, output_schema=dict, execute=lambda params, ctx: {"result": "found it"} +) +manual_tool = tool(name="exec_command", input_schema=dict, execute=False) +post_comment_tool = tool( + name="post_comment", input_schema=dict, output_schema=dict, execute=lambda params, ctx: {"ok": True} +) + + +async def test_stops_loop_instead_of_orphaned_function_call_followup() -> None: + client = QueuedClient( + [ + make_response( + "resp_mixed", + [ + function_call_item("call_auto_1", "auto_search", '{"query":"docs"}'), + function_call_item("call_manual_1", "exec_command", '{"command":"ls"}'), + ], + ) + ] + ) + + result = call_model(client, {"model": "test-model", "input": "do both things", "tools": [auto_tool, manual_tool]}) + response = await result.get_response() + + assert response["id"] == "resp_mixed" + assert len(client.beta.responses.requests) == 1 + + +async def test_still_loops_when_every_call_in_the_round_resolves() -> None: + client = QueuedClient( + [ + make_response("resp_auto", [function_call_item("call_auto_1", "auto_search", '{"query":"docs"}')]), + text_response("resp_final", "All done."), + ] + ) + + result = call_model(client, {"model": "test-model", "input": "search the docs", "tools": [auto_tool, manual_tool]}) + text = await result.get_text() + + assert text == "All done." + assert len(client.beta.responses.requests) == 2 + followup_input = client.beta.responses.requests[1]["input"] + fn_call_output = next((i for i in followup_input if i.get("type") == "function_call_output"), None) + assert fn_call_output["callId"] == "call_auto_1" + assert "found it" in fn_call_output["output"] + + +async def test_retries_once_then_accepts_empty_final_after_a_completed_tool_round() -> None: + client = QueuedClient( + [ + make_response("resp_tool_call", [function_call_item("call_abc", "post_comment", '{"body":"lgtm"}')]), + empty_response("resp_empty_1"), + empty_response("resp_empty_2"), + ] + ) + + result = call_model(client, {"model": "test-model", "input": "review", "tools": [post_comment_tool]}) + + text = await result.get_text() + assert text == "" + assert len(client.beta.responses.requests) == 3 + + response = await result.get_response() + assert response["id"] == "resp_empty_2" + assert response["output"] == [] + + +async def test_returns_text_when_the_empty_final_retry_succeeds() -> None: + client = QueuedClient( + [ + make_response("resp_tool_call", [function_call_item("call_abc", "post_comment", '{"body":"lgtm"}')]), + empty_response("resp_empty"), + text_response("resp_retry_text", "Done posting."), + ] + ) + + text = await call_model(client, {"model": "test-model", "input": "review", "tools": [post_comment_tool]}).get_text() + + assert text == "Done posting." + assert len(client.beta.responses.requests) == 3 + + +async def test_retry_forces_tool_choice_none_while_keeping_tools_in_request() -> None: + client = QueuedClient( + [ + make_response("resp_tool_call", [function_call_item("call_abc", "post_comment", '{"body":"lgtm"}')]), + empty_response("resp_empty"), + text_response("resp_retry_text", "Done posting."), + ] + ) + + await call_model(client, {"model": "test-model", "input": "review", "tools": [post_comment_tool]}).get_text() + + followup_request = client.beta.responses.requests[1] + assert "tools" in followup_request + assert followup_request.get("tool_choice") != "none" + + retry_request = client.beta.responses.requests[2] + assert "tools" in retry_request + assert retry_request["tool_choice"] == "none" + assert retry_request["input"] == followup_request["input"] + + +async def test_throws_on_empty_final_when_strict_final_response_is_true() -> None: + client = QueuedClient( + [ + make_response("resp_tool_call", [function_call_item("call_abc", "post_comment", '{"body":"lgtm"}')]), + empty_response(), + ] + ) + + raised = False + try: + await call_model( + client, + { + "model": "test-model", + "input": "review", + "tools": [post_comment_tool], + "strict_final_response": True, + }, + ).get_text() + except ValueError as error: + raised = True + assert "Invalid final response: empty or invalid output" in str(error) + + assert raised + assert len(client.beta.responses.requests) == 2 + + +async def test_still_throws_on_empty_output_when_no_tool_rounds_completed() -> None: + client = QueuedClient([empty_response()]) + + raised = False + try: + await call_model(client, {"model": "test-model", "input": "hello"}).get_text() + except ValueError: + raised = True + + assert raised + assert len(client.beta.responses.requests) == 1 + + +async def test_does_not_send_client_only_fields_to_the_api() -> None: + client = QueuedClient([text_response("resp_text", "hi")]) + + await call_model( + client, + { + "model": "test-model", + "input": "hello", + "strict_final_response": True, + "allow_final_response": True, + }, + ).get_text() + + request = client.beta.responses.requests[0] + for key in ( + "strict_final_response", + "allow_final_response", + "stop_when", + "shared_context_schema", + "on_turn_start", + "on_turn_end", + "hooks", + ): + assert key not in request diff --git a/upstreamer-changelog.md b/upstreamer-changelog.md index 49fd01a..879fb99 100644 --- a/upstreamer-changelog.md +++ b/upstreamer-changelog.md @@ -1,6 +1,18 @@ # openrouter-agent Changelog -## Latest Sync +## 0.8.0 Sync + +- **Lifecycle hooks system** (the headline feature of this release): `HooksManager`, `HookName`, `HookEntry`, and nine built-in lifecycle hooks — `PreToolUse`, `PostToolUse`, `PostToolUseFailure`, `UserPromptSubmit`, `Stop`, `PermissionRequest`, `SessionStart`, `SessionEnd`, and `PostModelCall`. Pass a `HooksManager` instance, or an inline `{hook_name: [HookEntry(...)]}` dict of built-in hooks, via `hooks=` on `call_model`. + - `PreToolUse` can block a tool call or mutate its input before execution; `PermissionRequest` can pre-empt the approval gate with `allow`/`deny`/`ask_user`; `UserPromptSubmit` can reject or rewrite the user's prompt before the first request goes out; `Stop` can force the loop past a `stop_when` hit (as a zero-cost in-memory retry, no extra model request) and inject a follow-up prompt. + - `SessionStart`/`SessionEnd` fire exactly once per run, paired even when a no-tools stream raises; `SessionEnd` carries aggregated token/cost usage totals across every model call in the run. `PostModelCall` fires once per model request with a `turn_type` label (`initial`, `resume`, `tool_round`, `final`, `retry`) and per-call usage. + - A single `HooksManager` instance is safe to share across concurrent `call_model` runs — session identity is threaded per emit rather than stored as shared mutable state. +- **Versioned conversation-state serialization**: `serialize_conversation_state` / `deserialize_conversation_state` / `CONVERSATION_STATE_VERSION` for durable, cross-process state storage. Deserializing a version-less legacy blob normalizes to version 1; a future/unknown version raises `UnsupportedStateVersionError`, and malformed or missing-field JSON raises `InvalidStateError`, instead of a store silently misinterpreting an incompatible shape. The existing `StateAccessor` `load()`/`save()` contract is unchanged — these helpers are opt-in. +- **Unresolved manual tool calls now pause cleanly**: when the model calls a manual tool (`execute=False`, no `on_tool_called`) and there is no way to auto-resolve it, the run now stops with conversation status `"awaiting_client_tools"` and the unresolved calls available via `get_pending_tool_calls()` / `get_state()`, instead of silently dropping the call. This also applies to a mixed round of auto-executable and manual calls: the auto-executable outputs are still persisted before the pause. +- **`allow_final_response` is now on by default**: when a `stop_when` condition halts the loop mid-tool-call, `call_model` now makes one more turn with `tool_choice="none"` by default (tools stay in the request so the prompt-cache prefix survives) instead of requiring the caller to opt in. Bare `True` or omitting the option appends a new default directive (`DEFAULT_FINAL_RESPONSE_DIRECTIVE`) as a user message so models that emit tool-call syntax as text don't leak an unparsed call into the final answer; a non-empty string still overrides the wording, `""` appends nothing, and `False` disables the extra turn entirely. +- **Empty final responses after a completed tool round are now tolerated**: some models intermittently return an empty final turn right after a tool call was effectively the answer. That case is now retried once (forcing `tool_choice="none"`) and, if still empty, accepted rather than raised as an error. Pass `strict_final_response=True` to restore the old strict behavior. A run with no completed tool rounds still raises on an empty/invalid final response as before. +- **MCP-style tool result discrimination**: `mark_mcp()` / `is_mcp_tool()` let a tool be branded as originating from a remote MCP-style server without changing its execution or wire shape; tool results and `tool.result` stream events now carry a `source: "client" | "mcp"` field so consumers can tell precisely-typed local results apart from untyped remote ones. + +## Previous Sync (0.7.2) - Completed the Python port against the current `@openrouter/agent` 0.7.2 surface, keeping the package focused on the Responses API, tool orchestration, streaming, state, approval/HITL, tool context, stop conditions, and Claude/Chat compatibility. - Strengthened stateful pause/resume behavior so approval and HITL pauses persist the model tool-call turn and resume with `function_call` before `function_call_output`, matching upstream multi-turn semantics. diff --git a/uv.lock b/uv.lock index b05afc1..d2bad58 100644 --- a/uv.lock +++ b/uv.lock @@ -24,9 +24,9 @@ resolution-markers = [ "python_full_version < '3.10'", ] dependencies = [ - { name = "exceptiongroup", marker = "python_full_version < '3.10'" }, - { name = "idna", marker = "python_full_version < '3.10'" }, - { name = "typing-extensions", marker = "python_full_version < '3.10'" }, + { name = "exceptiongroup" }, + { name = "idna" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/96/f0/5eb65b2bb0d09ac6776f2eb54adee6abe8228ea05b20a5ad0e4945de8aac/anyio-4.12.1.tar.gz", hash = "sha256:41cfcc3a4c85d3f05c932da7c26d0201ac36f72abd4435ba90d0464a3ffed703", size = 228685, upload-time = "2026-01-06T11:45:21.246Z" } wheels = [ @@ -42,9 +42,9 @@ resolution-markers = [ "python_full_version >= '3.10' and python_full_version < '3.15'", ] dependencies = [ - { name = "exceptiongroup", marker = "python_full_version == '3.10.*'" }, - { name = "idna", marker = "python_full_version >= '3.10'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.10' and python_full_version < '3.13'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3b/72/5562aabb8dd7181e8e860622a38bea08d17842b99ecd4c91f84ac95251b0/anyio-4.14.1.tar.gz", hash = "sha256:8d648a3544c1a700e3ff78615cd679e4c5c3f149904287e73687b2596963629e", size = 254831, upload-time = "2026-06-24T20:56:06.017Z" } wheels = [ @@ -74,7 +74,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -337,7 +337,7 @@ wheels = [ [[package]] name = "openrouter-agent" -version = "0.7.2" +version = "0.8.0" source = { editable = "." } dependencies = [ { name = "openrouter" }, From 51ce402a48641aafb8eeb70dcf7a58de4339acac Mon Sep 17 00:00:00 2001 From: Luke Parke <5702154+LukasParke@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:22:53 -0500 Subject: [PATCH 6/7] chore: switch port model to Claude Opus latest OPENCODE_MODEL repo variable updated to openrouter/~anthropic/claude-opus-latest to match; contract frontmatter, port.env.example, PORTING.md, and the workflow comment all now reference the same alias. Co-Authored-By: Claude Fable 5 --- .github/workflows/upstreamer-port.yaml | 2 +- .upstreamer/port.env.example | 2 +- .upstreamer/upstreamer.md | 2 +- PORTING.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/upstreamer-port.yaml b/.github/workflows/upstreamer-port.yaml index 0527bac..0f8e7ab 100644 --- a/.github/workflows/upstreamer-port.yaml +++ b/.github/workflows/upstreamer-port.yaml @@ -55,7 +55,7 @@ jobs: env: # Provide these in repo settings: # Secret OPENROUTER_API_KEY — sk-or-... key opencode uses for inference - # Variable OPENCODE_MODEL — e.g. openrouter/~anthropic/claude-sonnet-latest + # Variable OPENCODE_MODEL — e.g. openrouter/~anthropic/claude-opus-latest OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} OPENCODE_MODEL: ${{ vars.OPENCODE_MODEL }} UPSTREAMER_TIMEOUT_SECONDS: 7200 diff --git a/.upstreamer/port.env.example b/.upstreamer/port.env.example index 01d8578..49b1b2c 100644 --- a/.upstreamer/port.env.example +++ b/.upstreamer/port.env.example @@ -17,7 +17,7 @@ OPENROUTER_API_KEY= # # This is a load-bearing SDK port with a strict parity eval — use a strong coding # model. Suggested starting point: -OPENCODE_MODEL=openrouter/~anthropic/claude-sonnet-latest +OPENCODE_MODEL=openrouter/~anthropic/claude-opus-latest # Optional: hard wall-clock cap for one run, in seconds. A full-surface # reconciliation is much slower than an incremental delta port. diff --git a/.upstreamer/upstreamer.md b/.upstreamer/upstreamer.md index af87d1f..90191cf 100644 --- a/.upstreamer/upstreamer.md +++ b/.upstreamer/upstreamer.md @@ -1,7 +1,7 @@ --- upstream: OpenRouterTeam/typescript-agent downstream: OpenRouterTeam/python-agent -model: openrouter/~anthropic/claude-sonnet-latest +model: openrouter/~anthropic/claude-opus-latest --- # Python Port Contract — `@openrouter/agent` → `openrouter-agent` diff --git a/PORTING.md b/PORTING.md index 6b22c92..fb4db6f 100644 --- a/PORTING.md +++ b/PORTING.md @@ -96,7 +96,7 @@ Two values, same names locally and in CI: | Name | Where | What | |------|-------|------| | `OPENROUTER_API_KEY` | local: `.upstreamer/port.env` · CI: repo **secret** | `sk-or-…` key opencode uses for inference | -| `OPENCODE_MODEL` | local: `.upstreamer/port.env` · CI: repo **variable** | e.g. `openrouter/~anthropic/claude-sonnet-latest` | +| `OPENCODE_MODEL` | local: `.upstreamer/port.env` · CI: repo **variable** | e.g. `openrouter/~anthropic/claude-opus-latest` | The wrapper writes the key into `~/.local/share/opencode/auth.json` so headless runs work without the interactive `opencode /connect` flow. From a219e42116826cdfe6c9bc1a8a8da33a3ba8bdc4 Mon Sep 17 00:00:00 2001 From: Luke Parke <5702154+LukasParke@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:32:42 -0500 Subject: [PATCH 7/7] ci: drop the external-token requirement from this repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two changes so the pipeline runs entirely on the native GITHUB_TOKEN: - Cron and blank manual dispatches now resolve the latest published @openrouter/agent version from the public npm registry and port that release tag, instead of porting upstream main HEAD. This makes the weekly cron fully equivalent to the repository_dispatch fast path (same tag either way), so the cross-repo PAT dispatch in typescript-agent becomes a latency optimization, not a requirement — and it also fixes cron runs violating the 'ports track releases' principle by syncing to mid-flight upstream commits. - PRs opened with the native GITHUB_TOKEN don't trigger other workflows (GitHub's recursion guard), which was the standard argument for a PAT here. workflow_dispatch is exempt from that guard, so the port job now explicitly dispatches ci.yaml at the upstreamer/sync branch after opening the PR. Requires actions: write, added to the permissions block. No PAT or external secret is referenced anywhere in this repo now; the only secrets are OPENROUTER_API_KEY (inference) and the built-in token. Co-Authored-By: Claude Fable 5 --- .github/workflows/upstreamer-port.yaml | 35 +++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/.github/workflows/upstreamer-port.yaml b/.github/workflows/upstreamer-port.yaml index 0f8e7ab..17317c6 100644 --- a/.github/workflows/upstreamer-port.yaml +++ b/.github/workflows/upstreamer-port.yaml @@ -29,6 +29,7 @@ on: permissions: contents: write pull-requests: write + actions: write # to dispatch ci.yaml onto the generated PR branch concurrency: group: upstreamer-port @@ -51,6 +52,24 @@ jobs: - name: Set up language toolchain uses: ./.github/actions/port-toolchain + # Ports track published releases, not upstream main. When no ref arrives + # (cron, or a manual dispatch with the input left blank), resolve the + # latest published @openrouter/agent version from the public npm registry + # and port its release tag. This makes the cron fully equivalent to the + # repository_dispatch fast path — same tag either way — so the pipeline + # works with no cross-repo token at all if the dispatch is unavailable. + - name: Resolve target ref + id: target + run: | + set -euo pipefail + REF="${{ inputs.ref || github.event.client_payload.ref }}" + if [ -z "$REF" ]; then + VERSION="$(curl -fsSL 'https://registry.npmjs.org/@openrouter%2Fagent/latest' | python3 -c 'import json,sys; print(json.load(sys.stdin)["version"])')" + REF="@openrouter/agent@${VERSION}" + echo "No ref provided — resolved latest npm release: $REF" + fi + echo "ref=$REF" >> "$GITHUB_OUTPUT" + - name: Run port env: # Provide these in repo settings: @@ -65,10 +84,8 @@ jobs: echo "::error::OPENROUTER_API_KEY secret is not set. See .upstreamer/port.env.example." exit 1 fi - args=() + args=(--ref "${{ steps.target.outputs.ref }}") [ "${{ inputs.force }}" = "true" ] && args+=(--force) - REF="${{ inputs.ref || github.event.client_payload.ref }}" - [ -n "$REF" ] && args+=(--ref "$REF") ./scripts/upstream "${args[@]}" - name: Check for changes @@ -96,6 +113,7 @@ jobs: fi - name: Open PR + id: open-pr if: steps.diff.outputs.changed == 'true' uses: peter-evans/create-pull-request@v6 with: @@ -124,6 +142,17 @@ jobs: `.upstreamer/state.yaml` did not advance, the eval did not pass and this PR must not be merged as-is. + # Events created with the native GITHUB_TOKEN deliberately do not trigger + # other workflows (GitHub's recursion guard), so the PR opened above gets + # no CI checks on its own. workflow_dispatch is exempt from that guard: + # kick ci.yaml at the PR branch explicitly. This keeps the whole pipeline + # on the native token — no PAT anywhere in this repo. + - name: Trigger CI on the port PR + if: steps.diff.outputs.changed == 'true' && steps.open-pr.outputs.pull-request-operation != 'none' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh workflow run ci.yaml --repo "$GITHUB_REPOSITORY" --ref upstreamer/sync + - name: Upload logs if: always() uses: actions/upload-artifact@v4