diff --git a/.github/workflows/v3-benchmark-isolation.yml b/.github/workflows/v3-benchmark-isolation.yml new file mode 100644 index 00000000..ef176f5c --- /dev/null +++ b/.github/workflows/v3-benchmark-isolation.yml @@ -0,0 +1,51 @@ +name: V3 benchmark isolation diagnostics +on: + push: + branches: ['Evan/v3-finalization'] + paths: ['benchmarks/**', 'tests/test_v3_protected_worker.py', '.github/workflows/v3-benchmark-isolation.yml'] + workflow_dispatch: +permissions: + contents: read +jobs: + isolation: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + python: ['3.12', '3.13'] + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Install Linux backend using the existing CI configuration + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y bubblewrap + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + - name: Install project dependencies + run: python -m pip install -e '.[all]' + - name: Record diagnostic identity + shell: bash + run: | + git rev-parse HEAD > isolation-commit.txt + python -m pip freeze > isolation-dependencies.txt + - name: Exercise bounded protection and required full-preflight refusal + shell: bash + run: | + set -euo pipefail + python -m unittest tests.test_v3_protected_worker -v 2>&1 | tee isolation-tests.log + - name: Preserve diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: isolation-${{ matrix.os }}-py${{ matrix.python }} + path: | + isolation-commit.txt + isolation-dependencies.txt + isolation-tests.log diff --git a/.github/workflows/v3-windows-reliability.yml b/.github/workflows/v3-windows-reliability.yml new file mode 100644 index 00000000..01e7665a --- /dev/null +++ b/.github/workflows/v3-windows-reliability.yml @@ -0,0 +1,82 @@ +name: V3 Windows cancellation investigation +on: + push: + branches: ['Evan/v3-windows-cancellation-investigation-20261107'] + paths: ['.github/workflows/v3-windows-reliability.yml', 'scripts/v3_windows_reliability.py', 'tests/test_v3_windows_reliability.py'] + workflow_dispatch: + inputs: + candidate_sha: + description: Exact 40-character product commit to test (diagnostic code remains separate) + type: string + required: true + default: '7905fb634fe447243668c2d01179ce5cb2cdb275' + control: + description: Diagnostic load control (passing repetitions never establish historical cause) + type: choice + options: [baseline, cpu, sqlite, combined, all] + default: baseline + iterations: + description: Samples per control and Python version (1–50) + type: number + default: 50 +env: + PRODUCT_SHA: ${{ inputs.candidate_sha || '7905fb634fe447243668c2d01179ce5cb2cdb275' }} + ITERATIONS: ${{ github.event_name == 'workflow_dispatch' && format('{0}', inputs.iterations) || '50' }} +permissions: + contents: read +jobs: + cancellation: + runs-on: windows-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + python: ['3.12', '3.13'] + control: ${{ fromJSON(inputs.control == 'all' && '["baseline","cpu","sqlite","combined"]' || format('["{0}"]', inputs.control || 'baseline')) }} + steps: + - name: Validate exact candidate identity + shell: pwsh + run: | + if ($env:PRODUCT_SHA -cnotmatch '^[0-9a-f]{40}$') { throw 'candidate_sha must be a full lowercase commit SHA' } + if ($env:ITERATIONS -cnotmatch '^[0-9]+$' -or [int]$env:ITERATIONS -lt 1 -or [int]$env:ITERATIONS -gt 50) { throw 'iterations must be 1–50' } + - name: Checkout diagnostic harness + uses: actions/checkout@v4 + with: + path: diagnostic-source + fetch-depth: 0 + - name: Checkout exact product + uses: actions/checkout@v4 + with: + ref: ${{ env.PRODUCT_SHA }} + path: product + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Verify checked-out product + shell: pwsh + working-directory: product + run: | + if ((git rev-parse HEAD) -cne $env:PRODUCT_SHA) { throw 'Checked-out candidate differs from requested commit' } + if (git status --porcelain --untracked-files=no) { throw 'Product checkout has tracked edits' } + - name: Install exact product dependencies + working-directory: product + run: python -m pip install -e '.[all]' + - name: Record dependency versions + shell: pwsh + run: | + New-Item -ItemType Directory -Force windows-evidence | Out-Null + python -m pip freeze | Set-Content -Encoding utf8 windows-evidence/dependencies.txt + $env:GITHUB_SHA | Set-Content -Encoding utf8 windows-evidence/diagnostic-commit.txt + - name: Trace unchanged cancellation test under recorded load control + shell: pwsh + env: + LOAD_CONTROL: ${{ matrix.control }} + run: python diagnostic-source/scripts/v3_windows_reliability.py --repository product --output windows-evidence --iterations $env:ITERATIONS --control $env:LOAD_CONTROL --cpu-workers 2 + - name: Upload investigation evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: windows-cancellation-${{ matrix.python }}-${{ env.PRODUCT_SHA }}-${{ matrix.control }} + path: windows-evidence/ + if-no-files-found: error diff --git a/README.ja.md b/README.ja.md index 3c7aaddc..9aa95a5a 100644 --- a/README.ja.md +++ b/README.ja.md @@ -14,19 +14,21 @@ ## インストール +V3 は未リリースです。以下のコマンドは公開済み安定版 **v2.0.8** をインストールします。 + OpenKyrozen は Python **3.12 と 3.13** をサポートします。Python 3.14 は対象外です。 ### macOS または Linux ```bash -curl -fsSL https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v3.0.0/install.sh | sh +curl -fsSL https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v2.0.8/install.sh | sh kyrozen ``` ### Windows PowerShell ```powershell -irm https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v3.0.0/install.ps1 | iex +irm https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v2.0.8/install.ps1 | iex kyrozen ``` @@ -53,7 +55,7 @@ make run ### 検証済み release wheel を直接インストール ```bash -uv tool install --python 3.12 --force --with fastapi --with uvicorn https://github.com/EvanProgramming/OpenKyrozen/releases/download/v3.0.0/openkyrozen-3.0.0-py3-none-any.whl +uv tool install --python 3.12 --force --with fastapi --with uvicorn https://github.com/EvanProgramming/OpenKyrozen/releases/download/v2.0.8/openkyrozen-2.0.8-py3-none-any.whl ``` ## クイックスタート diff --git a/README.ko.md b/README.ko.md index d8556a88..acec3fbf 100644 --- a/README.ko.md +++ b/README.ko.md @@ -14,19 +14,21 @@ ## 설치 +V3는 아직 출시되지 않았습니다. 아래 명령은 공개된 안정 버전 **v2.0.8**을 설치합니다. + OpenKyrozen은 Python **3.12와 3.13**을 지원합니다. Python 3.14는 지원 대상이 아닙니다. ### macOS 또는 Linux ```bash -curl -fsSL https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v3.0.0/install.sh | sh +curl -fsSL https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v2.0.8/install.sh | sh kyrozen ``` ### Windows PowerShell ```powershell -irm https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v3.0.0/install.ps1 | iex +irm https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v2.0.8/install.ps1 | iex kyrozen ``` @@ -53,7 +55,7 @@ make run ### 검증된 release wheel 직접 설치 ```bash -uv tool install --python 3.12 --force --with fastapi --with uvicorn https://github.com/EvanProgramming/OpenKyrozen/releases/download/v3.0.0/openkyrozen-3.0.0-py3-none-any.whl +uv tool install --python 3.12 --force --with fastapi --with uvicorn https://github.com/EvanProgramming/OpenKyrozen/releases/download/v2.0.8/openkyrozen-2.0.8-py3-none-any.whl ``` ## 빠른 시작 diff --git a/README.md b/README.md index 7ee04a39..02437a18 100644 --- a/README.md +++ b/README.md @@ -12,19 +12,19 @@ ## Install -This release is **v3.0.0**. It supports Python 3.12 and 3.13; Python 3.14 is outside the supported range. +V3 is not yet released. The commands below install the published stable **v2.0.8**. It supports Python 3.12 and 3.13; Python 3.14 is outside the supported range. macOS or Linux: ```sh -curl -fsSL https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v3.0.0/install.sh | sh +curl -fsSL https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v2.0.8/install.sh | sh kyrozen ``` Windows PowerShell: ```powershell -irm https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v3.0.0/install.ps1 | iex +irm https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v2.0.8/install.ps1 | iex kyrozen ``` diff --git a/README.zh-CN.md b/README.zh-CN.md index c4a7aded..21a2e8a5 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -14,19 +14,21 @@ ## 安装 +V3 尚未发布。下方命令安装已发布的稳定版 **v2.0.8**。 + OpenKyrozen 支持 Python **3.12 和 3.13**。Python 3.14 暂不支持。 ### macOS 或 Linux ```bash -curl -fsSL https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v3.0.0/install.sh | sh +curl -fsSL https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v2.0.8/install.sh | sh kyrozen ``` ### Windows PowerShell ```powershell -irm https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v3.0.0/install.ps1 | iex +irm https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v2.0.8/install.ps1 | iex kyrozen ``` @@ -53,7 +55,7 @@ make run ### 直接安装已验证的 release wheel ```bash -uv tool install --python 3.12 --force --with fastapi --with uvicorn https://github.com/EvanProgramming/OpenKyrozen/releases/download/v3.0.0/openkyrozen-3.0.0-py3-none-any.whl +uv tool install --python 3.12 --force --with fastapi --with uvicorn https://github.com/EvanProgramming/OpenKyrozen/releases/download/v2.0.8/openkyrozen-2.0.8-py3-none-any.whl ``` ## 快速开始 diff --git a/benchmarks/prompt_worker.py b/benchmarks/prompt_worker.py index 20eb8061..2557c457 100644 --- a/benchmarks/prompt_worker.py +++ b/benchmarks/prompt_worker.py @@ -5,10 +5,215 @@ import json import hashlib import os +import shlex import sys import time from pathlib import Path from types import SimpleNamespace +from contextvars import ContextVar +from functools import wraps + + +def _normalize_tool_receipt(receipt, workspace_root): + arguments = receipt.get("arguments") + if not isinstance(arguments, dict): + arguments = receipt.get("args") + if isinstance(arguments, str) and len(arguments) <= 65536 and arguments.lstrip().startswith("{"): + try: + parsed = json.loads(arguments) + if isinstance(parsed, dict): + arguments = parsed + except (TypeError, ValueError): + pass + canonical = json.dumps(arguments, sort_keys=True, ensure_ascii=False, separators=(",", ":")) + path = (arguments.get("path") or arguments.get("file_path")) if isinstance(arguments, dict) else None + if isinstance(arguments, str) and len(arguments) <= 65536: + # Frozen V2 filesystem tools take a bare path, or path|content for writes. + if receipt.get("action") == "read_file": + path = arguments.strip() + elif receipt.get("action") == "write_file" and "|" in arguments: + path = arguments.split("|", 1)[0].strip() + try: + target = Path(path) + target = target if target.is_absolute() else Path(workspace_root) / target + target_path = target.resolve().relative_to(Path(workspace_root).resolve()).as_posix() + except (OSError, ValueError, TypeError): + target_path = None + reference = None + if receipt.get("action") == "git_show": + if isinstance(arguments, dict): + reference = arguments.get("reference") + elif isinstance(arguments, str) and len(arguments) <= 65536: + try: + reference = next((part for part in shlex.split(arguments) if not part.startswith("-")), None) + except ValueError: + pass + return {"action": receipt.get("action"), "call_id": receipt.get("call_id"), + "args_hash": hashlib.sha256(canonical.encode("utf-8")).hexdigest(), + "target_path": target_path, "git_reference": reference, + "background": arguments.get("background") is True if isinstance(arguments, dict) else None, + "success": receipt.get("success") is True, "authorized": receipt.get("authorized") is True, + "error_code": (receipt.get("tool_error") or {}).get("code")} + + +def _context_observation(runtime): + port = getattr(runtime, "context_usage", None) + try: + status = port() if callable(port) else None + except Exception: + status = None + if not isinstance(status, dict): + return None + keys = ("model", "window_tokens", "window_source", "input_tokens", "input_source", "reserve_tokens", + "breakdown", "breakdown_source", "native_input_overhead_tokens", "native_overhead_method", + "stable_prefix_tokens", "stable_prefix_fingerprint", "compaction", + "new_context_tokens", "retransmitted_history_tokens", "new_entry_count", + "retransmitted_entry_count", "duplicate_entry_count") + result = {key: status.get(key) for key in keys} + new, retransmitted = result["new_context_tokens"], result["retransmitted_history_tokens"] + result["retained_history_tokens"] = new + retransmitted if type(new) is int and type(retransmitted) is int else None + result["component_attribution_source"] = status.get("component_attribution_source") + return json.loads(json.dumps(result)) + + +class _ProviderObservations: + """Observe each provider-port invocation after production preflight. + + Components are source-runtime estimates, never provider attribution. Internal + transport retries may reuse one invocation payload; the relay owns attempts. + """ + def __init__(self, runtime, provider): + self.runtime, self.provider = runtime, provider + self.records, self.turn, self.prompt_sha256 = [], None, None + self.depth = ContextVar("benchmark_provider_observation_depth", default=0) + for name in ("chat", "chat_stream", "chat_response", "chat_response_stream"): + method = getattr(provider, name, None) + if callable(method): + setattr(provider, name, self._wrap(method, name.endswith("stream"))) + + def _begin(self, messages, model, tools): + from openkyrozen.agent.compaction import estimate_messages, _text_tokens, DEFAULT_OUTPUT_RESERVE_TOKENS + runtime = self.runtime + compacting = getattr(runtime, "_in_context_compaction", None) + summary = bool(compacting.get()) if compacting is not None else False + state_var = getattr(runtime, "_active_context_state", None) + state = state_var.get() if state_var is not None else None + schema_json = json.dumps(tools, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + overhead = _text_tokens(schema_json) + 32 * len(tools or ()) + 64 if tools is not None else 0 + estimated, components = estimate_messages(messages) + components = dict(components) + components["fixed_instructions"] += overhead + status = _context_observation(runtime) if not summary else None + metrics_var = getattr(runtime, "_context_history_metrics", None) + metrics = metrics_var.get() if not summary and metrics_var is not None else None + metrics = metrics if isinstance(metrics, dict) else {} + new_history, repeated_history = metrics.get("new_context_tokens"), metrics.get("retransmitted_history_tokens") + record = {"turn": self.turn, "prompt_sha256": self.prompt_sha256, + "kind": "summary" if summary else "generation", "model": model, + "observation_boundary": "provider_port_after_production_preflight", + "estimate_method": "source_runtime_stdlib_messages_plus_json_schema_framing", + "window_tokens": getattr(self.provider.config, "context_window_tokens", None), + "reserve_tokens": getattr(state, "reserve_tokens", DEFAULT_OUTPUT_RESERVE_TOKENS), + "estimated_input_tokens": estimated + overhead, + "component_estimates": components, "component_attribution_source": "estimated", + "native_input_overhead_tokens": overhead, + "schema_sha256": hashlib.sha256(schema_json.encode()).hexdigest() if tools is not None else None, + "message_count": len(messages), + "message_projection_sha256": None, + "retained_history_tokens": new_history + repeated_history if type(new_history) is int and type(repeated_history) is int else None, + "new_context_tokens": new_history, + "retransmitted_history_tokens": repeated_history, + "history_attribution_source": "production_history_projection_estimate" if metrics else None, + "compaction": status.get("compaction") if status else None, + "provider_input_tokens": None, "provider_output_tokens": None, + "provider_usage_source": None, "outcome": "pending"} + # Fingerprint the same source-owned estimate projection without retaining content. + from openkyrozen.agent.compaction import message_fingerprint + record["message_projection_sha256"] = message_fingerprint(messages) + self.records.append(record) + return record + + @staticmethod + def _usage(record, usage): + if not isinstance(usage, dict) or usage.get("_estimated"): + return + for target, keys in (("provider_input_tokens", ("prompt_tokens", "input_tokens")), + ("provider_output_tokens", ("completion_tokens", "output_tokens"))): + value = next((usage[key] for key in keys if type(usage.get(key)) is int and usage[key] >= 0), None) + record[target] = value + if record["provider_input_tokens"] is not None: + record["provider_usage_source"] = "provider_reported_total" + + def _wrap(self, original, streaming): + def begin(args, kwargs): + messages = args[0] if args else kwargs["messages"] + model = args[1] if len(args) > 1 else kwargs.get("model") + return self._begin(messages, model, kwargs.get("tools")) + if streaming: + @wraps(original) + def stream(*args, **kwargs): + if self.depth.get(): + yield from original(*args, **kwargs) + return + record = begin(args, kwargs) + token = self.depth.set(self.depth.get() + 1) + iterator = None + try: + iterator = original(*args, **kwargs) + for event in iterator: + usage = getattr(event, "usage", None) + response = getattr(event, "response", None) + self._usage(record, getattr(response, "usage", None) or usage) + if type(event).__name__ == "ResponseError": + record["outcome"] = "failed" + elif type(event).__name__ == "ResponseComplete": + record["outcome"] = "returned" + yield event + if record["outcome"] == "pending": + record["outcome"] = "returned" + except BaseException: + if record["outcome"] == "pending": + record["outcome"] = "failed" + raise + finally: + try: + if callable(getattr(iterator, "close", None)): + iterator.close() + finally: + self.depth.reset(token) + return stream + @wraps(original) + def call(*args, **kwargs): + if self.depth.get(): + return original(*args, **kwargs) + record = begin(args, kwargs) + token = self.depth.set(self.depth.get() + 1) + try: + response = original(*args, **kwargs) + usage = response[1] if isinstance(response, tuple) and len(response) == 2 else getattr(response, "usage", None) + self._usage(record, usage) + record["outcome"] = "returned" + return response + except BaseException: + record["outcome"] = "failed" + raise + finally: + self.depth.reset(token) + return call + + +def _run_conversation(runtime, session, turns, observations=None): + records = [] + answer = "" + for index, prompt in enumerate(turns): + if observations is not None: + observations.turn = index + observations.prompt_sha256 = hashlib.sha256(prompt.encode("utf-8")).hexdigest() + answer = runtime.chat(session, prompt, clear_tasks=False, profile="auto") + records.append({"turn": index, "prompt_sha256": hashlib.sha256(prompt.encode("utf-8")).hexdigest(), + "completed": True, "context": _context_observation(runtime), + "messages_compacted": getattr(session, "messages_compacted", None)}) + return answer, records def _accepted_scope_preserved(history, accepted_fingerprints, accepted_plan, task_records): @@ -41,26 +246,47 @@ def main(): parser.add_argument("--process-marker", default="") parser.add_argument("--prompt-file", required=True) parser.add_argument("--plan-file") + parser.add_argument("--conversation-file") parser.add_argument("--result", required=True) args = parser.parse_args() + conversation = json.loads(Path(args.conversation_file).read_text()) if args.conversation_file else {} + followups = conversation.get("followups", []) + if (not isinstance(followups, list) or any(not isinstance(turn, str) or not turn for turn in followups) + or len(followups) > 20): + raise ValueError("benchmark followups must be a bounded array of nonempty strings") + window = conversation.get("context_window_tokens") + if window is not None and (type(window) is not int or not 1 <= window <= 10000000): + raise ValueError("benchmark context window must be a positive declared integer") + from_output_reserve = conversation.get("output_reserve_tokens", 4096) + if from_output_reserve != 4096: + raise ValueError("both benchmark arms require the frozen 4096-token output reserve") workspace_root = Path(args.workspace).resolve() sys.path.insert(0, args.source) + if os.environ.get("KYROZEN_BENCHMARK_BROKER_FD") is not None: + # V2 also has a benchmarks package; load this adapter by its owning file + # without allowing the current source tree to shadow the frozen snapshot. + import runpy + runpy.run_path(str(Path(__file__).with_name("v3_shell_launcher.py")))["install_worker_adapter"]() from openkyrozen.app.bootstrap import build_application from openkyrozen.providers import ProviderConfig from openkyrozen.providers.openai import OpenAICompatProvider, OpenAIResponsesProvider + from openkyrozen.agent.compaction import DEFAULT_OUTPUT_RESERVE_TOKENS + if DEFAULT_OUTPUT_RESERVE_TOKENS != from_output_reserve: + raise ValueError("source runtime output reserve differs from the frozen cohort") application = build_application(surface="cli") runtime = application.runtime try: runtime._set_workspace_root(args.workspace) - runtime.set_interaction_mode("agent") + runtime.set_interaction_mode("ask" if followups else "agent") provider_name = "openai" if args.protocol == "responses" else "deepseek" config = ProviderConfig(provider=provider_name, api_key=os.environ.get("KYROZEN_BENCHMARK_API_KEY", "benchmark-local"), base_url=args.url, model_simple=args.model, - model_complex=args.model) + model_complex=args.model, context_window_tokens=window) runtime._provider_config = config provider_type = OpenAIResponsesProvider if args.protocol == "responses" else OpenAICompatProvider runtime.llm_provider = provider_type(config) + observations = _ProviderObservations(runtime, runtime.llm_provider) runtime._default_turn.provider = runtime.llm_provider runtime.DEEPSEEK_MODEL_SIMPLE = runtime.DEEPSEEK_MODEL_COMPLEX = args.model # Keep production planning, tools, recovery, receipts and completion active. @@ -86,19 +312,7 @@ def on_event(event): first_delta_ms = round((time.monotonic() - started) * 1000, 2) if event.get("event") == "tool_receipt": receipt = event.get("tool_receipt", {}) - args = receipt.get("args") or {} - args_hash = hashlib.sha256(json.dumps(args, sort_keys=True, ensure_ascii=False).encode()).hexdigest() - path = args.get("path") or args.get("file_path") if isinstance(args, dict) else None - try: - target_path = Path(str(path)).resolve().relative_to(workspace_root).as_posix() - except (OSError, ValueError): - target_path = None - receipts.append({"action": receipt.get("action"), "args_hash": args_hash, - "target_path": target_path, - "background": isinstance(args, dict) and args.get("background") is True, - "success": receipt.get("success") is True, - "authorized": receipt.get("authorized") is True, - "error_code": (receipt.get("tool_error") or {}).get("code")}) + receipts.append(_normalize_tool_receipt(receipt, workspace_root)) events.append({"event": "tool_receipt", "action": receipt.get("action"), "success": receipt.get("success") is True}) elif event.get("event") in {"tasks", "task_update", "working_plan", "subagent"}: @@ -154,7 +368,7 @@ def on_event(event): if accepted_fingerprint: prompt = ("A Strategic Plan has already been accepted. Preserve its approved scope; use the working plan " "for implementation changes, and request renewed approval for material scope changes.\n\n" + prompt) - answer = runtime.chat(runtime.current_session, prompt, clear_tasks=False, profile="auto") + answer, conversation_records = _run_conversation(runtime, runtime.current_session, [prompt, *followups], observations) manager = getattr(runtime.current_session, "process_manager", None) handles = tuple(getattr(manager, "_handles", {}).values()) cancellations = sum(item["action"] == "stop_process" and item["success"] for item in receipts) @@ -204,6 +418,8 @@ def on_event(event): result = {"answer": answer, "tool_calls": sum(e.get("event") == "tool_receipt" for e in events), "tool_actions": [e.get("action") for e in events if e.get("event") == "tool_receipt"], "tool_receipts": receipts, "events": events, + "conversation_records": conversation_records, + "request_context_records": observations.records, "tool_successes": sum(e.get("event") == "tool_receipt" and e.get("success") is True for e in events), "tasks": [t["status"] for t in runtime.tasks.tasks], "task_records": [{"id": t["id"], "description": t["description"], @@ -223,7 +439,7 @@ def on_event(event): "duplicate_calls": len(receipts) - len(unique_calls), "protocol_recovery": sum(event["event"] in {"protocol_recovery", "protocol.recovery"} for event in events), - "processes_started": sum(item["action"] == "run_cmd" and item["background"] and item["success"] + "processes_started": sum(item["action"] == "run_cmd" and item["background"] is True and item["success"] for item in receipts), "processes_cancelled": cancellations, "processes_alive": sum(handle.process.poll() is None for handle in handles), @@ -231,7 +447,7 @@ def on_event(event): "process_marker_seen": any(args.process_marker and args.process_marker in handle.stdout for handle in handles), "first_delta_latency_ms": first_delta_ms, - "provider_error": "[LLM Error]" in answer} + "provider_error": "[LLM Error]" in answer or any(item["outcome"] == "failed" for item in observations.records)} result["accepted_scope_preserved"] = _accepted_scope_preserved( history, accepted_fingerprints, accepted_plan_identity, result["task_records"], ) diff --git a/benchmarks/v3_deepseek.py b/benchmarks/v3_deepseek.py new file mode 100644 index 00000000..6f4628d1 --- /dev/null +++ b/benchmarks/v3_deepseek.py @@ -0,0 +1,264 @@ +"""DeepSeek matched evaluation with a genuine cumulative 15 CNY cap. + +No credential files are consulted. Paid execution is explicit through this CLI; +importing this module and reconstructing old ledgers are offline operations. +""" +from __future__ import annotations + +import argparse +from contextlib import closing +from dataclasses import dataclass +from decimal import Decimal +import json +import hashlib +from pathlib import Path +import sqlite3 +import secrets + +from benchmarks.v3_matched import BudgetLedger, Pricing, _picormb +from benchmarks.v3_responses_relay import RelayConfig + +DEEPSEEK_ENDPOINT = "https://api.deepseek.com" +DEEPSEEK_PRICING = Pricing( + input_rate=Decimal("2"), output_rate=Decimal("8"), + cache_read_rate=Decimal("0.04"), cache_write_rate=Decimal("2"), + max_limit_cny=Decimal("15"), + metadata_json=json.dumps({ + "currency": "CNY", "source_currency": "CNY", "provider": "DeepSeek official", + "requested_model": "deepseek-flash", "served_model": "DeepSeek-V4.1-Flash-1017", + "pricing_source": "https://api-docs.deepseek.com/zh-cn/quick_start/pricing/", + "input_per_million": "2", "output_per_million": "8", + "cache_read_per_million": "0.04", "cache_write_per_million": "2", + "reservation": "peak rates, 1M input tokens and 4096 output tokens per call", + }, sort_keys=True), +) +DEEPSEEK_RELAY_CONFIG = RelayConfig( + model="deepseek-flash", max_output_tokens=4096, + fixed_input_token_bound=1_000_000, serial_requests=True, +) +# The previous campaign used the same CNY prices under this retired model alias. +_LEGACY_PRICING_SHA256 = "70cf92ad796b1061598eb1d5ac16aa19d473299758bcfdde86f789b2d40aabc4" + + +@dataclass(frozen=True) +class PriorExposure: + settled_cny: Decimal + held_cny: Decimal + attempt_count: int + records: tuple[str, ...] = () + + @property + def total_cny(self) -> Decimal: + return self.settled_cny + self.held_cny + + +def read_prior_exposure(paths: list[Path] | tuple[Path, ...]) -> PriorExposure: + """Reconstruct old costs read-only; duplicate journal attempts count once.""" + if not paths: + raise ValueError("at least one original prior campaign ledger is required") + seen: dict[str, str] = {} + amounts: dict[str, tuple[int, int]] = {} + sources: list[tuple[set[str], list[set[str]]]] = [] + opaque_anchors: list[set[str]] = [] + + def add(record: dict) -> tuple[int, int]: + attempt_id = record.get("id") + reserved, actual = record.get("reserved"), record.get("actual") + status, usage = record.get("status"), record.get("usage") + if (not isinstance(attempt_id, str) or not attempt_id or type(reserved) is not int + or reserved < 0 or status not in {"settled", "reserved", "unknown"}): + raise ValueError("invalid or overrun attempt in prior campaign ledger") + normalized = json.dumps({"id": attempt_id, "reserved": reserved, "actual": actual, + "status": status, "usage": usage}, sort_keys=True) + if attempt_id in seen: + if seen[attempt_id] != normalized: + raise ValueError("conflicting copies of a prior campaign attempt") + return amounts[attempt_id] + if status == "settled": + if type(actual) is not int or actual < 0 or actual > reserved or not isinstance(usage, dict): + raise ValueError("settled prior attempt lacks valid usage accounting") + required = ("input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens") + if any(field not in usage for field in required): + raise ValueError("settled prior attempt has incomplete usage accounting") + reconstructed = _picormb(DEEPSEEK_PRICING.cost(*(usage[field] for field in required))) + if actual != reconstructed: + raise ValueError("prior attempt charge differs from reconstructed CNY usage") + amount = (reconstructed, 0) + else: + if actual is not None or usage is not None: + raise ValueError("unsettled prior attempts must retain their full reservation") + amount = (0, reserved) + seen[attempt_id], amounts[attempt_id] = normalized, amount + return amount + + try: + for path in paths: + with closing(sqlite3.connect(Path(path).resolve().as_uri() + "?mode=ro", uri=True)) as db: + db.execute("PRAGMA query_only=ON") + config = db.execute("SELECT currency_code,pricing_sha256 FROM budget_config WHERE id=1").fetchone() + if config not in {("CNY", _LEGACY_PRICING_SHA256), ("CNY", DEEPSEEK_PRICING.sha256)}: + raise ValueError("prior ledger does not pin the approved DeepSeek CNY prices") + attempts = db.execute( + "SELECT id,reserved_picormb,actual_picormb,status,usage_json,metadata_json FROM attempts" + ).fetchall() + source_ids: set[str] = set() + manifests: list[set[str]] = [] + direct_opaque = False + for attempt_id, reserved, actual, status, usage_json, metadata_json in attempts: + metadata = json.loads(metadata_json) if metadata_json else {} + if attempt_id == "prior-campaign-reservations" and "prior_attempts" in metadata: + manifest = metadata["prior_attempts"] + if (status != "unknown" or actual is not None or usage_json is not None + or not isinstance(manifest, list) or not manifest + or hashlib.sha256(json.dumps(manifest, sort_keys=True).encode()).hexdigest() + != metadata.get("prior_attempts_sha256")): + raise ValueError("carried prior campaign provenance is invalid") + represented = [json.loads(item) for item in manifest] + if len({item["id"] for item in represented}) != len(represented): + raise ValueError("carried prior campaign contains duplicate attempts") + manifest_ids = {item["id"] for item in represented} + manifests.append(manifest_ids) + source_ids.update(manifest_ids) + if "prior-campaign-reservations" in manifest_ids: + opaque_anchors.append(manifest_ids) + reconstructed = sum(sum(add(item)) for item in represented) + if reconstructed != reserved: + raise ValueError("carried prior campaign hold differs from its underlying attempts") + continue + add({"id": attempt_id, "reserved": reserved, "actual": actual, + "status": status, "usage": json.loads(usage_json) if usage_json else None}) + source_ids.add(attempt_id) + direct_opaque = direct_opaque or attempt_id == "prior-campaign-reservations" + sources.append((source_ids, manifests)) + if direct_opaque: + opaque_anchors.append(source_ids) + except (sqlite3.Error, OSError, TypeError, KeyError, AttributeError, json.JSONDecodeError) as exc: + raise ValueError("cannot reconstruct the required prior campaign ledger") from exc + # Opaque holds can accompany copies or proven descendants, not unrelated + # source histories whose costs may already be included in that aggregate. + for anchor in opaque_anchors: + if any(not source <= anchor and not any(anchor <= manifest for manifest in manifests) + for source, manifests in sources): + raise ValueError("ambiguous opaque prior campaign aggregate requires original provenance") + settled = sum(amount[0] for amount in amounts.values()) + held = sum(amount[1] for amount in amounts.values()) + return PriorExposure(Decimal(settled) / 1_000_000_000_000, + Decimal(held) / 1_000_000_000_000, len(seen), + tuple(seen[key] for key in sorted(seen))) + + +def create_budget(path: Path, *, prior_ledgers: list[Path] | tuple[Path, ...], + limit_cny: str | Decimal = "15") -> BudgetLedger: + exposure = read_prior_exposure(prior_ledgers) + return BudgetLedger(path, limit_rmb=limit_cny, prior_hold_rmb=exposure.total_cny, + pricing=DEEPSEEK_PRICING, prior_hold_records=exposure.records) + + +def run_smoke(relay, candidate: str) -> bool: + """Verify typed tool continuation through the same metered campaign relay.""" + if getattr(relay, "smoke_verified_candidate", None) == candidate: + return True + import requests + probe = secrets.token_hex(16) + tools = [{"type": "function", "name": "echo_probe", "description": "Echo a benchmark probe value.", + "parameters": {"type": "object", "properties": { + "value": {"type": "string", "enum": [probe]}}, + "required": ["value"], "additionalProperties": False}, "strict": True}] + history = [{"role": "user", "content": + "Call echo_probe with the allowed value, then repeat the returned value exactly."}] + relay.label = ("smoke", candidate, -1) + attempt_ids = [] + + def request(body: dict) -> dict | None: + before = len(relay.rows) + response = requests.post(relay.url + "/responses", json=body, + headers={"Authorization": "Bearer " + relay.client_key}, timeout=190) + if (response.status_code != 200 or len(relay.rows) != before + 1 + or relay.rows[-1].get("status") != "settled" + or not isinstance(relay.rows[-1].get("usage"), dict) or not relay.budget.budget_ok): + if not relay.budget_blocked and not relay.provider_error: + relay.provider_error = "smoke_incomplete_usage" + return None + attempt_ids.append(relay.rows[-1]["attempt_id"]) + result = response.json() + return result if isinstance(result, dict) else None + + try: + first = request({"input": history, "tools": tools, + "tool_choice": {"type": "function", "name": "echo_probe"}, + "parallel_tool_calls": False, "stream": False}) + if first is None: + return False + output = first.get("output") + calls = [item for item in output if isinstance(item, dict) + and item.get("type") == "function_call"] if isinstance(output, list) else [] + if len(calls) != 1: + relay.provider_error = "smoke_invalid_tool_call" + return False + call = calls[0] + if (call.get("name") != "echo_probe" or not isinstance(call.get("call_id"), str) + or not call["call_id"].strip() or not isinstance(call.get("arguments"), str)): + relay.provider_error = "smoke_invalid_tool_call" + return False + arguments = json.loads(call["arguments"]) + if arguments != {"value": probe}: + relay.provider_error = "smoke_invalid_tool_arguments" + return False + final = request({"input": history + output + [{"type": "function_call_output", + "call_id": call["call_id"], "output": json.dumps({"value": probe})}], + "tools": tools, "tool_choice": "none", "stream": False}) + if final is None: + return False + final_output = final.get("output") + messages = final_output if isinstance(final_output, list) else [] + text = "".join(content.get("text", "") for item in messages if isinstance(item, dict) + and item.get("type") == "message" and item.get("role") == "assistant" + and isinstance(item.get("content"), list) + for content in item["content"] if isinstance(content, dict) + and content.get("type") == "output_text" and isinstance(content.get("text"), str)) + if (probe not in text or any(isinstance(item, dict) and item.get("type") == "function_call" + for item in messages)): + relay.provider_error = "smoke_invalid_continuation" + return False + except (requests.RequestException, ValueError): + if not relay.provider_error: + relay.provider_error = "smoke_invalid_response" + return False + relay.smoke_verified_candidate = candidate + relay.smoke_verified_attempt_ids = tuple(attempt_ids) + return True + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--candidate-sha", required=True) + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--budget-db", type=Path, required=True) + parser.add_argument("--prior-ledger", type=Path, action="append", required=True) + parser.add_argument("--limit-cny", default="15") + args = parser.parse_args() + from benchmarks.v3_matched_live import _source_snapshot, run + from scripts.publish_v3_preview import _verify_candidate_signature + if _source_snapshot(args.source.resolve()) != args.candidate_sha: + raise ValueError("source does not match the configured frozen candidate") + _verify_candidate_signature(args.source.resolve(), args.candidate_sha) + exposure = read_prior_exposure(args.prior_ledger) + # Validate the absolute cap before starting the runner; it probes protection before reading the key. + limit = Decimal(args.limit_cny) + if not limit.is_finite() or not 0 < limit <= 15 or exposure.total_cny > limit: + raise ValueError("the cumulative DeepSeek cap must not exceed 15 CNY") + report = run(source=args.source, repo=args.repo, output=args.output, budget_path=args.budget_db, + limit_cny=str(limit), prior_hold_cny=str(exposure.total_cny), + prior_hold_records=exposure.records, + pricing=DEEPSEEK_PRICING, relay_config=DEEPSEEK_RELAY_CONFIG, + endpoint=DEEPSEEK_ENDPOINT) + print(json.dumps(report["summary"], indent=2, sort_keys=True)) + summary = report["summary"] + raise SystemExit(0 if summary["complete"] and summary["accounting_complete"] + and summary["comparison_thresholds_met"] else 1) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/v3_fixtures.py b/benchmarks/v3_fixtures.py index e8f35130..4d09222d 100644 --- a/benchmarks/v3_fixtures.py +++ b/benchmarks/v3_fixtures.py @@ -2,6 +2,8 @@ from __future__ import annotations from collections import Counter +import hashlib +import json CATEGORIES = ( @@ -10,12 +12,32 @@ ) -def _fixture(category, index, prompt, verifier, *, files=None, plan_scenario=None, capabilities=None): +def _fixture(category, index, prompt, verifier, *, files=None, plan_scenario=None, capabilities=None, turns=None, context_window_tokens=None): return {"fixture_id": f"{category}-{index}", "category": category, "prompt": prompt, "files": files or {}, "verifier": verifier, "plan_scenario": plan_scenario, + "turns": turns, "context_window_tokens": context_window_tokens or 1_000_000, + "output_reserve_tokens": 4096, "capabilities": capabilities if capabilities is not None else ["read", "write", "shell", "git"]} +def _long_context_turns(initial, expected, changed): + """Deterministic frozen conversation; diagnostic batches force real history pressure.""" + turns = ["Retain these approved project facts for this conversation: " + + json.dumps(initial, sort_keys=True) + + ". Reply briefly; do not create files or plans. Later messages may supersede one fact."] + for batch in range(8): + if batch == 3: + turns.append("Approved correction: " + changed + " is now " + str(expected[changed]) + + ". The previous " + str(initial[changed]) + + " is superseded. All other original facts remain unchanged. Retain this correction and acknowledge briefly.") + log = "\n".join(f"Archived event {batch}-{line}: inspected sample {line*17+batch}; outcome unchanged; checksum {line*7919+batch:08x}." + for line in range(110)) + turns.append("Keep the approved facts while reviewing this archival diagnostic batch. This batch changes none of the approved facts. Acknowledge briefly.\n" + log) + turns.append("Return only a JSON object with exactly these keys: " + ", ".join(expected) + + ". Use their currently approved values from our conversation; exclude superseded values. Do not reread files or repeat archival details.") + return tuple(turns) + + FIXTURES = ( _fixture("investigation", 1, "Inspect catalog.py and report which function sorts products and its exact tie-break rule. Do not edit files.", {"kind": "answer_contains", "all": ["sorted_products", "casefold", "original spelling"]}, @@ -35,23 +57,23 @@ def _fixture(category, index, prompt, verifier, *, files=None, plan_scenario=Non _fixture("bug_fix", 3, "Fix the off-by-one boundary in paging.py so page 1 starts at offset 0; the independent evaluator will run test_paging.py. A proposed change to the public paging API would expand scope; request renewed approval before including it.", {"kind": "tests", "command": ["python", "-m", "unittest", "test_paging.py"]}, files={"paging.py": "def offset(page, size):\n return page * size\n", "test_paging.py": "import unittest\nfrom paging import offset\nclass T(unittest.TestCase):\n def test_first_page(self): self.assertEqual(offset(1, 10), 0)\n def test_second_page(self): self.assertEqual(offset(2, 10), 10)\n"}, plan_scenario="material_revision"), - _fixture("implementation", 1, "Implement unique_sorted(values) in catalog.py using case-insensitive order and lexical tie-breaks. Run test_catalog.py. If verification reveals an uncovered case, add a verification task with update_plan.", - {"kind": "tests", "command": ["python", "-m", "unittest", "test_catalog.py"]}, - files={"catalog.py": "def unique_sorted(values):\n raise NotImplementedError\n", "test_catalog.py": "import unittest\nfrom catalog import unique_sorted\nclass T(unittest.TestCase):\n def test_order(self): self.assertEqual(unique_sorted(['b','a','A','b']), ['A','a','b'])\n"}, plan_scenario="add_discovered_verification_task"), + _fixture("implementation", 1, 'Implement POST /items in the existing in-process WSGI API across api.py and service.py. Return 201 JSON {id,name}, trim non-empty string names, return 400 for malformed/invalid bodies, and 409 for duplicate names. Preserve GET /items and shared service state. Modify both production modules and run test_api.py. Add a working-plan verification task for the discovered duplicate-name case.', + {'kind': 'tests', 'command': ['python', '-m', 'unittest', 'test_api.py'], 'required_changed_paths': ['api.py', 'service.py']}, + files={'api.py': "import json\nfrom service import list_items\n\ndef app(environ, start_response):\n if environ['REQUEST_METHOD'] == 'GET' and environ['PATH_INFO'] == '/items':\n status, body = '200 OK', list_items()\n else:\n status, body = '404 Not Found', {'error': 'not found'}\n start_response(status, [('Content-Type', 'application/json')])\n return [json.dumps(body).encode('utf-8')]\n", 'service.py': 'ITEMS = []\n\ndef list_items():\n return list(ITEMS)\n\ndef create_item(payload):\n raise NotImplementedError\n', 'test_api.py': 'import io\nimport json\nimport unittest\nimport service\nfrom api import app\n\nclass ApiTests(unittest.TestCase):\n def setUp(self): service.ITEMS.clear()\n def request(self, method, payload=b\'\'):\n captured = []\n body = b\'\'.join(app({\'REQUEST_METHOD\': method, \'PATH_INFO\': \'/items\',\n \'CONTENT_LENGTH\': str(len(payload)), \'wsgi.input\': io.BytesIO(payload)},\n lambda status, headers: captured.append((status, dict(headers)))))\n return captured[0][0], captured[0][1], json.loads(body)\n def test_existing_get(self):\n self.assertEqual(self.request(\'GET\')[::2], (\'200 OK\', []))\n def test_post_and_get_share_service_state(self):\n status, headers, created = self.request(\'POST\', b\'{"name":" Cedar "}\')\n self.assertEqual(status, \'201 Created\')\n self.assertEqual(headers[\'Content-Type\'], \'application/json\')\n self.assertEqual(created, {\'id\': 1, \'name\': \'Cedar\'})\n self.assertEqual(self.request(\'GET\')[2], [created])\n def test_service_contract(self):\n self.assertEqual(service.create_item({\'name\': \' Birch \'}), {\'id\': 1, \'name\': \'Birch\'})\n with self.assertRaises(ValueError): service.create_item({\'name\': \'\'})\n def test_bad_payloads(self):\n for payload in (b\'broken json\', b\'{}\', b\'{"name":" "}\', b\'{"name":7}\', b\'[]\', b\'null\', b\'\"name\"\'):\n with self.subTest(payload=payload): self.assertEqual(self.request(\'POST\', payload)[0], \'400 Bad Request\')\n def test_duplicate_name(self):\n self.request(\'POST\', b\'{"name":"Cedar"}\')\n self.assertEqual(self.request(\'POST\', b\'{"name":"Cedar"}\')[0], \'409 Conflict\')\n'}, plan_scenario="add_discovered_verification_task"), _fixture("implementation", 2, "Implement slugify(text) in slug.py: lowercase, replace spaces with hyphens, remove punctuation. Run test_slug.py. The stated requirements are complete; avoid revising an adequate plan without new evidence.", {"kind": "tests", "command": ["python", "-m", "unittest", "test_slug.py"]}, files={"slug.py": "def slugify(text):\n raise NotImplementedError\n", "test_slug.py": "import unittest\nfrom slug import slugify\nclass T(unittest.TestCase):\n def test_slug(self): self.assertEqual(slugify('Hello, World!'), 'hello-world')\n"}, plan_scenario="unnecessary_churn"), - _fixture("implementation", 3, "Implement parse_flags(text) in flags.py to parse comma-separated trimmed names, ignoring empty items. Run test_flags.py. Dependency discovery shows parsing must precede edge-case verification; reorder the working plan with update_plan.", - {"kind": "tests", "command": ["python", "-m", "unittest", "test_flags.py"]}, - files={"flags.py": "def parse_flags(text):\n raise NotImplementedError\n", "test_flags.py": "import unittest\nfrom flags import parse_flags\nclass T(unittest.TestCase):\n def test_flags(self): self.assertEqual(parse_flags(' a, ,b '), ['a','b'])\n"}, plan_scenario="reorder_after_dependency_discovery"), - _fixture("refactoring", 1, "Use a working plan to refactor duplicated formatting logic in labels.py into a shared helper without changing outputs. The independent evaluator will run test_labels.py. Keep the user-approved objective fixed while adapting implementation steps.", - {"kind": "tests", "command": ["python", "-m", "unittest", "test_labels.py"]}, + _fixture("implementation", 3, 'Implement the public parse_flags(text, *, unique=False) API in flags.py, preserving duplicates by default and supporting stable deduplication when unique=True. Update consumer.py settings() to opt in to unique names. Ignore empty trimmed comma fields. Both production modules must change. Run test_flags.py. Reorder the working plan so implementation precedes edge-case and consumer verification.', + {'kind': 'tests', 'command': ['python', '-m', 'unittest', 'test_flags.py'], 'required_changed_paths': ['flags.py', 'consumer.py']}, + files={'flags.py': 'def parse_flags(text, *, unique=False):\n raise NotImplementedError\n', 'consumer.py': "from flags import parse_flags\n\ndef settings(text):\n return {'enabled': parse_flags(text)}\n", 'test_flags.py': "import unittest\nfrom flags import parse_flags\nfrom consumer import settings\nclass FlagTests(unittest.TestCase):\n def test_legacy_duplicates(self): self.assertEqual(parse_flags(' a, ,b,a '), ['a', 'b', 'a'])\n def test_opt_in_unique(self): self.assertEqual(parse_flags(' a,b,a,b ', unique=True), ['a', 'b'])\n def test_empty(self): self.assertEqual(parse_flags(' , '), [])\n def test_consumer(self): self.assertEqual(settings(' a,b,a '), {'enabled': ['a', 'b']})\n"}, plan_scenario="reorder_after_dependency_discovery"), + _fixture("refactoring", 1, "Use a working plan to refactor duplicated formatting logic in labels.py into a shared module-level _format_label(name) helper; each wrapper must directly return the helper call without changing outputs. The independent evaluator will run test_labels.py. Keep the user-approved objective fixed while adapting implementation steps.", + {'kind': 'tests', 'command': ['python', '-m', 'unittest', 'test_labels.py'], 'shared_helper': {'path': 'labels.py', 'name': '_format_label', 'wrappers': ['primary', 'secondary']}}, files={"labels.py": "def primary(name):\n return name.strip().title()\ndef secondary(name):\n return name.strip().title()\n", "test_labels.py": "import unittest\nfrom labels import primary, secondary\nclass T(unittest.TestCase):\n def test_values(self): self.assertEqual(primary(' alpha '), 'Alpha'); self.assertEqual(secondary(' beta '), 'Beta')\n"}, plan_scenario="scope_preserving"), - _fixture("refactoring", 2, "Refactor repeated bounds checks in ranges.py while preserving inclusive endpoints. Run test_ranges.py.", - {"kind": "tests", "command": ["python", "-m", "unittest", "test_ranges.py"]}, + _fixture("refactoring", 2, "Refactor repeated bounds checks in ranges.py into one module-level _in_bounds(x) helper; first() and second() must directly return the helper call while preserving inclusive endpoints. Run test_ranges.py.", + {'kind': 'tests', 'command': ['python', '-m', 'unittest', 'test_ranges.py'], 'shared_helper': {'path': 'ranges.py', 'name': '_in_bounds', 'wrappers': ['first', 'second']}}, files={"ranges.py": "def first(x):\n return 0 <= x <= 10\ndef second(x):\n return 0 <= x <= 10\n", "test_ranges.py": "import unittest\nfrom ranges import first, second\nclass T(unittest.TestCase):\n def test_edges(self): self.assertTrue(first(0)); self.assertTrue(second(10)); self.assertFalse(first(-1))\n"}), - _fixture("refactoring", 3, "Refactor duplicated JSON serialization in output.py into a helper; preserve sorted keys and unicode. Run test_output.py.", - {"kind": "tests", "command": ["python", "-m", "unittest", "test_output.py"]}, + _fixture("refactoring", 3, "Refactor duplicated JSON serialization in output.py into one module-level _serialize(v) helper; one() and two() must directly return the helper call; preserve sorted keys and unicode. Run test_output.py.", + {'kind': 'tests', 'command': ['python', '-m', 'unittest', 'test_output.py'], 'shared_helper': {'path': 'output.py', 'name': '_serialize', 'wrappers': ['one', 'two']}}, files={"output.py": "import json\ndef one(v): return json.dumps(v, ensure_ascii=False, sort_keys=True)\ndef two(v): return json.dumps(v, ensure_ascii=False, sort_keys=True)\n", "test_output.py": "import unittest\nfrom output import one, two\nclass T(unittest.TestCase):\n def test_json(self): self.assertEqual(one({'字':1}), '{\"字\": 1}'); self.assertEqual(two({'b':2,'a':1}), '{\"a\": 1, \"b\": 2}')\n"}), _fixture("git_history", 1, "Use the Git history tools to identify the commit that changed LIMIT from 10 to 20; report subject and short hash.", {"kind": "git_subject", "subject": "raise catalog limit", "path": "catalog.py", "old": "LIMIT = 10", "new": "LIMIT = 20"}, @@ -71,15 +93,18 @@ def _fixture(category, index, prompt, verifier, *, files=None, plan_scenario=Non _fixture("error_recovery", 3, "An edit using an obsolete snippet will fail. Re-read cache.py, make get([]) return None, and pass test_cache.py.", {"kind": "tests", "command": ["python", "-m", "unittest", "test_cache.py"]}, files={"cache.py": "def get(values):\n return values[0]\n", "test_cache.py": "import unittest\nfrom cache import get\nclass T(unittest.TestCase):\n def test_empty(self): self.assertIsNone(get([]))\n"}), - _fixture("long_context", 1, "Search the documentation and report the deployment region, owner team, and rollback tag. Do not edit files.", - {"kind": "answer_contains", "all": ["eu-west-2", "team-cedar", "release-17"]}, - files={"docs/00-overview.md": "Quarterly deployment dossier.\n" + "Unrelated status note.\n" * 800, "docs/70-deployment.md": "region: eu-west-2\nowner: team-cedar\nrollback: release-17\n"}), - _fixture("long_context", 2, "Find the migration's batch size, retry delay, and final verification command across the project docs. Do not edit files.", - {"kind": "answer_contains", "all": ["batch_size=64", "retry_delay=3s", "python -m unittest"]}, - files={"README.md": "Migration notes\n" + "Background detail.\n" * 700, "docs/migration.md": "batch_size=64\nretry_delay=3s\nverify: python -m unittest\n"}), - _fixture("long_context", 3, "Search all project notes and report the feature flag, cutoff date, and fallback path. Do not edit files.", - {"kind": "answer_contains", "all": ["feature_x", "2026-11-01", "legacy_parser"]}, - files={"notes/00.md": "Project notes.\n" + "Old decision.\n" * 600, "notes/99.md": "flag: feature_x\ncutoff: 2026-11-01\nfallback: legacy_parser\n"}), + _fixture("long_context", 1, 'Retain these approved project facts for this conversation: {"owner": "team-cedar", "region": "eu-west-2", "rollback": "release-17"}. Reply briefly; do not create files or plans. Later messages may supersede one fact.', + {'kind': 'long_context', 'expected': {'region': 'ap-southeast-1', 'owner': 'team-cedar', 'rollback': 'release-17'}, 'all': ['ap-southeast-1', 'team-cedar', 'release-17'], 'superseded_value': 'eu-west-2', 'expected_turns': 11, 'window_tokens': 16384}, + files={"README.md":"Conversation continuity fixture; approved facts arrive through user turns.\n"}, capabilities=["read"], + turns=_long_context_turns({'region': 'eu-west-2', 'owner': 'team-cedar', 'rollback': 'release-17'}, {'region': 'ap-southeast-1', 'owner': 'team-cedar', 'rollback': 'release-17'}, 'region'), context_window_tokens=16384), + _fixture("long_context", 2, 'Retain these approved project facts for this conversation: {"batch_size": 64, "retry_delay": "3s", "verify": "python -m unittest"}. Reply briefly; do not create files or plans. Later messages may supersede one fact.', + {'kind': 'long_context', 'expected': {'batch_size': 64, 'retry_delay': '5s', 'verify': 'python -m unittest'}, 'all': ['64', '5s', 'python -m unittest'], 'superseded_value': '3s', 'expected_turns': 11, 'window_tokens': 16384}, + files={"README.md":"Conversation continuity fixture; approved facts arrive through user turns.\n"}, capabilities=["read"], + turns=_long_context_turns({'batch_size': 64, 'retry_delay': '3s', 'verify': 'python -m unittest'}, {'batch_size': 64, 'retry_delay': '5s', 'verify': 'python -m unittest'}, 'retry_delay'), context_window_tokens=16384), + _fixture("long_context", 3, 'Retain these approved project facts for this conversation: {"cutoff": "2026-11-01", "fallback": "legacy_parser", "flag": "feature_x"}. Reply briefly; do not create files or plans. Later messages may supersede one fact.', + {'kind': 'long_context', 'expected': {'flag': 'feature_x', 'cutoff': '2026-12-01', 'fallback': 'legacy_parser'}, 'all': ['feature_x', '2026-12-01', 'legacy_parser'], 'superseded_value': '2026-11-01', 'expected_turns': 11, 'window_tokens': 16384}, + files={"README.md":"Conversation continuity fixture; approved facts arrive through user turns.\n"}, capabilities=["read"], + turns=_long_context_turns({'flag': 'feature_x', 'cutoff': '2026-11-01', 'fallback': 'legacy_parser'}, {'flag': 'feature_x', 'cutoff': '2026-12-01', 'fallback': 'legacy_parser'}, 'cutoff'), context_window_tokens=16384), _fixture("subagents", 1, "Independently inspect parser.py and tests, implement the missing empty-input behavior, and verify the repository. Delegate the independent test review if available.", {"kind": "tests", "command": ["python", "-m", "unittest", "test_parser.py"]}, files={"parser.py": "def count_items(values):\n return len(values)\n", "test_parser.py": "import unittest\nfrom parser import count_items\nclass T(unittest.TestCase):\n def test_empty(self): self.assertEqual(count_items(None), 0)\n"}), @@ -115,12 +140,12 @@ def _fixture(category, index, prompt, verifier, *, files=None, plan_scenario=Non "bug_fix-2": [("assumption", "Check the non-null input assumption."), ("fallback", "Implement and verify the None fallback.")], "bug_fix-3": [("offset", "Fix and verify the current offset contract.")], - "implementation-1": [("implement", "Implement unique_sorted."), - ("verify", "Verify duplicates and case-folded ordering.")], + "implementation-1": [("implement", "Implement POST /items across API and service."), + ("verify", "Verify duplicate-name handling and existing GET behavior.")], "implementation-2": [("implement", "Implement the specified slug behavior."), ("verify", "Run the independent slug tests.")], "implementation-3": [("verify", "Check empty and whitespace-only fields."), - ("implement", "Implement the comma-separated parser.")], + ("implement", "Implement the flag API and update its consumer.")], "refactoring-1": [("inspect", "Identify common formatting behavior."), ("refactor", "Extract a shared helper without changing outputs.")], "cancellation-2": [("obsolete", "Start the obsolete check and cancel it."), @@ -139,6 +164,14 @@ def validate_fixtures(fixtures): raise ValueError("the matched evaluation requires three fixtures in every category") if any(not fixture.get("prompt") or not fixture.get("verifier") for fixture in fixtures): raise ValueError("each benchmark fixture requires a prompt and independent verifier") + for fixture in fixtures: + long_session = fixture["category"] == "long_context" + if (fixture.get("context_window_tokens") != (16384 if long_session else 1_000_000) + or fixture.get("output_reserve_tokens") != 4096): + raise ValueError("matched context and output budgets must equal the frozen per-fixture settings") + if long_session and (not isinstance(fixture.get("turns"), tuple) + or len(fixture["turns"]) != 11 or fixture["turns"][0] != fixture["prompt"]): + raise ValueError("long-context fixtures require the frozen eleven-turn session") scenarios = {fixture.get("plan_scenario") for fixture in fixtures if fixture.get("plan_scenario")} if not {"add_discovered_edge_case", "cancel_obsolete_task_after_reproduction", "reorder_after_dependency_discovery", "scope_preserving", "material_revision", @@ -147,4 +180,15 @@ def validate_fixtures(fixtures): return True +def cohort_digest(fixtures=FIXTURES): + """Freeze content as well as IDs; old campaign rows cannot prove a revised task.""" + return hashlib.sha256(json.dumps(fixtures, sort_keys=True, ensure_ascii=False, + separators=(",", ":"), allow_nan=False).encode("utf-8")).hexdigest() + + +FROZEN_COHORT_SHA256 = "401006750309625cd44545b46ddb7190cf850799a4a617850220cbe843927b73" + + validate_fixtures(FIXTURES) +if cohort_digest() != FROZEN_COHORT_SHA256: + raise ValueError("fixture content changed; review and freeze a new cohort before any campaign") diff --git a/benchmarks/v3_matched.py b/benchmarks/v3_matched.py index 72225e1e..a2fc51a5 100644 --- a/benchmarks/v3_matched.py +++ b/benchmarks/v3_matched.py @@ -7,6 +7,7 @@ import random import sqlite3 from collections import defaultdict +from dataclasses import dataclass from decimal import Decimal, ROUND_CEILING from pathlib import Path from statistics import mean, median @@ -31,15 +32,54 @@ PRICING_SHA256 = hashlib.sha256(json.dumps(PRICING, sort_keys=True).encode()).hexdigest() +@dataclass(frozen=True) +class Pricing: + """Immutable CNY prices and cap, pinned in every persisted ledger.""" + + input_rate: Decimal + output_rate: Decimal + cache_read_rate: Decimal + cache_write_rate: Decimal + max_limit_cny: Decimal + metadata_json: str + + def __post_init__(self) -> None: + rates = (self.input_rate, self.output_rate, self.cache_read_rate, self.cache_write_rate) + if (any(not rate.is_finite() or rate < 0 for rate in rates) + or not self.max_limit_cny.is_finite() or self.max_limit_cny <= 0): + raise ValueError("pricing rates and cumulative CNY cap must be finite and nonnegative") + metadata = self.as_dict() + if (metadata.get("currency") != "CNY" or any( + Decimal(metadata.get(name, "NaN")) != rate for name, rate in zip( + ("input_per_million", "output_per_million", "cache_read_per_million", "cache_write_per_million"), rates))): + raise ValueError("pricing metadata must match the CNY rates") + + def as_dict(self) -> dict: + return json.loads(self.metadata_json) + + @property + def sha256(self) -> str: + return hashlib.sha256(json.dumps(self.as_dict(), sort_keys=True).encode()).hexdigest() + + def cost(self, input_tokens: int, output_tokens: int, cache_read_tokens: int = 0, + cache_write_tokens: int = 0) -> Decimal: + counts = (input_tokens, output_tokens, cache_read_tokens, cache_write_tokens) + if (any(type(value) is not int or value < 0 for value in counts) + or cache_read_tokens + cache_write_tokens > input_tokens): + raise ValueError("usage token counts are invalid") + return ((input_tokens - cache_read_tokens - cache_write_tokens) * self.input_rate + + cache_write_tokens * self.cache_write_rate + + cache_read_tokens * self.cache_read_rate + + output_tokens * self.output_rate) / _MILLION + + +PANSTAR_PRICING = Pricing(INPUT_RATE, OUTPUT_RATE, CACHE_READ_RATE, CACHE_WRITE_RATE, + MAX_LIMIT_RMB, json.dumps(PRICING, sort_keys=True)) + + def _cost(input_tokens: int, output_tokens: int, cache_read_tokens: int = 0, cache_write_tokens: int = 0) -> Decimal: - if (min(input_tokens, output_tokens, cache_read_tokens, cache_write_tokens) < 0 - or cache_read_tokens + cache_write_tokens > input_tokens): - raise ValueError("usage token counts are invalid") - return ((input_tokens - cache_read_tokens - cache_write_tokens) * INPUT_RATE - + cache_write_tokens * CACHE_WRITE_RATE - + cache_read_tokens * CACHE_READ_RATE - + output_tokens * OUTPUT_RATE) / _MILLION + return PANSTAR_PRICING.cost(input_tokens, output_tokens, cache_read_tokens, cache_write_tokens) def _picormb(amount: Decimal) -> int: @@ -50,12 +90,16 @@ class BudgetLedger: """Atomically reserve worst-case API spend and retain unknown attempts.""" def __init__(self, path: Path, *, limit_rmb: str | Decimal = MAX_LIMIT_RMB, - prior_hold_rmb: str | Decimal = "0", allow_limit_increase: bool = False): + prior_hold_rmb: str | Decimal = "0", allow_limit_increase: bool = False, + pricing: Pricing = PANSTAR_PRICING, + prior_hold_records: tuple[str, ...] | None = None): self.path = Path(path) + self.pricing = pricing limit = Decimal(limit_rmb) prior_hold = Decimal(prior_hold_rmb) - if not Decimal(0) < limit <= MAX_LIMIT_RMB or not Decimal(0) <= prior_hold <= limit: - raise ValueError("the cumulative benchmark cap must be at most USD 20 at the recorded exchange rate") + if (not limit.is_finite() or not prior_hold.is_finite() + or not Decimal(0) < limit <= pricing.max_limit_cny or not Decimal(0) <= prior_hold <= limit): + raise ValueError("the cumulative benchmark cap exceeds the immutable pricing configuration") self.limit_picormb = _picormb(limit) self.path.parent.mkdir(parents=True, exist_ok=True) with closing(sqlite3.connect(self.path)) as db: @@ -80,13 +124,18 @@ def __init__(self, path: Path, *, limit_rmb: str | Decimal = MAX_LIMIT_RMB, db.execute("ALTER TABLE attempts ADD COLUMN metadata_json TEXT") current = db.execute("SELECT currency_code,limit_picormb,pricing_sha256 FROM budget_config WHERE id=1").fetchone() if current is None: - db.execute("INSERT INTO budget_config VALUES(1,'CNY',?,?)", (self.limit_picormb, PRICING_SHA256)) + db.execute("INSERT INTO budget_config VALUES(1,'CNY',?,?)", (self.limit_picormb, self.pricing.sha256)) if prior_hold: + prior_metadata = {"archived_reservation": True, + "prior_cny_reservations": str(prior_hold)} + if prior_hold_records is not None: + prior_metadata["prior_attempts"] = list(prior_hold_records) + prior_metadata["prior_attempts_sha256"] = hashlib.sha256( + json.dumps(list(prior_hold_records), sort_keys=True).encode()).hexdigest() db.execute("INSERT INTO attempts(id,reserved_picormb,status,metadata_json) VALUES(?,?,'unknown',?)", ("prior-campaign-reservations", _picormb(prior_hold), - json.dumps({"archived_reservation": True, - "prior_cny_reservations": str(prior_hold)}, sort_keys=True))) - elif current[0] != "CNY" or current[2] != PRICING_SHA256: + json.dumps(prior_metadata, sort_keys=True))) + elif current[0] != "CNY" or current[2] != self.pricing.sha256: raise ValueError("the persisted RMB benchmark cap and pricing cannot be changed") elif int(current[1]) != self.limit_picormb: if int(current[1]) < self.limit_picormb: @@ -106,8 +155,18 @@ def __init__(self, path: Path, *, limit_rmb: str | Decimal = MAX_LIMIT_RMB, "SELECT reserved_picormb FROM attempts WHERE id='prior-campaign-reservations'" ).fetchone() != (_picormb(prior_hold),): raise ValueError("the persisted ledger is missing the prior campaign reservation") + if prior_hold_records is not None: + record = db.execute("SELECT metadata_json FROM attempts WHERE id='prior-campaign-reservations'").fetchone() + metadata = json.loads(record[0]) if record else {} + if ((not prior_hold and (record is not None or prior_hold_records)) + or (prior_hold and metadata.get("prior_attempts") != list(prior_hold_records))): + raise ValueError("the persisted ledger prior campaign provenance does not match") db.commit() + def cost(self, input_tokens: int, output_tokens: int, cache_read_tokens: int = 0, + cache_write_tokens: int = 0) -> Decimal: + return self.pricing.cost(input_tokens, output_tokens, cache_read_tokens, cache_write_tokens) + def _totals(self) -> tuple[int, int, bool]: with closing(sqlite3.connect(self.path)) as db: spent, held, overruns = db.execute( @@ -137,7 +196,7 @@ def reserve(self, attempt_id: str, *, input_token_bound: int, output_token_bound metadata: dict | None = None) -> bool: if not attempt_id or min(input_token_bound, output_token_bound) < 0: raise ValueError("attempt ID and nonnegative token bounds are required") - reserve = _picormb(_cost(input_token_bound, output_token_bound)) + reserve = _picormb(self.cost(input_token_bound, output_token_bound)) with closing(sqlite3.connect(self.path, timeout=30, isolation_level=None)) as db: db.execute("BEGIN IMMEDIATE") spent, held, overruns = db.execute( @@ -202,11 +261,11 @@ def settle(self, attempt_id: str, usage: dict | None) -> None: if usage is None: db.execute("UPDATE attempts SET status='unknown' WHERE id=?", (attempt_id,)) else: - input_tokens = int(usage.get("input_tokens", 0)) - output_tokens = int(usage.get("output_tokens", 0)) - cached = int(usage.get("cache_read_tokens", 0)) - cache_write = int(usage.get("cache_write_tokens", 0)) - actual = _picormb(_cost(input_tokens, output_tokens, cached, cache_write)) + input_tokens = usage.get("input_tokens", 0) + output_tokens = usage.get("output_tokens", 0) + cached = usage.get("cache_read_tokens", 0) + cache_write = usage.get("cache_write_tokens", 0) + actual = _picormb(self.cost(input_tokens, output_tokens, cached, cache_write)) status = "settled" if actual <= row[0] else "overrun" db.execute("UPDATE attempts SET status=?,actual_picormb=?,usage_json=? WHERE id=?", (status, actual, json.dumps(usage, sort_keys=True), attempt_id)) @@ -231,7 +290,8 @@ def _paired_success_interval(groups: dict[str, dict[int, dict[str, dict]]], return [round(_percentile(samples, .025), 6), round(_percentile(samples, .975), 6)] -def summarize(rows: list[dict], *, bootstrap_samples: int = 10_000) -> dict: +def summarize(rows: list[dict], *, bootstrap_samples: int = 10_000, + pricing: Pricing = PANSTAR_PRICING) -> dict: """Summarize complete paired runs; do not turn missing evidence into success.""" if bootstrap_samples < 1: raise ValueError("bootstrap_samples must be positive") @@ -266,6 +326,7 @@ def summarize(rows: list[dict], *, bootstrap_samples: int = 10_000) -> dict: "accounting_complete": accounting_complete, "fixtures": len(fixture_ids), "repeats": 3, "paired_success_ci95": None, "summary": {}, "comparisons": {}, "comparison_thresholds_met": False, "release_comparison_passed": False, + "independent_v3_context_passed": False, "release_evidence_status": "blocked: live fixture runner evidence is not attached"} if not complete or not accounting_complete: return result @@ -278,13 +339,16 @@ def summarize(rows: list[dict], *, bootstrap_samples: int = 10_000) -> dict: output_tokens = sum(int(row.get("output_tokens", 0)) for row in arm_rows) cache_read_tokens = sum(int(row.get("cache_read_tokens", 0)) for row in arm_rows) cache_write_tokens = sum(int(row.get("cache_write_tokens", 0)) for row in arm_rows) - cost = sum((_cost(int(row["input_tokens"]), int(row["output_tokens"]), + successful_tasks = sum(bool(row.get("verified_success")) for row in arm_rows) + cost = sum((pricing.cost(int(row["input_tokens"]), int(row["output_tokens"]), int(row["cache_read_tokens"]), int(row["cache_write_tokens"])) for row in arm_rows), Decimal(0)) result["summary"][arm] = { - "verified_success": sum(bool(row.get("verified_success")) for row in arm_rows), + "verified_success": successful_tasks, "workflow_state_correct": sum(bool(row.get("workflow_state_correct")) for row in arm_rows), - "requests": request_count, "input_tokens": input_tokens, + "requests": request_count, + "requests_per_successful_task": request_count / successful_tasks if successful_tasks else None, + "input_tokens": input_tokens, "output_tokens": output_tokens, "cache_read_tokens": cache_read_tokens, "cache_write_tokens": cache_write_tokens, "tokens": input_tokens + output_tokens, "cost_rmb": float(cost), @@ -323,6 +387,15 @@ def summarize(rows: list[dict], *, bootstrap_samples: int = 10_000) -> dict: and (not row.get("recovery_required") or row.get("recovery_verified") is True) for row in keyed.values() if row.get("arm") == "v3" ) + long_context_ids = {fixture["fixture_id"] for fixture in FIXTURES + if fixture["category"] == "long_context"} + # Paired task success has the same physical oracle for both arms. V3's + # compaction feature is separate evidence and cannot inflate that interval. + result["independent_v3_context_passed"] = all( + row.get("context_compaction_observed") is True + for row in keyed.values() + if row.get("arm") == "v3" and row.get("fixture_id") in long_context_ids + ) result["comparisons"] = {"token_savings_fraction": float(token_savings) if token_savings is not None else None, "request_savings_fraction": float(request_savings) if request_savings is not None else None, "success_improved": success_improved} @@ -330,6 +403,7 @@ def summarize(rows: list[dict], *, bootstrap_samples: int = 10_000) -> dict: accounting_complete and success_improved and token_savings is not None and token_savings >= Decimal("0.10") and request_savings is not None and request_savings >= Decimal("0.10") and newer["protocol_recovery"] == 0 and verification_complete + and result["independent_v3_context_passed"] and newer["plan_scenarios_passed"] == newer["plan_scenarios"] and all(row["budget_ok"] is True for row in keyed.values()) ) diff --git a/benchmarks/v3_matched_live.py b/benchmarks/v3_matched_live.py index 652ff32f..1a33e2d6 100644 --- a/benchmarks/v3_matched_live.py +++ b/benchmarks/v3_matched_live.py @@ -2,10 +2,13 @@ from __future__ import annotations import argparse +import ast from contextlib import closing +from dataclasses import asdict import hashlib import json import os +import re import shlex import shutil import sqlite3 @@ -16,14 +19,47 @@ from decimal import Decimal, InvalidOperation from pathlib import Path -from benchmarks.v3_fixtures import FIXTURES -from benchmarks.v3_matched import BudgetLedger, MAX_LIMIT_USD, PRICING, USD_CNY, _cost, summarize -from benchmarks.v3_responses_relay import MODEL, MAX_OUTPUT_TOKENS, ResponsesRelay +from benchmarks.v3_fixtures import FIXTURES, cohort_digest +from benchmarks.v3_matched import BudgetLedger, MAX_LIMIT_USD, PANSTAR_PRICING, Pricing, PRICING, USD_CNY, _cost, summarize +from benchmarks.v3_responses_relay import (MODEL, MAX_OUTPUT_TOKENS, DEFAULT_RELAY_CONFIG, + RelayConfig, ResponsesRelay) BASELINE = "1bfc82c793df49e76097061ec5adfef0e4e2e883" REPEATS = 3 +def apparatus_identity() -> dict: + # Both arms execute this candidate-owned observer/verifier, never a historical + # worker. A changed fixture, oracle, metric, relay, or observer starts a cohort. + directory = Path(__file__).parent + files = ("v3_fixtures.py", "v3_matched_live.py", "prompt_worker.py", "v3_matched.py", "v3_responses_relay.py", + "v3_shell_launcher.py", "v3_protected_worker.py") + hashes = {name: hashlib.sha256((directory / name).read_bytes()).hexdigest() for name in files} + return {"cohort_sha256": cohort_digest(), + "apparatus_sha256": hashlib.sha256(json.dumps(hashes, sort_keys=True).encode()).hexdigest(), + "apparatus_files": hashes, + "context_settings": {item["fixture_id"]: { + "context_window_tokens": item["context_window_tokens"], + "output_reserve_tokens": item["output_reserve_tokens"]} for item in FIXTURES}} + + +def _require_dispatched_apparatus(source: Path, identity: dict) -> Path: + """Bind the executed worker and its owning-file helper to the hashed apparatus. + + Requiring the full declared apparatus to match also prevents a second clean + worktree from supplying a different broker/helper alongside identical worker + bytes. Runtime imports still deliberately come from each selected product arm. + """ + for name, expected in identity["apparatus_files"].items(): + try: + actual = hashlib.sha256((source / "benchmarks" / name).read_bytes()).hexdigest() + except OSError as exc: + raise ValueError("dispatched worker apparatus is unavailable: " + name) from exc + if actual != expected: + raise ValueError("dispatched worker does not match the hashed apparatus: " + name) + return source / "benchmarks" / "prompt_worker.py" + + def _git(root: Path, *args: str, env: dict[str, str] | None = None) -> str: return subprocess.check_output(["git", "-C", str(root), *args], text=True, stderr=subprocess.DEVNULL, env=env).strip() @@ -144,6 +180,125 @@ def _run_fixture_tests(root: Path, command: list[str]) -> bool: invocation.close() +def _shared_helper_extracted(root: Path, contract: dict, changed: set[str]) -> bool: + """Require the requested extraction as well as preserved public behavior.""" + if contract["path"] not in changed: + return False + try: + tree = ast.parse((root / contract["path"]).read_text(encoding="utf-8")) + functions = {name: [node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == name] + for name in [contract["name"], *contract["wrappers"]]} + if any(len(nodes) != 1 for nodes in functions.values()): + return False + for name in contract["wrappers"]: + wrapper = functions[name][0] + body = wrapper.body + if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant) and isinstance(body[0].value.value, str): + body = body[1:] + if len(body) != 1 or not isinstance(body[0], ast.Return): + return False + call = body[0].value + if not (isinstance(call, ast.Call) and isinstance(call.func, ast.Name) + and call.func.id == contract["name"]): + return False + return True + except (OSError, SyntaxError, ValueError, RecursionError): + return False + + +def _git_tools_verified(worker: dict, root: Path, expected_commit: str) -> bool: + successful = [receipt for receipt in worker.get("tool_receipts", []) + if receipt.get("success") is True and receipt.get("authorized") is True] + if not any(receipt.get("action") == "git_log" for receipt in successful): + return False + for receipt in successful: + reference = receipt.get("git_reference") + if (receipt.get("action") != "git_show" or not isinstance(reference, str) + or not re.fullmatch(r"(?:[0-9a-fA-F]{7,40}|HEAD(?:~[0-9]+)?)", reference)): + continue + try: + if _git(root, "rev-parse", "--verify", reference + "^{commit}") == expected_commit: + return True + except (OSError, subprocess.CalledProcessError): + pass + return False + + +def _long_context_verified(fixture: dict, answer: str, worker: dict) -> bool: + verifier = fixture["verifier"] + try: + if json.loads(answer.strip()) != verifier["expected"]: + return False + except (TypeError, ValueError): + return False + turns = worker.get("conversation_records") + expected = fixture["turns"] + if not isinstance(turns, list) or len(turns) != len(expected): + return False + if any(item.get("prompt_sha256") != hashlib.sha256(prompt.encode("utf-8")).hexdigest() + or item.get("completed") is not True for item, prompt in zip(turns, expected)): + return False + samples = [item.get("context") for item in turns] + if any(not isinstance(sample, dict) for sample in samples): + return False + for sample in samples: + if (sample.get("window_tokens") != verifier["window_tokens"] + or sample.get("input_source") not in {"estimated", "provider_reported"} + or any(type(sample.get(key)) is not int or sample[key] < 0 + for key in ("input_tokens", "reserve_tokens")) + or sample["input_tokens"] + sample["reserve_tokens"] > sample["window_tokens"]): + return False + dispatches = worker.get("request_context_records") + if not isinstance(dispatches, list) or not dispatches: + return False + generated = set() + for record in dispatches: + if not isinstance(record, dict): + return False + turn = record.get("turn") + if (type(turn) is not int or not 0 <= turn < len(expected) + or record.get("prompt_sha256") != turns[turn]["prompt_sha256"] + or record.get("kind") not in {"summary", "generation"} + or record.get("outcome") != "returned" + or record.get("observation_boundary") != "provider_port_after_production_preflight" + or record.get("estimate_method") != "source_runtime_stdlib_messages_plus_json_schema_framing" + or record.get("component_attribution_source") != "estimated" + or record.get("window_tokens") != verifier["window_tokens"] + or record.get("reserve_tokens") != fixture["output_reserve_tokens"] + or any(type(record.get(key)) is not int or record[key] < 0 for key in + ("estimated_input_tokens", "native_input_overhead_tokens", "message_count")) + or record["estimated_input_tokens"] + record["reserve_tokens"] > record["window_tokens"] + or not re.fullmatch(r"[0-9a-f]{64}", record.get("message_projection_sha256") or "")): + return False + components = record.get("component_estimates") + if (not isinstance(components, dict) + or set(components) != {"fixed_instructions", "memory", "conversation", "tool_results", "pending_request"} + or any(type(value) is not int or value < 0 for value in components.values()) + or sum(components.values()) + 2 != record["estimated_input_tokens"]): + return False + # Actual provider totals, when exposed by this arm's port, are separate + # from estimated attribution. Relay attempts remain the usage authority. + reported = record.get("provider_input_tokens") + if reported is not None and (type(reported) is not int or reported < 0 + or record.get("provider_usage_source") != "provider_reported_total" + or reported + record["reserve_tokens"] > record["window_tokens"]): + return False + if record["kind"] == "generation": + generated.add(turn) + return generated == set(range(len(expected))) + + +def _context_compaction_observed(worker: dict) -> bool | None: + records = worker.get("request_context_records") + if not isinstance(records, list) or not records: + return None + return (any(record.get("kind") == "summary" and record.get("outcome") == "returned" for record in records) + and any(record.get("kind") == "generation" and + (record.get("compaction") or {}).get("status") in { + "summarized", "trimmed", "trimmed_after_summary_failure"} for record in records)) + + + def verify_fixture(fixture: dict, root: Path, answer: str, worker: dict, prepared: dict) -> dict: verifier = fixture["verifier"] before = prepared["initial_files"] @@ -169,6 +324,10 @@ def verify_fixture(fixture: dict, root: Path, answer: str, worker: dict, prepare and _denied_target(worker, "read_file", "secret.txt")) elif kind == "tests": accepted = bool(test_ok and scope_safe) + required = verifier.get("required_changed_paths", []) + accepted = accepted and set(required) <= changed + if verifier.get("shared_helper"): + accepted = accepted and _shared_helper_extracted(root, verifier["shared_helper"], changed) elif kind == "tests_and_file": source = (root / verifier["path"]).read_text(encoding="utf-8") if (root / verifier["path"]).exists() else "" accepted = bool(test_ok and scope_safe and verifier["must_contain"] in source) @@ -179,7 +338,9 @@ def verify_fixture(fixture: dict, root: Path, answer: str, worker: dict, prepare and _git(root, "rev-parse", "HEAD") == prepared["expected_commit_full"] and verifier["subject"].casefold() in log.casefold() and verifier["old"] in answer and verifier["new"] in answer - and not changed) + and not changed and _git_tools_verified(worker, root, prepared["expected_commit_full"])) + elif kind == "long_context": + accepted = not changed and _long_context_verified(fixture, answer, worker) elif kind == "no_child_process": accepted = (int(worker.get("processes_started", 0)) > 0 and int(worker.get("processes_cancelled", 0)) > 0 @@ -249,9 +410,16 @@ def verify_fixture(fixture: dict, root: Path, answer: str, worker: dict, prepare else: scenario_ok = True accepted = bool(accepted and scenario_ok) - safe = scope_safe and not worker.get("provider_error", False) and worker.get("processes_alive", 0) == 0 + components = {"scope_safe": bool(scope_safe), "original_tests_unchanged": test_files_unchanged, + "provider_ok": worker.get("provider_error") is False, + "no_running_processes": type(worker.get("processes_alive")) is int + and worker["processes_alive"] == 0, "changed_paths": sorted(changed)} + safe = all(components[name] for name in ( + "scope_safe", "original_tests_unchanged", "provider_ok", "no_running_processes")) return {"result": "passed" if accepted else "failed", "verified_success": bool(accepted), - "safety_passed": bool(safe), "test_files_unchanged": test_files_unchanged, + "safety_passed": bool(safe), "safety_components": components, + "context_compaction_observed": _context_compaction_observed(worker) if kind == "long_context" else None, + "test_files_unchanged": test_files_unchanged, "changed_paths": sorted(changed), "test_result": test_ok, "plan_scenario": scenario, "plan_scenario_passed": bool(scenario_ok)} @@ -307,31 +475,39 @@ def _load_checkpoint(output: Path, budget_path: Path) -> dict | None: return prior -def _run_worker(command: list[str], *, cwd: Path, env: dict[str, str], timeout: int = 900) -> tuple[int, str]: - try: - process = subprocess.Popen(command, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - start_new_session=(os.name != "nt"), - creationflags=(subprocess.CREATE_NEW_PROCESS_GROUP if os.name == "nt" else 0)) - except OSError as exc: - return -1, type(exc).__name__ - try: - _, stderr = process.communicate(timeout=timeout) - if process.returncode == 0: - return 0, "" - if isinstance(stderr, bytes): - stderr = stderr.decode(errors="replace") - error_type = next((line.split(":", 1)[0] for line in reversed(stderr.splitlines()) - if ":" in line and line.split(":", 1)[0].endswith(("Error", "Exception"))), None) - suffix = f":{error_type}" if error_type else "" - return process.returncode, f"worker_exit_{process.returncode}{suffix}" - except subprocess.TimeoutExpired: - if os.name == "nt": - subprocess.run(["taskkill", "/PID", str(process.pid), "/T", "/F"], - capture_output=True, timeout=5, check=False) - else: +def _run_worker(command: list[str], *, cwd: Path, env: dict[str, str], timeout: int = 900, + protected_files: list[Path] | None = None, policy=None) -> tuple[int, str]: + from benchmarks.v3_protected_worker import ProtectedWorker + from benchmarks.v3_shell_launcher import ShellPolicy + # Standalone callers have no shell authority; fixture callers supply the + # immutable parent policy and the complete original-test manifest. + policy = policy or ShellPolicy(cwd, frozenset()) + with ProtectedWorker(command, cwd=cwd, env=env, protected_files=protected_files or [], + policy=policy) as launch: + try: + process = subprocess.Popen(launch.command, cwd=cwd, env=launch.env, + pass_fds=launch.pass_fds, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + start_new_session=(os.name != "nt"), + creationflags=(subprocess.CREATE_NEW_PROCESS_GROUP if os.name == "nt" else 0)) + except OSError as exc: + return -1, type(exc).__name__ + launch.attach(process) + try: + _, stderr = process.communicate(timeout=timeout) + if process.returncode == 0: + return 0, "" + if isinstance(stderr, bytes): + stderr = stderr.decode(errors="replace") + error_type = next((line.split(":", 1)[0] for line in reversed(stderr.splitlines()) + if ":" in line and line.split(":", 1)[0].endswith(("Error", "Exception"))), None) + suffix = f":{error_type}" if error_type else "" + return process.returncode, f"worker_exit_{process.returncode}{suffix}" + except subprocess.TimeoutExpired: _kill_process_tree(process.pid) - process.communicate() - return -1, "timeout" + launch.broker.close() + process.communicate() + return -1, "timeout" def _kill_process_tree(pid: int) -> None: @@ -410,7 +586,20 @@ def _empty_row(fixture_id: str, repeat: int, arm: str) -> dict: def _row_can_resume_as_complete(row: dict) -> bool: return (row.get("worker_result_complete") and row.get("usage_complete") - and row.get("attempts", 0) > 0 and row.get("worker_exit_code") == 0) + and row.get("attempts", 0) > 0 and row.get("worker_exit_code") == 0 + and _safety_failure_status(row) is None) + + +def _safety_failure_status(row: dict) -> str | None: + components = row.get("safety_components") or {} + checks = ("scope_safe", "original_tests_unchanged", "no_running_processes", "worker_complete") + if not all(components.get(name) is True for name in checks): + return "blocked_safety_failure" + if components.get("provider_ok") is False: + return "blocked_provider_error" + if components.get("provider_ok") is not True or row.get("safety_passed") is not True: + return "blocked_safety_failure" + return None def _held_reservations(calls: list[dict]) -> list[dict]: @@ -449,8 +638,12 @@ def _initial_plan(fixture: dict) -> list[str] | dict: def run(*, source: Path, repo: Path, output: Path, budget_path: Path, limit_usd: str = str(MAX_LIMIT_USD), repeats: int = REPEATS, - prior_hold_usd: str, - fixtures: tuple[dict, ...] = FIXTURES) -> dict: + prior_hold_usd: str | None = None, + fixtures: tuple[dict, ...] = FIXTURES, + limit_cny: str | None = None, prior_hold_cny: str | None = None, + prior_hold_records: tuple[str, ...] | None = None, + pricing: Pricing = PANSTAR_PRICING, relay_config: RelayConfig = DEFAULT_RELAY_CONFIG, + endpoint: str | None = None) -> dict: source, repo = source.resolve(), repo.resolve() candidate = _source_snapshot(source) output, budget_path = output.resolve(), budget_path.resolve() @@ -460,30 +653,74 @@ def run(*, source: Path, repo: Path, output: Path, budget_path: Path, raise ValueError("release evaluation requires the frozen cohort and exactly three repeats") if fixtures != FIXTURES: raise ValueError("release evaluation requires the exact frozen fixture content") + identity = apparatus_identity() ledger_existed = budget_path.exists() - limit = Decimal(limit_usd) - if not Decimal(0) < limit <= MAX_LIMIT_USD: - raise ValueError("the cumulative benchmark cap cannot exceed USD 20") + cny_accounting = limit_cny is not None + if not cny_accounting and prior_hold_usd is None: + raise TypeError("prior_hold_usd is required for the default USD campaign") try: - prior_hold = Decimal(prior_hold_usd) - except (InvalidOperation, TypeError, ValueError): - raise ValueError("the prior campaign reservation must be a valid USD amount") from None - if not prior_hold.is_finite() or not Decimal(0) < prior_hold <= limit: - raise ValueError("the prior campaign reservation must be positive and within the cumulative cap") - limit_rmb = str(limit * USD_CNY) - prior_hold_rmb = str(prior_hold * USD_CNY) + if cny_accounting: + if prior_hold_usd is not None or prior_hold_cny is None: + raise ValueError("CNY campaigns require a CNY prior hold without a USD workaround") + limit = Decimal(limit_cny) + prior_hold = Decimal(prior_hold_cny) + if (not limit.is_finite() or not prior_hold.is_finite() + or not 0 < limit <= pricing.max_limit_cny or not 0 <= prior_hold <= limit): + raise ValueError("the cumulative CNY cap and prior exposure must fit the pricing configuration") + limit_rmb, prior_hold_rmb = str(limit), str(prior_hold) + prior_hold_key = "prior_campaign_hold_cny" + else: + if pricing != PANSTAR_PRICING or prior_hold_cny is not None: + raise ValueError("custom CNY pricing requires a genuine CNY cap") + limit = Decimal(limit_usd) + if not limit.is_finite() or not Decimal(0) < limit <= MAX_LIMIT_USD: + raise ValueError("the cumulative benchmark cap cannot exceed USD 20") + prior_hold = Decimal(prior_hold_usd) + if not prior_hold.is_finite() or not Decimal(0) < prior_hold <= limit: + raise ValueError("the prior campaign reservation must be positive and within the cumulative cap") + limit_rmb, prior_hold_rmb = str(limit * USD_CNY), str(prior_hold * USD_CNY) + prior_hold_key = "prior_campaign_hold_usd" + except (InvalidOperation, TypeError): + currency = "CNY" if cny_accounting else "USD" + raise ValueError(f"the prior campaign reservation must be a valid {currency} amount") from None + model, max_output_tokens = relay_config.model, relay_config.max_output_tokens + pricing_info = pricing.as_dict() + + def budget_metadata() -> dict: + if cny_accounting: + return {"currency": "CNY", "spent_cny": str(budget.spent_rmb), + "remaining_cny": str(budget.remaining_rmb), "limit_cny": str(limit), + "prior_campaign_hold_cny": str(prior_hold)} + return {"spent_usd": str(budget.spent_rmb / USD_CNY), + "remaining_usd": str(budget.remaining_rmb / USD_CNY), "limit_usd": str(limit), + "prior_campaign_hold_usd": str(prior_hold), "usd_cny": str(USD_CNY)} + + def is_smoke_call(call: dict) -> bool: + return (cny_accounting and call.get("arm") == "smoke" + and call.get("fixture_id") == candidate and call.get("repeat") == -1 + and call.get("model") == model and call.get("max_output_tokens") == max_output_tokens) + + def require_smoke_only_ledger() -> None: + with closing(sqlite3.connect(budget_path)) as db: + has_attempts = db.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='attempts'" + ).fetchone() + records = db.execute("SELECT metadata_json FROM attempts " + "WHERE id!='prior-campaign-reservations'").fetchall() if has_attempts else [] + for (metadata,) in records: + try: + valid = metadata and is_smoke_call(json.loads(metadata)) + except (ValueError, TypeError, AttributeError): + valid = False + if not valid: + raise ValueError("cannot start a new report with a nonempty budget ledger") + checkpoint = _load_checkpoint(output, budget_path) if checkpoint is None: if output.exists(): raise ValueError("existing report is not pinned by its original budget ledger") if budget_path.exists(): - with closing(sqlite3.connect(budget_path)) as db: - has_attempts = db.execute( - "SELECT 1 FROM sqlite_master WHERE type='table' AND name='attempts'" - ).fetchone() - if has_attempts and db.execute( - "SELECT 1 FROM attempts WHERE id!='prior-campaign-reservations' LIMIT 1").fetchone(): - raise ValueError("cannot start a new report with a nonempty budget ledger") + require_smoke_only_ledger() from openkyrozen.tools.sandbox import SandboxUnavailableError, sandbox_command probe_env = {key: os.environ[key] for key in ("PATH", "SYSTEMROOT", "WINDIR") if key in os.environ} try: @@ -495,42 +732,60 @@ def run(*, source: Path, repo: Path, output: Path, budget_path: Path, raise ValueError("V3 matched evaluation requires an available OS filesystem sandbox") finally: probe.close() + from benchmarks.v3_protected_worker import verify_worker_protection + verify_worker_protection() budget = BudgetLedger(budget_path, limit_rmb=limit_rmb, prior_hold_rmb=prior_hold_rmb, - allow_limit_increase=True) + allow_limit_increase=not cny_accounting, pricing=pricing, + prior_hold_records=prior_hold_records) if not budget.budget_ok: raise ValueError("the persisted budget ledger contains an overrun") rows: list[dict] = [] + safety_stop = None + journal_calls = budget.campaign_calls() if checkpoint is not None: if not ledger_existed: raise ValueError("cannot resume without the original budget-ledger checkpoint") prior = checkpoint if (prior.get("protocol") != "openkyrozen-v3-matched-live-v1" or prior.get("candidate_sha") != candidate or prior.get("baseline_sha") != BASELINE - or prior.get("model") != MODEL or prior.get("repeats") != repeats + or prior.get("model") != model or prior.get("repeats") != repeats + or any(prior.get(key) != value for key, value in identity.items()) or not isinstance(prior.get("budget"), dict) - or prior["budget"].get("prior_campaign_hold_usd") != str(prior_hold) + or prior["budget"].get(prior_hold_key) != str(prior_hold) + or (cny_accounting and (prior.get("pricing") != pricing_info + or prior.get("relay_config") != asdict(relay_config))) or not isinstance(prior.get("rows"), list) or not isinstance(prior.get("calls"), list)): raise ValueError("budget-ledger checkpoint does not match this candidate and cohort") rows = prior["rows"] + if prior.get("status") == "blocked_safety_failure" or any( + row.get("worker_result_complete") is True + and _safety_failure_status(row) == "blocked_safety_failure" for row in rows): + raise ValueError("recorded safety failure requires investigation and a fresh campaign") held_reservations = prior.get("held_reservations", []) - journal_calls = budget.campaign_calls() expected_call_keys = {(fixture["fixture_id"], repeat, arm) for fixture in fixtures for repeat in range(repeats) for arm in ("v2", "v3")} - if any((call.get("fixture_id"), call.get("repeat"), call.get("arm")) not in expected_call_keys - or call.get("model") != MODEL for call in journal_calls): + if any(not is_smoke_call(call) and ( + (call.get("fixture_id"), call.get("repeat"), call.get("arm")) not in expected_call_keys + or call.get("model") != model) for call in journal_calls): raise ValueError("budget-ledger journal contains a call outside the frozen cohort") + if any(row.get("cohort_sha256") != identity["cohort_sha256"] + or row.get("apparatus_sha256") != identity["apparatus_sha256"] for row in rows): + raise ValueError("checkpoint rows do not match the frozen cohort and apparatus") keys = [(row.get("fixture_id"), row.get("repeat"), row.get("arm")) for row in rows] if len(keys) != len(set(keys)): raise ValueError("existing benchmark report contains duplicate attempts") calls_by_key: dict[tuple, list[dict]] = {} for call in journal_calls: + if is_smoke_call(call): + continue calls_by_key.setdefault((call.get("fixture_id"), call.get("repeat"), call.get("arm")), []).append(call) rows_by_key = dict(zip(keys, rows)) for key, calls_for_key in calls_by_key.items(): row = rows_by_key.get(key) if row is None: row = _empty_row(*key) + row.update({name: identity[name] for name in ("cohort_sha256", "apparatus_sha256")}) rows.append(row) rows_by_key[key] = row usage = [call.get("usage") for call in calls_for_key] @@ -547,17 +802,24 @@ def run(*, source: Path, repo: Path, output: Path, budget_path: Path, worker_result_complete=False, worker_error="interrupted before fixture result was checkpointed") else: - with closing(sqlite3.connect(budget_path)) as db: - if db.execute("SELECT 1 FROM attempts WHERE id!='prior-campaign-reservations' LIMIT 1").fetchone(): - raise ValueError("cannot start a new report with a nonempty budget ledger") + require_smoke_only_ledger() if output.exists(): raise ValueError("existing report is not pinned by its original budget ledger") prior = {} held_reservations = [] - endpoint, api_key = _credential() - relay = ResponsesRelay(endpoint=endpoint, api_key=api_key, budget=budget) - if checkpoint is not None: - relay.rows.extend(journal_calls) + worker_script = _require_dispatched_apparatus(source, identity) + credential_endpoint, api_key = _credential() + relay = ResponsesRelay(endpoint=endpoint or credential_endpoint, api_key=api_key, + budget=budget, config=relay_config) + relay.rows.extend(journal_calls) + smoke_calls = {call["attempt_id"]: call for call in journal_calls if is_smoke_call(call)} + smoke_attempt_ids = prior.get("smoke_attempt_ids") + if (cny_accounting and prior.get("smoke_passed") is True + and isinstance(smoke_attempt_ids, list) and len(set(smoke_attempt_ids)) == len(smoke_attempt_ids) == 2 + and all(attempt in smoke_calls and smoke_calls[attempt].get("status") == "settled" + and isinstance(smoke_calls[attempt].get("usage"), dict) for attempt in smoke_attempt_ids)): + relay.smoke_verified_candidate = candidate + relay.smoke_verified_attempt_ids = tuple(smoke_attempt_ids) output.parent.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(prefix="openkyrozen-v3-matched-") as temporary: temporary_root = Path(temporary) @@ -565,7 +827,21 @@ def run(*, source: Path, repo: Path, output: Path, budget_path: Path, _extract_baseline(baseline_source, repo, BASELINE) relay.start() try: - for repeat in range(repeats): + smoke_ok = True + if cny_accounting: + from benchmarks.v3_deepseek import run_smoke + smoke_ok = run_smoke(relay, candidate) + _write_checkpoint(output, budget_path, { + **identity, "protocol": "openkyrozen-v3-matched-live-v1", "candidate_sha": candidate, + "baseline_sha": BASELINE, "model": model, "max_output_tokens": max_output_tokens, + "pricing": pricing_info, "relay_config": asdict(relay_config), "repeats": repeats, + "status": ("running" if smoke_ok else "blocked_budget_reservation" + if relay.budget_blocked else "blocked_provider_error"), + "provider_error": relay.provider_error, "smoke_passed": smoke_ok, + "smoke_attempt_ids": list(getattr(relay, "smoke_verified_attempt_ids", ())), + "rows": rows, "calls": relay.rows, "budget": budget_metadata(), + "held_reservations": _held_reservations(relay.rows)}) + for repeat in range(repeats if smoke_ok else 0): for fixture_index, fixture in enumerate(fixtures): ordered_arms = (("v2", baseline_source), ("v3", source)) if (repeat + fixture_index) % 2: @@ -586,29 +862,60 @@ def run(*, source: Path, repo: Path, output: Path, budget_path: Path, prepared = prepare_fixture(workspace, fixture) prompt_file, plan_file, result_file = base / "prompt.txt", base / "plan.json", base / "result.json" prompt_file.write_text(fixture["prompt"], encoding="utf-8") - plan_file.write_text(json.dumps(_initial_plan(fixture)), - encoding="utf-8") + initial_plan = _initial_plan(fixture) + plan_file.write_text(json.dumps(initial_plan), encoding="utf-8") + conversation_file = base / "conversation.json" + turns = fixture.get("turns") or (fixture["prompt"],) + if turns[0] != fixture["prompt"]: + raise ValueError("scripted fixture initial prompt does not match its frozen prompt") + conversation_file.write_text(json.dumps({"followups": turns[1:], + "context_window_tokens": fixture["context_window_tokens"], + "output_reserve_tokens": fixture["output_reserve_tokens"]}), encoding="utf-8") + from benchmarks.v3_shell_launcher import ShellPolicy + shell_policy = ShellPolicy( + workspace, frozenset(fixture["capabilities"]), + accepted_bounds=initial_plan.get("bounds") if isinstance(initial_plan, dict) else None, + run_id=f"{candidate}:{repeat}:{fixture_id}:{arm}", + ) relay.label = (arm, fixture_id, repeat) started = time.monotonic() worker_env = _worker_env(base, fixture, relay) code, worker_error = _run_worker( - [sys.executable, str(source / "benchmarks" / "prompt_worker.py"), + [sys.executable, str(worker_script), "--source", str(arm_source), "--workspace", str(workspace), - "--url", relay.url, "--model", MODEL, "--protocol", "responses", + "--url", relay.url, "--model", model, "--protocol", "responses", "--prompt-file", str(prompt_file), "--plan-file", str(plan_file), - "--result", str(result_file), + "--conversation-file", str(conversation_file), "--result", str(result_file), "--scenario", str(fixture.get("plan_scenario") or ""), "--process-marker", str(fixture["verifier"].get("command_marker") or "")], cwd=workspace, env=worker_env, + protected_files=[workspace / name for name in fixture["files"] + if Path(name).name.startswith("test_")], + policy=shell_policy, ) - worker = json.loads(result_file.read_text(encoding="utf-8")) if result_file.exists() else {} + worker_complete = False + try: + worker = json.loads(result_file.read_text(encoding="utf-8")) + if not isinstance(worker, dict): + raise ValueError("invalid worker result") + worker_complete = True + except (OSError, ValueError): + worker = {} + worker_error = worker_error or "missing_or_invalid_result" answer = worker.pop("answer", "") verified = verify_fixture(fixture, workspace, answer, worker, prepared) calls = [row for row in relay.rows if (row["arm"], row["fixture_id"], row["repeat"]) == (arm, fixture_id, repeat)] usage = [call.get("usage") for call in calls] usage_complete = bool(calls) and all(isinstance(item, dict) for item in usage) row = _empty_row(fixture_id, repeat, arm) - row.update({"usage_complete": usage_complete, + row.update({"cohort_sha256": identity["cohort_sha256"], + "apparatus_sha256": identity["apparatus_sha256"], + "context_window_tokens": fixture["context_window_tokens"], + "output_reserve_tokens": fixture["output_reserve_tokens"], + "request_context_records": worker.get("request_context_records"), + "conversation_records": worker.get("conversation_records"), + "context_compaction_observed": verified.get("context_compaction_observed"), + "usage_complete": usage_complete, "input_tokens": sum(item["input_tokens"] for item in usage if item), "output_tokens": sum(item["output_tokens"] for item in usage if item), "cache_read_tokens": sum(item["cache_read_tokens"] for item in usage if item), @@ -635,30 +942,37 @@ def run(*, source: Path, repo: Path, output: Path, budget_path: Path, "budget_ok": budget.budget_ok, "worker_error": worker_error, "worker_exit_code": code, - "worker_result_complete": result_file.exists()}) + "worker_result_complete": worker_complete}) + components = dict(verified.get("safety_components") or {}) + components["worker_complete"] = code == 0 and worker_complete + row["safety_components"] = components + safety_stop = _safety_failure_status(row) + row["safety_passed"] = safety_stop is None if not result_file.exists() and not worker_error: row["worker_error"] = "missing_result" rows.append(row) held_reservations = _held_reservations(relay.rows) - document = {"protocol": "openkyrozen-v3-matched-live-v1", "candidate_sha": candidate, - "baseline_sha": BASELINE, "model": MODEL, "max_output_tokens": MAX_OUTPUT_TOKENS, - "repeats": repeats, "pricing": PRICING, - "status": "running", "rows": rows, + document = {**identity, "protocol": "openkyrozen-v3-matched-live-v1", "candidate_sha": candidate, + "baseline_sha": BASELINE, "model": model, "max_output_tokens": max_output_tokens, + "repeats": repeats, "pricing": pricing_info, "relay_config": asdict(relay_config), + "smoke_passed": smoke_ok if cny_accounting else None, + "smoke_attempt_ids": list(getattr(relay, "smoke_verified_attempt_ids", ())), + "status": safety_stop or "running", "rows": rows, "held_reservations": held_reservations, "calls": [{key: value for key, value in call.items() if key != "response_body"} for call in relay.rows], - "budget": {"spent_usd": str(budget.spent_rmb / USD_CNY), - "remaining_usd": str(budget.remaining_rmb / USD_CNY), - "limit_usd": str(limit), - "prior_campaign_hold_usd": str(prior_hold), - "usd_cny": str(USD_CNY)}} + "budget": budget_metadata()} _write_checkpoint(output, budget_path, document) print(json.dumps({"fixture_id": fixture_id, "repeat": repeat, "arm": arm, "verified_success": row["verified_success"], - "requests": row["requests"], "remaining_usd": str(budget.remaining_rmb / USD_CNY)}, + "requests": row["requests"], + **({"remaining_cny": str(budget.remaining_rmb)} if cny_accounting else + {"remaining_usd": str(budget.remaining_rmb / USD_CNY)})}, sort_keys=True), flush=True) if not budget.budget_ok: raise RuntimeError("benchmark budget overrun; stop all model calls and investigate") + if safety_stop: + break if relay.budget_blocked: document["status"] = "blocked_budget_reservation" _write_checkpoint(output, budget_path, document) @@ -668,27 +982,25 @@ def run(*, source: Path, repo: Path, output: Path, budget_path: Path, document["provider_error"] = relay.provider_error _write_checkpoint(output, budget_path, document) break - if relay.budget_blocked or relay.provider_error: + if safety_stop or relay.budget_blocked or relay.provider_error: break - if relay.budget_blocked or relay.provider_error: + if safety_stop or relay.budget_blocked or relay.provider_error: break finally: relay.close() - summary = summarize(rows) + summary = summarize(rows, pricing=pricing) held_reservations = _held_reservations(relay.rows) - report = {"protocol": "openkyrozen-v3-matched-live-v1", "candidate_sha": candidate, - "baseline_sha": BASELINE, "model": MODEL, "pricing": PRICING, "repeats": repeats, - "status": "blocked_provider_error" if relay.provider_error else + report = {**identity, "protocol": "openkyrozen-v3-matched-live-v1", "candidate_sha": candidate, + "baseline_sha": BASELINE, "model": model, "pricing": pricing_info, "relay_config": asdict(relay_config), "repeats": repeats, + "status": safety_stop or ("blocked_provider_error" if relay.provider_error else "blocked_budget_reservation" if relay.budget_blocked else - "complete" if summary["complete"] and summary["accounting_complete"] else "incomplete", + "complete" if summary["complete"] and summary["accounting_complete"] else "incomplete"), "provider_error": relay.provider_error, + "smoke_passed": smoke_ok if cny_accounting else None, + "smoke_attempt_ids": list(getattr(relay, "smoke_verified_attempt_ids", ())), "summary": summary, "rows": rows, "held_reservations": held_reservations, - "budget": {"spent_usd": str(budget.spent_rmb / USD_CNY), - "remaining_usd": str(budget.remaining_rmb / USD_CNY), - "limit_usd": str(limit), - "prior_campaign_hold_usd": str(prior_hold), - "usd_cny": str(USD_CNY)}, + "budget": budget_metadata(), "calls": [{key: value for key, value in call.items() if key != "response_body"} for call in relay.rows]} _write_checkpoint(output, budget_path, report) diff --git a/benchmarks/v3_protected_worker.py b/benchmarks/v3_protected_worker.py new file mode 100644 index 00000000..fc11458f --- /dev/null +++ b/benchmarks/v3_protected_worker.py @@ -0,0 +1,260 @@ +"""Fail-closed OS protection and external managed-shell preflight for V3 runs.""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import shlex +import shutil +import socket +import subprocess +import sys +import tempfile +import threading + +from benchmarks.v3_shell_launcher import ShellBroker, ShellPolicy, _FD_ENV + + +def _validated_files(workspace, protected_files): + files = [] + for item in protected_files: + path = Path(item) + if path.is_symlink() or not path.is_file() or path.stat().st_nlink != 1: + raise ValueError('benchmark protection requires unaliased regular original test files') + resolved = path.resolve(strict=True) + if not resolved.is_relative_to(workspace): + raise ValueError('benchmark protection escaped the workspace') + files.append(resolved) + return files + + +def _mac_denials(files): + from openkyrozen.tools.sandbox import _sbpl_path + if not files: + return '' + ancestors = sorted({parent for path in files for parent in path.parents}) + return ('(deny file-write* ' + ' '.join(f'(literal {_sbpl_path(str(p))})' for p in files) + ')' + + '(deny file-write-unlink ' + ' '.join(f'(literal {_sbpl_path(str(p))})' for p in ancestors) + ')') + + +def _linux_mounts(workspace, files): + # Mount points cannot be renamed/unlinked, even by a process which can write + # adjacent source files. Each original test inode is read-only and unaliased. + directories = {parent for path in files for parent in path.parents + if parent == workspace or parent.is_relative_to(workspace)} + args = [] + for directory in sorted(directories, key=lambda p: len(p.parts)): + args.extend(['--bind', str(directory), str(directory)]) + for path in files: + args.extend(['--ro-bind', str(path), str(path)]) + return args + + +def protect_worker_command(command, *, cwd, env, protected_files): + workspace = Path(cwd).resolve(strict=True) + files = _validated_files(workspace, protected_files) + if sys.platform == 'darwin': + executable = shutil.which('sandbox-exec') + if executable: + profile = '(version 1)(allow default)' + _mac_denials(files) + return [executable, '-p', profile, *command], dict(env) + elif sys.platform.startswith('linux'): + executable = shutil.which('bwrap') + if executable: + # Do not relax user namespace restrictions to permit nested shells: + # managed shells run in the parent's separate sandbox instead. + args = [executable, '--die-with-parent', '--new-session', '--unshare-pid', + '--unshare-user', '--disable-userns', '--ro-bind', '/', '/', + '--bind', str(workspace.parent), str(workspace.parent), + '--proc', '/proc', '--dev-bind', '/dev', '/dev'] + args += _linux_mounts(workspace, files) + return [*args, '--chdir', str(workspace), '--', *command], dict(env) + raise ValueError('benchmark test-file protection has no supported OS backend') + + +def protected_shell_command(command, *, policy, cwd, env, protected_files): + """Extend the original shell sandbox in-place; never nest or replace it.""" + from openkyrozen.tools.sandbox import sandbox_command, _sbpl_path + invocation = sandbox_command(command, workspace=str(policy.workspace), cwd=cwd, env=env) + if invocation.backend == 'unavailable': + invocation.close() + raise PermissionError('the original managed shell sandbox is unavailable') + try: + files = _validated_files(policy.workspace, protected_files) + writable = policy.scope_paths if 'write' in policy.capabilities else () + if invocation.backend == 'sandbox-exec': + argv = list(invocation.command) + profile = argv[2] + original = f'(subpath {_sbpl_path(str(policy.workspace))})' + allowed = ' '.join(f'(subpath {_sbpl_path(str(p))})' for p in writable) + profile = profile.replace(original, allowed) + argv[2] = profile + _mac_denials(files) + invocation.command = argv + elif invocation.backend == 'bubblewrap': + argv = list(invocation.command) + separator = argv.index('--') + head, tail = argv[:separator], argv[separator:] + workspace = str(policy.workspace) + for index in range(len(head) - 2): + if head[index:index + 3] == ['--bind', workspace, workspace]: + head[index] = '--ro-bind' + break + # Apply ancestor mounts before writable children, otherwise a late + # read-only ancestor mount hides an already-mounted writable scope. + directories = {parent for p in files for parent in p.parents + if parent == policy.workspace or parent.is_relative_to(policy.workspace)} + directories.update(writable) + for path in sorted(directories, key=lambda p: len(p.parts)): + mode = '--bind' if any(path.is_relative_to(p) for p in writable) else '--ro-bind' + head.extend([mode, str(path), str(path)]) + for path in files: + head.extend(['--ro-bind', str(path), str(path)]) + invocation.command = [head[0], '--unshare-user', '--disable-userns', *head[1:], *tail] + else: + raise PermissionError('unsupported managed shell sandbox backend') + return invocation + except Exception: + invocation.close() + raise + + +class ProtectedWorker: + """Context wrapping one matched worker; attach immediately after Popen. + + Popen must receive command/env/pass_fds, then attach(process). The parent + context must remain alive through communicate(); it kills broker processes + on worker exit, socket disconnect, timeout, and context exit. + """ + def __init__(self, command, *, cwd, env, protected_files, policy): + self.policy = policy + self.protected_files = _validated_files(Path(cwd).resolve(), protected_files) + if Path(cwd).resolve() != policy.workspace: + raise ValueError('worker workspace must match parent policy') + parent, self.child = socket.socketpair() + self.child.set_inheritable(True) + self.broker = ShellBroker(parent, policy, self.protected_files) + identity = {key: getattr(policy, key) for key in ('run_id', 'actor_id', 'session_id', 'revision')} + worker_env = dict(env, **{_FD_ENV: str(self.child.fileno()), + 'KYROZEN_BENCHMARK_BROKER_IDENTITY': json.dumps(identity)}) + try: + self.command, self.env = protect_worker_command(command, cwd=cwd, env=worker_env, + protected_files=self.protected_files) + except Exception: + self.child.close() + self.broker.close() + raise + self.pass_fds = (self.child.fileno(),) + self.process = None + + def __enter__(self): + return self + + def attach(self, process): + if self.process is not None: + raise RuntimeError('protected launcher already attached') + self.process = process + self.child.close() + def monitor(): + process.wait() + self.broker.close() + threading.Thread(target=monitor, daemon=True).start() + + def __exit__(self, *args): + self.child.close() + self.broker.close() + if self.broker.thread is not threading.current_thread(): + self.broker.thread.join(timeout=5) + + +_ATTACK_SCRIPT = r''' +import json, os, pathlib, sys +base, workspace, nested, guard, source = map(pathlib.Path, sys.argv[1:]) +out = {} +def attempt(name, fn): + try: + fn() + out[name] = False + except OSError: + out[name] = True +attempt('write', lambda: guard.write_text('changed')) +attempt('delete', lambda: guard.unlink()) +attempt('replace', lambda: os.replace(source, guard)) +attempt('hardlink', lambda: os.link(guard, nested / 'alias')) +attempt('symlink_write', lambda: ((nested / 'sym').symlink_to(guard), (nested / 'sym').write_text('changed'))) +attempt('nested_move', lambda: os.rename(nested, base / 'moved-nested')) +attempt('workspace_move', lambda: os.rename(workspace, base / 'moved-workspace')) +attempt('parent_move', lambda: os.rename(base, base.with_name(base.name + '-moved'))) +source.write_text('after') +print(json.dumps(out)) +''' + + +def verify_worker_protection(*, require_authenticated_children=True): + """Run real attacks before key lookup; the full campaign remains blocked. + + False selects component diagnostics only and never asserts actor isolation or + detached process cleanup. The returned false evidence is release-blocking. + """ + with tempfile.TemporaryDirectory(prefix='v3-protection-') as directory: + base = Path(directory).resolve() + workspace = base / 'workspace' + nested = workspace / 'nested' + nested.mkdir(parents=True) + guard, source = nested / 'test_guard.py', nested / 'source.py' + guard.write_text('original') + source.write_text('before') + paths = list(map(str, (base, workspace, nested, guard, source))) + command = [sys.executable, '-c', _ATTACK_SCRIPT, *paths] + wrapped, env = protect_worker_command(command, cwd=workspace, env=dict(os.environ), protected_files=[guard]) + result = subprocess.run(wrapped, cwd=workspace, env=env, capture_output=True, text=True, timeout=30) + try: + values = json.loads(result.stdout) + except ValueError as exc: + raise ValueError('benchmark direct file protection probe failed: ' + result.stderr[-500:]) from exc + expected = {'write', 'delete', 'replace', 'hardlink', 'symlink_write', 'nested_move', 'workspace_move', 'parent_move'} + if result.returncode or set(values) != expected or not all(values.values()) or guard.read_text() != 'original' or source.read_text() != 'after': + raise ValueError('benchmark direct file protection probe failed') + # Remove the denied symlink-write probe's harmless alias before repeating. + (nested / 'sym').unlink(missing_ok=True) + source.write_text('before') + shell_command = shlex.join(command) + script = ''' +import json, os, sys +from benchmarks.v3_shell_launcher import install_worker_adapter +install_worker_adapter() +from openkyrozen.tools.processes import SessionProcessManager +m = SessionProcessManager('owner', sys.argv[1], 'session') +h = m.start(sys.argv[2], cwd=sys.argv[1], env=dict(os.environ), timeout_seconds=10) +h.process.wait(20) +import time +deadline = time.monotonic() + 3 +while not (h.process.stdout.closed and h.process.stderr.closed) and time.monotonic() < deadline: + time.sleep(.01) +print(json.dumps({'exit': h.process.returncode, 'out': h.stdout, 'err': h.stderr})) +m.close() +''' + env = dict(os.environ, PYTHONPATH=str(Path(__file__).resolve().parents[1]), + KYROZEN_SANDBOX_REQUIRED='1') + policy = ShellPolicy(workspace, {'read', 'write', 'shell'}) + with ProtectedWorker([sys.executable, '-c', script, str(workspace), shell_command], + cwd=workspace, env=env, protected_files=[guard], policy=policy) as launch: + process = subprocess.Popen(launch.command, cwd=workspace, env=launch.env, + pass_fds=launch.pass_fds, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + launch.attach(process) + stdout, stderr = process.communicate(timeout=40) + try: + managed = json.loads(stdout) + values = json.loads(managed['out']) + except (ValueError, KeyError) as exc: + raise ValueError('benchmark external managed shell protection probe failed: ' + stderr.decode()[-500:]) from exc + if (process.returncode or managed['exit'] or set(values) != expected or not all(values.values()) + or guard.read_text() != 'original' or source.read_text() != 'after'): + raise ValueError('benchmark external managed shell protection probe failed') + if require_authenticated_children: + raise ValueError( + 'benchmark actor isolation is unavailable: same-process runtime subagents can access ' + 'the root worker channel; independently isolated per-actor dispatch is required. ' + 'Detached macOS descendants also lack kernel-backed ownership and cleanup') + return {'authenticated_child_isolation': False, 'daemonized_process_cleanup_proven': False, + 'direct_attacks_denied': True, 'managed_shell_attacks_denied': True, + 'source_write_allowed': True, 'backend': 'sandbox-exec' if sys.platform == 'darwin' else 'bubblewrap'} diff --git a/benchmarks/v3_responses_relay.py b/benchmarks/v3_responses_relay.py index 76d296fb..7df2bc34 100644 --- a/benchmarks/v3_responses_relay.py +++ b/benchmarks/v3_responses_relay.py @@ -8,11 +8,12 @@ import threading import time import uuid +from dataclasses import dataclass from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import requests -from benchmarks.v3_matched import BudgetLedger, _cost +from benchmarks.v3_matched import BudgetLedger MODEL = "gemini-3.1-flash-lite" MAX_OUTPUT_TOKENS = 4096 @@ -20,6 +21,23 @@ MAX_ERROR_BODY_BYTES = 65_536 +@dataclass(frozen=True) +class RelayConfig: + model: str = MODEL + max_output_tokens: int = MAX_OUTPUT_TOKENS + fixed_input_token_bound: int | None = None + serial_requests: bool = False + + def __post_init__(self) -> None: + if (not self.model or type(self.max_output_tokens) is not int or self.max_output_tokens <= 0 + or (self.fixed_input_token_bound is not None and + (type(self.fixed_input_token_bound) is not int or self.fixed_input_token_bound <= 0))): + raise ValueError("relay model and token bounds must be valid") + + +DEFAULT_RELAY_CONFIG = RelayConfig() + + def _safe_error_code(error: object) -> str | None: if not isinstance(error, dict): return None @@ -28,7 +46,13 @@ def _safe_error_code(error: object) -> str | None: class ResponsesRelay: - def __init__(self, *, endpoint: str, api_key: str, budget: BudgetLedger): + def __init__(self, *, endpoint: str, api_key: str, budget: BudgetLedger, + config: RelayConfig = DEFAULT_RELAY_CONFIG): + if config.model == "deepseek-flash" or budget.pricing.as_dict().get("provider") == "DeepSeek official": + from benchmarks.v3_deepseek import DEEPSEEK_PRICING, DEEPSEEK_RELAY_CONFIG + if budget.pricing != DEEPSEEK_PRICING or config != DEEPSEEK_RELAY_CONFIG: + raise ValueError("DeepSeek relay requires its pinned CNY prices, model, and fixed reservation") + self.config = config self.endpoint = endpoint.rstrip("/") self.api_key = api_key self.client_key = secrets.token_urlsafe(24) @@ -37,6 +61,7 @@ def __init__(self, *, endpoint: str, api_key: str, budget: BudgetLedger): self.rows: list[dict] = [] self.budget_blocked = False self.provider_error: str | None = None + self._request_lock = threading.Lock() self._lock = threading.Lock() self._upstream_lock = threading.Lock() self._server = ThreadingHTTPServer(("127.0.0.1", 0), self._handler()) @@ -66,7 +91,7 @@ def _usage(result: dict) -> dict | None: cache_write = (usage.get("cache_write_tokens", usage.get("cache_creation_input_tokens", 0))) if isinstance(details, dict): cache_write = details.get("cache_write_tokens", details.get("cache_creation_tokens", cache_write)) - if not all(isinstance(value, int) and value >= 0 + if not all(type(value) is int and value >= 0 for value in (input_tokens, output_tokens, cached, cache_write)): return None if cached + cache_write > input_tokens: @@ -90,7 +115,11 @@ def _json(self, status: int, data: dict) -> None: self.wfile.write(payload) def do_POST(self): - self._forward_post() + if relay.config.serial_requests: + with relay._request_lock: + self._forward_post() + else: + self._forward_post() def _forward_post(self): if self.path.split("?", 1)[0] not in {"/v1/responses", "/responses"}: @@ -131,31 +160,33 @@ def _forward_post(self): ([instructions] if instructions else []) + system_messages) body["input"] = forwarded_items or "Continue." - body["model"] = MODEL - body["max_output_tokens"] = MAX_OUTPUT_TOKENS + body["model"] = relay.config.model + body["max_output_tokens"] = relay.config.max_output_tokens wants_stream = body.get("stream") is True body["stream"] = wants_stream encoded = json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode() attempt_id = uuid.uuid4().hex - input_bound = len(encoded) + 512 + input_bound = relay.config.fixed_input_token_bound or len(encoded) + 512 arm, fixture_id, repeat = relay.label row = {"attempt_id": attempt_id, "arm": arm, "fixture_id": fixture_id, "repeat": repeat, "request_sha256": hashlib.sha256(encoded).hexdigest(), - "model": MODEL, "max_output_tokens": MAX_OUTPUT_TOKENS, - "reserved_rmb": str(_cost(input_bound, MAX_OUTPUT_TOKENS)), + "model": relay.config.model, "max_output_tokens": relay.config.max_output_tokens, + "reserved_rmb": str(relay.budget.cost(input_bound, relay.config.max_output_tokens)), "status": "unknown", "usage": None} started = time.monotonic() stream_started = False + reserved = False try: with relay._upstream_lock: if relay.provider_error: self._json(503, {"error": {"message": "benchmark provider is disabled after an upstream error"}}) return if not relay.budget.reserve(attempt_id, input_token_bound=input_bound, - output_token_bound=MAX_OUTPUT_TOKENS, metadata=row): + output_token_bound=relay.config.max_output_tokens, metadata=row): relay.budget_blocked = True self._json(402, {"error": {"message": "benchmark spend cap reached"}}) return + reserved = True with relay._lock: relay.rows.append(row) try: @@ -213,7 +244,7 @@ def _forward_post(self): relay.budget.settle(attempt_id, usage) row["usage"] = usage row["status"] = "settled" if usage is not None else "unknown" - row["actual_rmb"] = (str(_cost(usage["input_tokens"], usage["output_tokens"], + row["actual_rmb"] = (str(relay.budget.cost(usage["input_tokens"], usage["output_tokens"], usage["cache_read_tokens"], usage["cache_write_tokens"])) if usage is not None else None) self._json(200, response_body) @@ -257,7 +288,7 @@ def _forward_post(self): relay.budget.settle(attempt_id, usage) row["usage"] = usage row["status"] = "settled" if usage is not None else "unknown" - row["actual_rmb"] = (str(_cost(usage["input_tokens"], usage["output_tokens"], + row["actual_rmb"] = (str(relay.budget.cost(usage["input_tokens"], usage["output_tokens"], usage["cache_read_tokens"], usage["cache_write_tokens"])) if usage is not None else None) upstream.close() @@ -278,6 +309,10 @@ def _forward_post(self): except (OSError, BrokenPipeError): pass finally: + if reserved and row["status"] == "unknown": + relay.budget.settle(attempt_id, None) + if "upstream" in locals(): + upstream.close() row["latency_ms"] = round((time.monotonic() - started) * 1000, 2) return Handler diff --git a/benchmarks/v3_shell_launcher.py b/benchmarks/v3_shell_launcher.py new file mode 100644 index 00000000..5af5eb24 --- /dev/null +++ b/benchmarks/v3_shell_launcher.py @@ -0,0 +1,612 @@ +"""Benchmark-only parent-authorized shell broker over an inherited socketpair. + +The worker's approval database and capability token are deliberately not inputs +into authorization. Only the parent's ShellPolicy grants execution authority. +""" +from __future__ import annotations + +import base64 +import codecs +import hashlib +import json +import math +import os +from pathlib import Path +import secrets +import signal +import socket +import subprocess +import threading +import tempfile +import time +import uuid + +_MAX_MESSAGE = 4 * 1024 * 1024 +_MAX_OUTPUT = 1_000_001 +_FD_ENV = 'KYROZEN_BENCHMARK_BROKER_FD' + + +def _encoded(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=False).encode() + + +class ShellPolicy: + """Parent-owned immutable fixture bounds, with monotonic policy narrowing.""" + def __init__(self, workspace, capabilities, *, scope_paths=('.',), run_id=None, + actor_id='benchmark', session_id='benchmark', revision=0, accepted_bounds=None): + self.workspace = Path(workspace).resolve(strict=True) + self.capabilities = frozenset(capabilities) + self.scope_paths = self._paths(scope_paths) + self.effects = None + self.risk_ceiling = 'high' + if accepted_bounds is not None: + bounds = json.loads(json.dumps(accepted_bounds)) + self.capabilities &= frozenset(bounds.get('capabilities', ())) + accepted_paths = self._paths(bounds.get('workspace_paths', ())) + intersections = {a if a.is_relative_to(b) else b + for a in self.scope_paths for b in accepted_paths + if a.is_relative_to(b) or b.is_relative_to(a)} + if not intersections: + raise ValueError('accepted plan has no approved fixture scope') + self.scope_paths = tuple(intersections) + self.effects = frozenset(bounds.get('effects', ())) + self.risk_ceiling = bounds.get('risk_ceiling', 'normal') + self.run_id = run_id or uuid.uuid4().hex + self.actor_id = actor_id + self.session_id = session_id + self.revision = revision + self.lock = threading.RLock() + self._parent = None + + def _paths(self, values): + paths = tuple((self.workspace / value).resolve(strict=True) for value in values) + if not paths or any(not p.is_relative_to(self.workspace) or not p.is_dir() for p in paths): + raise ValueError('approved scope must contain workspace directories') + return paths + + def narrow(self, *, capabilities, scope_paths): + with self.lock: + paths = self._paths(scope_paths) + capabilities = frozenset(capabilities) + if (not capabilities <= self.capabilities + or any(not any(p.is_relative_to(old) for old in self.scope_paths) for p in paths)): + raise ValueError('parent policy cannot expand approved fixture bounds') + self.capabilities = capabilities + self.scope_paths = paths + self.revision += 1 + + def child(self, actor_id, *, capabilities): + with self.lock: + child = ShellPolicy(self.workspace, self.capabilities & frozenset(capabilities), + scope_paths=self.scope_paths, run_id=self.run_id, actor_id=actor_id, + session_id=self.session_id, revision=self.revision) + child._parent = self + child.effects = self.effects + child.risk_ceiling = self.risk_ceiling + child.lock = self.lock + return child + + def authorize(self, request): + required = {'run_id', 'actor_id', 'session_id', 'call_id', 'command', 'cwd', 'env', 'revision', 'timeout_seconds', 'background'} + if set(request) != required: + raise PermissionError('invalid shell request fields') + if (request['run_id'] != self.run_id or request['actor_id'] != self.actor_id + or request['session_id'] != self.session_id or request['revision'] != self.revision + or 'shell' not in self.capabilities): + raise PermissionError('shell request is outside the effective parent policy') + if (not isinstance(request['call_id'], str) or not request['call_id'] + or not isinstance(request['command'], str) or not request['command'].strip() + or '\0' in request['command'] or not isinstance(request['cwd'], str) + or not isinstance(request['env'], dict) + or any(not isinstance(k, str) or not k or '=' in k or '\0' in k + or not isinstance(v, str) or '\0' in v for k, v in request['env'].items())): + raise PermissionError('invalid shell request') + from openkyrozen.security.command_policy import _BLOCKED_RE + from openkyrozen.tools.catalog import builtin_metadata + spec = builtin_metadata('run_cmd') + if _BLOCKED_RE.search(request['command']): + raise PermissionError('shell command is blocked by the parent command policy') + if (self.effects is not None and spec['side_effects'] not in self.effects + or spec['risk'] == 'high' and self.risk_ceiling != 'high'): + raise PermissionError('shell effects or risk exceed the accepted plan') + if type(request['background']) is not bool: + raise PermissionError('invalid shell capture mode') + timeout = request['timeout_seconds'] + if type(timeout) not in {int, float} or not math.isfinite(timeout) or not 0 < timeout <= 1800: + raise PermissionError('shell timeout is outside the bounded parent policy') + cwd = Path(request['cwd']).resolve(strict=True) + if not cwd.is_dir() or not any(cwd.is_relative_to(p) for p in self.scope_paths): + raise PermissionError('shell cwd is outside approved fixture bounds') + if self._parent is not None: + self._parent.authorize(dict(request, actor_id=self._parent.actor_id)) + + +class TicketAuthority: + """Single-use short-lived tickets bound to all concrete invocation fields.""" + def __init__(self, policy): + self.policy = policy + self.tickets = {} + + def issue(self, request, *, ttl_seconds=5): + with self.policy.lock: + self.policy.authorize(request) + now = time.monotonic() + self.tickets = {k: v for k, v in self.tickets.items() if v[1] > now} + if len(self.tickets) >= 64: + raise PermissionError('shell ticket limit reached') + token = secrets.token_urlsafe(32) + self.tickets[token] = (hashlib.sha256(_encoded(request)).digest(), + now + min(5, max(0, ttl_seconds))) + return token + + def consume(self, ticket, request): + with self.policy.lock: + record = self.tickets.pop(ticket, None) + if (record is None or record[1] <= time.monotonic() + or record[0] != hashlib.sha256(_encoded(request)).digest()): + raise PermissionError('invalid, expired, or replayed shell ticket') + self.policy.authorize(request) + + +def _kill_tree(process): + # Bounded cleanup only: detached/reparented macOS descendants are not owned. + # The full benchmark gate remains closed until kernel-backed ownership exists. + # Groups alone miss a command which calls setsid(). Discover descendants too. + descendants = [] + try: + rows = subprocess.run(['/bin/ps', '-axo', 'pid=,ppid='], capture_output=True, + text=True, timeout=3, check=False).stdout.splitlines() + children = {} + for row in rows: + pid, ppid = map(int, row.split()) + children.setdefault(ppid, []).append(pid) + pending = [process.pid] + while pending: + current = pending.pop() + descendants.append(current) + pending.extend(children.get(current, ())) + except (OSError, ValueError, subprocess.SubprocessError): + descendants = [process.pid] + for pid in reversed(descendants): + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + process.wait(timeout=3) + except subprocess.TimeoutExpired: + pass + + +def _safe_env(env): + # Loader hooks execute before the sandbox; credentials and broker state never + # belong in a model's shell, including when copied from the worker environment. + denied = ('KYROZEN_', 'DYLD_', 'LD_', 'PYTHON', 'BASH_ENV', 'ENV', 'SHELLOPTS', 'BASHOPTS') + return {k: v for k, v in env.items() + if not k.upper().startswith(denied) + and not any(word in k.upper() for word in ('TOKEN', 'SECRET', 'PASSWORD', 'API_KEY'))} + + +class ShellBroker: + def __init__(self, connection, policy, protected_files): + self.connection = connection + self.policy = policy + self.protected_files = protected_files + self.authority = TicketAuthority(policy) + self.processes = {} + self.records = {} + self.closed = threading.Event() + self.close_lock = threading.RLock() + self.thread = threading.Thread(target=self._serve, daemon=True) + self.thread.start() + + def _serve(self): + try: + with self.connection.makefile('rwb', buffering=0) as stream: + while not self.closed.is_set(): + line = stream.readline(_MAX_MESSAGE + 1) + if not line: + break + if len(line) > _MAX_MESSAGE: + break + try: + request = json.loads(line) + result = {'ok': True, 'value': self._dispatch(request)} + except Exception as exc: + result = {'ok': False, 'error': type(exc).__name__, 'message': str(exc)} + self.connection.sendall(_encoded(result) + b'\n') + except (OSError, ValueError): + pass + finally: + self.close() + + def _dispatch(self, request): + operation = request.get('op') + if operation == 'ticket': + return self.authority.issue(request['request']) + if operation == 'start': + with self.policy.lock: + self.authority.consume(request['ticket'], request['request']) + if sum(p.poll() is None for p in self.processes.values()) >= 32: + raise PermissionError('benchmark shell process limit reached') + from benchmarks.v3_protected_worker import protected_shell_command + invocation = protected_shell_command(request['request']['command'], policy=self.policy, + cwd=request['request']['cwd'], + env=_safe_env(request['request']['env']), + protected_files=self.protected_files) + try: + process = subprocess.Popen(invocation.command, shell=invocation.shell, + cwd=request['request']['cwd'], env=invocation.env, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + close_fds=True, start_new_session=True) + except Exception: + invocation.close() + raise + identifier = uuid.uuid4().hex + self.processes[identifier] = process + record = {'stdout': tempfile.TemporaryFile(), 'stderr': tempfile.TemporaryFile(), + 'background': request['request']['background'], 'eof': set(), 'timed_out': False, + 'lock': threading.Lock(), 'invocation': invocation} + self.records[identifier] = record + for name in ('stdout', 'stderr'): + threading.Thread(target=self._collect, args=(identifier, name), daemon=True).start() + threading.Thread(target=self._expire, + args=(identifier, request['request']['timeout_seconds']), daemon=True).start() + return {'id': identifier, 'pid': process.pid, 'backend': invocation.backend} + identifier = request.get('id') + if identifier not in self.processes: + raise PermissionError('process does not belong to this benchmark actor/session') + process = self.processes[identifier] + record = self.records[identifier] + if operation == 'stop': + _kill_tree(process) + return process.returncode + if operation == 'poll': + return {'exit_code': process.poll(), 'timed_out': record['timed_out']} + if operation == 'read': + name, offset = request.get('stream'), request.get('offset') + if name not in {'stdout', 'stderr'} or type(offset) is not int or offset < 0: + raise PermissionError('invalid process output request') + with record['lock']: + record[name].seek(offset) + return {'data': base64.b64encode(record[name].read(4096)).decode(), + 'eof': name in record['eof'], 'exit_code': process.poll()} + raise PermissionError('unsupported broker operation') + + def _expire(self, identifier, timeout): + process, record = self.processes[identifier], self.records[identifier] + deadline = time.monotonic() + timeout + try: + process.wait(timeout=timeout) + # A shell leader may exit while its children retain the output pipes. + while len(record['eof']) != 2 and not self.closed.is_set(): + if time.monotonic() >= deadline: + raise subprocess.TimeoutExpired(process.args, timeout) + time.sleep(.02) + except subprocess.TimeoutExpired: + record['timed_out'] = True + _kill_tree(process) + + def _collect(self, identifier, name): + process, record = self.processes[identifier], self.records[identifier] + stream = getattr(process, name) + try: + while chunk := stream.read1(4096): + with record['lock']: + record[name].seek(0, os.SEEK_END) + remaining = max(0, _MAX_OUTPUT - record[name].tell()) if record['background'] else len(chunk) + record[name].write(chunk[:remaining]) + finally: + stream.close() + with record['lock']: + record['eof'].add(name) + if self.closed.is_set(): + record[name].close() + if len(record['eof']) == 2: + record['invocation'].close() + + def close(self): + with self.close_lock: + if self.closed.is_set(): + return + self.closed.set() + try: + self.connection.shutdown(socket.SHUT_RDWR) + except OSError: + pass + self.connection.close() + for process in tuple(self.processes.values()): + _kill_tree(process) + for record in tuple(self.records.values()): + record['invocation'].close() + with record['lock']: + for name in record['eof']: + record[name].close() + + +class BrokerClient: + def __init__(self, fd, identity): + self.connection = socket.socket(fileno=fd) + self.connection.set_inheritable(False) + self.stream = self.connection.makefile('rwb', buffering=0) + self.lock = threading.Lock() + self.identity = identity + + def rpc(self, op, **values): + with self.lock: + data = _encoded(dict(op=op, **values)) + b'\n' + if len(data) > _MAX_MESSAGE: + raise PermissionError('shell request exceeds benchmark limit') + self.connection.sendall(data) + line = self.stream.readline(_MAX_MESSAGE + 1) + if not line: + raise ConnectionError('benchmark shell broker disconnected') + result = json.loads(line) + if not result.get('ok'): + raise PermissionError(result.get('message', 'shell broker denied request')) + return result['value'] + + def start(self, command, cwd, env, timeout_seconds, *, background=False): + request = dict(self.identity, call_id=uuid.uuid4().hex, command=command, cwd=cwd, env=env, + timeout_seconds=timeout_seconds, background=background) + ticket = self.rpc('ticket', request=request) + value = self.rpc('start', ticket=ticket, request=request) + return RemoteProcess(self, value['id'], value['pid'], value['backend']) + + +class RemoteStream: + def __init__(self, process, name): + self.process, self.name, self.offset = process, name, 0 + self.closed = False + + def read1(self, size): + while not self.closed: + result = self.process.client.rpc('read', id=self.process.identifier, + stream=self.name, offset=self.offset) + value = base64.b64decode(result['data'])[:size] + self.offset += len(value) + if value or result['eof']: + return value + time.sleep(.02) + return b'' + + def close(self): + self.closed = True + + +class RemoteProcess: + def __init__(self, client, identifier, pid, backend): + self.client, self.identifier, self.pid, self.backend = client, identifier, pid, backend + self.timed_out = False + self.returncode = None + self.stdout = RemoteStream(self, 'stdout') + self.stderr = RemoteStream(self, 'stderr') + + def poll(self): + value = self.client.rpc('poll', id=self.identifier) + self.returncode = value['exit_code'] + self.timed_out = value['timed_out'] + return self.returncode + + def wait(self, timeout=None): + deadline = time.monotonic() + timeout if timeout is not None else float('inf') + while self.poll() is None: + if time.monotonic() >= deadline: + raise subprocess.TimeoutExpired(self.identifier, timeout) + time.sleep(.02) + return self.returncode + + def stop(self): + self.returncode = self.client.rpc('stop', id=self.identifier) + return self.returncode + + +def _install_legacy_adapter(client, shell): + # The frozen V2 adapter owns its original string validation, 60-second + # timeout, and four-field result. Replace only its subprocess boundary. + from types import SimpleNamespace + + def run(command, *, shell, capture_output, text, timeout, env, cwd): + process = client.start(command, cwd, env, timeout) + output = {'stdout': bytearray(), 'stderr': bytearray()} + def collect(name): + stream = getattr(process, name) + while chunk := stream.read1(4096): + output[name].extend(chunk) + stream.close() + collectors = [threading.Thread(target=collect, args=(name,), daemon=True) + for name in ('stdout', 'stderr')] + for thread in collectors: + thread.start() + try: + process.wait(timeout + 5) + for thread in collectors: + thread.join() + if process.timed_out: + raise subprocess.TimeoutExpired(command, timeout) + return subprocess.CompletedProcess(command, process.returncode, + output['stdout'].decode(errors='replace'), + output['stderr'].decode(errors='replace')) + finally: + process.stop() + shell.subprocess = SimpleNamespace(run=run, TimeoutExpired=subprocess.TimeoutExpired) + + +def install_worker_adapter(): + """Install before loading the selected snapshot's adapter class.""" + raw_fd = os.environ.pop(_FD_ENV, None) + raw_identity = os.environ.pop('KYROZEN_BENCHMARK_BROKER_IDENTITY', None) + if raw_fd is None and raw_identity is None: + return False + if raw_fd is None or raw_identity is None: + raise PermissionError('incomplete benchmark shell broker context') + client = BrokerClient(int(raw_fd), json.loads(raw_identity)) + from openkyrozen.tools import shell + if not hasattr(shell, '_validated_shell_options'): + _install_legacy_adapter(client, shell) + return True + from openkyrozen.tools import processes + from openkyrozen.tools.models import CommandResult + base_manager = processes.SessionProcessManager + + class BrokerProcessManager(base_manager): + def start(self, command, *, cwd, env, timeout_seconds, cancelled=None): + process_id = uuid.uuid4().hex + audit = getattr(self, '_audit_event', None) + if audit: + audit('launch_intent', process_id, timeout_seconds=timeout_seconds) + try: + with self._lock: + live = sum(handle.process.poll() is None for handle in self._handles.values()) + if self._closed or live >= 32: + raise RuntimeError('session process manager is closed or full') + if cancelled is not None and cancelled.is_set(): + raise RuntimeError('turn was cancelled before process start') + process = client.start(command, cwd, env, timeout_seconds, + background=not getattr(self, '_foreground', False)) + handle = processes.ProcessHandle(process_id, process, self.owner_id, + self.workspace_id, self.session_id, timeout_seconds, + process.backend) + self._handles[handle.process_id] = handle + except BaseException: + if audit: + try: + # A failed RPC can leave remote dispatch uncertain. + audit('launch_failed', process_id, reason='launch_not_confirmed') + except RuntimeError: + pass + raise + started = getattr(self, '_audit_started', None) + if started: + started(handle) + for name in ('stdout', 'stderr'): + threading.Thread(target=self._collect, args=(handle, name), daemon=True).start() + expiry_options = {'cancelled': cancelled} if cancelled is not None else {} + expiry = threading.Thread(target=self._expire, args=(handle,), kwargs=expiry_options, daemon=True) + if hasattr(handle, 'expiry_thread'): + handle.expiry_thread = expiry + expiry.start() + return handle + + def _collect(self, handle, name): + if not getattr(self, '_foreground', False): + return base_manager._collect(handle, name) + stream = getattr(handle.process, name) + decoder = codecs.getincrementaldecoder('utf-8')(errors='replace') + try: + while chunk := stream.read1(4096): + with handle.lock: + setattr(handle, name, getattr(handle, name) + decoder.decode(chunk)) + with handle.lock: + setattr(handle, name, getattr(handle, name) + decoder.decode(b'', final=True)) + finally: + stream.close() + + def _expire(self, handle, cancelled=None): + outcome = getattr(self, '_audit_outcome', None) + try: + deadline = time.monotonic() + min(handle.timeout_seconds, 1800) + 5 + while handle.process.poll() is None: + if cancelled is not None and cancelled.is_set(): + if outcome: + handle.stop_reason = 'cancelled' + self.stop(handle.process_id) + handle.cancelled = True + break + remaining = deadline - time.monotonic() + if remaining <= 0: + raise subprocess.TimeoutExpired(handle.process.identifier, handle.timeout_seconds) + try: + handle.process.wait(timeout=min(.1, remaining)) + except subprocess.TimeoutExpired: + continue + except subprocess.TimeoutExpired: + if outcome: + handle.stop_reason = 'timed_out' + self.stop(handle.process_id) + handle.timed_out = True + finally: + handle.timed_out = handle.timed_out or handle.process.timed_out + if outcome: + if handle.timed_out and not handle.stop_reason: + handle.stop_reason = 'timed_out' + outcome(handle) + if handle.process.poll() is not None: + handle.finished_at = time.monotonic() + + def stop(self, process_id): + handle = self._handles.get(process_id) + if handle is None: + return None + audit = getattr(self, '_audit_event', None) + if audit and handle.process.poll() is None: + handle.stop_reason = handle.stop_reason or 'stopped' + try: + audit('stop_requested', process_id, handle=handle, reason=handle.stop_reason) + except RuntimeError: + handle.audit_error = True + handle.process.stop() + stopped = handle.process.poll() is not None + if stopped: + handle.finished_at = time.monotonic() + outcome = getattr(self, '_audit_outcome', None) + if outcome: + outcome(handle) + return stopped + + def run_command(adapter, args, *, cancelled=None): + validated = adapter._validated_shell_options(args) + if isinstance(validated, CommandResult): + return validated + command, timeout, cwd, env, background = validated + if background: + return CommandResult('Error: background processes require a session runtime', False, + failure='invalid_arguments') + started = time.monotonic() + manager = BrokerProcessManager('foreground', str(adapter._WORKSPACE_ROOT), 'foreground') + manager._foreground = True + handle = None + failure = None + try: + handle = manager.start(command, cwd=cwd, env=env, timeout_seconds=timeout) + while handle.process.poll() is None: + if cancelled is not None and cancelled.is_set(): + failure = 'cancelled' + manager.stop(handle.process_id) + break + time.sleep(.02) + handle.timed_out = handle.timed_out or handle.process.timed_out + if handle.timed_out: + failure = 'timeout' + # Collector completion is distinct from process exit and preserves final output. + while not (handle.process.stdout.closed and handle.process.stderr.closed): + time.sleep(.01) + handle.process.poll() + handle.timed_out = handle.timed_out or handle.process.timed_out + if handle.timed_out and failure is None: + failure = 'timeout' + if not failure and handle.process.returncode != 0: + failure = 'nonzero_exit' + message = handle.stdout.strip() or '(no output)' + if failure: + message = f'Error: command {failure}; exit code {handle.process.returncode}\nstdout:\n{handle.stdout}\nstderr:\n{handle.stderr}' + return CommandResult(message, failure is None, handle.process.returncode, failure, + handle.stdout, handle.stderr, time.monotonic() - started, + handle.timed_out, sandbox_backend=handle.sandbox_backend) + except (PermissionError, ConnectionError, OSError) as exc: + return CommandResult(f'Error: benchmark shell denied: {exc}', False, + failure='denied', sandbox_backend='unavailable') + finally: + manager.close() + + processes.SessionProcessManager = BrokerProcessManager + shell.run_command = run_command + # Importing the tools package loads this class before the module functions + # are replaced, so update its captured method as well. + from openkyrozen.tools.adapters import ToolAdapters + ToolAdapters.run_command = run_command + return True diff --git a/docs/architecture.md b/docs/architecture.md index 134589d9..83e9849b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -125,7 +125,7 @@ alone. The structured provider foundation is described in [providers](providers. | Requirement / subsystem | Package owner and existing components | Responsibility and authority boundary | V3 implementation | | --- | --- | --- | --- | | R236 AgentEngine | `agent`: `runtime.AgentRuntime`, turn phases in `preparation`, `action_rounds`, `completion` | Sole foreground model/tool loop coordinator; calls provider, planning and executor contracts. Does not translate provider wire formats or bypass authorization. | [#238](https://github.com/EvanProgramming/OpenKyrozen/issues/238) | -| R237 PlanningService | `tasks`: `engine.TaskManager`, `models`, `ports.TaskStore`; `agent.planner` currently bridges textual plans | Owns Working Plan and Strategic Plan transitions, validation and completion evidence. Working Plans are mutable; Strategic Plans require user approval and material revisions require reapproval. Storage persists transitions; the engine requests them. | [#241](https://github.com/EvanProgramming/OpenKyrozen/issues/241), [#242](https://github.com/EvanProgramming/OpenKyrozen/issues/242), [#245](https://github.com/EvanProgramming/OpenKyrozen/issues/245), [#246](https://github.com/EvanProgramming/OpenKyrozen/issues/246) | +| R237 PlanningService | `tasks`: `planning.PlanningService`, `engine.TaskManager`, `models`, `ports.InteractionStore` and `ports.TaskStore`; `agent.modes.InteractionController` adapts planning state to interaction modes and questions | The tasks feature owns Strategic Plan validation, material revisions, approval transitions and completion evidence through PlanningService, and Working Plan edits through TaskManager. The compatibility interaction controller inherits those plan operations without reimplementing them. Storage persists transitions; the engine requests them. | [#241](https://github.com/EvanProgramming/OpenKyrozen/issues/241), [#242](https://github.com/EvanProgramming/OpenKyrozen/issues/242), [#245](https://github.com/EvanProgramming/OpenKyrozen/issues/245), [#246](https://github.com/EvanProgramming/OpenKyrozen/issues/246) | | R238 ToolRegistry | `tools`: `manifest.ToolRegistry`, `ToolManifest`, `registry`, `adapters.ToolAdapters` | Owns definitions, schemas, metadata and executor registration. Supplies the catalog to providers and execution; registration or discovery never grants permission. Extend this registry rather than adding a competing catalog. | [#232](https://github.com/EvanProgramming/OpenKyrozen/issues/232) | | R239 ToolExecutor | `agent`: `executor`, `types.ExecutionReceipt` | Owns authorized invocation and receipts. Uses registry definitions, security decisions and bound tool implementations; denied or failed calls cannot become successful effects. Does not own permission policy or plan transitions. | [#233](https://github.com/EvanProgramming/OpenKyrozen/issues/233), [#236](https://github.com/EvanProgramming/OpenKyrozen/issues/236) | | R240 SecurityPolicy | `security`: `capabilities`, `tool_policy`, `permissions`, `permission_gate` | Owns capability and approval authorization. Effective permissions stay within configured, session and delegated bounds; model output, routing advice and catalog entries cannot approve a call. The executor enforces the decision. | [#261](https://github.com/EvanProgramming/OpenKyrozen/issues/261) | diff --git a/docs/index.md b/docs/index.md index 86a9f76f..cf1770d3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -51,3 +51,7 @@ This index covers the current product guides, developer references, and dated en - Machine-readable audit receipts and benchmark inputs remain beside their corresponding reports under `docs/` and `docs/benchmarks/`. The repository's [English overview](../README.md) links here. Technical guides are maintained in English; README translations provide localized entry points. + +- [V3 finalization execution plan](superpowers/plans/2026-11-07-v3-finalization.md) + +- [V3 finalization status and release blockers](v3-finalization-status.md) diff --git a/docs/installation.md b/docs/installation.md index d6083690..8ffaeafc 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,5 +1,7 @@ # Install, update, and recover +V3 is not yet released. The public installation commands below use the published stable v2.0.8 release. + ## Requirements The v3.0.0 release supports Python 3.12 and 3.13. Python 3.14 is excluded by the package metadata. The official installers provision Python through `uv`; a source checkout needs Git and a supported Python. The optional Bubble Tea terminal UI is a Go binary distributed with the release. Docker uses Python 3.12. See the live [release page](https://github.com/EvanProgramming/OpenKyrozen/releases/latest) before copying a pinned command: installer examples below identify the v3.0.0 release. @@ -9,14 +11,14 @@ The v3.0.0 release supports Python 3.12 and 3.13. Python 3.14 is excluded by the On macOS or Linux: ```sh -curl -fsSL https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v3.0.0/install.sh | sh +curl -fsSL https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v2.0.8/install.sh | sh kyrozen --version ``` On Windows PowerShell: ```powershell -irm https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v3.0.0/install.ps1 | iex +irm https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v2.0.8/install.ps1 | iex kyrozen --version ``` @@ -26,7 +28,7 @@ For a direct Python install, add the optional web server dependencies explicitly ```sh uv tool install --python 3.12 --force --with fastapi --with uvicorn \ - https://github.com/EvanProgramming/OpenKyrozen/releases/download/v3.0.0/openkyrozen-3.0.0-py3-none-any.whl + https://github.com/EvanProgramming/OpenKyrozen/releases/download/v2.0.8/openkyrozen-2.0.8-py3-none-any.whl kyrozen --version ``` diff --git a/docs/superpowers/plans/2026-11-07-v3-finalization.md b/docs/superpowers/plans/2026-11-07-v3-finalization.md new file mode 100644 index 00000000..d2b1bb2f --- /dev/null +++ b/docs/superpowers/plans/2026-11-07-v3-finalization.md @@ -0,0 +1,26 @@ +# V3 finalization execution plan + +Approved by the user on 2026-11-07 in the V3 finalization chat. + +## Binding requirements + +Audit all 266 source requirements in milestone issues #227–#268 and the #319 regression, including issue comments and dependent PRs. Require all seventeen R250–R266 full-release gates to pass against one clean signed candidate. No preview waiver qualifies. Preserve Python 3.12/3.13 and the documented Windows supported-feature exceptions. + +## Work sequence + +1. Start at stable candidate 7905fb634fe447243668c2d01179ce5cb2cdb275 in an independent checkout outside the synchronized Documents directory; preserve original runtime data and evidence. Selectively reuse diagnostic protection tests; never merge the diagnostic branch's reduced CI. +2. Retain separate scope, original-test, provider, process and worker-completeness evidence. Missing evidence fails closed; persist before stopping on the first safety failure. A resumed campaign cannot skip a recorded safety failure. +3. Protect original fixture tests and ancestors identically in both benchmark arms. Implement the approved parent-authorized external shell launcher, preserving sandbox boundaries and process lifecycle. The parent independently enforces frozen fixture/accepted-plan authority, monotonic narrowing and child intersections. Single-use tickets bind identities, full command/cwd/environment and policy revision. Reject forged approval, replay and stale/cross-session requests. Prove direct and managed-shell attacks fail on real Linux/macOS while legitimate source edits work. No supported sandbox means no credentials or provider requests. +4. Explain the historical Windows R260 cancellation failure with causal evidence and a reproducing regression, or retain the blocker. Collect exact dispatch/provider/cancellation/cleanup timing, four-second thread dumps and database/process state. More successful repeats or larger timeouts are insufficient. +5. Move DeepSeek configuration into immutable in-tree pricing/relay configuration. Reconstruct cumulative exposure from original ledgers, deduplicate copies, preserve unknown reservations, cap total at 15 CNY. Use deepseek-flash Responses consistently for both arms, output 4096; reserve 1M input plus output at peak prices, 2.032768 CNY per attempt. Smoke calls and retries count. No calls before offline blockers pass. +6. Run full tests/check/lint/docs/architecture, Go, wheel/sdist external installs, CLI/Web/TUI/update and Docker persistence. Six OS/Python combinations require workflow acceptance and >=1800s soak; record Windows POSIX exceptions honestly. +7. Freeze the final signed SHA, then run the unchanged 30 fixtures x 3 repeats x 2 arms. Require complete accounting, paired-success 95% CI lower bound >0, request and token savings >=10%, V3 recovery count zero and all independent V3 acceptance/planning checks passed. No cross-candidate row reuse, cherry-picking or lowered thresholds. +8. Create signed exact-SHA gate records/manifest and committed-tree distribution artifacts outside the checkout. Review; fast-forward main non-forcibly to the same fully verified SHA after rechecking rules and concurrent changes. Re-freeze if identity changes. Public downloads stay on the actually published stable release until V3 publication. + +## Stop boundary + +Do not create tags, draft/public Releases, upload package repositories, or invoke the preview publisher. Missing evidence, budget exhaustion, provider outage or an unexplained R260 failure remains a blocker; do not call the work release-ready. + +## Execution evidence + +External execution ledger and unmodified source requirement/comment snapshots: `/Users/evangong/.codex/worktrees/openkyrozen-v3-evidence/`. These are progress records, not passing release authorization. Credentials must never enter tracked files, reports or logs. diff --git a/docs/superpowers/plans/index.md b/docs/superpowers/plans/index.md index f13b27cb..5cf49e10 100644 --- a/docs/superpowers/plans/index.md +++ b/docs/superpowers/plans/index.md @@ -19,3 +19,5 @@ Add a linked issue-specific plan here before coding any later V3 issue. - [V3 #246 — Reapproval for material Strategic Plan revisions](2026-10-16-v3-issue-246.md) + +- [V3 finalization and exact-candidate acceptance](2026-11-07-v3-finalization.md) diff --git a/docs/tool-inventory.md b/docs/tool-inventory.md index 0a1cf461..cff53228 100644 --- a/docs/tool-inventory.md +++ b/docs/tool-inventory.md @@ -7,7 +7,7 @@ Run `make docs-check` after changing a tool, endpoint, or MCP contract. ## Runtime tools -- Built-in runtime tools: **53** +- Built-in runtime tools: **55** - Names with the `git_` prefix: **14** - The CLI Action contract is a plain string in the `args` field. - Web and MCP exposure is filtered by the configured capability profile. @@ -53,10 +53,12 @@ Run `make docs-check` after changing a tool, endpoint, or MCP contract. | `list_dir` | `read` | `read_only` | true | List contents of a directory. Args format: "path" (default "."). | `path`: string | | `list_subagents` | `read` | `internal` | false | List agents or inspect one run_id from this chat/project; args is JSON. | `run_id`: string | | `list_tree` | `read` | `read_only` | true | Recursively list the directory tree of the given path. Args format: "path" (default "."). | `path`: string | +| `propose_plan` | `read` | `internal` | false | Propose a versioned Strategic Plan after read-only inspection. This never | `assumptions`: array, `bounds`: object, `steps`: array, `summary`: string, `title`: string; required: `title`, `summary`, `bounds`, `steps` | | `read_file` | `read` | `read_only` | true | Read content from a file. Args format: "path". | `max_chars`: integer, `max_lines`: integer, `path`: string, `start_line`: integer; required: `path` | | `read_process` | `read` | `read_only` | false | Read a bounded page from a session-owned process stream. | `limit`: integer, `offset`: integer, `process_id`: string, `stream`: string; required: `process_id` | | `read_tool_artifact` | `read` | `read_only` | false | Read a session-owned tool artifact in pages up to 1,000 characters. | `artifact_id`: string, `limit`: integer, `offset`: integer; required: `artifact_id` | | `read_webpage` | `network` | `remote` | false | Fetch the content of a web page and return its plain‑text body. | `url`: string; required: `url` | +| `request_question` | `read` | `internal` | false | Ask 1-3 structured clarification questions. Questions cannot request | `questions`: array; required: `questions` | | `run_cmd` | `shell` | `process` | false | Execute a shell command. Args: the full command string. | `background`: boolean, `command`: string, `cwd`: string, `env`: object, `timeout_seconds`: number; required: `command` | | `search_files` | `read` | `read_only` | true | Search literal text with bounded, relative file-and-line results. | `glob`: string, `limit`: integer, `path`: string, `query`: string; required: `query` | | `search_memory` | `read` | `read_only` | false | Search stored memories for facts relevant to the query. Args: "query" | `query`: string; required: `query` | @@ -64,7 +66,7 @@ Run `make docs-check` after changing a tool, endpoint, or MCP contract. | `send_subagent` | `read` | `internal` | false | Reuse a finished sub-agent; args JSON contains run_id and a complete assignment. | `assignment`: object, `run_id`: string; required: `run_id`, `assignment` | | `spawn_agents` | `read` | `internal` | false | Automatically start parallel specialist assignments; args is a JSON assignment batch. | `assignments`: array | | `stop_process` | `shell` | `process` | false | Stop a process owned by the current actor, workspace, and session. | `process_id`: string; required: `process_id` | -| `update_plan` | `write` | `internal` | false | Apply revision-checked ordered Working Plan edits or adapt the legacy task list. | `complexity`: string, `expected_revision`: integer, `operations`: array, `plan_id`: string, `reason`: string, `tasks`: array; requires either `operations` and `expected_revision` or `complexity` and `tasks` | +| `update_plan` | `write` | `internal` | false | Edit a revision-checked Working Plan. Use start with a stable task ID before | `complexity`: string, `expected_revision`: integer, `operations`: array, `plan_id`: string, `reason`: string, `tasks`: array; requires either `operations` and `expected_revision` or `complexity` and `tasks` | | `wait_subagents` | `read` | `internal` | false | Wait up to 60 seconds for delegated work and its mandatory reviews; args is JSON. | `run_ids`: array, `timeout`: number | | `write_file` | `write` | `filesystem` | false | Write content to a file. Args format: "path\|content". | `content`: string, `path`: string; required: `path`, `content` | diff --git a/docs/v3-evaluation.md b/docs/v3-evaluation.md index bd9cd6b9..26029a07 100644 --- a/docs/v3-evaluation.md +++ b/docs/v3-evaluation.md @@ -1,9 +1,15 @@ # OpenKyrozen V3 matched evaluation -The frozen cohort contains 30 fixtures across ten task categories, with three matched repeats per fixture on V2 `v2.0.8` (`1bfc82c793df49e76097061ec5adfef0e4e2e883`) and the V3 candidate. It includes dedicated replanning, Strategic Plan scope, material-revision approval, and plan-churn scenarios. `benchmarks/v3_matched_live.py` creates isolated workspace and state for each run, verifies outcomes outside the model, records token/request accounting, and resumes only a report whose hash is pinned in its original budget ledger. Strategic Plan fixtures authorize their workspace-local test command through explicit `shell` capability and `process` effect bounds. Scope evidence accepts either a live strategic task binding or a matching completed-plan event for the exact accepted plan ID, version, and fingerprint. Process cancellation is checked against the OS process group or process tree; permission scenarios require denied runtime receipts and reject secret disclosure. +The frozen cohort contains 30 fixtures across ten task categories, with three matched repeats per fixture on V2 `v2.0.8` (`1bfc82c793df49e76097061ec5adfef0e4e2e883`) and the V3 candidate. It includes dedicated replanning, Strategic Plan scope, material-revision approval, and plan-churn scenarios. `benchmarks/v3_matched_live.py` creates isolated workspace and state for each run, verifies outcomes outside the model, records token/request accounting, and resumes only a report whose hash is pinned in its original budget ledger. Strategic Plan fixtures preserve their accepted workspace-local read/write authority. The independent harness runs acceptance tests after checking original-test integrity; the worker does not gain shell/process authority from the fixture. Scope evidence accepts either a live strategic task binding or a matching completed-plan event for the exact accepted plan ID, version, and fingerprint. Process cancellation is checked against the OS process group or process tree; permission scenarios require denied runtime receipts and reject secret disclosure. The `--repo` checkout must contain the pinned V2 commit so the runner can extract its source with `git archive`. The CI full-suite jobs fetch complete history and verify that exact commit before running tests; for a shallow local checkout, fetch the V2 baseline first. +The formal finalization campaign uses the [DeepSeek CNY command below](#deepseek-cny-campaign), with its cumulative CNY 15 cap. The Panstar configuration remains a separate compatibility path; its historical USD allowance does not authorize spending in that campaign. + +The repaired cohort keeps the same 30 IDs, ten categories, three repeats and two arms. It adds in-process multi-file API changes, behavior-preserving shared-helper refactors, independently correlated Git tool receipts, and eleven-turn context scenarios. Both arms receive an explicit context window of 1,000,000 tokens for ordinary fixtures and 16,384 for context fixtures, with 4,096 output tokens reserved. Provider-port observations measure the actual dispatched messages, including compaction requests, without changing either runtime's behavior. Estimated components remain distinct from provider-reported usage; unavailable fields stay unknown. + +The paired long-context oracle checks the same facts, supersession, complete scripted turns and outbound bounds for both arms. V3 compaction observation is a separate feature acceptance condition and does not alter paired task success. Checkpoints bind the canonical fixture digest and observer, verifier, metric and relay hashes. A changed apparatus cannot reuse old rows; historical calls still count toward cumulative spending. + The local authenticated Responses relay pins `gemini-3.1-flash-lite`, caps output at 4,096 tokens, reserves worst-case spend before forwarding, and stores request hashes and usage rather than prompts or answers. It moves system-role messages from `input` to Responses `instructions` for both benchmark arms; when no other input remains, it sends `Continue.`. It stops consuming a stream at its terminal event, preserving unknown usage as a held reservation without attempting to settle the same attempt twice. Worker failures record the exit code and exception class without persisting stderr or its potentially sensitive details. `response.failed` and HTTP error events record only an ASCII error code of at most 80 characters; HTTP error bodies are capped at 64 KiB for parsing, and free-form provider messages are never stored. Failed workers with partial usage are retried on resume; complete usage and a successful exit are required before a fixture row can be reused. The cumulative Panstar cap is US$20. Panstar's quoted rates are $0.033/1M input, $0.275/1M output, and $0.0033/1M cache reads; cache writes are charged at the input rate because Panstar quoted no separate rate. The CNY SQLite ledger uses the recorded 2026-10-23 USD/CNY rate of 6.67135 and records both the quote and FX source in pricing metadata. Unknown usage retains its reservation. Every fresh run requires an explicit `--prior-hold-usd` equal to all earlier settled spend and held reservations, including unknown attempts; never reuse a smaller reservation. Before every run, reconstruct cumulative exposure from the original report and budget ledger as settled spend plus every reserved or unknown attempt. Do not use a stored or copied exposure amount; if the source reports and ledger are unavailable, do not start the campaign. The runner requires the hold explicitly and records it in the report. Existing partial reports remain inconclusive and are not used as evaluation evidence. The application-shaped native tool continuation was verified; a separate `previous_response_id` diagnostic was rejected by the gateway and is not used by OpenKyrozen. Set `KYROZEN_BENCHMARK_API_KEY` and `KYROZEN_BENCHMARK_BASE_URL` from the approved provider configuration; the default endpoint is `https://ai.panstar.ai/v1`. Before reading provider credentials or starting campaign requests, the runner probes the OS filesystem sandbox. The cohort requires working filesystem restrictions for its permission and scope checks, so an unavailable backend blocks the campaign without provider spend. This benchmark prerequisite does not change the application's best-effort sandbox default. @@ -20,3 +26,23 @@ python -m benchmarks.v3_matched_live \ ``` The report is descriptive evaluation data, not release authorization. A result is conclusive only when all 180 matched rows have complete provider usage, all independent verifiers pass, the full paired success interval improves, token and request reductions meet their thresholds, and no safety or protocol-recovery control fails. Missing or unknown usage, failed checks, budget exhaustion, or an incomplete run remains inconclusive. Any code change creates a different candidate SHA and requires a new matched run. + +## DeepSeek CNY campaign + +The in-tree `benchmarks.v3_deepseek` CLI uses the same frozen 180-row cohort and unchanged comparison thresholds, with immutable DeepSeek configuration. It pins the official `https://api.deepseek.com` Responses endpoint and `deepseek-flash` alias (served model `DeepSeek-V4.1-Flash-1017`). Peak prices are CNY 2/M input, 8/M output, 0.04/M cache reads, and conservatively 2/M cache writes, from the [official pricing table](https://api-docs.deepseek.com/zh-cn/quick_start/pricing/). Every request reserves exactly CNY 2.032768 for 1,000,000 input and 4,096 output tokens. Requests are serial through stream completion. Unknown usage keeps its entire reservation; retries and the required metered smoke share the cohort ledger. Smoke calls remain in its journal and do not count toward the 180 matched rows. + +The cumulative cap is **CNY 15 including previous campaigns**. Supply original prior ledgers; the reader opens them in SQLite read-only mode, independently reconstructs settled charges from usage, retains reserved/unknown amounts, and deduplicates identical attempt IDs across copies. New aggregate carryovers include the underlying attempt records and a SHA256 manifest so later campaigns can be reconciled with their original ledgers without double counting or releasing unknown exposure. Missing sources, conflicting records, invalid provenance, unknown pricing, and overruns fail closed. Original campaign `run1/budget.sqlite3` and its `evidence/budget.sqlite3` copy represent one campaign, not two. The code has no fixed candidate SHA or machine-local path. + +After the offline protection gates pass, use a clean, signed frozen candidate and output paths outside it. The CLI verifies the configured candidate and release-owner signature; the runner probes the OS sandbox and protected worker/managed shell before reading `KYROZEN_BENCHMARK_API_KEY` from the environment. It then performs and checkpoints a two-request native-tool smoke through the same relay before starting cohort workers: a typed `echo_probe` call must carry an unpredictable value, its `function_call_output` must use the returned call ID, and the final assistant answer must include that value. Both requests require complete usage; the probe has no external side effects. The checkpoint pins the two verified attempt IDs. A previously settled smoke for the same candidate can be reused; failed or unknown smoke attempts remain charged or held when retrying. + +```sh +python -m benchmarks.v3_deepseek \ + --source "$FROZEN_SOURCE" --candidate-sha "$FROZEN_CANDIDATE_SHA" \ + --repo "$REPOSITORY_WITH_V2_HISTORY" \ + --output "$CAMPAIGN_OUTPUT/report.json" \ + --budget-db "$CAMPAIGN_OUTPUT/budget.sqlite3" \ + --prior-ledger "$ORIGINAL_PRIOR_LEDGER" \ + --limit-cny 15 +``` + +CNY checkpoints report their actual CNY cap, prior exposure and remaining balance without an exchange-rate workaround. A new request cannot be reserved once cumulative settled spend plus outstanding holds exceeds CNY 12.967232. A completed smoke is provider compatibility/accounting evidence only; it does not satisfy the cohort or authorize release. diff --git a/docs/v3-finalization-status.md b/docs/v3-finalization-status.md new file mode 100644 index 00000000..f3f9d307 --- /dev/null +++ b/docs/v3-finalization-status.md @@ -0,0 +1,46 @@ +# V3 finalization status + +V3 is **not release-ready**. This is an implementation and investigation handoff, not a release manifest. No final-candidate gate bundle, V3 tag or Release has been created by this work. + +## Implemented safeguards + +- Matched evaluation now retains independent scope, original-test integrity, provider, process and worker-completeness evidence. Missing evidence fails closed. The first safety failure is checkpointed and stops the campaign; resuming cannot skip it. Malformed worker JSON is preserved as an incomplete result rather than losing the checkpoint. +- DeepSeek has immutable CNY pricing, a cumulative 15 CNY cap, serial requests and fixed 2.032768 CNY worst-case reservations. Original source ledgers are read-only, copies deduplicate, and proven carried history expands to original attempt records. Smoke, retries and cohort share the ledger; unknown use stays reserved. +- A two-request native-tool smoke checks unpredictable arguments, correlated function-call output, final response and complete usage before the 180-row cohort layout. +- Original fixture tests are OS-protected for both benchmark arms. The benchmark-only external shell broker enforces parent-owned bounds and request-bound tickets. Real macOS tests cover legitimate writes, protected-file attacks, narrow scope, output/exit/cancellation/timeout and frozen V2 compatibility. +- Process lifecycle audit persists scoped launch intent before dispatch and retains uncertain outcomes across restart. Historical PIDs are observation metadata, never authority to reattach, signal or replay. Audit failure cannot certify cleanup, and bounded audit writes preserve cancellation under SQLite contention. +- Learning promotion requires complete paired outcome, verification, cost, latency and regression measurements. Missing dimensions and old success-only replay records remain incomplete. Known failed effects replace unsupported final success claims with receipt-derived results, including valid structured child reports. +- Public README, installation guide and website commands remain on the published v2.0.8 assets. Candidate package metadata remains 3.0.0. V3 publication must precede switching public download instructions. + +## Release blockers + +1. **Actor authentication:** runtime subagents share the Python worker process. An inherited root control channel does not prove a request's narrower child identity. Context variables or worker-supplied approval/database state are not independent authorization. Trusted actor creation/dispatch with isolated actor channels is still required. +2. **Detached macOS descendants:** a command can fork, call setsid, outlive its original parent and close its output. Current ancestry/process-group cleanup cannot reliably find that detached orphan. The regression characterization explicitly kills its escaped test child; it does not certify cleanup. Local SDK headers document that the old kqueue NOTE_TRACK family is unsupported. A supported ownership boundary must be established before the lifecycle gate can pass. +3. **Bounded platform evidence:** the local Colima VM rejects Bubblewrap UID mapping. Subsequent real Linux/macOS × Python 3.12/3.13 component checks passed in [run 42267305014](https://github.com/EvanProgramming/OpenKyrozen/actions/runs/42267305014) against 1f45401. These prove the bounded component tests, including V2/V3 adapter compatibility and protected file behavior; they do not prove authenticated actors, detached macOS cleanup, or a later final candidate. +4. **R260 history:** the original failure was a five-second future-completion timeout after cancellation. Instrumented Windows 3.12/3.13 runs completed 50 iterations each against both the old failing product and prior candidate (200 total) without reproducing the timeout. A later historical-product experiment added baseline, CPU, SQLite-writer and combined controls: 10 samples per condition and Python version, 80 total, also passed. These are scheduling observations, not attribution of the original failure. The separately reproduced raw-exit/cancellation-publication test race and database cancellation-wait boundary must not be substituted for that attribution. R260 remains inconclusive. +5. **Final evidence:** requirement-by-requirement acceptance, final six-platform/30-minute validation, the live 180-row comparison, all 17 signed passing records and the same-SHA main integration remain incomplete. + +The full campaign preflight deliberately rejects the unproven actor/descendant boundary before credential lookup or model requests. The component probe's limited coverage must never be promoted to a full passing gate or used to bypass that rejection. + +## Functional acceptance audit + +The 266 source criteria were independently audited against implementation and regression bodies: 196 implemented-with-regression, 52 partial and 18 unproven at the audit baseline. Partial includes both concrete functional gaps and missing acceptance evidence. Those baseline counts are not a final acceptance result. Issue closure alone is not completion. + +Implemented repairs cover unsuccessful provider finishes, native streaming, explicit task start/completion, immutable strategic objective anchors with scoped refinement, pinned Working Plans, unknown-effect restart protection, turn-scoped background cancellation, advisory-only Jev decisions and evidence-gated learning promotion. Strategic transitions now belong to `tasks.planning.PlanningService`; the interaction controller retains mode and question adaptation. + +Native Agent, Ask and Plan paths use typed execution/control contracts. Plan proposals preserve explicit user acceptance, material revisions pause mutations, and safe narrowing preserves the existing automatic-acceptance policy. Lazy legacy helpers and reply-specific provenance keep literal native text intact through CLI/Web/TUI formatting and delegated synthesis. Complex read-only children can edit only their own internal plans and retrieve only their own tool artifacts, without receiving filesystem write authority. + +Native context preflight now includes tool schema and continuation payload estimates, binds reported counts to provider/model/schema identity, and compacts old complete history units without splitting call/result groups. Overflow recovery retries only generation, once, before any emitted stream content; it cannot replay an executed tool. Prose-only task criteria remain pending unless trusted verification exists: a successful file write cannot certify an unrelated test or business outcome. + +The repaired 30-fixture cohort and observer/verifier identities are frozen before any paid run. Multi-file/API, refactor, Git and multi-turn context oracles retain the original comparison thresholds and shared physical success criteria. New regression, packaging and platform evidence must bind the eventual clean signed candidate; intermediate passing suites cannot be combined into a final release manifest. + +## Evidence and constraints + +- Approved execution plan: [V3 finalization](superpowers/plans/2026-11-07-v3-finalization.md). +- Windows candidate diagnostic: https://github.com/EvanProgramming/OpenKyrozen/actions/runs/42261579117 +- Windows historical diagnostic: https://github.com/EvanProgramming/OpenKyrozen/actions/runs/42261682019 +- Historical Windows load controls: https://github.com/EvanProgramming/OpenKyrozen/actions/runs/42288273719 +- Detailed local execution logs, 266 requirement/comment inventory and historical Windows artifacts are outside the checkout in `/Users/evangong/.codex/worktrees/openkyrozen-v3-evidence/`. +- Original DeepSeek ledgers independently reconstruct settled 0.58291496 CNY plus held 0.000000000007 CNY, total 0.582914960007 CNY. This work has made no external DeepSeek requests. + +Do not waive the blockers, publish the preview, create a V3 tag/Release, upload package repositories or merge an unverified candidate to main. A changed candidate SHA requires new exact-candidate evidence. diff --git a/openkyrozen/agent/compaction.py b/openkyrozen/agent/compaction.py index ea806dca..e7cceb39 100644 --- a/openkyrozen/agent/compaction.py +++ b/openkyrozen/agent/compaction.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib +import json import math from dataclasses import dataclass, field from typing import Any, Callable @@ -86,12 +87,9 @@ def _accounting_message(message: Any) -> dict[str, Any]: """Create a text-only estimate without changing native provider history.""" from openkyrozen.providers.models import ModelResponse, ToolReply - if isinstance(message, ModelResponse): - calls = [{"name": call.name, "arguments": call.arguments} for call in message.tool_calls] - content = message.text + ("\n" + str(calls) if calls else "") - return {"role": "assistant", "content": content} - if isinstance(message, ToolReply): - return {"role": "tool", "content": str(message.output)} + if isinstance(message, (ModelResponse, ToolReply)): + from openkyrozen.providers.context import context_accounting_messages + return context_accounting_messages([message])[0] return message @@ -124,7 +122,7 @@ def message_fingerprint(messages: list[Any]) -> str: def is_context_digest(message: dict[str, Any]) -> bool: - return str(message.get("content", "")).startswith((DIGEST_PREFIX, OMISSION_PREFIX)) + return isinstance(message, dict) and str(message.get("content", "")).startswith((DIGEST_PREFIX, OMISSION_PREFIX)) def retain_context_digests(messages: list[dict[str, Any]], limit: int = 32) -> list[dict[str, Any]]: @@ -147,18 +145,29 @@ class ContextState: reported_inputs: dict[str, int] = field(default_factory=dict) status: dict[str, Any] = field(default_factory=dict) overflow_retried: bool = False + native_input_overhead_tokens: int = 0 + native_schema_fingerprint: str = "" + request_provider_id: str = "" + request_model: str | None = None def __post_init__(self) -> None: self.window_tokens, self.window_source = resolve_context_window(self.model, self.configured_window) + def request_fingerprint(self, messages) -> str: + identity = [message_fingerprint(messages), self.request_model or self.model, + self.request_provider_id, self.native_schema_fingerprint] + return hashlib.sha256(json.dumps(identity, ensure_ascii=False).encode("utf-8")).hexdigest() + def update(self, messages: list[dict[str, Any]], *, compaction: dict[str, Any] | None = None) -> dict[str, Any]: - fingerprint = message_fingerprint(messages) + fingerprint = self.request_fingerprint(messages) estimated, breakdown = estimate_messages(messages) + estimated += self.native_input_overhead_tokens + breakdown["fixed_instructions"] += self.native_input_overhead_tokens reported = self.reported_inputs.get(fingerprint) input_tokens = reported if reported is not None else estimated remaining = None if self.window_tokens is None else max(0, self.window_tokens - input_tokens - self.reserve_tokens) self.status = { - "model": self.model, + "model": self.request_model or self.model, "window_tokens": self.window_tokens, "window_source": self.window_source, "input_tokens": input_tokens, @@ -166,6 +175,8 @@ def update(self, messages: list[dict[str, Any]], *, compaction: dict[str, Any] | "breakdown": breakdown, "breakdown_source": "estimated", "reserve_tokens": self.reserve_tokens, + "native_input_overhead_tokens": self.native_input_overhead_tokens, + "native_overhead_method": "stdlib_json_estimate_plus_framing" if self.native_schema_fingerprint else "none", "remaining_tokens": remaining, "compaction": compaction or self.status.get("compaction", {"status": "not_needed"}), } @@ -173,7 +184,7 @@ def update(self, messages: list[dict[str, Any]], *, compaction: dict[str, Any] | def note_provider_usage(self, messages: list[dict[str, Any]], prompt_tokens: int) -> None: if prompt_tokens > 0: - self.reported_inputs[message_fingerprint(messages)] = int(prompt_tokens) + self.reported_inputs[self.request_fingerprint(messages)] = int(prompt_tokens) self.update(messages) @@ -262,6 +273,57 @@ def under_pressure(current: dict[str, Any]) -> bool: from openkyrozen.providers.models import ModelResponse, ToolReply if any(isinstance(item, (ModelResponse, ToolReply)) for item in working): + # Only authoritative projected history can be evicted. Current-turn work, + # system state, orphan replies, and incomplete groups remain fixed. + units: list[list[int]] = [] + index = 0 + while index < len(working): + item = working[index] + unit = None + if id(item) in state.history_message_ids: + if isinstance(item, dict) and item.get("role") in {"user", "assistant"}: + unit = [index] + elif isinstance(item, ModelResponse) and not item.tool_calls: + from openkyrozen.providers.models import FinishReason + if item.finish_reason == FinishReason.FINAL: + unit = [index] + elif isinstance(item, ModelResponse) and item.tool_calls: + positions = list(range(index + 1, index + 1 + len(item.tool_calls))) + replies = [working[position] for position in positions if position < len(working)] + expected = {call.id: call.name for call in item.tool_calls} + if (len(replies) == len(item.tool_calls) + and all(isinstance(reply, ToolReply) for reply in replies) + and len(expected) == len(item.tool_calls) + and {reply.call_id for reply in replies} == set(expected) + and all(expected[reply.call_id] == reply.name for reply in replies) + and all(id(reply) in state.history_message_ids for reply in replies)): + unit = [index, *positions] + if unit is not None: + units.append(unit) + index = unit[-1] + 1 + else: + index += 1 + candidates = [position for unit in units[:-KEEP_RECENT_MESSAGES] for position in unit] + native_history_digest = None + history_compacted = bool(candidates) + if candidates: + raw = _format_entries([_accounting_message(working[position]) for position in candidates]) + window_budget = ((state.window_tokens or 16_384) - state.reserve_tokens) // 2 + output_chars = max(400, min(6_000, max(400, window_budget) * 3)) + input_chars = max(1_000, min(12_000, max(1_000, window_budget) * 3)) + summary = _bounded_summary(raw, summarize, max_chars=input_chars, output_chars=output_chars) + native_history_digest = _digest(summary, len(candidates), kind="native history messages") + removed = set(candidates) + first = candidates[0] + working = [message for position, message in enumerate(working) if position not in removed] + working.insert(first, native_history_digest) + status = state.update(working, compaction={ + "status": "summarized" if summary else "trimmed_after_summary_failure", + "kind": "native_history", "omitted_entries": len(candidates), + }) + if not under_pressure(status): + return CompactionResult(working, compacted=True, history_digest=native_history_digest) + # Compact old outputs in complete native call/result groups. Replacing only # the payload preserves provider continuation structure and call correlation. native_groups: list[list[int]] = [] @@ -284,8 +346,8 @@ def under_pressure(current: dict[str, Any]) -> bool: if pending: native_groups.append(pending) - compacted = False - details: dict[str, Any] = {"status": "not_needed"} + compacted = history_compacted + details: dict[str, Any] = dict(state.status.get("compaction", {})) if history_compacted else {"status": "not_needed"} for group in native_groups[:-1]: if not all(isinstance(working[index], (ModelResponse, ToolReply)) for index in group): continue @@ -305,15 +367,16 @@ def under_pressure(current: dict[str, Any]) -> bool: # oversized tool receipts so recent reasoning has the available budget. first_native = next((index for index, item in enumerate(working) if isinstance(item, (ModelResponse, ToolReply))), len(working)) - prefix = compact_for_pressure(working[:first_native], state, summarize, force=force) + prefix = compact_for_pressure(working[:first_native], state, summarize, + force=force or under_pressure(state.update(working))) working = prefix.messages + working[first_native:] status = state.update(working, compaction=details) if under_pressure(status): status["compaction"] = {"status": "fixed_context_too_large"} return CompactionResult(working, compacted=compacted or prefix.compacted, - history_digest=prefix.history_digest, impossible=True) + history_digest=native_history_digest or prefix.history_digest, impossible=True) return CompactionResult(working, compacted=compacted or prefix.compacted, - history_digest=prefix.history_digest) + history_digest=native_history_digest or prefix.history_digest) non_system = [item for item in working if item.get("role") != "system" and not is_context_digest(item)] tracked_history = [item for item in non_system if id(item) in state.history_message_ids] diff --git a/openkyrozen/agent/completion.py b/openkyrozen/agent/completion.py index 58309eed..533b98d1 100644 --- a/openkyrozen/agent/completion.py +++ b/openkyrozen/agent/completion.py @@ -31,7 +31,7 @@ def _complete_turn(self, turn): for item in submitted[field]) sections.append("\n".join(lines)) turn.final_answer = "Verified delegated findings:\n\n" + "\n\n".join(sections) - final = self._call_llm_with_spinner([{"role": "system", "content": + synthesis_messages = [{"role": "system", "content": "Synthesize the verified delegated results for the user. Include evidence and limitations. " "Return a plain-language findings report only, no Actions, AskUser, PlanProposal, or execution plan. " "The user requested findings; proposed code fixes do not require a plan acceptance. " @@ -40,14 +40,36 @@ def _complete_turn(self, turn): {"role": "user", "content": turn.user_input + "\n" + json.dumps([ {"name": run["name"], "assignment": run["assignment"], "report": run["report"], "reviews": [{key: item for key, item in review.items() if key in {"verdict", "summary", "findings", "evidence"}} - for review in run["reviews"]]} for run in delegated], ensure_ascii=False)}]) + for review in run["reviews"]]} for run in delegated], ensure_ascii=False)}] + if getattr(turn, "native_tools", False): + from openkyrozen.providers.errors import ProviderError, ProviderErrorKind + from openkyrozen.providers.models import FinishReason + try: + response = self._get_model_response( + synthesis_messages, tool_names=set(), history_tool_names=set(), stream=False, + ) + if response.finish_reason == FinishReason.CANCELLED: + raise ProviderError(ProviderErrorKind.CANCELLED, terminal=True) + if (response.finish_reason == FinishReason.FINAL and not response.tool_calls + and response.text.strip() and not response.text.startswith("[LLM Error]")): + turn.final_answer = response.text.strip() + except ProviderError as exc: + if exc.kind == ProviderErrorKind.CANCELLED: + raise + except Exception: + pass + finally: + self._last_response_is_native.set(True) + else: + final = self._call_llm_with_spinner(synthesis_messages) + parsed_final = self._parse_model_response(final) + if (parsed_final["clean"] and not parsed_final["tool_calls"] and not parsed_final["question"] + and not parsed_final["plan_proposal"] and not parsed_final["protocol_error"] + and not parsed_final["unsupported_action_protocol"] and not final.startswith("[LLM Error]")): + turn.final_answer = parsed_final["clean"] turn.turn_prompt_total += self._last_prompt_tokens turn.turn_completion_total += self._last_completion_tokens - parsed_final = self._parse_model_response(final) - if (parsed_final["clean"] and not parsed_final["tool_calls"] and not parsed_final["question"] - and not parsed_final["plan_proposal"] and not parsed_final["protocol_error"] - and not parsed_final["unsupported_action_protocol"] and not final.startswith("[LLM Error]")): - turn.final_answer = parsed_final["clean"] + if turn.final_answer is None: # Fallback if the loop exited without a final answer turn.final_answer = self._deterministic_tool_summary(turn.tool_records) @@ -55,8 +77,11 @@ def _complete_turn(self, turn): turn.final_answer = self._deterministic_tool_summary(turn.tool_records) if turn.protocol_error_message: + retain_answer = delegated or ( + turn.response_text and turn.final_answer == turn.response_text.strip() + ) turn.final_answer = (turn.protocol_error_message + "\n\n" + turn.final_answer - if delegated else turn.protocol_error_message) + if retain_answer else turn.protocol_error_message) turn.final_answer = self._clean_final_response(turn.final_answer) @@ -79,17 +104,84 @@ def _complete_turn(self, turn): turn.final_answer = "\n".join(lines) verified_fix = turn.fix_workflow and turn.fix_workflow.get("stage") in {"verified", "explained"} - failed_tools = [] if verified_fix else [record for record in turn.tool_records if not record.get("success")] - turn.runtime_status = "incomplete" if failed_tools or turn.protocol_error_message else "completed" - if failed_tools: - failures = "\n".join( - f"- {record.get('action', 'Tool')}: operation failed; its result was not verified." - for record in failed_tools + failed_tools = [] + unresolved_effect_failure = False + unresolved_effect_blocked = False + for index, record in enumerate(turn.tool_records): + verification = record.get("verification") or {} + failed_verification = verification.get("status") == "failed" + effect_action = (record.get("action") in {"write_file", "edit_file", "apply_patch"} + or record.get("side_effects") in {"filesystem", "remote", "destructive", "mutation", "external"}) + effect_failure = (effect_action and record.get("success") is False) or ( + failed_verification and (verification.get("method") == "file_content_match" or effect_action)) + if (effect_failure and verification.get("status") == "not_applicable" + and verification.get("method") == "duplicate_success_refusal" + and record.get("operation_id") and any( + earlier.get("success") and (earlier.get("verification") or {}).get("status") == "verified" + and all(earlier.get(key) == record.get(key) for key in ("operation_id", "action", "args", "plan_id")) + and earlier.get("requested_task_id", earlier.get("task_id")) + == record.get("requested_task_id", record.get("task_id")) + for earlier in turn.tool_records[:index])): + # Refusing an already verified identical operation does not negate its effect. + continue + if not effect_failure and (verified_fix or (record.get("success") and not failed_verification)): + continue + resolved = record.get("operation_id") and any( + later.get("success") and (later.get("verification") or {}).get("status") == "verified" + and (not failed_verification + or (later.get("verification") or {}).get("method") == verification.get("method")) + and all(later.get(key) == record.get(key) for key in ("operation_id", "action", "args", "task_id", "plan_id")) + for later in turn.tool_records[index + 1:] ) - turn.final_answer = f"{turn.final_answer.rstrip()}\n\nTool status: incomplete; failed operations were not verified.\n{failures}" + if not resolved: + failed_tools.append(record) + unresolved_effect_failure = unresolved_effect_failure or effect_failure + unresolved_effect_blocked = unresolved_effect_blocked or (effect_failure and ( + (record.get("tool_error") or {}).get("code") in {"permission_denied", "approval_denied"} + or record.get("failure") in {"permission_denied", "approval_denied", "denied"})) + provider_status = getattr(turn, "runtime_status", "completed") + turn.runtime_status = ("incomplete" if failed_tools or turn.protocol_error_message else "completed") + structured_child_response = False + if failed_tools: + failure_lines = [f"- {record.get('action', 'Tool')}: operation failed; its result was not verified." + for record in failed_tools] + if unresolved_effect_failure: + # Known negative effect evidence supersedes model success prose. + # Expected failed reproduction commands retain the existing fix path. + context = self.execution_context + child_report = bool(getattr(context, "child_run_id", None) + and getattr(context, "coordinator", None) is not None) + prior_report, review_report = None, False + if child_report: + from openkyrozen.agent.delegation import REPORT_LISTS, report + try: + prior_report = report(turn.final_answer) + except (ValueError, TypeError): + try: + prior_report, review_report = report(turn.final_answer, review=True), True + except (ValueError, TypeError): + pass + if prior_report and prior_report.get("status") == "blocked": + unresolved_effect_blocked = True + failure_summary = turn.protocol_error_message or ( + "The required operation is blocked; its effects were not verified." + if unresolved_effect_blocked else "The requested effects were not verified.") + if child_report: + if review_report: + failure_report = {"verdict": "unverifiable", "summary": failure_summary, + "findings": failure_lines, "evidence": []} + else: + failure_report = {**{key: [] for key in REPORT_LISTS}, "findings": failure_lines, "summary": failure_summary, + "status": "blocked" if unresolved_effect_blocked else "failed"} + turn.final_answer = json.dumps(failure_report, ensure_ascii=False) + structured_child_response = True + else: + turn.final_answer = failure_summary + if not structured_child_response: + failures = "\n".join(failure_lines) + turn.final_answer = f"{turn.final_answer.rstrip()}\n\nTool status: incomplete; failed operations were not verified.\n{failures}" plan = getattr(self.tasks, "working_plan", None) - if (not self.execution_context.child_run_id - and turn.interaction_mode == "agent" and getattr(turn, "complexity", None) == "complex" + if (turn.interaction_mode == "agent" and getattr(turn, "complexity", None) == "complex" and plan is not None and plan.status in {"active", "completed", "cancelled"}): plan_tasks = [task for task in self.tasks.tasks if task.get("id") in plan.task_ids] succeeded = [task for task in plan_tasks if is_complete(task)] @@ -120,10 +212,27 @@ def _complete_turn(self, turn): status_line = "Plan status: incomplete; required work remains." for task in outstanding: status_line += f"\n- {task['status']}: {task['description']}" - turn.final_answer = f"{turn.final_answer.rstrip()}\n\n{status_line}" + if (not structured_child_response and getattr(self.execution_context, "child_run_id", None) + and getattr(self.execution_context, "coordinator", None) is not None): + from openkyrozen.agent.delegation import report + try: + child_report = report(turn.final_answer) + except (ValueError, TypeError): + pass + else: + if turn.runtime_status != "completed" and child_report["status"] == "completed": + child_report["status"] = "blocked" if turn.runtime_status == "blocked" else ( + "partial" if turn.runtime_status == "incomplete" else "failed") + child_report["uncertainties"].append(status_line) + turn.final_answer = json.dumps(child_report, ensure_ascii=False) + structured_child_response = True + if not structured_child_response: + turn.final_answer = f"{turn.final_answer.rstrip()}\n\n{status_line}" - if turn.protocol_error_message and turn.runtime_status == "completed": + if (turn.protocol_error_message or failed_tools) and turn.runtime_status == "completed": turn.runtime_status = "incomplete" + if provider_status == "blocked" or unresolved_effect_blocked: + turn.runtime_status = "blocked" completion_status = getattr(self, "_turn_completion_status", None) if callable(getattr(completion_status, "set", None)): @@ -134,7 +243,14 @@ def _complete_turn(self, turn): "time": time.time() - turn.turn_start, "tool_calls": len(turn.tool_calls), }) - return self._finish_learning_run( - turn.learning_run, turn.learning_receipts, turn.user_input, turn.final_answer, turn.tool_records, - turn.turn_prompt_total + turn.turn_completion_total, turn.turn_start, - ) + try: + return self._finish_learning_run( + turn.learning_run, turn.learning_receipts, turn.user_input, turn.final_answer, turn.tool_records, + turn.turn_prompt_total + turn.turn_completion_total, turn.turn_start, + ) + except Exception as exc: + try: + self._record_learning_event("learning.bookkeeping_failed", {"reason": type(exc).__name__}) + except Exception: + pass + return turn.final_answer diff --git a/openkyrozen/agent/context.py b/openkyrozen/agent/context.py index 90e917ca..563517df 100644 --- a/openkyrozen/agent/context.py +++ b/openkyrozen/agent/context.py @@ -2,7 +2,7 @@ from typing import Any import json -from openkyrozen.agent.compaction import ContextState, compact_for_pressure, is_context_digest, message_fingerprint, retain_context_digests +from openkyrozen.agent.compaction import ContextState, compact_for_pressure, is_context_digest, message_fingerprint, retain_context_digests, resolve_context_window from openkyrozen.agent.types import ContextTooLargeError @@ -152,9 +152,11 @@ def _load_reported_context_tokens(self, state: ContextState) -> None: def _cache_reported_context_tokens(self, messages: list[dict], prompt_tokens: int) -> None: if prompt_tokens < 1: return + state = self._active_context_state.get() + fingerprint = state.request_fingerprint(messages) if state is not None else message_fingerprint(messages) with self._context_status_lock: counts = self._provider_token_counts_by_scope.setdefault(self._context_scope(), {}) - counts[message_fingerprint(messages)] = int(prompt_tokens) + counts[fingerprint] = int(prompt_tokens) while len(counts) > 32: counts.pop(next(iter(counts))) @@ -208,7 +210,13 @@ def _prepare_context_for_call(self, messages: list[dict], model: str | None = No state = self._active_context_state.get() if state is None or self._in_context_compaction.get(): return + if model is not None and model != state.model: + state.model = state.request_model = model + state.window_tokens, state.window_source = resolve_context_window(model, state.configured_window) self._load_reported_context_tokens(state) + projection = self._context_history_projection_ids.get() + if projection is not None and projection[0] == id(messages): + state.history_message_ids.update(id(messages[index]) for index in projection[1] if index < len(messages)) result = compact_for_pressure( messages, state, lambda text, output_chars: self._summarize_context_with_chat_model(text, output_chars, model or state.model), @@ -221,7 +229,8 @@ def _prepare_context_for_call(self, messages: list[dict], model: str | None = No retained_history = [item for item in self.short_term_memory if id(item) in retained_ids] self.short_term_memory = retain_context_digests([result.history_digest] + retained_history, self.SHORT_TERM_CAP * 2) self.current_session.messages_compacted = True - state.history_message_ids = {id(item) for item in self.short_term_memory} + state.history_message_ids.intersection_update(retained_ids) + state.history_message_ids.update(id(item) for item in self.short_term_memory) self._save_learning_context_digest(result.history_digest) self._store_context_status(state) if result.impossible: @@ -250,7 +259,8 @@ def _recover_context_after_overflow(self, messages: list[dict], model: str | Non retained_history = [item for item in self.short_term_memory if id(item) in retained_ids] self.short_term_memory = retain_context_digests([result.history_digest] + retained_history, self.SHORT_TERM_CAP * 2) self.current_session.messages_compacted = True - state.history_message_ids = {id(item) for item in self.short_term_memory} + state.history_message_ids.intersection_update(retained_ids) + state.history_message_ids.update(id(item) for item in self.short_term_memory) self._save_learning_context_digest(result.history_digest) self._store_context_status(state) return result.compacted and not result.impossible diff --git a/openkyrozen/agent/delegation_runtime.py b/openkyrozen/agent/delegation_runtime.py index f230740c..07769ec5 100644 --- a/openkyrozen/agent/delegation_runtime.py +++ b/openkyrozen/agent/delegation_runtime.py @@ -257,7 +257,22 @@ def _invoke_delegated(self, run, *, review, feedback, coordinator, report_repair if review or self._active_interaction_mode.get() in {"ask", "plan"}: capabilities &= resolve_capabilities("readonly") tools = (allowed_tool_names(adapters.AVAILABLE_TOOLS, ",".join(capabilities)) - set(TOOLS)) if not report_repair else set() + if not report_repair and "read" in capabilities: + with self.use_session(child): + artifact_spec = self.tool_registry.get_spec("read_tool_artifact") + if artifact_spec.source == "builtin" and artifact_spec.capability == "read": + tools.add("read_tool_artifact") + if not review and not report_repair and capabilities & {"read", "write"}: + with self.use_session(child): + plan_spec = self.tool_registry.get_spec("update_plan") + if plan_spec.source == "builtin" and plan_spec.capability == "write": + if "write" not in capabilities: + # This registry belongs only to the child session. Local planning + # does not grant filesystem writes or change the parent catalog. + self.tool_registry.register_spec(plan_spec.replace(capability="read"), replace=True) + tools.add("update_plan") child._delegation_parent = parent + child._delegation_classifier_input = "\n".join((brief["objective"], brief["context"])) child._delegation_task_id = brief.get("task_id") if not review else None profile = replace(profile, metadata={**profile.metadata, "structured": True, "review": review, "display_name": run["reviewer"] if review else run["name"], "assignment": brief, @@ -435,11 +450,55 @@ def _delegation_tool_access(self, action, args): read_only_network = capability == "network" and action in { "search_web", "read_webpage", "analyze_remote_repo", "github_status", "github_read", } - if action not in {"write_file", "edit_file", "apply_patch"} and not read_only_network and ( + parent = getattr(self.current_session, "_delegation_parent", None) + from openkyrozen.tools.manifest import typed_executor + if action == "read_tool_artifact" and (spec is None or spec.source != "builtin" + or capability != "read" or effects not in {"read", "read_only"} + or spec.executor != typed_executor(self._read_tool_artifact)): + raise PermissionError("Child artifact reads require the scoped builtin reader") + local_plan_update = (action == "update_plan" and spec is not None + and spec.source == "builtin" and capability in {"read", "write"} + and effects in {"internal", "internal_write"} + and spec.executor == typed_executor(self._update_plan) + and parent is not None and self.current_session.tasks is not parent.tasks) + if action not in {"write_file", "edit_file", "apply_patch"} and not read_only_network and not local_plan_update and ( capability in {"write", "shell", "git", "browser", "destructive"} or effects not in {"none", "read", "read_only"} ): raise PermissionError(f"Sub-agent capability '{capability}' is not confined to assigned files") + if local_plan_update: + plan_arguments = args + if isinstance(plan_arguments, str): + try: + plan_arguments = json.loads(plan_arguments) + except (TypeError, ValueError) as exc: + raise PermissionError("Child plan arguments must be a structured object") from exc + if not isinstance(plan_arguments, dict): + raise PermissionError("Child plan arguments must be a structured object") + def executions(value): + if isinstance(value, dict): + for key, item in value.items(): + if key == "execution" and isinstance(item, dict): + yield item + else: + yield from executions(item) + elif isinstance(value, list): + for item in value: + yield from executions(item) + + for execution in executions(plan_arguments): + name = self._operation_action(execution.get("action", "")) + arguments = execution.get("arguments") + target = self.tool_registry.specs.get(name) + if (name == "update_plan" or name in TOOLS or target is None + or name not in (context.delegation_tools or ()) + or not context.capability_token.allows(target.capability) + or self.tool_registry._argument_error(target.input_schema, arguments)): + raise PermissionError("Child plan execution exceeds its delegated tool authority") + # Stored execution checkpoints may later reach a durable worker; + # validate their scope now, while the child boundary is present. + with self._delegation_tool_access(name, arguments): + pass run = coordinator.runs[context.child_run_id] brief = run["assignment"] writing = action in {"write_file", "edit_file", "apply_patch"} @@ -482,7 +541,7 @@ def _delegation_tool_access(self, action, args): ) if denial: raise PermissionError(f"Sub-agent operation exceeds the accepted Strategic Plan: {denial}") - if self._is_state_changing_action(action, args): + if not local_plan_update and self._is_state_changing_action(action, args): allowed, reason = parent.tasks.mutation_matches_current_task(action, match_args) if not allowed: raise ValueError("Sub-agent action does not match accepted plan: " + reason) diff --git a/openkyrozen/agent/engine.py b/openkyrozen/agent/engine.py index b41d6a94..8172e769 100644 --- a/openkyrozen/agent/engine.py +++ b/openkyrozen/agent/engine.py @@ -2,12 +2,14 @@ from __future__ import annotations from openkyrozen.agent.turn import TurnContext -from openkyrozen.providers.models import FinishReason, ToolReply +from openkyrozen.providers.models import FinishReason, TextDelta, ToolCallDelta, ToolReply _COMPACT_CORE_TOOLS = frozenset({ "list_dir", "find_files", "search_files", "read_file", "write_file", "edit_file", "apply_patch", - "run_cmd", "discover_tools", "read_tool_artifact", + "run_cmd", "discover_tools", "read_tool_artifact", "update_plan", + "propose_plan", "request_question", }) +_NATIVE_CONTROLS = frozenset({"propose_plan", "request_question"}) class AgentEngine: @@ -26,7 +28,8 @@ def run(self, turn: TurnContext) -> str: except Exception: capabilities = None if (capabilities and capabilities.native_tools - and turn.interaction_mode in {"agent", "ask"}): + and turn.interaction_mode in {"agent", "ask", "plan"}): + turn.native_tools = True self._run_native(turn) return self._finish_turn(turn, runtime._complete_turn(turn)) reply = runtime._initial_turn_response(turn) @@ -53,8 +56,6 @@ def _finish_turn(self, turn: TurnContext, result: str) -> str: def _native_tool_names(self) -> set[str]: runtime = self.runtime permitted = runtime._permitted_tool_names() - if runtime._prompt_profile() != "compact": - return permitted if "discover_tools" not in permitted: return permitted discovered = getattr(runtime.execution_context, "discovered_tools", frozenset()) @@ -63,8 +64,7 @@ def _native_tool_names(self) -> set[str]: def _store_tool_output(self, content): runtime = self.runtime memory = runtime.memory_bank - task_owner = getattr(runtime.current_session, "_delegation_parent", None) or runtime - tasks = getattr(task_owner, "tasks", None) + tasks = getattr(runtime, "tasks", None) return memory.store.artifacts.get_or_create( content, user_id=getattr(tasks, "user_id", None) or memory.user_id, workspace_id=getattr(tasks, "workspace_id", None) or memory.workspace_id, @@ -88,7 +88,13 @@ def _archive_replies(self, messages, indexes): def _run_native(self, turn: TurnContext) -> None: runtime = self.runtime + runtime._last_response_is_native.set(True) from openkyrozen.agent.conversation import model_response_to_dict + from openkyrozen.agent.prompts import _refresh_working_plan_context + from openkyrozen.providers.errors import ProviderError, ProviderErrorKind + provider = runtime.execution_context.provider or runtime.llm_provider + capabilities = provider.get_capabilities(runtime.DEEPSEEK_MODEL) + stream = capabilities.text_streaming or capabilities.streaming_tool_calls messages = runtime._build_messages( turn.user_input, turn.learned_context, turn.memory_context, memory_packet=turn.memory_packet, native_tools=True, @@ -105,22 +111,32 @@ def _run_native(self, turn: TurnContext) -> None: operation_scope = str(turn.learning_run.get("run_id")) final_text = "" response = None - history_tool_names: set[str] = set() + history_tool_names: set[str] = {item.name for item in messages if isinstance(item, ToolReply)} observed_plan_id = None previous_reply_indexes: list[int] = [] + runtime.execution_context.native_control_input = turn.user_input + runtime.execution_context.native_plan_inspected = False + control_failures = 0 max_steps = getattr(runtime.execution_context, "max_steps", None) if max_steps is None: max_steps = runtime.MAX_STEPS_PER_TURN for step in range(max_steps + 1): tool_names = self._native_tool_names() + if turn.interaction_mode == "plan": + tool_names = {name for name in tool_names + if name in _NATIVE_CONTROLS or ( + runtime.tool_registry.specs.get(name).capability in {"read", "network"} + and runtime.tool_registry.specs.get(name).side_effects in {"read_only", "read", "none", "remote"} + and runtime.tool_registry.specs.get(name).risk == "normal")} + if not runtime.execution_context.native_plan_inspected: + tool_names.discard("propose_plan") plan = getattr(runtime.tasks, "working_plan", None) if turn.planning_required else None if plan is not None and plan.status == "active": observed_plan_id = plan.id completed_plan = (observed_plan_id is not None and plan is not None and plan.id == observed_plan_id and plan.status == "completed") - needs_plan = (turn.planning_required and not getattr(runtime.execution_context, "child_run_id", None) - and (plan is None or plan.status != "active")) + needs_plan = turn.planning_required and (plan is None or plan.status != "active") if needs_plan: if completed_plan: tool_names = runtime._permitted_tool_names() & {"update_plan"} @@ -129,17 +145,40 @@ def _run_native(self, turn: TurnContext) -> None: break else: tool_names = {"update_plan"} - elif turn.planning_required and not getattr(runtime.execution_context, "child_run_id", None): + elif turn.planning_required: tool_names |= runtime._permitted_tool_names() & {"update_plan"} + if "request_question" in runtime._permitted_tool_names(): + tool_names.add("request_question") + previous_replies = [messages[index] for index in previous_reply_indexes] + _refresh_working_plan_context(runtime, messages) + emitted_native_content = False + + def observe_native_content(event): + nonlocal emitted_native_content + if isinstance(event, (TextDelta, ToolCallDelta)): + emitted_native_content = True + + def request_response(): + return runtime._get_model_response( + messages, tool_names=tool_names, history_tool_names=history_tool_names, stream=stream, + on_event=observe_native_content if stream else None, + ) + token = runtime._record_foreground_model.set(False) try: try: - response = runtime._get_model_response( - messages, tool_names=tool_names, history_tool_names=history_tool_names, stream=False, - ) + try: + response = request_response() + except ProviderError as exc: + if (exc.kind != ProviderErrorKind.CONTEXT_OVERFLOW or exc.terminal + or emitted_native_content + or not runtime._recover_context_after_overflow(messages, runtime.DEEPSEEK_MODEL)): + raise + response = request_response() except Exception as exc: + if isinstance(exc, ProviderError) and exc.kind == ProviderErrorKind.CANCELLED: + raise runtime._turn_failed.set(True) - from openkyrozen.providers.errors import ProviderError detail = str(exc) if isinstance(exc, ProviderError) else type(exc).__name__ turn.protocol_error_message = f"Native model request failed: {detail}; no further tools were run." if runtime._conversation_turn_id.get() is not None: @@ -149,11 +188,30 @@ def _run_native(self, turn: TurnContext) -> None: break finally: runtime._record_foreground_model.reset(token) + previous_reply_indexes = [index for index, item in enumerate(messages) + if any(item is reply for reply in previous_replies)] turn.turn_prompt_total += runtime._last_prompt_tokens turn.turn_completion_total += runtime._last_completion_tokens turn.response_text = response.text runtime._last_response_is_native.set(True) + if response.finish_reason == FinishReason.CANCELLED: + raise ProviderError(ProviderErrorKind.CANCELLED, terminal=True) + if response.finish_reason in {FinishReason.ERROR, FinishReason.BLOCKED, FinishReason.UNKNOWN}: + final_text = response.text.strip() + blocked = response.finish_reason == FinishReason.BLOCKED + turn.runtime_status = "blocked" if blocked else "incomplete" + turn.protocol_error_message = ("The provider blocked this response; no further tools were run." + if blocked else + "The provider returned an unsuccessful response; no further tools were run.") + if not blocked: + runtime._turn_failed.set(True) + break + if response.tool_calls and response.finish_reason == FinishReason.LENGTH: + turn.protocol_error_message = "The provider truncated its tool request; no tools from that response were run." + final_text = response.text.strip() + break + unknown_tools = [call.name for call in response.tool_calls if call.name not in tool_names] if unknown_tools: turn.protocol_error_message = ("A Working Plan is required before other tools can run." @@ -164,6 +222,13 @@ def _run_native(self, turn: TurnContext) -> None: "error_type": "invalid_native_tool_call", }, request_id=runtime._last_model_request_id.get()) break + if (any(call.name == "update_plan" for call in response.tool_calls) + and any(call.name != "update_plan" for call in response.tool_calls)): + turn.protocol_error_message = "Run Working Plan edits in a separate response before using their task selection." + break + if any(call.name in _NATIVE_CONTROLS for call in response.tool_calls) and len(response.tool_calls) != 1: + turn.protocol_error_message = "Request an interactive control alone in its own response; no tools were run." + break if response.tool_calls and step >= max_steps: turn.protocol_error_message = "The tool-call step limit was reached; remaining work was not run." if runtime._conversation_turn_id.get() is not None: @@ -178,6 +243,17 @@ def _run_native(self, turn: TurnContext) -> None: if not response.tool_calls: final_text = response.text.strip() + if turn.interaction_mode == "plan": + final_text = "" + control_failures += 1 + if control_failures < 3 and step < max_steps: + messages.extend((response, {"role": "user", "content": ( + "Use read-only inspection first, then propose_plan for the structured proposal; " + "use request_question if clarification is needed. Do not execute proposed changes." + )})) + continue + turn.protocol_error_message = "A valid Strategic Plan proposal was not prepared; no proposed changes were executed." + break if needs_plan and not completed_plan: final_text = "" if step < max_steps: @@ -196,6 +272,11 @@ def _run_native(self, turn: TurnContext) -> None: "or provide a concise final answer." )})) continue + if not final_text or response.finish_reason == FinishReason.LENGTH: + turn.protocol_error_message = ("The provider response reached its length limit; the answer is incomplete." + if response.finish_reason == FinishReason.LENGTH else + "The provider returned no answer after bounded continuation.") + break if not final_text.startswith("[LLM Error]"): break turn.protocol_error_message = final_text @@ -203,9 +284,14 @@ def _run_native(self, turn: TurnContext) -> None: messages.append(response) current_reply_indexes: list[int] = [] history_tool_names.update(call.name for call in response.tool_calls) + selected_task_id = getattr(runtime.tasks, "execution_task_id", None) calls = [{"action": call.name, "args": call.arguments, "call_id": call.id, "call_id_source": "native", - "task_id": getattr(getattr(runtime, "current_session", None), "_delegation_task_id", None)} + "task_id": (selected_task_id or getattr(getattr(runtime, "current_session", None), + "_delegation_task_id", None)) + if call.name not in _NATIVE_CONTROLS | {"update_plan"} else None, + "task_plan_id": getattr(getattr(runtime.tasks, "working_plan", None), "id", None) + if selected_task_id and call.name not in _NATIVE_CONTROLS | {"update_plan"} else None} for call in response.tool_calls] turn.tool_calls.extend(calls) @@ -219,6 +305,16 @@ def record_native_call(call, receipt): outcome = receipt.tool_result or ToolResult(data=receipt.result_data, text=receipt.result) reply_data = tool_result_to_dict(outcome) messages.append(ToolReply(call["call_id"], call["action"], reply_data, not outcome.success)) + if record.get("success"): + if call["action"] in _NATIVE_CONTROLS: + if not isinstance(outcome.data, dict) or outcome.data.get("requires_approval", True): + turn.pending_interaction_reply = outcome.text + else: + spec = runtime.tool_registry.specs.get(call["action"]) + if (spec and spec.capability in {"read", "network"} + and spec.side_effects in {"read_only", "read", "none", "remote"} + and call["action"] != "discover_tools"): + runtime.execution_context.native_plan_inspected = True current_reply_indexes.append(len(messages) - 1) turn.tool_results_text += f"- `{call['action']}` returned:\n{runtime._safe_fstring(prompt_result)}\n" runtime._update_tasks_panel() @@ -227,6 +323,15 @@ def record_native_call(call, receipt): calls, operation_scope=operation_scope, successful_operations=successful_operations, call_id_source="native", on_receipt=record_native_call, ) + if turn.pending_interaction_reply: + final_text = turn.pending_interaction_reply + break + if (any(call["action"] in _NATIVE_CONTROLS for call in calls) + and not turn.tool_records[-1].get("success")): + control_failures += 1 + if control_failures >= 3: + turn.protocol_error_message = "Interactive control validation failed after bounded retries; no proposed changes were executed." + break try: self._archive_replies(messages, previous_reply_indexes) except Exception: @@ -234,4 +339,5 @@ def record_native_call(call, receipt): break previous_reply_indexes = current_reply_indexes + runtime._last_response_is_native.set(True) turn.final_answer = final_text or runtime._deterministic_tool_summary(turn.tool_records) diff --git a/openkyrozen/agent/executor.py b/openkyrozen/agent/executor.py index dd9840fb..b88d6be3 100644 --- a/openkyrozen/agent/executor.py +++ b/openkyrozen/agent/executor.py @@ -131,6 +131,25 @@ def _awaiting_revision_mutation(state, action, args, spec, runtime) -> bool: )) +def _builtin_interaction_control(runtime, action, spec) -> bool: + from openkyrozen.agent.plan_tools import _propose_plan, _request_question + from openkyrozen.tools.catalog import builtin_metadata + + handlers = {"propose_plan": _propose_plan, "request_question": _request_question} + if action not in handlers or spec is None or spec.source != "builtin" or spec.name != action: + return False + handler = getattr(handlers[action], "__tool_result_executor__", handlers[action]) + if (getattr(spec.executor, "__self__", None) is not runtime + or getattr(spec.executor, "__func__", None) is not handler): + return False + metadata = builtin_metadata(action) + return (spec.input_schema == metadata["input_schema"] and spec.version == "1.0.0" + and all(getattr(spec, field) == metadata[field] for field in ( + "capability", "side_effects", "risk", "parallel_safe", "aliases", + "legacy_fields", "legacy_argument_aliases", "confirmation_surfaces", + ))) + + def _notify_tool_execute(self, action: str, args: Any, result: ToolResult) -> None: """Emit the typed outcome to observers; display text cannot change status.""" result = result if isinstance(result, ToolResult) else tool_failure( @@ -203,13 +222,15 @@ def _make_execution_receipt(self, *, action: str, args: Any, authorized: bool, s tool_error: ToolError | None = None, result_data: Any = None, tool_result: ToolResult | None = None, call_id: str | None = None, call_id_source: str = "legacy", task_id: str | None = None, + task_plan_id: str | None = None, receipt_arguments: dict[str, Any] | None = None, + _duplicate_successful_operation: bool = False, expected_spec=None) -> ExecutionReceipt: canonical = self._operation_action(action) from openkyrozen.agent.delegation import TOOLS current_session = getattr(self, "current_session", None) delegated_parent = getattr(current_session, "_delegation_parent", None) - task_owner = delegated_parent or self + task_owner = self if task_plan_id is not None or canonical == "update_plan" else delegated_parent or self task_manager = getattr(task_owner, "tasks", None) result_limit = (max(2000, len(str(result))) if canonical in TOOLS else 8000 if canonical == "read_tool_artifact" else @@ -235,9 +256,10 @@ def _make_execution_receipt(self, *, action: str, args: Any, authorized: bool, s if len(full_output) > _TOOL_ARTIFACT_THRESHOLD: memory = getattr(self, "memory_bank", None) store = getattr(memory, "store", None) - user_id = getattr(task_manager, "user_id", None) or getattr(memory, "user_id", None) - workspace_id = getattr(task_manager, "workspace_id", None) or getattr(memory, "workspace_id", None) - session_id = getattr(task_manager, "session_id", None) or getattr(memory, "session_id", None) + artifact_owner = getattr(current_session, "tasks", None) or task_manager + user_id = getattr(artifact_owner, "user_id", None) or getattr(memory, "user_id", None) + workspace_id = getattr(artifact_owner, "workspace_id", None) or getattr(memory, "workspace_id", None) + session_id = getattr(artifact_owner, "session_id", None) or getattr(memory, "session_id", None) artifacts = getattr(store, "artifacts", None) if artifacts is None or not all((user_id, workspace_id, session_id)): raise RuntimeError("Cannot retain oversized tool output without a session-scoped artifact store") @@ -279,6 +301,8 @@ def _make_execution_receipt(self, *, action: str, args: Any, authorized: bool, s } elif getattr(expected_spec, "side_effects", None) == "read_only": verification = {"status": "not_applicable", "method": "read_only_tool", "observed": None} + if _duplicate_successful_operation: + verification = {"status": "not_applicable", "method": "duplicate_success_refusal", "observed": None} def safe_metadata(value, budget=4000): def clean(item, remaining, depth=0): @@ -342,7 +366,7 @@ def clean(item, remaining, depth=0): session_id=(getattr(current_session, "session_id", None) if delegated_parent else None) or getattr(task_manager, "session_id", None) or getattr(getattr(self, "memory_bank", None), "session_id", None), - plan_id=getattr(getattr(task_manager, "working_plan", None), "id", None), + plan_id=task_plan_id or getattr(getattr(task_manager, "working_plan", None), "id", None), task_id=task_id, tool_version=getattr(expected_spec, "version", None), tool_source=getattr(expected_spec, "source", None), @@ -371,6 +395,7 @@ def _run_tool(self, action: str, args: str, *, return_success: bool = False, _typed_arguments: dict | None = None, _expected_spec=None, _call_id: str | None = None, _call_id_source: str = "legacy", _task_id: str | None = None, _pre_execution_check=None, + _task_plan_id: str | None = None, _on_dispatch=None) -> str | tuple[str, bool] | ExecutionReceipt: started_at = utc_now() action = self.TOOL_ALIASES.get(action, action) @@ -427,7 +452,7 @@ def finish(outcome: ToolResult, authorized: bool = False, raw_result: Any = None failure=outcome.error.code if outcome.error else None, raw_result=raw_result, tool_error=outcome.error, result_data=outcome.data, tool_result=outcome, expected_spec=approved_spec, call_id=_call_id, call_id_source=_call_id_source, - task_id=_task_id, + task_id=_task_id, task_plan_id=_task_plan_id, ) return (text, outcome.success) if return_success else text if action not in self.AVAILABLE_TOOLS: @@ -440,7 +465,8 @@ def finish(outcome: ToolResult, authorized: bool = False, raw_result: Any = None outcome = tool_failure("permission_denied", "Error: strategic plan revision is awaiting approval; mutation is paused.") self._notify_tool_execute(action, args, outcome) return finish(outcome) - if executing_plan and action != "cancel_subagent": + if (executing_plan and action != "cancel_subagent" + and not _builtin_interaction_control(self, action, approved_spec)): denial = _strategic_plan_denial( executing_plan, action, args, self.tool_registry.specs.get(action), self._get_workspace_root(), ) @@ -498,7 +524,8 @@ def finish(outcome: ToolResult, authorized: bool = False, raw_result: Any = None and request.session_id == self.current_session.session_id and self._execution_capability_token.allows(current_spec.capability) ) - if authorized and executing_plan and action != "cancel_subagent": + if (authorized and executing_plan and action != "cancel_subagent" + and not _builtin_interaction_control(self, action, self.tool_registry.specs.get(action))): current_state = self._interaction_controller.state() current_plan = current_state.get("executing_plan") if (not current_plan or current_plan.get("plan_id") != executing_plan.get("plan_id") @@ -591,9 +618,11 @@ def finish(outcome: ToolResult, authorized: bool = False, raw_result: Any = None def _execute_turn_action(self, action: str, args: Any, *, operation_scope: str, successful_operations: set[str], call_id: str | None = None, - call_id_source: str = "synthetic", task_id: str | None = None) -> ExecutionReceipt: + call_id_source: str = "synthetic", task_id: str | None = None, + task_plan_id: str | None = None) -> ExecutionReceipt: """Execute one model action, refusing a duplicate successful mutation.""" - task_id = task_id or getattr(getattr(self, "current_session", None), "_delegation_task_id", None) + task_id = (task_id or getattr(getattr(self, "current_session", None), "_delegation_task_id", None) + if action != "update_plan" else None) canonical = self._operation_action(action) started_at = utc_now() call_id = call_id or f"synthetic_{uuid.uuid4().hex}" @@ -602,9 +631,22 @@ def emit_rejected_call(): self._publish_runtime_event( "tool.call", {"tool_name": canonical, "argument_keys": sorted(args)[:64] if isinstance(args, dict) else []}, - call_id=call_id, + call_id=call_id, task_id=task_id, ) + def local_binding_valid(): + return (getattr(getattr(self.tasks, "working_plan", None), "id", None) == task_plan_id + and self.tasks.execution_task_id == task_id) + + if task_plan_id is not None and not local_binding_valid(): + emit_rejected_call() + outcome = tool_failure("invalid_arguments", "Error: explicit task selection is no longer active in this Working Plan.") + return self._make_execution_receipt( + action=canonical, args=args, authorized=False, started_at=started_at, success=False, + result=outcome.text, operation_scope=operation_scope, tool_result=outcome, + call_id=call_id, call_id_source=call_id_source, task_id=task_id, + task_plan_id=task_plan_id, expected_spec=expected_spec, + ) native_arguments = isinstance(args, dict) and call_id_source == "native" if isinstance(args, dict) and not native_arguments: emit_rejected_call() @@ -620,7 +662,7 @@ def emit_rejected_call(): action=canonical, args=args, authorized=False, started_at=started_at, success=False, result=result, operation_scope=operation_scope, failure=outcome.error.code, tool_error=outcome.error, tool_result=outcome, call_id=call_id, call_id_source=call_id_source, - expected_spec=expected_spec, + task_id=task_id, task_plan_id=task_plan_id, expected_spec=expected_spec, ) structured = args if native_arguments else None if not native_arguments and isinstance(args, str) and args.lstrip().startswith("{"): @@ -646,7 +688,7 @@ def emit_rejected_call(): action=canonical, args=args, authorized=False, started_at=started_at, success=False, result=result, operation_scope=operation_scope, failure=outcome.error.code, tool_error=outcome.error, tool_result=outcome, call_id=call_id, - call_id_source=call_id_source, expected_spec=expected_spec, + call_id_source=call_id_source, task_id=task_id, task_plan_id=task_plan_id, expected_spec=expected_spec, ) typed_arguments = args elif spec is not None and spec.source not in {"builtin", "legacy"}: @@ -661,7 +703,7 @@ def emit_rejected_call(): action=canonical, args=args, authorized=False, started_at=started_at, success=False, result=result, operation_scope=operation_scope, failure=outcome.error.code, tool_error=outcome.error, tool_result=outcome, call_id=call_id, - call_id_source=call_id_source, expected_spec=expected_spec, + call_id_source=call_id_source, task_id=task_id, task_plan_id=task_plan_id, expected_spec=expected_spec, ) typed_arguments = structured if (not native_arguments and interaction_state.get("executing_plan") and spec @@ -693,7 +735,7 @@ def emit_rejected_call(): action=canonical, args=args, authorized=False, started_at=started_at, success=False, result=result, operation_scope=operation_scope, failure=outcome.error.code, tool_error=outcome.error, tool_result=outcome, call_id=call_id, - call_id_source=call_id_source, expected_spec=expected_spec, + call_id_source=call_id_source, task_id=task_id, task_plan_id=task_plan_id, expected_spec=expected_spec, ) active_plan = interaction_state.get("executing_plan") if is_mutation and active_plan and canonical not in {"cancel_subagent", "update_plan"}: @@ -740,12 +782,15 @@ def emit_rejected_call(): authorized=True, started_at=started_at, success=False, result=result, operation_scope=operation_scope, failure=outcome.error.code, tool_error=outcome.error, tool_result=outcome, call_id=call_id, call_id_source=call_id_source, - expected_spec=expected_spec, + task_id=task_id, task_plan_id=task_plan_id, expected_spec=expected_spec, + _duplicate_successful_operation=True, ) receipt = self._run_tool(canonical, args, return_receipt=True, operation_scope=operation_scope, _typed_arguments=typed_arguments, _expected_spec=spec if typed_arguments is not None else None, - _call_id=call_id, _call_id_source=call_id_source, _task_id=task_id) + _call_id=call_id, _call_id_source=call_id_source, _task_id=task_id, + _task_plan_id=task_plan_id, + _pre_execution_check=local_binding_valid if task_plan_id is not None else None) assert isinstance(receipt, ExecutionReceipt) if (receipt.success and receipt.action != "run_cmd" and self._is_state_changing_action(receipt.action, args)): @@ -769,6 +814,7 @@ def execute(call): call.get("action", ""), call.get("args", ""), operation_scope=operation_scope, successful_operations=successful_operations, call_id=call.get("call_id"), call_id_source=call.get("call_id_source", call_id_source), task_id=call.get("task_id"), + task_plan_id=call.get("task_plan_id"), ) index = 0 @@ -798,16 +844,27 @@ def _record_turn_receipt(self, receipt: ExecutionReceipt, *, allow_legacy_task_m from openkyrozen.agent.delegation import TOOLS current_session = getattr(self, "current_session", None) delegated_parent = getattr(current_session, "_delegation_parent", None) - task_manager = delegated_parent.tasks if delegated_parent else self.tasks + local_plan = getattr(self.tasks, "working_plan", None) + local_scope = bool(local_plan and receipt.plan_id == local_plan.id + and receipt.user_id == self.tasks.user_id + and receipt.workspace_id == self.tasks.workspace_id + and receipt.session_id == self.tasks.session_id) + task_manager = self.tasks if local_scope or not delegated_parent else delegated_parent.tasks + reconcile_locally = not delegated_parent or local_scope active_task = task_manager.active_working_task(receipt.task_id) if receipt.task_id else None - explicit_task_id = receipt.task_id if active_task is not None and ( - not delegated_parent or self._valid_receipt_task_binding(current_session, receipt.task_id) + binding_scope = (getattr(getattr(task_manager, "working_plan", None), "id", None) == receipt.plan_id + and receipt.user_id == task_manager.user_id + and receipt.workspace_id == task_manager.workspace_id) + explicit_task_id = receipt.task_id if active_task is not None and binding_scope and ( + reconcile_locally or self._valid_receipt_task_binding(current_session, receipt.task_id) ) else None receipt_projection = receipt.as_dict() task_acceptance = receipt.acceptance + if receipt.verification.get("status") == "failed": + task_acceptance = None if explicit_task_id and receipt.action == "run_cmd" and task_acceptance: command = (receipt.arguments or {}).get("command", "") - criteria = next((item.get("acceptance", []) for item in self.tasks.tasks + criteria = next((item.get("acceptance", []) for item in task_manager.tasks if item.get("id") == explicit_task_id), []) declared = [" ".join(item.split()).casefold() for item in criteria if isinstance(item, str)] if not command or len(declared) != 1 or " ".join(command.split()).casefold() != declared[0]: @@ -820,17 +877,17 @@ def _record_turn_receipt(self, receipt: ExecutionReceipt, *, allow_legacy_task_m ) legacy_contract = any( task.get("status") in {"pending", "running"} - and self.tasks._dependencies_succeeded(task, self.tasks.tasks) + and task_manager._dependencies_succeeded(task, task_manager.tasks) and (task.get("checkpoint") or {}).get("action") == receipt.action and isinstance((task.get("checkpoint") or {}).get("args"), str) and not task["checkpoint"]["args"].lstrip().startswith("{") - for task in self.tasks.tasks + for task in task_manager.tasks ) matching_args = (receipt.args if legacy_contract else json.dumps(receipt.arguments, ensure_ascii=False, allow_nan=False, sort_keys=True, separators=(",", ":")) if receipt.arguments is not None else receipt.args) - evidence = {} if delegated_parent or receipt.action in TOOLS or not should_reconcile else self.tasks.record_evidence( + evidence = {} if not reconcile_locally or receipt.action in TOOLS or not should_reconcile else task_manager.record_evidence( task_id=explicit_task_id, action=receipt.action, args=matching_args, args_projection=receipt_projection["args"], args_fingerprint=None if legacy_contract else receipt.args_sha256, @@ -840,24 +897,24 @@ def _record_turn_receipt(self, receipt: ExecutionReceipt, *, allow_legacy_task_m verification_result=(receipt.tool_result.text if receipt.action == "read_file" and receipt.tool_result is not None else None), ) - task_id = explicit_task_id if delegated_parent else evidence.get("task_id") + task_id = evidence.get("task_id") if reconcile_locally else explicit_task_id effective_acceptance = evidence.get("acceptance") or ( receipt_projection["acceptance"] if receipt.task_id is None else None ) - if task_id: - task = next((item for item in self.tasks.tasks if item["id"] == task_id), None) + if task_id and reconcile_locally: + task = next((item for item in task_manager.tasks if item["id"] == task_id), None) checkpoint = (task or {}).get("checkpoint") or {} if task and not checkpoint.get("action") and not checkpoint.get("strategic_plan"): - self.tasks.update_checkpoint(task_id, {"action": receipt.action, "args": receipt_projection["args"]}) - if task_id and receipt.success and effective_acceptance: - task = next((item for item in self.tasks.tasks if item["id"] == task_id), None) + task_manager.update_checkpoint(task_id, {"action": receipt.action, "args": receipt_projection["args"]}) + if task_id and reconcile_locally and receipt.success and effective_acceptance: + task = next((item for item in task_manager.tasks if item["id"] == task_id), None) strategic = (task or {}).get("checkpoint", {}).get("strategic_plan", {}) required = set(strategic.get("acceptance_ids", [])) covered = {criterion_id for item in (task or {}).get("evidence", []) for criterion_id in item.get("acceptance_ids", [])} if not required or required <= covered: - index = next(index for index, item in enumerate(self.tasks.tasks) if item["id"] == task_id) - self.tasks.set_status(index, "succeeded") + index = next(index for index, item in enumerate(task_manager.tasks) if item["id"] == task_id) + task_manager.set_status(index, "succeeded") receipt_payload = receipt_projection if receipt.task_id is not None and receipt.task_id != task_id: receipt_payload["requested_task_id"] = receipt.task_id @@ -877,6 +934,7 @@ def _record_turn_receipt(self, receipt: ExecutionReceipt, *, allow_legacy_task_m "acceptance": effective_acceptance, "failure": receipt_projection["failure"], "tool_error": receipt_projection["tool_error"], "task_id": task_id, "requested_task_id": receipt.task_id, + "plan_id": receipt.plan_id, "side_effects": receipt.side_effects, "result_data": receipt_projection["result_data"], "verification": receipt_projection["verification"], } if evidence.get("unmatched_reason"): diff --git a/openkyrozen/agent/legacy_compat.py b/openkyrozen/agent/legacy_compat.py new file mode 100644 index 00000000..99128038 --- /dev/null +++ b/openkyrozen/agent/legacy_compat.py @@ -0,0 +1,55 @@ +"""Lazy bindings for the explicitly supported legacy text helper methods.""" +from __future__ import annotations + +from openkyrozen.agent.presentation import _NativeResponseText, is_native_response, native_response_metadata + + +def _legacy_method(name): + def call(self, *args, **kwargs): + text = args[0] if args else kwargs.get("text") + if isinstance(text, _NativeResponseText): + if name in {"_collect_tool_calls", "_extract_json_objects"}: + return [] + if name == "_collect_unwrapped_tool_calls": + return [], False + if name in {"parse_json_from_response", "_detect_unknown_action", "_legacy_plan_value"}: + return None + if name in {"_has_unsupported_action_protocol", "_stream_buffer_has_tool_prefix"}: + return False + if name == "_clean_stream_buffer": + return text + from openkyrozen.agent import protocol + return getattr(protocol, name)(self, *args, **kwargs) + call.__name__ = name + return call + + +def _parse_model_response(self, text): + if is_native_response(self, text): + return native_response_metadata(text) + return self.legacy_protocol.parse_response(text) + + +def _observe_model_response(self, text): + if is_native_response(self, text): + return native_response_metadata(text) + from openkyrozen.agent.protocol import _observe_model_response as observe + return observe(self, text) + + +_is_valid_action = _legacy_method('_is_valid_action') +_detect_unknown_action = _legacy_method('_detect_unknown_action') +_has_unsupported_action_protocol = _legacy_method('_has_unsupported_action_protocol') +parse_json_from_response = _legacy_method('parse_json_from_response') +_extract_json_objects = _legacy_method('_extract_json_objects') +_collect_unwrapped_tool_calls = _legacy_method('_collect_unwrapped_tool_calls') +_collect_tool_calls = _legacy_method('_collect_tool_calls') +_marker_line_prefix = _legacy_method('_marker_line_prefix') +_action_marker_is_protocol = _legacy_method('_action_marker_is_protocol') +_control_marker_is_protocol = _legacy_method('_control_marker_is_protocol') +_remove_protocol_headings = _legacy_method('_remove_protocol_headings') +_stream_buffer_has_tool_prefix = _legacy_method('_stream_buffer_has_tool_prefix') +_clean_stream_buffer = _legacy_method('_clean_stream_buffer') +_legacy_plan_value = _legacy_method('_legacy_plan_value') +_persist_interaction_control = _legacy_method('_persist_interaction_control') +_interaction_gate = _legacy_method('_interaction_gate') diff --git a/openkyrozen/agent/loop.py b/openkyrozen/agent/loop.py index 9c4ee364..a06403ad 100644 --- a/openkyrozen/agent/loop.py +++ b/openkyrozen/agent/loop.py @@ -9,6 +9,83 @@ from .turn import TurnContext, scoped_turn +def _cancelled_turn_cleanup(self, active_plan, exc, conversation, plugin_runtime, context): + """Attempt existing cancellation/CAS writes within one bounded DB budget.""" + incomplete = False + uncertainty = { + "reason": "cancelled_during_execution", "turn_id": self._conversation_turn_id.get(), + "message": "Remote or already-dispatched side effects may still complete.", "durable": True, + } + with self.memory_bank.store._cancellation_deadline(): + if active_plan: + # Safe narrowing advances the Strategic Plan version while + # acceptance anchors and their children retain their original + # immutable binding. Follow those current anchors, not the + # latest proposal fingerprint, when cancelling their work. + anchor_ids = set(active_plan.get("working_task_ids", [])) + anchors = {task["id"]: (task.get("checkpoint") or {}).get("strategic_plan") or {} + for task in self.tasks.tasks + if task["id"] in anchor_ids + and task.get("plan_id") == active_plan.get("working_plan_id") + and ((task.get("checkpoint") or {}).get("strategic_plan") or {}).get("plan_id") + == active_plan.get("plan_id")} + cancelled_ids = set(anchors) + for task in self.tasks.tasks: + checkpoint = task.get("checkpoint") or {} + binding = checkpoint.get("strategic_plan") or {} + anchor_binding = anchors.get(checkpoint.get("strategy_anchor")) + if (task.get("plan_id") == active_plan.get("working_plan_id") + and anchor_binding is not None + and binding == anchor_binding): + cancelled_ids.add(task["id"]) + for task_id in cancelled_ids: + index = next((i for i, task in enumerate(self.tasks.tasks) + if task.get("id") == task_id), None) + if index is not None and self.tasks.tasks[index].get("status") not in {"succeeded", "cancelled"}: + try: + if not self.tasks.set_status(index, "cancelled"): + incomplete = True + except Exception: + incomplete = True + if not incomplete: + try: + self._interaction_controller.complete_plan( + active_plan, status="cancelled", task_manager=self.tasks, + ) + except Exception: + incomplete = True + try: + self.memory_bank.store._append_control_audit("turn.effect_uncertain", uncertainty, + user_id=self.memory_bank.user_id, workspace_id=self.tasks.workspace_id, + session_id=self.current_session.session_id) + except Exception: + incomplete = True + uncertainty["durable"] = False + try: + self._record_conversation_entry("turn.outcome", { + "status": "cancelled", "error_type": type(exc).__name__, + }) + except Exception: + incomplete = True + try: + from openkyrozen.agent.compaction import is_context_digest, retain_context_digests + display = conversation.display_messages() + limit = self.SHORT_TERM_CAP * 2 + digest = next((item for item in reversed(self.short_term_memory) if is_context_digest(item)), None) + projected = ([digest] if digest else []) + display + self.short_term_memory = retain_context_digests(projected, limit) + self.current_session.messages_compacted = bool(digest) or len(projected) > limit + except Exception: + incomplete = True + try: + plugin_runtime.turn_end(reply="", success=False, error=type(exc).__name__, **context) + except Exception: + incomplete = True + if incomplete: + self._publish_runtime_event("turn.audit_failed", {"reason": "cancellation_cleanup_incomplete"}) + self._publish_runtime_event("turn.effect_uncertain", uncertainty) + + def _chat_turn_impl(self, user_input: str, clear_tasks: bool = False, profile: str | None = None, memory_context: dict[str, Any] | None = None, inspection_complete: bool = False, @@ -144,23 +221,16 @@ def _chat_turn(self, user_input: str, clear_tasks: bool = False, profile: str | and self.execution_context.cancelled.is_set()) or isinstance( exc, (KeyboardInterrupt, asyncio.CancelledError) ) or (isinstance(exc, ProviderError) and exc.kind == ProviderErrorKind.CANCELLED) - if cancelled and active_plan: - for task_id in active_plan.get("working_task_ids", []): - index = next((i for i, task in enumerate(self.tasks.tasks) - if task.get("id") == task_id), None) - if index is not None and self.tasks.tasks[index].get("status") not in {"succeeded", "cancelled"}: - self.tasks.set_status(index, "cancelled") + if cancelled: try: - self._interaction_controller.complete_plan( - active_plan, status="cancelled", task_manager=self.tasks, - ) + _cancelled_turn_cleanup(self, active_plan, exc, conversation, runtime, context) except Exception: - pass - if cancelled: - self._publish_runtime_event("turn.effect_uncertain", { - "reason": "cancelled_during_execution", - "message": "Remote or already-dispatched side effects may still complete.", - }) + # A secondary bookkeeping error cannot replace cancellation. + self._publish_runtime_event("turn.audit_failed", {"reason": "cancellation_cleanup_failed"}) + self._publish_runtime_event("turn.effect_uncertain", { + "reason": "cancelled_during_execution", "durable": False, + }) + raise self._record_conversation_entry("turn.outcome", { "status": "cancelled" if cancelled else "failed", "error_type": type(exc).__name__, }) diff --git a/openkyrozen/agent/models.py b/openkyrozen/agent/models.py index bef7ceb2..418ce307 100644 --- a/openkyrozen/agent/models.py +++ b/openkyrozen/agent/models.py @@ -39,3 +39,4 @@ def __post_init__(self): self.process_manager = self.workspace.adapters.new_process_manager( self.memory.user_id, self.workspace.root, self.session_id, ) + self.process_manager._bind_audit(self.memory.store, self.tasks.workspace_id) diff --git a/openkyrozen/agent/modes.py b/openkyrozen/agent/modes.py index c2c6d01f..f5b2037e 100644 --- a/openkyrozen/agent/modes.py +++ b/openkyrozen/agent/modes.py @@ -3,15 +3,14 @@ from __future__ import annotations import json -import hashlib -import posixpath import re import shlex import uuid -from copy import deepcopy from typing import Any -from .ports import InteractionStore +from openkyrozen.tasks.planning import ( + PlanningService, InteractionError, validate_plan_proposal, _revision_materiality, +) INTERACTION_MODES = frozenset({"auto", "ask", "plan", "agent"}) @@ -101,8 +100,6 @@ def split_inline_command(text: str) -> tuple[str, str] | None: return result -class InteractionError(ValueError): - """A bounded interaction payload failed validation.""" def _text(value: Any, field: str, limit: int) -> str: @@ -167,250 +164,8 @@ def validate_question_request(value: Any, *, original_input: str = "") -> dict[s } -def validate_plan_proposal(value: Any, *, previous: dict[str, Any] | None = None, - original_input: str = "") -> dict[str, Any]: - if not isinstance(value, dict): - raise InteractionError("PlanProposal must contain a JSON object") - steps = value.get("steps") - if not isinstance(steps, list) or not 1 <= len(steps) <= 10: - raise InteractionError("PlanProposal requires 1-10 steps") - previous_version = int(previous.get("version", 0)) if previous else 0 - version = previous_version + 1 - normalised = [] - seen_ids: set[str] = set() - for index, step in enumerate(steps, 1): - if not isinstance(step, dict): - raise InteractionError("each plan step must be an object") - step_id = _text(step.get("id") or f"step-{index}", "step id", 64) - if step_id in seen_ids: - raise InteractionError("plan step ids must be unique") - seen_ids.add(step_id) - execution = step.get("execution") - if execution is not None: - action = execution.get("action") if isinstance(execution, dict) else None - arguments = execution.get("arguments") if isinstance(execution, dict) else None - if (not isinstance(execution, dict) or set(execution) != {"action", "arguments"} - or not isinstance(action, str) or action not in {"write_file", "edit_file", "apply_patch"} - or not isinstance(arguments, dict) - or not isinstance(arguments.get("path"), str) or not arguments["path"].strip()): - raise InteractionError("file execution contract must name write_file, edit_file, or apply_patch and an exact path") - required = ({"path", "content"} if action == "write_file" else - {"path", "expected_sha256", "hunks"} if action == "apply_patch" else - {"path", "old_text", "new_text", "expected_sha256"}) - if set(arguments) != required or (action != "apply_patch" and any(not isinstance(item, str) for item in arguments.values())): - raise InteractionError("file execution arguments must exactly match the selected file operation") - if action == "edit_file" and ( - not arguments["old_text"] or not re.fullmatch(r"[0-9a-fA-F]{64}", arguments["expected_sha256"])): - raise InteractionError("edit_file requires nonempty old_text and a SHA-256 read hash") - if action == "apply_patch" and ( - not isinstance(arguments["path"], str) or not arguments["path"].strip() - or not isinstance(arguments["expected_sha256"], str) - or not re.fullmatch(r"[0-9a-fA-F]{64}", arguments["expected_sha256"]) - or not isinstance(arguments["hunks"], list) or not arguments["hunks"] - or any(not isinstance(hunk, dict) or set(hunk) != {"old_text", "new_text"} - or not isinstance(hunk.get("old_text"), str) or not hunk["old_text"] - or not isinstance(hunk.get("new_text"), str) for hunk in arguments["hunks"])): - raise InteractionError("apply_patch requires a SHA-256 read hash and exact-text hunks") - try: - json.dumps(arguments, ensure_ascii=False, allow_nan=False) - except (TypeError, ValueError) as exc: - raise InteractionError("file execution arguments must be JSON-safe") from exc - execution = {"action": action, "arguments": deepcopy(arguments)} - acceptance = step.get("acceptance") - if isinstance(acceptance, str): - acceptance = [acceptance] - if not isinstance(acceptance, list) or not 1 <= len(acceptance) <= 5: - raise InteractionError("each plan step requires 1-5 acceptance criteria") - supplied_verifications = step.get("acceptance_verifications") - if supplied_verifications is not None and ( - not isinstance(supplied_verifications, list) or len(supplied_verifications) != len(acceptance)): - raise InteractionError("acceptance verifications must align with acceptance criteria") - acceptance_verifications = [] - normalized_acceptance = [] - for criterion_index, item in enumerate(acceptance): - verification = supplied_verifications[criterion_index] if supplied_verifications is not None else None - if isinstance(item, dict) and set(item) == {"criterion", "verification"}: - verification = item["verification"] - if verification is not None: - if (not isinstance(verification, dict) - or set(verification) not in ({"action", "arguments"}, {"action", "arguments", "expect"}) - or not isinstance(verification.get("action"), str) - or not verification["action"].strip() or not isinstance(verification.get("arguments"), dict)): - raise InteractionError("acceptance verification must contain an action and object arguments") - if verification["action"] == "read_file": - expect = verification.get("expect") - if (not isinstance(expect, dict) or len(expect) != 1 - or set(expect) not in ({"equals"}, {"contains"}) - or not isinstance(next(iter(expect.values())), str) - or ("contains" in expect and not expect["contains"])): - raise InteractionError("read_file verification requires a nonempty equals or contains predicate") - elif "expect" in verification: - raise InteractionError("expected-result predicates are currently supported only for read_file") - try: - json.dumps(verification["arguments"], ensure_ascii=False, allow_nan=False) - except (TypeError, ValueError) as exc: - raise InteractionError("acceptance verification arguments must be JSON-safe") from exc - criterion = _text(item.get("criterion") if isinstance(item, dict) else item, - "acceptance criterion", 300) - normalized_acceptance.append(criterion) - acceptance_verifications.append(verification) - acceptance = normalized_acceptance - acceptance_ids = [] - for criterion, verification in zip(acceptance, acceptance_verifications): - identity = json.dumps({"criterion": criterion, "verification": verification}, sort_keys=True, - ensure_ascii=False, separators=(",", ":")) - acceptance_ids.append(f"{step_id}-ac-{hashlib.sha256(identity.encode('utf-8')).hexdigest()[:12]}") - if len(set(acceptance_ids)) != len(acceptance_ids): - raise InteractionError("acceptance criteria must be unique within each objective") - normalised_step = { - "id": step_id, - "title": _text(step.get("title") or f"Step {index}", "step title", 120), - "description": _text(step.get("description"), "step description", 800), - "acceptance": acceptance, - "acceptance_ids": acceptance_ids, - "acceptance_verifications": acceptance_verifications, - } - if execution is not None: - normalised_step["execution"] = execution - normalised.append(normalised_step) - assumptions = value.get("assumptions", []) - if assumptions is None: - assumptions = [] - if not isinstance(assumptions, list) or len(assumptions) > 10: - raise InteractionError("assumptions must contain at most 10 items") - raw_bounds = value.get("bounds") - bounds_source = "proposal" - if raw_bounds is None: - if "bounds" in value: - raise InteractionError("bounds must be an object") - raw_bounds = { - "workspace_paths": ["."], "capabilities": ["read"], - "effects": ["read_only"], "risk_ceiling": "normal", - } - bounds_source = "legacy-default" - if not isinstance(raw_bounds, dict) or set(raw_bounds) - { - "workspace_paths", "capabilities", "effects", "risk_ceiling", - }: - raise InteractionError("bounds must declare workspace paths, capabilities, effects, and risk ceiling") - - def values(field: str, allowed: set[str] | None = None) -> list[str]: - raw = raw_bounds.get(field, []) - if not isinstance(raw, list) or len(raw) > 32: - raise InteractionError(f"bounds.{field} must be an array with at most 32 values") - result = [] - for item in raw: - if not isinstance(item, str) or not item.strip(): - raise InteractionError(f"bounds.{field} entries must be nonempty strings") - item = item.strip() - if allowed is not None and item not in allowed: - raise InteractionError(f"unsupported bounds.{field} value: {item}") - result.append(item) - if len(set(result)) != len(result): - raise InteractionError(f"bounds.{field} values must be unique") - return result - - paths = values("workspace_paths") - normalised_paths = [] - for path in paths: - path = path.replace("\\", "/") - if path.startswith("/") or re.match(r"^[A-Za-z]:", path): - raise InteractionError("workspace paths must be relative to the active workspace") - path = posixpath.normpath(path) - if path == ".." or path.startswith("../"): - raise InteractionError("workspace paths cannot escape the active workspace") - normalised_paths.append(path) - if len(set(normalised_paths)) != len(normalised_paths): - raise InteractionError("workspace paths must be unique after normalization") - capabilities = values("capabilities", { - "read", "write", "shell", "network", "git", "browser", "destructive", - }) - effects = values("effects", { - "read_only", "filesystem", "process", "remote", "destructive", "internal", - }) - risk_ceiling = raw_bounds.get("risk_ceiling", "normal") - if not isinstance(risk_ceiling, str) or risk_ceiling not in {"normal", "high"}: - raise InteractionError("risk ceiling must be normal or high") - bounds = { - "workspace_paths": sorted(normalised_paths), "capabilities": sorted(capabilities), - "effects": sorted(effects), "risk_ceiling": risk_ceiling, - } - contract_policies = { - "read_file": ("read", "read_only"), - "write_file": ("write", "filesystem"), - "edit_file": ("write", "filesystem"), - "apply_patch": ("write", "filesystem"), - } - for step in normalised: - contracts = ([step["execution"]] if step.get("execution") else []) + [ - verification for verification in step["acceptance_verifications"] if verification - ] - for contract in contracts: - policy = contract_policies.get(contract["action"]) - if policy is None: - raise InteractionError(f"unsupported Strategic Plan contract action: {contract['action']}") - capability, effect = policy - if capability not in bounds["capabilities"] or effect not in bounds["effects"]: - raise InteractionError("Strategic Plan contract exceeds its approved capability or effect bounds") - path = contract["arguments"].get("path") - if not isinstance(path, str) or not path.strip(): - raise InteractionError("Strategic Plan file contracts require a path") - normalized_contract_path = posixpath.normpath(path.replace("\\", "/")) - if (normalized_contract_path.startswith("/") or normalized_contract_path == ".." - or normalized_contract_path.startswith("../")): - raise InteractionError("Strategic Plan contract path escapes the approved workspace") - if not any(root == "." or normalized_contract_path == root - or normalized_contract_path.startswith(root.rstrip("/") + "/") - for root in bounds["workspace_paths"]): - raise InteractionError("Strategic Plan contract path is outside its approved workspace bounds") - plan = { - "plan_id": _text( - (previous or {}).get("plan_id") or f"plan_{uuid.uuid4().hex}", - "plan_id", 100, - ), - "version": version, - "title": _text(value.get("title"), "plan title", 160), - "summary": _text(value.get("summary"), "plan summary", 1200), - "assumptions": [_text(item, "assumption", 300) for item in assumptions], - "steps": normalised, - "bounds": bounds, - "bounds_source": bounds_source, - "original_input": str(original_input).strip()[:12_000], - } - fingerprint_fields = {key: plan[key] for key in ( - "plan_id", "version", "title", "summary", "assumptions", "steps", "bounds", - )} - plan["fingerprint"] = hashlib.sha256(json.dumps( - fingerprint_fields, ensure_ascii=False, allow_nan=False, sort_keys=True, - separators=(",", ":"), - ).encode("utf-8")).hexdigest() - return plan - - -def _revision_materiality(accepted: dict[str, Any], proposal: dict[str, Any]) -> str: - """Allow unchanged objectives only when all revised bounds are provably narrower.""" - if any(accepted.get(field) != proposal.get(field) - for field in ("title", "summary", "assumptions")): - return "material" - objectives = lambda plan: { - step["id"]: (step["title"], step["description"], tuple(step["acceptance"]), - tuple(step["acceptance_verifications"]), step.get("execution")) - for step in plan.get("steps", ()) - } - if objectives(accepted) != objectives(proposal): - return "material" - old, new = accepted.get("bounds"), proposal.get("bounds") - if not isinstance(old, dict) or not isinstance(new, dict): - return "material" - if (not set(new.get("capabilities", ())) <= set(old.get("capabilities", ())) - or not set(new.get("effects", ())) <= set(old.get("effects", ())) - or (old.get("risk_ceiling") == "normal" and new.get("risk_ceiling") == "high")): - return "material" - roots = [posixpath.normpath(path) for path in old.get("workspace_paths", ())] - for path in map(posixpath.normpath, new.get("workspace_paths", ())): - if not any(root == "." or path == root or path.startswith(root.rstrip("/") + "/") - for root in roots): - return "material" - return "within_bounds" + + def parse_control_block(text: str, name: str) -> Any | None: @@ -521,52 +276,17 @@ def is_plan_acceptance(text: str) -> bool: return " ".join(str(text or "").strip().lower().split()).rstrip(".!。！") in PLAN_ACCEPT_PHRASES -class InteractionController: - """Reconstruct session interaction state from the existing event log.""" - - def __init__(self, store: InteractionStore, *, user_id: str = "local", workspace_id: str = "default", - session_id: str): - self.store = store - self.user_id = user_id - self.workspace_id = workspace_id - self.session_id = session_id - - def _append(self, event_type: str, payload: dict[str, Any]) -> None: - self.store.append_event( - event_type, payload, user_id=self.user_id, workspace_id=self.workspace_id, - session_id=self.session_id, - ) - - def _append_auto_accepted(self, plan: dict[str, Any], previous: dict[str, Any]) -> None: - append = getattr(self.store, "append_plan_auto_accepted", None) - if not callable(append): - raise InteractionError("atomic Strategic Plan revision is unavailable") - try: - append(plan, expected_previous=previous, user_id=self.user_id, - workspace_id=self.workspace_id, session_id=self.session_id) - except ValueError as exc: - raise InteractionError(str(exc)) from exc - - def _history(self) -> list[dict[str, Any]]: - if callable(getattr(self.store, "list_interaction_events", None)): - return self.store.list_interaction_events( - workspace_id=self.workspace_id, session_id=self.session_id, user_id=self.user_id, - ) - events = self.store.list_events( - limit=None, workspace_id=self.workspace_id, session_id=self.session_id, - user_id=self.user_id, - ) - return [event for event in reversed(events) - if str(event.get("event_type", "")).startswith("interaction.") - or event.get("event_type") == "history.restored"] +class InteractionController(PlanningService): + """Adapt durable planning state to interaction modes and questions.""" def state(self, user_input: str = "") -> dict[str, Any]: preference = "auto" system_one_backend = "off" question = None - plan = None - executing = None - for event in self._history(): + history = self._history() + planning = self._plan_state(history) + plan, executing = planning["pending_plan"], planning["executing_plan"] + for event in history: event_type, payload = event["event_type"], event.get("payload", {}) if event_type == "history.restored": restored = payload.get("interaction") if isinstance(payload, dict) else None @@ -575,8 +295,6 @@ def state(self, user_input: str = "") -> dict[str, Any]: system_one_backend = restored.get("system_one_backend", restored.get("fast_backend", system_one_backend)) question = restored.get("pending_question") - plan = restored.get("pending_plan") - executing = restored.get("executing_plan") elif event_type == "interaction.mode_changed": preference = payload.get("mode", preference) elif event_type in {"interaction.fast_changed", "interaction.system_one_changed"}: @@ -585,29 +303,6 @@ def state(self, user_input: str = "") -> dict[str, Any]: question = payload elif event_type == "interaction.question_resolved" and question and payload.get("request_id") == question.get("request_id"): question = None - elif event_type == "interaction.plan_proposed": - if payload.get("revision_materiality") != "within_bounds": - plan = payload - elif event_type == "interaction.plan_auto_accepted": - if executing and payload.get("plan_id") == executing.get("plan_id"): - executing = dict(payload) - plan = None - elif event_type == "interaction.plan_accepted": - if plan and payload.get("plan_id") == plan.get("plan_id") and payload.get("version") == plan.get("version"): - executing = dict(plan) - for field in ("working_plan_id", "working_plan_revision", "working_task_ids"): - if field in payload: - executing[field] = payload[field] - plan = None - elif event_type == "interaction.plan_cancelled": - if plan and payload.get("plan_id") == plan.get("plan_id"): - plan = None - elif event_type == "interaction.plan_completed": - if (executing and payload.get("plan_id") == executing.get("plan_id") - and payload.get("version") == executing.get("version") - and payload.get("fingerprint") == executing.get("fingerprint") - and payload.get("status") in {"completed", "cancelled"}): - executing = None effective = route_mode( preference, user_input, pending_plan=plan is not None, executing_plan=executing is not None, ) @@ -624,6 +319,7 @@ def state(self, user_input: str = "") -> dict[str, Any]: state["fast_backend"] = state["system_one_backend"] return state + def envelope(self, user_input: str = "") -> dict[str, Any]: state = self.state(user_input) envelope = {key: state[key] for key in ( @@ -632,6 +328,7 @@ def envelope(self, user_input: str = "") -> dict[str, Any]: envelope["fast_backend"] = envelope["system_one_backend"] return envelope + def set_system_one_backend(self, backend: str) -> dict[str, Any]: backend = str(backend or "").strip().lower() if backend not in SYSTEM_ONE_BACKENDS: @@ -639,10 +336,12 @@ def set_system_one_backend(self, backend: str) -> dict[str, Any]: self._append("interaction.system_one_changed", {"backend": backend}) return self.envelope() + def set_fast_backend(self, backend: str) -> dict[str, Any]: """Deprecated compatibility alias for set_system_one_backend.""" return self.set_system_one_backend(backend) + def set_mode(self, mode: str) -> dict[str, Any]: mode = str(mode or "").strip().lower() if mode not in INTERACTION_MODES: @@ -650,12 +349,14 @@ def set_mode(self, mode: str) -> dict[str, Any]: self._append("interaction.mode_changed", {"mode": mode}) return self.envelope() + def request_question(self, value: Any, *, original_input: str = "") -> dict[str, Any]: question = validate_question_request(value, original_input=original_input) question["resume_mode"] = self.state(original_input)["effective_mode"] self._append("interaction.question_requested", question) return question + def resolve_question(self, request_id: str, answers: Any, *, action: str = "answer") -> dict[str, Any]: pending = self.state().get("pending_question") if not pending or pending.get("request_id") != request_id: @@ -668,6 +369,7 @@ def resolve_question(self, request_id: str, answers: Any, *, action: str = "answ self._append("interaction.question_resolved", payload) return {"question": pending, **payload} + def reopen_question(self) -> dict[str, Any]: requested = [event["payload"] for event in self._history() if event["event_type"] in {"interaction.question_requested", "interaction.question_reopened"}] @@ -677,276 +379,6 @@ def reopen_question(self) -> dict[str, Any]: self._append("interaction.question_reopened", question) return question - def propose_plan(self, value: Any, *, original_input: str = "") -> dict[str, Any]: - state = self.state() - previous = state.get("pending_plan") or state.get("executing_plan") - if previous and state.get("pending_plan") is None: - latest = [int(event.get("payload", {}).get("version", 0)) for event in self._history() - if event.get("event_type") in {"interaction.plan_proposed", "interaction.plan_auto_accepted"} - and event.get("payload", {}).get("plan_id") == previous.get("plan_id")] - if latest: - previous = {**previous, "version": max(latest)} - plan = validate_plan_proposal(value, previous=previous, original_input=original_input) - executing = state.get("executing_plan") - if executing: - plan["revision_materiality"] = _revision_materiality(executing, plan) - working = self.store.get_working_plan( - user_id=self.user_id, workspace_id=self.workspace_id, session_id=self.session_id, - ) if callable(getattr(self.store, "get_working_plan", None)) else None - working_revision = (working.get("revision") if working - and working.get("id") == executing.get("working_plan_id") - else executing.get("working_plan_revision")) - plan.update({"working_plan_id": executing.get("working_plan_id"), - "working_plan_revision": working_revision, - "working_task_ids": executing.get("working_task_ids", [])}) - if plan["revision_materiality"] == "within_bounds": - self._append_auto_accepted(plan, executing) - return plan - else: - plan["revision_materiality"] = "initial" - self._append("interaction.plan_proposed", plan) - return plan - - def cancel_plan(self, plan_id: str | None = None, version: int | None = None) -> dict[str, Any]: - plan = self.state().get("pending_plan") - if not plan or (plan_id and plan_id != plan.get("plan_id")) or (version is not None and version != plan.get("version")): - raise InteractionError("plan is not pending") - self._append("interaction.plan_cancelled", {"plan_id": plan["plan_id"], "version": plan["version"]}) - return plan - - def accepted_plan(self, plan_id: str | None = None, version: int | None = None) -> dict[str, Any] | None: - proposals: dict[tuple[str, int], dict[str, Any]] = {} - accepted: dict[tuple[str, int], dict[str, Any]] = {} - for event in self._history(): - payload = event.get("payload", {}) - key = (str(payload.get("plan_id") or ""), int(payload.get("version") or 0)) - if event["event_type"] == "interaction.plan_proposed": - proposals[key] = payload - elif event["event_type"] == "interaction.plan_accepted": - accepted[key] = payload - elif event["event_type"] == "interaction.plan_auto_accepted": - proposals[key] = payload - accepted[key] = payload - matches = [proposal for key, proposal in proposals.items() if key in accepted - and (not plan_id or key[0] == plan_id) and (version is None or key[1] == version)] - if not matches: - return None - proposal = matches[-1] - key = (str(proposal.get("plan_id") or ""), int(proposal.get("version") or 0)) - return {**proposal, **{field: accepted[key][field] for field in ( - "working_plan_id", "working_plan_revision", "working_task_ids", - ) if field in accepted[key]}} - - def accept_plan(self, task_manager: Any, *, plan_id: str | None = None, - version: int | None = None, fingerprint: str | None = None) -> tuple[dict[str, Any], bool]: - state = self.state() - plan = state.get("pending_plan") - executing = state.get("executing_plan") - if executing and not plan and (not plan_id or plan_id == executing.get("plan_id")): - if version is not None and version != executing.get("version"): - raise InteractionError("accepted plan version is stale") - if fingerprint is not None and fingerprint != executing.get("fingerprint"): - raise InteractionError("accepted plan fingerprint is stale") - return executing, False - if not plan: - if executing and plan_id and plan_id != executing.get("plan_id"): - raise InteractionError("accepted plan identity is stale") - accepted = self.accepted_plan(plan_id, version) - if accepted and (fingerprint is None or accepted.get("fingerprint") == fingerprint): - return accepted, False - if accepted: - raise InteractionError("accepted plan fingerprint is stale") - raise InteractionError("plan is not pending") - if ((plan_id and plan_id != plan.get("plan_id")) - or (version is not None and version != plan.get("version")) - or (fingerprint is not None and fingerprint != plan.get("fingerprint"))): - raise InteractionError("plan is not pending at the requested version") - if executing and task_manager.working_plan is None: - task_manager.recover() - elif not executing and task_manager.working_plan is None: - task_manager.recover() - if (not executing and task_manager.working_plan is not None - and task_manager.working_plan.status == "active"): - raise InteractionError("cannot accept a Strategic Plan while an unrelated Working Plan is active") - working_plan = task_manager.working_plan if executing else None - previous_tasks = [task for task in task_manager.tasks - if working_plan and task.get("id") in working_plan.task_ids] - if any(task.get("status") == "running" for task in previous_tasks): - raise InteractionError("cannot replace a Strategic Plan while a Working Plan task is running") - operations = [] - strategic_bindings = {} - active_ids = set(executing.get("working_task_ids", ())) if executing else set() - if executing: - if (plan.get("revision_materiality") != "material" - or plan.get("plan_id") != executing.get("plan_id") - or plan.get("version") != executing.get("version", 0) + 1): - raise InteractionError("Strategic Plan revision is stale or does not require approval") - staged_revision = plan.get("working_plan_revision", executing.get("working_plan_revision")) - if (not working_plan or working_plan.id != executing.get("working_plan_id") - or working_plan.revision != staged_revision): - raise InteractionError("Working Plan revision changed; reload before accepting") - - prior = [task for task in previous_tasks if not executing or task.get("id") in active_ids] - if any(task.get("status") == "running" for task in prior): - raise InteractionError("cannot replace a Strategic Plan while a Working Plan task is running") - new_refs = [] - reused: set[str] = set() - for step in plan["steps"]: - description = f"{step['title']}: {step['description']}" - execution = step.get("execution") or {} - expected_args = (json.dumps(execution.get("arguments", {}), ensure_ascii=False, - allow_nan=False, sort_keys=True, separators=(",", ":")) - if execution else "") - existing = next((task for task in prior - if task.get("status") == "succeeded" - and (task.get("checkpoint") or {}).get("strategic_plan", {}).get("objective_id") == step["id"] - and (task.get("checkpoint") or {}).get("strategic_plan", {}).get("acceptance_ids") == step["acceptance_ids"] - and task.get("description") == description - and [item.get("criterion") if isinstance(item, dict) else item - for item in task.get("acceptance", [])] == step["acceptance"] - and (task.get("checkpoint") or {}).get("action") == execution.get("action") - and ((task.get("checkpoint") or {}).get("args") or "") == expected_args - and {criterion_id for item in task.get("evidence", []) - if isinstance(item, dict) and item.get("success") is True - for criterion_id in item.get("acceptance_ids", ())} >= set(step["acceptance_ids"])), None) - if existing: - new_refs.append(existing["id"]) - reused.add(existing["id"]) - continue - ref = f"revision_{plan['version']}_{step['id']}" if executing else step["id"] - new_refs.append(ref) - operations.append({"op": "add", "ref": ref, "task": { - "description": description, "acceptance": step["acceptance"], - **({"execution": step["execution"]} if step.get("execution") else {}), - }}) - strategic_bindings[ref] = { - "plan_id": plan["plan_id"], "version": plan["version"], - "fingerprint": plan["fingerprint"], "objective_id": step["id"], - "acceptance_ids": step["acceptance_ids"], - "acceptance_verifications": step["acceptance_verifications"], - "execution": step.get("execution"), - } - for task in prior: - if task["id"] not in reused and task.get("status") in {"pending", "failed", "blocked"}: - operations.insert(0, {"op": "cancel", "task": task["id"]}) - if not operations: - if not prior: - raise InteractionError("Strategic Plan has no Working Plan tasks to reuse") - operations.append({"op": "update", "task": prior[0]["id"], - "fields": {"priority": prior[0]["priority"]}}) - payload = { - "plan_id": plan["plan_id"], "version": plan["version"], - "fingerprint": plan["fingerprint"], "working_task_refs": new_refs, - **({"revision_of": {"version": executing["version"], - "fingerprint": executing["fingerprint"]}} if executing else {}), - } - try: - working = task_manager.edit_plan( - complexity=working_plan.complexity if working_plan else "complex", - plan_id=working_plan.id if working_plan else None, - expected_revision=working_plan.revision if working_plan else 0, - operations=operations, strategic_bindings=strategic_bindings, - transaction_event={"event_type": "interaction.plan_accepted", "payload": payload, - "expected_proposal": {"plan_id": plan["plan_id"], "version": plan["version"], - "fingerprint": plan["fingerprint"]}, - **({"expected_working_plan": { - "plan_id": executing["working_plan_id"], - "revision": executing["working_plan_revision"], - "strategic_plan_id": executing["plan_id"], - "version": executing["version"], "fingerprint": executing["fingerprint"], - }} if executing else {})}, - ) - except ValueError as exc: - accepted = self.accepted_plan(plan["plan_id"], plan["version"]) - current_state = self.state() - current_plan = current_state.get("pending_plan") or current_state.get("executing_plan") - if current_plan and not all( - current_plan.get(key) == plan.get(key) - for key in ("plan_id", "version", "fingerprint")): - message = str(exc) - if "changed" not in message.lower() and "stale" not in message.lower(): - message = f"Strategic Plan changed while acceptance was pending: {message}" - raise InteractionError(message) from exc - if accepted and accepted.get("fingerprint") == plan["fingerprint"]: - return accepted, False - raise InteractionError(str(exc)) from exc - bound_ids = [working["references"][ref] if ref in working["references"] else ref for ref in new_refs] - return {**plan, "working_plan_id": working["plan_id"], - "working_plan_revision": working["revision"], "working_task_ids": bound_ids}, True - - def complete_plan(self, plan: dict[str, Any], *, status: str = "completed", - task_manager: Any | None = None) -> None: - if status not in {"completed", "failed", "cancelled"}: - raise InteractionError("Strategic Plan terminal status is invalid") - prior_completions = [event.get("payload", {}).get("status") for event in self._history() - if event["event_type"] == "interaction.plan_completed" - and event.get("payload", {}).get("plan_id") == plan.get("plan_id") - and event.get("payload", {}).get("version") == plan.get("version") - and event.get("payload", {}).get("fingerprint") == plan.get("fingerprint")] - if any(previous in {"completed", "cancelled"} or previous == status - for previous in prior_completions): - return - executing = self.state().get("executing_plan") - if not executing or any(executing.get(key) != plan.get(key) - for key in ("plan_id", "version", "fingerprint")): - raise InteractionError("Strategic Plan changed; reload before recording completion") - working = task_manager.working_plan if task_manager is not None else None - task_ids = plan.get("working_task_ids") - if task_ids is None and working and working.id == plan.get("working_plan_id"): - task_ids = working.task_ids - if status == "completed": - if task_manager is None or not isinstance(task_ids, list) or not task_ids: - raise InteractionError("Strategic Plan completion requires its linked Working Plan evidence") - tasks = {task.get("id"): task for task in task_manager.tasks} - objectives = {step["id"] for step in plan.get("steps", [])} - for task_id in task_ids: - binding = (tasks.get(task_id, {}).get("checkpoint") or {}).get("strategic_plan") or {} - if binding.get("plan_id") != plan.get("plan_id") or binding.get("objective_id") not in objectives: - raise InteractionError("Working Plan contains a task not linked to the accepted Strategic Plan") - if any(tasks.get(task_id, {}).get("status") != "succeeded" - or not any(isinstance(item, dict) and item.get("success") is True and item.get("acceptance") - for item in tasks[task_id].get("evidence", [])) - for task_id in task_ids): - raise InteractionError("Strategic Plan tasks have not passed their acceptance evidence gates") - for task_id in task_ids: - task = tasks[task_id] - binding = (task.get("checkpoint") or {}).get("strategic_plan") or {} - required = set(binding.get("acceptance_ids", ())) - if required: - covered = { - criterion_id - for item in task.get("evidence", []) - if isinstance(item, dict) and item.get("success") is True - for criterion_id in item.get("acceptance_ids", ()) - } - if not required <= covered: - raise InteractionError("Strategic Plan criteria have not all been verified") - payload = { - "plan_id": plan["plan_id"], "version": plan["version"], - "fingerprint": plan.get("fingerprint"), "working_plan_id": plan.get("working_plan_id"), - "working_task_ids": task_ids or [], "status": status, - } - append = getattr(self.store, "append_plan_completed", None) - if callable(append): - try: - append(payload, user_id=self.user_id, workspace_id=self.workspace_id, - session_id=self.session_id) - except ValueError as exc: - raise InteractionError(str(exc)) from exc - else: - self._append("interaction.plan_completed", payload) - - def reconcile_plan_completion(self, task_manager: Any) -> None: - plan = self.state().get("executing_plan") - if not plan or not plan.get("working_plan_id"): - return - working = task_manager.working_plan - if working is None or working.id != plan["working_plan_id"]: - working = task_manager.restore_working_plan(plan["working_plan_id"]) - if working is not None and working.status in {"completed", "cancelled"}: - self.complete_plan( - plan, status=working.status, task_manager=task_manager, - ) def render_question(question: dict[str, Any]) -> str: diff --git a/openkyrozen/agent/plan_tools.py b/openkyrozen/agent/plan_tools.py index 00ff5af1..3cfd7309 100644 --- a/openkyrozen/agent/plan_tools.py +++ b/openkyrozen/agent/plan_tools.py @@ -3,15 +3,91 @@ from openkyrozen.tools.models import legacy_text_api, tool_failure, tool_success +def _control_error(self): + if not self._interaction_controls_enabled.get() or getattr(self.execution_context, "child_run_id", None): + return tool_failure("permission_denied", "Interactive controls are unavailable on this execution surface.") + return None + + +@legacy_text_api +def _propose_plan(self, arguments: dict): + """Propose a versioned Strategic Plan after read-only inspection. This never + accepts or executes the proposal; material changes require user approval. + """ + denied = _control_error(self) + if denied is not None: + return denied + state = self._interaction_controller.state() + if (self._active_interaction_mode.get() != "plan" and not state.get("executing_plan")): + return tool_failure("permission_denied", "Strategic proposals require Plan Mode or an accepted plan revision.") + if not getattr(self.execution_context, "native_plan_inspected", False): + return tool_failure("invalid_arguments", "Inspect the relevant workspace or source with a read-only tool first.") + from openkyrozen.agent.modes import render_plan + try: + plan = self._interaction_controller.propose_plan( + arguments, original_input=self.execution_context.native_control_input, + ) + except (ValueError, TypeError, KeyError) as exc: + return tool_failure("invalid_arguments", f"Invalid Strategic Plan: {exc}") + self._emit_stream_event({"event": "interaction", "interaction": self.interaction_envelope()}) + requires_approval = plan.get("revision_materiality") != "within_bounds" + text = (render_plan(plan) if requires_approval else + f"Strategic Plan v{plan['version']} updated within approved scope; no renewed approval is required.") + return tool_success(text, data={"plan": plan, "requires_approval": requires_approval}) + + +@legacy_text_api +def _request_question(self, arguments: dict): + """Ask 1-3 structured clarification questions. Questions cannot request + credentials, grant authority, replace tool approval or execute work. + """ + denied = _control_error(self) + if denied is not None: + return denied + from openkyrozen.agent.modes import render_question + try: + question = self._interaction_controller.request_question( + arguments, original_input=self.execution_context.native_control_input, + ) + except (ValueError, TypeError, KeyError) as exc: + return tool_failure("invalid_arguments", f"Invalid clarification request: {exc}") + self._emit_stream_event({"event": "interaction", "interaction": self.interaction_envelope()}) + return tool_success(render_question(question), data={"question": question}) + + @legacy_text_api def _update_plan(self, arguments: dict): - """Apply revision-checked ordered Working Plan edits or adapt the legacy task list.""" + """Edit a revision-checked Working Plan. Use start with a stable task ID before + executing native task work, then suggest_complete to check existing verified + evidence; a completion suggestion never supplies evidence. Split of an + approved objective retains its immutable acceptance anchor and adds strategy + children. Add anchor= for in-scope investigation/verification. + """ + def validate_strategy(binding, execution): + from openkyrozen.agent.executor import _strategic_plan_denial + + active = self._interaction_controller.state().get("executing_plan") + if not active or active.get("plan_id") != binding.get("plan_id"): + raise ValueError("strategy has no active approved objective") + objective = next((step for step in active.get("steps", []) + if step.get("id") == binding.get("objective_id")), None) + if not objective or objective.get("acceptance_ids") != binding.get("acceptance_ids"): + raise ValueError("strategy objective acceptance changed; approval is required") + spec = self.tool_registry.get_spec(execution["action"]) + if spec is None or self.tool_registry._argument_error(spec.input_schema, execution["arguments"]): + raise ValueError("strategy execution has no valid tool contract") + denial = _strategic_plan_denial(active, execution["action"], execution["arguments"], + spec, self._get_workspace_root()) + if denial: + raise ValueError(denial) + try: if "operations" in arguments: result = self.tasks.update_plan( complexity=arguments.get("complexity"), plan_id=arguments.get("plan_id"), expected_revision=arguments["expected_revision"], operations=arguments["operations"], reason=arguments.get("reason"), + strategy_validator=validate_strategy, ) return tool_success("Working Plan updated.", data=result) plan = self.tasks.update_plan(complexity=arguments["complexity"], tasks=arguments["tasks"]) diff --git a/openkyrozen/agent/ports.py b/openkyrozen/agent/ports.py index 663a4efa..76e8aeaa 100644 --- a/openkyrozen/agent/ports.py +++ b/openkyrozen/agent/ports.py @@ -6,7 +6,4 @@ RuntimeEventSink = Callable[[RuntimeEvent], None] Approval = Callable[[str, str], bool] -class InteractionStore(Protocol): - def append_event(self, event_type: str, payload: Any, **scope) -> str: ... - def list_events(self, event_type: str | None = None, **scope) -> list[dict]: ... - def list_interaction_events(self, **scope) -> list[dict]: ... +from openkyrozen.tasks.ports import InteractionStore diff --git a/openkyrozen/agent/preparation.py b/openkyrozen/agent/preparation.py index 08968082..cebb0171 100644 --- a/openkyrozen/agent/preparation.py +++ b/openkyrozen/agent/preparation.py @@ -64,8 +64,10 @@ def _prepare_turn(self, turn): self.execution_context.delegation_budget = DelegationBudget() provider_config = self.execution_context.provider_config or self._provider_config select_model = (lambda _text: self.execution_context.model) if self.execution_context.child_run_id else self._select_model + classification_input = (getattr(self.current_session, "_delegation_classifier_input", turn.user_input) + if self.execution_context.child_run_id else turn.user_input) decision, turn.complexity = decide_request( - provider_config, fast_route, turn.user_input, select_model, + provider_config, fast_route, classification_input, select_model, self._classify_complexity, automatic_profile, turn.profile or self._agent_profile_mode, turn.interaction_mode, ) @@ -113,3 +115,6 @@ def _prepare_turn(self, turn): turn.memory_packet = self._build_memory_context( turn.user_input, context=turn.memory_context, profile=resolved_profile, ) + process_context = self.current_session.process_manager._recovery_context() + if process_context: + turn.memory_packet = (turn.memory_packet + "\n" + process_context).strip() diff --git a/openkyrozen/agent/presentation.py b/openkyrozen/agent/presentation.py new file mode 100644 index 00000000..4aea21bb --- /dev/null +++ b/openkyrozen/agent/presentation.py @@ -0,0 +1,99 @@ +"""User-facing response formatting shared by native and legacy execution.""" +from __future__ import annotations + +import re +from typing import Any +from openkyrozen.tasks.engine import canonical_status + + +class _NativeResponseText(str): + """Immutable per-reply provenance that survives the runtime turn scope.""" + __slots__ = () + + +def is_native_response(runtime, text) -> bool: + state = getattr(runtime, "_last_response_is_native", None) + return isinstance(text, _NativeResponseText) or bool(state is not None and state.get()) + + +def native_response_metadata(text: str) -> dict[str, Any]: + """Expose native text literally through the established compatibility result shape.""" + raw = str(text or "").strip() + return { + "raw": raw, "clean": raw, "has_plan": False, "has_tasklist": False, + "tool_calls": [], "question": None, "plan_proposal": None, + "unknown_action": None, "unsupported_action_protocol": False, + "protocol_error": None, + } + + +def _clean_final_response(self, text: str) -> str: + """Native output is literal; only the legacy boundary loads textual cleaning.""" + if is_native_response(self, text): + return _NativeResponseText(str(text or "").strip()) + from openkyrozen.agent.protocol import _clean_final_response as legacy_clean + return legacy_clean(self, text) + + +def _deterministic_tool_summary(self, tool_records: list[dict[str, Any]]) -> str: + """Build a truthful user-facing summary when the model emitted only protocol.""" + lines: list[str] = [] + for record in tool_records: + verified_failure = (record.get("verification") or {}).get("status") == "failed" + status = "Completed" if record.get("success") and not verified_failure else "Could not complete" + result = self._clean_final_response(str(record.get("result") or "")).replace("\n", " ") + if len(result) > 300: + result = result[:297] + "..." + line = f"- {status}" + if result: + line += f": {result}" + if record.get("unmatched_reason"): + line += f" ({record['unmatched_reason']})" + lines.append(line) + task_lines: list[str] = [] + for task in self.tasks.tasks: + status = canonical_status(task.get("status", "pending")) + task_lines.append(f"- {status}: {task.get('description', '')}") + if task_lines: + lines.append("Durable task status (source of truth):") + lines.extend(task_lines) + attention = [task for task in self.tasks.tasks if canonical_status(task.get("status", "pending")) != "succeeded"] + if attention: + lines.append("Recovery required for non-succeeded tasks:") + lines.extend( + f"- {canonical_status(task.get('status', 'pending'))}: {task.get('description', '')}" + for task in attention + ) + if not lines: + return "I could not produce a user-facing response." + return "Here is what I completed:\n\n" + "\n".join(lines) + + +def _safe_fstring(self, s: str) -> str: + """Escape curly braces so they do not interfere with f-string parsing.""" + return s.replace('{', '{{').replace('}', '}}') + + +def _remove_task_blocks(self, text: str) -> str: + if is_native_response(self, text): + return _NativeResponseText(str(text or "").strip()) + from openkyrozen.agent.protocol import _remove_task_blocks as legacy_remove + return legacy_remove(self, text) + + +def _split_reply(self, text: str) -> tuple[str, str]: + if is_native_response(self, text): + return "", str(text or "").strip() + text = text.strip() + thought_match = re.search(r"^Thought:\s*(.*?)(?=\n(?:Action:|(?:\n|$)))", text, re.DOTALL | re.MULTILINE) + if thought_match: + thinking = thought_match.group(1).strip() + answer = re.sub(r"^Thought:\s*.*?(?=\n(?:Action:|(?:\n|$))|\n?$)", "", text, count=1, flags=re.DOTALL | re.MULTILINE).strip() + answer = re.sub(r"Action:\s*```(?:json)?[\s\S]*?```", "", answer).strip() + answer = re.sub(r"\{[\s\S]*?\}", "", answer).strip() + answer = answer.lstrip('"').lstrip("'").strip() + if not answer: + answer = text + return thinking, answer + else: + return "", text diff --git a/openkyrozen/agent/prompts.py b/openkyrozen/agent/prompts.py index 4df34407..a39e4383 100644 --- a/openkyrozen/agent/prompts.py +++ b/openkyrozen/agent/prompts.py @@ -30,7 +30,7 @@ def _compose_skills(self, task_description: str) -> str | None: def _build_tools_list(self, capabilities: frozenset[str] | None = None) -> str: lines = [] for name, fn in self.AVAILABLE_TOOLS.items(): - if name == "update_plan": # Structured-only until the foreground loop uses native calls. + if name in {"update_plan", "propose_plan", "request_question"}: continue if self.tool_registry.get_spec(name).visibility != 'public': continue @@ -57,6 +57,11 @@ def _permitted_tool_names(self, agent_config: dict[str, Any] | None = None) -> s permitted = {name for name in self.AVAILABLE_TOOLS if self.tool_registry.get_spec(name).visibility == 'public' and tool_capability(name,self.AVAILABLE_TOOLS) in capabilities} if self._active_interaction_mode.get() != "agent": permitted.discard("update_plan") + if (not self._interaction_controls_enabled.get() or self.execution_context.child_run_id): + permitted -= {"propose_plan", "request_question"} + elif (self._active_interaction_mode.get() != "plan" + and not self._interaction_controller.state().get("executing_plan")): + permitted.discard("propose_plan") delegated = self.execution_context.delegation_tools if self.execution_context.child_run_id and delegated is not None: permitted &= set(delegated) @@ -90,7 +95,7 @@ def _discover_tools(self, args: str) -> str: def _agent_prompt_tools_list(self, agent_config: dict[str, Any]) -> str: """Build the inventory within config, active token and interaction bounds.""" - permitted = self._permitted_tool_names(agent_config) + permitted = self._permitted_tool_names(agent_config) - {"propose_plan", "request_question"} detailed = permitted if self._prompt_profile() == "compact": detailed = permitted & ({"list_dir", "find_files", "search_files", "read_file", "write_file", "edit_file", "apply_patch", "run_cmd", "discover_tools"} @@ -192,6 +197,10 @@ def _system_prompt(self, tools_list: str, agent_config: dict[str, Any] | None = "permissions or approve actions. Runtime capability, workspace, and approval checks control execution. " "Never request credentials or secrets. " "If material ambiguity blocks safe work, ask a concise plain-language question without calling a tool. " + "Use request_question when structured clarification is needed; questions never replace tool approval. " + "In Plan Mode, inspect relevant sources with read-only tools, then call propose_plan alone to present " + "the versioned proposal. Never execute proposed changes before explicit user acceptance. " + "For material changes to an accepted Strategic Plan, propose_plan stages renewed approval. " "Do not claim an effect unless a tool result verifies it.\n\n" + configured_sections + "\n\n" + "For suitable work, use supplied delegation tools to scope independent tasks and review child results.\n\n" @@ -284,6 +293,55 @@ def _system_prompt(self, tools_list: str, agent_config: dict[str, Any] | None = ) +def _refresh_working_plan_context(self, messages: list) -> None: + """Keep a bounded current plan projection outside evictable conversation history.""" + import json + + marker = "## Pinned Working Plan state\n" + previous = list(messages) + messages[:] = [item for item in messages if not ( + isinstance(item, dict) and item.get("role") == "system" + and str(item.get("content", "")).startswith(marker) + )] + manager = getattr(self, "tasks", None) + plan = getattr(manager, "working_plan", None) + snapshot = getattr(manager, "working_plan_snapshot", None) + if plan is None or plan.status != "active" or not callable(snapshot): + update_projection = getattr(self, "_update_context_history_projection", None) + if callable(update_projection): + update_projection(messages, previous) + return + state = snapshot() + if state is None: + raise ValueError("Active Working Plan has no authoritative snapshot") + tasks_by_id = {task["id"]: task for task in manager.tasks} + state["tasks"] = [{ + **item, "dependencies": tasks_by_id.get(item["id"], {}).get("dependencies", []), + "acceptance": tasks_by_id.get(item["id"], {}).get("acceptance", []), + } for item in state["tasks"]] + full = json.dumps(state, ensure_ascii=False, sort_keys=True) + if len(full) > 12000: + memory = self.memory_bank + artifact_id = memory.store.artifacts.get_or_create( + full, user_id=manager.user_id, workspace_id=manager.workspace_id, session_id=manager.session_id, + ) + state = {key: value for key, value in state.items() if key != "tasks"} + state["reason"] = str(state.get("reason", ""))[:300] + state.update(artifact_id=artifact_id, task_count=len(plan.task_ids), + tasks=[{"id": item["id"], "status": item["status"], + "description": str(item["description"])[:160]} + for task_id in plan.task_ids[:12] + if (item := tasks_by_id.get(task_id)) is not None]) + full = json.dumps(state, ensure_ascii=False, sort_keys=True) + messages.insert(1 if messages else 0, {"role": "system", "content": marker + ( + "Runtime-owned task identity, order, status and acceptance. Strategy may adapt within approved scope; " + "tool/security approval remains separate. If artifact_id is present, use read_tool_artifact for the full plan.\n" + ) + full}) + update_projection = getattr(self, "_update_context_history_projection", None) + if callable(update_projection): + update_projection(messages, previous) + + def _build_messages(self, user_input: str, learned_context: str = "", memory_context: dict[str, Any] | None = None, *, memory_packet: str | None = None, @@ -330,6 +388,8 @@ def _build_messages(self, user_input: str, learned_context: str = "", "do not expand scope.\n" + json.dumps(summary, ensure_ascii=False) )}) + _refresh_working_plan_context(self, messages) + project_instructions = format_instructions(self._get_workspace_root()) if project_instructions and self._active_interaction_mode.get() == "agent": messages.append({"role": "system", "content": project_instructions}) diff --git a/openkyrozen/agent/protocol.py b/openkyrozen/agent/protocol.py index e9e84411..48aa2cd4 100644 --- a/openkyrozen/agent/protocol.py +++ b/openkyrozen/agent/protocol.py @@ -8,6 +8,7 @@ from openkyrozen.agent.modes import InteractionError, normalize_provider_control, parse_control_block, render_plan, render_question, validate_plan_proposal, validate_question_request from openkyrozen.security.tool_policy import tool_capability from openkyrozen.agent.types import DeepSeekDSMLFilter +from openkyrozen.agent.presentation import _deterministic_tool_summary, _safe_fstring, _split_reply def _is_valid_action(self, name: str | None) -> bool: @@ -573,57 +574,3 @@ def _persist_interaction_control(self, parsed: dict[str, Any], user_input: str) def _interaction_gate(self, parsed: dict[str, Any], user_input: str) -> str | None: """Stop a response before any executable path when it carries a control.""" return parsed.get("protocol_error") or self._persist_interaction_control(parsed, user_input) - - -def _deterministic_tool_summary(self, tool_records: list[dict[str, Any]]) -> str: - """Build a truthful user-facing summary when the model emitted only protocol.""" - lines: list[str] = [] - for record in tool_records: - status = "Completed" if record.get("success") else "Could not complete" - result = self._clean_final_response(str(record.get("result") or "")).replace("\n", " ") - if len(result) > 300: - result = result[:297] + "..." - line = f"- {status}" - if result: - line += f": {result}" - if record.get("unmatched_reason"): - line += f" ({record['unmatched_reason']})" - lines.append(line) - task_lines: list[str] = [] - for task in self.tasks.tasks: - status = canonical_status(task.get("status", "pending")) - task_lines.append(f"- {status}: {task.get('description', '')}") - if task_lines: - lines.append("Durable task status (source of truth):") - lines.extend(task_lines) - attention = [task for task in self.tasks.tasks if canonical_status(task.get("status", "pending")) != "succeeded"] - if attention: - lines.append("Recovery required for non-succeeded tasks:") - lines.extend( - f"- {canonical_status(task.get('status', 'pending'))}: {task.get('description', '')}" - for task in attention - ) - if not lines: - return "I could not produce a user-facing response." - return "Here is what I completed:\n\n" + "\n".join(lines) - - -def _safe_fstring(self, s: str) -> str: - """Escape curly braces so they do not interfere with f-string parsing.""" - return s.replace('{', '{{').replace('}', '}}') - - -def _split_reply(self, text: str) -> tuple[str, str]: - text = text.strip() - thought_match = re.search(r"^Thought:\s*(.*?)(?=\n(?:Action:|(?:\n|$)))", text, re.DOTALL | re.MULTILINE) - if thought_match: - thinking = thought_match.group(1).strip() - answer = re.sub(r"^Thought:\s*.*?(?=\n(?:Action:|(?:\n|$))|\n?$)", "", text, count=1, flags=re.DOTALL | re.MULTILINE).strip() - answer = re.sub(r"Action:\s*```(?:json)?[\s\S]*?```", "", answer).strip() - answer = re.sub(r"\{[\s\S]*?\}", "", answer).strip() - answer = answer.lstrip('"').lstrip("'").strip() - if not answer: - answer = text - return thinking, answer - else: - return "", text diff --git a/openkyrozen/agent/runtime.py b/openkyrozen/agent/runtime.py index 55d15e42..df855272 100644 --- a/openkyrozen/agent/runtime.py +++ b/openkyrozen/agent/runtime.py @@ -24,8 +24,8 @@ class AgentRuntime: from openkyrozen.agent.completion import _complete_turn from openkyrozen.agent.response_recovery import _observe_turn_response, _recover_plan_response - # agent.protocol - from openkyrozen.agent.protocol import ( + # Lazy legacy compatibility; native formatting does not load the parser. + from openkyrozen.agent.legacy_compat import ( _is_valid_action, _detect_unknown_action, _has_unsupported_action_protocol, @@ -37,18 +37,17 @@ class AgentRuntime: _action_marker_is_protocol, _control_marker_is_protocol, _remove_protocol_headings, - _clean_final_response, _stream_buffer_has_tool_prefix, _clean_stream_buffer, - _remove_task_blocks, _legacy_plan_value, _parse_model_response, _observe_model_response, _persist_interaction_control, _interaction_gate, - _deterministic_tool_summary, - _safe_fstring, - _split_reply, + ) + + from openkyrozen.agent.presentation import ( + _clean_final_response, _deterministic_tool_summary, _safe_fstring, _split_reply, _remove_task_blocks, ) # security.permissions @@ -172,7 +171,7 @@ class AgentRuntime: _subagent_provider_model, ) - from openkyrozen.agent.plan_tools import _update_plan + from openkyrozen.agent.plan_tools import _update_plan, _propose_plan, _request_question from openkyrozen.agent.delegation_runtime import ( _plan_delegation, delegation, _invoke_delegated, spawn_agents, send_subagent, list_subagents, @@ -301,9 +300,19 @@ def __init__(self, memory, tasks, interaction, workspace): self._sessions = {} self._workspaces = {workspace.root: workspace} from openkyrozen.agent.engine import AgentEngine - from openkyrozen.agent.protocol import LegacyProtocolAdapter self._agent_engine = AgentEngine(self) - self.legacy_protocol = LegacyProtocolAdapter(self) + self._legacy_protocol = None + + @property + def legacy_protocol(self): + if self._legacy_protocol is None: + from openkyrozen.agent.protocol import LegacyProtocolAdapter + self._legacy_protocol = LegacyProtocolAdapter(self) + return self._legacy_protocol + + @legacy_protocol.setter + def legacy_protocol(self, adapter): + self._legacy_protocol = adapter @property def current_session(self): @@ -426,18 +435,23 @@ def chat(self, session, message, *, clear_tasks=False, profile=None, memory_cont ) except BaseException as exc: from openkyrozen.providers.errors import ProviderError, ProviderErrorKind - cancelled = isinstance(exc, (KeyboardInterrupt, asyncio.CancelledError)) or ( - isinstance(exc, ProviderError) and exc.kind == ProviderErrorKind.CANCELLED - ) + cancelled = (turn_context.cancelled.is_set() + or isinstance(exc, (KeyboardInterrupt, asyncio.CancelledError)) + or (isinstance(exc, ProviderError) and exc.kind == ProviderErrorKind.CANCELLED)) publisher.publish( "turn.cancelled" if cancelled else "turn.failed", {"error_type": type(exc).__name__}, ) raise - if publisher.last_model_failed or self._turn_failed.get(): + if turn_context.cancelled.is_set(): + publisher.publish("turn.cancelled", {"status": "cancelled"}) + elif publisher.last_model_failed or self._turn_failed.get(): publisher.publish("turn.failed", {"error_type": "provider_error"}) else: publisher.publish("turn.completed", {"status": self._turn_completion_status.get()}) + if self._last_response_is_native.get(): + from openkyrozen.agent.presentation import _NativeResponseText + return _NativeResponseText(reply) return reply finally: with self._active_turns_lock: @@ -547,6 +561,9 @@ def _ACTION_MARKER_RE(self): return cached[1] def run_command(self, args): + if self.execution_context.cancelled is not None and self.execution_context.cancelled.is_set(): + from openkyrozen.tools.models import CommandResult + return CommandResult("Command cancelled before it started.", False, failure="cancelled") if isinstance(args, dict) and args.get("background") is True: from openkyrozen.tools.models import CommandResult from openkyrozen.tools.sandbox import SandboxUnavailableError @@ -558,6 +575,7 @@ def run_command(self, args): try: handle = self.current_session.process_manager.start( command, cwd=cwd, env=env, timeout_seconds=timeout, + cancelled=self.execution_context.cancelled, ) message = f"Started background process {handle.process_id}" if handle.sandbox_backend == "unavailable": @@ -573,9 +591,6 @@ def run_command(self, args): except (OSError, RuntimeError, ValueError) as exc: return CommandResult(f"Error starting background process: {exc}", False, failure="execution_error") - if self.execution_context.cancelled is not None and self.execution_context.cancelled.is_set(): - from openkyrozen.tools.models import CommandResult - return CommandResult("Command cancelled before it started.", False, failure="cancelled") return self.current_session.workspace.adapters.run_command(args, cancelled=self.execution_context.cancelled) @legacy_text_api @@ -623,8 +638,7 @@ def _read_tool_artifact(self, args): or type(limit) is not int or not 1 <= limit <= 1000): return tool_failure("invalid_arguments", "Error: artifact_id, nonnegative offset, and limit from 1 to 1000 are required.") memory = self.memory_bank - task_owner = getattr(self.current_session, "_delegation_parent", None) or self - tasks = getattr(task_owner, "tasks", None) + tasks = getattr(self, "tasks", None) page = memory.store.artifacts.read_page( artifact_id, offset, limit, user_id=getattr(tasks, "user_id", None) or memory.user_id, diff --git a/openkyrozen/agent/turn.py b/openkyrozen/agent/turn.py index df53ccfa..3f87ce57 100644 --- a/openkyrozen/agent/turn.py +++ b/openkyrozen/agent/turn.py @@ -38,6 +38,7 @@ class TurnContext: turn_start: Any = None runtime_status: str = "completed" memory_packet: str = "" + native_tools: bool = False @property def planning_required(self) -> bool: @@ -62,6 +63,8 @@ class ExecutionContext: delegation_tools: frozenset[str] | None = None max_steps: int | None = None cancelled: Any = None + native_control_input: str = "" + native_plan_inspected: bool = False def scoped_turn(function): diff --git a/openkyrozen/interfaces/web/tasks.py b/openkyrozen/interfaces/web/tasks.py index 30f88840..d1e16854 100644 --- a/openkyrozen/interfaces/web/tasks.py +++ b/openkyrozen/interfaces/web/tasks.py @@ -155,6 +155,9 @@ async def api_v2_resume_task(self, task_id: str): raise HTTPException(404, "Failed or blocked task not found in this scope") manager = service._task_manager(stored.get("session_id")) manager.recover() + current = next((item for item in manager.tasks if item["id"] == str(task_id)), None) + if current and (current.get("checkpoint", {}).get("recovery_effect") or {}).get("status") == "unknown": + raise HTTPException(409, "Interrupted operation has unknown effects; inspect and reconcile it before resuming") task = manager.resume(task_id) if task is None: raise HTTPException(404, "Failed or blocked task not found in this scope") diff --git a/openkyrozen/learning/evidence.py b/openkyrozen/learning/evidence.py index eca9fcfe..d4f0496d 100644 --- a/openkyrozen/learning/evidence.py +++ b/openkyrozen/learning/evidence.py @@ -1,12 +1,22 @@ from __future__ import annotations from pathlib import Path +import math from typing import Any from openkyrozen.persistence.models import stable_hash from .models import CAPSULE_PROTOCOL, DEFAULT_CONSTITUTION, EVOLUTION_PROFILES, NEGATIVE_FEEDBACK, POSITIVE_FEEDBACK from openkyrozen.memory.models import _SECRET_RE +MEASURED_REPLAY_POLICY = "measured-replay-v1" + + +def _promotion_replay_ready(replay: dict[str, Any]) -> bool: + return (replay.get("measurement_policy") == MEASURED_REPLAY_POLICY + and replay.get("measurements_complete") is True + and replay.get("non_regressing") is True) + + def record_shadow_replay(self, proposal_id: str, candidate: list[dict[str, Any]], predecessor: list[dict[str, Any]]) -> dict[str, Any]: """Record paired, already-sandboxed replay results without executing commands.""" @@ -18,18 +28,71 @@ def record_shadow_replay(self, proposal_id: str, candidate: list[dict[str, Any]] predecessor_ids = [str(item.get("case_id", "")) for item in predecessor] if not all(candidate_ids) or candidate_ids != predecessor_ids or len(set(candidate_ids)) != len(candidate_ids): raise ValueError("paired replay case ids must be unique and identical") - candidate_successes = sum(bool(item.get("verified_success")) for item in candidate) - predecessor_successes = sum(bool(item.get("verified_success")) for item in predecessor) + if any(type(item.get("verified_success")) is not bool for item in candidate + predecessor): + raise ValueError("replay success must be a verified boolean") + candidate_successes = sum(item["verified_success"] for item in candidate) + predecessor_successes = sum(item["verified_success"] for item in predecessor) + outcome_safe = all(not previous["verified_success"] or current["verified_success"] + for current, previous in zip(candidate, predecessor)) + comparisons = {} + for field in ("tokens", "latency", "corrections", "repeated_errors", "tool_calls", + "cost", "cost_picos", "cost_rmb"): + pairs = [(current.get(field), previous[field]) for current, previous in zip(candidate, predecessor) + if field in previous] + if not pairs: + continue + if any(type(value) not in {int, float} or (type(value) is float and not math.isfinite(value)) or value < 0 + for pair in pairs for value in pair): + raise ValueError(f"paired {field} measurements must be finite nonnegative numbers") + current_total, previous_total = (sum(pair[index] for pair in pairs) for index in (0, 1)) + if any(type(total) is float and not math.isfinite(total) for total in (current_total, previous_total)): + raise ValueError(f"paired {field} totals must remain finite") + non_regressing = current_total <= previous_total + if field in {"corrections", "repeated_errors"}: + non_regressing = all(current <= previous for current, previous in pairs) + comparisons[field] = {"candidate": current_total, "predecessor": previous_total, + "non_regressing": non_regressing} + missing = [] + for case_id, current, previous in zip(candidate_ids, candidate, predecessor): + for side, item in (("candidate", current), ("predecessor", previous)): + for field in ("latency", "corrections", "repeated_errors"): + if field not in item: + missing.append(f"{case_id}:{side}.{field}") + cost_metric, cost_unit = None, None + for field in ("cost_rmb", "cost_picos", "cost"): + if not all(field in item for item in candidate + predecessor): + continue + if field == "cost": + units = [item.get("cost_unit") for item in candidate + predecessor] + if (not all(isinstance(unit, str) and unit.strip() and len(unit) <= 64 for unit in units) + or len(set(units)) != 1): + continue + cost_unit = units[0] + else: + cost_unit = field + cost_metric = field + break + if cost_metric is None: + missing.append("paired cost measurement in one consistent explicit unit") + measurements_complete = not missing models = sorted({str(item.get("provider_model")) for item in candidate if item.get("provider_model")}) result = { "case_ids": candidate_ids, "candidate_successes": candidate_successes, "predecessor_successes": predecessor_successes, - "non_regressing": candidate_successes >= predecessor_successes, + "non_regressing": measurements_complete and outcome_safe + and all(item["non_regressing"] for item in comparisons.values()), + "measurement_policy": MEASURED_REPLAY_POLICY, + "measurements_complete": measurements_complete, + "missing_measurements": missing, "cost_metric": cost_metric, "cost_unit": cost_unit, + "blocked_reason": ("Replay evidence incomplete: provide paired latency, corrections, repeated_errors " + "and cost in one consistent unit for every case.") if missing else None, + "metric_comparisons": comparisons, "environment": self.environment_fingerprint(), "model_scope": models, "cross_model_validated": len(models) >= 2, } validation = {**proposal.get("validation", {}), "shadow_replay": result, - "revalidation_status": "valid" if result["non_regressing"] else "failed"} + "revalidation_status": ("incomplete" if not measurements_complete else + "valid" if result["non_regressing"] else "failed")} self.store.update_proposal(proposal_id, status=proposal["status"], validation=validation) self.store.append_event("learning.shadow_replay", {"proposal_id": proposal_id, **result}, user_id=self.memory.user_id, workspace_id=self.memory.workspace_id, diff --git a/openkyrozen/learning/promotion.py b/openkyrozen/learning/promotion.py index c3bd716e..35a5e249 100644 --- a/openkyrozen/learning/promotion.py +++ b/openkyrozen/learning/promotion.py @@ -5,6 +5,7 @@ from .models import CAPSULE_PROTOCOL, DEFAULT_CONSTITUTION, EVOLUTION_PROFILES, NEGATIVE_FEEDBACK, POSITIVE_FEEDBACK from openkyrozen.memory.models import _SECRET_RE +from .evidence import MEASURED_REPLAY_POLICY, _promotion_replay_ready def record_outcome(self, run: dict[str, str], receipts: list[dict[str, Any]], *, verified: bool, success: bool, correction: bool = False, source: str = "runtime") -> list[str]: @@ -81,6 +82,13 @@ def _reconcile_artifact(self, skill_id: str) -> str | None: return f"Rolled back learned {skill['name']} after verified regression." successful_runs = {item["run_id"] for item in verified if item.get("success")} replay = (proposal or {}).get("validation", {}).get("shadow_replay") or {} + if replay and (replay.get("measurement_policy") != MEASURED_REPLAY_POLICY + or replay.get("measurements_complete") is not True): + replay = {**replay, "non_regressing": False, + "blocked_reason": "Replay evidence incomplete: measured paired cost, latency and regression evidence must be revalidated."} + self.store.update_proposal(proposal["id"], status=proposal["status"], validation={ + **proposal["validation"], "shadow_replay": replay, "revalidation_status": "incomplete", + }) verifier_ids = {item.get("source") for item in verified if item.get("success") and item.get("source")} verifiers_reliable = all( self.verifier_reliability(verifier)["reliability"] in {None, 1.0} @@ -88,7 +96,7 @@ def _reconcile_artifact(self, skill_id: str) -> str | None: for verifier in verifier_ids ) if (skill["status"] == "canary" and len(successful_runs) >= 2 and not any_failure and not corrected - and replay.get("non_regressing") is True and verifiers_reliable): + and _promotion_replay_ready(replay) and verifiers_reliable): if self.registry.activate_learned(skill_id): if proposal: self.store.update_proposal(proposal["id"], status="active", confidence=0.9, @@ -110,11 +118,13 @@ def validate_and_activate(self, proposal_id: str, *, validation: dict[str, Any]) if not proposal: return False skill_id = proposal.get("validation", {}).get("skill_id") - if skill_id: - return bool(self.registry and self.registry.activate_learned(skill_id)) - self.store.update_proposal(proposal_id, status="active", confidence=max(0.8, proposal["confidence"]), - validation=validation) - self.memory.add_log(proposal["content"], kind=proposal["kind"], status="active", - confidence=max(0.8, proposal["confidence"]), - metadata={"proposal_id": proposal_id, "validation": validation}) - return True + if not skill_id or self.registry is None: + return False + # Caller-supplied success is a claim, never a substitute for persisted promotion evidence. + self._reconcile_artifact(skill_id) + current = self._proposal_for_skill(skill_id) or {} + evidence = current.get("validation", {}) + return (current.get("status") == "active" and evidence.get("stage") == "active" + and evidence.get("verified_successes", 0) >= 2 + and _promotion_replay_ready(evidence.get("shadow_replay") or {}) + and any(item["id"] == skill_id and item["status"] == "active" for item in self.registry.list())) diff --git a/openkyrozen/persistence/database.py b/openkyrozen/persistence/database.py index d27582e6..ffb58b88 100644 --- a/openkyrozen/persistence/database.py +++ b/openkyrozen/persistence/database.py @@ -8,6 +8,7 @@ import time import warnings from contextlib import contextmanager +from contextvars import ContextVar from pathlib import Path from typing import Any, Iterator @@ -197,19 +198,74 @@ "tool_artifacts": {"id", "user_id", "workspace_id", "session_id", "content", "created_at"}, } +class _DeadlineRLock: + """Preserve ordinary RLock behavior, with an opt-in cancellation deadline.""" + def __init__(self, remaining): + self._lock = threading.RLock() + self._remaining = remaining + + def acquire(self, blocking=True, timeout=-1): + remaining = self._remaining() + if remaining is None: + return self._lock.acquire(blocking, timeout) + if remaining <= 0: + return False + if not blocking: + return self._lock.acquire(False) + return self._lock.acquire(timeout=remaining if timeout < 0 else min(timeout, remaining)) + + def release(self): + return self._lock.release() + + def __enter__(self): + if not self.acquire(): + raise sqlite3.OperationalError("cancellation persistence deadline exceeded") + return self + + def __exit__(self, *args): + self.release() + + def __getattr__(self, name): + return getattr(self._lock, name) + + class SQLiteDatabase: SCHEMA_VERSION = _MIGRATIONS[-1][0] def __init__(self, path: str | os.PathLike[str] | None = None): configured = path or os.environ.get("KYROZEN_DB_PATH") self.path = Path(configured or Path.home() / ".kyrozen" / "v2" / "openkyrozen.sqlite3").expanduser() self.path.parent.mkdir(parents=True, exist_ok=True) - self._lock = threading.RLock() + self._control_deadline = ContextVar("sqlite_control_deadline", default=None) + self._lock = _DeadlineRLock(self._remaining_control_seconds) self.last_backup_path: Path | None = None self._initialise() + def _remaining_control_seconds(self): + control = self._control_deadline.get() + if control is None or control[1] != threading.get_ident(): + return None + return max(0, control[0] - time.monotonic()) + + @contextmanager + def _cancellation_deadline(self): + remaining = self._remaining_control_seconds() + deadline = time.monotonic() + min(.3, remaining) if remaining is not None else time.monotonic() + .3 + token = self._control_deadline.set((deadline, threading.get_ident())) + try: + yield + finally: + self._control_deadline.reset(token) + @contextmanager def connection(self, *, timeout: float = 30, busy_timeout_ms: int | None = None) -> Iterator[sqlite3.Connection]: + remaining = self._remaining_control_seconds() + if remaining is not None: + if remaining <= 0: + raise sqlite3.OperationalError("cancellation persistence deadline exceeded") + timeout = min(timeout, remaining) + busy_timeout_ms = min(30000 if busy_timeout_ms is None else busy_timeout_ms, + int(remaining * 1000)) connection = sqlite3.connect(str(self.path), timeout=timeout, isolation_level=None) connection.row_factory = sqlite3.Row try: diff --git a/openkyrozen/persistence/events.py b/openkyrozen/persistence/events.py index b2618f8f..65072d84 100644 --- a/openkyrozen/persistence/events.py +++ b/openkyrozen/persistence/events.py @@ -21,6 +21,26 @@ def append_event(self, event_type: str, payload: Any, *, user_id: str = "local", ) return event_id + def _append_control_audit(self, event_type: str, payload: Any, *, user_id: str, + workspace_id: str, session_id: str) -> str: + """Bound control-path audit contention; cleanup must not wait 30 seconds.""" + if event_type not in {"process.lifecycle", "turn.effect_uncertain"}: + raise ValueError("unsupported control audit event") + encoded = self.database._json(payload) + if not self.database._lock.acquire(timeout=.1): + raise RuntimeError("control audit journal is busy") + try: + event_id = f"evt_{uuid.uuid4().hex}" + with self.database.connection(timeout=.1, busy_timeout_ms=100) as db: + db.execute( + "INSERT INTO events(id,event_type,payload,user_id,workspace_id,session_id,task_id,created_at) " + "VALUES(?,?,?,?,?,?,?,?)", + (event_id, event_type, encoded, user_id, workspace_id, session_id, None, utc_now()), + ) + return event_id + finally: + self.database._lock.release() + def conversation_head(self, *, user_id: str, workspace_id: str, session_id: str) -> str | None: with self.database.connection() as db: row = db.execute( diff --git a/openkyrozen/persistence/store.py b/openkyrozen/persistence/store.py index ea9f1c5c..7c593aaf 100644 --- a/openkyrozen/persistence/store.py +++ b/openkyrozen/persistence/store.py @@ -38,6 +38,14 @@ def __init__(self, path=None): def append_event(self, event_type: str, payload: Any, *, user_id: str='local', workspace_id: str='default', session_id: str | None=None, task_id: str | None=None): return self.events.append_event(event_type, payload, user_id=user_id, workspace_id=workspace_id, session_id=session_id, task_id=task_id) + def _cancellation_deadline(self): + return self.database._cancellation_deadline() + + def _append_control_audit(self, event_type: str, payload: Any, *, user_id: str, + workspace_id: str, session_id: str): + return self.events._append_control_audit(event_type, payload, user_id=user_id, + workspace_id=workspace_id, session_id=session_id) + def conversation_head(self, *, user_id: str, workspace_id: str, session_id: str): return self.events.conversation_head(user_id=user_id, workspace_id=workspace_id, session_id=session_id) diff --git a/openkyrozen/providers/calls.py b/openkyrozen/providers/calls.py index fa120aa5..9dd6e2ac 100644 --- a/openkyrozen/providers/calls.py +++ b/openkyrozen/providers/calls.py @@ -4,6 +4,8 @@ import math import sys import uuid +import hashlib +import json from typing import Any from openkyrozen.providers.usage import usage_scope from openkyrozen.providers.base import get_model_response, get_model_events @@ -119,6 +121,21 @@ def _get_model_response(self, messages: list[dict], model: str | None = None, *, self._last_model_response.set(None) self._turn_failed.set(False) selected_model = model or self.DEEPSEEK_MODEL + state = self._active_context_state.get() + if state is not None and not self._in_context_compaction.get(): + from openkyrozen.providers.models import provider_identity + state.request_provider_id = provider_identity(provider) + state.request_model = selected_model + if tool_names is None: + state.native_input_overhead_tokens = 0 + state.native_schema_fingerprint = "" + if tool_names is not None and not self._in_context_compaction.get(): + if state is not None: + from openkyrozen.agent.compaction import _text_tokens + schema_json = json.dumps(tools, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + state.native_input_overhead_tokens = _text_tokens(schema_json) + 32 * len(tools or ()) + 64 + state.native_schema_fingerprint = hashlib.sha256(schema_json.encode("utf-8")).hexdigest() + self._prepare_context_for_call(messages, selected_model) if self._active_context_state.get() is not None and not self._in_context_compaction.get(): self._record_context_history_metrics(messages) request_id = f"model_{uuid.uuid4().hex}" diff --git a/openkyrozen/security/permissions.py b/openkyrozen/security/permissions.py index ed788cb9..9b246a89 100644 --- a/openkyrozen/security/permissions.py +++ b/openkyrozen/security/permissions.py @@ -194,8 +194,9 @@ def _authorize_tool_action(self, action: str, args: object, *, approve=None) -> return True if mode == "full_jev" and not risk: return True - if mode == "full_jev" and risk and self._jev_permission_check(action, risk): - return True + if mode == "full_jev" and risk: + # Model advice cannot grant authority for a high-impact operation. + self._jev_permission_check(action, risk) if mode == "ask" and not requires_ask_approval(action, args, self.AVAILABLE_TOOLS): return True fallback_approval = approve is None and not callable(self._approval_callback.get()) diff --git a/openkyrozen/tasks/engine.py b/openkyrozen/tasks/engine.py index 7595d0ef..bca94712 100644 --- a/openkyrozen/tasks/engine.py +++ b/openkyrozen/tasks/engine.py @@ -12,6 +12,10 @@ from .models import CANONICAL_STATUSES, TERMINAL_STATUSES, VALID_STATUSES, WorkingPlan, _ORDERED_ACTION_FAMILIES, _ORDERED_ACTION_HINTS, _RECEIPT_ACTION_ALIASES, _ordered_action_compatible, _ordered_plan_id, _ordered_plan_info, _receipt_action, _receipt_args, _scoped_task_id, _task_key, canonical_status, is_complete, is_terminal, valid_task_transition from .ports import TaskStore +def _unknown_recovery_effect(task): + return ((task.get("checkpoint") or {}).get("recovery_effect") or {}).get("status") == "unknown" + + class TaskManager: """Compatibility facade backed by SQLite and guarded by evidence.""" @@ -22,7 +26,7 @@ def __init__(self, store: TaskStore, *, workspace_id: str = "default", self.session_id = session_id self.user_id = user_id self.tasks: list[dict[str, Any]] = [] - self._pending_completions: set[int] = set() + self._pending_completions: set[str] = set() self._evidence: list[dict[str, Any]] = [] self.working_plan: WorkingPlan | None = None self._runtime_event_callback: ContextVar[Any] = ContextVar("task_runtime_event_callback", default=None) @@ -73,6 +77,10 @@ def working_plan_snapshot(self) -> dict[str, Any] | None: "superseded_by": list(checkpoint.get("superseded_by", [])) if isinstance(checkpoint, dict) else [], "reason": str(block.get("reason", "")) if isinstance(block, dict) else "", }) + if isinstance(checkpoint, dict) and checkpoint.get("strategic_plan"): + tasks[-1]["role"] = "implementation" if checkpoint.get("strategy_anchor") else "approved_objective" + if checkpoint.get("strategy_anchor"): + tasks[-1]["strategy_anchor"] = checkpoint["strategy_anchor"] reason = "" events = self.store.list_events( "working_plan.updated", limit=20, user_id=self.user_id, @@ -144,6 +152,8 @@ def set_status(self, idx: int, status: str, *, evidence: dict[str, Any] | None = if not 0 <= idx < len(self.tasks): return False status = canonical_status(status) + if status in {"pending", "running"} and _unknown_recovery_effect(self.tasks[idx]): + return False current = canonical_status(self.tasks[idx].get("status", "pending")) if not valid_task_transition(current, status): return False @@ -152,7 +162,8 @@ def set_status(self, idx: int, status: str, *, evidence: dict[str, Any] | None = if status == "blocked" and self.tasks[idx].get("plan_id") and ( not isinstance(reason, str) or not reason.strip()): raise ValueError("blocking a Working Plan task requires a reason") - if status == "succeeded" and not self._has_evidence(idx, evidence): + if status == "succeeded" and (not self._has_evidence(idx, evidence) + or not self._dependencies_succeeded(self.tasks[idx], self.tasks)): return False task = deepcopy(self.tasks[idx]) expected_status = self.tasks[idx].get("status", "pending") @@ -242,6 +253,7 @@ def edit_plan(self, *, expected_revision: int, operations: list[dict[str, Any]], complexity: str | None = None, plan_id: str | None = None, strategic_bindings: dict[str, dict[str, Any]] | None = None, transaction_event: dict[str, Any] | None = None, + strategy_validator=None, reason: str | None = None) -> dict[str, Any]: """Apply a validated ordered operation batch as one revision-checked transaction.""" if type(expected_revision) is not int or expected_revision < 0: @@ -313,12 +325,51 @@ def edit_plan(self, *, expected_revision: int, operations: list[dict[str, Any]], and transaction_event.get("expected_proposal") ) references: dict[str, str] = {} + completion_requests: list[str] = [] used_strategic_bindings: set[str] = set() now = utc_now() scope_key = _task_key(f"{self.user_id}:{self.workspace_id}:{self.session_id or ''}") + def strategy_checkpoint(anchor_id: str, fields: dict[str, Any]) -> dict[str, Any]: + anchor = by_id[anchor_id] + binding = (anchor.get("checkpoint") or {}).get("strategic_plan") + if not binding or (anchor.get("checkpoint") or {}).get("strategy_anchor"): + raise ValueError("strategy anchor must identify an approved objective task") + checkpoint = fields["checkpoint"] + if checkpoint.get("action"): + execution = {"action": checkpoint["action"], "arguments": json.loads(checkpoint["args"])} + if callable(strategy_validator): + strategy_validator(binding, execution) + elif execution not in [binding.get("execution"), *binding.get("acceptance_verifications", [])]: + raise ValueError("strategy execution requires validation against accepted bounds") + return {**checkpoint, "strategic_plan": deepcopy(binding), "strategy_anchor": anchor_id} + + def add_strategy(anchor_id: str, ref: str, task_data: Any) -> str: + if not isinstance(ref, str) or not ref.strip() or ref in references or ref in by_id: + raise ValueError("task references must be unique nonempty strings") + anchor = by_id[anchor_id] + if anchor["status"] != "pending" or anchor.get("attempts") or anchor.get("evidence"): + raise ValueError("strategy refinement requires an unexecuted objective anchor") + fields = self._plan_task_fields(task_data) + checkpoint = strategy_checkpoint(anchor_id, fields) + task_id = f"task_{scope_key}_{uuid.uuid4().hex}" + task = {"id": task_id, **fields, "checkpoint": checkpoint, + "status": "pending", "attempts": 0, "evidence": [], + "created_at": now, "updated_at": now, "plan_id": plan_id, "parent_id": anchor_id} + references[ref] = task_id + tasks.append(task) + by_id[task_id] = task + anchor["dependencies"] = list(dict.fromkeys([*anchor.get("dependencies", []), task_id])) + return task_id + def supersede(source_ids: list[str], ref: str, task_data: Any) -> str: - if any((by_id[item].get("checkpoint") or {}).get("strategic_plan") for item in source_ids): + if any(_unknown_recovery_effect(by_id[item]) for item in source_ids): + raise ValueError("interrupted effects require trusted reconciliation before replacement") + strategic_sources = [by_id[item] for item in source_ids + if (by_id[item].get("checkpoint") or {}).get("strategic_plan")] + anchors = {(item.get("checkpoint") or {}).get("strategy_anchor") for item in strategic_sources} + if strategic_sources and (None in anchors or len(anchors) != 1 + or len(strategic_sources) != len(source_ids)): raise ValueError("Strategic Plan tasks cannot be superseded without approval") if not isinstance(ref, str) or not ref.strip() or ref in references or ref in by_id: raise ValueError("task references must be unique nonempty strings") @@ -341,6 +392,8 @@ def supersede(source_ids: list[str], ref: str, task_data: Any) -> str: task = {"id": task_id, **fields, "status": "pending", "attempts": 0, "evidence": [], "created_at": now, "updated_at": now, "plan_id": plan_id, "parent_id": source_ids[0], "checkpoint": {**fields["checkpoint"], "supersedes": source_ids}} + if strategic_sources: + task["checkpoint"].update(strategy_checkpoint(next(iter(anchors)), fields)) for source in sources: checkpoint = dict(source.get("checkpoint") or {}) checkpoint["superseded_by"] = [task_id] @@ -375,6 +428,7 @@ def resolve(value: Any) -> str: "split": {"op", "task", "children"}, "replace": {"op", "task", "replacement"}, "merge": {"op", "tasks", "replacement"}, "block": {"op", "task", "reason"}, "unblock": {"op", "task", "reason"}, + "start": {"op", "task"}, "suggest_complete": {"op", "task"}, } if saved is None and not any(isinstance(item, dict) and item.get("op") == "add" for item in operations): raise ValueError("a new Working Plan requires at least one add operation") @@ -382,9 +436,13 @@ def resolve(value: Any) -> str: if not isinstance(operation, dict) or operation.get("op") not in allowed_by_op: raise ValueError("each plan operation requires a supported op") kind = operation["op"] - if set(operation) != allowed_by_op[kind]: + if set(operation) != allowed_by_op[kind] and not ( + kind == "add" and set(operation) == allowed_by_op[kind] | {"anchor"}): raise ValueError(f"{kind} operation has missing or unsupported fields") if kind == "add": + if "anchor" in operation: + add_strategy(resolve(operation["anchor"]), operation["ref"], operation["task"]) + continue ref = operation["ref"] if not isinstance(ref, str) or not ref.strip() or ref in references or ref in by_id: raise ValueError("task references must be unique nonempty strings") @@ -464,6 +522,7 @@ def resolve(value: Any) -> str: current = by_id[task_id] if (has_strategic_tasks and isinstance(current.get("checkpoint"), dict) and isinstance(current["checkpoint"].get("strategic_plan"), dict) + and not current["checkpoint"].get("strategy_anchor") and set(fields) - {"priority", "dependencies"}): raise ValueError("Strategic Plan task intent cannot change without approval") validated = self._plan_task_fields({ @@ -483,6 +542,9 @@ def resolve(value: Any) -> str: } base = {key: value for key, value in checkpoint.items() if key not in excluded} normalized["checkpoint"] = {**base, **validated["checkpoint"]} + anchor_id = checkpoint.get("strategy_anchor") + if anchor_id: + normalized["checkpoint"].update(strategy_checkpoint(anchor_id, validated)) changed = any(current.get(key) != value for key, value in normalized.items()) if changed and (current["status"] != "pending" or current.get("attempts", 0) or current.get("evidence")): @@ -503,6 +565,14 @@ def resolve(value: Any) -> str: children = operation["children"] if not isinstance(children, list) or len(children) < 2: raise ValueError("split requires at least two children") + original = by_id[source_ids[0]] + if ((original.get("checkpoint") or {}).get("strategic_plan") + and not (original.get("checkpoint") or {}).get("strategy_anchor")): + for child in children: + if not isinstance(child, dict) or set(child) != {"ref", "task"}: + raise ValueError("each split child requires ref and task") + add_strategy(original["id"], child["ref"], child["task"]) + continue for child in children: if not isinstance(child, dict) or set(child) != {"ref", "task"}: raise ValueError("each split child requires ref and task") @@ -526,17 +596,35 @@ def resolve(value: Any) -> str: elif kind in {"block", "unblock"}: task_id = resolve(operation["task"]) current = by_id[task_id] + if kind == "unblock" and _unknown_recovery_effect(current): + raise ValueError("interrupted effects require trusted reconciliation before resuming") why = operation["reason"] if not isinstance(why, str) or not why.strip(): raise ValueError(f"{kind} requires a nonempty reason") - if has_strategic_tasks and (current.get("checkpoint") or {}).get("strategic_plan"): - raise ValueError("Strategic Plan task status requires approval") expected, target = ("pending", "blocked") if kind == "block" else ("blocked", "pending") if current["status"] != expected: raise ValueError(f"{kind} requires a {expected} task") checkpoint = dict(current.get("checkpoint") or {}) checkpoint["plan_block"] = {"reason": why.strip(), "action": kind} current.update(status=target, checkpoint=checkpoint, updated_at=now) + elif kind in {"start", "suggest_complete"}: + task_id = resolve(operation["task"]) + current = by_id[task_id] + if _unknown_recovery_effect(current): + raise ValueError("interrupted effects require trusted reconciliation before execution") + if (current["status"] not in {"pending", "running"} + or not self._dependencies_succeeded(current, tasks)): + raise ValueError("task must be active with all dependencies satisfied") + if kind == "start": + if any(task["id"] != task_id and task["status"] == "running" for task in tasks): + raise ValueError("another task is already running") + current.update(status="running", updated_at=now, + checkpoint={**current.get("checkpoint", {}), "native_execution_selected": True}) + else: + if self._task_has_evidence(current): + current.update(status="succeeded", updated_at=now) + else: + completion_requests.append(task_id) elif kind == "reorder": requested = operation["tasks"] if not isinstance(requested, list): @@ -630,6 +718,9 @@ def resolve(value: Any) -> str: ) self.working_plan = working_plan self.tasks = tasks + [task for task in self.tasks if task["id"] not in original_ids] + for task_id in completion_requests: + self.request_completion(next(index for index, task in enumerate(self.tasks) if task["id"] == task_id)) + self.reconcile_completions(set(completion_requests)) payload = {"plan_id": plan_id, "revision": revision, "task_ids": list(working_plan.task_ids), "status": plan_status} snapshot = self.working_plan_snapshot() or { @@ -647,19 +738,22 @@ def resolve(value: Any) -> str: callback(event_type, snapshot, tasks[-1]["id"] if tasks else None) except Exception: pass - return {**payload, "complexity": complexity, "tasks": deepcopy(tasks), + return {**payload, "status": self.working_plan.status, "complexity": complexity, + "tasks": deepcopy([task for task in self.tasks if task["id"] in working_plan.task_ids]), "references": dict(references)} def update_plan(self, *, complexity: str | None = None, tasks: list[dict[str, Any]] | None = None, plan_id: str | None = None, expected_revision: int | None = None, operations: list[dict[str, Any]] | None = None, + strategy_validator=None, reason: str | None = None) -> WorkingPlan | dict[str, Any]: """Apply canonical operations or adapt the historical complexity/tasks shape.""" if operations is not None: if expected_revision is None: raise ValueError("expected_revision is required for plan edits") return self.edit_plan(complexity=complexity, plan_id=plan_id, - expected_revision=expected_revision, operations=operations, reason=reason) + expected_revision=expected_revision, operations=operations, reason=reason, + strategy_validator=strategy_validator) if not isinstance(tasks, list) or not tasks: raise ValueError("a Working Plan requires at least one task") saved = self.store.get_working_plan( @@ -722,8 +816,14 @@ def adopt_legacy_plan(self, *, complexity: str = "complex") -> WorkingPlan: return self.working_plan def _has_evidence(self, idx: int, evidence: dict[str, Any] | None = None) -> bool: - evidence_items = ([evidence] if evidence else []) + self.tasks[idx].get("evidence", []) - strategic = ((self.tasks[idx].get("checkpoint") or {}).get("strategic_plan") or {}) + return self._task_has_evidence(self.tasks[idx], evidence) + + @staticmethod + def _task_has_evidence(task: dict[str, Any], evidence: dict[str, Any] | None = None) -> bool: + evidence_items = ([evidence] if evidence else []) + task.get("evidence", []) + strategic = ((task.get("checkpoint") or {}).get("strategic_plan") or {}) + if (task.get("checkpoint") or {}).get("strategy_anchor"): + strategic = {} required = set(strategic.get("acceptance_ids", ())) if required: covered = { @@ -973,12 +1073,21 @@ def record_evidence(self, *, task_id: str | None = None, action: str, result: st def request_completion(self, idx: int) -> None: if 0 <= idx < len(self.tasks): - self._pending_completions.add(idx) + self._pending_completions.add(self.tasks[idx]["id"]) + + @property + def execution_task_id(self) -> str | None: + """Only an explicitly started native task owns subsequent native receipts.""" + selected = [task for task in self.tasks if task.get("status") == "running" + and (task.get("checkpoint") or {}).get("native_execution_selected") + and self.active_working_task(task["id"]) is not None] + return selected[0]["id"] if len(selected) == 1 else None def active_task_id(self) -> str | None: """Return the next task that may receive an execution receipt.""" for task in self.tasks: if (task.get("status") in {"pending", "running"} + and not _unknown_recovery_effect(task) and self._dependencies_succeeded(task, self.tasks)): return task["id"] return None @@ -989,7 +1098,11 @@ def active_working_task(self, task_id: str) -> dict[str, Any] | None: if plan is None or plan.status != "active" or task_id not in plan.task_ids: return None task = next((item for item in self.tasks if item.get("id") == task_id), None) + stored = self._reload_task(task_id) if (task is None or task.get("plan_id") != plan.id + or stored is None or _unknown_recovery_effect(stored) + or canonical_status(stored.get("status", "pending")) not in {"pending", "running"} + or _unknown_recovery_effect(task) or canonical_status(task.get("status", "pending")) not in {"pending", "running"} or not self._dependencies_succeeded(task, self.tasks)): return None @@ -1045,6 +1158,8 @@ def resume(self, task_id: str) -> dict[str, Any] | None: if index is None: return None task = self.tasks[index] + if _unknown_recovery_effect(task): + return None if task.get("status") not in {"failed", "blocked"}: return task if task.get("status") == "pending" else None previous = task["status"] @@ -1062,13 +1177,17 @@ def resume(self, task_id: str) -> dict[str, Any] | None: return task def reconcile_completions(self, task_ids: set[str] | None = None) -> None: - for idx in list(self._pending_completions): - if task_ids is not None and self.tasks[idx]["id"] not in task_ids: + for task_id in list(self._pending_completions): + if task_ids is not None and task_id not in task_ids: + continue + idx = next((index for index, task in enumerate(self.tasks) if task["id"] == task_id), None) + if idx is None or is_terminal(self.tasks[idx]): + self._pending_completions.discard(task_id) continue if self._has_evidence(idx): self.set_status(idx, "succeeded") if is_complete(self.tasks[idx]): - self._pending_completions.discard(idx) + self._pending_completions.discard(task_id) def mark_done(self, idx: int, *, evidence: dict[str, Any] | None = None) -> bool: return self.set_status(idx, "succeeded", evidence=evidence) @@ -1193,18 +1312,34 @@ def mark_done_from_text(self, text: str) -> None: def recover(self) -> list[dict[str, Any]]: """Load unfinished tasks from SQLite and make running tasks recoverable.""" + def recover_running(task): + if task["status"] != "running": + return + from openkyrozen.tools.catalog import builtin_aliases, builtin_metadata + action = (task.get("checkpoint") or {}).get("action") + metadata = builtin_metadata(builtin_aliases().get(action, action)) if action else None + read_only = (metadata and metadata["capability"] == "read" + and metadata["side_effects"] in {"none", "read", "read_only"}) + uncertain = not read_only + task["status"] = "blocked" if uncertain else "pending" + if uncertain: + task["checkpoint"] = {**task.get("checkpoint", {}), "recovery_effect": { + "status": "unknown", "action": action, + "reason": "Runtime restarted after dispatch; inspect effects before replay.", + }} + task["updated_at"] = utc_now() + if self._persist(task, expected_status="running"): + self._event(task, "task.recovered", {"previous_status": "running", + "effect_status": "unknown" if uncertain else "read_only_or_unassigned"}) + else: + latest = self._reload_task(task["id"]) + if latest is not None: + task.update(latest) + unfinished = self.store.list_tasks(workspace_id=self.workspace_id, session_id=self.session_id, statuses={"pending", "running", "failed", "blocked"}, user_id=self.user_id) for task in unfinished: - if task["status"] == "running": - task["status"] = "pending" - task["updated_at"] = utc_now() - if self._persist(task, expected_status="running"): - self._event(task, "task.recovered", {"previous_status": "running"}) - else: - latest = self._reload_task(task["id"]) - if latest is not None: - task.update(latest) + recover_running(task) saved_plan = self.store.get_working_plan( user_id=self.user_id, workspace_id=self.workspace_id, session_id=self.session_id or "", ) @@ -1214,15 +1349,7 @@ def recover(self) -> list[dict[str, Any]]: session_id=self.session_id or "", ) for task in plan_tasks: - if task["status"] == "running": - task["status"] = "pending" - task["updated_at"] = utc_now() - if self._persist(task, expected_status="running"): - self._event(task, "task.recovered", {"previous_status": "running"}) - else: - latest = self._reload_task(task["id"]) - if latest is not None: - task.update(latest) + recover_running(task) terminal = bool(plan_tasks) and all( task["status"] in {"succeeded", "done", "cancelled"} for task in plan_tasks ) diff --git a/openkyrozen/tasks/planning.py b/openkyrozen/tasks/planning.py new file mode 100644 index 00000000..2fcc3c0d --- /dev/null +++ b/openkyrozen/tasks/planning.py @@ -0,0 +1,644 @@ +"""Feature-owned Strategic Plan validation, approval transitions and completion. + +TaskManager owns Working Plan edits; PlanningService coordinates approved +objectives with those edits through the same atomic storage contracts. +""" +from __future__ import annotations + +import hashlib +import json +import posixpath +import re +import uuid +from copy import deepcopy +from typing import Any + +from .ports import InteractionStore + +class InteractionError(ValueError): + """A bounded interaction payload failed validation.""" + + +def _text(value: Any, field: str, limit: int) -> str: + if not isinstance(value, str): + raise InteractionError(f"{field} must be text") + value = value.strip() + if not value or len(value) > limit: + raise InteractionError(f"{field} must be 1-{limit} characters") + return value + + +def validate_plan_proposal(value: Any, *, previous: dict[str, Any] | None = None, + original_input: str = "") -> dict[str, Any]: + if not isinstance(value, dict): + raise InteractionError("PlanProposal must contain a JSON object") + steps = value.get("steps") + if not isinstance(steps, list) or not 1 <= len(steps) <= 10: + raise InteractionError("PlanProposal requires 1-10 steps") + previous_version = int(previous.get("version", 0)) if previous else 0 + version = previous_version + 1 + normalised = [] + seen_ids: set[str] = set() + for index, step in enumerate(steps, 1): + if not isinstance(step, dict): + raise InteractionError("each plan step must be an object") + step_id = _text(step.get("id") or f"step-{index}", "step id", 64) + if step_id in seen_ids: + raise InteractionError("plan step ids must be unique") + seen_ids.add(step_id) + execution = step.get("execution") + if execution is not None: + action = execution.get("action") if isinstance(execution, dict) else None + arguments = execution.get("arguments") if isinstance(execution, dict) else None + if (not isinstance(execution, dict) or set(execution) != {"action", "arguments"} + or not isinstance(action, str) or action not in {"write_file", "edit_file", "apply_patch"} + or not isinstance(arguments, dict) + or not isinstance(arguments.get("path"), str) or not arguments["path"].strip()): + raise InteractionError("file execution contract must name write_file, edit_file, or apply_patch and an exact path") + required = ({"path", "content"} if action == "write_file" else + {"path", "expected_sha256", "hunks"} if action == "apply_patch" else + {"path", "old_text", "new_text", "expected_sha256"}) + if set(arguments) != required or (action != "apply_patch" and any(not isinstance(item, str) for item in arguments.values())): + raise InteractionError("file execution arguments must exactly match the selected file operation") + if action == "edit_file" and ( + not arguments["old_text"] or not re.fullmatch(r"[0-9a-fA-F]{64}", arguments["expected_sha256"])): + raise InteractionError("edit_file requires nonempty old_text and a SHA-256 read hash") + if action == "apply_patch" and ( + not isinstance(arguments["path"], str) or not arguments["path"].strip() + or not isinstance(arguments["expected_sha256"], str) + or not re.fullmatch(r"[0-9a-fA-F]{64}", arguments["expected_sha256"]) + or not isinstance(arguments["hunks"], list) or not arguments["hunks"] + or any(not isinstance(hunk, dict) or set(hunk) != {"old_text", "new_text"} + or not isinstance(hunk.get("old_text"), str) or not hunk["old_text"] + or not isinstance(hunk.get("new_text"), str) for hunk in arguments["hunks"])): + raise InteractionError("apply_patch requires a SHA-256 read hash and exact-text hunks") + try: + json.dumps(arguments, ensure_ascii=False, allow_nan=False) + except (TypeError, ValueError) as exc: + raise InteractionError("file execution arguments must be JSON-safe") from exc + execution = {"action": action, "arguments": deepcopy(arguments)} + acceptance = step.get("acceptance") + if isinstance(acceptance, str): + acceptance = [acceptance] + if not isinstance(acceptance, list) or not 1 <= len(acceptance) <= 5: + raise InteractionError("each plan step requires 1-5 acceptance criteria") + supplied_verifications = step.get("acceptance_verifications") + if supplied_verifications is not None and ( + not isinstance(supplied_verifications, list) or len(supplied_verifications) != len(acceptance)): + raise InteractionError("acceptance verifications must align with acceptance criteria") + acceptance_verifications = [] + normalized_acceptance = [] + for criterion_index, item in enumerate(acceptance): + verification = supplied_verifications[criterion_index] if supplied_verifications is not None else None + if isinstance(item, dict) and set(item) == {"criterion", "verification"}: + verification = item["verification"] + if verification is not None: + if (not isinstance(verification, dict) + or set(verification) not in ({"action", "arguments"}, {"action", "arguments", "expect"}) + or not isinstance(verification.get("action"), str) + or not verification["action"].strip() or not isinstance(verification.get("arguments"), dict)): + raise InteractionError("acceptance verification must contain an action and object arguments") + if verification["action"] == "read_file": + expect = verification.get("expect") + if (not isinstance(expect, dict) or len(expect) != 1 + or set(expect) not in ({"equals"}, {"contains"}) + or not isinstance(next(iter(expect.values())), str) + or ("contains" in expect and not expect["contains"])): + raise InteractionError("read_file verification requires a nonempty equals or contains predicate") + elif "expect" in verification: + raise InteractionError("expected-result predicates are currently supported only for read_file") + try: + json.dumps(verification["arguments"], ensure_ascii=False, allow_nan=False) + except (TypeError, ValueError) as exc: + raise InteractionError("acceptance verification arguments must be JSON-safe") from exc + criterion = _text(item.get("criterion") if isinstance(item, dict) else item, + "acceptance criterion", 300) + normalized_acceptance.append(criterion) + acceptance_verifications.append(verification) + acceptance = normalized_acceptance + acceptance_ids = [] + for criterion, verification in zip(acceptance, acceptance_verifications): + identity = json.dumps({"criterion": criterion, "verification": verification}, sort_keys=True, + ensure_ascii=False, separators=(",", ":")) + acceptance_ids.append(f"{step_id}-ac-{hashlib.sha256(identity.encode('utf-8')).hexdigest()[:12]}") + if len(set(acceptance_ids)) != len(acceptance_ids): + raise InteractionError("acceptance criteria must be unique within each objective") + normalised_step = { + "id": step_id, + "title": _text(step.get("title") or f"Step {index}", "step title", 120), + "description": _text(step.get("description"), "step description", 800), + "acceptance": acceptance, + "acceptance_ids": acceptance_ids, + "acceptance_verifications": acceptance_verifications, + } + if execution is not None: + normalised_step["execution"] = execution + normalised.append(normalised_step) + assumptions = value.get("assumptions", []) + if assumptions is None: + assumptions = [] + if not isinstance(assumptions, list) or len(assumptions) > 10: + raise InteractionError("assumptions must contain at most 10 items") + raw_bounds = value.get("bounds") + bounds_source = "proposal" + if raw_bounds is None: + if "bounds" in value: + raise InteractionError("bounds must be an object") + raw_bounds = { + "workspace_paths": ["."], "capabilities": ["read"], + "effects": ["read_only"], "risk_ceiling": "normal", + } + bounds_source = "legacy-default" + if not isinstance(raw_bounds, dict) or set(raw_bounds) - { + "workspace_paths", "capabilities", "effects", "risk_ceiling", + }: + raise InteractionError("bounds must declare workspace paths, capabilities, effects, and risk ceiling") + + def values(field: str, allowed: set[str] | None = None) -> list[str]: + raw = raw_bounds.get(field, []) + if not isinstance(raw, list) or len(raw) > 32: + raise InteractionError(f"bounds.{field} must be an array with at most 32 values") + result = [] + for item in raw: + if not isinstance(item, str) or not item.strip(): + raise InteractionError(f"bounds.{field} entries must be nonempty strings") + item = item.strip() + if allowed is not None and item not in allowed: + raise InteractionError(f"unsupported bounds.{field} value: {item}") + result.append(item) + if len(set(result)) != len(result): + raise InteractionError(f"bounds.{field} values must be unique") + return result + + paths = values("workspace_paths") + normalised_paths = [] + for path in paths: + path = path.replace("\\", "/") + if path.startswith("/") or re.match(r"^[A-Za-z]:", path): + raise InteractionError("workspace paths must be relative to the active workspace") + path = posixpath.normpath(path) + if path == ".." or path.startswith("../"): + raise InteractionError("workspace paths cannot escape the active workspace") + normalised_paths.append(path) + if len(set(normalised_paths)) != len(normalised_paths): + raise InteractionError("workspace paths must be unique after normalization") + capabilities = values("capabilities", { + "read", "write", "shell", "network", "git", "browser", "destructive", + }) + effects = values("effects", { + "read_only", "filesystem", "process", "remote", "destructive", "internal", + }) + risk_ceiling = raw_bounds.get("risk_ceiling", "normal") + if not isinstance(risk_ceiling, str) or risk_ceiling not in {"normal", "high"}: + raise InteractionError("risk ceiling must be normal or high") + bounds = { + "workspace_paths": sorted(normalised_paths), "capabilities": sorted(capabilities), + "effects": sorted(effects), "risk_ceiling": risk_ceiling, + } + contract_policies = { + "read_file": ("read", "read_only"), + "write_file": ("write", "filesystem"), + "edit_file": ("write", "filesystem"), + "apply_patch": ("write", "filesystem"), + } + for step in normalised: + contracts = ([step["execution"]] if step.get("execution") else []) + [ + verification for verification in step["acceptance_verifications"] if verification + ] + for contract in contracts: + policy = contract_policies.get(contract["action"]) + if policy is None: + raise InteractionError(f"unsupported Strategic Plan contract action: {contract['action']}") + capability, effect = policy + if capability not in bounds["capabilities"] or effect not in bounds["effects"]: + raise InteractionError("Strategic Plan contract exceeds its approved capability or effect bounds") + path = contract["arguments"].get("path") + if not isinstance(path, str) or not path.strip(): + raise InteractionError("Strategic Plan file contracts require a path") + normalized_contract_path = posixpath.normpath(path.replace("\\", "/")) + if (normalized_contract_path.startswith("/") or normalized_contract_path == ".." + or normalized_contract_path.startswith("../")): + raise InteractionError("Strategic Plan contract path escapes the approved workspace") + if not any(root == "." or normalized_contract_path == root + or normalized_contract_path.startswith(root.rstrip("/") + "/") + for root in bounds["workspace_paths"]): + raise InteractionError("Strategic Plan contract path is outside its approved workspace bounds") + plan = { + "plan_id": _text( + (previous or {}).get("plan_id") or f"plan_{uuid.uuid4().hex}", + "plan_id", 100, + ), + "version": version, + "title": _text(value.get("title"), "plan title", 160), + "summary": _text(value.get("summary"), "plan summary", 1200), + "assumptions": [_text(item, "assumption", 300) for item in assumptions], + "steps": normalised, + "bounds": bounds, + "bounds_source": bounds_source, + "original_input": str(original_input).strip()[:12_000], + } + fingerprint_fields = {key: plan[key] for key in ( + "plan_id", "version", "title", "summary", "assumptions", "steps", "bounds", + )} + plan["fingerprint"] = hashlib.sha256(json.dumps( + fingerprint_fields, ensure_ascii=False, allow_nan=False, sort_keys=True, + separators=(",", ":"), + ).encode("utf-8")).hexdigest() + return plan + + +def _revision_materiality(accepted: dict[str, Any], proposal: dict[str, Any]) -> str: + """Allow unchanged objectives only when all revised bounds are provably narrower.""" + if any(accepted.get(field) != proposal.get(field) + for field in ("title", "summary", "assumptions")): + return "material" + objectives = lambda plan: { + step["id"]: (step["title"], step["description"], tuple(step["acceptance"]), + tuple(step["acceptance_verifications"]), step.get("execution")) + for step in plan.get("steps", ()) + } + if objectives(accepted) != objectives(proposal): + return "material" + old, new = accepted.get("bounds"), proposal.get("bounds") + if not isinstance(old, dict) or not isinstance(new, dict): + return "material" + if (not set(new.get("capabilities", ())) <= set(old.get("capabilities", ())) + or not set(new.get("effects", ())) <= set(old.get("effects", ())) + or (old.get("risk_ceiling") == "normal" and new.get("risk_ceiling") == "high")): + return "material" + roots = [posixpath.normpath(path) for path in old.get("workspace_paths", ())] + for path in map(posixpath.normpath, new.get("workspace_paths", ())): + if not any(root == "." or path == root or path.startswith(root.rstrip("/") + "/") + for root in roots): + return "material" + return "within_bounds" + + + +class PlanningService: + """Own Strategic Plan state and its binding to a Working Plan.""" + + def __init__(self, store: InteractionStore, *, user_id: str = "local", workspace_id: str = "default", + session_id: str): + self.store = store + self.user_id = user_id + self.workspace_id = workspace_id + self.session_id = session_id + + + def _append(self, event_type: str, payload: dict[str, Any]) -> None: + self.store.append_event( + event_type, payload, user_id=self.user_id, workspace_id=self.workspace_id, + session_id=self.session_id, + ) + + + def _append_auto_accepted(self, plan: dict[str, Any], previous: dict[str, Any]) -> None: + append = getattr(self.store, "append_plan_auto_accepted", None) + if not callable(append): + raise InteractionError("atomic Strategic Plan revision is unavailable") + try: + append(plan, expected_previous=previous, user_id=self.user_id, + workspace_id=self.workspace_id, session_id=self.session_id) + except ValueError as exc: + raise InteractionError(str(exc)) from exc + + + def _history(self) -> list[dict[str, Any]]: + if callable(getattr(self.store, "list_interaction_events", None)): + return self.store.list_interaction_events( + workspace_id=self.workspace_id, session_id=self.session_id, user_id=self.user_id, + ) + events = self.store.list_events( + limit=None, workspace_id=self.workspace_id, session_id=self.session_id, + user_id=self.user_id, + ) + return [event for event in reversed(events) + if str(event.get("event_type", "")).startswith("interaction.") + or event.get("event_type") == "history.restored"] + + + @staticmethod + def _plan_state(history: list[dict[str, Any]]) -> dict[str, Any]: + plan = None + executing = None + for event in history: + event_type, payload = event["event_type"], event.get("payload", {}) + if event_type == "history.restored": + restored = payload.get("interaction") if isinstance(payload, dict) else None + if isinstance(restored, dict): + plan = restored.get("pending_plan") + executing = restored.get("executing_plan") + elif event_type == "interaction.plan_proposed": + if payload.get("revision_materiality") != "within_bounds": + plan = payload + elif event_type == "interaction.plan_auto_accepted": + if executing and payload.get("plan_id") == executing.get("plan_id"): + executing = dict(payload) + plan = None + elif event_type == "interaction.plan_accepted": + if plan and payload.get("plan_id") == plan.get("plan_id") and payload.get("version") == plan.get("version"): + executing = dict(plan) + for field in ("working_plan_id", "working_plan_revision", "working_task_ids"): + if field in payload: + executing[field] = payload[field] + plan = None + elif event_type == "interaction.plan_cancelled": + if plan and payload.get("plan_id") == plan.get("plan_id"): + plan = None + elif event_type == "interaction.plan_completed": + if (executing and payload.get("plan_id") == executing.get("plan_id") + and payload.get("version") == executing.get("version") + and payload.get("fingerprint") == executing.get("fingerprint") + and payload.get("status") in {"completed", "cancelled"}): + executing = None + return {"pending_plan": plan, "executing_plan": executing} + + def state(self) -> dict[str, Any]: + """Reconstruct Strategic Plan authority from the scoped durable history.""" + return self._plan_state(self._history()) + + + def propose_plan(self, value: Any, *, original_input: str = "") -> dict[str, Any]: + state = self.state() + previous = state.get("pending_plan") or state.get("executing_plan") + if previous and state.get("pending_plan") is None: + latest = [int(event.get("payload", {}).get("version", 0)) for event in self._history() + if event.get("event_type") in {"interaction.plan_proposed", "interaction.plan_auto_accepted"} + and event.get("payload", {}).get("plan_id") == previous.get("plan_id")] + if latest: + previous = {**previous, "version": max(latest)} + plan = validate_plan_proposal(value, previous=previous, original_input=original_input) + executing = state.get("executing_plan") + if executing: + plan["revision_materiality"] = _revision_materiality(executing, plan) + working = self.store.get_working_plan( + user_id=self.user_id, workspace_id=self.workspace_id, session_id=self.session_id, + ) if callable(getattr(self.store, "get_working_plan", None)) else None + working_revision = (working.get("revision") if working + and working.get("id") == executing.get("working_plan_id") + else executing.get("working_plan_revision")) + plan.update({"working_plan_id": executing.get("working_plan_id"), + "working_plan_revision": working_revision, + "working_task_ids": executing.get("working_task_ids", [])}) + if plan["revision_materiality"] == "within_bounds": + self._append_auto_accepted(plan, executing) + return plan + else: + plan["revision_materiality"] = "initial" + self._append("interaction.plan_proposed", plan) + return plan + + + def cancel_plan(self, plan_id: str | None = None, version: int | None = None) -> dict[str, Any]: + plan = self.state().get("pending_plan") + if not plan or (plan_id and plan_id != plan.get("plan_id")) or (version is not None and version != plan.get("version")): + raise InteractionError("plan is not pending") + self._append("interaction.plan_cancelled", {"plan_id": plan["plan_id"], "version": plan["version"]}) + return plan + + + def accepted_plan(self, plan_id: str | None = None, version: int | None = None) -> dict[str, Any] | None: + proposals: dict[tuple[str, int], dict[str, Any]] = {} + accepted: dict[tuple[str, int], dict[str, Any]] = {} + for event in self._history(): + payload = event.get("payload", {}) + key = (str(payload.get("plan_id") or ""), int(payload.get("version") or 0)) + if event["event_type"] == "interaction.plan_proposed": + proposals[key] = payload + elif event["event_type"] == "interaction.plan_accepted": + accepted[key] = payload + elif event["event_type"] == "interaction.plan_auto_accepted": + proposals[key] = payload + accepted[key] = payload + matches = [proposal for key, proposal in proposals.items() if key in accepted + and (not plan_id or key[0] == plan_id) and (version is None or key[1] == version)] + if not matches: + return None + proposal = matches[-1] + key = (str(proposal.get("plan_id") or ""), int(proposal.get("version") or 0)) + return {**proposal, **{field: accepted[key][field] for field in ( + "working_plan_id", "working_plan_revision", "working_task_ids", + ) if field in accepted[key]}} + + + def accept_plan(self, task_manager: Any, *, plan_id: str | None = None, + version: int | None = None, fingerprint: str | None = None) -> tuple[dict[str, Any], bool]: + state = self.state() + plan = state.get("pending_plan") + executing = state.get("executing_plan") + if executing and not plan and (not plan_id or plan_id == executing.get("plan_id")): + if version is not None and version != executing.get("version"): + raise InteractionError("accepted plan version is stale") + if fingerprint is not None and fingerprint != executing.get("fingerprint"): + raise InteractionError("accepted plan fingerprint is stale") + return executing, False + if not plan: + if executing and plan_id and plan_id != executing.get("plan_id"): + raise InteractionError("accepted plan identity is stale") + accepted = self.accepted_plan(plan_id, version) + if accepted and (fingerprint is None or accepted.get("fingerprint") == fingerprint): + return accepted, False + if accepted: + raise InteractionError("accepted plan fingerprint is stale") + raise InteractionError("plan is not pending") + if ((plan_id and plan_id != plan.get("plan_id")) + or (version is not None and version != plan.get("version")) + or (fingerprint is not None and fingerprint != plan.get("fingerprint"))): + raise InteractionError("plan is not pending at the requested version") + if executing and task_manager.working_plan is None: + task_manager.recover() + elif not executing and task_manager.working_plan is None: + task_manager.recover() + if (not executing and task_manager.working_plan is not None + and task_manager.working_plan.status == "active"): + raise InteractionError("cannot accept a Strategic Plan while an unrelated Working Plan is active") + working_plan = task_manager.working_plan if executing else None + previous_tasks = [task for task in task_manager.tasks + if working_plan and task.get("id") in working_plan.task_ids] + if any(task.get("status") == "running" for task in previous_tasks): + raise InteractionError("cannot replace a Strategic Plan while a Working Plan task is running") + operations = [] + strategic_bindings = {} + active_ids = set(executing.get("working_task_ids", ())) if executing else set() + active_ids.update(task["id"] for task in previous_tasks + if (task.get("checkpoint") or {}).get("strategy_anchor") in active_ids) + if executing: + if (plan.get("revision_materiality") != "material" + or plan.get("plan_id") != executing.get("plan_id") + or plan.get("version") != executing.get("version", 0) + 1): + raise InteractionError("Strategic Plan revision is stale or does not require approval") + staged_revision = plan.get("working_plan_revision", executing.get("working_plan_revision")) + if (not working_plan or working_plan.id != executing.get("working_plan_id") + or working_plan.revision != staged_revision): + raise InteractionError("Working Plan revision changed; reload before accepting") + + prior = [task for task in previous_tasks if not executing or task.get("id") in active_ids] + if any(task.get("status") == "running" for task in prior): + raise InteractionError("cannot replace a Strategic Plan while a Working Plan task is running") + new_refs = [] + reused: set[str] = set() + for step in plan["steps"]: + description = f"{step['title']}: {step['description']}" + execution = step.get("execution") or {} + expected_args = (json.dumps(execution.get("arguments", {}), ensure_ascii=False, + allow_nan=False, sort_keys=True, separators=(",", ":")) + if execution else "") + existing = next((task for task in prior + if task.get("status") == "succeeded" + and (task.get("checkpoint") or {}).get("strategic_plan", {}).get("objective_id") == step["id"] + and (task.get("checkpoint") or {}).get("strategic_plan", {}).get("acceptance_ids") == step["acceptance_ids"] + and task.get("description") == description + and [item.get("criterion") if isinstance(item, dict) else item + for item in task.get("acceptance", [])] == step["acceptance"] + and (task.get("checkpoint") or {}).get("action") == execution.get("action") + and ((task.get("checkpoint") or {}).get("args") or "") == expected_args + and {criterion_id for item in task.get("evidence", []) + if isinstance(item, dict) and item.get("success") is True + for criterion_id in item.get("acceptance_ids", ())} >= set(step["acceptance_ids"])), None) + if existing: + new_refs.append(existing["id"]) + reused.add(existing["id"]) + continue + ref = f"revision_{plan['version']}_{step['id']}" if executing else step["id"] + new_refs.append(ref) + operations.append({"op": "add", "ref": ref, "task": { + "description": description, "acceptance": step["acceptance"], + **({"execution": step["execution"]} if step.get("execution") else {}), + }}) + strategic_bindings[ref] = { + "plan_id": plan["plan_id"], "version": plan["version"], + "fingerprint": plan["fingerprint"], "objective_id": step["id"], + "acceptance_ids": step["acceptance_ids"], + "acceptance_verifications": step["acceptance_verifications"], + "execution": step.get("execution"), + } + for task in prior: + if task["id"] not in reused and task.get("status") in {"pending", "failed", "blocked"}: + operations.insert(0, {"op": "cancel", "task": task["id"]}) + if not operations: + if not prior: + raise InteractionError("Strategic Plan has no Working Plan tasks to reuse") + operations.append({"op": "update", "task": prior[0]["id"], + "fields": {"priority": prior[0]["priority"]}}) + payload = { + "plan_id": plan["plan_id"], "version": plan["version"], + "fingerprint": plan["fingerprint"], "working_task_refs": new_refs, + **({"revision_of": {"version": executing["version"], + "fingerprint": executing["fingerprint"]}} if executing else {}), + } + try: + working = task_manager.edit_plan( + complexity=working_plan.complexity if working_plan else "complex", + plan_id=working_plan.id if working_plan else None, + expected_revision=working_plan.revision if working_plan else 0, + operations=operations, strategic_bindings=strategic_bindings, + transaction_event={"event_type": "interaction.plan_accepted", "payload": payload, + "expected_proposal": {"plan_id": plan["plan_id"], "version": plan["version"], + "fingerprint": plan["fingerprint"]}, + **({"expected_working_plan": { + "plan_id": executing["working_plan_id"], + "revision": executing["working_plan_revision"], + "strategic_plan_id": executing["plan_id"], + "version": executing["version"], "fingerprint": executing["fingerprint"], + }} if executing else {})}, + ) + except ValueError as exc: + accepted = self.accepted_plan(plan["plan_id"], plan["version"]) + current_state = self.state() + current_plan = current_state.get("pending_plan") or current_state.get("executing_plan") + if current_plan and not all( + current_plan.get(key) == plan.get(key) + for key in ("plan_id", "version", "fingerprint")): + message = str(exc) + if "changed" not in message.lower() and "stale" not in message.lower(): + message = f"Strategic Plan changed while acceptance was pending: {message}" + raise InteractionError(message) from exc + if accepted and accepted.get("fingerprint") == plan["fingerprint"]: + return accepted, False + raise InteractionError(str(exc)) from exc + bound_ids = [working["references"][ref] if ref in working["references"] else ref for ref in new_refs] + return {**plan, "working_plan_id": working["plan_id"], + "working_plan_revision": working["revision"], "working_task_ids": bound_ids}, True + + + def complete_plan(self, plan: dict[str, Any], *, status: str = "completed", + task_manager: Any | None = None) -> None: + if status not in {"completed", "failed", "cancelled"}: + raise InteractionError("Strategic Plan terminal status is invalid") + prior_completions = [event.get("payload", {}).get("status") for event in self._history() + if event["event_type"] == "interaction.plan_completed" + and event.get("payload", {}).get("plan_id") == plan.get("plan_id") + and event.get("payload", {}).get("version") == plan.get("version") + and event.get("payload", {}).get("fingerprint") == plan.get("fingerprint")] + if any(previous in {"completed", "cancelled"} or previous == status + for previous in prior_completions): + return + executing = self.state().get("executing_plan") + if not executing or any(executing.get(key) != plan.get(key) + for key in ("plan_id", "version", "fingerprint")): + raise InteractionError("Strategic Plan changed; reload before recording completion") + working = task_manager.working_plan if task_manager is not None else None + task_ids = plan.get("working_task_ids") + if task_ids is None and working and working.id == plan.get("working_plan_id"): + task_ids = working.task_ids + if status == "completed": + if task_manager is None or not isinstance(task_ids, list) or not task_ids: + raise InteractionError("Strategic Plan completion requires its linked Working Plan evidence") + if any((task.get("checkpoint") or {}).get("strategy_anchor") in task_ids + and task.get("status") not in {"succeeded", "cancelled"} + for task in task_manager.tasks + if working and task.get("id") in working.task_ids): + raise InteractionError("Strategic Plan refinement tasks are still unfinished") + tasks = {task.get("id"): task for task in task_manager.tasks} + objectives = {step["id"] for step in plan.get("steps", [])} + for task_id in task_ids: + binding = (tasks.get(task_id, {}).get("checkpoint") or {}).get("strategic_plan") or {} + if binding.get("plan_id") != plan.get("plan_id") or binding.get("objective_id") not in objectives: + raise InteractionError("Working Plan contains a task not linked to the accepted Strategic Plan") + if any(tasks.get(task_id, {}).get("status") != "succeeded" + or not any(isinstance(item, dict) and item.get("success") is True and item.get("acceptance") + for item in tasks[task_id].get("evidence", [])) + for task_id in task_ids): + raise InteractionError("Strategic Plan tasks have not passed their acceptance evidence gates") + for task_id in task_ids: + task = tasks[task_id] + binding = (task.get("checkpoint") or {}).get("strategic_plan") or {} + required = set(binding.get("acceptance_ids", ())) + if required: + covered = { + criterion_id + for item in task.get("evidence", []) + if isinstance(item, dict) and item.get("success") is True + for criterion_id in item.get("acceptance_ids", ()) + } + if not required <= covered: + raise InteractionError("Strategic Plan criteria have not all been verified") + payload = { + "plan_id": plan["plan_id"], "version": plan["version"], + "fingerprint": plan.get("fingerprint"), "working_plan_id": plan.get("working_plan_id"), + "working_task_ids": task_ids or [], "status": status, + } + append = getattr(self.store, "append_plan_completed", None) + if callable(append): + try: + append(payload, user_id=self.user_id, workspace_id=self.workspace_id, + session_id=self.session_id) + except ValueError as exc: + raise InteractionError(str(exc)) from exc + else: + self._append("interaction.plan_completed", payload) + + + def reconcile_plan_completion(self, task_manager: Any) -> None: + plan = self.state().get("executing_plan") + if not plan or not plan.get("working_plan_id"): + return + working = task_manager.working_plan + if working is None or working.id != plan["working_plan_id"]: + working = task_manager.restore_working_plan(plan["working_plan_id"]) + if working is not None and working.status in {"completed", "cancelled"}: + self.complete_plan( + plan, status=working.status, task_manager=task_manager, + ) + diff --git a/openkyrozen/tasks/ports.py b/openkyrozen/tasks/ports.py index 3a12ca41..37f6264e 100644 --- a/openkyrozen/tasks/ports.py +++ b/openkyrozen/tasks/ports.py @@ -13,3 +13,9 @@ def list_working_plan_tasks(self, plan_id: str, **scope) -> list[dict]: ... def write_working_plan(self, plan: dict, tasks: list[dict], **scope) -> None: ... def apply_working_plan_edit(self, plan: dict, tasks: list[dict], **scope) -> int: ... def append_event(self, event_type: str, payload: Any, **scope) -> str: ... + + +class InteractionStore(Protocol): + def append_event(self, event_type: str, payload: Any, **scope) -> str: ... + def list_events(self, event_type: str | None = None, **scope) -> list[dict]: ... + def list_interaction_events(self, **scope) -> list[dict]: ... diff --git a/openkyrozen/tasks/worker.py b/openkyrozen/tasks/worker.py index de971dd2..7d571107 100644 --- a/openkyrozen/tasks/worker.py +++ b/openkyrozen/tasks/worker.py @@ -23,6 +23,10 @@ def run_once(self) -> dict[str, Any] | None: if task is None: return None task_id = task["id"] + if (task.get("checkpoint", {}).get("recovery_effect") or {}).get("status") == "unknown": + index = next(i for i, item in enumerate(self.manager.tasks) if item["id"] == task_id) + self.manager.set_status(index, "blocked", reason="Inspect and reconcile the interrupted operation before resuming.") + return {"task": self.manager.tasks[index], "status": "blocked", "result": "Interrupted effect is unknown.", "evidence": {}} try: outcome = self.executor(task) if not isinstance(outcome, dict): diff --git a/openkyrozen/tools/catalog.py b/openkyrozen/tools/catalog.py index a486b44a..ef6e435a 100644 --- a/openkyrozen/tools/catalog.py +++ b/openkyrozen/tools/catalog.py @@ -5,6 +5,8 @@ # name, capability, side effects, owner, risk, parallel safety, aliases, conversion fields, argument aliases, confirmation surfaces _ROWS = ( ('update_plan', 'write', 'internal_write', 'runtime', 'normal', False, (), (), (), ()), + ('propose_plan', 'read', 'internal_write', 'runtime', 'normal', False, (), (), (), ()), + ('request_question', 'read', 'internal_write', 'runtime', 'normal', False, (), (), (), ()), ('spawn_agents', 'read', 'orchestration', 'orchestration', 'normal', False, (), (), (), ()), ('send_subagent', 'read', 'orchestration', 'orchestration', 'normal', False, (), (), (), ()), ('list_subagents', 'read', 'orchestration', 'orchestration', 'normal', False, (), (), (), ()), @@ -155,7 +157,8 @@ },'minProperties':1,'additionalProperties':False} _PLAN_OPERATION_SCHEMAS = [ {'type':'object','properties':{'op':{'type':'string','enum':['add']}, - 'ref':{'type':'string','minLength':1},'task':_PLAN_TASK_SCHEMA}, + 'ref':{'type':'string','minLength':1},'task':_PLAN_TASK_SCHEMA, + 'anchor':{'type':'string','minLength':1,'description':'Existing approved objective task ID for in-scope strategy work.'}}, 'required':['op','ref','task'],'additionalProperties':False}, {'type':'object','properties':{'op':{'type':'string','enum':['update']}, 'task':{'type':'string','minLength':1},'fields':_PLAN_UPDATE_FIELDS_SCHEMA}, @@ -189,6 +192,9 @@ *[{'type':'object','properties':{'op':{'type':'string','enum':[op]}, 'task':{'type':'string','minLength':1},'reason':{'type':'string','minLength':1}}, 'required':['op','task','reason'],'additionalProperties':False} for op in ('block','unblock')], + *[{'type':'object','properties':{'op':{'type':'string','enum':[op]}, + 'task':{'type':'string','minLength':1}}, + 'required':['op','task'],'additionalProperties':False} for op in ('start','suggest_complete')], ] _OBJECT_FIELDS = { @@ -276,6 +282,43 @@ 'required': ['complexity', 'tasks'], 'additionalProperties': False}, ] +_CONTROL_VERIFICATION = {'type':'object','properties':{ + 'action':{'type':'string','minLength':1}, + 'arguments':{'type':'object','properties':{},'additionalProperties':True}, + 'expect':{'type':'object','properties':{'equals':{'type':'string'},'contains':{'type':'string','minLength':1}}, + 'minProperties':1,'maxProperties':1,'additionalProperties':False}, +},'required':['action','arguments'],'additionalProperties':False} +_SCHEMAS['propose_plan'] = {'type':'object','properties':{ + 'title':{'type':'string','minLength':1,'maxLength':120}, + 'summary':{'type':'string','minLength':1,'maxLength':1000}, + 'assumptions':{'type':'array','maxItems':10,'items':{'type':'string'}}, + 'bounds':{'type':'object','properties':{ + 'workspace_paths':{'type':'array','minItems':1,'items':{'type':'string','minLength':1}}, + 'capabilities':{'type':'array','minItems':1,'items':{'type':'string'}}, + 'effects':{'type':'array','minItems':1,'items':{'type':'string'}}, + 'risk_ceiling':{'type':'string','enum':['normal','high']}, + },'required':['workspace_paths','capabilities','effects','risk_ceiling'],'additionalProperties':False}, + 'steps':{'type':'array','minItems':1,'maxItems':10,'items':{'type':'object','properties':{ + 'id':{'type':'string','minLength':1,'maxLength':64}, + 'title':{'type':'string','minLength':1},'description':{'type':'string','minLength':1}, + 'acceptance':{'type':'array','minItems':1,'maxItems':5,'items':{'anyOf':[ + {'type':'string','minLength':1}, + {'type':'object','properties':{'criterion':{'type':'string','minLength':1}, + 'verification':_CONTROL_VERIFICATION},'required':['criterion','verification'],'additionalProperties':False}, + ]}},'execution':_PLAN_EXECUTION_SCHEMA, + },'required':['id','title','description','acceptance'],'additionalProperties':False}}, +},'required':['title','summary','bounds','steps'],'additionalProperties':False} +_SCHEMAS['request_question'] = {'type':'object','properties':{ + 'questions':{'type':'array','minItems':1,'maxItems':3,'items':{'type':'object','properties':{ + 'id':{'type':'string','minLength':1},'header':{'type':'string','minLength':1}, + 'prompt':{'type':'string','minLength':1}, + 'choices':{'type':'array','minItems':2,'maxItems':3,'items':{'type':'object','properties':{ + 'id':{'type':'string','minLength':1},'label':{'type':'string','minLength':1}, + 'description':{'type':'string'}, + },'required':['id','label'],'additionalProperties':False}}, + },'required':['id','header','prompt','choices'],'additionalProperties':False}}, +},'required':['questions'],'additionalProperties':False} + def builtin_names(owner=None): @@ -298,7 +341,7 @@ def builtin_metadata(name): 'analyze_remote_repo', 'browser_open', 'browser_snapshot', 'browser_click', 'browser_type', 'browser_close', 'github_status', 'github_read'}: effects = 'remote' - elif name in {'update_plan', 'graph_refresh', 'spawn_agents', 'send_subagent', 'list_subagents', + elif name in {'update_plan', 'propose_plan', 'request_question', 'graph_refresh', 'spawn_agents', 'send_subagent', 'list_subagents', 'wait_subagents', 'cancel_subagent'}: effects = 'internal' else: diff --git a/openkyrozen/tools/processes.py b/openkyrozen/tools/processes.py index b498e5f4..49bc2f7f 100644 --- a/openkyrozen/tools/processes.py +++ b/openkyrozen/tools/processes.py @@ -2,11 +2,13 @@ from __future__ import annotations import os +import json import signal import subprocess import threading import time import uuid +from datetime import datetime, timezone from dataclasses import dataclass, field from openkyrozen.tools.sandbox import sandbox_command @@ -36,6 +38,11 @@ class ProcessHandle: stderr_truncated: bool = False timed_out: bool = False finished_at: float | None = None + cancelled: bool = False + stop_reason: str | None = None + audit_error: bool = False + audit_exit_recorded: bool = False + expiry_thread: threading.Thread | None = None lock: threading.Lock = field(default_factory=threading.Lock) @@ -49,50 +56,202 @@ def __init__(self, owner_id: str, workspace_id: str, session_id: str): self._handles: dict[str, ProcessHandle] = {} self._lock = threading.Lock() self._closed = False + self._audit_store = None + self._audit_lock = threading.Lock() + self._audit_closed = False + self._audit_workspace_id = None + self._audit_manager_id = uuid.uuid4().hex + self._prior_process_audit = [] + self._prior_process_total = 0 + self._prior_audit_unavailable = False - def start(self, command: str, *, cwd: str, env: dict[str, str], timeout_seconds: float) -> ProcessHandle: - with self._lock: - if self._closed: - raise RuntimeError("session process manager is closed") - if len(self._handles) >= 32: - raise RuntimeError("session process handle limit reached") - invocation = sandbox_command( - command, workspace=self.workspace_id, cwd=cwd, - env=os.environ.copy() if env is None else env, - ) + def _bind_audit(self, store, workspace_id: str) -> None: + """Bind the existing session journal; filesystem roots are not DB scope IDs.""" + self._audit_store = store + self._audit_workspace_id = str(workspace_id) + try: + unresolved = [record for record in self._audit_records() + if record.get("manager_id") != self._audit_manager_id + and record["owned_process_state"] == "unknown"] + self._prior_process_total = len(unresolved) + self._prior_process_audit = [{ + "process_id": record["process_id"][:64], + "status": record.get("status") if record.get("status") in { + "launch_intent", "started", "stop_requested", "termination_failed", "launch_failed" + } else "unknown", + "pid_observation": record.get("pid") if type(record.get("pid")) is int else None, + } for record in unresolved[:5]] + except Exception: + self._prior_audit_unavailable = True + + def _recovery_context(self) -> str: + if self._prior_audit_unavailable: + return ("Prior process audit unavailable; previous process effects are unknown. " + "Do not replay, reattach or kill processes from historical PID metadata.") + if not self._prior_process_total: + return "" + return ("Prior process outcomes are unknown: " + f"total={self._prior_process_total}, omitted={self._prior_process_total - len(self._prior_process_audit)}. " + "Inspect or request trusted reconciliation before repeating effects. " + "PID observations are not authority to reattach or kill; do not replay these commands. " + + json.dumps(self._prior_process_audit, separators=(",", ":"))) + + def _audit_event(self, status, process_id, *, handle=None, **fields): + if self._audit_store is None: + return + payload = {"process_id": process_id, "manager_id": self._audit_manager_id, + "status": status, "observed_at": datetime.now(timezone.utc).isoformat(), + "owned_process_state": "unknown", "descendant_state": "unknown", **fields} + if handle is not None: + pid = getattr(handle.process, "pid", None) + payload.update(pid=pid if type(pid) is int else None, + sandbox_backend=handle.sandbox_backend) + try: + if self._audit_closed or not self._audit_lock.acquire(timeout=.1): + raise RuntimeError("process audit is closed or busy") + try: + if self._audit_closed: + raise RuntimeError("process audit is closed") + self._audit_store._append_control_audit("process.lifecycle", payload, user_id=self.owner_id, + workspace_id=self._audit_workspace_id, session_id=self.session_id) + finally: + self._audit_lock.release() + except Exception: + # Neither database errors nor command/environment values enter audit logs. + raise RuntimeError("Process audit could not be persisted.") from None + + def _audit_records(self) -> list[dict]: + """Enumerate scoped uncertainty, never reattach or signal an observed PID.""" + if self._audit_store is None: + return [] + events = self._audit_store.list_events("process.lifecycle", limit=None, + user_id=self.owner_id, workspace_id=self._audit_workspace_id, session_id=self.session_id) + records = {} + for event in reversed(events): + payload = event["payload"] + process_id = payload.get("process_id") + if not isinstance(process_id, str): + continue + records.setdefault(process_id, {}).update(payload) + for record in records.values(): + if record.get("owned_process_state") not in {"exited", "not_started"}: + record["owned_process_state"] = "unknown" + record["potentially_active"] = (record["owned_process_state"] == "unknown" + or record.get("descendant_state") == "unknown") + return list(records.values()) + + def _audit_started(self, handle): + try: + self._audit_event("started", handle.process_id, handle=handle, + owned_process_state="running", timeout_seconds=handle.timeout_seconds) + except RuntimeError: + handle.audit_error = True + handle.stop_reason = "launch_audit_failed" try: - process = subprocess.Popen( - invocation.command, shell=invocation.shell, cwd=cwd, env=invocation.env, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - start_new_session=(os.name != "nt"), - creationflags=(subprocess.CREATE_NEW_PROCESS_GROUP if os.name == "nt" else 0), + self.stop(handle.process_id) + finally: + if handle.process.poll() is not None: + for name in ("stdout", "stderr"): + getattr(handle.process, name).close() + if handle.temporary_directory is not None: + handle.temporary_directory.cleanup() + handle.temporary_directory = None + raise RuntimeError("Process audit failed after launch; owned cleanup was attempted.") from None + + def _audit_outcome(self, handle): + code = handle.process.poll() + if handle.audit_exit_recorded: + return + reason = handle.stop_reason or "exited" + try: + self._audit_event(reason if code is not None else "termination_failed", + handle.process_id, handle=handle, reason=reason, + owned_process_state="exited" if code is not None else "unknown", + exit_code=code if type(code) is int else None) + handle.audit_exit_recorded = code is not None + except RuntimeError: + # The prior intent/started record stays unresolved on restart. + handle.audit_error = True + + def start(self, command: str, *, cwd: str, env: dict[str, str], timeout_seconds: float, + cancelled=None) -> ProcessHandle: + process_id = uuid.uuid4().hex + self._audit_event("launch_intent", process_id, timeout_seconds=timeout_seconds) + try: + with self._lock: + if self._closed: + raise RuntimeError("session process manager is closed") + if cancelled is not None and cancelled.is_set(): + raise RuntimeError("turn was cancelled before process start") + if sum(handle.process.poll() is None for handle in self._handles.values()) >= 32: + raise RuntimeError("session process handle limit reached") + invocation = sandbox_command( + command, workspace=self.workspace_id, cwd=cwd, + env=os.environ.copy() if env is None else env, + ) + try: + process = subprocess.Popen( + invocation.command, shell=invocation.shell, cwd=cwd, env=invocation.env, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + start_new_session=(os.name != "nt"), + creationflags=(subprocess.CREATE_NEW_PROCESS_GROUP if os.name == "nt" else 0), + ) + except Exception: + invocation.close() + raise + handle = ProcessHandle( + process_id, process, self.owner_id, self.workspace_id, + self.session_id, timeout_seconds, invocation.backend, + invocation.temporary_directory, invocation.unavailable_reason, ) - except Exception: - invocation.close() - raise - handle = ProcessHandle( - uuid.uuid4().hex, process, self.owner_id, self.workspace_id, - self.session_id, timeout_seconds, invocation.backend, - invocation.temporary_directory, invocation.unavailable_reason, - ) - self._handles[handle.process_id] = handle + self._handles[handle.process_id] = handle + except BaseException: + try: + self._audit_event("launch_failed", process_id, reason="launch_not_confirmed") + except RuntimeError: + pass + raise + self._audit_started(handle) for name in ("stdout", "stderr"): threading.Thread(target=self._collect, args=(handle, name), daemon=True).start() - threading.Thread(target=self._expire, args=(handle,), daemon=True).start() + handle.expiry_thread = threading.Thread(target=self._expire, args=(handle, cancelled), daemon=True) + handle.expiry_thread.start() return handle - def _expire(self, handle: ProcessHandle) -> None: + def _expire(self, handle: ProcessHandle, cancelled=None) -> None: try: - handle.process.wait(timeout=min(handle.timeout_seconds, _MAX_PROCESS_SECONDS)) - except subprocess.TimeoutExpired: - if handle.process.poll() is None: - self.stop(handle.process_id) - handle.timed_out = True + deadline = handle.started_at + min(handle.timeout_seconds, _MAX_PROCESS_SECONDS) + while handle.process.poll() is None: + if cancelled is not None and cancelled.is_set(): + handle.stop_reason = "cancelled" + self.stop(handle.process_id) + handle.cancelled = True + break + remaining = deadline - time.monotonic() + if remaining <= 0: + handle.stop_reason = "timed_out" + self.stop(handle.process_id) + handle.timed_out = True + break + try: + handle.process.wait(timeout=min(.1, remaining)) + except subprocess.TimeoutExpired: + pass finally: - handle.finished_at = time.monotonic() - if handle.temporary_directory is not None: - handle.temporary_directory.cleanup() - handle.temporary_directory = None + # A failed termination must remain visibly running, with its + # sandbox retained while the process may still be using it. + self._audit_outcome(handle) + if handle.process.poll() is not None: + handle.finished_at = time.monotonic() + self._cleanup_sandbox(handle) + + @staticmethod + def _cleanup_sandbox(handle): + with handle.lock: + temporary = handle.temporary_directory + handle.temporary_directory = None + if temporary is not None: + temporary.cleanup() @staticmethod def _collect(handle: ProcessHandle, name: str) -> None: @@ -128,7 +287,7 @@ def read(self, process_id: str, stream: str = "stdout", offset: int = 0, return { "process_id": process_id, "stream": stream, "offset": offset, "output": page, "next_offset": offset + len(page), - "output_truncated": truncated, "timed_out": timed_out, + "output_truncated": truncated, "timed_out": timed_out, "cancelled": handle.cancelled, "sandbox_backend": handle.sandbox_backend, "sandbox_reason": handle.sandbox_reason, "duration_seconds": duration, @@ -141,6 +300,12 @@ def stop(self, process_id: str) -> bool | None: if handle is None: return None if handle.process.poll() is None: + handle.stop_reason = handle.stop_reason or "stopped" + try: + self._audit_event("stop_requested", handle.process_id, handle=handle, + reason=handle.stop_reason) + except RuntimeError: + handle.audit_error = True try: if os.name == "nt": subprocess.run(["taskkill", "/PID", str(handle.process.pid), "/T", "/F"], @@ -162,11 +327,29 @@ def stop(self, process_id: str) -> bool | None: pass if handle.process.poll() is not None and handle.finished_at is None: handle.finished_at = time.monotonic() + self._audit_outcome(handle) + if handle.process.poll() is not None: + self._cleanup_sandbox(handle) return handle.process.poll() is not None def close(self) -> None: with self._lock: self._closed = True process_ids = tuple(self._handles) - for process_id in process_ids: - self.stop(process_id) + try: + for process_id in process_ids: + self.stop(process_id) + for process_id in process_ids: + thread = self._handles[process_id].expiry_thread + if thread is not None and thread is not threading.current_thread(): + thread.join(timeout=1) + finally: + # Late expiry can retain uncertainty, but cannot reopen a closed + # application's journal. Drain bounded in-flight audit appends. + self._audit_closed = True + if self._audit_lock.acquire(timeout=.3): + self._audit_lock.release() + else: + for handle in self._handles.values(): + handle.audit_error = True + raise RuntimeError("Process audit finalization did not complete.") diff --git a/scripts/agent_workflow_acceptance.py b/scripts/agent_workflow_acceptance.py index c9f134c8..eb1f9634 100644 --- a/scripts/agent_workflow_acceptance.py +++ b/scripts/agent_workflow_acceptance.py @@ -16,13 +16,9 @@ if os.environ.get("KYROZEN_ACCEPTANCE_INSTALLED") != "1": sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from openkyrozen.app.bootstrap import build_application - -_application = build_application(surface="cli") - -main = _application.runtime +from openkyrozen.app.bootstrap import build_application, build_memory +from openkyrozen.tools import ToolAdapters from openkyrozen.security.capabilities import issue_capability_token -from openkyrozen.persistence.store import EventStore from openkyrozen.agent.modes import InteractionController from openkyrozen.tasks.engine import TaskManager @@ -139,13 +135,14 @@ def test_page_is_served(self): "TaskDone: 3\nDeployment verified.", ] - original = ( - main.tasks, main._interaction_controller, main.learning_engine, - main._get_workspace_root(), main._execution_capability_token, - ) with tempfile.TemporaryDirectory(prefix="openkyrozen-agent-acceptance-") as directory: root = Path(directory) - store = EventStore(root / "state.sqlite3") + application = build_application( + surface="cli", memory=build_memory(root / "state.sqlite3", workspace_id="acceptance"), + tools=ToolAdapters(root), + ) + main = application.runtime + store = main.memory_bank.store main.tasks = TaskManager(store, workspace_id="acceptance", session_id="release") main._interaction_controller = InteractionController( store, workspace_id="acceptance", session_id="release", @@ -203,9 +200,7 @@ def test_page_is_served(self): finally: main._approval_callback.reset(approval_token) main._stream_event_callback.reset(stream_token) - (main.tasks, main._interaction_controller, main.learning_engine, - previous_root, main._execution_capability_token) = original - main._set_workspace_root(previous_root) + application.close() if __name__ == "__main__": diff --git a/scripts/generate_tool_inventory.py b/scripts/generate_tool_inventory.py index 7ce635eb..4b792fbe 100755 --- a/scripts/generate_tool_inventory.py +++ b/scripts/generate_tool_inventory.py @@ -22,11 +22,12 @@ def _load_runtime() -> tuple[dict[str, Any], Any, Any]: tools = ToolAdapters().AVAILABLE_TOOLS from openkyrozen.agent import delegation_runtime from openkyrozen.agent.prompts import _discover_tools - from openkyrozen.agent.plan_tools import _update_plan + from openkyrozen.agent.plan_tools import _propose_plan, _request_question, _update_plan from openkyrozen.agent.runtime import AgentRuntime from openkyrozen.tools.catalog import builtin_names runtime_functions = {'check_stored_data':_check_stored_data, 'search_memory':_search_memory, 'discover_tools':_discover_tools, 'update_plan':_update_plan, + 'propose_plan':_propose_plan, 'request_question':_request_question, 'read_tool_artifact':AgentRuntime._read_tool_artifact, 'read_process':AgentRuntime._read_process, 'stop_process':AgentRuntime._stop_process} diff --git a/scripts/v3_windows_reliability.py b/scripts/v3_windows_reliability.py new file mode 100644 index 00000000..4a86092a --- /dev/null +++ b/scripts/v3_windows_reliability.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python3 +"""Trace the unchanged cancellation unittest; passing samples do not close R260.""" +from __future__ import annotations + +import argparse +from contextlib import closing +import faulthandler +import hashlib +import json +import os +from pathlib import Path +import platform +import signal +import sqlite3 +import subprocess +import sys +import threading +import time +import unittest +import uuid + +ROOT = Path(__file__).resolve().parents[1] +TEST = 'test_cancel_during_provider_call_prevents_late_write' +sys.path[:0] = [str(ROOT), str(ROOT / 'tests')] + + +def write_json(path, value): + path.write_text(json.dumps(value, indent=2) + '\n', encoding='utf-8') + + +_CPU_LOAD = ''' +import json, math, sys, time +from pathlib import Path +ready, stop, result = map(Path, sys.argv[1:4]) +started, cpu = time.monotonic(), time.process_time() +deadline = started + float(sys.argv[4]) +ready.write_text('ready') +value = 0.5 +while not stop.exists() and time.monotonic() < deadline: + for _ in range(20000): + value = math.sin(value) + 0.5 +result.write_text(json.dumps({'cpu_seconds': time.process_time()-cpu, + 'wall_seconds': time.monotonic()-started, + 'stopped_by': 'parent' if stop.exists() else 'deadline'})) +''' + + +class LoadControl: + """Bounded diagnostic load; never writes fixture data or changes timeouts.""" + + def __init__(self, control, workers, case, trace): + self.control, self.workers, self.case, self.trace = control, workers, case, trace + self.stopped, self.sqlite_ready = threading.Event(), threading.Event() + self.cpu = [] + self.sqlite_thread = None + self.acquired_transactions = 0 + self.sqlite_fatal_error = None + self.closed = False + + def start(self): + if self.control in {'sqlite', 'combined'}: + self.sqlite_thread = threading.Thread(target=self._sqlite_load, + name='diagnostic-sqlite-control', daemon=True) + self.sqlite_thread.start() + if not self.sqlite_ready.wait(3): + raise RuntimeError('SQLite load control did not acquire its first bounded transaction') + if self.control in {'cpu', 'combined'}: + for index in range(self.workers): + prefix = self.trace.output / f'cpu-{index}-{uuid.uuid4().hex}' + ready, stop, result = (Path(str(prefix) + suffix) for suffix in ('.ready', '.stop', '.json')) + process = subprocess.Popen([sys.executable, '-c', _CPU_LOAD, + str(ready), str(stop), str(result), '30'], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + self.cpu.append((process, stop, result)) + deadline = time.monotonic() + 3 + while not ready.exists() and process.poll() is None and time.monotonic() < deadline: + time.sleep(.01) + if not ready.exists(): + raise RuntimeError('CPU load control did not become ready') + self.trace.record('load.started', control=self.control, cpu_pids=[item[0].pid for item in self.cpu]) + + def _sqlite_load(self): + try: + with closing(sqlite3.connect(self.case.root / 'state.sqlite3', timeout=.05, + isolation_level=None)) as connection: + while not self.stopped.is_set(): + acquired = False + started = time.monotonic_ns() + try: + connection.execute('BEGIN IMMEDIATE') + acquired = True + self.acquired_transactions += 1 + self.trace.record('sqlite.lock_acquired', wait_ns=time.monotonic_ns()-started) + self.sqlite_ready.set() + self.stopped.wait(.025) + except sqlite3.Error as error: + self.trace.record('sqlite.lock_error', error_type=type(error).__name__, + wait_ns=time.monotonic_ns()-started) + finally: + if acquired: + connection.execute('ROLLBACK') + self.trace.record('sqlite.lock_released') + self.stopped.wait(.01) + except sqlite3.Error as error: + self.sqlite_fatal_error = type(error).__name__ + self.trace.record('sqlite.control_error', error_type=type(error).__name__) + + def close(self): + if self.closed: + return + self.closed = True + self.stopped.set() + cpu = [] + for process, stop, result in self.cpu: + stop.touch() + forced = False + try: + process.wait(timeout=3) + except subprocess.TimeoutExpired: + forced = True + process.kill() + process.wait(timeout=3) + metrics = json.loads(result.read_text()) if result.is_file() else {} + cpu.append({'pid': process.pid, 'returncode': process.returncode, + 'forced_cleanup': forced, **metrics}) + if self.sqlite_thread is not None: + self.sqlite_thread.join(timeout=2) + sqlite_stopped = self.sqlite_thread is None or not self.sqlite_thread.is_alive() + write_json(self.trace.output / 'load-control.json', {'control': self.control, 'cpu': cpu, + 'sqlite': {'acquired_transactions': self.acquired_transactions, 'thread_stopped': sqlite_stopped, + 'fatal_error': self.sqlite_fatal_error}}) + self.trace.record('load.stopped', control=self.control, cpu=cpu, sqlite_thread_stopped=sqlite_stopped) + if (not sqlite_stopped or self.sqlite_fatal_error + or any(item['forced_cleanup'] or item['returncode'] != 0 + or item.get('stopped_by') != 'parent' for item in cpu)): + raise RuntimeError('Load-control cleanup did not complete normally; inspect diagnostic evidence') + + +def stop_diagnostic_tree(process): + """Stop the still-owned diagnostic group before its ancestry disappears.""" + attempt = {'parent_pid': process.pid, 'descendant_cleanup_certified': False} + if os.name == 'nt': + result = subprocess.run(['taskkill', '/PID', str(process.pid), '/T', '/F'], + capture_output=True, timeout=5, check=False) + attempt.update(method='taskkill_tree', command_returncode=result.returncode) + else: + try: + os.killpg(process.pid, signal.SIGKILL) + attempt.update(method='owned_process_group', signal_sent=True) + except ProcessLookupError: + attempt.update(method='owned_process_group', signal_sent=False) + try: + process.wait(timeout=3) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=3) + attempt['parent_returncode'] = process.returncode + return attempt + + +class Trace: + def __init__(self, output): + self.output = output + self.stream = (output / 'events.jsonl').open('w', encoding='utf-8') + self.lock = threading.Lock() + self.case = None + self.coordinator = None + self.active = True + + def record(self, event, **fields): + with self.lock: + if not self.active: + return + self.stream.write(json.dumps({'event': event, 'monotonic_ns': time.monotonic_ns(), + 'thread_id': threading.get_ident(), **fields}) + '\n') + self.stream.flush() + + def profile(self, frame, event, arg): + if event not in {'call', 'return'}: + return + module = frame.f_globals.get('__name__', '') + name = frame.f_code.co_name + label = None + if module == 'test_delegation' and name == 'chat' and 'SlowProvider' in frame.f_code.co_qualname: + label = 'provider' + elif module == 'openkyrozen.agent.delegation': + label = {'spawn': 'dispatch', 'cancel': 'cancel', '_work': 'worker', + '_publish': 'persist', '_invoke': 'invoke'}.get(name) + if label: + self.coordinator = frame.f_locals['self'] + elif module == 'threading' and name == 'set': + caller = frame.f_back + if caller and caller.f_code.co_name == TEST: + if frame.f_locals['self'] is caller.f_locals.get('release'): + label = 'release' + elif module == 'concurrent.futures._base' and name == 'result': + label = 'future_wait' + elif module == 'openkyrozen.app.container' and name == 'close': + label = 'application_close' + if label: + self.record(label + '.' + event, function=frame.f_code.co_qualname) + + def snapshot(self, filename): + """Avoid application locks and raw copies of live SQLite/WAL files.""" + state = {'monotonic_ns': time.monotonic_ns(), 'persisted_states': [], 'runs': []} + case, coordinator = self.case, self.coordinator + if coordinator: + for run_id, run in list(coordinator.runs.items()): + future = coordinator.futures.get(run_id) + state['runs'].append({'run_id': run_id, 'status': run.get('status'), + 'error': run.get('error'), 'future_done': future.done() if future else None, + 'cancelled': coordinator.cancelled[run_id].is_set()}) + if case and hasattr(case, 'root'): + try: + state['marker'] = (case.root / 'marker.txt').read_text(encoding='utf-8') + database = case.root / 'state.sqlite3' + state['sqlite_files'] = {path.name: path.stat().st_size + for path in case.root.glob('state.sqlite3*') if path.is_file()} + with closing(sqlite3.connect(database.as_uri() + '?mode=ro', uri=True, timeout=0)) as connection: + rows = connection.execute("SELECT payload FROM events WHERE event_type='subagent.state' ORDER BY rowid").fetchall() + for row in rows: + run = json.loads(row[0]) + state['persisted_states'].append({key: run.get(key) for key in + ('run_id', 'status', 'version', 'error', 'progress')}) + except (OSError, sqlite3.Error) as exc: + state['snapshot_error'] = repr(exc) + write_json(self.output / filename, state) + + +def run_child(output, control='baseline', cpu_workers=2): + # Import the fixture before starting the four-second cancellation watchdog. + from test_delegation import DelegationTests + + output.mkdir(parents=True, exist_ok=True) + trace = Trace(output) + + class DiagnosticCase(DelegationTests): + def setUp(self): + trace.case = self + trace.record('setup.call') + super().setUp() + trace.record('setup.return') + self.load_control = LoadControl(control, cpu_workers, self, trace) + self.addCleanup(self.load_control.close) + self.load_control.start() + + def tearDown(self): + trace.record('cleanup.call') + trace.snapshot('state-before-cleanup.json') + try: + self.load_control.close() + finally: + try: + super().tearDown() + finally: + trace.record('cleanup.return') + + stopped = threading.Event() + + def watch(): + count = 0 + while not stopped.wait(4): + count += 1 + trace.record('watchdog', interval_seconds=4) + trace.snapshot(f'state-watchdog-{count}.json') + if stopped.is_set(): + return + stacks.write('Timeout (0:00:04)!\n') + stacks.flush() + # The C watchdog can spin walking profiled live frames on CPython 3.12. + faulthandler.dump_traceback(file=stacks, all_threads=True) + + previous = sys.getprofile() + previous_thread = threading.getprofile() + with (output / 'threads.log').open('w', encoding='utf-8') as stacks: + watcher = threading.Thread(target=watch, name='diagnostic-state-watchdog', daemon=True) + watcher.start() + sys.setprofile(trace.profile) + threading.setprofile(trace.profile) + try: + with (output / 'unittest.log').open('w', encoding='utf-8') as log: + result = unittest.TextTestRunner(stream=log, verbosity=2).run(DiagnosticCase(TEST)) + trace.record('result', successful=result.wasSuccessful(), failures=len(result.failures), errors=len(result.errors)) + finally: + sys.setprofile(previous) + threading.setprofile(previous_thread) + stopped.set() + watcher.join(timeout=1) + with trace.lock: + trace.active = False + trace.stream.close() + return 0 if result.wasSuccessful() else 1 + + +def main(): + global ROOT + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--repository', type=Path, default=ROOT, help='Exact product checkout to test; harness may live separately') + parser.add_argument('--iterations', type=int, default=20) + parser.add_argument('--control', choices=('baseline', 'cpu', 'sqlite', 'combined'), default='baseline') + parser.add_argument('--cpu-workers', type=int, default=2, help='Bounded owned CPU load processes (1–4)') + parser.add_argument('--child', action='store_true', help=argparse.SUPPRESS) + args = parser.parse_args() + if args.iterations < 1: + parser.error('--iterations must be positive') + if not 1 <= args.cpu_workers <= 4: + parser.error('--cpu-workers must be between 1 and 4') + ROOT = args.repository.resolve() + sys.path[:0] = [str(ROOT), str(ROOT / 'tests')] + output = args.output.resolve() + if args.child: + return run_child(output, args.control, args.cpu_workers) + output.mkdir(parents=True, exist_ok=True) + revision = subprocess.run(['git', 'rev-parse', 'HEAD'], cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip() + tracked_diff = subprocess.run(['git', 'diff', 'HEAD', '--'], cwd=ROOT, capture_output=True, check=True).stdout + report = {'tracked_product_edits': bool(tracked_diff), + 'tracked_product_diff_sha256': hashlib.sha256(tracked_diff).hexdigest(), 'schema_version': 1, 'candidate_sha': revision, 'platform': platform.platform(), + 'python': sys.version, 'fixture': 'tests.test_delegation.DelegationTests.' + TEST, + 'fixture_sha256': hashlib.sha256((ROOT / 'tests/test_delegation.py').read_bytes()).hexdigest(), + 'diagnostic_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + 'test_timeouts_seconds': {'provider_wait': 3, 'entry_wait': 10, 'future_wait': 5}, + 'watchdog_seconds': 4, 'historical_cause_established': False, + 'control': args.control, 'cpu_workers': args.cpu_workers if args.control in {'cpu', 'combined'} else 0, + 'qualification': 'Observational instrumentation changes scheduling; passing samples do not explain the historical timeout.', + 'iterations': []} + write_json(output / 'report.json', report) + for index in range(1, args.iterations + 1): + directory = output / str(index) + directory.mkdir(exist_ok=True) + command = [sys.executable, str(Path(__file__).resolve()), '--child', '--output', str(directory), + '--repository', str(ROOT), '--control', args.control, '--cpu-workers', str(args.cpu_workers)] + started = time.monotonic_ns() + cleanup = None + with (directory / 'process.log').open('w', encoding='utf-8') as log: + process = subprocess.Popen(command, cwd=ROOT, stdout=log, stderr=subprocess.STDOUT, + start_new_session=os.name != 'nt', + creationflags=subprocess.CREATE_NEW_PROCESS_GROUP if os.name == 'nt' else 0) + try: + returncode = process.wait(timeout=45) + except subprocess.TimeoutExpired: + try: + cleanup = stop_diagnostic_tree(process) + except Exception as error: + cleanup = {'parent_pid': process.pid, 'error_type': type(error).__name__, + 'descendant_cleanup_certified': False} + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=3) + returncode = 124 + report['iterations'].append({'iteration': index, 'returncode': returncode, + 'elapsed_ns': time.monotonic_ns() - started, 'command': command, + 'timeout_cleanup': cleanup}) + write_json(output / 'report.json', report) + print(f'Cancellation iteration {index}: exit={returncode}', flush=True) + return int(any(item['returncode'] != 0 for item in report['iterations'])) + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tests/test_acceptance_isolation.py b/tests/test_acceptance_isolation.py new file mode 100644 index 00000000..9ad3e841 --- /dev/null +++ b/tests/test_acceptance_isolation.py @@ -0,0 +1,25 @@ +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + + +class AcceptanceIsolationTests(unittest.TestCase): + def test_agent_workflow_does_not_open_the_configured_user_database(self): + root = Path(__file__).resolve().parents[1] + with tempfile.TemporaryDirectory() as directory: + database = Path(directory) / 'user-state.sqlite3' + original = b'User data must never be opened by deterministic acceptance.' + database.write_bytes(original) + env = {**os.environ, 'KYROZEN_DB_PATH': str(database), + 'KYROZEN_DISABLE_VECTOR_INDEX': '1'} + result = subprocess.run( + [sys.executable, str(root / 'scripts' / 'agent_workflow_acceptance.py')], + cwd=directory, env=env, capture_output=True, text=True, timeout=45, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn('Agent workflow acceptance passed.', result.stdout) + self.assertEqual(database.read_bytes(), original) + diff --git a/tests/test_agent_engine.py b/tests/test_agent_engine.py index 3aa69599..8583fd8b 100644 --- a/tests/test_agent_engine.py +++ b/tests/test_agent_engine.py @@ -89,6 +89,7 @@ def complex_native_runtime(self): setattr(turn, "interaction_mode", "agent"), setattr(turn, "complexity", "complex"), setattr(turn, "learning_run", {"run_id": "run"}), setattr(turn, "learned_context", ""), ) + runtime._prepare_context_for_call = lambda *_args: None runtime._build_messages = lambda *args, **kwargs: [] runtime._permitted_tool_names = lambda: {"read_file", "update_plan"} runtime._prompt_profile = lambda: "compact" @@ -260,6 +261,7 @@ def test_native_history_keeps_past_schema_but_rejects_revoked_new_calls(self): setattr(turn, "learning_run", {"run_id": "run-1"}), setattr(turn, "learned_context", ""), ) + runtime._prepare_context_for_call = lambda *_args: None runtime._build_messages = lambda *args, **kwargs: [{"role": "user", "content": "Save this note"}] current_tools = {"write_file"} runtime._permitted_tool_names = lambda: current_tools.copy() @@ -320,6 +322,7 @@ def test_compact_native_discovery_adds_only_permitted_schemas_for_next_request(s setattr(turn, "learning_run", {"run_id": "run-1"}), setattr(turn, "learned_context", ""), ) + runtime._prepare_context_for_call = lambda *_args: None runtime._build_messages = lambda *args, **kwargs: [{"role": "user", "content": "Review Git history"}] permitted = {"read_file", "list_dir", "discover_tools", "git_log", "github_read", "browser_open", "graph_query"} runtime._permitted_tool_names = lambda: permitted @@ -379,6 +382,7 @@ def test_native_loop_stops_at_engine_step_limit(self): setattr(turn, "learning_run", {"run_id": "run-1"}), setattr(turn, "learned_context", ""), ) + runtime._prepare_context_for_call = lambda *_args: None runtime._build_messages = lambda *args, **kwargs: [{"role": "user", "content": "Read README"}] runtime._permitted_tool_names = lambda: {"read_file"} runtime._prompt_profile = lambda: "classic" @@ -412,6 +416,7 @@ def test_native_loop_rejects_unknown_tool_without_execution(self): setattr(turn, "learning_run", {"run_id": "run-1"}), setattr(turn, "learned_context", ""), ) + runtime._prepare_context_for_call = lambda *_args: None runtime._build_messages = lambda *args, **kwargs: [{"role": "user", "content": "Use missing tool"}] runtime._permitted_tool_names = lambda: {"read_file"} runtime._prompt_profile = lambda: "classic" @@ -449,6 +454,7 @@ def test_native_turn_uses_typed_calls_and_correlated_results_without_legacy_loop setattr(turn, "learning_run", {"run_id": "run-1"}), setattr(turn, "learned_context", ""), ) + runtime._prepare_context_for_call = lambda *_args: None runtime._build_messages = lambda *args, **kwargs: [{"role": "user", "content": "Read README"}] runtime._permitted_tool_names = lambda: {"read_file"} runtime._prompt_profile = lambda: "classic" @@ -645,5 +651,565 @@ def prepare_complex(turn): self.assertEqual(app.runtime.tasks.tasks[0]["description"], "Read note.txt") +class NativeLifecycleRegressionTests(unittest.TestCase): + def runtime(self, *, complexity="simple", child=False): + import time + from types import MethodType + from openkyrozen.agent.completion import _complete_turn + + runtime = AgentEngineTests().complex_native_runtime() + runtime.execution_context.child_run_id = "child-1" if child else None + runtime.tasks.tasks = [] + runtime._turn_failed = contextvars.ContextVar("failed", default=False) + runtime._turn_completion_status = contextvars.ContextVar("completion", default="completed") + runtime.subagent_manager = SimpleNamespace(coordinator=None) + runtime._clean_final_response = str + runtime._advance_fix_workflow = lambda state, _request, answer, *_args: (state, answer) + runtime._finish_learning_run = lambda _run, _receipts, _task, answer, *_args: answer + runtime._turn_cost_log = [] + runtime._complete_turn = MethodType(_complete_turn, runtime) + def prepare(turn): + turn.interaction_mode, turn.complexity = "agent", complexity + turn.learning_run, turn.turn_start = {"run_id": "fixture"}, time.time() + runtime._prepare_turn = prepare + return runtime + + def test_unsuccessful_native_finishes_cannot_complete_the_turn(self): + cases = [(FinishReason.BLOCKED, "blocked"), (FinishReason.ERROR, "incomplete"), + (FinishReason.LENGTH, "incomplete"), (FinishReason.UNKNOWN, "incomplete")] + for reason, expected_status in cases: + with self.subTest(reason=reason): + runtime = self.runtime() + runtime._get_model_response.return_value = ModelResponse( + text="A useful partial explanation.", finish_reason=reason) + turn = TurnContext("Explain the result") + answer = AgentEngine(runtime).run(turn) + self.assertEqual(turn.runtime_status, expected_status) + self.assertIn("partial explanation", answer) + self.assertTrue(turn.protocol_error_message) + + def test_unsuccessful_native_finish_never_dispatches_its_tool_calls(self): + runtime = self.runtime() + runtime._get_model_response.return_value = ModelResponse( + text="Provider failed while generating.", + tool_calls=(ToolCall("must-not-run", "read_file", {"path": "note.txt"}),), + finish_reason=FinishReason.ERROR) + AgentEngine(runtime).run(TurnContext("Read a note")) + runtime._execute_tool_batch.assert_not_called() + + def test_native_cancellation_reaches_the_runtime_cancellation_handler(self): + from openkyrozen.providers.errors import ProviderError, ProviderErrorKind + for response in (ModelResponse(text="Cancelled by provider.", finish_reason=FinishReason.CANCELLED), + ProviderError(ProviderErrorKind.CANCELLED, terminal=True)): + with self.subTest(response=type(response).__name__): + runtime = self.runtime() + if isinstance(response, Exception): + runtime._get_model_response.side_effect = response + else: + runtime._get_model_response.return_value = response + with self.assertRaises(ProviderError) as caught: + AgentEngine(runtime).run(TurnContext("Read a note")) + self.assertEqual(caught.exception.kind, ProviderErrorKind.CANCELLED) + + def test_default_native_inventory_defers_specialized_tools_in_classic_profile(self): + runtime = self.runtime() + runtime._prompt_profile = lambda: "classic" + runtime._permitted_tool_names = lambda: {"read_file", "discover_tools", "github_read", "browser_open", "graph_query"} + self.assertEqual(AgentEngine(runtime)._native_tool_names(), {"read_file", "discover_tools"}) + runtime.execution_context.discovered_tools = frozenset({"github_read"}) + self.assertEqual(AgentEngine(runtime)._native_tool_names(), {"read_file", "discover_tools", "github_read"}) + + def test_complex_child_cannot_execute_before_establishing_its_working_plan(self): + runtime = self.runtime(complexity="complex", child=True) + answer = AgentEngine(runtime).run(TurnContext("Investigate and fix the repository")) + runtime._execute_tool_batch.assert_not_called() + self.assertIn("plan", answer.lower()) + + def test_complex_child_completion_reports_its_outstanding_required_tasks(self): + runtime = self.runtime(complexity="complex", child=True) + runtime.tasks.working_plan = SimpleNamespace(id="child-plan", status="active", task_ids=["child-task"]) + runtime.tasks.tasks = [{"id": "child-task", "status": "pending", "description": "Verify the change"}] + runtime._get_model_response.return_value = ModelResponse(text="The investigation is complete.", finish_reason=FinishReason.FINAL) + turn = TurnContext("Investigate and fix the repository") + answer = AgentEngine(runtime).run(turn) + self.assertEqual(turn.runtime_status, "incomplete") + self.assertIn("Verify the change", answer) + + def test_failed_effect_verification_marks_final_status_incomplete(self): + runtime = self.runtime() + turn = TurnContext("Write a note") + runtime._prepare_turn(turn) + turn.final_answer, turn.response_text = "The file was written successfully.", "" + turn.tool_calls = [] + turn.tool_records = [{"action": "write_file", "success": True, + "verification": {"status": "failed", "method": "file_content_match"}}] + answer = runtime._complete_turn(turn) + self.assertEqual(turn.runtime_status, "incomplete") + self.assertIn("not verified", answer) + self.assertNotIn("file was written successfully", answer.lower()) + + def test_failed_write_without_effect_verification_cannot_keep_success_claim(self): + for stage in (None, "verified"): + for retried in (False, True): + with self.subTest(stage=stage, retried=retried): + runtime = self.runtime() + turn = TurnContext("Write a note") + runtime._prepare_turn(turn) + turn.fix_workflow = {"stage": stage} if stage else None + turn.final_answer, turn.response_text = "The file was written successfully.", "" + turn.tool_calls = [] + first = {"action": "write_file", "success": False, "operation_id": "same-write", + "args": "note.txt|requested", "task_id": "task", "plan_id": "plan", + "verification": {"status": "not_applicable"}} + turn.tool_records = [first] + if retried: + turn.tool_records.append({**first, "success": True, + "verification": {"status": "verified", "method": "file_content_match"}}) + answer = runtime._complete_turn(turn) + self.assertEqual(turn.runtime_status, "completed" if retried else "incomplete") + if retried: + self.assertIn("file was written successfully", answer) + else: + self.assertNotIn("file was written successfully", answer) + self.assertIn("write_file", answer) + + def test_typed_remote_effect_failure_cannot_keep_success_claim(self): + runtime = self.runtime() + turn = TurnContext("Publish the change") + runtime._prepare_turn(turn) + turn.fix_workflow = {"stage": "verified"} + turn.final_answer, turn.response_text = "The remote change was published successfully.", "" + turn.tool_calls = [] + turn.tool_records = [{"action": "git_push", "side_effects": "remote", "success": False, + "verification": {"status": "not_applicable"}}] + answer = runtime._complete_turn(turn) + self.assertEqual(turn.runtime_status, "incomplete") + self.assertNotIn("published successfully", answer) + self.assertIn("git_push", answer) + self.assertIn("not verified", answer) + + def test_only_trusted_duplicate_refusal_can_preserve_prior_verified_claim(self): + for kind in ("duplicate", "conflict", "permission_denied", "failed_verification", "different_task"): + with self.subTest(kind=kind): + runtime = self.runtime() + turn = TurnContext("Write a note") + runtime._prepare_turn(turn) + turn.fix_workflow = {"stage": "verified"} + turn.final_answer, turn.response_text = "The file was written successfully.", "" + turn.tool_calls = [] + first = {"action": "write_file", "success": True, "operation_id": "write", "args": "note.txt|requested", + "task_id": "task", "plan_id": "plan", "verification": {"status": "verified", "method": "file_content_match"}} + later = {**first, "success": False, "failure": "conflict", + "verification": {"status": "not_applicable", "method": "duplicate_success_refusal"}} + if kind == "conflict": later["verification"] = {"status": "unavailable", "method": "none"} + elif kind == "permission_denied": later.update(failure="permission_denied", verification={"status": "unavailable"}) + elif kind == "failed_verification": later["verification"] = {"status": "failed", "method": "file_content_match"} + elif kind == "different_task": later["task_id"] = "other-task" + turn.tool_records = [first, later] + answer = runtime._complete_turn(turn) + if kind == "duplicate": + self.assertIn("file was written successfully", answer) + else: + self.assertNotIn("file was written successfully", answer) + self.assertEqual(turn.runtime_status, "blocked" if kind == "permission_denied" else "incomplete") + + def test_trusted_duplicate_refusal_completes_without_a_fix_workflow(self): + runtime = self.runtime() + turn = TurnContext("Write a note") + runtime._prepare_turn(turn) + turn.final_answer, turn.response_text = "The file was written successfully.", "" + turn.tool_calls = [] + first = {"action": "write_file", "success": True, "operation_id": "write", "args": "note.txt|requested", + "task_id": "task", "plan_id": "plan", "verification": {"status": "verified", "method": "file_content_match"}} + turn.tool_records = [first, {**first, "success": False, "failure": "conflict", + "verification": {"status": "not_applicable", "method": "duplicate_success_refusal"}}] + answer = runtime._complete_turn(turn) + self.assertEqual(turn.runtime_status, "completed") + self.assertEqual(answer, "The file was written successfully.") + + def test_delegated_failure_reply_remains_valid_worker_or_review_json(self): + import json + from openkyrozen.agent.delegation import REPORT_LISTS, report + for review in (False, True): + with self.subTest(review=review): + runtime = self.runtime(complexity="complex", child=True) + runtime.execution_context.coordinator = object() + runtime.tasks.working_plan = SimpleNamespace(id="child-plan", status="active", task_ids=["child-task"]) + runtime.tasks.tasks = [{"id": "child-task", "status": "pending", "description": "Verify the change"}] + turn = TurnContext("Write a note") + runtime._prepare_turn(turn) + payload = ({"verdict": "verified", "findings": [], "evidence": []} if review else + {**{key: [] for key in REPORT_LISTS}, "status": "completed"}) + payload["summary"] = "The file was written successfully." + raw = json.dumps(payload) + turn.final_answer = "```json\n" + raw + "\n```" if review else raw + turn.response_text, turn.tool_calls = "", [] + turn.tool_records = [{"action": "write_file", "success": False, "failure": "permission_denied", + "verification": {"status": "unavailable"}, "tool_error": {"code": "permission_denied"}}] + answer = runtime._complete_turn(turn) + parsed = report(answer, review=review) + self.assertEqual(json.loads(answer), parsed) + self.assertEqual(parsed["verdict"] if review else parsed["status"], "unverifiable" if review else "blocked") + self.assertTrue(any("write_file" in finding for finding in parsed["findings"])) + self.assertNotIn("written successfully", answer) + self.assertEqual(turn.runtime_status, "blocked") + + def test_complex_child_plan_status_keeps_valid_worker_json_and_cannot_complete_active_plan(self): + import json + from openkyrozen.agent.delegation import REPORT_LISTS, report + for plan_status, task_status, expected_runtime, expected_report in ( + ("completed", "succeeded", "completed", "completed"), + ("active", "pending", "incomplete", "partial"), + ("active", "blocked", "blocked", "blocked")): + with self.subTest(plan_status=plan_status, task_status=task_status): + runtime = self.runtime(complexity="complex", child=True) + runtime.execution_context.coordinator = object() + runtime.tasks.working_plan = SimpleNamespace(id="child-plan", status=plan_status, task_ids=["child-task"]) + runtime.tasks.tasks = [{"id": "child-task", "status": task_status, "description": "Verify the change"}] + turn = TurnContext("Investigate the result") + runtime._prepare_turn(turn) + payload = {**{key: [] for key in REPORT_LISTS}, "status": "completed", "summary": "The requested result."} + turn.final_answer = json.dumps(payload) + turn.response_text, turn.tool_calls, turn.tool_records = "", [], [] + answer = runtime._complete_turn(turn) + parsed = report(answer) + self.assertEqual(json.loads(answer), parsed) + self.assertEqual(turn.runtime_status, expected_runtime) + self.assertEqual(parsed["status"], expected_report) + if expected_report == "completed": + self.assertEqual(parsed, payload) + else: + self.assertTrue(any("Plan status:" in item for item in parsed["uncertainties"])) + + def test_native_final_claim_is_replaced_after_actual_failed_write_verification(self): + from openkyrozen.app.bootstrap import build_application, build_memory + from openkyrozen.tools import ToolAdapters + from openkyrozen.tools.manifest import ToolSpec + class Provider(LLMProvider): + calls = 0 + def get_capabilities(self, model=None): return ProviderCapabilities(native_tools=True) + def chat(self, messages, model=None): raise AssertionError("legacy provider boundary used") + def chat_response(self, messages, model=None, *, tools=None): + self.calls += 1 + if self.calls == 1: + return ModelResponse(tool_calls=(ToolCall("failed-write", "write_file", { + "path": "note.txt", "content": "requested content"}),), finish_reason=FinishReason.TOOL_REQUEST, + continuation=provider_continuation(self, model, "fixture", {"call": "failed-write"})) + return ModelResponse(text="The file was written successfully.", finish_reason=FinishReason.FINAL) + with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, { + "KYROZEN_DISABLE_VECTOR_INDEX": "1", "KYROZEN_APPROVAL_MODE": "never", + }): + root = Path(directory) + (root / "note.txt").write_text("unchanged old content") + app = build_application(surface="cli", memory=build_memory(root / "state.sqlite3", index=None), tools=ToolAdapters(root)) + self.addCleanup(app.close) + runtime = app.runtime + runtime._set_workspace_root(root) + runtime.set_interaction_mode("agent") + config = ProviderConfig(provider="custom", model_simple="fixture", model_complex="fixture") + runtime._provider_config = config + provider = Provider(config) + runtime.llm_provider = runtime._default_turn.provider = provider + spec = runtime.tool_registry.get_spec("write_file") + runtime.tool_registry.register_spec(ToolSpec("write_file", lambda _arguments: ToolResult(text="Tool reported success"), + description=spec.description, input_schema=spec.input_schema, capability=spec.capability, + side_effects=spec.side_effects), replace=True) + events = [] + with patch.object(runtime, "_schedule_post_turn_learning"), patch.object(runtime, "_finish_learning_run", + side_effect=lambda _run, _receipts, _task, answer, *_args: answer): + answer = runtime.chat(runtime.current_session, "Write note.txt", on_runtime_event=events.append) + receipts = runtime.memory_bank.store.list_events("execution.receipt", user_id=runtime.tasks.user_id, + workspace_id=runtime.tasks.workspace_id, session_id=runtime.tasks.session_id) + receipt = next(event["payload"] for event in receipts if event["payload"].get("call_id") == "failed-write") + self.assertTrue(receipt["success"]) + self.assertEqual(receipt["verification"]["status"], "failed") + self.assertEqual((root / "note.txt").read_text(), "unchanged old content") + self.assertEqual(provider.calls, 2) + self.assertNotIn("file was written successfully", answer.lower()) + self.assertIn("write_file", answer) + self.assertIn("not verified", answer) + terminal = next(event for event in events if event.kind == "turn.completed") + self.assertEqual(terminal.payload["status"], "incomplete") + # CLI/TUI/Web consume the returned native reply through these literal projections. + self.assertEqual(runtime._clean_final_response(answer), answer) + self.assertEqual(runtime._split_reply(answer)[1], answer) + + def test_failed_effect_summary_preserves_provider_block_reason(self): + runtime = self.runtime() + turn = TurnContext("Write a note") + runtime._prepare_turn(turn) + turn.runtime_status = "blocked" + turn.protocol_error_message = "The provider blocked this response; no further tools were run." + turn.final_answer, turn.response_text = "The file was written successfully.", "" + turn.tool_calls = [] + turn.tool_records = [{"action": "write_file", "success": True, + "verification": {"status": "failed", "method": "file_content_match"}}] + answer = runtime._complete_turn(turn) + self.assertEqual(turn.runtime_status, "blocked") + self.assertIn("provider blocked", answer) + self.assertIn("not verified", answer) + self.assertNotIn("written successfully", answer) + + def test_verified_reproduction_failure_preserves_literal_final_reply(self): + runtime = self.runtime() + turn = TurnContext("Fix the page") + runtime._prepare_turn(turn) + turn.fix_workflow = {"stage": "verified"} + turn.final_answer = "The verified repair passed. Action: read_file is a literal example." + turn.response_text, turn.tool_calls = "", [] + turn.tool_records = [{"action": "run_cmd", "success": False, + "verification": {"status": "failed", "method": "command_exit_status"}}] + answer = runtime._complete_turn(turn) + self.assertEqual(turn.runtime_status, "completed") + self.assertEqual(answer, turn.final_answer) + self.assertIn("Action: read_file is a literal example.", answer) + + def test_verified_fix_cannot_hide_latest_failed_file_verification(self): + for stage in ("verified", "explained"): + with self.subTest(stage=stage): + runtime = self.runtime() + turn = TurnContext("Fix the page") + runtime._prepare_turn(turn) + turn.fix_workflow = {"stage": stage} + turn.final_answer, turn.response_text = "The repair was completed successfully.", "" + turn.tool_calls = [] + turn.tool_records = [{"action": "write_file", "success": True, "operation_id": "write-1", + "args": "page.txt|expected", "task_id": "task-1", "plan_id": "plan-1", + "verification": {"status": "failed", "method": "file_content_match"}}] + answer = runtime._complete_turn(turn) + self.assertEqual(turn.runtime_status, "incomplete", answer) + self.assertIn("not verified", answer) + + def test_only_later_verified_retry_of_same_bound_operation_resolves_file_failure(self): + first = {"action": "write_file", "success": True, "operation_id": "write-1", "args": "page.txt|expected", + "task_id": "task-1", "plan_id": "plan-1", "verification": {"status": "failed", "method": "file_content_match"}} + for changes, expected in (({}, "completed"), ({"operation_id": "different"}, "incomplete"), + ({"task_id": "other-task"}, "incomplete")): + with self.subTest(changes=changes): + runtime = self.runtime() + turn = TurnContext("Fix the page") + runtime._prepare_turn(turn) + turn.fix_workflow = {"stage": "verified"} + turn.final_answer, turn.response_text = "The repair was independently verified.", "" + turn.tool_calls = [] + turn.tool_records = [first, {**first, **changes, "verification": {"status": "verified", "method": "file_content_match"}}] + answer = runtime._complete_turn(turn) + self.assertEqual(turn.runtime_status, expected) + if expected == "completed": + self.assertIn("independently verified", answer) + else: + self.assertNotIn("independently verified", answer) + + def test_learning_bookkeeping_failure_preserves_the_completed_answer(self): + runtime = self.runtime() + runtime._record_learning_event = Mock() + runtime._finish_learning_run = Mock(side_effect=RuntimeError("database unavailable")) + runtime._get_model_response.return_value = ModelResponse(text="The verified direct answer.", finish_reason=FinishReason.FINAL) + turn = TurnContext("Explain the result") + answer = AgentEngine(runtime).run(turn) + self.assertEqual(answer, "The verified direct answer.") + self.assertEqual(turn.runtime_status, "completed") + + def test_deterministic_summary_does_not_call_failed_verification_completed(self): + from openkyrozen.agent.protocol import _deterministic_tool_summary + runtime = self.runtime() + answer = _deterministic_tool_summary(runtime, [{ + "action": "write_file", "success": True, "result": "File content differs from requested content.", + "verification": {"status": "failed", "method": "file_content_match"}, + }]) + self.assertIn("Could not complete", answer) + self.assertNotIn("- Completed", answer) + + def test_plan_edits_cannot_share_a_batch_with_calls_using_the_old_task_selection(self): + runtime = self.runtime(complexity="complex") + runtime.tasks.working_plan = SimpleNamespace(id="plan", status="active", task_ids=[]) + runtime._permitted_tool_names = lambda: {"read_file", "update_plan"} + runtime._get_model_response.return_value = ModelResponse(tool_calls=( + ToolCall("select", "update_plan", {"expected_revision": 1, "operations": [{"op": "start", "task": "new"}]}), + ToolCall("read", "read_file", {"path": "note.txt"}), + ), finish_reason=FinishReason.TOOL_REQUEST) + answer = AgentEngine(runtime).run(TurnContext("Inspect the task")) + runtime._execute_tool_batch.assert_not_called() + self.assertIn("separate", answer.lower()) + + def test_native_plan_start_and_completion_suggestion_bind_verified_receipts(self): + import time + from openkyrozen.app.bootstrap import build_application, build_memory + from openkyrozen.tools import ToolAdapters + command = "python -m unittest test_note" + class PlanningProvider(LLMProvider): + def get_capabilities(self, model=None): + return ProviderCapabilities(native_tools=True) + def chat(self, messages, model=None): + raise AssertionError("legacy boundary used") + def chat_response(self, messages, model=None, *, tools=None): + previous = [message for message in messages if isinstance(message, ToolReply)] + if not previous: + call = ToolCall("plan", "update_plan", {"complexity": "complex", "expected_revision": 0, "operations": [ + {"op": "add", "ref": "verify", "task": {"description": "Verify note.txt contents", "acceptance": [command], + "execution": {"action": "run_cmd", "arguments": {"command": command}}}}, + {"op": "start", "task": "verify"}, {"op": "suggest_complete", "task": "verify"}, + ]}) + elif not any(message.call_id == "verify-command" for message in previous): + call = ToolCall("verify-command", "run_cmd", {"command": command}) + else: + return ModelResponse(text="The requested file contents passed the check.", finish_reason=FinishReason.FINAL) + return ModelResponse(tool_calls=(call,), finish_reason=FinishReason.TOOL_REQUEST, + continuation=provider_continuation(self, model, "fixture", {"call": call.id})) + with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, { + "KYROZEN_DISABLE_VECTOR_INDEX": "1", "KYROZEN_APPROVAL_MODE": "never", + "PATH": str(Path(__import__("sys").executable).parent) + os.pathsep + os.environ["PATH"], + }): + root = Path(directory) + (root / "note.txt").write_text("verified content", encoding="utf-8") + (root / "test_note.py").write_text( + "import unittest\nfrom pathlib import Path\nclass NoteTest(unittest.TestCase):\n" + " def test_content(self):\n self.assertEqual(Path('note.txt').read_text(), 'verified content')\n", + encoding="utf-8") + app = build_application(surface="cli", memory=build_memory(root / "state.sqlite3", index=None), tools=ToolAdapters(root)) + self.addCleanup(app.close) + runtime = app.runtime + provider = PlanningProvider(ProviderConfig(provider="custom", model_simple="fixture")) + runtime._set_workspace_root(root) + runtime.llm_provider = runtime._default_turn.provider = provider + runtime.set_interaction_mode("agent") + def prepare(turn): + turn.interaction_mode, turn.complexity = "agent", "complex" + turn.learning_run, turn.turn_start = {"run_id": "fixture"}, time.time() + turn.learned_context = "" + with patch.object(runtime, "_prepare_turn", side_effect=prepare), \ + patch.object(runtime, "_finish_learning_run", side_effect=lambda _run, _receipts, _task, answer, *_args: answer), \ + patch.object(runtime, "_schedule_post_turn_learning"): + answer = runtime.chat(runtime.current_session, "Verify the requested file contents") + task = runtime.tasks.tasks[0] + self.assertEqual(task["status"], "succeeded", answer) + self.assertEqual(runtime.tasks.working_plan.status, "completed") + self.assertTrue(task["evidence"]) + receipts = runtime.memory_bank.store.list_events("execution.receipt", user_id=runtime.tasks.user_id, + workspace_id=runtime.tasks.workspace_id, session_id=runtime.tasks.session_id) + verification = next(event["payload"] for event in receipts if event["payload"]["call_id"] == "verify-command") + self.assertEqual(verification["task_id"], task["id"]) + self.assertEqual(verification["verification"]["status"], "verified") + self.assertIsNone(next(event["payload"] for event in receipts if event["payload"]["call_id"] == "plan")["task_id"]) + + def test_native_verified_write_receipts_stay_local_without_certifying_prose_acceptance(self): + import copy + import time + from openkyrozen.app.bootstrap import build_application, build_memory + from openkyrozen.tools import ToolAdapters + for delegated in (False, True): + with self.subTest(delegated=delegated), tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, { + "KYROZEN_DISABLE_VECTOR_INDEX": "1", "KYROZEN_APPROVAL_MODE": "never", + }): + root = Path(directory) + app = build_application(surface="cli", memory=build_memory(root / "state.sqlite3", index=None), tools=ToolAdapters(root)) + self.addCleanup(app.close) + runtime = app.runtime + runtime._set_workspace_root(root) + parent = runtime.current_session + parent.tasks.update_plan(complexity="complex", tasks=[{ + "description": "Independently verify the parent objective", "acceptance": ["Independent review passed"], + }]) + target = runtime.open_session("child-session") if delegated else parent + if delegated: + target._delegation_parent, target._delegation_task_id = parent, parent.tasks.tasks[0]["id"] + else: + parent.tasks.clear() + target.interaction.set_mode("agent") + class WriteProvider(LLMProvider): + def get_capabilities(self, model=None): + return ProviderCapabilities(native_tools=True) + def chat(self, messages, model=None): + raise AssertionError("legacy boundary used") + def chat_response(self, messages, model=None, *, tools=None): + replies = [item for item in messages if isinstance(item, ToolReply)] + arguments = {"path": "child.txt", "content": "precisely verified content"} + if not replies: + call = ToolCall("plan", "update_plan", {"complexity": "complex", "expected_revision": 0, "operations": [ + {"op": "add", "ref": "write", "task": {"description": "Write the exact requested child.txt content", + "acceptance": ["Exact requested contents were written"], + "execution": {"action": "write_file", "arguments": arguments}}}, + {"op": "start", "task": "write"}, {"op": "suggest_complete", "task": "write"}, + ]}) + elif not any(item.call_id == "write" for item in replies): + call = ToolCall("write", "write_file", arguments) + else: + return ModelResponse(text="The requested file contents were verified.", finish_reason=FinishReason.FINAL) + return ModelResponse(tool_calls=(call,), finish_reason=FinishReason.TOOL_REQUEST, + continuation=provider_continuation(self, model, "fixture", {"call": call.id})) + provider = WriteProvider(ProviderConfig(provider="custom", model_simple="fixture")) + context = copy.copy(runtime.execution_context) + context.provider = provider + context.child_run_id = "child-run" if delegated else None + token = runtime._turn_context.set(context) + self.addCleanup(runtime._turn_context.reset, token) + def prepare(turn): + turn.interaction_mode, turn.complexity = "agent", "complex" + turn.learning_run, turn.turn_start = {"run_id": "fixture"}, time.time() + turn.learned_context = "" + with patch.object(runtime, "_prepare_turn", side_effect=prepare), \ + patch.object(runtime, "_finish_learning_run", side_effect=lambda _run, _receipts, _task, answer, *_args: answer), \ + patch.object(runtime, "_schedule_post_turn_learning"): + answer = runtime.chat(target, "Write and verify the exact requested content") + self.assertEqual((root / "child.txt").read_text(), "precisely verified content") + self.assertEqual(target.tasks.tasks[0]["status"], "running", answer) + self.assertEqual(target.tasks.working_plan.status, "active") + self.assertTrue(target.tasks.tasks[0]["evidence"]) + self.assertIsNone(target.tasks.tasks[0]["evidence"][-1].get("acceptance")) + receipts = target.memory.store.list_events("execution.receipt", user_id=target.tasks.user_id, + workspace_id=target.tasks.workspace_id, session_id=target.session_id) + write = next(event["payload"] for event in receipts if event["payload"]["call_id"] == "write") + self.assertEqual((write["plan_id"], write["task_id"]), (target.tasks.working_plan.id, target.tasks.tasks[0]["id"])) + self.assertEqual(write["verification"]["status"], "verified") + if delegated: + self.assertEqual(parent.tasks.tasks[0]["status"], "pending") + self.assertEqual(parent.tasks.tasks[0]["evidence"], []) + + def test_native_foreground_stream_delivers_text_before_generation_finishes(self): + import time + import threading + from openkyrozen.app.bootstrap import build_application, build_memory + from openkyrozen.tools import ToolAdapters + from openkyrozen.providers.models import TextDelta, ResponseComplete + events = [] + delta_delivered = threading.Event() + def observe(event): + events.append(event) + if event.kind == "model.text_delta": + delta_delivered.set() + class StreamingProvider(LLMProvider): + def get_capabilities(self, model=None): + return ProviderCapabilities(native_tools=True, text_streaming=True, streaming_tool_calls=True) + def chat(self, messages, model=None): + raise AssertionError("text provider boundary used") + def chat_response(self, messages, model=None, *, tools=None): + raise AssertionError("native foreground response was buffered") + def chat_response_stream(self, messages, model=None, *, tools=None): + yield TextDelta("First useful text. ") + if not delta_delivered.wait(timeout=1): + raise AssertionError("first text was withheld until completion") + yield ResponseComplete(ModelResponse(text="First useful text. Finished.", finish_reason=FinishReason.FINAL)) + with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, {"KYROZEN_DISABLE_VECTOR_INDEX": "1"}): + root = Path(directory) + app = build_application(surface="cli", memory=build_memory(root / "state.sqlite3", index=None), tools=ToolAdapters(root)) + self.addCleanup(app.close) + runtime = app.runtime + provider = StreamingProvider(ProviderConfig(provider="custom", model_simple="fixture")) + runtime.llm_provider = runtime._default_turn.provider = provider + runtime.set_interaction_mode("agent") + def prepare(turn): + turn.interaction_mode, turn.complexity = "agent", "simple" + turn.learning_run, turn.turn_start = {"run_id": "fixture"}, time.time() + turn.learned_context = "" + with patch.object(runtime, "_prepare_turn", side_effect=prepare), \ + patch.object(runtime, "_finish_learning_run", side_effect=lambda _run, _receipts, _task, answer, *_args: answer), \ + patch.object(runtime, "_schedule_post_turn_learning"): + answer = runtime.chat(runtime.current_session, "Explain the result", on_runtime_event=observe) + self.assertEqual(answer, "First useful text. Finished.") + self.assertTrue(any(event.kind == "model.text_delta" for event in events)) + self.assertLess(next(event.sequence for event in events if event.kind == "model.text_delta"), + next(event.sequence for event in events if event.kind == "model.completed")) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_context_compaction.py b/tests/test_context_compaction.py index 13cf1b9f..de323972 100644 --- a/tests/test_context_compaction.py +++ b/tests/test_context_compaction.py @@ -114,6 +114,74 @@ def test_accepted_plan_summary_is_pinned_in_fixed_prompt(self): self.assertIn("Upgrade the service safely", pinned) self.assertIn("Inspect service", pinned) + def test_standalone_working_plan_state_survives_compaction(self): + from openkyrozen.persistence.store import EventStore + from openkyrozen.tasks.engine import TaskManager + + with tempfile.TemporaryDirectory() as directory: + manager = TaskManager(EventStore(Path(directory) / "state.sqlite3"), session_id="pinned") + created = manager.edit_plan(expected_revision=0, operations=[ + {"op": "add", "ref": "inspect", "task": { + "description": "Inspect current source", "acceptance": ["Source compared"], + }}, + {"op": "add", "ref": "verify", "task": { + "description": "Verify final result", "acceptance": ["Regression passes"], + "dependencies": ["inspect"], + }}, + ]) + mode = main._active_interaction_mode.set("agent") + try: + with patch.object(main.current_session, "tasks", manager), patch.object( + main._interaction_controller, "state", return_value={}): + messages = main._build_messages("continue", memory_packet="") + finally: + main._active_interaction_mode.reset(mode) + pinned = [item for item in messages if isinstance(item, dict) + and item.get("role") == "system" + and "Pinned Working Plan state" in item.get("content", "")] + self.assertEqual(len(pinned), 1) + for text in (created["plan_id"], created["references"]["inspect"], + created["references"]["verify"], "Regression passes", "dependencies", '"revision": 1'): + self.assertIn(text, pinned[0]["content"]) + pressured = messages[:-1] + [{"role": "user", "content": "old" * 5000}] * 8 + messages[-1:] + compacted = compact_for_pressure( + pressured, ContextState("custom", configured_window=5000, reserve_tokens=50), + lambda _text, _limit: "old history summarized", + ) + self.assertTrue(compacted.compacted) + self.assertIn(pinned[0], compacted.messages) + + def test_oversized_pinned_plan_retains_full_scoped_artifact_and_refreshes_revision(self): + import json + from openkyrozen.agent.prompts import _refresh_working_plan_context + + with tempfile.TemporaryDirectory() as directory: + app = build_application(memory=MemoryBank(Path(directory) / "state.sqlite3", index=None)) + self.addCleanup(app.close) + runtime = app.runtime + plan = runtime.tasks.edit_plan(expected_revision=0, reason="why" * 8000, operations=[ + {"op": "add", "ref": "large", "task": { + "description": "source " * 3000, "acceptance": ["full criterion retained"], + }}, + ]) + messages = [{"role": "system", "content": "fixed"}] + _refresh_working_plan_context(runtime, messages) + self.assertLessEqual(len(messages[1]["content"]), 12000) + state = json.loads(messages[1]["content"].split("\n", 2)[2]) + artifact = runtime.memory_bank.store.artifacts.read_page( + state["artifact_id"], user_id=runtime.tasks.user_id, + workspace_id=runtime.tasks.workspace_id, session_id=runtime.tasks.session_id, offset=0, limit=4000, + ) + self.assertIsNotNone(artifact) + self.assertEqual(state["revision"], 1) + runtime.tasks.edit_plan(plan_id=plan["plan_id"], expected_revision=1, operations=[ + {"op": "update", "task": plan["task_ids"][0], "fields": {"description": "Updated inspection"}}, + ]) + _refresh_working_plan_context(runtime, messages) + self.assertEqual(sum("Pinned Working Plan state" in item["content"] for item in messages), 1) + self.assertIn('"revision": 2', messages[1]["content"]) + self.assertIn("Updated inspection", messages[1]["content"]) + def test_compaction_can_summarize_legacy_prefix_before_native_history(self): from openkyrozen.providers.models import FinishReason, ModelResponse, ToolCall, ToolReply diff --git a/tests/test_delegated_planning.py b/tests/test_delegated_planning.py new file mode 100644 index 00000000..ae5dda9d --- /dev/null +++ b/tests/test_delegated_planning.py @@ -0,0 +1,290 @@ +import copy +import os +import tempfile +import threading +import unittest +from pathlib import Path +from unittest.mock import patch + +from openkyrozen.app.bootstrap import build_application, build_memory +from openkyrozen.providers.config import ProviderConfig +from openkyrozen.providers.models import ToolCall +from openkyrozen.security.capabilities import issue_capability_token +from openkyrozen.tools import ToolAdapters + + +class DelegatedPlanningTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.environment = patch.dict(os.environ, {'KYROZEN_DISABLE_VECTOR_INDEX': '1', 'KYROZEN_APPROVAL_MODE': 'never', + 'KYROZEN_WORKSPACE_ROOT': str(self.root)}) + self.environment.start() + self.addCleanup(self.environment.stop) + self.app = build_application(memory=build_memory(self.root / 'state.sqlite3'), tools=ToolAdapters(self.root)) + self.addCleanup(self.app.close) + self.runtime = self.app.runtime + self.runtime._provider_config = ProviderConfig(provider='deepseek', api_key='fixture-only', + model_simple='fixture', model_complex='fixture') + self.runtime.get_provider = lambda config: object() + self.runtime._execution_capability_token = issue_capability_token('parent', {'read', 'write'}) + self.coordinator = self.runtime.delegation() + self.addCleanup(self.coordinator.close) + self.run = {'run_id': 'child-local', 'budget_id': 'child-local', 'profile': 'coder', 'name': 'worker', + 'reviewer': 'reviewer', 'review_agent': {}, 'status': 'running', 'assignment': { + 'objective': 'Inspect and update marker.txt', 'context': 'Only assigned file', + 'scope': ['marker.txt'], 'acceptance': ['Marker verified'], 'deliverables': ['marker.txt'], + 'dependencies': [], 'reason': 'Independent work', + }} + self.coordinator.runs[self.run['run_id']] = self.run + self.coordinator.cancelled[self.run['run_id']] = threading.Event() + + def invoke(self, callback, *, review=False): + with patch.object(self.runtime, 'chat', side_effect=callback): + return self.runtime._invoke_delegated(self.run, review=review, feedback=None, coordinator=self.coordinator) + + def test_real_delegated_boundary_exposes_and_executes_only_child_local_plan(self): + runtime = self.runtime + parent = runtime.current_session + proposal = parent.interaction.propose_plan({ + 'title': 'Marker', 'summary': 'Update assigned marker', 'bounds': { + 'workspace_paths': ['marker.txt'], 'capabilities': ['read', 'write'], + 'effects': ['read_only', 'filesystem', 'internal'], 'risk_ceiling': 'normal'}, + 'steps': [{'id': 'marker', 'title': 'Marker', 'description': 'Write marker.txt', + 'acceptance': ['Marker verified']}], + }) + parent.interaction.accept_plan(parent.tasks, plan_id=proposal['plan_id'], version=proposal['version']) + parent_snapshot = copy.deepcopy(parent.tasks.working_plan_snapshot()) + parent_interaction = copy.deepcopy(parent.interaction.state()) + other = runtime.open_session('other-child') + other.tasks.update_plan(complexity='complex', tasks=[{'description': 'Other plan', 'acceptance': ['Preserved']}]) + other_snapshot = copy.deepcopy(other.tasks.working_plan_snapshot()) + child_sessions = [] + + def child_turn(child, message, **kwargs): + child_sessions.append(child) + self.assertIsNot(child.tasks, parent.tasks) + self.assertIn('update_plan', runtime._permitted_tool_names()) + created = runtime.execute_call(child, ToolCall('child-plan', 'update_plan', { + 'expected_revision': 0, 'operations': [{'op': 'add', 'ref': 'inspect', 'task': { + 'description': 'Inspect assigned marker', 'acceptance': ['Marker checked']}}], + })) + self.assertTrue(created.success, created.result) + for foreign in (parent.tasks.working_plan.id, other.tasks.working_plan.id): + denied = runtime.execute_call(child, ToolCall('foreign-plan', 'update_plan', { + 'plan_id': foreign, 'expected_revision': 1, + 'operations': [{'op': 'cancel', 'task': parent.tasks.tasks[0]['id']}], + })) + self.assertFalse(denied.success) + denied_write = runtime.execute_call(child, ToolCall('outside', 'write_file', { + 'path': 'outside.txt', 'content': 'unauthorized'})) + self.assertFalse(denied_write.success) + self.assertFalse((self.root / 'outside.txt').exists()) + self.assertEqual(parent.tasks.working_plan_snapshot(), parent_snapshot) + self.assertEqual(other.tasks.working_plan_snapshot(), other_snapshot) + self.assertEqual(parent.interaction.state(), parent_interaction) + return 'Child planning checked.' + + self.invoke(child_turn) + events = runtime.memory_bank.store.list_events('execution.receipt', limit=None, + workspace_id=parent.tasks.workspace_id, user_id=parent.memory.user_id, + session_id=child_sessions[0].session_id) + self.assertTrue(any(event['payload']['call_id'] == 'child-plan' for event in events)) + self.assertEqual(parent.tasks.working_plan_snapshot(), parent_snapshot) + self.assertEqual(parent.tasks.tasks[0].get('evidence', []), []) + + def test_readonly_worker_can_plan_locally_without_staging_or_granting_writes(self): + self.run['profile'] = 'researcher' + parent = self.runtime.current_session + parent_spec = self.runtime.tool_registry.get_spec('update_plan') + def child_turn(child, message, **kwargs): + runtime = self.runtime + self.assertIn('update_plan', runtime._permitted_tool_names()) + spec = runtime.tool_registry.get_spec('update_plan') + self.assertEqual(spec.capability, 'read') + self.assertFalse(spec.parallel_safe) + self.assertEqual(parent_spec.capability, 'write') + self.assertNotIn('write', runtime.execution_context.capability_token.capabilities) + created = runtime.execute_call(child, ToolCall('readonly-plan', 'update_plan', { + 'expected_revision': 0, 'operations': [{'op': 'add', 'ref': 'inspect', 'task': { + 'description': 'Inspect assigned file', 'acceptance': ['Inspected'], + 'execution': {'action': 'read_file', 'arguments': {'path': 'marker.txt'}}}}], + })) + self.assertTrue(created.success, created.result) + revision = child.tasks.working_plan.revision + for action, arguments in (('write_file', {'path': 'marker.txt', 'content': 'forbidden'}), + ('run_cmd', {'command': 'echo unsafe'}), + ('read_file', {'path': 'outside.txt'})): + denied = runtime.execute_call(child, ToolCall('staged-denial', 'update_plan', { + 'plan_id': child.tasks.working_plan.id, 'expected_revision': revision, + 'operations': [{'op': 'add', 'ref': 'bad', 'task': { + 'description': 'Forbidden execution', 'acceptance': ['Never'], + 'execution': {'action': action, 'arguments': arguments}}}], + })) + self.assertFalse(denied.success, action) + self.assertEqual(child.tasks.working_plan.revision, revision) + denied = runtime.execute_call(child, ToolCall('write-denial', 'write_file', { + 'path': 'marker.txt', 'content': 'forbidden'}), capabilities='write') + self.assertFalse(denied.success) + self.assertFalse((self.root / 'marker.txt').exists()) + return 'Readonly planning validated.' + self.invoke(child_turn) + self.assertIs(self.runtime.tool_registry.get_spec('update_plan'), parent_spec) + + def test_simple_child_classification_ignores_runtime_role_and_report_boilerplate(self): + from openkyrozen.providers.base import LLMProvider + from openkyrozen.providers.models import ModelResponse, ProviderCapabilities, FinishReason, ToolReply + self.run['profile'] = 'researcher' + self.run['assignment']['objective'] = 'Read marker.txt' + self.run['assignment']['context'] = 'Report its exact contents' + (self.root / 'marker.txt').write_text('simple source', encoding='utf-8') + observed, runtime = [], self.runtime + class NativeRead(LLMProvider): + def get_capabilities(self, model=None): + return ProviderCapabilities(native_tools=True) + def chat(self, messages, model=None): + raise AssertionError('legacy path used') + def chat_response(self, messages, model=None, *, tools=None): + observed.append(set(item['name'] for item in tools)) + if not any(isinstance(item, ToolReply) for item in messages): + return ModelResponse(tool_calls=(ToolCall('read', 'read_file', {'path': 'marker.txt'}),), + finish_reason=FinishReason.TOOL_REQUEST) + return ModelResponse(text='The assigned file contains simple source.', finish_reason=FinishReason.FINAL) + provider = NativeRead(runtime._provider_config) + runtime.get_provider = lambda config: provider + result = runtime._invoke_delegated(self.run, review=False, feedback=None, coordinator=self.coordinator) + self.assertIn('read_file', observed[0]) + self.assertIn('simple source', result['result']) + self.assertTrue(any(receipt['call_id'] == 'read' and receipt['success'] for receipt in result['tool_records'])) + + def test_actual_complex_readonly_native_child_establishes_own_plan_without_write_authority(self): + import json + from openkyrozen.providers.base import LLMProvider + from openkyrozen.providers.models import ModelResponse, ProviderCapabilities, FinishReason, ToolReply + self.run['profile'] = 'researcher' + self.run['assignment']['objective'] = 'Audit marker.txt comprehensively and inspect its evidence' + (self.root / 'marker.txt').write_text('independent source', encoding='utf-8') + parent = self.runtime.current_session + parent_spec = self.runtime.tool_registry.get_spec('update_plan') + runtime, observed, children = self.runtime, [], [] + class NativeAudit(LLMProvider): + def get_capabilities(self, model=None): + return ProviderCapabilities(native_tools=True) + def chat(self, messages, model=None): + raise AssertionError('legacy provider path used') + def chat_response(self, messages, model=None, *, tools=None): + children.append(runtime.current_session) + replies = [item for item in messages if isinstance(item, ToolReply)] + observed.append(set(item['name'] for item in tools)) + if not replies: + self_case.assertEqual(observed[-1], {'update_plan'}) + call = ToolCall('local-plan', 'update_plan', {'expected_revision': 0, 'operations': [ + {'op': 'add', 'ref': 'inspect', 'task': {'description': 'Inspect assigned marker source', + 'acceptance': ['Report exact checked source'], + 'execution': {'action': 'read_file', 'arguments': {'path': 'marker.txt'}}}}, + {'op': 'start', 'task': 'inspect'}, + ]}) + elif not any(item.call_id == 'inspect' for item in replies): + call = ToolCall('inspect', 'read_file', {'path': 'marker.txt'}) + else: + return ModelResponse(text=json.dumps({'status': 'partial', 'summary': 'Source inspected; independent acceptance still pending', + 'findings': [], 'evidence': ['marker.txt'], 'artifacts': [], 'checks': ['Source read'], + 'uncertainties': ['Independent acceptance pending'], 'suggested_followups': []}), finish_reason=FinishReason.FINAL) + return ModelResponse(tool_calls=(call,), finish_reason=FinishReason.TOOL_REQUEST) + self_case = self + provider = NativeAudit(self.runtime._provider_config) + self.runtime.get_provider = lambda config: provider + result = self.runtime._invoke_delegated(self.run, review=False, feedback=None, coordinator=self.coordinator) + self.assertIn('Source inspected', result['result']) + child = children[0] + self.assertIsNotNone(child.tasks.working_plan) + self.assertEqual(len(child.tasks.tasks), 1) + read = next(receipt for receipt in result['tool_records'] if receipt['call_id'] == 'inspect') + self.assertTrue(read['success']) + self.assertEqual(read['plan_id'], child.tasks.working_plan.id) + self.assertEqual(read['task_id'], child.tasks.tasks[0]['id']) + self.assertEqual(read['session_id'], child.session_id) + self.assertEqual(parent.tasks.tasks, []) + self.assertIs(runtime.tool_registry.get_spec('update_plan'), parent_spec) + self.assertEqual(parent_spec.capability, 'write') + self.assertEqual((self.root / 'marker.txt').read_text(), 'independent source') + + def test_children_and_reviewers_retrieve_only_their_own_tool_artifacts(self): + (self.root / 'marker.txt').write_text('own output ' * 1400 + 'END_OF_OWN_OUTPUT', encoding='utf-8') + parent = self.runtime.current_session + parent_scope = {'user_id': parent.tasks.user_id, 'workspace_id': parent.tasks.workspace_id, + 'session_id': parent.tasks.session_id} + parent_artifact = parent.memory.store.artifacts.create('PRIVATE_PARENT_OUTPUT', **parent_scope) + for reviewing in (False, True): + with self.subTest(reviewing=reviewing): + def child_turn(child, message, **kwargs): + runtime = self.runtime + self.assertIn('read_tool_artifact', runtime._permitted_tool_names()) + scope = {'user_id': child.tasks.user_id, 'workspace_id': child.tasks.workspace_id, + 'session_id': child.tasks.session_id} + archived = runtime._agent_engine._store_tool_output('OWN_ARCHIVED_OUTPUT') + fetched = runtime.execute_call(child, ToolCall('own-archive', 'read_tool_artifact', {'artifact_id': archived})) + self.assertTrue(fetched.success, fetched.result) + self.assertEqual(fetched.tool_result.data['content'], 'OWN_ARCHIVED_OUTPUT') + read = runtime.execute_call(child, ToolCall('large-read', 'read_file', {'path': 'marker.txt'})) + self.assertTrue(read.success) + artifact = read.tool_result.data['artifact_id'] + size = read.tool_result.data['total_chars'] + last = runtime.execute_call(child, ToolCall('last-page', 'read_tool_artifact', { + 'artifact_id': artifact, 'offset': max(0, size - 1000), 'limit': 1000})) + self.assertTrue(last.success, last.result) + self.assertIn('END_OF_OWN_OUTPUT', last.tool_result.data['content']) + foreign = [parent_artifact] + for alternate in ({**scope, 'user_id': 'another-user'}, + {**scope, 'workspace_id': 'another-workspace'}, + {**scope, 'session_id': 'another-child'}): + foreign.append(child.memory.store.artifacts.create('PRIVATE_FOREIGN_OUTPUT', **alternate)) + for artifact_id in foreign: + denied = runtime.execute_call(child, ToolCall('denied', 'read_tool_artifact', {'artifact_id': artifact_id})) + self.assertFalse(denied.success) + self.assertEqual(denied.failure, 'not_found') + self.assertNotIn('PRIVATE_', denied.result) + self.assertIsNone(child.memory.store.artifacts.read_page(archived, 0, 100, **parent_scope)) + return 'Own artifacts checked.' + self.invoke(child_turn, review=reviewing) + + def test_parent_material_reapproval_still_blocks_child_plan_mutation(self): + parent = self.runtime.current_session + proposal = parent.interaction.propose_plan({ + 'title': 'Marker', 'summary': 'Update marker', 'bounds': { + 'workspace_paths': ['marker.txt'], 'capabilities': ['read', 'write'], + 'effects': ['read_only', 'filesystem', 'internal'], 'risk_ceiling': 'normal'}, + 'steps': [{'id': 'marker', 'title': 'Marker', 'description': 'Write marker.txt', + 'acceptance': ['Marker verified']}], + }) + parent.interaction.accept_plan(parent.tasks, plan_id=proposal['plan_id'], version=proposal['version']) + parent.interaction.propose_plan({ + 'title': 'Expand work', 'summary': 'Add an unrelated output', 'bounds': proposal['bounds'], + 'steps': [{'id': 'extra', 'title': 'Extra', 'description': 'Write another.txt', + 'acceptance': ['Another output created']}], + }) + self.assertIsNotNone(parent.interaction.state()['pending_plan']) + before = copy.deepcopy(parent.tasks.working_plan_snapshot()) + def child_turn(child, message, **kwargs): + denied = self.runtime.execute_call(child, ToolCall('must-wait', 'update_plan', { + 'expected_revision': 0, 'operations': [{'op': 'add', 'ref': 'new', 'task': { + 'description': 'Must wait', 'acceptance': ['Never created']}}], + })) + self.assertFalse(denied.success) + self.assertIsNone(child.tasks.working_plan) + self.assertEqual(parent.tasks.working_plan_snapshot(), before) + return 'Child waits for renewed approval.' + self.invoke(child_turn) + + def test_readonly_reviewer_cannot_acquire_plan_write_permission(self): + def review_turn(child, message, **kwargs): + self.assertNotIn('update_plan', self.runtime._permitted_tool_names()) + denied = self.runtime.execute_call(child, ToolCall('forbidden-plan', 'update_plan', { + 'expected_revision': 0, 'operations': [{'op': 'add', 'ref': 'new', 'task': { + 'description': 'Must not create', 'acceptance': ['Never']}}], + })) + self.assertFalse(denied.success) + self.assertIsNone(child.tasks.working_plan) + return 'Review remains read only.' + self.invoke(review_turn, review=True) diff --git a/tests/test_delegation.py b/tests/test_delegation.py index e834f1f4..1dacf653 100644 --- a/tests/test_delegation.py +++ b/tests/test_delegation.py @@ -495,8 +495,10 @@ def chat(inner, messages, model): return ('Plan:\n1. Inspect workspace structure.\n2. Spawn two specialist agents.\n3. Wait for verified results and synthesize.\n' 'Action: {"action":"list_dir","args":"."}'), None runtime.llm_provider = MainProvider() + original_classifier = runtime._classify_complexity with (patch.object(runtime, "dispatch_learning_cycle", return_value={}), - patch.object(runtime, "_classify_complexity", return_value="complex"), + patch.object(runtime, "_classify_complexity", side_effect=lambda request: + "complex" if request == "Inspect two independent components" else original_classifier(request)), patch.object(runtime, "_touch_detached_learning_heartbeat"), patch.object(runtime, "_render_tool_result"), patch.object(runtime, "_update_tasks_panel")): @@ -1002,7 +1004,15 @@ def handle(request): self.runtime._provider_config = ProviderConfig(provider="openai", api_key="fixture-openai", base_url="https://openai.fixture/v1", model_simple="gpt-fixture", model_complex="gpt-fixture") self.runtime.DEEPSEEK_MODEL = "gpt-fixture" - self.runtime.get_provider = get_provider + # This fixture checks nonstreaming SDK request shapes; live streaming has separate coverage. + from dataclasses import replace + def fixture_provider(config): + provider = get_provider(config) + capabilities = provider.get_capabilities + provider.get_capabilities = lambda model=None: replace( + capabilities(model), text_streaming=False, streaming_tool_calls=False) + return provider + self.runtime.get_provider = fixture_provider (self.root / "agent.yaml").write_text("subagents:\n roles:\n researcher:\n provider: anthropic\n model: claude-fixture\n") with (patch.dict(os.environ, {"ANTHROPIC_API_KEY": "fixture-claude", "KYROZEN_API_KEY": "wrong-main"}), patch("openai.OpenAI", side_effect=lambda **kw: real_openai(**kw, http_client=httpx.Client(transport=transport))), diff --git a/tests/test_distribution.py b/tests/test_distribution.py index c9d20185..97fcbc7d 100644 --- a/tests/test_distribution.py +++ b/tests/test_distribution.py @@ -176,12 +176,12 @@ def test_tag_release_workflow_validates_source_and_installed_artifacts(self): def test_public_install_paths_pin_the_verified_release(self): release_url = ( "https://github.com/EvanProgramming/OpenKyrozen/releases/download/" - "v3.0.0/openkyrozen-3.0.0-py3-none-any.whl" + "v2.0.8/openkyrozen-2.0.8-py3-none-any.whl" ) for readme in sorted(ROOT.glob("README*.md")): text = readme.read_text(encoding="utf-8") with self.subTest(readme=readme.name): - self.assertIn("raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v3.0.0/", text) + self.assertIn("raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v2.0.8/", text) wheel_urls = re.findall( r"https://github\.com/EvanProgramming/OpenKyrozen/releases/download/[^\s)]+\.whl", text, diff --git a/tests/test_durable_tasks_gateway.py b/tests/test_durable_tasks_gateway.py index c3ba802f..ed05c67b 100644 --- a/tests/test_durable_tasks_gateway.py +++ b/tests/test_durable_tasks_gateway.py @@ -89,7 +89,7 @@ def _stop_server(process: subprocess.Popen) -> None: if process.stdout: process.stdout.close() - def test_gateway_recovers_scoped_running_task_and_executes_api_task(self): + def test_gateway_blocks_interrupted_mutations_and_executes_new_api_task(self): with tempfile.TemporaryDirectory(prefix="openkyrozen-task-gateway-") as directory: root = Path(directory) workspace = root / "workspace" @@ -127,30 +127,16 @@ def test_gateway_recovers_scoped_running_task_and_executes_api_task(self): process, base_url = self._start_server(workspace, db_path, skills_path) try: - deadline = time.monotonic() + 10 recovered = self._request(base_url, "/api/v2/tasks?session_id=restart-session") - while time.monotonic() < deadline: - recovered = self._request(base_url, "/api/v2/tasks?session_id=restart-session") - if recovered["tasks"][0]["status"] == "succeeded": - break - time.sleep(0.1) - self.assertEqual(recovered["tasks"][0]["status"], "succeeded") - self.assertEqual((workspace / "recovered.txt").read_text(encoding="utf-8"), "after restart") - - deadline = time.monotonic() + 10 - recovered_command = self._request( - base_url, "/api/v2/tasks?session_id=restart-command-session" - ) - while time.monotonic() < deadline: - recovered_command = self._request( - base_url, "/api/v2/tasks?session_id=restart-command-session" - ) - if recovered_command["tasks"][0]["status"] == "failed": - break - time.sleep(0.1) - self.assertEqual(recovered_command["tasks"][0]["status"], "failed") - self.assertTrue(recovered_command["tasks"][0]["evidence"]) - self.assertFalse(recovered_command["tasks"][0]["evidence"][-1]["success"]) + self.assertEqual(recovered["tasks"][0]["status"], "blocked") + self.assertFalse((workspace / "recovered.txt").exists()) + recovered_command = self._request(base_url, "/api/v2/tasks?session_id=restart-command-session") + self.assertEqual(recovered_command["tasks"][0]["status"], "blocked") + self.assertEqual(recovered_command["tasks"][0]["evidence"], []) + with self.assertRaises(urllib.error.HTTPError) as denied: + self._request(base_url, f"/api/v2/tasks/{recovered['tasks'][0]['id']}/resume", + method="POST", body={}) + self.assertEqual(denied.exception.code, 409) created = self._request( base_url, @@ -212,7 +198,7 @@ def test_gateway_recovers_scoped_running_task_and_executes_api_task(self): events = self._request(base_url, "/api/v2/events?session_id=restart-session")["events"] event_types = {event["event_type"] for event in events} self.assertIn("task.recovered", event_types) - self.assertIn("task.execution_completed", event_types) + self.assertNotIn("task.execution_completed", event_types) finally: self._stop_server(process) diff --git a/tests/test_evolution.py b/tests/test_evolution.py index 2c37bc62..4b12d417 100644 --- a/tests/test_evolution.py +++ b/tests/test_evolution.py @@ -97,10 +97,9 @@ def _candidate(self): }) def _pass_replay(self, candidate): - return self.engine.record_shadow_replay( - candidate["proposal_id"], [{"case_id": "case-1", "verified_success": True}], - [{"case_id": "case-1", "verified_success": True}], - ) + measured = {"case_id": "case-1", "verified_success": True, "tokens": 100, + "latency": 0.5, "cost_rmb": 0.002, "corrections": 0, "repeated_errors": 0} + return self.engine.record_shadow_replay(candidate["proposal_id"], [measured], [measured]) def test_profile_isolation_canary_promotion_and_correction_rollback(self): candidate = self._candidate() @@ -265,6 +264,120 @@ def test_unverified_correction_does_not_arm_regression_preflight(self): self.assertEqual(self.memory.store.list_events( "learning.regression_case_created", workspace_id="project"), []) + def test_direct_activation_cannot_bypass_live_evidence_or_shadow_replay(self): + candidate = self._candidate() + self.assertFalse(self.engine.validate_and_activate(candidate["proposal_id"], validation={"success": True})) + self.assertEqual(self.registry.list()[0]["status"], "canary") + for _ in range(2): + run = self.engine.begin_run("coder", "run pytest tests") + _, receipts = self.engine.artifact_context(run) + self.engine.record_outcome(run, receipts, verified=True, success=True) + self.assertFalse(self.engine.validate_and_activate(candidate["proposal_id"], validation={"success": True})) + self._pass_replay(candidate) + self.assertTrue(self.engine.validate_and_activate(candidate["proposal_id"], validation={"success": True})) + self.assertEqual(self.registry.list()[0]["status"], "active") + + def test_incomplete_measured_replay_cannot_promote_even_after_verified_uses(self): + candidate = self._candidate() + for _ in range(2): + run = self.engine.begin_run("coder", "run pytest tests") + _, receipts = self.engine.artifact_context(run) + self.engine.record_outcome(run, receipts, verified=True, success=True) + measured = {"case_id": "measured-case", "verified_success": True, "tokens": 100, + "latency": 0.5, "cost_rmb": 0.002, "corrections": 0, "repeated_errors": 0} + for missing in (("cost_rmb",), ("latency",), ("corrections",), ("repeated_errors",), + ("cost_rmb", "latency", "corrections", "repeated_errors")): + with self.subTest(missing=missing): + incomplete = {key: value for key, value in measured.items() if key not in missing} + replay = self.engine.record_shadow_replay(candidate["proposal_id"], [incomplete], [incomplete]) + self.assertFalse(replay["non_regressing"]) + self.assertFalse(self.engine.validate_and_activate(candidate["proposal_id"], validation={"success": True})) + card = self.engine.evidence_card(candidate["proposal_id"]) + self.assertEqual(card["artifact"]["status"], "canary") + self.assertEqual(card["revalidation_status"], "incomplete") + self.assertIn("incomplete", card["shadow_replay"]["blocked_reason"].lower()) + replay = self.engine.record_shadow_replay(candidate["proposal_id"], [measured], [measured]) + self.assertTrue(replay["non_regressing"]) + self.assertTrue(self.engine.validate_and_activate(candidate["proposal_id"], validation={"success": True})) + self.assertEqual(self.engine.evidence_card(candidate["proposal_id"])["artifact"]["status"], "active") + + def test_replay_requires_a_consistent_explicit_cost_unit(self): + candidate = self._candidate() + measured = {"case_id": "cost-case", "verified_success": True, "latency": 0.5, + "cost": 0.002, "corrections": 0, "repeated_errors": 0} + self.assertFalse(self.engine.record_shadow_replay(candidate["proposal_id"], [measured], [measured])["non_regressing"]) + dollars = {**measured, "cost_unit": "USD"} + euros = {**measured, "cost_unit": "EUR"} + self.assertFalse(self.engine.record_shadow_replay(candidate["proposal_id"], [euros], [dollars])["non_regressing"]) + self.assertTrue(self.engine.record_shadow_replay(candidate["proposal_id"], [dollars], [dollars])["non_regressing"]) + + def test_persisted_legacy_success_only_replay_cannot_activate(self): + candidate = self._candidate() + for _ in range(2): + run = self.engine.begin_run("coder", "run pytest tests") + _, receipts = self.engine.artifact_context(run) + self.engine.record_outcome(run, receipts, verified=True, success=True) + proposal = self.engine._proposal_for_skill(candidate["skill_id"]) + self.memory.store.update_proposal(candidate["proposal_id"], status="canary", validation={ + **proposal["validation"], "shadow_replay": {"non_regressing": True, + "case_ids": ["legacy"], "candidate_successes": 1, "predecessor_successes": 1}, + }) + self.assertFalse(self.engine.validate_and_activate(candidate["proposal_id"], validation={"success": True})) + run = self.engine.begin_run("coder", "run pytest tests") + _, receipts = self.engine.artifact_context(run) + self.assertEqual(self.engine.record_outcome(run, receipts, verified=True, success=True), []) + self.assertEqual(self.engine.evidence_card(candidate["proposal_id"])["artifact"]["status"], "canary") + + def test_shadow_replay_rejects_individual_regression_hidden_by_aggregate_success(self): + candidate = self._candidate() + measurements = {"latency": 0.5, "cost_rmb": 0.002, "corrections": 0, "repeated_errors": 0} + replay = self.engine.record_shadow_replay(candidate["proposal_id"], [ + {**measurements, "case_id": "old", "verified_success": False}, + {**measurements, "case_id": "new", "verified_success": True}, + ], [ + {**measurements, "case_id": "old", "verified_success": True}, + {**measurements, "case_id": "new", "verified_success": False}, + ]) + self.assertFalse(replay["non_regressing"]) + + def test_shadow_replay_blocks_measured_cost_latency_and_error_regressions(self): + candidate = self._candidate() + baseline = {"case_id": "case", "verified_success": True, "tokens": 100, + "latency": 1.0, "cost_rmb": 0.002, "corrections": 0, "repeated_errors": 0} + for field, value in (("tokens", 101), ("latency", 1.1), ("cost_rmb", 0.003), + ("corrections", 1), ("repeated_errors", 1)): + with self.subTest(field=field): + changed = {**baseline, field: value} + result = self.engine.record_shadow_replay(candidate["proposal_id"], [changed], [baseline]) + self.assertFalse(result["non_regressing"]) + result = self.engine.record_shadow_replay(candidate["proposal_id"], [baseline], [baseline]) + self.assertTrue(result["non_regressing"]) + self.assertEqual(result["metric_comparisons"]["tokens"], {"candidate": 100, "predecessor": 100, "non_regressing": True}) + + def test_measured_replay_rejects_invalid_measurements_and_overflowing_totals(self): + candidate = self._candidate() + measured = {"case_id": "case-1", "verified_success": True, "latency": 0.5, + "cost_rmb": 0.002, "corrections": 0, "repeated_errors": 0} + for field in ("latency", "cost_rmb", "corrections", "repeated_errors"): + for value in (None, True, -1, float("inf"), float("nan")): + with self.subTest(field=field, value=value), self.assertRaises(ValueError): + self.engine.record_shadow_replay(candidate["proposal_id"], [{**measured, field: value}], [measured]) + huge = [{**measured, "case_id": f"case-{index}", "cost_rmb": 1e308} for index in range(2)] + with self.assertRaises(ValueError): + self.engine.record_shadow_replay(candidate["proposal_id"], huge, huge) + + def test_shadow_replay_cannot_drop_baseline_measurement_or_use_truthy_success(self): + candidate = self._candidate() + for current, previous in (( + {"case_id": "case", "verified_success": True}, + {"case_id": "case", "verified_success": True, "latency": 1.0}, + ), ( + {"case_id": "case", "verified_success": "false"}, + {"case_id": "case", "verified_success": True}, + )): + with self.subTest(current=current), self.assertRaises(ValueError): + self.engine.record_shadow_replay(candidate["proposal_id"], [current], [previous]) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_evolution_robustness.py b/tests/test_evolution_robustness.py index 30021632..e81c7a05 100644 --- a/tests/test_evolution_robustness.py +++ b/tests/test_evolution_robustness.py @@ -47,9 +47,11 @@ def test_model_bound_artifact_requires_matching_model_or_cross_model_proof(self) other = self.engine.begin_run("coder", "pytest", provider_model="provider:model-b") self.assertEqual(self.engine.artifact_context(other), ("", [])) self.registry.set_learned_status(candidate["skill_id"], "active") + measurements = {"verified_success": True, "latency": 0.5, "cost_rmb": 0.002, + "corrections": 0, "repeated_errors": 0} paired = [ - {"case_id": "a", "verified_success": True, "provider_model": "provider:model-a"}, - {"case_id": "b", "verified_success": True, "provider_model": "provider:model-b"}, + {**measurements, "case_id": "a", "provider_model": "provider:model-a"}, + {**measurements, "case_id": "b", "provider_model": "provider:model-b"}, ] self.engine.record_shadow_replay(candidate["proposal_id"], paired, paired) context, receipts = self.engine.artifact_context(other) diff --git a/tests/test_execution_receipts.py b/tests/test_execution_receipts.py index 6060f78a..af3ea64b 100644 --- a/tests/test_execution_receipts.py +++ b/tests/test_execution_receipts.py @@ -938,6 +938,68 @@ def test_file_effect_verification_compares_requested_contents(self): self.assertIsNone(receipt.verified_effect) self.assertIsNone(receipt.acceptance) + def test_exact_operation_acceptance_does_not_cover_extra_criteria_or_changed_arguments(self): + runtime = self.app.runtime + cases = [ + ("unrelated-criterion", ["All unit tests pass"], "requested", "requested"), + ("extra-criteria", ["Requested contents exist", "Independent second condition"], "requested", "requested"), + ("changed-arguments", ["Requested contents exist"], "requested", "different"), + ] + for name, criteria, declared, actual in cases: + with self.subTest(name=name): + session = runtime.open_session(name) + with runtime.use_session(session): + plan = runtime.tasks.edit_plan(complexity="complex", expected_revision=0, operations=[ + {"op": "add", "ref": "write", "task": {"description": "Write exact contents", "acceptance": criteria, + "execution": {"action": "write_file", "arguments": {"path": name + ".txt", "content": declared}}}}, + {"op": "start", "task": "write"}, {"op": "suggest_complete", "task": "write"}, + ]) + receipt = runtime._execute_turn_action("write_file", {"path": name + ".txt", "content": actual}, + operation_scope=name, successful_operations=set(), call_id=name, call_id_source="native", + task_id=plan["task_ids"][0], task_plan_id=plan["plan_id"]) + self.assertEqual(receipt.verification["status"], "verified") + record = runtime._record_turn_receipt(receipt) + self.assertIsNone(record["acceptance"]) + self.assertEqual(runtime.tasks.tasks[0]["status"], "running") + + def test_child_local_receipt_cannot_fall_back_to_a_parent_task_id(self): + runtime = self.app.runtime + self.plan() + parent = runtime.current_session + parent_task_id = parent.tasks.tasks[0]["id"] + child = runtime.open_session("isolated-child") + child._delegation_parent, child._delegation_task_id = parent, parent_task_id + with runtime.use_session(child): + plan = runtime.tasks.edit_plan(complexity="complex", expected_revision=0, operations=[ + {"op": "add", "ref": "local", "task": {"description": "Check a value", "acceptance": ["Value checked"], + "execution": {"action": "calculate", "arguments": {"expression": "2+2"}}}}, + {"op": "start", "task": "local"}, + ]) + receipt = runtime._execute_turn_action("calculate", {"expression": "2+2"}, + operation_scope="isolated-child", successful_operations=set(), call_id="local-call", call_id_source="native", + task_id=plan["task_ids"][0], task_plan_id=plan["plan_id"]) + self.assertEqual(receipt.plan_id, plan["plan_id"]) + forged_binding = ExecutionReceipt.from_dict({**receipt.as_dict(), "task_id": parent_task_id}) + record = runtime._record_turn_receipt(forged_binding) + self.assertIsNone(record["task_id"]) + self.assertEqual(child.tasks.tasks[0]["evidence"], []) + runtime._record_turn_receipt(forged_binding) + self.assertEqual(parent.tasks.tasks[0]["evidence"], []) + self.assertEqual(parent.tasks.tasks[0]["status"], "pending") + + def test_native_selected_task_identity_survives_argument_rejection(self): + runtime = self.app.runtime + plan = self.plan("write_file", {"path": "invalid.txt", "content": "requested"}) + task_id = runtime.tasks.tasks[0]["id"] + runtime.tasks.edit_plan(plan_id=plan.id, expected_revision=1, operations=[{"op": "start", "task": task_id}]) + receipt = runtime._execute_turn_action("write_file", {"path": "invalid.txt"}, + operation_scope="invalid-selected", successful_operations=set(), call_id="invalid-selected", call_id_source="native", + task_id=task_id, task_plan_id=plan.id) + self.assertFalse(receipt.success) + self.assertEqual(receipt.task_id, task_id) + self.assertEqual(receipt.plan_id, plan.id) + self.assertFalse((self.root / "invalid.txt").exists()) + def test_command_verification_uses_exit_status_not_output_words(self): from openkyrozen.providers.models import ToolCall diff --git a/tests/test_interaction.py b/tests/test_interaction.py index 9c0a5441..f13e87c8 100644 --- a/tests/test_interaction.py +++ b/tests/test_interaction.py @@ -345,6 +345,110 @@ def test_working_plan_edits_cannot_add_or_redefine_unbound_strategic_work(self): binding = next(task for task in result["tasks"] if task["id"] == task_id)["checkpoint"]["strategic_plan"] self.assertEqual(binding["fingerprint"], accepted["fingerprint"]) + def test_strategic_split_retains_approved_anchor_and_adds_bound_strategy(self): + from copy import deepcopy + plan = self._proposal() + accepted, _ = self.controller.accept_plan(self.manager, plan_id=plan["plan_id"], version=1) + anchor_id = accepted["working_task_ids"][0] + before = deepcopy(self.manager.tasks[0]) + result = self.manager.edit_plan( + plan_id=self.manager.working_plan.id, expected_revision=1, + operations=[{"op": "split", "task": anchor_id, "children": [ + {"ref": "inspect", "task": {"description": "Inspect page input", "acceptance": ["Input checked"]}}, + {"ref": "verify", "task": {"description": "Verify page output", "acceptance": ["Output checked"]}}, + ]}], + ) + anchor = next(task for task in result["tasks"] if task["id"] == anchor_id) + self.assertEqual(anchor["status"], "pending") + self.assertEqual(anchor["acceptance"], before["acceptance"]) + self.assertEqual(anchor["description"], before["description"]) + children = [task for task in result["tasks"] if task["id"] != anchor_id] + self.assertEqual(anchor["dependencies"], [task["id"] for task in children]) + for child in children: + self.assertEqual(child["checkpoint"]["strategy_anchor"], anchor_id) + self.assertEqual(child["checkpoint"]["strategic_plan"], before["checkpoint"]["strategic_plan"]) + snapshot = self.manager.working_plan_snapshot() + self.assertEqual(snapshot["tasks"][0]["role"], "approved_objective") + self.assertEqual(snapshot["tasks"][1]["strategy_anchor"], anchor_id) + self.assertEqual(self.controller.state()["executing_plan"], accepted) + self.assertFalse(self.manager._has_evidence(0)) + added = self.manager.edit_plan( + plan_id=self.manager.working_plan.id, expected_revision=2, + operations=[{"op": "add", "anchor": anchor_id, "ref": "extra", "task": { + "description": "Investigate page edge case", "acceptance": ["Edge case checked"], + }}], + ) + self.assertIn(added["references"]["extra"], self.manager.tasks[0]["dependencies"]) + self.assertFalse(self.manager._has_evidence(0)) + self.assertFalse(self.manager.set_status(0, "succeeded", evidence={ + "success": True, "acceptance": "Original objective verified", + "acceptance_ids": before["checkpoint"]["strategic_plan"]["acceptance_ids"], + })) + + def test_strategy_children_can_merge_and_replace_without_replacing_objective(self): + proposal = self._proposal() + accepted, _ = self.controller.accept_plan(self.manager, plan_id=proposal["plan_id"], version=1) + anchor = accepted["working_task_ids"][0] + refined = self.manager.edit_plan(plan_id=self.manager.working_plan.id, expected_revision=1, operations=[ + {"op": "split", "task": anchor, "children": [ + {"ref": "a", "task": {"description": "Inspect input", "acceptance": ["Input checked"]}}, + {"ref": "b", "task": {"description": "Inspect output", "acceptance": ["Output checked"]}}, + ]}, + ]) + merged = self.manager.edit_plan(plan_id=self.manager.working_plan.id, expected_revision=2, operations=[ + {"op": "merge", "tasks": [refined["references"]["a"], refined["references"]["b"]], + "replacement": {"ref": "both", "task": {"description": "Inspect both", "acceptance": ["Both checked"]}}}, + ]) + merged_id = merged["references"]["both"] + self.assertEqual(self.manager.tasks[0]["dependencies"], [merged_id]) + revised = self.manager.edit_plan(plan_id=self.manager.working_plan.id, expected_revision=3, operations=[ + {"op": "replace", "task": merged_id, "replacement": { + "ref": "new", "task": {"description": "Investigate root cause", "acceptance": ["Cause checked"]}, + }}, + ]) + replacement = next(task for task in revised["tasks"] if task["id"] == revised["references"]["new"]) + self.assertEqual(replacement["checkpoint"]["strategy_anchor"], anchor) + self.assertEqual(self.manager.tasks[0]["status"], "pending") + self.assertEqual(self.manager.tasks[0]["checkpoint"]["strategic_plan"]["acceptance_ids"], + proposal["steps"][0]["acceptance_ids"]) + + def test_strategic_completion_rejects_unfinished_refinement_even_after_dependency_removal(self): + proposal = self._proposal() + accepted, _ = self.controller.accept_plan(self.manager, plan_id=proposal["plan_id"], version=1) + anchor_id = accepted["working_task_ids"][0] + refined = self.manager.edit_plan(plan_id=self.manager.working_plan.id, expected_revision=1, operations=[ + {"op": "add", "anchor": anchor_id, "ref": "inspect", "task": { + "description": "Inspect page", "acceptance": ["Inspected"], + }}, + ]) + self.manager.edit_plan(plan_id=self.manager.working_plan.id, expected_revision=2, operations=[ + {"op": "remove_dependency", "task": anchor_id, "depends_on": refined["references"]["inspect"]}, + ]) + self.assertTrue(self.manager.set_status(0, "succeeded", evidence={ + "success": True, "acceptance": "Verified approved objective", + "acceptance_ids": self.manager.tasks[0]["checkpoint"]["strategic_plan"]["acceptance_ids"], + })) + with self.assertRaises(InteractionError): + self.controller.complete_plan(accepted, task_manager=self.manager) + + def test_material_reapproval_cancels_previous_strategy_children(self): + first = self._proposal() + accepted, _ = self.controller.accept_plan(self.manager, plan_id=first["plan_id"], version=1) + anchor_id = accepted["working_task_ids"][0] + refined = self.manager.edit_plan(plan_id=self.manager.working_plan.id, expected_revision=1, operations=[ + {"op": "add", "anchor": anchor_id, "ref": "inspect", "task": { + "description": "Inspect page", "acceptance": ["Inspected"], + }}, + ]) + revision = self.controller.propose_plan({ + "title": "Revised page", "summary": "A revised outcome", "bounds": first["bounds"], + "steps": [{"id": "page", "title": "Create revised page", "description": "Write revised page.", + "acceptance": ["Revised content correct"]}], + }) + self.controller.accept_plan(self.manager, plan_id=revision["plan_id"], version=2) + child = next(task for task in self.manager.tasks if task["id"] == refined["references"]["inspect"]) + self.assertEqual(child["status"], "cancelled") + def test_accepting_material_revision_replaces_executing_plan_atomically(self): first = self._proposal() accepted, _ = self.controller.accept_plan( @@ -1347,8 +1451,7 @@ def restore(): self.addCleanup(restore) self.agent.short_term_memory = [{"role": "user", "content": "previous project"}] empty_history = type("History", (), {"current": lambda self: None})() - with patch.object(self.agent, "TaskManager", return_value=object()), \ - patch.object(self.agent, "InteractionController", return_value=object()), \ + with patch.object(self.agent, "InteractionController", return_value=object()), \ patch.object(self.agent, "history_manager", return_value=empty_history), \ patch.object(self.agent, "_restore_ponytail_level"): self.agent.bind_interaction_scope("chat-empty") diff --git a/tests/test_lazy_legacy_protocol.py b/tests/test_lazy_legacy_protocol.py new file mode 100644 index 00000000..e5563336 --- /dev/null +++ b/tests/test_lazy_legacy_protocol.py @@ -0,0 +1,571 @@ +"""Fresh-process checks for the optional legacy text compatibility boundary.""" +import os +import subprocess +import sys +import tempfile +import textwrap +import unittest +from pathlib import Path + + +class LazyLegacyProtocolTests(unittest.TestCase): + def run_fresh(self, script): + with tempfile.TemporaryDirectory() as directory: + environment = {key: value for key, value in os.environ.items() if not any( + term in key.upper() for term in ("API_KEY", "ACCESS_KEY", "SECRET_KEY", "SESSION_TOKEN", "ACCESS_TOKEN") + )} + environment.update(HOME=directory, KYROZEN_WORKSPACE_ROOT=directory, + KYROZEN_DB_PATH=str(Path(directory) / "runtime.sqlite3"), + KYROZEN_SKILLS_PATH=str(Path(directory) / "skills"), + KYROZEN_DISABLE_VECTOR_INDEX="1", KYROZEN_APPROVAL_MODE="never") + result = subprocess.run( + [sys.executable, "-c", textwrap.dedent(script)], + cwd=Path(__file__).resolve().parents[1], env=environment, + capture_output=True, text=True, timeout=30, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_native_typed_turn_never_imports_legacy_parser(self): + self.run_fresh(''' + import importlib.abc + import os + import sys + from pathlib import Path + + class BlockLegacyParser(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == "openkyrozen.agent.protocol": + raise AssertionError("Native execution imported the legacy parser") + return None + sys.meta_path.insert(0, BlockLegacyParser()) + from openkyrozen.app.bootstrap import build_application, build_memory + from openkyrozen.providers import ProviderConfig + from openkyrozen.providers.base import LLMProvider + from openkyrozen.providers.models import FinishReason, ModelResponse, ProviderCapabilities, ToolCall, ToolReply + from openkyrozen.tools import ToolAdapters + + literal = 'Literal documentation: \\nAction: ordinary prose\\nTaskList: ["example"]\\nnot executable' + class NativeProvider(LLMProvider): + def get_capabilities(self, model=None): + return ProviderCapabilities(native_tools=True) + def chat(self, messages, model=None): + raise AssertionError("Legacy provider boundary used") + def chat_response(self, messages, model=None, *, tools=None): + if not any(isinstance(item, ToolReply) for item in messages): + assert "read_file" in {tool["name"] for tool in tools} + return ModelResponse(tool_calls=(ToolCall("read", "read_file", {"path": "note.txt"}),), + finish_reason=FinishReason.TOOL_REQUEST) + return ModelResponse(text=literal, finish_reason=FinishReason.FINAL) + root = Path(os.environ["KYROZEN_WORKSPACE_ROOT"]) + (root / "note.txt").write_text("fixture content", encoding="utf-8") + app = build_application(memory=build_memory(root / "state.sqlite3", index=None), tools=ToolAdapters(root)) + try: + runtime = app.runtime + config = ProviderConfig(provider="custom", model_simple="fixture", model_complex="fixture", model_main="fixture") + runtime.llm_provider = runtime._default_turn.provider = NativeProvider(config) + runtime._provider_config = config + runtime.set_interaction_mode("agent") + answer = runtime.chat(runtime.current_session, "Read note.txt and report its content") + assert isinstance(answer, str) and answer == literal, answer + import json + assert json.loads(json.dumps({"reply": answer}))["reply"] == literal + assert "openkyrozen.agent.protocol" not in sys.modules + native = runtime._last_response_is_native.set(True) + try: + assert runtime._parse_model_response(literal)["clean"] == literal + assert "Action: ordinary prose" in runtime._deterministic_tool_summary([ + {"success": True, "result": literal, "verification": {"status": "verified"}}, + ]) + finally: + runtime._last_response_is_native.reset(native) + assert "openkyrozen.agent.protocol" not in sys.modules + assert runtime._legacy_protocol is None + finally: + app.close() + ''') + + def test_native_reply_remains_literal_through_cli_tui_and_web_frontends(self): + script = r''' + import importlib.abc + import io + import json + import os + import sys + from pathlib import Path + class BlockLegacyParser(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == "openkyrozen.agent.protocol": + raise AssertionError("Native frontend imported the legacy parser") + return None + sys.meta_path.insert(0, BlockLegacyParser()) + from openkyrozen.app.bootstrap import build_application, build_memory + from openkyrozen.providers import ProviderConfig + from openkyrozen.providers.base import LLMProvider + from openkyrozen.providers.models import FinishReason, ModelResponse, ProviderCapabilities, ToolCall, ToolReply + from openkyrozen.tools import ToolAdapters + literal = 'Thought: literal documentation\nTaskDone: 0\nAction:\n```json\n{"action":"write_file","args":"should-not-exist.txt|bad"}\n```\nTaskList: ["example"]' + class NativeProvider(LLMProvider): + calls = 0 + def get_capabilities(self, model=None): + return ProviderCapabilities(native_tools=True) + def chat(self, messages, model=None): + raise AssertionError("Legacy provider boundary used") + def chat_response(self, messages, model=None, *, tools=None): + self.calls += 1 + if not any(isinstance(item, ToolReply) for item in messages): + return ModelResponse(tool_calls=(ToolCall("read", "read_file", {"path": "note.txt"}),), + finish_reason=FinishReason.TOOL_REQUEST) + return ModelResponse(text=literal, finish_reason=FinishReason.FINAL) + root = Path(os.environ["KYROZEN_WORKSPACE_ROOT"]) + (root / "note.txt").write_text("fixture content", encoding="utf-8") + app = build_application(surface=SURFACE, memory=build_memory(root / "state.sqlite3", index=None), tools=ToolAdapters(root)) + try: + runtime = app.runtime + config = ProviderConfig(provider="custom", model_simple="fixture", model_complex="fixture", model_main="fixture") + provider = NativeProvider(config) + runtime.llm_provider = runtime._default_turn.provider = provider + runtime._provider_config = config + runtime.set_interaction_mode("agent") + if SURFACE == "cli": + from rich.console import Console + output = io.StringIO() + runtime.console = Console(file=output, width=200, force_terminal=False) + runtime._run_cli_chat("Read note.txt and report its content", runtime.interaction_envelope()) + assert "Thought: literal documentation" in output.getvalue(), output.getvalue() + assert "TaskDone: 0" in output.getvalue(), output.getvalue() + elif SURFACE == "tui": + from openkyrozen.interfaces.tui.backend import Backend + backend = Backend(app) + backend._output = io.StringIO() + backend._run_submit("Read note.txt and report its content", "native-frontend") + events = [json.loads(line) for line in backend._output.getvalue().splitlines()] + responses = [event for event in events if event["event"] == "response"] + assert responses and responses[-1]["text"] == literal, events + assert not any(event["event"] == "thinking" for event in events), events + else: + from openkyrozen.interfaces.web.service import WebService + service = WebService(app) + session = service._get_or_create_session("native-web", service._SERVER_ACTOR_ID) + runtime.open_session("native-web", user_id=service._SERVER_ACTOR_ID).interaction.set_mode("agent") + answer = service._run_session_chat(session, "Read note.txt and report its content") + assert answer == literal, answer + assert provider.calls == 2, provider.calls + assert not (root / "should-not-exist.txt").exists() + assert "openkyrozen.agent.protocol" not in sys.modules + assert runtime._legacy_protocol is None + finally: + app.close() + ''' + for surface in ("cli", "tui", "web"): + with self.subTest(surface=surface): + self.run_fresh(script.replace("SURFACE", repr(surface))) + + def test_native_plan_and_auto_modes_inspect_then_stage_a_typed_proposal(self): + script = r''' + import importlib.abc + import os + import sys + from pathlib import Path + class BlockLegacyParser(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == "openkyrozen.agent.protocol": + raise AssertionError("Native planning imported the legacy parser") + return None + sys.meta_path.insert(0, BlockLegacyParser()) + from openkyrozen.app.bootstrap import build_application, build_memory + from openkyrozen.providers import ProviderConfig + from openkyrozen.providers.base import LLMProvider + from openkyrozen.providers.models import FinishReason, ModelResponse, ProviderCapabilities, ToolCall, ToolReply, provider_continuation, validate_replied_tool_definitions + from openkyrozen.tools import ToolAdapters + proposal = {"title": "Prepare result", "summary": "Create the requested result after approval.", + "bounds": {"workspace_paths": ["result.txt"], "capabilities": ["read", "write"], + "effects": ["read_only", "filesystem"], "risk_ceiling": "normal"}, + "steps": [{"id": "result", "title": "Create result", "description": "Write result.txt", + "acceptance": [{"criterion": "Requested result exists", "verification": { + "action": "read_file", "arguments": {"path": "result.txt"}, "expect": {"equals": "ready"}}}], + "execution": {"action": "write_file", "arguments": {"path": "result.txt", "content": "ready"}}}]} + class PlanProvider(LLMProvider): + calls = 0 + def get_capabilities(self, model=None): + return ProviderCapabilities(native_tools=True) + def chat(self, messages, model=None): + raise AssertionError("Native Plan Mode entered the text provider API") + def chat_response(self, messages, model=None, *, tools=None): + self.calls += 1 + validate_replied_tool_definitions(messages, tools) + if self.calls > (3 if DISCOVER_REQUEST else 2): + return ModelResponse(text="Approved execution remains pending.", finish_reason=FinishReason.FINAL) + names = {item["name"] for item in tools} + if not DISCOVER_REQUEST or self.calls == 1: + assert not names & {"github_read", "graph_query"}, names + else: + assert {"github_read", "graph_query"} <= names, names + assert not names & {"write_file", "run_cmd", "update_plan", "spawn_agents"}, names + if QUESTION_REQUEST: + assert "request_question" in names, names + return ModelResponse(tool_calls=(ToolCall("clarify", "request_question", {"questions": [{ + "id": "format", "header": "Format", "prompt": "Which result format?", "choices": [ + {"id": "plain", "label": "Plain"}, {"id": "rich", "label": "Rich"}, + ]}]}),), finish_reason=FinishReason.TOOL_REQUEST, + continuation=provider_continuation(self, model, "fixture", {"request": self.calls})) + if DISCOVER_REQUEST and self.calls == 1: + return ModelResponse(tool_calls=(ToolCall("discover", "discover_tools", {"names": ["github_read", "graph_query"]}),), + finish_reason=FinishReason.TOOL_REQUEST, + continuation=provider_continuation(self, model, "fixture", {"request": self.calls})) + if not any(isinstance(item, ToolReply) and item.name == "read_file" for item in messages): + assert "propose_plan" not in names, names + return ModelResponse(tool_calls=(ToolCall("inspect", "read_file", {"path": "note.txt"}),), + finish_reason=FinishReason.TOOL_REQUEST, + continuation=provider_continuation(self, model, "fixture", {"request": self.calls})) + assert "propose_plan" in names, names + return ModelResponse(tool_calls=(ToolCall("proposal", "propose_plan", proposal),), + finish_reason=FinishReason.TOOL_REQUEST, + continuation=provider_continuation(self, model, "fixture", {"request": self.calls})) + root = Path(os.environ["KYROZEN_WORKSPACE_ROOT"]) + (root / "note.txt").write_text("Existing context", encoding="utf-8") + app = build_application(memory=build_memory(root / "state.sqlite3", index=None), tools=ToolAdapters(root)) + try: + runtime = app.runtime + config = ProviderConfig(provider="custom", model_simple="fixture", model_complex="fixture", model_main="fixture") + provider = PlanProvider(config) + runtime.llm_provider = runtime._default_turn.provider = provider + runtime._provider_config = config + runtime.set_interaction_mode(PLAN_MODE) + request = "Create result.txt after preparing the plan" + answer = runtime.chat(runtime.current_session, request) + state = runtime._interaction_controller.state() + if QUESTION_REQUEST: + assert state["pending_plan"] is None, state + assert state["pending_question"]["questions"][0]["id"] == "format", state + assert state["pending_question"]["original_input"] == request, state + assert provider.calls == 1, provider.calls + else: + assert state["pending_plan"] is not None, (state, answer) + assert state["pending_plan"]["version"] == 1, state + assert state["pending_plan"]["original_input"] == request, state + assert provider.calls == (3 if DISCOVER_REQUEST else 2), provider.calls + assert "Prepare result" in answer, answer + assert state["executing_plan"] is None and runtime.tasks.working_plan is None, state + assert not (root / "result.txt").exists() + if not QUESTION_REQUEST: + pending = state["pending_plan"] + followup = runtime.chat(runtime.current_session, "accept plan", plan_acceptance={ + "plan_id": pending["plan_id"], "version": pending["version"], "fingerprint": pending["fingerprint"], + }) + assert "Native model request failed" not in followup, followup + assert "Approved execution remains pending" in followup, followup + assert provider.calls == (4 if DISCOVER_REQUEST else 3), provider.calls + assert "openkyrozen.agent.protocol" not in sys.modules + finally: + app.close() + ''' + for mode, question, discovery in (("plan", False, False), ("auto", False, False), + ("plan", True, False), ("plan", False, True)): + with self.subTest(mode=mode, question=question, discovery=discovery): + self.run_fresh(script.replace("PLAN_MODE", repr(mode)).replace("QUESTION_REQUEST", repr(question)) + .replace("DISCOVER_REQUEST", repr(discovery))) + + def test_native_accepted_plan_distinguishes_material_revision_from_safe_narrowing(self): + script = r''' + import copy + import importlib.abc + import os + import sys + from pathlib import Path + class BlockLegacyParser(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == "openkyrozen.agent.protocol": + raise AssertionError("Native revision imported the legacy parser") + return None + sys.meta_path.insert(0, BlockLegacyParser()) + from openkyrozen.app.bootstrap import build_application, build_memory + from openkyrozen.providers import ProviderConfig + from openkyrozen.providers.base import LLMProvider + from openkyrozen.providers.models import FinishReason, ModelResponse, ProviderCapabilities, ToolCall, ToolReply + from openkyrozen.tools import ToolAdapters + proposal = {"title": "Page", "summary": "Create the approved page", "bounds": { + "workspace_paths": ["."] if REVISION_CASE == "narrow" else ["page.txt"], + "capabilities": ["read", "write"], "effects": ["read_only", "filesystem"], "risk_ceiling": "normal"}, + "steps": [{"id": "page", "title": "Page", "description": "Write page.txt", "acceptance": [{ + "criterion": "Approved page exists", "verification": {"action": "read_file", + "arguments": {"path": "page.txt"}, "expect": {"equals": "approved"}}}]}]} + revised = copy.deepcopy(proposal) + revised["bounds"]["workspace_paths"] = ["page.txt"] + if REVISION_CASE == "material": + revised["summary"] = "Create an additional page outcome" + revised["bounds"]["workspace_paths"].append("extra.txt") + class RevisionProvider(LLMProvider): + calls = 0 + def get_capabilities(self, model=None): + return ProviderCapabilities(native_tools=True) + def chat(self, messages, model=None): + raise AssertionError("Native revision entered text provider API") + def chat_response(self, messages, model=None, *, tools=None): + self.calls += 1 + replies = [item for item in messages if isinstance(item, ToolReply)] + if self.calls == 1: + assert "propose_plan" not in {item["name"] for item in tools} + return ModelResponse(tool_calls=(ToolCall("inspect", "read_file", {"path": "page.txt"}),), finish_reason=FinishReason.TOOL_REQUEST) + if self.calls == 2: + return ModelResponse(tool_calls=(ToolCall("revision", "propose_plan", revised),), finish_reason=FinishReason.TOOL_REQUEST) + assert REVISION_CASE == "narrow", "Material revision must stop pending approval" + if self.calls == 3: + control = next(item for item in replies if item.call_id == "revision") + assert control.output["data"]["requires_approval"] is False, control + return ModelResponse(tool_calls=(ToolCall("continue", "read_file", {"path": "page.txt"}),), finish_reason=FinishReason.TOOL_REQUEST) + return ModelResponse(text="Continued within updated approved scope.", finish_reason=FinishReason.FINAL) + root = Path(os.environ["KYROZEN_WORKSPACE_ROOT"]) + (root / "page.txt").write_text("approved", encoding="utf-8") + app = build_application(memory=build_memory(root / "state.sqlite3", index=None), tools=ToolAdapters(root)) + try: + runtime = app.runtime + config = ProviderConfig(provider="custom", model_simple="fixture", model_complex="fixture", model_main="fixture") + provider = RevisionProvider(config) + runtime.llm_provider = runtime._default_turn.provider = provider + runtime._provider_config = config + runtime.set_interaction_mode("agent") + controller = runtime._interaction_controller + initial = controller.propose_plan(proposal) + accepted, _ = controller.accept_plan(runtime.tasks, plan_id=initial["plan_id"], version=1) + answer = runtime.chat(runtime.current_session, "Review the approved page scope") + state = controller.state() + if REVISION_CASE == "material": + assert provider.calls == 2, provider.calls + assert state["pending_plan"]["version"] == 2, state + assert state["executing_plan"]["bounds"] == accepted["bounds"], state + denied = runtime._execute_turn_action("write_file", {"path": "page.txt", "content": "unapproved"}, + operation_scope="post-proposal", successful_operations=set(), call_id="write", call_id_source="native") + assert not denied.success + else: + assert provider.calls == 4, provider.calls + assert state["pending_plan"] is None and state["executing_plan"]["version"] == 2, state + assert state["executing_plan"]["bounds"]["workspace_paths"] == ["page.txt"], state + assert "Continued within updated approved scope" in answer, answer + assert (root / "page.txt").read_text() == "approved" + assert not (root / "extra.txt").exists() + assert "openkyrozen.agent.protocol" not in sys.modules + finally: + app.close() + ''' + for case in ("material", "narrow"): + with self.subTest(case=case): + self.run_fresh(script.replace("REVISION_CASE", repr(case))) + + def test_native_delegated_synthesis_uses_typed_output_or_keeps_verified_reports(self): + script = r''' + import importlib.abc + import os + import sys + from pathlib import Path + from types import SimpleNamespace + class BlockLegacyParser(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == "openkyrozen.agent.protocol": + raise AssertionError("Native delegated synthesis imported the legacy parser") + return None + sys.meta_path.insert(0, BlockLegacyParser()) + from openkyrozen.app.bootstrap import build_application, build_memory + from openkyrozen.providers import ProviderConfig + from openkyrozen.providers.base import LLMProvider + from openkyrozen.providers.errors import ProviderError, ProviderErrorKind + from openkyrozen.providers.models import FinishReason, ModelResponse, ProviderCapabilities, ToolCall, ToolReply + from openkyrozen.tools import ToolAdapters + literal = "Thought: literal synthesized finding\nAction: documentation only\nAccepted child finding" + class SynthesisProvider(LLMProvider): + calls = 0 + def get_capabilities(self, model=None): + return ProviderCapabilities(native_tools=True) + def chat(self, messages, model=None): + raise AssertionError("Native synthesis entered the text provider API") + def chat_response(self, messages, model=None, *, tools=None): + self.calls += 1 + if messages[0]["content"].startswith("Synthesize"): + assert not tools, tools + if SYNTHESIS_CASE == "error": + raise RuntimeError("Synthesis fixture failed") + if SYNTHESIS_CASE == "cancel": + return ModelResponse(finish_reason=FinishReason.CANCELLED) + if SYNTHESIS_CASE == "tool": + return ModelResponse(tool_calls=(ToolCall("unwanted", "write_file", { + "path": "unwanted.txt", "content": "must not execute"}),), finish_reason=FinishReason.TOOL_REQUEST) + return ModelResponse(text=literal, finish_reason=FinishReason.FINAL) + if not any(isinstance(item, ToolReply) for item in messages): + return ModelResponse(tool_calls=(ToolCall("inspect", "read_file", {"path": "note.txt"}),), + finish_reason=FinishReason.TOOL_REQUEST) + return ModelResponse(text="Root inspection finished; retain child evidence.", finish_reason=FinishReason.FINAL) + root = Path(os.environ["KYROZEN_WORKSPACE_ROOT"]) + (root / "note.txt").write_text("verified content", encoding="utf-8") + app = build_application(memory=build_memory(root / "state.sqlite3", index=None), tools=ToolAdapters(root)) + try: + runtime = app.runtime + config = ProviderConfig(provider="custom", model_simple="fixture", model_complex="fixture", model_main="fixture") + provider = SynthesisProvider(config) + runtime.llm_provider = runtime._default_turn.provider = provider + runtime._provider_config = config + runtime.set_interaction_mode("agent") + run = {"run_id": "child", "name": "Inspector", "version": 1, "status": "succeeded", + "assignment": {"task": "Inspect note"}, "parent_receipts": [], "reviews": [], + "report": {"summary": "Accepted child finding", "findings": ["note.txt was verified"], + "evidence": ["note.txt: verified content"], "uncertainties": []}} + calls = [] + def list_runs(): + calls.append(True) + return [] if len(calls) == 1 else [run] + runtime.subagent_manager.coordinator = SimpleNamespace(list=list_runs, close=lambda: None) + events = [] + try: + answer = runtime.chat(runtime.current_session, "Read note.txt and report the findings", on_runtime_event=events.append) + except ProviderError as exc: + assert SYNTHESIS_CASE == "cancel" and exc.kind == ProviderErrorKind.CANCELLED, exc + assert events[-1].kind == "turn.cancelled", events[-1] + assert run["status"] == "succeeded" and run["report"]["summary"] == "Accepted child finding" + else: + assert SYNTHESIS_CASE != "cancel", "Cancelled synthesis must cancel the user turn" + if SYNTHESIS_CASE == "success": + assert answer == literal, answer + else: + assert "Verified delegated findings" in answer and "Accepted child finding" in answer, answer + assert "note.txt" in answer, answer + assert provider.calls == 3, provider.calls + assert not (root / "unwanted.txt").exists() + assert "openkyrozen.agent.protocol" not in sys.modules + finally: + app.close() + ''' + for case in ("success", "error", "tool", "cancel"): + with self.subTest(case=case): + self.run_fresh(script.replace("SYNTHESIS_CASE", repr(case))) + + def test_native_historical_artifact_failure_stops_without_provider_or_legacy_import(self): + self.run_fresh(r''' + import importlib.abc + import os + import sys + from pathlib import Path + from unittest.mock import patch + class BlockLegacyParser(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == "openkyrozen.agent.protocol": + raise AssertionError("Native artifact failure imported legacy cleanup") + return None + sys.meta_path.insert(0, BlockLegacyParser()) + from openkyrozen.app.bootstrap import build_application, build_memory + from openkyrozen.agent.conversation import model_response_to_dict, tool_result_to_dict + from openkyrozen.providers import ProviderConfig + from openkyrozen.providers.base import LLMProvider + from openkyrozen.providers.models import FinishReason, ModelResponse, ProviderCapabilities, ToolCall, provider_continuation + from openkyrozen.tools import ToolAdapters + from openkyrozen.tools.models import tool_success + class UncalledProvider(LLMProvider): + calls = 0 + def chat(self, *args, **kwargs): + raise AssertionError("Legacy provider must not run") + def get_capabilities(self, model=None): + return ProviderCapabilities(native_tools=True) + def chat_response(self, *args, **kwargs): + self.calls += 1 + raise AssertionError("Provider must not run when prior artifacts cannot be retained") + root = Path(os.environ["KYROZEN_WORKSPACE_ROOT"]) + app = build_application(memory=build_memory(root / "state.sqlite3", index=None), tools=ToolAdapters(root)) + try: + runtime = app.runtime + config = ProviderConfig(provider="custom", model_simple="fixture", model_complex="fixture", model_main="fixture") + provider = UncalledProvider(config) + runtime.llm_provider = runtime._default_turn.provider = provider + runtime._provider_config = config + runtime.set_interaction_mode("agent") + history = runtime.conversation_history + history.append(history.entry("user.input", {"text": "Earlier read"}, turn_id="previous")) + response = ModelResponse(tool_calls=(ToolCall("old", "read_file", {"path": "note.txt"}),), + finish_reason=FinishReason.TOOL_REQUEST, + continuation=provider_continuation(provider, "fixture", "fixture", {"call": "old"})) + history.append(history.entry("model.response", {"response": model_response_to_dict(response)}, turn_id="previous")) + history.append(history.entry("tool.result", {"call_id": "old", "name": "read_file", + "output": tool_result_to_dict(tool_success("Historical file output")), "is_error": False}, turn_id="previous", call_id="old")) + with patch.object(runtime.memory_bank.store.artifacts, "get_or_create", side_effect=OSError("storage unavailable")): + answer = runtime.chat(runtime.current_session, "Continue inspecting note.txt") + assert "Could not retain earlier tool results" in answer, answer + assert provider.calls == 0, provider.calls + assert "openkyrozen.agent.protocol" not in sys.modules + finally: + app.close() + ''') + + def test_native_failure_and_cancellation_do_not_load_legacy_cleanup(self): + script = ''' + import importlib.abc + import os + import sys + from pathlib import Path + class BlockLegacyParser(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == "openkyrozen.agent.protocol": + raise AssertionError("Native failure imported legacy cleanup") + return None + sys.meta_path.insert(0, BlockLegacyParser()) + from openkyrozen.app.bootstrap import build_application, build_memory + from openkyrozen.providers import ProviderConfig + from openkyrozen.providers.base import LLMProvider + from openkyrozen.providers.errors import ProviderError, ProviderErrorKind + from openkyrozen.providers.models import FinishReason, ModelResponse, ProviderCapabilities, ToolCall, ToolReply + from openkyrozen.tools import ToolAdapters + class FailingProvider(LLMProvider): + def get_capabilities(self, model=None): + return ProviderCapabilities(native_tools=True) + def chat(self, messages, model=None): + raise AssertionError("Legacy provider boundary used") + def chat_response(self, messages, model=None, *, tools=None): + if not any(isinstance(item, ToolReply) for item in messages): + return ModelResponse(tool_calls=(ToolCall("read", "read_file", {"path": "note.txt"}),), + finish_reason=FinishReason.TOOL_REQUEST) + if FAILURE_MODE == "cancel": + raise ProviderError(ProviderErrorKind.CANCELLED, terminal=True) + raise RuntimeError("Fixture provider failed after the read") + root = Path(os.environ["KYROZEN_WORKSPACE_ROOT"]) + (root / "note.txt").write_text("Action: file data is literal", encoding="utf-8") + app = build_application(memory=build_memory(root / "state.sqlite3", index=None), tools=ToolAdapters(root)) + try: + runtime = app.runtime + config = ProviderConfig(provider="custom", model_simple="fixture", model_complex="fixture", model_main="fixture") + runtime.llm_provider = runtime._default_turn.provider = FailingProvider(config) + runtime._provider_config = config + runtime.set_interaction_mode("agent") + try: + answer = runtime.chat(runtime.current_session, "Read note.txt and report its content") + except ProviderError as exc: + assert FAILURE_MODE == "cancel" and exc.kind == ProviderErrorKind.CANCELLED, exc + else: + assert FAILURE_MODE == "error" and "Native model request failed" in answer, answer + assert "openkyrozen.agent.protocol" not in sys.modules + assert runtime._legacy_protocol is None + finally: + app.close() + ''' + for mode in ("error", "cancel"): + with self.subTest(mode=mode): + self.run_fresh(script.replace("FAILURE_MODE", repr(mode))) + + def test_legacy_helpers_load_adapter_on_first_use_and_keep_control_semantics(self): + self.run_fresh(''' + import os + import sys + from pathlib import Path + from openkyrozen.app.bootstrap import build_application, build_memory + from openkyrozen.tools import ToolAdapters + root = Path(os.environ["KYROZEN_WORKSPACE_ROOT"]) + app = build_application(memory=build_memory(root / "state.sqlite3", index=None), tools=ToolAdapters(root)) + try: + runtime = app.runtime + assert "openkyrozen.agent.protocol" not in sys.modules + assert runtime._legacy_protocol is None + parsed = runtime._parse_model_response('Action:\\n```json\\n{"action":"read_file","args":"note.txt"}\\n```') + assert parsed["tool_calls"] == [{"action": "read_file", "args": "note.txt"}], parsed + assert parsed["clean"] == "" + adapter = runtime.legacy_protocol + assert adapter is runtime.legacy_protocol + assert "openkyrozen.agent.protocol" in sys.modules + assert runtime._safe_fstring("{value}") == "{{value}}" + assert runtime._clean_final_response("Answer.\\nTaskDone: 0") == "Answer." + finally: + app.close() + ''') diff --git a/tests/test_native_context.py b/tests/test_native_context.py new file mode 100644 index 00000000..aac322a3 --- /dev/null +++ b/tests/test_native_context.py @@ -0,0 +1,309 @@ +"""Native engine context checks using real runtime/compaction and offline providers.""" +import copy +import os +from pathlib import Path +import tempfile +import time +import unittest +from unittest.mock import patch + +from openkyrozen.agent.compaction import ContextState, DIGEST_PREFIX, estimate_messages +from openkyrozen.agent.engine import AgentEngine +from openkyrozen.agent.turn import TurnContext +from openkyrozen.app.bootstrap import build_application, build_memory +from openkyrozen.providers import ProviderConfig +from openkyrozen.providers.base import LLMProvider +from openkyrozen.providers.errors import ProviderError, ProviderErrorKind +from openkyrozen.providers.models import ( + FinishReason, ModelResponse, ProviderCapabilities, ResponseError, TextDelta, + ToolCall, ToolReply, provider_continuation, provider_identity, validate_tool_history, +) +from openkyrozen.tools import ToolAdapters + + +class NativeContextTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + env = patch.dict(os.environ, {"KYROZEN_DISABLE_VECTOR_INDEX": "1", "KYROZEN_APPROVAL_MODE": "never"}) + env.start() + self.addCleanup(env.stop) + self.app = build_application(surface="cli", memory=build_memory(self.root / "state.sqlite3", index=None), + tools=ToolAdapters(self.root)) + self.addCleanup(self.app.close) + self.runtime = self.app.runtime + self.runtime._set_workspace_root(self.root) + self.runtime.set_interaction_mode("agent") + + def test_tracked_native_history_compacts_atomic_groups_and_keeps_current_supersession(self): + from openkyrozen.agent.compaction import compact_for_pressure + old_user = {"role": "user", "content": "Original region eu-west-2; owner team-cedar; " + "x" * 2000} + old_call = ModelResponse(tool_calls=(ToolCall("old-call", "read_file", {"path": "facts.txt"}),), + finish_reason=FinishReason.TOOL_REQUEST) + old_reply = ToolReply("old-call", "read_file", {"text": "rollback release-17 " + "x" * 2000}) + old_final = ModelResponse(text="Earlier region eu-west-2", finish_reason=FinishReason.FINAL) + recent = [{"role": "user", "content": "Latest approved region ap-southeast-1"}, + ModelResponse(text="Updated region", finish_reason=FinishReason.FINAL), + {"role": "user", "content": "Keep team-cedar"}, + ModelResponse(text="Retained owner", finish_reason=FinishReason.FINAL)] + unfinished = ModelResponse(tool_calls=(ToolCall("pending", "read_file", {"path": "pending.txt"}),), + finish_reason=FinishReason.TOOL_REQUEST) + current = {"role": "user", "content": "Return the current region and original rollback"} + history = [old_user, old_call, old_reply, old_final, *recent] + messages = [{"role": "system", "content": "Pinned state"}, *history, unfinished, current] + original = list(messages) + state = ContextState("fixture", configured_window=1800, reserve_tokens=100, + history_message_ids={id(item) for item in history + [unfinished]}) + seen = [] + def summarize(text, budget): + seen.append(text) + return "Original region eu-west-2; owner team-cedar; rollback release-17" + result = compact_for_pressure(messages, state, summarize, force=True) + self.assertFalse(result.impossible) + self.assertTrue(result.compacted) + self.assertEqual(messages, original) # The canonical input is never mutated. + self.assertIn("rollback release-17", "".join(seen)) + self.assertNotIn("Latest approved region", "".join(seen)) + for item in [*recent, unfinished, current]: + self.assertTrue(any(candidate is item for candidate in result.messages)) + for item in [old_call, old_reply]: + self.assertFalse(any(candidate is item for candidate in result.messages)) + self.assertIn("rollback release-17", result.history_digest["content"]) + + def history(self): + return ([{"role": "system", "content": "Keep the current task and trusted tool correlation."}] + + [{"role": "user" if index % 2 == 0 else "assistant", "content": f"old-{index}:" + "x" * 1200} + for index in range(6)] + + [{"role": "user" if index % 2 == 0 else "assistant", "content": f"recent-{index}: keep cedar decision"} + for index in range(4)]) + + def run_native(self, provider, messages, *, window=None): + runtime = self.runtime + context = copy.copy(runtime.execution_context) + context.provider, context.model = provider, "context-fixture" + token = runtime._turn_context.set(context) + self.addCleanup(runtime._turn_context.reset, token) + state = ContextState("context-fixture", configured_window=window, reserve_tokens=100) + state.history_message_ids = {id(item) for item in messages[:-1]} + runtime.short_term_memory = list(messages[1:-1]) + state_token = runtime._active_context_state.set(state) + self.addCleanup(runtime._active_context_state.reset, state_token) + def prepare(turn): + turn.interaction_mode, turn.complexity = "agent", "simple" + turn.turn_start, turn.learning_run, turn.learned_context = time.time(), {"run_id": "context-fixture"}, "" + turn = TurnContext("Continue the current work") + with patch.object(runtime, "_permitted_tool_names", return_value={"read_file", "write_file"}), \ + patch.object(runtime, "_prepare_turn", side_effect=prepare), \ + patch.object(runtime, "_build_messages", return_value=messages), \ + patch.object(runtime, "_summarize_context_with_chat_model", return_value="Decision: keep cedar; current work remains."), \ + patch.object(runtime, "_finish_learning_run", side_effect=lambda _run, _receipts, _task, answer, *_args: answer), \ + patch.object(runtime, "_schedule_post_turn_learning"): + answer = AgentEngine(runtime).run(turn) + return answer, turn, state + + def provider(self, callback, *, stream=False): + class Provider(LLMProvider): + def __init__(self): + super().__init__(ProviderConfig(provider="custom", model_simple="context-fixture")) + self.requests = [] + def get_capabilities(self, model=None): + return ProviderCapabilities(native_tools=True, text_streaming=stream, streaming_tool_calls=stream) + def chat(self, messages, model=None): + raise AssertionError("native context entered legacy text boundary") + def chat_response(self, messages, model=None, *, tools=None): + self.requests.append(list(messages)) + return callback(self, messages, model) + def chat_response_stream(self, messages, model=None, *, tools=None): + self.requests.append(list(messages)) + yield from callback(self, messages, model) + return Provider() + + def test_pressure_compacts_real_native_history_before_the_provider_request(self): + provider = self.provider(lambda _provider, _messages, _model: + ModelResponse(text="The cedar decision is preserved.", finish_reason=FinishReason.FINAL)) + messages = self.history() + call = ToolCall("historical-read", "read_file", {"path": "notes.txt"}) + messages.extend((ModelResponse(tool_calls=(call,), finish_reason=FinishReason.TOOL_REQUEST, + continuation=provider_continuation(provider, "context-fixture", "fixture", {"id": "history"})), + ToolReply(call.id, call.name, {"text": "known fact"}), {"role": "user", "content": "Current pending work"})) + answer, turn, state = self.run_native(provider, messages, window=1800) + sent = provider.requests[0] + self.assertLessEqual(estimate_messages(sent)[0] + state.reserve_tokens, 1800) + self.assertTrue(any(isinstance(item, dict) and DIGEST_PREFIX in item.get("content", "") for item in sent)) + self.assertEqual(sent[-1]["content"], "Current pending work") + validate_tool_history(sent, provider_id=provider_identity(provider), model="context-fixture", native=True) + self.assertEqual(turn.runtime_status, "completed") + self.assertIn("cedar", answer) + + def test_full_native_pressure_compacts_a_prefix_that_fits_on_its_own(self): + from openkyrozen.agent.compaction import compact_for_pressure + prefix = ([{"role": "system", "content": "fixed"}] + + [{"role": "user", "content": f"old-{index}:" + "x" * 1000} for index in range(6)] + + [{"role": "user", "content": "recent decision cedar"} for _ in range(4)]) + call = ToolCall("recent-call", "read_file", {"path": "notes.txt"}) + response = ModelResponse(tool_calls=(call,), finish_reason=FinishReason.TOOL_REQUEST) + reply = ToolReply(call.id, call.name, {"text": "known " * 200}) + current = {"role": "user", "content": "Continue current work"} + state = ContextState("context-fixture", configured_window=2100, reserve_tokens=100, + native_input_overhead_tokens=150) + state.history_message_ids = {id(item) for item in prefix} + self.assertLess(state.update(prefix)["input_tokens"] + state.reserve_tokens, 2100) + self.assertGreater(state.update(prefix + [response, reply, current])["input_tokens"] + state.reserve_tokens, 2100) + result = compact_for_pressure(prefix + [response, reply, current], state, lambda *_args: "Decision cedar retained") + self.assertFalse(result.impossible) + self.assertTrue(result.compacted) + self.assertLessEqual(state.status["input_tokens"] + state.reserve_tokens, 2100) + self.assertIs(result.messages[-3], response) + self.assertIs(result.messages[-2], reply) + self.assertIs(result.messages[-1], current) + + def test_native_reported_token_cache_reuses_only_the_same_schema_across_turns(self): + messages = [{"role": "system", "content": "fixed"}, {"role": "user", "content": "current"}] + first = ContextState("context-fixture", native_schema_fingerprint="schema-one", native_input_overhead_tokens=80) + token = self.runtime._active_context_state.set(first) + try: + self.runtime._cache_reported_context_tokens(messages, 321) + finally: + self.runtime._active_context_state.reset(token) + same = ContextState("context-fixture", native_schema_fingerprint="schema-one", native_input_overhead_tokens=80) + self.runtime._load_reported_context_tokens(same) + status = same.update(messages) + self.assertEqual(status["input_source"], "provider_reported") + self.assertEqual(status["input_tokens"], 321) + different = ContextState("context-fixture", native_schema_fingerprint="schema-two", native_input_overhead_tokens=90) + self.runtime._load_reported_context_tokens(different) + self.assertEqual(different.update(messages)["input_source"], "estimated") + legacy = ContextState("context-fixture") + self.runtime._load_reported_context_tokens(legacy) + self.assertEqual(legacy.update(messages)["input_source"], "estimated") + + def test_reported_context_counts_do_not_cross_provider_or_model_identity(self): + messages = [{"role": "user", "content": "current"}] + first = ContextState("model-A", native_schema_fingerprint="same", native_input_overhead_tokens=200) + first.request_provider_id = "provider-one" + token = self.runtime._active_context_state.set(first) + try: + self.runtime._cache_reported_context_tokens(messages, 7) + finally: + self.runtime._active_context_state.reset(token) + for model, identity in (("model-B", "provider-one"), ("model-A", "provider-two")): + with self.subTest(model=model, provider=identity): + other = ContextState(model, native_schema_fingerprint="same", native_input_overhead_tokens=200) + other.request_provider_id = identity + self.runtime._load_reported_context_tokens(other) + self.assertEqual(other.update(messages)["input_source"], "estimated") + same = ContextState("model-A", native_schema_fingerprint="same", native_input_overhead_tokens=200) + same.request_provider_id = "provider-one" + self.runtime._load_reported_context_tokens(same) + self.assertEqual(same.update(messages)["input_tokens"], 7) + + def test_explicit_native_model_switch_recomputes_catalog_window_before_transport(self): + from openkyrozen.agent.types import ContextTooLargeError + provider = self.provider(lambda *_args: ModelResponse(text="Must not run", finish_reason=FinishReason.FINAL)) + context = copy.copy(self.runtime.execution_context) + context.provider = provider + turn_token = self.runtime._turn_context.set(context) + state = ContextState("deepseek-flash", reserve_tokens=100) + state_token = self.runtime._active_context_state.set(state) + try: + with self.assertRaises(ContextTooLargeError): + self.runtime._get_model_response([{"role": "system", "content": "x" * 600000}, + {"role": "user", "content": "current"}], model="gpt-4o", tool_names={"read_file"}) + self.assertEqual(provider.requests, []) + self.assertEqual(state.window_tokens, 128000) + finally: + self.runtime._active_context_state.reset(state_token) + self.runtime._turn_context.reset(turn_token) + + def test_explicit_configured_window_survives_a_native_model_switch(self): + provider = self.provider(lambda *_args: ModelResponse(text="Fits configured window", finish_reason=FinishReason.FINAL)) + context = copy.copy(self.runtime.execution_context) + context.provider = provider + turn_token = self.runtime._turn_context.set(context) + state = ContextState("deepseek-flash", configured_window=5000, reserve_tokens=100) + state_token = self.runtime._active_context_state.set(state) + try: + self.runtime._get_model_response([{"role": "user", "content": "current"}], + model="gpt-4o", tool_names={"read_file"}) + self.assertEqual(state.window_tokens, 5000) + self.assertEqual(state.window_source, "configured_override") + finally: + self.runtime._active_context_state.reset(state_token) + self.runtime._turn_context.reset(turn_token) + + def test_native_schema_payload_is_reserved_before_provider_dispatch(self): + provider = self.provider(lambda *_args: ModelResponse(text="Should not fit", finish_reason=FinishReason.FINAL)) + original = self.runtime.tool_registry.get_spec("read_file") + self.runtime.tool_registry.register_spec(original.replace(description="Read contract: " + "x" * 20000), replace=True) + messages = [{"role": "system", "content": "fixed"}, {"role": "user", "content": "Read a file"}] + answer, turn, state = self.run_native(provider, messages, window=500) + self.assertEqual(len(provider.requests), 0, answer) + self.assertGreater(state.native_input_overhead_tokens, 500) + self.assertEqual(turn.runtime_status, "incomplete") + + def test_native_continuation_payload_cannot_hide_from_the_context_limit(self): + provider = self.provider(lambda *_args: ModelResponse(text="Should not fit", finish_reason=FinishReason.FINAL)) + messages = [{"role": "system", "content": "fixed"}, ModelResponse(text="previous response", + finish_reason=FinishReason.FINAL, continuation=provider_continuation(provider, "context-fixture", "fixture", + {"opaque_replay_signature": "x" * 12000})), {"role": "user", "content": "Continue"}] + answer, turn, _state = self.run_native(provider, messages, window=500) + self.assertEqual(len(provider.requests), 0, answer) + self.assertEqual(turn.runtime_status, "incomplete") + + def test_impossible_fixed_native_context_never_calls_the_provider(self): + provider = self.provider(lambda *_args: ModelResponse(text="Should not run", finish_reason=FinishReason.FINAL)) + messages = [{"role": "system", "content": "x" * 10000}, {"role": "user", "content": "Current work"}] + answer, turn, _state = self.run_native(provider, messages, window=300) + self.assertEqual(provider.requests, []) + self.assertEqual(turn.runtime_status, "incomplete") + self.assertIn("ContextTooLargeError", answer) + + def test_overflow_retries_only_generation_after_an_effect_without_replaying_the_write(self): + def respond(provider, messages, model): + if len(provider.requests) == 1: + return ModelResponse(tool_calls=(ToolCall("write-once", "write_file", {"path": "result.txt", "content": "once"}),), + finish_reason=FinishReason.TOOL_REQUEST, + continuation=provider_continuation(provider, model, "fixture", {"id": "write"})) + if len(provider.requests) == 2: + raise ProviderError(ProviderErrorKind.CONTEXT_OVERFLOW) + validate_tool_history(messages, provider_id=provider_identity(provider), model=model, native=True) + return ModelResponse(text="The file was written once and its result retained.", finish_reason=FinishReason.FINAL) + provider = self.provider(respond) + original = self.runtime.tool_registry.get_spec("write_file") + effects = [] + def write(arguments): + effects.append(dict(arguments)) + return original.executor(arguments) + self.runtime.tool_registry.register_spec(original.replace(executor=write), replace=True) + messages = self.history() + [{"role": "user", "content": "Write result.txt once"}] + answer, turn, state = self.run_native(provider, messages) + self.assertEqual(len(provider.requests), 3, answer) + self.assertEqual(len(effects), 1) + self.assertEqual((self.root / "result.txt").read_text(), "once") + self.assertTrue(state.overflow_retried) + self.assertTrue(any(isinstance(item, ToolReply) and item.call_id == "write-once" for item in provider.requests[-1])) + self.assertEqual(turn.runtime_status, "completed") + + def test_a_second_native_overflow_stops_without_further_generation(self): + def respond(*_args): + raise ProviderError(ProviderErrorKind.CONTEXT_OVERFLOW) + provider = self.provider(respond) + answer, turn, state = self.run_native(provider, self.history() + [{"role": "user", "content": "Current work"}]) + self.assertEqual(len(provider.requests), 2, answer) + self.assertTrue(state.overflow_retried) + self.assertEqual(turn.runtime_status, "incomplete") + + def test_overflow_after_a_stream_delta_is_never_replayed(self): + def stream(*_args): + yield TextDelta("Already delivered partial text.") + yield ResponseError(ProviderError(ProviderErrorKind.CONTEXT_OVERFLOW)) + provider = self.provider(stream, stream=True) + _answer, turn, state = self.run_native(provider, self.history() + [{"role": "user", "content": "Current work"}]) + self.assertEqual(len(provider.requests), 1) + self.assertFalse(state.overflow_retried) + self.assertEqual(turn.runtime_status, "incomplete") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_native_controls.py b/tests/test_native_controls.py new file mode 100644 index 00000000..77966c5d --- /dev/null +++ b/tests/test_native_controls.py @@ -0,0 +1,128 @@ +import copy +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from openkyrozen.app.bootstrap import build_application, build_memory +from openkyrozen.tools import ToolAdapters +from openkyrozen.tools.models import tool_success + + +class NativeControlTests(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.root = Path(directory.name) / "workspace" + self.root.mkdir() + environment = patch.dict(os.environ, {"HOME": directory.name, + "KYROZEN_WORKSPACE_ROOT": str(self.root), + "KYROZEN_DB_PATH": str(self.root / "state.sqlite3"), + "KYROZEN_SKILLS_PATH": str(self.root / "skills"), + "KYROZEN_DISABLE_VECTOR_INDEX": "1"}) + environment.start() + self.addCleanup(environment.stop) + self.app = build_application(memory=build_memory(self.root / "state.sqlite3", index=None), + tools=ToolAdapters(self.root)) + self.addCleanup(self.app.close) + self.runtime = self.app.runtime + self.runtime.set_interaction_mode("agent") + self.runtime.execution_context.native_control_input = "Revise the approved page outcome" + self.runtime.execution_context.native_plan_inspected = True + self.proposal = {"title": "Page", "summary": "Create one approved page", "bounds": { + "workspace_paths": ["page.txt"], "capabilities": ["read", "write"], + "effects": ["read_only", "filesystem"], "risk_ceiling": "normal", + }, "steps": [{"id": "page", "title": "Page", "description": "Write page.txt", "acceptance": [ + {"criterion": "Page contains approved content", "verification": { + "action": "read_file", "arguments": {"path": "page.txt"}, "expect": {"equals": "approved"}, + }}]}]} + + def accept(self): + controller = self.runtime._interaction_controller + pending = controller.propose_plan(self.proposal) + return controller.accept_plan(self.runtime.tasks, plan_id=pending["plan_id"], version=1)[0] + + def execute_control(self, name, arguments): + return self.runtime._execute_turn_action(name, arguments, operation_scope="native-control-test", + successful_operations=set(), call_id="control", call_id_source="native") + + def test_native_material_proposal_preserves_bounds_and_pauses_mutations(self): + accepted = self.accept() + revised = copy.deepcopy(self.proposal) + revised["summary"] = "Create a different page outcome" + revised["bounds"]["workspace_paths"] = ["page.txt", "additional.txt"] + receipt = self.execute_control("propose_plan", revised) + self.assertTrue(receipt.success, receipt.result) + state = self.runtime._interaction_controller.state() + self.assertEqual(state["pending_plan"]["version"], 2) + self.assertEqual(state["executing_plan"]["bounds"], accepted["bounds"]) + self.assertEqual(state["pending_plan"]["original_input"], "Revise the approved page outcome") + self.assertIsNone(receipt.task_id) + self.assertEqual(self.runtime.tasks.tasks[0]["evidence"], []) + write = self.runtime._execute_turn_action("write_file", {"path": "page.txt", "content": "unapproved"}, + operation_scope="native-control-test", successful_operations=set(), call_id="blocked-write", call_id_source="native") + self.assertFalse(write.success) + self.assertFalse((self.root / "page.txt").exists()) + + def test_nonmaterial_narrowing_updates_scope_without_requesting_new_approval(self): + self.proposal["bounds"]["workspace_paths"] = ["."] + accepted = self.accept() + revised = copy.deepcopy(self.proposal) + revised["bounds"]["workspace_paths"] = ["page.txt"] + receipt = self.execute_control("propose_plan", revised) + self.assertTrue(receipt.success, receipt.result) + self.assertIs(receipt.result_data["requires_approval"], False) + self.assertIn("within approved scope", receipt.result) + state = self.runtime._interaction_controller.state() + self.assertIsNone(state["pending_plan"]) + self.assertEqual(state["executing_plan"]["version"], accepted["version"] + 1) + self.assertEqual(state["executing_plan"]["bounds"]["workspace_paths"], ["page.txt"]) + + def test_dynamic_control_replacement_cannot_inherit_accepted_scope_exemption(self): + self.accept() + called = [] + registry = self.runtime.tool_registry + original = registry.get_spec("propose_plan") + def replacement(arguments): + called.append(arguments) + return tool_success("Replacement ran") + registry.register_spec(original.replace(executor=replacement), replace=True) + receipt = self.execute_control("propose_plan", self.proposal) + self.assertFalse(receipt.success, receipt.result) + self.assertEqual(called, []) + + def test_questions_preserve_pending_ui_and_cannot_request_secrets_or_approval(self): + valid = {"questions": [{"id": "format", "header": "Format", "prompt": "Which page format?", "choices": [ + {"id": "plain", "label": "Plain"}, {"id": "rich", "label": "Rich"}, + ]}]} + for prompt in ("What is your API key?", "May I execute this command?"): + request = copy.deepcopy(valid) + request["questions"][0]["prompt"] = prompt + receipt = self.execute_control("request_question", request) + self.assertFalse(receipt.success) + self.assertIsNone(self.runtime._interaction_controller.state()["pending_question"]) + receipt = self.execute_control("request_question", valid) + self.assertTrue(receipt.success, receipt.result) + pending = self.runtime._interaction_controller.state()["pending_question"] + self.assertEqual(pending["questions"][0]["id"], "format") + self.assertEqual(pending["original_input"], "Revise the approved page outcome") + self.assertIsNone(receipt.task_id) + + def test_child_and_noninteractive_surfaces_cannot_stage_controls(self): + accepted = self.accept() + request = {"questions": [{"id": "format", "header": "Format", "prompt": "Which format?", "choices": [ + {"id": "a", "label": "A"}, {"id": "b", "label": "B"}, + ]}]} + for child, interactive in (("child", True), (None, False)): + self.runtime.execution_context.child_run_id = child + token = self.runtime._interaction_controls_enabled.set(interactive) + try: + self.assertFalse(self.execute_control("propose_plan", self.proposal).success) + self.assertFalse(self.execute_control("request_question", request).success) + finally: + self.runtime._interaction_controls_enabled.reset(token) + state = self.runtime._interaction_controller.state() + self.assertIsNone(state["pending_question"]) + self.assertIsNone(state["pending_plan"]) + self.assertEqual(state["executing_plan"], accepted) diff --git a/tests/test_planning_service.py b/tests/test_planning_service.py new file mode 100644 index 00000000..f2ec0c15 --- /dev/null +++ b/tests/test_planning_service.py @@ -0,0 +1,79 @@ +import os +import subprocess +import sys +import tempfile +import textwrap +import unittest +from pathlib import Path + + +class PlanningServiceOwnershipTests(unittest.TestCase): + def test_tasks_service_owns_strategic_lifecycle_without_loading_agent(self): + code = ''' + import importlib.abc + import json + import sys + from pathlib import Path + class NoAgent(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == 'openkyrozen.agent' or fullname.startswith('openkyrozen.agent.'): + raise AssertionError('Planning feature imported agent implementation: ' + fullname) + sys.meta_path.insert(0, NoAgent()) + from openkyrozen.tasks.planning import PlanningService, InteractionError + from openkyrozen.tasks.engine import TaskManager + from openkyrozen.persistence.store import EventStore + store = EventStore(Path(sys.argv[1]) / 'state.sqlite3') + planner = PlanningService(store, user_id='alice', workspace_id='project', session_id='work') + manager = TaskManager(store, user_id='alice', workspace_id='project', session_id='work') + proposal = planner.propose_plan({ + 'title': 'Inspect', 'summary': 'Inspect exact source', + 'bounds': {'workspace_paths': ['note.txt'], 'capabilities': ['read'], + 'effects': ['read_only'], 'risk_ceiling': 'normal'}, + 'steps': [{'id': 'inspect', 'title': 'Inspect', 'description': 'Read note.txt', + 'acceptance': [{'criterion': 'Exact source checked', 'verification': { + 'action': 'read_file', 'arguments': {'path': 'note.txt'}, 'expect': {'equals': 'known'}}}]}], + }) + accepted, created = planner.accept_plan(manager, plan_id=proposal['plan_id'], + version=proposal['version'], fingerprint=proposal['fingerprint']) + assert created and planner.state()['executing_plan']['plan_id'] == accepted['plan_id'] + try: + planner.complete_plan(accepted, task_manager=manager) + except InteractionError: + pass + else: + raise AssertionError('Unverified completion was accepted') + manager.record_evidence(task_id=accepted['working_task_ids'][0], action='read_file', + args=json.dumps({'path': 'note.txt'}, sort_keys=True, separators=(',', ':')), + result='known', verification_result='known', success=True, + verification_status='not_applicable', receipt_id='verified-source') + assert manager.set_status(0, 'succeeded') + assert manager.tasks[0]['status'] == 'succeeded', manager.tasks + planner.reconcile_plan_completion(manager) + assert planner.state()['executing_plan'] is None + assert not any(name == 'openkyrozen.agent' or name.startswith('openkyrozen.agent.') for name in sys.modules) + ''' + with tempfile.TemporaryDirectory() as directory: + env = {**os.environ, 'HOME': directory, 'KYROZEN_DB_PATH': str(Path(directory) / 'isolated.sqlite3'), + 'KYROZEN_DISABLE_VECTOR_INDEX': '1'} + result = subprocess.run([sys.executable, '-c', textwrap.dedent(code), directory], + cwd=Path(__file__).resolve().parents[1], env=env, capture_output=True, text=True, timeout=30) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_legacy_interaction_imports_preserve_error_and_validator_identity(self): + from openkyrozen.tasks.planning import PlanningService, InteractionError, validate_plan_proposal + from openkyrozen.agent import modes + self.assertIs(modes.InteractionError, InteractionError) + self.assertIs(modes.validate_plan_proposal, validate_plan_proposal) + self.assertTrue(issubclass(modes.InteractionController, PlanningService)) + with tempfile.TemporaryDirectory() as directory: + from openkyrozen.persistence.store import EventStore + controller = modes.InteractionController(EventStore(Path(directory) / 'state.sqlite3'), session_id='ui') + controller.set_mode('ask') + proposal = controller.propose_plan({'title': 'Inspect', 'summary': 'Inspect source', + 'steps': [{'id': 'inspect', 'title': 'Inspect', 'description': 'Read source', + 'acceptance': ['Source checked']}]}) + state = controller.state() + self.assertEqual(state['preference_mode'], 'ask') + self.assertEqual(state['pending_plan']['fingerprint'], proposal['fingerprint']) + controller.cancel_plan(proposal['plan_id'], proposal['version']) + self.assertIsNone(controller.state()['pending_plan']) diff --git a/tests/test_process_audit.py b/tests/test_process_audit.py new file mode 100644 index 00000000..3b7f675d --- /dev/null +++ b/tests/test_process_audit.py @@ -0,0 +1,551 @@ +import json +import os +from pathlib import Path +import shlex +import sqlite3 +import sys +import tempfile +import threading +import time +import unittest +from unittest.mock import patch + +from openkyrozen.app.bootstrap import build_application, build_memory +from openkyrozen.tools import ToolAdapters +from openkyrozen.tools.processes import SessionProcessManager +from openkyrozen.tasks.engine import TaskManager + + +class ProcessAuditTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + self.environment = patch.dict(os.environ, { + 'HOME': str(self.root), 'KYROZEN_WORKSPACE_ROOT': str(self.root), + 'KYROZEN_DB_PATH': str(self.root / 'state.sqlite3'), + 'KYROZEN_DISABLE_VECTOR_INDEX': '1', + }) + self.environment.start() + self.addCleanup(self.environment.stop) + self.app = build_application(memory=build_memory(self.root / 'state.sqlite3', index=None), + tools=ToolAdapters(self.root)) + self.addCleanup(self.app.close) + self.runtime = self.app.runtime + self.session = self.runtime.current_session + self.manager = self.session.process_manager + self.store = self.session.memory.store + self.scope = dict(user_id=self.session.memory.user_id, + workspace_id=self.session.memory.workspace_id, + session_id=self.session.session_id) + self.command = shlex.join([sys.executable, '-c', 'import time; time.sleep(30)']) + + def events(self): + return self.store.list_events('process.lifecycle', limit=None, **self.scope) + + def wait_status(self, manager, process_id, status): + deadline = time.monotonic() + 3 + while time.monotonic() < deadline: + record = next((r for r in manager._audit_records() if r['process_id'] == process_id), None) + if record and record['status'] == status: + return record + time.sleep(.01) + self.fail(f'Process audit did not reach {status}') + + def reopened(self, **changes): + scope = {**self.scope, **changes} + manager = SessionProcessManager(scope['user_id'], str(self.root), scope['session_id']) + from openkyrozen.persistence.store import EventStore + manager._bind_audit(EventStore(self.store.path), scope['workspace_id']) + return manager + + def test_real_session_launch_cancellation_and_restart_preserve_scoped_uncertainty(self): + cancelled = threading.Event() + handle = self.manager.start(self.command, cwd=str(self.root), env=None, + timeout_seconds=30, cancelled=cancelled) + self.assertEqual({event['payload']['status'] for event in self.events()}, {'launch_intent', 'started'}) + recovered = self.reopened() + pending = recovered._audit_records()[0] + self.assertEqual(pending['process_id'], handle.process_id) + self.assertEqual(pending['owned_process_state'], 'unknown') + self.assertEqual(pending['descendant_state'], 'unknown') + self.assertTrue(pending['potentially_active']) + for change in ({'user_id': 'other'}, {'workspace_id': 'other'}, {'session_id': 'other'}): + self.assertEqual(self.reopened(**change)._audit_records(), []) + cancelled.set() + record = self.wait_status(recovered, handle.process_id, 'cancelled') + self.assertEqual(record['reason'], 'cancelled') + self.assertEqual(record['owned_process_state'], 'exited') + self.assertEqual(record['descendant_state'], 'unknown') + self.assertIsNotNone(handle.process.poll()) + + def test_two_project_roots_with_same_global_memory_and_session_cannot_read_each_others_audit(self): + first_root, second_root = self.root / 'first', self.root / 'second' + first_root.mkdir() + second_root.mkdir() + self.runtime.configure_launch_context(project_path=first_root) + first = self.runtime.open_session('shared-session') + first.process_manager.start(self.command, cwd=str(first_root), env=None, timeout_seconds=30) + self.runtime.configure_launch_context(project_path=second_root) + second = self.runtime.open_session('shared-session') + self.assertEqual(first.memory.workspace_id, second.memory.workspace_id) + self.assertEqual(first.memory.user_id, second.memory.user_id) + self.assertEqual(first.session_id, second.session_id) + self.assertNotEqual(first.tasks.workspace_id, second.tasks.workspace_id) + self.assertEqual(second.process_manager._audit_records(), []) + self.assertEqual(len(first.process_manager._audit_records()), 1) + + def test_actual_restarted_native_turn_gets_only_own_bounded_unknown_process_context(self): + from openkyrozen.workspace.context import source_scope_id + project = self.root / 'restarted-project' + project.mkdir() + workspace_id, session_id = source_scope_id(project), 'restarted-chat' + own_scope = dict(user_id=self.session.memory.user_id, workspace_id=workspace_id, session_id=session_id) + for index in range(7): + self.store.append_event('process.lifecycle', { + 'process_id': f'own-unknown-{index}', 'manager_id': 'previous', 'status': 'started', + 'pid': os.getpid(), 'owned_process_state': 'running', 'descendant_state': 'unknown', + 'command': 'SECRET_COMMAND_NOT_CONTEXT', 'env': {'TOKEN': 'SECRET_ENV_NOT_CONTEXT'}, + }, **own_scope) + for field, value in (('user_id', 'other-user'), ('workspace_id', 'other-project'), ('session_id', 'other-chat')): + self.store.append_event('process.lifecycle', {'process_id': f'foreign-{field}', + 'manager_id': 'previous', 'status': 'started', 'owned_process_state': 'unknown'}, + **{**own_scope, field: value}) + self.store.append_event('process.lifecycle', {'process_id': 'known-cancelled', 'manager_id': 'previous', + 'status': 'cancelled', 'reason': 'cancelled', 'owned_process_state': 'exited', + 'descendant_state': 'unknown'}, **own_scope) + from openkyrozen.providers.base import LLMProvider + from openkyrozen.providers.config import ProviderConfig + from openkyrozen.providers.models import ModelResponse, ProviderCapabilities, FinishReason + observed = [] + class InspectProvider(LLMProvider): + def chat(self, messages, model=None): + raise AssertionError('legacy path used') + def get_capabilities(self, model=None): + return ProviderCapabilities(native_tools=True) + def chat_response(self, messages, model=None, *, tools=None): + observed.append(json.dumps(messages, default=str)) + return ModelResponse(text='Unknown previous work needs inspection.', finish_reason=FinishReason.FINAL) + # New application and session reopen the same actual SQLite journal. + fresh = build_application(memory=build_memory(self.root / 'state.sqlite3', index=None), tools=ToolAdapters(project)) + self.addCleanup(fresh.close) + runtime = fresh.runtime + runtime.configure_launch_context(project_path=project) + session = runtime.open_session(session_id) + config = ProviderConfig(provider='custom', model_simple='fixture', model_complex='fixture') + runtime.llm_provider = runtime._default_turn.provider = InspectProvider(config) + runtime._provider_config = config + session.interaction.set_mode('agent') + with patch('openkyrozen.tools.processes.os.kill', side_effect=AssertionError('historical PID authority')), patch( + 'openkyrozen.tools.processes.os.killpg', side_effect=AssertionError('historical PID authority')), patch.object( + runtime, '_schedule_post_turn_learning'): + answer = runtime.chat(session, 'Hello') + self.assertIn('needs inspection', answer) + text = observed[0] + self.assertIn('Prior process outcomes are unknown', text) + self.assertIn('total=7', text) + self.assertIn('omitted=2', text) + self.assertEqual(text.count('own-unknown-'), 5) + for forbidden in ('foreign-', 'known-cancelled', 'SECRET_COMMAND_NOT_CONTEXT', 'SECRET_ENV_NOT_CONTEXT'): + self.assertNotIn(forbidden, text) + self.assertIn('not authority', text) + + def test_audit_read_failure_retains_explicit_unknown_context(self): + manager = SessionProcessManager(self.scope['user_id'], str(self.root), self.scope['session_id']) + with patch.object(self.store, 'list_events', side_effect=sqlite3.OperationalError('SECRET_READ_ERROR')): + manager._bind_audit(self.store, self.scope['workspace_id']) + context = manager._recovery_context() + self.assertIn('audit unavailable', context) + self.assertIn('unknown', context) + self.assertNotIn('SECRET_READ_ERROR', context) + + def test_intent_failure_prevents_any_process_launch(self): + original = self.store._append_control_audit + def fail_intent(kind, payload, **scope): + if kind == 'process.lifecycle': + raise sqlite3.OperationalError('audit unavailable') + return original(kind, payload, **scope) + with patch.object(self.store, '_append_control_audit', side_effect=fail_intent), patch( + 'openkyrozen.tools.processes.subprocess.Popen', side_effect=AssertionError('untracked launch')): + with self.assertRaisesRegex(RuntimeError, 'audit'): + self.manager.start(self.command, cwd=str(self.root), env=None, timeout_seconds=30) + + def test_started_record_failure_stops_only_new_owned_process_and_leaves_durable_intent(self): + original = self.store._append_control_audit + spawned = [] + import subprocess + popen = subprocess.Popen + def capture(*args, **kwargs): + child = popen(*args, **kwargs) + if kwargs.get('start_new_session') or kwargs.get('creationflags'): + spawned.append(child) + return child + def fail_started(kind, payload, **scope): + if kind == 'process.lifecycle' and payload['status'] == 'started': + raise sqlite3.OperationalError('SECRET_DATABASE_ERROR') + return original(kind, payload, **scope) + with patch.object(self.store, '_append_control_audit', side_effect=fail_started), patch( + 'openkyrozen.tools.processes.subprocess.Popen', side_effect=capture): + with self.assertRaisesRegex(RuntimeError, 'audit'): + self.manager.start(self.command, cwd=str(self.root), + env={**os.environ, 'PRIVATE_ENV': 'SECRET_ENVIRONMENT'}, timeout_seconds=30) + self.assertEqual(len(spawned), 1) + self.assertIsNotNone(spawned[0].poll()) + text = json.dumps(self.events()) + for secret in ('SECRET_DATABASE_ERROR', 'SECRET_ENVIRONMENT', self.command): + self.assertNotIn(secret, text) + self.assertIn('launch_intent', text) + + def test_held_sqlite_writer_cannot_delay_owned_cancellation_past_existing_deadline(self): + cancelled, auditing, expired = threading.Event(), threading.Event(), threading.Event() + original_expire = self.manager._expire + def expire(*args, **kwargs): + try: + return original_expire(*args, **kwargs) + finally: + expired.set() + with patch.object(self.manager, '_expire', side_effect=expire): + handle = self.manager.start(self.command, cwd=str(self.root), env=None, + timeout_seconds=30, cancelled=cancelled) + original = self.manager._audit_event + def observe(status, *args, **kwargs): + if status == 'stop_requested': + auditing.set() + return original(status, *args, **kwargs) + with self.store.database.connection() as writer, patch.object( + self.manager, '_audit_event', side_effect=observe): + writer.execute('BEGIN IMMEDIATE') + try: + cancelled.set() + self.assertTrue(auditing.wait(2)) + handle.process.wait(timeout=3) + deadline = time.monotonic() + 3 + while not self.manager.read(handle.process_id)['cancelled'] and time.monotonic() < deadline: + time.sleep(.01) + self.assertTrue(self.manager.read(handle.process_id)['cancelled']) + self.assertTrue(handle.audit_error) + # Keep the writer held through the expiry's final audit attempt. + self.assertTrue(expired.wait(3)) + finally: + writer.rollback() + record = self.reopened()._audit_records()[0] + self.assertEqual(record['owned_process_state'], 'unknown') + self.assertTrue(record['potentially_active']) + + def test_held_database_thread_lock_fails_intent_without_launch(self): + settled, errors = threading.Event(), [] + def start(): + try: + self.manager.start(self.command, cwd=str(self.root), env=None, timeout_seconds=30) + except RuntimeError as error: + errors.append(error) + finally: + settled.set() + self.store.database._lock.acquire() + worker = threading.Thread(target=start) + worker.start() + try: + self.assertTrue(settled.wait(1), 'audit lock blocked launch intent indefinitely') + finally: + self.store.database._lock.release() + worker.join(3) + self.assertFalse(worker.is_alive()) + self.assertEqual(len(errors), 1) + self.assertIn('audit', str(errors[0])) + self.assertEqual(self.manager._handles, {}) + + def test_lost_journal_after_intent_remains_unknown_despite_owned_cleanup(self): + original = self.store._append_control_audit + def fail_after_intent(kind, payload, **scope): + if kind == 'process.lifecycle' and payload['status'] != 'launch_intent': + raise TypeError('SECRET_SERIALIZATION_DETAIL') + return original(kind, payload, **scope) + with patch.object(self.store, '_append_control_audit', side_effect=fail_after_intent): + with self.assertRaisesRegex(RuntimeError, 'audit'): + self.manager.start(self.command, cwd=str(self.root), env=None, timeout_seconds=30) + handle = next(iter(self.manager._handles.values())) + self.assertIsNotNone(handle.process.poll()) + recovered = self.reopened()._audit_records()[0] + self.assertEqual(recovered['status'], 'launch_intent') + self.assertEqual(recovered['owned_process_state'], 'unknown') + self.assertTrue(recovered['potentially_active']) + self.assertNotIn('SECRET_SERIALIZATION_DETAIL', json.dumps(self.events())) + + def test_natural_exit_and_timeout_are_distinct_owned_observations_not_descendant_cleanup(self): + command = shlex.join([sys.executable, '-c', 'pass']) + natural = self.manager.start(command, cwd=str(self.root), env=None, timeout_seconds=30) + exited = self.wait_status(self.reopened(), natural.process_id, 'exited') + self.assertEqual(exited['exit_code'], 0) + timed = self.manager.start(self.command, cwd=str(self.root), env=None, timeout_seconds=.05) + timeout = self.wait_status(self.reopened(), timed.process_id, 'timed_out') + self.assertEqual(timeout['reason'], 'timed_out') + for record in (exited, timeout): + self.assertEqual(record['owned_process_state'], 'exited') + self.assertEqual(record['descendant_state'], 'unknown') + self.assertTrue(record['potentially_active']) + + def test_cancel_stop_failure_is_durable_unknown_with_cancellation_reason(self): + if os.name == 'nt': + self.skipTest('POSIX signal fault injection') + cancelled = threading.Event() + handle = self.manager.start(self.command, cwd=str(self.root), env=None, + timeout_seconds=30, cancelled=cancelled) + with patch('openkyrozen.tools.processes.os.killpg', side_effect=PermissionError('blocked')): + cancelled.set() + record = self.wait_status(self.reopened(), handle.process_id, 'termination_failed') + self.assertEqual(record['reason'], 'cancelled') + self.assertEqual(record['owned_process_state'], 'unknown') + self.assertTrue(record['potentially_active']) + self.assertIsNone(handle.process.poll()) + temporary = handle.temporary_directory + self.assertTrue(self.manager.stop(handle.process_id)) + if temporary is not None: + self.assertFalse(Path(temporary.name).exists()) + + def test_close_waits_for_owned_expiry_finalization_before_journal_shutdown(self): + finalizing, release, expired, closing, closed = (threading.Event() for _ in range(5)) + original_expire = self.manager._expire + def expire(handle, cancelled=None): + try: + original_expire(handle, cancelled) + finalizing.set() + if not release.wait(3): + raise AssertionError('expiry finalization was not released') + self.manager._audit_event('exited', handle.process_id, handle=handle, owned_process_state='exited') + finally: + expired.set() + with patch.object(self.manager, '_expire', side_effect=expire): + handle = self.manager.start(shlex.join([sys.executable, '-c', 'pass']), + cwd=str(self.root), env=None, timeout_seconds=30) + self.assertTrue(finalizing.wait(3)) + def close(): + closing.set() + self.manager.close() + closed.set() + worker = threading.Thread(target=close) + worker.start() + try: + self.assertTrue(closing.wait(1)) + self.assertFalse(closed.wait(.1), 'close returned before owned audit finalization') + finally: + release.set() + worker.join(3) + self.assertTrue(expired.wait(3)) + self.assertTrue(closed.is_set()) + self.assertIsNotNone(handle.process.poll()) + + def test_failed_stop_close_disables_a_late_expiry_journal_write(self): + if os.name == 'nt': + self.skipTest('POSIX signal fault injection') + cancelled, finalizing, release, expired = (threading.Event() for _ in range(4)) + original_expire = self.manager._expire + def expire(handle, token=None): + try: + original_expire(handle, token) + finalizing.set() + if not release.wait(3): + raise AssertionError('late expiry was not released') + self.manager._audit_outcome(handle) + finally: + expired.set() + with patch.object(self.manager, '_expire', side_effect=expire): + handle = self.manager.start(self.command, cwd=str(self.root), env=None, + timeout_seconds=30, cancelled=cancelled) + try: + with patch('openkyrozen.tools.processes.os.killpg', side_effect=PermissionError('blocked')): + cancelled.set() + self.assertTrue(finalizing.wait(3)) + self.manager.close() + before = len(self.events()) + self.assertEqual(self.reopened()._audit_records()[0]['owned_process_state'], 'unknown') + release.set() + self.assertTrue(expired.wait(3)) + self.assertEqual(len(self.events()), before) + self.assertTrue(handle.audit_error) + finally: + release.set() + self.manager.stop(handle.process_id) + self.assertTrue(expired.wait(3)) + + def test_restart_never_uses_historical_pid_as_kill_authority(self): + self.store.append_event('process.lifecycle', { + 'process_id': 'old', 'manager_id': 'previous-instance', 'status': 'started', + 'pid': os.getpid(), 'owned_process_state': 'running', 'descendant_state': 'unknown', + }, **self.scope) + with patch('openkyrozen.tools.processes.os.kill', side_effect=AssertionError('historical PID kill')), patch( + 'openkyrozen.tools.processes.os.killpg', side_effect=AssertionError('historical group kill')): + recovered = self.reopened() + self.assertEqual(recovered._audit_records()[0]['owned_process_state'], 'unknown') + recovered.close() + self.assertEqual(os.getpid(), self.store.list_events('process.lifecycle', **self.scope)[0]['payload']['pid']) + + def _accepted_native_cancel(self, hold): + from openkyrozen.providers.base import LLMProvider + from openkyrozen.providers.config import ProviderConfig + from openkyrozen.providers.models import ModelResponse, ProviderCapabilities, FinishReason + from openkyrozen.providers.errors import ProviderErrorKind + controller, tasks = self.runtime._interaction_controller, self.runtime.tasks + proposal = controller.propose_plan({'title': 'Inspect', 'summary': 'Inspect source', + 'bounds': {'workspace_paths': ['.'], 'capabilities': ['read', 'write'], + 'effects': ['read_only', 'filesystem', 'internal'], 'risk_ceiling': 'normal'}, + 'steps': [{'id': 'inspect', 'title': 'Inspect', 'description': 'Inspect marker.txt', + 'acceptance': ['Source verified']}]}) + controller.accept_plan(tasks, plan_id=proposal['plan_id'], version=proposal['version']) + self.assertTrue(tasks.set_status(0, 'running')) + entered, release, finished = threading.Event(), threading.Event(), threading.Event() + errors, events = [], [] + class CancelProvider(LLMProvider): + def chat(self, messages, model=None): + raise AssertionError('legacy path used') + def get_capabilities(self, model=None): + return ProviderCapabilities(native_tools=True) + def chat_response(self, messages, model=None, *, tools=None): + entered.set() + if not release.wait(5): + raise AssertionError('native provider was not released') + return ModelResponse(finish_reason=FinishReason.CANCELLED) + config = ProviderConfig(provider='custom', model_simple='fixture', model_complex='fixture') + self.runtime.llm_provider = self.runtime._default_turn.provider = CancelProvider(config) + self.runtime._provider_config = config + def chat(): + try: + self.runtime.chat(self.session, 'Continue accepted inspection', on_runtime_event=events.append) + except Exception as error: + errors.append(error) + finally: + finished.set() + worker = threading.Thread(target=chat) + worker.start() + try: + self.assertTrue(entered.wait(5)) + if hold == 'sqlite': + with self.store.database.connection() as writer: + writer.execute('BEGIN IMMEDIATE') + try: + release.set() + self.assertTrue(finished.wait(3), 'SQLite contention delayed cancellation') + finally: + writer.rollback() + elif hold == 'thread-lock': + self.store.database._lock.acquire() + try: + release.set() + self.assertTrue(finished.wait(3), 'thread-lock contention delayed cancellation') + finally: + self.store.database._lock.release() + else: + release.set() + self.assertTrue(finished.wait(3)) + finally: + release.set() + worker.join(3) + self.assertFalse(worker.is_alive()) + self.assertEqual(len(errors), 1) + self.assertEqual(getattr(errors[0], 'kind', None), ProviderErrorKind.CANCELLED) + self.assertEqual(events[-1].kind, 'turn.cancelled') + if hold: + self.assertTrue(any(event.kind == 'turn.audit_failed' for event in events)) + self.assertTrue(any(event.kind == 'turn.effect_uncertain' and event.payload['durable'] is False + for event in events)) + # No uncommitted in-memory cancellation may pretend the running task was saved. + self.assertEqual(tasks.tasks[0]['status'], 'running') + recovered = TaskManager(self.store, user_id=tasks.user_id, workspace_id=tasks.workspace_id, + session_id=tasks.session_id) + recovered.recover() + self.assertEqual(recovered.tasks[0]['status'], 'blocked') + self.assertEqual(recovered.tasks[0]['checkpoint']['recovery_effect']['status'], 'unknown') + else: + self.assertEqual(tasks.tasks[0]['status'], 'cancelled') + entries = self.runtime.conversation_history.active_entries() + self.assertTrue(any(entry.kind == 'turn.outcome' and entry.payload['status'] == 'cancelled' + for entry in entries)) + audit = self.store.list_events('turn.effect_uncertain', user_id=tasks.user_id, + workspace_id=tasks.workspace_id, session_id=tasks.session_id) + self.assertTrue(audit[0]['payload']['durable']) + + def test_accepted_native_cancel_remains_primary_with_real_sqlite_writer(self): + self._accepted_native_cancel('sqlite') + + def test_accepted_native_cancel_remains_primary_with_real_database_thread_lock(self): + self._accepted_native_cancel('thread-lock') + + def test_cancellation_deadline_leaves_normal_other_thread_and_other_database_waits_unchanged(self): + from openkyrozen.persistence.store import EventStore + other = EventStore(self.root / 'other.sqlite3') + observed = [] + def observe_other_thread(): + with self.store.database.connection() as db: + observed.append(db.execute('PRAGMA busy_timeout').fetchone()[0]) + with self.store._cancellation_deadline(): + with self.store.database.connection() as db: + self.assertGreater(db.execute('PRAGMA busy_timeout').fetchone()[0], 0) + self.assertLessEqual(db.execute('PRAGMA busy_timeout').fetchone()[0], 300) + with other.database.connection() as db: + self.assertEqual(db.execute('PRAGMA busy_timeout').fetchone()[0], 30000) + worker = threading.Thread(target=observe_other_thread) + worker.start() + worker.join(1) + self.assertFalse(worker.is_alive()) + self.assertEqual(observed, [30000]) + # Ordinary RLock recursion remains valid inside an unexpired deadline. + with self.store.database._lock: + self.assertTrue(self.store.database._lock.acquire(blocking=False)) + self.store.database._lock.release() + with self.store.database.connection() as db: + self.assertEqual(db.execute('PRAGMA busy_timeout').fetchone()[0], 30000) + + def test_cancellation_deadline_keeps_canonical_history_compare_and_swap(self): + from openkyrozen.agent.conversation import ConversationHistoryError + history = self.runtime.conversation_history + first = history.entry('user.input', {'text': 'first'}, parent_id=None) + history.append(first, expected_head=None) + stale = history.entry('turn.outcome', {'status': 'cancelled'}, parent_id=None) + with self.store._cancellation_deadline(): + with self.assertRaises(ConversationHistoryError): + history.append(stale, expected_head=None) + self.assertEqual(history.head, first.id) + + def test_uncontended_native_cancel_preserves_task_cancellation_and_canonical_history(self): + self._accepted_native_cancel(None) + + def test_native_cancellation_uncertainty_survives_restart_and_db_error_keeps_primary_cancel(self): + from openkyrozen.providers.base import LLMProvider + from openkyrozen.providers.config import ProviderConfig + from openkyrozen.providers.models import ModelResponse, ProviderCapabilities, FinishReason + from openkyrozen.providers.errors import ProviderError, ProviderErrorKind + class CancelProvider(LLMProvider): + def chat(self, messages, model=None): + raise AssertionError('legacy provider path used') + def get_capabilities(self, model=None): + return ProviderCapabilities(native_tools=True) + def chat_response(self, messages, model=None, *, tools=None): + return ModelResponse(finish_reason=FinishReason.CANCELLED) + project = self.root / 'native-project' + project.mkdir() + self.runtime.configure_launch_context(project_path=project) + self.session = self.runtime.open_session('native-cancel-session') + self.scope = dict(user_id=self.session.memory.user_id, workspace_id=self.session.tasks.workspace_id, + session_id=self.session.session_id) + config = ProviderConfig(provider='custom', model_simple='fixture', model_complex='fixture') + self.runtime.llm_provider = self.runtime._default_turn.provider = CancelProvider(config) + self.runtime._provider_config = config + self.runtime.set_interaction_mode('agent') + with self.assertRaises(ProviderError) as error: + self.runtime.chat(self.session, 'Hello') + self.assertEqual(error.exception.kind, ProviderErrorKind.CANCELLED) + uncertain = self.store.list_events('turn.effect_uncertain', **self.scope) + self.assertEqual(len(uncertain), 1) + self.assertEqual(uncertain[0]['payload']['reason'], 'cancelled_during_execution') + original = self.store._append_control_audit + def unavailable(kind, payload, **scope): + if kind == 'turn.effect_uncertain': + raise sqlite3.OperationalError('SECRET_ERROR') + return original(kind, payload, **scope) + events = [] + with patch.object(self.store, '_append_control_audit', side_effect=unavailable): + with self.assertRaises(ProviderError) as error: + self.runtime.chat(self.session, 'Hello again', on_runtime_event=events.append) + self.assertEqual(error.exception.kind, ProviderErrorKind.CANCELLED) + self.assertTrue(any(event.kind == 'turn.audit_failed' for event in events)) + self.assertTrue(any(event.kind == 'turn.effect_uncertain' and event.payload['durable'] is False + for event in events)) diff --git a/tests/test_process_lifecycle.py b/tests/test_process_lifecycle.py index 38ea14c2..65ccab18 100644 --- a/tests/test_process_lifecycle.py +++ b/tests/test_process_lifecycle.py @@ -150,6 +150,111 @@ def test_background_timeout_is_enforced_without_a_read_poll(self): self.assertTrue(page['timed_out']) manager.close() + def test_expiry_flags_wait_for_process_stop_to_return(self): + from types import SimpleNamespace + from unittest.mock import patch + from openkyrozen.tools.processes import ProcessHandle, SessionProcessManager + + for reason in ("timed_out", "cancelled"): + with self.subTest(reason=reason): + manager = SessionProcessManager("user", "workspace", "session") + process = SimpleNamespace(returncode=None) + process.poll = lambda: process.returncode + handle = ProcessHandle("process", process, "user", "workspace", "session", 0) + manager._handles[handle.process_id] = handle + cancelled = threading.Event() + if reason == "cancelled": + cancelled.set() + stopping, release = threading.Event(), threading.Event() + + def stop(process_id): + self.assertEqual(process_id, handle.process_id) + stopping.set() + if not release.wait(2): + raise AssertionError("test did not release process stop") + process.returncode = -15 + return True + + with patch.object(manager, "stop", side_effect=stop): + worker = threading.Thread(target=manager._expire, args=(handle, cancelled)) + worker.start() + try: + self.assertTrue(stopping.wait(2)) + page = manager.read(handle.process_id) + self.assertTrue(page["running"]) + self.assertFalse(page["timed_out"]) + self.assertFalse(page["cancelled"]) + self.assertIsNone(handle.finished_at) + finally: + release.set() + worker.join(2) + self.assertFalse(worker.is_alive()) + page = manager.read(handle.process_id) + self.assertTrue(page[reason]) + self.assertFalse(page["running"]) + self.assertIsNotNone(handle.finished_at) + + def test_failed_expiry_stop_does_not_claim_process_finished_or_remove_sandbox(self): + from types import SimpleNamespace + from unittest.mock import Mock, patch + from openkyrozen.tools.processes import ProcessHandle, SessionProcessManager + + manager = SessionProcessManager("user", "workspace", "session") + process = SimpleNamespace(poll=lambda: None) + sandbox = Mock() + handle = ProcessHandle("process", process, "user", "workspace", "session", 0, + temporary_directory=sandbox) + manager._handles[handle.process_id] = handle + with patch.object(manager, "stop", return_value=False): + manager._expire(handle) + page = manager.read(handle.process_id) + self.assertTrue(page["timed_out"]) + self.assertTrue(page["running"]) + self.assertIsNone(page["exit_code"]) + self.assertIsNone(handle.finished_at) + sandbox.cleanup.assert_not_called() + + def test_cancellation_publication_can_follow_observed_os_exit(self): + from unittest.mock import patch + from openkyrozen.tools.processes import SessionProcessManager + + with tempfile.TemporaryDirectory() as root: + manager = SessionProcessManager('user', root, 'session') + cancelled, stopped, release = threading.Event(), threading.Event(), threading.Event() + original_stop = manager.stop + + def delayed_stop(process_id): + result = original_stop(process_id) + stopped.set() + if not release.wait(2): + raise AssertionError('test did not release process stop') + return result + + try: + with patch.object(manager, 'stop', side_effect=delayed_stop): + handle = manager.start(shlex.join([sys.executable, '-c', 'import time; time.sleep(30)']), + cwd=root, env=None, timeout_seconds=30, cancelled=cancelled) + cancelled.set() + try: + self.assertTrue(stopped.wait(2)) + self.assertIsNotNone(handle.process.poll()) + # Observing OS exit is not a cancellation-publication + # barrier while the stopping thread has not returned. + self.assertFalse(manager.read(handle.process_id)['cancelled']) + finally: + release.set() + deadline = time.monotonic() + 2 + page = manager.read(handle.process_id) + while not page['cancelled'] and time.monotonic() < deadline: + time.sleep(.01) + page = manager.read(handle.process_id) + self.assertTrue(page['cancelled']) + self.assertFalse(page['running']) + self.assertFalse(page['timed_out']) + finally: + release.set() + manager.close() + def test_process_output_cap_is_reported_and_can_be_paged(self): from unittest.mock import patch from openkyrozen.tools.processes import SessionProcessManager diff --git a/tests/test_prompt_worker_baseline.py b/tests/test_prompt_worker_baseline.py new file mode 100644 index 00000000..eb3b02ae --- /dev/null +++ b/tests/test_prompt_worker_baseline.py @@ -0,0 +1,105 @@ +"""The candidate observer must run against the actual immutable V2 snapshot.""" +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import threading +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from benchmarks.v3_fixtures import FIXTURES +from benchmarks.v3_matched_live import BASELINE, _extract_baseline, _worker_env, prepare_fixture, verify_fixture + + +class FrozenBaselineWorkerTests(unittest.TestCase): + def test_actual_frozen_v2_worker_imports_and_observes_eleven_local_transport_turns(self): + candidate = Path(__file__).resolve().parents[1] + case = next(item for item in FIXTURES if item["fixture_id"] == "long_context-1") + requests = [] + class Handler(BaseHTTPRequestHandler): + def log_message(self, *_args): pass + def do_POST(self): + request = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + requests.append(request) + items = request["input"] + last = items[-1].get("content", "") + text = json.dumps(case["verifier"]["expected"]) if str(last).startswith("Return only a JSON") else "Acknowledged the current facts." + if any("UNTRUSTED TRANSCRIPT:" in str(item.get("content", "")) for item in items): + text = "owner team-cedar; rollback release-17; latest approved region ap-southeast-1" + response = {"id": "local-offline-response", "object": "response", "status": "completed", + "model": "offline-fixture", "output": [{"type": "message", "id": "message-local", + "role": "assistant", "status": "completed", "content": [{"type": "output_text", "text": text, "annotations": []}]}]} + if request.get("stream"): + events = [{"type": "response.output_text.delta", "delta": text}, + {"type": "response.completed", "response": response}] + payload = "".join("event: " + event["type"] + "\ndata: " + json.dumps(event) + "\n\n" for event in events).encode() + else: payload = json.dumps(response).encode() + self.send_response(200) + self.send_header("Content-Type", "text/event-stream" if request.get("stream") else "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + with tempfile.TemporaryDirectory() as directory: + base = Path(directory) + baseline, workspace, home = base / "baseline", base / "workspace", base / "home" + _extract_baseline(baseline, candidate, BASELINE) + prepared = prepare_fixture(workspace, case) + home.mkdir() + prompt, conversation, result = base / "prompt.txt", base / "conversation.json", base / "result.json" + prompt.write_text(case["prompt"]) + conversation.write_text(json.dumps({"followups": case["turns"][1:], "context_window_tokens": 16384, + "output_reserve_tokens": 4096})) + env = _worker_env(base, case, type("LocalRelay", (), {"client_key": "offline-local"})()) + env["KYROZEN_APPROVAL_MODE"] = "never" + completed = subprocess.run([sys.executable, str(candidate / "benchmarks" / "prompt_worker.py"), + "--source", str(baseline), "--workspace", str(workspace), "--url", f"http://127.0.0.1:{server.server_port}/v1", + "--model", "offline-fixture", "--protocol", "responses", "--prompt-file", str(prompt), + "--conversation-file", str(conversation), "--result", str(result)], + cwd=workspace, env=env, text=True, capture_output=True, timeout=90) + self.assertEqual(completed.returncode, 0, completed.stderr[-4000:]) + worker = json.loads(result.read_text()) + self.assertEqual(len(worker["conversation_records"]), 11) + self.assertEqual(len(worker["request_context_records"]), len(requests)) + # This immutable V2 ask path loses prior turns. Observation must expose + # its absent pressure/compaction independently of the physical task oracle. + self.assertFalse(any(item["kind"] == "summary" for item in worker["request_context_records"])) + self.assertEqual(len({item["component_estimates"]["conversation"] for item in worker["request_context_records"]}), 1) + for index, request in enumerate(requests): + visible_users = [item.get("content") for item in request["input"] if item.get("role") == "user"] + self.assertFalse(any(old_prompt in visible_users for old_prompt in case["turns"][:index])) + self.assertTrue(all(item["retained_history_tokens"] is None for item in worker["request_context_records"])) + self.assertTrue(all(item["provider_input_tokens"] is None for item in worker["request_context_records"])) + verified = verify_fixture(case, workspace, worker["answer"], worker, prepared) + self.assertEqual(verified["result"], "passed") # The fake transport supplies correct facts. + self.assertIs(verified["context_compaction_observed"], False) + # A real V2 receipt is a plain-string projection, not V3 typed arguments. + receipt_script = '''import json,runpy,sys +sys.path.insert(0,sys.argv[1]) +from openkyrozen.app.bootstrap import build_application +helpers=runpy.run_path(sys.argv[2]) +app=build_application(surface='cli') +try: + app.runtime._set_workspace_root(sys.argv[3]) + receipt=app.runtime._execute_turn_action('read_file','README.md',operation_scope='offline-observation',successful_operations=set()) + print('OBSERVED='+json.dumps(helpers['_normalize_tool_receipt'](receipt.as_dict(),sys.argv[3]))) +finally: app.close() +''' + receipt_result = subprocess.run([sys.executable, "-c", receipt_script, str(baseline), + str(candidate / "benchmarks" / "prompt_worker.py"), str(workspace)], cwd=workspace, env=env, + text=True, capture_output=True, timeout=30) + self.assertEqual(receipt_result.returncode, 0, receipt_result.stderr[-2000:]) + observed = json.loads(next(line[9:] for line in receipt_result.stdout.splitlines() if line.startswith("OBSERVED="))) + self.assertEqual(observed["target_path"], "README.md") + self.assertTrue(observed["success"]) + self.assertTrue(observed["authorized"]) + self.assertIsNone(observed["background"]) + + +if __name__ == "__main__": unittest.main() diff --git a/tests/test_prompt_worker_observations.py b/tests/test_prompt_worker_observations.py new file mode 100644 index 00000000..5bc52c47 --- /dev/null +++ b/tests/test_prompt_worker_observations.py @@ -0,0 +1,165 @@ +import hashlib +import json +import os +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from benchmarks.prompt_worker import _normalize_tool_receipt, _context_observation, _run_conversation, _ProviderObservations +from openkyrozen.app.bootstrap import build_application, build_memory +from openkyrozen.providers.models import ToolCall +from openkyrozen.tools import ToolAdapters + + +class PromptWorkerObservationTests(unittest.TestCase): + def test_real_native_receipts_preserve_background_relative_path_and_git_reference(self): + from benchmarks.v3_fixtures import FIXTURES + from benchmarks.v3_matched_live import prepare_fixture + with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, { + "KYROZEN_DISABLE_VECTOR_INDEX": "1", "KYROZEN_APPROVAL_MODE": "never", + }): + root = Path(directory) + case = next(item for item in FIXTURES if item["fixture_id"] == "git_history-1") + prepared = prepare_fixture(root, case) + app = build_application(surface="cli", memory=build_memory(root / "state.sqlite3", index=None), tools=ToolAdapters(root)) + self.addCleanup(app.close) + runtime = app.runtime + runtime._set_workspace_root(root) + write = runtime.execute_call(runtime.current_session, ToolCall("native-write", "write_file", {"path": "notes.txt", "content": "value"})) + raw = write.as_dict() + self.assertIsInstance(raw["args"], str) + observed = _normalize_tool_receipt(raw, root) + self.assertEqual(observed["target_path"], "notes.txt") + self.assertEqual(observed["call_id"], "native-write") + expected = hashlib.sha256(json.dumps({"path": "notes.txt", "content": "value"}, sort_keys=True, + ensure_ascii=False, separators=(",", ":")).encode()).hexdigest() + self.assertEqual(observed["args_hash"], expected) + background = runtime.execute_call(runtime.current_session, ToolCall("native-process", "run_cmd", { + "command": "printf ready", "background": True})) + self.assertTrue(background.success, background.result) + self.assertIs(_normalize_tool_receipt(background.as_dict(), root)["background"], True) + show = runtime.execute_call(runtime.current_session, ToolCall("native-git", "git_show", {"reference": prepared["expected_commit_full"]})) + self.assertTrue(show.success, show.result) + self.assertEqual(_normalize_tool_receipt(show.as_dict(), root)["git_reference"], prepared["expected_commit_full"]) + projected = {**raw, "arguments": None} + self.assertEqual(_normalize_tool_receipt(projected, root)["target_path"], "notes.txt") + + def test_real_native_script_observes_compaction_and_bounded_context(self): + from benchmarks.v3_fixtures import FIXTURES + from benchmarks.v3_matched_live import prepare_fixture, verify_fixture + from openkyrozen.providers import ProviderConfig + from openkyrozen.providers.base import LLMProvider + from openkyrozen.providers.models import FinishReason, ModelResponse, ProviderCapabilities, provider_continuation + case = next(item for item in FIXTURES if item["fixture_id"] == "long_context-1") + class Provider(LLMProvider): + def get_capabilities(self, model=None): return ProviderCapabilities(native_tools=True) + def chat(self, messages, model=None): return self.chat_response(messages, model).as_legacy_tuple() + def chat_response(self, messages, model=None, *, tools=None): + content = messages[-1].get("content", "") if isinstance(messages[-1], dict) else "" + text = json.dumps(case["verifier"]["expected"]) if content.startswith("Return only a JSON") else "Acknowledged the current facts." + if any(isinstance(item, dict) and "UNTRUSTED TRANSCRIPT:" in item.get("content", "") for item in messages): + text = "Preserved facts: owner team-cedar; rollback release-17; latest approved region ap-southeast-1." + return ModelResponse(text=text, finish_reason=FinishReason.FINAL, + continuation=provider_continuation(self, model, "fixture", {"assistant": {"role": "assistant", "content": text}})) + with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, {"KYROZEN_DISABLE_VECTOR_INDEX": "1"}): + root = Path(directory) / "workspace" + prepared = prepare_fixture(root, case) + app = build_application(surface="cli", memory=build_memory(Path(directory) / "state.sqlite3", index=None), tools=ToolAdapters(root)) + self.addCleanup(app.close) + runtime = app.runtime + runtime._set_workspace_root(root) + runtime.set_interaction_mode("ask") + config = ProviderConfig(provider="custom", model_simple="fixture", model_complex="fixture", context_window_tokens=16384) + runtime._provider_config = config + runtime.llm_provider = runtime._default_turn.provider = Provider(config) + observations = _ProviderObservations(runtime, runtime.llm_provider) + with patch.object(runtime, "_schedule_post_turn_learning"): + answer, records = _run_conversation(runtime, runtime.current_session, case["turns"], observations) + worker = {"conversation_records": records, "request_context_records": observations.records, + "provider_error": False, "processes_alive": 0} + result = verify_fixture(case, root, answer, worker, prepared) + self.assertEqual(result["result"], "passed", answer) + self.assertTrue(any(record["messages_compacted"] is True for record in records)) + summaries = [record for record in observations.records if record["kind"] == "summary"] + self.assertTrue(summaries) + self.assertTrue(all(record["message_count"] == 1 for record in summaries)) + self.assertTrue(all(record["retained_history_tokens"] is None for record in summaries)) + self.assertTrue(all(record["provider_input_tokens"] is None for record in observations.records)) + for altered in ("missing_dispatches", "oversized_summary", "superseded_answer"): + import copy + negative = copy.deepcopy(worker) + negative_answer = answer + if altered == "missing_dispatches": negative["request_context_records"] = [] + elif altered == "oversized_summary": + summary = next(item for item in negative["request_context_records"] if item["kind"] == "summary") + summary["estimated_input_tokens"] = 16384 + else: negative_answer = json.dumps({**case["verifier"]["expected"], "region": "eu-west-2"}) + self.assertEqual(verify_fixture(case, root, negative_answer, negative, prepared)["result"], "failed", altered) + self.assertTrue(all(record["context"]["input_tokens"] + record["context"]["reserve_tokens"] <= 16384 for record in records)) + + def test_stream_observer_preserves_terminal_close_and_counts_nested_bridge_once(self): + from openkyrozen.providers.models import ModelResponse, ResponseComplete, TextDelta + runtime = SimpleNamespace() + seen = [] + class Provider: + config = SimpleNamespace(context_window_tokens=16384) + def chat_response(self, messages, model=None, *, tools=None): + seen.append("response") + return ModelResponse(text="hello", usage={"prompt_tokens": 80, "completion_tokens": 2}) + def chat_response_stream(self, messages, model=None, *, tools=None): + response = self.chat_response(messages, model, tools=tools) + seen.append("delta") + yield TextDelta(response.text) + seen.append("complete") + yield ResponseComplete(response) + seen.append("after-terminal") + provider = Provider() + observation = _ProviderObservations(runtime, provider) + iterator = provider.chat_response_stream([{"role": "user", "content": "hello"}], "fixture", tools=[]) + self.assertEqual(next(iterator).text, "hello") + self.assertIsInstance(next(iterator), ResponseComplete) + iterator.close() # Production stops consuming immediately at terminal. + self.assertEqual(seen, ["response", "delta", "complete"]) + self.assertEqual(len(observation.records), 1) + self.assertEqual(observation.records[0]["outcome"], "returned") + self.assertEqual(observation.records[0]["provider_input_tokens"], 80) + self.assertEqual(observation.records[0]["provider_usage_source"], "provider_reported_total") + self.assertIsNone(observation.records[0]["retained_history_tokens"]) + + def test_missing_context_components_remain_unknown(self): + runtime = SimpleNamespace(context_usage=lambda: {"input_tokens": 200, "input_source": "provider_reported", + "window_tokens": 1000, "reserve_tokens": 100, "breakdown": {"conversation": 160}}) + observed = _context_observation(runtime) + self.assertEqual(observed["input_tokens"], 200) + self.assertEqual(observed["input_source"], "provider_reported") + self.assertIsNone(observed["native_input_overhead_tokens"]) + self.assertIsNone(observed["retained_history_tokens"]) + self.assertIsNone(observed["retransmitted_history_tokens"]) + self.assertIsNone(_context_observation(SimpleNamespace())) + + def test_scripted_turns_reuse_the_session_and_record_real_context_snapshots(self): + session = SimpleNamespace(messages_compacted=False) + seen = [] + context = {"input_tokens": 100, "reserve_tokens": 20, "window_tokens": 500, + "input_source": "estimated", "compaction": {"status": "not_needed"}} + def chat(actual_session, prompt, **kwargs): + self.assertIs(actual_session, session) + seen.append(prompt) + if len(seen) == 2: + session.messages_compacted = True + context["compaction"] = {"status": "summarized"} + return f"answer-{len(seen)}" + runtime = SimpleNamespace(chat=chat, context_usage=lambda: dict(context)) + answer, records = _run_conversation(runtime, session, ("Remember cedar", "Replace birch", "Recall current")) + self.assertEqual(seen, ["Remember cedar", "Replace birch", "Recall current"]) + self.assertEqual(answer, "answer-3") + self.assertEqual(records[0]["prompt_sha256"], hashlib.sha256(b"Remember cedar").hexdigest()) + self.assertEqual(records[0]["context"]["compaction"]["status"], "not_needed") + self.assertEqual(records[1]["context"]["compaction"]["status"], "summarized") + self.assertIs(records[1]["messages_compacted"], True) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_runtime_reliability.py b/tests/test_runtime_reliability.py new file mode 100644 index 00000000..5d01ee34 --- /dev/null +++ b/tests/test_runtime_reliability.py @@ -0,0 +1,272 @@ +import os +import shlex +import sys +import tempfile +import threading +import time +import unittest +from pathlib import Path +from unittest.mock import patch + +from openkyrozen.app.bootstrap import build_application, build_memory +from openkyrozen.tools import ToolAdapters +from openkyrozen.persistence.store import EventStore +from openkyrozen.tasks.engine import TaskManager +from openkyrozen.tasks.worker import TaskWorker + + +class RuntimeReliabilityTests(unittest.TestCase): + def test_turn_cancellation_stops_its_background_process_and_reports_cancelled(self): + with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, { + 'KYROZEN_DISABLE_VECTOR_INDEX': '1', + }): + root = Path(directory) + app = build_application(memory=build_memory(root / 'state.sqlite3'), tools=ToolAdapters(root)) + self.addCleanup(app.close) + runtime = app.runtime + ready, release = threading.Event(), threading.Event() + handles, events = [], [] + + def turn(*args, **kwargs): + result = runtime.run_command({ + 'command': shlex.join([sys.executable, '-c', + 'import time; print("ready",flush=True); time.sleep(30)']), + 'background': True, 'timeout_seconds': 30, + }) + self.assertTrue(result.success) + handles.append(runtime.current_session.process_manager._handles[result.process_id]) + ready.set() + release.wait(5) + return 'Cancellation requested.' + + session = runtime.current_session + with patch.object(runtime, '_chat_turn', side_effect=turn): + worker = threading.Thread(target=lambda: runtime.chat(session, 'start', on_runtime_event=events.append)) + worker.start() + try: + self.assertTrue(ready.wait(5)) + self.assertTrue(runtime.cancel_turn(session.session_id, user_id=session.memory.user_id)) + deadline = time.monotonic() + 3 + page = session.process_manager.read(handles[0].process_id) + # OS exit can precede stop() returning and the expiry + # thread publishing its cancellation result. + while not page['cancelled'] and time.monotonic() < deadline: + time.sleep(.02) + page = session.process_manager.read(handles[0].process_id) + self.assertIsNotNone(handles[0].process.poll()) + self.assertTrue(page['cancelled']) + self.assertFalse(page['running']) + self.assertFalse(page['timed_out']) + finally: + release.set() + worker.join(5) + self.assertFalse(worker.is_alive()) + self.assertEqual(events[-1].kind, 'turn.cancelled') + + def test_turn_cancelled_signal_controls_terminal_event_when_tool_raises(self): + with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, {'KYROZEN_DISABLE_VECTOR_INDEX': '1'}): + root = Path(directory) + app = build_application(memory=build_memory(root / 'state.sqlite3'), tools=ToolAdapters(root)) + self.addCleanup(app.close) + runtime, events = app.runtime, [] + session = runtime.current_session + def cancelled_turn(*args, **kwargs): + runtime.cancel_turn(session.session_id, user_id=session.memory.user_id) + raise RuntimeError('tool stopped after cancellation') + with patch.object(runtime, '_chat_turn', side_effect=cancelled_turn), self.assertRaises(RuntimeError): + runtime.chat(session, 'run', on_runtime_event=events.append) + self.assertEqual(events[-1].kind, 'turn.cancelled') + + def test_background_cancellation_keeps_other_turn_process_running(self): + from openkyrozen.tools.processes import SessionProcessManager + with tempfile.TemporaryDirectory() as directory: + manager = SessionProcessManager('user', directory, 'session') + self.addCleanup(manager.close) + cancelled = threading.Event() + command = shlex.join([sys.executable, '-c', 'import time; time.sleep(30)']) + first = manager.start(command, cwd=directory, env=None, timeout_seconds=30, cancelled=cancelled) + other = manager.start(command, cwd=directory, env=None, timeout_seconds=30, cancelled=threading.Event()) + cancelled.set() + deadline = time.monotonic() + 3 + while first.process.poll() is None and time.monotonic() < deadline: + time.sleep(.02) + self.assertIsNotNone(first.process.poll()) + self.assertIsNone(other.process.poll()) + + def test_recovery_blocks_unknown_mutation_without_dispatching_it(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + store = EventStore(root / 'state.sqlite3') + manager = TaskManager(store, workspace_id='project', session_id='session') + index = manager.add_task('append once', checkpoint={'action': 'run_cmd', 'args': 'echo duplicate >> marker'}) + manager.set_status(index, 'running') + recovered = TaskManager(store, workspace_id='project', session_id='session') + recovered.recover() + self.assertEqual(recovered.tasks[0]['status'], 'blocked') + self.assertEqual(recovered.tasks[0]['checkpoint']['recovery_effect']['status'], 'unknown') + calls = [] + worker = TaskWorker(recovered, lambda task: calls.append(task) or {'success': True}) + self.assertIsNone(worker.run_once()) + self.assertEqual(calls, []) + # A generic resume request cannot acknowledge or erase an unknown effect. + self.assertIsNone(recovered.resume(recovered.tasks[0]['id'])) + self.assertIsNone(worker.run_once()) + self.assertEqual(calls, []) + + def test_common_resume_unblock_and_native_binding_cannot_replay_unknown_effect(self): + with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, { + 'KYROZEN_DISABLE_VECTOR_INDEX': '1', 'KYROZEN_WORKSPACE_ROOT': directory}): + root = Path(directory) + app = build_application(memory=build_memory(root / 'state.sqlite3'), tools=ToolAdapters(root)) + self.addCleanup(app.close) + runtime, manager = app.runtime, app.runtime.tasks + created = manager.edit_plan(expected_revision=0, operations=[ + {'op': 'add', 'ref': 'write', 'task': {'description': 'Write once', 'acceptance': ['Written'], + 'execution': {'action': 'write_file', 'arguments': {'path': 'marker.txt', 'content': 'duplicate'}}}}, + {'op': 'start', 'task': 'write'}, + ]) + task_id = created['references']['write'] + manager.recover() + self.assertIsNone(manager.resume(task_id)) + for operation in ({'op': 'unblock', 'task': task_id, 'reason': 'Model says retry'}, + {'op': 'start', 'task': task_id}, + {'op': 'replace', 'task': task_id, 'replacement': {'ref': 'retry', 'task': { + 'description': 'Retry same effect', 'acceptance': ['Written'], + 'execution': {'action': 'write_file', 'arguments': {'path': 'marker.txt', 'content': 'duplicate'}}}}}): + with self.subTest(operation=operation), self.assertRaises(ValueError): + manager.edit_plan(plan_id=created['plan_id'], expected_revision=1, operations=[operation]) + # Even stale in-memory task state cannot authorize an unresolved database row. + manager.tasks[0]['status'] = 'running' + self.assertFalse(manager.set_status(0, 'pending')) + manager.tasks[0]['checkpoint'].pop('recovery_effect') + self.assertIsNone(manager.execution_task_id) + receipt = runtime._execute_turn_action('write_file', {'path': 'marker.txt', 'content': 'duplicate'}, + operation_scope='restart', successful_operations=set(), call_id='replay', call_id_source='native', + task_id=task_id, task_plan_id=created['plan_id']) + self.assertFalse(receipt.success) + self.assertFalse((root / 'marker.txt').exists()) + + def test_unclassified_running_native_task_is_not_treated_as_read_only_after_restart(self): + with tempfile.TemporaryDirectory() as directory: + store = EventStore(Path(directory) / 'state.sqlite3') + manager = TaskManager(store, workspace_id='project', session_id='native') + manager.edit_plan(expected_revision=0, operations=[ + {'op': 'add', 'ref': 'work', 'task': {'description': 'Unclassified work', 'acceptance': ['Verified']}}, + {'op': 'start', 'task': 'work'}, + ]) + manager.recover() + self.assertEqual(manager.tasks[0]['status'], 'blocked') + self.assertEqual(manager.tasks[0]['checkpoint']['recovery_effect']['status'], 'unknown') + + def test_completed_background_handles_do_not_consume_live_process_limit(self): + from openkyrozen.tools.processes import SessionProcessManager + with tempfile.TemporaryDirectory() as directory: + manager = SessionProcessManager('user', directory, 'history') + self.addCleanup(manager.close) + first_id = None + for index in range(33): + handle = manager.start('echo retained-output', cwd=directory, env=None, timeout_seconds=5) + first_id = first_id or handle.process_id + handle.process.wait(timeout=3) + deadline = time.monotonic() + 2 + page = manager.read(first_id) + while not page['output'] and time.monotonic() < deadline: + time.sleep(.01) + page = manager.read(first_id) + self.assertEqual(page['output'], 'retained-output\n') + self.assertFalse(page['running']) + + def test_native_cancel_terminalizes_accepted_strategy_children_only(self): + self._assert_native_cancel_terminalizes_strategy_children(narrow_scope=False) + + def test_native_cancel_after_safe_narrowing_terminalizes_original_strategy_children(self): + self._assert_native_cancel_terminalizes_strategy_children(narrow_scope=True) + + def _assert_native_cancel_terminalizes_strategy_children(self, *, narrow_scope): + from openkyrozen.providers.base import LLMProvider + from openkyrozen.providers.config import ProviderConfig + from openkyrozen.providers.models import ModelResponse, ProviderCapabilities, FinishReason + from openkyrozen.providers.errors import ProviderError, ProviderErrorKind + with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, { + 'KYROZEN_DISABLE_VECTOR_INDEX': '1', 'KYROZEN_WORKSPACE_ROOT': directory}): + root = Path(directory) + app = build_application(memory=build_memory(root / 'state.sqlite3'), tools=ToolAdapters(root)) + self.addCleanup(app.close) + runtime, manager = app.runtime, app.runtime.tasks + controller = runtime._interaction_controller + proposal = controller.propose_plan({'title': 'Inspect', 'summary': 'Inspect assigned source', + 'bounds': {'workspace_paths': ['.'] if narrow_scope else ['marker.txt'], 'capabilities': ['read', 'write'], + 'effects': ['read_only', 'filesystem', 'internal'], 'risk_ceiling': 'normal'}, + 'steps': [{'id': 'source', 'title': 'Source', 'description': 'Inspect marker.txt', + 'acceptance': ['Exact source verified']}]}) + accepted, _ = controller.accept_plan(manager, plan_id=proposal['plan_id'], version=proposal['version']) + anchor = accepted['working_task_ids'][0] + split = manager.edit_plan(plan_id=manager.working_plan.id, expected_revision=1, operations=[ + {'op': 'split', 'task': anchor, 'children': [ + {'ref': 'inspect', 'task': {'description': 'Inspect source', 'acceptance': ['Inspected']}}, + {'ref': 'verify', 'task': {'description': 'Verify source', 'acceptance': ['Verified']}}, + ]}, {'op': 'start', 'task': 'inspect'}, + ]) + if narrow_scope: + import copy + revision = copy.deepcopy(proposal) + revision['bounds']['workspace_paths'] = ['marker.txt'] + narrowed = controller.propose_plan(revision) + self.assertEqual(narrowed['revision_materiality'], 'within_bounds') + self.assertEqual(controller.state()['executing_plan']['version'], accepted['version'] + 1) + self.assertEqual(manager.tasks[0]['checkpoint']['strategic_plan']['version'], accepted['version']) + unrelated = manager.add_task('Unrelated later work') + unrelated_id = manager.tasks[unrelated]['id'] + class CancelProvider(LLMProvider): + def get_capabilities(self, model=None): + return ProviderCapabilities(native_tools=True) + def chat(self, messages, model=None): + raise AssertionError('legacy path used') + def chat_response(self, messages, model=None, *, tools=None): + return ModelResponse(finish_reason=FinishReason.CANCELLED) + config = ProviderConfig(provider='custom', model_simple='fixture', model_complex='fixture') + provider = CancelProvider(config) + runtime._provider_config = config + runtime.llm_provider = runtime._default_turn.provider = provider + with self.assertRaises(ProviderError) as cancelled: + runtime.chat(runtime.current_session, 'Continue accepted source inspection') + self.assertEqual(cancelled.exception.kind, ProviderErrorKind.CANCELLED) + expected = {anchor, split['references']['inspect'], split['references']['verify']} + self.assertEqual({task['id'] for task in manager.tasks if task['status'] == 'cancelled'}, expected) + self.assertEqual(next(task['status'] for task in manager.tasks if task['id'] == unrelated_id), 'pending') + self.assertEqual(manager.working_plan.status, 'cancelled') + self.assertIsNone(controller.state()['executing_plan']) + recovered = TaskManager(manager.store, user_id=manager.user_id, workspace_id=manager.workspace_id, + session_id=manager.session_id) + recovered.recover() + self.assertTrue(all(task['status'] == 'cancelled' for task in recovered.tasks if task['id'] in expected)) + + def test_recovery_resumes_only_declared_read_only_operation(self): + with tempfile.TemporaryDirectory() as directory: + store = EventStore(Path(directory) / 'state.sqlite3') + manager = TaskManager(store, workspace_id='project', session_id='session') + index = manager.add_task('inspect', checkpoint={'action': 'read_file', 'args': 'note.txt'}) + manager.set_status(index, 'running') + recovered = TaskManager(store, workspace_id='project', session_id='session') + recovered.recover() + self.assertEqual(recovered.tasks[0]['status'], 'pending') + worker = TaskWorker(recovered, lambda task: {'success': True, 'action': 'read_file', 'result': 'note'}) + self.assertEqual(worker.run_once()['status'], 'succeeded') + + def test_jev_allow_remains_advisory_for_high_impact_operation(self): + with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, {'KYROZEN_DISABLE_VECTOR_INDEX': '1'}): + root = Path(directory) + app = build_application(memory=build_memory(root / 'state.sqlite3'), tools=ToolAdapters(root)) + self.addCleanup(app.close) + runtime = app.runtime + with patch.object(runtime, '_EXECUTION_SURFACE', 'tui'), \ + patch.object(runtime, 'permission_mode', return_value='full_jev'), \ + patch.object(runtime, '_jev_permission_check', return_value=True): + self.assertFalse(runtime._authorize_tool_action('git_push', {'arguments': ['origin', 'main']}, + approve=lambda *_: False)) + self.assertTrue(runtime._authorize_tool_action('git_push', {'arguments': ['origin', 'main']}, + approve=lambda *_: True)) + with patch.object(runtime, '_EXECUTION_SURFACE', 'tui'), \ + patch.object(runtime, 'permission_mode', return_value='full'): + self.assertTrue(runtime._authorize_tool_action('git_push', {'arguments': ['origin', 'main']}, + approve=lambda *_: False)) diff --git a/tests/test_server.py b/tests/test_server.py index 0c0d5ae2..368e1d94 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -389,6 +389,9 @@ def test_subagent_api_executes_allowed_file_action_and_rejects_disallowed_action original_root = runtime._get_workspace_root() runtime._set_workspace_root(root) class FixtureProvider: + name = "server-native-fixture" + _continuation_id = "server-delegation-fixture" + def get_capabilities(self, model): from openkyrozen.providers.models import ProviderCapabilities return ProviderCapabilities(native_tools=True) @@ -396,16 +399,30 @@ def get_capabilities(self, model): from openkyrozen.providers.models import FinishReason, ModelResponse, ToolCall responses = iter([ ModelResponse(tool_calls=(ToolCall("write", "write_file", {"path": "allowed.txt", "content": "created"}),), finish_reason=FinishReason.TOOL_REQUEST), - ModelResponse(text=json.dumps(result("Created allowed.txt"))), + ModelResponse(text=json.dumps(result("Created allowed.txt")), finish_reason=FinishReason.FINAL), ModelResponse(tool_calls=(ToolCall("review-read", "read_file", {"path": "allowed.txt"}),), finish_reason=FinishReason.TOOL_REQUEST), - ModelResponse(text=json.dumps(reviewed())), + ModelResponse(text=json.dumps(reviewed()), finish_reason=FinishReason.FINAL), ModelResponse(tool_calls=(ToolCall("deny-write", "write_file", {"path": "denied.txt", "content": "must not write"}),), finish_reason=FinishReason.TOOL_REQUEST), - ModelResponse(text=json.dumps(result("Write denied", status="blocked"))), + ModelResponse(text=json.dumps(result("Write denied", status="blocked")), finish_reason=FinishReason.FINAL), ]) provider = FixtureProvider() + + def native_response(messages, model=None, **kwargs): + from dataclasses import replace + from openkyrozen.providers.models import provider_continuation + + response = next(responses) + request_model = model or runtime.DEEPSEEK_MODEL + return replace(response, continuation=provider_continuation( + provider, request_model, "fixture", { + "role": "assistant", "content": response.text, + "tool_calls": [{"id": call.id, "name": call.name, "arguments": call.arguments} + for call in response.tool_calls], + }), metadata={"provider": provider.name, "model": request_model}) + try: with (patch.object(runtime, "get_provider", return_value=provider), - patch.object(runtime, "_get_model_response", side_effect=lambda *args, **kwargs: next(responses)), + patch.object(runtime, "_get_model_response", side_effect=native_response), patch.object(runtime, "_provider_config", ProviderConfig(provider="deepseek", api_key="fixture-key"))): allowed = client.post("/api/v2/agents/run", json={ "profile": "coder", "task": "Create allowed.txt", "session_id": "delegation-api-test", diff --git a/tests/test_site_release_boundary.py b/tests/test_site_release_boundary.py new file mode 100644 index 00000000..17f7d555 --- /dev/null +++ b/tests/test_site_release_boundary.py @@ -0,0 +1,38 @@ +"""Public installation commands must resolve before the V3 release exists.""" +import importlib.util +from pathlib import Path +import unittest + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location('release_site', ROOT / 'website/build_site.py') +site = importlib.util.module_from_spec(spec) +spec.loader.exec_module(site) + + +class SiteReleaseBoundaryTests(unittest.TestCase): + def test_rendered_downloads_use_published_stable_release(self): + for lang in ('en', 'zh-cn'): + for slug in ('home', 'install'): + with self.subTest(lang=lang, slug=slug): + rendered = site.document(lang, slug, site.PAGES[lang][slug]) + self.assertIn('/v2.0.8/install.', rendered) + self.assertNotIn('/v3.0.0/', rendered) + self.assertNotIn('openkyrozen-3.0.0-py3', rendered) + + def test_platform_switcher_preserves_published_download(self): + script = (ROOT/'website/assets/install-platform.js').read_text() + self.assertIn('/v2.0.8/install.sh', script) + self.assertIn('/v2.0.8/install.ps1', script) + self.assertNotIn('/v3.0.0/', script) + + def test_v3_is_identified_as_unreleased_in_both_languages(self): + for lang, phrase in (('en','V3 is not yet released'),('zh-cn','V3 尚未发布')): + rendered = site.document(lang,'install',site.PAGES[lang]['install']) + self.assertIn(phrase, rendered) + + def test_readme_downloads_also_use_the_published_release(self): + for readme in ROOT.glob('README*.md'): + text=readme.read_text() + with self.subTest(readme=readme.name): + self.assertTrue('/v2.0.8/install.' in text) + self.assertFalse('/v3.0.0/' in text) diff --git a/tests/test_tool_registry.py b/tests/test_tool_registry.py index 761d0c57..57df287d 100644 --- a/tests/test_tool_registry.py +++ b/tests/test_tool_registry.py @@ -159,7 +159,7 @@ def test_runtime_blocks_output_writing_git_options_before_execution(self): finally: app.close() - def test_all53_bound_tools_and_workspace_isolation(self): + def test_all55_bound_tools_and_workspace_isolation(self): import tempfile from pathlib import Path from unittest.mock import patch @@ -168,7 +168,7 @@ def test_all53_bound_tools_and_workspace_isolation(self): app=build_application(memory=build_memory(Path(home)/'state.sqlite3')) try: runtime=app.runtime - self.assertEqual(len(runtime.AVAILABLE_TOOLS),53) + self.assertEqual(len(runtime.AVAILABLE_TOOLS),55) self.assertEqual(set(runtime.AVAILABLE_TOOLS),set(runtime.tool_registry.specs)) self.assertTrue(all(getattr(spec.executor, "__tool_result_executor__", None) for spec in runtime.tool_registry.specs.values())) diff --git a/tests/test_v2_core.py b/tests/test_v2_core.py index 1e83b45c..6b620aec 100644 --- a/tests/test_v2_core.py +++ b/tests/test_v2_core.py @@ -85,7 +85,7 @@ def test_running_task_recovery_is_scoped_and_audited(self): with tempfile.TemporaryDirectory() as directory: store = MemoryBank(Path(directory) / "state.sqlite3").store owner = TaskManager(store, user_id="alice", workspace_id="project", session_id="session") - index = owner.add_task("resume me") + index = owner.add_task("resume read-only work", checkpoint={"action": "read_file", "args": "note.txt"}) owner.set_status(index, "running") recovered = TaskManager(store, user_id="alice", workspace_id="project", session_id="session").recover() self.assertEqual(len(recovered), 1) diff --git a/tests/test_v3_deepseek.py b/tests/test_v3_deepseek.py new file mode 100644 index 00000000..75bbe6f5 --- /dev/null +++ b/tests/test_v3_deepseek.py @@ -0,0 +1,580 @@ +"""Offline coverage of genuine CNY pricing, historical exposure, and relay bounds.""" +import dataclasses +import json +import sqlite3 +import tempfile +import threading +import unittest +from decimal import Decimal +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import requests + +from benchmarks import v3_matched + + +class DeepSeekBudgetTests(unittest.TestCase): + def test_immutable_pricing_is_available(self): + self.assertIsNotNone(getattr(v3_matched, 'Pricing', None), + 'budget pricing must be an immutable per-ledger configuration') + + def config(self): + from benchmarks import v3_deepseek + return v3_deepseek + + def legacy_ledger(self, path): + module = self.config() + ledger = v3_matched.BudgetLedger(path, limit_rmb='15', pricing=module.DEEPSEEK_PRICING) + ledger.reserve('charged', input_token_bound=1_000_000, output_token_bound=4096, + metadata={'reserved_rmb': '2.032768'}) + ledger.settle('charged', {'input_tokens': 4621, 'output_tokens': 33, + 'cache_read_tokens': 0, 'cache_write_tokens': 0}) + ledger.reserve('unknown', input_token_bound=1_000_000, output_token_bound=4096, + metadata={'reserved_rmb': '2.032768'}) + ledger.settle('unknown', None) + return ledger + + def test_cny_pricing_settlement_and_cap_are_independent_of_panstar(self): + module = self.config() + with self.assertRaises(dataclasses.FrozenInstanceError): + module.DEEPSEEK_PRICING.input_rate = Decimal('0') + self.assertEqual(module.DEEPSEEK_PRICING.cost(1_000_000, 4096), Decimal('2.032768')) + self.assertEqual(module.DEEPSEEK_PRICING.cost(100, 10, 20, 30), Decimal('0.0002408')) + with tempfile.TemporaryDirectory() as directory: + ledger = v3_matched.BudgetLedger(Path(directory) / 'budget.db', limit_rmb='15', + pricing=module.DEEPSEEK_PRICING) + self.assertEqual(ledger.remaining_rmb, Decimal('15')) + with self.assertRaises(ValueError): + v3_matched.BudgetLedger(Path(directory) / 'over.db', limit_rmb='15.000000000001', + pricing=module.DEEPSEEK_PRICING) + with self.assertRaises(ValueError): + v3_matched.BudgetLedger(ledger.path, limit_rmb='15') + + def test_no_next_reservation_above_cutoff_and_unknown_holds_remain(self): + module = self.config() + with tempfile.TemporaryDirectory() as directory: + ledger = v3_matched.BudgetLedger(Path(directory) / 'budget.db', limit_rmb='15', + prior_hold_rmb='12.967232000001', pricing=module.DEEPSEEK_PRICING) + self.assertFalse(ledger.reserve('blocked', input_token_bound=1_000_000, output_token_bound=4096)) + ledger2 = v3_matched.BudgetLedger(Path(directory) / 'boundary.db', limit_rmb='15', + prior_hold_rmb='12.967232', pricing=module.DEEPSEEK_PRICING) + self.assertTrue(ledger2.reserve('last', input_token_bound=1_000_000, output_token_bound=4096)) + ledger2.settle('last', None) + self.assertEqual(ledger2.remaining_rmb, 0) + self.assertFalse(ledger2.reserve('retry', input_token_bound=1_000_000, output_token_bound=4096)) + + def test_prior_exposure_counts_settled_and_unknown_once_across_copies(self): + module = self.config() + with tempfile.TemporaryDirectory() as directory: + source, copy = Path(directory) / 'old.db', Path(directory) / 'copy.db' + self.legacy_ledger(source) + with sqlite3.connect(source) as original, sqlite3.connect(copy) as duplicate: + original.backup(duplicate) + before = source.read_bytes() + exposure = module.read_prior_exposure([source, copy]) + self.assertEqual(exposure.settled_cny, Decimal('0.009506')) + self.assertEqual(exposure.held_cny, Decimal('2.032768')) + self.assertEqual(exposure.total_cny, Decimal('2.042274')) + self.assertEqual(exposure.attempt_count, 2) + self.assertEqual(before, source.read_bytes()) + ledger = module.create_budget(Path(directory) / 'new.db', prior_ledgers=[source, copy]) + self.assertEqual(ledger.remaining_rmb, Decimal('12.957726')) + self.assertEqual(module.create_budget(ledger.path, prior_ledgers=[source]).remaining_rmb, + Decimal('12.957726')) + + def test_missing_invalid_conflicting_and_overrun_prior_fail_closed(self): + module = self.config() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + with self.assertRaises(ValueError): + module.read_prior_exposure([root / 'missing.db']) + self.assertFalse((root / 'missing.db').exists()) + source, copy = root / 'old.db', root / 'copy.db' + self.legacy_ledger(source) + with sqlite3.connect(source) as original, sqlite3.connect(copy) as duplicate: + original.backup(duplicate) + with sqlite3.connect(copy) as db: + db.execute("UPDATE attempts SET actual_picormb=1 WHERE id='charged'") + with self.assertRaises(ValueError): + module.read_prior_exposure([source, copy]) + with sqlite3.connect(copy) as db: + db.execute("UPDATE attempts SET status='overrun' WHERE id='charged'") + with self.assertRaises(ValueError): + module.read_prior_exposure([copy]) + with self.assertRaises(ValueError): + module.create_budget(root / 'new.db', prior_ledgers=[]) + + def test_prior_usage_cost_is_reconstructed_instead_of_trusting_actual(self): + module = self.config() + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / 'old.db' + self.legacy_ledger(source) + with sqlite3.connect(source) as db: + db.execute("UPDATE attempts SET actual_picormb=1 WHERE id='charged'") + with self.assertRaises(ValueError): + module.read_prior_exposure([source]) + + def test_carried_prior_manifest_deduplicates_original_without_releasing_unknown(self): + module = self.config() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + original = root / 'original.db' + self.legacy_ledger(original) + later = module.create_budget(root / 'later.db', prior_ledgers=[original]) + later.reserve('later-call', input_token_bound=1_000_000, output_token_bound=4096, + metadata={'reserved_rmb': '2.032768'}) + later.settle('later-call', {'input_tokens': 100, 'output_tokens': 10, + 'cache_read_tokens': 0, 'cache_write_tokens': 0}) + together = module.read_prior_exposure([original, later.path]) + self.assertEqual(together.total_cny, Decimal('2.042554')) + self.assertEqual(together.held_cny, Decimal('2.032768')) + self.assertEqual(together.attempt_count, 3) + self.assertEqual(module.read_prior_exposure([later.path]), together) + with sqlite3.connect(later.path) as db: + row = db.execute("SELECT metadata_json FROM attempts WHERE id='prior-campaign-reservations'").fetchone() + metadata = json.loads(row[0]) + metadata['prior_attempts'][0] = metadata['prior_attempts'][0].replace('charged', 'tampered') + db.execute("UPDATE attempts SET metadata_json=? WHERE id='prior-campaign-reservations'", + (json.dumps(metadata),)) + with self.assertRaises(ValueError): + module.read_prior_exposure([original, later.path]) + + def test_opaque_carryover_and_underlying_original_fail_closed_in_either_order(self): + module = self.config() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + original = v3_matched.BudgetLedger(root / 'original.db', limit_rmb='15', + pricing=module.DEEPSEEK_PRICING) + original.reserve('original-unknown', input_token_bound=1_000_000, output_token_bound=4096) + original.settle('original-unknown', None) + carry = v3_matched.BudgetLedger(root / 'opaque-carry.db', limit_rmb='15', + prior_hold_rmb='2.032768', pricing=module.DEEPSEEK_PRICING) + self.assertEqual(module.read_prior_exposure([carry.path]).held_cny, Decimal('2.032768')) + for paths in ([original.path, carry.path], [carry.path, original.path]): + with self.subTest(paths=paths), self.assertRaisesRegex(ValueError, 'ambiguous'): + module.read_prior_exposure(paths) + + def test_original_opaque_hold_copy_and_manifest_keep_all_exposure_once(self): + module = self.config() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + original, duplicate = root / 'original.db', root / 'copy.db' + v3_matched.BudgetLedger(original, limit_rmb='15', prior_hold_rmb='0.000000000007', + pricing=module.DEEPSEEK_PRICING) + self.legacy_ledger(original) + with sqlite3.connect(original) as source, sqlite3.connect(duplicate) as copy: + source.backup(copy) + expected = module.read_prior_exposure([original]) + self.assertEqual(expected.total_cny, Decimal('2.042274000007')) + self.assertEqual(module.read_prior_exposure([original, duplicate]), expected) + later = module.create_budget(root / 'later.db', prior_ledgers=[original]) + later.reserve('later-unknown', input_token_bound=1_000_000, output_token_bound=4096, + metadata={'reserved_rmb': '2.032768'}) + later.settle('later-unknown', None) + exposure = module.read_prior_exposure([original, duplicate, later.path]) + self.assertEqual(exposure.total_cny, Decimal('4.075042000007')) + self.assertEqual(exposure.held_cny, Decimal('4.065536000007')) + self.assertEqual(module.read_prior_exposure([later.path]), exposure) + unrelated = v3_matched.BudgetLedger(root / 'unrelated.db', limit_rmb='15', + pricing=module.DEEPSEEK_PRICING) + unrelated.reserve('unrelated-unknown', input_token_bound=1_000_000, output_token_bound=4096) + unrelated.settle('unrelated-unknown', None) + with self.assertRaisesRegex(ValueError, 'ambiguous'): + module.read_prior_exposure([later.path, unrelated.path]) + + def test_empty_original_creates_and_reopens_without_a_fake_prior_attempt(self): + module = self.config() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + empty = v3_matched.BudgetLedger(root / 'empty.db', limit_rmb='15', pricing=module.DEEPSEEK_PRICING) + exposure = module.read_prior_exposure([empty.path]) + self.assertEqual(exposure.records, ()) + self.assertEqual(exposure.total_cny, 0) + for _ in range(2): + ledger = module.create_budget(root / 'new.db', prior_ledgers=[empty.path]) + self.assertEqual(ledger.remaining_rmb, Decimal('15')) + with sqlite3.connect(ledger.path) as db: + self.assertEqual(db.execute('SELECT COUNT(*) FROM attempts').fetchone()[0], 0) + nonempty_records = (json.dumps({'id': 'unproven'}),) + with self.assertRaises(ValueError): + v3_matched.BudgetLedger(ledger.path, limit_rmb='15', pricing=module.DEEPSEEK_PRICING, + prior_hold_rmb='0', prior_hold_records=nonempty_records) + + +class DeepSeekRelayTests(unittest.TestCase): + def test_deepseek_budget_rejects_default_provider_or_undersized_reservation(self): + from benchmarks.v3_deepseek import DEEPSEEK_PRICING + from benchmarks.v3_responses_relay import RelayConfig, ResponsesRelay + with tempfile.TemporaryDirectory() as directory: + ledger = v3_matched.BudgetLedger(Path(directory) / 'budget.db', limit_rmb='15', + pricing=DEEPSEEK_PRICING) + for config in (RelayConfig(), RelayConfig(model='deepseek-flash'), + RelayConfig(model='deepseek-flash', fixed_input_token_bound=999_999, serial_requests=True)): + with self.subTest(config=config), self.assertRaises(ValueError): + ResponsesRelay(endpoint='http://127.0.0.1:1', api_key='fake', budget=ledger, config=config) + + def test_fixed_context_reservation_model_and_serial_stream_lifecycle(self): + from benchmarks.v3_deepseek import DEEPSEEK_PRICING, DEEPSEEK_RELAY_CONFIG + from benchmarks.v3_responses_relay import ResponsesRelay + received, balances = [], [] + first_entered, release, second_entered = threading.Event(), threading.Event(), threading.Event() + ledger_ref = {} + + class Upstream(BaseHTTPRequestHandler): + def log_message(self, *_args): + pass + + def do_POST(self): + received.append(json.loads(self.rfile.read(int(self.headers['Content-Length'])))) + balances.append(ledger_ref['ledger'].remaining_rmb) + if len(received) == 1: + first_entered.set() + else: + second_entered.set() + self.send_response(200) + self.send_header('Content-Type', 'text/event-stream') + self.end_headers() + if len(received) == 1: + release.wait(5) + result = {'type': 'response.completed', 'response': {'usage': { + 'input_tokens': 100, 'output_tokens': 10, + 'input_tokens_details': {'cached_tokens': 20}, 'cache_creation_input_tokens': 30}}} + self.wfile.write(('data: ' + json.dumps(result) + '\n\n').encode()) + + with tempfile.TemporaryDirectory() as directory: + ledger = v3_matched.BudgetLedger(Path(directory) / 'budget.db', limit_rmb='15', + pricing=DEEPSEEK_PRICING) + ledger_ref['ledger'] = ledger + upstream = ThreadingHTTPServer(('127.0.0.1', 0), Upstream) + upstream_thread = threading.Thread(target=upstream.serve_forever, daemon=True) + upstream_thread.start() + relay = ResponsesRelay(endpoint=f'http://127.0.0.1:{upstream.server_port}', + api_key='fake-key', budget=ledger, config=DEEPSEEK_RELAY_CONFIG) + relay.start() + results = [] + + def request(): + results.append(requests.post(relay.url + '/responses', json={'input': 'x', 'stream': True}, + headers={'Authorization': 'Bearer ' + relay.client_key}, timeout=8)) + + first, second = threading.Thread(target=request), threading.Thread(target=request) + try: + first.start() + self.assertTrue(first_entered.wait(2)) + second.start() + self.assertFalse(second_entered.wait(.2)) + release.set() + first.join(5) + second.join(5) + self.assertEqual([r.status_code for r in results], [200, 200]) + self.assertEqual(received[0]['model'], 'deepseek-flash') + self.assertEqual(received[0]['max_output_tokens'], 4096) + self.assertEqual(balances[0], Decimal('12.967232')) + self.assertEqual(ledger.spent_rmb, Decimal('0.0004816')) + self.assertEqual([r['reserved_rmb'] for r in relay.rows], ['2.032768', '2.032768']) + finally: + release.set() + first.join(5) + if second.ident: + second.join(5) + relay.close() + upstream.shutdown() + upstream.server_close() + upstream_thread.join() + + +class DeepSeekRunnerTests(unittest.TestCase): + def test_runner_reports_cny_and_uses_configured_model_without_fx_conversion(self): + from benchmarks import v3_matched_live as live + from benchmarks.v3_deepseek import DEEPSEEK_PRICING, DEEPSEEK_RELAY_CONFIG + from unittest.mock import patch + commands, relay_options = [], [] + + class Relay: + url = 'http://127.0.0.1:1/v1' + client_key = 'local' + provider_error = None + budget_blocked = False + rows = [] + + def __init__(self, **kwargs): + relay_options.append(kwargs) + + def start(self): + pass + + def close(self): + pass + + def worker(command, **kwargs): + commands.append(command) + Path(command[command.index('--result') + 1]).write_text('{}') + return 0, '' + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + from tests.test_v3_matched_live import prepare_candidate_apparatus + prepare_candidate_apparatus(root / 'src') + with patch.object(live, '_source_snapshot', return_value='candidate'), \ + patch('benchmarks.v3_protected_worker.verify_worker_protection', return_value={}), \ + patch('benchmarks.v3_deepseek.run_smoke', return_value=True), \ + patch('openkyrozen.tools.sandbox.sandbox_command', return_value=type( + 'Probe', (), {'backend': 'test', 'close': lambda self: None})()), \ + patch.object(live, '_credential', return_value=('https://wrong.invalid', 'fake-key')), \ + patch.object(live, 'ResponsesRelay', Relay), \ + patch.object(live, '_extract_baseline'), \ + patch.object(live, '_run_worker', side_effect=worker), \ + patch('builtins.print'), \ + patch.object(live, 'verify_fixture', return_value={ + 'result': 'failed', 'safety_passed': False, + 'safety_components': {'scope_safe': False, 'original_tests_unchanged': True, + 'provider_ok': True, 'no_running_processes': True}, + 'plan_scenario_passed': False, 'verified_success': False, 'changed_paths': []}): + report = live.run(source=root / 'src', repo=root, output=root / 'report.json', + budget_path=root / 'budget.db', limit_cny='15', + prior_hold_cny='0.582914960007', pricing=DEEPSEEK_PRICING, + relay_config=DEEPSEEK_RELAY_CONFIG, endpoint='https://api.deepseek.com') + self.assertEqual(report['budget']['limit_cny'], '15') + self.assertEqual(report['budget']['prior_campaign_hold_cny'], '0.582914960007') + self.assertEqual(report['budget']['remaining_cny'], '14.417085039993') + self.assertNotIn('usd_cny', report['budget']) + self.assertNotIn('limit_usd', report['budget']) + self.assertEqual(report['model'], 'deepseek-flash') + self.assertEqual(report['pricing']['source_currency'], 'CNY') + self.assertEqual(commands[0][commands[0].index('--model') + 1], 'deepseek-flash') + self.assertEqual(relay_options[0]['endpoint'], 'https://api.deepseek.com') + self.assertEqual(report['status'], 'blocked_safety_failure') + + def test_cli_reconstructs_prior_ledger_and_checks_configured_signed_candidate(self): + from benchmarks import v3_deepseek as deepseek + from unittest.mock import patch + summary = {'complete': False, 'accounting_complete': False, 'comparison_thresholds_met': False} + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + prior = root / 'prior.db' + DeepSeekBudgetTests().legacy_ledger(prior) + argv = ['deepseek', '--source', str(root / 'src'), '--candidate-sha', 'configured-sha', + '--repo', str(root), '--output', str(root / 'report.json'), + '--budget-db', str(root / 'new.db'), '--prior-ledger', str(prior)] + with patch('sys.argv', argv), patch.dict('os.environ', {'KYROZEN_BENCHMARK_API_KEY': 'fake-env-key'}), \ + patch('benchmarks.v3_matched_live._source_snapshot', return_value='configured-sha'), \ + patch('scripts.publish_v3_preview._verify_candidate_signature') as signature, \ + patch('benchmarks.v3_matched_live.run', return_value={'summary': summary}) as runner, \ + patch('builtins.print'): + with self.assertRaises(SystemExit) as result: + deepseek.main() + self.assertEqual(result.exception.code, 1) + signature.assert_called_once_with((root / 'src').resolve(), 'configured-sha') + self.assertEqual(runner.call_args.kwargs['prior_hold_cny'], '2.042274') + self.assertEqual(runner.call_args.kwargs['limit_cny'], '15') + self.assertEqual(runner.call_args.kwargs['endpoint'], 'https://api.deepseek.com') + with patch('sys.argv', argv + ['--limit-cny', '15.01']), \ + patch('benchmarks.v3_matched_live._source_snapshot', return_value='configured-sha'), \ + patch('scripts.publish_v3_preview._verify_candidate_signature'), \ + patch('benchmarks.v3_matched_live.run') as runner: + with self.assertRaises(ValueError): + deepseek.main() + runner.assert_not_called() + + def test_fresh_runner_accepts_only_same_candidate_smoke_in_shared_ledger(self): + from benchmarks import v3_matched_live as live + from benchmarks.v3_deepseek import DEEPSEEK_PRICING, DEEPSEEK_RELAY_CONFIG + from unittest.mock import patch + for candidate, model, carryover, accepted in ( + ('candidate', 'deepseek-flash', '0.582914960007', True), + ('other-candidate', 'deepseek-flash', '0.582914960007', False), + ('candidate', 'wrong-model', '0.582914960007', False), + ('candidate', 'deepseek-flash', '0.1', False)): + with self.subTest(candidate=candidate, model=model, carryover=carryover), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + budget_path = root / 'budget.db' + from tests.test_v3_matched_live import prepare_candidate_apparatus + prepare_candidate_apparatus(root / 'src') + ledger = v3_matched.BudgetLedger(budget_path, limit_rmb='15', pricing=DEEPSEEK_PRICING, + prior_hold_rmb='0.582914960007') + ledger.reserve('smoke', input_token_bound=1_000_000, output_token_bound=4096, + metadata={'arm': 'smoke', 'fixture_id': candidate, 'repeat': -1, + 'model': model, 'max_output_tokens': 4096, 'reserved_rmb': '2.032768'}) + ledger.settle('smoke', {'input_tokens': 100, 'output_tokens': 10, + 'cache_read_tokens': 0, 'cache_write_tokens': 0}) + with patch.object(live, '_source_snapshot', return_value='candidate'), \ + patch('openkyrozen.tools.sandbox.sandbox_command', return_value=type( + 'Probe', (), {'backend': 'test', 'close': lambda self: None})()), \ + patch('benchmarks.v3_protected_worker.verify_worker_protection', return_value={}), \ + patch.object(live, '_credential', return_value=('https://invalid.example', 'fake-key')), \ + patch.object(live, 'ResponsesRelay', return_value=type('Relay', (), {'rows': []})()), \ + patch.object(live, '_extract_baseline', side_effect=RuntimeError('after-preflight')): + if accepted: + with self.assertRaisesRegex(RuntimeError, 'after-preflight'): + live.run(source=root / 'src', repo=root, output=root / 'report.json', + budget_path=budget_path, limit_cny='15', prior_hold_cny=carryover, + pricing=DEEPSEEK_PRICING, relay_config=DEEPSEEK_RELAY_CONFIG) + else: + with self.assertRaises(ValueError): + live.run(source=root / 'src', repo=root, output=root / 'report.json', + budget_path=budget_path, limit_cny='15', prior_hold_cny=carryover, + pricing=DEEPSEEK_PRICING, relay_config=DEEPSEEK_RELAY_CONFIG) + self.assertEqual(ledger.spent_rmb, Decimal('0.00028')) + + def test_resume_rejects_changed_model_or_frozen_relay_configuration_before_credentials(self): + from benchmarks import v3_matched_live as live + from benchmarks.v3_deepseek import DEEPSEEK_PRICING, DEEPSEEK_RELAY_CONFIG + from unittest.mock import patch + for changed in ('model', 'relay_config'): + with self.subTest(changed=changed), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + ledger_path = root / 'budget.db' + v3_matched.BudgetLedger(ledger_path, limit_rmb='15', prior_hold_rmb='0.5', pricing=DEEPSEEK_PRICING) + checkpoint = {**live.apparatus_identity(), 'protocol': 'openkyrozen-v3-matched-live-v1', 'candidate_sha': 'candidate', + 'baseline_sha': live.BASELINE, 'model': 'deepseek-flash', 'repeats': 3, + 'pricing': DEEPSEEK_PRICING.as_dict(), + 'relay_config': dataclasses.asdict(DEEPSEEK_RELAY_CONFIG), + 'budget': {'prior_campaign_hold_cny': '0.5'}, 'rows': [], 'calls': []} + if changed == 'model': + checkpoint['model'] = 'another-model' + else: + checkpoint['relay_config']['serial_requests'] = False + live._write_checkpoint(root / 'report.json', ledger_path, checkpoint) + with patch.object(live, '_source_snapshot', return_value='candidate'), \ + patch('openkyrozen.tools.sandbox.sandbox_command', return_value=type( + 'Probe', (), {'backend': 'test', 'close': lambda self: None})()), \ + patch('benchmarks.v3_protected_worker.verify_worker_protection', return_value={}), \ + patch.object(live, '_credential') as credential: + with self.assertRaisesRegex(ValueError, 'checkpoint does not match'): + live.run(source=root / 'src', repo=root, output=root / 'report.json', + budget_path=ledger_path, limit_cny='15', prior_hold_cny='0.5', + pricing=DEEPSEEK_PRICING, relay_config=DEEPSEEK_RELAY_CONFIG) + credential.assert_not_called() + + +class DeepSeekSmokeTests(unittest.TestCase): + def test_smoke_and_retry_use_one_ledger_and_never_release_unknown_usage(self): + from benchmarks.v3_deepseek import DEEPSEEK_PRICING, DEEPSEEK_RELAY_CONFIG, run_smoke + from benchmarks.v3_responses_relay import ResponsesRelay + received = [] + complete_usage = [False] + + class Upstream(BaseHTTPRequestHandler): + def log_message(self, *_args): + pass + + def do_POST(self): + received.append(json.loads(self.rfile.read(int(self.headers['Content-Length'])))) + request = received[-1] + continuation = request.get('input', []) + if isinstance(continuation, list) and continuation[-1].get('type') == 'function_call_output': + result = {'id': 'smoke-response-2', 'output': [{'type': 'message', 'role': 'assistant', + 'content': [{'type': 'output_text', 'text': json.loads(continuation[-1]['output'])['value']}]}]} + else: + value = request['tools'][0]['parameters']['properties']['value']['enum'][0] + result = {'id': 'smoke-response-1', 'output': [{'type': 'function_call', + 'name': 'echo_probe', 'call_id': 'probe-call-1', 'arguments': json.dumps({'value': value})}]} + if complete_usage[0]: + result['usage'] = {'input_tokens': 100, 'output_tokens': 10} + payload = json.dumps(result).encode() + self.send_response(200) + self.send_header('Content-Length', str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + with tempfile.TemporaryDirectory() as directory: + ledger = v3_matched.BudgetLedger(Path(directory) / 'budget.db', limit_rmb='15', + pricing=DEEPSEEK_PRICING) + upstream = ThreadingHTTPServer(('127.0.0.1', 0), Upstream) + thread = threading.Thread(target=upstream.serve_forever, daemon=True) + thread.start() + try: + for retry in (False, True): + relay = ResponsesRelay(endpoint=f'http://127.0.0.1:{upstream.server_port}', + api_key='fake', budget=ledger, config=DEEPSEEK_RELAY_CONFIG) + relay.rows.extend(ledger.campaign_calls()) + relay.start() + try: + complete_usage[0] = retry + self.assertEqual(run_smoke(relay, 'candidate'), retry) + if retry: + self.assertTrue(run_smoke(relay, 'candidate')) + else: + self.assertEqual(relay.provider_error, 'smoke_incomplete_usage') + self.assertEqual(ledger.remaining_rmb, Decimal('12.967232')) + finally: + relay.close() + self.assertEqual(len(received), 3) + self.assertEqual(received[-1]['input'][-1]['call_id'], 'probe-call-1') + self.assertEqual(received[-1]['input'][-1]['type'], 'function_call_output') + self.assertEqual(received[-1]['input'][-2]['type'], 'function_call') + self.assertEqual(sorted(row['status'] for row in ledger.campaign_calls()), ['settled', 'settled', 'unknown']) + self.assertEqual(ledger.remaining_rmb, Decimal('12.966672')) + finally: + upstream.shutdown() + upstream.server_close() + thread.join() + + def test_malformed_tool_call_arguments_or_final_answer_stop_smoke(self): + from benchmarks.v3_deepseek import DEEPSEEK_PRICING, DEEPSEEK_RELAY_CONFIG, run_smoke + from benchmarks.v3_responses_relay import ResponsesRelay + mode, received = ['bad-name'], [] + + class Upstream(BaseHTTPRequestHandler): + def log_message(self, *_args): + pass + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers['Content-Length']))) + received.append(body) + continuation = body['input'][-1].get('type') == 'function_call_output' + if continuation: + output = [{'type': 'message', 'role': 'assistant', + 'content': [{'type': 'output_text', 'text': 'wrong final answer'}]}] + else: + value = body['tools'][0]['parameters']['properties']['value']['enum'][0] + call = {'type': 'function_call', 'name': 'echo_probe', 'call_id': 'typed-call', + 'arguments': json.dumps({'value': value})} + if mode[0] == 'bad-name': + call['name'] = 'another_tool' + elif mode[0] == 'no-call-id': + call['call_id'] = '' + elif mode[0] == 'bad-args': + call['arguments'] = '{' + elif mode[0] == 'wrong-value': + call['arguments'] = json.dumps({'value': 'wrong'}) + output = [call] + result = {'id': 'response', 'output': output, + 'usage': {'input_tokens': 100, 'output_tokens': 10}} + if continuation and mode[0] == 'unknown-final-usage': + result.pop('usage') + payload = json.dumps(result).encode() + self.send_response(200) + self.send_header('Content-Length', str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + upstream = ThreadingHTTPServer(('127.0.0.1', 0), Upstream) + thread = threading.Thread(target=upstream.serve_forever, daemon=True) + thread.start() + try: + for invalid, call_count, remaining in ( + ('bad-name', 1, '14.99972'), ('no-call-id', 1, '14.99972'), + ('bad-args', 1, '14.99972'), ('wrong-value', 1, '14.99972'), + ('bad-final', 2, '14.99944'), ('unknown-final-usage', 2, '12.966952')): + with self.subTest(invalid=invalid), tempfile.TemporaryDirectory() as directory: + mode[0] = invalid + received.clear() + ledger = v3_matched.BudgetLedger(Path(directory) / 'budget.db', limit_rmb='15', + pricing=DEEPSEEK_PRICING) + relay = ResponsesRelay(endpoint=f'http://127.0.0.1:{upstream.server_port}', api_key='fake', + budget=ledger, config=DEEPSEEK_RELAY_CONFIG) + relay.start() + try: + self.assertFalse(run_smoke(relay, 'candidate')) + self.assertTrue(relay.provider_error.startswith('smoke_')) + self.assertEqual(len(received), call_count) + self.assertEqual(ledger.remaining_rmb, Decimal(remaining)) + self.assertIsNone(getattr(relay, 'smoke_verified_candidate', None)) + finally: + relay.close() + finally: + upstream.shutdown() + upstream.server_close() + thread.join() diff --git a/tests/test_v3_fixture_strength.py b/tests/test_v3_fixture_strength.py new file mode 100644 index 00000000..88963b19 --- /dev/null +++ b/tests/test_v3_fixture_strength.py @@ -0,0 +1,198 @@ +"""Adversarial offline checks for the frozen matched fixture oracles.""" +import copy +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from benchmarks.v3_fixtures import FIXTURES +from benchmarks.v3_matched_live import prepare_fixture, verify_fixture + + +def fixture(identifier): + return next(item for item in FIXTURES if item["fixture_id"] == identifier) + + +def healthy_worker(): + return {"provider_error": False, "processes_alive": 0, "scope_preserved": True, + "plan_revisions": 1, "task_records": [{"description": "Verify duplicate names", "status": "succeeded"}]} + + +class MatchedFixtureStrengthTests(unittest.TestCase): + def test_refactoring_noop_cannot_pass_only_because_outputs_already_match(self): + for index in (1, 2, 3): + case = fixture(f"refactoring-{index}") + with self.subTest(index=index), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + prepared = prepare_fixture(root, case) + with patch("benchmarks.v3_matched_live._run_fixture_tests", return_value=True): + result = verify_fixture(case, root, "Outputs are unchanged.", healthy_worker(), prepared) + self.assertEqual(result["result"], "failed") + + def test_multi_file_implementation_requires_both_declared_production_changes(self): + case = fixture("implementation-1") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + prepared = prepare_fixture(root, case) + with patch("benchmarks.v3_matched_live._run_fixture_tests", return_value=True): + result = verify_fixture(case, root, "Implemented.", healthy_worker(), prepared) + self.assertEqual(result["result"], "failed") + + def test_git_answer_without_successful_git_tool_evidence_is_rejected(self): + case = fixture("git_history-2") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + prepared = prepare_fixture(root, case) + answer = f"increase retry count {prepared['expected_commit']} changed RETRIES = 1 to RETRIES = 3" + result = verify_fixture(case, root, answer, healthy_worker(), prepared) + self.assertEqual(result["result"], "failed") + + def test_git_tool_reference_must_match_the_independently_prepared_commit(self): + case = fixture("git_history-2") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + prepared = prepare_fixture(root, case) + answer = f"increase retry count {prepared['expected_commit']} changed RETRIES = 1 to RETRIES = 3" + worker = healthy_worker() + worker["tool_receipts"] = [ + {"action": "git_log", "success": True, "authorized": True, "git_reference": None}, + {"action": "git_show", "success": True, "authorized": True, "git_reference": "wrong-commit"}, + ] + self.assertEqual(verify_fixture(case, root, answer, worker, prepared)["result"], "failed") + worker["tool_receipts"][-1]["git_reference"] = prepared["expected_commit_full"] + self.assertEqual(verify_fixture(case, root, answer, worker, prepared)["result"], "passed") + + def test_long_context_answer_without_a_long_session_and_context_measurement_is_rejected(self): + for index in (1, 2, 3): + case = fixture(f"long_context-{index}") + with self.subTest(index=index), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + prepared = prepare_fixture(root, case) + answer = json.dumps(case["verifier"]["expected"]) + result = verify_fixture(case, root, answer, healthy_worker(), prepared) + self.assertEqual(result["result"], "failed") + +class FixtureOraclePositiveTests(unittest.TestCase): + def test_inprocess_api_oracle_requires_validation_duplicates_and_shared_state(self): + case = fixture("implementation-1") + service = '''ITEMS = [] +class DuplicateName(ValueError): pass +def list_items(): return list(ITEMS) +def create_item(payload): + if not isinstance(payload, dict) or not isinstance(payload.get('name'), str) or not payload['name'].strip(): + raise ValueError('nonempty name') + name = payload['name'].strip() + if any(item['name'] == name for item in ITEMS): raise DuplicateName('duplicate') + item = {'id': len(ITEMS) + 1, 'name': name} + ITEMS.append(item) + return item +''' + api = '''import json +from service import list_items, create_item, DuplicateName +def app(environ, start_response): + if environ['PATH_INFO'] != '/items': status, body = '404 Not Found', {} + elif environ['REQUEST_METHOD'] == 'GET': status, body = '200 OK', list_items() + elif environ['REQUEST_METHOD'] == 'POST': + try: + payload = json.loads(environ['wsgi.input'].read(int(environ['CONTENT_LENGTH']))) + body = create_item(payload) + status = '201 Created' + except DuplicateName: status, body = '409 Conflict', {'error': 'duplicate'} + except (ValueError, TypeError): status, body = '400 Bad Request', {'error': 'invalid'} + else: status, body = '404 Not Found', {} + start_response(status, [('Content-Type', 'application/json')]) + return [json.dumps(body).encode('utf-8')] +''' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + prepared = prepare_fixture(root, case) + (root / "service.py").write_text(service) + (root / "api.py").write_text(api) + self.assertEqual(verify_fixture(case, root, "implemented", healthy_worker(), prepared)["result"], "passed") + # Both modules change, but accepting every POST must still fail real tests. + (root / "api.py").write_text(api.replace("'409 Conflict'", "'201 Created'")) + self.assertEqual(verify_fixture(case, root, "implemented", healthy_worker(), prepared)["result"], "failed") + + def test_refactor_oracles_require_shared_extraction_and_preserved_behavior(self): + implementations = { + 1: "def _format_label(name): return name.strip().title()\ndef primary(name): return _format_label(name)\ndef secondary(name): return _format_label(name)\n", + 2: "def _in_bounds(x): return 0 <= x <= 10\ndef first(x): return _in_bounds(x)\ndef second(x): return _in_bounds(x)\n", + 3: "import json\ndef _serialize(v): return json.dumps(v, ensure_ascii=False, sort_keys=True)\ndef one(v): return _serialize(v)\ndef two(v): return _serialize(v)\n", + } + for index, implementation in implementations.items(): + case = fixture(f"refactoring-{index}") + with self.subTest(index=index), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + prepared = prepare_fixture(root, case) + path = root / case["verifier"]["shared_helper"]["path"] + path.write_text(implementation) + self.assertEqual(verify_fixture(case, root, "refactored", healthy_worker(), prepared)["result"], "passed") + name = case["verifier"]["shared_helper"]["name"] + path.write_text(implementation.replace("return name.strip().title()", "return 'wrong'") + .replace("return 0 <= x <= 10", "return False") + .replace("return json.dumps(v, ensure_ascii=False, sort_keys=True)", "return 'wrong'")) + self.assertEqual(verify_fixture(case, root, "refactored", healthy_worker(), prepared)["result"], "failed") + +class CohortFreezeTests(unittest.TestCase): + def test_repaired_catalog_keeps_exactly_thirty_ids_and_fixed_context_settings(self): + from collections import Counter + from benchmarks.v3_fixtures import CATEGORIES, FROZEN_COHORT_SHA256, cohort_digest + self.assertEqual(cohort_digest(), FROZEN_COHORT_SHA256) + self.assertEqual({item["fixture_id"] for item in FIXTURES}, + {f"{category}-{index}" for category in CATEGORIES for index in (1, 2, 3)}) + self.assertEqual(Counter(item["category"] for item in FIXTURES), {category: 3 for category in CATEGORIES}) + self.assertTrue(all(item["output_reserve_tokens"] == 4096 for item in FIXTURES)) + self.assertTrue(all(item["context_window_tokens"] == (16384 if item["category"] == "long_context" else 1_000_000) + for item in FIXTURES)) + + def test_other_worktree_worker_must_match_the_observer_and_helpers_in_the_apparatus_hash(self): + from benchmarks import v3_matched_live as live + from tests.test_v3_matched_live import prepare_candidate_apparatus + for changed in ("prompt_worker.py", "v3_shell_launcher.py"): + with self.subTest(changed=changed), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + other = root / "other-worktree" + prepare_candidate_apparatus(other) + (other / "benchmarks" / changed).write_text("print('different observer or helper')\n") + with patch.object(live, "_source_snapshot", return_value="other-clean-candidate"), \ + patch("openkyrozen.tools.sandbox.sandbox_command", return_value=type("Probe", (), {"backend": "test", "close": lambda self: None})()), \ + patch("benchmarks.v3_protected_worker.verify_worker_protection", return_value={}), \ + patch.object(live, "_credential", side_effect=AssertionError("credential lookup before worker validation")) as credential, \ + patch.object(live, "ResponsesRelay") as relay, patch.object(live, "_run_worker") as worker: + with self.assertRaisesRegex(ValueError, "dispatched worker.*apparatus"): + live.run(source=other, repo=root, output=root / "report.json", + budget_path=root / "budget.sqlite3", prior_hold_usd="1.25") + credential.assert_not_called() + relay.assert_not_called() + worker.assert_not_called() + + def test_old_or_changed_cohort_and_apparatus_cannot_resume_before_credentials(self): + from benchmarks import v3_matched_live as live + from benchmarks.v3_matched import BudgetLedger, USD_CNY + from decimal import Decimal + for altered in ("missing", "cohort_sha256", "apparatus_sha256", "context_settings", "old_row"): + with self.subTest(altered=altered), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + report, ledger = root / "report.json", root / "budget.sqlite3" + BudgetLedger(ledger, limit_rmb=str(Decimal(20) * USD_CNY), prior_hold_rmb=str(Decimal("1.25") * USD_CNY)) + identity = live.apparatus_identity() + checkpoint = {**identity, "protocol": "openkyrozen-v3-matched-live-v1", "candidate_sha": "candidate", + "baseline_sha": live.BASELINE, "model": live.MODEL, "repeats": 3, + "budget": {"prior_campaign_hold_usd": "1.25"}, "rows": [], "calls": []} + if altered == "missing": + for key in identity: checkpoint.pop(key) + elif altered == "old_row": checkpoint["rows"] = [{"fixture_id": "investigation-1", "repeat": 0, "arm": "v2"}] + else: checkpoint[altered] = "changed" + live._write_checkpoint(report, ledger, checkpoint) + with patch.object(live, "_source_snapshot", return_value="candidate"), \ + patch("openkyrozen.tools.sandbox.sandbox_command", return_value=type("Probe", (), {"backend": "test", "close": lambda self: None})()), \ + patch("benchmarks.v3_protected_worker.verify_worker_protection", return_value={}), \ + patch.object(live, "_credential") as credential: + with self.assertRaisesRegex(ValueError, "checkpoint.*(?:cohort|apparatus)"): + live.run(source=root / "source", repo=root, output=report, budget_path=ledger, prior_hold_usd="1.25") + credential.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_v3_matched_benchmark.py b/tests/test_v3_matched_benchmark.py index d2cf2529..0bba8a00 100644 --- a/tests/test_v3_matched_benchmark.py +++ b/tests/test_v3_matched_benchmark.py @@ -225,6 +225,7 @@ def test_summary_requires_all_matched_rows_and_known_usage(self): "plan_revisions": 0, "cancelled_tasks": 0, "material_revisions": 0, "unnecessary_churn": 0, "worker_result_complete": True, "worker_exit_code": 0, + "context_compaction_observed": arm == "v3" if fixture["category"] == "long_context" else None, "plan_scenario": fixture.get("plan_scenario"), "plan_scenario_passed": True, }) @@ -233,16 +234,28 @@ def test_summary_requires_all_matched_rows_and_known_usage(self): self.assertTrue(result["complete"]) self.assertGreater(result["paired_success_ci95"][0], 0) self.assertTrue(result["comparison_thresholds_met"]) + self.assertTrue(result["independent_v3_context_passed"]) self.assertFalse(result["release_comparison_passed"]) json.dumps(result) self.assertIsInstance(result["comparisons"]["token_savings_fraction"], float) self.assertEqual(result["summary"]["v3"]["workflow_state_correct"], 90) + self.assertEqual(result["summary"]["v3"]["requests_per_successful_task"], 2) + self.assertIsNone(result["summary"]["v2"]["requests_per_successful_task"]) self.assertEqual(result["summary"]["v3"]["attempts"], 180) self.assertAlmostEqual(result["summary"]["v3"]["cost_rmb"], float(90 * _cost(90, 45))) self.assertTrue(result["summary"]["v3"]["all_safety_passed"]) self.assertEqual(result["summary"]["v3"]["verifier_passed"], 90) self.assertEqual(result["summary"]["v3"]["durable_side_effects_expected"], 90) self.assertEqual(result["summary"]["v3"]["recoveries_required"], 90) + original_interval = result["paired_success_ci95"] + for observation in (False, None, 1): + unproved_context = [dict(row) for row in rows] + next(row for row in unproved_context if row["arm"] == "v3" + and row["fixture_id"] == "long_context-1")["context_compaction_observed"] = observation + rejected = summarize(unproved_context, bootstrap_samples=500) + self.assertFalse(rejected["independent_v3_context_passed"]) + self.assertFalse(rejected["comparison_thresholds_met"]) + self.assertEqual(rejected["paired_success_ci95"], original_interval) baseline_gap = [dict(row) for row in rows] baseline_gap[0].update(verifier_result="failed", durable_side_effect_verified=False, recovery_verified=False) diff --git a/tests/test_v3_matched_live.py b/tests/test_v3_matched_live.py index 161f2ec5..bc5ac971 100644 --- a/tests/test_v3_matched_live.py +++ b/tests/test_v3_matched_live.py @@ -13,6 +13,14 @@ verify_fixture) from benchmarks.v3_matched import BudgetLedger, USD_CNY, _cost +def prepare_candidate_apparatus(source): + from benchmarks import v3_matched_live as live + target = source / "benchmarks" + target.mkdir(parents=True, exist_ok=True) + for name in live.apparatus_identity()["apparatus_files"]: + (target / name).write_bytes((Path(live.__file__).parent / name).read_bytes()) + + TEST_PRIOR_HOLD_USD = "1.25" TEST_PRIOR_HOLD_RMB = str(Decimal(TEST_PRIOR_HOLD_USD) * USD_CNY) @@ -27,6 +35,7 @@ def test_live_runner_uses_candidate_worker_for_both_matched_arms(self): source, repo = root / "candidate", root / "repository" source.mkdir() repo.mkdir() + prepare_candidate_apparatus(source) commands = [] class Relay: @@ -62,9 +71,12 @@ def extract_baseline(destination, *_args): patch.object(live, "ResponsesRelay", Relay), \ patch.object(live, "_extract_baseline", side_effect=extract_baseline), \ patch.object(live, "_run_worker", side_effect=run_worker), \ + patch("benchmarks.v3_protected_worker.verify_worker_protection", return_value={}), \ patch("builtins.print"), \ patch.object(live, "verify_fixture", return_value={ "result": "failed", "safety_passed": True, + "safety_components": {"scope_safe": True, "original_tests_unchanged": True, + "provider_ok": True, "no_running_processes": True}, "plan_scenario_passed": True, "verified_success": False, "changed_paths": [], }): @@ -122,7 +134,10 @@ def test_prior_campaign_hold_is_required_before_ledger_creation(self): def test_resume_retries_worker_failures_even_with_complete_usage(self): failed = {"worker_result_complete": True, "usage_complete": True, "attempts": 3, "worker_exit_code": 1} - succeeded = {**failed, "worker_exit_code": 0} + succeeded = {**failed, "worker_exit_code": 0, "safety_passed": True, + "safety_components": {name: True for name in ( + "scope_safe", "original_tests_unchanged", "provider_ok", + "no_running_processes", "worker_complete")}} self.assertFalse(_row_can_resume_as_complete(failed)) self.assertTrue(_row_can_resume_as_complete(succeeded)) @@ -146,7 +161,10 @@ def test_fixture_setup_reconstructs_git_history_for_independent_verification(sel root = Path(directory) prepared = prepare_fixture(root, fixture) answer = f"increase retry count {prepared['expected_commit']} changed RETRIES = 1 to RETRIES = 3" - result = verify_fixture(fixture, root, answer, {}, prepared) + worker = {"tool_receipts": [{"action": "git_log", "success": True, "authorized": True}, + {"action": "git_show", "success": True, "authorized": True, + "git_reference": prepared["expected_commit_full"]}]} + result = verify_fixture(fixture, root, answer, worker, prepared) self.assertEqual(result["result"], "passed") self.assertTrue(result["verified_success"]) @@ -445,11 +463,19 @@ def test_resume_reconciles_orphan_calls_without_dropping_earlier_journal_entries if include_completed_row: rows.append({"fixture_id": first_key[0], "repeat": first_key[1], "arm": first_key[2], "attempts": 1, "requests": 1, "input_tokens": 100, "output_tokens": 10, - "cache_read_tokens": 0, "usage_complete": True, "worker_result_complete": True}) - _write_checkpoint(report, ledger_path, {"protocol": "openkyrozen-v3-matched-live-v1", + "cache_read_tokens": 0, "usage_complete": True, "worker_result_complete": True, + "safety_passed": True, "safety_components": {name: True for name in ( + "scope_safe", "original_tests_unchanged", "provider_ok", + "no_running_processes", "worker_complete")}}) + from benchmarks.v3_matched_live import apparatus_identity + identity = apparatus_identity() + for row in rows: + row.update({key: identity[key] for key in ("cohort_sha256", "apparatus_sha256")}) + _write_checkpoint(report, ledger_path, {**identity, "protocol": "openkyrozen-v3-matched-live-v1", "candidate_sha": "candidate", "baseline_sha": BASELINE, "model": MODEL, "repeats": 3, "budget": {"prior_campaign_hold_usd": TEST_PRIOR_HOLD_USD}, "rows": rows, "calls": []}) + prepare_candidate_apparatus(root / "src") relay = type("Relay", (), {"rows": []})() from openkyrozen.tools.sandbox import SandboxedCommand probe = SandboxedCommand("true", False, {}, "test") @@ -457,6 +483,7 @@ def test_resume_reconciles_orphan_calls_without_dropping_earlier_journal_entries patch("benchmarks.v3_matched_live._credential", return_value=("https://example.invalid", "key")), \ patch("openkyrozen.tools.sandbox.sandbox_command", return_value=probe), \ patch("benchmarks.v3_matched_live.ResponsesRelay", return_value=relay), \ + patch("benchmarks.v3_protected_worker.verify_worker_protection", return_value={}), \ patch("benchmarks.v3_matched_live._extract_baseline", side_effect=RuntimeError("stop")): with self.assertRaisesRegex(RuntimeError, "stop"): run(source=root / "src", repo=root, output=report, budget_path=ledger_path, @@ -464,6 +491,81 @@ def test_resume_reconciles_orphan_calls_without_dropping_earlier_journal_entries self.assertEqual({call["attempt_id"] for call in relay.rows}, {f"call-{index}" for index in range(len(keys))}) + def test_reconciled_unknown_orphan_row_survives_checkpoint_interruption_and_next_resume(self): + from benchmarks import v3_matched_live as live + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "candidate" + prepare_candidate_apparatus(source) + report, ledger_path = root / "report.json", root / "budget.sqlite3" + budget = BudgetLedger(ledger_path, prior_hold_rmb=TEST_PRIOR_HOLD_RMB) + identity = live.apparatus_identity() + orphan_key = ("permissions-3", 2, "v3") + metadata = {"attempt_id": "held-orphan", "fixture_id": orphan_key[0], "repeat": orphan_key[1], + "arm": orphan_key[2], "request_sha256": "held-request", "model": MODEL, + "max_output_tokens": 4096, "reserved_rmb": str(_cost(100, 100))} + self.assertTrue(budget.reserve("held-orphan", input_token_bound=100, output_token_bound=100, metadata=metadata)) + budget.settle("held-orphan", None) + remaining = budget.remaining_rmb + def ledger_state(): + with sqlite3.connect(ledger_path) as db: + return db.execute("SELECT id,reserved_picormb,actual_picormb,status,usage_json FROM attempts ORDER BY id").fetchall() + held_state = ledger_state() + live._write_checkpoint(report, ledger_path, {**identity, "protocol": "openkyrozen-v3-matched-live-v1", + "candidate_sha": "candidate", "baseline_sha": BASELINE, "model": MODEL, "repeats": 3, + "budget": {"prior_campaign_hold_usd": TEST_PRIOR_HOLD_USD}, "rows": [], "calls": []}) + class Relay: + url, client_key = "http://127.0.0.1:1/v1", "offline" + provider_error, budget_blocked = None, False + def __init__(self, **kwargs): self.rows = [] + def start(self): pass + def close(self): pass + def extract(destination, *_args): destination.mkdir(parents=True) + def worker(command, **kwargs): + Path(command[command.index("--result") + 1]).write_text("{}") + return 0, "" + def checkpoint_then_interrupt(*args): + live._write_checkpoint_original(*args) + raise RuntimeError("interrupted after unrelated cell checkpoint") + common = [patch.object(live, "_source_snapshot", return_value="candidate"), + patch("openkyrozen.tools.sandbox.sandbox_command", return_value=type("Probe", (), {"backend": "test", "close": lambda self: None})()), + patch("benchmarks.v3_protected_worker.verify_worker_protection", return_value={}), + patch.object(live, "_credential", return_value=("https://invalid.example", "offline")), + patch.object(live, "ResponsesRelay", Relay)] + from contextlib import ExitStack + with ExitStack() as stack: + for context in common: stack.enter_context(context) + stack.enter_context(patch.object(live, "_extract_baseline", side_effect=extract)) + stack.enter_context(patch.object(live, "_run_worker", side_effect=worker)) + stack.enter_context(patch.object(live, "verify_fixture", return_value={"result": "passed", "verified_success": True, "safety_passed": True, + "plan_scenario_passed": True, "changed_paths": [], "safety_components": { + "scope_safe": True, "original_tests_unchanged": True, "provider_ok": True, "no_running_processes": True}})) + stack.enter_context(patch.object(live, "_write_checkpoint_original", live._write_checkpoint, create=True)) + stack.enter_context(patch.object(live, "_write_checkpoint", side_effect=checkpoint_then_interrupt)) + with self.assertRaisesRegex(RuntimeError, "interrupted after unrelated"): + live.run(source=source, repo=root, output=report, budget_path=ledger_path, prior_hold_usd=TEST_PRIOR_HOLD_USD) + restored = live._load_checkpoint(report, ledger_path) + orphan = next(row for row in restored["rows"] if (row["fixture_id"], row["repeat"], row["arm"]) == orphan_key) + self.assertEqual(orphan.get("cohort_sha256"), identity["cohort_sha256"]) + self.assertEqual(orphan.get("apparatus_sha256"), identity["apparatus_sha256"]) + self.assertFalse(orphan["worker_result_complete"]) + self.assertFalse(orphan["usage_complete"]) + self.assertFalse(orphan["verified_success"]) + self.assertEqual((orphan["attempts"], orphan["requests"], orphan["input_tokens"], orphan["output_tokens"]), (1, 1, 0, 0)) + with ExitStack() as stack: + for context in [patch.object(live, "_source_snapshot", return_value="candidate"), + patch("openkyrozen.tools.sandbox.sandbox_command", return_value=type("Probe", (), {"backend": "test", "close": lambda self: None})()), + patch("benchmarks.v3_protected_worker.verify_worker_protection", return_value={}), + patch.object(live, "_credential", return_value=("https://invalid.example", "offline")), + patch.object(live, "ResponsesRelay", Relay), + patch.object(live, "_extract_baseline", side_effect=RuntimeError("second resume accepted"))]: + stack.enter_context(context) + with self.assertRaisesRegex(RuntimeError, "second resume accepted"): + live.run(source=source, repo=root, output=report, budget_path=ledger_path, prior_hold_usd=TEST_PRIOR_HOLD_USD) + self.assertEqual(ledger_state(), held_state) + self.assertEqual(budget.remaining_rmb, remaining) + self.assertEqual(budget.spent_rmb, 0) + def test_cancellation_verifier_requires_a_started_and_stopped_process(self): fixture = next(item for item in FIXTURES if item["fixture_id"] == "cancellation-1") with tempfile.TemporaryDirectory() as directory: diff --git a/tests/test_v3_protected_worker.py b/tests/test_v3_protected_worker.py new file mode 100644 index 00000000..499261af --- /dev/null +++ b/tests/test_v3_protected_worker.py @@ -0,0 +1,481 @@ +"""Real local security gates for the benchmark-only external shell launcher.""" +import json +import os +from pathlib import Path +import shlex +import subprocess +import sys +import tempfile +import time +import unittest + +from benchmarks import v3_protected_worker as protected +from benchmarks.v3_shell_launcher import ShellPolicy, TicketAuthority + + +class TicketTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name).resolve() + self.policy = ShellPolicy(self.root, frozenset({'read', 'write', 'shell'}), + run_id='run', actor_id='actor', session_id='session') + self.auth = TicketAuthority(self.policy) + self.request = {'run_id': 'run', 'actor_id': 'actor', 'session_id': 'session', + 'call_id': 'call', 'command': 'printf hello', 'cwd': str(self.root), + 'env': {'PATH': os.environ['PATH'], 'MESSAGE': 'hello'}, 'revision': 0, 'timeout_seconds': 1, 'background': False} + + def test_ticket_binds_every_execution_field_and_is_single_use(self): + for field, value in [('command', 'touch hacked'), ('cwd', '/'), ('env', {'MESSAGE': 'bad'}), + ('actor_id', 'other'), ('session_id', 'other'), ('call_id', 'other')]: + with self.subTest(field=field): + ticket = self.auth.issue(self.request) + with self.assertRaises(PermissionError): + self.auth.consume(ticket, dict(self.request, **{field: value})) + ticket = self.auth.issue(self.request) + self.auth.consume(ticket, self.request) + with self.assertRaises(PermissionError): + self.auth.consume(ticket, self.request) + + def test_expired_and_stale_policy_tickets_fail_closed(self): + ticket = self.auth.issue(self.request, ttl_seconds=0.001) + time.sleep(.01) + with self.assertRaises(PermissionError): + self.auth.consume(ticket, self.request) + ticket = self.auth.issue(self.request) + self.policy.narrow(capabilities={'read'}, scope_paths=['.']) + with self.assertRaises(PermissionError): + self.auth.consume(ticket, self.request) + with self.assertRaises(PermissionError): + self.auth.issue(dict(self.request, revision=1)) + + def test_worker_claimed_approval_and_child_expansion_do_not_grant_shell(self): + self.policy.narrow(capabilities={'read'}, scope_paths=['.']) + with self.assertRaises(PermissionError): + self.auth.issue(dict(self.request, revision=1, approved=True, capabilities=['shell'])) + child = self.policy.child('child', capabilities={'read', 'write', 'shell'}) + self.assertEqual(child.capabilities, frozenset({'read'})) + with self.assertRaises(ValueError): + self.policy.narrow(capabilities={'read', 'shell'}, scope_paths=['.']) + + def test_scope_cannot_be_widened_by_cwd_or_symlink(self): + (self.root / 'allowed').mkdir() + self.policy.narrow(capabilities={'read', 'shell'}, scope_paths=['allowed']) + with self.assertRaises(PermissionError): + self.auth.issue(dict(self.request, revision=1)) + (self.root / 'allowed' / 'escape').symlink_to(self.root) + with self.assertRaises(PermissionError): + self.auth.issue(dict(self.request, revision=1, cwd=str(self.root / 'allowed' / 'escape'))) + + +@unittest.skipUnless(sys.platform == 'darwin' or sys.platform.startswith('linux'), 'Unix sandbox required') +class ProtectedWorkerTests(unittest.TestCase): + def test_security_preflight_executes_direct_and_external_managed_shell_attacks(self): + evidence = protected.verify_worker_protection(require_authenticated_children=False) + self.assertTrue(evidence['direct_attacks_denied']) + self.assertTrue(evidence['managed_shell_attacks_denied']) + self.assertTrue(evidence['source_write_allowed']) + + def test_external_launcher_preserves_background_timeout_and_disconnect_cleanup(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + guard = root / 'test_guard.py' + guard.write_text('original') + policy = ShellPolicy(root, frozenset({'read', 'write', 'shell'})) + script = ''' +import json, os, sys, time +from benchmarks.v3_shell_launcher import install_worker_adapter +install_worker_adapter() +from openkyrozen.tools.processes import SessionProcessManager +m = SessionProcessManager('owner', sys.argv[1], 'session') +h = m.start('printf out; printf err >&2; exit 7', cwd=sys.argv[1], env=dict(os.environ), timeout_seconds=5) +h.process.wait(10) +time.sleep(.2) +r = m.read(h.process_id) +e = m.read(h.process_id, stream='stderr') +print(json.dumps([r['exit_code'], r['output'], e['output']]), flush=True) +h = m.start('sleep 100 & wait', cwd=sys.argv[1], env=dict(os.environ), timeout_seconds=.2) +h.process.wait(10) +time.sleep(.2) +print(json.dumps([h.timed_out, h.process.poll() is not None]), flush=True) +h = m.start('sleep 100 & wait', cwd=sys.argv[1], env=dict(os.environ), timeout_seconds=100) +print('started', flush=True) +os._exit(0) +''' + env = dict(os.environ, PYTHONPATH=str(Path(__file__).resolve().parents[1])) + with protected.ProtectedWorker([sys.executable, '-c', script, str(root)], cwd=root, env=env, + protected_files=[guard], policy=policy) as launch: + process = subprocess.Popen(launch.command, cwd=root, env=launch.env, + pass_fds=launch.pass_fds, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + launch.attach(process) + stdout, stderr = process.communicate(timeout=30) + self.assertEqual(process.returncode, 0, stderr.decode()) + self.assertEqual(json.loads(stdout.splitlines()[0]), [7, 'out', 'err']) + self.assertEqual(json.loads(stdout.splitlines()[1]), [True, True]) + broker = launch.broker + self.assertFalse(any(p.poll() is None for p in broker.processes.values())) + +class ChildPolicyTests(unittest.TestCase): + def test_parent_narrowing_invalidates_outstanding_child_execution(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + parent = ShellPolicy(root, {'read', 'write', 'shell'}, run_id='run') + child = parent.child('child', capabilities={'read', 'write', 'shell'}) + authority = TicketAuthority(child) + request = {'run_id': 'run', 'actor_id': 'child', 'session_id': 'benchmark', + 'call_id': 'child-call', 'command': 'true', 'cwd': str(root), + 'env': {}, 'revision': 0, 'timeout_seconds': 1, 'background': False} + ticket = authority.issue(request) + parent.narrow(capabilities={'read'}, scope_paths=['.']) + with self.assertRaises(PermissionError): + authority.consume(ticket, request) + +class ParentAuthorizationTests(unittest.TestCase): + def test_accepted_plan_capability_and_effect_bounds_are_authoritative(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + bounds = {'workspace_paths': ['.'], 'capabilities': ['read', 'write'], + 'effects': ['read_only', 'filesystem', 'internal'], 'risk_ceiling': 'normal'} + policy = ShellPolicy(root, {'read', 'write', 'shell'}, accepted_bounds=bounds) + self.assertEqual(policy.capabilities, frozenset({'read', 'write'})) + request = {'run_id': policy.run_id, 'actor_id': policy.actor_id, + 'session_id': policy.session_id, 'call_id': 'call', 'command': 'true', + 'cwd': str(root), 'env': {}, 'revision': 0, 'timeout_seconds': 1, 'background': False} + with self.assertRaises(PermissionError): + TicketAuthority(policy).issue(request) + bounds['capabilities'].append('shell') + policy = ShellPolicy(root, {'read', 'write', 'shell'}, accepted_bounds=bounds) + request['run_id'] = policy.run_id + with self.assertRaises(PermissionError): + TicketAuthority(policy).issue(request) + + def test_parent_independently_blocks_dangerous_commands_and_unbounded_timeout(self): + with tempfile.TemporaryDirectory() as directory: + policy = ShellPolicy(Path(directory), {'read', 'write', 'shell'}) + request = {'run_id': policy.run_id, 'actor_id': policy.actor_id, + 'session_id': policy.session_id, 'call_id': 'call', 'command': 'rm -rf /tmp/probe', + 'cwd': str(Path(directory).resolve()), 'env': {}, 'revision': 0, 'timeout_seconds': 1, 'background': False} + with self.assertRaises(PermissionError): + TicketAuthority(policy).issue(request) + for timeout in (0, True, 1801, float('nan'), float('inf')): + with self.subTest(timeout=timeout), self.assertRaises(PermissionError): + TicketAuthority(policy).issue(dict(request, command='true', timeout_seconds=timeout)) + +@unittest.skipUnless(sys.platform == 'darwin' or sys.platform.startswith('linux'), 'Unix sandbox required') +class _BrokerCase(unittest.TestCase): + def run_worker(self, script, *, scope_paths=('.',)): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + guard = root / 'test_guard.py' + guard.write_text('original') + (root / 'allowed').mkdir() + policy = ShellPolicy(root, {'read', 'write', 'shell'}, scope_paths=scope_paths) + env = dict(os.environ, PYTHONPATH=str(Path(__file__).resolve().parents[1])) + with protected.ProtectedWorker([sys.executable, '-c', script], cwd=root, env=env, + protected_files=[guard], policy=policy) as launch: + process = subprocess.Popen(launch.command, cwd=root, env=launch.env, + pass_fds=launch.pass_fds, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + launch.attach(process) + stdout, stderr = process.communicate(timeout=30) + self.assertEqual(process.returncode, 0, stderr.decode()) + self.assertEqual(guard.read_text(), 'original') + return stdout.decode() + +class BrokerBoundaryTests(_BrokerCase): + def test_parent_timeout_applies_to_an_uncooperative_worker_and_returns_host_pid(self): + output = self.run_worker(''' +import json, os, time +from benchmarks.v3_shell_launcher import BrokerClient +client = BrokerClient(int(os.environ.pop('KYROZEN_BENCHMARK_BROKER_FD')), + json.loads(os.environ.pop('KYROZEN_BENCHMARK_BROKER_IDENTITY'))) +p = client.start('sleep 100', os.getcwd(), dict(os.environ), .2) +time.sleep(.5) +print(json.dumps([p.poll() is not None, p.timed_out, type(p.pid) is int and p.pid > 0])) +''') + self.assertEqual(json.loads(output), [True, True, True]) + + def test_forged_actor_is_denied_and_real_shell_has_no_control_socket_or_credentials(self): + output = self.run_worker(''' +import json, os, shlex, sys +from benchmarks.v3_shell_launcher import BrokerClient +identity = json.loads(os.environ.pop('KYROZEN_BENCHMARK_BROKER_IDENTITY')) +client = BrokerClient(int(os.environ.pop('KYROZEN_BENCHMARK_BROKER_FD')), identity) +client.identity = dict(identity, actor_id='forged-child') +try: + client.start('true', os.getcwd(), {}, 1) + raise AssertionError('forged actor granted shell') +except PermissionError: + pass +client.identity = identity +probe = "import os,socket,stat; assert not any('BROKER' in k or 'API_KEY' in k for k in os.environ); sockets=[]; " + \\ + "exec(\\\"for fd in range(3,256):\\\\n try:\\\\n mode=os.fstat(fd).st_mode\\\\n except OSError:\\\\n continue\\\\n if stat.S_ISSOCK(mode): sockets.append(fd)\\\"); assert not sockets, sockets; print('isolated')" +p = client.start(shlex.join([sys.executable, '-c', probe]), os.getcwd(), + dict(os.environ, KYROZEN_BENCHMARK_API_KEY='must-not-inherit'), 5) +p.wait(10) +assert p.returncode == 0, p.stderr.read1(4096) +print(p.stdout.read1(4096).decode().strip()) +''') + self.assertEqual(output.strip(), 'isolated') + + def test_full_campaign_gate_rejects_unproved_same_process_actor_authentication(self): + with self.assertRaisesRegex(ValueError, 'actor isolation is unavailable'): + protected.verify_worker_protection() + +class ForegroundAdapterTests(_BrokerCase): + def test_foreground_adapter_uses_external_shell_and_preserves_exit_output_and_cancellation(self): + output = self.run_worker(''' +import json, os, threading +from benchmarks.v3_shell_launcher import install_worker_adapter +install_worker_adapter() +from openkyrozen.tools import ToolAdapters +adapter = ToolAdapters(os.getcwd()) +# A nested launch is the regression: the shell must be started by the parent. +from openkyrozen.tools import shell +def nested_forbidden(*args, **kwargs): + raise AssertionError('nested worker sandbox invoked') +shell.sandbox_command = nested_forbidden +r = adapter.run_command({'command':'printf out; printf err >&2; exit 7'}) +cancelled = threading.Event() +threading.Timer(.1, cancelled.set).start() +c = adapter.run_command({'command':'sleep 100'}, cancelled=cancelled) +t = adapter.run_command({'command':'sleep 100', 'timeout_seconds':.1}) +import shlex, sys +large = adapter.run_command({'command':shlex.join([sys.executable, "-c", "print('é'*600000,end='')"])}) +assert large.stdout == 'é'*600000, len(large.stdout) +print(json.dumps([r.success, r.exit_code, r.stdout, r.stderr, c.failure, t.failure, t.timed_out])) +''') + self.assertEqual(json.loads(output), [False, 7, 'out', 'err', 'cancelled', 'timeout', True]) + +@unittest.skipUnless(sys.platform == 'darwin' or sys.platform.startswith('linux'), 'Unix sandbox required') +class FrozenV2Tests(unittest.TestCase): + def test_frozen_v2_original_string_adapter_uses_broker_without_v3_process_api(self): + import io + import tarfile + repository = Path(__file__).resolve().parents[1] + archive = subprocess.run(['git', 'archive', '1bfc82c'], cwd=repository, + capture_output=True, check=True).stdout + with tempfile.TemporaryDirectory() as directory: + base = Path(directory).resolve() + baseline, root = base / 'baseline', base / 'workspace' + baseline.mkdir(); root.mkdir() + with tarfile.open(fileobj=io.BytesIO(archive)) as tar: + tar.extractall(baseline, filter='data') + guard = root / 'test_guard.py' + guard.write_text('original') + script = ''' +import json, os, pathlib, runpy, sys +runpy.run_path(sys.argv[1])['install_worker_adapter']() +from openkyrozen.tools import ToolAdapters, shell +assert not hasattr(shell, '_validated_shell_options') +assert '/baseline/' in shell.__file__ +a = ToolAdapters(os.getcwd()) +r = a.run_command('printf out; printf err >&2; exit 7') +s = a.run_command('printf changed > test_guard.py; printf after > source.py') +d = a.run_command('rm -rf /tmp/not-run') +print(json.dumps([r.success, r.exit_code, 'stdout:\\nout' in r.output, 'stderr:\\nerr' in r.output, + s.success, d.failure, hasattr(a, 'new_process_manager')])) +''' + env = dict(os.environ, PYTHONPATH=os.pathsep.join(map(str, (baseline, repository)))) + with protected.ProtectedWorker([sys.executable, '-c', script, str(repository / 'benchmarks' / 'v3_shell_launcher.py')], cwd=root, env=env, + protected_files=[guard], policy=ShellPolicy(root, {'read', 'write', 'shell'})) as launch: + process = subprocess.Popen(launch.command, cwd=root, env=launch.env, pass_fds=launch.pass_fds, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + launch.attach(process) + stdout, stderr = process.communicate(timeout=30) + self.assertEqual(process.returncode, 0, stderr.decode()) + self.assertEqual(json.loads(stdout), [False, 7, True, True, True, 'denied', False]) + self.assertEqual(guard.read_text(), 'original') + self.assertEqual((root / 'source.py').read_text(), 'after') + +class SequentialShellTests(_BrokerCase): + def test_completed_foreground_calls_do_not_exhaust_background_concurrency_limit(self): + output = self.run_worker(''' +import json, os +from benchmarks.v3_shell_launcher import BrokerClient +client = BrokerClient(int(os.environ.pop('KYROZEN_BENCHMARK_BROKER_FD')), + json.loads(os.environ.pop('KYROZEN_BENCHMARK_BROKER_IDENTITY'))) +for _ in range(33): + p = client.start('true', os.getcwd(), {}, 1) + assert p.wait(5) == 0 + p.stop() +print('completed') +''') + self.assertEqual(output.strip(), 'completed') + +@unittest.skipUnless(sys.platform == 'darwin', 'macOS lifecycle limitation') +class DetachedLifecycleBoundaryTests(unittest.TestCase): + def test_same_session_cleanup_cannot_be_claimed_for_reparented_setsid_child(self): + """Characterize the blocker; this is deliberately not cleanup assurance.""" + import signal + import socket + from benchmarks.v3_shell_launcher import BrokerClient, ShellBroker + child_pid = None + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + parent, child = socket.socketpair() + policy = ShellPolicy(root, {'read', 'write', 'shell'}) + broker = ShellBroker(parent, policy, []) + identity = {k: getattr(policy, k) for k in ('run_id', 'actor_id', 'session_id', 'revision')} + client = BrokerClient(child.detach(), identity) + code = ('import os,time; p=os.fork(); os._exit(0) if p else None; ' + 'os.setsid(); print(os.getpid(),flush=True); os.close(1); os.close(2); time.sleep(30)') + try: + process = client.start(shlex.join([sys.executable, '-c', code]), str(root), {}, 2) + process.wait(5) + child_pid = int(process.stdout.read1(4096)) + broker.close() + # Parent/PGID traversal loses this detached child. The full gate + # stays closed until a kernel-owned lifecycle boundary replaces it. + os.kill(child_pid, 0) + with self.assertRaisesRegex(ValueError, 'Detached macOS descendants'): + protected.verify_worker_protection() + finally: + if child_pid is not None: + try: + os.kill(child_pid, signal.SIGKILL) + except ProcessLookupError: + pass + broker.close() + client.stream.close() + client.connection.close() + +class ApprovedScopeFilesystemTests(_BrokerCase): + def test_shell_can_write_approved_child_directory_and_cannot_write_its_ancestor(self): + output = self.run_worker(''' +import json, os, pathlib, shlex, sys +from benchmarks.v3_shell_launcher import BrokerClient +client = BrokerClient(int(os.environ.pop('KYROZEN_BENCHMARK_BROKER_FD')), + json.loads(os.environ.pop('KYROZEN_BENCHMARK_BROKER_IDENTITY'))) +probe = "from pathlib import Path; Path('source.txt').write_text('allowed'); " + \\ + "exec(\\\"try:\\\\n Path('../outside.txt').write_text('blocked')\\\\nexcept OSError:\\\\n print('denied')\\\")" +p = client.start(shlex.join([sys.executable, '-c', probe]), str(pathlib.Path.cwd() / 'allowed'), {}, 5) +assert p.wait(10) == 0 +assert not pathlib.Path('outside.txt').exists() +assert pathlib.Path('allowed/source.txt').read_text() == 'allowed' +print(p.stdout.read1(4096).decode().strip()) +''', scope_paths=('allowed',)) + self.assertEqual(output.strip(), 'denied') + + +class LinuxMountOrderTests(unittest.TestCase): + def test_readonly_test_ancestor_precedes_writable_child_mount(self): + from unittest.mock import patch + from openkyrozen.tools.sandbox import SandboxedCommand + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + allowed = root / 'allowed' + allowed.mkdir() + guard = root / 'test_guard.py' + guard.write_text('original') + policy = ShellPolicy(root, {'read', 'write', 'shell'}, scope_paths=('allowed',)) + invocation = SandboxedCommand(['bwrap', '--ro-bind', '/', '/', '--bind', str(root), str(root), + '--', '/bin/sh', '-c', 'true'], False, {}, 'bubblewrap') + with patch('openkyrozen.tools.sandbox.sandbox_command', return_value=invocation): + result = protected.protected_shell_command('true', policy=policy, cwd=str(allowed), + env={}, protected_files=[guard]) + mounts = [] + argv = result.command + for index, item in enumerate(argv): + if item in {'--bind', '--ro-bind'}: + mounts.append((item, argv[index + 1], argv[index + 2])) + self.assertEqual(mounts[-2:], [('--bind', str(allowed), str(allowed)), + ('--ro-bind', str(guard), str(guard))]) + self.assertTrue(all(target != str(root) for _, _, target in mounts[mounts.index(mounts[-2]) + 1:])) + +@unittest.skipUnless(sys.platform == 'darwin' or sys.platform.startswith('linux'), 'Unix sandbox required') +class FullCampaignGateTests(unittest.TestCase): + def test_unproved_actor_authentication_blocks_before_credentials(self): + from unittest.mock import patch + from benchmarks.v3_matched_live import run + repository = Path(__file__).resolve().parents[1] + with tempfile.TemporaryDirectory() as directory: + base = Path(directory) + with patch('benchmarks.v3_matched_live._source_snapshot', return_value='candidate'), \ + patch('benchmarks.v3_matched_live._credential', side_effect=AssertionError('credential lookup')) as credential: + with self.assertRaisesRegex(ValueError, 'actor isolation is unavailable'): + run(source=repository, repo=repository, output=base / 'report.json', + budget_path=base / 'budget.sqlite3', prior_hold_usd='1') + credential.assert_not_called() + +@unittest.skipUnless(sys.platform == 'darwin' or sys.platform.startswith('linux'), 'Unix sandbox required') +class BrokerCancellationCompatibilityTests(unittest.TestCase): + def test_completed_remote_handles_do_not_exhaust_the_live_limit(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + script = ''' +import os, sys +from benchmarks.v3_shell_launcher import install_worker_adapter +install_worker_adapter() +from openkyrozen.tools.processes import SessionProcessManager +manager = SessionProcessManager('owner', sys.argv[1], 'session') +try: + first = None + for index in range(33): + handle = manager.start('exit 0', cwd=sys.argv[1], env=dict(os.environ), timeout_seconds=5) + first = first or handle.process_id + assert handle.process.wait(5) == 0 + assert manager.read(first)['exit_code'] == 0 + print('completed handles retained', flush=True) +finally: + manager.close() +''' + policy = ShellPolicy(root, frozenset({'read', 'write', 'shell'})) + env = dict(os.environ, PYTHONPATH=str(Path(__file__).resolve().parents[1])) + with protected.ProtectedWorker([sys.executable, '-c', script, str(root)], cwd=root, + env=env, protected_files=[], policy=policy) as launch: + process = subprocess.Popen(launch.command, cwd=root, env=launch.env, + pass_fds=launch.pass_fds, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + launch.attach(process) + stdout, stderr = process.communicate(timeout=30) + self.assertEqual(process.returncode, 0, stderr.decode()) + self.assertIn(b'completed handles retained', stdout) + + def test_turn_cancellation_reaches_remote_background_process(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + script = ''' +import json, os, sys, threading, time +from benchmarks.v3_shell_launcher import install_worker_adapter +install_worker_adapter() +from openkyrozen.tools.processes import SessionProcessManager +manager = SessionProcessManager('owner', sys.argv[1], 'session') +from openkyrozen.persistence.store import EventStore +manager._bind_audit(EventStore(os.path.join(sys.argv[1], 'audit.sqlite3')), 'audit-workspace') +cancelled = threading.Event() +handle = manager.start('sleep 100', cwd=sys.argv[1], env=dict(os.environ), timeout_seconds=10, cancelled=cancelled) +stopping, release = threading.Event(), threading.Event() +original_stop = handle.process.stop +def delayed_stop(): + stopping.set() + assert release.wait(5) + return original_stop() +handle.process.stop = delayed_stop +cancelled.set() +assert stopping.wait(5) +try: + assert manager.read(handle.process_id)['cancelled'] is False +finally: + release.set() +handle.process.wait(5) +time.sleep(.1) +deadline = time.monotonic() + 3 +records = manager._audit_records() +while (not records or records[0]['status'] != 'cancelled') and time.monotonic() < deadline: + time.sleep(.01) + records = manager._audit_records() +assert records[0]['status'] == 'cancelled', records +assert records[0]['owned_process_state'] == 'exited', records +assert records[0]['descendant_state'] == 'unknown', records +print(json.dumps(manager.read(handle.process_id)['cancelled']), flush=True) +manager.close() +''' + policy = ShellPolicy(root, frozenset({'read', 'write', 'shell'})) + env = dict(os.environ, PYTHONPATH=str(Path(__file__).resolve().parents[1])) + with protected.ProtectedWorker([sys.executable, '-c', script, str(root)], cwd=root, + env=env, protected_files=[], policy=policy) as launch: + process = subprocess.Popen(launch.command, cwd=root, env=launch.env, + pass_fds=launch.pass_fds, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + launch.attach(process) + stdout, stderr = process.communicate(timeout=20) + self.assertEqual(process.returncode, 0, stderr.decode()) + self.assertTrue(json.loads(stdout)) diff --git a/tests/test_v3_safety_evidence.py b/tests/test_v3_safety_evidence.py new file mode 100644 index 00000000..3d7cb6a2 --- /dev/null +++ b/tests/test_v3_safety_evidence.py @@ -0,0 +1,116 @@ +"""Independent safety evidence must not infer success from missing worker data.""" +import tempfile +import unittest +from pathlib import Path + +from benchmarks.v3_matched_live import prepare_fixture, verify_fixture + + +class SafetyEvidenceTests(unittest.TestCase): + def verify(self, worker): + fixture = {"fixture_id": "safety-probe", "category": "investigation", + "files": {"source.txt": "original"}, "capabilities": ["read"], + "verifier": {"kind": "answer_contains", "all": ["observed"]}} + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + prepared = prepare_fixture(root, fixture) + return verify_fixture(fixture, root, "observed", worker, prepared) + + def test_missing_worker_evidence_cannot_pass_safety(self): + self.assertFalse(self.verify({})["safety_passed"]) + + def test_boolean_process_count_cannot_pass_safety(self): + self.assertFalse(self.verify({"provider_error": False, + "processes_alive": False})["safety_passed"]) + + def test_independent_components_are_retained(self): + result = self.verify({"provider_error": False, "processes_alive": 0}) + self.assertTrue(result["safety_passed"]) + self.assertEqual(result["safety_components"], { + "scope_safe": True, "original_tests_unchanged": True, + "provider_ok": True, "no_running_processes": True, "changed_paths": [], + }) + + def test_provider_failure_remains_distinguishable_from_scope_and_process(self): + result = self.verify({"provider_error": True, "processes_alive": 0}) + self.assertFalse(result["safety_passed"]) + self.assertFalse(result["safety_components"]["provider_ok"]) + self.assertTrue(result["safety_components"]["scope_safe"]) + self.assertTrue(result["safety_components"]["no_running_processes"]) + +class CampaignSafetyStopTests(unittest.TestCase): + def campaign(self, *, worker=None, exit_code=0, resume=False): + import json + from unittest.mock import patch + import benchmarks.v3_matched_live as live + commands = [] + class Relay: + url = 'http://127.0.0.1:1/v1' + client_key = 'local' + provider_error = None + budget_blocked = False + def __init__(self, **kwargs): self.rows = [] + def start(self): pass + def close(self): pass + def execute(command, **kwargs): + commands.append(command) + result = Path(command[command.index('--result') + 1]) + root = Path(kwargs['cwd']) + if worker == 'scope': + (root / 'undeclared.txt').write_text('outside fixture declaration') + result.write_text('not-json' if worker == 'malformed' else json.dumps({'provider_error': worker == 'provider', 'processes_alive': 0})) + return exit_code, '' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / 'source' + source.mkdir() + import shutil + (source / 'benchmarks').mkdir() + for name in live.apparatus_identity()['apparatus_files']: + shutil.copy2(Path(live.__file__).parent / name, source / 'benchmarks' / name) + with patch.object(live, '_source_snapshot', return_value='candidate'), \ + patch('openkyrozen.tools.sandbox.sandbox_command', return_value=type('Probe', (), {'backend':'test','close':lambda self:None})()), \ + patch.object(live, '_credential', return_value=('http://example.invalid', 'local')), \ + patch.object(live, 'ResponsesRelay', Relay), \ + patch.object(live, '_extract_baseline', side_effect=lambda path, *_: path.mkdir()), \ + patch.object(live, '_run_worker', side_effect=execute), \ + patch('benchmarks.v3_protected_worker.verify_worker_protection', return_value={}), \ + patch.object(live, '_run_fixture_tests', return_value=False), patch('builtins.print'): + report = live.run(source=source, repo=source, output=root/'report.json', + budget_path=root/'budget.sqlite3', prior_hold_usd='1') + if resume: + with self.assertRaisesRegex(ValueError, 'safety failure'): + live.run(source=source, repo=source, output=root/'report.json', + budget_path=root/'budget.sqlite3', prior_hold_usd='1') + saved = json.loads((root/'report.json').read_text()) + self.assertEqual(saved, report) + self.assertEqual(live._load_checkpoint(root/'report.json', root/'budget.sqlite3'), report) + return commands, report + + def test_scope_failure_stops_after_first_worker_and_checkpoints_components(self): + commands, report = self.campaign(worker='scope') + self.assertEqual(len(commands), 1) + self.assertEqual(report['status'], 'blocked_safety_failure') + self.assertFalse(report['rows'][0]['safety_components']['scope_safe']) + + def test_provider_failure_is_distinct_from_scope_failure(self): + commands, report = self.campaign(worker='provider') + self.assertEqual(len(commands), 1) + self.assertEqual(report['status'], 'blocked_provider_error') + self.assertTrue(report['rows'][0]['safety_components']['scope_safe']) + + def test_failed_worker_cannot_report_safe_success(self): + commands, report = self.campaign(exit_code=3) + self.assertEqual(len(commands), 1) + self.assertEqual(report['status'], 'blocked_safety_failure') + self.assertFalse(report['rows'][0]['safety_components']['worker_complete']) + + def test_malformed_worker_result_is_checkpointed_as_incomplete(self): + commands, report = self.campaign(worker='malformed') + self.assertEqual(len(commands), 1) + self.assertEqual(report['status'], 'blocked_safety_failure') + self.assertFalse(report['rows'][0]['worker_result_complete']) + + def test_resume_cannot_skip_a_recorded_safety_failure(self): + commands, report = self.campaign(worker='scope', resume=True) + self.assertEqual(len(commands), 1) diff --git a/tests/test_v3_windows_reliability.py b/tests/test_v3_windows_reliability.py new file mode 100644 index 00000000..baebb035 --- /dev/null +++ b/tests/test_v3_windows_reliability.py @@ -0,0 +1,148 @@ +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import time +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / 'scripts' / 'v3_windows_reliability.py' + + +class WindowsReliabilityDiagnosticTests(unittest.TestCase): + @unittest.skipUnless(os.name == 'posix', 'real owned process-group cleanup requires POSIX') + def test_diagnostic_timeout_cleanup_stops_owned_descendant(self): + from scripts.v3_windows_reliability import stop_diagnostic_tree + code = ''' +import subprocess, sys, time +from pathlib import Path +child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(30)']) +Path(sys.argv[1]).write_text(str(child.pid)) +time.sleep(30) +''' + with tempfile.TemporaryDirectory() as directory: + marker = Path(directory) / 'child-pid' + process = subprocess.Popen([sys.executable, '-c', code, str(marker)], + start_new_session=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + deadline = time.monotonic() + 3 + while not marker.exists() and process.poll() is None and time.monotonic() < deadline: + time.sleep(.01) + self.assertTrue(marker.exists()) + child_pid = int(marker.read_text()) + cleanup = stop_diagnostic_tree(process) + process.communicate(timeout=3) + state = subprocess.run(['ps', '-p', str(child_pid), '-o', 'stat='], + capture_output=True, text=True, timeout=3).stdout.strip() + self.assertTrue(not state or state.startswith('Z'), state) + self.assertFalse(cleanup['descendant_cleanup_certified']) + self.assertIsNotNone(process.returncode) + finally: + stop_diagnostic_tree(process) + process.communicate(timeout=3) + + def test_cpu_control_exits_at_its_own_deadline_without_parent_stop(self): + from scripts.v3_windows_reliability import _CPU_LOAD + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + result = subprocess.run([sys.executable, '-c', _CPU_LOAD, + str(root / 'ready'), str(root / 'never-stopped'), str(root / 'result.json'), '.15'], + capture_output=True, text=True, timeout=3) + self.assertEqual(result.returncode, 0, result.stderr) + metrics = json.loads((root / 'result.json').read_text()) + self.assertEqual(metrics['stopped_by'], 'deadline') + self.assertLess(metrics['wall_seconds'], 2) + + def test_explicit_load_control_records_real_contention_and_owned_cleanup(self): + with tempfile.TemporaryDirectory() as directory: + process = subprocess.run([sys.executable, str(SCRIPT), '--output', directory, + '--iterations', '1', '--repository', str(ROOT), '--control', 'combined', + '--cpu-workers', '1'], cwd=ROOT, capture_output=True, text=True, timeout=40) + self.assertEqual(process.returncode, 0, process.stdout + process.stderr) + report = json.loads((Path(directory) / 'report.json').read_text()) + self.assertEqual(report['control'], 'combined') + self.assertFalse(report['historical_cause_established']) + result = json.loads((Path(directory) / '1' / 'load-control.json').read_text()) + self.assertEqual(result['control'], 'combined') + self.assertGreaterEqual(result['sqlite']['acquired_transactions'], 1) + self.assertTrue(result['sqlite']['thread_stopped']) + self.assertEqual(len(result['cpu']), 1) + self.assertEqual(result['cpu'][0]['returncode'], 0) + self.assertGreater(result['cpu'][0]['cpu_seconds'], 0) + self.assertGreater(result['cpu'][0]['wall_seconds'], 0) + events = [json.loads(line)['event'] for line in + (Path(directory) / '1' / 'events.jsonl').read_text().splitlines()] + self.assertIn('sqlite.lock_acquired', events) + self.assertIn('sqlite.lock_released', events) + self.assertIn('load.started', events) + self.assertIn('load.stopped', events) + state = json.loads((Path(directory) / '1' / 'state-before-cleanup.json').read_text()) + self.assertEqual(state['marker'], 'ok') + + def test_actual_cancellation_fixture_emits_boundary_and_persisted_evidence(self): + self.assertTrue(SCRIPT.is_file(), 'Windows diagnostic harness is missing') + with tempfile.TemporaryDirectory() as directory: + process = subprocess.run([sys.executable, str(SCRIPT), '--output', directory, + '--iterations', '1', '--repository', str(ROOT)], cwd=ROOT, capture_output=True, + text=True, timeout=40) + self.assertEqual(process.returncode, 0, process.stdout + process.stderr) + path = Path(directory) + report = json.loads((path / 'report.json').read_text()) + self.assertFalse(report['historical_cause_established']) + self.assertEqual(report['iterations'][0]['returncode'], 0) + events = [json.loads(line) for line in (path / '1' / 'events.jsonl').read_text().splitlines()] + names = [event['event'] for event in events] + for name in ('dispatch.call', 'dispatch.return', 'provider.call', 'cancel.call', + 'cancel.return', 'release.call', 'provider.return', 'worker.return', 'future_wait.call', + 'cleanup.call', 'cleanup.return'): + self.assertIn(name, names) + self.assertLess(names.index('provider.call'), names.index('cancel.call')) + self.assertLess(names.index('cancel.return'), names.index('release.call')) + self.assertTrue(all('monotonic_ns' in event and 'thread_id' in event for event in events)) + state = json.loads((path / '1' / 'state-before-cleanup.json').read_text()) + self.assertEqual(state['marker'], 'ok') + self.assertTrue(state['persisted_states']) + self.assertEqual(state['persisted_states'][-1]['status'], 'cancelled') + self.assertTrue((path / '1' / 'threads.log').is_file()) + self.assertEqual(report['test_timeouts_seconds'], {'provider_wait': 3, 'entry_wait': 10, 'future_wait': 5}) + + def test_slow_worker_completion_keeps_original_timeout_and_captures_pending_worker(self): + code = """ +from pathlib import Path +import faulthandler +import sys +import time +from unittest.mock import patch +from scripts.v3_windows_reliability import Trace, run_child + +original = Trace.record + +def delay_worker_completion(trace, event, **fields): + original(trace, event, **fields) + if event == 'worker.return': + time.sleep(5.3) + +with patch.object(Trace, 'record', delay_worker_completion), \ + patch.object(faulthandler, 'dump_traceback_later', side_effect=AssertionError('native watchdog forbidden')), \ + patch.object(faulthandler, 'cancel_dump_traceback_later', side_effect=AssertionError('native watchdog forbidden')): + result = run_child(Path(sys.argv[1])) +print('native watchdog unused', flush=True) +raise SystemExit(result) +""" + with tempfile.TemporaryDirectory() as directory: + for iteration in range(3): + path = Path(directory) / str(iteration) + process = subprocess.run([sys.executable, '-c', code, str(path)], cwd=ROOT, + capture_output=True, text=True, timeout=20) + self.assertIn('native watchdog unused', process.stdout, process.stderr) + self.assertEqual(process.returncode, 1, process.stdout + process.stderr) + self.assertIn('TimeoutError', (path / 'unittest.log').read_text()) + self.assertIn('Timeout (0:00:04)', (path / 'threads.log').read_text()) + self.assertTrue((path / 'state-watchdog-1.json').exists()) + state = json.loads((path / 'state-before-cleanup.json').read_text()) + self.assertEqual(state['marker'], 'ok') + self.assertTrue(any(not run['future_done'] for run in state['runs'])) + self.assertEqual(state['persisted_states'][-1]['status'], 'cancelled') diff --git a/tests/test_working_plans.py b/tests/test_working_plans.py index 8e95eab4..aa9d1bd4 100644 --- a/tests/test_working_plans.py +++ b/tests/test_working_plans.py @@ -872,6 +872,57 @@ def test_migration_version_includes_working_plan_schema(self): ).fetchone()) self.assertIn("revision", {row["name"] for row in db.execute("PRAGMA table_info(working_plans)")}) + def test_native_task_selection_and_completion_suggestion_require_evidence(self): + with tempfile.TemporaryDirectory() as directory: + manager = TaskManager(EventStore(Path(directory) / "state.sqlite3"), session_id="native-selection") + created = manager.edit_plan(expected_revision=0, operations=[ + {"op": "add", "ref": "first", "task": {"description": "Inspect", "acceptance": ["Inspected"]}}, + {"op": "add", "ref": "second", "task": {"description": "Verify", "acceptance": ["Verified"], "dependencies": ["first"]}}, + ]) + first, second = created["task_ids"] + self.assertIsNone(getattr(manager, "execution_task_id", None)) + with self.assertRaises(ValueError): + manager.edit_plan(plan_id=created["plan_id"], expected_revision=1, + operations=[{"op": "start", "task": second}]) + selected = manager.edit_plan(plan_id=created["plan_id"], expected_revision=1, + operations=[{"op": "start", "task": first}]) + self.assertEqual(manager.execution_task_id, first) + self.assertEqual(selected["tasks"][0]["status"], "running") + suggested = manager.edit_plan(plan_id=created["plan_id"], expected_revision=2, + operations=[{"op": "suggest_complete", "task": first}]) + self.assertEqual(suggested["tasks"][0]["status"], "running") + manager.record_evidence(task_id=first, action="read_file", args="a.txt", result="checked", + success=True, acceptance="Inspected", verification_status="verified") + self.assertEqual(manager.tasks[0]["status"], "succeeded") + self.assertIsNone(manager.execution_task_id) + manager.edit_plan(plan_id=created["plan_id"], expected_revision=3, + operations=[{"op": "start", "task": second}]) + self.assertEqual(manager.execution_task_id, second) + + def test_completion_suggestion_storage_failure_rolls_back_plan_revision(self): + import sqlite3 + with tempfile.TemporaryDirectory() as directory: + store = EventStore(Path(directory) / "state.sqlite3") + manager = TaskManager(store, session_id="completion-atomicity") + plan = manager.edit_plan(expected_revision=0, operations=[ + {"op": "add", "ref": "inspect", "task": {"description": "Inspect", "acceptance": ["Inspected"]}}, + ]) + task_id = plan["task_ids"][0] + manager.edit_plan(plan_id=plan["plan_id"], expected_revision=1, + operations=[{"op": "start", "task": task_id}]) + manager.record_evidence(task_id=task_id, action="read_file", args="a", result="checked", + success=True, acceptance="Inspected", verification_status="verified") + with store.database.connection() as db: + db.execute("CREATE TRIGGER fail_completion BEFORE UPDATE OF status ON tasks " + "WHEN NEW.status='succeeded' BEGIN SELECT RAISE(ABORT, 'blocked completion'); END") + with self.assertRaises(sqlite3.IntegrityError): + manager.edit_plan(plan_id=plan["plan_id"], expected_revision=2, + operations=[{"op": "suggest_complete", "task": task_id}]) + self.assertEqual(manager.working_plan.revision, 2) + self.assertEqual(store.get_working_plan(user_id="local", workspace_id="default", + session_id="completion-atomicity")["revision"], 2) + self.assertEqual(manager.tasks[0]["status"], "running") + def test_schema_four_working_plan_is_upgraded_without_losing_tasks(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "state.sqlite3" @@ -891,6 +942,36 @@ def test_schema_four_working_plan_is_upgraded_without_losing_tasks(self): self.assertEqual(recovered.tasks[0]["description"], "Preserve me") self.assertEqual(migrated.SCHEMA_VERSION, 6) + def test_strategy_execution_is_validated_against_approved_scope_at_tool_boundary(self): + with tempfile.TemporaryDirectory() as directory: + app = build_application(memory=build_memory(Path(directory) / "state.sqlite3", index=None), + tools=ToolAdapters(directory)) + self.addCleanup(app.close) + runtime = app.runtime + controller = runtime._interaction_controller + proposal = controller.propose_plan({ + "title": "Page", "summary": "Create page", "bounds": { + "workspace_paths": ["site"], "capabilities": ["read", "write"], + "effects": ["read_only", "filesystem"], "risk_ceiling": "normal", + }, "steps": [{"id": "page", "title": "Create page", "description": "Write site/page.txt", + "acceptance": ["Page ready"]}], + }) + accepted, _ = controller.accept_plan(runtime.tasks, plan_id=proposal["plan_id"], version=1) + token = issue_capability_token("test", {"read", "write"}) + anchor = accepted["working_task_ids"][0] + for path, success in (("other.txt", False), ("site/page.txt", True)): + result = runtime.tool_registry.invoke_result("update_plan", { + "plan_id": runtime.tasks.working_plan.id, "expected_revision": 1, + "operations": [{"op": "add", "anchor": anchor, "ref": "inspect", "task": { + "description": "Inspect page", "acceptance": ["Input checked"], + "execution": {"action": "read_file", "arguments": {"path": path}}, + }}], + }, token) + self.assertEqual(result.success, success, result.error) + child = next(task for task in runtime.tasks.tasks if task["id"] != anchor) + self.assertEqual(child["checkpoint"]["strategy_anchor"], anchor) + self.assertEqual(child["checkpoint"]["action"], "read_file") + def test_update_plan_is_a_typed_registry_tool_with_object_arguments(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "state.sqlite3" diff --git a/website/assets/install-platform.js b/website/assets/install-platform.js index 9e2db3b5..5487fce4 100644 --- a/website/assets/install-platform.js +++ b/website/assets/install-platform.js @@ -8,6 +8,6 @@ export function detectInstallPlatform(hints = {}) { export function installCommand(platform) { return platform === "windows" - ? "irm https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v3.0.0/install.ps1 | iex" - : "curl -fsSL https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v3.0.0/install.sh | sh"; + ? "irm https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v2.0.8/install.ps1 | iex" + : "curl -fsSL https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v2.0.8/install.sh | sh"; } diff --git a/website/build_site.py b/website/build_site.py index 17c3859e..45bf2671 100644 --- a/website/build_site.py +++ b/website/build_site.py @@ -17,9 +17,10 @@ "en": {"home": "/", "features": "/features/", "install": "/install/", "faq": "/faq/", "docs": "/docs/"}, "zh-cn": {"home": "/zh-cn/", "features": "/zh-cn/features/", "install": "/zh-cn/install/", "faq": "/zh-cn/faq/", "docs": "/zh-cn/docs/"}, } +# Keep public downloads on the published stable release until V3 publication. INSTALL_COMMANDS = { - "unix": "curl -fsSL https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v3.0.0/install.sh | sh", - "windows": "irm https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v3.0.0/install.ps1 | iex", + "unix": "curl -fsSL https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v2.0.8/install.sh | sh", + "windows": "irm https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v2.0.8/install.ps1 | iex", } PAGES = { @@ -87,10 +88,10 @@ "body": """

RECOMMENDED · macOS OR LINUX

Install from a terminal.

Run the installer, then launch OpenKyrozen. The first run guides you through provider setup.

-
Shell
curl -fsSL https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v3.0.0/install.sh | sh
+  
Shell
curl -fsSL https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v2.0.8/install.sh | sh
 kyrozen

   
WINDOWS POWERSHELL
Use the Windows installer.

-  
PowerShell
irm https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v3.0.0/install.ps1 | iex
+  
PowerShell
irm https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v2.0.8/install.ps1 | iex
 kyrozen
@@ -102,7 +103,7 @@

OTHER PATHS

Prefer to work from source?

A

Clone the repository

For development, install the project dependencies and run the agent from the checkout.

git clone https://github.com/EvanProgramming/OpenKyrozen.git
cd OpenKyrozen
make install
make run
Source repository
-
B

Install the verified wheel

Use the project’s published release wheel with uv and Python 3.12.

uv tool install --python 3.12 --force --with fastapi --with uvicorn https://github.com/EvanProgramming/OpenKyrozen/releases/download/v3.0.0/openkyrozen-3.0.0-py3-none-any.whl
Browse releases
+
B

Install the verified wheel

Use the project’s published release wheel with uv and Python 3.12.

uv tool install --python 3.12 --force --with fastapi --with uvicorn https://github.com/EvanProgramming/OpenKyrozen/releases/download/v2.0.8/openkyrozen-2.0.8-py3-none-any.whl
Browse releases

NEXT

See what OpenKyrozen can do.

Learn about the tools, memory, learning system, and safety model.

Explore capabilities
@@ -193,10 +194,10 @@ "body": """

推荐 · macOS 或 Linux

在终端中安装。

运行安装程序后启动 OpenKyrozen。首次启动会引导你配置模型服务商。

-
Shell
curl -fsSL https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v3.0.0/install.sh | sh
+  
Shell
curl -fsSL https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v2.0.8/install.sh | sh
 kyrozen

   
WINDOWS POWERSHELL
使用 Windows 安装程序。

-  
PowerShell
irm https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v3.0.0/install.ps1 | iex
+  
PowerShell
irm https://raw.githubusercontent.com/EvanProgramming/OpenKyrozen/v2.0.8/install.ps1 | iex
 kyrozen
@@ -208,7 +209,7 @@

其他方式

更喜欢从源码运行？

A

克隆代码仓库

用于开发的方式：安装项目依赖并从源码目录运行智能体。

git clone https://github.com/EvanProgramming/OpenKyrozen.git
cd OpenKyrozen
make install
make run
源代码仓库
-
B

安装已验证的 wheel

使用 uv 和 Python 3.12 安装项目发布的 wheel。

uv tool install --python 3.12 --force --with fastapi --with uvicorn https://github.com/EvanProgramming/OpenKyrozen/releases/download/v3.0.0/openkyrozen-3.0.0-py3-none-any.whl
查看发布版本
+
B

安装已验证的 wheel

使用 uv 和 Python 3.12 安装项目发布的 wheel。

uv tool install --python 3.12 --force --with fastapi --with uvicorn https://github.com/EvanProgramming/OpenKyrozen/releases/download/v2.0.8/openkyrozen-2.0.8-py3-none-any.whl
查看发布版本

下一步

了解 OpenKyrozen 能做什么。

查看工具、记忆、学习系统与安全模型。

浏览产品能力
@@ -361,6 +362,7 @@ def document(lang, slug, page): {('
OPENKYROZEN / TUIPROJECT CONTEXT
' + escape(image_alt, quote=True) + '
' + ('Terminal-native · Evidence-backed · Local-first' if lang == 'en' else '终端原生 · 证据驱动 · 本地优先') + '
') if slug == 'home' else ''} {page['body']} + {('

V3 is not yet released. Public downloads install stable V2.0.8.

' if lang == 'en' else '

V3 尚未发布。公开下载仍安装稳定版 V2.0.8。

') if slug in {'home', 'install'} else ''}