diff --git a/docs/architecture/rfcs/typescript-control-plane-migration-v0.md b/docs/architecture/rfcs/typescript-control-plane-migration-v0.md index 7ff5f4b33c..c82b3b03cb 100644 --- a/docs/architecture/rfcs/typescript-control-plane-migration-v0.md +++ b/docs/architecture/rfcs/typescript-control-plane-migration-v0.md @@ -2090,6 +2090,16 @@ daemon command. CLI and App surfaces consume the same lifecycle projection (`running`, `stopped`, or `unavailable`) and stable diagnostic code. Raw stderr, tokens, local paths, and private runtime metadata are not projected. +Node launch observation remains in the existing Python transport adapter; it is +not a second control-plane decision owner. Startup and doctor share its bounded +probe and distinguish unknown compatibility after a timeout or launch failure +from a parsed unsupported version. A later cold-start failure during deep doctor +must retain the same diagnosis and recovery, even after its initial probe +succeeded; recovery is verified by retrying the actual request. The +[host diagnostic contract](../../reference/protocols/host-integration-surface-v0.md#managed-node-startup-diagnostics) +defines the budgets, stable codes and recovery. This repairs readiness diagnosis; +sustained runtime and whole-task performance require separate evidence. + The runtime fingerprint includes every executed TS module and contract. An upgrade starts a runtime for the new fingerprint; an old process can finish in-flight work and exits on idle. Requests carry stable effect identities, so a diff --git a/docs/reference/protocols/host-integration-surface-v0.md b/docs/reference/protocols/host-integration-surface-v0.md index 0fd1c322db..c282bd4af8 100644 --- a/docs/reference/protocols/host-integration-surface-v0.md +++ b/docs/reference/protocols/host-integration-surface-v0.md @@ -209,6 +209,39 @@ Optional projections such as `task_graph_projection_v0`, inputs to a host integration. They do not add graph write authority, launch workers, change quota gates, or create a new source of truth. +### Managed Node startup diagnostics + +The Effect transport's version probe and process readiness each have a bounded +15-second budget; the version probe previously allowed only 2 seconds. This +changes cold startup tolerance and the worst-case failed-probe wait, not work +Turn deadlines or the Node.js 22.22.3 compatibility floor. Warm requests reuse +the serving runtime without a new version probe. `loopx doctor` still observes +the current launcher independently, so replacing PATH is visible immediately. + +Startup, doctor and guided start-goal share the same probe diagnosis: + +A successful initial doctor check does not guarantee a later cold-start probe +will succeed. Deep doctor preserves the later probe's diagnostic and recovery +below, with host-permission recovery taking precedence. Other runtime startup +failures retain their existing recovery advice. + +| Diagnostic code | Observation | Recovery | +| --- | --- | --- | +| `node_unavailable` | Node missing or a parsed version below the floor | Install or activate a qualified Node on PATH | +| `node_probe_timeout` | No completed version observation within the budget | Check host load and the launcher; compatibility remains unknown | +| `node_probe_launch_failed` | The launcher could not be executed | Repair the launcher on PATH | +| `node_probe_exit_failed` | The probe exited unsuccessfully | Repair the launcher on PATH | +| `node_probe_invalid_version` | Successful exit without a valid version | Verify the launcher's version output | +| `runtime_host_permission_denied` | Host denied the probe | Retry through host-approved runtime access | + +After recovery, run `loopx doctor --deep` and retry the original request. Failed +probes do not dispatch semantic work, grant capability authority, rewrite Goal +state or publish raw output, stderr or executable paths. Timeout and cancellation +kill and reap the probe child; cancellation propagates to the caller. +The request transport does not automatically repeat a failed Node probe before +the caller can recover. Existing retries for other startup and safe transport +failures remain in place. + ## Controlled Writes Writes must be CLI-equivalent, idempotent where possible, and fail closed when diff --git a/loopx/cli_commands/start_goal.py b/loopx/cli_commands/start_goal.py index bac4899ee2..028cf6b2b6 100644 --- a/loopx/cli_commands/start_goal.py +++ b/loopx/cli_commands/start_goal.py @@ -16,6 +16,7 @@ EffectRuntimeStartupError, MINIMUM_NODE_VERSION_TEXT, ) +from ..control_plane.runtime.node_probe import node_probe_remediation from ._host_thread import current_host_thread_id PrintPayload = Callable[ @@ -32,11 +33,6 @@ _FINE_GRAINED_PREFIX = re.compile(r"\A--fine-grained(?P(?:\s[\s\S]*)?)\Z") _EFFECT_RUNTIME_STARTUP_REMEDIATION_BY_CODE = { - "node_unavailable": ( - f"Install or activate Node.js {MINIMUM_NODE_VERSION_TEXT} or newer " - "on PATH, then run `loopx doctor --deep` and retry " - "`loopx start-goal --guided`." - ), "startup_lock_timeout": ( "Another LoopX TypeScript control-plane runtime may be starting. Run " "`loopx doctor --deep`, wait for the active startup to settle, and retry " @@ -69,6 +65,9 @@ def _effect_runtime_startup_recommended_action( diagnostic_code: str, message: str, ) -> str: + node_action = node_probe_remediation(diagnostic_code) + if node_action is not None: + return f"{node_action} Then retry `loopx start-goal --guided`." if diagnostic_code == "invalid_idle_timeout": # The TypeScript runtime owns the idle-timeout validation rules and # publishes them as the typed startup message, so the projection must diff --git a/loopx/control_plane/effect_runtime.py b/loopx/control_plane/effect_runtime.py index 7b40718cd3..52a4068f12 100644 --- a/loopx/control_plane/effect_runtime.py +++ b/loopx/control_plane/effect_runtime.py @@ -3,9 +3,7 @@ import hashlib import json import os -import re import secrets -import shutil import socket import subprocess import tempfile @@ -22,6 +20,14 @@ from ..file_lock import process_is_alive from .runtime.file_reads import iter_binary_file_reads +from .runtime.node_probe import ( + HOST_PERMISSION_RECOMMENDATION, + MINIMUM_NODE_VERSION as MINIMUM_NODE_VERSION, + MINIMUM_NODE_VERSION_TEXT as MINIMUM_NODE_VERSION_TEXT, + STARTUP_READY_TIMEOUT_SECONDS as STARTUP_READY_TIMEOUT_SECONDS, + node_probe_remediation, + probe_node as _probe_node, +) from .content_digest import BARE_SHA256_PATTERN EFFECT_RUNTIME_REQUEST_SCHEMA_VERSION = "loopx_effect_runtime_request_v0" @@ -31,8 +37,6 @@ EFFECT_RUNTIME_STARTUP_ERROR_SCHEMA_VERSION = ( "loopx_effect_runtime_startup_error_v0" ) -MINIMUM_NODE_VERSION = (22, 22, 3) -MINIMUM_NODE_VERSION_TEXT = ".".join(str(part) for part in MINIMUM_NODE_VERSION) MAX_RESPONSE_BYTES = 2 * 1024 * 1024 MAX_REQUEST_BYTES = 2 * 1024 * 1024 MAX_LOCAL_SNAPSHOT_BYTES = 64 * 1024 * 1024 @@ -47,7 +51,6 @@ }) MAX_STARTUP_DIAGNOSTIC_BYTES = 8 * 1024 STARTUP_LOCK_TIMEOUT_SECONDS = 15.0 -STARTUP_READY_TIMEOUT_SECONDS = 15.0 STARTUP_POLL_SECONDS = 0.025 RUNTIME_LOCATOR_PERMISSION_RETRIES = 3 RUNTIME_RETRY_SETTLE_SECONDS = 0.25 @@ -58,7 +61,6 @@ # turns an in-flight write into an avoidable ambiguous response. CANONICAL_AUTHORITY_WRITE_TIMEOUT_SECONDS = 45.0 CANONICAL_AUTHORITY_READ_TIMEOUT_SECONDS = 15.0 -_NODE_VERSION_RE = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)(?:[-+].*)?$") _RUNTIME_SOURCE_SUFFIXES = frozenset({".json", ".ts"}) _RuntimeSourceSnapshot = tuple[tuple[str, int, int, int], ...] @@ -227,11 +229,7 @@ def __init__(self, message: str, *, diagnostic_code: str) -> None: class EffectRuntimeHostPermissionError(EffectRuntimeStartupError): """The host denied local runtime access before any request was dispatched.""" - recommended_action = ( - "retry the same registry, Goal, Agent and Turn through host-approved " - "local runtime access; do not enable optional capabilities, replace " - "authority or spend until the guard succeeds" - ) + recommended_action = HOST_PERMISSION_RECOMMENDATION def __init__(self) -> None: super().__init__( @@ -241,6 +239,10 @@ def __init__(self) -> None: ) +class EffectRuntimeNodeProbeError(EffectRuntimeStartupError): + """A pre-dispatch toolchain observation requires caller recovery.""" + + class EffectRuntimeResponseAmbiguous(EffectRuntimeStartupError): """The request may have executed even though its response was lost.""" @@ -397,38 +399,16 @@ def _runtime_server_path() -> Path: def _node_executable() -> str: - status, executable, _version = _probe_node() - if status != "ready" or executable is None: - raise EffectRuntimeStartupError( - f"LoopX Effect runtime requires Node.js {MINIMUM_NODE_VERSION_TEXT} " - "or newer", - diagnostic_code="node_unavailable", + probe = _probe_node(timeout=STARTUP_READY_TIMEOUT_SECONDS) + if not probe.ready: + if probe.diagnostic_code == "runtime_host_permission_denied": + raise EffectRuntimeHostPermissionError() + raise EffectRuntimeNodeProbeError( + probe.failure_message, + diagnostic_code=str(probe.diagnostic_code), ) - return executable - - -def _probe_node() -> tuple[str, str | None, str | None]: - executable = shutil.which("node") - if executable is None: - return "missing", None, None - try: - completed = subprocess.run( - [executable, "--version"], - check=False, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - timeout=2, - ) - except (OSError, subprocess.TimeoutExpired): - return "probe_failed", executable, None - match = _NODE_VERSION_RE.fullmatch(completed.stdout.strip()) - version = tuple(int(part) for part in match.groups()) if match else None - if completed.returncode != 0 or version is None: - return "probe_failed", executable, None - version_text = ".".join(str(part) for part in version) - if version < MINIMUM_NODE_VERSION: - return "unsupported", executable, version_text - return "ready", executable, version_text + assert probe.executable is not None + return probe.executable def _pid_is_alive(value: object) -> bool: @@ -1044,6 +1024,7 @@ def effect_runtime_request( EffectRuntimeRemoteError, EffectRuntimeResponseAmbiguous, EffectRuntimeHostPermissionError, + EffectRuntimeNodeProbeError, ): raise except EffectRuntimeStartupError as exc: @@ -1137,10 +1118,11 @@ def _sqlite_restart_recommendation(identity: Mapping[str, Any] | None) -> str | def collect_effect_runtime_readiness(*, deep: bool = False) -> dict[str, object]: """Report whether the managed TS Effect runtime can serve control-plane work.""" - status, _executable, version = _probe_node() - ready = status == "ready" + probe = _probe_node(timeout=STARTUP_READY_TIMEOUT_SECONDS) + status, version = probe.status, probe.version + ready = probe.ready runtime_state = "unavailable" - runtime_diagnostic_code: str | None = None + runtime_diagnostic_code: str | None = probe.diagnostic_code runtime_identity: dict[str, Any] | None = None host_permission_error: EffectRuntimeHostPermissionError | None = None if ready: @@ -1190,12 +1172,8 @@ def collect_effect_runtime_readiness(*, deep: bool = False) -> dict[str, object] if host_permission_error is not None else None if ready - else ( - f"Install Node.js {MINIMUM_NODE_VERSION_TEXT} or newer, then " - "rerun `loopx doctor --deep`." - if status in {"missing", "unsupported"} - else "Repair Node.js on PATH, then rerun `loopx doctor --deep`." - ) + else probe.recommended_action + or "Repair the packaged LoopX runtime, then rerun `loopx doctor --deep`." ), } if ready: @@ -1231,7 +1209,8 @@ def collect_effect_runtime_readiness(*, deep: bool = False) -> dict[str, object] "recommended_action": ( exc.recommended_action if isinstance(exc, EffectRuntimeHostPermissionError) - else "Run `loopx doctor --deep` again after any concurrent startup " + else node_probe_remediation(diagnostic_code) + or "Run `loopx doctor --deep` again after any concurrent startup " "finishes. If the same diagnostic code remains, reinstall LoopX " "and verify Node.js before retrying." ), diff --git a/loopx/control_plane/runtime/node_probe.py b/loopx/control_plane/runtime/node_probe.py new file mode 100644 index 0000000000..527fbdd1ca --- /dev/null +++ b/loopx/control_plane/runtime/node_probe.py @@ -0,0 +1,130 @@ +"""Physical Node launch observation for the existing Effect transport. + +This adapter does not select capabilities or authorize work. Startup and doctor +share its observations and public-safe remediation; neither interprets a failed +version probe as an unsupported version. +""" +from __future__ import annotations + +import re +import shutil +import subprocess +from dataclasses import dataclass +from enum import StrEnum + +MINIMUM_NODE_VERSION = (22, 22, 3) +MINIMUM_NODE_VERSION_TEXT = ".".join(str(part) for part in MINIMUM_NODE_VERSION) +STARTUP_READY_TIMEOUT_SECONDS = 15.0 +HOST_PERMISSION_RECOMMENDATION = ( + "retry the same registry, Goal, Agent and Turn through host-approved " + "local runtime access; do not enable optional capabilities, replace " + "authority or spend until the guard succeeds" +) +_VERSION_RE = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)(?:[-+].*)?$") + + +class NodeProbeOutcome(StrEnum): + """Local process facts, projected onto the existing readiness statuses.""" + + READY = "ready" + MISSING = "missing" + UNSUPPORTED = "unsupported" + TIMED_OUT = "timed_out" + PERMISSION_DENIED = "permission_denied" + LAUNCH_FAILED = "launch_failed" + EXIT_FAILED = "exit_failed" + INVALID_VERSION = "invalid_version" + + +_FAILURES = { + NodeProbeOutcome.MISSING: ( + "node_unavailable", f"LoopX Effect runtime requires Node.js {MINIMUM_NODE_VERSION_TEXT} or newer on PATH", + ), + NodeProbeOutcome.UNSUPPORTED: ( + "node_unavailable", f"LoopX Effect runtime requires Node.js {MINIMUM_NODE_VERSION_TEXT} or newer", + ), + NodeProbeOutcome.TIMED_OUT: ( + "node_probe_timeout", "Node.js version probe exceeded the bounded startup budget; compatibility is unknown", + ), + NodeProbeOutcome.PERMISSION_DENIED: ( + "runtime_host_permission_denied", "Host permission denied the Node.js version probe before request dispatch", + ), + NodeProbeOutcome.LAUNCH_FAILED: ( + "node_probe_launch_failed", "Node.js version probe could not be launched; compatibility is unknown", + ), + NodeProbeOutcome.EXIT_FAILED: ( + "node_probe_exit_failed", "Node.js version probe exited unsuccessfully; compatibility is unknown", + ), + NodeProbeOutcome.INVALID_VERSION: ( + "node_probe_invalid_version", "Node.js version probe returned no valid version; compatibility is unknown", + ), +} +_REMEDIATION = { + "node_unavailable": f"Install or activate Node.js {MINIMUM_NODE_VERSION_TEXT} or newer on PATH, then run `loopx doctor --deep`.", + "node_probe_timeout": "Check host load and the Node.js launcher on PATH, then rerun `loopx doctor --deep`. A timed-out probe does not establish version incompatibility.", + "node_probe_launch_failed": "Repair the Node.js launcher on PATH, then rerun `loopx doctor --deep`.", + "node_probe_exit_failed": "Repair the Node.js launcher on PATH, then rerun `loopx doctor --deep`.", + "node_probe_invalid_version": "Verify that the Node.js launcher on PATH returns a valid version, then rerun `loopx doctor --deep`.", + "runtime_host_permission_denied": HOST_PERMISSION_RECOMMENDATION, +} + + +def node_probe_remediation(diagnostic_code: str) -> str | None: + return _REMEDIATION.get(diagnostic_code) + + +@dataclass(frozen=True) +class NodeProbe: + outcome: NodeProbeOutcome + executable: str | None = None + version: str | None = None + + @property + def ready(self) -> bool: + return self.outcome is NodeProbeOutcome.READY + + @property + def status(self) -> str: + if self.outcome in {NodeProbeOutcome.READY, NodeProbeOutcome.MISSING, NodeProbeOutcome.UNSUPPORTED}: + return self.outcome.value + return "probe_failed" + + @property + def diagnostic_code(self) -> str | None: + failure = _FAILURES.get(self.outcome) + return failure[0] if failure else None + + @property + def failure_message(self) -> str: + return _FAILURES[self.outcome][1] + + @property + def recommended_action(self) -> str | None: + return node_probe_remediation(self.diagnostic_code or "") + + +def probe_node(*, timeout: float = STARTUP_READY_TIMEOUT_SECONDS) -> NodeProbe: + executable = shutil.which("node") + if executable is None: + return NodeProbe(NodeProbeOutcome.MISSING) + try: + # run kills and waits for the probe child on timeout/cancellation. No + # retry, version cache, shell or semantic request is involved. + completed = subprocess.run( + [executable, "--version"], check=False, capture_output=True, + text=True, encoding="utf-8", errors="replace", timeout=timeout, + ) + except subprocess.TimeoutExpired: + return NodeProbe(NodeProbeOutcome.TIMED_OUT, executable) + except PermissionError: + return NodeProbe(NodeProbeOutcome.PERMISSION_DENIED, executable) + except OSError: + return NodeProbe(NodeProbeOutcome.LAUNCH_FAILED, executable) + if completed.returncode != 0: + return NodeProbe(NodeProbeOutcome.EXIT_FAILED, executable) + match = _VERSION_RE.fullmatch(completed.stdout.strip()) + if match is None: + return NodeProbe(NodeProbeOutcome.INVALID_VERSION, executable) + version = tuple(int(part) for part in match.groups()) + outcome = NodeProbeOutcome.UNSUPPORTED if version < MINIMUM_NODE_VERSION else NodeProbeOutcome.READY + return NodeProbe(outcome, executable, ".".join(str(part) for part in version)) diff --git a/tests/control_plane/test_effect_runtime_host_permission.py b/tests/control_plane/test_effect_runtime_host_permission.py index 0f04345907..b302780143 100644 --- a/tests/control_plane/test_effect_runtime_host_permission.py +++ b/tests/control_plane/test_effect_runtime_host_permission.py @@ -11,6 +11,7 @@ from loopx.cli_commands.quota_failure_report import quota_failure_payload from loopx.cli import build_parser, main from loopx.control_plane import effect_runtime +from loopx.control_plane.runtime.node_probe import NodeProbe, NodeProbeOutcome def _locator(tmp_path: Path, monkeypatch) -> tuple[Path, dict[str, object]]: @@ -240,7 +241,7 @@ def test_readiness_preserves_locator_permission_diagnostic_and_recovery( tmp_path: Path, monkeypatch, deep: bool, ) -> None: monkeypatch.setattr(effect_runtime, "_runtime_dir", lambda: tmp_path) - monkeypatch.setattr(effect_runtime, "_probe_node", lambda: ("ready", "node", "24.0.0")) + monkeypatch.setattr(effect_runtime, "_probe_node", lambda **_: NodeProbe(NodeProbeOutcome.READY, "node", "24.0.0")) locator = effect_runtime._runtime_info_path(effect_runtime._runtime_fingerprint()) original = Path.read_text @@ -322,7 +323,7 @@ def test_deep_readiness_connection_denial_keeps_host_recovery( tmp_path: Path, monkeypatch, ) -> None: _locator(tmp_path, monkeypatch) - monkeypatch.setattr(effect_runtime, "_probe_node", lambda: ("ready", "node", "24.0.0")) + monkeypatch.setattr(effect_runtime, "_probe_node", lambda **_: NodeProbe(NodeProbeOutcome.READY, "node", "24.0.0")) def denied(*_args, **_kwargs): raise PermissionError(errno.EPERM, "private connection details") diff --git a/tests/control_plane/test_effect_runtime_node_probe.py b/tests/control_plane/test_effect_runtime_node_probe.py new file mode 100644 index 0000000000..87884f1473 --- /dev/null +++ b/tests/control_plane/test_effect_runtime_node_probe.py @@ -0,0 +1,276 @@ +"""A failed version observation is not evidence of an unsupported runtime.""" +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from loopx.cli_commands.start_goal import _effect_runtime_startup_failure_payload +from loopx.control_plane import effect_runtime +from loopx.control_plane.runtime import node_probe + + +@pytest.mark.parametrize( + ("failure", "code", "action"), + [ + (subprocess.TimeoutExpired("node", 15), "node_probe_timeout", "host load"), + (OSError("private executable path"), "node_probe_launch_failed", "launcher"), + (PermissionError("private executable path"), "runtime_host_permission_denied", "host-approved"), + (SimpleNamespace(returncode=1, stdout="v24.0.0\n"), "node_probe_exit_failed", "launcher"), + (SimpleNamespace(returncode=0, stdout="private malformed output"), "node_probe_invalid_version", "launcher"), + ], +) +def test_node_failure_survives_startup_and_readiness_projection(monkeypatch, failure, code, action): + monkeypatch.setattr(node_probe.shutil, "which", lambda _: "node") + + def run(*_args, **_kwargs): + if isinstance(failure, Exception): + raise failure + return failure + + monkeypatch.setattr(effect_runtime.subprocess, "run", run) + with pytest.raises(effect_runtime.EffectRuntimeStartupError) as raised: + effect_runtime._node_executable() + assert raised.value.diagnostic_code == code + assert "requires Node.js" not in str(raised.value) + assert "private" not in str(raised.value) + readiness = effect_runtime.collect_effect_runtime_readiness(deep=True) + assert readiness["status"] == "probe_failed" + assert readiness["semantic_probe"] == "not_run" + assert readiness["runtime_lifecycle"]["diagnostic_code"] == code + assert action in str(readiness["recommended_action"]) + assert "private" not in str(readiness) + guided = _effect_runtime_startup_failure_payload(raised.value) + assert guided["diagnostic_code"] == code + assert action in str(guided["recommended_action"]) + + + +@pytest.mark.parametrize( + ("outcome", "code", "action"), + [ + (node_probe.NodeProbeOutcome.TIMED_OUT, "node_probe_timeout", "host load"), + (node_probe.NodeProbeOutcome.LAUNCH_FAILED, "node_probe_launch_failed", "Repair the Node.js launcher"), + (node_probe.NodeProbeOutcome.EXIT_FAILED, "node_probe_exit_failed", "Repair the Node.js launcher"), + (node_probe.NodeProbeOutcome.INVALID_VERSION, "node_probe_invalid_version", "returns a valid version"), + (node_probe.NodeProbeOutcome.UNSUPPORTED, "node_unavailable", "Install or activate Node.js 22.22.3"), + (node_probe.NodeProbeOutcome.PERMISSION_DENIED, "runtime_host_permission_denied", "host-approved"), + ], +) +def test_deep_readiness_preserves_later_node_probe_recovery(tmp_path, monkeypatch, outcome, code, action): + observations = iter([ + node_probe.NodeProbe(node_probe.NodeProbeOutcome.READY, "node", "24.21.0"), + node_probe.NodeProbe(outcome, "node"), + ]) + monkeypatch.setattr(effect_runtime, "_probe_node", lambda **_: next(observations)) + monkeypatch.setattr(effect_runtime, "_runtime_dir", lambda: tmp_path) + monkeypatch.setattr(effect_runtime, "_runtime_fingerprint", lambda: "fixture") + monkeypatch.setattr(effect_runtime, "_request_with_info", lambda **_: pytest.fail("no dispatch")) + result = effect_runtime.collect_effect_runtime_readiness(deep=True) + assert result["ready"] is False + assert result["status"] == "probe_failed" + assert result["semantic_probe"] == "failed" + assert result["detected_node_version"] == "24.21.0" + assert result["runtime_lifecycle"]["diagnostic_code"] == code + assert action in result["recommended_action"] + assert not list(tmp_path.glob("start-*.lock")) + + +def test_deep_readiness_keeps_non_node_runtime_recovery(tmp_path, monkeypatch): + monkeypatch.setattr(effect_runtime, "_probe_node", lambda **_: node_probe.NodeProbe( + node_probe.NodeProbeOutcome.READY, "node", "24.21.0", + )) + monkeypatch.setattr(effect_runtime, "_runtime_dir", lambda: tmp_path) + monkeypatch.setattr(effect_runtime, "_runtime_fingerprint", lambda: "fixture") + + def fail(*_args, **_kwargs): + raise effect_runtime.EffectRuntimeStartupError("unavailable", diagnostic_code="runtime_start_failed") + + monkeypatch.setattr(effect_runtime, "effect_runtime_result", fail) + result = effect_runtime.collect_effect_runtime_readiness(deep=True) + assert result["runtime_lifecycle"]["diagnostic_code"] == "runtime_start_failed" + assert "concurrent startup" in result["recommended_action"] + assert "reinstall LoopX" in result["recommended_action"] + + +def test_node_probe_uses_the_existing_bounded_startup_budget(monkeypatch): + observed = [] + monkeypatch.setattr(node_probe.shutil, "which", lambda _: "node") + + def run(*_args, **kwargs): + observed.append(kwargs["timeout"]) + return SimpleNamespace(returncode=0, stdout="v22.22.3\n") + + monkeypatch.setattr(effect_runtime.subprocess, "run", run) + assert effect_runtime._node_executable() == "node" + assert observed == [effect_runtime.STARTUP_READY_TIMEOUT_SECONDS] + + +@pytest.mark.parametrize("failure", [ + subprocess.TimeoutExpired("node", 15), + SimpleNamespace(returncode=1, stdout=""), + SimpleNamespace(returncode=0, stdout="invalid"), + SimpleNamespace(returncode=0, stdout="v20.0.0"), +]) +def test_request_does_not_repeat_a_failed_pre_dispatch_node_probe(tmp_path, monkeypatch, failure): + calls = [] + monkeypatch.setattr(node_probe.shutil, "which", lambda _: "node") + monkeypatch.setattr(effect_runtime, "_runtime_dir", lambda: tmp_path) + monkeypatch.setattr(effect_runtime, "_runtime_fingerprint", lambda: "fixture") + + def run(*_args, **_kwargs): + calls.append("probe") + if isinstance(failure, Exception): + raise failure + return failure + + monkeypatch.setattr(node_probe.subprocess, "run", run) + monkeypatch.setattr(effect_runtime, "_request_with_info", lambda **_: pytest.fail("no dispatch")) + with pytest.raises(effect_runtime.EffectRuntimeNodeProbeError): + effect_runtime.effect_runtime_result("runtime.ping", {}) + assert calls == ["probe"] + assert not list(tmp_path.glob("start-*.lock")) + + +def test_probe_cancellation_is_not_converted_into_version_failure(monkeypatch): + monkeypatch.setattr(node_probe.shutil, "which", lambda _: "node") + + def interrupt(*_args, **_kwargs): + raise KeyboardInterrupt + + monkeypatch.setattr(effect_runtime.subprocess, "run", interrupt) + with pytest.raises(KeyboardInterrupt): + effect_runtime._node_executable() + + +def _launcher(tmp_path: Path, body: str) -> Path: + launcher = tmp_path / "node" + launcher.write_text(f"#!{sys.executable}\n" + body, encoding="utf-8") + launcher.chmod(0o700) + return launcher + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX executable launcher fixture") +def test_slow_compatible_node_serves_real_requests_and_deep_readiness(tmp_path, monkeypatch): + actual_node = shutil.which("node") + assert actual_node is not None + launcher = _launcher(tmp_path, ( + "import os, sys, time\ntime.sleep(2.6)\n" + f"os.execv({actual_node!r}, [{actual_node!r}, *sys.argv[1:]])\n" + )) + monkeypatch.setattr(node_probe.shutil, "which", lambda _: str(launcher)) + monkeypatch.setattr(effect_runtime, "_runtime_dir", lambda: tmp_path / "runtime") + for name in ("TMPDIR", "TEMP", "TMP"): + monkeypatch.setenv(name, str(tmp_path)) + monkeypatch.setenv("LOOPX_EFFECT_RUNTIME_IDLE_MS", "60000") + started = False + try: + ping = effect_runtime.effect_runtime_result("runtime.ping", {}) + started = True + assert ping["ready"] is True + assert effect_runtime.effect_runtime_result("runtime.ping", {})["pid"] == ping["pid"] + readiness = effect_runtime.collect_effect_runtime_readiness(deep=True) + assert readiness["ready"] is True + assert readiness["semantic_probe"] == "passed" + assert readiness["runtime_lifecycle"]["state"] == "running" + assert readiness["runtime_lifecycle"]["diagnostic_code"] is None + assert readiness["runtime_identity"]["sqlite_authority_qualified"] is True + finally: + stopped = effect_runtime.restart_effect_runtime() + if started: + assert stopped["stopped"] is True + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX executable launcher fixture") +def test_timed_out_probe_reaps_child_and_can_recover(tmp_path, monkeypatch): + actual_node = shutil.which("node") + assert actual_node is not None + pid_file = tmp_path / "pid" + launcher = _launcher(tmp_path, ( + "import os, time\nfrom pathlib import Path\n" + f"Path({str(pid_file)!r}).write_text(str(os.getpid()))\ntime.sleep(30)\n" + )) + monkeypatch.setattr(node_probe.shutil, "which", lambda _: str(launcher)) + probe = node_probe.probe_node(timeout=5) + assert probe.diagnostic_code == "node_probe_timeout" + assert pid_file.exists() + with pytest.raises(ProcessLookupError): + os.kill(int(pid_file.read_text()), 0) + _launcher(tmp_path, f"import os, sys\nos.execv({actual_node!r}, [{actual_node!r}, *sys.argv[1:]])\n") + assert node_probe.probe_node().ready + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX executable launcher fixture") +def test_cancelling_real_startup_reaps_probe_and_releases_lock(tmp_path): + pid_file = tmp_path / "pid" + launcher = _launcher(tmp_path, ( + "import os, time\nfrom pathlib import Path\n" + f"Path({str(pid_file)!r}).write_text(str(os.getpid()))\ntime.sleep(30)\n" + )) + program = ( + "import os, signal, threading, time\n" + "from pathlib import Path\n" + "from loopx.control_plane import effect_runtime\n" + "from loopx.control_plane.runtime import node_probe\n" + f"node_probe.shutil.which = lambda _: {str(launcher)!r}\n" + f"effect_runtime._runtime_dir = lambda: Path({str(tmp_path / 'runtime')!r})\n" + "def cancel():\n" + f" while not Path({str(pid_file)!r}).exists(): time.sleep(0.01)\n" + " os.kill(os.getpid(), signal.SIGINT)\n" + "threading.Thread(target=cancel, daemon=True).start()\n" + "try: effect_runtime.effect_runtime_result('runtime.ping', {})\n" + "except KeyboardInterrupt: pass\n" + "else: raise AssertionError('cancellation swallowed')\n" + ) + subprocess.run([sys.executable, "-c", program], check=True, timeout=10) + with pytest.raises(ProcessLookupError): + os.kill(int(pid_file.read_text()), 0) + assert not list((tmp_path / "runtime").glob("start-*.lock")) + assert not list((tmp_path / "runtime").glob("runtime-*.json")) + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX executable launcher fixture") +def test_deep_readiness_second_probe_timeout_recovers_real_request(tmp_path, monkeypatch): + actual_node = shutil.which("node") + assert actual_node is not None + counter, pid_file = tmp_path / "probe-count", tmp_path / "probe-pid" + launcher = _launcher(tmp_path, ( + "import os, sys, time\nfrom pathlib import Path\n" + "if sys.argv[1:] == ['--version']:\n" + f" counter = Path({str(counter)!r})\n" + " count = int(counter.read_text()) + 1 if counter.exists() else 1\n" + " counter.write_text(str(count))\n" + " if count > 1:\n" + f" Path({str(pid_file)!r}).write_text(str(os.getpid()))\n" + " time.sleep(30)\n" + f"os.execv({actual_node!r}, [{actual_node!r}, *sys.argv[1:]])\n" + )) + monkeypatch.setattr(node_probe.shutil, "which", lambda _: str(launcher)) + monkeypatch.setattr(effect_runtime, "_runtime_dir", lambda: tmp_path / "runtime") + for name in ("TMPDIR", "TEMP", "TMP"): + monkeypatch.setenv(name, str(tmp_path)) + monkeypatch.setenv("LOOPX_EFFECT_RUNTIME_IDLE_MS", "60000") + result = effect_runtime.collect_effect_runtime_readiness(deep=True) + assert counter.read_text() == "2" + assert result["ready"] is False and result["semantic_probe"] == "failed" + assert result["runtime_lifecycle"]["diagnostic_code"] == "node_probe_timeout" + assert "host load" in result["recommended_action"] + assert "reinstall" not in result["recommended_action"] + with pytest.raises(ProcessLookupError): + os.kill(int(pid_file.read_text()), 0) + assert not list((tmp_path / "runtime").glob("start-*.lock")) + assert not list((tmp_path / "runtime").glob("runtime-*.json")) + _launcher(tmp_path, f"import os, sys\nos.execv({actual_node!r}, [{actual_node!r}, *sys.argv[1:]])\n") + try: + ping = effect_runtime.effect_runtime_result("runtime.ping", {}) + assert ping["ready"] is True + recovered = effect_runtime.collect_effect_runtime_readiness(deep=True) + assert recovered["ready"] is True and recovered["semantic_probe"] == "passed" + assert recovered["runtime_identity"]["sqlite_authority_qualified"] is True + finally: + assert effect_runtime.restart_effect_runtime()["stopped"] is True diff --git a/tests/control_plane/test_turn_journal_runtime_readiness.py b/tests/control_plane/test_turn_journal_runtime_readiness.py index 9877af979e..d667f43fb7 100644 --- a/tests/control_plane/test_turn_journal_runtime_readiness.py +++ b/tests/control_plane/test_turn_journal_runtime_readiness.py @@ -7,6 +7,7 @@ import pytest from loopx.control_plane import effect_runtime +from loopx.control_plane.runtime import node_probe from loopx.doctor import collect_doctor, render_doctor_markdown @@ -187,7 +188,7 @@ def test_runtime_source_churn_has_a_stable_readiness_diagnostic( monkeypatch: pytest.MonkeyPatch, ) -> None: scans = _install_persistent_stat_read_churn(tmp_path, monkeypatch) - monkeypatch.setattr(effect_runtime.shutil, "which", lambda _name: "node") + monkeypatch.setattr(node_probe.shutil, "which", lambda _name: "node") monkeypatch.setattr( effect_runtime.subprocess, "run", @@ -228,7 +229,7 @@ def test_runtime_request_source_churn_raises_a_stable_startup_diagnostic( def test_missing_node_blocks_the_typescript_control_plane_and_is_actionable( monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr(effect_runtime.shutil, "which", lambda _name: None) + monkeypatch.setattr(node_probe.shutil, "which", lambda _name: None) result = effect_runtime.collect_effect_runtime_readiness() @@ -244,7 +245,7 @@ def test_missing_node_request_raises_startup_diagnostic( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(effect_runtime, "_runtime_dir", lambda: tmp_path) - monkeypatch.setattr(effect_runtime.shutil, "which", lambda _name: None) + monkeypatch.setattr(node_probe.shutil, "which", lambda _name: None) with pytest.raises(effect_runtime.EffectRuntimeStartupError) as error: effect_runtime.effect_runtime_result("runtime.ping", {}) @@ -258,7 +259,7 @@ def test_old_node_is_reported_without_running_semantic_probe( version: str, monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr(effect_runtime.shutil, "which", lambda _name: "node") + monkeypatch.setattr(node_probe.shutil, "which", lambda _name: "node") monkeypatch.setattr( effect_runtime.subprocess, "run", @@ -277,7 +278,7 @@ def test_current_node_standard_probe_does_not_execute_rule( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(effect_runtime, "_runtime_dir", lambda: tmp_path) - monkeypatch.setattr(effect_runtime.shutil, "which", lambda _name: "node") + monkeypatch.setattr(node_probe.shutil, "which", lambda _name: "node") monkeypatch.setattr( effect_runtime.subprocess, "run", @@ -304,7 +305,7 @@ def test_deep_probe_executes_packaged_semantics( monkeypatch: pytest.MonkeyPatch, ) -> None: calls: list[tuple[str, dict[str, Any]]] = [] - monkeypatch.setattr(effect_runtime.shutil, "which", lambda _name: "node") + monkeypatch.setattr(node_probe.shutil, "which", lambda _name: "node") monkeypatch.setattr( effect_runtime.subprocess, "run", @@ -338,7 +339,7 @@ def request(method: str, params: dict[str, Any]) -> dict[str, object]: def test_deep_probe_failure_is_public_safe_and_actionable( monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr(effect_runtime.shutil, "which", lambda _name: "node") + monkeypatch.setattr(node_probe.shutil, "which", lambda _name: "node") monkeypatch.setattr( effect_runtime.subprocess, "run", diff --git a/tests/test_kunluncode_goal_mode.py b/tests/test_kunluncode_goal_mode.py index fec72b0ccb..576c0fea84 100644 --- a/tests/test_kunluncode_goal_mode.py +++ b/tests/test_kunluncode_goal_mode.py @@ -97,10 +97,10 @@ def fake_cli(command, **kwargs): return subprocess.CompletedProcess(command, 0, "not-json", "") mock_goal_cli(fake_cli) - status, executable, version = effect_runtime._probe_node() - assert status == "ready" - assert executable is not None - assert version is not None + probe = effect_runtime._probe_node() + assert probe.ready + assert probe.executable is not None + assert probe.version is not None def test_claude_and_kunluncode_bind_distinct_agents(tmp_path: Path) -> None: