From 7794203fcb439616a77d62e365f6d3f5228155d4 Mon Sep 17 00:00:00 2001 From: ChakraFusion <112588066+ChakraFusion@users.noreply.github.com> Date: Tue, 6 Oct 2026 07:59:13 +0200 Subject: [PATCH] Planned llama restarts opt-in; failsafe watchdog; one Hawk and one llama-server only Incident 2026-10-05: Hawk restarted llama-server on a fixed cadence (every 3.5 h) without any switch, and after Hawk itself died nothing was left to bring either back. - planned restarts only when llama_kill_degraded is on and the average decode speed is below llama_kill_tps; no time cadence by default (0); a dead server is always started again - hawk ensure / hawk install-watchdog: Task Scheduler (pythonw, every 2 min) or cron starts a stopped Hawk and a llama-server down for two checks - OS file locks: one monitor, one dashboard, one llama-server start at a time (the alive check is repeated under the lock) Co-Authored-By: Claude Opus 5.5 --- .gitignore | 3 ++ README.md | 27 ++++++++-- opencode_hawk/cli.py | 97 ++++++++++++++++++++++++++++++++++ opencode_hawk/coordinator.py | 33 ++++++++++++ opencode_hawk/dashboard.py | 3 ++ opencode_hawk/llama_restart.py | 29 ++++++++-- opencode_hawk/probes.py | 12 +++++ tests/selftest_coordinator.py | 64 ++++++++++++++++++++++ tests/selftest_dashboard.py | 15 ++++++ 9 files changed, 273 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index defcea8..a2fbcfe 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,6 @@ __pycache__/ build/ dist/ *.egg-info/ + +# Single-instance locks (monitor, dashboard, llama-start) +*.lock diff --git a/README.md b/README.md index e5a79b6..abf8c4f 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,8 @@ Update with `pipx upgrade opencode-hawk`. | `hawk dashboard` | The web dashboard only (`--port`, `--open`) | | `hawk install-task` | Print the `schtasks` (Windows) or crontab (Linux) entry for `hawk poll` | | `hawk install-plugin` | Install the optional llama-restart plugin into OpenCode (`--remove` uninstalls) | +| `hawk ensure` | Failsafe: start Hawk if it is not running; start llama-server if it stays down | +| `hawk install-watchdog` | Run `hawk ensure` every 2 minutes via Task Scheduler (no window) or cron (`--remove`) | | `hawk where` | Show the folders Hawk uses | ### From a source checkout @@ -172,13 +174,28 @@ hawk install-plugin It installs `local-hawk-llama-restart.ts` into OpenCode's global plugin folder (`~/.config/opencode/plugins/`; restart OpenCode once to load it). After a -compaction it asks the Hawk dashboard whether a restart is due (uptime cadence or -slow decode at deep context) and triggers it at the next idle moment; the +compaction it asks the Hawk dashboard whether a restart is due and triggers it at +the next idle moment. A planned restart is due only when you enabled it +(`llama_kill_degraded`, "Kill llama when degraded" in the dashboard) and the average +decode speed is below `llama_kill_tps`; there is no time-based restart unless you +set `llama_restart_cadence_min`. A dead server is always started again. The dashboard refuses while a request is in flight and restarts the server with its exact command line; if that start fails, it falls back to `llama_bat_path`, so a -restart never leaves the server down. Without a running dashboard the plugin does -nothing. Set -`HAWK_DASHBOARD_URL` if the dashboard is not on `http://127.0.0.1:8765`. +restart never leaves the server down; only one start runs at a time, so there is +never a second llama-server. Without a running dashboard the plugin does nothing. +Set `HAWK_DASHBOARD_URL` if the dashboard is not on `http://127.0.0.1:8765`. + +## Failsafe: keep Hawk running + +```sh +hawk install-watchdog +``` + +registers `hawk ensure` with Task Scheduler (Windows, every 2 minutes, no window) or +prints the cron line (Linux). It starts Hawk again if the monitor or dashboard is not +running, and starts llama-server via `llama_bat_path` if it stays down for two checks +while Hawk runs. Only one Hawk monitor and one dashboard can run at a time; a second +one exits at once. ## Platform notes diff --git a/opencode_hawk/cli.py b/opencode_hawk/cli.py index 7452eb2..6a6ce60 100644 --- a/opencode_hawk/cli.py +++ b/opencode_hawk/cli.py @@ -2,6 +2,7 @@ from __future__ import annotations import argparse +import os import shutil import subprocess import sys @@ -103,6 +104,93 @@ def cmd_install_task(_args) -> int: return coordinator.main(["--install-task"]) or 0 +WATCHDOG_TASK = "HawkWatchdog" + + +def _hawk_processes(): + """Running Hawk monitor / dashboard processes (psutil), as {'monitor': [pids], 'dashboard': [pids], 'run': [pids]}.""" + import psutil + found = {"monitor": [], "dashboard": [], "run": []} + for p in psutil.process_iter(["pid", "cmdline"]): + argv = [str(a) for a in (p.info.get("cmdline") or [])] + if p.info["pid"] == os.getpid() or not any("opencode_hawk" in a or a.lower().endswith(("hawk", "hawk.exe")) + for a in argv): + continue + for role in ("monitor", "dashboard", "run"): + if role in argv: + found[role].append(p.info["pid"]) + return found + + +def _url_ok(url: str) -> bool: + import urllib.request + try: + with urllib.request.urlopen(url, timeout=4) as r: + return 200 <= r.status < 500 + except Exception: + return False + + +def cmd_ensure(args) -> int: + """Failsafe (run every few minutes by `hawk install-watchdog`): a stopped Hawk is started again, outside any + app's job; a llama-server that stays down across two checks while Hawk runs is started via llama_bat_path. + 2026-10-05: Hawk and llama-server died with the app that had started them, and the night's run stopped.""" + import json + from . import coordinator + home = coordinator.home_dir() + procs = _hawk_processes() + dash_ok = _url_ok("http://127.0.0.1:%d/" % args.port) + if not procs["monitor"] or not dash_ok: + import psutil + for pid in procs["monitor"] + procs["dashboard"] + procs["run"]: + try: + psutil.Process(pid).kill() # a half-running Hawk: start clean, never twice + except Exception: + pass + log = open(home / "hawk-console.log", "ab") + coordinator.spawn_independent([sys.executable, "-m", "opencode_hawk", "run", "--no-open", "--port", str(args.port)], + stdout=log, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL) + coordinator.log("WATCHDOG: Hawk was not running (monitor %s, dashboard %s); started it" + % ("up" if procs["monitor"] else "down", "up" if dash_ok else "down")) + return 0 + cfg = coordinator.load_config() + state_file = home / "watchdog.json" + try: + state = json.loads(state_file.read_text(encoding="utf-8")) + except Exception: + state = {} + llama_ok = _url_ok(coordinator.llama_api_base(cfg) + "/health") + misses = 0 if llama_ok else int(state.get("llama_misses") or 0) + 1 + if misses >= 2 and (cfg.get("llama_bat_path") or "").strip(): + coordinator.log("WATCHDOG: llama-server down for %d checks while Hawk runs; starting it" % misses) + if coordinator.llama_respawn(cfg): + misses = 0 + state_file.write_text(json.dumps({"llama_misses": misses}), encoding="utf-8") + return 0 + + +def cmd_install_watchdog(args) -> int: + """Register `hawk ensure` with the OS scheduler: Task Scheduler on Windows (pythonw, no window), cron elsewhere.""" + if os.name != "nt": + print("# add to `crontab -e`:") + print("*/%d * * * * %s -m opencode_hawk ensure" % (args.every, sys.executable)) + return 0 + if args.remove: + r = subprocess.run(["schtasks", "/Delete", "/F", "/TN", WATCHDOG_TASK], capture_output=True, text=True) + print((r.stdout or r.stderr).strip()) + return r.returncode + pyw = Path(sys.executable).with_name("pythonw.exe") + exe = pyw if pyw.exists() else Path(sys.executable) + tr = '"%s" -m opencode_hawk ensure' % exe + r = subprocess.run(["schtasks", "/Create", "/F", "/TN", WATCHDOG_TASK, "/SC", "MINUTE", "/MO", str(args.every), + "/TR", tr, "/RL", "LIMITED"], capture_output=True, text=True) + print((r.stdout or r.stderr).strip()) + if r.returncode == 0: + print("watchdog: every %d min `hawk ensure` starts Hawk if it is down (remove: hawk install-watchdog --remove)" + % args.every) + return r.returncode + + def cmd_run(args) -> int: """Monitor and dashboard as two child processes; Ctrl+C stops both.""" py = [sys.executable, "-m", "opencode_hawk"] @@ -161,6 +249,15 @@ def build_parser() -> argparse.ArgumentParser: sub.add_parser("install-task", help="print the Task Scheduler / cron entry").set_defaults(func=cmd_install_task) + p = sub.add_parser("ensure", help="failsafe: start Hawk if it is not running; start llama-server if it stays down") + p.add_argument("--port", type=int, default=8765) + p.set_defaults(func=cmd_ensure) + + p = sub.add_parser("install-watchdog", help="run `hawk ensure` every 2 minutes (Task Scheduler / cron)") + p.add_argument("--remove", action="store_true", help="remove the watchdog task") + p.add_argument("--every", type=int, default=2, help="minutes between checks (default 2)") + p.set_defaults(func=cmd_install_watchdog) + p = sub.add_parser("install-plugin", help="install the llama-restart plugin into OpenCode") p.add_argument("--dir", default=None, help="plugin folder (default: OpenCode's global plugins folder)") p.add_argument("--remove", action="store_true", help="uninstall it") diff --git a/opencode_hawk/coordinator.py b/opencode_hawk/coordinator.py index 2093384..b75104e 100644 --- a/opencode_hawk/coordinator.py +++ b/opencode_hawk/coordinator.py @@ -771,6 +771,36 @@ def creation_flags() -> int: return 0 +_HELD_LOCKS = {} + + +def hold_lock(name: str): + """Take the OS file lock `/.lock` for the rest of this process (released by the OS when the + process ends, also on a crash). Returns True when this process holds it, False when another one does.""" + if name in _HELD_LOCKS: + return True + f = open(home_dir() / ("%s.lock" % name), "a+") + try: + if os.name == "nt": + import msvcrt + f.seek(0) + msvcrt.locking(f.fileno(), msvcrt.LK_NBLCK, 1) + else: + import fcntl + fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + f.close() + return False + _HELD_LOCKS[name] = f + return True + + +def release_lock(name: str) -> None: + f = _HELD_LOCKS.pop(name, None) + if f: + f.close() + + CREATE_BREAKAWAY_FROM_JOB = 0x01000000 @@ -1602,6 +1632,9 @@ def main(argv=None): return 0 if args.monitor: + if not args.dry_run and not hold_lock("monitor"): + log("another Hawk monitor is already running; this one exits (one instance only)") + return 3 interval = args.interval or int(cfg.get("poll_minutes", DEFAULT_POLL_MINUTES)) log("monitor loop every %d min (dry_run=%s)" % (interval, args.dry_run)) stop_perm = threading.Event() diff --git a/opencode_hawk/dashboard.py b/opencode_hawk/dashboard.py index 1de9aab..c92907a 100644 --- a/opencode_hawk/dashboard.py +++ b/opencode_hawk/dashboard.py @@ -2065,6 +2065,9 @@ def main(argv=None) -> int: args = ap.parse_args(argv) if args.self_test: return self_test() + if not coordinator.hold_lock("dashboard"): + print("another Hawk dashboard is already running; this one exits (one instance only)") + return 3 new_topic = coordinator.ensure_ntfy_topic(coordinator.load_config()) if new_topic: print("ntfy: generated topic %s for this install" % new_topic) diff --git a/opencode_hawk/llama_restart.py b/opencode_hawk/llama_restart.py index a02ffba..d6f36f9 100644 --- a/opencode_hawk/llama_restart.py +++ b/opencode_hawk/llama_restart.py @@ -47,7 +47,10 @@ LOG_FN = "llama-server.out.log" DEFAULTS = { - "llama_restart_cadence_min": 210, + # Planned restarts (not the dead-server respawn, which is always on) happen only when the user enabled them + # (llama_kill_degraded) and the average decode speed is below llama_kill_tps. A time-based cadence is off (0) + # unless set; it also needs llama_kill_degraded. + "llama_restart_cadence_min": 0, "llama_restart_tps_floor": 14.0, "llama_restart_depth_floor": 30000, "llama_restart_window_s": 900, @@ -222,13 +225,24 @@ def due_reason(now_ms: int, last_at_ms, cadence_s, degraded: bool) -> dict: """Union of time + degradation triggers. Returns {'due': bool, 'reasons': [str]}.""" reasons = [] - if last_at_ms and (now_ms - last_at_ms) >= cadence_s * 1000: + if cadence_s and last_at_ms and (now_ms - last_at_ms) >= cadence_s * 1000: reasons.append("time") if degraded: reasons.append("degraded") return {"due": bool(reasons), "reasons": reasons} +def planned_due(cfg, now_ms: int, last_at_ms, deg: dict) -> dict: + """Planned restart: only when enabled (llama_kill_degraded) and the average decode speed is below + llama_kill_tps (or, if configured, the uptime cadence ran out). Disabled -> never due; a dead or unhealthy + server is handled separately by status() and always restarted.""" + if not cfg_get(cfg, "llama_kill_degraded"): + return {"due": False, "reasons": []} + tps = deg.get("avg_tps") + slow = bool(deg.get("degraded")) and tps is not None and tps < cfg_get(cfg, "llama_kill_tps") + return due_reason(now_ms, last_at_ms, cfg_get(cfg, "llama_restart_cadence_min") * 60, slow) + + def kill_decision(cfg, deg: dict, deg_since_ms, now_ms: int) -> dict: """Decide the degradation-kill (webUI 'Kill llama when degraded'). @@ -270,7 +284,7 @@ def status(cfg) -> dict: now = _now_ms() cad = cfg_get(cfg, "llama_restart_cadence_min") * 60 deg = degraded_signal(cfg) - due = due_reason(now, last, cad, deg["degraded"]) + due = planned_due(cfg, now, last, deg) # Liveness-validated state: a dead pid must read as restart_due even when # the state file says healthy (the 2026-09-25 16:21 gap); an alive pid # whose /health probe refuses is treated the same (half-dead survivor). @@ -460,8 +474,13 @@ def restart(cfg, reason: str = "manual") -> dict: ok_shutdown = _graceful_shutdown(cfg, pid) if not ok_shutdown and _pid_alive(pid): return {"ok": False, "reason": "shutdown_failed"} # still running: nothing lost - # From here the server is gone: whatever happens, never leave it down. - new_pid = _respawn(spec) + # From here the server is gone: whatever happens, never leave it down. One start at a time (llama-start). + if not coordinator.hold_lock("llama-start"): + return {"ok": False, "reason": "another start in progress"} + try: + new_pid = None if coordinator.llama_server_alive(cfg) else _respawn(spec) + finally: + coordinator.release_lock("llama-start") if not new_pid: from . import probes if probes.llama_respawn(cfg): diff --git a/opencode_hawk/probes.py b/opencode_hawk/probes.py index fea8a43..0b1d88e 100644 --- a/opencode_hawk/probes.py +++ b/opencode_hawk/probes.py @@ -237,6 +237,18 @@ def llama_respawn(cfg, timeout_s=None) -> bool: already present this is a no-op that reports True - the liveness watchdog and the monitor supervisor both run this path and must never double- spawn an extra instance.""" + # One start at a time across every Hawk process (monitor, dashboard, `hawk ensure`): two starts racing past the + # alive check would run two llama-servers. The alive check is repeated under the lock. + if not core.hold_lock("llama-start"): + core.log("LLAMA-DOWN: another start of llama-server is in progress; skipping") + return core.llama_server_alive(cfg) + try: + return _llama_respawn_locked(cfg, timeout_s) + finally: + core.release_lock("llama-start") + + +def _llama_respawn_locked(cfg, timeout_s=None) -> bool: if core.llama_server_alive(cfg): core.log("LLAMA-DOWN: server already alive (pid %s); skipping respawn" % core.llama_pid(cfg)) diff --git a/tests/selftest_coordinator.py b/tests/selftest_coordinator.py index d260b3b..5345b1c 100644 --- a/tests/selftest_coordinator.py +++ b/tests/selftest_coordinator.py @@ -614,6 +614,70 @@ def mkllama_idle_test(cpu_frac, slots_processing, gate_slots, "OK" if ok_m else "MISMATCH", "escalation note written to session DB as synthetic user msg + text part")) + # N0000: one instance only: a lock another process holds is refused; free again once that process ends + import subprocess as _sp1, tempfile as _tf1 + _lockhome = Path(_tf1.mkdtemp(prefix="hawk-lock-")) + _old_home1 = os.environ.get("COORD_HOME") + os.environ["COORD_HOME"] = str(_lockhome) + _holder = _sp1.Popen([sys.executable, "-c", + "import sys, time; sys.path.insert(0, %r); from opencode_hawk import coordinator as c; " + "print(c.hold_lock('monitor'), flush=True); time.sleep(6)" % str(Path(__file__).resolve().parent.parent)], + stdout=_sp1.PIPE, text=True, env=dict(os.environ)) + try: + _held_by_child = _holder.stdout.readline().strip() == "True" + _refused = hawk.hold_lock("monitor") is False + finally: + _holder.wait(timeout=20) + _free_after = hawk.hold_lock("monitor") is True + hawk.release_lock("monitor") + if _old_home1 is None: + os.environ.pop("COORD_HOME", None) + else: + os.environ["COORD_HOME"] = _old_home1 + shutil.rmtree(_lockhome, ignore_errors=True) + ok_lock = _held_by_child and _refused and _free_after + results.append(("single_instance_lock", "pass", "pass" if ok_lock else "fail", ok_lock, + "a second Hawk (or llama start) is refused while one runs", [])) + print("%-22s expect=%-9s got=%-9s %s" % ("single_instance_lock", "pass", "pass" if ok_lock else "fail", + "OK" if ok_lock else "MISMATCH")) + + # N000: the failsafe (`hawk ensure`): Hawk down -> started; llama down -> started at the second check + import tempfile as _tf0 + from opencode_hawk import cli as _cli0 + _home0 = Path(_tf0.mkdtemp(prefix="hawk-ensure-")) + _old_home0 = os.environ.get("COORD_HOME") + os.environ["COORD_HOME"] = str(_home0) + (_home0 / "config.json").write_text(json.dumps({"llama_bat_path": "C:/llama/start.bat"}), encoding="utf-8") + _spawned0, _respawned0 = [], [] + _saved0 = (_cli0._hawk_processes, _cli0._url_ok, hawk.spawn_independent, hawk.llama_respawn) + class _Args0: + port = 8765 + try: + hawk.spawn_independent = lambda argv, **kw: _spawned0.append(argv) + hawk.llama_respawn = lambda cfg, timeout_s=None: (_respawned0.append(1) or True) + _cli0._hawk_processes = lambda: {"monitor": [], "dashboard": [], "run": []} + _cli0._url_ok = lambda url: False + _cli0.cmd_ensure(_Args0()) + ok_e1 = len(_spawned0) == 1 and _spawned0[0][-4:] == ["run", "--no-open", "--port", "8765"] + _cli0._hawk_processes = lambda: {"monitor": [1], "dashboard": [2], "run": [3]} + _cli0._url_ok = lambda url: "8765" in url # dashboard up, llama down + _cli0.cmd_ensure(_Args0()) + ok_e2 = len(_respawned0) == 0 and len(_spawned0) == 1 # one miss: wait + _cli0.cmd_ensure(_Args0()) + ok_e3 = len(_respawned0) == 1 # second miss: start llama + finally: + _cli0._hawk_processes, _cli0._url_ok, hawk.spawn_independent, hawk.llama_respawn = _saved0 + if _old_home0 is None: + os.environ.pop("COORD_HOME", None) + else: + os.environ["COORD_HOME"] = _old_home0 + shutil.rmtree(_home0, ignore_errors=True) + ok_ens = ok_e1 and ok_e2 and ok_e3 + results.append(("ensure_failsafe", "pass", "pass" if ok_ens else "fail", ok_ens, + "hawk ensure starts a stopped Hawk; starts llama after two missed checks", [])) + print("%-22s expect=%-9s got=%-9s %s" % ("ensure_failsafe", "pass", "pass" if ok_ens else "fail", + "OK" if ok_ens else "MISMATCH")) + # N00: llama-server is started outside the starter's job (Windows), falling back when breakaway is refused import subprocess as _sp _calls = [] diff --git a/tests/selftest_dashboard.py b/tests/selftest_dashboard.py index 2722db4..afb6faa 100644 --- a/tests/selftest_dashboard.py +++ b/tests/selftest_dashboard.py @@ -789,6 +789,18 @@ def _msg(i, sid, done, tin, tout, role="assistant", reason=0): # llama_restart: due-reason computation (pure) from opencode_hawk import llama_restart + # planned restarts: never while disabled; when enabled only below the set average tps; no time cadence by default + _deg_slow = {"degraded": True, "avg_tps": 8.0} + _deg_ok = {"degraded": True, "avg_tps": 20.0} + _day = 24 * 3600 * 1000 + check("llama_planned_off_by_default", + not llama_restart.planned_due({}, 10 * _day, 1, _deg_slow)["due"]) + check("llama_planned_needs_slow_tps", + llama_restart.planned_due({"llama_kill_degraded": True, "llama_kill_tps": 10.0}, 10 * _day, 1, _deg_slow)["due"] + and not llama_restart.planned_due({"llama_kill_degraded": True, "llama_kill_tps": 10.0}, 10 * _day, 1, _deg_ok)["due"]) + check("llama_planned_no_time_cadence_by_default", + not llama_restart.planned_due({"llama_kill_degraded": True}, 10 * _day, 1, {"degraded": False, "avg_tps": None})["due"]) + # llama_restart.restart: a server without /shutdown (404) that outlives the forced kill by a few seconds # (2026-10-05: a 56 GB host cache) still gets a new process - the restart never leaves llama down. import urllib.error as _ue @@ -819,6 +831,8 @@ def _no_shutdown(req, timeout=5): if hasattr(llama_restart, "_unlock"): llama_restart._unlock = lambda *a, **k: None coordinator.llama_slots = lambda cfg: {"is_processing": False} + _saved_alive_srv = coordinator.llama_server_alive + coordinator.llama_server_alive = lambda cfg: False llama_restart.urllib.request.urlopen = _no_shutdown llama_restart.subprocess.run = _fake_run # Windows: taskkill from opencode_hawk import hawk_linux as _hl @@ -832,6 +846,7 @@ def _no_shutdown(req, timeout=5): for n, v in _saved.items(): setattr(llama_restart, n, v) coordinator.llama_slots = _saved_slots + coordinator.llama_server_alive = _saved_alive_srv llama_restart.urllib.request.urlopen = _saved_urlopen llama_restart.subprocess.run = _saved_run try: