Planned llama restarts opt-in; failsafe watchdog; single instance

.gitignore

@@ -39,3 +39,6 @@ __pycache__/
 build/
 dist/
 *.egg-info/
+
+# Single-instance locks (monitor, dashboard, llama-start)
+*.lock

README.md

@@ -111,6 +111,8 @@ Update with `pipx upgrade opencode-hawk`.
 | `hawk dashboard` | The web dashboard only (`--port`, `--open`) |
 | `hawk install-task` | Print the `schtasks` (Windows) or crontab (Linux) entry for `hawk poll` |
 | `hawk install-plugin` | Install the optional llama-restart plugin into OpenCode (`--remove` uninstalls) |
+| `hawk ensure` | Failsafe: start Hawk if it is not running; start llama-server if it stays down |
+| `hawk install-watchdog` | Run `hawk ensure` every 2 minutes via Task Scheduler (no window) or cron (`--remove`) |
 | `hawk where` | Show the folders Hawk uses |
 
 ### From a source checkout
@@ -172,13 +174,28 @@ hawk install-plugin
 
 It installs `local-hawk-llama-restart.ts` into OpenCode's global plugin folder
 (`~/.config/opencode/plugins/`; restart OpenCode once to load it). After a
-compaction it asks the Hawk dashboard whether a restart is due (uptime cadence or
-slow decode at deep context) and triggers it at the next idle moment; the
+compaction it asks the Hawk dashboard whether a restart is due and triggers it at
+the next idle moment. A planned restart is due only when you enabled it
+(`llama_kill_degraded`, "Kill llama when degraded" in the dashboard) and the average
+decode speed is below `llama_kill_tps`; there is no time-based restart unless you
+set `llama_restart_cadence_min`. A dead server is always started again. The
 dashboard refuses while a request is in flight and restarts the server with its
 exact command line; if that start fails, it falls back to `llama_bat_path`, so a
-restart never leaves the server down. Without a running dashboard the plugin does
-nothing. Set
-`HAWK_DASHBOARD_URL` if the dashboard is not on `http://127.0.0.1:8765`.
+restart never leaves the server down; only one start runs at a time, so there is
+never a second llama-server. Without a running dashboard the plugin does nothing.
+Set `HAWK_DASHBOARD_URL` if the dashboard is not on `http://127.0.0.1:8765`.
+
+## Failsafe: keep Hawk running
+
+```sh
+hawk install-watchdog
+```
+
+registers `hawk ensure` with Task Scheduler (Windows, every 2 minutes, no window) or
+prints the cron line (Linux). It starts Hawk again if the monitor or dashboard is not
+running, and starts llama-server via `llama_bat_path` if it stays down for two checks
+while Hawk runs. Only one Hawk monitor and one dashboard can run at a time; a second
+one exits at once.
 
 ## Platform notes
 

opencode_hawk/cli.py

@@ -2,6 +2,7 @@
 from __future__ import annotations
 
 import argparse
+import os
 import shutil
 import subprocess
 import sys
@@ -103,6 +104,93 @@ def cmd_install_task(_args) -> int:
     return coordinator.main(["--install-task"]) or 0
 
 
+WATCHDOG_TASK = "HawkWatchdog"
+
+
+def _hawk_processes():
+    """Running Hawk monitor / dashboard processes (psutil), as {'monitor': [pids], 'dashboard': [pids], 'run': [pids]}."""
+    import psutil
+    found = {"monitor": [], "dashboard": [], "run": []}
+    for p in psutil.process_iter(["pid", "cmdline"]):
+        argv = [str(a) for a in (p.info.get("cmdline") or [])]
+        if p.info["pid"] == os.getpid() or not any("opencode_hawk" in a or a.lower().endswith(("hawk", "hawk.exe"))
+                                                    for a in argv):
+            continue
+        for role in ("monitor", "dashboard", "run"):
+            if role in argv:
+                found[role].append(p.info["pid"])
+    return found
+
+
+def _url_ok(url: str) -> bool:
+    import urllib.request
+    try:
+        with urllib.request.urlopen(url, timeout=4) as r:
+            return 200 <= r.status < 500
+    except Exception:
+        return False
+
+
+def cmd_ensure(args) -> int:
+    """Failsafe (run every few minutes by `hawk install-watchdog`): a stopped Hawk is started again, outside any
+    app's job; a llama-server that stays down across two checks while Hawk runs is started via llama_bat_path.
+    2026-10-05: Hawk and llama-server died with the app that had started them, and the night's run stopped."""
+    import json
+    from . import coordinator
+    home = coordinator.home_dir()
+    procs = _hawk_processes()
+    dash_ok = _url_ok("http://127.0.0.1:%d/" % args.port)
+    if not procs["monitor"] or not dash_ok:
+        import psutil
+        for pid in procs["monitor"] + procs["dashboard"] + procs["run"]:
+            try:
+                psutil.Process(pid).kill()  # a half-running Hawk: start clean, never twice
+            except Exception:
+                pass
+        log = open(home / "hawk-console.log", "ab")
+        coordinator.spawn_independent([sys.executable, "-m", "opencode_hawk", "run", "--no-open", "--port", str(args.port)],
+                                      stdout=log, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL)
+        coordinator.log("WATCHDOG: Hawk was not running (monitor %s, dashboard %s); started it"
+                        % ("up" if procs["monitor"] else "down", "up" if dash_ok else "down"))
+        return 0
+    cfg = coordinator.load_config()
+    state_file = home / "watchdog.json"
+    try:
+        state = json.loads(state_file.read_text(encoding="utf-8"))
+    except Exception:
+        state = {}
+    llama_ok = _url_ok(coordinator.llama_api_base(cfg) + "/health")
+    misses = 0 if llama_ok else int(state.get("llama_misses") or 0) + 1
+    if misses >= 2 and (cfg.get("llama_bat_path") or "").strip():
+        coordinator.log("WATCHDOG: llama-server down for %d checks while Hawk runs; starting it" % misses)
+        if coordinator.llama_respawn(cfg):
+            misses = 0
+    state_file.write_text(json.dumps({"llama_misses": misses}), encoding="utf-8")
+    return 0
+
+
+def cmd_install_watchdog(args) -> int:
+    """Register `hawk ensure` with the OS scheduler: Task Scheduler on Windows (pythonw, no window), cron elsewhere."""
+    if os.name != "nt":
+        print("# add to `crontab -e`:")
+        print("*/%d * * * * %s -m opencode_hawk ensure" % (args.every, sys.executable))
+        return 0
+    if args.remove:
+        r = subprocess.run(["schtasks", "/Delete", "/F", "/TN", WATCHDOG_TASK], capture_output=True, text=True)
+        print((r.stdout or r.stderr).strip())
+        return r.returncode
+    pyw = Path(sys.executable).with_name("pythonw.exe")
+    exe = pyw if pyw.exists() else Path(sys.executable)
+    tr = '"%s" -m opencode_hawk ensure' % exe
+    r = subprocess.run(["schtasks", "/Create", "/F", "/TN", WATCHDOG_TASK, "/SC", "MINUTE", "/MO", str(args.every),
+                        "/TR", tr, "/RL", "LIMITED"], capture_output=True, text=True)
+    print((r.stdout or r.stderr).strip())
+    if r.returncode == 0:
+        print("watchdog: every %d min `hawk ensure` starts Hawk if it is down (remove: hawk install-watchdog --remove)"
+              % args.every)
+    return r.returncode
+
+
 def cmd_run(args) -> int:
     """Monitor and dashboard as two child processes; Ctrl+C stops both."""
     py = [sys.executable, "-m", "opencode_hawk"]
@@ -161,6 +249,15 @@ def build_parser() -> argparse.ArgumentParser:
 
     sub.add_parser("install-task", help="print the Task Scheduler / cron entry").set_defaults(func=cmd_install_task)
 
+    p = sub.add_parser("ensure", help="failsafe: start Hawk if it is not running; start llama-server if it stays down")
+    p.add_argument("--port", type=int, default=8765)
+    p.set_defaults(func=cmd_ensure)
+
+    p = sub.add_parser("install-watchdog", help="run `hawk ensure` every 2 minutes (Task Scheduler / cron)")
+    p.add_argument("--remove", action="store_true", help="remove the watchdog task")
+    p.add_argument("--every", type=int, default=2, help="minutes between checks (default 2)")
+    p.set_defaults(func=cmd_install_watchdog)
+
     p = sub.add_parser("install-plugin", help="install the llama-restart plugin into OpenCode")
     p.add_argument("--dir", default=None, help="plugin folder (default: OpenCode's global plugins folder)")
     p.add_argument("--remove", action="store_true", help="uninstall it")

opencode_hawk/coordinator.py

@@ -771,6 +771,36 @@ def creation_flags() -> int:
     return 0
 
 
+_HELD_LOCKS = {}
+
+
+def hold_lock(name: str):
+    """Take the OS file lock `<home>/<name>.lock` for the rest of this process (released by the OS when the
+    process ends, also on a crash). Returns True when this process holds it, False when another one does."""
+    if name in _HELD_LOCKS:
+        return True
+    f = open(home_dir() / ("%s.lock" % name), "a+")
+    try:
+        if os.name == "nt":
+            import msvcrt
+            f.seek(0)
+            msvcrt.locking(f.fileno(), msvcrt.LK_NBLCK, 1)
+        else:
+            import fcntl
+            fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except OSError:
+        f.close()
+        return False
+    _HELD_LOCKS[name] = f
+    return True
+
+
+def release_lock(name: str) -> None:
+    f = _HELD_LOCKS.pop(name, None)
+    if f:
+        f.close()
+
+
 CREATE_BREAKAWAY_FROM_JOB = 0x01000000
 
 
@@ -1602,6 +1632,9 @@ def main(argv=None):
         return 0
 
     if args.monitor:
+        if not args.dry_run and not hold_lock("monitor"):
+            log("another Hawk monitor is already running; this one exits (one instance only)")
+            return 3
         interval = args.interval or int(cfg.get("poll_minutes", DEFAULT_POLL_MINUTES))
         log("monitor loop every %d min (dry_run=%s)" % (interval, args.dry_run))
         stop_perm = threading.Event()

opencode_hawk/dashboard.py

@@ -2065,6 +2065,9 @@ def main(argv=None) -> int:
     args = ap.parse_args(argv)
     if args.self_test:
         return self_test()
+    if not coordinator.hold_lock("dashboard"):
+        print("another Hawk dashboard is already running; this one exits (one instance only)")
+        return 3
     new_topic = coordinator.ensure_ntfy_topic(coordinator.load_config())
     if new_topic:
         print("ntfy: generated topic %s for this install" % new_topic)

opencode_hawk/llama_restart.py

@@ -47,7 +47,10 @@
 LOG_FN = "llama-server.out.log"
 
 DEFAULTS = {
-    "llama_restart_cadence_min": 210,
+    # Planned restarts (not the dead-server respawn, which is always on) happen only when the user enabled them
+    # (llama_kill_degraded) and the average decode speed is below llama_kill_tps. A time-based cadence is off (0)
+    # unless set; it also needs llama_kill_degraded.
+    "llama_restart_cadence_min": 0,
     "llama_restart_tps_floor": 14.0,
     "llama_restart_depth_floor": 30000,
     "llama_restart_window_s": 900,
@@ -222,13 +225,24 @@ def due_reason(now_ms: int, last_at_ms, cadence_s, degraded: bool) -> dict:
     """Union of time + degradation triggers.
     Returns {'due': bool, 'reasons': [str]}."""
     reasons = []
-    if last_at_ms and (now_ms - last_at_ms) >= cadence_s * 1000:
+    if cadence_s and last_at_ms and (now_ms - last_at_ms) >= cadence_s * 1000:
         reasons.append("time")
     if degraded:
         reasons.append("degraded")
     return {"due": bool(reasons), "reasons": reasons}
 
 
+def planned_due(cfg, now_ms: int, last_at_ms, deg: dict) -> dict:
+    """Planned restart: only when enabled (llama_kill_degraded) and the average decode speed is below
+    llama_kill_tps (or, if configured, the uptime cadence ran out). Disabled -> never due; a dead or unhealthy
+    server is handled separately by status() and always restarted."""
+    if not cfg_get(cfg, "llama_kill_degraded"):
+        return {"due": False, "reasons": []}
+    tps = deg.get("avg_tps")
+    slow = bool(deg.get("degraded")) and tps is not None and tps < cfg_get(cfg, "llama_kill_tps")
+    return due_reason(now_ms, last_at_ms, cfg_get(cfg, "llama_restart_cadence_min") * 60, slow)
+
+
 def kill_decision(cfg, deg: dict, deg_since_ms, now_ms: int) -> dict:
     """Decide the degradation-kill (webUI 'Kill llama when degraded').
 
@@ -270,7 +284,7 @@ def status(cfg) -> dict:
     now = _now_ms()
     cad = cfg_get(cfg, "llama_restart_cadence_min") * 60
     deg = degraded_signal(cfg)
-    due = due_reason(now, last, cad, deg["degraded"])
+    due = planned_due(cfg, now, last, deg)
     # Liveness-validated state: a dead pid must read as restart_due even when
     # the state file says healthy (the 2026-09-25 16:21 gap); an alive pid
     # whose /health probe refuses is treated the same (half-dead survivor).
@@ -460,8 +474,13 @@ def restart(cfg, reason: str = "manual") -> dict:
         ok_shutdown = _graceful_shutdown(cfg, pid)
         if not ok_shutdown and _pid_alive(pid):
             return {"ok": False, "reason": "shutdown_failed"}  # still running: nothing lost
-        # From here the server is gone: whatever happens, never leave it down.
-        new_pid = _respawn(spec)
+        # From here the server is gone: whatever happens, never leave it down. One start at a time (llama-start).
+        if not coordinator.hold_lock("llama-start"):
+            return {"ok": False, "reason": "another start in progress"}
+        try:
+            new_pid = None if coordinator.llama_server_alive(cfg) else _respawn(spec)
+        finally:
+            coordinator.release_lock("llama-start")
         if not new_pid:
             from . import probes
             if probes.llama_respawn(cfg):

opencode_hawk/probes.py

@@ -237,6 +237,18 @@ def llama_respawn(cfg, timeout_s=None) -> bool:
     already present this is a no-op that reports True - the liveness watchdog
     and the monitor supervisor both run this path and must never double-
     spawn an extra instance."""
+    # One start at a time across every Hawk process (monitor, dashboard, `hawk ensure`): two starts racing past the
+    # alive check would run two llama-servers. The alive check is repeated under the lock.
+    if not core.hold_lock("llama-start"):
+        core.log("LLAMA-DOWN: another start of llama-server is in progress; skipping")
+        return core.llama_server_alive(cfg)
+    try:
+        return _llama_respawn_locked(cfg, timeout_s)
+    finally:
+        core.release_lock("llama-start")
+
+
+def _llama_respawn_locked(cfg, timeout_s=None) -> bool:
     if core.llama_server_alive(cfg):
         core.log("LLAMA-DOWN: server already alive (pid %s); skipping respawn"
             % core.llama_pid(cfg))

tests/selftest_coordinator.py

@@ -614,6 +614,70 @@ def mkllama_idle_test(cpu_frac, slots_processing, gate_slots,
         "OK" if ok_m else "MISMATCH",
         "escalation note written to session DB as synthetic user msg + text part"))
 
+    # N0000: one instance only: a lock another process holds is refused; free again once that process ends
+    import subprocess as _sp1, tempfile as _tf1
+    _lockhome = Path(_tf1.mkdtemp(prefix="hawk-lock-"))
+    _old_home1 = os.environ.get("COORD_HOME")
+    os.environ["COORD_HOME"] = str(_lockhome)
+    _holder = _sp1.Popen([sys.executable, "-c",
+                          "import sys, time; sys.path.insert(0, %r); from opencode_hawk import coordinator as c; "
+                          "print(c.hold_lock('monitor'), flush=True); time.sleep(6)" % str(Path(__file__).resolve().parent.parent)],
+                         stdout=_sp1.PIPE, text=True, env=dict(os.environ))
+    try:
+        _held_by_child = _holder.stdout.readline().strip() == "True"
+        _refused = hawk.hold_lock("monitor") is False
+    finally:
+        _holder.wait(timeout=20)
+    _free_after = hawk.hold_lock("monitor") is True
+    hawk.release_lock("monitor")
+    if _old_home1 is None:
+        os.environ.pop("COORD_HOME", None)
+    else:
+        os.environ["COORD_HOME"] = _old_home1
+    shutil.rmtree(_lockhome, ignore_errors=True)
+    ok_lock = _held_by_child and _refused and _free_after
+    results.append(("single_instance_lock", "pass", "pass" if ok_lock else "fail", ok_lock,
+                    "a second Hawk (or llama start) is refused while one runs", []))
+    print("%-22s expect=%-9s got=%-9s %s" % ("single_instance_lock", "pass", "pass" if ok_lock else "fail",
+                                            "OK" if ok_lock else "MISMATCH"))
+
+    # N000: the failsafe (`hawk ensure`): Hawk down -> started; llama down -> started at the second check
+    import tempfile as _tf0
+    from opencode_hawk import cli as _cli0
+    _home0 = Path(_tf0.mkdtemp(prefix="hawk-ensure-"))
+    _old_home0 = os.environ.get("COORD_HOME")
+    os.environ["COORD_HOME"] = str(_home0)
+    (_home0 / "config.json").write_text(json.dumps({"llama_bat_path": "C:/llama/start.bat"}), encoding="utf-8")
+    _spawned0, _respawned0 = [], []
+    _saved0 = (_cli0._hawk_processes, _cli0._url_ok, hawk.spawn_independent, hawk.llama_respawn)
+    class _Args0:
+        port = 8765
+    try:
+        hawk.spawn_independent = lambda argv, **kw: _spawned0.append(argv)
+        hawk.llama_respawn = lambda cfg, timeout_s=None: (_respawned0.append(1) or True)
+        _cli0._hawk_processes = lambda: {"monitor": [], "dashboard": [], "run": []}
+        _cli0._url_ok = lambda url: False
+        _cli0.cmd_ensure(_Args0())
+        ok_e1 = len(_spawned0) == 1 and _spawned0[0][-4:] == ["run", "--no-open", "--port", "8765"]
+        _cli0._hawk_processes = lambda: {"monitor": [1], "dashboard": [2], "run": [3]}
+        _cli0._url_ok = lambda url: "8765" in url  # dashboard up, llama down
+        _cli0.cmd_ensure(_Args0())
+        ok_e2 = len(_respawned0) == 0 and len(_spawned0) == 1  # one miss: wait
+        _cli0.cmd_ensure(_Args0())
+        ok_e3 = len(_respawned0) == 1  # second miss: start llama
+    finally:
+        _cli0._hawk_processes, _cli0._url_ok, hawk.spawn_independent, hawk.llama_respawn = _saved0
+        if _old_home0 is None:
+            os.environ.pop("COORD_HOME", None)
+        else:
+            os.environ["COORD_HOME"] = _old_home0
+        shutil.rmtree(_home0, ignore_errors=True)
+    ok_ens = ok_e1 and ok_e2 and ok_e3
+    results.append(("ensure_failsafe", "pass", "pass" if ok_ens else "fail", ok_ens,
+                    "hawk ensure starts a stopped Hawk; starts llama after two missed checks", []))
+    print("%-22s expect=%-9s got=%-9s %s" % ("ensure_failsafe", "pass", "pass" if ok_ens else "fail",
+                                            "OK" if ok_ens else "MISMATCH"))
+
     # N00: llama-server is started outside the starter's job (Windows), falling back when breakaway is refused
     import subprocess as _sp
     _calls = []