diff --git a/docs/spec/SPEC-011-daemon-architecture.md b/docs/spec/SPEC-011-daemon-architecture.md index 42f2f3be8..283f1445f 100644 --- a/docs/spec/SPEC-011-daemon-architecture.md +++ b/docs/spec/SPEC-011-daemon-architecture.md @@ -243,6 +243,11 @@ restores the pending input under its recorded authority when the reminder arrives. A completed turn, partial text, or possible tool effect produces no restart reminder. +The daemon gives session drain 20 seconds within a 30-second shutdown phase. +The CLI allows 45 seconds before forced termination. The generated systemd +unit allows 60 seconds. A container should set +`terminationGracePeriodSeconds` to at least 60 seconds. + `netclaw daemon status` checks the PID file and verifies the process is alive. Reports: running/stopped, PID, uptime, port, number of active sessions. diff --git a/feeds/skills/.system/files/netclaw-operations/SKILL.md b/feeds/skills/.system/files/netclaw-operations/SKILL.md index d1eb69083..d971969ef 100644 --- a/feeds/skills/.system/files/netclaw-operations/SKILL.md +++ b/feeds/skills/.system/files/netclaw-operations/SKILL.md @@ -3,7 +3,7 @@ name: netclaw-operations description: "REQUIRED when the user asks about scheduling, reminders, cron jobs, timers, background jobs, diagnostics, troubleshooting, MCP tools, daemon health, identity updates, or Netclaw capabilities and self-maintenance." metadata: author: netclaw - version: "2.75.2" + version: "2.75.3" --- # Netclaw Operations diff --git a/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md b/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md index 7d6bbe72f..93d423602 100644 --- a/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md +++ b/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md @@ -88,6 +88,15 @@ Slack health reads the live Socket Mode state. The connection supervisor checks A disconnect emits `channel.disconnected`. The supervisor retries with exponential backoff up to five minutes. A successful recovery emits `channel.reconnected`. +A graceful stop can resume a safe model turn through a standard reminder. +The reminder expires ten minutes after the interruption. A completed reply, +partial reply, or possible tool effect leaves the session quiet. A crash does +not create a restart reminder. + +The daemon gives session drain 20 seconds within a 30-second shutdown phase. +The CLI allows 45 seconds, and the generated systemd unit allows 60 seconds. +A pod should set `terminationGracePeriodSeconds` to at least 60 seconds. + If webhook notifications are configured, daemon crash paths emit `daemon.crashing` operational alerts with context (PID, reason, and latest known session/turn snapshot when available). diff --git a/src/Netclaw.Cli.Tests/Cli/DaemonManagerGracefulShutdownTests.cs b/src/Netclaw.Cli.Tests/Cli/DaemonManagerGracefulShutdownTests.cs index b06d2594f..87ec73694 100644 --- a/src/Netclaw.Cli.Tests/Cli/DaemonManagerGracefulShutdownTests.cs +++ b/src/Netclaw.Cli.Tests/Cli/DaemonManagerGracefulShutdownTests.cs @@ -16,7 +16,7 @@ namespace Netclaw.Cli.Tests.Cli; /// Covers the canary daemon-stop finding: systemctl --user stop netclaw.service landed /// in failed (Result: signal) because 's SIGTERM /// grace period (previously a hardcoded 10s) was far shorter than the ~200s the daemon's own -/// Akka CoordinatedShutdown session-drain phase is deliberately allotted — so the CLI itself +/// Akka CoordinatedShutdown session-drain phase had been allotted — so the CLI itself /// gave up and force-killed the daemon long before a legitimately slow (in-flight LLM call) /// graceful shutdown could finish. It still died, so `netclaw daemon stop` (ExecStop) reported /// success, but via SIGKILL rather than a clean exit — exactly what systemd's @@ -24,7 +24,7 @@ namespace Netclaw.Cli.Tests.Cli; /// /// These tests exercise the two testable halves of the fix: (1) the internal /// poll now honors an injected -/// end-to-end (not just for its deadline math), so the up-to-200s +/// end-to-end (not just for its deadline math), so the bounded /// wait can be driven with a instead of a real sleep; and /// (2) the generated systemd unit's TimeoutStopSec= stays in lockstep with /// so systemd itself never SIGKILLs the whole @@ -49,7 +49,7 @@ public async Task WaitForExitAsync_ReturnsTrue_Immediately_WhenProcessAlreadyExi var manager = new DaemonManager(_paths, TimeProvider.System); using var exited = StartAndWaitForRealExit(); - var result = await manager.WaitForExitAsync(exited, TimeSpan.FromSeconds(200), CancellationToken.None); + var result = await manager.WaitForExitAsync(exited, DaemonConfig.GracefulShutdownBudget, CancellationToken.None); Assert.True(result); } diff --git a/src/Netclaw.Cli/Daemon/DaemonManager.cs b/src/Netclaw.Cli/Daemon/DaemonManager.cs index e56b7a7b3..3d8f6a9fa 100644 --- a/src/Netclaw.Cli/Daemon/DaemonManager.cs +++ b/src/Netclaw.Cli/Daemon/DaemonManager.cs @@ -180,8 +180,8 @@ await http.PostAsync( // Wait for graceful exit. DaemonConfig.GracefulShutdownBudget matches the daemon's own // Akka CoordinatedShutdown "before-service-unbind" phase timeout — where session // draining (SessionDrainHelper.DrainAsync) actually happens — so the CLI does not give - // up on a daemon that is still legitimately draining in-flight sessions (TurnLlmTimeout - // defaults to 3 minutes). See DaemonConfig.GracefulShutdownBudget remarks for the full + // up on a daemon that is still draining in-flight sessions. See + // DaemonConfig.GracefulShutdownBudget remarks for the full // layering this must respect: bounded drain < Akka phase timeout < this budget + the // grace window below < systemd's TimeoutStopSec= (netclaw-dev/netclaw#1664, #1665). var exitedWithinBudget = await WaitForExitAsync(process, DaemonConfig.GracefulShutdownBudget, cancellationToken); @@ -192,7 +192,7 @@ await http.PostAsync( // exactly this same budget boundary, and still needs to finish tearing down (actor // system termination, PID file cleanup) afterward. Poll a short additional grace // window before escalating to SIGKILL — production evidence (#1665) showed a - // daemon force-killed ~100ms from a clean exit because the CLI escalated the + // daemon was force-killed ~100ms from a clean exit because the CLI escalated the // instant its budget elapsed, with no headroom at all. exitedDuringGraceWindow = await WaitForExitAsync(process, DaemonConfig.CliForceKillGraceWindow, cancellationToken); } @@ -229,8 +229,8 @@ await http.PostAsync( if (exitedDuringGraceWindow) { // Clean exit — no kill needed — but worth surfacing: the daemon used its full - // graceful-shutdown budget, which usually means a session was still mid-LLM-call - // at shutdown. + // graceful-shutdown budget, which can mean that a session task did not stop + // after cancellation. return new DaemonResult(true, $"Daemon stopped (was PID {pid}); exited during the " + $"{DaemonConfig.CliForceKillGraceWindow.TotalSeconds:F0}s grace window after the " + @@ -241,7 +241,7 @@ await http.PostAsync( return new DaemonResult(true, $"Daemon stopped (was PID {pid}), but did not exit gracefully within " + $"{DaemonConfig.CliForceKillBudget.TotalSeconds:F0}s (budget + grace window) and had to be " + - "force-killed. This usually means a session was still mid-LLM-call at shutdown; if it " + + "force-killed. A session task or teardown step may have stalled; if it " + "recurs, check for stuck sessions before stopping the daemon."); } @@ -686,7 +686,7 @@ private static bool SendSignal(int pid, Signal signal) /// /// Polls until it exits or elapses. - /// Internal (not private) so tests can drive the up-to-200-second graceful-shutdown wait + /// Internal (not private) so tests can drive the bounded graceful-shutdown wait /// via an injected without a real wall-clock sleep: the poll /// delay is scheduled against (matching this repo's virtualized- /// timer convention, e.g. ConfigWatcherService), not a bare Task.Delay(ms). diff --git a/src/Netclaw.Configuration.Tests/DaemonConfigTests.cs b/src/Netclaw.Configuration.Tests/DaemonConfigTests.cs index a361fa6c0..59062f58c 100644 --- a/src/Netclaw.Configuration.Tests/DaemonConfigTests.cs +++ b/src/Netclaw.Configuration.Tests/DaemonConfigTests.cs @@ -325,12 +325,10 @@ public void ShutdownBudgetLayering_cli_force_kill_budget_stays_under_systemd_tim [Fact] public void ShutdownBudgetLayering_matches_the_documented_second_values() { - // Pins the concrete values referenced in netclaw-dev/netclaw#1665's evidence trail - // (200s phase timeout, 230s TimeoutStopSec) so a change to any constant is a visible, - // deliberate diff rather than a silent drift. - Assert.Equal(TimeSpan.FromSeconds(190), DaemonConfig.BoundedDrainTimeout); - Assert.Equal(TimeSpan.FromSeconds(200), DaemonConfig.GracefulShutdownBudget); - Assert.Equal(TimeSpan.FromSeconds(215), DaemonConfig.CliForceKillBudget); - Assert.Equal(TimeSpan.FromSeconds(230), DaemonConfig.SystemdTimeoutStopSec); + // Pin the agreed stop limits so a later change to any layer stays visible. + Assert.Equal(TimeSpan.FromSeconds(20), DaemonConfig.BoundedDrainTimeout); + Assert.Equal(TimeSpan.FromSeconds(30), DaemonConfig.GracefulShutdownBudget); + Assert.Equal(TimeSpan.FromSeconds(45), DaemonConfig.CliForceKillBudget); + Assert.Equal(TimeSpan.FromSeconds(60), DaemonConfig.SystemdTimeoutStopSec); } } diff --git a/src/Netclaw.Configuration/DaemonConfig.cs b/src/Netclaw.Configuration/DaemonConfig.cs index ffe4d8260..67d98c16b 100644 --- a/src/Netclaw.Configuration/DaemonConfig.cs +++ b/src/Netclaw.Configuration/DaemonConfig.cs @@ -21,10 +21,9 @@ public sealed record DaemonConfig public const int DefaultPort = 5199; /// - /// Worst-case time the daemon's graceful shutdown drain is allotted before something gives - /// up and forces termination. Sized to comfortably exceed SessionConfig.TurnLlmTimeout's - /// default (3 minutes) so a session mid-LLM-call during shutdown can finish draining instead - /// of being interrupted. + /// Worst-case time for the daemon's graceful shutdown phase. The session actor gives a + /// model call a short completion grace, then confirms cancellation before it returns a + /// restart reminder. Other work remains under the bounded drain deadline. /// /// Single source of truth shared by every shutdown-timing surface that must stay in /// lockstep (netclaw-dev/netclaw#1664, #1665): @@ -48,7 +47,7 @@ public sealed record DaemonConfig /// SIGKILL race the daemon could not win (evidence: a production daemon force-killed ~100ms /// from a clean exit). /// - public static readonly TimeSpan GracefulShutdownBudget = TimeSpan.FromSeconds(200); + public static readonly TimeSpan GracefulShutdownBudget = TimeSpan.FromSeconds(30); /// /// Safety margin subtracted from to produce diff --git a/src/Netclaw.Daemon.Tests/Services/DaemonRestartCoordinatorTests.cs b/src/Netclaw.Daemon.Tests/Services/DaemonRestartCoordinatorTests.cs index dcd5abe2d..7eb1a3a38 100644 --- a/src/Netclaw.Daemon.Tests/Services/DaemonRestartCoordinatorTests.cs +++ b/src/Netclaw.Daemon.Tests/Services/DaemonRestartCoordinatorTests.cs @@ -215,14 +215,9 @@ public async Task SessionDrainHelper_propagates_caller_cancellation() [Fact] public async Task SessionDrainHelper_daemon_stop_bound_times_out_instead_of_hanging_when_a_session_never_acks() { - // Mirrors the daemon-stop CoordinatedShutdown drain task wired in Program.cs - // (netclaw-dev/netclaw#1664): a session whose in-flight turn is parked on interactive - // tool approval never acks PrepareForDaemonRestart. Previously this call passed - // CancellationToken.None for the operation token and hung until Akka's own 200s - // before-service-unbind phase timeout abandoned the task. The bounded CTS below — - // sized from DaemonConfig.BoundedDrainTimeout (GracefulShutdownBudget minus - // DrainSafetyMargin) and driven by TimeProvider exactly as Program.cs constructs it — - // must make the drain complete with a timed-out result well before that. + // This test mirrors the daemon-stop drain task in Program.cs. A session task can + // ignore cancellation and fail to acknowledge drain. The bounded CTS must end the + // drain before Akka's phase timeout abandons the task. var time = new FakeTimeProvider(); var activeIds = new[] { "slack/approval-parked" }; var drain = new DrainControl(activeIds, activeIds); // never acknowledged diff --git a/src/Netclaw.Daemon/Program.cs b/src/Netclaw.Daemon/Program.cs index 4a5b05010..78a569acf 100644 --- a/src/Netclaw.Daemon/Program.cs +++ b/src/Netclaw.Daemon/Program.cs @@ -1065,11 +1065,9 @@ static void ConfigureDaemonServices( { // Prevent coordinated shutdown from calling Environment.Exit(), // which would kill the process before the restart loop can iterate. - // The before-service-unbind phase needs a generous timeout (DaemonConfig. - // GracefulShutdownBudget) because sessions mid-LLM-call (TurnLlmTimeout defaults to - // 3 minutes) must finish before passivation can begin. See DaemonConfig. - // GracefulShutdownBudget remarks for the full set of surfaces this must stay in - // lockstep with. + // The before-service-unbind phase gives session drain time to confirm model + // cancellation and write restart reminders. DaemonConfig keeps this phase, + // the CLI wait, and the systemd stop timeout in order. akkaBuilder.AddHocon( DaemonShutdownConfiguration.BuildCoordinatedShutdownHocon(DaemonConfig.GracefulShutdownBudget), HoconAddMode.Prepend); @@ -1120,8 +1118,7 @@ static void ConfigureDaemonServices( // Runs in an early CoordinatedShutdown phase while actors are still alive. // If DaemonRestartCoordinator already drained sessions (config reload), the ingress // gate will be closed and this task skips its drain to avoid double-draining. - // The phase timeout (DaemonConfig.GracefulShutdownBudget) is generous because - // sessions mid-LLM-call must finish before passivation can begin. + // The phase timeout lets sessions confirm model cancellation before passivation. var cs = CoordinatedShutdown.Get(system); var sessionManager = registry.Get(); var ingressGate = sp.GetRequiredService(); @@ -1144,8 +1141,8 @@ static void ConfigureDaemonServices( // the phase timeout itself fires and abandons this task outright. // netclaw-dev/netclaw#1664: a session parked on interactive tool approval // never acks PrepareForDaemonRestart, so an unbounded wait here (previously - // CancellationToken.None, CancellationToken.None) hung for the full 200s - // phase timeout with no timeout of its own, leaking the abandoned drain task. + // CancellationToken.None, CancellationToken.None) had left the drain task + // active until the phase timeout, with no separate drain deadline. using var drainDeadlineCts = new CancellationTokenSource(DaemonConfig.BoundedDrainTimeout, tp); var drainResult = await SessionDrainHelper.DrainAsync(