diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index 84b0bda89..10164761d 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -446,6 +446,14 @@ If post-drain MCP teardown remains material, review open PR [#1681](https://github.com/netclaw-dev/netclaw/pull/1681) before a new fix. Keep the unexplained 26-second service gap as a separate investigation. +The isolated persistent-home check on 2026-09-19 used an active model turn. +The baseline stopped in 7.47 seconds and reached the listener after 10.05 seconds. +The candidate stopped in 7.47 seconds and reached the listener after 9.96 seconds. +Both stops kept their resume candidates in a mode `0600` manifest. +The shorter limits affect stalled drains. They did not change this normal stop path. +Set the drain, phase, CLI, and systemd limits to 20, 30, 45, and 60 seconds. +The actor, daemon, and CLI tests cover the accepted input and duplicate action boundaries. + At this review date, open PR [#1893](https://github.com/netclaw-dev/netclaw/pull/1893) addresses a model timeout inside one process. Open PR [#1987](https://github.com/netclaw-dev/netclaw/pull/1987) addresses busy diff --git a/docs/spec/SPEC-011-daemon-architecture.md b/docs/spec/SPEC-011-daemon-architecture.md index f0fd95e8e..5eb2aad50 100644 --- a/docs/spec/SPEC-011-daemon-architecture.md +++ b/docs/spec/SPEC-011-daemon-architecture.md @@ -232,6 +232,11 @@ Any graceful stop can save a candidate in an atomic restart manifest. A candidat expires ten minutes after the first interruption. A normal stop hours before a later start leaves the session quiet. A crash does not create a candidate or repeat a claimed model call. +The daemon gives session drain 20 seconds within a 30 second shutdown phase. +The CLI allows 45 seconds before forced termination. The generated systemd unit allows 60 seconds. +A container operator should set `terminationGracePeriodSeconds` to at least 60. +This period gives the daemon its full stop budget to write the manifest and exit. + After channel startup, the daemon checks every recorded requester against the current channel ACL. It then restores a supported channel output subscriber. The actor checks the deadline, input IDs, and recorded authority again. diff --git a/feeds/skills/.system/files/netclaw-operations/SKILL.md b/feeds/skills/.system/files/netclaw-operations/SKILL.md index 57e1f52b7..24150dce5 100644 --- a/feeds/skills/.system/files/netclaw-operations/SKILL.md +++ b/feeds/skills/.system/files/netclaw-operations/SKILL.md @@ -3,7 +3,7 @@ name: netclaw-operations description: "REQUIRED when the user asks about scheduling, reminders, cron jobs, timers, background jobs, diagnostics, troubleshooting, MCP tools, daemon health, identity updates, or Netclaw capabilities and self-maintenance." metadata: author: netclaw - version: "2.74.9" + version: "2.74.10" --- # Netclaw Operations diff --git a/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md b/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md index 3a72c9e18..beb09c9cf 100644 --- a/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md +++ b/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md @@ -94,6 +94,9 @@ The daemon requires a live output route before it starts the model call. The candidate expires ten minutes after the first interruption, even after another short stop. A completed reply without accepted queued input leaves the session quiet. A crash does not schedule automatic resume. +The daemon gives session drain 20 seconds within a 30 second shutdown phase. +The CLI allows 45 seconds, and the generated systemd unit allows 60 seconds. +A pod should set `terminationGracePeriodSeconds` to at least 60 for the full graceful stop budget. If webhook notifications are configured, daemon crash paths emit `daemon.crashing` operational alerts with context (PID, reason, and latest known diff --git a/src/Netclaw.Cli.Tests/Cli/DaemonManagerGracefulShutdownTests.cs b/src/Netclaw.Cli.Tests/Cli/DaemonManagerGracefulShutdownTests.cs index b06d2594f..87ec73694 100644 --- a/src/Netclaw.Cli.Tests/Cli/DaemonManagerGracefulShutdownTests.cs +++ b/src/Netclaw.Cli.Tests/Cli/DaemonManagerGracefulShutdownTests.cs @@ -16,7 +16,7 @@ namespace Netclaw.Cli.Tests.Cli; /// Covers the canary daemon-stop finding: systemctl --user stop netclaw.service landed /// in failed (Result: signal) because 's SIGTERM /// grace period (previously a hardcoded 10s) was far shorter than the ~200s the daemon's own -/// Akka CoordinatedShutdown session-drain phase is deliberately allotted — so the CLI itself +/// Akka CoordinatedShutdown session-drain phase had been allotted — so the CLI itself /// gave up and force-killed the daemon long before a legitimately slow (in-flight LLM call) /// graceful shutdown could finish. It still died, so `netclaw daemon stop` (ExecStop) reported /// success, but via SIGKILL rather than a clean exit — exactly what systemd's @@ -24,7 +24,7 @@ namespace Netclaw.Cli.Tests.Cli; /// /// These tests exercise the two testable halves of the fix: (1) the internal /// poll now honors an injected -/// end-to-end (not just for its deadline math), so the up-to-200s +/// end-to-end (not just for its deadline math), so the bounded /// wait can be driven with a instead of a real sleep; and /// (2) the generated systemd unit's TimeoutStopSec= stays in lockstep with /// so systemd itself never SIGKILLs the whole @@ -49,7 +49,7 @@ public async Task WaitForExitAsync_ReturnsTrue_Immediately_WhenProcessAlreadyExi var manager = new DaemonManager(_paths, TimeProvider.System); using var exited = StartAndWaitForRealExit(); - var result = await manager.WaitForExitAsync(exited, TimeSpan.FromSeconds(200), CancellationToken.None); + var result = await manager.WaitForExitAsync(exited, DaemonConfig.GracefulShutdownBudget, CancellationToken.None); Assert.True(result); } diff --git a/src/Netclaw.Cli/Daemon/DaemonManager.cs b/src/Netclaw.Cli/Daemon/DaemonManager.cs index e56b7a7b3..3d8f6a9fa 100644 --- a/src/Netclaw.Cli/Daemon/DaemonManager.cs +++ b/src/Netclaw.Cli/Daemon/DaemonManager.cs @@ -180,8 +180,8 @@ await http.PostAsync( // Wait for graceful exit. DaemonConfig.GracefulShutdownBudget matches the daemon's own // Akka CoordinatedShutdown "before-service-unbind" phase timeout — where session // draining (SessionDrainHelper.DrainAsync) actually happens — so the CLI does not give - // up on a daemon that is still legitimately draining in-flight sessions (TurnLlmTimeout - // defaults to 3 minutes). See DaemonConfig.GracefulShutdownBudget remarks for the full + // up on a daemon that is still draining in-flight sessions. See + // DaemonConfig.GracefulShutdownBudget remarks for the full // layering this must respect: bounded drain < Akka phase timeout < this budget + the // grace window below < systemd's TimeoutStopSec= (netclaw-dev/netclaw#1664, #1665). var exitedWithinBudget = await WaitForExitAsync(process, DaemonConfig.GracefulShutdownBudget, cancellationToken); @@ -192,7 +192,7 @@ await http.PostAsync( // exactly this same budget boundary, and still needs to finish tearing down (actor // system termination, PID file cleanup) afterward. Poll a short additional grace // window before escalating to SIGKILL — production evidence (#1665) showed a - // daemon force-killed ~100ms from a clean exit because the CLI escalated the + // daemon was force-killed ~100ms from a clean exit because the CLI escalated the // instant its budget elapsed, with no headroom at all. exitedDuringGraceWindow = await WaitForExitAsync(process, DaemonConfig.CliForceKillGraceWindow, cancellationToken); } @@ -229,8 +229,8 @@ await http.PostAsync( if (exitedDuringGraceWindow) { // Clean exit — no kill needed — but worth surfacing: the daemon used its full - // graceful-shutdown budget, which usually means a session was still mid-LLM-call - // at shutdown. + // graceful-shutdown budget, which can mean that a session task did not stop + // after cancellation. return new DaemonResult(true, $"Daemon stopped (was PID {pid}); exited during the " + $"{DaemonConfig.CliForceKillGraceWindow.TotalSeconds:F0}s grace window after the " + @@ -241,7 +241,7 @@ await http.PostAsync( return new DaemonResult(true, $"Daemon stopped (was PID {pid}), but did not exit gracefully within " + $"{DaemonConfig.CliForceKillBudget.TotalSeconds:F0}s (budget + grace window) and had to be " + - "force-killed. This usually means a session was still mid-LLM-call at shutdown; if it " + + "force-killed. A session task or teardown step may have stalled; if it " + "recurs, check for stuck sessions before stopping the daemon."); } @@ -686,7 +686,7 @@ private static bool SendSignal(int pid, Signal signal) /// /// Polls until it exits or elapses. - /// Internal (not private) so tests can drive the up-to-200-second graceful-shutdown wait + /// Internal (not private) so tests can drive the bounded graceful-shutdown wait /// via an injected without a real wall-clock sleep: the poll /// delay is scheduled against (matching this repo's virtualized- /// timer convention, e.g. ConfigWatcherService), not a bare Task.Delay(ms). diff --git a/src/Netclaw.Configuration.Tests/DaemonConfigTests.cs b/src/Netclaw.Configuration.Tests/DaemonConfigTests.cs index a361fa6c0..59062f58c 100644 --- a/src/Netclaw.Configuration.Tests/DaemonConfigTests.cs +++ b/src/Netclaw.Configuration.Tests/DaemonConfigTests.cs @@ -325,12 +325,10 @@ public void ShutdownBudgetLayering_cli_force_kill_budget_stays_under_systemd_tim [Fact] public void ShutdownBudgetLayering_matches_the_documented_second_values() { - // Pins the concrete values referenced in netclaw-dev/netclaw#1665's evidence trail - // (200s phase timeout, 230s TimeoutStopSec) so a change to any constant is a visible, - // deliberate diff rather than a silent drift. - Assert.Equal(TimeSpan.FromSeconds(190), DaemonConfig.BoundedDrainTimeout); - Assert.Equal(TimeSpan.FromSeconds(200), DaemonConfig.GracefulShutdownBudget); - Assert.Equal(TimeSpan.FromSeconds(215), DaemonConfig.CliForceKillBudget); - Assert.Equal(TimeSpan.FromSeconds(230), DaemonConfig.SystemdTimeoutStopSec); + // Pin the agreed stop limits so a later change to any layer stays visible. + Assert.Equal(TimeSpan.FromSeconds(20), DaemonConfig.BoundedDrainTimeout); + Assert.Equal(TimeSpan.FromSeconds(30), DaemonConfig.GracefulShutdownBudget); + Assert.Equal(TimeSpan.FromSeconds(45), DaemonConfig.CliForceKillBudget); + Assert.Equal(TimeSpan.FromSeconds(60), DaemonConfig.SystemdTimeoutStopSec); } } diff --git a/src/Netclaw.Configuration/DaemonConfig.cs b/src/Netclaw.Configuration/DaemonConfig.cs index ffe4d8260..a764ae07d 100644 --- a/src/Netclaw.Configuration/DaemonConfig.cs +++ b/src/Netclaw.Configuration/DaemonConfig.cs @@ -21,10 +21,9 @@ public sealed record DaemonConfig public const int DefaultPort = 5199; /// - /// Worst-case time the daemon's graceful shutdown drain is allotted before something gives - /// up and forces termination. Sized to comfortably exceed SessionConfig.TurnLlmTimeout's - /// default (3 minutes) so a session mid-LLM-call during shutdown can finish draining instead - /// of being interrupted. + /// Worst-case time for the daemon's graceful shutdown phase. The session actor gives a + /// model call a short completion grace, then confirms cancellation before it grants a + /// restart candidate. Other work remains under the bounded drain deadline. /// /// Single source of truth shared by every shutdown-timing surface that must stay in /// lockstep (netclaw-dev/netclaw#1664, #1665): @@ -48,7 +47,7 @@ public sealed record DaemonConfig /// SIGKILL race the daemon could not win (evidence: a production daemon force-killed ~100ms /// from a clean exit). /// - public static readonly TimeSpan GracefulShutdownBudget = TimeSpan.FromSeconds(200); + public static readonly TimeSpan GracefulShutdownBudget = TimeSpan.FromSeconds(30); /// /// Safety margin subtracted from to produce diff --git a/src/Netclaw.Daemon.Tests/Services/DaemonRestartCoordinatorTests.cs b/src/Netclaw.Daemon.Tests/Services/DaemonRestartCoordinatorTests.cs index 0e548af4f..c5f9e3760 100644 --- a/src/Netclaw.Daemon.Tests/Services/DaemonRestartCoordinatorTests.cs +++ b/src/Netclaw.Daemon.Tests/Services/DaemonRestartCoordinatorTests.cs @@ -197,14 +197,9 @@ public async Task SessionDrainHelper_propagates_caller_cancellation() [Fact] public async Task SessionDrainHelper_daemon_stop_bound_times_out_instead_of_hanging_when_a_session_never_acks() { - // Mirrors the daemon-stop CoordinatedShutdown drain task wired in Program.cs - // (netclaw-dev/netclaw#1664): a session whose in-flight turn is parked on interactive - // tool approval never acks PrepareForDaemonRestart. Previously this call passed - // CancellationToken.None for the operation token and hung until Akka's own 200s - // before-service-unbind phase timeout abandoned the task. The bounded CTS below — - // sized from DaemonConfig.BoundedDrainTimeout (GracefulShutdownBudget minus - // DrainSafetyMargin) and driven by TimeProvider exactly as Program.cs constructs it — - // must make the drain complete with a timed-out result well before that. + // This test mirrors the daemon-stop drain task in Program.cs. A session task can + // ignore cancellation and fail to acknowledge drain. The bounded CTS must end the + // drain before Akka's phase timeout abandons the task. var time = new FakeTimeProvider(); var activeIds = new[] { "slack/approval-parked" }; var drain = new DrainControl(activeIds, activeIds); // never acknowledged diff --git a/src/Netclaw.Daemon/Program.cs b/src/Netclaw.Daemon/Program.cs index af0629e7a..e8b4f8fd0 100644 --- a/src/Netclaw.Daemon/Program.cs +++ b/src/Netclaw.Daemon/Program.cs @@ -1064,11 +1064,9 @@ static void ConfigureDaemonServices( { // Prevent coordinated shutdown from calling Environment.Exit(), // which would kill the process before the restart loop can iterate. - // The before-service-unbind phase needs a generous timeout (DaemonConfig. - // GracefulShutdownBudget) because sessions mid-LLM-call (TurnLlmTimeout defaults to - // 3 minutes) must finish before passivation can begin. See DaemonConfig. - // GracefulShutdownBudget remarks for the full set of surfaces this must stay in - // lockstep with. + // The before-service-unbind phase gives session drain time to confirm model + // cancellation and write restart candidates. DaemonConfig keeps this phase, + // the CLI wait, and the systemd stop timeout in order. akkaBuilder.AddHocon( DaemonShutdownConfiguration.BuildCoordinatedShutdownHocon(DaemonConfig.GracefulShutdownBudget), HoconAddMode.Prepend); @@ -1119,8 +1117,7 @@ static void ConfigureDaemonServices( // Runs in an early CoordinatedShutdown phase while actors are still alive. // If DaemonRestartCoordinator already drained sessions (config reload), the ingress // gate will be closed and this task skips its drain to avoid double-draining. - // The phase timeout (DaemonConfig.GracefulShutdownBudget) is generous because - // sessions mid-LLM-call must finish before passivation can begin. + // The phase timeout lets sessions confirm model cancellation before passivation. var cs = CoordinatedShutdown.Get(system); var sessionManager = registry.Get(); var ingressGate = sp.GetRequiredService(); @@ -1143,8 +1140,8 @@ static void ConfigureDaemonServices( // the phase timeout itself fires and abandons this task outright. // netclaw-dev/netclaw#1664: a session parked on interactive tool approval // never acks PrepareForDaemonRestart, so an unbounded wait here (previously - // CancellationToken.None, CancellationToken.None) hung for the full 200s - // phase timeout with no timeout of its own, leaking the abandoned drain task. + // CancellationToken.None, CancellationToken.None) had left the drain task + // active until the phase timeout, with no separate drain deadline. using var drainDeadlineCts = new CancellationTokenSource(DaemonConfig.BoundedDrainTimeout, tp); var drainResult = await SessionDrainHelper.DrainAsync(