Add tests which are verifying comp_req__launch_man__launcher_exit_shutdown

.bazelrc

@@ -72,6 +72,8 @@ build:x86_64-linux --extra_toolchains=@score_toolchains_rust//toolchains/ferroce
 
 test:x86_64-linux --//config:integration_mode=docker
 test:x86_64-linux --//config:unit_mode=host
+# Show a failing test's log (incl. the crash-dump banner) in the console.
+test:x86_64-linux --test_output=errors
 
 # Target configuration for CPU:AArch64|OS:Linux build (do not use it in case of system toolchains!)
 build:arm64-linux --config=stub
@@ -181,3 +183,11 @@ test:tsan --test_tag_filters=-no-tsan
 test:tsan --build_tests_only
 test:tsan --cxxopt=-Wno-maybe-uninitialized
 test:tsan --cxxopt=-Wno-redundant-move
+
+# Core-dump capture — opt in with --config=core_dump. The env var gates the
+# plugin at runtime; the build flag selects the gdb-equipped debug image so cores
+# can be analysed in-container and a backtrace is auto-captured. -c dbg is implied
+# so the auto-captured backtrace carries file/line symbols.
+build:core_dump --compilation_mode=dbg
+test:core_dump --test_env=SCORE_ENABLE_CORE_DUMP=1
+test:core_dump --//config:core_dump=True

.gitignore

@@ -71,6 +71,10 @@ target/
 tests/**/*.html
 tests/**/*.xml
 
+# Backup of the kernel core_pattern saved by the integration test core-dump
+# capture (--config=core_dump); auto-removed after restore
+/.original_core_pattern
+
 # IDE Code files
 *.orig
 .venv_docs

MODULE.bazel

@@ -19,6 +19,7 @@ bazel_dep(name = "rules_python", version = "1.8.5")
 bazel_dep(name = "rules_rust", version = "0.68.2-score")
 bazel_dep(name = "rules_cc", version = "0.2.17")
 bazel_dep(name = "rules_oci", version = "2.3.0")
+bazel_dep(name = "rules_distroless", version = "0.8.0")
 bazel_dep(name = "rules_shell", version = "0.6.1")
 bazel_dep(name = "aspect_rules_lint", version = "2.3.0")
 bazel_dep(name = "buildifier_prebuilt", version = "8.5.1")
@@ -175,6 +176,20 @@ oci.pull(
 )
 use_repo(oci, "debian-test-runtime", "debian-test-runtime_linux_amd64")
 
+# gdb (+ dependency closure) layered onto the debug test image; see
+# tests/utils/environments/x86_64-linux/gdb_apt.yaml. Regenerate the lockfile
+# with: bazel run @gdb_apt//:lock
+apt = use_extension("@rules_distroless//apt:extensions.bzl", "apt")
+apt.install(
+    name = "gdb_apt",
+    lock = "//tests/utils/environments/x86_64-linux:gdb_apt.lock.json",
+    manifest = "//tests/utils/environments/x86_64-linux:gdb_apt.yaml",
+    # Normalize to a merged-usr layout so the layer does not clobber the base
+    # image's /bin, /lib, ... usr-merge symlinks (which would break /bin/sh).
+    mergedusr = True,
+)
+use_repo(apt, "gdb_apt")
+
 bazel_dep(name = "score_baselibs", version = "0.2.10")
 
 # Hedron's Compile Commands Extractor for Bazel

config/BUILD

@@ -50,6 +50,29 @@ config_setting(
     },
 )
 
+# Opt in with --config=core_dump: selects the gdb-equipped debug test image so
+# core dumps can be analysed in-container (see .bazelrc, tests/integration/readme.md).
+bool_flag(
+    name = "core_dump",
+    build_setting_default = False,
+)
+
+config_setting(
+    name = "core_dump_enabled",
+    flag_values = {
+        ":core_dump": "True",
+    },
+)
+
+# Specialization of :integration_docker; select() prefers it when both match.
+config_setting(
+    name = "integration_docker_core_dump",
+    flag_values = {
+        ":integration_mode": "docker",
+        ":core_dump": "True",
+    },
+)
+
 # How to run unit tests:
 #
 # - qemu: in a QEMU virtual machine

score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp

@@ -14,6 +14,8 @@
 #include <ctime>
 
 #include <score/span.hpp>
+#include <algorithm>
+#include <chrono>
 #include <functional>
 #include <type_traits>
 #include <variant>
@@ -542,6 +544,23 @@ void Graph::forceKillProcesses()
     }
 }
 
+std::chrono::milliseconds Graph::getMaxTerminationTimeout()
+{
+    std::chrono::milliseconds max_timeout{0};
+    for (const auto& component : nodes_)
+    {
+        if (const ProcessInfoNode* process = std::get_if<ProcessInfoNode>(&component))
+        {
+            // Only processes with a live OS process still to stop count
+            if (process->getPid() > 0 && process->getState() < ProcessState::kTerminated)
+            {
+                max_timeout = std::max(max_timeout, process->getTerminationTimeout());
+            }
+        }
+    }
+    return max_timeout;
+}
+
 void Graph::updateCancelMessage()
 {
     ControlClientCode code = getPendingEvent();

score/launch_manager/src/daemon/src/process_group_manager/details/graph.hpp

@@ -288,6 +288,10 @@ class Graph final
     /// @brief For forced shutdown, kill all leftover processes
     void forceKillProcesses();
 
+    /// @brief Returns the largest configured shutdown_timeout across all running processes
+    /// @return The timeout in milliseconds, or zero if there are no live processes to stop.
+    std::chrono::milliseconds getMaxTerminationTimeout();
+
   private:
     /// @brief Helper function to identify a node with ready state "Terminated" from the legacy configuration
     bool nodeHasTerminatedDeps(IdentifierHash pg_name, uint32_t node_index);

score/launch_manager/src/daemon/src/process_group_manager/details/graph_UT.cpp

@@ -607,4 +607,65 @@ TEST_F(GraphUtilitiesTest, gettersSetters)
     EXPECT_LE(graph_time, after_time);
 }
 
+class GraphMaxTerminationTimeoutTest : public GraphTest
+{
+  protected:
+    uint32_t SetConfig() override
+    {
+        auto procs = generateProcessComponents(3);
+        auto count = procs.size();
+        procs[0].deployment_config.shutdown_timeout_ms = 1500;
+        procs[0].component_properties.application_profile.is_self_terminating = true;
+        procs[1].deployment_config.shutdown_timeout_ms = 500;
+        procs[2].deployment_config.shutdown_timeout_ms = 5000;
+        auto rts = generateRunTargets(2);
+        rts[1].depends_on = {procs[0].name, procs[1].name};
+        rts[2].depends_on = {procs[2].name};
+        const auto config = ConfigBuilder{}
+                                .setComponents(std::move(procs))
+                                .setRunTargets(std::move(rts))
+                                .setInitialRunTarget("Startup")
+                                .setFallbackRunTarget(std::move(fallback))
+                                .build();
+        config_.initialize(config);
+
+        return count;
+    }
+};
+
+TEST_F(GraphMaxTerminationTimeoutTest, ignoresNodesWithoutLiveProcess)
+{
+    RecordProperty(
+        "Description",
+        "Test that getMaxTerminationTimeout returns the max shutdown_timeout over running processes and ignores "
+        "never-started (pid == 0) nodes");
+
+    // No process started yet, so there is nothing to wait on.
+    EXPECT_EQ(graph_.getMaxTerminationTimeout(), 0ms);
+
+    // Bring up RunTarget0 (proc0 + proc1); proc2, with the largest timeout, stays idle in RunTarget1.
+    completeTransition(state_name(run_target_name(0)));
+
+    // Max over the two live processes; proc2's 5000 ms is ignored because it never started.
+    EXPECT_EQ(graph_.getMaxTerminationTimeout(), 1500ms);
+}
+
+TEST_F(GraphMaxTerminationTimeoutTest, ignoresTerminatedProcesses)
+{
+    RecordProperty(
+        "Description",
+        "Test that getMaxTerminationTimeout ignores processes that have already terminated, even if they carry the "
+        "largest shutdown_timeout");
+
+    completeTransition(state_name(run_target_name(0)));
+    ASSERT_EQ(graph_.getMaxTerminationTimeout(), 1500ms);
+
+    // proc0 is a self-terminating one-shot with the largest timeout; it exits on its own
+    // (status 0) and stays kTerminated, so it no longer needs to be waited on at shutdown.
+    static_cast<void>(graph_.getProcessInfoNode(0)->tryHandleTermination(0));
+
+    // Only proc1 remains live, so its timeout bounds the wait.
+    EXPECT_EQ(graph_.getMaxTerminationTimeout(), 500ms);
+}
+
 }  // namespace score::mw::lifecycle::internal

score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.cpp

@@ -432,6 +432,11 @@ score::mw::lifecycle::ProcessState ProcessInfoNode::getState() const
     return process_state_.load();
 }
 
+std::chrono::milliseconds ProcessInfoNode::getTerminationTimeout() const
+{
+    return config_ != nullptr ? config_->pgm_config_.termination_timeout_ms_ : std::chrono::milliseconds{0};
+}
+
 uint32_t ProcessInfoNode::getIndex() const
 {
     return process_index_;

score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.hpp

@@ -21,6 +21,7 @@
 #include "score/mw/launch_manager/supervision_control_client/isupervision_event_publisher.hpp"
 #include <score/stop_token.hpp>
 #include <atomic>
+#include <chrono>
 
 namespace score::mw::lifecycle::internal
 {
@@ -99,6 +100,9 @@ class ProcessInfoNode final : public IComponent
     /// @return The current state of this process.
     score::mw::lifecycle::ProcessState getState() const;
 
+    /// @return The configured shutdown_timeout for this process, or zero
+    std::chrono::milliseconds getTerminationTimeout() const;
+
     /// @return The ControlClientChannel for this process, or nullptr if none exists.
     ControlClientChannelP getControlClientChannel() const;
 

score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.cpp

@@ -153,10 +153,17 @@ void ProcessGroupManager::deinitialize()
     process_monitor_.reset();
     alive_monitor_thread_->stop();
     configuration_.deinitialize();
-    process_groups_.clear();
 
+    // Stop and join the worker threads BEFORE destroying the process groups.
+    // Worker threads run ProcessInfoNode::doWork(), which dereferences its Graph
+    // (nodeExecuted(), getState(), ...) via a raw back-pointer. If a transition is
+    // still completing on a worker thread (e.g. an in-progress switch to Off that
+    // is allowed to continue during shutdown), destroying the graphs first would be
+    // a use-after-free.
     thread_pool_.reset();
     worker_jobs_.reset();
+
+    process_groups_.clear();
     process_map_.reset();
 }
 
@@ -237,14 +244,16 @@ bool ProcessGroupManager::initializeProcessGroups()
             const auto* states = configuration_.getListOfProcessGroupStates(pg_name).value_or(nullptr);
             const uint32_t num_run_targets = states ? static_cast<uint32_t>(states->size()) : 0U;
 
-            process_groups_.push_back(std::make_shared<Graph>(
-                num_processes + num_run_targets,
-                &configuration_,
-                worker_jobs_,
-                &process_interface_,
-                process_map_,
-                *supervision_control_notifier_.get(),
-                this));
+            process_groups_.push_back(
+
+                std::make_shared<Graph>(
+                    num_processes + num_run_targets,
+                    &configuration_,
+                    worker_jobs_,
+                    &process_interface_,
+                    process_map_,
+                    *supervision_control_notifier_.get(),
+                    this));
         }
     }
     else
@@ -321,6 +330,7 @@ bool ProcessGroupManager::run()
     bool overflow_logged = false;
 
     if (result)
+    {
         while (!em_cancelled.load())
         {
             // Wait for something to happen...
@@ -350,6 +360,8 @@ bool ProcessGroupManager::run()
 
             watchdog_->serviceWatchdog();
         }
+        LM_LOG_INFO() << "ProcessGroupManager::run() - received SIGTERM, exiting";
+    }
 
     allProcessGroupsOff();
 
@@ -457,15 +469,24 @@ void ProcessGroupManager::allProcessGroupsOff()
     }
 
     LM_LOG_DEBUG() << "Wait for all process groups to complete the transition";
-    if (!waitForStateCompletion(GraphState::kInTransition, 1000))
+
+    // Bound the whole transition-to-Off wait by the slowest still-running process's
+    // shutdown_timeout (plus the SIGKILL grace), so every component's configured
+    // timeout is honoured. Processes deactivate in parallel.
+    const auto off_transition_timeout = graph.getMaxTerminationTimeout() + kMaxSigKillDelay;
+    if (!waitForStateCompletion(GraphState::kInTransition, static_cast<int32_t>(off_transition_timeout.count())))
     {
+        // Last resort: a process ignored even SIGKILL within its budget. Force-kill
+        // whatever is left and tear down the worker pool so shutdown can still proceed.
         LM_LOG_ERROR() << "NOTE: Transition to Off state timed out";
         thread_pool_->stop();
 
         for (auto& pg : process_groups_)
         {
             pg->forceKillProcesses();
         }
+
+        thread_pool_.reset();
     }
 }
 

score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.hpp

@@ -259,7 +259,8 @@ class ProcessGroupManager final : public ITransitionResultPublisher
     /// @brief Send all process groups to the "Off" state
     /// @details cancel any Graph for a process group not in the "Off" state, wait for up to 2 seconds for all graphs
     /// to be no longer in the `kCancelled` state, start a transition of remaining process groups to "Off" state,
-    /// and finally wait for up to a second for all graphs to complete.
+    /// and finally wait for all graphs to complete. The final wait is bounded by the largest configured per-process
+    /// shutdown_timeout (plus the SIGKILL grace) so each component's individual shutdown_timeout is respected.
     /// @warning Side effect: Depending if it is needed to forcefully terminate processes, worker jobs might be stopped
     /// after this call
     void allProcessGroupsOff();

tests/integration/lm_shutdown_during_rt_switch/BUILD

@@ -0,0 +1,56 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+load("@rules_cc//cc:cc_binary.bzl", "cc_binary")
+load("@rules_cc//cc:cc_library.bzl", "cc_library")
+load("//tests/utils/bazel:integration.bzl", "integration_test")
+
+cc_library(
+    name = "lm_shutdown_common",
+    hdrs = ["common.hpp"],
+)
+
+cc_binary(
+    name = "control_client_test_driver",
+    srcs = ["control_client_test_driver.cpp"],
+    deps = [
+        ":lm_shutdown_common",
+        "//score/launch_manager:control_cc",
+        "//score/launch_manager:lifecycle_cc",
+        "//tests/utils/test_helper",
+        "@googletest//:gtest_main",
+    ],
+)
+
+cc_binary(
+    name = "component_c",
+    srcs = ["component_c.cpp"],
+    deps = [
+        ":lm_shutdown_common",
+        "//score/launch_manager:lifecycle_cc",
+        "//tests/utils/test_helper",
+        "@googletest//:gtest_main",
+    ],
+)
+
+integration_test(
+    name = "lm_shutdown_during_rt_switch",
+    timeout = "short",
+    srcs = ["lm_shutdown_during_rt_switch.py"],
+    binaries = [
+        "//tests/utils/test_helper:process_hanging_on_sigterm",
+        ":component_c",
+        ":control_client_test_driver",
+        "//score/launch_manager",
+    ],
+    config = ":lm_shutdown_during_rt_switch.json",
+)