diff --git a/.bazelrc b/.bazelrc index 2a35375ed..7452c4571 100644 --- a/.bazelrc +++ b/.bazelrc @@ -72,6 +72,8 @@ build:x86_64-linux --extra_toolchains=@score_toolchains_rust//toolchains/ferroce test:x86_64-linux --//config:integration_mode=docker test:x86_64-linux --//config:unit_mode=host +# Show a failing test's log (incl. the crash-dump banner) in the console. +test:x86_64-linux --test_output=errors # Target configuration for CPU:AArch64|OS:Linux build (do not use it in case of system toolchains!) build:arm64-linux --config=stub @@ -181,3 +183,11 @@ test:tsan --test_tag_filters=-no-tsan test:tsan --build_tests_only test:tsan --cxxopt=-Wno-maybe-uninitialized test:tsan --cxxopt=-Wno-redundant-move + +# Core-dump capture — opt in with --config=core_dump. The env var gates the +# plugin at runtime; the build flag selects the gdb-equipped debug image so cores +# can be analysed in-container and a backtrace is auto-captured. -c dbg is implied +# so the auto-captured backtrace carries file/line symbols. +build:core_dump --compilation_mode=dbg +test:core_dump --test_env=SCORE_ENABLE_CORE_DUMP=1 +test:core_dump --//config:core_dump=True diff --git a/.gitignore b/.gitignore index f3a58c2f5..5f3da465e 100644 --- a/.gitignore +++ b/.gitignore @@ -71,6 +71,10 @@ target/ tests/**/*.html tests/**/*.xml +# Backup of the kernel core_pattern saved by the integration test core-dump +# capture (--config=core_dump); auto-removed after restore +/.original_core_pattern + # IDE Code files *.orig .venv_docs diff --git a/MODULE.bazel b/MODULE.bazel index 44cb0f648..7a42c2ecf 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -19,6 +19,7 @@ bazel_dep(name = "rules_python", version = "1.8.5") bazel_dep(name = "rules_rust", version = "0.68.2-score") bazel_dep(name = "rules_cc", version = "0.2.17") bazel_dep(name = "rules_oci", version = "2.3.0") +bazel_dep(name = "rules_distroless", version = "0.8.0") bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "aspect_rules_lint", version = "2.3.0") bazel_dep(name = "buildifier_prebuilt", version = "8.5.1") @@ -175,6 +176,20 @@ oci.pull( ) use_repo(oci, "debian-test-runtime", "debian-test-runtime_linux_amd64") +# gdb (+ dependency closure) layered onto the debug test image; see +# tests/utils/environments/x86_64-linux/gdb_apt.yaml. Regenerate the lockfile +# with: bazel run @gdb_apt//:lock +apt = use_extension("@rules_distroless//apt:extensions.bzl", "apt") +apt.install( + name = "gdb_apt", + lock = "//tests/utils/environments/x86_64-linux:gdb_apt.lock.json", + manifest = "//tests/utils/environments/x86_64-linux:gdb_apt.yaml", + # Normalize to a merged-usr layout so the layer does not clobber the base + # image's /bin, /lib, ... usr-merge symlinks (which would break /bin/sh). + mergedusr = True, +) +use_repo(apt, "gdb_apt") + bazel_dep(name = "score_baselibs", version = "0.2.10") # Hedron's Compile Commands Extractor for Bazel diff --git a/config/BUILD b/config/BUILD index 1b432cad5..4d3e19721 100644 --- a/config/BUILD +++ b/config/BUILD @@ -50,6 +50,29 @@ config_setting( }, ) +# Opt in with --config=core_dump: selects the gdb-equipped debug test image so +# core dumps can be analysed in-container (see .bazelrc, tests/integration/readme.md). +bool_flag( + name = "core_dump", + build_setting_default = False, +) + +config_setting( + name = "core_dump_enabled", + flag_values = { + ":core_dump": "True", + }, +) + +# Specialization of :integration_docker; select() prefers it when both match. +config_setting( + name = "integration_docker_core_dump", + flag_values = { + ":integration_mode": "docker", + ":core_dump": "True", + }, +) + # How to run unit tests: # # - qemu: in a QEMU virtual machine diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp b/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp index c503c84ef..abfa8920c 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp @@ -14,6 +14,8 @@ #include #include +#include +#include #include #include #include @@ -542,6 +544,23 @@ void Graph::forceKillProcesses() } } +std::chrono::milliseconds Graph::getMaxTerminationTimeout() +{ + std::chrono::milliseconds max_timeout{0}; + for (const auto& component : nodes_) + { + if (const ProcessInfoNode* process = std::get_if(&component)) + { + // Only processes with a live OS process still to stop count + if (process->getPid() > 0 && process->getState() < ProcessState::kTerminated) + { + max_timeout = std::max(max_timeout, process->getTerminationTimeout()); + } + } + } + return max_timeout; +} + void Graph::updateCancelMessage() { ControlClientCode code = getPendingEvent(); diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/graph.hpp b/score/launch_manager/src/daemon/src/process_group_manager/details/graph.hpp index ba383ed08..4849623d4 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/graph.hpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/graph.hpp @@ -288,6 +288,10 @@ class Graph final /// @brief For forced shutdown, kill all leftover processes void forceKillProcesses(); + /// @brief Returns the largest configured shutdown_timeout across all running processes + /// @return The timeout in milliseconds, or zero if there are no live processes to stop. + std::chrono::milliseconds getMaxTerminationTimeout(); + private: /// @brief Helper function to identify a node with ready state "Terminated" from the legacy configuration bool nodeHasTerminatedDeps(IdentifierHash pg_name, uint32_t node_index); diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/graph_UT.cpp b/score/launch_manager/src/daemon/src/process_group_manager/details/graph_UT.cpp index 2f4181357..5391b4a73 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/graph_UT.cpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/graph_UT.cpp @@ -607,4 +607,65 @@ TEST_F(GraphUtilitiesTest, gettersSetters) EXPECT_LE(graph_time, after_time); } +class GraphMaxTerminationTimeoutTest : public GraphTest +{ + protected: + uint32_t SetConfig() override + { + auto procs = generateProcessComponents(3); + auto count = procs.size(); + procs[0].deployment_config.shutdown_timeout_ms = 1500; + procs[0].component_properties.application_profile.is_self_terminating = true; + procs[1].deployment_config.shutdown_timeout_ms = 500; + procs[2].deployment_config.shutdown_timeout_ms = 5000; + auto rts = generateRunTargets(2); + rts[1].depends_on = {procs[0].name, procs[1].name}; + rts[2].depends_on = {procs[2].name}; + const auto config = ConfigBuilder{} + .setComponents(std::move(procs)) + .setRunTargets(std::move(rts)) + .setInitialRunTarget("Startup") + .setFallbackRunTarget(std::move(fallback)) + .build(); + config_.initialize(config); + + return count; + } +}; + +TEST_F(GraphMaxTerminationTimeoutTest, ignoresNodesWithoutLiveProcess) +{ + RecordProperty( + "Description", + "Test that getMaxTerminationTimeout returns the max shutdown_timeout over running processes and ignores " + "never-started (pid == 0) nodes"); + + // No process started yet, so there is nothing to wait on. + EXPECT_EQ(graph_.getMaxTerminationTimeout(), 0ms); + + // Bring up RunTarget0 (proc0 + proc1); proc2, with the largest timeout, stays idle in RunTarget1. + completeTransition(state_name(run_target_name(0))); + + // Max over the two live processes; proc2's 5000 ms is ignored because it never started. + EXPECT_EQ(graph_.getMaxTerminationTimeout(), 1500ms); +} + +TEST_F(GraphMaxTerminationTimeoutTest, ignoresTerminatedProcesses) +{ + RecordProperty( + "Description", + "Test that getMaxTerminationTimeout ignores processes that have already terminated, even if they carry the " + "largest shutdown_timeout"); + + completeTransition(state_name(run_target_name(0))); + ASSERT_EQ(graph_.getMaxTerminationTimeout(), 1500ms); + + // proc0 is a self-terminating one-shot with the largest timeout; it exits on its own + // (status 0) and stays kTerminated, so it no longer needs to be waited on at shutdown. + static_cast(graph_.getProcessInfoNode(0)->tryHandleTermination(0)); + + // Only proc1 remains live, so its timeout bounds the wait. + EXPECT_EQ(graph_.getMaxTerminationTimeout(), 500ms); +} + } // namespace score::mw::lifecycle::internal diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.cpp b/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.cpp index 806a11d9c..01fd8f347 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.cpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.cpp @@ -432,6 +432,11 @@ score::mw::lifecycle::ProcessState ProcessInfoNode::getState() const return process_state_.load(); } +std::chrono::milliseconds ProcessInfoNode::getTerminationTimeout() const +{ + return config_ != nullptr ? config_->pgm_config_.termination_timeout_ms_ : std::chrono::milliseconds{0}; +} + uint32_t ProcessInfoNode::getIndex() const { return process_index_; diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.hpp b/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.hpp index 8310330a8..237b4e0b1 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.hpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.hpp @@ -21,6 +21,7 @@ #include "score/mw/launch_manager/supervision_control_client/isupervision_event_publisher.hpp" #include #include +#include namespace score::mw::lifecycle::internal { @@ -99,6 +100,9 @@ class ProcessInfoNode final : public IComponent /// @return The current state of this process. score::mw::lifecycle::ProcessState getState() const; + /// @return The configured shutdown_timeout for this process, or zero + std::chrono::milliseconds getTerminationTimeout() const; + /// @return The ControlClientChannel for this process, or nullptr if none exists. ControlClientChannelP getControlClientChannel() const; diff --git a/score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.cpp b/score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.cpp index 358fa9e11..6caaabb92 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.cpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.cpp @@ -153,10 +153,17 @@ void ProcessGroupManager::deinitialize() process_monitor_.reset(); alive_monitor_thread_->stop(); configuration_.deinitialize(); - process_groups_.clear(); + // Stop and join the worker threads BEFORE destroying the process groups. + // Worker threads run ProcessInfoNode::doWork(), which dereferences its Graph + // (nodeExecuted(), getState(), ...) via a raw back-pointer. If a transition is + // still completing on a worker thread (e.g. an in-progress switch to Off that + // is allowed to continue during shutdown), destroying the graphs first would be + // a use-after-free. thread_pool_.reset(); worker_jobs_.reset(); + + process_groups_.clear(); process_map_.reset(); } @@ -237,14 +244,16 @@ bool ProcessGroupManager::initializeProcessGroups() const auto* states = configuration_.getListOfProcessGroupStates(pg_name).value_or(nullptr); const uint32_t num_run_targets = states ? static_cast(states->size()) : 0U; - process_groups_.push_back(std::make_shared( - num_processes + num_run_targets, - &configuration_, - worker_jobs_, - &process_interface_, - process_map_, - *supervision_control_notifier_.get(), - this)); + process_groups_.push_back( + + std::make_shared( + num_processes + num_run_targets, + &configuration_, + worker_jobs_, + &process_interface_, + process_map_, + *supervision_control_notifier_.get(), + this)); } } else @@ -321,6 +330,7 @@ bool ProcessGroupManager::run() bool overflow_logged = false; if (result) + { while (!em_cancelled.load()) { // Wait for something to happen... @@ -350,6 +360,8 @@ bool ProcessGroupManager::run() watchdog_->serviceWatchdog(); } + LM_LOG_INFO() << "ProcessGroupManager::run() - received SIGTERM, exiting"; + } allProcessGroupsOff(); @@ -457,8 +469,15 @@ void ProcessGroupManager::allProcessGroupsOff() } LM_LOG_DEBUG() << "Wait for all process groups to complete the transition"; - if (!waitForStateCompletion(GraphState::kInTransition, 1000)) + + // Bound the whole transition-to-Off wait by the slowest still-running process's + // shutdown_timeout (plus the SIGKILL grace), so every component's configured + // timeout is honoured. Processes deactivate in parallel. + const auto off_transition_timeout = graph.getMaxTerminationTimeout() + kMaxSigKillDelay; + if (!waitForStateCompletion(GraphState::kInTransition, static_cast(off_transition_timeout.count()))) { + // Last resort: a process ignored even SIGKILL within its budget. Force-kill + // whatever is left and tear down the worker pool so shutdown can still proceed. LM_LOG_ERROR() << "NOTE: Transition to Off state timed out"; thread_pool_->stop(); @@ -466,6 +485,8 @@ void ProcessGroupManager::allProcessGroupsOff() { pg->forceKillProcesses(); } + + thread_pool_.reset(); } } diff --git a/score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.hpp b/score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.hpp index fdb998869..33cde73da 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.hpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.hpp @@ -259,7 +259,8 @@ class ProcessGroupManager final : public ITransitionResultPublisher /// @brief Send all process groups to the "Off" state /// @details cancel any Graph for a process group not in the "Off" state, wait for up to 2 seconds for all graphs /// to be no longer in the `kCancelled` state, start a transition of remaining process groups to "Off" state, - /// and finally wait for up to a second for all graphs to complete. + /// and finally wait for all graphs to complete. The final wait is bounded by the largest configured per-process + /// shutdown_timeout (plus the SIGKILL grace) so each component's individual shutdown_timeout is respected. /// @warning Side effect: Depending if it is needed to forcefully terminate processes, worker jobs might be stopped /// after this call void allProcessGroupsOff(); diff --git a/tests/integration/lm_shutdown_during_rt_switch/BUILD b/tests/integration/lm_shutdown_during_rt_switch/BUILD new file mode 100644 index 000000000..d5b2c0d52 --- /dev/null +++ b/tests/integration/lm_shutdown_during_rt_switch/BUILD @@ -0,0 +1,56 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("//tests/utils/bazel:integration.bzl", "integration_test") + +cc_library( + name = "lm_shutdown_common", + hdrs = ["common.hpp"], +) + +cc_binary( + name = "control_client_test_driver", + srcs = ["control_client_test_driver.cpp"], + deps = [ + ":lm_shutdown_common", + "//score/launch_manager:control_cc", + "//score/launch_manager:lifecycle_cc", + "//tests/utils/test_helper", + "@googletest//:gtest_main", + ], +) + +cc_binary( + name = "component_c", + srcs = ["component_c.cpp"], + deps = [ + ":lm_shutdown_common", + "//score/launch_manager:lifecycle_cc", + "//tests/utils/test_helper", + "@googletest//:gtest_main", + ], +) + +integration_test( + name = "lm_shutdown_during_rt_switch", + timeout = "short", + srcs = ["lm_shutdown_during_rt_switch.py"], + binaries = [ + "//tests/utils/test_helper:process_hanging_on_sigterm", + ":component_c", + ":control_client_test_driver", + "//score/launch_manager", + ], + config = ":lm_shutdown_during_rt_switch.json", +) diff --git a/tests/integration/lm_shutdown_during_rt_switch/common.hpp b/tests/integration/lm_shutdown_during_rt_switch/common.hpp new file mode 100644 index 000000000..4e45fe3dd --- /dev/null +++ b/tests/integration/lm_shutdown_during_rt_switch/common.hpp @@ -0,0 +1,34 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +#ifndef SCORE_TESTS_INTEGRATION_LM_SHUTDOWN_COMMON_HPP +#define SCORE_TESTS_INTEGRATION_LM_SHUTDOWN_COMMON_HPP + +#include + +/// @brief Written by component_a when it has reported running (run_target_a is +/// active). +constexpr std::string_view a_started = "component_a_started"; + +/// @brief Written by component_a when it starts being terminated (i.e. the +/// switch away from run_target_a has begun). component_a then stalls, which +/// keeps the run-target switch in progress and gives the test a deterministic +/// window in which to send SIGTERM to the launch manager. +constexpr std::string_view a_terminating = "component_a_terminating"; + +/// @brief Written by component_c when it starts. component_c belongs only to +/// run_target_c, so this file must NEVER appear: a SIGTERM to the launch manager +/// during the switch must cancel the pending activation of run_target_c. +constexpr std::string_view c_started = "component_c_started"; + +#endif // SCORE_TESTS_INTEGRATION_LM_SHUTDOWN_COMMON_HPP diff --git a/tests/integration/lm_shutdown_during_rt_switch/component_c.cpp b/tests/integration/lm_shutdown_during_rt_switch/component_c.cpp new file mode 100644 index 000000000..299ef0385 --- /dev/null +++ b/tests/integration/lm_shutdown_during_rt_switch/component_c.cpp @@ -0,0 +1,46 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include +#include + +#include "common.hpp" +#include "tests/utils/test_helper/test_helper.hpp" +#include + +// component_c belongs only to run_target_c. Because the switch to run_target_c +// must be cancelled by the SIGTERM sent to the launch manager, this process must +// never be launched. Should it ever start, it records `c_started`, which makes +// both the control client and the Python-side assertions fail. +TEST(LmShutdownDuringRtSwitch, ComponentC) +{ + TEST_STEP("Report running") + { + // This code should be never executed. In Python code there is also an assertion + // that component_c must not be started (i.e. c_started should not exist). + // This is a second line of defense in case the Python code is not executed or fails to detect the problem. + ADD_FAILURE() << "component_c must never be started"; + + EXPECT_TRUE(touch_file(c_started)); + score::mw::lifecycle::report_running(); + } + + while (!TestRunner::exitRequested) + { + pause(); + } +} + +int main() +{ + return TestRunner(__FILE__).RunTests(); +} diff --git a/tests/integration/lm_shutdown_during_rt_switch/control_client_test_driver.cpp b/tests/integration/lm_shutdown_during_rt_switch/control_client_test_driver.cpp new file mode 100644 index 000000000..e3c2b8207 --- /dev/null +++ b/tests/integration/lm_shutdown_during_rt_switch/control_client_test_driver.cpp @@ -0,0 +1,76 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include +#include +#include + +#include "common.hpp" +#include "tests/utils/test_helper/test_helper.hpp" +#include +#include + +// The Launch Manager shall exit after performing a shutdown - stopping all the +// processes it owns in dependency order - when requested (i.e. when it receives +// a SIGTERM). A shutdown request takes priority over an in-progress run-target +// switch, which must therefore be cancelled. +// +// This control client activates run_target_a and then requests a switch to +// run_target_c. component_a (only part of run_target_a) stalls while it is being +// terminated during that switch, so the switch is still in progress when the +// test sends a SIGTERM to the launch manager from the Python side. The launch +// manager must then cancel the pending switch (component_c, only part of +// run_target_c, must never start) and shut everything down. +TEST(LmShutdownDuringRtSwitch, ControlClient) +{ + score::mw::lifecycle::ControlClient client{}; + ASSERT_TRUE(check_clean({test_end_location, a_started, a_terminating, c_started})); + + TEST_STEP("Report running") + { + score::mw::lifecycle::report_running(); + } + + TEST_STEP("Activate run_target_a") + { + score::cpp::stop_token stop_token; + auto result = client.ActivateRunTarget("run_target_a").Get(stop_token); + EXPECT_TRUE(result.has_value()) << "Activating run_target_a failed: " << result.error().Message(); + EXPECT_TRUE(std::filesystem::exists(a_started)) << "component_a was not started"; + } + + TEST_STEP("Request switch to run_target_c") + { + // Fire-and-forget: this transition is expected to be cancelled by an + // external SIGTERM to the launch manager, so we must not wait for a + // result. The launch manager will shut this process down instead of ever + // completing the switch. + client.ActivateRunTarget("run_target_c"); + } + + // Block until the launch manager terminates us as part of its own shutdown. + while (!TestRunner::exitRequested) + { + pause(); + } + + TEST_STEP("Verify run_target_c was never activated") + { + EXPECT_FALSE(std::filesystem::exists(c_started)) + << "run_target_c must not be activated: a SIGTERM to the launch manager must cancel the pending switch"; + } +} + +int main() +{ + return TestRunner(__FILE__, TerminationBehavior::kWait, TerminationNotification::kTestEnd).RunTests(); +} diff --git a/tests/integration/lm_shutdown_during_rt_switch/lm_shutdown_during_rt_switch.json b/tests/integration/lm_shutdown_during_rt_switch/lm_shutdown_during_rt_switch.json new file mode 100644 index 000000000..9376cba98 --- /dev/null +++ b/tests/integration/lm_shutdown_during_rt_switch/lm_shutdown_during_rt_switch.json @@ -0,0 +1,114 @@ +{ + "schema_version": 1, + "defaults": { + "deployment_config": { + "bin_dir": "/tmp/tests/lm_shutdown_during_rt_switch", + "ready_timeout": 1.0, + "shutdown_timeout": 1.0, + "ready_recovery_action": { + "restart": { + "number_of_attempts": 0 + } + }, + "recovery_action": { + "switch_run_target": { + "run_target": "fallback_run_target" + } + }, + "environmental_variables": { + "LD_LIBRARY_PATH": "/opt/lib" + }, + "sandbox": { + "uid": 0, + "gid": 0, + "scheduling_policy": "SCHED_OTHER", + "scheduling_priority": 0 + } + }, + "component_properties": { + "application_profile": { + "application_type": "Reporting", + "is_self_terminating": false, + "alive_supervision": { + "reporting_cycle": 0.1, + "min_indications": 1, + "max_indications": 3, + "failed_cycles_tolerance": 1 + } + }, + "ready_condition": { + "process_state": "Running" + } + } + }, + "components": { + "component_initial": { + "component_properties": { + "binary_name": "control_client_test_driver", + "application_profile": { + "application_type": "State_Manager", + "alive_supervision": { + "min_indications": 0 + } + } + }, + "deployment_config": { + "ready_timeout": 1.0, + "shutdown_timeout": 1.0, + "environmental_variables": { + "PROCESSIDENTIFIER": "control_client_test_driver" + } + } + }, + "component_a": { + "component_properties": { + "binary_name": "process_hanging_on_sigterm" + }, + "deployment_config": { + "shutdown_timeout": 5.0, + "environmental_variables": { + "PROCESSIDENTIFIER": "component_a" + } + } + }, + "component_c": { + "component_properties": { + "binary_name": "component_c" + }, + "deployment_config": { + "environmental_variables": { + "PROCESSIDENTIFIER": "component_c" + } + } + } + }, + "run_targets": { + "Startup": { + "depends_on": [ + "component_initial" + ] + }, + "run_target_a": { + "depends_on": [ + "component_initial", + "component_a" + ] + }, + "run_target_c": { + "depends_on": [ + "component_initial", + "component_c" + ] + }, + "Off": { + "depends_on": [] + } + }, + "initial_run_target": "Startup", + "alive_supervision": { + "evaluation_cycle": 0.05 + }, + "fallback_run_target": { + "depends_on": [] + } +} diff --git a/tests/integration/lm_shutdown_during_rt_switch/lm_shutdown_during_rt_switch.py b/tests/integration/lm_shutdown_during_rt_switch/lm_shutdown_during_rt_switch.py new file mode 100644 index 000000000..52cd0141d --- /dev/null +++ b/tests/integration/lm_shutdown_during_rt_switch/lm_shutdown_during_rt_switch.py @@ -0,0 +1,62 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +from tests.utils.testing_utils.run_until_file_deployed import run_until_file_deployed +from tests.utils.testing_utils.setup_test import setup_test +from tests.utils.testing_utils.test_results import assert_test_results +from attribute_plugin import add_test_properties + + +@add_test_properties( + fully_verifies=[], + partially_verifies=[ + "comp_req__launch_man__launcher_exit_shutdown", + ], + test_type="requirements-based", + derivation_technique="requirements-analysis", +) +def test_lm_shutdown(target, setup_test, assert_test_results, remote_test_dir): + """ + Objective: Verifies that the Launch Manager exits after performing a shutdown + (stopping all processes it owns) when requested via SIGTERM, and that this + shutdown takes priority over an in-progress run-target switch (the switch is + cancelled). + + The control client activates run_target_a and then requests a switch to + run_target_c. component_a (only part of run_target_a) stalls while it is being + terminated during the switch, keeping the switch in progress. That window is + signalled by the file `component_a_terminating`, at which point the launch + manager is sent a SIGTERM. + + Expected Behaviour: The launch manager cancels the pending switch - so + component_c (only part of run_target_c) is never started - stops all the + processes it owns, and exits cleanly. + """ + + new_config_path = str(remote_test_dir / "etc/lm_shutdown_during_rt_switch.bin") + a_terminating = remote_test_dir / "component_a_terminating" + + # Run until `component_a_terminating` is deployed so we can request shutdown during + # the transition to run target c + run_until_file_deployed( + target=target, + binary_path=str(remote_test_dir / "launch_manager"), + file_path=a_terminating, + cwd=str(remote_test_dir), + args=["-c", new_config_path], + timeout_s=10.0, + ) + + # component_c never runs (the pending switch was cancelled), so it produces no XML + # result; component_a and the control client shut down gracefully. The control + # client additionally asserts that run_target_c was never activated. + assert_test_results({"control_client_test_driver.xml", "component_a.xml"}) diff --git a/tests/integration/lm_shutdown_during_switch_to_off/BUILD b/tests/integration/lm_shutdown_during_switch_to_off/BUILD new file mode 100644 index 000000000..88512b86c --- /dev/null +++ b/tests/integration/lm_shutdown_during_switch_to_off/BUILD @@ -0,0 +1,44 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("//tests/utils/bazel:integration.bzl", "integration_test") + +cc_library( + name = "lm_shutdown_common", + hdrs = ["common.hpp"], +) + +cc_binary( + name = "control_client_test_driver", + srcs = ["control_client_test_driver.cpp"], + deps = [ + ":lm_shutdown_common", + "//score/launch_manager:control_cc", + "//score/launch_manager:lifecycle_cc", + "//tests/utils/test_helper", + "@googletest//:gtest_main", + ], +) + +integration_test( + name = "lm_shutdown_during_switch_to_off", + timeout = "short", + srcs = ["lm_shutdown_during_switch_to_off.py"], + binaries = [ + "//tests/utils/test_helper:process_hanging_on_sigterm", + ":control_client_test_driver", + "//score/launch_manager", + ], + config = ":lm_shutdown_during_switch_to_off.json", +) diff --git a/tests/integration/lm_shutdown_during_switch_to_off/common.hpp b/tests/integration/lm_shutdown_during_switch_to_off/common.hpp new file mode 100644 index 000000000..f3b5eb946 --- /dev/null +++ b/tests/integration/lm_shutdown_during_switch_to_off/common.hpp @@ -0,0 +1,30 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +#ifndef SCORE_TESTS_INTEGRATION_LM_SHUTDOWN_DURING_SWITCH_TO_OFF_COMMON_HPP +#define SCORE_TESTS_INTEGRATION_LM_SHUTDOWN_DURING_SWITCH_TO_OFF_COMMON_HPP + +#include + +/// @brief Written by component_a when it has reported running (run_target_a is +/// active). +constexpr std::string_view a_started = "component_a_started"; + +/// @brief Written by component_a when it starts being terminated (i.e. the +/// switch away from run_target_a - here, the switch to the "Off" run target - +/// has begun). component_a then stalls, which keeps the run-target switch in +/// progress and gives the test a deterministic window in which to send SIGTERM +/// to the launch manager. +constexpr std::string_view a_terminating = "component_a_terminating"; + +#endif // SCORE_TESTS_INTEGRATION_LM_SHUTDOWN_DURING_SWITCH_TO_OFF_COMMON_HPP diff --git a/tests/integration/lm_shutdown_during_switch_to_off/control_client_test_driver.cpp b/tests/integration/lm_shutdown_during_switch_to_off/control_client_test_driver.cpp new file mode 100644 index 000000000..9a8187e62 --- /dev/null +++ b/tests/integration/lm_shutdown_during_switch_to_off/control_client_test_driver.cpp @@ -0,0 +1,76 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include +#include +#include + +#include "common.hpp" +#include "tests/utils/test_helper/test_helper.hpp" +#include +#include + +// The Launch Manager shall exit after performing a shutdown - stopping all the +// processes it owns in dependency order - when requested (i.e. when it receives +// a SIGTERM). +// +// This variant differs from lm_shutdown_during_rt_switch: instead of switching +// to another (non-Off) run target, the control client explicitly switches to +// the "Off" run target. component_a (part of run_target_a) stalls while it is +// being terminated during that switch, so the switch to Off is still in progress +// when the test sends a SIGTERM to the launch manager from the Python side. +// +// Because the process group is ALREADY heading to Off, the SIGTERM-triggered +// shutdown must simply let that in-progress switch to Off continue to completion +// - it must NOT cancel the explicit switch to Off and redo it. Either way the +// launch manager must end up stopping everything it owns and exit cleanly. +TEST(LmShutdownDuringSwitchToOff, ControlClient) +{ + score::mw::lifecycle::ControlClient client{}; + ASSERT_TRUE(check_clean({test_end_location, a_started, a_terminating})); + + const auto pid = getpid(); + const std::string step_msg = "Report running with pid == " + std::to_string(pid); + + TEST_STEP(step_msg) + { + score::mw::lifecycle::report_running(); + } + + TEST_STEP("Activate run_target_a") + { + score::cpp::stop_token stop_token; + auto result = client.ActivateRunTarget("run_target_a").Get(stop_token); + EXPECT_TRUE(result.has_value()) << "Activating run_target_a failed: " << result.error().Message(); + EXPECT_TRUE(std::filesystem::exists(a_started)) << "component_a was not started"; + } + + TEST_STEP("Request switch to Off") + { + // Fire-and-forget: switching to the "Off" run target terminates this + // control client too (it is not part of "Off"), so we must not wait for a + // result. The launch manager will shut this process down as part of the + // switch to Off. + client.ActivateRunTarget("Off"); + } + + // Block until the launch manager terminates us as part of its own shutdown. + while (!TestRunner::exitRequested) + { + pause(); + } +} + +int main() +{ + return TestRunner(__FILE__, TerminationBehavior::kContinue, TerminationNotification::kTestEnd).RunTests(); +} diff --git a/tests/integration/lm_shutdown_during_switch_to_off/lm_shutdown_during_switch_to_off.json b/tests/integration/lm_shutdown_during_switch_to_off/lm_shutdown_during_switch_to_off.json new file mode 100644 index 000000000..ac67c6af7 --- /dev/null +++ b/tests/integration/lm_shutdown_during_switch_to_off/lm_shutdown_during_switch_to_off.json @@ -0,0 +1,98 @@ +{ + "schema_version": 1, + "defaults": { + "deployment_config": { + "bin_dir": "/tmp/tests/lm_shutdown_during_switch_to_off", + "ready_timeout": 1.0, + "shutdown_timeout": 1.0, + "ready_recovery_action": { + "restart": { + "number_of_attempts": 0 + } + }, + "recovery_action": { + "switch_run_target": { + "run_target": "fallback_run_target" + } + }, + "environmental_variables": { + "LD_LIBRARY_PATH": "/opt/lib" + }, + "sandbox": { + "uid": 0, + "gid": 0, + "scheduling_policy": "SCHED_OTHER", + "scheduling_priority": 0 + } + }, + "component_properties": { + "application_profile": { + "application_type": "Reporting", + "is_self_terminating": false, + "alive_supervision": { + "reporting_cycle": 0.1, + "min_indications": 1, + "max_indications": 3, + "failed_cycles_tolerance": 1 + } + }, + "ready_condition": { + "process_state": "Running" + } + } + }, + "components": { + "component_initial": { + "component_properties": { + "binary_name": "control_client_test_driver", + "application_profile": { + "application_type": "State_Manager", + "alive_supervision": { + "min_indications": 0 + } + } + }, + "deployment_config": { + "ready_timeout": 1.0, + "shutdown_timeout": 5.0, + "environmental_variables": { + "PROCESSIDENTIFIER": "control_client_test_driver" + } + } + }, + "component_a": { + "component_properties": { + "binary_name": "process_hanging_on_sigterm" + }, + "deployment_config": { + "shutdown_timeout": 5.0, + "environmental_variables": { + "PROCESSIDENTIFIER": "component_a" + } + } + } + }, + "run_targets": { + "Startup": { + "depends_on": [ + "component_initial" + ] + }, + "run_target_a": { + "depends_on": [ + "component_initial", + "component_a" + ] + }, + "Off": { + "depends_on": [] + } + }, + "initial_run_target": "Startup", + "alive_supervision": { + "evaluation_cycle": 0.05 + }, + "fallback_run_target": { + "depends_on": [] + } +} diff --git a/tests/integration/lm_shutdown_during_switch_to_off/lm_shutdown_during_switch_to_off.py b/tests/integration/lm_shutdown_during_switch_to_off/lm_shutdown_during_switch_to_off.py new file mode 100644 index 000000000..a35641533 --- /dev/null +++ b/tests/integration/lm_shutdown_during_switch_to_off/lm_shutdown_during_switch_to_off.py @@ -0,0 +1,64 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +from tests.utils.testing_utils.run_until_file_deployed import run_until_file_deployed +from tests.utils.testing_utils.setup_test import setup_test +from tests.utils.testing_utils.test_results import assert_test_results +from attribute_plugin import add_test_properties + + +@add_test_properties( + fully_verifies=[], + partially_verifies=[ + "comp_req__launch_man__launcher_exit_shutdown", + ], + test_type="requirements-based", + derivation_technique="requirements-analysis", +) +def test_lm_shutdown(target, setup_test, assert_test_results, remote_test_dir): + """ + Objective: Verifies that the Launch Manager exits after performing a shutdown + (stopping all processes it owns) when a SIGTERM arrives while an explicit switch + to the "Off" run target is already in progress. + + The control client activates run_target_a and then explicitly requests a switch + to the "Off" run target. component_a (part of run_target_a) stalls while it is + being terminated during that switch, keeping the switch to Off in progress. That + window is signalled by the file `component_a_terminating`, at which point the + launch manager is sent a SIGTERM. + + Expected Behaviour: The launch manager lets the in-progress switch to Off + continue, stops all the processes it owns, and exits cleanly. It honours each + component's shutdown_timeout, so component_a - which stalls for less than its + shutdown_timeout - exits gracefully (producing its XML result) rather than being + force-terminated. + """ + + new_config_path = str(remote_test_dir / "etc/lm_shutdown_during_switch_to_off.bin") + a_terminating = remote_test_dir / "component_a_terminating" + + # Run until `component_a_terminating` is deployed so we can send SIGTERM + # to launch manager during the transition to Off + run_until_file_deployed( + target=target, + binary_path=str(remote_test_dir / "launch_manager"), + file_path=a_terminating, + cwd=str(remote_test_dir), + args=["-c", new_config_path], + timeout_s=10.0, + ) + + # Both processes are stopped gracefully as part of the switch to Off and produce + # their XML results: the control client is terminated when the switch to Off + # begins, and component_a exits within its shutdown_timeout (which the launch + # manager honours) instead of being force-terminated. + assert_test_results({"control_client_test_driver.xml", "component_a.xml"}) diff --git a/tests/integration/readme.md b/tests/integration/readme.md index 92d514dd3..2521ec3d2 100644 --- a/tests/integration/readme.md +++ b/tests/integration/readme.md @@ -23,3 +23,121 @@ Currently the following configs are supported: - `host` - `x86_64-linux` +## Crash dumps (core dumps) + +Core-dump capture is **opt-in**: add `--config=core_dump` to include the support. **Attention: This influences the kernel `core_pattern` value of your host system!** + + +How it works: +- `--config=core_dump` forwards `SCORE_ENABLE_CORE_DUMP=1` into the test + environment and sets the `//config:core_dump` build flag (see `.bazelrc`); the + shared pytest plugin keys off the env var, individual tests need no adaptions. +- The build flag selects a **debug image variant** (`score_itf_examples_debug`, + the normal image plus `gdb`) so cores can be analysed inside the container. + Normal runs keep the slim `score_itf_examples` image, unchanged. +- The sandbox container runs privileged with an unlimited core-file `ulimit` and + a read-write bind-mount of the workspace root. +- A shared fixture sets the kernel `core_pattern` to a sandbox-local path + (`/tmp/score_cores/core.%e.%p.%s.%t`). On teardown it symbolizes each core + **inside the container** (where the binary and matching libraries live) into a + `.bt.txt` backtrace, copies the cores and backtraces into the Bazel test + outputs, and restores the original `core_pattern`. +- Before changing `core_pattern`, the fixture mirrors the original value to + `.original_core_pattern` in the workspace root. The sandboxed test process sees + the source tree read-only, so this file is written from inside the privileged + container via the workspace bind-mount (hence it is root-owned). It is removed + again once the value is restored, so it exists only if a run is force-killed. + +Further technical limitations are described in [Important: the `core_pattern` is a global kernel setting](#important-the-core_pattern-is-a-global-kernel-setting) + +### Getting a crash dump + +Run the (crashing) test with `--config=core_dump`, disabling the cache so it +actually executes: +``` +bazel test //tests/integration/ --config=x86_64-linux --config=core_dump --nocache_test_results +``` + +If a crash dump was created, a `CRASH DUMP` section is printed right under the +pytest `FAILURES` section at the end of the run (the `x86_64-linux` config +enables `--test_output=errors`, so the failing log is shown automatically): +``` +=================================== FAILURES =================================== +... +================================== CRASH DUMP ================================== +CRASH DUMP HAS BEEN CREATED! See <.../test.outputs/cores> for details. + +core.launch_manager.42.6.1787209649: + Program terminated with signal SIGABRT, Aborted. + #0 0x... in ?? () from /lib/x86_64-linux-gnu/libc.so.6 + #1 0x... in raise () from /lib/x86_64-linux-gnu/libc.so.6 + #2 0x... in abort () from /lib/x86_64-linux-gnu/libc.so.6 + #3 0x... in at : + ... + Full backtrace (all threads): <.../cores/core.launch_manager.*.bt.txt> + Reopen in gdb inside the debug image: + docker run --rm -it -v <.../launch_manager>:/tmp/.../launch_manager:ro -v <.../cores>:/cores:ro score_itf_examples_debug:latest gdb /tmp/.../launch_manager /cores/core.launch_manager.* +=========================== short test summary info ============================ +``` +The crashing thread's stack is printed **inline** (symbolized inside the +container, so libraries match). The printed paths are absolute and +copy-pasteable. Core files are named `core....