diff --git a/.bazelrc b/.bazelrc index 2a35375ed..c5c061d29 100644 --- a/.bazelrc +++ b/.bazelrc @@ -72,6 +72,8 @@ build:x86_64-linux --extra_toolchains=@score_toolchains_rust//toolchains/ferroce test:x86_64-linux --//config:integration_mode=docker test:x86_64-linux --//config:unit_mode=host +# Show a failing test's log (incl. the crash-dump banner) in the console. +test:x86_64-linux --test_output=errors # Target configuration for CPU:AArch64|OS:Linux build (do not use it in case of system toolchains!) build:arm64-linux --config=stub diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp b/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp index c503c84ef..abfa8920c 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp @@ -14,6 +14,8 @@ #include #include +#include +#include #include #include #include @@ -542,6 +544,23 @@ void Graph::forceKillProcesses() } } +std::chrono::milliseconds Graph::getMaxTerminationTimeout() +{ + std::chrono::milliseconds max_timeout{0}; + for (const auto& component : nodes_) + { + if (const ProcessInfoNode* process = std::get_if(&component)) + { + // Only processes with a live OS process still to stop count + if (process->getPid() > 0 && process->getState() < ProcessState::kTerminated) + { + max_timeout = std::max(max_timeout, process->getTerminationTimeout()); + } + } + } + return max_timeout; +} + void Graph::updateCancelMessage() { ControlClientCode code = getPendingEvent(); diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/graph.hpp b/score/launch_manager/src/daemon/src/process_group_manager/details/graph.hpp index ba383ed08..4849623d4 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/graph.hpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/graph.hpp @@ -288,6 +288,10 @@ class Graph final /// @brief For forced shutdown, kill all leftover processes void forceKillProcesses(); + /// @brief Returns the largest configured shutdown_timeout across all running processes + /// @return The timeout in milliseconds, or zero if there are no live processes to stop. + std::chrono::milliseconds getMaxTerminationTimeout(); + private: /// @brief Helper function to identify a node with ready state "Terminated" from the legacy configuration bool nodeHasTerminatedDeps(IdentifierHash pg_name, uint32_t node_index); diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/graph_UT.cpp b/score/launch_manager/src/daemon/src/process_group_manager/details/graph_UT.cpp index 2f4181357..5391b4a73 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/graph_UT.cpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/graph_UT.cpp @@ -607,4 +607,65 @@ TEST_F(GraphUtilitiesTest, gettersSetters) EXPECT_LE(graph_time, after_time); } +class GraphMaxTerminationTimeoutTest : public GraphTest +{ + protected: + uint32_t SetConfig() override + { + auto procs = generateProcessComponents(3); + auto count = procs.size(); + procs[0].deployment_config.shutdown_timeout_ms = 1500; + procs[0].component_properties.application_profile.is_self_terminating = true; + procs[1].deployment_config.shutdown_timeout_ms = 500; + procs[2].deployment_config.shutdown_timeout_ms = 5000; + auto rts = generateRunTargets(2); + rts[1].depends_on = {procs[0].name, procs[1].name}; + rts[2].depends_on = {procs[2].name}; + const auto config = ConfigBuilder{} + .setComponents(std::move(procs)) + .setRunTargets(std::move(rts)) + .setInitialRunTarget("Startup") + .setFallbackRunTarget(std::move(fallback)) + .build(); + config_.initialize(config); + + return count; + } +}; + +TEST_F(GraphMaxTerminationTimeoutTest, ignoresNodesWithoutLiveProcess) +{ + RecordProperty( + "Description", + "Test that getMaxTerminationTimeout returns the max shutdown_timeout over running processes and ignores " + "never-started (pid == 0) nodes"); + + // No process started yet, so there is nothing to wait on. + EXPECT_EQ(graph_.getMaxTerminationTimeout(), 0ms); + + // Bring up RunTarget0 (proc0 + proc1); proc2, with the largest timeout, stays idle in RunTarget1. + completeTransition(state_name(run_target_name(0))); + + // Max over the two live processes; proc2's 5000 ms is ignored because it never started. + EXPECT_EQ(graph_.getMaxTerminationTimeout(), 1500ms); +} + +TEST_F(GraphMaxTerminationTimeoutTest, ignoresTerminatedProcesses) +{ + RecordProperty( + "Description", + "Test that getMaxTerminationTimeout ignores processes that have already terminated, even if they carry the " + "largest shutdown_timeout"); + + completeTransition(state_name(run_target_name(0))); + ASSERT_EQ(graph_.getMaxTerminationTimeout(), 1500ms); + + // proc0 is a self-terminating one-shot with the largest timeout; it exits on its own + // (status 0) and stays kTerminated, so it no longer needs to be waited on at shutdown. + static_cast(graph_.getProcessInfoNode(0)->tryHandleTermination(0)); + + // Only proc1 remains live, so its timeout bounds the wait. + EXPECT_EQ(graph_.getMaxTerminationTimeout(), 500ms); +} + } // namespace score::mw::lifecycle::internal diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.cpp b/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.cpp index 806a11d9c..01fd8f347 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.cpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.cpp @@ -432,6 +432,11 @@ score::mw::lifecycle::ProcessState ProcessInfoNode::getState() const return process_state_.load(); } +std::chrono::milliseconds ProcessInfoNode::getTerminationTimeout() const +{ + return config_ != nullptr ? config_->pgm_config_.termination_timeout_ms_ : std::chrono::milliseconds{0}; +} + uint32_t ProcessInfoNode::getIndex() const { return process_index_; diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.hpp b/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.hpp index 8310330a8..237b4e0b1 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.hpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.hpp @@ -21,6 +21,7 @@ #include "score/mw/launch_manager/supervision_control_client/isupervision_event_publisher.hpp" #include #include +#include namespace score::mw::lifecycle::internal { @@ -99,6 +100,9 @@ class ProcessInfoNode final : public IComponent /// @return The current state of this process. score::mw::lifecycle::ProcessState getState() const; + /// @return The configured shutdown_timeout for this process, or zero + std::chrono::milliseconds getTerminationTimeout() const; + /// @return The ControlClientChannel for this process, or nullptr if none exists. ControlClientChannelP getControlClientChannel() const; diff --git a/score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.cpp b/score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.cpp index 37fad6ea3..7c851d80d 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.cpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.cpp @@ -153,10 +153,14 @@ void ProcessGroupManager::deinitialize() process_monitor_.reset(); alive_monitor_thread_->stop(); configuration_.deinitialize(); - process_groups_.clear(); + // Join the worker threads before destroying the process groups: a worker may + // still be (de)activating a ProcessInfoNode owned by a graph, so tearing the + // graphs down first would be a use-after-free. worker_threads_.reset(); worker_jobs_.reset(); + + process_groups_.clear(); process_map_.reset(); } @@ -321,6 +325,7 @@ bool ProcessGroupManager::run() bool overflow_logged = false; if (result) + { while (!em_cancelled.load()) { // Wait for something to happen... @@ -350,6 +355,8 @@ bool ProcessGroupManager::run() watchdog_->serviceWatchdog(); } + LM_LOG_INFO() << "ProcessGroupManager::run() - received SIGTERM, exiting"; + } allProcessGroupsOff(); @@ -457,15 +464,23 @@ void ProcessGroupManager::allProcessGroupsOff() } LM_LOG_DEBUG() << "Wait for all process groups to complete the transition"; - if (!waitForStateCompletion(GraphState::kInTransition, 1000)) + + // Bound the whole transition-to-Off wait by the slowest still-running process's + // shutdown_timeout (plus the SIGKILL grace), so every component's configured + // timeout is honoured. Processes deactivate in parallel. + const auto off_transition_timeout = graph.getMaxTerminationTimeout() + kMaxSigKillDelay; + if (!waitForStateCompletion(GraphState::kInTransition, static_cast(off_transition_timeout.count()))) { + // Last resort: a process ignored even SIGKILL within its budget. Force-kill + // whatever is left and tear down the worker pool so shutdown can still proceed. LM_LOG_ERROR() << "NOTE: Transition to Off state timed out"; - worker_threads_->stop(); for (auto& pg : process_groups_) { pg->forceKillProcesses(); } + + worker_threads_.reset(); } } diff --git a/score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.hpp b/score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.hpp index fe077b5c2..163811400 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.hpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.hpp @@ -259,7 +259,8 @@ class ProcessGroupManager final : public ITransitionResultPublisher /// @brief Send all process groups to the "Off" state /// @details cancel any Graph for a process group not in the "Off" state, wait for up to 2 seconds for all graphs /// to be no longer in the `kCancelled` state, start a transition of remaining process groups to "Off" state, - /// and finally wait for up to a second for all graphs to complete. + /// and finally wait for all graphs to complete. The final wait is bounded by the largest configured per-process + /// shutdown_timeout (plus the SIGKILL grace) so each component's individual shutdown_timeout is respected. /// @warning Side effect: Depending if it is needed to forcefully terminate processes, worker jobs might be stopped /// after this call void allProcessGroupsOff(); diff --git a/tests/integration/lm_shutdown_during_rt_switch/BUILD b/tests/integration/lm_shutdown_during_rt_switch/BUILD new file mode 100644 index 000000000..d5b2c0d52 --- /dev/null +++ b/tests/integration/lm_shutdown_during_rt_switch/BUILD @@ -0,0 +1,56 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("//tests/utils/bazel:integration.bzl", "integration_test") + +cc_library( + name = "lm_shutdown_common", + hdrs = ["common.hpp"], +) + +cc_binary( + name = "control_client_test_driver", + srcs = ["control_client_test_driver.cpp"], + deps = [ + ":lm_shutdown_common", + "//score/launch_manager:control_cc", + "//score/launch_manager:lifecycle_cc", + "//tests/utils/test_helper", + "@googletest//:gtest_main", + ], +) + +cc_binary( + name = "component_c", + srcs = ["component_c.cpp"], + deps = [ + ":lm_shutdown_common", + "//score/launch_manager:lifecycle_cc", + "//tests/utils/test_helper", + "@googletest//:gtest_main", + ], +) + +integration_test( + name = "lm_shutdown_during_rt_switch", + timeout = "short", + srcs = ["lm_shutdown_during_rt_switch.py"], + binaries = [ + "//tests/utils/test_helper:process_hanging_on_sigterm", + ":component_c", + ":control_client_test_driver", + "//score/launch_manager", + ], + config = ":lm_shutdown_during_rt_switch.json", +) diff --git a/tests/integration/lm_shutdown_during_rt_switch/common.hpp b/tests/integration/lm_shutdown_during_rt_switch/common.hpp new file mode 100644 index 000000000..4e45fe3dd --- /dev/null +++ b/tests/integration/lm_shutdown_during_rt_switch/common.hpp @@ -0,0 +1,34 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +#ifndef SCORE_TESTS_INTEGRATION_LM_SHUTDOWN_COMMON_HPP +#define SCORE_TESTS_INTEGRATION_LM_SHUTDOWN_COMMON_HPP + +#include + +/// @brief Written by component_a when it has reported running (run_target_a is +/// active). +constexpr std::string_view a_started = "component_a_started"; + +/// @brief Written by component_a when it starts being terminated (i.e. the +/// switch away from run_target_a has begun). component_a then stalls, which +/// keeps the run-target switch in progress and gives the test a deterministic +/// window in which to send SIGTERM to the launch manager. +constexpr std::string_view a_terminating = "component_a_terminating"; + +/// @brief Written by component_c when it starts. component_c belongs only to +/// run_target_c, so this file must NEVER appear: a SIGTERM to the launch manager +/// during the switch must cancel the pending activation of run_target_c. +constexpr std::string_view c_started = "component_c_started"; + +#endif // SCORE_TESTS_INTEGRATION_LM_SHUTDOWN_COMMON_HPP diff --git a/tests/integration/lm_shutdown_during_rt_switch/component_c.cpp b/tests/integration/lm_shutdown_during_rt_switch/component_c.cpp new file mode 100644 index 000000000..299ef0385 --- /dev/null +++ b/tests/integration/lm_shutdown_during_rt_switch/component_c.cpp @@ -0,0 +1,46 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include +#include + +#include "common.hpp" +#include "tests/utils/test_helper/test_helper.hpp" +#include + +// component_c belongs only to run_target_c. Because the switch to run_target_c +// must be cancelled by the SIGTERM sent to the launch manager, this process must +// never be launched. Should it ever start, it records `c_started`, which makes +// both the control client and the Python-side assertions fail. +TEST(LmShutdownDuringRtSwitch, ComponentC) +{ + TEST_STEP("Report running") + { + // This code should be never executed. In Python code there is also an assertion + // that component_c must not be started (i.e. c_started should not exist). + // This is a second line of defense in case the Python code is not executed or fails to detect the problem. + ADD_FAILURE() << "component_c must never be started"; + + EXPECT_TRUE(touch_file(c_started)); + score::mw::lifecycle::report_running(); + } + + while (!TestRunner::exitRequested) + { + pause(); + } +} + +int main() +{ + return TestRunner(__FILE__).RunTests(); +} diff --git a/tests/integration/lm_shutdown_during_rt_switch/control_client_test_driver.cpp b/tests/integration/lm_shutdown_during_rt_switch/control_client_test_driver.cpp new file mode 100644 index 000000000..dcb24b9b9 --- /dev/null +++ b/tests/integration/lm_shutdown_during_rt_switch/control_client_test_driver.cpp @@ -0,0 +1,73 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include +#include +#include + +#include "common.hpp" +#include "tests/utils/test_helper/test_helper.hpp" +#include +#include + +// The Launch Manager shall exit after performing a shutdown - stopping all the +// processes it owns in dependency order - when requested (i.e. when it receives +// a SIGTERM). A shutdown request takes priority over an in-progress run-target +// switch, which must therefore be cancelled. +// +// This control client activates run_target_a and then requests a switch to +// run_target_c. component_a (only part of run_target_a) stalls while it is being +// terminated during that switch, so the switch is still in progress when the +// test sends a SIGTERM to the launch manager from the Python side. The launch +// manager must then cancel the pending switch (component_c, only part of +// run_target_c, must never start) and shut everything down. +TEST(LmShutdownDuringRtSwitch, ControlClient) +{ + score::mw::lifecycle::ControlClient client{}; + ASSERT_TRUE(check_clean({test_end_location, a_started, a_terminating, c_started})); + + TEST_STEP("Report running") + { + score::mw::lifecycle::report_running(); + } + + TEST_STEP("Activate run_target_a") + { + score::cpp::stop_token stop_token; + auto result = client.ActivateRunTarget("run_target_a").Get(stop_token); + EXPECT_TRUE(result.has_value()) << "Activating run_target_a failed: " << result.error().Message(); + EXPECT_TRUE(std::filesystem::exists(a_started)) << "component_a was not started"; + } + + TEST_STEP("Request switch to run_target_c") + { + // Fire-and-forget: this transition is expected to be cancelled by an + // external SIGTERM to the launch manager, so we must not wait for a + // result. The launch manager will shut this process down instead of ever + // completing the switch. + client.ActivateRunTarget("run_target_c"); + } + + // Block until the launch manager terminates us as part of its own shutdown. + TestRunner::waitForTermination(); + + TEST_STEP("Verify run_target_c was never activated") + { + EXPECT_FALSE(std::filesystem::exists(c_started)) + << "run_target_c must not be activated: a SIGTERM to the launch manager must cancel the pending switch"; + } +} + +int main() +{ + return TestRunner(__FILE__, TerminationBehavior::kWait, TerminationNotification::kTestEnd).RunTests(); +} diff --git a/tests/integration/lm_shutdown_during_rt_switch/lm_shutdown_during_rt_switch.json b/tests/integration/lm_shutdown_during_rt_switch/lm_shutdown_during_rt_switch.json new file mode 100644 index 000000000..9376cba98 --- /dev/null +++ b/tests/integration/lm_shutdown_during_rt_switch/lm_shutdown_during_rt_switch.json @@ -0,0 +1,114 @@ +{ + "schema_version": 1, + "defaults": { + "deployment_config": { + "bin_dir": "/tmp/tests/lm_shutdown_during_rt_switch", + "ready_timeout": 1.0, + "shutdown_timeout": 1.0, + "ready_recovery_action": { + "restart": { + "number_of_attempts": 0 + } + }, + "recovery_action": { + "switch_run_target": { + "run_target": "fallback_run_target" + } + }, + "environmental_variables": { + "LD_LIBRARY_PATH": "/opt/lib" + }, + "sandbox": { + "uid": 0, + "gid": 0, + "scheduling_policy": "SCHED_OTHER", + "scheduling_priority": 0 + } + }, + "component_properties": { + "application_profile": { + "application_type": "Reporting", + "is_self_terminating": false, + "alive_supervision": { + "reporting_cycle": 0.1, + "min_indications": 1, + "max_indications": 3, + "failed_cycles_tolerance": 1 + } + }, + "ready_condition": { + "process_state": "Running" + } + } + }, + "components": { + "component_initial": { + "component_properties": { + "binary_name": "control_client_test_driver", + "application_profile": { + "application_type": "State_Manager", + "alive_supervision": { + "min_indications": 0 + } + } + }, + "deployment_config": { + "ready_timeout": 1.0, + "shutdown_timeout": 1.0, + "environmental_variables": { + "PROCESSIDENTIFIER": "control_client_test_driver" + } + } + }, + "component_a": { + "component_properties": { + "binary_name": "process_hanging_on_sigterm" + }, + "deployment_config": { + "shutdown_timeout": 5.0, + "environmental_variables": { + "PROCESSIDENTIFIER": "component_a" + } + } + }, + "component_c": { + "component_properties": { + "binary_name": "component_c" + }, + "deployment_config": { + "environmental_variables": { + "PROCESSIDENTIFIER": "component_c" + } + } + } + }, + "run_targets": { + "Startup": { + "depends_on": [ + "component_initial" + ] + }, + "run_target_a": { + "depends_on": [ + "component_initial", + "component_a" + ] + }, + "run_target_c": { + "depends_on": [ + "component_initial", + "component_c" + ] + }, + "Off": { + "depends_on": [] + } + }, + "initial_run_target": "Startup", + "alive_supervision": { + "evaluation_cycle": 0.05 + }, + "fallback_run_target": { + "depends_on": [] + } +} diff --git a/tests/integration/lm_shutdown_during_rt_switch/lm_shutdown_during_rt_switch.py b/tests/integration/lm_shutdown_during_rt_switch/lm_shutdown_during_rt_switch.py new file mode 100644 index 000000000..611597b68 --- /dev/null +++ b/tests/integration/lm_shutdown_during_rt_switch/lm_shutdown_during_rt_switch.py @@ -0,0 +1,66 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +from tests.utils.testing_utils.run_until_file_deployed import run_until_file_deployed +from tests.utils.testing_utils.setup_test import setup_test +from tests.utils.testing_utils.test_results import assert_test_results +from attribute_plugin import add_test_properties + + +@add_test_properties( + fully_verifies=[], + partially_verifies=[ + "comp_req__launch_man__launcher_exit_shutdown", + ], + test_type="requirements-based", + derivation_technique="requirements-analysis", +) +def test_lm_shutdown(target, setup_test, assert_test_results, remote_test_dir): + """ + Objective: Verifies that the Launch Manager exits after performing a shutdown + (stopping all processes it owns) when requested via SIGTERM, and that this + shutdown takes priority over an in-progress run-target switch (the switch is + cancelled). + + The control client activates run_target_a and then requests a switch to + run_target_c. component_a (only part of run_target_a) stalls while it is being + terminated during the switch, keeping the switch in progress. That window is + signalled by the file `component_a_terminating`, at which point the launch + manager is sent a SIGTERM. + + Expected Behaviour: The launch manager cancels the pending switch - so + component_c (only part of run_target_c) is never started - stops all the + processes it owns, and exits cleanly. + """ + + new_config_path = str(remote_test_dir / "etc/lm_shutdown_during_rt_switch.bin") + a_terminating = remote_test_dir / "component_a_terminating" + + # Run the launch manager until component_a signals it is stalling mid-termination + # (file `component_a_terminating`): the switch to run_target_c is then in progress + # but run_target_c has not been activated yet. run_until_file_deployed stops the + # launch manager at that point by sending it a SIGTERM (to the launch manager + # process only, so it performs its own orderly shutdown) and asserts it exits + # cleanly (code 0). + run_until_file_deployed( + target=target, + binary_path=str(remote_test_dir / "launch_manager"), + file_path=a_terminating, + cwd=str(remote_test_dir), + args=["-c", new_config_path], + timeout_s=10.0, + ) + + # component_c never runs (the pending switch was cancelled), so it produces no XML + # result; component_a and the control client shut down gracefully. The control + # client additionally asserts that run_target_c was never activated. + assert_test_results({"control_client_test_driver.xml", "component_a.xml"}) diff --git a/tests/integration/lm_shutdown_during_switch_to_off/BUILD b/tests/integration/lm_shutdown_during_switch_to_off/BUILD new file mode 100644 index 000000000..88512b86c --- /dev/null +++ b/tests/integration/lm_shutdown_during_switch_to_off/BUILD @@ -0,0 +1,44 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("//tests/utils/bazel:integration.bzl", "integration_test") + +cc_library( + name = "lm_shutdown_common", + hdrs = ["common.hpp"], +) + +cc_binary( + name = "control_client_test_driver", + srcs = ["control_client_test_driver.cpp"], + deps = [ + ":lm_shutdown_common", + "//score/launch_manager:control_cc", + "//score/launch_manager:lifecycle_cc", + "//tests/utils/test_helper", + "@googletest//:gtest_main", + ], +) + +integration_test( + name = "lm_shutdown_during_switch_to_off", + timeout = "short", + srcs = ["lm_shutdown_during_switch_to_off.py"], + binaries = [ + "//tests/utils/test_helper:process_hanging_on_sigterm", + ":control_client_test_driver", + "//score/launch_manager", + ], + config = ":lm_shutdown_during_switch_to_off.json", +) diff --git a/tests/integration/lm_shutdown_during_switch_to_off/common.hpp b/tests/integration/lm_shutdown_during_switch_to_off/common.hpp new file mode 100644 index 000000000..f3b5eb946 --- /dev/null +++ b/tests/integration/lm_shutdown_during_switch_to_off/common.hpp @@ -0,0 +1,30 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +#ifndef SCORE_TESTS_INTEGRATION_LM_SHUTDOWN_DURING_SWITCH_TO_OFF_COMMON_HPP +#define SCORE_TESTS_INTEGRATION_LM_SHUTDOWN_DURING_SWITCH_TO_OFF_COMMON_HPP + +#include + +/// @brief Written by component_a when it has reported running (run_target_a is +/// active). +constexpr std::string_view a_started = "component_a_started"; + +/// @brief Written by component_a when it starts being terminated (i.e. the +/// switch away from run_target_a - here, the switch to the "Off" run target - +/// has begun). component_a then stalls, which keeps the run-target switch in +/// progress and gives the test a deterministic window in which to send SIGTERM +/// to the launch manager. +constexpr std::string_view a_terminating = "component_a_terminating"; + +#endif // SCORE_TESTS_INTEGRATION_LM_SHUTDOWN_DURING_SWITCH_TO_OFF_COMMON_HPP diff --git a/tests/integration/lm_shutdown_during_switch_to_off/control_client_test_driver.cpp b/tests/integration/lm_shutdown_during_switch_to_off/control_client_test_driver.cpp new file mode 100644 index 000000000..61c29b549 --- /dev/null +++ b/tests/integration/lm_shutdown_during_switch_to_off/control_client_test_driver.cpp @@ -0,0 +1,73 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include +#include +#include + +#include "common.hpp" +#include "tests/utils/test_helper/test_helper.hpp" +#include +#include + +// The Launch Manager shall exit after performing a shutdown - stopping all the +// processes it owns in dependency order - when requested (i.e. when it receives +// a SIGTERM). +// +// This variant differs from lm_shutdown_during_rt_switch: instead of switching +// to another (non-Off) run target, the control client explicitly switches to +// the "Off" run target. component_a (part of run_target_a) stalls while it is +// being terminated during that switch, so the switch to Off is still in progress +// when the test sends a SIGTERM to the launch manager from the Python side. +// +// Because the process group is ALREADY heading to Off, the SIGTERM-triggered +// shutdown must simply let that in-progress switch to Off continue to completion +// - it must NOT cancel the explicit switch to Off and redo it. Either way the +// launch manager must end up stopping everything it owns and exit cleanly. +TEST(LmShutdownDuringSwitchToOff, ControlClient) +{ + score::mw::lifecycle::ControlClient client{}; + ASSERT_TRUE(check_clean({test_end_location, a_started, a_terminating})); + + const auto pid = getpid(); + const std::string step_msg = "Report running with pid == " + std::to_string(pid); + + TEST_STEP(step_msg) + { + score::mw::lifecycle::report_running(); + } + + TEST_STEP("Activate run_target_a") + { + score::cpp::stop_token stop_token; + auto result = client.ActivateRunTarget("run_target_a").Get(stop_token); + EXPECT_TRUE(result.has_value()) << "Activating run_target_a failed: " << result.error().Message(); + EXPECT_TRUE(std::filesystem::exists(a_started)) << "component_a was not started"; + } + + TEST_STEP("Request switch to Off") + { + // Fire-and-forget: switching to the "Off" run target terminates this + // control client too (it is not part of "Off"), so we must not wait for a + // result. The launch manager will shut this process down as part of the + // switch to Off. + client.ActivateRunTarget("Off"); + } + + // Block until the launch manager terminates us as part of its own shutdown. + TestRunner::waitForTermination(); +} + +int main() +{ + return TestRunner(__FILE__, TerminationBehavior::kContinue, TerminationNotification::kTestEnd).RunTests(); +} diff --git a/tests/integration/lm_shutdown_during_switch_to_off/lm_shutdown_during_switch_to_off.json b/tests/integration/lm_shutdown_during_switch_to_off/lm_shutdown_during_switch_to_off.json new file mode 100644 index 000000000..ac67c6af7 --- /dev/null +++ b/tests/integration/lm_shutdown_during_switch_to_off/lm_shutdown_during_switch_to_off.json @@ -0,0 +1,98 @@ +{ + "schema_version": 1, + "defaults": { + "deployment_config": { + "bin_dir": "/tmp/tests/lm_shutdown_during_switch_to_off", + "ready_timeout": 1.0, + "shutdown_timeout": 1.0, + "ready_recovery_action": { + "restart": { + "number_of_attempts": 0 + } + }, + "recovery_action": { + "switch_run_target": { + "run_target": "fallback_run_target" + } + }, + "environmental_variables": { + "LD_LIBRARY_PATH": "/opt/lib" + }, + "sandbox": { + "uid": 0, + "gid": 0, + "scheduling_policy": "SCHED_OTHER", + "scheduling_priority": 0 + } + }, + "component_properties": { + "application_profile": { + "application_type": "Reporting", + "is_self_terminating": false, + "alive_supervision": { + "reporting_cycle": 0.1, + "min_indications": 1, + "max_indications": 3, + "failed_cycles_tolerance": 1 + } + }, + "ready_condition": { + "process_state": "Running" + } + } + }, + "components": { + "component_initial": { + "component_properties": { + "binary_name": "control_client_test_driver", + "application_profile": { + "application_type": "State_Manager", + "alive_supervision": { + "min_indications": 0 + } + } + }, + "deployment_config": { + "ready_timeout": 1.0, + "shutdown_timeout": 5.0, + "environmental_variables": { + "PROCESSIDENTIFIER": "control_client_test_driver" + } + } + }, + "component_a": { + "component_properties": { + "binary_name": "process_hanging_on_sigterm" + }, + "deployment_config": { + "shutdown_timeout": 5.0, + "environmental_variables": { + "PROCESSIDENTIFIER": "component_a" + } + } + } + }, + "run_targets": { + "Startup": { + "depends_on": [ + "component_initial" + ] + }, + "run_target_a": { + "depends_on": [ + "component_initial", + "component_a" + ] + }, + "Off": { + "depends_on": [] + } + }, + "initial_run_target": "Startup", + "alive_supervision": { + "evaluation_cycle": 0.05 + }, + "fallback_run_target": { + "depends_on": [] + } +} diff --git a/tests/integration/lm_shutdown_during_switch_to_off/lm_shutdown_during_switch_to_off.py b/tests/integration/lm_shutdown_during_switch_to_off/lm_shutdown_during_switch_to_off.py new file mode 100644 index 000000000..428998649 --- /dev/null +++ b/tests/integration/lm_shutdown_during_switch_to_off/lm_shutdown_during_switch_to_off.py @@ -0,0 +1,67 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +from tests.utils.testing_utils.run_until_file_deployed import run_until_file_deployed +from tests.utils.testing_utils.setup_test import setup_test +from tests.utils.testing_utils.test_results import assert_test_results +from attribute_plugin import add_test_properties + + +@add_test_properties( + fully_verifies=[], + partially_verifies=[ + "comp_req__launch_man__launcher_exit_shutdown", + ], + test_type="requirements-based", + derivation_technique="requirements-analysis", +) +def test_lm_shutdown(target, setup_test, assert_test_results, remote_test_dir): + """ + Objective: Verifies that the Launch Manager exits after performing a shutdown + (stopping all processes it owns) when a SIGTERM arrives while an explicit switch + to the "Off" run target is already in progress. + + The control client activates run_target_a and then explicitly requests a switch + to the "Off" run target. component_a (part of run_target_a) stalls while it is + being terminated during that switch, keeping the switch to Off in progress. That + window is signalled by the file `component_a_terminating`, at which point the + launch manager is sent a SIGTERM. + + Expected Behaviour: The launch manager lets the in-progress switch to Off + continue, stops all the processes it owns, and exits cleanly. It honours each + component's shutdown_timeout, so component_a - which stalls for less than its + shutdown_timeout - exits gracefully (producing its XML result) rather than being + force-terminated. + """ + + new_config_path = str(remote_test_dir / "etc/lm_shutdown_during_switch_to_off.bin") + a_terminating = remote_test_dir / "component_a_terminating" + + # Run the launch manager until component_a signals it is stalling mid-termination + # (file `component_a_terminating`): the explicit switch to Off is then in progress. + # run_until_file_deployed stops the launch manager at that point by sending it a + # SIGTERM (to the launch manager process only, so it performs its own orderly + # shutdown) and asserts it exits cleanly (code 0). + run_until_file_deployed( + target=target, + binary_path=str(remote_test_dir / "launch_manager"), + file_path=a_terminating, + cwd=str(remote_test_dir), + args=["-c", new_config_path], + timeout_s=10.0, + ) + + # Both processes are stopped gracefully as part of the switch to Off and produce + # their XML results: the control client is terminated when the switch to Off + # begins, and component_a exits within its shutdown_timeout (which the launch + # manager honours) instead of being force-terminated. + assert_test_results({"control_client_test_driver.xml", "component_a.xml"}) diff --git a/tests/integration/readme.md b/tests/integration/readme.md index 92d514dd3..46c0a8e2e 100644 --- a/tests/integration/readme.md +++ b/tests/integration/readme.md @@ -23,3 +23,74 @@ Currently the following configs are supported: - `host` - `x86_64-linux` +## Crash dumps (core dumps) + +When a binary under test (e.g. the launch manager) crashes with `SIGSEGV`, +`SIGABRT`, etc. inside the Docker sandbox, a core dump is captured +automatically. This is wired in commonly for every `integration_test`, so +individual tests need no changes. + +How it works: +- The sandbox container runs privileged with an unlimited core-file `ulimit`. +- A shared fixture sets the kernel `core_pattern` to a sandbox-local path + (`/tmp/score_cores/core.%e.%p.%s.%t`), copies any core dumps produced during + the test into the Bazel test outputs, and then restores the original + `core_pattern`. + +### Getting a crash dump + +Run the (crashing) test, disabling the cache so it actually executes: +``` +bazel test //tests/integration/ --config=x86_64-linux --nocache_test_results +``` + +If a crash dump was created, a `CRASH DUMP` section is printed right under the +pytest `FAILURES` section at the end of the run (the `x86_64-linux` config +enables `--test_output=errors`, so the failing log is shown automatically): +``` +=================================== FAILURES =================================== +... +================================== CRASH DUMP ================================== +CRASH DUMP HAS BEEN CREATED! See <.../test.outputs/cores> for details. + +To open it in gdb (build the crashing binary with -c dbg for symbols): + gdb <.../bin/.../launch_manager> "<.../test.outputs/cores/core.launch_manager.*>" +=========================== short test summary info ============================ +``` +The printed paths are absolute and copy-pasteable. Core files are named +`core....