Add tests which are verifying comp_req__launch_man__launcher_exit_shutdown

.bazelrc

@@ -72,6 +72,8 @@ build:x86_64-linux --extra_toolchains=@score_toolchains_rust//toolchains/ferroce
 
 test:x86_64-linux --//config:integration_mode=docker
 test:x86_64-linux --//config:unit_mode=host
+# Show a failing test's log (incl. the crash-dump banner) in the console.
+test:x86_64-linux --test_output=errors
 
 # Target configuration for CPU:AArch64|OS:Linux build (do not use it in case of system toolchains!)
 build:arm64-linux --config=stub

score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp

@@ -14,6 +14,8 @@
 #include <ctime>
 
 #include <score/span.hpp>
+#include <algorithm>
+#include <chrono>
 #include <functional>
 #include <type_traits>
 #include <variant>
@@ -542,6 +544,23 @@ void Graph::forceKillProcesses()
     }
 }
 
+std::chrono::milliseconds Graph::getMaxTerminationTimeout()
+{
+    std::chrono::milliseconds max_timeout{0};
+    for (const auto& component : nodes_)
+    {
+        if (const ProcessInfoNode* process = std::get_if<ProcessInfoNode>(&component))
+        {
+            // Only processes with a live OS process still to stop count
+            if (process->getPid() > 0 && process->getState() < ProcessState::kTerminated)
+            {
+                max_timeout = std::max(max_timeout, process->getTerminationTimeout());
+            }
+        }
+    }
+    return max_timeout;
+}
+
 void Graph::updateCancelMessage()
 {
     ControlClientCode code = getPendingEvent();

score/launch_manager/src/daemon/src/process_group_manager/details/graph.hpp

@@ -288,6 +288,10 @@ class Graph final
     /// @brief For forced shutdown, kill all leftover processes
     void forceKillProcesses();
 
+    /// @brief Returns the largest configured shutdown_timeout across all running processes
+    /// @return The timeout in milliseconds, or zero if there are no live processes to stop.
+    std::chrono::milliseconds getMaxTerminationTimeout();
+
   private:
     /// @brief Helper function to identify a node with ready state "Terminated" from the legacy configuration
     bool nodeHasTerminatedDeps(IdentifierHash pg_name, uint32_t node_index);

score/launch_manager/src/daemon/src/process_group_manager/details/graph_UT.cpp

@@ -607,4 +607,65 @@ TEST_F(GraphUtilitiesTest, gettersSetters)
     EXPECT_LE(graph_time, after_time);
 }
 
+class GraphMaxTerminationTimeoutTest : public GraphTest
+{
+  protected:
+    uint32_t SetConfig() override
+    {
+        auto procs = generateProcessComponents(3);
+        auto count = procs.size();
+        procs[0].deployment_config.shutdown_timeout_ms = 1500;
+        procs[0].component_properties.application_profile.is_self_terminating = true;
+        procs[1].deployment_config.shutdown_timeout_ms = 500;
+        procs[2].deployment_config.shutdown_timeout_ms = 5000;
+        auto rts = generateRunTargets(2);
+        rts[1].depends_on = {procs[0].name, procs[1].name};
+        rts[2].depends_on = {procs[2].name};
+        const auto config = ConfigBuilder{}
+                                .setComponents(std::move(procs))
+                                .setRunTargets(std::move(rts))
+                                .setInitialRunTarget("Startup")
+                                .setFallbackRunTarget(std::move(fallback))
+                                .build();
+        config_.initialize(config);
+
+        return count;
+    }
+};
+
+TEST_F(GraphMaxTerminationTimeoutTest, ignoresNodesWithoutLiveProcess)
+{
+    RecordProperty(
+        "Description",
+        "Test that getMaxTerminationTimeout returns the max shutdown_timeout over running processes and ignores "
+        "never-started (pid == 0) nodes");
+
+    // No process started yet, so there is nothing to wait on.
+    EXPECT_EQ(graph_.getMaxTerminationTimeout(), 0ms);
+
+    // Bring up RunTarget0 (proc0 + proc1); proc2, with the largest timeout, stays idle in RunTarget1.
+    completeTransition(state_name(run_target_name(0)));
+
+    // Max over the two live processes; proc2's 5000 ms is ignored because it never started.
+    EXPECT_EQ(graph_.getMaxTerminationTimeout(), 1500ms);
+}
+
+TEST_F(GraphMaxTerminationTimeoutTest, ignoresTerminatedProcesses)
+{
+    RecordProperty(
+        "Description",
+        "Test that getMaxTerminationTimeout ignores processes that have already terminated, even if they carry the "
+        "largest shutdown_timeout");
+
+    completeTransition(state_name(run_target_name(0)));
+    ASSERT_EQ(graph_.getMaxTerminationTimeout(), 1500ms);
+
+    // proc0 is a self-terminating one-shot with the largest timeout; it exits on its own
+    // (status 0) and stays kTerminated, so it no longer needs to be waited on at shutdown.
+    static_cast<void>(graph_.getProcessInfoNode(0)->tryHandleTermination(0));
+
+    // Only proc1 remains live, so its timeout bounds the wait.
+    EXPECT_EQ(graph_.getMaxTerminationTimeout(), 500ms);
+}
+
 }  // namespace score::mw::lifecycle::internal

score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.cpp

@@ -432,6 +432,11 @@ score::mw::lifecycle::ProcessState ProcessInfoNode::getState() const
     return process_state_.load();
 }
 
+std::chrono::milliseconds ProcessInfoNode::getTerminationTimeout() const
+{
+    return config_ != nullptr ? config_->pgm_config_.termination_timeout_ms_ : std::chrono::milliseconds{0};
+}
+
 uint32_t ProcessInfoNode::getIndex() const
 {
     return process_index_;

score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.hpp

@@ -21,6 +21,7 @@
 #include "score/mw/launch_manager/supervision_control_client/isupervision_event_publisher.hpp"
 #include <score/stop_token.hpp>
 #include <atomic>
+#include <chrono>
 
 namespace score::mw::lifecycle::internal
 {
@@ -99,6 +100,9 @@ class ProcessInfoNode final : public IComponent
     /// @return The current state of this process.
     score::mw::lifecycle::ProcessState getState() const;
 
+    /// @return The configured shutdown_timeout for this process, or zero
+    std::chrono::milliseconds getTerminationTimeout() const;
+
     /// @return The ControlClientChannel for this process, or nullptr if none exists.
     ControlClientChannelP getControlClientChannel() const;
 

score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.cpp

@@ -153,10 +153,14 @@ void ProcessGroupManager::deinitialize()
     process_monitor_.reset();
     alive_monitor_thread_->stop();
     configuration_.deinitialize();
-    process_groups_.clear();
 
+    // Join the worker threads before destroying the process groups: a worker may
+    // still be (de)activating a ProcessInfoNode owned by a graph, so tearing the
+    // graphs down first would be a use-after-free.
     worker_threads_.reset();
     worker_jobs_.reset();
+
+    process_groups_.clear();
     process_map_.reset();
 }
 
@@ -321,6 +325,7 @@ bool ProcessGroupManager::run()
     bool overflow_logged = false;
 
     if (result)
+    {
         while (!em_cancelled.load())
         {
             // Wait for something to happen...
@@ -350,6 +355,8 @@ bool ProcessGroupManager::run()
 
             watchdog_->serviceWatchdog();
         }
+        LM_LOG_INFO() << "ProcessGroupManager::run() - received SIGTERM, exiting";
+    }
 
     allProcessGroupsOff();
 
@@ -457,15 +464,23 @@ void ProcessGroupManager::allProcessGroupsOff()
     }
 
     LM_LOG_DEBUG() << "Wait for all process groups to complete the transition";
-    if (!waitForStateCompletion(GraphState::kInTransition, 1000))
+
+    // Bound the whole transition-to-Off wait by the slowest still-running process's
+    // shutdown_timeout (plus the SIGKILL grace), so every component's configured
+    // timeout is honoured. Processes deactivate in parallel.
+    const auto off_transition_timeout = graph.getMaxTerminationTimeout() + kMaxSigKillDelay;
+    if (!waitForStateCompletion(GraphState::kInTransition, static_cast<int32_t>(off_transition_timeout.count())))
     {
+        // Last resort: a process ignored even SIGKILL within its budget. Force-kill
+        // whatever is left and tear down the worker pool so shutdown can still proceed.
         LM_LOG_ERROR() << "NOTE: Transition to Off state timed out";
-        worker_threads_->stop();
 
         for (auto& pg : process_groups_)
         {
             pg->forceKillProcesses();
         }
+
+        worker_threads_.reset();
     }
 }
 

score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.hpp

@@ -259,7 +259,8 @@ class ProcessGroupManager final : public ITransitionResultPublisher
     /// @brief Send all process groups to the "Off" state
     /// @details cancel any Graph for a process group not in the "Off" state, wait for up to 2 seconds for all graphs
     /// to be no longer in the `kCancelled` state, start a transition of remaining process groups to "Off" state,
-    /// and finally wait for up to a second for all graphs to complete.
+    /// and finally wait for all graphs to complete. The final wait is bounded by the largest configured per-process
+    /// shutdown_timeout (plus the SIGKILL grace) so each component's individual shutdown_timeout is respected.
     /// @warning Side effect: Depending if it is needed to forcefully terminate processes, worker jobs might be stopped
     /// after this call
     void allProcessGroupsOff();

tests/integration/lm_shutdown_during_rt_switch/BUILD

@@ -0,0 +1,56 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+load("@rules_cc//cc:cc_binary.bzl", "cc_binary")
+load("@rules_cc//cc:cc_library.bzl", "cc_library")
+load("//tests/utils/bazel:integration.bzl", "integration_test")
+
+cc_library(
+    name = "lm_shutdown_common",
+    hdrs = ["common.hpp"],
+)
+
+cc_binary(
+    name = "control_client_test_driver",
+    srcs = ["control_client_test_driver.cpp"],
+    deps = [
+        ":lm_shutdown_common",
+        "//score/launch_manager:control_cc",
+        "//score/launch_manager:lifecycle_cc",
+        "//tests/utils/test_helper",
+        "@googletest//:gtest_main",
+    ],
+)
+
+cc_binary(
+    name = "component_c",
+    srcs = ["component_c.cpp"],
+    deps = [
+        ":lm_shutdown_common",
+        "//score/launch_manager:lifecycle_cc",
+        "//tests/utils/test_helper",
+        "@googletest//:gtest_main",
+    ],
+)
+
+integration_test(
+    name = "lm_shutdown_during_rt_switch",
+    timeout = "short",
+    srcs = ["lm_shutdown_during_rt_switch.py"],
+    binaries = [
+        "//tests/utils/test_helper:process_hanging_on_sigterm",
+        ":component_c",
+        ":control_client_test_driver",
+        "//score/launch_manager",
+    ],
+    config = ":lm_shutdown_during_rt_switch.json",
+)

tests/integration/lm_shutdown_during_rt_switch/common.hpp

@@ -0,0 +1,34 @@
+/********************************************************************************
+ * Copyright (c) 2026 Contributors to the Eclipse Foundation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ********************************************************************************/
+
+#ifndef SCORE_TESTS_INTEGRATION_LM_SHUTDOWN_COMMON_HPP
+#define SCORE_TESTS_INTEGRATION_LM_SHUTDOWN_COMMON_HPP
+
+#include <string_view>
+
+/// @brief Written by component_a when it has reported running (run_target_a is
+/// active).
+constexpr std::string_view a_started = "component_a_started";
+
+/// @brief Written by component_a when it starts being terminated (i.e. the
+/// switch away from run_target_a has begun). component_a then stalls, which
+/// keeps the run-target switch in progress and gives the test a deterministic
+/// window in which to send SIGTERM to the launch manager.
+constexpr std::string_view a_terminating = "component_a_terminating";
+
+/// @brief Written by component_c when it starts. component_c belongs only to
+/// run_target_c, so this file must NEVER appear: a SIGTERM to the launch manager
+/// during the switch must cancel the pending activation of run_target_c.
+constexpr std::string_view c_started = "component_c_started";
+
+#endif  // SCORE_TESTS_INTEGRATION_LM_SHUTDOWN_COMMON_HPP

tests/integration/lm_shutdown_during_rt_switch/component_c.cpp

@@ -0,0 +1,46 @@
+/********************************************************************************
+ * Copyright (c) 2026 Contributors to the Eclipse Foundation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ********************************************************************************/
+#include <gtest/gtest.h>
+#include <unistd.h>
+
+#include "common.hpp"
+#include "tests/utils/test_helper/test_helper.hpp"
+#include <score/mw/lifecycle/report_running.h>
+
+// component_c belongs only to run_target_c. Because the switch to run_target_c
+// must be cancelled by the SIGTERM sent to the launch manager, this process must
+// never be launched. Should it ever start, it records `c_started`, which makes
+// both the control client and the Python-side assertions fail.
+TEST(LmShutdownDuringRtSwitch, ComponentC)
+{
+    TEST_STEP("Report running")
+    {
+        // This code should be never executed. In Python code there is also an assertion
+        // that component_c must not be started (i.e. c_started should not exist).
+        // This is a second line of defense in case the Python code is not executed or fails to detect the problem.
+        ADD_FAILURE() << "component_c must never be started";
+
+        EXPECT_TRUE(touch_file(c_started));
+        score::mw::lifecycle::report_running();
+    }
+
+    while (!TestRunner::exitRequested)
+    {
+        pause();
+    }
+}
+
+int main()
+{
+    return TestRunner(__FILE__).RunTests();
+}