From 5891c5c14184abeea96b9fd8d8f3881111413f38 Mon Sep 17 00:00:00 2001 From: Timo Steuerwald Date: Tue, 28 Jul 2026 16:04:47 +0200 Subject: [PATCH 01/23] Mainly cherry picked commits from internal branch These commits have been squashed together to ease review. After cherry pick some conflicts have been solved wrong, this is why there are two additional commits also. For details please also have a look onto the internal branch in etas-eng feature/create-process-launch-fit-specification-unverified The original commit message of this commit: Add a log message for SIGTERM receival Move dir tests/integration/lm_shutdown to tests/integration/lm_shutdown_during_rt_switch Update references accordingly Add lm_shutdown_during_switch_to_off A test which verifies that a launch manager shutdown signalled via SIGTERM does not cancel existent switches to off. Currently does not fail, but should fail. As switch to off gets cancelled and reinitiated by launch manager code. Let test fail, as switch to off is cancelled by lm Revert to 1s timeout for lm shutdown & add some comments Still not 100% stable. Fix cherry pick odyssey Former fix for SIGSEGV of old branch --- .../process_group_manager.cpp | 29 ++- .../lm_shutdown_during_rt_switch/BUILD | 62 ++++++ .../lm_shutdown_during_rt_switch/common.hpp | 34 +++ .../component_a.cpp | 61 ++++++ .../component_c.cpp | 46 ++++ .../control_client_mock.cpp | 76 +++++++ .../lm_shutdown_during_rt_switch.json | 114 ++++++++++ .../lm_shutdown_during_rt_switch.py | 160 ++++++++++++++ .../lm_shutdown_during_switch_to_off/BUILD | 55 +++++ .../common.hpp | 30 +++ .../component_a.cpp | 67 ++++++ .../control_client_mock.cpp | 74 +++++++ .../lm_shutdown_during_switch_to_off.json | 98 +++++++++ .../lm_shutdown_during_switch_to_off.py | 196 ++++++++++++++++++ 14 files changed, 1093 insertions(+), 9 deletions(-) create mode 100644 tests/integration/lm_shutdown_during_rt_switch/BUILD create mode 100644 tests/integration/lm_shutdown_during_rt_switch/common.hpp create mode 100644 tests/integration/lm_shutdown_during_rt_switch/component_a.cpp create mode 100644 tests/integration/lm_shutdown_during_rt_switch/component_c.cpp create mode 100644 tests/integration/lm_shutdown_during_rt_switch/control_client_mock.cpp create mode 100644 tests/integration/lm_shutdown_during_rt_switch/lm_shutdown_during_rt_switch.json create mode 100644 tests/integration/lm_shutdown_during_rt_switch/lm_shutdown_during_rt_switch.py create mode 100644 tests/integration/lm_shutdown_during_switch_to_off/BUILD create mode 100644 tests/integration/lm_shutdown_during_switch_to_off/common.hpp create mode 100644 tests/integration/lm_shutdown_during_switch_to_off/component_a.cpp create mode 100644 tests/integration/lm_shutdown_during_switch_to_off/control_client_mock.cpp create mode 100644 tests/integration/lm_shutdown_during_switch_to_off/lm_shutdown_during_switch_to_off.json create mode 100644 tests/integration/lm_shutdown_during_switch_to_off/lm_shutdown_during_switch_to_off.py diff --git a/score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.cpp b/score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.cpp index 358fa9e11..1d1cd679a 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.cpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.cpp @@ -153,10 +153,17 @@ void ProcessGroupManager::deinitialize() process_monitor_.reset(); alive_monitor_thread_->stop(); configuration_.deinitialize(); - process_groups_.clear(); + // Stop and join the worker threads BEFORE destroying the process groups. + // Worker threads run ProcessInfoNode::doWork(), which dereferences its Graph + // (nodeExecuted(), getState(), ...) via a raw back-pointer. If a transition is + // still completing on a worker thread (e.g. an in-progress switch to Off that + // is allowed to continue during shutdown), destroying the graphs first would be + // a use-after-free. thread_pool_.reset(); worker_jobs_.reset(); + + process_groups_.clear(); process_map_.reset(); } @@ -237,14 +244,15 @@ bool ProcessGroupManager::initializeProcessGroups() const auto* states = configuration_.getListOfProcessGroupStates(pg_name).value_or(nullptr); const uint32_t num_run_targets = states ? static_cast(states->size()) : 0U; - process_groups_.push_back(std::make_shared( - num_processes + num_run_targets, - &configuration_, - worker_jobs_, - &process_interface_, - process_map_, - *supervision_control_notifier_.get(), - this)); + process_groups_.push_back( + std::make_shared( + num_processes + num_run_targets, + &configuration_, + worker_jobs_, + &process_interface_, + process_map_, + *supervision_control_notifier_.get(), + this)); } } else @@ -321,6 +329,7 @@ bool ProcessGroupManager::run() bool overflow_logged = false; if (result) + { while (!em_cancelled.load()) { // Wait for something to happen... @@ -350,6 +359,8 @@ bool ProcessGroupManager::run() watchdog_->serviceWatchdog(); } + LM_LOG_WARN() << "ProcessGroupManager::run() - received SIGTERM, exiting"; + } allProcessGroupsOff(); diff --git a/tests/integration/lm_shutdown_during_rt_switch/BUILD b/tests/integration/lm_shutdown_during_rt_switch/BUILD new file mode 100644 index 000000000..29ded4c31 --- /dev/null +++ b/tests/integration/lm_shutdown_during_rt_switch/BUILD @@ -0,0 +1,62 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("//tests/utils/bazel:integration.bzl", "integration_test") + +cc_library( + name = "lm_shutdown_common", + hdrs = ["common.hpp"], +) + +cc_binary( + name = "control_client_mock", + srcs = ["control_client_mock.cpp"], + deps = [ + ":lm_shutdown_common", + "//score/launch_manager:control_cc", + "//score/launch_manager:lifecycle_cc", + "//tests/utils/test_helper", + "@googletest//:gtest_main", + ], +) + +[ + cc_binary( + name = component, + srcs = [component + ".cpp"], + deps = [ + ":lm_shutdown_common", + "//score/launch_manager:lifecycle_cc", + "//tests/utils/test_helper", + "@googletest//:gtest_main", + ], + ) + for component in [ + "component_a", + "component_c", + ] +] + +integration_test( + name = "lm_shutdown_during_rt_switch", + timeout = "short", + srcs = ["lm_shutdown_during_rt_switch.py"], + binaries = [ + ":component_a", + ":component_c", + ":control_client_mock", + "//score/launch_manager", + ], + config = ":lm_shutdown_during_rt_switch.json", +) diff --git a/tests/integration/lm_shutdown_during_rt_switch/common.hpp b/tests/integration/lm_shutdown_during_rt_switch/common.hpp new file mode 100644 index 000000000..4e45fe3dd --- /dev/null +++ b/tests/integration/lm_shutdown_during_rt_switch/common.hpp @@ -0,0 +1,34 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +#ifndef SCORE_TESTS_INTEGRATION_LM_SHUTDOWN_COMMON_HPP +#define SCORE_TESTS_INTEGRATION_LM_SHUTDOWN_COMMON_HPP + +#include + +/// @brief Written by component_a when it has reported running (run_target_a is +/// active). +constexpr std::string_view a_started = "component_a_started"; + +/// @brief Written by component_a when it starts being terminated (i.e. the +/// switch away from run_target_a has begun). component_a then stalls, which +/// keeps the run-target switch in progress and gives the test a deterministic +/// window in which to send SIGTERM to the launch manager. +constexpr std::string_view a_terminating = "component_a_terminating"; + +/// @brief Written by component_c when it starts. component_c belongs only to +/// run_target_c, so this file must NEVER appear: a SIGTERM to the launch manager +/// during the switch must cancel the pending activation of run_target_c. +constexpr std::string_view c_started = "component_c_started"; + +#endif // SCORE_TESTS_INTEGRATION_LM_SHUTDOWN_COMMON_HPP diff --git a/tests/integration/lm_shutdown_during_rt_switch/component_a.cpp b/tests/integration/lm_shutdown_during_rt_switch/component_a.cpp new file mode 100644 index 000000000..3040c70ba --- /dev/null +++ b/tests/integration/lm_shutdown_during_rt_switch/component_a.cpp @@ -0,0 +1,61 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include +#include + +#include "common.hpp" +#include "tests/utils/test_helper/test_helper.hpp" +#include + +namespace +{ +/// @brief How long component_a stalls while it is being terminated. During a +/// run-target switch the launch manager runs the STOP phase (terminating +/// no-longer-needed processes) fully before the START phase (activating newly +/// needed processes). By stalling here, component_a keeps the switch in the STOP +/// phase, giving the test a deterministic window to send SIGTERM to the launch +/// manager before run_target_c could ever be activated. +/// +/// It must be larger than the time the test needs to observe `a_terminating` and +/// deliver the SIGTERM, and smaller than component_a's configured +/// shutdown_timeout so the process still exits gracefully (and writes its XML +/// result) rather than being SIGKILLed. +constexpr unsigned int kTerminationDelaySeconds = 2U; +} // namespace + +TEST(LmShutdownDuringRtSwitch, ComponentA) +{ + TEST_STEP("Report running") + { + EXPECT_TRUE(touch_file(a_started)) << "failed to deploy file"; + score::mw::lifecycle::report_running(); + } + + // Wait until the launch manager asks us to terminate (SIGTERM), which happens + // when the switch away from run_target_a begins. + while (!TestRunner::exitRequested) + { + pause(); + } + + TEST_STEP("Stall during termination to keep the run-target switch in progress") + { + EXPECT_TRUE(touch_file(a_terminating)) << "failed to deploy file"; + static_cast(sleep(kTerminationDelaySeconds)); + } +} + +int main() +{ + return TestRunner(__FILE__).RunTests(); +} diff --git a/tests/integration/lm_shutdown_during_rt_switch/component_c.cpp b/tests/integration/lm_shutdown_during_rt_switch/component_c.cpp new file mode 100644 index 000000000..299ef0385 --- /dev/null +++ b/tests/integration/lm_shutdown_during_rt_switch/component_c.cpp @@ -0,0 +1,46 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include +#include + +#include "common.hpp" +#include "tests/utils/test_helper/test_helper.hpp" +#include + +// component_c belongs only to run_target_c. Because the switch to run_target_c +// must be cancelled by the SIGTERM sent to the launch manager, this process must +// never be launched. Should it ever start, it records `c_started`, which makes +// both the control client and the Python-side assertions fail. +TEST(LmShutdownDuringRtSwitch, ComponentC) +{ + TEST_STEP("Report running") + { + // This code should be never executed. In Python code there is also an assertion + // that component_c must not be started (i.e. c_started should not exist). + // This is a second line of defense in case the Python code is not executed or fails to detect the problem. + ADD_FAILURE() << "component_c must never be started"; + + EXPECT_TRUE(touch_file(c_started)); + score::mw::lifecycle::report_running(); + } + + while (!TestRunner::exitRequested) + { + pause(); + } +} + +int main() +{ + return TestRunner(__FILE__).RunTests(); +} diff --git a/tests/integration/lm_shutdown_during_rt_switch/control_client_mock.cpp b/tests/integration/lm_shutdown_during_rt_switch/control_client_mock.cpp new file mode 100644 index 000000000..e3c2b8207 --- /dev/null +++ b/tests/integration/lm_shutdown_during_rt_switch/control_client_mock.cpp @@ -0,0 +1,76 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include +#include +#include + +#include "common.hpp" +#include "tests/utils/test_helper/test_helper.hpp" +#include +#include + +// The Launch Manager shall exit after performing a shutdown - stopping all the +// processes it owns in dependency order - when requested (i.e. when it receives +// a SIGTERM). A shutdown request takes priority over an in-progress run-target +// switch, which must therefore be cancelled. +// +// This control client activates run_target_a and then requests a switch to +// run_target_c. component_a (only part of run_target_a) stalls while it is being +// terminated during that switch, so the switch is still in progress when the +// test sends a SIGTERM to the launch manager from the Python side. The launch +// manager must then cancel the pending switch (component_c, only part of +// run_target_c, must never start) and shut everything down. +TEST(LmShutdownDuringRtSwitch, ControlClient) +{ + score::mw::lifecycle::ControlClient client{}; + ASSERT_TRUE(check_clean({test_end_location, a_started, a_terminating, c_started})); + + TEST_STEP("Report running") + { + score::mw::lifecycle::report_running(); + } + + TEST_STEP("Activate run_target_a") + { + score::cpp::stop_token stop_token; + auto result = client.ActivateRunTarget("run_target_a").Get(stop_token); + EXPECT_TRUE(result.has_value()) << "Activating run_target_a failed: " << result.error().Message(); + EXPECT_TRUE(std::filesystem::exists(a_started)) << "component_a was not started"; + } + + TEST_STEP("Request switch to run_target_c") + { + // Fire-and-forget: this transition is expected to be cancelled by an + // external SIGTERM to the launch manager, so we must not wait for a + // result. The launch manager will shut this process down instead of ever + // completing the switch. + client.ActivateRunTarget("run_target_c"); + } + + // Block until the launch manager terminates us as part of its own shutdown. + while (!TestRunner::exitRequested) + { + pause(); + } + + TEST_STEP("Verify run_target_c was never activated") + { + EXPECT_FALSE(std::filesystem::exists(c_started)) + << "run_target_c must not be activated: a SIGTERM to the launch manager must cancel the pending switch"; + } +} + +int main() +{ + return TestRunner(__FILE__, TerminationBehavior::kWait, TerminationNotification::kTestEnd).RunTests(); +} diff --git a/tests/integration/lm_shutdown_during_rt_switch/lm_shutdown_during_rt_switch.json b/tests/integration/lm_shutdown_during_rt_switch/lm_shutdown_during_rt_switch.json new file mode 100644 index 000000000..d1d538b17 --- /dev/null +++ b/tests/integration/lm_shutdown_during_rt_switch/lm_shutdown_during_rt_switch.json @@ -0,0 +1,114 @@ +{ + "schema_version": 1, + "defaults": { + "deployment_config": { + "bin_dir": "/tmp/tests/lm_shutdown_during_rt_switch", + "ready_timeout": 1.0, + "shutdown_timeout": 1.0, + "ready_recovery_action": { + "restart": { + "number_of_attempts": 0 + } + }, + "recovery_action": { + "switch_run_target": { + "run_target": "fallback_run_target" + } + }, + "environmental_variables": { + "LD_LIBRARY_PATH": "/opt/lib" + }, + "sandbox": { + "uid": 0, + "gid": 0, + "scheduling_policy": "SCHED_OTHER", + "scheduling_priority": 0 + } + }, + "component_properties": { + "application_profile": { + "application_type": "Reporting", + "is_self_terminating": false, + "alive_supervision": { + "reporting_cycle": 0.1, + "min_indications": 1, + "max_indications": 3, + "failed_cycles_tolerance": 1 + } + }, + "ready_condition": { + "process_state": "Running" + } + } + }, + "components": { + "component_initial": { + "component_properties": { + "binary_name": "control_client_mock", + "application_profile": { + "application_type": "State_Manager", + "alive_supervision": { + "min_indications": 0 + } + } + }, + "deployment_config": { + "ready_timeout": 1.0, + "shutdown_timeout": 1.0, + "environmental_variables": { + "PROCESSIDENTIFIER": "control_client_mock" + } + } + }, + "component_a": { + "component_properties": { + "binary_name": "component_a" + }, + "deployment_config": { + "shutdown_timeout": 5.0, + "environmental_variables": { + "PROCESSIDENTIFIER": "component_a" + } + } + }, + "component_c": { + "component_properties": { + "binary_name": "component_c" + }, + "deployment_config": { + "environmental_variables": { + "PROCESSIDENTIFIER": "component_c" + } + } + } + }, + "run_targets": { + "Startup": { + "depends_on": [ + "component_initial" + ] + }, + "run_target_a": { + "depends_on": [ + "component_initial", + "component_a" + ] + }, + "run_target_c": { + "depends_on": [ + "component_initial", + "component_c" + ] + }, + "Off": { + "depends_on": [] + } + }, + "initial_run_target": "Startup", + "alive_supervision": { + "evaluation_cycle": 0.05 + }, + "fallback_run_target": { + "depends_on": [] + } +} diff --git a/tests/integration/lm_shutdown_during_rt_switch/lm_shutdown_during_rt_switch.py b/tests/integration/lm_shutdown_during_rt_switch/lm_shutdown_during_rt_switch.py new file mode 100644 index 000000000..8a0824e15 --- /dev/null +++ b/tests/integration/lm_shutdown_during_rt_switch/lm_shutdown_during_rt_switch.py @@ -0,0 +1,160 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +import logging +import time + +from tests.utils.testing_utils.setup_test import setup_test +from tests.utils.testing_utils.test_results import assert_test_results +from attribute_plugin import add_test_properties + +logger = logging.getLogger(__name__) + + +def _wait_for_file(target, file_path, proc, timeout_s): + """Block until `file_path` exists on the target, or raise on timeout / early + exit of the launch manager process.""" + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + if not proc.is_running(): + raise RuntimeError( + f"Launch manager exited (code {proc.get_exit_code()}) before " + f"'{file_path}' appeared. Output:\n{proc.get_output()}" + ) + exit_code, _ = target.execute(f"test -f {file_path}") + if exit_code == 0: + return + time.sleep(0.05) + raise TimeoutError(f"'{file_path}' did not appear within {timeout_s}s") + + +def _pids_by_comm(target, name): + """Return the PIDs whose process name matches `name`. + + The test sandbox provides neither ``pgrep`` nor ``ps``, so processes are + located by scanning ``/proc//comm`` using only shell builtins and + ``cat``. Linux truncates the process name (``comm``) to 15 characters, so + the target name is truncated the same way before comparing. + """ + truncated = name[:15] + scan = ( + "for p in /proc/[0-9]*/comm; do " + 'c=$(cat "$p" 2>/dev/null) || continue; ' + f'if [ "$c" = "{truncated}" ]; then ' + "q=${p#/proc/}; echo ${q%/comm}; fi; " + "done" + ) + exit_code, stdout = target.execute(scan) + if exit_code != 0: + return [] + return [int(pid) for pid in stdout.decode().split()] + + +def _launch_manager_pid(target): + """Return the PID of the running launch_manager process, or None.""" + pids = _pids_by_comm(target, "launch_manager") + return pids[0] if pids else None + + +@add_test_properties( + fully_verifies=[], + partially_verifies=[ + "comp_req__lifecycle__launcher_exit_shutdown", + ], + test_type="requirements-based", + derivation_technique="requirements-analysis", +) +def test_lm_shutdown( + target, setup_test, assert_test_results, remote_test_dir, test_output_dir +): + """ + Objective: Verifies that the Launch Manager exits after performing a shutdown + (stopping all processes it owns) when requested via SIGTERM, and that this + shutdown takes priority over an in-progress run-target switch (the switch is + cancelled). + + The control client activates run_target_a and then requests a switch to + run_target_c. component_a (only part of run_target_a) stalls while it is being + terminated during the switch, keeping the switch in progress. At that point + the test sends a SIGTERM to the launch manager process (only that process, not + the whole group, so the launch manager performs its own orderly shutdown). + + Expected Behaviour: The launch manager cancels the pending switch - so + component_c (only part of run_target_c) is never started - stops all the + processes it owns, and exits cleanly. + """ + + new_config_path = str(remote_test_dir / "etc/lm_shutdown_during_rt_switch.bin") + + a_terminating = remote_test_dir / "component_a_terminating" + c_started = remote_test_dir / "component_c_started" + + proc = target.execute_async( + str(remote_test_dir / "launch_manager"), + args=["-c", new_config_path], + cwd=str(remote_test_dir), + ) + + try: + # Wait until the switch to run_target_c is underway: component_a is being + # terminated (and is now stalling), so run_target_c has not been activated + # yet. This state is signalled via file a_terminating. + # This is the window in which the shutdown request must win. + _wait_for_file(target, a_terminating, proc, timeout_s=10.0) + + # run_target_c must not have started yet at this point. + exit_code, _ = target.execute(f"test -f {c_started}") + # The assertion below could only fail if either the sleep in component_a's termination code is too short + # or component_a has been killed by launch manager, because it takes too long to react on SIGTERM. + assert exit_code != 0, ( + "run_target_c was activated before shutdown was requested - this should not happen" + ) + + # Request shutdown: send SIGTERM to the launch manager process only, so + # that the launch manager itself stops the processes it owns (rather than + # the OS terminating the whole process group directly). + lm_pid = _launch_manager_pid(target) + assert lm_pid is not None, "Could not find the running launch_manager process" + logger.info(f"Sending SIGTERM to launch_manager (pid {lm_pid})") + exit_code, _ = target.execute(f"kill -15 {lm_pid}") + assert exit_code == 0, "Failed to send SIGTERM to the launch manager" + + # The launch manager must exit after completing its shutdown. + deadline = time.monotonic() + 10.0 + while proc.is_running() and time.monotonic() < deadline: + time.sleep(0.1) + assert not proc.is_running(), ( + f"Launch manager did not exit after SIGTERM. Output:\n{proc.get_output()}" + ) + assert proc.get_exit_code() == 0, ( + f"Launch manager did not exit cleanly (code {proc.get_exit_code()}). " + f"Output:\n{proc.get_output()}" + ) + + # The pending switch must have been cancelled: run_target_c never activated. + exit_code, _ = target.execute(f"test -f {c_started}") + assert exit_code != 0, ( + "run_target_c was activated: the SIGTERM shutdown request must cancel the pending switch" + ) + + # The launch manager must have stopped all the processes it owns. + for binary in ("component_a", "component_c", "control_client_mock"): + running = _pids_by_comm(target, binary) + assert not running, ( + f"'{binary}' is still running (pids {running}) after launch manager shutdown" + ) + finally: + if proc.is_running(): + proc.stop() + + # component_c never runs, so it produces no XML result. + assert_test_results({"control_client_mock.xml", "component_a.xml"}) diff --git a/tests/integration/lm_shutdown_during_switch_to_off/BUILD b/tests/integration/lm_shutdown_during_switch_to_off/BUILD new file mode 100644 index 000000000..3dd1da6f1 --- /dev/null +++ b/tests/integration/lm_shutdown_during_switch_to_off/BUILD @@ -0,0 +1,55 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("//tests/utils/bazel:integration.bzl", "integration_test") + +cc_library( + name = "lm_shutdown_common", + hdrs = ["common.hpp"], +) + +cc_binary( + name = "control_client_mock", + srcs = ["control_client_mock.cpp"], + deps = [ + ":lm_shutdown_common", + "//score/launch_manager:control_cc", + "//score/launch_manager:lifecycle_cc", + "//tests/utils/test_helper", + "@googletest//:gtest_main", + ], +) + +cc_binary( + name = "component_a", + srcs = ["component_a.cpp"], + deps = [ + ":lm_shutdown_common", + "//score/launch_manager:lifecycle_cc", + "//tests/utils/test_helper", + "@googletest//:gtest_main", + ], +) + +integration_test( + name = "lm_shutdown_during_switch_to_off", + timeout = "short", + srcs = ["lm_shutdown_during_switch_to_off.py"], + binaries = [ + ":component_a", + ":control_client_mock", + "//score/launch_manager", + ], + config = ":lm_shutdown_during_switch_to_off.json", +) diff --git a/tests/integration/lm_shutdown_during_switch_to_off/common.hpp b/tests/integration/lm_shutdown_during_switch_to_off/common.hpp new file mode 100644 index 000000000..f3b5eb946 --- /dev/null +++ b/tests/integration/lm_shutdown_during_switch_to_off/common.hpp @@ -0,0 +1,30 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +#ifndef SCORE_TESTS_INTEGRATION_LM_SHUTDOWN_DURING_SWITCH_TO_OFF_COMMON_HPP +#define SCORE_TESTS_INTEGRATION_LM_SHUTDOWN_DURING_SWITCH_TO_OFF_COMMON_HPP + +#include + +/// @brief Written by component_a when it has reported running (run_target_a is +/// active). +constexpr std::string_view a_started = "component_a_started"; + +/// @brief Written by component_a when it starts being terminated (i.e. the +/// switch away from run_target_a - here, the switch to the "Off" run target - +/// has begun). component_a then stalls, which keeps the run-target switch in +/// progress and gives the test a deterministic window in which to send SIGTERM +/// to the launch manager. +constexpr std::string_view a_terminating = "component_a_terminating"; + +#endif // SCORE_TESTS_INTEGRATION_LM_SHUTDOWN_DURING_SWITCH_TO_OFF_COMMON_HPP diff --git a/tests/integration/lm_shutdown_during_switch_to_off/component_a.cpp b/tests/integration/lm_shutdown_during_switch_to_off/component_a.cpp new file mode 100644 index 000000000..26b9633eb --- /dev/null +++ b/tests/integration/lm_shutdown_during_switch_to_off/component_a.cpp @@ -0,0 +1,67 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include +#include + +#include "common.hpp" +#include "tests/utils/test_helper/test_helper.hpp" +#include + +namespace +{ +/// @brief How long component_a stalls while it is being terminated. During a +/// run-target switch the launch manager runs the STOP phase (terminating +/// no-longer-needed processes) fully before the START phase (activating newly +/// needed processes). By stalling here, component_a keeps the switch (to the +/// "Off" run target) in the STOP phase, giving the test a deterministic window +/// to send SIGTERM to the launch manager while the switch to Off is still in +/// progress. +/// +/// It must be comfortably larger than the time the test needs to observe +/// `a_terminating` and deliver the SIGTERM to the launch manager AND larger than +/// the launch manager's fixed shutdown grace period (see the NOTE in +/// ProcessGroupManager::allProcessGroupsOff): the shutdown does NOT respect the +/// per-process shutdown_timeout, so this stall outlives that grace period and +/// component_a is force-terminated (SIGKILLed) - it does not write an XML result. +/// It must also be smaller than component_a's configured shutdown_timeout so the +/// STOP job does not SIGKILL it on its own before the launch manager SIGTERM is +/// handled (which would end the switch to Off early and defeat the test). +constexpr unsigned int kTerminationDelaySeconds = 2U; +} // namespace + +TEST(LmShutdownDuringSwitchToOff, ComponentA) +{ + TEST_STEP("Report running") + { + EXPECT_TRUE(touch_file(a_started)) << "failed to deploy file"; + score::mw::lifecycle::report_running(); + } + + // Wait until the launch manager asks us to terminate (SIGTERM), which happens + // when the switch away from run_target_a (to the "Off" run target) begins. + while (!TestRunner::exitRequested) + { + pause(); + } + + TEST_STEP("Stall during termination to keep the run-target switch in progress") + { + EXPECT_TRUE(touch_file(a_terminating)) << "failed to deploy file"; + static_cast(sleep(kTerminationDelaySeconds)); + } +} + +int main() +{ + return TestRunner(__FILE__).RunTests(); +} diff --git a/tests/integration/lm_shutdown_during_switch_to_off/control_client_mock.cpp b/tests/integration/lm_shutdown_during_switch_to_off/control_client_mock.cpp new file mode 100644 index 000000000..8648d6a59 --- /dev/null +++ b/tests/integration/lm_shutdown_during_switch_to_off/control_client_mock.cpp @@ -0,0 +1,74 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include +#include +#include + +#include "common.hpp" +#include "tests/utils/test_helper/test_helper.hpp" +#include +#include + +// The Launch Manager shall exit after performing a shutdown - stopping all the +// processes it owns in dependency order - when requested (i.e. when it receives +// a SIGTERM). +// +// This variant differs from lm_shutdown_during_rt_switch: instead of switching +// to another (non-Off) run target, the control client explicitly switches to +// the "Off" run target. component_a (part of run_target_a) stalls while it is +// being terminated during that switch, so the switch to Off is still in progress +// when the test sends a SIGTERM to the launch manager from the Python side. +// +// Because the process group is ALREADY heading to Off, the SIGTERM-triggered +// shutdown must simply let that in-progress switch to Off continue to completion +// - it must NOT cancel the explicit switch to Off and redo it. Either way the +// launch manager must end up stopping everything it owns and exit cleanly. +TEST(LmShutdownDuringSwitchToOff, ControlClient) +{ + score::mw::lifecycle::ControlClient client{}; + ASSERT_TRUE(check_clean({test_end_location, a_started, a_terminating})); + + TEST_STEP("Report running") + { + score::mw::lifecycle::report_running(); + } + + TEST_STEP("Activate run_target_a") + { + score::cpp::stop_token stop_token; + auto result = client.ActivateRunTarget("run_target_a").Get(stop_token); + EXPECT_TRUE(result.has_value()) << "Activating run_target_a failed: " << result.error().Message(); + EXPECT_TRUE(std::filesystem::exists(a_started)) << "component_a was not started"; + } + + TEST_STEP("Request switch to Off") + { + // Fire-and-forget: switching to the "Off" run target terminates this + // control client too (it is not part of "Off"), so we must not wait for a + // result. The launch manager will shut this process down as part of the + // switch to Off. + client.ActivateRunTarget("Off"); + } + + // Block until the launch manager terminates us as part of the switch to Off / + // its own shutdown. + while (!TestRunner::exitRequested) + { + pause(); + } +} + +int main() +{ + return TestRunner(__FILE__, TerminationBehavior::kWait, TerminationNotification::kTestEnd).RunTests(); +} diff --git a/tests/integration/lm_shutdown_during_switch_to_off/lm_shutdown_during_switch_to_off.json b/tests/integration/lm_shutdown_during_switch_to_off/lm_shutdown_during_switch_to_off.json new file mode 100644 index 000000000..b1b9beafa --- /dev/null +++ b/tests/integration/lm_shutdown_during_switch_to_off/lm_shutdown_during_switch_to_off.json @@ -0,0 +1,98 @@ +{ + "schema_version": 1, + "defaults": { + "deployment_config": { + "bin_dir": "/tmp/tests/lm_shutdown_during_switch_to_off", + "ready_timeout": 1.0, + "shutdown_timeout": 1.0, + "ready_recovery_action": { + "restart": { + "number_of_attempts": 0 + } + }, + "recovery_action": { + "switch_run_target": { + "run_target": "fallback_run_target" + } + }, + "environmental_variables": { + "LD_LIBRARY_PATH": "/opt/lib" + }, + "sandbox": { + "uid": 0, + "gid": 0, + "scheduling_policy": "SCHED_OTHER", + "scheduling_priority": 0 + } + }, + "component_properties": { + "application_profile": { + "application_type": "Reporting", + "is_self_terminating": false, + "alive_supervision": { + "reporting_cycle": 0.1, + "min_indications": 1, + "max_indications": 3, + "failed_cycles_tolerance": 1 + } + }, + "ready_condition": { + "process_state": "Running" + } + } + }, + "components": { + "component_initial": { + "component_properties": { + "binary_name": "control_client_mock", + "application_profile": { + "application_type": "State_Manager", + "alive_supervision": { + "min_indications": 0 + } + } + }, + "deployment_config": { + "ready_timeout": 1.0, + "shutdown_timeout": 1.0, + "environmental_variables": { + "PROCESSIDENTIFIER": "control_client_mock" + } + } + }, + "component_a": { + "component_properties": { + "binary_name": "component_a" + }, + "deployment_config": { + "shutdown_timeout": 5.0, + "environmental_variables": { + "PROCESSIDENTIFIER": "component_a" + } + } + } + }, + "run_targets": { + "Startup": { + "depends_on": [ + "component_initial" + ] + }, + "run_target_a": { + "depends_on": [ + "component_initial", + "component_a" + ] + }, + "Off": { + "depends_on": [] + } + }, + "initial_run_target": "Startup", + "alive_supervision": { + "evaluation_cycle": 0.05 + }, + "fallback_run_target": { + "depends_on": [] + } +} diff --git a/tests/integration/lm_shutdown_during_switch_to_off/lm_shutdown_during_switch_to_off.py b/tests/integration/lm_shutdown_during_switch_to_off/lm_shutdown_during_switch_to_off.py new file mode 100644 index 000000000..bc7c7e14d --- /dev/null +++ b/tests/integration/lm_shutdown_during_switch_to_off/lm_shutdown_during_switch_to_off.py @@ -0,0 +1,196 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +import logging +import time + +from tests.utils.testing_utils.setup_test import setup_test +from tests.utils.testing_utils.test_results import assert_test_results +from attribute_plugin import add_test_properties + +logger = logging.getLogger(__name__) + + +def _wait_for_file(target, file_path, proc, timeout_s): + """Block until `file_path` exists on the target, or raise on timeout / early + exit of the launch manager process.""" + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + if not proc.is_running(): + raise RuntimeError( + f"Launch manager exited (code {proc.get_exit_code()}) before " + f"'{file_path}' appeared. Output:\n{proc.get_output()}" + ) + exit_code, _ = target.execute(f"test -f {file_path}") + if exit_code == 0: + return + time.sleep(0.05) + raise TimeoutError(f"'{file_path}' did not appear within {timeout_s}s") + + +def _pids_by_comm(target, name): + """Return the PIDs whose process name matches `name`. + + The test sandbox provides neither ``pgrep`` nor ``ps``, so processes are + located by scanning ``/proc//comm`` using only shell builtins and + ``cat``. Linux truncates the process name (``comm``) to 15 characters, so + the target name is truncated the same way before comparing. + """ + truncated = name[:15] + scan = ( + "for p in /proc/[0-9]*/comm; do " + 'c=$(cat "$p" 2>/dev/null) || continue; ' + f'if [ "$c" = "{truncated}" ]; then ' + "q=${p#/proc/}; echo ${q%/comm}; fi; " + "done" + ) + exit_code, stdout = target.execute(scan) + if exit_code != 0: + return [] + return [int(pid) for pid in stdout.decode().split()] + + +def _launch_manager_pid(target): + """Return the PID of the running launch_manager process, or None.""" + pids = _pids_by_comm(target, "launch_manager") + return pids[0] if pids else None + + +@add_test_properties( + fully_verifies=[], + partially_verifies=[ + "comp_req__lifecycle__launcher_exit_shutdown", + ], + test_type="requirements-based", + derivation_technique="requirements-analysis", +) +def test_lm_shutdown( + target, setup_test, assert_test_results, remote_test_dir, test_output_dir +): + """ + Objective: Verifies that the Launch Manager exits after performing a shutdown + (stopping all processes it owns) when requested via SIGTERM, and that a + SIGTERM that arrives while an explicit switch to the "Off" run target is + already in progress lets that switch to Off continue to completion (it must + NOT be cancelled and redone). + + The control client activates run_target_a and then explicitly requests a + switch to the "Off" run target. component_a (part of run_target_a) stalls + while it is being terminated during that switch, keeping the switch to Off in + progress. At that point the test sends a SIGTERM to the launch manager process + (only that process, not the whole group, so the launch manager performs its + own orderly shutdown). + + Expected Behaviour: The launch manager lets the in-progress switch to Off + continue - it must NOT cancel the explicit switch to Off and redo it - all the + processes it owns are stopped, and it exits cleanly. + + NOTE: the launch manager does NOT respect the per-process shutdown_timeout + during its own shutdown (see the NOTE in ProcessGroupManager:: + allProcessGroupsOff). It applies a single fixed grace period to the whole + in-progress transition, after which any process still terminating is + force-terminated (SIGKILLed). component_a deliberately stalls past that grace + period, so it is SIGKILLed and does NOT produce an XML result - this test + therefore does not assert on component_a's XML. + + Note: this test currently FAILS. It documents a launch manager defect: on + SIGTERM, allProcessGroupsOff() unconditionally cancels every process group - + including one that is already transitioning to Off - and then restarts the + transition to Off, instead of letting the in-progress switch to Off continue. + The test passes once that defect is fixed. + """ + + new_config_path = str(remote_test_dir / "etc/lm_shutdown_during_switch_to_off.bin") + + a_terminating = remote_test_dir / "component_a_terminating" + + proc = target.execute_async( + str(remote_test_dir / "launch_manager"), + args=["-c", new_config_path], + cwd=str(remote_test_dir), + ) + + try: + # Wait until the switch to Off is underway: component_a is being terminated + # (and is now stalling). This is signalled via file a_terminating and is + # the window in which the shutdown request arrives. + _wait_for_file(target, a_terminating, proc, timeout_s=10.0) + + # Request shutdown: send SIGTERM to the launch manager process only, so + # that the launch manager itself stops the processes it owns (rather than + # the OS terminating the whole process group directly). + lm_pid = _launch_manager_pid(target) + assert lm_pid is not None, "Could not find the running launch_manager process" + logger.info(f"Sending SIGTERM to launch_manager (pid {lm_pid})") + exit_code, _ = target.execute(f"kill -15 {lm_pid}") + assert exit_code == 0, "Failed to send SIGTERM to the launch manager" + + # The launch manager must exit after completing its shutdown. + deadline = time.monotonic() + 10.0 + while proc.is_running() and time.monotonic() < deadline: + time.sleep(0.1) + assert not proc.is_running(), ( + f"Launch manager did not exit after SIGTERM. Output:\n{proc.get_output()}" + ) + assert proc.get_exit_code() == 0, ( + f"Launch manager did not exit cleanly (code {proc.get_exit_code()}). " + f"Output:\n{proc.get_output()}" + ) + + # The launch manager must have stopped all the processes it owns. + for binary in ("component_a", "control_client_mock"): + running = _pids_by_comm(target, binary) + assert not running, ( + f"'{binary}' is still running (pids {running}) after launch manager shutdown" + ) + + # Core assertion: the explicit switch to Off must be CONTINUED, not + # cancelled and redone. + # + # The switch to Off was already in progress (MainPG in transition to Off) + # when the SIGTERM arrived, so the SIGTERM-triggered shutdown must simply + # let it finish. If instead the launch manager cancels that transition, the + # main process group logs a "kInTransition -> kCancelled" transition after + # the SIGTERM was received. That must not happen. (The only legitimate + # cancellation - the Startup -> run_target_a switch - happens before the + # SIGTERM, so scoping the check to after the SIGTERM marker excludes it.) + output = proc.get_output() + if isinstance(output, bytes): + output = output.decode(errors="replace") + + sigterm_marker = "received SIGTERM" + marker_index = output.find(sigterm_marker) + assert marker_index != -1, ( + f"Launch manager never logged receiving the SIGTERM.\nOutput:\n{output}" + ) + output_after_sigterm = output[marker_index:] + + cancelled_lines = [ + line + for line in output_after_sigterm.splitlines() + if "to kCancelled" in line and "MainPG" in line + ] + assert not cancelled_lines, ( + "The explicit switch to Off was cancelled and redone by the launch " + "manager during shutdown instead of being continued. Offending log " + "line(s):\n" + "\n".join(cancelled_lines) + "\n\n" + f"Full launch manager output after SIGTERM:\n{output_after_sigterm}" + ) + finally: + if proc.is_running(): + proc.stop() + + # component_a is force-terminated (SIGKILLed) during shutdown (its stall + # outlives the launch manager's fixed shutdown grace period), so it does not + # produce an XML result and is intentionally not asserted here. control_client + # is stopped gracefully as part of the switch to Off and produces its result. + assert_test_results({"control_client_mock.xml"}) From d555b8cfd2504013386bfb2cc03932cd1f943073 Mon Sep 17 00:00:00 2001 From: Timo Steuerwald Date: Tue, 11 Aug 2026 16:21:30 +0200 Subject: [PATCH 02/23] Add crashdump support in docker environment --- .bazelrc | 2 + tests/integration/readme.md | 71 +++++++++++++++ tests/utils/bazel/integration.bzl | 7 +- tests/utils/plugins/integration.py | 140 +++++++++++++++++++++++++++++ 4 files changed, 218 insertions(+), 2 deletions(-) diff --git a/.bazelrc b/.bazelrc index 2a35375ed..c5c061d29 100644 --- a/.bazelrc +++ b/.bazelrc @@ -72,6 +72,8 @@ build:x86_64-linux --extra_toolchains=@score_toolchains_rust//toolchains/ferroce test:x86_64-linux --//config:integration_mode=docker test:x86_64-linux --//config:unit_mode=host +# Show a failing test's log (incl. the crash-dump banner) in the console. +test:x86_64-linux --test_output=errors # Target configuration for CPU:AArch64|OS:Linux build (do not use it in case of system toolchains!) build:arm64-linux --config=stub diff --git a/tests/integration/readme.md b/tests/integration/readme.md index 92d514dd3..46c0a8e2e 100644 --- a/tests/integration/readme.md +++ b/tests/integration/readme.md @@ -23,3 +23,74 @@ Currently the following configs are supported: - `host` - `x86_64-linux` +## Crash dumps (core dumps) + +When a binary under test (e.g. the launch manager) crashes with `SIGSEGV`, +`SIGABRT`, etc. inside the Docker sandbox, a core dump is captured +automatically. This is wired in commonly for every `integration_test`, so +individual tests need no changes. + +How it works: +- The sandbox container runs privileged with an unlimited core-file `ulimit`. +- A shared fixture sets the kernel `core_pattern` to a sandbox-local path + (`/tmp/score_cores/core.%e.%p.%s.%t`), copies any core dumps produced during + the test into the Bazel test outputs, and then restores the original + `core_pattern`. + +### Getting a crash dump + +Run the (crashing) test, disabling the cache so it actually executes: +``` +bazel test //tests/integration/ --config=x86_64-linux --nocache_test_results +``` + +If a crash dump was created, a `CRASH DUMP` section is printed right under the +pytest `FAILURES` section at the end of the run (the `x86_64-linux` config +enables `--test_output=errors`, so the failing log is shown automatically): +``` +=================================== FAILURES =================================== +... +================================== CRASH DUMP ================================== +CRASH DUMP HAS BEEN CREATED! See <.../test.outputs/cores> for details. + +To open it in gdb (build the crashing binary with -c dbg for symbols): + gdb <.../bin/.../launch_manager> "<.../test.outputs/cores/core.launch_manager.*>" +=========================== short test summary info ============================ +``` +The printed paths are absolute and copy-pasteable. Core files are named +`core....