From 88800d853c6da8c5001213e7eb0c92a73d6fee4c Mon Sep 17 00:00:00 2001 From: Matt Morehouse Date: Fri, 18 Sep 2026 12:19:56 -0500 Subject: [PATCH 1/2] workloads/cln: patch coverage build to break deadlock It turns out that removing the SIGKILL from lightningd/subd.c can cause CLN to deadlock on shutdown, which holds up coverage reporting. We can resolve most of the deadlock cases by closing the subdaemon socket and allowing the subdaemon to gracefully shut down. --- workloads/cln/Dockerfile.coverage | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/workloads/cln/Dockerfile.coverage b/workloads/cln/Dockerfile.coverage index 4345c278..105a39f9 100644 --- a/workloads/cln/Dockerfile.coverage +++ b/workloads/cln/Dockerfile.coverage @@ -65,14 +65,15 @@ RUN git clone --depth 1 --branch ${CLN_VERSION} --recurse-submodules \ https://github.com/ElementsProject/lightning.git /cln WORKDIR /cln RUN pip3 install --break-system-packages mako -# Currently CLN does an unnecessary SIGKILL of per-peer subdaemons (openingd, -# channeld, etc.) during node shutdown. If the SIGKILL arrives before the -# subdaemon shuts down gracefully on its own, we lose coverage profile data -# (profraw files) for that subdaemon. +# When lightningd frees a per-peer subdaemon (openingd, channeld, etc.) that has +# not exited yet, it SIGKILLs it. SIGKILL skips the atexit handler that writes +# the profraw file, so we lose that subdaemon's entire coverage profile. # -# Remove the SIGKILL to ensure we get full coverage data. -RUN grep -q 'kill(sd->pid, SIGKILL)' lightningd/subd.c && \ - sed -i '/kill(sd->pid, SIGKILL)/d' lightningd/subd.c +# Close our end of its socket instead: the subdaemon then exits by itself and +# flushes its profile. Note that destroy_subd() goes on to wait for the exit +# with no timeout, so without this close it would block forever. +RUN grep -q 'kill(sd->pid, SIGKILL);' lightningd/subd.c && \ + sed -i 's/kill(sd->pid, SIGKILL);/sd->conn = tal_free(sd->conn);/' lightningd/subd.c # Reduce the block-polling interval from the 30s default to 2s. CLN only # notices newly confirmed transactions when this poll runs, so shorten the # interval to make block-related events available to scenarios promptly. From 6e62dceeeca0400580b168da554c9e8fb4a7f0ff Mon Sep 17 00:00:00 2001 From: Matt Morehouse Date: Fri, 18 Sep 2026 12:38:41 -0500 Subject: [PATCH 2/2] smite-scenarios: drop connection before target Fields are dropped in declaration order, so the first field is dropped before the second. This change ensures our connection to the peer is always closed before we attempt to shut down the target. This only matters for local mode -- when fuzzing in Nyx the snapshot is always restored before the scenario object is dropped anway. In local mode, it avoids a shutdown deadlock for CLN where our open connection prevents CLN from shutting down cleanly until connectd's keepalive ping fails and causes CLN to drop the connection itself. --- smite-scenarios/src/scenarios/encrypted_bytes.rs | 6 ++++-- smite-scenarios/src/scenarios/init.rs | 6 ++++-- smite-scenarios/src/scenarios/ir.rs | 7 +++++-- smite-scenarios/src/scenarios/noise.rs | 6 ++++-- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/smite-scenarios/src/scenarios/encrypted_bytes.rs b/smite-scenarios/src/scenarios/encrypted_bytes.rs index 0d235b99..f6c50de6 100644 --- a/smite-scenarios/src/scenarios/encrypted_bytes.rs +++ b/smite-scenarios/src/scenarios/encrypted_bytes.rs @@ -16,8 +16,10 @@ use crate::targets::Target; /// them over an encrypted Lightning connection. This can find parsing bugs or /// crashes from malformed messages. pub struct EncryptedBytesScenario { - target: T, + /// Declared before `target` so the peer connection closes first and doesn't + /// stall the target's shutdown. conn: NoiseConnection, + target: T, } impl Scenario for EncryptedBytesScenario { @@ -34,7 +36,7 @@ impl Scenario for EncryptedBytesScenario { // speeding up every subsequent iteration. ping_pong(&mut conn)?; - Ok(Self { target, conn }) + Ok(Self { conn, target }) } fn run(&mut self, input: &[u8]) -> ScenarioResult { diff --git a/smite-scenarios/src/scenarios/init.rs b/smite-scenarios/src/scenarios/init.rs index e81869d8..487ef89c 100644 --- a/smite-scenarios/src/scenarios/init.rs +++ b/smite-scenarios/src/scenarios/init.rs @@ -25,8 +25,10 @@ const TIMEOUT: Duration = Duration::from_secs(5); /// ping-pong on the same connection to ensure it has processed the data /// before checking for crashes. pub struct InitScenario { - target: T, + /// Declared before `target` so the peer connection closes first and doesn't + /// stall the target's shutdown. conn: NoiseConnection, + target: T, } impl Scenario for InitScenario { @@ -47,7 +49,7 @@ impl Scenario for InitScenario { // the target's init. let (conn, _) = handshake_with_target(&target, TIMEOUT)?; - Ok(Self { target, conn }) + Ok(Self { conn, target }) } fn run(&mut self, input: &[u8]) -> ScenarioResult { diff --git a/smite-scenarios/src/scenarios/ir.rs b/smite-scenarios/src/scenarios/ir.rs index 265b165b..827bfff0 100644 --- a/smite-scenarios/src/scenarios/ir.rs +++ b/smite-scenarios/src/scenarios/ir.rs @@ -20,11 +20,14 @@ use crate::targets::Target; /// mutators or generators; the executor panics on invariant violations /// (out-of-bounds variable refs, type mismatches, `MineBlocks(0)`, etc.). pub struct IrScenario> { - target: T, /// Executes IR programs and owns the connection, bitcoin-cli handle, /// program context, and the target's RPC handle. Created once before the /// snapshot and reused across fuzzing runs. + /// + /// Declared before `target` so the peer connection closes first and doesn't + /// stall the target's shutdown. executor: Executor, + target: T, // S is only used for static dispatch on S::setup(), not stored. _phantom: PhantomData, } @@ -36,8 +39,8 @@ impl> Scenario for IrScenario { let bitcoin_cli = target.bitcoin_cli().clone(); let executor = Executor::new(conn, bitcoin_cli, target.rpc(), context); Ok(Self { - target, executor, + target, _phantom: PhantomData, }) } diff --git a/smite-scenarios/src/scenarios/noise.rs b/smite-scenarios/src/scenarios/noise.rs index 3d96df1c..55762668 100644 --- a/smite-scenarios/src/scenarios/noise.rs +++ b/smite-scenarios/src/scenarios/noise.rs @@ -46,9 +46,11 @@ const FUZZ_STATIC_KEY: [u8; 32] = [ /// ping-pong on the sync connection ensures the target has some time to process /// the fuzz data before we check for crashes. pub struct NoiseScenario { - target: T, + /// Both peer connections are declared before `target` so the peer + /// connections close first and don't stall the target's shutdown. stream: TcpStream, sync_conn: NoiseConnection, + target: T, } impl NoiseScenario { @@ -245,9 +247,9 @@ impl Scenario for NoiseScenario { stream.set_write_timeout(Some(TIMEOUT))?; Ok(Self { - target, stream, sync_conn, + target, }) }