From ed0948512aef4e11031cce0a5299251b99f75d59 Mon Sep 17 00:00:00 2001 From: Matt Morehouse Date: Wed, 16 Sep 2026 15:44:01 -0500 Subject: [PATCH] workloads/cln: patch coverage build to break deadlock It turns out that removing the SIGKILL from lightningd/subd.c can cause CLN to deadlock on shutdown, which holds up coverage reporting. Add the SIGKILL back in, but first attempt to gracefully shutdown by closing the subdaemon socket and waiting 5s for it. The socket close handles most of the deadlock cases while preserving coverage data. The fallback SIGKILL causes some coverage data to be lost but prevents coverage runs from deadlocking forever. --- workloads/cln/Dockerfile.coverage | 15 ++++----- .../cln/patches/subd-coverage-shutdown.patch | 33 +++++++++++++++++++ 2 files changed, 40 insertions(+), 8 deletions(-) create mode 100644 workloads/cln/patches/subd-coverage-shutdown.patch diff --git a/workloads/cln/Dockerfile.coverage b/workloads/cln/Dockerfile.coverage index 4345c278..20ff4936 100644 --- a/workloads/cln/Dockerfile.coverage +++ b/workloads/cln/Dockerfile.coverage @@ -65,14 +65,13 @@ RUN git clone --depth 1 --branch ${CLN_VERSION} --recurse-submodules \ https://github.com/ElementsProject/lightning.git /cln WORKDIR /cln RUN pip3 install --break-system-packages mako -# Currently CLN does an unnecessary SIGKILL of per-peer subdaemons (openingd, -# channeld, etc.) during node shutdown. If the SIGKILL arrives before the -# subdaemon shuts down gracefully on its own, we lose coverage profile data -# (profraw files) for that subdaemon. -# -# Remove the SIGKILL to ensure we get full coverage data. -RUN grep -q 'kill(sd->pid, SIGKILL)' lightningd/subd.c && \ - sed -i '/kill(sd->pid, SIGKILL)/d' lightningd/subd.c +# When a per-peer subdaemon (openingd, channeld, etc.) dies mid-run (e.g. after +# a channel error), lightningd SIGKILLs it. SIGKILL skips the atexit handler +# that writes the profraw file, so we lose that subdaemon's entire coverage +# profile. Close its socket instead, which makes it exit by itself, only killing +# as a fallback. +COPY workloads/cln/patches/subd-coverage-shutdown.patch . +RUN git apply subd-coverage-shutdown.patch # Reduce the block-polling interval from the 30s default to 2s. CLN only # notices newly confirmed transactions when this poll runs, so shorten the # interval to make block-related events available to scenarios promptly. diff --git a/workloads/cln/patches/subd-coverage-shutdown.patch b/workloads/cln/patches/subd-coverage-shutdown.patch new file mode 100644 index 00000000..ed3bf6b1 --- /dev/null +++ b/workloads/cln/patches/subd-coverage-shutdown.patch @@ -0,0 +1,33 @@ +Let a dying subdaemon exit by itself before SIGKILLing it. + +SIGKILL skips the atexit handler that writes the subdaemon's LLVM profile data, +so a coverage build loses every line it executed. + +Closing our end of its socket gets it to exit on its own, flushing the profraw +file. This is what subd_shutdown() already does for the global daemons, so we +apply the same approach here for destroy_subd(). + +Closing is not always enough -- subdaemons can still block on the HSM fd, which +causes lightningd to wait several minutes for it. So keep the kill as a +fallback after 5s. + +--- a/lightningd/subd.c ++++ b/lightningd/subd.c +@@ -611,9 +611,17 @@ + if (!sd->must_not_exit) { + log_debug(sd->log, + "Status closed, but not exited. Killing"); ++ sd->conn = tal_free(sd->conn); ++ for (size_t i = 0; i < 500; i++) { ++ if (waitpid(sd->pid, &status, ++ WNOHANG) > 0) ++ goto reaped; ++ usleep(10000); ++ } + kill(sd->pid, SIGKILL); + } + waitpid(sd->pid, &status, 0); ++ reaped: + fail_if_subd_fails = false; + break; + case -1: