CI: switch ThreadSanitizer to a gcc-based, fully instrumented build

.github/actions/setup-deps/action.yml

@@ -6,7 +6,7 @@ inputs:
     description: 'GCC version to use'
     required: true
   env:
-    description: 'Spack environment name (latest, boost187)'
+    description: 'Spack environment name (latest, boost187, tsan)'
     default: 'latest'
   fresh:
     description: 'Use fresh concretization'
@@ -56,8 +56,11 @@ runs:
           # Push the gcc node now, BEFORE `spack compiler find` registers it as an
           # external -- externals are excluded from `buildcache push`. This is what
           # lets CI pull gcc from the cache (~1 min) instead of building it (~1 h).
+          # --allow-missing: when gcc itself was pulled from the cache, its
+          # build-time dependencies are not installed locally and would
+          # otherwise fail the push.
           spack mirror set --oci-username ${{ github.actor }} --oci-password-variable GITHUB_TOKEN ghcr-buildcache
-          spack buildcache push --unsigned ghcr-buildcache /$gcc_hash
+          spack buildcache push --unsigned --allow-missing ghcr-buildcache /$gcc_hash
         fi
         spack compiler find "$(spack location -i /$gcc_hash)"
         echo "::endgroup::"
@@ -66,6 +69,7 @@ runs:
       shell: spack-bash {0}
       run: |
         echo "::group::Install dependencies"
+        spack repo add "$GITHUB_WORKSPACE/test/ci/spack_repo/fairmq_ci"
         spack env create fairmq test/ci/spack-${{ inputs.env }}.yaml
         spack -e fairmq add gcc@${{ inputs.gcc }}
         spack -e fairmq config add "packages:all:require:'%gcc@${{ inputs.gcc }}'"

.github/workflows/buildcache.yml

@@ -8,6 +8,7 @@ on:
     branches: [dev, master]
     paths:
       - 'test/ci/spack-*.yaml'
+      - 'test/ci/spack_repo/**'
       - '.github/workflows/buildcache.yml'
       - '.github/actions/setup-deps/**'
 
@@ -30,6 +31,8 @@ jobs:
         include:
           - gcc: '15'
             env: 'boost187'
+          - gcc: '15'
+            env: 'tsan'
 
     steps:
       - uses: actions/checkout@v6
@@ -50,7 +53,10 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           spack -e fairmq mirror set --oci-username ${{ github.actor }} --oci-password-variable GITHUB_TOKEN ghcr-buildcache
-          spack -e fairmq buildcache push --unsigned ghcr-buildcache
+          # --allow-missing: build-time dependencies of specs that were
+          # satisfied from the buildcache are not installed locally and
+          # would otherwise fail the whole push.
+          spack -e fairmq buildcache push --unsigned --allow-missing ghcr-buildcache
 
   update-index:
     if: github.repository == 'FairRootGroup/FairMQ' && !cancelled()

.github/workflows/ci.yml

@@ -37,13 +37,13 @@ jobs:
           env: ${{ matrix.env }}
 
       - name: ccache
-        uses: hendrikmuhs/ccache-action@v1
+        uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23
         with:
           key: ${{ github.job }}-${{ matrix.env }}-gcc${{ matrix.gcc }}
           max-size: 500M
 
       - name: Configure and Build
-        uses: threeal/cmake-action@v2
+        uses: threeal/cmake-action@725d1314ccf9ea922805d7e3f9d9bcbca892b406 # v2.1.0
         with:
           generator: Ninja
           options: |
@@ -68,22 +68,23 @@ jobs:
     if: github.repository == 'FairRootGroup/FairMQ'
     name: ${{ matrix.sanitizer.name }}
     runs-on: ubuntu-24.04
-    env:
-      TSAN_OPTIONS: suppressions=${{ github.workspace }}/test/thread_sanitizer_suppressions.txt
     strategy:
       fail-fast: false
       matrix:
         sanitizer:
           - name: asan+lsan+ubsan
+            gcc: '14'
+            env: latest
             options: |
               ENABLE_SANITIZER_ADDRESS=ON
               ENABLE_SANITIZER_LEAK=ON
-              ENABLE_SANITIZER_UNDEFINED_BEHAVIOUR=ON
+              ENABLE_SANITIZER_UNDEFINED_BEHAVIOR=ON
             cxx-flags: -O1 -fno-omit-frame-pointer
           - name: tsan
+            gcc: '15'
+            env: tsan
             options: ENABLE_SANITIZER_THREAD=ON
-            cxx-compiler: clang++
-            cxx-flags: -fuse-ld=lld
+            cxx-flags: -fno-omit-frame-pointer
 
     steps:
       - uses: actions/checkout@v6
@@ -95,31 +96,64 @@ jobs:
       - name: Setup spack environment
         uses: ./.github/actions/setup-deps
         with:
-          gcc: '14'
+          gcc: ${{ matrix.sanitizer.gcc }}
+          env: ${{ matrix.sanitizer.env }}
 
       - name: ccache
-        uses: hendrikmuhs/ccache-action@v1
+        uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23
         with:
           key: ${{ github.job }}-${{ matrix.sanitizer.name }}
           max-size: 500M
 
-      - name: Install lld
+      - name: Locate instrumented libstdc++
         if: matrix.sanitizer.name == 'tsan'
-        run: sudo apt-get update && sudo apt-get install -y lld
+        shell: spack-bash {0}
+        # The test processes must load the tsan-instrumented libstdc++
+        # instead of the compiler's own (same soname, LD_LIBRARY_PATH beats
+        # the RUNPATH). Set per test via ctest, not at the job level: like
+        # any shared library built with gcc -fsanitize=thread it has
+        # unresolved __tsan_* symbols, so loading it into uninstrumented
+        # tools (cmake, ctest, ninja) would break them.
+        run: |
+          prefix=$(spack -e fairmq location -i libstdcxx-tsan)
+          echo "test_library_path=$prefix/lib" >> $GITHUB_ENV
 
       - name: Configure and Build
-        uses: threeal/cmake-action@v2
+        uses: threeal/cmake-action@725d1314ccf9ea922805d7e3f9d9bcbca892b406 # v2.1.0
         with:
           generator: Ninja
-          cxx-compiler: ${{ matrix.sanitizer.cxx-compiler }}
           cxx-flags: ${{ matrix.sanitizer.cxx-flags }}
           options: |
             CMAKE_BUILD_TYPE=Debug
             BUILD_TESTING=ON
             CMAKE_C_COMPILER_LAUNCHER=ccache
             CMAKE_CXX_COMPILER_LAUNCHER=ccache
+            FAIRMQ_TEST_LD_LIBRARY_PATH=${{ env.test_library_path }}
             ${{ matrix.sanitizer.options }}
 
+      - name: Verify tsan instrumentation wiring
+        if: matrix.sanitizer.name == 'tsan'
+        shell: spack-bash {0}
+        run: |
+          set -x
+          # the test environment must resolve libstdc++ to the instrumented copy
+          LD_LIBRARY_PATH=$test_library_path ldd build/test/testsuite_Channel \
+            | grep 'libstdc++' | tee /dev/stderr | grep -q libstdcxx-tsan
+          # libzmq must be instrumented
+          nm -D --undefined-only "$(spack -e fairmq location -i libzmq)/lib/libzmq.so" \
+            | grep -q __tsan_
+          # the instrumented libstdc++ must match the compiler release exactly,
+          # or binaries may reference GLIBCXX versions the runtime lacks
+          # (--color=never: the CI config forces SPACK_COLOR=always, which
+          # would wrap the version in ANSI escapes)
+          test "$(spack --color=never -e fairmq find --format '{version}' libstdcxx-tsan)" \
+            = "$(g++ -dumpfullversion)"
+          # the LD_LIBRARY_PATH prepend must reach the registered tests
+          # (via a file: grep -q quits on first match, and SIGPIPE on the
+          # large json output would fail the step under pipefail)
+          ctest --test-dir build --show-only=json-v1 > ctest-show-only.json
+          grep -q libstdcxx-tsan ctest-show-only.json
+
       - name: Test
         run: |
           # Region/segment tests mlock() shared memory; raise the locked-memory
@@ -146,7 +180,7 @@ jobs:
           gcc: '14'
 
       - name: ccache
-        uses: hendrikmuhs/ccache-action@v1
+        uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23
         with:
           key: ${{ github.job }}
           max-size: 500M

CMakeLists.txt

@@ -46,6 +46,13 @@ include(FairMQDependencies)
 
 
 # Targets ######################################################################
+if(FAIRMQ_TEST_LD_LIBRARY_PATH AND CMAKE_VERSION VERSION_LESS 3.22)
+  # The per-test injection relies on ctest's ENVIRONMENT_MODIFICATION, which
+  # older CMake silently drops -- the tests would run against the default
+  # runtime while looking green.
+  message(FATAL_ERROR "FAIRMQ_TEST_LD_LIBRARY_PATH requires CMake >= 3.22")
+endif()
+
 if(BUILD_FAIRMQ)
   add_subdirectory(fairmq)
 endif()

cmake/FairMQProjectSettings.cmake

@@ -54,8 +54,8 @@ endif()
 list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_LIBDIR}" isSystemDir)
 if("${isSystemDir}" STREQUAL "-1")
   if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
-    list(APPEND CMAKE_EXE_LINKER_FLAGS "-Wl,--enable-new-dtags")
-    list(APPEND CMAKE_SHARED_LINKER_FLAGS "-Wl,--enable-new-dtags")
+    string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--enable-new-dtags")
+    string(APPEND CMAKE_SHARED_LINKER_FLAGS " -Wl,--enable-new-dtags")
     list(PREPEND CMAKE_INSTALL_RPATH "$ORIGIN/../${PROJECT_INSTALL_LIBDIR}")
   elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
     list(PREPEND CMAKE_INSTALL_RPATH "@loader_path/../${PROJECT_INSTALL_LIBDIR}")

examples/CMakeLists.txt

@@ -12,6 +12,22 @@ set(test_script_prefix "test-ex")
 set(testsuite "Example")
 set(transports "zeromq" "shmem")
 
+# Environment for every example test (directory scope, so the subdirectories
+# see it too)
+set(env_mods)
+if(ENABLE_SANITIZER_LEAK AND CMAKE_VERSION VERSION_GREATER_EQUAL 3.22)
+  get_filename_component(lsan_supps "${CMAKE_SOURCE_DIR}/test/leak_sanitizer_suppressions.txt" ABSOLUTE)
+  list(APPEND env_mods "LSAN_OPTIONS=set:suppressions=${lsan_supps}")
+endif()
+# Run the example tests against an alternative runtime library directory as
+# well (see test/CMakeLists.txt). The test scripts only launch instrumented
+# binaries (bash itself does not link libstdc++). Note: unlike the
+# testsuites, the example binaries get no locale-cache warmup -- their
+# main() comes from the installed public header fairmq/runDevice.h.
+if(FAIRMQ_TEST_LD_LIBRARY_PATH)
+  list(APPEND env_mods "LD_LIBRARY_PATH=path_list_prepend:${FAIRMQ_TEST_LD_LIBRARY_PATH}")
+endif()
+
 function(add_example)
   cmake_parse_arguments(PARSE_ARGV 0 ARG
     "CONFIG;NO_TRANSPORT;NO_TEST"
@@ -29,11 +45,6 @@ function(add_example)
     message(FATAL_ERROR "NAME arg is required")
   endif()
 
-  if(ENABLE_SANITIZER_LEAK AND CMAKE_VERSION VERSION_GREATER_EQUAL 3.22)
-    get_filename_component(lsan_supps "${CMAKE_SOURCE_DIR}/test/leak_sanitizer_suppressions.txt" ABSOLUTE)
-    set(lsan_options "LSAN_OPTIONS=set:suppressions=${lsan_supps}")
-  endif()
-
   if(ARG_DEVICE)
     set(exe_targets)
     foreach(device IN LISTS ARG_DEVICE)
@@ -78,8 +89,8 @@ function(add_example)
       set(test "${testsuite}.${name}.${transport}")
       add_test(NAME ${test} COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${test_script} ${transport})
       set_tests_properties(${test} PROPERTIES TIMEOUT "30")
-      if(lsan_options)
-        set_tests_properties(${test} PROPERTIES ENVIRONMENT_MODIFICATION ${lsan_options})
+      if(env_mods)
+        set_tests_properties(${test} PROPERTIES ENVIRONMENT_MODIFICATION "${env_mods}")
       endif()
     else()
       foreach(transport IN LISTS transports)
@@ -88,16 +99,16 @@ function(add_example)
             set(test "${testsuite}.${name}.${variant}.${transport}")
             add_test(NAME ${test} COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${test_script} ${transport} ${variant})
             set_tests_properties(${test} PROPERTIES TIMEOUT "30")
-            if(lsan_options)
-              set_tests_properties(${test} PROPERTIES ENVIRONMENT_MODIFICATION ${lsan_options})
+            if(env_mods)
+              set_tests_properties(${test} PROPERTIES ENVIRONMENT_MODIFICATION "${env_mods}")
             endif()
           endforeach()
         else()
           set(test "${testsuite}.${name}.${transport}")
           add_test(NAME ${test} COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${test_script} ${transport})
           set_tests_properties(${test} PROPERTIES TIMEOUT "30")
-          if(lsan_options)
-            set_tests_properties(${test} PROPERTIES ENVIRONMENT_MODIFICATION ${lsan_options})
+          if(env_mods)
+            set_tests_properties(${test} PROPERTIES ENVIRONMENT_MODIFICATION "${env_mods}")
           endif()
         endif()
       endforeach()

examples/custom-controller/CMakeLists.txt

@@ -15,6 +15,8 @@ set_target_properties(${exe} PROPERTIES ENABLE_EXPORTS ON)
 set(test "${testsuite}.${name}")
 add_test(NAME ${test} COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${exe})
 set_tests_properties(${test} PROPERTIES TIMEOUT 30)
-if(lsan_options)
-  set_tests_properties(${test} PROPERTIES ENVIRONMENT_MODIFICATION ${lsan_options})
+# (the previous `if(lsan_options)` here could never fire -- that variable
+# only ever existed in the add_example() function scope)
+if(env_mods)
+  set_tests_properties(${test} PROPERTIES ENVIRONMENT_MODIFICATION "${env_mods}")
 endif()

fairmq/Channel.cxx

@@ -181,7 +181,8 @@ try {
 
     // validate channel name
     smatch m;
-    if (regex_search(fName, m, regex(R"([^a-zA-Z0-9\-_\[\]#])"))) {
+    static regex const invalidName(R"([^a-zA-Z0-9\-_\[\]#])");
+    if (regex_search(fName, m, invalidName)) {
         ss << "INVALID";
         LOG(debug) << ss.str();
         LOG(error) << "channel name contains illegal character: '" << m.str(0) << "', allowed characters are: a-z, A-Z, 0-9, -, _, [, ], #";

fairmq/tools/Process.cxx

@@ -18,6 +18,7 @@
 #include <chrono>
 #include <csignal>   // kill, signals
 #include <iostream>
+#include <mutex>
 #include <sstream>
 #include <stdexcept>
 #include <thread>
@@ -40,10 +41,18 @@ class LinePrinter
         , fPrefix(std::move(prefix))
     {}
 
-    // prints line with prefix on both cout (thread-safe) and output stream
+    // prints line with prefix on both cout and output stream
     void Print(const string& line)
     {
-        cout << fair::mq::tools::ToString(fPrefix, line, "\n") << flush;
+        // Serialize: the standard allows concurrent insertion on std::cout,
+        // but libstdc++ maintains the formatted-output state (e.g.
+        // ios_base::width) with plain reads and writes that constitute data
+        // races; the lock also keeps whole lines from interleaving.
+        static mutex sCoutMutex;
+        {
+            lock_guard<mutex> lock(sCoutMutex);
+            cout << fair::mq::tools::ToString(fPrefix, line, "\n") << flush;
+        }
         fOut << fPrefix << line << endl;
     }
 

test/CMakeLists.txt

@@ -24,14 +24,25 @@ if(definitions)
   set(definitions DEFINITIONS ${definitions})
 endif()
 
+set(test_environment)
 if(ENABLE_SANITIZER_LEAK)
   get_filename_component(lsan_supps "${CMAKE_CURRENT_SOURCE_DIR}/leak_sanitizer_suppressions.txt" ABSOLUTE)
-  set(environment ENVIRONMENT "LSAN_OPTIONS=set:suppressions=${lsan_supps}")
+  list(APPEND test_environment "LSAN_OPTIONS=set:suppressions=${lsan_supps}")
+endif()
+# Run the tests against an alternative runtime library directory, e.g. a
+# tsan-instrumented libstdc++. Set per test (not at the job level), because
+# such a library must only be loaded into instrumented executables.
+if(FAIRMQ_TEST_LD_LIBRARY_PATH)
+  list(APPEND test_environment "LD_LIBRARY_PATH=path_list_prepend:${FAIRMQ_TEST_LD_LIBRARY_PATH}")
+endif()
+if(test_environment)
+  set(environment ENVIRONMENT ${test_environment})
 endif()
 
 add_testhelper(runTestDevice
     SOURCES
     helper/runTestDevice.cxx
+    helper/LocaleWarmup.h
     helper/devices/TestPairLeft.h
     helper/devices/TestPairRight.h
     helper/devices/TestPollIn.h