CI: parallel test runner, CPU job on kkt, LDLᵀ compiled only when used

.github/workflows/ci.yml

@@ -11,11 +11,16 @@ on:
     - cron: '0 0 * * 0'
 
 jobs:
+  # Runs on the self-hosted kkt machine (more cores than a GitHub runner for the parallel test workers). The
+  # job id and the matrix stay as they were: the check name "test-github-cpuonly (ubuntu-latest, 1, x64)" is a
+  # required check of the branch ruleset.
   test-github-cpuonly:
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       SDS_TEST_GPU: '0'
-    runs-on: ${{ matrix.os }}
+      # ParallelTestRunner workers (test/runtests.jl): kkt has 192 threads and 2 TB, shared by its four runners
+      PTR_NUM_JOBS: '32'
+    runs-on: [self-hosted, linux, X64, kkt]
     strategy:
       fail-fast: false
       matrix:
@@ -28,7 +33,8 @@ jobs:
       - uses: julia-actions/setup-julia@latest
         with:
           version: ${{ matrix.julia-version }}
-      - uses: julia-actions/cache@v2
+      - name: Machine
+        run: nproc && free -g
       - uses: julia-actions/julia-buildpkg@latest
       - uses: julia-actions/julia-runtest@latest
       # the GPU jobs skip the CPU backend, so its coverage comes from here
@@ -45,6 +51,8 @@ jobs:
       # CPU-only jobs; testing them again here doubled the GPU jobs' time.
       SDS_TEST_CPU: '0'
       SDS_TEST_SKIP: 'test_aqua'
+      # ParallelTestRunner workers, each with its own CUDA context on the runner's GPU
+      PTR_NUM_JOBS: '12'
     strategy:
       fail-fast: false
       matrix:
@@ -62,6 +70,8 @@ jobs:
         with:
           version: ${{ matrix.julia-version }}
           arch: ${{ matrix.julia-arch }}
+      - name: Machine
+        run: nproc && free -g && (nvidia-smi -L || true)
       - name: Add CUDA.jl to the test environment
         if: matrix.backend == 'cuda'
         run: |

AGENTS.md

@@ -125,9 +125,9 @@ reviewing agent must pass. The moving parts:
 * Linux (WSL2). One NVIDIA RTX 4080 (16 GB); CUDA.jl is functional.
 * **No AMDGPU, oneAPI or Metal hardware here.** Local tests run on the
   KernelAbstractions CPU backend and on CUDA only. GitHub CI
-  (`.github/workflows/ci.yml`, modelled on `../ExaPF.jl`) runs the CPU suite on
-  GitHub runners and the GPU suite on the exanauts self-hosted runners labelled
-  `cuda` and `amdgpu`. oneAPI and Metal extension files are written by analogy
+  (`.github/workflows/ci.yml`, modelled on `../ExaPF.jl`) runs the CPU suite and
+  the GPU suite on the exanauts self-hosted runners (CPU suite: the `kkt`
+  machine; GPU suite: the runners labelled `cuda` and `amdgpu`). oneAPI and Metal extension files are written by analogy
   (task T23) and only precompile-checked.
 * CUDA.jl 6.x is split into packages: `CUDACore` (`CuArray`, `CUDABackend`),
   `cuSPARSE` (`CuSparseMatrixCSR`, `CuSparseMatrixCSC`), `cuBLAS`, `cuSOLVER`.
@@ -151,6 +151,10 @@ SDS_TEST_CPU=0 julia --project=. -e 'using Pkg; Pkg.test()'
 SDS_TEST_ONLY="test_symbolic_etree,test_options" julia --project=. -e 'using Pkg; Pkg.test()'
 SDS_TEST_SKIP="test_aqua" julia --project=. -e 'using Pkg; Pkg.test()'
 
+# the same with ParallelTestRunner arguments (prefix match, `!` excludes), and the number of workers
+julia --project=. -e 'using Pkg; Pkg.test(; test_args = ["test_symbolic", "!test_symbolic_etree"])'
+PTR_NUM_JOBS=4 julia --project=. -e 'using Pkg; Pkg.test()'
+
 # instantiate / update after editing Project.toml
 julia --project=. -e 'using Pkg; Pkg.instantiate()'
 
@@ -170,6 +174,17 @@ CUDA jobs run with `SDS_TEST_CPU=0 SDS_TEST_SKIP=test_aqua`, since the CPU-only
 jobs already cover the CPU backend and Aqua. A task's "passes on CPU and CUDA"
 still means the default run (both backends) on the owner's machine.
 
+The test files run in parallel through ParallelTestRunner.jl: each `test_*.jl` is
+evaluated in its own module on a pool of worker processes, after the packages,
+`test/utils.jl`, `test/matrices.jl`, `test/backends.jl` and `Random.seed!(666)`
+(`test/runtests.jl`). A test file therefore cannot use definitions from another
+test file; shared code belongs in the helpers. A worker's cold compilation, not the
+tests, sets the wall time, so the long files (`SPLIT_FILES` in `test/runtests.jl`)
+run once per element type (`test_api[Float64]`, …): `ELTYPES`, `REAL_ELTYPES` and
+`COMPLEX_ELTYPES` are then that part's subset, and a testset that does not loop over
+the element types runs in the `Float64` part only (`RUN_SHARED && @testset …`). In
+a split file, write every testset either over `ELTYPES` or behind `RUN_SHARED`.
+
 ## Code conventions
 
 * Julia style: 4-space indentation, no tabs, `snake_case` functions, `CamelCase`

src/numeric/factorize.jl

@@ -228,9 +228,19 @@ the factor is incomplete. Structure `"SPD"` (real) or `"HPD"`. Nothing is
 allocated on the device; the panels are bitwise reproducible for a fixed
 `impl` and backend (regimes A and B are deterministic by construction).
 """
-function factorize!(N::Numeric{T}, S::Symbolic, nzval::AbstractVector; impl::Symbol = :auto,
-                    opts::Options = Options()) where {T}
+function factorize!(N::Numeric, S::Symbolic, nzval::AbstractVector; impl::Symbol = :auto, opts::Options = Options())
     _is_ldlt_structure(S.structure) && return factorize_ldlt!(N, S, nzval; impl, opts)
+    return factorize_cholesky!(N, S, nzval; impl, opts)
+end
+
+"""
+    factorize_cholesky!(numeric, symbolic, nzval; impl = :auto, opts = Options()) -> info::Int
+
+The Cholesky path of [`factorize!`](@ref) (structures `"SPD"` and `"HPD"`),
+callable without compiling the LDLᵀ/LDLᴴ path ([`factorize_ldlt!`](@ref)).
+"""
+function factorize_cholesky!(N::Numeric{T}, S::Symbolic, nzval::AbstractVector; impl::Symbol = :auto,
+                             opts::Options = Options()) where {T}
     _check_numeric(N, S, nzval)
     p = _front_impls(N, S, impl)
     plan = N.plan

src/solver.jl

@@ -409,7 +409,7 @@ function _factorize!(solver::DirectSolver, p::Phase)
     end
     tic = time_ns()
     try
-        _numeric_phase!(N, S, solver.A.nzval, solver.options)
+        _numeric_phase!(solver)
     catch err
         if err isa InterruptedError
             # the panels are partly overwritten: back to "analyzed", a "factorization" must follow
@@ -446,9 +446,31 @@ _info_value(solver::DirectSolver) = solver.nbatch == 1 ? solver.info[1] : copy(s
 
 _elapsed(tic) = string(round((time_ns() - tic) / 1.0e6; digits = 3), " ms")
 
-# function barrier: concrete storage types for the numeric phase
-_numeric_phase!(N::Numeric, S::Symbolic, nzval::AbstractVector, opts::Options) = factorize!(N, S, nzval; opts)
-_numeric_phase!(N::Numeric, S::Symbolic, nzval::AbstractMatrix, opts::Options) = factorize!(N, S, vec(nzval); opts)
+# The numeric phase of the structure, behind a barrier: the structure is a run-time value, and a static call
+# would compile the Cholesky and the LDLᵀ/LDLᴴ paths with every solver, on the CPU backend all their kernels (most
+# of the test suite's compile time). The barrier takes the mutable handle only: the dynamic call boxes nothing
+# (an immutable `Numeric` or `Symbolic` argument would be copied to the heap on every call).
+function _numeric_phase!(solver::DirectSolver)
+    if _is_ldlt_structure(solver.structure)
+        Base.inferencebarrier(_numeric_phase_ldlt!)(solver)
+    else
+        Base.inferencebarrier(_numeric_phase_cholesky!)(solver)
+    end
+    return nothing
+end
+
+function _numeric_phase_cholesky!(solver::DirectSolver)
+    factorize_cholesky!(solver.numeric, solver.symbolic, _flat(solver.A.nzval); opts = solver.options)
+    return nothing
+end
+
+function _numeric_phase_ldlt!(solver::DirectSolver)
+    factorize_ldlt!(solver.numeric, solver.symbolic, _flat(solver.A.nzval); opts = solver.options)
+    return nothing
+end
+
+_flat(nzval::AbstractVector) = nzval
+_flat(nzval::AbstractMatrix) = vec(nzval)
 
 # user data of a right-hand side / solution: (array, transposed); an `n × nrhs × nbatch` array is the
 # `n × (nrhs nbatch)` matrix with the same memory (uniform batch)

test/Project.toml

@@ -6,6 +6,7 @@ KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c"
 Krylov = "ba0b0d4f-ebba-5204-a429-3ac8c609bfb7"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Metis = "2679e427-3c69-5b7f-982b-ece356f1e94b"
+ParallelTestRunner = "d3525ed8-44d0-4b2c-a655-542cee43accc"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
@@ -20,6 +21,7 @@ KernelAbstractions = "0.9"
 Krylov = "0.10"
 LinearAlgebra = "1.13"
 Metis = "1"
+ParallelTestRunner = "2"
 Printf = "1.13"
 Random = "1.13"
 SparseArrays = "1.13"

test/backends.jl

@@ -172,7 +172,14 @@ if AMDGPU_LOADED
                                                (n, n))
 end
 
-let gpus = String[]
+"""
+    print_backends()
+
+Print the backends under test and why GPU backends are missing (once per run, by
+`runtests.jl`).
+"""
+function print_backends()
+    gpus = String[]
     CUDA_FUNCTIONAL && push!(gpus, "CUDA ($(CUDA.name(CUDA.device())))")
     AMDGPU_FUNCTIONAL && push!(gpus, "ROCm ($(AMDGPU.device()))")
     CUDA_LOADED && !CUDA_FUNCTIONAL && push!(gpus, "CUDA installed but not functional: skipped")
@@ -181,6 +188,7 @@ let gpus = String[]
     TEST_CPU || push!(gpus, "CPU backend disabled by SDS_TEST_CPU=0")
     println("Backends under test: ", join(backend_name.(BACKENDS), ", "),
             isempty(gpus) ? "" : "  [" * join(gpus, "; ") * "]")
+    return nothing
 end
 
 """

test/runtests.jl

@@ -1,26 +1,18 @@
-# Test driver. Every `test_*.jl` file in this directory is a test file; run a
-# subset with SDS_TEST_ONLY="test_options,test_aqua", leave files out with
-# SDS_TEST_SKIP="test_aqua", skip the GPU backends with SDS_TEST_GPU=0 and the
-# CPU backend with SDS_TEST_CPU=0 (see backends.jl).
-
-using Test
-using Random
-using LinearAlgebra
-using SparseArrays
-using Aqua
-using KernelAbstractions
-using SparseDirectSolver
-using Metis  # activates SparseDirectSolverMetisExt (nested-dissection orderings, T05)
-using Krylov  # activates SparseDirectSolverKrylovExt (ir_mode = "fgmres", T18)
+# Test driver on ParallelTestRunner: every `test_*.jl` file in this directory is a test file and runs in its own
+# sandbox module on a pool of worker processes (one compilation of the solver per worker, the files in parallel); the
+# long files run once per element type (`SPLIT_FILES`).
+#
+# Run a subset with SDS_TEST_ONLY="test_options,test_aqua" or with test arguments (prefix match, `!name` excludes:
+# `Pkg.test(; test_args = ["test_symbolic", "!test_symbolic_etree"])`), leave files out with SDS_TEST_SKIP="test_aqua",
+# skip the GPU backends with SDS_TEST_GPU=0 and the CPU backend with SDS_TEST_CPU=0 (see backends.jl). The number of
+# workers is ParallelTestRunner's default (CPU threads and free memory); set it with `--jobs=N` or PTR_NUM_JOBS.
 
-const SDS = SparseDirectSolver
+using ParallelTestRunner
+using SparseDirectSolver
 
-include("utils.jl")
-include("matrices.jl")
-include("backends.jl")
+const TEST_DIR = @__DIR__
 
-const TEST_FILES = sort!([first(splitext(f)) for f in readdir(@__DIR__)
-                          if startswith(f, "test_") && endswith(f, ".jl")])
+const TEST_FILES = sort!([first(splitext(f)) for f in readdir(TEST_DIR) if startswith(f, "test_") && endswith(f, ".jl")])
 
 function test_file_list(var::String)
     names = [first(splitext(strip(s))) for s in split(get(ENV, var, ""), ',') if !isempty(strip(s))]
@@ -30,19 +22,69 @@ function test_file_list(var::String)
     return names
 end
 
-function selected_test_files()
-    only = test_file_list("SDS_TEST_ONLY")
-    skip = test_file_list("SDS_TEST_SKIP")
-    return setdiff(isempty(only) ? TEST_FILES : only, skip)
+# what every test file sees: the packages (the shared helpers come with the test, see `test_expr`)
+const init_code = quote
+    using Test
+    using Random
+    using LinearAlgebra
+    using SparseArrays
+    using Aqua
+    using KernelAbstractions
+    using SparseDirectSolver
+    using Metis  # activates SparseDirectSolverMetisExt (nested-dissection orderings, T05)
+    using Krylov  # activates SparseDirectSolverKrylovExt (ir_mode = "fgmres", T18)
+    const SDS = SparseDirectSolver
+end
+
+# the element types of the running test (`test/utils.jl`), set by the test before the helpers are included
+const init_worker_code = quote
+    const SDS_TEST_PART = Ref{Any}(nothing)
+end
+
+# Long test files run once per element type, "test_api[Float64]" etc., each part in a worker of its own: a
+# worker compiles the solver for every element type it tests, and that compilation, not the tests, sets the wall
+# time. The testsets that do not loop over the element types run in the Float64 part (`RUN_SHARED`).
+const SPLIT_FILES = ["test_api", "test_dense", "test_fgmres", "test_numeric_cholesky_a", "test_numeric_cholesky_b",
+                     "test_numeric_cholesky_c", "test_numeric_ldlt", "test_ported", "test_refinement", "test_solve",
+                     "test_ubatch"]
+const SPLIT_ELTYPES = (Float32, Float64, ComplexF32, ComplexF64)
+
+# the seed goes with the test, not into `init_code`: ParallelTestRunner seeds with 1 after `init_code`
+test_expr(name, part) = quote
+    Main.SDS_TEST_PART[] = $part
+    include($(joinpath(TEST_DIR, "utils.jl")))
+    include($(joinpath(TEST_DIR, "matrices.jl")))
+    include($(joinpath(TEST_DIR, "backends.jl")))
+    Random.seed!(666)
+    include($(joinpath(TEST_DIR, "$name.jl")))
 end
 
-@testset "SparseDirectSolver.jl" begin
-    for name in selected_test_files()
-        @testset "$name" begin
-            Random.seed!(666)
-            tic = time()
-            include("$name.jl")
-            println("$name: $(round(time() - tic; digits = 1)) s")
+testsuite = Dict{String, Expr}()
+for name in TEST_FILES
+    if name in SPLIT_FILES
+        for T in SPLIT_ELTYPES
+            testsuite["$name[$T]"] = test_expr(name, ((T,), T == Float64))
         end
+    else
+        testsuite[name] = test_expr(name, nothing)
     end
 end
+
+# SDS_TEST_ONLY/SDS_TEST_SKIP name test files: a split file stands for all its parts
+names_test(names, test) = any(n -> test == n || startswith(test, n * "["), names)
+
+args = parse_args(ARGS)
+if filter_tests!(testsuite, args)
+    only, skip = test_file_list("SDS_TEST_ONLY"), test_file_list("SDS_TEST_SKIP")
+    isempty(only) || filter!(t -> names_test(only, first(t)), testsuite)
+    filter!(t -> !names_test(skip, first(t)), testsuite)
+end
+
+# the backend banner once, from the main process (also fails early when no backend is left to test)
+module BackendBanner end
+Core.eval(BackendBanner, init_code)
+Core.eval(BackendBanner, :(include($(joinpath(TEST_DIR, "backends.jl")))))
+BackendBanner.print_backends()
+
+runtests(SparseDirectSolver, args; testsuite, init_code, init_worker_code,
+         history_key = get(ENV, "SDS_TEST_CPU", "1") == "0" ? "gpu" : nothing)

test/test_dense.jl

@@ -11,9 +11,11 @@ dense_tri(A, uplo::Char, diag::Char) =
 dense_part(C, uplo::Char) = uplo == 'L' ? tril(C) : triu(C)
 
 @testset "capability audit ($(backend_name(backend)))" for backend in BACKENDS
-    SDS.print_capabilities(stdout, backend)
-    table = sprint(io -> SDS.print_capabilities(io, backend))
-    @test occursin("generic_mul", table) && occursin("ComplexF64", table)
+    if RUN_SHARED   # the table covers every element type
+        SDS.print_capabilities(stdout, backend)
+        table = sprint(io -> SDS.print_capabilities(io, backend))
+        @test occursin("generic_mul", table) && occursin("ComplexF64", table)
+    end
     for T in ELTYPES
         caps = SDS.capabilities(backend, T)
         @test caps isa SDS.DenseCapabilities
@@ -255,7 +257,7 @@ end
     @test to_host(C) ≈ to_host(A) * to_host(A)
 end
 
-@testset "dense interface errors" begin
+RUN_SHARED && @testset "dense interface errors" begin
     A = zeros(4, 4)
     @test_throws InvalidValueError SDS.gemm!(A, A, A; impl = :bogus)
     @test_throws InvalidValueError SDS.gemm!(A, A, A; transA = :X)

test/test_fgmres.jl

@@ -7,8 +7,8 @@
 const FGMRES_STALL_PARAMS = (("pivot_pairs", "none"), ("pivot_epsilon", 1.0))
 fgmres_stall_matrix() = kkt_matrix(Float64, 150, 5, 0.0)
 
-@testset "FGMRES-IR where plain IR stalls ($(backend_name(backend)), $INT)" for backend in BACKENDS,
-                                                                                 INT in INTTYPES
+RUN_SHARED && @testset "FGMRES-IR where plain IR stalls ($(backend_name(backend)), $INT)" for backend in BACKENDS,
+                                                                                               INT in INTTYPES
     Random.seed!(666)
     K = fgmres_stall_matrix()
     b = K * rand(size(K, 1))
@@ -72,7 +72,8 @@ end
     end
 end
 
-@testset "FGMRES-IR: uniform batch ($(backend_name(backend)), $T)" for backend in BACKENDS, T in (Float64, ComplexF32)
+@testset "FGMRES-IR: uniform batch ($(backend_name(backend)), $T)" for backend in BACKENDS,
+                                                                       T in filter(in((Float64, ComplexF32)), ELTYPES)
     n, nb, nrhs = 120, 3, 2
     members = batch_members(random_symindef(T, n, 0.03), nb)
     solver = DirectSolver(api_batch_matrix(backend, members, 'L'), sym_structure(T), 'L')
@@ -95,7 +96,7 @@ end
     @test iszero(Xh[:, :, 1]) && iszero(Xh[:, :, 3])
 end
 
-@testset "FGMRES-IR: interrupt and missing Krylov ($(backend_name(backend)))" for backend in BACKENDS
+RUN_SHARED && @testset "FGMRES-IR: interrupt and missing Krylov ($(backend_name(backend)))" for backend in BACKENDS
     Random.seed!(666)
     K = fgmres_stall_matrix()
     b = K * rand(size(K, 1))

test/test_numeric_cholesky_a.jl

@@ -30,7 +30,7 @@ function subtree_move_rounds(S, v)
     return cld(m * (m + 1) ÷ 2, d)
 end
 
-@testset "plan: regime-A groups" begin
+RUN_SHARED && @testset "plan: regime-A groups" begin
     for (name, A, opts) in numeric_a_matrices(Float64)
         S, _, _, _, Nd, _ = numeric_setup(CPU(), A; opts)
         sc, plan = S.schedule, Nd.plan

test/test_numeric_cholesky_b.jl

@@ -17,7 +17,7 @@ numeric_b_allocated(N, S, nz) = @allocated SDS.factorize!(N, S, nz)
 
 nregime(S, r) = count(==(r), S.schedule.regime)
 
-@testset "plan: regime-B groups" begin
+RUN_SHARED && @testset "plan: regime-B groups" begin
     for opts in (NUMERIC_B_OPTS, NUMERIC_BC_OPTS, Options(subtree_budgets = Int[], regime_c_width = 128))
         A = laplacian3d(Float64, 10, 10, 10)
         S, _, _, _, Nd, _ = numeric_setup(CPU(), A; opts)

test/test_numeric_cholesky_c.jl

@@ -15,7 +15,7 @@ numeric_c_setup(backend, A, INT = Int32; opts = NUMERIC_C_OPTS, kw...) = numeric
 
 numeric_c_allocated(N, S, nz) = @allocated SDS.factorize!(N, S, nz)
 
-@testset "storage and plan" begin
+RUN_SHARED && @testset "storage and plan" begin
     A = laplacian2d(Float64, 20, 20)
     S, _, _, _, Nd, _ = numeric_c_setup(CPU(), A)
     @test Nd isa SDS.Numeric{Float64, Vector{Float64}, Vector{Int64}, Vector{Int32}}