diff --git a/.github/workflows/Aqua.yml b/.github/workflows/Aqua.yml index 3c3d7f0..af731a0 100644 --- a/.github/workflows/Aqua.yml +++ b/.github/workflows/Aqua.yml @@ -14,4 +14,7 @@ jobs: with: version: '1' - name: Aqua.jl - run: julia --color=yes -e 'using Pkg; Pkg.add("Aqua"); Pkg.develop(path="."); using Aqua, SparseDirectSolver; Aqua.test_all(SparseDirectSolver, ambiguities=false)' + # MadNLP is loaded so the SparseDirectSolverMadNLPExt extension assigns the + # exported SDSSolver/SDSSolverOptions globals: the undefined-exports check + # then validates them for real instead of flagging the declared-only state + run: julia --color=yes -e 'using Pkg; Pkg.add(["Aqua", "MadNLP"]); Pkg.develop(path="."); using Aqua, MadNLP, SparseDirectSolver; Aqua.test_all(SparseDirectSolver, ambiguities=false)' diff --git a/Project.toml b/Project.toml index e0d4446..f34afbe 100644 --- a/Project.toml +++ b/Project.toml @@ -12,13 +12,14 @@ GPUArrays = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" GPUArraysCore = "46192b85-c4d5-4398-a991-12ede77f4527" KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +Metis = "2679e427-3c69-5b7f-982b-ece356f1e94b" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [weakdeps] AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" CUDACore = "bd0ed864-bdfe-4181-a5ed-ce625a5fdea2" Krylov = "ba0b0d4f-ebba-5204-a429-3ac8c609bfb7" -Metis = "2679e427-3c69-5b7f-982b-ece356f1e94b" +MadNLP = "2621e9c9-9eb4-46b1-8089-e8c72242dfb6" cuBLAS = "182d3088-87b7-4494-8cad-fc6afaa545bc" cuSOLVER = "887afef0-6a32-4de5-add4-7827692ba8fc" cuSPARSE = "b26da814-b3bc-49ef-b0ee-c816305aa060" @@ -27,7 +28,7 @@ cuSPARSE = "b26da814-b3bc-49ef-b0ee-c816305aa060" SparseDirectSolverAMDGPUExt = "AMDGPU" SparseDirectSolverCUDAExt = ["CUDACore", "cuBLAS", "cuSOLVER", "cuSPARSE"] SparseDirectSolverKrylovExt = "Krylov" -SparseDirectSolverMetisExt = "Metis" +SparseDirectSolverMadNLPExt = "MadNLP" [compat] AMD = "0.5" @@ -41,6 +42,7 @@ GPUArraysCore = "0.2" KernelAbstractions = "0.9" Krylov = "0.10" LinearAlgebra = "1.13" +MadNLP = "0.8, 0.9, 0.10" Metis = "1" SparseArrays = "1.13" cuBLAS = "6" diff --git a/README.md b/README.md index 5e83efb..e630a55 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,8 @@ oneAPI, Metal and the KernelAbstractions CPU backend. **Status: v0.1 under construction.** Working today, on the CPU backend, on CUDA and on AMDGPU (ROCm): -* host symbolic analysis: AMD or nested dissection (METIS through the Metis - extension) ordering, elimination tree, supernodes with GPU-tuned +* host symbolic analysis: AMD or nested dissection (METIS) ordering, + elimination tree, supernodes with GPU-tuned amalgamation, static factor layout and device assembly maps; for symmetric indefinite matrices the analysis pairs structurally zero pivots with a partner (`pivot_pairs`, MA57-style compressed-graph ordering) so that the @@ -108,8 +108,7 @@ Pkg.add(url = "https://github.com/exanauts/SparseDirectSolver.jl") Loading CUDA.jl enables the CUDA extension (`CuSparseMatrixCSR`/`CuSparseMatrixCSC` constructors, cuBLAS/cuSOLVER dense kernels); loading AMDGPU.jl enables the AMDGPU extension (`ROCSparseMatrixCSR`/`ROCSparseMatrixCSC` constructors, -rocBLAS/rocSOLVER dense kernels; ROCm must be found by AMDGPU.jl). Loading Metis.jl enables nested -dissection ordering; without it the ordering is AMD. +rocBLAS/rocSOLVER dense kernels; ROCm must be found by AMDGPU.jl). ## Usage diff --git a/bench/ARCH-TUNING.md b/bench/ARCH-TUNING.md new file mode 100644 index 0000000..23df695 --- /dev/null +++ b/bench/ARCH-TUNING.md @@ -0,0 +1,107 @@ +# Per-architecture tuning of the fused factorization + +How the October 2026 campaign tuned the fused refactorization prototype per GPU +architecture: the knob inventory, the measurement discipline, the step-by-step +procedure for a new device, the mechanisms behind each knob, and the measured +champions. Workload throughout: the 78k-bus ACOPF condensed KKT +(`pglib_opf_case78484_epigrids`, n = 674562, nnz(L) ≈ 14.3M), tuned ordering +(METIS nseps = 4, seed = 3). + +## Ground rules (these produced every number below) + +1. **Gate every configuration.** A trial is valid only with `info == 0`, + elementwise factor delta ~1e-9 against the stock factorization, and the + residual matching stock. An `info == 0` check alone is insufficient: one + race produced a silently wrong factor (Δ ≈ 1e-5) with clean info. +2. **Interleave with the baseline.** Shared boxes drift by 2× within an hour; + A/B/A/B in one process is the only defensible comparison. Three pairs + minimum; treat sub-2% differences as noise. +3. **Correctness before speed on a new chip.** Run 20 gated refactorizations + (`bench/e2e/ldl_fused_race.jl`) before believing any timing. This is what caught + the sm_86/89/Blackwell/gfx906 completion-signal race that sm_70/80/90 hide. +4. **Occupancy arithmetic before submitting sweeps.** Most structural knobs are + decided by `blocks/SM = min(smem_per_SM ÷ smem_per_block, + threads_per_SM ÷ workgroup)`; compute it first and you can predict the sign + of the result (every prediction below held). + +## Knob inventory + +| knob | where | kind | mechanism | +|---|---|---|---| +| wide-front strategy (`wk`/`mseg`) | plan builder | runtime | host vendor calls vs in-kernel blocked wides + single merged launch; trades ~100 host launches against in-kernel spin residency | +| subtree budget (`subtree_budgets`) | solver option | runtime | regime-A local-memory class; smaller class = more subtree workgroups resident per SM/CU | +| subtree parallelism / amalgamation | solver options | runtime | subtree formation and supernode merging; saturated on every tested arch except amal 32→48 (small win on big-SM parts) | +| ordering (`nd_nseps`, `nd_seed`, `user_perm`) | solver/wrapper | runtime | fill/flops; arch-independent; every instance recomputes (no caching); pin with `user_perm` | +| panel width (`pw`) | plan + kernel `Val` | runtime per-plan | fronts ≤ pw factor in-kernel; packed panel = pw(pw+1)/2 · 8B of shared memory **for every block of the kernel** (one kernel = one shared size) | +| tile size (`tl`) | plan + kernel `Val` | runtime per-plan | syrk tile; shared halves = 2·32·tl · 8B; accumulators = tl²/WGF registers per thread | +| workgroup size (`SDSTuning.workgroup`) | tuning bundle | runtime (>= 128, multiple of 32: the tile roles' register accumulators) | waves per workgroup; latency hiding on wave64 AMD; occupancy on 1536-thread parts | +| spin backoff cap (`SDSTuning.spin_cap`) | tuning bundle | runtime | dependency-counter wake-up latency vs polling traffic | +| CUDA graph replay (`graph`) | run_fused kwarg | runtime | whole refactorization captured once, replayed as one launch | + +## Procedure for a new architecture + +1. **Correctness:** `bench/e2e/ldl_fused_race.jl` (20 gated refactorizations). + Any failure: stop, bisect kernel-vs-src-vs-config before timing anything. +2. **Phase profile:** `profile_refact()` — sync-inflated but reveals the split + (resets / subtrees / mega kernel / host wides / stats). The dominant phase + picks the first lever: host-wides share ≥50% ⇒ try merged wides first. +3. **Wide strategy:** `mseg_ab.jl`. Decision so far follows one attribute: + `MAX_THREADS_PER_MULTIPROCESSOR < 2048` (sm_86/89, Blackwell workstation) + or AMD ⇒ merged wins big (1.6–2.1×); 2048-thread parts (sm_70/80/90) keep + host wides (merged costs ~1.5 ms there). +4. **Knob sweep:** `tune_platform.jl` (budgets, sp, amal, cw, rows, flip-check + of the wide strategy), baseline-interleaved. Expect budgets to be the only + mover: pick the class whose size ≈ `LDS_per_SM ÷ desired_workgroups`. +5. **Structural (only where shared memory allows):** `pw_ab.jl`, `tl_ab.jl`. + Run the occupancy arithmetic first; on ≤100 KB/SM parts pw=96/tl=64 lose by + construction, on 228 KB (Hopper) they are free. +6. **Bundle knobs:** `SDSTuning.workgroup` ∈ {128, 256} and `SDSTuning.spin_cap` + ∈ {128, 16} (runtime, per plan; workgroup has a hard 128 floor). WGF=256 was −20% on wave64 AMD, flat-to-pending on + NVIDIA; the spin cap matters only on residency-thin parts. +7. **Graphs:** `graph_ab.jl` — small (~1%) and free; keep it on everywhere. +8. **Stack and re-validate:** combine the winners, three gated interleaved + pairs (`combo_ab.jl`), then 20 gated refactorizations of the final config. + +## Mechanisms, compressed + +- **2048- vs 1536-thread SMs is the single most predictive attribute.** It + decides the wide strategy, the subtree budget, and whether the completion + race (pre-fix) manifested at all. +- **One fused kernel means one shared-memory size.** Any panel/tile growth + taxes *all* roles' occupancy; it only pays where SM shared memory grew + faster than the panel (Hopper). +- **Launch overhead doesn't scale with silicon.** The faster the chip, the + larger the share of host launches and dependency-hop latency; hence merged + wides and graphs matter more on newer parts, flops-side knobs less. +- **Wave64 wants bigger workgroups.** 128 threads = 2 waves on AMD leaves too + little in-flight work per workgroup; 256 = 4 waves was −20%. + +## Measured champions (fused refactorization, gated, interleaved) + +| device | arch | threads/SM | config | SDS | cuDSS | +|---|---|---|---|---|---| +| Quadro GV100 | sm_70 | 2048 | host wides, bud 16K, graph | **24.1 ms** | 24.3 ms | +| A100-80GB | sm_80 | 2048 | host wides, bud 16K | 19–21 ms | 17.3–18.7 ms | +| H200 | sm_90 | 2048 | pending the structural matrix | 13.6 ms so far | 11.1 ms | +| L40S | sm_89 | 1536 | merged, bud 8K, graph | **15.2 ms** | 12.2 ms | +| RTX Pro 6000 | Blackwell | 1536 | merged, bud 8K | **15.8 ms** | 13.5 ms | +| RTX 3080 | sm_86 | 1536 | merged | 32.4 ms (10 GB: kernel-level only) | — | +| Radeon VII | gfx906 | wave64 | merged, bud 8K, WGF 256, cap 16 | **40.0 ms** | — | + +All of this is bundled: `SparseDirectSolver.Fused.SDSTuning` holds every knob as a runtime +field, `default_tuning(backend)` is the single per-architecture defaults table, +and `madnlp(m; ..., sds_tuning = SDSTuning(; workgroup = 256))` overrides it. +The portable kernel takes workgroup size and spin cap per plan — nothing is +compile-time or environment-dependent in the wrapper path. + +## Scripts + +The campaign's one-off sweep drivers (`race_repro.jl`, `fact_lib.jl`'s +`profile_refact`, `mseg_ab.jl`, `pw_ab.jl`, `tl_ab.jl`, `graph_ab.jl`, +`combo_ab.jl`, `tune_platform.jl`, `h200_matrix.jl`, `e2e/amd_tune.jl`) were +not shipped; the committed equivalents are `e2e/ldl_fused_race.jl` (gated +correctness), `e2e/ldl_fused_test.jl` (fused vs stock + same-strategy ratio) +and `e2e/amd_sweep.jl` (knob sweep). Read the procedure above with those +substitutions; CUDA-graph replay (step 7's `graph_ab.jl`) retired with the +consolidation cleanup (no graph kwarg at HEAD). All gates take the +matrix from `bench/data/` and the tuned permutation from `bench/perm_cand3.bin`. diff --git a/bench/PERFORMANCE-2026-10.md b/bench/PERFORMANCE-2026-10.md new file mode 100644 index 0000000..66fca70 --- /dev/null +++ b/bench/PERFORMANCE-2026-10.md @@ -0,0 +1,186 @@ +# SparseDirectSolver vs cuDSS across seven GPU platforms — October 2026 campaign + +Benchmark: refactorization of the condensed ACOPF KKT system of +`pglib_opf_case78484_epigrids` (n = 674562, nnz(L) ≈ 14.3M after the tuned +ordering), the operation an interior-point method repeats ~200 times per solve. +Every number below is a median of gated runs (info = 0, elementwise factor +delta ~1e-9 vs the stock factorization, residual matching stock), interleaved +A/B against a baseline in the same process. Methodology and knob mechanisms: +`bench/ARCH-TUNING.md`. + +## Headline results + +| device | arch | f64:f32 | cuDSS refact | SDS fused refact | ratio | SDS config (per-arch defaults) | +|---|---|---|---|---|---|---| +| Quadro GV100 | sm_70 | 1:2 | 24.3 ms | **24.1 ms** | **0.99×** | host wides, 16K budgets, graph | +| NVIDIA H200 | sm_90 | 1:2 | 11.1 ms | **12.0 ms** | 1.08× | host wides, 16K, 96-wide panels, graph | +| NVIDIA A100-80GB | sm_80 | 1:2 | 17.3–18.7 ms | 19–21 ms | ~1.1× | host wides, 16K | +| RTX Pro 6000 (Blackwell) | sm_120 | 1:64 | 13.5 ms | 15.8 ms | 1.17× | merged wides, 8K | +| NVIDIA L40S | sm_89 | 1:64 | 12.2 ms | 15.2 ms | 1.25× | merged wides, 8K, graph | +| RTX 3080 (10 GB) | sm_86 | 1:64 | — | 32.4 ms | — | merged wides (kernel-level; e2e needs ~17 GB) | +| AMD Radeon VII | gfx906 | 1:4 | n/a | 40.0 ms | — | merged wides, 8K, workgroup 256, spin cap 16 | + +Solve (triangular sweeps): SDS 3.61 ms vs cuDSS 3.77 ms on the GV100 — ahead. +End-to-end MadNLP (`SparseCondensedKKTSystem`, tol 1e-6): identical iteration +counts and objectives as cuDSS on every platform tested; AMD end-to-end linear +algebra improved 36.5 → 34.0 s from the per-arch defaults alone. + +Within one day of per-architecture tuning the campaign moved: L40S 34.1 → 15.2, +H200 13.6 → 12.0, Radeon VII 66 → 40.0, GV100 29.3 → 24.1 (the division-free +and triangle-enumeration kernel work, PR #125, is included in all rows). + +## Why the times look the way they do + +**The workload is scheduling-bound, not flop-bound.** One refactorization is +~1.2 GFLOP spread over ~92k mostly tiny fronts (median supernode width ~7–16). +The instruction stream is dominated by index arithmetic, gathers/scatters, +barriers, dependency counters and kernel-launch latency — all full-rate on +every chip — with f64 FMAs a small minority. Two measurements pin this: + +- H200 has ~50× the f64 throughput of an L40S, yet is only 1.27× faster for SDS + and 1.1× for cuDSS. Both solvers' times track *core clock*, not FLOP rate. +- cuDSS across chips orders as A100 (1.4 GHz) 17.3 > H200 (2.0 GHz) 11.1 ≈ + L40S (2.5 GHz) 12.2: the signature of a latency-bound computation. + +**Consequently, f64 on "fp32 chips" is nearly free for this problem class.** +On 1:64 parts (L40S, 3080, workstation Blackwell) the pure arithmetic is +~1–2 ms of a ~15 ms budget. The caveat marks the boundary: wherever the work +becomes dense-block-shaped the penalty returns (the stock path, which leans on +dense vendor kernels, spreads 3× between H200 and L40S where the fused pipeline +spreads 1.27×; large-supernode 3D problems would be far worse). Sparse +small-front power-grid KKTs are close to the best case for cheap consumer GPUs. + +**Tensor-core f64 cannot close the remaining gaps.** A role-ablation ceiling +measurement on the A100 shows that making the GEMM-shaped tile-syrk infinitely +fast saves ~0.75 ms, and freeing *all* narrow-front in-kernel compute saves +~5 ms of ~21. DMMA would at best double the speed of that subset, bounding the +realistic gain at ≤10% on Ampere/Hopper — and the chips furthest from cuDSS +(sm_86/89/120) have no f64 tensor cores at all. The DMMA kernel was therefore +not built. + +**Where cuDSS's remaining edge lives.** Feeding cuDSS any external permutation +(even its own, extracted and fed back, bit-identical fill) collapses its +refactorization ~12× — its speed is scheduler metadata generated inside its +reorderer, not ordering quality (our fill is slightly better than its own: +14.3M vs 15.55M by its own count). The residual 8–25% on Hopper/Ada/Blackwell +is that scheduling craftsmanship against clock-insensitive dependency-hop +latency in our counter-based design; the L40S ratio (1.25×) is inflated by +cuDSS being anomalously strong on high-clock mature-toolchain Ada — our own +times on L40S and Blackwell are nearly identical (15.2 vs 15.8). + +## What moved the needle, per class (measured, gated) + +- 2048-threads/SM NVIDIA (sm_70/80/90): host vendor wides; CUDA-graph replay + of the whole refactorization (−0.5 ms on H200); on Hopper only, 96-wide + in-kernel panels (−1.8 ms; occupancy-free in 228 KB/SM shared memory). +- 1536-threads/SM NVIDIA (sm_86/89/120): in-kernel blocked wide fronts with a + single merged launch (1.6–2.1×), 8 KB subtree budget (~−1 ms), graphs. +- AMD gfx906: merged wides, 8 KB budgets, workgroup 256 (wave64 wants 4 waves + per workgroup: −20%), spin cap 16 (−1 ms). +- Everywhere: division-free scale-first Cholesky column step and packed-triangle + trailing-update enumeration (PR #125: 29.3 → 24.4 ms on GV100 alone); the + nseps = 4 / seed = 3 METIS ordering (−6%, PR #124). Every GPU number in + this document was measured with nseps = 4. Two owner calls on 2026-10-12: + the shipped default is nseps = 1 (first-touch reordering is ~2x faster: 5 s + versus 11.5 s on the 78k; reproduce the scoreboard with + `sds_tuning = SDSTuning(; nd_nseps = 4)`), and the extension caches nothing — + the on-disk permutation cache the campaign used is removed; every solver + instance recomputes its ordering, and `sds_user_perm` pins one explicitly. + +Tried and rejected with measurements: inverse-based TRSM (2×), multi-stream +wides, subtree/mega-kernel overlap, spin-cap tuning on NVIDIA, tile-64 on every +chip, 96-panels on ≤100 KB/SM parts, nseps beyond 4, Scotch ordering (+20–60% +fill), cuDSS-extracted orderings. + +## LDL^T in the fused pipeline (added 2026-10-11) + +The mega-kernel gained a no-pivot LDL^T mode (real element types; the MadNLP +case): a compile-time `Val` switches the column phases, D stays on the shared +diagonal in flight, the factor keeps the stock LDL layout (unit-lower panels + +separate D), and the solve applies D between the unit-triangular sweeps. Wide +fronts factor through the VERBATIM Cholesky vendor path where the front's +pivots share a sign (A = CC^T = LDL^T with L = C diag(C)^-1, d = diag(C)^2; +the herk contribution block is bit-identical), with the in-kernel wide roles +as the mixed-sign fallback. Gated against stock pivot-N (info 0, factor +~1e-7, D ~5e-10, 20/20 repeats): + +| device | fused LDL | fused Cholesky | LDL multiplier | cuDSS LDL | +|---|---|---|---|---| +| Quadro GV100 | 28.6 ms | 25.8 ms | **1.106** | 33.8 ms (1.25x of its own Cholesky) | +| AMD Radeon VII | 41.6 ms | 39.6 ms | **1.051** | n/a | + +## CPU: the fast path (added 2026-10-11) + +`bench/e2e/SDSCPU.jl`: a task-parallel CPU refactorization over the stock +schedule - subtree groups as chunked tasks running the regime-A algorithm in +plain scalar Julia, regime-B/C fronts assembling themselves per task (cache +local) then LAPACK potrf + BLAS trsm/syrk with BLAS pinned to one thread, and +a single-thread specialization with no task machinery at all. CPU-specific +amalgamation: max_width 96 (-17% at 24 threads vs the GPU champion 48; the +GPU prefers 48 by +39%). Measured and ruled out with the reasons recorded: +DAG scheduling (correct, 6x slower: the CB stack's slot reuse IS the cache +locality) and single-socket NUMA pinning (2.5x slower: the fronts phase wants +both sockets' bandwidth). The phase is memory-bound past ~8 threads. +MA27/MA57 reference numbers come via libcoinhsl: the libHSL 2025 build STUBS +the HSL Archive codes (its ma27 symbols return success with zeroed output; +the subset build returns -26). + +## The full picture + +Refactorization of the 78k condensed KKT; one row per hardware; best +validated configuration each; all gated. + +| hardware | SDS LL^T | SDS LDL^T | best alternative (LL^T / LDL^T) | +|---|---|---|---| +| NVIDIA H200 | 12.0 | - | cuDSS 11.1 / - | +| NVIDIA L40S | 15.2 | - | cuDSS 12.2 / - | +| RTX Pro 6000 | 15.8 | - | cuDSS 13.5 / - | +| NVIDIA A100 | 19-21 | - | cuDSS 17.3-18.7 / - | +| Quadro GV100 | 24.1 | 28.6 | cuDSS 24.3 / 33.8 | +| RTX 3080 (kernel-level, 10 GB) | 32.4 | - | - | +| AMD Radeon VII | 39.6 | 41.6 | no vendor solver exists | +| 2x Xeon 8160, 24 threads | ~85* | - | - / MA86 76.5 | +| 2x Xeon 8160, 8 threads | 106 | - | - / MA86 151 | +| 2x Xeon 8160, 1 thread | 449 | - | CHOLMOD 966 / MA86 963 | + +More 1-thread references: MA57 1909 ms, MA27 2798 ms (both LDL^T, libcoinhsl, +committed drivers `e2e/ma27_bench.jl` / `e2e/ma57_bench.jl`; the MA86 numbers +came from a one-off HSL.jl raw-wrapper driver with MadNLP-matching controls, +not shipped; +MA27 on the augmented k2 system of the same problem: 3709 ms at n = 2.09M). +*Projected from the measured interleaved -16.8% (amalgamation 96) on the +quiet-box 101.5 ms; every other entry is directly measured. Solve times where +they matter: SDS GPU 3.4-3.6 ms, cuDSS 3.7-4.1 ms, SDS CPU 81 ms at 24 +threads, MA86 168 ms at 24 threads. + +## Correctness note + +All of the above sits on the completion-signal fix (`left == 0`): the fused +kernel's last-finisher test had used the atomic's *new* value since PR #107, +signaling one block early — invisible on 2048-thread parts, 20/20 failures on +sm_86/89/120 and ~25% on gfx906, including one *silent* wrong factor with +info = 0. Validated 0-failure on all seven platforms after the fix. Gate on +factor deltas, not info alone. + +## Reproduction + + +One command on a fresh GPU machine (NVIDIA or AMD, autodetected). The shell route +builds the `@sds` named project environment (everything from +`bench/e2e/Project.toml`, no ad-hoc `Pkg.add`) and is reusable afterwards with +`julia +1.13 --project=@sds bench/e2e/workflow.jl`: + +```bash +curl -fsSL https://raw.githubusercontent.com/exanauts/SparseDirectSolver.jl/main/bench/e2e/fresh_machine.sh | bash +``` + +No shell / no checkout wanted? The self-contained Julia variant: + +```bash +julia +1.13 --startup-file=no -i -e 'include(download("https://raw.githubusercontent.com/exanauts/SparseDirectSolver.jl/main/bench/e2e/fresh_machine.jl"))' +``` + +Kernel-level benchmarks: the scripts listed in `bench/ARCH-TUNING.md` against +`bench/data/kkt_pglib_opf_case78484_epigrids_condensed_10.mtx`. Per-arch +defaults: `SparseDirectSolver.Fused.default_tuning`; override any knob with +`madnlp(m; sds_tuning = SparseDirectSolver.Fused.SDSTuning(; ...))`. diff --git a/bench/README.md b/bench/README.md index 38c4610..5db4956 100644 --- a/bench/README.md +++ b/bench/README.md @@ -22,7 +22,9 @@ Baselines every later milestone is measured against (PLAN.md §5 M0, §7). The | `order_search.jl`, `get_cudss_perm.jl` | PR #107 ordering study: METIS knob sweep and schedule depth; extraction of cuDSS's reordering through the raw C API for replay under `user_perm` | | `ordering_chooser.jl` | T22 (issue #108): AMD vs ND per harness matrix and dump (schedule depth, fundamental and column-etree depth, nnz(L), flops) and the choice of the T22 and T05 cost models; host only, no GPU: `julia --project=bench bench/ordering_chooser.jl` (`SDS_BENCH_SUITESPARSE=0` skips MatrixDepot) | | `amd_proto.jl`, `amd/Project.toml` | the solve prototype on ROCm (ROCArray, raw-array constructor, `Threads.atomic_fence`); environment `bench/amd/` | -| `e2e/MadNLPSDS.jl`, `e2e/SDSProto.jl`, `e2e/run_gv100.jl`, `e2e/run_amd.jl` | MadNLP `AbstractLinearSolver` over the public API (prototype of the External task) and the prototype kernels packaged backend-portably; full ACOPF interior-point runs on CUDA and AMD; environment `bench/e2e/` (MadNLP, MadNLPGPU, ExaModels, ExaModelsPower) | +| `e2e/run_gv100.jl`, `e2e/run_amd.jl`, `e2e/fresh_machine.{jl,sh}` | full ACOPF interior-point runs through the MadNLP extension (`SparseDirectSolver.SDSSolver`, the fused pipeline of `src/fused.jl`) on CUDA and AMD, and the one-file fresh-machine workflow; environment `bench/e2e/` (MadNLP, MadNLPGPU, ExaModels, ExaModelsPower) | +| `e2e/ldl_fused_test.jl`, `e2e/ldl_fused_race.jl`, `e2e/amd_baseline.jl`, `e2e/amd_sweep.jl` | gates and timings of the fused Cholesky/LDLᵀ refactorization and the fused solve against the stock kernels on the 78k-bus condensed KKT (CUDA; the `amd_*` scripts on ROCm), see `PERFORMANCE-2026-10.md` and `ARCH-TUNING.md` | +| `e2e/SDSCPU.jl`, `e2e/cpu_fast_test.jl`, `e2e/hsl_cpu_bench.jl`, `e2e/ma27_bench.jl`, `e2e/ma57_bench.jl` | the task-parallel CPU refactorization prototype over the stock schedule and its HSL references (MA27/MA57 through libcoinhsl; the MA86/CHOLMOD numbers in PERFORMANCE-2026-10.md came from one-off drivers, not shipped) | | `pivot_pairs.jl` | 2×2 pivot pairs of the `S` analysis (issue #66): `pivot_pairs` = `none`/`default`/`all` on the K2 dumps and the KKT generators, with nnz(L), zero/perturbed/2×2 pivots, max abs L and factor error of the CPU reference LDLᵀ; package environment: `julia --project=. bench/pivot_pairs.jl [--only=case118,...] [--generators=false]` | ## Commands diff --git a/bench/comparison-gv100/comparison.md b/bench/comparison-gv100/comparison.md new file mode 100644 index 0000000..fdcd626 --- /dev/null +++ b/bench/comparison-gv100/comparison.md @@ -0,0 +1,238 @@ +# cuDSS vs SparseDirectSolver.jl + +Generated by `bench/compare_report.jl` from `bench/comparison/{cudss,sds}.csv` (`bench/compare.jl`). Times are BenchmarkTools medians per phase on a fresh solver; the ratio is SDS time / cuDSS time, so below 1 means SparseDirectSolver.jl is faster. Features whose task is not done in TASKS.md have blank SDS cells. + +![SDS/cuDSS time ratio per feature](comparison.png) + +## Overview + +Geometric mean of the SDS/cuDSS ratio over the matrices both solvers ran. + +| feature | task | status | matrices | analysis | factorization | refactorization | solve | +| --- | --- | --- | --- | --- | --- | --- | --- | +| Cholesky, Float64 | T13 | done | 16 | 0.80× | 2.81× | 4.63× | 5.43× | +| Cholesky, 16 right-hand sides | T12 | done | 16 | 0.80× | 3.63× | 4.66× | 3.68× | +| LDLᵀ, static pivoting | T15 | done | 27 | 1.59× | 6.01× | 8.01× | 5.09× | +| LDLᵀ + 2 refinement steps, K2 dumps | T16 | done | 11 | 4.60× | 7.39× | 9.57× | 24.47× | +| Uniform batch of 8, Cholesky | T17 | done | 11 | 0.56× | 1.21× | 1.84× | 3.69× | +| LU, unsymmetric | T19 | done | 2 | 1.20× | 11.99× | 11.41× | 10.83× | +| Schur complement, Cholesky | T20 | done | 11 | 0.62× | 2.88× | 4.31× | 4.56× | +| LDLᵀ + matching (algo5), K2 dumps | T21 | done | 11 | 0.84× | 3.11× | 4.39× | 4.91× | +| Non-uniform batch, condensed dumps | T22 | pending | | | | | | +| Partitioned-inverse solve (SDS algo1, cuDSS default) | T25 | pending | | | | | | +| Hybrid memory mode | T26 | pending | | | | | | +| Float32 factors, Float64 refinement | T27 | pending | | | | | | + +## Cholesky, Float64 + +`cholesky_f64`, T13 (done), structure `SPD`, Float64, nrhs = 1. + +| matrix | n | analysis cuDSS ms | SDS ms | ratio | factorization cuDSS ms | SDS ms | ratio | refactorization cuDSS ms | SDS ms | ratio | solve cuDSS ms | SDS ms | ratio | nnz(L) cuDSS | SDS | relres cuDSS | SDS | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| Boeing/bcsstk38 | 8032 | 65 | 121 | 1.85× | 3.1 | 27.4 | 8.84× | 2.81 | 27.4 | 9.75× | 0.642 | 2.67 | 4.16× | 7.99e+05 | 8.05e+05 | 2.14e-10 | 9.26e-11 | +| GHS_psdef/apache2 | 715176 | 7.71e+03 | 1.32e+04 | 1.71× | 265 | 763 | 2.88× | 264 | 752 | 2.85× | 9.94 | 86.8 | 8.73× | 1.45e+08 | 1.29e+08 | 1.79e-10 | 1.57e-10 | +| HB/bcsstk17 | 10974 | 73.4 | 151 | 2.06× | 4.27 | 30.7 | 7.19× | 2.91 | 30.7 | 10.55× | 0.67 | 4.95 | 7.38× | 1.08e+06 | 1.04e+06 | 3.91e-11 | 3.88e-11 | +| kkt_pglib_opf_case118_ieee_condensed_1 | 1088 | 19.9 | 10.6 | 0.53× | 0.848 | 1.16 | 1.36× | 0.315 | 1.14 | 3.62× | 0.227 | 1.28 | 5.64× | 1.52e+04 | 1.16e+04 | 3.64e-05 | 5.22e-05 | +| kkt_pglib_opf_case118_ieee_condensed_10 | 1088 | 19.9 | 10.6 | 0.53× | 3.17 | 1.15 | 0.36× | 0.314 | 1.13 | 3.60× | 0.225 | 1.35 | 5.99× | 1.52e+04 | 1.16e+04 | 1.38e-05 | 1.12e-05 | +| kkt_pglib_opf_case118_ieee_condensed_20 | 1088 | 19.9 | 10.8 | 0.54× | 0.623 | 1.14 | 1.82× | 0.316 | 1.13 | 3.57× | 0.226 | 1.27 | 5.62× | 1.52e+04 | 1.16e+04 | 0.0466 | 0.0455 | +| kkt_pglib_opf_case1354_pegase_condensed_1 | 11192 | 98.1 | 121 | 1.24× | 3.31 | 3.89 | 1.18× | 0.78 | 3.86 | 4.95× | 0.332 | 1.71 | 5.15× | 1.55e+05 | 1.21e+05 | 0.000312 | 0.00028 | +| kkt_pglib_opf_case1354_pegase_condensed_10 | 11192 | 98.1 | 121 | 1.23× | 3.26 | 3.87 | 1.19× | 0.778 | 3.83 | 4.93× | 0.328 | 1.63 | 4.95× | 1.55e+05 | 1.21e+05 | 0.000144 | 9.67e-05 | +| kkt_pglib_opf_case1354_pegase_condensed_20 | 11192 | 98.4 | 122 | 1.24× | 1.11 | 3.85 | 3.47× | 0.785 | 3.84 | 4.89× | 0.323 | 1.76 | 5.45× | 1.55e+05 | 1.21e+05 | 7.72e-05 | 5.03e-05 | +| kkt_pglib_opf_case14_ieee_condensed_1 | 118 | 10.8 | 1.37 | 0.13× | 0.184 | 0.409 | 2.23× | 0.166 | 0.414 | 2.50× | 0.139 | 0.65 | 4.68× | 1.27e+03 | 1.03e+03 | 1.13e-06 | 1.3e-06 | +| kkt_pglib_opf_case14_ieee_condensed_10 | 118 | 10.8 | 1.4 | 0.13× | 0.186 | 0.415 | 2.24× | 0.165 | 0.4 | 2.42× | 0.138 | 0.664 | 4.82× | 1.27e+03 | 1.03e+03 | 4.27e-05 | 7.85e-05 | +| kkt_pglib_opf_case14_ieee_condensed_11 | 118 | 10.8 | 1.39 | 0.13× | 0.186 | 0.401 | 2.16× | 0.167 | 0.384 | 2.30× | 0.139 | 0.661 | 4.76× | 1.27e+03 | 1.03e+03 | 9.6e-06 | 9.57e-06 | +| kkt_pglib_opf_case78484_epigrids_condensed_1 | 674562 | 6.57e+03 | 1.14e+04 | 1.73× | 24.9 | 167 | 6.71× | 24.5 | 167 | 6.82× | 3.79 | 18.3 | 4.82× | 1.56e+07 | 1.32e+07 | 0.153 | 0.141 | +| kkt_pglib_opf_case78484_epigrids_condensed_10 | 674562 | 6.56e+03 | 1.16e+04 | 1.76× | 24.8 | 167 | 6.73× | 24.4 | 167 | 6.82× | 3.79 | 18.2 | 4.82× | 1.56e+07 | 1.32e+07 | 0.0176 | 0.0147 | +| lap2d_300 | 90000 | 512 | 787 | 1.54× | 6.06 | 33.7 | 5.56× | 5.53 | 33.6 | 6.09× | 1.14 | 5.12 | 4.51× | 2.43e+06 | 2.47e+06 | 1.84e-12 | 1.76e-12 | +| lap3d_40 | 64000 | 637 | 904 | 1.42× | 44.3 | 303 | 6.83× | 43.2 | 303 | 7.01× | 2.44 | 17.5 | 7.17× | 1.75e+07 | 1.44e+07 | 8.21e-14 | 7.33e-14 | + +## Cholesky, 16 right-hand sides + +`cholesky_nrhs16`, T12 (done), structure `SPD`, Float64, nrhs = 16. + +| matrix | n | analysis cuDSS ms | SDS ms | ratio | factorization cuDSS ms | SDS ms | ratio | refactorization cuDSS ms | SDS ms | ratio | solve cuDSS ms | SDS ms | ratio | nnz(L) cuDSS | SDS | relres cuDSS | SDS | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| Boeing/bcsstk38 | 8032 | 61.5 | 120 | 1.95× | 3.12 | 27.4 | 8.77× | 2.81 | 27.4 | 9.72× | 0.888 | 3.23 | 3.63× | 7.99e+05 | 8.05e+05 | 7.31e-10 | 1.72e-10 | +| GHS_psdef/apache2 | 715176 | 7.67e+03 | 1.31e+04 | 1.71× | 265 | 750 | 2.83× | 264 | 751 | 2.84× | 43.9 | 176 | 4.00× | 1.45e+08 | 1.29e+08 | 1.82e-10 | 1.53e-10 | +| HB/bcsstk17 | 10974 | 74.1 | 151 | 2.03× | 3.21 | 30.7 | 9.57× | 2.91 | 30.7 | 10.55× | 0.967 | 5.86 | 6.05× | 1.08e+06 | 1.04e+06 | 4.57e-11 | 5.57e-11 | +| kkt_pglib_opf_case118_ieee_condensed_1 | 1088 | 19.9 | 10.6 | 0.53× | 0.858 | 1.16 | 1.35× | 0.319 | 1.14 | 3.58× | 0.308 | 1.36 | 4.42× | 1.52e+04 | 1.16e+04 | 5.34e-05 | 0.000108 | +| kkt_pglib_opf_case118_ieee_condensed_10 | 1088 | 19.9 | 10.6 | 0.53× | 0.616 | 1.14 | 1.85× | 0.322 | 1.13 | 3.51× | 0.307 | 1.38 | 4.51× | 1.52e+04 | 1.16e+04 | 1.48e-05 | 1.69e-05 | +| kkt_pglib_opf_case118_ieee_condensed_20 | 1088 | 19.9 | 10.7 | 0.54× | 0.624 | 1.15 | 1.84× | 0.316 | 1.13 | 3.58× | 0.306 | 1.37 | 4.47× | 1.52e+04 | 1.16e+04 | 0.108 | 0.0835 | +| kkt_pglib_opf_case1354_pegase_condensed_1 | 11192 | 101 | 121 | 1.20× | 1.09 | 3.9 | 3.57× | 0.771 | 3.86 | 5.01× | 0.612 | 1.89 | 3.08× | 1.55e+05 | 1.21e+05 | 0.000392 | 0.000291 | +| kkt_pglib_opf_case1354_pegase_condensed_10 | 11192 | 97.9 | 121 | 1.23× | 1.09 | 3.88 | 3.56× | 0.774 | 3.84 | 4.96× | 0.609 | 1.89 | 3.10× | 1.55e+05 | 1.21e+05 | 0.000152 | 0.000114 | +| kkt_pglib_opf_case1354_pegase_condensed_20 | 11192 | 103 | 124 | 1.21× | 1.09 | 3.87 | 3.54× | 0.767 | 3.84 | 5.01× | 0.609 | 1.9 | 3.11× | 1.55e+05 | 1.21e+05 | 0.000101 | 6.47e-05 | +| kkt_pglib_opf_case14_ieee_condensed_1 | 118 | 10.8 | 1.38 | 0.13× | 0.187 | 0.418 | 2.24× | 0.165 | 0.417 | 2.52× | 0.16 | 0.708 | 4.42× | 1.27e+03 | 1.03e+03 | 1.18e-06 | 1.47e-06 | +| kkt_pglib_opf_case14_ieee_condensed_10 | 118 | 10.8 | 1.39 | 0.13× | 0.187 | 0.42 | 2.24× | 0.167 | 0.419 | 2.51× | 0.164 | 0.706 | 4.30× | 1.27e+03 | 1.03e+03 | 6.75e-05 | 0.000143 | +| kkt_pglib_opf_case14_ieee_condensed_11 | 118 | 10.8 | 1.46 | 0.14× | 0.187 | 0.407 | 2.17× | 0.169 | 0.403 | 2.39× | 0.157 | 0.711 | 4.52× | 1.27e+03 | 1.03e+03 | 4.36e-05 | 2.29e-05 | +| kkt_pglib_opf_case78484_epigrids_condensed_1 | 674562 | 6.56e+03 | 1.19e+04 | 1.81× | 24.9 | 167 | 6.71× | 24.5 | 167 | 6.84× | 21.7 | 51.7 | 2.38× | 1.56e+07 | 1.32e+07 | 0.185 | 0.175 | +| kkt_pglib_opf_case78484_epigrids_condensed_10 | 674562 | 6.55e+03 | 1.17e+04 | 1.79× | 25.2 | 167 | 6.62× | 24.4 | 167 | 6.84× | 21.7 | 51.7 | 2.38× | 1.56e+07 | 1.32e+07 | 0.0178 | 0.0148 | +| lap2d_300 | 90000 | 510 | 787 | 1.55× | 6.08 | 33.6 | 5.53× | 5.51 | 33.6 | 6.11× | 4.23 | 9.99 | 2.36× | 2.43e+06 | 2.47e+06 | 1.85e-12 | 1.76e-12 | +| lap3d_40 | 64000 | 633 | 904 | 1.43× | 43.7 | 303 | 6.93× | 42.9 | 303 | 7.06× | 5.54 | 22.9 | 4.14× | 1.75e+07 | 1.44e+07 | 8.92e-14 | 6.39e-14 | + +## LDLᵀ, static pivoting + +`ldlt`, T15 (done), structure `S`, Float64, nrhs = 1. + +| matrix | n | analysis cuDSS ms | SDS ms | ratio | factorization cuDSS ms | SDS ms | ratio | refactorization cuDSS ms | SDS ms | ratio | solve cuDSS ms | SDS ms | ratio | nnz(L) cuDSS | SDS | relres cuDSS | SDS | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| Boeing/bcsstk38 | 8032 | 68 | 139 | 2.05× | 5.39 | 47.1 | 8.75× | 3.72 | 47.1 | 12.66× | 0.675 | 2.79 | 4.14× | 7.99e+05 | 8.05e+05 | 7.1e-11 | 1.77e-10 | +| GHS_psdef/apache2 | 715176 | 7.69e+03 | 1.33e+04 | 1.73× | 314 | 1.88e+03 | 5.99× | 313 | 1.87e+03 | 5.98× | 13.2 | 108 | 8.18× | 1.45e+08 | 1.29e+08 | 1.57e-10 | 1.42e-10 | +| HB/bcsstk17 | 10974 | 76 | 169 | 2.22× | 4.1 | 56.6 | 13.81× | 3.8 | 56.5 | 14.88× | 0.702 | 5.18 | 7.38× | 1.08e+06 | 1.04e+06 | 3.62e-11 | 4.16e-11 | +| kkt_pglib_opf_case118_ieee_condensed_1 | 1088 | 24.8 | 11.4 | 0.46× | 0.758 | 2.67 | 3.53× | 0.476 | 2.65 | 5.57× | 0.342 | 1.28 | 3.75× | 1.52e+04 | 1.16e+04 | 5.35e-05 | 0.000111 | +| kkt_pglib_opf_case118_ieee_condensed_10 | 1088 | 24.7 | 11.4 | 0.46× | 1.25 | 2.6 | 2.08× | 0.484 | 2.56 | 5.30× | 0.334 | 1.37 | 4.10× | 1.52e+04 | 1.16e+04 | 1.18e-05 | 1.01e-05 | +| kkt_pglib_opf_case118_ieee_condensed_20 | 1088 | 24.8 | 11.4 | 0.46× | 0.724 | 2.67 | 3.69× | 0.475 | 2.64 | 5.56× | 0.336 | 1.3 | 3.87× | 1.52e+04 | 1.16e+04 | 0.0353 | 0.0365 | +| kkt_pglib_opf_case118_ieee_k2_1 | 3185 | 27.9 | 134 | 4.80× | 0.901 | 2.66 | 2.95× | 0.367 | 2.63 | 7.18× | 0.221 | 1.58 | 7.13× | 2e+04 | 2.11e+04 | 3.8 | 1.05 | +| kkt_pglib_opf_case118_ieee_k2_10 | 3185 | 27.9 | 137 | 4.93× | 0.675 | 2.59 | 3.84× | 0.353 | 2.56 | 7.26× | 0.222 | 1.61 | 7.27× | 2e+04 | 2.06e+04 | 7.08 | 0.00737 | +| kkt_pglib_opf_case118_ieee_k2_20 | 3185 | 27.9 | 134 | 4.81× | 2.12 | 2.56 | 1.21× | 0.356 | 2.53 | 7.10× | 0.223 | 1.56 | 7.01× | 2e+04 | 2.06e+04 | 12.6 | 0.0238 | +| kkt_pglib_opf_case1354_pegase_condensed_1 | 11192 | 101 | 127 | 1.26× | 1.26 | 10.3 | 8.17× | 0.956 | 10.2 | 10.70× | 0.335 | 1.71 | 5.11× | 1.55e+05 | 1.21e+05 | 0.000291 | 0.000313 | +| kkt_pglib_opf_case1354_pegase_condensed_10 | 11192 | 102 | 128 | 1.26× | 1.26 | 10.4 | 8.23× | 0.951 | 10.4 | 10.90× | 0.334 | 1.77 | 5.31× | 1.55e+05 | 1.21e+05 | 0.000146 | 0.00011 | +| kkt_pglib_opf_case1354_pegase_condensed_20 | 11192 | 99.7 | 128 | 1.29× | 1.26 | 10.4 | 8.27× | 0.935 | 10.4 | 11.11× | 0.337 | 1.73 | 5.13× | 1.55e+05 | 1.21e+05 | 6.73e-05 | 5.62e-05 | +| kkt_pglib_opf_case1354_pegase_k2_1 | 33811 | 193 | 2.3e+03 | 11.90× | 1.99 | 10.4 | 5.22× | 1.37 | 10.3 | 7.50× | 0.444 | 2.35 | 5.30× | 2.02e+05 | 2.16e+05 | 91.7 | 16.9 | +| kkt_pglib_opf_case1354_pegase_k2_10 | 33811 | 195 | 2.37e+03 | 12.18× | 1.66 | 9.47 | 5.72× | 1.37 | 9.44 | 6.90× | 0.442 | 2.19 | 4.95× | 2.02e+05 | 2.2e+05 | 56.3 | 29.6 | +| kkt_pglib_opf_case1354_pegase_k2_20 | 33811 | 192 | 2.39e+03 | 12.41× | 1.7 | 10.1 | 5.97× | 1.37 | 10.1 | 7.36× | 0.545 | 2.22 | 4.07× | 2.02e+05 | 2.21e+05 | 1.16e+03 | 27.5 | +| kkt_pglib_opf_case14_ieee_condensed_1 | 118 | 13.8 | 1.53 | 0.11× | 0.284 | 0.632 | 2.22× | 0.289 | 0.628 | 2.17× | 0.248 | 0.696 | 2.81× | 1.27e+03 | 1.03e+03 | 1.07e-06 | 8.58e-07 | +| kkt_pglib_opf_case14_ieee_condensed_10 | 118 | 13.8 | 1.51 | 0.11× | 0.29 | 0.637 | 2.19× | 0.287 | 0.632 | 2.20× | 0.246 | 0.692 | 2.81× | 1.27e+03 | 1.03e+03 | 3.42e-05 | 1.82e-05 | +| kkt_pglib_opf_case14_ieee_condensed_11 | 118 | 13.8 | 1.51 | 0.11× | 0.296 | 0.641 | 2.17× | 0.283 | 0.638 | 2.25× | 0.25 | 0.669 | 2.68× | 1.27e+03 | 1.03e+03 | 1.04e-05 | 7.29e-06 | +| kkt_pglib_opf_case14_ieee_k2_1 | 347 | 15.5 | 7.26 | 0.47× | 0.313 | 0.957 | 3.06× | 0.302 | 0.96 | 3.18× | 0.269 | 1.03 | 3.83× | 2.05e+03 | 2.15e+03 | 0.518 | 7.57e-08 | +| kkt_pglib_opf_case14_ieee_k2_10 | 347 | 15.5 | 7.23 | 0.47× | 0.197 | 0.965 | 4.91× | 0.177 | 0.952 | 5.38× | 0.154 | 0.977 | 6.34× | 2.05e+03 | 2.16e+03 | 0.409 | 4.36e-05 | +| kkt_pglib_opf_case14_ieee_k2_15 | 347 | 12 | 5.62 | 0.47× | 0.197 | 1.58 | 8.02× | 0.185 | 1.57 | 8.47× | 0.152 | 1.13 | 7.41× | 2.05e+03 | 3.23e+03 | 0.698 | 0.123 | +| kkt_pglib_opf_case78484_epigrids_condensed_1 | 674562 | 6.55e+03 | 1.21e+04 | 1.85× | 30.8 | 535 | 17.36× | 30.3 | 535 | 17.62× | 4.1 | 20.1 | 4.90× | 1.56e+07 | 1.32e+07 | 0.165 | 0.114 | +| kkt_pglib_opf_case78484_epigrids_condensed_10 | 674562 | 6.59e+03 | 1.14e+04 | 1.72× | 39.2 | 510 | 13.01× | 29.8 | 509 | 17.10× | 4.1 | 20.2 | 4.92× | 1.56e+07 | 1.32e+07 | 0.0168 | 0.0121 | +| kkt_pglib_opf_case78484_epigrids_k2_1 | 2091669 | 2.01e+04 | 5.44e+05 | 27.03× | 48.1 | 2.29e+03 | 47.68× | 47.4 | 2.26e+03 | 47.70× | 13.3 | 81.3 | 6.12× | 1.85e+07 | 2.77e+07 | 6.46e+09 | 1.9e+04 | +| kkt_pglib_opf_case78484_epigrids_k2_10 | 2091669 | 2.02e+04 | 5.69e+05 | 28.16× | 49.9 | 2.25e+03 | 45.05× | 46.7 | 2.23e+03 | 47.73× | 13.3 | 79 | 5.94× | 1.85e+07 | 2.7e+07 | 2.55e+04 | 8.08e+03 | +| lap2d_300 | 90000 | 478 | 807 | 1.69× | 7.38 | 50 | 6.77× | 7.02 | 49.9 | 7.11× | 1.21 | 5.94 | 4.92× | 2.43e+06 | 2.47e+06 | 1.61e-12 | 1.58e-12 | +| lap3d_40 | 64000 | 610 | 925 | 1.52× | 53 | 515 | 9.72× | 52.9 | 515 | 9.73× | 2.53 | 23.2 | 9.19× | 1.75e+07 | 1.44e+07 | 7.58e-14 | 6.27e-14 | + +## LDLᵀ + 2 refinement steps, K2 dumps + +`ldlt_ir2`, T16 (done), structure `S`, Float64, nrhs = 1. + +| matrix | n | analysis cuDSS ms | SDS ms | ratio | factorization cuDSS ms | SDS ms | ratio | refactorization cuDSS ms | SDS ms | ratio | solve cuDSS ms | SDS ms | ratio | nnz(L) cuDSS | SDS | relres cuDSS | SDS | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| kkt_pglib_opf_case118_ieee_k2_1 | 3185 | 28.9 | 133 | 4.59× | 0.664 | 2.63 | 3.96× | 0.357 | 2.59 | 7.26× | 0.489 | 7.54 | 15.44× | 2e+04 | 2.11e+04 | 0.000464 | 1.72e-08 | +| kkt_pglib_opf_case118_ieee_k2_10 | 3185 | 27.8 | 138 | 4.94× | 0.67 | 2.55 | 3.81× | 0.359 | 2.52 | 7.03× | 0.487 | 7.4 | 15.19× | 2e+04 | 2.06e+04 | 0.228 | 6.76e-10 | +| kkt_pglib_opf_case118_ieee_k2_20 | 3185 | 27.9 | 132 | 4.75× | 0.676 | 2.52 | 3.73× | 0.362 | 2.48 | 6.86× | 0.486 | 7.61 | 15.64× | 2e+04 | 2.06e+04 | 0.0968 | 1.29e-09 | +| kkt_pglib_opf_case1354_pegase_k2_1 | 33811 | 195 | 2.22e+03 | 11.39× | 1.67 | 10.3 | 6.17× | 1.37 | 10.3 | 7.49× | 1.27 | 43.2 | 34.00× | 2.02e+05 | 2.16e+05 | 35.3 | 0.0178 | +| kkt_pglib_opf_case1354_pegase_k2_10 | 33811 | 191 | 2.35e+03 | 12.26× | 1.67 | 9.51 | 5.68× | 1.36 | 9.41 | 6.90× | 1.26 | 42.5 | 33.87× | 2.02e+05 | 2.2e+05 | 60.4 | 0.657 | +| kkt_pglib_opf_case1354_pegase_k2_20 | 33811 | 193 | 2.36e+03 | 12.21× | 2.2 | 10.1 | 4.57× | 1.37 | 10 | 7.34× | 1.27 | 44.8 | 35.25× | 2.02e+05 | 2.21e+05 | 102 | 0.0683 | +| kkt_pglib_opf_case14_ieee_k2_1 | 347 | 12 | 6.89 | 0.57× | 0.2 | 0.934 | 4.67× | 0.188 | 0.92 | 4.90× | 0.295 | 3.21 | 10.87× | 2.05e+03 | 2.15e+03 | 3.35e-05 | 5.98e-13 | +| kkt_pglib_opf_case14_ieee_k2_10 | 347 | 12 | 7.22 | 0.60× | 0.199 | 0.943 | 4.74× | 0.192 | 0.925 | 4.83× | 0.296 | 3.23 | 10.93× | 2.05e+03 | 2.16e+03 | 5.93e-07 | 3.12e-14 | +| kkt_pglib_opf_case14_ieee_k2_15 | 347 | 12 | 5.47 | 0.46× | 0.197 | 1.56 | 7.95× | 0.18 | 1.53 | 8.52× | 0.592 | 3.58 | 6.06× | 2.05e+03 | 3.23e+03 | 2.35e-07 | 1.77e-09 | +| kkt_pglib_opf_case78484_epigrids_k2_1 | 2091669 | 2.01e+04 | 4.93e+05 | 24.49× | 48 | 2.29e+03 | 47.69× | 47.2 | 2.27e+03 | 47.98× | 46.1 | 6.18e+03 | 133.91× | 1.85e+07 | 2.77e+07 | 6.01e+06 | 4.13e+05 | +| kkt_pglib_opf_case78484_epigrids_k2_10 | 2091669 | 2.01e+04 | 5.61e+05 | 27.85× | 47.4 | 2.26e+03 | 47.65× | 46.5 | 2.23e+03 | 48.04× | 46.2 | 6.06e+03 | 131.15× | 1.85e+07 | 2.7e+07 | 6.25e+05 | 1.99e+05 | + +## Uniform batch of 8, Cholesky + +`ubatch8`, T17 (done), structure `SPD`, Float64, nrhs = 1, uniform batch of 8. + +| matrix | n | analysis cuDSS ms | SDS ms | ratio | factorization cuDSS ms | SDS ms | ratio | refactorization cuDSS ms | SDS ms | ratio | solve cuDSS ms | SDS ms | ratio | nnz(L) cuDSS | SDS | relres cuDSS | SDS | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| kkt_pglib_opf_case118_ieee_condensed_1 | 1088 | 19.9 | 10.7 | 0.54× | 0.966 | 1.17 | 1.21× | 0.627 | 1.14 | 1.82× | 0.265 | 1.22 | 4.61× | 1.52e+04 | 1.16e+04 | 1.84e-05 | 0.000101 | +| kkt_pglib_opf_case118_ieee_condensed_10 | 1088 | 19.9 | 10.7 | 0.54× | 0.952 | 1.15 | 1.20× | 0.626 | 1.12 | 1.79× | 0.265 | 1.27 | 4.78× | 1.52e+04 | 1.16e+04 | 1.35e-05 | 1.03e-05 | +| kkt_pglib_opf_case118_ieee_condensed_20 | 1088 | 19.9 | 10.7 | 0.54× | 0.962 | 1.15 | 1.19× | 0.627 | 1.13 | 1.80× | 0.264 | 1.3 | 4.93× | 1.52e+04 | 1.16e+04 | 0.146 | 0.0495 | +| kkt_pglib_opf_case1354_pegase_condensed_1 | 11192 | 97.9 | 122 | 1.25× | 3.13 | 4.81 | 1.54× | 2.82 | 4.76 | 1.69× | 0.644 | 1.76 | 2.73× | 1.55e+05 | 1.21e+05 | 0.000317 | 0.000273 | +| kkt_pglib_opf_case1354_pegase_condensed_10 | 11192 | 98.8 | 120 | 1.22× | 3.27 | 4.74 | 1.45× | 2.82 | 4.7 | 1.67× | 0.639 | 1.74 | 2.72× | 1.55e+05 | 1.21e+05 | 0.000154 | 0.000104 | +| kkt_pglib_opf_case1354_pegase_condensed_20 | 11192 | 100 | 121 | 1.21× | 3.12 | 4.74 | 1.52× | 2.82 | 4.71 | 1.67× | 0.642 | 1.72 | 2.68× | 1.55e+05 | 1.21e+05 | 7.5e-05 | 5.43e-05 | +| kkt_pglib_opf_case14_ieee_condensed_1 | 118 | 10.8 | 1.37 | 0.13× | 0.484 | 0.4 | 0.83× | 0.172 | 0.364 | 2.11× | 0.131 | 0.627 | 4.78× | 1.27e+03 | 1.03e+03 | 3.24e-07 | 9.03e-07 | +| kkt_pglib_opf_case14_ieee_condensed_10 | 118 | 10.8 | 1.37 | 0.13× | 0.484 | 0.392 | 0.81× | 0.171 | 0.398 | 2.33× | 0.131 | 0.633 | 4.82× | 1.27e+03 | 1.03e+03 | 2.89e-05 | 3.97e-05 | +| kkt_pglib_opf_case14_ieee_condensed_11 | 118 | 10.8 | 1.44 | 0.13× | 0.487 | 0.375 | 0.77× | 0.173 | 0.371 | 2.15× | 0.131 | 0.643 | 4.89× | 1.27e+03 | 1.03e+03 | 8.59e-06 | 7.96e-06 | +| kkt_pglib_opf_case78484_epigrids_condensed_1 | 674562 | 6.6e+03 | 1.06e+04 | 1.60× | 170 | 282 | 1.66× | 169 | 282 | 1.67× | 25.2 | 66.6 | 2.65× | 1.56e+07 | 1.32e+07 | 0.173 | 0.135 | +| kkt_pglib_opf_case78484_epigrids_condensed_10 | 674562 | 6.58e+03 | 1.06e+04 | 1.62× | 170 | 279 | 1.65× | 169 | 279 | 1.65× | 25.2 | 66.7 | 2.65× | 1.56e+07 | 1.32e+07 | 0.0176 | 0.0149 | + +## LU, unsymmetric + +`lu`, T19 (done), structure `G`, Float64, nrhs = 1. + +| matrix | n | analysis cuDSS ms | SDS ms | ratio | factorization cuDSS ms | SDS ms | ratio | refactorization cuDSS ms | SDS ms | ratio | solve cuDSS ms | SDS ms | ratio | nnz(L) cuDSS | SDS | relres cuDSS | SDS | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| Rajat/rajat21 | 411676 | 3.76e+03 | 4.86e+03 | 1.29× | 42.2 | 646 | 15.29× | 51.5 | 646 | 12.54× | 4.32 | 32.7 | 7.55× | 6.89e+06 | 2.37e+06 | 3.34e+07 | 2.9e+03 | +| TSOPF/TSOPF_RS_b39_c7 | 14098 | 199 | 223 | 1.12× | 2.99 | 28.1 | 9.41× | 2.71 | 28.1 | 10.38× | 0.516 | 8.02 | 15.52× | 1.47e+06 | 5.22e+05 | 9.07e+29 | 20.3 | + +## Schur complement, Cholesky + +`schur`, T20 (done), structure `SPD`, Float64, nrhs = 1. + +| matrix | n | analysis cuDSS ms | SDS ms | ratio | factorization cuDSS ms | SDS ms | ratio | refactorization cuDSS ms | SDS ms | ratio | solve cuDSS ms | SDS ms | ratio | nnz(L) cuDSS | SDS | relres cuDSS | SDS | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| kkt_pglib_opf_case118_ieee_condensed_1 | 1088 | 24.9 | 11.6 | 0.47× | 0.709 | 1.57 | 2.21× | 0.335 | 1.5 | 4.47× | 0.142 | 0.754 | 5.33× | 1.96e+04 | 1.85e+04 | | | +| kkt_pglib_opf_case118_ieee_condensed_10 | 1088 | 19.7 | 11.8 | 0.60× | 0.647 | 1.58 | 2.44× | 0.34 | 1.52 | 4.46× | 0.143 | 0.735 | 5.15× | 1.96e+04 | 1.85e+04 | | | +| kkt_pglib_opf_case118_ieee_condensed_20 | 1088 | 19.7 | 11.7 | 0.60× | 3.38 | 1.5 | 0.44× | 0.353 | 1.48 | 4.18× | 0.159 | 0.675 | 4.25× | 1.96e+04 | 1.85e+04 | | | +| kkt_pglib_opf_case1354_pegase_condensed_1 | 11192 | 103 | 129 | 1.25× | 1.18 | 4.86 | 4.10× | 0.856 | 4.83 | 5.64× | 0.2 | 1.01 | 5.04× | 1.65e+05 | 1.31e+05 | | | +| kkt_pglib_opf_case1354_pegase_condensed_10 | 11192 | 99.9 | 128 | 1.28× | 1.18 | 4.83 | 4.11× | 0.851 | 4.81 | 5.64× | 0.213 | 0.973 | 4.57× | 1.65e+05 | 1.31e+05 | | | +| kkt_pglib_opf_case1354_pegase_condensed_20 | 11192 | 103 | 128 | 1.24× | 1.17 | 4.84 | 4.15× | 0.873 | 4.8 | 5.50× | 0.199 | 1.01 | 5.07× | 1.65e+05 | 1.31e+05 | | | +| kkt_pglib_opf_case14_ieee_condensed_1 | 118 | 10.8 | 1.61 | 0.15× | 0.218 | 0.504 | 2.31× | 0.197 | 0.464 | 2.36× | 0.115 | 0.405 | 3.52× | 1.51e+03 | 1.3e+03 | | | +| kkt_pglib_opf_case14_ieee_condensed_10 | 118 | 10.8 | 1.96 | 0.18× | 0.222 | 0.691 | 3.11× | 0.201 | 0.533 | 2.66× | 0.117 | 0.441 | 3.77× | 1.51e+03 | 1.3e+03 | | | +| kkt_pglib_opf_case14_ieee_condensed_11 | 118 | 10.8 | 2.01 | 0.19× | 0.222 | 0.545 | 2.46× | 0.195 | 0.508 | 2.60× | 0.116 | 0.435 | 3.75× | 1.51e+03 | 1.3e+03 | | | +| kkt_pglib_opf_case78484_epigrids_condensed_1 | 674562 | 6.65e+03 | 1.24e+04 | 1.86× | 26.7 | 166 | 6.22× | 26 | 166 | 6.37× | 1.89 | 9.71 | 5.14× | 1.57e+07 | 1.34e+07 | | | +| kkt_pglib_opf_case78484_epigrids_condensed_10 | 674562 | 6.66e+03 | 1.19e+04 | 1.79× | 26.8 | 165 | 6.18× | 26.2 | 165 | 6.30× | 1.89 | 9.59 | 5.06× | 1.57e+07 | 1.34e+07 | | | + +## LDLᵀ + matching (algo5), K2 dumps + +`ldlt_matching`, T21 (done), structure `S`, Float64, nrhs = 1. + +| matrix | n | analysis cuDSS ms | SDS ms | ratio | factorization cuDSS ms | SDS ms | ratio | refactorization cuDSS ms | SDS ms | ratio | solve cuDSS ms | SDS ms | ratio | nnz(L) cuDSS | SDS | relres cuDSS | SDS | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| kkt_pglib_opf_case118_ieee_k2_1 | 3185 | 39.3 | 30 | 0.76× | 6.89 | 3.38 | 0.49× | 0.841 | 3.36 | 4.00× | 0.279 | 1.81 | 6.47× | 7.41e+04 | 3.61e+04 | 3.1e-12 | 4.39e-12 | +| kkt_pglib_opf_case118_ieee_k2_10 | 3185 | 39.3 | 37.9 | 0.96× | 1.12 | 3.16 | 2.82× | 0.776 | 3.08 | 3.96× | 0.253 | 1.66 | 6.54× | 6.87e+04 | 3.47e+04 | 7.32e-13 | 7.39e-13 | +| kkt_pglib_opf_case118_ieee_k2_20 | 3185 | 39.5 | 31.3 | 0.79× | 2.16 | 3.52 | 1.63× | 0.809 | 3.48 | 4.30× | 0.261 | 1.76 | 6.74× | 6.93e+04 | 3.44e+04 | 5.12e-13 | 4.08e-13 | +| kkt_pglib_opf_case1354_pegase_k2_1 | 33811 | 255 | 349 | 1.37× | 3.75 | 16 | 4.28× | 3.1 | 16 | 5.16× | 0.524 | 2.53 | 4.83× | 7.23e+05 | 3.68e+05 | 0.00447 | 0.000311 | +| kkt_pglib_opf_case1354_pegase_k2_10 | 33811 | 259 | 374 | 1.44× | 3.34 | 14.8 | 4.43× | 3.03 | 14.7 | 4.86× | 0.517 | 2.48 | 4.80× | 7.21e+05 | 3.65e+05 | 0.00902 | 0.000577 | +| kkt_pglib_opf_case1354_pegase_k2_20 | 33811 | 260 | 419 | 1.61× | 3.49 | 14.8 | 4.23× | 3.17 | 14.7 | 4.64× | 0.526 | 2.26 | 4.29× | 7.23e+05 | 3.63e+05 | 0.0147 | 0.000394 | +| kkt_pglib_opf_case14_ieee_k2_1 | 347 | 13.6 | 4.13 | 0.30× | 0.33 | 1.23 | 3.73× | 0.313 | 1.19 | 3.80× | 0.17 | 0.832 | 4.89× | 7.11e+03 | 3.64e+03 | 7.95e-13 | 1.58e-12 | +| kkt_pglib_opf_case14_ieee_k2_10 | 347 | 13.1 | 3.95 | 0.30× | 0.295 | 1.04 | 3.54× | 0.283 | 1.02 | 3.61× | 0.159 | 0.811 | 5.10× | 6.76e+03 | 3.54e+03 | 5.73e-14 | 7.09e-14 | +| kkt_pglib_opf_case14_ieee_k2_15 | 347 | 13.1 | 4.07 | 0.31× | 0.301 | 1.06 | 3.53× | 0.286 | 1.06 | 3.70× | 0.157 | 0.857 | 5.46× | 6.76e+03 | 3.53e+03 | 9.6e-14 | 4.87e-14 | +| kkt_pglib_opf_case78484_epigrids_k2_1 | 2091669 | 2.43e+04 | 3.71e+04 | 1.53× | 130 | 714 | 5.49× | 129 | 685 | 5.31× | 14.4 | 43.1 | 2.99× | 7.45e+07 | 4.08e+07 | 1.06e+08 | 0.000612 | +| kkt_pglib_opf_case78484_epigrids_k2_10 | 2091669 | 2.48e+04 | 4.27e+04 | 1.73× | 132 | 760 | 5.74× | 132 | 726 | 5.51× | 14.4 | 49.9 | 3.46× | 7.41e+07 | 4.13e+07 | 6.03e+09 | 0.000205 | + +## Non-uniform batch, condensed dumps + +`nubatch`, T22 (pending), structure `SPD`, Float64, nrhs = 1. + +| matrix | n | analysis cuDSS ms | SDS ms | ratio | factorization cuDSS ms | SDS ms | ratio | refactorization cuDSS ms | SDS ms | ratio | solve cuDSS ms | SDS ms | ratio | nnz(L) cuDSS | SDS | relres cuDSS | SDS | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| batch_of_11 | 1386318 | 1.43e+04 | | | 53.1 | | | 50.9 | | | 11.9 | | | 3.16e+07 | | 0.14 | | + +## Partitioned-inverse solve (SDS algo1, cuDSS default) + +`solve_algo1`, T25 (pending), structure `SPD`, Float64, nrhs = 1. + +| matrix | n | analysis cuDSS ms | SDS ms | ratio | factorization cuDSS ms | SDS ms | ratio | refactorization cuDSS ms | SDS ms | ratio | solve cuDSS ms | SDS ms | ratio | nnz(L) cuDSS | SDS | relres cuDSS | SDS | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| Boeing/bcsstk38 | 8032 | 61.4 | | | 5.15 | | | 2.81 | | | 0.647 | | | 7.99e+05 | | 2.53e-10 | | +| GHS_psdef/apache2 | 715176 | 7.66e+03 | | | 264 | | | 264 | | | 9.97 | | | 1.45e+08 | | 1.79e-10 | | +| HB/bcsstk17 | 10974 | 77.1 | | | 3.22 | | | 2.9 | | | 0.671 | | | 1.08e+06 | | 4.07e-11 | | +| kkt_pglib_opf_case118_ieee_condensed_1 | 1088 | 19.9 | | | 0.635 | | | 0.328 | | | 0.245 | | | 1.52e+04 | | 6.49e-05 | | +| kkt_pglib_opf_case118_ieee_condensed_10 | 1088 | 19.9 | | | 0.66 | | | 0.318 | | | 0.233 | | | 1.52e+04 | | 1.06e-05 | | +| kkt_pglib_opf_case118_ieee_condensed_20 | 1088 | 19.9 | | | 0.646 | | | 0.316 | | | 0.225 | | | 1.52e+04 | | 0.0836 | | +| kkt_pglib_opf_case1354_pegase_condensed_1 | 11192 | 100 | | | 1.1 | | | 0.784 | | | 0.345 | | | 1.55e+05 | | 0.000326 | | +| kkt_pglib_opf_case1354_pegase_condensed_10 | 11192 | 98 | | | 1.1 | | | 0.777 | | | 0.325 | | | 1.55e+05 | | 0.000152 | | +| kkt_pglib_opf_case1354_pegase_condensed_20 | 11192 | 98.2 | | | 1.87 | | | 0.764 | | | 0.326 | | | 1.55e+05 | | 8.09e-05 | | +| kkt_pglib_opf_case14_ieee_condensed_1 | 118 | 10.8 | | | 0.181 | | | 0.161 | | | 0.137 | | | 1.27e+03 | | 2.55e-07 | | +| kkt_pglib_opf_case14_ieee_condensed_10 | 118 | 10.8 | | | 0.181 | | | 0.16 | | | 0.136 | | | 1.27e+03 | | 3.97e-05 | | +| kkt_pglib_opf_case14_ieee_condensed_11 | 118 | 10.8 | | | 0.183 | | | 0.16 | | | 0.135 | | | 1.27e+03 | | 2.24e-05 | | +| kkt_pglib_opf_case78484_epigrids_condensed_1 | 674562 | 6.55e+03 | | | 25.1 | | | 24.4 | | | 3.8 | | | 1.56e+07 | | 0.155 | | +| kkt_pglib_opf_case78484_epigrids_condensed_10 | 674562 | 6.56e+03 | | | 25 | | | 24.4 | | | 3.8 | | | 1.56e+07 | | 0.0178 | | +| lap2d_300 | 90000 | 479 | | | 5.83 | | | 5.5 | | | 1.13 | | | 2.43e+06 | | 1.84e-12 | | +| lap3d_40 | 64000 | 612 | | | 46.3 | | | 42.9 | | | 2.41 | | | 1.75e+07 | | 8.21e-14 | | + +## Hybrid memory mode + +`hybrid_memory`, T26 (pending), structure `SPD`, Float64, nrhs = 1. + +| matrix | n | analysis cuDSS ms | SDS ms | ratio | factorization cuDSS ms | SDS ms | ratio | refactorization cuDSS ms | SDS ms | ratio | solve cuDSS ms | SDS ms | ratio | nnz(L) cuDSS | SDS | relres cuDSS | SDS | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| GHS_psdef/apache2 | 715176 | 7.71e+03 | | | 265 | | | 264 | | | 9.97 | | | 1.45e+08 | | 1.79e-10 | | +| kkt_pglib_opf_case1354_pegase_condensed_1 | 11192 | 99.2 | | | 1.16 | | | 0.781 | | | 0.327 | | | 1.55e+05 | | 0.000321 | | +| kkt_pglib_opf_case1354_pegase_condensed_10 | 11192 | 98.4 | | | 1.09 | | | 0.764 | | | 0.325 | | | 1.55e+05 | | 0.000146 | | +| kkt_pglib_opf_case1354_pegase_condensed_20 | 11192 | 98.2 | | | 1.09 | | | 0.764 | | | 0.327 | | | 1.55e+05 | | 7.71e-05 | | + +## Float32 factors, Float64 refinement + +`mixed_precision`, T27 (pending), structure `SPD`, Float64, nrhs = 1, no cuDSS counterpart. + +Not run yet. + +## Runs + +* cuDSS 0.8.0 on Quadro GV100, repository 8a452f0+dirty, 2026-10-08. +* SparseDirectSolver.jl 0.1.0 on Quadro GV100, repository 8a452f0+dirty, 2026-10-09. diff --git a/bench/comparison-gv100/comparison.png b/bench/comparison-gv100/comparison.png new file mode 100644 index 0000000..7f35d6c Binary files /dev/null and b/bench/comparison-gv100/comparison.png differ diff --git a/bench/comparison-gv100/cudss.csv b/bench/comparison-gv100/cudss.csv new file mode 100644 index 0000000..2993b45 --- /dev/null +++ b/bench/comparison-gv100/cudss.csv @@ -0,0 +1,127 @@ +solver,feature,task,matrix,structure,n,nnz,T,nrhs,nbatch,analysis_s,factorization_s,refactorization_s,solve_s,analysis_min_s,factorization_min_s,refactorization_min_s,solve_min_s,lu_nnz,flops,nsuperpanels,relres,samples,status,device,version,git_sha,date +cudss,cholesky_f64,T13,Boeing/bcsstk38,SPD,8032,355460,Float64,1,1,0.064999539,0.003095827,0.002807449,0.00064201,0.060300336,0.003094385,0.002787795,0.000633514,798965,130393272,0,2.1362732900373895e-10,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_f64,T13,GHS_psdef/apache2,SPD,715176,4817870,Float64,1,1,7.705200361,0.265192644,0.264030786,0.009940697,7.673937595,0.26461381,0.26359667,0.009909215,144624555,228522934388,21,1.789585234492284e-10,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_f64,T13,HB/bcsstk17,SPD,10974,428650,Float64,1,1,0.073375949,0.004270767,0.00290683,0.0006698,0.072978439,0.003191313,0.002881145,0.00066438,1077096,167140496,0,3.9062363897516044e-11,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_f64,T13,kkt_pglib_opf_case118_ieee_condensed_1,SPD,1088,12860,Float64,1,1,0.019904901,0.000847561,0.000315462,0.000227018,0.019889125,0.000752629,0.00031421,0.000224813,15200,272110,0,3.6400229766023885e-5,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_f64,T13,kkt_pglib_opf_case118_ieee_condensed_10,SPD,1088,12860,Float64,1,1,0.019932614,0.003169784,0.000314018,0.000225447,0.019878686,0.000619376,0.000311993,0.000222115,15200,272110,0,1.3801629270858276e-5,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_f64,T13,kkt_pglib_opf_case118_ieee_condensed_20,SPD,1088,12860,Float64,1,1,0.019918299,0.000622902,0.000315915,0.000225592,0.019908202,0.000622268,0.000311162,0.000221053,15200,272110,0,0.046572725699223064,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_f64,T13,kkt_pglib_opf_case1354_pegase_condensed_1,SPD,11192,136724,Float64,1,1,0.098072346,0.003308438,0.000780033,0.000332307,0.097833601,0.001068959,0.000759791,0.000324678,155201,3329818,0,0.0003122503179414002,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_f64,T13,kkt_pglib_opf_case1354_pegase_condensed_10,SPD,11192,136724,Float64,1,1,0.098123155,0.003255224,0.000777904,0.000328261,0.097911642,0.001098421,0.000770667,0.000326027,155201,3329818,0,0.0001440999030191872,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_f64,T13,kkt_pglib_opf_case1354_pegase_condensed_20,SPD,11192,136724,Float64,1,1,0.098373033,0.001108749,0.000785104,0.000323303,0.09768608,0.001083165,0.000775855,0.000321844,155201,3329818,0,7.716585113021588e-5,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_f64,T13,kkt_pglib_opf_case14_ieee_condensed_1,SPD,118,1282,Float64,1,1,0.010794459,0.000183804,0.000166006,0.000138962,0.010768329,0.00018147,0.000163696,0.000137338,1268,15255,0,1.133774974192117e-6,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_f64,T13,kkt_pglib_opf_case14_ieee_condensed_10,SPD,118,1282,Float64,1,1,0.010771854,0.000185644,0.000165182,0.000137817,0.01076058,0.000182147,0.000162779,0.000136982,1268,15255,0,4.2683144076881134e-5,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_f64,T13,kkt_pglib_opf_case14_ieee_condensed_11,SPD,118,1282,Float64,1,1,0.010788836,0.000185848,0.000167302,0.000138632,0.010760669,0.000184022,0.000163737,0.000136028,1268,15255,0,9.603204227640593e-6,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_f64,T13,kkt_pglib_opf_case78484_epigrids_condensed_1,SPD,674562,7936414,Float64,1,1,6.569747828,0.024928917,0.024497581,0.003794454,6.565949293,0.024630689,0.024240592,0.003779946,15554630,1469673073,0,0.15320816427651932,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_f64,T13,kkt_pglib_opf_case78484_epigrids_condensed_10,SPD,674562,7936414,Float64,1,1,6.559852601,0.024785275,0.024442903,0.00378581,6.555690484,0.024720824,0.024313386,0.003769443,15554630,1469673073,0,0.017640397463792364,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_f64,T13,lap2d_300,SPD,90000,448800,Float64,1,1,0.511977805,0.00605892,0.005527636,0.001135328,0.509539828,0.006048013,0.005490201,0.001119964,2428175,317918802,0,1.839041722707578e-12,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_f64,T13,lap3d_40,SPD,64000,438400,Float64,1,1,0.636986724,0.044330413,0.043235135,0.002436321,0.633926916,0.043726974,0.042923833,0.002420955,17527585,21925367162,2,8.214178435163057e-14,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_nrhs16,T12,Boeing/bcsstk38,SPD,8032,355460,Float64,16,1,0.061471931,0.003120425,0.002814637,0.000888397,0.060393866,0.003088085,0.00279852,0.000881736,798965,130393272,0,7.312915754528645e-10,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_nrhs16,T12,GHS_psdef/apache2,SPD,715176,4817870,Float64,16,1,7.67057382,0.265487784,0.264479484,0.043925428,7.66344769,0.264683488,0.264052015,0.043869501,144624555,228522934388,21,1.820498473327344e-10,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_nrhs16,T12,HB/bcsstk17,SPD,10974,428650,Float64,16,1,0.074142441,0.003214796,0.002909549,0.000967388,0.073110239,0.003196917,0.002897053,0.000955089,1077096,167140496,0,4.569264917777994e-11,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_nrhs16,T12,kkt_pglib_opf_case118_ieee_condensed_1,SPD,1088,12860,Float64,16,1,0.01993137,0.000858345,0.00031932,0.000308104,0.019910232,0.000621672,0.00031272,0.000307734,15200,272110,0,5.338209377985221e-5,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_nrhs16,T12,kkt_pglib_opf_case118_ieee_condensed_10,SPD,1088,12860,Float64,16,1,0.019927306,0.000616389,0.000321717,0.000306928,0.019903724,0.000611295,0.000314366,0.00030421,15200,272110,0,1.476213682565357e-5,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_nrhs16,T12,kkt_pglib_opf_case118_ieee_condensed_20,SPD,1088,12860,Float64,16,1,0.019901973,0.000624219,0.000315621,0.000306222,0.019889022,0.000615057,0.000314325,0.000305106,15200,272110,0,0.10751442137043135,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_nrhs16,T12,kkt_pglib_opf_case1354_pegase_condensed_1,SPD,11192,136724,Float64,16,1,0.100619743,0.001093081,0.000770569,0.000612108,0.097689822,0.001075405,0.000765337,0.000610837,155201,3329818,0,0.00039227116120409367,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_nrhs16,T12,kkt_pglib_opf_case1354_pegase_condensed_10,SPD,11192,136724,Float64,16,1,0.097932691,0.001090326,0.000773761,0.000608779,0.097679818,0.001075651,0.000764398,0.000606592,155201,3329818,0,0.00015201008697926846,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_nrhs16,T12,kkt_pglib_opf_case1354_pegase_condensed_20,SPD,11192,136724,Float64,16,1,0.102889969,0.001094201,0.000766814,0.000608712,0.0977185,0.001087189,0.000763271,0.000606791,155201,3329818,0,0.00010068328246352301,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_nrhs16,T12,kkt_pglib_opf_case14_ieee_condensed_1,SPD,118,1282,Float64,16,1,0.010788424,0.0001869,0.000165172,0.000160293,0.010772978,0.000182064,0.000164049,0.000158274,1268,15255,0,1.1788676976102841e-6,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_nrhs16,T12,kkt_pglib_opf_case14_ieee_condensed_10,SPD,118,1282,Float64,16,1,0.010773573,0.000187376,0.000167218,0.000164322,0.010766672,0.000184082,0.000164678,0.000156629,1268,15255,0,6.74639749002417e-5,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_nrhs16,T12,kkt_pglib_opf_case14_ieee_condensed_11,SPD,118,1282,Float64,16,1,0.010786874,0.00018738,0.000169023,0.000157363,0.010764581,0.000186726,0.000163493,0.000156317,1268,15255,0,4.361938442869111e-5,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_nrhs16,T12,kkt_pglib_opf_case78484_epigrids_condensed_1,SPD,674562,7936414,Float64,16,1,6.558993518,0.024933914,0.024452937,0.021738587,6.54706538,0.024719955,0.024360924,0.021731193,15554630,1469673073,0,0.18491721316818224,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_nrhs16,T12,kkt_pglib_opf_case78484_epigrids_condensed_10,SPD,674562,7936414,Float64,16,1,6.551954923,0.025160527,0.024389361,0.021696459,6.537958221,0.024883277,0.02410197,0.021678022,15554630,1469673073,0,0.017803190298986197,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_nrhs16,T12,lap2d_300,SPD,90000,448800,Float64,16,1,0.50953286,0.006082697,0.005508531,0.004233276,0.506456792,0.006021627,0.005491258,0.004203431,2428175,317918802,0,1.85099289228328e-12,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,cholesky_nrhs16,T12,lap3d_40,SPD,64000,438400,Float64,16,1,0.633487459,0.043724999,0.042879773,0.00553785,0.632878803,0.043327293,0.04264924,0.005532498,17527585,21925367162,2,8.923873831750635e-14,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,Boeing/bcsstk38,S,8032,355460,Float64,1,1,0.067995294,0.005386153,0.003722227,0.000674731,0.064795458,0.004046682,0.003712027,0.000665692,798965,130393272,0,7.095110795999743e-11,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,GHS_psdef/apache2,S,715176,4817870,Float64,1,1,7.692427036,0.31421505,0.313168655,0.013242784,7.680199359,0.313370765,0.312672296,0.010266615,144624555,228522934388,21,1.5714827763179924e-10,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,HB/bcsstk17,S,10974,428650,Float64,1,1,0.075980347,0.004098096,0.003800979,0.000702457,0.073191007,0.004068608,0.003782377,0.000698348,1077096,167140496,0,3.620145186669755e-11,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,kkt_pglib_opf_case118_ieee_condensed_1,S,1088,12860,Float64,1,1,0.02479056,0.00075765,0.000476361,0.000341503,0.024642442,0.000749136,0.000474447,0.000330093,15200,272110,0,5.347143297202352e-5,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,kkt_pglib_opf_case118_ieee_condensed_10,S,1088,12860,Float64,1,1,0.024710709,0.001247059,0.00048427,0.00033407,0.024145238,0.000725976,0.000472481,0.000330291,15200,272110,0,1.1804295252269577e-5,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,kkt_pglib_opf_case118_ieee_condensed_20,S,1088,12860,Float64,1,1,0.024779173,0.000724469,0.000474722,0.000336129,0.02306361,0.000722715,0.000466935,0.000231762,15200,272110,0,0.03525196872933484,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,kkt_pglib_opf_case118_ieee_k2_1,S,3185,17819,Float64,1,1,0.027865382,0.000900682,0.000366523,0.000221214,0.027854128,0.000674385,0.000356181,0.000218452,20022,183597,0,3.8030590409483858,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,kkt_pglib_opf_case118_ieee_k2_10,S,3185,17819,Float64,1,1,0.027855486,0.00067458,0.000353115,0.000221696,0.027819393,0.000663412,0.000352056,0.000220132,20022,183597,0,7.079756082666193,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,kkt_pglib_opf_case118_ieee_k2_20,S,3185,17819,Float64,1,1,0.02787731,0.002115359,0.000355938,0.000222703,0.027834165,0.000673622,0.000350903,0.000217551,20022,183597,0,12.602316227857361,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,kkt_pglib_opf_case1354_pegase_condensed_1,S,11192,136724,Float64,1,1,0.100938476,0.001257747,0.000956269,0.000334675,0.097583314,0.001236441,0.000948125,0.000332555,155201,3329818,0,0.0002908822632750786,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,kkt_pglib_opf_case1354_pegase_condensed_10,S,11192,136724,Float64,1,1,0.101767095,0.001263516,0.000950561,0.000333983,0.098002716,0.001252254,0.000941244,0.000332073,155201,3329818,0,0.00014598259519248754,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,kkt_pglib_opf_case1354_pegase_condensed_20,S,11192,136724,Float64,1,1,0.099662938,0.00126018,0.000935404,0.000336735,0.09758536,0.00124212,0.000932609,0.000333946,155201,3329818,0,6.734010129294929e-5,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,kkt_pglib_opf_case1354_pegase_k2_1,S,33811,188063,Float64,1,1,0.193403739,0.001993517,0.001373378,0.000443645,0.190886708,0.001681208,0.00135643,0.000441605,202300,2326076,0,91.66111975922111,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,kkt_pglib_opf_case1354_pegase_k2_10,S,33811,188063,Float64,1,1,0.194554961,0.001657702,0.001367976,0.000442099,0.19058838,0.001646434,0.001362801,0.00043945,202300,2326076,0,56.25745695508454,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,kkt_pglib_opf_case1354_pegase_k2_20,S,33811,188063,Float64,1,1,0.192233958,0.001697756,0.001366222,0.000544606,0.190243067,0.001637264,0.001360275,0.000539555,202300,2326076,0,1161.712231353449,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,kkt_pglib_opf_case14_ieee_condensed_1,S,118,1282,Float64,1,1,0.013826236,0.000283835,0.000288991,0.000247964,0.013613704,0.000281858,0.000283299,0.000243473,1268,15255,0,1.0681731870758567e-6,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,kkt_pglib_opf_case14_ieee_condensed_10,S,118,1282,Float64,1,1,0.013782103,0.000290472,0.000287486,0.00024617,0.013587431,0.00028591,0.000281524,0.000235839,1268,15255,0,3.422759649059638e-5,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,kkt_pglib_opf_case14_ieee_condensed_11,S,118,1282,Float64,1,1,0.013801446,0.000295628,0.00028326,0.000250089,0.011939036,0.000282852,0.000174698,0.000138706,1268,15255,0,1.0395022961616498e-5,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,kkt_pglib_opf_case14_ieee_k2_1,S,347,1933,Float64,1,1,0.015522355,0.000312568,0.000301807,0.000269411,0.01521209,0.000294962,0.000196666,0.000253857,2048,15401,0,0.5183719709317326,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,kkt_pglib_opf_case14_ieee_k2_10,S,347,1933,Float64,1,1,0.015506586,0.000196536,0.000177053,0.000154012,0.011969701,0.000191117,0.000172493,0.000147806,2048,15401,0,0.40930955551753356,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,kkt_pglib_opf_case14_ieee_k2_15,S,347,1933,Float64,1,1,0.01196723,0.000197224,0.000185496,0.000151948,0.011942183,0.000193018,0.000172107,0.000146943,2048,15401,0,0.6984149551898986,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,kkt_pglib_opf_case78484_epigrids_condensed_1,S,674562,7936414,Float64,1,1,6.550120728,0.030794914,0.030339761,0.004102392,6.535125794,0.030600378,0.030214068,0.004092383,15554630,1469673073,0,0.1653027084607183,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,kkt_pglib_opf_case78484_epigrids_condensed_10,S,674562,7936414,Float64,1,1,6.587338911,0.039189939,0.029786881,0.00409503,6.567422007,0.030298178,0.029730191,0.004092355,15554630,1469673073,0,0.016786953777290756,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,kkt_pglib_opf_case78484_epigrids_k2_1,S,2091669,11770277,Float64,1,1,20.135416745,0.048056047,0.047432841,0.013292728,20.103790517,0.048037854,0.047253105,0.013288609,18487523,1126504689,0,6.460082247725613e9,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,kkt_pglib_opf_case78484_epigrids_k2_10,S,2091669,11770277,Float64,1,1,20.189487168,0.049858881,0.046670979,0.013296709,20.186852302,0.047257879,0.046403743,0.013294756,18487523,1126504689,0,25462.219071956744,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,lap2d_300,S,90000,448800,Float64,1,1,0.477512827,0.007379456,0.007018509,0.001207784,0.476539939,0.007308426,0.006984016,0.001204902,2428175,317918802,0,1.61303491560536e-12,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt,T15,lap3d_40,S,64000,438400,Float64,1,1,0.610084058,0.052980959,0.052927336,0.00252809,0.608093738,0.052851339,0.052624788,0.002514351,17527585,21925367162,2,7.584958771133913e-14,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt_ir2,T16,kkt_pglib_opf_case118_ieee_k2_1,S,3185,17819,Float64,1,1,0.02892164,0.000663892,0.000357033,0.000488512,0.028393736,0.000641686,0.000356167,0.000487101,20022,183597,0,0.00046392459990464896,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt_ir2,T16,kkt_pglib_opf_case118_ieee_k2_10,S,3185,17819,Float64,1,1,0.027843207,0.000670253,0.000358952,0.000487193,0.027802062,0.000656622,0.000355342,0.000484993,20022,183597,0,0.22758605104464297,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt_ir2,T16,kkt_pglib_opf_case118_ieee_k2_20,S,3185,17819,Float64,1,1,0.027867682,0.000676097,0.000361574,0.000486177,0.027823705,0.000672555,0.000356135,0.000485622,20022,183597,0,0.09677373860807566,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt_ir2,T16,kkt_pglib_opf_case1354_pegase_k2_1,S,33811,188063,Float64,1,1,0.194632649,0.001672484,0.001372266,0.001270318,0.190566428,0.001657897,0.0013662,0.001263929,202300,2326076,0,35.33858037725026,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt_ir2,T16,kkt_pglib_opf_case1354_pegase_k2_10,S,33811,188063,Float64,1,1,0.191298433,0.001673817,0.001364573,0.0012561,0.190354016,0.001656279,0.001357053,0.001251077,202300,2326076,0,60.3616177747509,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt_ir2,T16,kkt_pglib_opf_case1354_pegase_k2_20,S,33811,188063,Float64,1,1,0.192917399,0.00220178,0.001367437,0.001269921,0.191081623,0.001654622,0.001364258,0.00126502,202300,2326076,0,102.1933447147118,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt_ir2,T16,kkt_pglib_opf_case14_ieee_k2_1,S,347,1933,Float64,1,1,0.011993739,0.000199956,0.000187901,0.000295168,0.011981606,0.000198231,0.000179672,0.000293734,2048,15401,0,3.3489990651009444e-5,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt_ir2,T16,kkt_pglib_opf_case14_ieee_k2_10,S,347,1933,Float64,1,1,0.011971948,0.000199032,0.000191736,0.000295773,0.011962934,0.000192538,0.000186454,0.000292191,2048,15401,0,5.930303501549897e-7,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt_ir2,T16,kkt_pglib_opf_case14_ieee_k2_15,S,347,1933,Float64,1,1,0.012023509,0.000196718,0.00017989,0.000591859,0.011975607,0.000189583,0.000178893,0.000478884,2048,15401,0,2.345177591123125e-7,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt_ir2,T16,kkt_pglib_opf_case78484_epigrids_k2_1,S,2091669,11770277,Float64,1,1,20.112399214,0.047975381,0.047230189,0.046148024,20.109384425,0.047880775,0.047080481,0.046146146,18487523,1126504689,0,6.007876121713949e6,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt_ir2,T16,kkt_pglib_opf_case78484_epigrids_k2_10,S,2091669,11770277,Float64,1,1,20.139223728,0.047387988,0.0464877,0.046171126,20.114626884,0.047177315,0.046399778,0.046168266,18487523,1126504689,0,624982.1624599457,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ubatch8,T17,kkt_pglib_opf_case118_ieee_condensed_1,SPD,1088,12860,Float64,1,8,0.01989601,0.000966453,0.000626797,0.000265194,0.019887087,0.000958631,0.000623008,0.000261771,15200,272110,0,1.8363235438901045e-5,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ubatch8,T17,kkt_pglib_opf_case118_ieee_condensed_10,SPD,1088,12860,Float64,1,8,0.019884612,0.000951745,0.000625773,0.000264797,0.019877792,0.000949169,0.000621443,0.000262925,15200,272110,0,1.345081993104875e-5,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ubatch8,T17,kkt_pglib_opf_case118_ieee_condensed_20,SPD,1088,12860,Float64,1,8,0.019900921,0.000962175,0.000626513,0.000264073,0.01986564,0.000952103,0.000618663,0.000262085,15200,272110,0,0.14564745236862903,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ubatch8,T17,kkt_pglib_opf_case1354_pegase_condensed_1,SPD,11192,136724,Float64,1,8,0.097907974,0.00313261,0.002819404,0.000644275,0.097687329,0.00312355,0.0028141,0.000637277,155201,3329818,0,0.0003172817104599787,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ubatch8,T17,kkt_pglib_opf_case1354_pegase_condensed_10,SPD,11192,136724,Float64,1,8,0.098787161,0.003273355,0.002818335,0.000639203,0.097733001,0.003131657,0.002810796,0.000637315,155201,3329818,0,0.00015393530689036456,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ubatch8,T17,kkt_pglib_opf_case1354_pegase_condensed_20,SPD,11192,136724,Float64,1,8,0.099987148,0.003124558,0.002816451,0.000642467,0.098133461,0.003116961,0.002811553,0.000641675,155201,3329818,0,7.501207161680047e-5,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ubatch8,T17,kkt_pglib_opf_case14_ieee_condensed_1,SPD,118,1282,Float64,1,8,0.010763196,0.000483853,0.00017245,0.000131254,0.01075772,0.00047716,0.000171641,0.000129438,1268,15255,0,3.2410069963424963e-7,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ubatch8,T17,kkt_pglib_opf_case14_ieee_condensed_10,SPD,118,1282,Float64,1,8,0.010771724,0.000483832,0.000170503,0.000131414,0.010754394,0.000479808,0.000169235,0.000127559,1268,15255,0,2.890238760292203e-5,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ubatch8,T17,kkt_pglib_opf_case14_ieee_condensed_11,SPD,118,1282,Float64,1,8,0.010771224,0.000486736,0.000172864,0.000131314,0.010763194,0.000475516,0.000170353,0.000127937,1268,15255,0,8.59108212268016e-6,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ubatch8,T17,kkt_pglib_opf_case78484_epigrids_condensed_1,SPD,674562,7936414,Float64,1,8,6.600159711,0.169619573,0.168990309,0.02516346,6.577470383,0.169318322,0.168702357,0.025155759,15554630,1469673073,0,0.1730107744294277,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ubatch8,T17,kkt_pglib_opf_case78484_epigrids_condensed_10,SPD,674562,7936414,Float64,1,8,6.58190498,0.16963485,0.16897053,0.025189291,6.572253252,0.169469144,0.168636538,0.025155273,15554630,1469673073,0,0.017620693567056124,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,lu,T19,Rajat/rajat21,G,411676,1893370,Float64,1,1,3.760955242,0.042227432,0.051466691,0.004321971,3.756541239,0.04217017,0.041566336,0.004305047,6891584,5212013513,2,3.3446160745542027e7,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,lu,T19,TSOPF/TSOPF_RS_b39_c7,G,14098,252446,Float64,1,1,0.198757584,0.002992305,0.002706635,0.000516478,0.189276376,0.002973669,0.002336257,0.000419731,1470000,95459011,0,9.065082662757548e29,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,schur,T20,kkt_pglib_opf_case118_ieee_condensed_1,SPD,1088,12860,Float64,1,1,0.024913144,0.000709016,0.000335253,0.000141573,0.024693926,0.000643579,0.000333355,0.000140826,19631,622719,0,,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,schur,T20,kkt_pglib_opf_case118_ieee_condensed_10,SPD,1088,12860,Float64,1,1,0.019667036,0.000646568,0.00034047,0.000142735,0.019644995,0.000642655,0.000335594,0.000138995,19631,622719,0,,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,schur,T20,kkt_pglib_opf_case118_ieee_condensed_20,SPD,1088,12860,Float64,1,1,0.019662181,0.003375067,0.000353236,0.000158735,0.019656524,0.000644608,0.000351723,0.00015267,19631,622719,0,,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,schur,T20,kkt_pglib_opf_case1354_pegase_condensed_1,SPD,11192,136724,Float64,1,1,0.102914783,0.001184699,0.000856357,0.000199825,0.099328185,0.001157263,0.000845528,0.00019652,165293,4462458,0,,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,schur,T20,kkt_pglib_opf_case1354_pegase_condensed_10,SPD,11192,136724,Float64,1,1,0.099912438,0.001176289,0.000851303,0.000213028,0.098696906,0.001161892,0.000843421,0.000195846,165293,4462458,0,,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,schur,T20,kkt_pglib_opf_case1354_pegase_condensed_20,SPD,11192,136724,Float64,1,1,0.103438608,0.0011666,0.000872899,0.000198979,0.099176884,0.001149781,0.000856134,0.000197089,165293,4462458,0,,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,schur,T20,kkt_pglib_opf_case14_ieee_condensed_1,SPD,118,1282,Float64,1,1,0.010824409,0.000217768,0.000196828,0.000115117,0.010806719,0.000213545,0.000195777,0.000113825,1511,23110,0,,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,schur,T20,kkt_pglib_opf_case14_ieee_condensed_10,SPD,118,1282,Float64,1,1,0.010804046,0.000222327,0.000200725,0.000117017,0.01079681,0.000214837,0.000197695,0.000116594,1511,23110,0,,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,schur,T20,kkt_pglib_opf_case14_ieee_condensed_11,SPD,118,1282,Float64,1,1,0.010808502,0.000221754,0.000195238,0.000116053,0.010794239,0.000218073,0.000193025,0.000112981,1511,23110,0,,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,schur,T20,kkt_pglib_opf_case78484_epigrids_condensed_1,SPD,674562,7936414,Float64,1,1,6.654767636,0.02672269,0.026034983,0.00188739,6.635976284,0.02647081,0.025874026,0.00188514,15684550,1535218182,0,,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,schur,T20,kkt_pglib_opf_case78484_epigrids_condensed_10,SPD,674562,7936414,Float64,1,1,6.663699089,0.026782814,0.026240783,0.001894198,6.631643407,0.026410566,0.025951568,0.001879394,15684550,1535218182,0,,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt_matching,T21,kkt_pglib_opf_case118_ieee_k2_1,S,3185,17819,Float64,1,1,0.039344498,0.006893134,0.000840739,0.000279313,0.039311132,0.001144329,0.000830219,0.000277414,74055,1719497,0,3.096359832996682e-12,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt_matching,T21,kkt_pglib_opf_case118_ieee_k2_10,S,3185,17819,Float64,1,1,0.039281211,0.001119014,0.000776291,0.00025324,0.039235731,0.001110362,0.000764843,0.000246123,68745,1375382,0,7.32438723711953e-13,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt_matching,T21,kkt_pglib_opf_case118_ieee_k2_20,S,3185,17819,Float64,1,1,0.039489489,0.002156693,0.000809461,0.000261249,0.039464889,0.001109838,0.000787427,0.000254414,69327,1420392,0,5.118284964993496e-13,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt_matching,T21,kkt_pglib_opf_case1354_pegase_k2_1,S,33811,188063,Float64,1,1,0.25479207,0.003750261,0.003102918,0.00052364,0.252158143,0.003388753,0.003075973,0.000519532,722739,19670743,0,0.004465636024104995,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt_matching,T21,kkt_pglib_opf_case1354_pegase_k2_10,S,33811,188063,Float64,1,1,0.259436602,0.003337769,0.003026416,0.000516658,0.257835716,0.003320784,0.002997937,0.000501249,721043,19593864,0,0.009017295843210123,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt_matching,T21,kkt_pglib_opf_case1354_pegase_k2_20,S,33811,188063,Float64,1,1,0.259625817,0.003490517,0.003169599,0.000526145,0.258757625,0.003459986,0.003166825,0.00052344,722809,20002915,0,0.014734104642288831,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt_matching,T21,kkt_pglib_opf_case14_ieee_k2_1,S,347,1933,Float64,1,1,0.013598215,0.000329659,0.000312903,0.000170033,0.013589951,0.000328392,0.00031078,0.000168061,7113,128434,0,7.947478337577263e-13,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt_matching,T21,kkt_pglib_opf_case14_ieee_k2_10,S,347,1933,Float64,1,1,0.01313347,0.0002954,0.000283216,0.000158968,0.013097347,0.000294308,0.000280314,0.000155135,6759,109990,0,5.727190044672952e-14,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt_matching,T21,kkt_pglib_opf_case14_ieee_k2_15,S,347,1933,Float64,1,1,0.01312555,0.000300924,0.000285693,0.000157005,0.013120504,0.00029578,0.000274053,0.000154251,6759,109990,0,9.600360225395852e-14,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt_matching,T21,kkt_pglib_opf_case78484_epigrids_k2_1,S,2091669,11770277,Float64,1,1,24.283621775,0.130159927,0.129033433,0.014387223,24.200518312,0.129948858,0.12865281,0.014365654,74458257,9755130703,1,1.0648194923708136e8,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,ldlt_matching,T21,kkt_pglib_opf_case78484_epigrids_k2_10,S,2091669,11770277,Float64,1,1,24.755046464,0.132382525,0.131667264,0.01440876,24.732709336,0.131917925,0.131388521,0.014407203,74099251,9149931556,0,6.030894130405046e9,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,nubatch,T22,batch_of_11,SPD,1386318,16325426,Float64,1,11,14.308207497,0.0531000865,0.050884753,0.011942352,14.275163242,0.051818723,0.050730738,0.011607893,31610573,2961811566,0,0.1397725825222732,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,solve_algo1,T25,Boeing/bcsstk38,SPD,8032,355460,Float64,1,1,0.061369036,0.005154184,0.002812466,0.000647379,0.060370511,0.003116153,0.002788745,0.000638874,798965,130393272,0,2.5262402558995575e-10,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,solve_algo1,T25,GHS_psdef/apache2,SPD,715176,4817870,Float64,1,1,7.662890782,0.264344909,0.263808368,0.009972529,7.657023408,0.264003001,0.263565996,0.009948966,144624555,228522934388,21,1.7911624245464745e-10,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,solve_algo1,T25,HB/bcsstk17,SPD,10974,428650,Float64,1,1,0.077095733,0.003223938,0.002902235,0.000670563,0.074189462,0.003207079,0.002883373,0.000668283,1077096,167140496,0,4.072206182927878e-11,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,solve_algo1,T25,kkt_pglib_opf_case118_ieee_condensed_1,SPD,1088,12860,Float64,1,1,0.019915174,0.000635043,0.000328476,0.000244901,0.019885786,0.000632642,0.000322932,0.00023533,15200,272110,0,6.49190082013908e-5,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,solve_algo1,T25,kkt_pglib_opf_case118_ieee_condensed_10,SPD,1088,12860,Float64,1,1,0.019905916,0.000660427,0.00031832,0.000232963,0.019879124,0.000637467,0.000315341,0.000229648,15200,272110,0,1.0586332080992323e-5,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,solve_algo1,T25,kkt_pglib_opf_case118_ieee_condensed_20,SPD,1088,12860,Float64,1,1,0.019919045,0.000646282,0.000315667,0.000224978,0.019878974,0.000627826,0.000310148,0.000224074,15200,272110,0,0.08364877259972306,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,solve_algo1,T25,kkt_pglib_opf_case1354_pegase_condensed_1,SPD,11192,136724,Float64,1,1,0.100059918,0.001101694,0.000784128,0.000344643,0.098062309,0.001093538,0.00077387,0.000340673,155201,3329818,0,0.00032647171843520325,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,solve_algo1,T25,kkt_pglib_opf_case1354_pegase_condensed_10,SPD,11192,136724,Float64,1,1,0.098022162,0.001098909,0.000777447,0.000324935,0.097509098,0.001088139,0.000765761,0.000323791,155201,3329818,0,0.00015155541972935533,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,solve_algo1,T25,kkt_pglib_opf_case1354_pegase_condensed_20,SPD,11192,136724,Float64,1,1,0.098210503,0.001869596,0.000763548,0.000325882,0.097553836,0.001099985,0.000758762,0.000323197,155201,3329818,0,8.09332660617376e-5,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,solve_algo1,T25,kkt_pglib_opf_case14_ieee_condensed_1,SPD,118,1282,Float64,1,1,0.010782465,0.000180521,0.000160768,0.000137462,0.010780482,0.000177534,0.000157681,0.000134985,1268,15255,0,2.5523739978163924e-7,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,solve_algo1,T25,kkt_pglib_opf_case14_ieee_condensed_10,SPD,118,1282,Float64,1,1,0.010781446,0.000181174,0.000160358,0.000135785,0.010775508,0.00018058,0.000157039,0.000132384,1268,15255,0,3.970028097796526e-5,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,solve_algo1,T25,kkt_pglib_opf_case14_ieee_condensed_11,SPD,118,1282,Float64,1,1,0.010781611,0.000182748,0.000159871,0.000135357,0.010762812,0.000179281,0.000158657,0.000132794,1268,15255,0,2.2408207312837266e-5,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,solve_algo1,T25,kkt_pglib_opf_case78484_epigrids_condensed_1,SPD,674562,7936414,Float64,1,1,6.548531987,0.025056876,0.024397298,0.003797007,6.534366244,0.025007475,0.024270737,0.003788212,15554630,1469673073,0,0.1550372951473611,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,solve_algo1,T25,kkt_pglib_opf_case78484_epigrids_condensed_10,SPD,674562,7936414,Float64,1,1,6.555541464,0.025006189,0.024390393,0.00380228,6.521930772,0.024826131,0.024320309,0.003797189,15554630,1469673073,0,0.01784798114335486,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,solve_algo1,T25,lap2d_300,SPD,90000,448800,Float64,1,1,0.478682047,0.005828295,0.005501706,0.001127642,0.474946472,0.005793938,0.005483949,0.001122702,2428175,317918802,0,1.8409761962962026e-12,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,solve_algo1,T25,lap3d_40,SPD,64000,438400,Float64,1,1,0.612230893,0.046349087,0.042943001,0.002405895,0.607710771,0.043108715,0.042598433,0.002384604,17527585,21925367162,2,8.212102600018684e-14,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,hybrid_memory,T26,GHS_psdef/apache2,SPD,715176,4817870,Float64,1,1,7.70506488,0.265084578,0.264339981,0.009967425,7.667524954,0.264209574,0.263693662,0.009895246,144624555,228522934388,21,1.7927381362612363e-10,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,hybrid_memory,T26,kkt_pglib_opf_case1354_pegase_condensed_1,SPD,11192,136724,Float64,1,1,0.099167279,0.001155923,0.000780751,0.000327217,0.098157455,0.00112852,0.000775967,0.000324759,155201,3329818,0,0.0003211398966657904,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,hybrid_memory,T26,kkt_pglib_opf_case1354_pegase_condensed_10,SPD,11192,136724,Float64,1,1,0.098429994,0.001094082,0.000764456,0.000324657,0.098037716,0.001075773,0.000762489,0.000321517,155201,3329818,0,0.00014588378153839013,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 +cudss,hybrid_memory,T26,kkt_pglib_opf_case1354_pegase_condensed_20,SPD,11192,136724,Float64,1,1,0.098248812,0.001091936,0.000763968,0.000326925,0.097969211,0.001062511,0.000757538,0.000324255,155201,3329818,0,7.711183317052271e-5,5,ok,Quadro GV100,0.8.0,8a452f0+dirty,2026-10-08 diff --git a/bench/comparison-gv100/sds.csv b/bench/comparison-gv100/sds.csv new file mode 100644 index 0000000..32e137f --- /dev/null +++ b/bench/comparison-gv100/sds.csv @@ -0,0 +1,106 @@ +solver,feature,task,matrix,structure,n,nnz,T,nrhs,nbatch,analysis_s,factorization_s,refactorization_s,solve_s,analysis_min_s,factorization_min_s,refactorization_min_s,solve_min_s,lu_nnz,flops,nsuperpanels,relres,samples,status,device,version,git_sha,date +sds,cholesky_f64,T13,Boeing/bcsstk38,SPD,8032,355460,Float64,1,1,0.120552143,0.027352636,0.027368117,0.002670297,0.120377661,0.02732593,0.027353798,0.002665236,804613,1.40793119e8,604,9.256449560312438e-11,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_f64,T13,GHS_psdef/apache2,SPD,715176,4817870,Float64,1,1,13.182377305,0.762535062,0.751876987,0.0868015235,13.177971507,0.750580889,0.751450807,0.086605455,129200941,1.66491403722e11,123689,1.5722430494202702e-10,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_f64,T13,HB/bcsstk17,SPD,10974,428650,Float64,1,1,0.150839113,0.03069238,0.030671192,0.004945988,0.150087154,0.030682034,0.030656582,0.004936868,1043601,1.6795197e8,1300,3.879533234176818e-11,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_f64,T13,kkt_pglib_opf_case118_ieee_condensed_1,SPD,1088,12860,Float64,1,1,0.010606828,0.001156095,0.001143098,0.001279769,0.010569375,0.001150362,0.001136232,0.001198555,11634,219042.0,303,5.222025477933645e-5,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_f64,T13,kkt_pglib_opf_case118_ieee_condensed_10,SPD,1088,12860,Float64,1,1,0.01060236,0.001146235,0.001130388,0.001350593,0.010583828,0.001139243,0.001125229,0.001247264,11634,219042.0,303,1.1226840171824022e-5,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_f64,T13,kkt_pglib_opf_case118_ieee_condensed_20,SPD,1088,12860,Float64,1,1,0.010806016,0.00113675,0.001127652,0.001267871,0.010793754,0.001134056,0.001121926,0.001236722,11634,219042.0,303,0.04550257439586628,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_f64,T13,kkt_pglib_opf_case1354_pegase_condensed_1,SPD,11192,136724,Float64,1,1,0.121499416,0.003891132,0.003860202,0.0017117,0.121257144,0.003874857,0.003853298,0.001646812,120598,2.787739e6,3134,0.00027950947524883757,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_f64,T13,kkt_pglib_opf_case1354_pegase_condensed_10,SPD,11192,136724,Float64,1,1,0.121058902,0.003870154,0.003834331,0.001625535,0.120834869,0.003860384,0.00383257,0.001586506,120598,2.787739e6,3134,9.670379220068758e-5,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_f64,T13,kkt_pglib_opf_case1354_pegase_condensed_20,SPD,11192,136724,Float64,1,1,0.121779,0.003852464,0.003841462,0.001761983,0.120841167,0.003847468,0.003830444,0.001591176,120598,2.787739e6,3134,5.0320897327844414e-5,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_f64,T13,kkt_pglib_opf_case14_ieee_condensed_1,SPD,118,1282,Float64,1,1,0.001371058,0.000409174,0.000414341,0.000650187,0.001366171,0.000402977,0.000397114,0.000626933,1027,15193.0,33,1.3024460439372e-6,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_f64,T13,kkt_pglib_opf_case14_ieee_condensed_10,SPD,118,1282,Float64,1,1,0.001403775,0.000414988,0.000399576,0.000664279,0.001375632,0.00040834,0.000383617,0.00065606,1027,15193.0,33,7.849096497706581e-5,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_f64,T13,kkt_pglib_opf_case14_ieee_condensed_11,SPD,118,1282,Float64,1,1,0.001392112,0.000401451,0.000384428,0.000660541,0.001385388,0.000379083,0.00036729,0.000640335,1027,15193.0,33,9.566132794313988e-6,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_f64,T13,kkt_pglib_opf_case78484_epigrids_condensed_1,SPD,674562,7936414,Float64,1,1,11.39592042,0.167181721,0.167000827,0.018278896,11.379577478,0.16708902,0.166939561,0.018175981,13229878,1.460147624e9,92878,0.1408833643782678,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_f64,T13,kkt_pglib_opf_case78484_epigrids_condensed_10,SPD,674562,7936414,Float64,1,1,11.561199556,0.166808538,0.166588513,0.018236268,11.493055784,0.166705359,0.166530652,0.018148033,13229878,1.460147624e9,92878,0.014721495527528015,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_f64,T13,lap2d_300,SPD,90000,448800,Float64,1,1,0.786551701,0.033677439,0.033642062,0.005122703,0.782576703,0.033660191,0.033608611,0.005105254,2465905,3.73611053e8,20495,1.7636637752817264e-12,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_f64,T13,lap3d_40,SPD,64000,438400,Float64,1,1,0.904454186,0.302910217,0.302992142,0.017475988,0.903884596,0.302857127,0.302913604,0.017440669,14387160,1.6401480134e10,11066,7.331775892090179e-14,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_nrhs16,T12,Boeing/bcsstk38,SPD,8032,355460,Float64,16,1,0.120165413,0.027372073,0.027362566,0.003225435,0.119892154,0.027348462,0.027358886,0.003207286,804613,1.40793119e8,604,1.7152744268072052e-10,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_nrhs16,T12,GHS_psdef/apache2,SPD,715176,4817870,Float64,16,1,13.106805662,0.750245755,0.750782279,0.1757323445,13.030285961,0.749551966,0.750308348,0.17567332,129200941,1.66491403722e11,123689,1.5328573062454928e-10,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_nrhs16,T12,HB/bcsstk17,SPD,10974,428650,Float64,16,1,0.150627255,0.030749535,0.030692929,0.005856249,0.149754479,0.030719154,0.030656329,0.005852641,1043601,1.6795197e8,1300,5.569372145755773e-11,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_nrhs16,T12,kkt_pglib_opf_case118_ieee_condensed_1,SPD,1088,12860,Float64,16,1,0.010618159,0.001161781,0.001143779,0.001362515,0.010570918,0.001154351,0.001142774,0.001330248,11634,219042.0,303,0.00010755450830763632,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_nrhs16,T12,kkt_pglib_opf_case118_ieee_condensed_10,SPD,1088,12860,Float64,16,1,0.010589889,0.001141448,0.001130113,0.001384478,0.01057038,0.001135931,0.001125284,0.001315917,11634,219042.0,303,1.686504924342648e-5,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_nrhs16,T12,kkt_pglib_opf_case118_ieee_condensed_20,SPD,1088,12860,Float64,16,1,0.010714106,0.001146878,0.001130957,0.001369345,0.010709696,0.001142023,0.001125448,0.001280475,11634,219042.0,303,0.08353519383870289,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_nrhs16,T12,kkt_pglib_opf_case1354_pegase_condensed_1,SPD,11192,136724,Float64,16,1,0.120869361,0.003899001,0.00386239,0.001885509,0.120707173,0.003890023,0.003858145,0.001884123,120598,2.787739e6,3134,0.000290839088756989,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_nrhs16,T12,kkt_pglib_opf_case1354_pegase_condensed_10,SPD,11192,136724,Float64,16,1,0.120946362,0.003881252,0.003839912,0.001888608,0.120659073,0.003876482,0.003828834,0.001885673,120598,2.787739e6,3134,0.00011395735370074328,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_nrhs16,T12,kkt_pglib_opf_case1354_pegase_condensed_20,SPD,11192,136724,Float64,16,1,0.124067041,0.003869651,0.003838629,0.001895083,0.120721603,0.003865359,0.003829064,0.001893634,120598,2.787739e6,3134,6.469766480530265e-5,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_nrhs16,T12,kkt_pglib_opf_case14_ieee_condensed_1,SPD,118,1282,Float64,16,1,0.001376015,0.000418178,0.000416953,0.000708094,0.001349841,0.000387303,0.000410475,0.000659111,1027,15193.0,33,1.4669401778060025e-6,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_nrhs16,T12,kkt_pglib_opf_case14_ieee_condensed_10,SPD,118,1282,Float64,16,1,0.00138583,0.000419815,0.000419469,0.000706337,0.001380185,0.000400278,0.000413485,0.000672063,1027,15193.0,33,0.0001433207307265809,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_nrhs16,T12,kkt_pglib_opf_case14_ieee_condensed_11,SPD,118,1282,Float64,16,1,0.001458948,0.000407039,0.000403371,0.000711212,0.00138564,0.000402325,0.00038875,0.000672666,1027,15193.0,33,2.287471322954381e-5,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_nrhs16,T12,kkt_pglib_opf_case78484_epigrids_condensed_1,SPD,674562,7936414,Float64,16,1,11.892419375,0.167209264,0.167160728,0.051685954,11.733826689,0.167140135,0.167045172,0.051639447,13229878,1.460147624e9,92878,0.17490474662872965,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_nrhs16,T12,kkt_pglib_opf_case78484_epigrids_condensed_10,SPD,674562,7936414,Float64,16,1,11.698498546,0.166664854,0.166806857,0.05166284,11.66376224,0.166658107,0.166643254,0.051558498,13229878,1.460147624e9,92878,0.014838990900026515,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_nrhs16,T12,lap2d_300,SPD,90000,448800,Float64,16,1,0.787440591,0.033648477,0.033636843,0.009992964,0.782363099,0.033614452,0.033627881,0.009985229,2465905,3.73611053e8,20495,1.7551365679010265e-12,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,cholesky_nrhs16,T12,lap3d_40,SPD,64000,438400,Float64,16,1,0.903641034,0.302850684,0.302790275,0.022929803,0.900641179,0.302766952,0.302723608,0.022915374,14387160,1.6401480134e10,11066,6.393126409263393e-14,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,Boeing/bcsstk38,S,8032,355460,Float64,1,1,0.139457403,0.047137904,0.047139377,0.002793786,0.138141835,0.047120499,0.047119263,0.002790434,804613,1.40793119e8,604,1.7711758486535893e-10,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,GHS_psdef/apache2,S,715176,4817870,Float64,1,1,13.327237068,1.881395037,1.872439185,0.108333181,13.190786188,1.870593545,1.871694284,0.1077258,129200941,1.66491403722e11,123689,1.4199000161870498e-10,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,HB/bcsstk17,S,10974,428650,Float64,1,1,0.168992591,0.056597524,0.056540523,0.005181688,0.167982066,0.056578628,0.056522948,0.005169287,1043601,1.6795197e8,1300,4.1559816576559667e-11,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,kkt_pglib_opf_case118_ieee_condensed_1,S,1088,12860,Float64,1,1,0.011421366,0.002674556,0.002652751,0.001280267,0.01138793,0.002661307,0.002647014,0.001204064,11634,219042.0,303,0.00011067414872254996,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,kkt_pglib_opf_case118_ieee_condensed_10,S,1088,12860,Float64,1,1,0.011423965,0.002596075,0.002564408,0.001368159,0.011420599,0.002586818,0.002561249,0.001289981,11634,219042.0,303,1.0098509199371433e-5,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,kkt_pglib_opf_case118_ieee_condensed_20,S,1088,12860,Float64,1,1,0.011412615,0.002670909,0.002640223,0.001301021,0.011384109,0.002663594,0.002633847,0.001232991,11634,219042.0,303,0.03653673168373129,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,kkt_pglib_opf_case118_ieee_k2_1,S,3185,17819,Float64,1,1,0.133685016,0.00266032,0.002630395,0.001578266,0.133536942,0.002644116,0.002621213,0.001468907,21110,309387.0,1949,1.0497379713880406,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,kkt_pglib_opf_case118_ieee_k2_10,S,3185,17819,Float64,1,1,0.137365589,0.002592005,0.002564113,0.00161094,0.136904767,0.00258784,0.002553468,0.001504504,20620,293225.0,1933,0.007367428249487967,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,kkt_pglib_opf_case118_ieee_k2_20,S,3185,17819,Float64,1,1,0.13411635,0.002556855,0.002526008,0.001561675,0.133741871,0.00255262,0.002517668,0.001532335,20585,291971.0,1921,0.02377509361760076,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,kkt_pglib_opf_case1354_pegase_condensed_1,S,11192,136724,Float64,1,1,0.127104329,0.010269646,0.010228912,0.001709178,0.126999441,0.010254498,0.010222402,0.001652775,120598,2.787739e6,3134,0.00031260415821256834,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,kkt_pglib_opf_case1354_pegase_condensed_10,S,11192,136724,Float64,1,1,0.128279613,0.010400772,0.010359449,0.001773729,0.128012578,0.010398937,0.010350482,0.001699094,120598,2.787739e6,3134,0.00010962331529376899,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,kkt_pglib_opf_case1354_pegase_condensed_20,S,11192,136724,Float64,1,1,0.128346992,0.010427674,0.010389245,0.001728075,0.127559529,0.010418853,0.010372195,0.001645416,120598,2.787739e6,3134,5.61855767764813e-5,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,kkt_pglib_opf_case1354_pegase_k2_1,S,33811,188063,Float64,1,1,2.300845163,0.010397631,0.010301982,0.002352618,2.271566288,0.010327289,0.010282605,0.002269306,215953,3.404739e6,19624,16.86505375658475,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,kkt_pglib_opf_case1354_pegase_k2_10,S,33811,188063,Float64,1,1,2.369604452,0.009474101,0.009444524,0.002189766,2.362094908,0.009421863,0.009430605,0.001950363,220146,3.494638e6,19496,29.592568263683376,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,kkt_pglib_opf_case1354_pegase_k2_20,S,33811,188063,Float64,1,1,2.386248679,0.010141572,0.010061586,0.002216623,2.380943219,0.010110682,0.010036939,0.002104045,221265,3.589937e6,19586,27.464961108569845,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,kkt_pglib_opf_case14_ieee_condensed_1,S,118,1282,Float64,1,1,0.001531721,0.000631503,0.00062755,0.000696176,0.001519781,0.000626687,0.000618583,0.000670879,1027,15193.0,33,8.584792622513327e-7,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,kkt_pglib_opf_case14_ieee_condensed_10,S,118,1282,Float64,1,1,0.001514594,0.000637155,0.000631808,0.000692245,0.001508239,0.000629796,0.000626565,0.000666878,1027,15193.0,33,1.8227592930364502e-5,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,kkt_pglib_opf_case14_ieee_condensed_11,S,118,1282,Float64,1,1,0.001514208,0.000641183,0.000637945,0.000669331,0.001503039,0.000637011,0.000634373,0.000628424,1027,15193.0,33,7.285134631994358e-6,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,kkt_pglib_opf_case14_ieee_k2_1,S,347,1933,Float64,1,1,0.007260417,0.000956653,0.000959676,0.001031235,0.00716654,0.000952119,0.000948519,0.000960213,2147,26857.0,191,7.56501219709915e-8,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,kkt_pglib_opf_case14_ieee_k2_10,S,347,1933,Float64,1,1,0.007231382,0.000965322,0.000952053,0.000976629,0.007172865,0.000956793,0.000939512,0.00091597,2162,27300.0,190,4.358854013999979e-5,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,kkt_pglib_opf_case14_ieee_k2_15,S,347,1933,Float64,1,1,0.005622011,0.001581089,0.00157055,0.001126502,0.005548268,0.001575894,0.001562471,0.001062511,3228,55307.0,93,0.12291892791595775,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,kkt_pglib_opf_case78484_epigrids_condensed_1,S,674562,7936414,Float64,1,1,12.145061869,0.534500019,0.534510309,0.020120094,12.051618741,0.534268912,0.534088957,0.019999071,13229878,1.460147624e9,92878,0.11446907429950352,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,kkt_pglib_opf_case78484_epigrids_condensed_10,S,674562,7936414,Float64,1,1,11.354121783,0.509691604,0.509470361,0.020163244,11.25643297,0.509224644,0.508852129,0.020118776,13229878,1.460147624e9,92878,0.01211048724463874,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,kkt_pglib_opf_case78484_epigrids_k2_1,S,2091669,11770277,Float64,1,1,544.184425544,2.291313762,2.262621671,0.081347085,544.184425544,2.291313762,2.262621671,0.081347085,27679254,3.483284013e9,1065590,19024.91260700019,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,kkt_pglib_opf_case78484_epigrids_k2_10,S,2091669,11770277,Float64,1,1,568.522920264,2.245916307,2.227702107,0.078953366,568.522920264,2.245916307,2.227702107,0.078953366,26981678,3.321955977e9,1072441,8076.652504677691,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,lap2d_300,S,90000,448800,Float64,1,1,0.80661722,0.049958339,0.049892496,0.005941042,0.798079976,0.049859193,0.049857334,0.005927889,2465905,3.73611053e8,20495,1.5831401088970504e-12,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt,T15,lap3d_40,S,64000,438400,Float64,1,1,0.925215039,0.514851877,0.514736208,0.023231701,0.920230895,0.514688701,0.514690942,0.023095141,14387160,1.6401480134e10,11066,6.268221293995084e-14,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt_ir2,T16,kkt_pglib_opf_case118_ieee_k2_1,S,3185,17819,Float64,1,1,0.132679524,0.002628153,0.002593405,0.007544311,0.132625342,0.002616599,0.002586111,0.007301088,21110,309387.0,1949,1.723074193830837e-8,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt_ir2,T16,kkt_pglib_opf_case118_ieee_k2_10,S,3185,17819,Float64,1,1,0.137503414,0.002554394,0.002524221,0.007400224,0.135772307,0.002550376,0.002515114,0.007196744,20620,293225.0,1933,6.7640064199324e-10,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt_ir2,T16,kkt_pglib_opf_case118_ieee_k2_20,S,3185,17819,Float64,1,1,0.132286468,0.002520172,0.002481883,0.007605299,0.132224939,0.002519115,0.002474888,0.007442212,20585,291971.0,1921,1.2947291252892672e-9,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt_ir2,T16,kkt_pglib_opf_case1354_pegase_k2_1,S,33811,188063,Float64,1,1,2.217442237,0.010325673,0.010275411,0.043186849,2.216077417,0.010299914,0.010265082,0.042635247,215953,3.404739e6,19624,0.017750159885650485,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt_ir2,T16,kkt_pglib_opf_case1354_pegase_k2_10,S,33811,188063,Float64,1,1,2.345685202,0.009508835,0.009410134,0.042545104,2.325695495,0.009462429,0.009398681,0.041881617,220146,3.494638e6,19496,0.6570472496657765,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt_ir2,T16,kkt_pglib_opf_case1354_pegase_k2_20,S,33811,188063,Float64,1,1,2.355560707,0.010064501,0.010030353,0.044761543,2.339760716,0.010059975,0.009999677,0.042352261,221265,3.589937e6,19586,0.06828417335155713,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt_ir2,T16,kkt_pglib_opf_case14_ieee_k2_1,S,347,1933,Float64,1,1,0.006885796,0.000933718,0.000920403,0.003208788,0.006850033,0.000928613,0.000916952,0.002877138,2147,26857.0,191,5.982293844014453e-13,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt_ir2,T16,kkt_pglib_opf_case14_ieee_k2_10,S,347,1933,Float64,1,1,0.007224579,0.000943056,0.000925405,0.003232148,0.006991115,0.000941702,0.000920709,0.003150655,2162,27300.0,190,3.1228109103936516e-14,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt_ir2,T16,kkt_pglib_opf_case14_ieee_k2_15,S,347,1933,Float64,1,1,0.005472133,0.001564496,0.001532793,0.003583936,0.005470145,0.001556246,0.0015256,0.003416448,3228,55307.0,93,1.7748050109314768e-9,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt_ir2,T16,kkt_pglib_opf_case78484_epigrids_k2_1,S,2091669,11770277,Float64,1,1,492.543342475,2.288080771,2.266092572,6.179852167,492.543342475,2.288080771,2.266092572,6.179852167,27679254,3.483284013e9,1065590,413210.59402041807,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt_ir2,T16,kkt_pglib_opf_case78484_epigrids_k2_10,S,2091669,11770277,Float64,1,1,560.895980505,2.257807617,2.233202878,6.055435734,560.895980505,2.257807617,2.233202878,6.055435734,26981678,3.321955977e9,1072441,198838.2851063324,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ubatch8,T17,kkt_pglib_opf_case118_ieee_condensed_1,SPD,1088,12860,Float64,1,8,0.010716307,0.001166713,0.001138332,0.001222882,0.010696689,0.00116102,0.001133994,0.001187076,11634,219042.0,303,0.00010104375340815254,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ubatch8,T17,kkt_pglib_opf_case118_ieee_condensed_10,SPD,1088,12860,Float64,1,8,0.010698793,0.001146504,0.00111887,0.001265265,0.010679513,0.001145161,0.001117142,0.001166747,11634,219042.0,303,1.0349938409324512e-5,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ubatch8,T17,kkt_pglib_opf_case118_ieee_condensed_20,SPD,1088,12860,Float64,1,8,0.01068926,0.001147693,0.001127314,0.001300959,0.010674193,0.001141381,0.001109532,0.001194165,11634,219042.0,303,0.049543485447857034,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ubatch8,T17,kkt_pglib_opf_case1354_pegase_condensed_1,SPD,11192,136724,Float64,1,8,0.121963246,0.0048112,0.004763052,0.001756736,0.121538005,0.004796032,0.004756466,0.001713231,120598,2.787739e6,3134,0.0002731547182023334,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ubatch8,T17,kkt_pglib_opf_case1354_pegase_condensed_10,SPD,11192,136724,Float64,1,8,0.120205725,0.004735406,0.004699771,0.001738534,0.120189917,0.004723772,0.004688649,0.00171778,120598,2.787739e6,3134,0.00010385635197390319,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ubatch8,T17,kkt_pglib_opf_case1354_pegase_condensed_20,SPD,11192,136724,Float64,1,8,0.121103624,0.004736976,0.00470779,0.001722145,0.120870034,0.004725836,0.004691405,0.001702469,120598,2.787739e6,3134,5.426740935200775e-5,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ubatch8,T17,kkt_pglib_opf_case14_ieee_condensed_1,SPD,118,1282,Float64,1,8,0.001370376,0.000399645,0.000364177,0.000627066,0.001355017,0.000382537,0.000352477,0.000592023,1027,15193.0,33,9.02775987371821e-7,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ubatch8,T17,kkt_pglib_opf_case14_ieee_condensed_10,SPD,118,1282,Float64,1,8,0.00137079,0.000391975,0.000397682,0.000633156,0.001356803,0.000361839,0.000341036,0.000623391,1027,15193.0,33,3.967768497890147e-5,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ubatch8,T17,kkt_pglib_opf_case14_ieee_condensed_11,SPD,118,1282,Float64,1,8,0.001442404,0.000375451,0.000371205,0.000642504,0.001400478,0.000352169,0.000350836,0.000602103,1027,15193.0,33,7.957223897687373e-6,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ubatch8,T17,kkt_pglib_opf_case78484_epigrids_condensed_1,SPD,674562,7936414,Float64,1,8,10.564337059,0.282249885,0.28221379,0.066586488,10.474850407,0.282239832,0.282126906,0.06654996,13229878,1.460147624e9,92878,0.1347119787622275,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ubatch8,T17,kkt_pglib_opf_case78484_epigrids_condensed_10,SPD,674562,7936414,Float64,1,8,10.632434292,0.279080556,0.279054825,0.066697878,10.385429998,0.278939789,0.278737639,0.066527344,13229878,1.460147624e9,92878,0.014872275311897479,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,lu,T19,Rajat/rajat21,G,411676,1893370,Float64,1,1,4.860309426,0.645647935,0.645649068,0.03265029,4.83578451,0.645467356,0.645074514,0.032612655,2368670,3.30845977e8,293600,2897.502934900568,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,lu,T19,TSOPF/TSOPF_RS_b39_c7,G,14098,252446,Float64,1,1,0.223003767,0.028148503,0.028094018,0.008016142,0.222681504,0.028136391,0.028082958,0.007706749,521509,2.960869e7,1463,20.31996632124525,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,schur,T20,kkt_pglib_opf_case118_ieee_condensed_1,SPD,1088,12860,Float64,1,1,0.011604986,0.00156573,0.001498207,0.000754075,0.010698601,0.001507169,0.001490625,0.000725448,18509,856762.0,286,,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,schur,T20,kkt_pglib_opf_case118_ieee_condensed_10,SPD,1088,12860,Float64,1,1,0.011826993,0.001579681,0.00151887,0.000734836,0.010924512,0.001489812,0.001506284,0.000642146,18509,856762.0,286,,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,schur,T20,kkt_pglib_opf_case118_ieee_condensed_20,SPD,1088,12860,Float64,1,1,0.011747004,0.001497446,0.001478163,0.000675327,0.011252922,0.001487535,0.001476533,0.000632345,18509,856762.0,286,,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,schur,T20,kkt_pglib_opf_case1354_pegase_condensed_1,SPD,11192,136724,Float64,1,1,0.129011332,0.004857331,0.004832757,0.001007342,0.12876364,0.004850741,0.004829855,0.000914209,130659,3.880025e6,3038,,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,schur,T20,kkt_pglib_opf_case1354_pegase_condensed_10,SPD,11192,136724,Float64,1,1,0.128199323,0.004834117,0.004805071,0.000972504,0.128010646,0.004830641,0.004795837,0.000932209,130659,3.880025e6,3038,,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,schur,T20,kkt_pglib_opf_case1354_pegase_condensed_20,SPD,11192,136724,Float64,1,1,0.128033181,0.004844143,0.004802303,0.001009712,0.127918414,0.004818055,0.004799913,0.00094076,130659,3.880025e6,3038,,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,schur,T20,kkt_pglib_opf_case14_ieee_condensed_1,SPD,118,1282,Float64,1,1,0.001614949,0.000503574,0.000464102,0.000405355,0.001611919,0.000494729,0.000432404,0.000383044,1295,26710.0,41,,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,schur,T20,kkt_pglib_opf_case14_ieee_condensed_10,SPD,118,1282,Float64,1,1,0.001960494,0.000691014,0.00053343,0.000441382,0.001729352,0.000603785,0.000488116,0.000384475,1295,26710.0,41,,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,schur,T20,kkt_pglib_opf_case14_ieee_condensed_11,SPD,118,1282,Float64,1,1,0.002011787,0.000544964,0.000507617,0.000435057,0.001684258,0.000500984,0.000483771,0.000388337,1295,26710.0,41,,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,schur,T20,kkt_pglib_opf_case78484_epigrids_condensed_1,SPD,674562,7936414,Float64,1,1,12.410262259,0.166091555,0.165967174,0.009708475,11.539297014,0.16602199,0.165819877,0.009605366,13412152,1.574393366e9,92880,,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,schur,T20,kkt_pglib_opf_case78484_epigrids_condensed_10,SPD,674562,7936414,Float64,1,1,11.905122141,0.165490362,0.165426999,0.009590388,11.05688113,0.165409286,0.165333068,0.009542288,13412152,1.574393366e9,92880,,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt_matching,T21,kkt_pglib_opf_case118_ieee_k2_1,S,3185,17819,Float64,1,1,0.029965945,0.003377787,0.003359909,0.001807678,0.029688657,0.003371207,0.00333607,0.001794968,36051,804423.0,967,4.3870200453287496e-12,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt_matching,T21,kkt_pglib_opf_case118_ieee_k2_10,S,3185,17819,Float64,1,1,0.037866617,0.00315721,0.003075062,0.001655907,0.035025453,0.003123342,0.003066808,0.001605482,34745,736481.0,977,7.390913948749555e-13,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt_matching,T21,kkt_pglib_opf_case118_ieee_k2_20,S,3185,17819,Float64,1,1,0.031306893,0.003519754,0.003477257,0.001760432,0.031115294,0.00350784,0.003470681,0.001684608,34416,716465.0,958,4.079528236177513e-13,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt_matching,T21,kkt_pglib_opf_case1354_pegase_k2_1,S,33811,188063,Float64,1,1,0.349380411,0.016041513,0.015998054,0.002529232,0.348727966,0.016020456,0.015986305,0.002490666,368439,9.653775e6,10107,0.0003105473373811897,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt_matching,T21,kkt_pglib_opf_case1354_pegase_k2_10,S,33811,188063,Float64,1,1,0.37401418,0.014774064,0.0147168,0.002481575,0.372274754,0.014746512,0.014702448,0.002389407,365264,9.232339e6,10005,0.0005767412534663728,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt_matching,T21,kkt_pglib_opf_case1354_pegase_k2_20,S,33811,188063,Float64,1,1,0.418816669,0.014762478,0.014709024,0.002255933,0.397828064,0.014728176,0.014665113,0.00214779,363249,9.027806e6,9985,0.0003938848608988164,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt_matching,T21,kkt_pglib_opf_case14_ieee_k2_1,S,347,1933,Float64,1,1,0.004134506,0.001229769,0.00118775,0.000832032,0.003800447,0.001162479,0.001160925,0.000808524,3640,70473.0,94,1.5812450595789091e-12,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt_matching,T21,kkt_pglib_opf_case14_ieee_k2_10,S,347,1933,Float64,1,1,0.003948099,0.001044611,0.001021806,0.00081142,0.003627007,0.001025478,0.001011042,0.000690192,3538,67232.0,95,7.092497394976035e-14,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt_matching,T21,kkt_pglib_opf_case14_ieee_k2_15,S,347,1933,Float64,1,1,0.004066277,0.001061147,0.001055969,0.00085672,0.003727402,0.001053497,0.001049534,0.000743509,3528,65385.0,94,4.867689420023049e-14,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt_matching,T21,kkt_pglib_opf_case78484_epigrids_k2_1,S,2091669,11770277,Float64,1,1,37.085829853,0.714066126,0.684838357,0.0430870005,36.854035493,0.707652829,0.684438613,0.042809795,40788177,5.555490355e9,528760,0.0006120507791675268,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 +sds,ldlt_matching,T21,kkt_pglib_opf_case78484_epigrids_k2_10,S,2091669,11770277,Float64,1,1,42.7278848305,0.759721849,0.726015239,0.0499032055,42.391122382,0.754187903,0.725944949,0.049645292,41287746,5.828056215e9,526392,0.00020452564982090558,5,ok,Quadro GV100,0.1.0,8a452f0+dirty,2026-10-09 diff --git a/bench/e2e/MadNLPSDS.jl b/bench/e2e/MadNLPSDS.jl deleted file mode 100644 index 71746f3..0000000 --- a/bench/e2e/MadNLPSDS.jl +++ /dev/null @@ -1,179 +0,0 @@ -# MadNLP linear-solver interface to SparseDirectSolver.jl (prototype). -# Backend-agnostic: accepts any device CSC (CUSPARSE.CuSparseMatrixCSC, -# rocSPARSE.ROCSparseMatrixCSC) whose colPtr/rowVal/nzVal are device vectors. -# The CSC arrays of MadNLP's lower-triangular KKT matrix are read as the CSR -# of the upper triangle, exactly as MadNLPGPU's CUDSSSolver does (view 'U'). -module MadNLPSDS - -using LinearAlgebra, SparseArrays -using MadNLP -using SparseDirectSolver -const SDS = SparseDirectSolver -const KA = SDS.KernelAbstractions - -include("SDSProto.jl") -using .SDSProto - -Base.@kwdef mutable struct SDSSolverOptions <: MadNLP.AbstractOptions - sds_reordering::String = "algo4" # native ND (see PR #107: the auto chooser picks AMD here) - sds_regime_c_width::Int = 32 - sds_regime_c_rows::Int = 256 - sds_subtree_parallelism::Int = 16384 - sds_subtree_budgets::Vector{Int} = [16384] -end - -mutable struct SDSSolver{T} <: MadNLP.AbstractLinearSolver{T} - inner::Any # SDS.DirectSolver (type elided: backend-parametric) - tril::Any # the device CSC whose nzVal MadNLP updates in place - x::Any - b::Any - factorized_once::Bool - last_info::Int - opt::SDSSolverOptions - logger::MadNLP.MadNLPLogger -end - -function SDSSolver( - csc; - opt = SDSSolverOptions(), - logger = MadNLP.MadNLPLogger(), - ) - T = eltype(csc.nzVal) - n = size(csc, 1) - s = SDS.DirectSolver(csc.colPtr, csc.rowVal, csc.nzVal, "SPD", 'U') - s.options.regime_c_width = opt.sds_regime_c_width - s.options.regime_c_rows = opt.sds_regime_c_rows - s.options.subtree_parallelism = opt.sds_subtree_parallelism - s.options.subtree_budgets = copy(opt.sds_subtree_budgets) - SDS.setparam!(s, "reordering_alg", opt.sds_reordering) - x = similar(csc.nzVal, n) - b = similar(csc.nzVal, n) - SDS.execute!("analysis", s, x, b; asynchronous = false) - return SDSSolver{T}(s, csc, x, b, false, 0, opt, logger) -end - -function MadNLP.factorize!(M::SDSSolver) - M.inner.A.nzval === M.tril.nzVal || - SDS.update!(M.inner, M.tril.colPtr, M.tril.rowVal, M.tril.nzVal) - phase = M.factorized_once ? "refactorization" : "factorization" - SDS.execute!(phase, M.inner, M.x, M.b; asynchronous = false) - M.factorized_once = true - M.last_info = Int(SDS.getparam(M.inner, "info")) - return M -end - -function MadNLP.solve_linear_system!(M::SDSSolver{T}, xb) where {T} - copyto!(M.b, xb) - SDS.execute!("solve", M.inner, M.x, M.b; asynchronous = false) - copyto!(xb, M.x) - return xb -end - -# older MadNLP spellings route to the same implementation -MadNLP.solve!(M::SDSSolver, xb::AbstractVector) = MadNLP.solve_linear_system!(M, xb) - -MadNLP.input_type(::Type{SDSSolver}) = :csc -MadNLP.default_options(::Type{SDSSolver}) = SDSSolverOptions() -MadNLP.is_inertia(::SDSSolver) = true -function MadNLP.inertia(M::SDSSolver) - n = size(M.tril, 1) - return M.last_info == 0 ? (n, 0, 0) : (n - 2, 1, 1) -end -MadNLP.improve!(::SDSSolver) = false -MadNLP.introduce(::SDSSolver) = "SparseDirectSolver.jl" -MadNLP.is_supported(::Type{SDSSolver}, ::Type{Float32}) = true -MadNLP.is_supported(::Type{SDSSolver}, ::Type{Float64}) = true - -# --------------------------------------------------------------------------- -# prototype-kernel wrapper: fused factorization + algo1/fused solve (PR #107) -mutable struct SDSProtoSolver{T} <: MadNLP.AbstractLinearSolver{T} - inner::Any - tril::Any - x::Any - b::Any - nzval::Any - fp::Any # fused factorization plan - fz::Any # fused solve segments - inv11::Any - inv_ptr::Any - invert!::Any - last_info::Int - opt::SDSSolverOptions - logger::MadNLP.MadNLPLogger -end - -function SDSProtoSolver( - csc; - opt = SDSSolverOptions(), - logger = MadNLP.MadNLPLogger(), - ) - T = eltype(csc.nzVal) - n = size(csc, 1) - backend = KA.get_backend(csc.nzVal) - SDSProto.BACKEND[] = backend - s = SDS.DirectSolver(csc.colPtr, csc.rowVal, csc.nzVal, "SPD", 'U') - s.options.regime_c_width = 32 - s.options.regime_c_rows = 64 # prototype kernels own BOTH phases: no coupling conflict - s.options.subtree_parallelism = 16384 - s.options.subtree_budgets = [16384] - SDS.setparam!(s, "reordering_alg", opt.sds_reordering) - x = similar(csc.nzVal, n) - b = similar(csc.nzVal, n) - SDS.execute!("analysis", s, x, b; asynchronous = false) - SDS.execute!("factorization", s, x, b; asynchronous = false) - SDS.execute!("solve", s, x, b; asynchronous = false) # allocates the solve workspace - nzval = SDS._factor_values(s) - fp = SDSProto.build_fused_plan(s) - inv11, inv_ptr, invertf, _ = SDSProto.build_inverse(s) - # solve-side plan state (module globals, single solver instance) - plan0 = s.workspace.plan - nodes0 = s.symbolic.schedule.group_nodes - stp = Array(s.symbolic.subtree_ptr) - sub_ids = Int32[] - for k in eachindex(plan0.kind) - plan0.kind[k] == SDS.SOLVE_SUBTREES || continue - append!(sub_ids, Int32.(nodes0[plan0.first[k]:plan0.last[k]])) - end - sort!(sub_ids; by = e -> stp[e + 1] - stp[e], rev = true) - SDSProto.SSUB[] = SDSProto.DVEC(sub_ids) - SDSProto.NSUB[] = length(sub_ids) - SDSProto.INV11[] = inv11 - SDSProto.INVPTR[] = inv_ptr - fz1 = SDSProto.build_fused(s; kstart = 1, wcap = 256) - segs = [(fz1, 64)] - fz = (segs, true) - M = SDSProtoSolver{T}(s, csc, x, b, nzval, fp, fz, inv11, inv_ptr, invertf, 0, opt, logger) - MadNLP.factorize!(M) # prime the fused path once - return M -end - -function MadNLP.factorize!(M::SDSProtoSolver) - M.last_info = Int(SDSProto.refact_fused!(M.inner, M.nzval, M.fp)) - M.invert!() - return M -end - -function MadNLP.solve_linear_system!(M::SDSProtoSolver{T}, xb) where {T} - copyto!(M.b, xb) - SDSProto.algo2_solve!(M.x, M.inner, M.b, M.inv11, M.inv_ptr, M.fz) - copyto!(xb, M.x) - KA.synchronize(KA.get_backend(xb)) - return xb -end - -MadNLP.solve!(M::SDSProtoSolver, xb::AbstractVector) = MadNLP.solve_linear_system!(M, xb) -MadNLP.input_type(::Type{SDSProtoSolver}) = :csc -MadNLP.default_options(::Type{SDSProtoSolver}) = SDSSolverOptions() -MadNLP.is_inertia(::SDSProtoSolver) = true -function MadNLP.inertia(M::SDSProtoSolver) - n = size(M.tril, 1) - return M.last_info == 0 ? (n, 0, 0) : (n - 2, 1, 1) -end -MadNLP.improve!(::SDSProtoSolver) = false -MadNLP.introduce(::SDSProtoSolver) = "SparseDirectSolver.jl (PR #107 prototype kernels)" -MadNLP.is_supported(::Type{SDSProtoSolver}, ::Type{Float32}) = true -MadNLP.is_supported(::Type{SDSProtoSolver}, ::Type{Float64}) = true - -export SDSSolver, SDSSolverOptions, SDSProtoSolver - -end # module diff --git a/bench/e2e/SDSCPU.jl b/bench/e2e/SDSCPU.jl new file mode 100644 index 0000000..b02a508 --- /dev/null +++ b/bench/e2e/SDSCPU.jl @@ -0,0 +1,327 @@ +# CPU fast path for the refactorization (prototype, stock src untouched): the +# stock schedule's groups processed in order, with +# - subtree groups: one Julia task per subtree, the serial multifrontal stack +# in a per-task scratch (the regime-A algorithm in plain scalar Julia); +# - regime-B/C groups: the stock assembly kernels (KA CPU, internally +# threaded), then one task per front running LAPACK potrf + BLAS trsm/syrk +# on the dense panel views (BLAS pinned to one thread; task-level +# parallelism is ours). +# Cholesky ("SPD") first; the potrf-derived LDL conversion drops in later. +module SDSCPU + +using SparseDirectSolver +using LinearAlgebra +using Base.Threads +using Printf +const SDS = SparseDirectSolver + +_packed(i, j, n) = SDS._packed(i, j, n) + +# split a:b into at most n contiguous chunks +function _chunks(a::Int, b::Int, n::Int) + len = b - a + 1 + len <= 0 && return UnitRange{Int}[] + c = min(n, len) + sz = cld(len, c) + return [ (a + (i - 1) * sz):min(a + i * sz - 1, b) for i in 1:cld(len, sz) ] +end + +# serial regime-A subtree: the GPU kernel's algorithm on a plain scratch vector +function _subtree!(factor::Vector{T}, stack::Vector{T}, info, nzval, S, t, buf::Vector{T}) where {T} + stn = S.subtree_nodes + @inbounds for q in S.subtree_ptr[t]:(S.subtree_ptr[t + 1] - 1) + v = Int(stn[q]) + f = Int(S.front_nrows[v]) + w = Int(S.front_ncols[v]) + m = f - w + lf = Int(S.local_front[v]) - 1 + # zero + scatter A + fill!(view(buf, (lf + 1):(lf + f * (f + 1) ÷ 2)), zero(T)) + p0 = Int(S.front_ptr[v]) + for k in Int(S.amap_ptr[v]):(Int(S.amap_ptr[v + 1]) - 1) + p = Int(S.amap_src[k]) + off = S.amap[p] + loc = Int(abs(off)) - p0 + j = loc ÷ f + 1 + i = loc - (j - 1) * f + 1 + buf[lf + _packed(i, j, f)] += SDS._amap_value(T, nzval[p], off) + end + # extend-add the children (their packed CBs live lower in the scratch) + for kc in Int(S.child_ptr[v]):(Int(S.child_ptr[v + 1]) - 1) + c = Int(S.child_list[kc]) + mc = Int(S.front_nrows[c]) - Int(S.front_ncols[c]) + lc = Int(S.local_cb[c]) - 1 + r0 = Int(S.relind_ptr[c]) - 1 + for jj in 1:mc + rj = Int(S.relind[r0 + jj]) + for ii in jj:mc + ri = Int(S.relind[r0 + ii]) + buf[lf + _packed(ri, rj, f)] += buf[lc + _packed(ii, jj, mc)] + end + end + end + # right-looking Cholesky of the first w columns (whole packed front) + st = Int32(0) + for j in 1:w + dj = real(buf[lf + _packed(j, j, f)]) + if !(dj > 0) + st = Int32(j) + break + end + c = sqrt(dj) + buf[lf + _packed(j, j, f)] = c + rd = inv(c) + base = lf + _packed(j, j, f) # column j is contiguous below the diagonal + @simd for i in 1:(f - j) + buf[base + i] *= rd + end + for k in (j + 1):f + lkj = conj(buf[base + k - j]) + bk = lf + _packed(k, k, f) - k + @simd for i in k:f + buf[bk + i] -= buf[base + i - j] * lkj + end + end + end + # write the panel (zero strict upper), status, and the root CB to the stack + for j in 1:w, i in 1:f + factor[p0 + (j - 1) * f + i - 1] = i >= j ? buf[lf + _packed(i, j, f)] : zero(T) + end + st != 0 && (info[v] = st) + lc = Int(S.local_cb[v]) - 1 + src = lf + w * (2 * f - w + 1) ÷ 2 + if lc < 0 # subtree root: CB to the update stack + c0 = Int(S.cb_ptr[v]) + c0 > 0 && copyto!(stack, c0, buf, src + 1, m * (m + 1) ÷ 2) + else # move the CB down (src > dst: forward copy) + copyto!(buf, lc + 1, buf, src + 1, m * (m + 1) ÷ 2) + end + end + return nothing +end + +# extend-add of all children into parent columns rj ∈ cols (panel) and CB columns +# rj - w ∈ cols .- w; disjoint across column slices, so safe to run sliced in parallel +function _extend_cols!(factor::Vector{T}, stack::Vector{T}, S, v, f, w, m, p0, c0, cols) where {T} + @inbounds for kc in Int(S.child_ptr[v]):(Int(S.child_ptr[v + 1]) - 1) + c = Int(S.child_list[kc]) + cc = Int(S.cb_ptr[c]) + cc > 0 || continue + mc = Int(S.front_nrows[c]) - Int(S.front_ncols[c]) + r0 = Int(S.relind_ptr[c]) - 1 + for jj in 1:mc + rj = Int(S.relind[r0 + jj]) + rj in cols || continue + if rj <= w + b = p0 + (rj - 1) * f - 1 + for ii in jj:mc + factor[b + Int(S.relind[r0 + ii])] += stack[cc + _packed(ii, jj, mc) - 1] + end + else + bj = c0 + _packed(rj - w, rj - w, m) - 1 - (rj - w) + for ii in jj:mc + ri = Int(S.relind[r0 + ii]) + stack[bj + ri - w] += stack[cc + _packed(ii, jj, mc) - 1] + end + end + end + end + return nothing +end + +# one regime-B/C front: assembly (zero + scatter A + extend-add, serial and cache-local) +# then LAPACK potrf + BLAS trsm + syrk, CB packed into the stack +function _front_blas!(factor::Vector{T}, stack::Vector{T}, work::Vector{T}, info, S, nzval, v, + par::Bool = false) where {T} + f = Int(S.front_nrows[v]) + w = Int(S.front_ncols[v]) + m = f - w + p0 = Int(S.front_ptr[v]) + c0 = Int(S.cb_ptr[v]) + @inbounds begin + if par + @sync begin + for r in _chunks(1, f * w, 4 * nthreads()) + Threads.@spawn fill!(view(factor, (p0 + first(r) - 1):(p0 + last(r) - 1)), zero(T)) + end + c0 > 0 && for r in _chunks(1, m * (m + 1) ÷ 2, 2 * nthreads()) + Threads.@spawn fill!(view(stack, (c0 + first(r) - 1):(c0 + last(r) - 1)), zero(T)) + end + end + else + fill!(view(factor, p0:(p0 + f * w - 1)), zero(T)) + c0 > 0 && fill!(view(stack, c0:(c0 + m * (m + 1) ÷ 2 - 1)), zero(T)) + end + for k in Int(S.amap_ptr[v]):(Int(S.amap_ptr[v + 1]) - 1) + p = Int(S.amap_src[k]) + off = S.amap[p] + factor[Int(abs(off))] += SDS._amap_value(T, nzval[p], off) + end + if par + @sync for r in _chunks(1, f, 4 * nthreads()) + Threads.@spawn _extend_cols!(factor, stack, S, v, f, w, m, p0, c0, r) + end + else + _extend_cols!(factor, stack, S, v, f, w, m, p0, c0, 1:f) + end + end + P = reshape(view(factor, p0:(p0 + f * w - 1)), f, w) + F11 = view(P, 1:w, 1:w) + _, st = LAPACK.potrf!('L', F11) + if st != 0 + @inbounds info[v] = Int32(st) + return nothing + end + m > 0 || return nothing + F21 = view(P, (w + 1):f, 1:w) + BLAS.trsm!('R', 'L', 'C', 'N', one(T), F11, F21) + c0 > 0 || return nothing + C = reshape(view(work, 1:(m * m)), m, m) + T <: Real ? BLAS.syrk!('L', 'N', -one(T), F21, zero(T), C) : + BLAS.herk!('L', 'N', -one(real(T)), F21, zero(real(T)), C) + if par + @sync for r in _chunks(1, m, 4 * nthreads()) + Threads.@spawn @inbounds for jj in r + b = c0 + _packed(jj, jj, m) - 1 - jj + @simd for ii in jj:m + stack[b + ii] += C[ii, jj] + end + end + end + else + @inbounds for jj in 1:m + b = c0 + _packed(jj, jj, m) - 1 - jj + @simd for ii in jj:m + stack[b + ii] += C[ii, jj] + end + end + end + return nothing +end + +""" + CPUCtx(s) + +Persistent buffers for [`refact_cpu!`](@ref): per-task scratch and workspace +pools (task-safe channels; threadid-indexed buffers break under task +migration). +""" +struct CPUCtx{T} + spool::Channel{Vector{T}} + wpool::Channel{Vector{T}} +end + +function CPUCtx(s) + N = s.numeric + T = eltype(N.factor) + plan = N.plan + maxloc = isempty(plan.sub_local) ? 0 : maximum(plan.sub_local) + spool = Channel{Vector{T}}(nthreads()) + wpool = Channel{Vector{T}}(nthreads() + 1) # +1: refact_cpu! holds one for the + for _ in 1:nthreads() # serial levels while tasks draw the rest + put!(spool, Vector{T}(undef, cld(maxloc, sizeof(T)))) + put!(wpool, Vector{T}(undef, length(N.work))) + end + put!(wpool, Vector{T}(undef, length(N.work))) + return CPUCtx{T}(spool, wpool) +end + +""" + refact_cpu!(s, nzval, ctx = CPUCtx(s)) -> info + +Task-parallel CPU refactorization of a factorized "SPD" solver on the CPU +backend, writing the stock factor layout (panels + packed CB stack). Wide +levels run one task per chunk of fronts with BLAS pinned to one thread; thin +levels (the root chain) run serially under fully threaded BLAS. +""" +function refact_cpu!(s, nzval, ctx::CPUCtx = CPUCtx(s)) + N, S = s.numeric, s.symbolic + plan = N.plan + T = eltype(N.factor) + nblas = BLAS.get_num_threads() + nt = nthreads() + fill!(N.info, Int32(0)) + spool, wpool = ctx.spool, ctx.wpool + nch = 8 * nt # chunked tasks: spawn cost amortized + tsub = 0.0; tblas = 0.0 + wk0 = take!(wpool) # serial-level workspace + if nt == 1 # single-thread specialization: no tasks + buf0 = take!(spool) + try + BLAS.set_num_threads(1) + tsub = @elapsed for k in eachindex(plan.sub_first) + for q in plan.sub_first[k]:plan.sub_last[k] + _subtree!(N.factor, N.stack, N.info, nzval, S, Int(S.group_nodes[q]), buf0) + end + end + tblas = @elapsed for k in eachindex(plan.group_first) + for q in plan.group_first[k]:plan.group_last[k] + _front_blas!(N.factor, N.stack, wk0, N.info, S, nzval, Int(S.group_nodes[q])) + end + end + finally + put!(spool, buf0) + put!(wpool, wk0) + BLAS.set_num_threads(nblas) + end + haskey(ENV, "SDSCPU_STAGES") && + @printf(" stages: subtrees %7.1f ms fronts %7.1f ms\n", tsub * 1e3, tblas * 1e3) + SDS.assemble_schur!(N, S, nzval) + return maximum(Array(N.info)) == 0 ? 0 : 1 + end + try + BLAS.set_num_threads(1) + tsub = @elapsed for k in eachindex(plan.sub_first) + a, b = plan.sub_first[k], plan.sub_last[k] + @sync for r in _chunks(a, b, nch) + Threads.@spawn begin + buf = take!(spool) + try + for q in r + _subtree!(N.factor, N.stack, N.info, nzval, S, Int(S.group_nodes[q]), buf) + end + finally + put!(spool, buf) + end + end + end + end + for k in eachindex(plan.group_first) + a, b = plan.group_first[k], plan.group_last[k] + if b - a + 1 <= max(2, nt ÷ 8) # thin level: serial fronts, threaded BLAS + tblas += @elapsed begin + BLAS.set_num_threads(nt) + # par = true (sliced assembly) measured SLOWER here: _extend_cols! rescans + # every child column per slice, O(slices x columns); a per-slice column + # index built at plan time would be needed to make it pay + for q in a:b + _front_blas!(N.factor, N.stack, wk0, N.info, S, nzval, Int(S.group_nodes[q]), false) + end + BLAS.set_num_threads(1) + end + else + tblas += @elapsed @sync for r in _chunks(a, b, nch) + Threads.@spawn begin + wk = take!(wpool) + try + for q in r + _front_blas!(N.factor, N.stack, wk, N.info, S, nzval, Int(S.group_nodes[q])) + end + finally + put!(wpool, wk) + end + end + end + end + end + finally + put!(wpool, wk0) + BLAS.set_num_threads(nblas) + end + haskey(ENV, "SDSCPU_STAGES") && + @printf(" stages: subtrees %7.1f ms fronts %7.1f ms\n", tsub * 1e3, tblas * 1e3) + SDS.assemble_schur!(N, S, nzval) + return maximum(Array(N.info)) == 0 ? 0 : 1 +end + +end # module diff --git a/bench/e2e/SDSProto.jl b/bench/e2e/SDSProto.jl deleted file mode 100644 index d51fcbc..0000000 --- a/bench/e2e/SDSProto.jl +++ /dev/null @@ -1,1338 +0,0 @@ -# Prototype kernels of PR #107 assembled as a backend-portable module, so the -# MadNLP wrapper can run the fused factorization and the algo1/fused solve on -# CUDA and AMDGPU alike. Single-solver-instance (module-global plan state). -module SDSProto - -using LinearAlgebra, SparseArrays, Statistics, Printf -using SparseDirectSolver -const SDS = SparseDirectSolver -const KA = SDS.KernelAbstractions -const Atomix = SDS.Atomix -using .KA: @kernel, @index, @localmem, @synchronize, @Const, @uniform - -const BACKEND = Ref{Any}(nothing) -DVEC(v) = (out = KA.allocate(BACKEND[], eltype(v), size(v)); copyto!(out, v); out) -DVEC(::Type{T}, v) where {T} = (out = KA.allocate(BACKEND[], T, length(v)); copyto!(out, T.(v)); out) - - - -const WG = 32 -const MW = 256 # max front width on the inverse path -const WGF = 128 # workgroup size of the fused kernels -const CHWG = 1024 # workgroup size of the chain kernels -const NSTRIP = 4 # strip parallelism of the chain kernels - -# --------------------------------------------------------------------------- -# inversion: thread j of the front's workgroup computes column j of inv(L11) -# (non-unit lower triangular), written col-major at inv_ptr[v] -@kernel function invert_l11_kernel!(inv11, @Const(factor), @Const(inv_ptr), @Const(list), - @Const(front_ptr), @Const(front_nrows), @Const(front_ncols), ::Val{WGV}) where {WGV} - li = @index(Local, Linear) - g = @index(Group, Linear) - @inbounds begin - v = list[g] - f = Int(front_nrows[v]) - w = Int(front_ncols[v]) - p0 = Int(front_ptr[v]) - 1 - ip = Int(inv_ptr[v]) - 1 - for j in li:WGV:w - cb = ip + (j - 1) * w - for k in 1:(j - 1) - inv11[cb + k] = zero(eltype(inv11)) - end - for k in j:w - acc = k == j ? one(eltype(inv11)) : zero(eltype(inv11)) - for m in j:(k - 1) - acc -= factor[p0 + (m - 1) * f + k] * inv11[cb + m] - end - inv11[cb + k] = acc / factor[p0 + (k - 1) * f + k] - end - end - end -end - -# --------------------------------------------------------------------------- -# forward: z = inv(L11) y per front, then the atomic L21 update (stock scatter) -@kernel function fwd1_kernel!(Y, @Const(factor), @Const(inv11), @Const(inv_ptr), @Const(list), first, count, - @Const(subtree_ptr), @Const(subtree_nodes), @Const(super_ptr), @Const(rowptr), - @Const(rowval), @Const(front_ptr), @Const(front_nrows), @Const(front_ncols), - ::Val{SUB}, ::Val{WGV}) where {SUB, WGV} - li = @index(Local, Linear) - g = @index(Group, Linear) - t = @localmem Float64 (MW,) - @inbounds begin - e = list[first + g - 1] - a0 = SUB ? Int(subtree_ptr[e]) : 0 - cnt = SUB ? Int(subtree_ptr[e + 1]) - a0 : 1 - limit = SUB ? Int(super_ptr[subtree_nodes[a0 + cnt - 1] + 1]) - 1 : 0 - for k in 1:cnt - v = SUB ? subtree_nodes[a0 + k - 1] : e - f = Int(front_nrows[v]) - w = Int(front_ncols[v]) - p0 = Int(front_ptr[v]) - 1 - rp = Int(rowptr[v]) - 1 - c0 = Int(super_ptr[v]) - 1 - ip = Int(inv_ptr[v]) - 1 - w > MW && continue - # t = inv(L11) * Y[c0+1 : c0+w] - for kk in li:WGV:w - acc = zero(Float64) - for j in 1:w - acc += inv11[ip + (j - 1) * w + kk] * Y[c0 + j] - end - t[kk] = acc - end - @synchronize - for kk in li:WGV:w - Y[c0 + kk] = t[kk] - end - # L21 update from t (Y rows above `limit` belong to other workgroups: atomic) - for i in (w + li):WGV:f - acc = zero(Float64) - for j in 1:w - acc += factor[p0 + (j - 1) * f + i] * t[j] - end - row = Int(rowval[rp + i]) - if row > limit - Atomix.@atomic Y[row] += -acc - else - Y[row] -= acc - end - end - @synchronize - end - end -end - -# --------------------------------------------------------------------------- -# backward: gather from finished ancestors, then x = inv(L11)ᵀ z per front -@kernel function bwd1_kernel!(Y, @Const(factor), @Const(inv11), @Const(inv_ptr), @Const(list), first, count, - @Const(subtree_ptr), @Const(subtree_nodes), @Const(super_ptr), @Const(rowptr), - @Const(rowval), @Const(front_ptr), @Const(front_nrows), @Const(front_ncols), - ::Val{SUB}, ::Val{WGV}) where {SUB, WGV} - li = @index(Local, Linear) - g = @index(Group, Linear) - t = @localmem Float64 (MW,) - @inbounds begin - e = list[first + g - 1] - a0 = SUB ? Int(subtree_ptr[e]) : 0 - cnt = SUB ? Int(subtree_ptr[e + 1]) - a0 : 1 - for k in 1:cnt - v = SUB ? subtree_nodes[a0 + cnt - k] : e # reverse processing order - f = Int(front_nrows[v]) - w = Int(front_ncols[v]) - p0 = Int(front_ptr[v]) - 1 - rp = Int(rowptr[v]) - 1 - c0 = Int(super_ptr[v]) - 1 - ip = Int(inv_ptr[v]) - 1 - w > MW && continue - # t = Y[c0+k] - L21ᵀ x(rows below) - for kk in li:WGV:w - acc = zero(Float64) - col = p0 + (kk - 1) * f - for i in (w + 1):f - acc += factor[col + i] * Y[rowval[rp + i]] - end - t[kk] = Y[c0 + kk] - acc - end - @synchronize - # x = inv(L11)ᵀ t (inv col-major: invᵀ[k, j] = inv11[(k-1)w + j], contiguous in j) - for kk in li:WGV:w - acc = zero(Float64) - cb = ip + (kk - 1) * w - for j in 1:w - acc += inv11[cb + j] * t[j] - end - Y[c0 + kk] = acc - end - @synchronize - end - end -end - -# --------------------------------------------------------------------------- -function build_inverse(s) - S = s.symbolic - sc = S.schedule - reg = Array(sc.regime) - width = sc.width - ns = length(reg) - inv_ptr_h = zeros(Int64, ns + 1) - p = 1 - for v in 1:ns - inv_ptr_h[v] = p - if width[v] <= MW - p += width[v]^2 - else - @assert reg[v] == SDS.REGIME_C "front $v: width $(width[v]) > $MW but not regime C" - end - end - inv_ptr_h[ns + 1] = p - inv11 = KA.zeros(BACKEND[], Float64, p - 1) - inv_ptr = DVEC(inv_ptr_h) - ab = DVEC(Int32.(findall(v -> width[v] <= MW, 1:ns))) - backend = KA.get_backend(inv11) - invert! = () -> invert_l11_kernel!(backend, WG)(inv11, s.numeric.factor, inv_ptr, ab, S.front_ptr, - S.front_nrows, S.front_ncols, Val(WG); - ndrange = WG * length(ab)) - return inv11, inv_ptr, invert!, sizeof(Float64) * (p - 1) -end - -function algo1_solve!(x, s, b, inv11, inv_ptr) - S, N, ws = s.symbolic, s.numeric, s.workspace - bm = SDS.batch_map(N; nrhs = 1) - SDS.permute_rhs!(ws.Y, b, S.perm; transposed = false, bm) - plan = ws.plan - nodes = S.schedule.group_nodes - widths = S.schedule.width - p = SDS._solve_impls(N, S, :auto) - backend = KA.get_backend(ws.Y) - Yv = vec(ws.Y) - args(first, count) = (Yv, N.factor, inv11, inv_ptr, S.group_nodes, first, count, S.subtree_ptr, - S.subtree_nodes, S.super_ptr, S.rowptr, S.rowval, S.front_ptr, S.front_nrows, - S.front_ncols) - for k in eachindex(plan.kind) - a, b_ = plan.first[k], plan.last[k] - if plan.kind[k] == SDS.SOLVE_DENSE - any(widths[nodes[q]] <= MW for q in a:b_) && - fwd1_kernel!(backend, WG)(args(a, b_ - a + 1)..., Val(false), Val(WG); ndrange = WG * (b_ - a + 1)) - for q in a:b_ - widths[nodes[q]] > MW && SDS._fwd_dense!(ws, S, N, nodes[q], 1, false, p, SDS._SV_CHOLESKY) - end - else - sub = plan.kind[k] == SDS.SOLVE_SUBTREES - fwd1_kernel!(backend, WG)(args(a, b_ - a + 1)..., Val(sub), Val(WG); ndrange = WG * (b_ - a + 1)) - end - end - for k in reverse(eachindex(plan.kind)) - a, b_ = plan.first[k], plan.last[k] - if plan.kind[k] == SDS.SOLVE_DENSE - for q in b_:-1:a - widths[nodes[q]] > MW && SDS._bwd_dense!(ws, S, N, nodes[q], 1, p, SDS._SV_CHOLESKY) - end - any(widths[nodes[q]] <= MW for q in a:b_) && - bwd1_kernel!(backend, WG)(args(a, b_ - a + 1)..., Val(false), Val(WG); ndrange = WG * (b_ - a + 1)) - else - sub = plan.kind[k] == SDS.SOLVE_SUBTREES - bwd1_kernel!(backend, WG)(args(a, b_ - a + 1)..., Val(sub), Val(WG); ndrange = WG * (b_ - a + 1)) - end - end - SDS.unpermute_solution!(x, ws.Y, S.perm; transposed = false, bm) - return x -end - - - -# --------------------------------------------------------------------------- -# fused sweeps (algo2): one launch per direction over every front of width -# <= MW below the first wide front, etree dependency counters instead of -# level-batched launches. CUDA-only (threadfence, resident-grid spinning). -@kernel function fwd2_kernel!(Y, @Const(factor), @Const(inv11), @Const(inv_ptr), - @Const(flist), @Const(slist), nsub, @Const(subtree_ptr), @Const(subtree_nodes), - @Const(snparent), arrived, @Const(nchild), @Const(super_ptr), - @Const(rowptr), @Const(rowval), @Const(front_ptr), @Const(front_nrows), - @Const(front_ncols), ::Val{WGV}) where {WGV} - li = @index(Local, Linear) - g = @index(Group, Linear) - yb = @localmem Float64 (MW,) - t = @localmem Float64 (MW,) - @inbounds if g <= nsub - e = Int(slist[g]) # regime-A subtree: serial walk, then signal - a0 = Int(subtree_ptr[e]) - cnt = Int(subtree_ptr[e + 1]) - a0 - root = Int(subtree_nodes[a0 + cnt - 1]) - limit = Int(super_ptr[root + 1]) - 1 - for k in 1:cnt - v = Int(subtree_nodes[a0 + k - 1]) - f = Int(front_nrows[v]) - w = Int(front_ncols[v]) - p0 = Int(front_ptr[v]) - 1 - rp = Int(rowptr[v]) - 1 - c0 = Int(super_ptr[v]) - 1 - ip = Int(inv_ptr[v]) - 1 - for kk in li:WGV:w - acc = zero(Float64) - for j in 1:w - acc += inv11[ip + (j - 1) * w + kk] * Y[c0 + j] - end - t[kk] = acc - end - @synchronize - for kk in li:WGV:w - Y[c0 + kk] = t[kk] - end - for i in (w + li):WGV:f - acc = zero(Float64) - for j in 1:w - acc += factor[p0 + (j - 1) * f + i] * t[j] - end - row = Int(rowval[rp + i]) - if row > limit - Atomix.@atomic Y[row] += -acc - else - Y[row] -= acc - end - end - @synchronize - end - if li == 1 - Threads.atomic_fence() - pv = Int(snparent[root]) - pv > 0 && (Atomix.@atomic arrived[pv] += Int32(1)) - end - else begin - v = Int(flist[g - nsub]) - f = Int(front_nrows[v]) - w = Int(front_ncols[v]) - p0 = Int(front_ptr[v]) - 1 - rp = Int(rowptr[v]) - 1 - c0 = Int(super_ptr[v]) - 1 - ip = Int(inv_ptr[v]) - 1 - if li == 1 - tgt = nchild[v] - pause = Int32(4) - while (Atomix.@atomic arrived[v] += Int32(0)) < tgt - z = 0.0 - for _ in 1:pause - z += 1.0 - end - z < 0 && (yb[1] = z) # keep the backoff loop alive - pause = min(pause << 1, Int32(128)) - end - end - @synchronize - Threads.atomic_fence() - for kk in li:WGV:w # coherent read of the front's columns - yb[kk] = Atomix.@atomic Y[c0 + kk] += 0.0 - end - @synchronize - for kk in li:WGV:w # t = inv(L11) * y - acc = zero(Float64) - for j in 1:w - acc += inv11[ip + (j - 1) * w + kk] * yb[j] - end - t[kk] = acc - end - @synchronize - for kk in li:WGV:w - Y[c0 + kk] = t[kk] - end - for i in (w + li):WGV:f # L21 update, atomic scatter - acc = zero(Float64) - for j in 1:w - acc += factor[p0 + (j - 1) * f + i] * t[j] - end - Atomix.@atomic Y[Int(rowval[rp + i])] += -acc - end - @synchronize - if li == 1 - Threads.atomic_fence() - pv = Int(snparent[v]) - pv > 0 && (Atomix.@atomic arrived[pv] += Int32(1)) - end - end end -end - -@kernel function bwd2_kernel!(Y, @Const(factor), @Const(inv11), @Const(inv_ptr), - @Const(flist), @Const(slist), nsub, @Const(subtree_ptr), @Const(subtree_nodes), - @Const(snparent), done, @Const(super_ptr), @Const(rowptr), - @Const(rowval), @Const(front_ptr), @Const(front_nrows), @Const(front_ncols), - ::Val{WGV}) where {WGV} - li = @index(Local, Linear) - g = @index(Group, Linear) - xb = @localmem Float64 (WGV,) - t = @localmem Float64 (MW,) - t2 = @localmem Float64 (MW,) - @inbounds if g > length(flist) - e = Int(slist[g - length(flist)]) # regime-A subtree: wait for the root's parent, walk down - a0 = Int(subtree_ptr[e]) - cnt = Int(subtree_ptr[e + 1]) - a0 - root = Int(subtree_nodes[a0 + cnt - 1]) - if li == 1 - pv = Int(snparent[root]) - if pv > 0 - pause = Int32(4) - while (Atomix.@atomic done[pv] += Int32(0)) == Int32(0) - z = 0.0 - for _ in 1:pause - z += 1.0 - end - z < 0 && (t[1] = z) - pause = min(pause << 1, Int32(128)) - end - end - end - @synchronize - Threads.atomic_fence() - for k in cnt:-1:1 - v = Int(subtree_nodes[a0 + k - 1]) - f = Int(front_nrows[v]) - w = Int(front_ncols[v]) - p0 = Int(front_ptr[v]) - 1 - rp = Int(rowptr[v]) - 1 - c0 = Int(super_ptr[v]) - 1 - ip = Int(inv_ptr[v]) - 1 - for kk in li:WGV:w - acc = zero(Float64) - col = p0 + (kk - 1) * f - for i in (w + 1):f - acc += factor[col + i] * Y[Int(rowval[rp + i])] - end - t2[kk] = Y[c0 + kk] - acc - end - @synchronize - for kk in li:WGV:w - acc = zero(Float64) - cb = ip + (kk - 1) * w - for j in 1:w - acc += inv11[cb + j] * t2[j] - end - Y[c0 + kk] = acc - end - @synchronize - end - else begin - v = Int(flist[length(flist) - g + 1]) # reverse topo order: parents get smaller block ids - f = Int(front_nrows[v]) - w = Int(front_ncols[v]) - p0 = Int(front_ptr[v]) - 1 - rp = Int(rowptr[v]) - 1 - c0 = Int(super_ptr[v]) - 1 - ip = Int(inv_ptr[v]) - 1 - if li == 1 - pv = Int(snparent[v]) - if pv > 0 - pause = Int32(4) - while (Atomix.@atomic done[pv] += Int32(0)) == Int32(0) - z = 0.0 - for _ in 1:pause - z += 1.0 - end - z < 0 && (t[1] = z) - pause = min(pause << 1, Int32(128)) - end - end - end - @synchronize - Threads.atomic_fence() - for kk in li:WGV:w - t[kk] = zero(Float64) - end - @synchronize - i0 = w + 1 - while i0 <= f # chunked coherent gather of ancestor solutions - ii = i0 + li - 1 - if ii <= f - xb[li] = Atomix.@atomic Y[Int(rowval[rp + ii])] += 0.0 - end - @synchronize - m = min(WGV, f - i0 + 1) - for kk in li:WGV:w - acc = t[kk] - col = p0 + (kk - 1) * f + i0 - 1 - for c in 1:m - acc += factor[col + c] * xb[c] - end - t[kk] = acc - end - @synchronize - i0 += WGV - end - for kk in li:WGV:w # z = y - L21' x, then x = inv(L11)' z - t2[kk] = Y[c0 + kk] - t[kk] - end - @synchronize - for kk in li:WGV:w - acc = zero(Float64) - cb = ip + (kk - 1) * w - for j in 1:w - acc += inv11[cb + j] * t2[j] - end - Y[c0 + kk] = acc - end - @synchronize - if li == 1 - Threads.atomic_fence() - Atomix.@atomic done[v] += Int32(1) - end - end end -end - - -# chain kernels: ONE workgroup walks the (near-serial) tall-front chain at the -# top of the tree in plan order. Launched after the fused bottom tier, so every -# input is already in Y (stream order): no spins, no fences, no atomics — the -# single block owns every row it touches; plain loads/stores + block barriers. -@kernel function chain_fwd_kernel!(Y, @Const(factor), @Const(inv11), @Const(inv_ptr), @Const(clist), - @Const(super_ptr), @Const(rowptr), @Const(rowval), @Const(front_ptr), - @Const(front_nrows), @Const(front_ncols), ::Val{WGV}) where {WGV} - li = @index(Local, Linear) - yb = @localmem Float64 (MW,) - t = @localmem Float64 (MW,) - tp = @localmem Float64 (MW * NSTRIP,) - @inbounds for q in 1:length(clist) - v = Int(clist[q]) - f = Int(front_nrows[v]) - w = Int(front_ncols[v]) - p0 = Int(front_ptr[v]) - 1 - rp = Int(rowptr[v]) - 1 - c0 = Int(super_ptr[v]) - 1 - ip = Int(inv_ptr[v]) - 1 - for kk in li:WGV:w - yb[kk] = Y[c0 + kk] - end - @synchronize - for kk in li:WGV:w - acc = zero(Float64) - for j in 1:w - acc += inv11[ip + (j - 1) * w + kk] * yb[j] - end - t[kk] = acc - end - @synchronize - for kk in li:WGV:w - Y[c0 + kk] = t[kk] - end - for i in (w + li):WGV:f - acc = zero(Float64) - for j in 1:w - acc += factor[p0 + (j - 1) * f + i] * t[j] - end - Y[Int(rowval[rp + i])] -= acc - end - @synchronize - end -end - -@kernel function chain_bwd_kernel!(Y, @Const(factor), @Const(inv11), @Const(inv_ptr), @Const(clist), - @Const(super_ptr), @Const(rowptr), @Const(rowval), @Const(front_ptr), - @Const(front_nrows), @Const(front_ncols), ::Val{WGV}) where {WGV} - li = @index(Local, Linear) - t = @localmem Float64 (MW,) - t2 = @localmem Float64 (MW,) - tp = @localmem Float64 (MW * NSTRIP,) - @inbounds for q in length(clist):-1:1 - v = Int(clist[q]) - f = Int(front_nrows[v]) - w = Int(front_ncols[v]) - p0 = Int(front_ptr[v]) - 1 - rp = Int(rowptr[v]) - 1 - c0 = Int(super_ptr[v]) - 1 - ip = Int(inv_ptr[v]) - 1 - kk = (li - 1) % w + 1 # strip-parallel gather: t = L21' x - st = (li - 1) ÷ w + 1 - if st <= NSTRIP - acc = zero(Float64) - col = p0 + (kk - 1) * f - for i in (w + st):NSTRIP:f - acc += factor[col + i] * Y[Int(rowval[rp + i])] - end - tp[(st - 1) * MW + kk] = acc - end - @synchronize - for k2 in li:WGV:w - acc = Y[c0 + k2] - for st2 in 1:NSTRIP - acc -= tp[(st2 - 1) * MW + k2] - end - t2[k2] = acc - end - @synchronize - kk2 = (li - 1) % w + 1 # strip-parallel GEMV: x = inv(L11)' t2 - st2b = (li - 1) ÷ w + 1 - if st2b <= NSTRIP - acc = zero(Float64) - cb = ip + (kk2 - 1) * w - for j in st2b:NSTRIP:w - acc += inv11[cb + j] * t2[j] - end - tp[(st2b - 1) * MW + kk2] = acc - end - @synchronize - for k2 in li:WGV:w - acc = zero(Float64) - for st3 in 1:NSTRIP - acc += tp[(st3 - 1) * MW + k2] - end - t[k2] = acc - end - @synchronize - for k2 in li:WGV:w - Y[c0 + k2] = t[k2] - end - @synchronize - end -end - -# fused segment: the maximal run of plan entries from `kstart` in which every -# front has width <= wcap (regime-A subtree entries break a run) -function build_fused(s; kstart::Int = 1, wcap::Int = MW, merged::Bool = (kstart == 1)) - S = s.symbolic - ws = s.workspace - plan = ws.plan - nodes = S.schedule.group_nodes - widths = S.schedule.width - narrow(k) = plan.kind[k] != SDS.SOLVE_SUBTREES && - all(widths[nodes[q]] <= wcap for q in plan.first[k]:plan.last[k]) - i1 = kstart - while i1 <= length(plan.kind) && !narrow(i1) - i1 += 1 - end - i1 > length(plan.kind) && return nothing - i2 = i1 - while i2 + 1 <= length(plan.kind) && narrow(i2 + 1) - i2 += 1 - end - flist_h = Int32[nodes[q] for k in i1:i2 for q in plan.first[k]:plan.last[k]] - ns = length(widths) - in_range = falses(ns) - for v in flist_h - in_range[v] = true - end - child_ptr_h = Array(S.child_ptr) - child_list_h = Array(S.child_list) - nchild_h = Int32[child_ptr_h[v + 1] - child_ptr_h[v] for v in 1:ns] - stp_h = Array(S.subtree_ptr) - stn_h = Array(S.subtree_nodes) - live_root = falses(ns) # roots of subtrees merged into this launch (the merged tier only) - if merged - for e in 1:(length(stp_h) - 1) - live_root[stn_h[stp_h[e + 1] - 1]] = true - end - end - arrived_base_h = zeros(Int32, ns) - for v in 1:ns - in_range[v] || continue - nc = child_ptr_h[v + 1] - child_ptr_h[v] - cnt = 0 - for k in 1:nc - c = child_list_h[child_ptr_h[v] + k - 1] - (in_range[c] || live_root[c]) || (cnt += 1) - end - arrived_base_h[v] = Int32(cnt) - end - done_base_h = Int32[in_range[v] ? 0 : 1 for v in 1:ns] - @printf "fused: entries %d:%d of %d, %d fronts\n" i1 i2 length(plan.kind) length(flist_h) - return (; i1, i2, flist = DVEC(flist_h), arrived = KA.zeros(BACKEND[], Int32, ns), - arrived_base = DVEC(arrived_base_h), done = KA.zeros(BACKEND[], Int32, ns), - done_base = DVEC(done_base_h), nchild = DVEC(nchild_h), nfused = length(flist_h)) -end - -function run_fused!(fz, wg, Yv, s, backend) - S, N = s.symbolic, s.numeric - if wg > 512 - chain_fwd_kernel!(backend, CHWG)(Yv, N.factor, INV11[], INVPTR[], fz.flist, S.super_ptr, S.rowptr, - S.rowval, S.front_ptr, S.front_nrows, S.front_ncols, Val(CHWG); - ndrange = CHWG) - return nothing - end - copyto!(fz.arrived, fz.arrived_base) - if wg == 64 - fwd2_kernel!(backend, 64)(Yv, N.factor, INV11[], INVPTR[], fz.flist, SSUB[], NSUB[], S.subtree_ptr, - S.subtree_nodes, S.snparent, fz.arrived, fz.nchild, - S.super_ptr, S.rowptr, S.rowval, S.front_ptr, S.front_nrows, S.front_ncols, - Val(64); ndrange = 64 * (fz.nfused + NSUB[])) - else - error("unused") - end -end - -function run_fused_bwd!(fz, wg, Yv, s, backend) - S, N = s.symbolic, s.numeric - if wg > 512 - chain_bwd_kernel!(backend, CHWG)(Yv, N.factor, INV11[], INVPTR[], fz.flist, S.super_ptr, S.rowptr, - S.rowval, S.front_ptr, S.front_nrows, S.front_ncols, Val(CHWG); - ndrange = CHWG) - return nothing - end - copyto!(fz.done, fz.done_base) - if wg == 64 - bwd2_kernel!(backend, 64)(Yv, N.factor, INV11[], INVPTR[], fz.flist, SSUB[], NSUB[], S.subtree_ptr, - S.subtree_nodes, S.snparent, fz.done, - S.super_ptr, S.rowptr, S.rowval, S.front_ptr, S.front_nrows, S.front_ncols, - Val(64); ndrange = 64 * (fz.nfused + NSUB[])) - else - error("unused") - end -end - -const INV11 = Ref{Any}() -const INVPTR = Ref{Any}() -const SSUB = Ref{Any}() # subtree ids sorted by size, longest first -const NSUB = Ref{Int}(0) - -function algo2_solve!(x, s, b, inv11, inv_ptr, fz; debug::Bool = false) - S, N, ws = s.symbolic, s.numeric, s.workspace - bm = SDS.batch_map(N; nrhs = 1) - SDS.permute_rhs!(ws.Y, b, S.perm; transposed = false, bm) - plan = ws.plan - nodes = S.schedule.group_nodes - widths = S.schedule.width - p = SDS._solve_impls(N, S, :auto) - backend = KA.get_backend(ws.Y) - Yv = vec(ws.Y) - args(first, count) = (Yv, N.factor, inv11, inv_ptr, S.group_nodes, first, count, S.subtree_ptr, - S.subtree_nodes, S.super_ptr, S.rowptr, S.rowval, S.front_ptr, S.front_nrows, - S.front_ncols) - fwd_entry(k) = begin - a, b_ = plan.first[k], plan.last[k] - if plan.kind[k] == SDS.SOLVE_SUBTREES - elseif plan.kind[k] == SDS.SOLVE_DENSE - any(widths[nodes[q]] <= MW for q in a:b_) && - fwd1_kernel!(backend, WG)(args(a, b_ - a + 1)..., Val(false), Val(WG); ndrange = WG * (b_ - a + 1)) - for q in a:b_ - widths[nodes[q]] > MW && SDS._fwd_dense!(ws, S, N, nodes[q], 1, false, p, SDS._SV_CHOLESKY) - end - else - fwd1_kernel!(backend, WG)(args(a, b_ - a + 1)..., Val(plan.kind[k] == SDS.SOLVE_SUBTREES), Val(WG); - ndrange = WG * (b_ - a + 1)) - end - end - bwd_entry(k) = begin - a, b_ = plan.first[k], plan.last[k] - if plan.kind[k] == SDS.SOLVE_SUBTREES - elseif plan.kind[k] == SDS.SOLVE_DENSE - for q in b_:-1:a - widths[nodes[q]] > MW && SDS._bwd_dense!(ws, S, N, nodes[q], 1, p, SDS._SV_CHOLESKY) - end - any(widths[nodes[q]] <= MW for q in a:b_) && - bwd1_kernel!(backend, WG)(args(a, b_ - a + 1)..., Val(false), Val(WG); ndrange = WG * (b_ - a + 1)) - else - bwd1_kernel!(backend, WG)(args(a, b_ - a + 1)..., Val(plan.kind[k] == SDS.SOLVE_SUBTREES), Val(WG); - ndrange = WG * (b_ - a + 1)) - end - end - INV11[] = inv11; INVPTR[] = inv_ptr - segs, covered = fz - k = 1 - for (fzk, wg) in segs - while k < fzk.i1 - fwd_entry(k); k += 1 - end - run_fused!(fzk, wg, Yv, s, backend) - k = fzk.i2 + 1 - end - while k <= length(plan.kind) - fwd_entry(k); k += 1 - end - debug && (KA.synchronize(BACKEND[]); println(" algo2: forward done"); flush(stdout)) - k = length(plan.kind) - for (fzk, wg) in Iterators.reverse(segs) - while k > fzk.i2 - bwd_entry(k); k -= 1 - end - run_fused_bwd!(fzk, wg, Yv, s, backend) - k = fzk.i1 - 1 - end - while k >= 1 - bwd_entry(k); k -= 1 - end - debug && (KA.synchronize(BACKEND[]); println(" algo2: backward done"); flush(stdout)) - SDS.unpermute_solution!(x, ws.Y, S.perm; transposed = false, bm) - return x -end - - - - - - -const SYRK_IN = 64 -const TILE = 32 -const NL64 = 2080 # packed 64x64 lower triangle - -const ROLE_PANEL_B = Int8(1) -const ROLE_PANEL_C = Int8(2) -const ROLE_TRSM = Int8(3) -const ROLE_TILE = Int8(4) -const ROLE_EA = Int8(5) - -# spin with backoff on an Int32 cell until pred(value) holds; thread 1 only -@inline function _spin_until_eq!(arr, idx, target, scratch) - pause = Int32(4) - while (Atomix.@atomic arr[idx] += Int32(0)) != target - z = 0.0 - for _ in 1:pause - z += 1.0 - end - z < 0 && (scratch[1] = z) - pause = min(pause << 1, Int32(128)) - end - return nothing -end - -@kernel function fused_fact_kernel!(factor, stack, pcb, info, @Const(nzval), @Const(amap), @Const(amap_ptr), - @Const(amap_src), @Const(cb2), @Const(wt), @Const(role), @Const(ra), @Const(rb), @Const(rc), - @Const(nchild), arrived, ea_left, panel_done, trsm_left, tiles_left, - cb_done, @Const(snparent), @Const(front_ptr), @Const(front_nrows), - @Const(front_ncols), @Const(cb_ptr), @Const(child_ptr), - @Const(child_list), @Const(relind_ptr), @Const(relind), base, - ::Val{WG}) where {WG} - @uniform TT = eltype(factor) - @uniform RT = real(eltype(factor)) - li = @index(Local, Linear) - G = Int(base) + @index(Group, Linear) - sh = @localmem TT (NL64,) - st = @localmem Int32 (1,) - piv = @localmem RT (1,) - fscr = @localmem Float64 (1,) - @inbounds begin - r = role[G] - if r == ROLE_EA - v = Int(ra[G]) # parent - c = Int(rb[G]) # child - q0 = (Int(rc[G]) - 1) * 16384 - if li == 1 - _spin_until_eq!(cb_done, c, Int32(1), fscr) - end - @synchronize - Threads.atomic_fence() - cpriv = Int(cb2[c]) # child CB: private (mega) or stock (subtree root) - cb = cpriv > 0 ? cpriv : Int(cb_ptr[c]) - mc = Int(front_nrows[c]) - Int(front_ncols[c]) - fp = Int(front_nrows[v]) - wp = Int(front_ncols[v]) - mp = fp - wp - pp = Int(front_ptr[v]) - cp = Int(cb2[v]) # parent CB is always private - r0 = Int(relind_ptr[c]) - 1 - qe = min(q0 + 16384, mc * mc) - 1 - for q in (q0 + li - 1):WG:qe - jj = q ÷ mc + 1 - ii = q - (jj - 1) * mc + 1 - if ii >= jj - ri = Int(relind[r0 + ii]) - rj = Int(relind[r0 + jj]) - x = cpriv > 0 ? pcb[cb + SDS._packed(ii, jj, mc) - 1] : - stack[cb + SDS._packed(ii, jj, mc) - 1] - if rj <= wp - Atomix.@atomic factor[pp + (rj - 1) * fp + ri - 1] += x - else - Atomix.@atomic pcb[cp + SDS._packed(ri - wp, rj - wp, mp) - 1] += x - end - end - end - @synchronize - if li == 1 - Threads.atomic_fence() - Atomix.@atomic ea_left[v] += Int32(-1) - end - elseif r == ROLE_PANEL_B || r == ROLE_PANEL_C - v = Int(ra[G]) - if li == 1 - if r == ROLE_PANEL_B - _spin_until_eq!(arrived, v, nchild[v], fscr) - else - _spin_until_eq!(ea_left, v, Int32(0), fscr) - end - end - @synchronize - Threads.atomic_fence() - if r == ROLE_PANEL_B - fz = Int(front_nrows[v]) - wz = Int(front_ncols[v]) - pz = Int(front_ptr[v]) - for q in (li - 1):WG:(fz * wz - 1) # zero the panel; private CBs are pre-zeroed - factor[pz + q] = zero(TT) - end - @synchronize - SDS._scatter_front!(factor, nzval, amap, amap_ptr, amap_src, v, - SDS._member_shift(front_ptr, v, 1, 1), li, Val(WG)) - @synchronize - nc = Int(child_ptr[v + 1]) - Int(child_ptr[v]) - for kc in 1:nc - c = Int(child_list[child_ptr[v] + kc - 1]) - cpr = Int(cb2[c]) - cbc = cpr > 0 ? cpr : Int(cb_ptr[c]) - if cbc > 0 - mc = Int(front_nrows[c]) - Int(front_ncols[c]) - r0 = Int(relind_ptr[c]) - 1 - mp = fz - wz - cp2 = Int(cb2[v]) - for q in (li - 1):WG:(mc * mc - 1) - jj = q ÷ mc + 1 - ii = q - (jj - 1) * mc + 1 - if ii >= jj - ri = Int(relind[r0 + ii]) - rj = Int(relind[r0 + jj]) - x = cpr > 0 ? pcb[cbc + SDS._packed(ii, jj, mc) - 1] : - stack[cbc + SDS._packed(ii, jj, mc) - 1] - if rj <= wz - factor[pz + (rj - 1) * fz + ri - 1] += x - else - pcb[cp2 + SDS._packed(ri - wz, rj - wz, mp) - 1] += x - end - end - end - end - @synchronize - end - end - SDS._front_load!(sh, st, factor, v, li, front_ptr, front_nrows, front_ncols, Val(64), Val(WG)) - @synchronize - w = Int(front_ncols[v]) - for j in 1:w - SDS._front_chol_update!(sh, st, piv, j, v, li, front_ncols, Val(64), Val(WG)) - @synchronize - SDS._front_chol_scale!(sh, st, piv, j, v, li, front_ncols, Val(64), Val(WG)) - @synchronize - end - f = Int(front_nrows[v]) - p0 = Int(front_ptr[v]) - for q in (li - 1):WG:(w * w - 1) - j = q ÷ w + 1 - i = q - (j - 1) * w + 1 - i >= j && (factor[p0 + (j - 1) * f + i - 1] = sh[SDS._packed(i, j, 64)]) - end - m = f - w - if st[1] == Int32(0) && m <= SYRK_IN - for i in (w + li):WG:f - for kk in 1:w - x = factor[p0 + (kk - 1) * f + i - 1] - for j in 1:(kk - 1) - x -= factor[p0 + (j - 1) * f + i - 1] * sh[SDS._packed(kk, j, 64)] - end - factor[p0 + (kk - 1) * f + i - 1] = x / sh[SDS._packed(kk, kk, 64)] - end - end - end - li == 1 && (info[v] = st[1]) - @synchronize - if m <= SYRK_IN - SDS._front_syrk!(factor, pcb, st, v, li, front_ptr, front_nrows, front_ncols, cb2, - Val(WG)) - @synchronize - if li == 1 - Threads.atomic_fence() - Atomix.@atomic cb_done[v] += Int32(1) - pv = Int(snparent[v]) - pv > 0 && (Atomix.@atomic arrived[pv] += Int32(1)) - end - else - if li == 1 - Threads.atomic_fence() - Atomix.@atomic panel_done[v] += Int32(1) - end - end - elseif r == ROLE_TRSM - v = Int(ra[G]) - if li == 1 - _spin_until_eq!(panel_done, v, Int32(1), fscr) - end - @synchronize - Threads.atomic_fence() - f = Int(front_nrows[v]) - w = Int(front_ncols[v]) - p0 = Int(front_ptr[v]) - for q in li:WG:(w * (w + 1) ÷ 2) - j = 1 - acc = w - qq = q - while qq > acc - qq -= acc - acc -= 1 - j += 1 - end - i = j + qq - 1 - sh[SDS._packed(i, j, 64)] = factor[p0 + (j - 1) * f + i - 1] - end - @synchronize - if info[v] == Int32(0) - i = w + (Int(rb[G]) - 1) * WG + li - if i <= f - for kk in 1:w - x = factor[p0 + (kk - 1) * f + i - 1] - for j in 1:(kk - 1) - x -= factor[p0 + (j - 1) * f + i - 1] * sh[SDS._packed(kk, j, 64)] - end - factor[p0 + (kk - 1) * f + i - 1] = x / sh[SDS._packed(kk, kk, 64)] - end - end - end - @synchronize - if li == 1 - Threads.atomic_fence() - left = (Atomix.@atomic trsm_left[v] += Int32(-1)) - if left == Int32(1) && tiles_left[v] == Int32(0) - Atomix.@atomic cb_done[v] += Int32(1) - pv = Int(snparent[v]) - pv > 0 && (Atomix.@atomic arrived[pv] += Int32(1)) - end - end - elseif r == ROLE_TILE - v = Int(ra[G]) - if li == 1 - _spin_until_eq!(trsm_left, v, Int32(0), fscr) - end - @synchronize - Threads.atomic_fence() - f = Int(front_nrows[v]) - w = Int(front_ncols[v]) - m = f - w - p0 = Int(front_ptr[v]) + w - 1 - c0 = Int(cb2[v]) - i0 = (Int(rb[G]) - 1) * TILE - j0 = (Int(rc[G]) - 1) * TILE - ok = info[v] == Int32(0) && c0 > 0 - acc1 = zero(TT); acc2 = zero(TT); acc3 = zero(TT); acc4 = zero(TT) - acc5 = zero(TT); acc6 = zero(TT); acc7 = zero(TT); acc8 = zero(TT) - kk0 = 0 - while kk0 < w # k in two halves: Ai/Aj share sh (2 x 1024) - kb = min(32, w - kk0) - if ok - for q in (li - 1):WG:(TILE * kb - 1) - rr = q % TILE + 1 - cc = q ÷ TILE + 1 - row = i0 + rr - sh[rr + (cc - 1) * TILE] = row <= m ? factor[p0 + (kk0 + cc - 1) * f + row] : zero(TT) - row2 = j0 + rr - sh[1024 + rr + (cc - 1) * TILE] = - row2 <= m ? factor[p0 + (kk0 + cc - 1) * f + row2] : zero(TT) - end - end - @synchronize - if ok - for (slot, q) in enumerate((li - 1):WG:(TILE * TILE - 1)) - rr = q % TILE + 1 - cc = q ÷ TILE + 1 - a = zero(TT) - for kk in 1:kb - a += sh[rr + (kk - 1) * TILE] * sh[1024 + cc + (kk - 1) * TILE] - end - slot == 1 && (acc1 += a); slot == 2 && (acc2 += a) - slot == 3 && (acc3 += a); slot == 4 && (acc4 += a) - slot == 5 && (acc5 += a); slot == 6 && (acc6 += a) - slot == 7 && (acc7 += a); slot == 8 && (acc8 += a) - end - end - @synchronize - kk0 += 32 - end - if ok - pw = Int(wt[v]) - if pw == 0 - for (slot, q) in enumerate((li - 1):WG:(TILE * TILE - 1)) - rr = q % TILE + 1 - cc = q ÷ TILE + 1 - ii = i0 + rr - jj = j0 + cc - if ii <= m && jj <= m && ii >= jj - a = slot == 1 ? acc1 : slot == 2 ? acc2 : slot == 3 ? acc3 : slot == 4 ? acc4 : - slot == 5 ? acc5 : slot == 6 ? acc6 : slot == 7 ? acc7 : acc8 - pcb[c0 + SDS._packed(ii, jj, m) - 1] -= a - end - end - else # write the finished CB entry through into the parent - fpw = Int(front_nrows[pw]) - wpw = Int(front_ncols[pw]) - mpw = fpw - wpw - ppw = Int(front_ptr[pw]) - cpw = Int(cb2[pw]) - r0w = Int(relind_ptr[v]) - 1 - for (slot, q) in enumerate((li - 1):WG:(TILE * TILE - 1)) - rr = q % TILE + 1 - cc = q ÷ TILE + 1 - ii = i0 + rr - jj = j0 + cc - if ii <= m && jj <= m && ii >= jj - a = slot == 1 ? acc1 : slot == 2 ? acc2 : slot == 3 ? acc3 : slot == 4 ? acc4 : - slot == 5 ? acc5 : slot == 6 ? acc6 : slot == 7 ? acc7 : acc8 - x = pcb[c0 + SDS._packed(ii, jj, m) - 1] - a # children's extend-adds + own syrk - ri = Int(relind[r0w + ii]) - rj = Int(relind[r0w + jj]) - if rj <= wpw - factor[ppw + (rj - 1) * fpw + ri - 1] += x - else - pcb[cpw + SDS._packed(ri - wpw, rj - wpw, mpw) - 1] += x - end - end - end - end - end - @synchronize - if li == 1 - Threads.atomic_fence() - left = (Atomix.@atomic tiles_left[v] += Int32(-1)) - if left == Int32(1) - Atomix.@atomic cb_done[v] += Int32(1) - pv = Int(snparent[v]) - pv > 0 && (Atomix.@atomic arrived[pv] += Int32(1)) - Int(wt[v]) > 0 && (Atomix.@atomic ea_left[Int(wt[v])] += Int32(-1)) - end - end - end - end -end - -@kernel function panel_zero_kernel!(factor, @Const(list), @Const(front_ptr), @Const(front_nrows), - @Const(front_ncols), ::Val{WG}) where {WG} - li = @index(Local, Linear) - G = @index(Group, Linear) - @inbounds begin - v = Int(list[G]) - f = Int(front_nrows[v]) - w = Int(front_ncols[v]) - p0 = Int(front_ptr[v]) - for q in (li - 1):WG:(f * w - 1) - factor[p0 + q] = zero(eltype(factor)) - end - end -end - -@kernel function waiter_kernel!(ea_left, v, fscr_unused) - li = @index(Local, Linear) - if li == 1 - pause = Int32(4) - @inbounds while (Atomix.@atomic ea_left[v] += Int32(0)) != Int32(0) - z = 0.0 - for _ in 1:pause - z += 1.0 - end - z < 0 && (ea_left[v] = Int32(0)) - pause = min(pause << 1, Int32(128)) - end - end -end - -@kernel function signal_kernel!(cb_done, arrived, @Const(snparent), v) - li = @index(Local, Linear) - @inbounds if li == 1 - Threads.atomic_fence() - Atomix.@atomic cb_done[v] += Int32(1) - pv = Int(snparent[v]) - pv > 0 && (Atomix.@atomic arrived[pv] += Int32(1)) - end -end - -# subtree launcher with a custom (sorted) list, local memory sized by eltype -function launch_subtrees_sorted!(N, S, nzval, list, lb, backend) - SDS._with_local_bytes(Int(lb)) do LBv - _sorted_subtree_launch(N, S, nzval, list, LBv, Val(SDS.SUBTREE_WORKGROUP), backend) - end -end -function _sorted_subtree_launch(N::SDS.Numeric{T}, S, nzval, list, ::Val{LB}, ::Val{WG}, backend) where {T, LB, WG} - bm = SDS.batch_map(N; first = 1) - k! = SDS.subtree_cholesky_kernel!(backend, WG) - k!(N.factor, N.stack, N.info, nzval, S.amap, S.amap_ptr, S.amap_src, list, bm, S.subtree_ptr, - S.subtree_nodes, S.front_ptr, S.front_nrows, S.front_ncols, S.cb_ptr, S.local_front, S.local_cb, - S.child_ptr, S.child_list, S.relind_ptr, S.relind, - Val((LB - SDS.SUBTREE_LOCAL_RESERVE) ÷ sizeof(T)), Val(WG); ndrange = WG * length(list) * bm.nact) - return nothing -end - -# --------------------------------------------------------------------------- -# host: the topologically ordered role list + counter bases + stream-2 program -function build_fused_plan(s) - S = s.symbolic - plan = s.numeric.plan - nodes = S.schedule.group_nodes - rows_h = S.schedule.rows - width_h = S.schedule.width - cbp = Array(S.cb_ptr) - chp = Array(S.child_ptr) - chl = Array(S.child_list) - stp = Array(S.subtree_ptr) - stn = Array(S.subtree_nodes) - ns = length(width_h) - - covered = falses(ns) # fronts factored by the subtree launches - for q in eachindex(stn) - covered[stn[q]] = true - end - - role = Int8[]; ra = Int32[]; rb = Int32[]; rc = Int32[] - wides = Int[] - ea_left = zeros(Int32, ns) - trsm_left = zeros(Int32, ns) - tiles_left = zeros(Int32, ns) - arrived_base = zeros(Int32, ns) - cb_done_base = zeros(Int32, ns) - for v in 1:ns - covered[v] && (cb_done_base[v] = 1) - end - - # proposal B: a child writes its CB through into the parent when the parent is a - # pre-assembled C-group front and the child is its ONLY CB child (any kind) - cset = Set{Int}() - for k in eachindex(plan.group_first) - plan.group_width[k] > 0 && continue - for q in plan.group_first[k]:plan.group_last[k] - push!(cset, Int(nodes[q])) - end - end - wt_h = zeros(Int32, ns) - nwt = 0; ewt = 0 - for v in 1:ns - (covered[v] || cbp[v] == 0) && continue - width_h[v] <= 64 || continue # wide children have no tiles to settle the debt - m = rows_h[v] - width_h[v] - m > SYRK_IN || continue - pv = Int(S.partition.snparent[v]) - (pv > 0 && pv in cset) || continue - count(c -> cbp[c] > 0, chl[chp[pv]:(chp[pv + 1] - 1)]) == 1 || continue - wt_h[v] = Int32(pv) - nwt += 1; ewt += m * (m + 1) ÷ 2 - end - println("write-through: ", nwt, " fronts, ", round(ewt / 1e6; digits = 2), "M CB entries"); flush(stdout) - - emit_ea(v) = for kc in chp[v]:(chp[v + 1] - 1) - c = chl[kc] - cbp[c] > 0 || continue - if wt_h[c] == v - ea_left[v] += 1 # settled by the child's last write-through tile - continue - end - mc = rows_h[c] - width_h[c] - for ch in 1:cld(mc * mc, 16384) - push!(role, ROLE_EA); push!(ra, Int32(v)); push!(rb, Int32(c)); push!(rc, Int32(ch)) - ea_left[v] += 1 - end - end - emit_split(v) = begin - m = rows_h[v] - width_h[v] - m > SYRK_IN || return nothing - for ch in 1:cld(m, WGF) - push!(role, ROLE_TRSM); push!(ra, Int32(v)); push!(rb, Int32(ch)); push!(rc, Int32(0)) - trsm_left[v] += 1 - end - if cbp[v] > 0 - T = cld(m, TILE) - for tjj in 1:T, tii in tjj:T - push!(role, ROLE_TILE); push!(ra, Int32(v)); push!(rb, Int32(tii)); push!(rc, Int32(tjj)) - tiles_left[v] += 1 - end - end - return nothing - end - - segments = NTuple{3, Any}[] # (base, count, wides-at-boundary) - seg_base = 0 - for k in eachindex(plan.group_first) - a, b = plan.group_first[k], plan.group_last[k] - if plan.group_width[k] > 0 - for q in a:b - v = nodes[q] - push!(role, ROLE_PANEL_B); push!(ra, Int32(v)); push!(rb, Int32(0)); push!(rc, Int32(0)) - end - for q in a:b - emit_split(nodes[q]) - end - else - gwides = Int[] - for q in a:b - emit_ea(nodes[q]) - end - for q in a:b - v = nodes[q] - if width_h[v] <= 64 - push!(role, ROLE_PANEL_C); push!(ra, Int32(v)); push!(rb, Int32(0)); push!(rc, Int32(0)) - else - push!(gwides, v) - end - end - for q in a:b - width_h[nodes[q]] <= 64 && emit_split(nodes[q]) - end - if !isempty(gwides) - push!(segments, (seg_base, length(role) - seg_base, gwides)) - seg_base = length(role) - append!(wides, gwides) - end - end - end - push!(segments, (seg_base, length(role) - seg_base, Int[])) - - cb2_h = zeros(Int64, ns) - off = 1 - for v in 1:ns - (covered[v] || cbp[v] == 0) && continue - m = rows_h[v] - width_h[v] - cb2_h[v] = off - off += m * (m + 1) ÷ 2 - end - czero = Int32[] # C-group fronts: panel zeroed by a prelaunch kernel - for k in eachindex(plan.group_first) - plan.group_width[k] > 0 && continue - append!(czero, Int32.(nodes[plan.group_first[k]:plan.group_last[k]])) - end - nchild = Int32[chp[v + 1] - chp[v] for v in 1:ns] - for v in 1:ns - cnt = 0 - for kc in chp[v]:(chp[v + 1] - 1) - covered[chl[kc]] && (cnt += 1) - end - arrived_base[v] = Int32(cnt) - end - - stp2 = stp - ssub = Vector{Any}() - for k in eachindex(plan.sub_first) - ids = [nodes[q] for q in plan.sub_first[k]:plan.sub_last[k]] - sort!(ids; by = e -> stp2[e + 1] - stp2[e], rev = true) - push!(ssub, DVEC(ids)) - end - - println("segments: ", length(segments)); flush(stdout) - return (; role = DVEC(role), ra = DVEC(ra), rb = DVEC(rb), rc = DVEC(rc), - nblocks = length(role), wides, ssub, segments, - cb2 = DVEC(Int32.(cb2_h)), cb2_h, pcb = KA.zeros(BACKEND[], Float64, off - 1), - czero = DVEC(czero), wt = DVEC(wt_h), - nchild = DVEC(nchild), - arrived_base = DVEC(arrived_base), ea_base = DVEC(ea_left), - trsm_base = DVEC(trsm_left), tiles_base = DVEC(tiles_left), - cb_base = DVEC(cb_done_base), - arrived = KA.zeros(BACKEND[], Int32, ns), ea = KA.zeros(BACKEND[], Int32, ns), - trsm = KA.zeros(BACKEND[], Int32, ns), tiles = KA.zeros(BACKEND[], Int32, ns), - pdone = KA.zeros(BACKEND[], Int32, ns), cbd = KA.zeros(BACKEND[], Int32, ns), - s2 = nothing) -end - -function refact_fused!(s, nzval, fp; nstreams::Int = 1) - N, S = s.numeric, s.symbolic - plan = N.plan - backend = KA.get_backend(N.factor) - p = SDS._front_impls(N, S, :auto) - copyto!(fp.arrived, fp.arrived_base) - copyto!(fp.ea, fp.ea_base) - copyto!(fp.trsm, fp.trsm_base) - copyto!(fp.tiles, fp.tiles_base) - copyto!(fp.cbd, fp.cb_base) - fill!(fp.pdone, Int32(0)) - for k in eachindex(plan.sub_first) - launch_subtrees_sorted!(N, S, nzval, fp.ssub[k], plan.sub_local[k], backend) - end - nodes = S.schedule.group_nodes - fill!(fp.pcb, 0.0) - isempty(fp.czero) || - panel_zero_kernel!(backend, WGF)(N.factor, fp.czero, S.front_ptr, S.front_nrows, S.front_ncols, - Val(WGF); ndrange = WGF * length(fp.czero)) - for k in eachindex(plan.group_first) # scatter A into every C-group front (panel only) - plan.group_width[k] > 0 && continue - a, b = plan.group_first[k], plan.group_last[k] - SDS.scatter_A!(N, S, nzval, a, b - a + 1) - end - for (base, count, gwides) in fp.segments - count > 0 && - fused_fact_kernel!(backend, WGF)(N.factor, N.stack, fp.pcb, N.info, nzval, S.amap, S.amap_ptr, - S.amap_src, fp.cb2, fp.wt, fp.role, fp.ra, fp.rb, fp.rc, fp.nchild, - fp.arrived, fp.ea, fp.pdone, fp.trsm, fp.tiles, fp.cbd, - S.snparent, S.front_ptr, S.front_nrows, S.front_ncols, - S.cb_ptr, S.child_ptr, S.child_list, S.relind_ptr, S.relind, - Int32(base), Val(WGF); ndrange = WGF * count) - if true - for v in gwides - SDS._factor_panel_c!(N.factor, fp.pcb, N.work, N.info, v, S.layout.panel_ptr[v], - S.schedule.rows[v], S.schedule.width[v], fp.cb2_h[v], p) - signal_kernel!(backend, 32)(fp.cbd, fp.arrived, S.snparent, v; ndrange = 32) - end - elseif false - nothing - end - end - SDS.assemble_schur!(N, S, nzval) - SDS.cholesky_stats!(N, S) - return SDS._numeric_info!(N, S, true) -end - - -end # module diff --git a/bench/e2e/amd_baseline.jl b/bench/e2e/amd_baseline.jl new file mode 100644 index 0000000..bf7b65c --- /dev/null +++ b/bench/e2e/amd_baseline.jl @@ -0,0 +1,63 @@ +# Radeon VII (gfx906) baseline: fused Cholesky (campaign champion knobs) and the +# first fused-LDL run on AMD, gated, interleaved. +# ~/.juliaup/bin/julia +1.13 --startup-file=no --project=~/.julia/environments/sdsamd \ +# bench/e2e/amd_baseline.jl +using SparseDirectSolver, SparseArrays, LinearAlgebra, Random, Statistics, Printf +using AMDGPU +const SDS = SparseDirectSolver +const SDSFused = SparseDirectSolver.Fused +const BD = joinpath(@__DIR__, "..") +include(joinpath(BD, "matrices.jl")) +using .BenchMatrices: read_mtx + +A = SparseMatrixCSC{Float64, Int}(sparse(read_mtx(joinpath(BD, "data", + "kkt_pglib_opf_case78484_epigrids_condensed_10.mtx")))) +n = size(A, 1) +Lh = tril(A) +Random.seed!(666) +bh = rand(n) +S3 = (p = Vector{Int32}(undef, n); read!(joinpath(BD, "perm_cand3.bin"), p); Int.(p)) +sync() = AMDGPU.synchronize() + +# build via plain device arrays (AMDGPU has no CuSparseMatrixCSR; DirectSolver takes raw triplets) +function mks(structure) + Lt = sparse(transpose(Lh)) # CSC of upper = CSR of lower + colptr = ROCArray(Vector{Int}(Lt.colptr)) + rowval = ROCArray(Vector{Int}(Lt.rowval)) + nzval = ROCArray(Vector{Float64}(Lt.nzval)) + s = DirectSolver(colptr, rowval, nzval, structure, 'L') + s.options.subtree_parallelism = 16384 + s.options.subtree_budgets = [8192] # campaign AMD champion + s.options.regime_c_width = 32 + s.options.regime_c_rows = 64 + s.options.amalgamation = (max_width = 48, zero_fraction = 0.25, min_width = 8) + SDS.setparam!(s, "user_perm", S3) + structure == "S" && SDS.setparam!(s, "pivot_type", 'N') + bd = ROCArray(bh); xd = similar(bd) + ex(ph) = SDS.execute!(ph, s, xd, bd; asynchronous = false) + ex("analysis"); ex("factorization"); ex("refactorization"); ex("solve"); sync() + return s, bd, xd +end + +say(x...) = (println(x...); flush(stdout)) +say("building solvers…") +sc, bdc, xdc = mks("SPD") +sl, bdl, xdl = mks("S") +fac0 = Array(sl.numeric.factor); d0 = Array(sl.numeric.d) +nzc = SDS._factor_values(sc); nzl = SDS._factor_values(sl) +# campaign AMD champion: merged in-kernel wides, wgf 256, spin cap 16 +fpc = SDSFused.build_fused_plan(sc; inkernel_wides = true, merge_segments = true, wgf = 256, scap = 16) +fpl = SDSFused.build_fused_plan(sl; ldl = true, merge_segments = true, wgf = 256, scap = 16) +infoc = SDSFused.refact_fused!(sc, nzc, fpc); sync() +infol = SDSFused.refact_fused!(sl, nzl, fpl); sync() +df = maximum(abs, Array(sl.numeric.factor) - fac0) / maximum(abs, fac0) +dd = maximum(abs, Array(sl.numeric.d) - d0) / maximum(abs, d0) +@printf "gates: chol info %d ldl info %d Δfactor %.2e Δd %.2e\n" infoc infol df dd +(infoc == 0 && infol == 0 && df < 1e-6 && dd < 1e-6) || error("AMD gate FAILED") +tc = Float64[]; tl = Float64[] +for r in 1:7 + push!(tc, @elapsed (SDSFused.refact_fused!(sc, nzc, fpc); sync())) + push!(tl, @elapsed (SDSFused.refact_fused!(sl, nzl, fpl); sync())) +end +@printf "gfx906 interleaved chol %7.2f ms ldl %7.2f ms ratio %.3f\n" median(tc)*1e3 median(tl)*1e3 median(tl)/median(tc) +say("done") diff --git a/bench/e2e/amd_sweep.jl b/bench/e2e/amd_sweep.jl new file mode 100644 index 0000000..c008e9c --- /dev/null +++ b/bench/e2e/amd_sweep.jl @@ -0,0 +1,83 @@ +# gfx906 knob sweep around the champion config (chol fused refact, gated, each +# config interleaved against the champion in the same process). +# julia +1.13 --project=~/.julia/environments/sdsamd bench/e2e/amd_sweep.jl +using SparseDirectSolver, SparseArrays, LinearAlgebra, Random, Statistics, Printf +using AMDGPU +const SDS = SparseDirectSolver +const SDSFused = SparseDirectSolver.Fused +const BD = joinpath(@__DIR__, "..") +include(joinpath(BD, "matrices.jl")) +using .BenchMatrices: read_mtx + +A = SparseMatrixCSC{Float64, Int}(sparse(read_mtx(joinpath(BD, "data", + "kkt_pglib_opf_case78484_epigrids_condensed_10.mtx")))) +n = size(A, 1) +Lh = tril(A) +Random.seed!(666) +bh = rand(n) +S3 = (p = Vector{Int32}(undef, n); read!(joinpath(BD, "perm_cand3.bin"), p); Int.(p)) +sync() = AMDGPU.synchronize() +say(x...) = (println(x...); flush(stdout)) + +function mks(; budgets = [8192], amal = (max_width = 48, zero_fraction = 0.25, min_width = 8)) + Lt = sparse(transpose(Lh)) + s = DirectSolver(ROCArray(Vector{Int}(Lt.colptr)), ROCArray(Vector{Int}(Lt.rowval)), + ROCArray(Vector{Float64}(Lt.nzval)), "SPD", 'L') + s.options.subtree_parallelism = 16384 + s.options.subtree_budgets = budgets + s.options.regime_c_width = 32 + s.options.regime_c_rows = 64 + s.options.amalgamation = amal + SDS.setparam!(s, "user_perm", S3) + bd = ROCArray(bh); xd = similar(bd) + ex(ph) = SDS.execute!(ph, s, xd, bd; asynchronous = false) + ex("analysis"); ex("factorization"); ex("refactorization"); ex("solve"); sync() + return s, Array(s.numeric.factor) # stock factor = per-solver gate reference +end + +# champion baseline solver (fixed) for interleaving +sb, facb = mks() +nzb = SDS._factor_values(sb) +fpb = SDSFused.build_fused_plan(sb; inkernel_wides = true, merge_segments = true, wgf = 256, scap = 16) +SDSFused.refact_fused!(sb, nzb, fpb); sync() + +function trial(label, s, fac0, fp) + nz = SDS._factor_values(s) + info = SDSFused.refact_fused!(s, nz, fp); sync() + df = maximum(abs, Array(s.numeric.factor) - fac0) / maximum(abs, fac0) + ok = info == 0 && df < 1e-6 + tb = Float64[]; tt = Float64[] + for r in 1:5 + push!(tb, @elapsed (SDSFused.refact_fused!(sb, nzb, fpb); sync())) + push!(tt, @elapsed (SDSFused.refact_fused!(s, nz, fp); sync())) + end + @printf "%-34s %s trial %7.2f ms champion %7.2f ms delta %+6.1f%%\n" label (ok ? "OK " : "BAD") median(tt)*1e3 median(tb)*1e3 (median(tt)/median(tb) - 1)*100 + flush(stdout) +end + +# plan-level knobs on the SAME solver/symbolic (cheap) +for (wgf, scap) in ((256, 8), (256, 32), (256, 64), (512, 16), (128, 16)) + fp = SDSFused.build_fused_plan(sb; inkernel_wides = true, merge_segments = true, wgf = wgf, scap = scap) + trial("wgf=$wgf scap=$scap", sb, facb, fp) +end + +# symbolic-level knobs (re-analysis each) +for budgets in ([4096], [16384]) + s, f0 = mks(; budgets) + fp = SDSFused.build_fused_plan(s; inkernel_wides = true, merge_segments = true, wgf = 256, scap = 16) + trial("budgets=$budgets", s, f0, fp) +end +for mw in (64, 96) + s, f0 = mks(; amal = (max_width = mw, zero_fraction = 0.25, min_width = 8)) + fp = SDSFused.build_fused_plan(s; inkernel_wides = true, merge_segments = true, wgf = 256, scap = 16) + trial("amal max_width=$mw", s, f0, fp) +end +say("done") + +# EA chunk size (plan-level) +for eac in (8192, 32768, 65536) + fp = SDSFused.build_fused_plan(sb; inkernel_wides = true, merge_segments = true, wgf = 256, + scap = 16, ea_chunk = eac) + trial("ea_chunk=$eac", sb, facb, fp) +end +say("done2") diff --git a/bench/e2e/cpu_fast_test.jl b/bench/e2e/cpu_fast_test.jl new file mode 100644 index 0000000..96ffcd4 --- /dev/null +++ b/bench/e2e/cpu_fast_test.jl @@ -0,0 +1,43 @@ +# CPU fast-path prototype: gate vs stock refactorization + timing, 78k KKT. +# julia +1.13 -t --project=bench bench/e2e/cpu_fast_test.jl +using SparseDirectSolver, SparseArrays, LinearAlgebra, Random, Statistics, Printf +const SDS = SparseDirectSolver +include(joinpath(@__DIR__, "SDSCPU.jl")) +using .SDSCPU +const BD = joinpath(@__DIR__, "..") +include(joinpath(BD, "matrices.jl")) +using .BenchMatrices: read_mtx + +A = SparseMatrixCSC{Float64, Int}(sparse(read_mtx(joinpath(BD, "data", + "kkt_pglib_opf_case78484_epigrids_condensed_10.mtx")))) +n = size(A, 1) +Lt = sparse(transpose(tril(A))) +Random.seed!(666) +bh = rand(n) +S3 = (p = Vector{Int32}(undef, n); read!(joinpath(BD, "perm_cand3.bin"), p); Int.(p)) + +s = DirectSolver(Vector{Int}(Lt.colptr), Vector{Int}(Lt.rowval), Vector{Float64}(Lt.nzval), "SPD", 'L') +s.options.subtree_parallelism = 16384 +s.options.subtree_budgets = [16384] +# CPU wants wider supernodes than the GPU champion (48): amal 96 = -17% at 24t, +# interleaved (fewer, bigger fronts = fewer memory passes + BLAS3-friendlier) +s.options.amalgamation = (max_width = parse(Int, get(ENV, "SDSCPU_AMAL", "96")), + zero_fraction = 0.25, min_width = 8) +SDS.setparam!(s, "user_perm", S3) +x = zeros(n); b = copy(bh) +ex(ph) = SDS.execute!(ph, s, x, b; asynchronous = false) +ex("analysis"); ex("factorization"); ex("refactorization") +tr0 = median([(@elapsed ex("refactorization")) for _ in 1:3]) +fac0 = copy(s.numeric.factor) +nz = SDS._factor_values(s) + +ctx = SDSCPU.CPUCtx(s) +info = SDSCPU.refact_cpu!(s, nz, ctx) +df = maximum(abs, s.numeric.factor - fac0) / maximum(abs, fac0) +@printf "threads %d gate: info %d Δfactor %.2e\n" Threads.nthreads() info df +(info == 0 && df < 1e-6) || error("CPU fast path gate FAILED") +tr1 = median([(@elapsed SDSCPU.refact_cpu!(s, nz, ctx)) for _ in 1:5]) +ex("solve") +rel = norm(bh - A * x) / norm(bh) +@printf "threads %d stock refact %8.1f ms fast refact %8.1f ms speedup %.2fx relres(stock solve) %.2e\n" Threads.nthreads() tr0*1e3 tr1*1e3 tr0/tr1 rel +println("done") diff --git a/bench/e2e/fresh_machine.jl b/bench/e2e/fresh_machine.jl new file mode 100644 index 0000000..4eb8199 --- /dev/null +++ b/bench/e2e/fresh_machine.jl @@ -0,0 +1,93 @@ +# MadNLP + SparseDirectSolver workflow on a fresh GPU machine, one file: +# +# julia +1.13 --startup-file=no fresh_machine.jl +# julia +1.13 --startup-file=no -i fresh_machine.jl (drop into a REPL after) +# +# Installs everything into its own environment (~/.sds-fresh-), downloads +# the pglib case on first use, then solves the 78k-bus ACOPF with MadNLP on the +# GPU using SparseDirectSolver (the SDSSolver wrapper) as the linear solver. +# The vendor is autodetected; force it with SDS_BACKEND=cuda|amdgpu. (AMD needs a +# system ROCm installation.) + +import Pkg +# vendor detection must RUN the tool, not just find it: cluster nodes often have +# nvidia-smi on PATH without an NVIDIA driver (and vice versa) +_works(cmd) = try + success(pipeline(cmd; stdout = devnull, stderr = devnull)) +catch + false +end +# device files first: they answer correctly even when NVML/nvidia-smi is wedged +# (driver updated under a loaded module — a real failure mode); tools as fallback +_has_nvidia_gpu() = !isempty(filter(f -> occursin(r"^nvidia\d+$", f), readdir("/dev"))) || + try occursin("GPU", read(`nvidia-smi -L`, String)) catch; false end +_has_amd_gpu() = ispath("/dev/kfd") || _works(`rocm-smi`) || _works(`rocminfo`) +const BACKEND = get(ENV, "SDS_BACKEND") do + _has_nvidia_gpu() ? "cuda" : + _has_amd_gpu() ? "amdgpu" : + error("no GPU found (/dev/nvidia*, /dev/kfd, nvidia-smi, rocm-smi all absent); set SDS_BACKEND=cuda|amdgpu") +end +Pkg.activate(joinpath(homedir(), ".sds-fresh-" * BACKEND)) +Pkg.add(["MadNLP", "MadNLPGPU", "ExaModels", "ExaModelsPower", "Metis"]) +Pkg.add(BACKEND == "cuda" ? ["CUDA", "CUDSS"] : ["AMDGPU"]) +# SDS as a proper dev checkout (editable, canonical path), not a frozen Pkg.add; +# an existing checkout is updated to the branch head (local edits block the pull loudly) +const SDS_DEV = joinpath(homedir(), ".julia", "dev", "SparseDirectSolver") +const SDS_REF = "main" +if isdir(SDS_DEV) + # the checkout may be shared (NFS home) with a concurrent run: retry once on + # git lock contention, then proceed on the existing tree with a warning + ok = success(`git -C $SDS_DEV fetch origin $SDS_REF`) || + (sleep(5); success(`git -C $SDS_DEV fetch origin $SDS_REF`)) + if ok + run(`git -C $SDS_DEV checkout $SDS_REF`) + run(`git -C $SDS_DEV merge --ff-only FETCH_HEAD`) + else + @warn "could not update $SDS_DEV (concurrent run?); continuing on the existing checkout" + end +else + run(`git clone --branch $SDS_REF https://github.com/exanauts/SparseDirectSolver.jl $SDS_DEV`) +end +Pkg.develop(path = SDS_DEV) +Pkg.instantiate() + +using SparseDirectSolver, MadNLP, MadNLPGPU, ExaModels, ExaModelsPower +using Metis +if BACKEND == "cuda" + using CUDA, CUDSS # CUDSS must be LOADED: MadNLPGPU's CUDA extension (the GPU +else # condensed-KKT constructors) activates only with it present + using AMDGPU +end + +# the MadNLP linear-solver wrapper (SDSSolver: the fused kernels with per-arch +# tuning), shipped with the package under bench/e2e +# the MadNLP interface is the SparseDirectSolverMadNLPExt package extension + +println("device: ", BACKEND == "cuda" ? CUDA.name(CUDA.device()) : string(AMDGPU.device())) + +m, _ = ac_opf_model("pglib_opf_case78484_epigrids.m"; + backend = BACKEND == "cuda" ? CUDABackend() : ROCBackend()) + +# SparseCondensedKKTSystem is required (the SPD Cholesky path); MadNLP defaults +# equality/fixed-variable treatment correctly for it. tol: condensed default is 1e-4. +# +# All performance knobs live in one bundle: pass sds_tuning = SparseDirectSolver.Fused.SDSTuning(; ...) +# to override the measured per-architecture defaults (see bench/ARCH-TUNING.md); +# sds_user_perm = perm (1-based, length n) bypasses the built-in reordering entirely. +solve() = madnlp(m; linear_solver = SDSSolver, + kkt_system = MadNLP.SparseCondensedKKTSystem, tol = 1e-6) + +t1 = @elapsed sol = solve() +println("\n== first solve (pays Julia compilation): ", round(t1; digits = 1), " s") +println(" status = ", sol.status, " iters = ", sol.counters.k, " obj = ", sol.objective) +for f in propertynames(sol.counters) + v = getfield(sol.counters, f) + v isa Real && endswith(String(f), "_time") && f !== :start_time && + println(" ", f, " = ", round(v; digits = 2), " s") +end +# the honest speed is the warm number: same process, nothing cached between the +# two solves except compiled code -- the reordering and symbolic analysis run in +# full both times (the extension caches nothing) +t2 = @elapsed sol2 = solve() +println("== second solve (warm): ", round(t2; digits = 1), " s", + " iters = ", sol2.counters.k, " obj = ", sol2.objective) diff --git a/bench/e2e/fresh_machine.sh b/bench/e2e/fresh_machine.sh new file mode 100755 index 0000000..03c386a --- /dev/null +++ b/bench/e2e/fresh_machine.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Fresh-machine test of the MadNLP + SparseDirectSolver workflow, on the @sds named +# PROJECT environment — no ad-hoc Pkg.add: the environment's Project.toml is generated +# from bench/e2e/Project.toml with SparseDirectSolver wired to the local checkout, and +# Pkg.instantiate() provides everything from the project file. +# +# curl -fsSL https://raw.githubusercontent.com/exanauts/SparseDirectSolver.jl/main/bench/e2e/fresh_machine.sh | bash +# +# When a terminal is attached, the workflow ends in an interactive REPL (julia -i) +# with `m`, `sol`, `sol2` and `solve()` defined; detached (nohup/sbatch, no tty) it +# just runs and exits. After one run the environment is reusable from anywhere: +# +# julia +1.13 --startup-file=no --project=@sds bench/e2e/workflow.jl +# +# Knobs (env vars): +# SDS_BRANCH=main git ref to test +# SDS_DIR=$HOME/sds-test checkout location +# SDS_BACKEND=cuda|amdgpu GPU vendor override (autodetected in workflow.jl) +# RUN_TESTS=1 also run the package test suite first (long) +# +# Needs: git, curl, network, and a GPU + driver. NVIDIA: CUDA toolkit and cuDSS come +# as artifacts. AMD: a system ROCm installation is required. +# NO `exec < /dev/null` here: under `curl | bash` the script itself arrives on stdin, +# and redirecting it away silently discards the rest of the script (measured: bash +# executes only its first read-ahead buffer and exits 0). Children that might read +# stdin get a per-command `< /dev/null` instead. +set -euo pipefail + +SDS_BRANCH="${SDS_BRANCH:-main}" +SDS_DIR="${SDS_DIR:-$HOME/sds-test}" + +# --- 1. Julia 1.13 via juliaup ----------------------------------------------- +if ! command -v juliaup >/dev/null 2>&1 && [ ! -x "$HOME/.juliaup/bin/juliaup" ]; then + curl -fsSL https://install.julialang.org | sh -s -- --yes < /dev/null +fi +export PATH="$HOME/.juliaup/bin:$PATH" +juliaup add 1.13 2>/dev/null || true +JL="julia +1.13 --startup-file=no" +$JL --version + +# --- 2. Repository ------------------------------------------------------------ +if [ ! -d "$SDS_DIR/.git" ]; then + git clone --branch "$SDS_BRANCH" https://github.com/exanauts/SparseDirectSolver.jl.git "$SDS_DIR" +fi +cd "$SDS_DIR" +git fetch origin "$SDS_BRANCH" +git checkout "$SDS_BRANCH" +git merge --ff-only "origin/$SDS_BRANCH" || true +echo "== testing $(git log --oneline -1)" + +# --- 3. The @sds environment, generated from the project file (no Pkg.add) ----- +$JL -e ' +using TOML +dir = pwd() +proj = TOML.parsefile(joinpath(dir, "bench", "e2e", "Project.toml")) +get!(proj, "sources", Dict{String, Any}())["SparseDirectSolver"] = Dict("path" => dir) +env = joinpath(DEPOT_PATH[1], "environments", "sds") +mkpath(env) +open(io -> TOML.print(io, proj), joinpath(env, "Project.toml"), "w") +println("== @sds environment: SparseDirectSolver -> ", dir) +' +$JL --project=@sds -e 'using Pkg; Pkg.instantiate()' + +# --- 4. Optional: the package test suite --------------------------------------- +if [ "${RUN_TESTS:-0}" = "1" ]; then + echo "== package test suite" + $JL --project=. -e 'using Pkg; Pkg.instantiate(); Pkg.test()' < /dev/null +fi + +# --- 5. The workflow: 78k-bus ACOPF, two solves, timing split ------------------ +# with a terminal attached, end in a REPL (m, sol, sol2, solve() are defined) +echo "== MadNLP + SDS workflow (downloads the pglib case on first use)" +if [ -e /dev/tty ] && [ -t 1 ]; then + exec $JL -i --project=@sds bench/e2e/workflow.jl < /dev/tty +else + $JL --project=@sds bench/e2e/workflow.jl + echo "== fresh-machine run complete" +fi diff --git a/bench/e2e/hsl_cpu_bench.jl b/bench/e2e/hsl_cpu_bench.jl new file mode 100644 index 0000000..bd98cc7 --- /dev/null +++ b/bench/e2e/hsl_cpu_bench.jl @@ -0,0 +1,56 @@ +# CPU state-of-the-art baseline on the 78k KKT: MA27, MA57 (serial), MA97 (OpenMP), +# via the registered MadNLPHSL wrappers (default options = each solver's own ordering). +# julia +1 --startup-file=no --project=~/.julia/environments/hslbench bench/e2e/hsl_cpu_bench.jl +using SparseArrays, LinearAlgebra, Random, Statistics, Printf +using MadNLPHSL, HSL +import MadNLP: factorize!, solve! +const BD = joinpath(@__DIR__, "..") +include(joinpath(BD, "matrices.jl")) +using .BenchMatrices: read_mtx + +A = SparseMatrixCSC{Float64, Int}(sparse(read_mtx(joinpath(BD, "data", + "kkt_pglib_opf_case78484_epigrids_condensed_10.mtx")))) +n = size(A, 1) +L = tril(A) +csc32 = SparseMatrixCSC{Float64, Int32}(Int32(n), Int32(n), Int32.(L.colptr), Int32.(L.rowval), L.nzval) +Random.seed!(666) +bh = rand(n) +say(x...) = (println(x...); flush(stdout)) + +function bench(label, M) + tf = @elapsed factorize!(M) + tr = median([(@elapsed factorize!(M)) for _ in 1:5]) + x = copy(bh); solve!(M, x) + ts = median([begin x .= bh; @elapsed solve!(M, x) end for _ in 1:5]) + x .= bh; solve!(M, x) + rel = norm(bh - A * x) / norm(bh) + @printf "%-12s first %8.1f ms refact %8.1f ms solve %6.1f ms relres %.2e\n" label tf*1e3 tr*1e3 ts*1e3 rel + flush(stdout) +end + +try +say("MA27 (serial)…") +ta = @elapsed M27 = MadNLPHSL.Ma27Solver(csc32; opt = MadNLPHSL.Ma27Options(ma27_liw_init_factor = 15.0, ma27_la_init_factor = 30.0, ma27_meminc_factor = 3.0)) +say(" init ", round(ta; digits = 2), " s") +bench("MA27", M27) +catch e + say("MA27 FAILED: ", typeof(e)) +end + +try +say("MA57 (serial + BLAS)…") +ta = @elapsed M57 = MadNLPHSL.Ma57Solver(csc32) +say(" init ", round(ta; digits = 2), " s") +bench("MA57", M57) +catch e + say("MA57 FAILED: ", typeof(e)) +end + +say("MA97 (OpenMP)…") +ta = @elapsed M97 = MadNLPHSL.Ma97Solver(csc32) +say(" init ", round(ta; digits = 2), " s") +for nt in (1, 8, 24) + HSL.omp_set_num_threads(Cint(nt)) + bench("MA97 nt=$nt", M97) +end +say("done") diff --git a/bench/e2e/ldl_fused_race.jl b/bench/e2e/ldl_fused_race.jl new file mode 100644 index 0000000..757ffee --- /dev/null +++ b/bench/e2e/ldl_fused_race.jl @@ -0,0 +1,61 @@ +# 20 gated fused-LDL refactorizations (race validation) + the in-kernel-wides +# Cholesky control that isolates the wide-strategy cost from the LDL cost. +# CUDA_VISIBLE_DEVICES=1 julia +1.13 --project=bench bench/e2e/ldl_fused_race.jl +using SparseDirectSolver, SparseArrays, LinearAlgebra, Random, Statistics, Printf +using CUDA, CUDA.CUSPARSE +const SDS = SparseDirectSolver +const SDSFused = SparseDirectSolver.Fused +const BD = joinpath(@__DIR__, "..") +include(joinpath(BD, "matrices.jl")) +using .BenchMatrices: read_mtx + +A = SparseMatrixCSC{Float64, Int}(sparse(read_mtx(joinpath(BD, "data", + "kkt_pglib_opf_case78484_epigrids_condensed_10.mtx")))) +n = size(A, 1) +Lh = tril(A) +Random.seed!(666) +bh = rand(n) +S3 = (p = Vector{Int32}(undef, n); read!(joinpath(BD, "perm_cand3.bin"), p); Int.(p)) + +function setup(structure) + bd = CuArray(bh); xd = similar(bd) + s = DirectSolver(CuSparseMatrixCSR(Lh), structure, 'L') + SDS.setparam!(s, "user_perm", S3) + structure == "S" && SDS.setparam!(s, "pivot_type", 'N') + ex(ph) = SDS.execute!(ph, s, xd, bd; asynchronous = false) + ex("analysis"); ex("factorization"); ex("refactorization"); ex("solve") + CUDA.synchronize() + return s +end + +s = setup("S") +fac0 = Array(s.numeric.factor) +d0 = Array(s.numeric.d) +mref = maximum(abs, fac0) +nz = SDS._factor_values(s) +fp = SDSFused.build_fused_plan(s; ldl = true, wides_potrf = true, wgf = 128, scap = 128) +nbad = 0 +for t in 1:20 + info = SDSFused.refact_fused!(s, nz, fp); CUDA.synchronize() + df = maximum(abs, Array(s.numeric.factor) - fac0) / mref + dd = maximum(abs, Array(s.numeric.d) - d0) / maximum(abs, d0) + ok = info == 0 && df < 1e-6 && dd < 1e-6 + ok || (global nbad += 1) + @printf "run %2d %s info %d Δfactor %.2e Δd %.2e\n" t (ok ? "OK " : "BAD") info df dd + flush(stdout) +end +println("bad: ", nbad, " of 20") +nbad == 0 || error("fused LDL race gate FAILED: $nbad of 20 runs bad") # a gate must exit nonzero + +# Cholesky with the SAME wide strategy as the LDL plan above (wides_potrf forces +# HOST wides, so the matching control is the host-wides Cholesky plan -- the +# in-kernel plan used before was a different strategy and confounded the ratio) +s2 = setup("SPD") +nz2 = SDS._factor_values(s2) +fp2h = SDSFused.build_fused_plan(s2; wgf = 128, scap = 128) # host wides (default) +SDSFused.refact_fused!(s2, nz2, fp2h) +tr_chol_host = median([CUDA.@elapsed SDSFused.refact_fused!(s2, nz2, fp2h) for _ in 1:7]) +tr_ldl = median([CUDA.@elapsed SDSFused.refact_fused!(s, nz, fp) for _ in 1:7]) +@printf "fused SPD (host wides) refact %8.2f ms\n" tr_chol_host*1e3 +@printf "fused S/N (host wides) refact %8.2f ms ratio same-strategy %.3f\n" tr_ldl*1e3 tr_ldl/tr_chol_host +println("done") diff --git a/bench/e2e/ldl_fused_test.jl b/bench/e2e/ldl_fused_test.jl new file mode 100644 index 0000000..56139ce --- /dev/null +++ b/bench/e2e/ldl_fused_test.jl @@ -0,0 +1,84 @@ +# Fused no-pivot LDL vs stock LDL (pivot 'N') on the 78k ACOPF condensed KKT, plus +# the fused-Cholesky control in the same process. +# CUDA_VISIBLE_DEVICES=1 julia +1.13 --project=bench bench/e2e/ldl_fused_test.jl +using SparseDirectSolver, SparseArrays, LinearAlgebra, Random, Statistics, Printf +using CUDA, CUDA.CUSPARSE +const SDS = SparseDirectSolver +const SDSFused = SparseDirectSolver.Fused + +const BD = joinpath(@__DIR__, "..") +include(joinpath(BD, "matrices.jl")) +using .BenchMatrices: read_mtx + +A = SparseMatrixCSC{Float64, Int}(sparse(read_mtx(joinpath(BD, "data", + "kkt_pglib_opf_case78484_epigrids_condensed_10.mtx")))) +n = size(A, 1) +Lh = tril(A) +Random.seed!(666) +bh = rand(n) +S3 = (p = Vector{Int32}(undef, n); read!(joinpath(BD, "perm_cand3.bin"), p); Int.(p)) + +function setup(structure) + bd = CuArray(bh); xd = similar(bd) + s = DirectSolver(CuSparseMatrixCSR(Lh), structure, 'L') + SDS.setparam!(s, "user_perm", S3) + structure == "S" && SDS.setparam!(s, "pivot_type", 'N') + ex(ph) = SDS.execute!(ph, s, xd, bd; asynchronous = false) + ex("analysis"); ex("factorization"); ex("refactorization"); ex("solve") + CUDA.synchronize() + return s, ex, bd, xd +end + +say(x...) = (println(x...); flush(stdout)) + +# --- stock LDL pivot-N: reference factor/d + timing --- +s, ex, bd, xd = setup("S") +tr_stock = median([CUDA.@elapsed ex("refactorization") for _ in 1:7]) +fac0 = Array(s.numeric.factor) +d0 = Array(s.numeric.d) +@printf "stock S/N refact %8.2f ms info %d\n" tr_stock*1e3 SDS.getparam(s, "info") +flush(stdout) + +# --- fused LDL on the same solver --- +nzval = SDS._factor_values(s) +fp = SDSFused.build_fused_plan(s; ldl = true, wgf = 128, scap = 128) +info = SDSFused.refact_fused!(s, nzval, fp) +CUDA.synchronize() +fac1 = Array(s.numeric.factor) +d1 = Array(s.numeric.d) +df = maximum(abs, fac1 - fac0) / maximum(abs, fac0) +dd = maximum(abs, d1 - d0) / maximum(abs, d0) +@printf "fused S/N info %d Δfactor %.2e Δd %.2e\n" info df dd +(info == 0 && df < 1e-6 && dd < 1e-6) || error("fused LDL gate FAILED") +tr_fused = median([CUDA.@elapsed SDSFused.refact_fused!(s, nzval, fp) for _ in 1:7]) +@printf "fused S/N refact %8.2f ms\n" tr_fused*1e3 +flush(stdout) + +# --- fused solve (inverse path + D pass) --- +inv11, inv_ptr, invertf, _ = SDSFused.build_inverse(s) +fz1 = SDSFused.build_fused(s; kstart = 1, wcap = 256) +fz = ([(fz1, 64)], true) +invertf() +x2 = similar(bd) +SDSFused.algo2_solve!(x2, s, bd, inv11, inv_ptr, fz) +CUDA.synchronize() +rel = norm(bh - A * Array(x2)) / norm(bh) +@printf "fused S/N solve relres %.2e\n" rel +# gate against the stock solve's residual on the same system (x holds the stock +# solution at this point), not an absolute threshold a zero vector nearly meets +rel_stock = norm(bh - A * Array(xd)) / norm(bh) +rel <= 10 * rel_stock + 1e-12 || + error("fused LDL solve gate FAILED: relres $rel vs stock $rel_stock") +ts_fused = median([CUDA.@elapsed SDSFused.algo2_solve!(x2, s, bd, inv11, inv_ptr, fz) for _ in 1:7]) +@printf "fused S/N solve %8.2f ms\n" ts_fused*1e3 +flush(stdout) + +# --- fused Cholesky control (fresh solver, same matrix as SPD) --- +s2, ex2, bd2, xd2 = setup("SPD") +nz2 = SDS._factor_values(s2) +fp2 = SDSFused.build_fused_plan(s2; wgf = 128, scap = 128) +info2 = SDSFused.refact_fused!(s2, nz2, fp2) +tr_chol = median([CUDA.@elapsed SDSFused.refact_fused!(s2, nz2, fp2) for _ in 1:7]) +@printf "fused SPD refact %8.2f ms info %d (control)\n" tr_chol*1e3 info2 +@printf "RATIO fused LDL / fused Cholesky = %.3f\n" tr_fused / tr_chol +say("done") diff --git a/bench/e2e/ma27_bench.jl b/bench/e2e/ma27_bench.jl new file mode 100644 index 0000000..2aab4e2 --- /dev/null +++ b/bench/e2e/ma27_bench.jl @@ -0,0 +1,88 @@ +# MA27 (serial multifrontal, 1983) on the 78k KKT, via libcoinhsl (the libHSL +# 2025 build STUBS the Archive codes: its ma27*_ symbols return instantly with +# zeroed output, and the subset build returns -26 "not available"). +# julia +1 --startup-file=no --project=bench bench/e2e/ma27_bench.jl +using SparseArrays, LinearAlgebra, Random, Statistics, Printf +const LIB = "/usr/local/lib/libcoinhsl.so" +const BD = joinpath(@__DIR__, "..") +include(joinpath(BD, "matrices.jl")) +using .BenchMatrices: read_mtx + +MAT = get(ENV, "MA27_MAT", "kkt_pglib_opf_case78484_epigrids_condensed_10.mtx") +A = SparseMatrixCSC{Float64, Int}(sparse(read_mtx(joinpath(BD, "data", MAT)))) +println("matrix: ", MAT, " n ", size(A, 1), " nnz(tril) ", nnz(tril(A))) +n = Int32(size(A, 1)) +L = tril(A) +nz = Int32(nnz(L)) +I32 = Vector{Int32}(undef, nz); J32 = Vector{Int32}(undef, nz) +let k = 0 + for j in 1:size(L, 2), p in L.colptr[j]:(L.colptr[j + 1] - 1) + k += 1 + I32[k] = Int32(L.rowval[p]) + J32[k] = Int32(j) + end +end +vals = L.nzval +Random.seed!(666) +bh = rand(Float64, Int(n)) +say(x...) = (println(x...); flush(stdout)) + +icntl = zeros(Int32, 30); cntl = zeros(Float64, 5) +@ccall LIB.ma27id_(icntl::Ptr{Int32}, cntl::Ptr{Float64})::Cvoid +icntl[1:2] .= 0 +cntl[1] = 1e-8 # pivtol as MadNLP's IPM default +info = zeros(Int32, 20) + +liw = Int32(8 * (2 * Int(nz) + 3 * Int(n) + 1)) +iw = Vector{Int32}(undef, liw) +ikeep = zeros(Int32, 3n) +iw1 = zeros(Int32, 2n) +nsteps = Int32[0]; iflag = Int32[0]; ops = Float64[0.0] +ta = @elapsed @ccall LIB.ma27ad_(n::Ref{Int32}, nz::Ref{Int32}, I32::Ptr{Int32}, J32::Ptr{Int32}, + iw::Ptr{Int32}, liw::Ref{Int32}, ikeep::Ptr{Int32}, iw1::Ptr{Int32}, + nsteps::Ptr{Int32}, iflag::Ptr{Int32}, icntl::Ptr{Int32}, + cntl::Ptr{Float64}, info::Ptr{Int32}, ops::Ptr{Float64})::Cvoid +say("analysis ", round(ta; digits = 2), " s info[1] ", info[1], " forecast la ", info[5], + " forecast liw ", info[6], " nsteps ", nsteps[1]) +info[1] >= 0 || error("ma27ad failed: info = $(info[1:4])") + +a = Vector{Float64}(undef, max(5 * Int(info[5]), 2 * Int(nz))) +resize!(iw, max(5 * Int(info[6]), 2 * Int(nz))) +maxfrt = Int32[0] +function factor!() + copyto!(a, 1, vals, 1, Int(nz)) + @ccall LIB.ma27bd_(n::Ref{Int32}, nz::Ref{Int32}, I32::Ptr{Int32}, J32::Ptr{Int32}, + a::Ptr{Float64}, Int32(length(a))::Ref{Int32}, iw::Ptr{Int32}, + Int32(length(iw))::Ref{Int32}, ikeep::Ptr{Int32}, nsteps::Ptr{Int32}, + maxfrt::Ptr{Int32}, iw1::Ptr{Int32}, icntl::Ptr{Int32}, cntl::Ptr{Float64}, + info::Ptr{Int32})::Cvoid + return Int(info[1]) +end +tf = 0.0; st = 0 +for attempt in 1:4 + global tf = @elapsed global st = factor!() + say("factor attempt ", attempt, ": info[1] ", st, " info[2] ", info[2], " maxfrt ", maxfrt[1]) + if st == -3 + resize!(iw, 2 * max(length(iw), Int(info[2]))) + elseif st == -4 + resize!(a, 2 * max(length(a), Int(info[2]))) + else + break + end +end +st >= 0 || error("ma27bd failed: info = $(info[1:4])") +say("first factor ", round(tf * 1e3; digits = 1), " ms factor-real info[9] ", info[9], + " factor-int info[10] ", info[10], " ncmpa info[11] ", info[11], " twos info[14] ", info[14]) +tr = median([(@elapsed factor!()) for _ in 1:5]) + +w = Vector{Float64}(undef, Int(maxfrt[1])) +x = copy(bh) +sv!() = @ccall LIB.ma27cd_(n::Ref{Int32}, a::Ptr{Float64}, Int32(length(a))::Ref{Int32}, + iw::Ptr{Int32}, Int32(length(iw))::Ref{Int32}, w::Ptr{Float64}, + maxfrt::Ptr{Int32}, x::Ptr{Float64}, iw1::Ptr{Int32}, nsteps::Ptr{Int32}, + icntl::Ptr{Int32}, info::Ptr{Int32})::Cvoid +sv!() +rel = norm(bh - A * x) / norm(bh) +ts = median([begin x .= bh; @elapsed sv!() end for _ in 1:5]) +@printf "MA27 refact %8.1f ms solve %6.1f ms relres %.2e\n" tr*1e3 ts*1e3 rel +say("done") diff --git a/bench/e2e/ma57_bench.jl b/bench/e2e/ma57_bench.jl new file mode 100644 index 0000000..8cb1692 --- /dev/null +++ b/bench/e2e/ma57_bench.jl @@ -0,0 +1,84 @@ +# MA57 (serial multifrontal with BLAS3, MA27's successor) via libcoinhsl. +# julia +1 --startup-file=no bench/e2e/ma57_bench.jl +using SparseArrays, LinearAlgebra, Random, Statistics, Printf +const LIB = "/usr/local/lib/libcoinhsl.so" +const BD = joinpath(@__DIR__, "..") +include(joinpath(BD, "matrices.jl")) +using .BenchMatrices: read_mtx + +MAT = get(ENV, "MA57_MAT", "kkt_pglib_opf_case78484_epigrids_condensed_10.mtx") +A = SparseMatrixCSC{Float64, Int}(sparse(read_mtx(joinpath(BD, "data", MAT)))) +println("matrix: ", MAT, " n ", size(A, 1), " nnz(tril) ", nnz(tril(A))) +n = Int32(size(A, 1)) +L = tril(A) +ne = Int32(nnz(L)) +I32 = Vector{Int32}(undef, ne); J32 = Vector{Int32}(undef, ne) +let k = 0 + for j in 1:size(L, 2), p in L.colptr[j]:(L.colptr[j + 1] - 1) + k += 1 + I32[k] = Int32(L.rowval[p]) + J32[k] = Int32(j) + end +end +vals = L.nzval +Random.seed!(666) +bh = rand(Float64, Int(n)) +say(x...) = (println(x...); flush(stdout)) + +cntl = zeros(Float64, 5); icntl = zeros(Int32, 20) +@ccall LIB.ma57id_(cntl::Ptr{Float64}, icntl::Ptr{Int32})::Cvoid +icntl[5] = 0 # print level +cntl[1] = 1e-8 # pivtol, as MadNLP +info = zeros(Int32, 40); rinfo = zeros(Float64, 20) +lkeep = Int32(7 * Int(n) + Int(ne) + max(Int(n), Int(ne)) + 42) +keep = zeros(Int32, lkeep) +iwork = zeros(Int32, 5 * Int(n)) +ta = @elapsed @ccall LIB.ma57ad_(n::Ref{Int32}, ne::Ref{Int32}, I32::Ptr{Int32}, J32::Ptr{Int32}, + lkeep::Ref{Int32}, keep::Ptr{Int32}, iwork::Ptr{Int32}, + icntl::Ptr{Int32}, info::Ptr{Int32}, rinfo::Ptr{Float64})::Cvoid +say("analysis ", round(ta; digits = 2), " s info[1] ", info[1], " forecast fact ", info[9], + " forecast ifact ", info[10]) +info[1] >= 0 || error("ma57ad failed: info = $(info[1:4])") + +lfact = Int32(max(3 * Int(info[9]), 2 * Int(ne))) +fact = Vector{Float64}(undef, lfact) +lifact = Int32(max(3 * Int(info[10]), 2 * Int(ne))) +ifact = Vector{Int32}(undef, lifact) +iwn = zeros(Int32, n) +function factor!() + @ccall LIB.ma57bd_(n::Ref{Int32}, ne::Ref{Int32}, vals::Ptr{Float64}, fact::Ptr{Float64}, + Int32(length(fact))::Ref{Int32}, ifact::Ptr{Int32}, + Int32(length(ifact))::Ref{Int32}, lkeep::Ref{Int32}, keep::Ptr{Int32}, + iwn::Ptr{Int32}, icntl::Ptr{Int32}, cntl::Ptr{Float64}, info::Ptr{Int32}, + rinfo::Ptr{Float64})::Cvoid + return Int(info[1]) +end +tf = 0.0; st = 0 +for attempt in 1:4 + global tf = @elapsed global st = factor!() + say("factor attempt ", attempt, ": info[1] ", st, " info[2] ", info[2]) + if st == -3 + resize!(fact, 2 * max(length(fact), Int(info[17]))) + elseif st == -4 + resize!(ifact, 2 * max(length(ifact), Int(info[18]))) + else + break + end +end +st >= 0 || error("ma57bd failed: info = $(info[1:4])") +say("first factor ", round(tf * 1e3; digits = 1), " ms factor-real info[14] ", info[14], + " negpiv info[24] ", info[24]) +tr = median([(@elapsed factor!()) for _ in 1:5]) + +x = copy(bh) +wrk = zeros(Float64, n) +sv!() = @ccall LIB.ma57cd_(Int32(1)::Ref{Int32}, n::Ref{Int32}, fact::Ptr{Float64}, + Int32(length(fact))::Ref{Int32}, ifact::Ptr{Int32}, + Int32(length(ifact))::Ref{Int32}, Int32(1)::Ref{Int32}, x::Ptr{Float64}, + n::Ref{Int32}, wrk::Ptr{Float64}, n::Ref{Int32}, iwn::Ptr{Int32}, + icntl::Ptr{Int32}, info::Ptr{Int32})::Cvoid +sv!() +rel = norm(bh - A * x) / norm(bh) +ts = median([begin x .= bh; @elapsed sv!() end for _ in 1:5]) +@printf "MA57 refact %8.1f ms solve %6.1f ms relres %.2e\n" tr*1e3 ts*1e3 rel +say("done") diff --git a/bench/e2e/run_amd.jl b/bench/e2e/run_amd.jl index 3d3fe4f..c5d6e24 100644 --- a/bench/e2e/run_amd.jl +++ b/bench/e2e/run_amd.jl @@ -8,8 +8,8 @@ using AMDGPU using MadNLP, MadNLPGPU using ExaModels, ExaModelsPower -include(joinpath(@__DIR__, "MadNLPSDS.jl")) -using .MadNLPSDS +using SparseDirectSolver +# the MadNLP interface is the SparseDirectSolverMadNLPExt package extension const CASE = "pglib_opf_case78484_epigrids.m" @@ -31,6 +31,5 @@ end println("building model on ROCm…"); flush(stdout) model, _ = ac_opf_model(CASE; backend = ROCBackend()) -run_one("SDS/AMD stock", model, SDSSolver) -run_one("SDS/AMD proto", model, SDSProtoSolver) +run_one("SDS/AMD", model, SDSSolver) println("done") diff --git a/bench/e2e/run_gv100.jl b/bench/e2e/run_gv100.jl index 8db2016..45a107e 100644 --- a/bench/e2e/run_gv100.jl +++ b/bench/e2e/run_gv100.jl @@ -8,8 +8,8 @@ using CUDA, CUDSS using MadNLP, MadNLPGPU using ExaModels, ExaModelsPower -include(joinpath(@__DIR__, "MadNLPSDS.jl")) -using .MadNLPSDS +using SparseDirectSolver +# the MadNLP interface is the SparseDirectSolverMadNLPExt package extension const CASE = "pglib_opf_case78484_epigrids.m" @@ -24,7 +24,6 @@ function run_one(label, model, linear_solver; warm = true) c = sol.counters @printf "%-16s status %-22s iters %3d obj %.8e\n" label string(sol.status) c.k sol.objective @printf " wall %7.2f s solver %7.2f s linsolve %7.2f s eval %7.2f s init %7.2f s\n" c.total_time c.solver_time c.linear_solver_time c.eval_function_time c.init_time - @printf " factorizations %d backsolves %d\n" c.factorization_cnt c.backsolve_cnt flush(stdout) return sol end @@ -32,5 +31,5 @@ end println("building model on CUDA…"); flush(stdout) model, _ = ac_opf_model(CASE; backend = CUDABackend()) run_one("cuDSS", model, MadNLPGPU.CUDSSSolver) -run_one("SDS-proto", model, SDSProtoSolver) +run_one("SDS", model, SDSSolver) println("done") diff --git a/bench/e2e/workflow.jl b/bench/e2e/workflow.jl new file mode 100644 index 0000000..a2f12be --- /dev/null +++ b/bench/e2e/workflow.jl @@ -0,0 +1,50 @@ +# The MadNLP + SparseDirectSolver 78k-bus ACOPF workflow, run on a project +# environment — the @sds named environment that fresh_machine.sh generates, or the +# in-repo bench/e2e one (its [sources] entry wires SparseDirectSolver either way, +# so `Pkg.instantiate()` is the only setup): +# +# julia +1.13 --startup-file=no --project=@sds bench/e2e/workflow.jl +# julia +1.13 --startup-file=no --project=bench/e2e bench/e2e/workflow.jl +# +# fresh_machine.jl inlines the same flow for a machine without a checkout. +# Backend is autodetected (SDS_BACKEND=cuda|amdgpu +# overrides); device files are consulted first, since nvidia-smi can be wedged by +# a driver updated under a loaded kernel module. +const BACKEND = get(ENV, "SDS_BACKEND") do + !isempty(filter(f -> occursin(r"^nvidia\d+$", f), readdir("/dev"))) ? "cuda" : + ispath("/dev/kfd") ? "amdgpu" : + error("no GPU found (/dev/nvidia*, /dev/kfd absent); set SDS_BACKEND=cuda|amdgpu") +end + +using SparseDirectSolver, MadNLP, MadNLPGPU, ExaModels, ExaModelsPower, Metis +if BACKEND == "cuda" + using CUDA, CUDSS # CUDSS must be LOADED: MadNLPGPU's CUDA extension (the GPU +else # condensed-KKT constructors) activates only with it present + using AMDGPU +end + +println("device: ", BACKEND == "cuda" ? CUDA.name(CUDA.device()) : string(AMDGPU.device())) + +m, _ = ac_opf_model("pglib_opf_case78484_epigrids.m"; + backend = BACKEND == "cuda" ? CUDABackend() : ROCBackend()) + +# SparseCondensedKKTSystem is required (the SPD Cholesky path). All performance +# knobs live in one bundle: sds_tuning = SparseDirectSolver.Fused.SDSTuning(; ...) +# overrides the per-architecture defaults (bench/ARCH-TUNING.md). +solve() = madnlp(m; linear_solver = SDSSolver, + kkt_system = MadNLP.SparseCondensedKKTSystem, tol = 1e-6) + +t1 = @elapsed sol = solve() +println("\n== first solve (pays Julia compilation): ", round(t1; digits = 1), " s") +println(" status = ", sol.status, " iters = ", sol.counters.k, " obj = ", sol.objective) +for f in propertynames(sol.counters) + v = getfield(sol.counters, f) + v isa Real && endswith(String(f), "_time") && f !== :start_time && + println(" ", f, " = ", round(v; digits = 2), " s") +end +# the honest speed is the warm number: same process, nothing carried between the +# two solves except compiled code — the reordering and symbolic analysis run in full +# both times (the extension caches nothing) +t2 = @elapsed sol2 = solve() +println("== second solve (warm): ", round(t2; digits = 1), " s", + " iters = ", sol2.counters.k, " obj = ", sol2.objective) diff --git a/docs/src/index.md b/docs/src/index.md index 6803149..94978b4 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -25,8 +25,8 @@ oneAPI, Metal and the KernelAbstractions CPU backend. ## Features -* **Analysis on the host**: AMD or nested dissection (METIS, through the Metis - extension) ordering, elimination tree, supernodes with GPU-tuned +* **Analysis on the host**: AMD or nested dissection (METIS) ordering, + elimination tree, supernodes with GPU-tuned amalgamation, a static factor layout and device assembly maps. For symmetric indefinite matrices the analysis pairs structurally zero pivots with a partner so that the in-front pivoting can form the 2×2 blocks of KKT systems. @@ -59,8 +59,8 @@ Pkg.add(url = "https://github.com/exanauts/SparseDirectSolver.jl") Loading CUDA.jl enables the CUDA extension (`CuSparseMatrixCSR` and `CuSparseMatrixCSC` inputs, cuBLAS and cuSOLVER dense kernels); loading AMDGPU.jl enables the AMDGPU extension (`ROCSparseMatrixCSR` and -`ROCSparseMatrixCSC` inputs, rocBLAS and rocSOLVER dense kernels). Loading Metis.jl -enables nested dissection ordering; Krylov.jl enables FGMRES refinement. +`ROCSparseMatrixCSC` inputs, rocBLAS and rocSOLVER dense kernels); loading +Krylov.jl enables FGMRES refinement. ## Where to go next diff --git a/docs/src/man/parameters.md b/docs/src/man/parameters.md index 2f792ca..55d7712 100644 --- a/docs/src/man/parameters.md +++ b/docs/src/man/parameters.md @@ -32,7 +32,7 @@ docstring. The ones that matter most in practice: | parameter | values | effect | | :--- | :--- | :--- | -| `reordering_alg` | `"default"`, `"algo1"`–`"algo5"` | automatic (AMD or, with `using Metis`, ND, whichever has the lower flops + 1e8 × supernodal schedule depth), AMD (`"algo3"`), nested dissection (`"algo4"`, needs `using Metis`), natural (`"algo5"`); see [`SparseDirectSolver.ReorderingAlg`](@ref) | +| `reordering_alg` | `"default"`, `"algo1"`–`"algo5"` | automatic (AMD or ND, whichever has the lower flops + 1e8 × supernodal schedule depth), AMD (`"algo3"`), nested dissection (`"algo4"`), natural (`"algo5"`); see [`SparseDirectSolver.ReorderingAlg`](@ref) | | `factorization_alg` | `"default"`, `"algo1"`, `"algo2"` | automatic, small-front kernels only, vendor dense calls for large fronts | | `matching_alg` | `"default"` (none), `"algo1"`–`"algo6"` | MC64 jobs 1–5 (`"algo6"`: automatic) before the ordering | | `pivot_type` | `'A'`, `'N'`, `'D'`, `'L'`, `'B'` | automatic, none, diagonal, local block, Bunch–Kaufman; global pivoting (`'C'`, `'R'`) is not supported | diff --git a/ext/SparseDirectSolverCUDAExt.jl b/ext/SparseDirectSolverCUDAExt.jl index d7c05f1..f551b5b 100644 --- a/ext/SparseDirectSolverCUDAExt.jl +++ b/ext/SparseDirectSolverCUDAExt.jl @@ -79,6 +79,14 @@ therefore stop at the 48 KiB class on CUDA (issue #60). """ SparseDirectSolver.max_local_bytes(::CUDABackend) = 49152 +""" + max_threads_per_multiprocessor(::CUDABackend) -> Int + +`CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR` of the current device. +""" +SparseDirectSolver.max_threads_per_multiprocessor(::CUDABackend) = + Int(CUDACore.attribute(CUDACore.device(), CUDACore.DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR)) + # --------------------------------------------------------------------------- # public API (T13, PLAN §3.1): solver constructors, update!, the LinearAlgebra layer diff --git a/ext/SparseDirectSolverMadNLPExt.jl b/ext/SparseDirectSolverMadNLPExt.jl new file mode 100644 index 0000000..81e1882 --- /dev/null +++ b/ext/SparseDirectSolverMadNLPExt.jl @@ -0,0 +1,157 @@ +# MadNLP linear-solver interface to SparseDirectSolver.jl, as a package +# extension (loads when MadNLP and SparseDirectSolver are both present). +# Backend-agnostic: accepts any device CSC (CUSPARSE.CuSparseMatrixCSC, +# rocSPARSE.ROCSparseMatrixCSC) whose colPtr/rowVal/nzVal are device vectors. +# The CSC arrays of MadNLP's lower-triangular KKT matrix are read as the CSR +# of the upper triangle, exactly as MadNLPGPU's CUDSSSolver does (view 'U'). +# Access from user code: SparseDirectSolver.SDSSolver (a declared global of the +# parent package this extension assigns in its __init__, the MadNLPGPU idiom). +module SparseDirectSolverMadNLPExt + +using LinearAlgebra, SparseArrays +using MadNLP +using SparseDirectSolver +const SDS = SparseDirectSolver +const KA = SDS.KernelAbstractions +const Fused = SDS.Fused +const SDSTuning = Fused.SDSTuning + +Base.@kwdef mutable struct SDSSolverOptions <: MadNLP.AbstractOptions + # :cholesky (SPD; the condensed-KKT default) or :ldl (no-pivot LDL^T: factors + # indefinite matrices and reports TRUE inertia from the signs of D) + sds_factorization::Symbol = :cholesky + sds_reordering::String = "algo4" # native ND + # ALL performance knobs live in the tuning bundle; nothing (the default) + # selects the measured per-architecture profile from Fused.default_tuning + sds_tuning::Union{Nothing, SDSTuning} = nothing + # a user permutation bypasses the reordering entirely (1-based, length n) + sds_user_perm::Union{Nothing, Vector{Int}} = nothing +end + +# shared solver construction knobs: the custom reordering (nd knobs or a user +# permutation) and the amalgamation, applied before the analysis. No caching of +# any kind: every solver instance recomputes its ordering (owner call, 2026-10-12); +# pin one explicitly with sds_user_perm. +function _apply_reordering!(s, opt::SDSSolverOptions, t::SDSTuning) + s.options.amalgamation = t.amalgamation + if opt.sds_user_perm !== nothing + SDS.setparam!(s, "user_perm", opt.sds_user_perm) + return nothing + end + SDS.setparam!(s, "reordering_alg", opt.sds_reordering) + SDS.setparam!(s, "nd_nseps", t.nd_nseps) + SDS.setparam!(s, "nd_seed", t.nd_seed) + return nothing +end + +""" + SDSSolver(csc; opt = SDSSolverOptions(), logger = MadNLP.MadNLPLogger()) + +The MadNLP linear solver backed by the fused SparseDirectSolver pipeline +(factorization mega-kernel + partitioned-inverse solve), with per-architecture +tuning from `Fused.default_tuning`. Available as `SparseDirectSolver.SDSSolver` +(exported) once MadNLP is loaded. +""" +mutable struct SDSSolver{T} <: MadNLP.AbstractLinearSolver{T} + inner::Any + tril::Any + x::Any + b::Any + nzval::Any + fp::Any # fused factorization plan + fz::Any # fused solve segments + inv11::Any + inv_ptr::Any + invert!::Any + last_info::Int + opt::SDSSolverOptions + logger::MadNLP.MadNLPLogger +end + +function SDSSolver( + csc; + opt = SDSSolverOptions(), + logger = MadNLP.MadNLPLogger(), + ) + T = eltype(csc.nzVal) + n = size(csc, 1) + backend = KA.get_backend(csc.nzVal) + t = Fused.resolve_tuning(something(opt.sds_tuning, Fused.default_tuning(backend)), backend) + ldl = opt.sds_factorization === :ldl + s = SDS.DirectSolver(csc.colPtr, csc.rowVal, csc.nzVal, ldl ? "S" : "SPD", 'U') + ldl && SDS.setparam!(s, "pivot_type", 'N') + # no-pivot LDL never consumes 2x2 pivot pairs, and "S" analysis pays dearly for + # detecting them on saddle systems (augmented 78k KKT, n = 2.09M: analysis 316 s + # with the default pair detection, 64 s without -- measured on a GV100) + ldl && SDS.setparam!(s, "pivot_pairs", "none") + s.options.regime_c_width = t.regime_c_width + s.options.regime_c_rows = t.regime_c_rows # the fused kernels own BOTH phases: no coupling conflict + s.options.subtree_parallelism = t.subtree_parallelism + s.options.subtree_budgets = copy(t.subtree_budgets) + _apply_reordering!(s, opt, t) + x = similar(csc.nzVal, n) + b = similar(csc.nzVal, n) + SDS.execute!("analysis", s, x, b; asynchronous = false) + SDS.execute!("factorization", s, x, b; asynchronous = false) + SDS.execute!("solve", s, x, b; asynchronous = false) # allocates the solve workspace + nzval = SDS._factor_values(s) + merged = t.wides === :merged + # LDL wide fronts run the verbatim Cholesky vendor path (wides_potrf): valid while a + # front's pivots share a sign; a sign flip fails potrf -> info != 0 -> MadNLP regularizes + fp = Fused.build_fused_plan(s; ldl, inkernel_wides = merged, merge_segments = merged, + wides_potrf = !merged, wgf = t.workgroup, scap = t.spin_cap) + inv11, inv_ptr, invertf, _ = Fused.build_inverse(s) + # the solve plan: one merged dependency-counter sweep over the whole tree + fz1 = Fused.build_fused(s; kstart = 1, wcap = 256) + fz1 === nothing && throw(SDS.NotSupportedError("fused solve: no plan entry with fronts of width <= 256")) + fz = ([(fz1, 64)], true) + M = SDSSolver{T}(s, csc, x, b, nzval, fp, fz, inv11, inv_ptr, invertf, 0, opt, logger) + MadNLP.factorize!(M) # prime the fused path once + return M +end + +function MadNLP.factorize!(M::SDSSolver) + M.last_info = Int(Fused.refact_fused!(M.inner, M.nzval, M.fp)) + M.invert!() + return M +end + +function _solve!(M::SDSSolver, xb) + copyto!(M.b, xb) + Fused.algo2_solve!(M.x, M.inner, M.b, M.inv11, M.inv_ptr, M.fz) + copyto!(xb, M.x) + KA.synchronize(KA.get_backend(xb)) + return xb +end + +MadNLP.solve!(M::SDSSolver, xb::AbstractVector) = _solve!(M, xb) +# the dev MadNLP routes through solve_linear_system!; the registered 0.8 does not have it +isdefined(MadNLP, :solve_linear_system!) && + @eval MadNLP.solve_linear_system!(M::SDSSolver, xb) = _solve!(M, xb) +MadNLP.input_type(::Type{SDSSolver}) = :csc +MadNLP.default_options(::Type{SDSSolver}) = SDSSolverOptions() +MadNLP.is_inertia(::SDSSolver) = true +# a failed factorization is reported as an indefinite inertia, so that MadNLP +# regularizes and factorizes again +function MadNLP.inertia(M::SDSSolver) + n = size(M.tril, 1) + if M.opt.sds_factorization === :ldl + # failure tuple must sum to n (the inertia contract), same as the Cholesky branch + M.last_info == 0 || return (n - 2, 1, 1) + l = count(x -> x < zero(x), view(M.inner.numeric.d, 1:n)) # true inertia from D + return (n - l, 0, l) + end + return M.last_info == 0 ? (n, 0, 0) : (n - 2, 1, 1) +end +MadNLP.improve!(::SDSSolver) = false +MadNLP.introduce(::SDSSolver) = "SparseDirectSolver.jl (fused pipeline)" +MadNLP.is_supported(::Type{SDSSolver}, ::Type{Float32}) = true +MadNLP.is_supported(::Type{SDSSolver}, ::Type{Float64}) = true + +function __init__() + setglobal!(SparseDirectSolver, :SDSSolver, SDSSolver) + setglobal!(SparseDirectSolver, :SDSSolverOptions, SDSSolverOptions) + return nothing +end + +end # module diff --git a/ext/SparseDirectSolverMetisExt.jl b/ext/SparseDirectSolverMetisExt.jl deleted file mode 100644 index b83251b..0000000 --- a/ext/SparseDirectSolverMetisExt.jl +++ /dev/null @@ -1,28 +0,0 @@ -""" - SparseDirectSolverMetisExt - -Nested-dissection ordering for SparseDirectSolver (PLAN §2.3 step 2). Loading -Metis.jl activates CliqueTrees' own Metis extension; this extension only -registers the algorithm object in `SparseDirectSolver.ND_PROVIDER`. -""" -module SparseDirectSolverMetisExt - -using SparseDirectSolver -using CliqueTrees -using Metis - -# METIS_NodeND with `ufactor = nd_ubfactor` (-1: METIS default). `nd_nlevels` is -# cuDSS' *minimum* number of dissection levels; METIS_NodeND dissects recursively -# until the parts are small, so the minimum holds whenever the graph is large -# enough to be split that often. CliqueTrees' level-capped `ND{S}` (AMD below the -# cap) was 4–13× slower and gave 15–80% more fill on Laplacians (T05 Report). -function nd_algorithm(nlevels::Integer, ubfactor::Integer) - return CliqueTrees.METIS(; ufactor = ubfactor) -end - -function __init__() - SparseDirectSolver.ND_PROVIDER[] = nd_algorithm - return nothing -end - -end # module SparseDirectSolverMetisExt diff --git a/src/SparseDirectSolver.jl b/src/SparseDirectSolver.jl index cd9e3d8..03a8bda 100644 --- a/src/SparseDirectSolver.jl +++ b/src/SparseDirectSolver.jl @@ -16,8 +16,10 @@ using KernelAbstractions using LinearAlgebra using SparseArrays -# AMD.jl activates CliqueTrees' AMD extension (`CliqueTrees.AMD()` orderings) +# AMD.jl and Metis.jl activate CliqueTrees' AMD and METIS extensions +# (`CliqueTrees.AMD()` and `CliqueTrees.METIS()` orderings) import AMD +import Metis export SparseDirectSolverError, NotSupportedError, InvalidValueError, FactorizationError, InterruptedError export CONFIG_PARAMETERS, DATA_PARAMETERS, CUDSS08_DATA_PARAMETERS, EXTRA_PARAMETERS @@ -49,6 +51,7 @@ include("symbolic/pairs.jl") include("symbolic/supernodes.jl") include("symbolic/schedule.jl") include("symbolic/layout.jl") +include("symbolic/device.jl") include("symbolic/maps.jl") # matching and scaling (PLAN §1.3, milestone M9), host @@ -81,6 +84,28 @@ include("solve/refinement.jl") include("solver.jl") include("generic.jl") +# the fused pipeline (factorization mega-kernel + partitioned-inverse solve) +# and its per-architecture tuning bundle; consumed by the MadNLP extension +include("fused.jl") + +# The MadNLP linear-solver type (SDSSolver) and its options type live in the +# SparseDirectSolverMadNLPExt package extension (they subtype MadNLP types, so +# they cannot be defined here). The standard idiom — as MadNLPGPU does for +# CUDSSSolver — is a declared global the extension assigns in its __init__: +# once MadNLP is loaded in the session, +# +# using MadNLP, SparseDirectSolver +# madnlp(model; linear_solver = SDSSolver, +# kkt_system = MadNLP.SparseCondensedKKTSystem) +# +# Options go as madnlp keywords (sds_factorization = :ldl, sds_reordering, +# sds_tuning = SparseDirectSolver.Fused.SDSTuning(; ...), sds_user_perm). +# Before MadNLP is loaded the names exist but are unassigned (UndefVarError +# on use), exactly like MadNLPGPU.CUDSSSolver without CUDA. +global SDSSolver +global SDSSolverOptions +export SDSSolver, SDSSolverOptions + __init__() = _init_log_level() end # module SparseDirectSolver diff --git a/src/dense/capabilities.jl b/src/dense/capabilities.jl index 40a30ff..ea5f5b5 100644 --- a/src/dense/capabilities.jl +++ b/src/dense/capabilities.jl @@ -99,6 +99,16 @@ per workgroup, 64 KiB on CDNA). max_local_bytes(backend) = 49152 max_local_bytes(::KernelAbstractions.CPU) = typemax(Int) +""" + max_threads_per_multiprocessor(backend) -> Int + +Resident work items per compute unit of the device `backend` runs on, `0` when +the backend does not report it. The CUDA extension reads the device attribute +(2048 on sm_70/80/90, 1536 on sm_86/89/120); the fused pipeline's +per-architecture tuning (`Fused.default_tuning`) keys on it. +""" +max_threads_per_multiprocessor(backend) = 0 + # --------------------------------------------------------------------------- # probes diff --git a/src/fused.jl b/src/fused.jl new file mode 100644 index 0000000..f692394 --- /dev/null +++ b/src/fused.jl @@ -0,0 +1,2098 @@ +# The fused pipeline (PR #107 kernels + the October 2026 campaign), assembled +# as a backend-portable submodule: the fused factorization (Cholesky and +# no-pivot LDL^T) and the partitioned-inverse/fused solve, on CUDA and AMDGPU +# alike. The plans are built on the solver's backend and carry their own device +# state (no module globals). Consumed by the MadNLP package extension +# (ext/SparseDirectSolverMadNLPExt.jl); the tuning bundle `SDSTuning` / +# `default_tuning` lives here because it parameterizes these kernels, not the +# MadNLP glue. +module Fused + +using LinearAlgebra +const SDS = Base.parentmodule(@__MODULE__) # the enclosing SparseDirectSolver module +const KA = SDS.KernelAbstractions +const Atomix = SDS.Atomix +using .KA: @kernel, @index, @localmem, @synchronize, @Const, @uniform + +""" + SDSTuning + +Every performance knob of the fused pipeline in one immutable bundle, so a +platform's tuning is a single value (see bench/ARCH-TUNING.md for how each +field was measured and the per-architecture champions). All fields are +runtime parameters — no environment variables, no recompilation. +""" +Base.@kwdef struct SDSTuning + wides::Symbol = :auto # :auto | :host | :merged + subtree_budgets::Vector{Int} = [16384] # regime-A local-memory class(es) + subtree_parallelism::Int = 16384 + regime_c_width::Int = 32 + regime_c_rows::Int = 64 + amalgamation::NamedTuple = (max_width = 48, zero_fraction = 0.25, min_width = 8) + workgroup::Int = 128 # fused-kernel workgroup size + spin_cap::Int = 128 # dependency-counter backoff cap + # METIS knobs. nseps = 1 by default: first-touch reordering latency wins (owner call, + # 2026-10-12); nseps = 4 is the 78k champion (-6% refactorization via lower fill, PR #124) + # at ~2x the reordering time -- opt in through `sds_tuning` when factor time dominates. + nd_nseps::Int = 1 + nd_seed::Int = 3 +end + +""" + default_tuning(backend) -> SDSTuning + +The measured per-architecture defaults — the single place they live. +2048-threads/SM NVIDIA parts (sm_70/80/90): host wides, 16K budgets. +1536-thread parts (sm_86/89, Blackwell workstation): merged wides, 8K budgets. +AMD (gfx906-measured): merged, 8K, workgroup 256, spin cap 16. +""" +function default_tuning(backend) + # NVIDIA parts report their resident-thread count through the CUDA extension + threads = SDS.max_threads_per_multiprocessor(backend) + if threads > 0 + return threads < 2048 ? SDSTuning(; wides = :merged, subtree_budgets = [8192]) : + SDSTuning(; wides = :host) + end + return SDSTuning(; wides = :merged, subtree_budgets = [8192], workgroup = 256, spin_cap = 16) +end + +""" + resolve_tuning(t::SDSTuning, backend) -> SDSTuning + +`t` with `wides = :auto` replaced by the wide-front strategy of +`default_tuning(backend)`; any other `t` is returned as is. +""" +function resolve_tuning(t::SDSTuning, backend) + t.wides === :auto || return t + return SDSTuning(default_tuning(backend).wides, t.subtree_budgets, t.subtree_parallelism, + t.regime_c_width, t.regime_c_rows, t.amalgamation, t.workgroup, t.spin_cap, + t.nd_nseps, t.nd_seed) +end + +# device copy of a host integer vector (empty vectors included) +_dvec(backend, v) = SDS._device_vec(backend, eltype(v), v) + +const WG = 32 +const MW = 256 # max front width on the inverse path +const WGF = 128 # default workgroup size; the fused kernel takes it per plan (fp.wgf) +const CHWG = 1024 # workgroup size of the chain kernels +const NSTRIP = 4 # strip parallelism of the chain kernels + +# --------------------------------------------------------------------------- +# inversion: thread j of the front's workgroup computes column j of inv(L11) +# (non-unit lower triangular), written col-major at inv_ptr[v] +@kernel function invert_l11_kernel!(inv11, @Const(factor), @Const(inv_ptr), @Const(list), + @Const(front_ptr), @Const(front_nrows), @Const(front_ncols), ::Val{WGV}) where {WGV} + li = @index(Local, Linear) + g = @index(Group, Linear) + @inbounds begin + v = list[g] + f = Int(front_nrows[v]) + w = Int(front_ncols[v]) + p0 = Int(front_ptr[v]) - 1 + ip = Int(inv_ptr[v]) - 1 + for j in li:WGV:w + cb = ip + (j - 1) * w + for k in 1:(j - 1) + inv11[cb + k] = zero(eltype(inv11)) + end + for k in j:w + acc = k == j ? one(eltype(inv11)) : zero(eltype(inv11)) + for m in j:(k - 1) + acc -= factor[p0 + (m - 1) * f + k] * inv11[cb + m] + end + inv11[cb + k] = acc / factor[p0 + (k - 1) * f + k] + end + end + end +end + +# --------------------------------------------------------------------------- +# forward: z = inv(L11) y per front, then the atomic L21 update (stock scatter) +@kernel function fwd1_kernel!(Y, @Const(factor), @Const(inv11), @Const(inv_ptr), @Const(list), first, count, + @Const(subtree_ptr), @Const(subtree_nodes), @Const(super_ptr), @Const(rowptr), + @Const(rowval), @Const(front_ptr), @Const(front_nrows), @Const(front_ncols), + ::Val{SUB}, ::Val{WGV}) where {SUB, WGV} + @uniform TT = eltype(Y) + li = @index(Local, Linear) + g = @index(Group, Linear) + t = @localmem TT (MW,) + @inbounds begin + e = list[first + g - 1] + a0 = SUB ? Int(subtree_ptr[e]) : 0 + cnt = SUB ? Int(subtree_ptr[e + 1]) - a0 : 1 + limit = SUB ? Int(super_ptr[subtree_nodes[a0 + cnt - 1] + 1]) - 1 : 0 + for k in 1:cnt + v = SUB ? subtree_nodes[a0 + k - 1] : e + f = Int(front_nrows[v]) + w = Int(front_ncols[v]) + p0 = Int(front_ptr[v]) - 1 + rp = Int(rowptr[v]) - 1 + c0 = Int(super_ptr[v]) - 1 + ip = Int(inv_ptr[v]) - 1 + w > MW && continue + # t = inv(L11) * Y[c0+1 : c0+w] + for kk in li:WGV:w + acc = zero(TT) + for j in 1:w + acc += inv11[ip + (j - 1) * w + kk] * Y[c0 + j] + end + t[kk] = acc + end + @synchronize + for kk in li:WGV:w + Y[c0 + kk] = t[kk] + end + # L21 update from t (Y rows above `limit` belong to other workgroups: atomic) + for i in (w + li):WGV:f + acc = zero(TT) + for j in 1:w + acc += factor[p0 + (j - 1) * f + i] * t[j] + end + row = Int(rowval[rp + i]) + if row > limit + Atomix.@atomic Y[row] += -acc + else + Y[row] -= acc + end + end + @synchronize + end + end +end + +# --------------------------------------------------------------------------- +# backward: gather from finished ancestors, then x = inv(L11)ᵀ z per front +@kernel function bwd1_kernel!(Y, @Const(factor), @Const(inv11), @Const(inv_ptr), @Const(list), first, count, + @Const(subtree_ptr), @Const(subtree_nodes), @Const(super_ptr), @Const(rowptr), + @Const(rowval), @Const(front_ptr), @Const(front_nrows), @Const(front_ncols), + ::Val{SUB}, ::Val{WGV}) where {SUB, WGV} + @uniform TT = eltype(Y) + li = @index(Local, Linear) + g = @index(Group, Linear) + t = @localmem TT (MW,) + @inbounds begin + e = list[first + g - 1] + a0 = SUB ? Int(subtree_ptr[e]) : 0 + cnt = SUB ? Int(subtree_ptr[e + 1]) - a0 : 1 + for k in 1:cnt + v = SUB ? subtree_nodes[a0 + cnt - k] : e # reverse processing order + f = Int(front_nrows[v]) + w = Int(front_ncols[v]) + p0 = Int(front_ptr[v]) - 1 + rp = Int(rowptr[v]) - 1 + c0 = Int(super_ptr[v]) - 1 + ip = Int(inv_ptr[v]) - 1 + w > MW && continue + # t = Y[c0+k] - L21ᵀ x(rows below) + for kk in li:WGV:w + acc = zero(TT) + col = p0 + (kk - 1) * f + for i in (w + 1):f + acc += factor[col + i] * Y[rowval[rp + i]] + end + t[kk] = Y[c0 + kk] - acc + end + @synchronize + # x = inv(L11)ᵀ t (inv col-major: invᵀ[k, j] = inv11[(k-1)w + j], contiguous in j) + for kk in li:WGV:w + acc = zero(TT) + cb = ip + (kk - 1) * w + for j in 1:w + acc += inv11[cb + j] * t[j] + end + Y[c0 + kk] = acc + end + @synchronize + end + end +end + +# --------------------------------------------------------------------------- +# per-front inverses of the L11 blocks (width <= MW) for the partitioned-inverse solve: +# returns the device buffer, its per-front offsets, a closure that (re)computes the +# inverses from the current factor, and the buffer size in bytes +function build_inverse(s) + S = s.symbolic + sc = S.schedule + reg = Array(sc.regime) + width = sc.width + ns = length(reg) + inv_ptr_h = zeros(Int64, ns + 1) + p = 1 + for v in 1:ns + inv_ptr_h[v] = p + if width[v] <= MW + p += width[v]^2 + else + @assert reg[v] == SDS.REGIME_C "front $v: width $(width[v]) > $MW but not regime C" + end + end + inv_ptr_h[ns + 1] = p + backend = KA.get_backend(s.numeric.factor) + T = eltype(s.numeric.factor) + inv11 = KA.zeros(backend, T, p - 1) + inv_ptr = _dvec(backend, inv_ptr_h) + ab = _dvec(backend, Int32.(findall(v -> width[v] <= MW, 1:ns))) + invert! = () -> invert_l11_kernel!(backend, WG)(inv11, s.numeric.factor, inv_ptr, ab, S.front_ptr, + S.front_nrows, S.front_ncols, Val(WG); + ndrange = WG * length(ab)) + return inv11, inv_ptr, invert!, sizeof(T) * (p - 1) +end + +# --------------------------------------------------------------------------- +# fused sweeps (algo2): one launch per direction over every front of width +# <= MW below the first wide front, etree dependency counters instead of +# level-batched launches. CUDA-only (threadfence, resident-grid spinning). +@kernel function fwd2_kernel!(Y, @Const(factor), @Const(inv11), @Const(inv_ptr), + @Const(flist), @Const(slist), nsub, @Const(subtree_ptr), @Const(subtree_nodes), + @Const(snparent), arrived, @Const(nchild), @Const(super_ptr), + @Const(rowptr), @Const(rowval), @Const(front_ptr), @Const(front_nrows), + @Const(front_ncols), ::Val{WGV}) where {WGV} + @uniform TT = eltype(Y) + li = @index(Local, Linear) + g = @index(Group, Linear) + yb = @localmem TT (MW,) + t = @localmem TT (MW,) + @inbounds if g <= nsub + e = Int(slist[g]) # regime-A subtree: serial walk, then signal + a0 = Int(subtree_ptr[e]) + cnt = Int(subtree_ptr[e + 1]) - a0 + root = Int(subtree_nodes[a0 + cnt - 1]) + limit = Int(super_ptr[root + 1]) - 1 + for k in 1:cnt + v = Int(subtree_nodes[a0 + k - 1]) + f = Int(front_nrows[v]) + w = Int(front_ncols[v]) + p0 = Int(front_ptr[v]) - 1 + rp = Int(rowptr[v]) - 1 + c0 = Int(super_ptr[v]) - 1 + ip = Int(inv_ptr[v]) - 1 + for kk in li:WGV:w + acc = zero(TT) + for j in 1:w + acc += inv11[ip + (j - 1) * w + kk] * Y[c0 + j] + end + t[kk] = acc + end + @synchronize + for kk in li:WGV:w + Y[c0 + kk] = t[kk] + end + for i in (w + li):WGV:f + acc = zero(TT) + for j in 1:w + acc += factor[p0 + (j - 1) * f + i] * t[j] + end + row = Int(rowval[rp + i]) + if row > limit + Atomix.@atomic Y[row] += -acc + else + Y[row] -= acc + end + end + @synchronize + end + if li == 1 + Threads.atomic_fence() + pv = Int(snparent[root]) + pv > 0 && (Atomix.@atomic arrived[pv] += Int32(1)) + end + else begin + v = Int(flist[g - nsub]) + f = Int(front_nrows[v]) + w = Int(front_ncols[v]) + p0 = Int(front_ptr[v]) - 1 + rp = Int(rowptr[v]) - 1 + c0 = Int(super_ptr[v]) - 1 + ip = Int(inv_ptr[v]) - 1 + if li == 1 + tgt = nchild[v] + pause = Int32(4) + while (Atomix.@atomic arrived[v] += Int32(0)) < tgt + z = Int32(0) + for _ in 1:pause + z += Int32(1) + end + z < Int32(0) && (yb[1] = TT(z)) # keep the backoff loop alive + pause = min(pause << 1, Int32(128)) + end + end + @synchronize + Threads.atomic_fence() + for kk in li:WGV:w # coherent read of the front's columns + yb[kk] = Atomix.@atomic Y[c0 + kk] += zero(TT) + end + @synchronize + for kk in li:WGV:w # t = inv(L11) * y + acc = zero(TT) + for j in 1:w + acc += inv11[ip + (j - 1) * w + kk] * yb[j] + end + t[kk] = acc + end + @synchronize + for kk in li:WGV:w + Y[c0 + kk] = t[kk] + end + for i in (w + li):WGV:f # L21 update, atomic scatter + acc = zero(TT) + for j in 1:w + acc += factor[p0 + (j - 1) * f + i] * t[j] + end + Atomix.@atomic Y[Int(rowval[rp + i])] += -acc + end + @synchronize + if li == 1 + Threads.atomic_fence() + pv = Int(snparent[v]) + pv > 0 && (Atomix.@atomic arrived[pv] += Int32(1)) + end + end end +end + +@kernel function bwd2_kernel!(Y, @Const(factor), @Const(inv11), @Const(inv_ptr), + @Const(flist), @Const(slist), nsub, @Const(subtree_ptr), @Const(subtree_nodes), + @Const(snparent), done, @Const(super_ptr), @Const(rowptr), + @Const(rowval), @Const(front_ptr), @Const(front_nrows), @Const(front_ncols), + ::Val{WGV}) where {WGV} + @uniform TT = eltype(Y) + li = @index(Local, Linear) + g = @index(Group, Linear) + xb = @localmem TT (WGV,) + t = @localmem TT (MW,) + t2 = @localmem TT (MW,) + @inbounds if g > length(flist) + e = Int(slist[g - length(flist)]) # regime-A subtree: wait for the root's parent, walk down + a0 = Int(subtree_ptr[e]) + cnt = Int(subtree_ptr[e + 1]) - a0 + root = Int(subtree_nodes[a0 + cnt - 1]) + if li == 1 + pv = Int(snparent[root]) + if pv > 0 + pause = Int32(4) + while (Atomix.@atomic done[pv] += Int32(0)) == Int32(0) + z = Int32(0) + for _ in 1:pause + z += Int32(1) + end + z < Int32(0) && (t[1] = TT(z)) + pause = min(pause << 1, Int32(128)) + end + end + end + @synchronize + Threads.atomic_fence() + for k in cnt:-1:1 + v = Int(subtree_nodes[a0 + k - 1]) + f = Int(front_nrows[v]) + w = Int(front_ncols[v]) + p0 = Int(front_ptr[v]) - 1 + rp = Int(rowptr[v]) - 1 + c0 = Int(super_ptr[v]) - 1 + ip = Int(inv_ptr[v]) - 1 + for kk in li:WGV:w + acc = zero(TT) + col = p0 + (kk - 1) * f + for i in (w + 1):f + acc += factor[col + i] * Y[Int(rowval[rp + i])] + end + t2[kk] = Y[c0 + kk] - acc + end + @synchronize + for kk in li:WGV:w + acc = zero(TT) + cb = ip + (kk - 1) * w + for j in 1:w + acc += inv11[cb + j] * t2[j] + end + Y[c0 + kk] = acc + end + @synchronize + end + else begin + v = Int(flist[length(flist) - g + 1]) # reverse topo order: parents get smaller block ids + f = Int(front_nrows[v]) + w = Int(front_ncols[v]) + p0 = Int(front_ptr[v]) - 1 + rp = Int(rowptr[v]) - 1 + c0 = Int(super_ptr[v]) - 1 + ip = Int(inv_ptr[v]) - 1 + if li == 1 + pv = Int(snparent[v]) + if pv > 0 + pause = Int32(4) + while (Atomix.@atomic done[pv] += Int32(0)) == Int32(0) + z = Int32(0) + for _ in 1:pause + z += Int32(1) + end + z < Int32(0) && (t[1] = TT(z)) + pause = min(pause << 1, Int32(128)) + end + end + end + @synchronize + Threads.atomic_fence() + for kk in li:WGV:w + t[kk] = zero(TT) + end + @synchronize + i0 = w + 1 + while i0 <= f # chunked coherent gather of ancestor solutions + ii = i0 + li - 1 + if ii <= f + xb[li] = Atomix.@atomic Y[Int(rowval[rp + ii])] += zero(TT) + end + @synchronize + m = min(WGV, f - i0 + 1) + for kk in li:WGV:w + acc = t[kk] + col = p0 + (kk - 1) * f + i0 - 1 + for c in 1:m + acc += factor[col + c] * xb[c] + end + t[kk] = acc + end + @synchronize + i0 += WGV + end + for kk in li:WGV:w # z = y - L21' x, then x = inv(L11)' z + t2[kk] = Y[c0 + kk] - t[kk] + end + @synchronize + for kk in li:WGV:w + acc = zero(TT) + cb = ip + (kk - 1) * w + for j in 1:w + acc += inv11[cb + j] * t2[j] + end + Y[c0 + kk] = acc + end + @synchronize + if li == 1 + Threads.atomic_fence() + Atomix.@atomic done[v] += Int32(1) + end + end end +end + + +# chain kernels: ONE workgroup walks the (near-serial) tall-front chain at the +# top of the tree in plan order. Launched after the fused bottom tier, so every +# input is already in Y (stream order): no spins, no fences, no atomics — the +# single block owns every row it touches; plain loads/stores + block barriers. +@kernel function chain_fwd_kernel!(Y, @Const(factor), @Const(inv11), @Const(inv_ptr), @Const(clist), + @Const(super_ptr), @Const(rowptr), @Const(rowval), @Const(front_ptr), + @Const(front_nrows), @Const(front_ncols), ::Val{WGV}) where {WGV} + @uniform TT = eltype(Y) + li = @index(Local, Linear) + yb = @localmem TT (MW,) + t = @localmem TT (MW,) + @inbounds for q in 1:length(clist) + v = Int(clist[q]) + f = Int(front_nrows[v]) + w = Int(front_ncols[v]) + p0 = Int(front_ptr[v]) - 1 + rp = Int(rowptr[v]) - 1 + c0 = Int(super_ptr[v]) - 1 + ip = Int(inv_ptr[v]) - 1 + for kk in li:WGV:w + yb[kk] = Y[c0 + kk] + end + @synchronize + for kk in li:WGV:w + acc = zero(TT) + for j in 1:w + acc += inv11[ip + (j - 1) * w + kk] * yb[j] + end + t[kk] = acc + end + @synchronize + for kk in li:WGV:w + Y[c0 + kk] = t[kk] + end + for i in (w + li):WGV:f + acc = zero(TT) + for j in 1:w + acc += factor[p0 + (j - 1) * f + i] * t[j] + end + Y[Int(rowval[rp + i])] -= acc + end + @synchronize + end +end + +@kernel function chain_bwd_kernel!(Y, @Const(factor), @Const(inv11), @Const(inv_ptr), @Const(clist), + @Const(super_ptr), @Const(rowptr), @Const(rowval), @Const(front_ptr), + @Const(front_nrows), @Const(front_ncols), ::Val{WGV}) where {WGV} + @uniform TT = eltype(Y) + li = @index(Local, Linear) + t = @localmem TT (MW,) + t2 = @localmem TT (MW,) + tp = @localmem TT (MW * NSTRIP,) + @inbounds for q in length(clist):-1:1 + v = Int(clist[q]) + f = Int(front_nrows[v]) + w = Int(front_ncols[v]) + p0 = Int(front_ptr[v]) - 1 + rp = Int(rowptr[v]) - 1 + c0 = Int(super_ptr[v]) - 1 + ip = Int(inv_ptr[v]) - 1 + kk = (li - 1) % w + 1 # strip-parallel gather: t = L21' x + st = (li - 1) ÷ w + 1 + if st <= NSTRIP + acc = zero(TT) + col = p0 + (kk - 1) * f + for i in (w + st):NSTRIP:f + acc += factor[col + i] * Y[Int(rowval[rp + i])] + end + tp[(st - 1) * MW + kk] = acc + end + @synchronize + for k2 in li:WGV:w + acc = Y[c0 + k2] + for st2 in 1:NSTRIP + acc -= tp[(st2 - 1) * MW + k2] + end + t2[k2] = acc + end + @synchronize + kk2 = (li - 1) % w + 1 # strip-parallel GEMV: x = inv(L11)' t2 + st2b = (li - 1) ÷ w + 1 + if st2b <= NSTRIP + acc = zero(TT) + cb = ip + (kk2 - 1) * w + for j in st2b:NSTRIP:w + acc += inv11[cb + j] * t2[j] + end + tp[(st2b - 1) * MW + kk2] = acc + end + @synchronize + for k2 in li:WGV:w + acc = zero(TT) + for st3 in 1:NSTRIP + acc += tp[(st3 - 1) * MW + k2] + end + t[k2] = acc + end + @synchronize + for k2 in li:WGV:w + Y[c0 + k2] = t[k2] + end + @synchronize + end +end + +# fused segment: the maximal run of plan entries from `kstart` in which every +# front has width <= wcap (regime-A subtree entries break a run) +function build_fused(s; kstart::Int = 1, wcap::Int = MW, merged::Bool = (kstart == 1)) + S = s.symbolic + ws = s.workspace + plan = ws.plan + nodes = S.schedule.group_nodes + widths = S.schedule.width + narrow(k) = plan.kind[k] != SDS.SOLVE_SUBTREES && + all(widths[nodes[q]] <= wcap for q in plan.first[k]:plan.last[k]) + i1 = kstart + while i1 <= length(plan.kind) && !narrow(i1) + i1 += 1 + end + i1 > length(plan.kind) && return nothing + i2 = i1 + while i2 + 1 <= length(plan.kind) && narrow(i2 + 1) + i2 += 1 + end + flist_h = Int32[nodes[q] for k in i1:i2 for q in plan.first[k]:plan.last[k]] + ns = length(widths) + in_range = falses(ns) + for v in flist_h + in_range[v] = true + end + child_ptr_h = Array(S.child_ptr) + child_list_h = Array(S.child_list) + nchild_h = Int32[child_ptr_h[v + 1] - child_ptr_h[v] for v in 1:ns] + stp_h = Array(S.subtree_ptr) + stn_h = Array(S.subtree_nodes) + live_root = falses(ns) # roots of subtrees merged into this launch (the merged tier only) + if merged + for e in 1:(length(stp_h) - 1) + live_root[stn_h[stp_h[e + 1] - 1]] = true + end + end + arrived_base_h = zeros(Int32, ns) + for v in 1:ns + in_range[v] || continue + nc = child_ptr_h[v + 1] - child_ptr_h[v] + cnt = 0 + for k in 1:nc + c = child_list_h[child_ptr_h[v] + k - 1] + (in_range[c] || live_root[c]) || (cnt += 1) + end + arrived_base_h[v] = Int32(cnt) + end + done_base_h = Int32[in_range[v] ? 0 : 1 for v in 1:ns] + # the regime-A subtrees the merged launch walks itself, longest first + sub_ids = Int32[] + if merged + for k in eachindex(plan.kind) + plan.kind[k] == SDS.SOLVE_SUBTREES || continue + append!(sub_ids, Int32.(nodes[plan.first[k]:plan.last[k]])) + end + sort!(sub_ids; by = e -> stp_h[e + 1] - stp_h[e], rev = true) + end + SDS._log(SDS.LOG_DEBUG, () -> "fused solve: plan entries $i1:$i2 of $(length(plan.kind)), " * + "$(length(flist_h)) fronts, $(length(sub_ids)) subtrees") + backend = KA.get_backend(s.numeric.factor) + return (; i1, i2, flist = _dvec(backend, flist_h), arrived = KA.zeros(backend, Int32, ns), + arrived_base = _dvec(backend, arrived_base_h), done = KA.zeros(backend, Int32, ns), + done_base = _dvec(backend, done_base_h), nchild = _dvec(backend, nchild_h), + nfused = length(flist_h), slist = _dvec(backend, sub_ids), nsub = length(sub_ids)) +end + +# one fused segment, forward sweep: `wg > 512` runs the single-workgroup chain kernel, +# `wg == 64` the dependency-counter sweep (the workgroup size the sweep is tuned for) +function run_fused!(fz, wg, Yv, s, inv11, inv_ptr, backend) + S, N = s.symbolic, s.numeric + if wg > 512 + chain_fwd_kernel!(backend, CHWG)(Yv, N.factor, inv11, inv_ptr, fz.flist, S.super_ptr, S.rowptr, + S.rowval, S.front_ptr, S.front_nrows, S.front_ncols, Val(CHWG); + ndrange = CHWG) + return nothing + end + wg == 64 || throw(SDS.InvalidValueError("fused solve: workgroup size $wg; expected 64 or > 512 (chain)")) + copyto!(fz.arrived, fz.arrived_base) + fwd2_kernel!(backend, 64)(Yv, N.factor, inv11, inv_ptr, fz.flist, fz.slist, fz.nsub, S.subtree_ptr, + S.subtree_nodes, S.snparent, fz.arrived, fz.nchild, + S.super_ptr, S.rowptr, S.rowval, S.front_ptr, S.front_nrows, S.front_ncols, + Val(64); ndrange = 64 * (fz.nfused + fz.nsub)) + return nothing +end + +# the backward counterpart of `run_fused!` +function run_fused_bwd!(fz, wg, Yv, s, inv11, inv_ptr, backend) + S, N = s.symbolic, s.numeric + if wg > 512 + chain_bwd_kernel!(backend, CHWG)(Yv, N.factor, inv11, inv_ptr, fz.flist, S.super_ptr, S.rowptr, + S.rowval, S.front_ptr, S.front_nrows, S.front_ncols, Val(CHWG); + ndrange = CHWG) + return nothing + end + wg == 64 || throw(SDS.InvalidValueError("fused solve: workgroup size $wg; expected 64 or > 512 (chain)")) + copyto!(fz.done, fz.done_base) + bwd2_kernel!(backend, 64)(Yv, N.factor, inv11, inv_ptr, fz.flist, fz.slist, fz.nsub, S.subtree_ptr, + S.subtree_nodes, S.snparent, fz.done, + S.super_ptr, S.rowptr, S.rowval, S.front_ptr, S.front_nrows, S.front_ncols, + Val(64); ndrange = 64 * (fz.nfused + fz.nsub)) + return nothing +end + +# the fused solve: `fz = (segments, covered)` with `segments` a vector of +# `(build_fused(...), wg)` pairs in plan order; plan entries outside the segments +# run through the level-batched kernels (`fwd1_kernel!`/`bwd1_kernel!`, vendor +# dense calls for fronts wider than MW) +function algo2_solve!(x, s, b, inv11, inv_ptr, fz) + S, N, ws = s.symbolic, s.numeric, s.workspace + bm = SDS.batch_map(N; nrhs = 1) + SDS.permute_rhs!(ws.Y, b, S.perm; transposed = false, bm) + plan = ws.plan + nodes = S.schedule.group_nodes + widths = S.schedule.width + p = SDS._solve_impls(N, S, :auto) + backend = KA.get_backend(ws.Y) + Yv = vec(ws.Y) + args(first, count) = (Yv, N.factor, inv11, inv_ptr, S.group_nodes, first, count, S.subtree_ptr, + S.subtree_nodes, S.super_ptr, S.rowptr, S.rowval, S.front_ptr, S.front_nrows, + S.front_ncols) + fwd_entry(k) = begin + a, b_ = plan.first[k], plan.last[k] + if plan.kind[k] == SDS.SOLVE_SUBTREES + elseif plan.kind[k] == SDS.SOLVE_DENSE + any(widths[nodes[q]] <= MW for q in a:b_) && + fwd1_kernel!(backend, WG)(args(a, b_ - a + 1)..., Val(false), Val(WG); ndrange = WG * (b_ - a + 1)) + for q in a:b_ + widths[nodes[q]] > MW && SDS._fwd_dense!(ws, S, N, nodes[q], 1, false, p, SDS._SV_CHOLESKY) + end + else + fwd1_kernel!(backend, WG)(args(a, b_ - a + 1)..., Val(plan.kind[k] == SDS.SOLVE_SUBTREES), Val(WG); + ndrange = WG * (b_ - a + 1)) + end + end + bwd_entry(k) = begin + a, b_ = plan.first[k], plan.last[k] + if plan.kind[k] == SDS.SOLVE_SUBTREES + elseif plan.kind[k] == SDS.SOLVE_DENSE + for q in b_:-1:a + widths[nodes[q]] > MW && SDS._bwd_dense!(ws, S, N, nodes[q], 1, p, SDS._SV_CHOLESKY) + end + any(widths[nodes[q]] <= MW for q in a:b_) && + bwd1_kernel!(backend, WG)(args(a, b_ - a + 1)..., Val(false), Val(WG); ndrange = WG * (b_ - a + 1)) + else + bwd1_kernel!(backend, WG)(args(a, b_ - a + 1)..., Val(plan.kind[k] == SDS.SOLVE_SUBTREES), Val(WG); + ndrange = WG * (b_ - a + 1)) + end + end + segs, _ = fz + k = 1 + for (fzk, wg) in segs + while k < fzk.i1 + fwd_entry(k); k += 1 + end + run_fused!(fzk, wg, Yv, s, inv11, inv_ptr, backend) + k = fzk.i2 + 1 + end + while k <= length(plan.kind) + fwd_entry(k); k += 1 + end + if SDS._is_ldlt_structure(S.structure) # LDL: apply D between the unit-triangular sweeps + Yv ./= view(N.d, 1:length(Yv)) + end + k = length(plan.kind) + for (fzk, wg) in Iterators.reverse(segs) + while k > fzk.i2 + bwd_entry(k); k -= 1 + end + run_fused_bwd!(fzk, wg, Yv, s, inv11, inv_ptr, backend) + k = fzk.i1 - 1 + end + while k >= 1 + bwd_entry(k); k -= 1 + end + SDS.unpermute_solution!(x, ws.Y, S.perm; transposed = false, bm) + return x +end + +const SYRK_IN = 64 +const TILE = 32 +const NL64 = 2080 # packed 64x64 lower triangle + +# --------------------------------------------------------------------------- +# no-pivot LDLᵀ mode of the fused kernel (real element types): the factor stores +# the UNIT lower panels and D goes to `d` (global factor-column indexing), the +# stock LDL layout, so the stock solve and the fused solve (one extra D pass +# between the sweeps) both work on it. A zero pivot sets info[v] to the local +# column and the front's consumers skip, exactly the Cholesky failure path; +# NEGATIVE pivots factor through (inertia for MadNLP). The mode is a compile-time +# `Val{LDL}` kernel parameter: the Cholesky instantiation is unchanged machine code. +# +# Panel convention while a front is in flight: the packed `sh` triangle keeps the +# pivot d_j ON THE DIAGONAL (the unit is implied), so the in-panel TRSM both +# reconstructs the unscaled column (`* sh[j,j]`) and divides by the pivot +# (`/ sh[kk,kk]`) with the same reads as Cholesky; write-out emits the unit +# diagonal. The TRSM role reloads the panel from the factor (diag = 1) and +# restores d into the diagonal slots from `d`; the TILE roles stage the 32 +# pivots of each k-block in sh[2049:2080] (free: NL64 - 2*1024) and fold them +# into the Ai tile only. + +# regime A: a dedicated no-pivot LDL variant of SDS.subtree_cholesky_kernel! (the stock +# kernel stays untouched; its CPU-launch allocation budget is load-bearing in the tests). +# Identical skeleton and assembly helpers; only the two column phases and the extension +# slots differ: the whole trailing triangle (CB included) takes the d-folded rank-1, so +# no separate finalize/CB pass exists, and d/piv/pivot_kind are filled as the stock LDL +# path (identity pivot order; a zero pivot sets the front status, as Cholesky). +const _SP_C0 = SDS._ST_CTL + 1 +const _SP_CTL = _SP_C0 + +@inline function _subtree_ldl_update!(buf, ctl, pv, j, li, d, piv, pivot_kind, ::Val{WG}) where {WG} + @inbounds begin + T = eltype(buf) + IT = eltype(ctl) + f = ctl[SDS._ST_F] + lf = ctl[SDS._ST_LF] - 1 + if ctl[SDS._ST_STATUS] == 0 + dk = real(buf[lf + SDS._packed(j, j, f)]) + if isfinite(dk) && !iszero(dk) # NaN/Inf must fail like potrf, not "factor" + if li == 1 + g = Int(ctl[_SP_C0]) + j - 1 + d[g] = dk + piv[g] = Int32(g) + pivot_kind[g] = SDS.PIVOT_KIND_1X1 + pv[1] = dk + end + rd = inv(dk) + for i in (j + li):WG:f + buf[lf + SDS._packed(i, j, f)] *= rd + end + elseif li == 1 + ctl[SDS._ST_STATUS] = j % IT + end + end + end + return nothing +end + +@inline function _subtree_ldl_scale!(buf, ctl, pv, j, li, ::Val{WG}) where {WG} + @inbounds begin + if ctl[SDS._ST_STATUS] == 0 + f = ctl[SDS._ST_F] + lf = ctl[SDS._ST_LF] - 1 + dk = pv[1] + li == 1 && (buf[lf + SDS._packed(j, j, f)] = one(eltype(buf))) + r = f - j + for q in (li - 1):WG:(r * (r + 1) ÷ 2 - 1) + a, b = SDS._tri_decode(q, r) + i = j + a + k = j + b + buf[lf + SDS._packed(i, k, f)] -= buf[lf + SDS._packed(i, j, f)] * dk * + buf[lf + SDS._packed(k, j, f)] + end + end + end + return nothing +end + +@kernel function subtree_ldl_kernel!(factor, stack, info, d, piv, pivot_kind, @Const(nzval), @Const(amap), + @Const(amap_ptr), @Const(amap_src), @Const(trees), bm, @Const(subtree_ptr), + @Const(subtree_nodes), @Const(super_ptr), @Const(front_ptr), + @Const(front_nrows), @Const(front_ncols), @Const(cb_ptr), + @Const(local_front), @Const(local_cb), @Const(child_ptr), + @Const(child_list), @Const(relind_ptr), @Const(relind), + ::Val{NE}, ::Val{WG}) where {NE, WG} + @uniform TT = eltype(factor) + @uniform IT = eltype(subtree_nodes) + li = @index(Local, Linear) + G = @index(Group, Linear) + buf = @localmem TT (NE,) + ctl = @localmem IT (_SP_CTL,) + pv = @localmem TT (1,) + if li == 1 + @inbounds t = trees[bm.first + SDS._bm_node(bm, G) - 1] + @inbounds ctl[SDS._ST_FIRST] = subtree_ptr[t] + @inbounds ctl[SDS._ST_COUNT] = subtree_ptr[t + 1] - subtree_ptr[t] + end + @synchronize + for k in 1:ctl[SDS._ST_COUNT] + if li == 1 + SDS._subtree_setup!(ctl, k, subtree_nodes, front_nrows, front_ncols, local_front, local_cb, + child_ptr) + @inbounds ctl[_SP_C0] = super_ptr[ctl[SDS._ST_NODE]] % IT + end + @synchronize + SDS._subtree_zero!(buf, ctl, li, Val(WG)) + @synchronize + SDS._subtree_scatter!(buf, ctl, SDS._mview(nzval, SDS._bm_gmember(bm, G), bm.nbatch), amap, amap_ptr, + amap_src, front_ptr, li, Val(WG)) + @synchronize + for kc in 1:ctl[SDS._ST_NCHILD] + SDS._subtree_extend_add!(buf, ctl, kc, child_ptr, child_list, front_nrows, front_ncols, local_cb, + relind_ptr, relind, li, Val(WG)) + @synchronize + end + for j in 1:ctl[SDS._ST_W] + _subtree_ldl_update!(buf, ctl, pv, j, li, SDS._mview(d, SDS._bm_gmember(bm, G), bm.nbatch), + SDS._mview(piv, SDS._bm_gmember(bm, G), bm.nbatch), + SDS._mview(pivot_kind, SDS._bm_gmember(bm, G), bm.nbatch), Val(WG)) + @synchronize + _subtree_ldl_scale!(buf, ctl, pv, j, li, Val(WG)) + @synchronize + end + mb = SDS._bm_gmember(bm, G) + SDS._subtree_write!(factor, SDS._mview(stack, mb, bm.nbatch), SDS._iview(info, mb, bm.nbatch), buf, ctl, + SDS.member_panels(front_ptr, mb, bm.nbatch), cb_ptr, li, Val(WG)) + @synchronize + for r in 1:ctl[SDS._ST_ROUNDS] + SDS._subtree_move!(buf, ctl, r, li, Val(WG)) + @synchronize + end + end +end + +@inline function _front_ldl_update!(sh, st, pv, j, v, li, front_ncols, d, g0, ::Val{WG}) where {WG} + @inbounds begin + w = Int(front_ncols[v]) + if j <= w && st[1] == Int32(0) + dk = real(sh[SDS._packed(j, j, 64)]) + if isfinite(dk) && !iszero(dk) # NaN/Inf must fail like potrf, not "factor" + if li == 1 + d[g0 + j] = dk + pv[1] = dk + end + rd = inv(dk) + for i in (j + li):WG:w + sh[SDS._packed(i, j, 64)] *= rd + end + elseif li == 1 + st[1] = Int32(j) + end + end + end + return nothing +end + +@inline function _front_ldl_scale!(sh, st, pv, j, v, li, front_ncols, ::Val{WG}) where {WG} + @inbounds begin + w = Int(front_ncols[v]) + if j <= w && st[1] == Int32(0) + dk = pv[1] + r = w - j + for q in (li - 1):WG:(r * (r + 1) ÷ 2 - 1) + a, b = SDS._tri_decode(q, r) + i = j + a + k = j + b + sh[SDS._packed(i, k, 64)] -= sh[SDS._packed(i, j, 64)] * dk * sh[SDS._packed(k, j, 64)] + end + end + end + return nothing +end + +# F22 ← F22 − (L21 D) L21ᵀ on the private CB; d_k read from the sh diagonal (still live) +@inline function _front_ldl_syrk!(factor, pcb, sh, st, v, li, front_ptr, front_nrows, front_ncols, cb2, + ::Val{WG}) where {WG} + @inbounds begin + TT = eltype(factor) + c0 = Int(cb2[v]) + if st[1] == Int32(0) && c0 > 0 + f = Int(front_nrows[v]) + w = Int(front_ncols[v]) + m = f - w + p0 = Int(front_ptr[v]) + w - 1 + for q in (li - 1):WG:(m * (m + 1) ÷ 2 - 1) + ii, jj = SDS._tri_decode(q, m) + acc = zero(TT) + for k in 1:w + acc += factor[p0 + (k - 1) * f + ii] * real(sh[SDS._packed(k, k, 64)]) * + factor[p0 + (k - 1) * f + jj] + end + pcb[c0 + SDS._packed(ii, jj, m) - 1] -= acc + end + end + end + return nothing +end + +const ROLE_PANEL_B = Int8(1) +const ROLE_PANEL_C = Int8(2) +const ROLE_TRSM = Int8(3) +const ROLE_TILE = Int8(4) +const ROLE_EA = Int8(5) +const ROLE_WPANEL = Int8(6) # wide fronts in-kernel: blocked 64-panel Cholesky +const ROLE_WTRSM = Int8(7) # (rb = panel k | rb = chunk, rc = k | rc = k*1024 + tjj) +const ROLE_WTILE = Int8(8) +const ROLE_WSIG = Int8(9) + +# spin with backoff on an Int32 cell until it equals `target`; thread 1 only +@inline function _spin_until_eq!(arr, idx, target, scratch, scap) + pause = min(Int32(4), scap) + while (Atomix.@atomic arr[idx] += Int32(0)) != target + z = Int32(0) # integer backoff: no Float64 in kernels (Metal) + for _ in 1:pause + z += Int32(1) + end + z < Int32(0) && (scratch[1] = z) + pause = min(pause << 1, scap) + end + return nothing +end + +@kernel function fused_fact_kernel!(factor, stack, pcb, info, @Const(nzval), @Const(amap), @Const(amap_ptr), + @Const(amap_src), @Const(cb2), @Const(wt), @Const(role), @Const(ra), @Const(rb), @Const(rc), + @Const(nchild), arrived, ea_left, panel_done, trsm_left, tiles_left, + @Const(tiles_base), + cb_done, @Const(snparent), @Const(front_ptr), @Const(front_nrows), + @Const(front_ncols), @Const(cb_ptr), @Const(child_ptr), + @Const(child_list), @Const(relind_ptr), @Const(relind), base, + @Const(wbase), wupd, wpdone2, wtrsml, wcbl, scap, eac, d, @Const(super_ptr), + ::Val{LDL}, ::Val{WG}) where {LDL, WG} + @uniform TT = eltype(factor) + @uniform RT = real(eltype(factor)) + li = @index(Local, Linear) + G = Int(base) + @index(Group, Linear) + sh = @localmem TT (NL64,) + st = @localmem Int32 (1,) + piv = @localmem RT (1,) + fscr = @localmem Int32 (1,) # spin-backoff scratch (integer: no Float64 in kernels) + @inbounds begin + r = role[G] + if r == ROLE_EA + v = Int(ra[G]) # parent + c = Int(rb[G]) # child + q0 = (Int(rc[G]) - 1) * Int(eac) + if li == 1 + _spin_until_eq!(cb_done, c, Int32(1), fscr, scap) + end + @synchronize + Threads.atomic_fence() + cpriv = Int(cb2[c]) # child CB: private (mega) or stock (subtree root) + cb = cpriv > 0 ? cpriv : Int(cb_ptr[c]) + mc = Int(front_nrows[c]) - Int(front_ncols[c]) + fp = Int(front_nrows[v]) + wp = Int(front_ncols[v]) + mp = fp - wp + pp = Int(front_ptr[v]) + cp = Int(cb2[v]) # parent CB is always private + r0 = Int(relind_ptr[c]) - 1 + qe = min(q0 + Int(eac), mc * mc) - 1 + for q in (q0 + li - 1):WG:qe + jj = q ÷ mc + 1 + ii = q - (jj - 1) * mc + 1 + if ii >= jj + ri = Int(relind[r0 + ii]) + rj = Int(relind[r0 + jj]) + x = cpriv > 0 ? pcb[cb + SDS._packed(ii, jj, mc) - 1] : + stack[cb + SDS._packed(ii, jj, mc) - 1] + if rj <= wp + Atomix.@atomic factor[pp + (rj - 1) * fp + ri - 1] += x + else + Atomix.@atomic pcb[cp + SDS._packed(ri - wp, rj - wp, mp) - 1] += x + end + end + end + @synchronize + Threads.atomic_fence() # ALL threads: each wave drains its own stores (gfx9) + @synchronize + if li == 1 + Atomix.@atomic ea_left[v] += Int32(-1) + end + elseif r == ROLE_PANEL_B || r == ROLE_PANEL_C + v = Int(ra[G]) + if li == 1 + if r == ROLE_PANEL_B + _spin_until_eq!(arrived, v, nchild[v], fscr, scap) + else + _spin_until_eq!(ea_left, v, Int32(0), fscr, scap) + end + end + @synchronize + Threads.atomic_fence() + if r == ROLE_PANEL_B + fz = Int(front_nrows[v]) + wz = Int(front_ncols[v]) + pz = Int(front_ptr[v]) + for q in (li - 1):WG:(fz * wz - 1) # zero the panel; private CBs are pre-zeroed + factor[pz + q] = zero(TT) + end + @synchronize + SDS._scatter_front!(factor, nzval, amap, amap_ptr, amap_src, v, + SDS._member_shift(front_ptr, v, 1, 1), li, Val(WG)) + @synchronize + nc = Int(child_ptr[v + 1]) - Int(child_ptr[v]) + for kc in 1:nc + c = Int(child_list[child_ptr[v] + kc - 1]) + cpr = Int(cb2[c]) + cbc = cpr > 0 ? cpr : Int(cb_ptr[c]) + if cbc > 0 + mc = Int(front_nrows[c]) - Int(front_ncols[c]) + r0 = Int(relind_ptr[c]) - 1 + mp = fz - wz + cp2 = Int(cb2[v]) + for q in (li - 1):WG:(mc * mc - 1) + jj = q ÷ mc + 1 + ii = q - (jj - 1) * mc + 1 + if ii >= jj + ri = Int(relind[r0 + ii]) + rj = Int(relind[r0 + jj]) + x = cpr > 0 ? pcb[cbc + SDS._packed(ii, jj, mc) - 1] : + stack[cbc + SDS._packed(ii, jj, mc) - 1] + if rj <= wz + factor[pz + (rj - 1) * fz + ri - 1] += x + else + pcb[cp2 + SDS._packed(ri - wz, rj - wz, mp) - 1] += x + end + end + end + end + @synchronize + end + end + SDS._front_load!(sh, st, factor, v, li, front_ptr, front_nrows, front_ncols, Val(64), Val(WG)) + @synchronize + w = Int(front_ncols[v]) + g0 = Int(super_ptr[v]) - 1 + for j in 1:w + if LDL + _front_ldl_update!(sh, st, piv, j, v, li, front_ncols, d, g0, Val(WG)) + else + SDS._front_chol_update!(sh, st, piv, j, v, li, front_ncols, Val(64), Val(WG)) + end + @synchronize + if LDL + _front_ldl_scale!(sh, st, piv, j, v, li, front_ncols, Val(WG)) + else + SDS._front_chol_scale!(sh, st, piv, j, v, li, front_ncols, Val(64), Val(WG)) + end + @synchronize + end + f = Int(front_nrows[v]) + p0 = Int(front_ptr[v]) + for q in (li - 1):WG:(w * w - 1) + j = q ÷ w + 1 + i = q - (j - 1) * w + 1 + i >= j && (factor[p0 + (j - 1) * f + i - 1] = (LDL && i == j) ? one(TT) : + sh[SDS._packed(i, j, 64)]) + end + m = f - w + if st[1] == Int32(0) && m <= SYRK_IN + for i in (w + li):WG:f + for kk in 1:w + x = factor[p0 + (kk - 1) * f + i - 1] + for j in 1:(kk - 1) + lij = factor[p0 + (j - 1) * f + i - 1] + x -= LDL ? lij * (real(sh[SDS._packed(j, j, 64)]) * sh[SDS._packed(kk, j, 64)]) : + lij * sh[SDS._packed(kk, j, 64)] + end + factor[p0 + (kk - 1) * f + i - 1] = x / sh[SDS._packed(kk, kk, 64)] + end + end + end + li == 1 && (info[v] = st[1]) + @synchronize + if m <= SYRK_IN + if LDL + _front_ldl_syrk!(factor, pcb, sh, st, v, li, front_ptr, front_nrows, front_ncols, cb2, + Val(WG)) + else + SDS._front_syrk!(factor, pcb, st, v, li, front_ptr, front_nrows, front_ncols, cb2, + Val(WG)) + end + @synchronize + Threads.atomic_fence() # ALL threads: each wave drains its own stores (gfx9) + @synchronize + if li == 1 + Atomix.@atomic cb_done[v] += Int32(1) + pv = Int(snparent[v]) + pv > 0 && (Atomix.@atomic arrived[pv] += Int32(1)) + end + else + Threads.atomic_fence() # ALL threads (gfx9) + @synchronize + if li == 1 + Atomix.@atomic panel_done[v] += Int32(1) + end + end + elseif r == ROLE_TRSM + v = Int(ra[G]) + if li == 1 + _spin_until_eq!(panel_done, v, Int32(1), fscr, scap) + end + @synchronize + Threads.atomic_fence() + f = Int(front_nrows[v]) + w = Int(front_ncols[v]) + p0 = Int(front_ptr[v]) + for q in li:WG:(w * (w + 1) ÷ 2) + j = 1 + acc = w + qq = q + while qq > acc + qq -= acc + acc -= 1 + j += 1 + end + i = j + qq - 1 + # LDL: the factor diagonal is unit; restore d in the same loop (a second + # loop would race this one on the diagonal slots) + sh[SDS._packed(i, j, 64)] = (LDL && i == j) ? TT(d[Int(super_ptr[v]) - 1 + j]) : + factor[p0 + (j - 1) * f + i - 1] + end + @synchronize + if info[v] == Int32(0) + i = w + (Int(rb[G]) - 1) * WG + li + if i <= f + for kk in 1:w + x = factor[p0 + (kk - 1) * f + i - 1] + for j in 1:(kk - 1) + lij = factor[p0 + (j - 1) * f + i - 1] + x -= LDL ? lij * (real(sh[SDS._packed(j, j, 64)]) * sh[SDS._packed(kk, j, 64)]) : + lij * sh[SDS._packed(kk, j, 64)] + end + factor[p0 + (kk - 1) * f + i - 1] = x / sh[SDS._packed(kk, kk, 64)] + end + end + end + @synchronize + Threads.atomic_fence() # ALL threads: each wave drains its own stores (gfx9) + @synchronize + if li == 1 + left = (Atomix.@atomic trsm_left[v] += Int32(-1)) + # tiles_base, not the live tiles_left: the live counter can reach 0 while + # this (stalled) last TRSM is between its decrement and this read, and the + # last tile signals too -- a double signal overshoots cb_done/arrived and + # the != spin loops of the consumers never terminate. The static "front + # has no tiles" predicate is what this check means. + if left == Int32(0) && tiles_base[v] == Int32(0) # new-value semantics: 0 = last finisher + Atomix.@atomic cb_done[v] += Int32(1) + pv = Int(snparent[v]) + pv > 0 && (Atomix.@atomic arrived[pv] += Int32(1)) + end + end + elseif r == ROLE_TILE + v = Int(ra[G]) + if li == 1 + _spin_until_eq!(trsm_left, v, Int32(0), fscr, scap) + end + @synchronize + Threads.atomic_fence() + f = Int(front_nrows[v]) + w = Int(front_ncols[v]) + m = f - w + p0 = Int(front_ptr[v]) + w - 1 + c0 = Int(cb2[v]) + i0 = (Int(rb[G]) - 1) * TILE + j0 = (Int(rc[G]) - 1) * TILE + ok = info[v] == Int32(0) && c0 > 0 + acc1 = zero(TT); acc2 = zero(TT); acc3 = zero(TT); acc4 = zero(TT) + acc5 = zero(TT); acc6 = zero(TT); acc7 = zero(TT); acc8 = zero(TT) + kk0 = 0 + while kk0 < w # k in two halves: Ai/Aj share sh (2 x 1024) + kb = min(32, w - kk0) + if LDL # stage the block's pivots; fold into Ai only + if ok + g0 = Int(super_ptr[v]) - 1 + for c in li:WG:kb + sh[2048 + c] = TT(d[g0 + kk0 + c]) + end + end + @synchronize + end + if ok + for q in (li - 1):WG:(TILE * kb - 1) + rr = q % TILE + 1 + cc = q ÷ TILE + 1 + row = i0 + rr + sh[rr + (cc - 1) * TILE] = row <= m ? + (LDL ? factor[p0 + (kk0 + cc - 1) * f + row] * sh[2048 + cc] : + factor[p0 + (kk0 + cc - 1) * f + row]) : zero(TT) + row2 = j0 + rr + sh[1024 + rr + (cc - 1) * TILE] = + row2 <= m ? factor[p0 + (kk0 + cc - 1) * f + row2] : zero(TT) + end + end + @synchronize + if ok + for (slot, q) in enumerate((li - 1):WG:(TILE * TILE - 1)) + rr = q % TILE + 1 + cc = q ÷ TILE + 1 + a = zero(TT) + for kk in 1:kb + a += sh[rr + (kk - 1) * TILE] * sh[1024 + cc + (kk - 1) * TILE] + end + slot == 1 && (acc1 += a); slot == 2 && (acc2 += a) + slot == 3 && (acc3 += a); slot == 4 && (acc4 += a) + slot == 5 && (acc5 += a); slot == 6 && (acc6 += a) + slot == 7 && (acc7 += a); slot == 8 && (acc8 += a) + end + end + @synchronize + kk0 += 32 + end + if ok + pw = Int(wt[v]) + if pw == 0 + for (slot, q) in enumerate((li - 1):WG:(TILE * TILE - 1)) + rr = q % TILE + 1 + cc = q ÷ TILE + 1 + ii = i0 + rr + jj = j0 + cc + if ii <= m && jj <= m && ii >= jj + a = slot == 1 ? acc1 : slot == 2 ? acc2 : slot == 3 ? acc3 : slot == 4 ? acc4 : + slot == 5 ? acc5 : slot == 6 ? acc6 : slot == 7 ? acc7 : acc8 + pcb[c0 + SDS._packed(ii, jj, m) - 1] -= a + end + end + else # write the finished CB entry through into the parent + fpw = Int(front_nrows[pw]) + wpw = Int(front_ncols[pw]) + mpw = fpw - wpw + ppw = Int(front_ptr[pw]) + cpw = Int(cb2[pw]) + r0w = Int(relind_ptr[v]) - 1 + for (slot, q) in enumerate((li - 1):WG:(TILE * TILE - 1)) + rr = q % TILE + 1 + cc = q ÷ TILE + 1 + ii = i0 + rr + jj = j0 + cc + if ii <= m && jj <= m && ii >= jj + a = slot == 1 ? acc1 : slot == 2 ? acc2 : slot == 3 ? acc3 : slot == 4 ? acc4 : + slot == 5 ? acc5 : slot == 6 ? acc6 : slot == 7 ? acc7 : acc8 + x = pcb[c0 + SDS._packed(ii, jj, m) - 1] - a # children's extend-adds + own syrk + ri = Int(relind[r0w + ii]) + rj = Int(relind[r0w + jj]) + if rj <= wpw + factor[ppw + (rj - 1) * fpw + ri - 1] += x + else + pcb[cpw + SDS._packed(ri - wpw, rj - wpw, mpw) - 1] += x + end + end + end + end + end + @synchronize + Threads.atomic_fence() # ALL threads: each wave drains its own stores (gfx9) + @synchronize + if li == 1 + left = (Atomix.@atomic tiles_left[v] += Int32(-1)) + if left == Int32(0) # Atomix += returns the NEW value: 0 = last finisher + Atomix.@atomic cb_done[v] += Int32(1) + pv = Int(snparent[v]) + pv > 0 && (Atomix.@atomic arrived[pv] += Int32(1)) + Int(wt[v]) > 0 && (Atomix.@atomic ea_left[Int(wt[v])] += Int32(-1)) + end + end + elseif r == ROLE_WPANEL + v = Int(ra[G]); k = Int(rb[G]) + sl = Int(wbase[v]) + k + if li == 1 + if k == 1 + _spin_until_eq!(ea_left, v, Int32(0), fscr, scap) + else + _spin_until_eq!(wupd, sl, Int32(0), fscr, scap) + end + end + @synchronize + Threads.atomic_fence() + f = Int(front_nrows[v]); w = Int(front_ncols[v]) + c0 = (k - 1) * 64 + kb = min(64, w - c0) + p0 = Int(front_ptr[v]) + li == 1 && (st[1] = Int32(0)) + @synchronize + for q in (li - 1):WG:(kb * kb - 1) + jj = q ÷ kb + 1 + ii = q - (jj - 1) * kb + 1 + ii >= jj && (sh[SDS._packed(ii, jj, 64)] = factor[p0 + (c0 + jj - 1) * f + c0 + ii - 1]) + end + @synchronize + for j in 1:kb + if li == 1 + dj = real(sh[SDS._packed(j, j, 64)]) + if LDL + if st[1] == Int32(0) && isfinite(dj) && !iszero(dj) + piv[1] = dj # d stays on the diagonal + d[Int(super_ptr[v]) - 1 + c0 + j] = dj + elseif st[1] == Int32(0) + st[1] = Int32(c0 + j) + piv[1] = one(RT) + end + else + if st[1] == Int32(0) && dj > zero(RT) + piv[1] = sqrt(dj) + sh[SDS._packed(j, j, 64)] = piv[1] + elseif st[1] == Int32(0) + st[1] = Int32(c0 + j) + piv[1] = one(RT) + end + end + end + @synchronize + pvt = piv[1] + if st[1] == Int32(0) + rpv = inv(pvt) + for i in (j + li):WG:kb + if LDL + sh[SDS._packed(i, j, 64)] *= rpv + else + sh[SDS._packed(i, j, 64)] /= pvt + end + end + end + @synchronize + if st[1] == Int32(0) + for q in (li - 1):WG:(kb * kb - 1) + jj = q ÷ kb + 1 + ii = q - (jj - 1) * kb + 1 + if ii >= jj && jj > j + sh[SDS._packed(ii, jj, 64)] -= LDL ? + sh[SDS._packed(ii, j, 64)] * pvt * sh[SDS._packed(jj, j, 64)] : + sh[SDS._packed(ii, j, 64)] * sh[SDS._packed(jj, j, 64)] + end + end + end + @synchronize + end + for q in (li - 1):WG:(kb * kb - 1) + jj = q ÷ kb + 1 + ii = q - (jj - 1) * kb + 1 + ii >= jj && (factor[p0 + (c0 + jj - 1) * f + c0 + ii - 1] = + (LDL && ii == jj) ? one(TT) : sh[SDS._packed(ii, jj, 64)]) + end + li == 1 && (k == 1 || st[1] != Int32(0)) && (info[v] = st[1]) + @synchronize + Threads.atomic_fence() # ALL threads: each wave drains its own stores (gfx9) + @synchronize + if li == 1 + Atomix.@atomic wpdone2[sl] += Int32(1) + end + elseif r == ROLE_WTRSM + v = Int(ra[G]); ch = Int(rb[G]); k = Int(rc[G]) + sl = Int(wbase[v]) + k + if li == 1 + _spin_until_eq!(wpdone2, sl, Int32(1), fscr, scap) + end + @synchronize + Threads.atomic_fence() + f = Int(front_nrows[v]); w = Int(front_ncols[v]) + c0 = (k - 1) * 64 + kb = min(64, w - c0) + p0 = Int(front_ptr[v]) + for q in li:WG:(kb * (kb + 1) ÷ 2) + j = 1 + acc = kb + qq = q + while qq > acc + qq -= acc + acc -= 1 + j += 1 + end + i = j + qq - 1 + # LDL: the factor diagonal is unit; restore d in the same loop (a second + # loop would race this one on the diagonal slots) + sh[SDS._packed(i, j, 64)] = (LDL && i == j) ? TT(d[Int(super_ptr[v]) - 1 + c0 + j]) : + factor[p0 + (c0 + j - 1) * f + c0 + i - 1] + end + @synchronize + if info[v] == Int32(0) + i = c0 + kb + (ch - 1) * WG + li + if i <= f + for kk in 1:kb + x = factor[p0 + (c0 + kk - 1) * f + i - 1] + for j in 1:(kk - 1) + lij = factor[p0 + (c0 + j - 1) * f + i - 1] + x -= LDL ? lij * (real(sh[SDS._packed(j, j, 64)]) * sh[SDS._packed(kk, j, 64)]) : + lij * sh[SDS._packed(kk, j, 64)] + end + factor[p0 + (c0 + kk - 1) * f + i - 1] = x / sh[SDS._packed(kk, kk, 64)] + end + end + end + @synchronize + Threads.atomic_fence() # ALL threads: each wave drains its own stores (gfx9) + @synchronize + if li == 1 + Atomix.@atomic wtrsml[sl] += Int32(-1) + end + elseif r == ROLE_WTILE + v = Int(ra[G]); tii = Int(rb[G]) + k = Int(rc[G]) ÷ 1024; tjj = Int(rc[G]) % 1024 + sl = Int(wbase[v]) + k + if li == 1 + _spin_until_eq!(wtrsml, sl, Int32(0), fscr, scap) + end + @synchronize + Threads.atomic_fence() + f = Int(front_nrows[v]); w = Int(front_ncols[v]) + m = f - w + c0 = (k - 1) * 64 + kb = min(64, w - c0) + p0 = Int(front_ptr[v]) + rb0 = c0 + kb + i0 = rb0 + (tii - 1) * TILE + j0 = rb0 + (tjj - 1) * TILE + ok = info[v] == Int32(0) + acc1 = zero(TT); acc2 = zero(TT); acc3 = zero(TT); acc4 = zero(TT) + acc5 = zero(TT); acc6 = zero(TT); acc7 = zero(TT); acc8 = zero(TT) + kk0 = 0 + while kk0 < kb + kbk = min(32, kb - kk0) + if LDL # stage the block's pivots; fold into Ai only + if ok + g0 = Int(super_ptr[v]) - 1 + c0 + for c in li:WG:kbk + sh[2048 + c] = TT(d[g0 + kk0 + c]) + end + end + @synchronize + end + if ok + for q in (li - 1):WG:(TILE * kbk - 1) + rr = q % TILE + 1 + cc = q ÷ TILE + 1 + row = i0 + rr + sh[rr + (cc - 1) * TILE] = row <= f ? + (LDL ? factor[p0 + (c0 + kk0 + cc - 1) * f + row - 1] * sh[2048 + cc] : + factor[p0 + (c0 + kk0 + cc - 1) * f + row - 1]) : zero(TT) + row2 = j0 + rr + sh[1024 + rr + (cc - 1) * TILE] = + row2 <= f ? factor[p0 + (c0 + kk0 + cc - 1) * f + row2 - 1] : zero(TT) + end + end + @synchronize + if ok + for (slot, q) in enumerate((li - 1):WG:(TILE * TILE - 1)) + rr = q % TILE + 1 + cc = q ÷ TILE + 1 + a = zero(TT) + for kk in 1:kbk + a += sh[rr + (kk - 1) * TILE] * sh[1024 + cc + (kk - 1) * TILE] + end + slot == 1 && (acc1 += a); slot == 2 && (acc2 += a) + slot == 3 && (acc3 += a); slot == 4 && (acc4 += a) + slot == 5 && (acc5 += a); slot == 6 && (acc6 += a) + slot == 7 && (acc7 += a); slot == 8 && (acc8 += a) + end + end + @synchronize + kk0 += 32 + end + if ok + c0cb = Int(cb2[v]) + for (slot, q) in enumerate((li - 1):WG:(TILE * TILE - 1)) + rr = q % TILE + 1 + cc = q ÷ TILE + 1 + ii = i0 + rr + jj = j0 + cc + if ii <= f && ii >= jj + a = slot == 1 ? acc1 : slot == 2 ? acc2 : slot == 3 ? acc3 : slot == 4 ? acc4 : + slot == 5 ? acc5 : slot == 6 ? acc6 : slot == 7 ? acc7 : acc8 + if jj <= w + Atomix.@atomic factor[p0 + (jj - 1) * f + ii - 1] -= a + elseif c0cb > 0 + Atomix.@atomic pcb[c0cb + SDS._packed(ii - w, jj - w, m) - 1] -= a + end + end + end + end + @synchronize + Threads.atomic_fence() # ALL threads: each wave drains its own stores (gfx9) + @synchronize + if li == 1 + if j0 < w + Atomix.@atomic wupd[Int(wbase[v]) + j0 ÷ 64 + 1] += Int32(-1) + end + if j0 + TILE > w + Atomix.@atomic wcbl[v] += Int32(-1) + end + end + elseif r == ROLE_WSIG + v = Int(ra[G]); K = Int(rb[G]) + sl = Int(wbase[v]) + K + if li == 1 + _spin_until_eq!(wpdone2, sl, Int32(1), fscr, scap) + _spin_until_eq!(wtrsml, sl, Int32(0), fscr, scap) + _spin_until_eq!(wcbl, v, Int32(0), fscr, scap) + Threads.atomic_fence() + Atomix.@atomic cb_done[v] += Int32(1) + pv = Int(snparent[v]) + pv > 0 && (Atomix.@atomic arrived[pv] += Int32(1)) + end + end + end +end + +# --------------------------------------------------------------------------- +# host-wide LDL path: the Cholesky vendor sequence (potrf/trsm/herk) with the +# diagonal blocks factored by a small kernel (no vendor no-pivot LDL exists), +# the SAME vendor TRSM with the unit-triangular flag, broadcast column scalings +# for D, and the CB through the SAME vendor GEMM against a D-scaled scratch. + +# one workgroup: no-pivot LDL of the kb×kb diagonal block at (c0b+1, c0b+1) of the +# f×w panel at p0; unit-L written back (diag = 1), D to d[g0 + c0b + j] +@kernel function wide_ldl_panel_kernel!(factor, d, info, v, p0, f, c0b, kb, g0, ::Val{WG}) where {WG} + li = @index(Local, Linear) + @uniform TT = eltype(factor) + sh = @localmem TT (NL64,) + st = @localmem Int32 (1,) + pv = @localmem TT (1,) + @inbounds begin + for q in li:WG:(kb * (kb + 1) ÷ 2) + j = 1 + acc = kb + qq = q + while qq > acc + qq -= acc + acc -= 1 + j += 1 + end + i = j + qq - 1 + sh[SDS._packed(i, j, 64)] = factor[p0 + (c0b + j - 1) * f + c0b + i - 1] + end + li == 1 && (st[1] = Int32(0)) + end + @synchronize + for j in 1:kb + @inbounds begin + if li == 1 + dk = real(sh[SDS._packed(j, j, 64)]) + if st[1] == Int32(0) && isfinite(dk) && !iszero(dk) + pv[1] = dk + d[g0 + c0b + j] = dk + elseif st[1] == Int32(0) + st[1] = Int32(c0b + j) + pv[1] = one(TT) + end + end + end + @synchronize + @inbounds begin + pvt = pv[1] + if st[1] == Int32(0) + rpv = inv(pvt) + for i in (j + li):WG:kb + sh[SDS._packed(i, j, 64)] *= rpv + end + end + end + @synchronize + @inbounds begin + pvt = pv[1] + if st[1] == Int32(0) + for q in (li - 1):WG:(kb * kb - 1) + jj = q ÷ kb + 1 + ii = q - (jj - 1) * kb + 1 + if ii >= jj && jj > j + sh[SDS._packed(ii, jj, 64)] -= sh[SDS._packed(ii, j, 64)] * pvt * sh[SDS._packed(jj, j, 64)] + end + end + end + end + @synchronize + end + @inbounds begin + for q in (li - 1):WG:(kb * kb - 1) + jj = q ÷ kb + 1 + ii = q - (jj - 1) * kb + 1 + ii >= jj && (factor[p0 + (c0b + jj - 1) * f + c0b + ii - 1] = + ii == jj ? one(TT) : sh[SDS._packed(ii, jj, 64)]) + end + li == 1 && st[1] != Int32(0) && info[v] == Int32(0) && (info[v] = st[1]) + end +end + +@kernel function zero_upper_kernel!(factor, p0, f, w) + q = @index(Global, Linear) + @inbounds begin + j = (q - 1) ÷ w + 1 + i = q - (j - 1) * w + i < j && (factor[p0 + (j - 1) * f + i - 1] = zero(eltype(factor))) + end +end + +# potrf-derived LDL of a positive-definite wide front: A = C Cᵀ = L D Lᵀ with +# L = C diag(C)⁻¹ and d = diag(C)², and F21_chol = L21 D^½ makes the Cholesky +# herk CB bit-identical to the LDL one — so the host path is the VERBATIM +# Cholesky vendor sequence plus three micro-kernels. Valid only where the +# front's pivots share a sign (always, for an SPD matrix factored as "S"). +@kernel function chol2ldl_d_kernel!(factor, d, p0, f, g0) + j = @index(Global, Linear) + @inbounds begin + c = real(factor[p0 + (j - 1) * f + j - 1]) + d[g0 + j] = c * c + end +end +@kernel function chol2ldl_scale_kernel!(factor, p0, f, w) # strict lower: /= diag of the column + q = @index(Global, Linear) + @inbounds begin + j = (Int(q) - 1) ÷ f + 1 + i = Int(q) - (j - 1) * f + i > j && (factor[p0 + (j - 1) * f + i - 1] /= real(factor[p0 + (j - 1) * f + j - 1])) + end +end +@kernel function chol2ldl_diag_kernel!(factor, p0, f) + j = @index(Global, Linear) + @inbounds factor[p0 + (j - 1) * f + j - 1] = one(eltype(factor)) +end + +function _factor_panel_c_ldl_potrf!(N, S, fp, v, p, backend) + f = Int(S.schedule.rows[v]) + w = Int(S.schedule.width[v]) + p0 = Int(S.layout.panel_ptr[v]) + g0 = fp.sptr_h[v] - 1 + SDS._factor_panel_c!(N.factor, fp.pcb, N.work, N.info, v, p0, f, w, fp.cb2_h[v], p) + chol2ldl_d_kernel!(backend, 128)(N.factor, N.d, p0, f, g0; ndrange = w) + chol2ldl_scale_kernel!(backend, 128)(N.factor, p0, f, w; ndrange = f * w) + chol2ldl_diag_kernel!(backend, 128)(N.factor, p0, f; ndrange = w) + return nothing +end + +# the host-wide front for LDL: blocked right-looking over 64-column panels, then one +# D-scaled GEMM for the contribution block (the Cholesky path's trsm/herk shape) +function _factor_panel_c_ldl!(N, S, fp, v, p, backend) + T = eltype(N.factor) + f = Int(S.schedule.rows[v]) + w = Int(S.schedule.width[v]) + p0 = Int(S.layout.panel_ptr[v]) + c0 = fp.cb2_h[v] + g0 = fp.sptr_h[v] - 1 + m = f - w + gi = SDS.select_impl(:gemm, N.factor, :auto) + P = reshape(view(N.factor, p0:(p0 + f * w - 1)), f, w) + for k in 1:cld(w, 64) + c0b = (k - 1) * 64 + kb = min(64, w - c0b) + wide_ldl_panel_kernel!(backend, 128)(N.factor, N.d, N.info, v, p0, f, c0b, kb, g0, Val(128); + ndrange = 128) + r1 = c0b + kb + 1 + r1 <= f || continue + Lblk = view(P, (c0b + 1):(c0b + kb), (c0b + 1):(c0b + kb)) + R = view(P, r1:f, (c0b + 1):(c0b + kb)) # A L1⁻ᵀ = W = L D + SDS._trsm_impl!(p.trsm, 'R', 'L', 'C', 'U', one(T), Lblk, R) + dvb = transpose(view(N.d, (g0 + c0b + 1):(g0 + c0b + kb))) + if r1 <= w # trailing panel columns + Wscr = reshape(view(fp.ldlw, 1:((f - r1 + 1) * kb)), f - r1 + 1, kb) + Wscr .= R # keep W; R becomes final L + R ./= dvb + Lc = view(P, r1:w, (c0b + 1):(c0b + kb)) + Ctr = view(P, r1:f, r1:w) + SDS._gemm_impl!(gi, 'N', 'T', -one(T), Wscr, Lc, one(T), Ctr) + else + R ./= dvb + end + end + zero_upper_kernel!(backend, 128)(N.factor, p0, f, w; ndrange = w * w) + (m > 0 && c0 > 0) || return nothing + L21 = view(P, (w + 1):f, 1:w) + W21 = reshape(view(fp.ldlw, 1:(m * w)), m, w) + W21 .= L21 .* transpose(view(N.d, (g0 + 1):(g0 + w))) + C = reshape(view(N.work, 1:(m * m)), m, m) + SDS._gemm_impl!(gi, 'N', 'T', -one(T), W21, L21, zero(T), C) + SDS.pack_add!(fp.pcb, c0, N.work, m) + return nothing +end + +@kernel function panel_zero_kernel!(factor, @Const(list), @Const(front_ptr), @Const(front_nrows), + @Const(front_ncols), ::Val{WG}) where {WG} + li = @index(Local, Linear) + G = @index(Group, Linear) + @inbounds begin + v = Int(list[G]) + f = Int(front_nrows[v]) + w = Int(front_ncols[v]) + p0 = Int(front_ptr[v]) + for q in (li - 1):WG:(f * w - 1) + factor[p0 + q] = zero(eltype(factor)) + end + end +end + +@kernel function signal_kernel!(cb_done, arrived, @Const(snparent), v) + li = @index(Local, Linear) + @inbounds if li == 1 + Threads.atomic_fence() + Atomix.@atomic cb_done[v] += Int32(1) + pv = Int(snparent[v]) + pv > 0 && (Atomix.@atomic arrived[pv] += Int32(1)) + end +end + +# subtree launcher with a custom (sorted) list, local memory sized by eltype +function launch_subtrees_sorted!(N, S, nzval, list, lb, backend, ldl::Bool = false) + SDS._with_local_bytes(Int(lb)) do LBv + _sorted_subtree_launch(N, S, nzval, list, Val(ldl), LBv, Val(SDS.SUBTREE_WORKGROUP), backend) + end +end +function _sorted_subtree_launch(N::SDS.Numeric{T}, S, nzval, list, ::Val{LDL}, ::Val{LB}, ::Val{WG}, + backend) where {T, LDL, LB, WG} + bm = SDS.batch_map(N; first = 1) + if LDL + # the schedule sized the "S" budget classes with the LDLT reserve: match it + k! = subtree_ldl_kernel!(backend, WG) + k!(N.factor, N.stack, N.info, N.d, N.piv, N.pivot_kind, nzval, S.amap, S.amap_ptr, S.amap_src, + list, bm, S.subtree_ptr, S.subtree_nodes, S.super_ptr, S.front_ptr, S.front_nrows, S.front_ncols, + S.cb_ptr, S.local_front, S.local_cb, S.child_ptr, S.child_list, S.relind_ptr, S.relind, + Val((LB - SDS.SUBTREE_LOCAL_RESERVE_LDLT) ÷ sizeof(T)), Val(WG); + ndrange = WG * length(list) * bm.nact) + else + k! = SDS.subtree_cholesky_kernel!(backend, WG) + k!(N.factor, N.stack, N.info, nzval, S.amap, S.amap_ptr, S.amap_src, list, bm, S.subtree_ptr, + S.subtree_nodes, S.front_ptr, S.front_nrows, S.front_ncols, S.cb_ptr, S.local_front, S.local_cb, + S.child_ptr, S.child_list, S.relind_ptr, S.relind, + Val((LB - SDS.SUBTREE_LOCAL_RESERVE) ÷ sizeof(T)), Val(WG); ndrange = WG * length(list) * bm.nact) + end + return nothing +end + +# --------------------------------------------------------------------------- +# host: the topologically ordered role list + counter bases + stream-2 program +function build_fused_plan(s; ldl::Bool = false, inkernel_wides::Union{Nothing, Bool} = nothing, + wides_potrf::Bool = false, merge_segments::Bool = false, + wgf::Int = WGF, scap::Integer = 128, ea_chunk::Integer = 16384) + S = s.symbolic + plan = s.numeric.plan + backend = KA.get_backend(s.numeric.factor) + T = eltype(s.numeric.factor) + ldl && (T <: Real || throw(SDS.NotSupportedError("fused LDL mode: real element types only"))) + # the tile roles keep cld(TILE*TILE, wgf) partial sums in exactly 8 registers per + # thread; a smaller workgroup needs more than 8 and would silently drop/reuse slots + (wgf >= 128 && wgf % 32 == 0) || + throw(SDS.InvalidValueError("fused plan: workgroup size $wgf; expected a multiple of 32, >= 128")) + # LDL defaults to in-kernel wides: the host-wide LDL path (_factor_panel_c_ldl!) is + # correct but needs ~5 launches per 64-column block where Cholesky's potrf is one + # vendor call -- measured SLOWER on the GV100 (55.5 vs 38.6 ms tuned). Pass + # inkernel_wides = false explicitly to use it. + inkernel_wides = inkernel_wides === nothing ? (ldl && !wides_potrf) : inkernel_wides + nodes = S.schedule.group_nodes + rows_h = S.schedule.rows + width_h = S.schedule.width + cbp = Array(S.cb_ptr) + chp = Array(S.child_ptr) + chl = Array(S.child_list) + stp = Array(S.subtree_ptr) + stn = Array(S.subtree_nodes) + ns = length(width_h) + + covered = falses(ns) # fronts factored by the subtree launches + for q in eachindex(stn) + covered[stn[q]] = true + end + + role = Int8[]; ra = Int32[]; rb = Int32[]; rc = Int32[] + wides = Int[] + ea_left = zeros(Int32, ns) + trsm_left = zeros(Int32, ns) + tiles_left = zeros(Int32, ns) + arrived_base = zeros(Int32, ns) + cb_done_base = zeros(Int32, ns) + for v in 1:ns + covered[v] && (cb_done_base[v] = 1) + end + + # proposal B: a child writes its CB through into the parent when the parent is a + # pre-assembled C-group front and the child is its ONLY CB child (any kind) + cset = Set{Int}() + for k in eachindex(plan.group_first) + plan.group_width[k] > 0 && continue + for q in plan.group_first[k]:plan.group_last[k] + push!(cset, Int(nodes[q])) + end + end + wt_h = zeros(Int32, ns) + nwt = 0; ewt = 0 + for v in 1:ns + (covered[v] || cbp[v] == 0) && continue + width_h[v] <= 64 || continue # wide children have no tiles to settle the debt + m = rows_h[v] - width_h[v] + m > SYRK_IN || continue + pv = Int(S.partition.snparent[v]) + (pv > 0 && pv in cset) || continue + count(c -> cbp[c] > 0, chl[chp[pv]:(chp[pv + 1] - 1)]) == 1 || continue + wt_h[v] = Int32(pv) + nwt += 1; ewt += m * (m + 1) ÷ 2 + end + SDS._log(SDS.LOG_DEBUG, () -> "fused plan: write-through $nwt fronts, $(round(ewt / 1e6; digits = 2))M CB entries") + + emit_ea(v) = for kc in chp[v]:(chp[v + 1] - 1) + c = chl[kc] + cbp[c] > 0 || continue + if wt_h[c] == v + ea_left[v] += 1 # settled by the child's last write-through tile + continue + end + mc = rows_h[c] - width_h[c] + for ch in 1:cld(mc * mc, ea_chunk) + push!(role, ROLE_EA); push!(ra, Int32(v)); push!(rb, Int32(c)); push!(rc, Int32(ch)) + ea_left[v] += 1 + end + end + emit_split(v) = begin + m = rows_h[v] - width_h[v] + m > SYRK_IN || return nothing + for ch in 1:cld(m, wgf) + push!(role, ROLE_TRSM); push!(ra, Int32(v)); push!(rb, Int32(ch)); push!(rc, Int32(0)) + trsm_left[v] += 1 + end + if cbp[v] > 0 + # ntl, not T: this closure assigns in function scope, and T is the element type + ntl = cld(m, TILE) + for tjj in 1:ntl, tii in tjj:ntl + push!(role, ROLE_TILE); push!(ra, Int32(v)); push!(rb, Int32(tii)); push!(rc, Int32(tjj)) + tiles_left[v] += 1 + end + end + return nothing + end + + # in-kernel wides: per-(front, panel) counter slots; host arithmetic MIRRORS the + # kernel's decrement conditions exactly (j0 < w -> wupd[kj]; j0 + TILE > w -> wcb[v]) + wbase_h = zeros(Int32, ns) + wupd_b = Int32[]; wtl_b = Int32[] + wcb_b = zeros(Int32, ns) + nwslots = 0 + emit_wide(v) = begin + f = rows_h[v]; w = width_h[v] + K = cld(w, 64) + wbase_h[v] = Int32(nwslots) + append!(wupd_b, zeros(Int32, K)); append!(wtl_b, zeros(Int32, K)) + for k in 1:K + push!(role, ROLE_WPANEL); push!(ra, Int32(v)); push!(rb, Int32(k)); push!(rc, Int32(0)) + c0 = (k - 1) * 64 + kb = min(64, w - c0) + nbelow = f - (c0 + kb) + nch = cld(nbelow, wgf) + for ch in 1:nch + push!(role, ROLE_WTRSM); push!(ra, Int32(v)); push!(rb, Int32(ch)); push!(rc, Int32(k)) + end + wtl_b[nwslots + k] = Int32(nch) + nt = cld(nbelow, TILE) + # the WTILE role packs rc = k * 1024 + tjj, so tjj must fit in 10 bits; + # beyond that the tile would silently decode as a different panel + nt <= 1023 || + throw(SDS.NotSupportedError("fused plan: wide front with $nbelow rows below " * + "the panel; the WTILE encoding holds cld(rows, $TILE) <= 1023")) + for tjj in 1:nt, tii in tjj:nt + push!(role, ROLE_WTILE); push!(ra, Int32(v)); push!(rb, Int32(tii)) + push!(rc, Int32(k * 1024 + tjj)) + j0 = c0 + kb + (tjj - 1) * TILE + j0 < w && (wupd_b[nwslots + j0 ÷ 64 + 1] += Int32(1)) + j0 + TILE > w && (wcb_b[v] += Int32(1)) + end + end + push!(role, ROLE_WSIG); push!(ra, Int32(v)); push!(rb, Int32(K)); push!(rc, Int32(0)) + nwslots += K + return nothing + end + + segments = NTuple{3, Any}[] # (base, count, wides-at-boundary) + seg_base = 0 + for k in eachindex(plan.group_first) + a, b = plan.group_first[k], plan.group_last[k] + if plan.group_width[k] > 0 + for q in a:b + v = nodes[q] + push!(role, ROLE_PANEL_B); push!(ra, Int32(v)); push!(rb, Int32(0)); push!(rc, Int32(0)) + end + for q in a:b + emit_split(nodes[q]) + end + else + gwides = Int[] + for q in a:b + emit_ea(nodes[q]) + end + for q in a:b + v = nodes[q] + if width_h[v] <= 64 + push!(role, ROLE_PANEL_C); push!(ra, Int32(v)); push!(rb, Int32(0)); push!(rc, Int32(0)) + else + push!(gwides, v) + end + end + for q in a:b + width_h[nodes[q]] <= 64 && emit_split(nodes[q]) + end + if !isempty(gwides) + if inkernel_wides + foreach(emit_wide, gwides) + gwides = Int[] + end + if !(inkernel_wides && merge_segments) # merged: one launch, no split + push!(segments, (seg_base, length(role) - seg_base, gwides)) + seg_base = length(role) + end + append!(wides, gwides) + end + end + end + push!(segments, (seg_base, length(role) - seg_base, Int[])) + + cb2_h = zeros(Int64, ns) + off = 1 + for v in 1:ns + (covered[v] || cbp[v] == 0) && continue + m = rows_h[v] - width_h[v] + cb2_h[v] = off + off += m * (m + 1) ÷ 2 + end + czero = Int32[] # C-group fronts: panel zeroed by a prelaunch kernel + for k in eachindex(plan.group_first) + plan.group_width[k] > 0 && continue + append!(czero, Int32.(nodes[plan.group_first[k]:plan.group_last[k]])) + end + nchild = Int32[chp[v + 1] - chp[v] for v in 1:ns] + for v in 1:ns + cnt = 0 + for kc in chp[v]:(chp[v + 1] - 1) + covered[chl[kc]] && (cnt += 1) + end + arrived_base[v] = Int32(cnt) + end + + stp2 = stp + ssub = Vector{Any}() + for k in eachindex(plan.sub_first) + ids = [nodes[q] for q in plan.sub_first[k]:plan.sub_last[k]] + sort!(ids; by = e -> stp2[e + 1] - stp2[e], rev = true) + push!(ssub, _dvec(backend, ids)) + end + + SDS._log(SDS.LOG_DEBUG, () -> "fused plan: $(length(role)) roles, $(length(segments)) segments, " * + "$(length(wides)) host wide fronts") + return (; role = _dvec(backend, role), ra = _dvec(backend, ra), rb = _dvec(backend, rb), + rc = _dvec(backend, rc), wides, ssub, segments, + cb2 = _dvec(backend, Int32.(cb2_h)), cb2_h, pcb = KA.zeros(backend, T, off - 1), + czero = _dvec(backend, czero), wt = _dvec(backend, wt_h), + nchild = _dvec(backend, nchild), + arrived_base = _dvec(backend, arrived_base), ea_base = _dvec(backend, ea_left), + trsm_base = _dvec(backend, trsm_left), tiles_base = _dvec(backend, tiles_left), + cb_base = _dvec(backend, cb_done_base), + arrived = KA.zeros(backend, Int32, ns), ea = KA.zeros(backend, Int32, ns), + trsm = KA.zeros(backend, Int32, ns), tiles = KA.zeros(backend, Int32, ns), + pdone = KA.zeros(backend, Int32, ns), cbd = KA.zeros(backend, Int32, ns), + wbase = _dvec(backend, wbase_h), wupd_b = _dvec(backend, wupd_b), wtl_b = _dvec(backend, wtl_b), + wcb_b = _dvec(backend, wcb_b), wupd = KA.zeros(backend, Int32, max(nwslots, 1)), + wtl = KA.zeros(backend, Int32, max(nwslots, 1)), wpd = KA.zeros(backend, Int32, max(nwslots, 1)), + wcb = KA.zeros(backend, Int32, ns), wgf, scap = Int32(scap), eac = Int32(ea_chunk), ldl, + ldl_potrf = ldl && wides_potrf, + sptr_h = Array(S.super_ptr), + ldlw = KA.zeros(backend, T, + ldl ? maximum(vcat([1], [max(rows_h[v] * 64, + (rows_h[v] - width_h[v]) * width_h[v]) + for v in wides])) : 1)) +end + +function refact_fused!(s, nzval, fp) + N, S = s.numeric, s.symbolic + plan = N.plan + backend = KA.get_backend(N.factor) + p = SDS._front_impls(N, S, :auto) + # per-front statuses start clean every refactorization: the narrow-panel kernels + # overwrite info[v] unconditionally, but the host-wide LDL kernels only SET it on + # failure -- without this reset one failed refactorization stays failed forever. + # (Before the subtree launches: kernels on this stream would otherwise be clobbered.) + fill!(view(N.info, 1:SDS.nsupernodes(S)), Int32(0)) + copyto!(fp.arrived, fp.arrived_base) + copyto!(fp.ea, fp.ea_base) + copyto!(fp.trsm, fp.trsm_base) + copyto!(fp.tiles, fp.tiles_base) + copyto!(fp.cbd, fp.cb_base) + fill!(fp.pdone, Int32(0)) + length(fp.wupd_b) > 0 && copyto!(fp.wupd, fp.wupd_b) + length(fp.wtl_b) > 0 && copyto!(fp.wtl, fp.wtl_b) + fill!(fp.wpd, Int32(0)) + copyto!(fp.wcb, fp.wcb_b) + for k in eachindex(plan.sub_first) + launch_subtrees_sorted!(N, S, nzval, fp.ssub[k], plan.sub_local[k], backend, fp.ldl) + end + nodes = S.schedule.group_nodes + fill!(fp.pcb, zero(eltype(fp.pcb))) + isempty(fp.czero) || + panel_zero_kernel!(backend, WGF)(N.factor, fp.czero, S.front_ptr, S.front_nrows, S.front_ncols, + Val(WGF); ndrange = WGF * length(fp.czero)) + for k in eachindex(plan.group_first) # scatter A into every C-group front (panel only) + plan.group_width[k] > 0 && continue + a, b = plan.group_first[k], plan.group_last[k] + SDS.scatter_A!(N, S, nzval, a, b - a + 1) + end + for (base, count, gwides) in fp.segments + count > 0 && + fused_fact_kernel!(backend, fp.wgf)(N.factor, N.stack, fp.pcb, N.info, nzval, S.amap, S.amap_ptr, + S.amap_src, fp.cb2, fp.wt, fp.role, fp.ra, fp.rb, fp.rc, fp.nchild, + fp.arrived, fp.ea, fp.pdone, fp.trsm, fp.tiles, fp.tiles_base, fp.cbd, + S.snparent, S.front_ptr, S.front_nrows, S.front_ncols, + S.cb_ptr, S.child_ptr, S.child_list, S.relind_ptr, S.relind, + Int32(base), fp.wbase, fp.wupd, fp.wpd, fp.wtl, fp.wcb, + fp.scap, fp.eac, N.d, S.super_ptr, Val(fp.ldl), Val(fp.wgf); + ndrange = fp.wgf * count) + for v in gwides # host-wide fronts at the segment boundary + if fp.ldl && fp.ldl_potrf + _factor_panel_c_ldl_potrf!(N, S, fp, v, p, backend) + elseif fp.ldl + _factor_panel_c_ldl!(N, S, fp, v, p, backend) + else + SDS._factor_panel_c!(N.factor, fp.pcb, N.work, N.info, v, S.layout.panel_ptr[v], + S.schedule.rows[v], S.schedule.width[v], fp.cb2_h[v], p) + end + signal_kernel!(backend, 32)(fp.cbd, fp.arrived, S.snparent, v; ndrange = 32) + end + end + SDS.assemble_schur!(N, S, nzval) + # the stats kernel ALSO reduces the per-front info into info[ns + 1], which + # _numeric_info! reads -- LDL needs that reduction too: without it a zero/NaN + # pivot sets a front's info while the reduced slot keeps reporting the stale + # success of an earlier factorization. In LDL mode N.stats' positive-pivot + # fields are Cholesky-interpretation garbage; inertia comes from the signs of + # d (the MadNLP extension), never from these stats. + SDS.cholesky_stats!(N, S) + return SDS._numeric_info!(N, S, true) +end + +end # module diff --git a/src/numeric/front.jl b/src/numeric/front.jl index 9652732..672a759 100644 --- a/src/numeric/front.jl +++ b/src/numeric/front.jl @@ -42,17 +42,9 @@ end if j <= w && st[1] == 0 ajj = real(L11[_packed(j, j, W)]) if ajj > 0 - d = sqrt(ajj) - li == 1 && (piv[1] = d) - r = w - j - for q in (li - 1):WG:(r * r - 1) - k = j + 1 + q ÷ r - i = j + 1 + q % r - if i >= k - lij = L11[_packed(i, j, W)] / d - lkj = L11[_packed(k, j, W)] / d - L11[_packed(i, k, W)] -= lij * conj(lkj) - end + rd = inv(sqrt(ajj)) # recomputed per work item: no barrier needed; + for i in (j + li):WG:w # strictly below the diagonal, since the diagonal + L11[_packed(i, j, W)] *= rd # is replaced in the second half end elseif li == 1 st[1] = Int32(j) # not positive (or NaN): stop, as LAPACK potrf @@ -62,16 +54,20 @@ end return nothing end -# column j, second half: scale the column by the pivot stored by the first half +# column j, second half: rank-1 update of the trailing block with the scaled column +# (multiply-add only; the packed-triangle enumeration replaces the guarded square) @inline function _front_chol_scale!(L11, st, piv, j, s, li, front_ncols, ::Val{W}, ::Val{WG}) where {W, WG} @inbounds begin w = front_ncols[s] if j <= w && st[1] == 0 - d = piv[1] - for i in (j + li):WG:w - L11[_packed(i, j, W)] /= d + li == 1 && (L11[_packed(j, j, W)] = sqrt(real(L11[_packed(j, j, W)]))) + r = w - j + for q in (li - 1):WG:(r * (r + 1) ÷ 2 - 1) + a, b = _tri_decode(q, r) + i = j + a + k = j + b + L11[_packed(i, k, W)] -= L11[_packed(i, j, W)] * conj(L11[_packed(k, j, W)]) end - li == 1 && (L11[_packed(j, j, W)] = d) end end return nothing diff --git a/src/numeric/subtree.jl b/src/numeric/subtree.jl index 27031dc..022425d 100644 --- a/src/numeric/subtree.jl +++ b/src/numeric/subtree.jl @@ -138,7 +138,10 @@ end return nothing end -# column j, first half: check the pivot, rank-1 update of the trailing front with the scaled column +# column j, first half: check the pivot and scale the column below the diagonal by it +# (every work item recomputes `rd` from local memory: cheaper than sharing it through +# a barrier, and it keeps the second half free of divisions; the diagonal itself is +# written in the second half, so no work item reads a value another is overwriting) @inline function _subtree_chol_update!(buf, ctl, piv, j, li, ::Val{WG}) where {WG} @inbounds begin if ctl[_ST_STATUS] == 0 @@ -146,17 +149,9 @@ end lf = ctl[_ST_LF] - 1 ajj = real(buf[lf + _packed(j, j, f)]) if ajj > 0 - d = sqrt(ajj) - li == 1 && (piv[1] = d) - r = f - j - for q in (li - 1):WG:(r * r - 1) - k = j + 1 + q ÷ r - i = j + 1 + q % r - if i >= k - lij = buf[lf + _packed(i, j, f)] / d - lkj = buf[lf + _packed(k, j, f)] / d - buf[lf + _packed(i, k, f)] -= lij * conj(lkj) - end + rd = inv(sqrt(ajj)) + for i in (j + li):WG:f + buf[lf + _packed(i, j, f)] *= rd end elseif li == 1 ctl[_ST_STATUS] = j % eltype(ctl) # not positive (or NaN): stop, as LAPACK potrf @@ -166,17 +161,23 @@ end return nothing end -# column j, second half: scale the column below the diagonal by the pivot +# column j, second half: rank-1 update of the trailing front with the scaled column +# (multiply-add only: the former divide-in-the-update doubled the flop cost in f64 +# divisions; the packed-triangle enumeration halves the work items of the old +# guarded full-square loop) @inline function _subtree_chol_scale!(buf, ctl, piv, j, li, ::Val{WG}) where {WG} @inbounds begin if ctl[_ST_STATUS] == 0 f = ctl[_ST_F] lf = ctl[_ST_LF] - 1 - d = piv[1] - for i in (j + li):WG:f - buf[lf + _packed(i, j, f)] /= d + li == 1 && (buf[lf + _packed(j, j, f)] = sqrt(real(buf[lf + _packed(j, j, f)]))) + r = f - j + for q in (li - 1):WG:(r * (r + 1) ÷ 2 - 1) + a, b = _tri_decode(q, r) + i = j + a + k = j + b + buf[lf + _packed(i, k, f)] -= buf[lf + _packed(i, j, f)] * conj(buf[lf + _packed(k, j, f)]) end - li == 1 && (buf[lf + _packed(j, j, f)] = d) end end return nothing diff --git a/src/options.jl b/src/options.jl index 399e6ae..f0fcded 100644 --- a/src/options.jl +++ b/src/options.jl @@ -17,7 +17,7 @@ const CONFIG_PARAMETERS = ("reordering_alg", "factorization_alg", "solve_alg", "hybrid_device_memory_limit", "use_cuda_register_memory", "host_nthreads", "hybrid_execute_mode", "pivot_epsilon_alg", "nd_nlevels", "ubatch_size", "ubatch_index", "use_superpanels", "device_count", "device_indices", - "schur_mode", "deterministic_mode", "nd_ubfactor") + "schur_mode", "deterministic_mode", "nd_ubfactor", "nd_nseps", "nd_seed") """ DATA_PARAMETERS @@ -127,6 +127,8 @@ defaults below. Keyword arguments are applied through [`setparam!`](@ref), so | `schur_mode` | `0` | off | | `deterministic_mode` | `0` | atomic forward solve allowed | | `nd_ubfactor` | `-1` | ordering library default | +| `nd_nseps` | `-1` | ordering library default (1 separator per split) | +| `nd_seed` | `-1` | ordering library default | | `ir_mode` | `IR_PLAIN` | plain iterative refinement | | `factor_precision` | `nothing` | factors in the input precision | | `amalgamation` | `(max_width = 32, zero_fraction = 0.25, min_width = 8)` | | @@ -176,6 +178,8 @@ mutable struct Options schur_mode::Int deterministic_mode::Int nd_ubfactor::Int + nd_nseps::Int + nd_seed::Int # configuration parameters beyond cuDSS (PLAN §1.7) ir_mode::IRMode factor_precision::Union{Nothing, DataType} @@ -202,7 +206,7 @@ mutable struct Options opts = new( REORDERING_DEFAULT, FACTORIZATION_DEFAULT, SOLVE_DEFAULT, MATCHING_NONE, 0, 0, 0.0, PIVOT_AUTO, 0.01, nothing, PIVOT_EPSILON_DEFAULT, -1, - 0, 0, 1, 0, 0, 10, 0, -1, 1, 0, 0, -1, + 0, 0, 1, 0, 0, 10, 0, -1, 1, 0, 0, -1, -1, -1, IR_PLAIN, nothing, DEFAULT_AMALGAMATION, SCHEDULE_AUTO, PIVOT_PAIRS_DEFAULT, PIVOT_PAIR_TOLERANCE, 64, 512, copy(DEFAULT_SUBTREE_BUDGETS), SUBTREE_PARALLELISM, 0, -1, nothing, nothing, nothing, nothing, nothing, nothing, @@ -286,6 +290,8 @@ const PARAMETER_SPECS = Dict{String, ParameterSpec}( "schur_mode" => ParameterSpec(:config, :port, :schur_mode), "deterministic_mode" => ParameterSpec(:config, :port, :deterministic_mode), "nd_ubfactor" => ParameterSpec(:config, :port, :nd_ubfactor), + "nd_nseps" => ParameterSpec(:config, :port, :nd_nseps), + "nd_seed" => ParameterSpec(:config, :port, :nd_seed), # CUDSS.jl data parameters "info" => ParameterSpec(:data, :solver, :none), "lu_nnz" => ParameterSpec(:data, :output, :none), @@ -415,11 +421,22 @@ for (field, lo, hi, expected) in ( (:schur_mode, 0, 1, "0 or 1"), (:deterministic_mode, 0, 1, "0 or 1"), (:nd_ubfactor, -1, typemax(Int), "-1 (library default) or an integer ≥ 0"), + # nd_nseps has a dedicated method below: 0 is in [-1, typemax] but invalid + (:nd_seed, -1, typemax(Int), "-1 (library default) or an integer ≥ 0"), ) @eval _parse_option(::Val{$(QuoteNode(field))}, value, _) = _parse_int($(String(field)), value, $lo, $hi, $expected) end +# nd_nseps = 0 lies inside the plain [-1, typemax] range but is not a METIS value; +# the documented domain is -1 (library default) or >= 1, so reject it loudly +# instead of silently coercing (the ordering code maps "< 1" to the default) +function _parse_option(::Val{:nd_nseps}, value, _) + v = _parse_int("nd_nseps", value, -1, typemax(Int), "-1 (library default) or an integer ≥ 1") + v == 0 && throw(InvalidValueError("nd_nseps = 0; expected -1 (library default) or an integer ≥ 1")) + return v +end + for field in (:ir_tol, :pivot_threshold, :pivot_pair_tolerance) @eval _parse_option(::Val{$(QuoteNode(field))}, value, _) = _parse_float($(String(field)), value) end diff --git a/src/solver.jl b/src/solver.jl index 3d70c81..9bfbf98 100644 --- a/src/solver.jl +++ b/src/solver.jl @@ -454,6 +454,10 @@ function _check_analysis_supported(solver::DirectSolver{T}) where {T} return nothing end +# the backend when the symbolic maps may be built on it (device.jl), else nothing +_device_backend(solver::DirectSolver) = + solver.backend isa KernelAbstractions.GPU ? solver.backend : nothing + function _reorder!(solver::DirectSolver{T}) where {T} _check_analysis_supported(solver) _poll_interrupt(solver.options.user_host_interrupt) @@ -468,7 +472,7 @@ function _reorder!(solver::DirectSolver{T}) where {T} solver.analysis_colval = m === nothing || m.symmetric ? solver.host_colval : matched_colval(m, solver.host_colval, A.index) P = SymmetricPattern(solver.host_rowptr, solver.analysis_colval, A.nrows, solver.structure; - view = _stored_view(solver), index = A.index) + view = _stored_view(solver), index = A.index, device = _device_backend(solver)) if solver.options.schur_mode == 1 # Schur complement mode: the Schur rows and columns last, no 2×2 pivot pairs flags = schur_flags(solver.options.user_schur_indices, A.nrows) @@ -535,7 +539,7 @@ function _symbolic!(solver::DirectSolver{T, INT}) where {T, INT} A = solver.A ord = solver.ordering P = SymmetricPattern(solver.host_rowptr, solver.analysis_colval, A.nrows, solver.structure; - view = _stored_view(solver), index = A.index) + view = _stored_view(solver), index = A.index, device = _device_backend(solver)) if solver.schur === nothing sp = supernode_partition(factor_pattern(P, ord), ord.perm, opts) schur = 0 @@ -548,7 +552,7 @@ function _symbolic!(solver::DirectSolver{T, INT}) where {T, INT} max_local = max_local_bytes(solver.backend)) layout = build_layout(sp, sc; ldlt = _is_ldlt_structure(solver.structure)) Sh = Symbolic(sp, sc, layout, solver.host_rowptr, solver.analysis_colval, A.nrows, solver.structure; - view = _stored_view(solver), index = A.index) + view = _stored_view(solver), index = A.index, device = _device_backend(solver)) nrhs = solver.workspace === nothing ? solver.nbatch : max_rhs(solver.workspace) Sd = adapt(solver.backend, Sh, INT) solver.host_symbolic = Sh diff --git a/src/symbolic/device.jl b/src/symbolic/device.jl new file mode 100644 index 0000000..f3a36b3 --- /dev/null +++ b/src/symbolic/device.jl @@ -0,0 +1,254 @@ +# Device construction of the symbolic maps (PLAN §2.3 step 6 hot spots). +# The host analysis stays the reference; a GPU solver routes three +# data-parallel pieces through KernelAbstractions kernels with element-exact +# results (the tests assert equality against the host originals): +# +# * `device_symmetric_pattern` — the both-triangle adjacency of +# `SymmetricPattern` by packed-key sort and deduplication; +# * `device_assembly_map` — `assembly_map` for the symmetric +# structures, one thread per CSR row, binary search per entry; +# * `device_group_amap` — `_group_amap` (unsigned) as a packed-key +# sort reproducing the host `(abs, neg, p)` order exactly. +# +# The packed 64-bit keys need `n`, `nnz` < 2^32 and factor offsets < 2^31; +# [`device_maps_supported`](@ref) checks the bounds and the callers fall back +# to the host route when it returns `false` (a larger-than-2^31-entry factor +# is a > 16 GiB panel buffer, beyond a single-GPU analysis anyway). + +""" + device_maps_supported(backend, structure, view, n, nnz, factor_len) -> Bool + +Whether the device symbolic-map routes apply: a GPU backend, a symmetric +structure (`"SPD"`, `"HPD"`, `"S"`, `"H"`), a triangular stored view, and +sizes that fit the packed 64-bit sort keys. +""" +function device_maps_supported(backend, structure, view, n::Integer, nnz::Integer, + factor_len::Integer) + backend isa KernelAbstractions.GPU || return false + s = _structure(structure) + (s == STRUCTURE_SPD || s == STRUCTURE_HPD || s == STRUCTURE_SYMMETRIC || + s == STRUCTURE_HERMITIAN) || return false + _matrix_view(view) == VIEW_FULL && return false + # the maps store values up to nnz + 1 / n / factor_len + 1 in Int32; admit only + # what Int32 holds (the host route serves the rest) + return n + 1 <= typemax(Int32) && nnz + 1 <= typemax(Int32) && factor_len + 1 <= typemax(Int32) +end + +# device vector of `x` as `I` +function _device_vec(backend, ::Type{I}, x) where {I <: Integer} + y = KernelAbstractions.allocate(backend, I, length(x)) + isempty(x) || copyto!(y, I.(x)) + return y +end + +# --------------------------------------------------------------------------- +# symmetric pattern + +@kernel function _pattern_keys_kernel!(keys, @Const(rowptr), @Const(colval), base::Int32, + upper::Bool) + r = @index(Global) + @inbounds for p in (rowptr[r] + (one(eltype(rowptr)) - base)):(rowptr[r + 1] - base) + c = Int64(colval[p]) + (1 - Int64(base)) + q = Int64(p) + # the host `_entry_filter`: diagonal dropped by the pattern, entries + # outside the declared triangle dropped by the filter + if c == r || (upper ? c < r : c > r) + keys[2q - 1] = typemax(UInt64) + keys[2q] = typemax(UInt64) + else + keys[2q - 1] = (UInt64(r) << 32) | UInt64(c) + keys[2q] = (UInt64(c) << 32) | UInt64(r) + end + end +end + +""" + device_symmetric_pattern(backend, rowptr, colval, n, view, index) -> (colptr, rowval) + +The `SymmetricPattern` arrays (1-based, sorted, deduplicated, no diagonal) +of the stored entries, built on `backend`. `rowptr`/`colval` are host or +device vectors with base `index`. +""" +function device_symmetric_pattern(backend, rowptr, colval, n::Integer, view, index) + m = length(colval) + base = _index_base(index) == INDEX_ONE ? Int32(1) : Int32(0) + upper = _matrix_view(view) == VIEW_UPPER + hrp = Array(rowptr) + hcv = Array(colval) + _host_pattern(hrp, hcv, n, index) # validation only: same errors as the host route + I = m + 1 <= typemax(Int32) && n + 1 <= typemax(Int32) ? Int32 : Int64 + drp = _device_vec(backend, I, hrp) + dcv = _device_vec(backend, I, hcv) + keys = KernelAbstractions.allocate(backend, UInt64, 2m) + n > 0 && _pattern_keys_kernel!(backend)(keys, drp, dcv, base, upper; ndrange = Int(n)) + KernelAbstractions.synchronize(backend) + isempty(keys) || sort!(keys) # empty: device sorts reject length 0 + hk = Array(keys) + nvalid = searchsortedfirst(hk, typemax(UInt64)) - 1 + colptr = zeros(Int, n + 1) + rowval = Vector{Int}(undef, nvalid) + colptr[1] = 1 + nv = 0 + prev = typemax(UInt64) + @inbounds for i in 1:nvalid + k = hk[i] + k == prev && continue # duplicate of a 'F' view + prev = k + nv += 1 + rowval[nv] = Int(k & 0xffffffff) + colptr[Int(k >> 32) + 1] += 1 + end + resize!(rowval, nv) + for j in 1:n + colptr[j + 1] += colptr[j] + end + return colptr, rowval +end + +# --------------------------------------------------------------------------- +# assembly map + +@kernel function _amap_kernel!(amap, @Const(rowptr), @Const(colval), @Const(iperm), + @Const(col2sn), @Const(snrowptr), @Const(snrowval), + @Const(superptr), @Const(panelptr), base::Int32, + upper::Bool, neg::Bool) + r = @index(Global) + T = eltype(amap) + @inbounds for p in (rowptr[r] + (one(eltype(rowptr)) - base)):(rowptr[r + 1] - base) + c = Int(colval[p]) + (1 - Int(base)) + if upper ? r <= c : r >= c + i = Int(iperm[r]) + j = Int(iperm[c]) + flip = i < j + if flip + i, j = j, i + end + s = Int(col2sn[j]) + lo = Int(snrowptr[s]) + hi = Int(snrowptr[s + 1]) - 1 + f = hi - lo + 1 + a = lo # binary search for i + b = hi + while a < b + mid = (a + b) >> 1 + if Int(snrowval[mid]) < i + a = mid + 1 + else + b = mid + end + end + if a <= hi && Int(snrowval[a]) == i + off = Int(panelptr[s]) + (j - Int(superptr[s])) * f + (a - lo + 1) - 1 + amap[p] = T(flip && neg ? -off : off) + else + # the host route throws "outside the symbolic factor" here; a kernel + # cannot, so it writes a sentinel the host wrapper checks once + amap[p] = typemax(T) + end + else + amap[p] = zero(T) + end + end +end + +""" + device_assembly_map(backend, sp, layout, rowptr, colval, n, structure; view, index) + -> amap::Vector{Int} + +[`assembly_map`](@ref) on `backend` for the symmetric structures with a +triangular stored view; identical output. The caller checks +[`device_maps_supported`](@ref). +""" +function device_assembly_map(backend, sp::SupernodePartition, layout::Layout, + rowptr, colval, n::Integer, structure; + view = VIEW_FULL, index = INDEX_ONE) + s = _structure(structure) + v = _matrix_view(view) + base = _index_base(index) == INDEX_ONE ? Int32(1) : Int32(0) + I = Int32 # bounds checked by the caller + drp = _device_vec(backend, I, Array(rowptr)) + dcv = _device_vec(backend, I, Array(colval)) + amap = KernelAbstractions.allocate(backend, I, length(colval)) + n > 0 && _amap_kernel!(backend)(amap, drp, dcv, _device_vec(backend, I, sp.iperm), + _device_vec(backend, I, sp.col2sn), + _device_vec(backend, I, sp.rowptr), + _device_vec(backend, I, sp.rowval), + _device_vec(backend, I, sp.super_ptr), + _device_vec(backend, I, layout.panel_ptr), + base, v == VIEW_UPPER, + _is_hermitian(s); ndrange = Int(n)) + KernelAbstractions.synchronize(backend) + h = Vector{Int}(Array(amap)) + bad = findfirst(==(Int(typemax(I))), h) + bad === nothing || + throw(InvalidValueError("entry at stored position $bad is outside the symbolic factor (pattern changed?)")) + return h +end + +# --------------------------------------------------------------------------- +# amap grouping + +@kernel function _group_keys_kernel!(keys, @Const(amap)) + p = @index(Global) + @inbounds begin + a = Int(amap[p]) + if a == 0 + keys[p] = typemax(UInt64) # dropped entries sort last + else + off = UInt64(a < 0 ? -a : a) + # the host order: (abs(amap[p]), amap[p] < 0, p) + keys[p] = (off << 33) | (UInt64(a < 0) << 32) | UInt64(p) + end + end +end + +@kernel function _group_count_kernel!(cnt, @Const(keys), @Const(panelptr), nvalid::Int64) + gi = @index(Global) + @inbounds if gi <= nvalid + off = Int((keys[gi] >> 33) & 0x7fffffff) + a = 1 # searchsortedlast(panel_ptr, off) + b = length(panelptr) + while a < b + mid = (a + b + 1) >> 1 + if Int(panelptr[mid]) <= off + a = mid + else + b = mid - 1 + end + end + KernelAbstractions.@atomic cnt[a + 1] += one(eltype(cnt)) + end +end + +""" + device_group_amap(backend, amap, layout, ns) -> (amap_ptr, order) + +`_group_amap` (unsigned) on `backend`: identical order and per-owner +counts. The signed (`"G"`) grouping stays on the host. +""" +function device_group_amap(backend, amap::Vector{Int}, layout::Layout, ns::Int) + m = length(amap) + damap = _device_vec(backend, Int32, amap) + keys = KernelAbstractions.allocate(backend, UInt64, m) + m > 0 && _group_keys_kernel!(backend)(keys, damap; ndrange = m) + KernelAbstractions.synchronize(backend) + isempty(keys) || sort!(keys) # empty: device sorts reject length 0 + hk = Array(keys) + nvalid = searchsortedfirst(hk, typemax(UInt64)) - 1 + order = Vector{Int}(undef, nvalid) + @inbounds for i in 1:nvalid + order[i] = Int(hk[i] & 0xffffffff) + end + cnt = KernelAbstractions.allocate(backend, Int32, ns + 1) + fill!(cnt, Int32(0)) + m > 0 && _group_count_kernel!(backend)(cnt, keys, _device_vec(backend, Int32, layout.panel_ptr), + Int64(nvalid); ndrange = m) + KernelAbstractions.synchronize(backend) + hc = Array(cnt) + amap_ptr = Vector{Int}(undef, ns + 1) + amap_ptr[1] = 1 + for s in 1:ns + amap_ptr[s + 1] = amap_ptr[s] + Int(hc[s + 1]) + end + return amap_ptr, order +end diff --git a/src/symbolic/maps.jl b/src/symbolic/maps.jl index 19f3013..7821b91 100644 --- a/src/symbolic/maps.jl +++ b/src/symbolic/maps.jl @@ -193,10 +193,19 @@ Assemble the host [`Symbolic`](@ref) from the supernodes, schedule and layout of a pattern and its user CSR arrays (only the pattern is read). """ function Symbolic(sp::SupernodePartition, sc::Schedule, layout::Layout, rowptr::AbstractVector{<:Integer}, - colval::AbstractVector{<:Integer}, n::Integer, structure; view = VIEW_FULL, index = INDEX_ONE) + colval::AbstractVector{<:Integer}, n::Integer, structure; view = VIEW_FULL, index = INDEX_ONE, + device = nothing) ns = nsupernodes(sp) - amap = assembly_map(sp, layout, rowptr, colval, n, structure; view, index) - amap_ptr, amap_src = _group_amap(amap, layout, ns, _structure(structure) == STRUCTURE_GENERAL) + signed = _structure(structure) == STRUCTURE_GENERAL + use_device = device !== nothing && !signed && + device_maps_supported(device, structure, view, n, length(colval), layout.factor_len) + if use_device + amap = device_assembly_map(device, sp, layout, rowptr, colval, n, structure; view, index) + amap_ptr, amap_src = device_group_amap(device, amap, layout, ns) + else + amap = assembly_map(sp, layout, rowptr, colval, n, structure; view, index) + amap_ptr, amap_src = _group_amap(amap, layout, ns, signed) + end child_ptr = zeros(Int, ns + 1) child_ptr[1] = 1 for s in 1:ns @@ -229,7 +238,7 @@ function Symbolic(sp::SupernodePartition, sc::Schedule, layout::Layout, rowptr:: end """ - symbolic_analysis(A::CSR, structure, view = 'F'; opts = Options(), T = eltype(A), max_local = typemax(Int)) + symbolic_analysis(A::CSR, structure, view = 'F'; opts = Options(), T = eltype(A), device = nothing, max_local = typemax(Int)) -> Symbolic{Int, Vector{Int}} The whole host analysis of PLAN §2.3: [`SymmetricPattern`](@ref), @@ -244,18 +253,18 @@ which bound the regime-A budgets ([`resolve_subtree_budgets`](@ref)); the default leaves them unbounded (host analyses, the CPU backend). """ function symbolic_analysis(A::CSR, structure, view = VIEW_FULL; opts::Options = Options(), T::Type = eltype(A), - max_local::Integer = typemax(Int)) + max_local::Integer = typemax(Int), device = nothing) A.nrows == A.ncols || throw(InvalidValueError("the matrix must be square, got $(A.nrows) × $(A.ncols)")) rowptr = Array(A.rowptr) colval = Array(A.colval) - P = SymmetricPattern(rowptr, colval, A.nrows, structure; view, index = A.index) + P = SymmetricPattern(rowptr, colval, A.nrows, structure; view, index = A.index, device) pp = analysis_pairs(P, rowptr, colval, A.nzval, A.nrows, structure, opts; view, index = A.index) ord = compute_ordering(P, opts; T, pp.pairs, pp.candidates) sp = supernode_partition(factor_pattern(P, ord), ord.perm, opts) sc = build_schedule(sp, opts, T; reserve = subtree_local_reserve(structure), elsize = schedule_elsize(structure, T), max_local) layout = build_layout(sp, sc; ldlt = _is_ldlt_structure(_structure(structure))) - return Symbolic(sp, sc, layout, rowptr, colval, A.nrows, structure; view, index = A.index) + return Symbolic(sp, sc, layout, rowptr, colval, A.nrows, structure; view, index = A.index, device) end function _index_vector(backend::KernelAbstractions.Backend, x::AbstractVector{<:Integer}, ::Type{INT}, diff --git a/src/symbolic/ordering.jl b/src/symbolic/ordering.jl index 05bc8f7..9433e52 100644 --- a/src/symbolic/ordering.jl +++ b/src/symbolic/ordering.jl @@ -1,33 +1,28 @@ # Symbolic step 2 (PLAN §2.3): fill-reducing ordering through CliqueTrees.jl, # user permutations, and the automatic AMD/ND choice by a cost model. # -# AMD and MMD are always available (CliqueTrees + AMD.jl). Nested dissection -# needs Metis.jl: `ext/SparseDirectSolverMetisExt.jl` stores the function that -# builds the CliqueTrees algorithm object in `ND_PROVIDER` when Metis is loaded. +# AMD and MMD come from CliqueTrees + AMD.jl, nested dissection from CliqueTrees' +# METIS extension (Metis.jl is a dependency of the package). """ - ND_PROVIDER - -`Ref` holding `nothing` or a function `(nd_nlevels, nd_ubfactor) -> alg` that -returns the CliqueTrees nested-dissection algorithm. Set by the Metis extension. -""" -const ND_PROVIDER = Ref{Any}(nothing) - + nd_algorithm(nd_nlevels, nd_ubfactor, nd_nseps, nd_seed) -> CliqueTrees.METIS + +The CliqueTrees nested-dissection algorithm object of the `nd_*` parameters: +`METIS_NodeND` with `ufactor = nd_ubfactor`, `nseps = nd_nseps` and +`seed = nd_seed` (`-1`: METIS defaults; `nseps < 1` and `seed < 0` fall back to +the defaults). `nd_nlevels` is cuDSS' *minimum* number of dissection levels; +`METIS_NodeND` dissects recursively until the parts are small, so the minimum +holds whenever the graph is large enough to be split that often. CliqueTrees' +level-capped `ND{S}` (AMD below the cap) was 4–13× slower and gave 15–80% more +fill on Laplacians (T05 Report). """ - nd_available() -> Bool - -Whether nested dissection can be used (Metis.jl is loaded, which activates -`SparseDirectSolverMetisExt`). -""" -nd_available() = ND_PROVIDER[] !== nothing - -function _nd_algorithm(opts::Options) - nd_available() || - throw(NotSupportedError("nested dissection (reordering_alg = \"algo4\") needs Metis.jl; " * - "run `using Metis` to load SparseDirectSolverMetisExt")) - return ND_PROVIDER[](opts.nd_nlevels, opts.nd_ubfactor) +function nd_algorithm(nlevels::Integer, ubfactor::Integer, nseps::Integer, seed::Integer) + return CliqueTrees.METIS(; ufactor = ubfactor, nseps = nseps >= 1 ? nseps : -1, + seed = seed >= 0 ? seed : -1) end +_nd_algorithm(opts::Options) = nd_algorithm(opts.nd_nlevels, opts.nd_ubfactor, opts.nd_nseps, opts.nd_seed) + """ OrderingCandidate @@ -51,8 +46,7 @@ Result of [`compute_ordering`](@ref): `sdepth` (supernodal schedule depth), `nsupernodes`, `cost` of the chosen ordering, `candidates::Vector{OrderingCandidate}` (every ordering evaluated, the chosen one included), `auto::Bool` (whether the automatic choice ran), - `level_flops` (the depth weight of [`ordering_cost`](@ref) used) and - `nd_available::Bool`; + `level_flops` (the depth weight of [`ordering_cost`](@ref) used); * `pairs::Vector{Tuple{Int,Int}}`: the 2×2 pivot candidate pairs `(partner, candidate)` the ordering was computed with (empty for no pair): `perm` puts each candidate right after its partner, and the symbolic factorization runs on @@ -66,7 +60,7 @@ struct Ordering alg_used::Symbol stats::@NamedTuple{nnz_L::Int, flops::Float64, nlevels::Int, sdepth::Int, nsupernodes::Int, cost::Float64, candidates::Vector{OrderingCandidate}, auto::Bool, level_flops::Float64, - nd_available::Bool, pair_rounds::Int} + pair_rounds::Int} pairs::Vector{Tuple{Int, Int}} end @@ -219,13 +213,14 @@ Fill-reducing ordering of the pattern `P` (PLAN §2.3 step 2): * `opts.user_perm` set: that permutation, validated, accepted 0- or 1-based and returned 1-based (`alg_used = :user`); * `reordering_alg = "algo5"`: natural ordering; `"algo3"`: AMD - (`CliqueTrees.AMD()`); `"algo4"`: nested dissection with Metis (needs - `using Metis`, otherwise [`NotSupportedError`](@ref)): `METIS_NodeND` with - `ufactor = nd_ubfactor` (`-1`: METIS default); `nd_nlevels` is the cuDSS - *minimum* number of dissection levels, which METIS' full recursion meets on - every graph large enough to be split that often; + (`CliqueTrees.AMD()`); `"algo4"`: nested dissection ([`nd_algorithm`](@ref)): + `METIS_NodeND` with `ufactor = nd_ubfactor`, `nseps = nd_nseps` and + `seed = nd_seed` (`-1`: METIS defaults; on fill-bound problems `nd_nseps = 4` + trades a slower ordering for less fill); `nd_nlevels` is the cuDSS *minimum* + number of dissection levels, which METIS' full recursion meets on every graph + large enough to be split that often; `"algo1"`/`"algo2"`: AMD on the symmetric pattern; -* `"default"`: AMD and, if Metis is loaded, ND are both evaluated with +* `"default"`: AMD and ND are both evaluated with [`evaluate_ordering`](@ref) and the one with the lower [`ordering_cost`](@ref) `flops + level_flops × sdepth` (supernodal schedule depth) is chosen; `stats.candidates` lists both with their schedule depths. @@ -289,8 +284,7 @@ function compute_ordering(P::SymmetricPattern, opts::Options; T::Type = Float64, opts.user_perm === nothing && throw(InvalidValueError("alg = :user needs opts.user_perm")) perms = [(:user, _validate_user_perm(opts.user_perm, P.n), fixed_pairs, 0)] elseif requested === :auto - algs = nd_available() ? (:amd, :nd) : (:amd,) - perms = [order_on(a) for a in algs] + perms = [order_on(a) for a in (:amd, :nd)] else perms = [order_on(requested)] end @@ -311,7 +305,7 @@ function compute_ordering(P::SymmetricPattern, opts::Options; T::Type = Float64, c = evaluated[best] stats = (nnz_L = c.nnz_L, flops = c.flops, nlevels = c.nlevels, sdepth = c.sdepth, nsupernodes = c.nsupernodes, cost = c.cost, candidates = evaluated, auto = requested === :auto, level_flops = Float64(level_flops), - nd_available = nd_available(), pair_rounds = rounds) + pair_rounds = rounds) return Ordering(perm, invperm(perm), alg_used, stats, used_pairs) end @@ -395,7 +389,7 @@ function compute_schur_ordering(P::SymmetricPattern, opts::Options, schur::Abstr e = evaluate_ordering(P, perm; T, level_flops, amalgamation = _amalgamation(opts)) stats = (nnz_L = e.nnz_L, flops = e.flops, nlevels = e.nlevels, sdepth = e.sdepth, nsupernodes = e.nsupernodes, cost = e.cost, candidates = candidates, auto = auto, level_flops = Float64(level_flops), - nd_available = nd_available(), pair_rounds = 0) + pair_rounds = 0) return Ordering(perm, invperm(perm), alg_used, stats, Tuple{Int, Int}[]) end diff --git a/src/symbolic/pattern.jl b/src/symbolic/pattern.jl index 59d97c1..b88f895 100644 --- a/src/symbolic/pattern.jl +++ b/src/symbolic/pattern.jl @@ -24,7 +24,7 @@ present). SymmetricPattern(n, colptr, rowval) SymmetricPattern(A::CSR, structure; view = 'F') - SymmetricPattern(rowptr, colval, n, structure; view = 'F', index = 'O') + SymmetricPattern(rowptr, colval, n, structure; view = 'F', index = 'O', device = nothing) The first form wraps arrays that already satisfy the invariants (checked). The others build the pattern from a CSR pattern, following PLAN §2.3 step 1: @@ -148,9 +148,13 @@ function _selected_entries(rp, cv, n, structure, view) end function SymmetricPattern(rowptr::AbstractVector{<:Integer}, colval::AbstractVector{<:Integer}, n::Integer, - structure; view = VIEW_FULL, index = INDEX_ONE) + structure; view = VIEW_FULL, index = INDEX_ONE, device = nothing) s = _structure(structure) v = _matrix_view(view) + if device !== nothing && device_maps_supported(device, s, v, n, length(colval), 0) + colptr_d, rowval_d = device_symmetric_pattern(device, rowptr, colval, n, v, index) + return SymmetricPattern(Int(n), colptr_d, rowval_d) + end rp, cv = _host_pattern(rowptr, colval, n, index) rows, cols, _ = _selected_entries(rp, cv, n, s, v) # off-diagonal entries in both orientations, then column-sort and deduplicate diff --git a/src/symbolic/schedule.jl b/src/symbolic/schedule.jl index 1a30d71..558ce9e 100644 --- a/src/symbolic/schedule.jl +++ b/src/symbolic/schedule.jl @@ -113,6 +113,22 @@ packed_length(m::Integer) = Int(m) * (Int(m) + 1) ÷ 2 # column j starts after (j - 1)(2m - j + 2)/2 entries @inline _packed(i, j, m) = (j - 1) * (2 * m - j + 2) ÷ 2 + i - j + 1 +# inverse of the packed lower-triangle enumeration: 0-based q in an r x r lower +# triangle (col-major, n = r(r+1)/2 entries) -> 1-based (a, b), a >= b. Enumerating +# only the triangle halves the work-item count of the guarded full-square loops. +# Integer isqrt (exact, and no Float64 in kernels: Metal has no fp64); the two +# boundary fixups are kept as a belt against any isqrt implementation quirk. +@inline function _tri_decode(q, r) + n = r * (r + 1) ÷ 2 + p = n - 1 - q + t = (isqrt(8 * p + 1) - 1) ÷ 2 + t * (t + 1) ÷ 2 > p && (t -= 1) + (t + 1) * (t + 2) ÷ 2 <= p && (t += 1) + b = r - t + a = r - (p - t * (t + 1) ÷ 2) + return a, b +end + """ takes_c_path(schedule, s) -> Bool diff --git a/src/types.jl b/src/types.jl index 40785e1..540de19 100644 --- a/src/types.jl +++ b/src/types.jl @@ -99,7 +99,7 @@ end ReorderingAlg `reordering_alg` values (cuDSS `cudssReorderingAlg_t`): `"default"` automatic -AMD/ND choice (ND needs `using Metis`) by flops plus a charge per level of the +AMD/ND choice by flops plus a charge per level of the supernodal schedule (the same on every backend), `"algo1"` BTF+COLAMD and `"algo2"` COLAMD (both fall back to the symmetric-pattern path), `"algo3"` AMD, `"algo4"` nested dissection, `"algo5"` natural ordering. diff --git a/test/Project.toml b/test/Project.toml index 8fe74e3..d3ff28c 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -5,7 +5,6 @@ DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab" KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c" Krylov = "ba0b0d4f-ebba-5204-a429-3ac8c609bfb7" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" -Metis = "2679e427-3c69-5b7f-982b-ece356f1e94b" ParallelTestRunner = "d3525ed8-44d0-4b2c-a655-542cee43accc" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" @@ -20,7 +19,6 @@ DelimitedFiles = "1" KernelAbstractions = "0.9" Krylov = "0.10" LinearAlgebra = "1.13" -Metis = "1" ParallelTestRunner = "2" Printf = "1.13" Random = "1.13" diff --git a/test/runtests.jl b/test/runtests.jl index 17da5af..312bdc3 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -32,7 +32,6 @@ const init_code = quote using Aqua using KernelAbstractions using SparseDirectSolver - using Metis # activates SparseDirectSolverMetisExt (nested-dissection orderings, T05) using Krylov # activates SparseDirectSolverKrylovExt (ir_mode = "fgmres", T18) const SDS = SparseDirectSolver end diff --git a/test/test_aqua.jl b/test/test_aqua.jl index dc73de5..57db540 100644 --- a/test/test_aqua.jl +++ b/test/test_aqua.jl @@ -6,4 +6,8 @@ # CUDA job), so give it more headroom; a real persistent task still # fails, it just takes longer to report. -Aqua.test_all(SparseDirectSolver; ambiguities = false, persistent_tasks = (tmax = 120,)) +# undefined_exports is excluded HERE only: SDSSolver/SDSSolverOptions are declared +# globals the MadNLP extension assigns, and MadNLP is not in the test environment. +# CI's Aqua workflow loads MadNLP and runs that check with the exports assigned. +Aqua.test_all(SparseDirectSolver; ambiguities = false, undefined_exports = false, + persistent_tasks = (tmax = 120,)) diff --git a/test/test_options.jl b/test/test_options.jl index 984c552..aa49dff 100644 --- a/test/test_options.jl +++ b/test/test_options.jl @@ -15,7 +15,7 @@ const CUDSS_JL_CONFIG_PARAMETERS = ("reordering_alg", "factorization_alg", "solv "hybrid_device_memory_limit", "use_cuda_register_memory", "host_nthreads", "hybrid_execute_mode", "pivot_epsilon_alg", "nd_nlevels", "ubatch_size", "ubatch_index", "use_superpanels", "device_count", "device_indices", - "schur_mode", "deterministic_mode", "nd_ubfactor") + "schur_mode", "deterministic_mode", "nd_ubfactor", "nd_nseps", "nd_seed") # Configuration parameters PLAN §1.3/§1.7 mark port or reinterpret: # name => (valid value, value of the wrong type). @@ -41,6 +41,8 @@ const PORTED_CONFIG_VALUES = Dict{String, Tuple{Any, Any}}( "schur_mode" => (1, "on"), "deterministic_mode" => (1, 1.0), "nd_ubfactor" => (30, 0.3), + "nd_nseps" => (4, 0.25), + "nd_seed" => (3, 0.3), # PLAN §1.7 "pivot_sign" => (Int8[1, -1, 0, 1], [1.0, -1.0]), "ir_mode" => ("fgmres", :fgmres), @@ -118,7 +120,7 @@ end ("reordering_alg", "algo6"), ("reordering_alg", 6), ("factorization_alg", "algo3"), ("solve_alg", "algo2"), ("matching_alg", "algo7"), ("pivot_epsilon_alg", "algo3"), ("use_superpanels", 2), ("ubatch_index", -2), ("ubatch_size", -1), - ("nd_ubfactor", -2), ("schedule", "fast"), ("ir_mode", "gmres"), + ("nd_ubfactor", -2), ("nd_nseps", -2), ("nd_seed", -2), ("schedule", "fast"), ("ir_mode", "gmres"), ("amalgamation", (min_width = 64,)), ("amalgamation", (zero_fraction = -0.1,)), ("amalgamation", (foo = 1,)), ("pivot_sign", [2, 0]), ("user_schur_indices", [0, 2]), ("ubatch_mask", [1, -1])) diff --git a/test/test_symbolic_device.jl b/test/test_symbolic_device.jl new file mode 100644 index 0000000..b34a6fc --- /dev/null +++ b/test/test_symbolic_device.jl @@ -0,0 +1,68 @@ +# Device construction of the symbolic maps (src/symbolic/device.jl): the +# kernels must produce ELEMENT-IDENTICAL results to the host originals, on +# every backend (the CPU backend runs the same kernels through KA). + +random_pattern_csr(n, density, view) = begin + A = random_spd(Float64, n, density) + T = view == 'U' ? triu(A) : tril(A) + C = SDS.CSR(SparseMatrixCSC{Float64, Int}(T)) + (Vector{Int}(C.rowptr), Vector{Int}(C.colval)) +end + +@testset "device symbolic maps ($(backend_name(backend)))" for backend in BACKENDS + for (n, density) in ((0, 0.0), (1, 0.0), (40, 0.15), (300, 0.02), (1500, 0.004)), + view in ('L', 'U'), structure in ("SPD", "S") + + rowptr, colval = random_pattern_csr(n, density, view) + # pattern: host route vs device route + Ph = SDS.SymmetricPattern(rowptr, colval, n, structure; view, index = 'O') + Pd = SDS.SymmetricPattern(rowptr, colval, n, structure; view, index = 'O', device = backend) + @test Ph == Pd # GPU: device route; CPU: gated to host + n == 0 && continue + # maps: build the partition once, then both map routes on it + opts = Options() + ord = SDS.compute_ordering(Ph, opts; alg = :amd) + sp = SDS.supernode_partition(SDS.factor_pattern(Ph, ord), ord.perm, opts) + sc = SDS.build_schedule(sp, opts, Float64; + reserve = SDS.subtree_local_reserve(structure), + elsize = SDS.schedule_elsize(structure, Float64)) + layout = SDS.build_layout(sp, sc; ldlt = structure == "S") + amap_h = SDS.assembly_map(sp, layout, rowptr, colval, n, structure; view, index = 'O') + ptr_h, src_h = SDS._group_amap(amap_h, layout, SDS.nsupernodes(sp)) + # the kernels run on every backend here (the solver gate excludes the + # CPU backend only as a performance choice; correctness is universal) + amap_d = SDS.device_assembly_map(backend, sp, layout, rowptr, colval, n, structure; + view, index = 'O') + ptr_d, src_d = SDS.device_group_amap(backend, amap_h, layout, SDS.nsupernodes(sp)) + cp_d, rv_d = SDS.device_symmetric_pattern(backend, rowptr, colval, n, view, 'O') + @test length(amap_h) > 0 # the comparisons are not vacuous + @test amap_d == amap_h + @test ptr_d == ptr_h + @test src_d == src_h + @test cp_d == Ph.colptr + @test rv_d == Ph.rowval + end + # end-to-end: an analyzed solver's stored maps equal the host originals + # recomputed from its own state (on GPU backends the stored maps came + # through the device route; on the CPU backend this is a host identity) + for T in eltypes_among((Float64,)) + A = random_spd(T, 200, 0.03) + solver = DirectSolver(api_matrix(backend, tril(A), Int32), spd_structure(T), 'L') + execute!("analysis", solver, nothing, nothing) + Sh = solver.host_symbolic + amap_ref = SDS.assembly_map(Sh.partition, Sh.layout, solver.host_rowptr, solver.analysis_colval, + 200, spd_structure(T); view = 'L', index = solver.A.index) + @test length(amap_ref) > 0 + @test Sh.amap == amap_ref + ptr_ref, src_ref = SDS._group_amap(amap_ref, Sh.layout, SDS.nsupernodes(Sh.partition)) + @test Sh.amap_ptr == ptr_ref + @test Sh.amap_src == src_ref + # the factorization still works through the device-built maps + execute!("factorization", solver, nothing, nothing) + b = rand(T, 200) + bd = to_device(backend, b) + xd = similar(bd) + execute!("solve", solver, xd, bd; asynchronous = false) + @test relres(A, to_host(xd), b) <= tol(T) + end +end diff --git a/test/test_symbolic_etree.jl b/test/test_symbolic_etree.jl index b4e68b9..23490e4 100644 --- a/test/test_symbolic_etree.jl +++ b/test/test_symbolic_etree.jl @@ -60,7 +60,6 @@ end lap2 = laplacian2d(30, 30) lap3 = laplacian3d(10, 10, 10) kkt = kkt_matrix(300, 100, 1.0e-8) - @test SDS.nd_available() for (name, A) in (("lap2d", lap2), ("lap3d", lap3), ("kkt", kkt)) n = size(A, 1) P = SDS.SymmetricPattern(SDS.CSR(A), "S"; view = 'F') @@ -111,8 +110,12 @@ end end end end - @test SDS.ND_PROVIDER[](10, 50).ufactor == 50 - @test SDS.ND_PROVIDER[](0, -1).ufactor == -1 + @test SDS.nd_algorithm(10, 50, -1, -1).ufactor == 50 + @test SDS.nd_algorithm(0, -1, -1, -1).ufactor == -1 + @test SDS.nd_algorithm(10, -1, 4, -1).nseps == 4 + @test SDS.nd_algorithm(10, -1, 0, -1).nseps == -1 + @test SDS.nd_algorithm(10, -1, 4, 3).seed == 3 + @test SDS.nd_algorithm(10, -1, 4, -1).seed == -1 @testset "user_perm" begin A = laplacian2d(7, 6) @@ -133,24 +136,6 @@ end @test thrown(() -> SDS.compute_ordering(P, Options(); alg = :colamd)) isa InvalidValueError end - @testset "without Metis" begin - A = laplacian2d(12, 12) - P = SDS.SymmetricPattern(SDS.CSR(A), "SPD"; view = 'F') - saved = SDS.ND_PROVIDER[] - try - SDS.ND_PROVIDER[] = nothing - @test !SDS.nd_available() - @test thrown(() -> SDS.compute_ordering(P, Options(reordering_alg = "algo4"))) isa NotSupportedError - ord = SDS.compute_ordering(P, Options()) - @test ord.alg_used === :amd - @test [c.alg for c in ord.stats.candidates] == [:amd] - @test !ord.stats.nd_available - finally - SDS.ND_PROVIDER[] = saved - end - @test SDS.nd_available() - end - @testset "empty and tiny patterns" begin for n in (0, 1) P = SDS.SymmetricPattern(SDS.CSR(sparse(1.0I, n, n)), "SPD") @@ -206,7 +191,7 @@ end P = SDS.SymmetricPattern(SDS.CSR(A), "S"; view = 'F') G = SparseMatrixCSC(P) amd = Vector{Int}(first(SDS.CliqueTrees.permutation(G; alg = SDS.CliqueTrees.AMD()))) - nd = Vector{Int}(first(SDS.CliqueTrees.permutation(G; alg = SDS.ND_PROVIDER[](10, -1)))) + nd = Vector{Int}(first(SDS.CliqueTrees.permutation(G; alg = SDS.nd_algorithm(10, -1, -1, -1)))) for (s, ref) in (("algo1", amd), ("algo2", amd), ("algo3", amd), ("algo4", nd)) ord = SDS.compute_ordering(P, Options(reordering_alg = s)) @test ord.perm == ref diff --git a/test/test_symbolic_pairs.jl b/test/test_symbolic_pairs.jl index ce7267a..bac6395 100644 --- a/test/test_symbolic_pairs.jl +++ b/test/test_symbolic_pairs.jl @@ -294,7 +294,7 @@ end opts = Options(pivot_pairs = "none", user_perm = opts.user_perm, reordering_alg = opts.reordering_alg))) end - # nested dissection (Metis is loaded by the test driver) and MMD also order the compressed graph + # nested dissection and MMD also order the compressed graph P, _, _ = pairs_ordering(A) pairs = SDS.pivot_pairs(P, C, sym_structure(T); view = 'L') for alg in (:nd, :mmd, :auto)