From df82c9e118be343ac017d2206361ccf87daaacee Mon Sep 17 00:00:00 2001 From: logan-nc <6198372+logan-nc@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:28:20 -0400 Subject: [PATCH 1/9] GPEC - NEW FEATURE - Record per-stage runtimes in gpec.h5 under Info/Runtimes main_from_inputs already timed every pipeline stage but only logged the durations. Persist them under Info/Runtimes/ (Float64 seconds, annotated with long_name/units) so an output file answers "where did the time go" on its own. Only the stages that ran are recorded; timings are informational, never a regression quantity. Co-Authored-By: Claude Fable 5 --- docs/development/hdf5-conventions.md | 2 +- src/GeneralizedPerturbedEquilibrium.jl | 66 ++++++++++++++++++++++---- test/runtests_fullruns.jl | 3 ++ 3 files changed, 62 insertions(+), 9 deletions(-) diff --git a/docs/development/hdf5-conventions.md b/docs/development/hdf5-conventions.md index b24610d2a..1048293e4 100644 --- a/docs/development/hdf5-conventions.md +++ b/docs/development/hdf5-conventions.md @@ -27,7 +27,7 @@ Top level (10 groups): | Group | Contents | |---|---| -| `Info/` | Run metadata: `git_version`, mode-number ranges (`mpert`, `mlow`, …, `mn_index`), `psilim`, `qlim` | +| `Info/` | Run metadata: `git_version`, mode-number ranges (`mpert`, `mlow`, …, `mn_index`), `psilim`, `qlim`, `Runtimes/` (per-stage wall-clock seconds) | | `Input/` | Rerun snapshot: `gpec_toml_raw`, `RawInputs/{Equilibrium, ForcingTerms, Coils/}` | | `Equilibrium/` | Scalars (β, q₀, q95, …) plus `Profiles/` (1-D: xs, 2piF, mu0p, dVdpsi, q) and `Geometry/` (2-D: rcoords, offset, nu, jac) | | `ForceFreeStates/` | `Solutions/ForwardIntegration/` (u-solutions), `Solutions/GalerkinIntegration/` (`Solution/`, `Match/`, `msing`), `EulerLagrangeMatrices/{Ideal,Kinetic}`, `FreeBoundaryStability/`, `EdgeScan/` | diff --git a/src/GeneralizedPerturbedEquilibrium.jl b/src/GeneralizedPerturbedEquilibrium.jl index 656316b75..cbd5528f2 100755 --- a/src/GeneralizedPerturbedEquilibrium.jl +++ b/src/GeneralizedPerturbedEquilibrium.jl @@ -67,6 +67,7 @@ const H5_RAW_EQUILIBRIUM = "Input/RawInputs/Equilibrium" const H5_RAW_FORCING = "Input/RawInputs/ForcingTerms" const H5_RAW_COILS = "Input/RawInputs/Coils" const H5_GIT_VERSION = "Info/git_version" +const H5_RUNTIMES = "Info/Runtimes" include("HDF5Schema.jl") include("Rerun.jl") @@ -181,6 +182,8 @@ function main_from_inputs( preloaded_coil_sets::Union{Nothing,Vector{ForcingTerms.CoilSet}}=nothing ) total_start = time() + # Per-stage wall-clock seconds, written to Info/Runtimes at the end of the run. + runtimes = Vector{Pair{String,Float64}}() # ---------------------------------------------------------------- # Equilibrium @@ -279,7 +282,9 @@ function main_from_inputs( @info "Two-pass psi grid: $(length(psi_nodes)) knots, $(length(mandatory)) rational surfaces pinned (n=$nstring)" end - @info "Equilibrium construction completed in $(@sprintf("%.3f", time() - equil_start)) s" + equil_dt = time() - equil_start + push!(runtimes, "equilibrium" => equil_dt) + @info "Equilibrium construction completed in $(@sprintf("%.3f", equil_dt)) s" # Early exit if user only requested equilibrium setup if equil.config.force_termination @@ -498,7 +503,9 @@ function main_from_inputs( if ctrl.gal_flag gal_start = time() gal_data = galerkin_solve(ctrl, equil, ffit, intr; vac_data=ctrl.vac_flag ? vac_data : nothing) - @info "Galerkin solve completed in $(@sprintf("%.3f", time() - gal_start)) s" + gal_dt = time() - gal_start + push!(runtimes, "galerkin" => gal_dt) + @info "Galerkin solve completed in $(@sprintf("%.3f", gal_dt)) s" end if ctrl.write_outputs_to_HDF5 @@ -518,7 +525,9 @@ function main_from_inputs( @info "Results written to $(ctrl.HDF5_filename)" end - @info "Force-Free States completed in $(@sprintf("%.3f", time() - ffs_start)) s" + ffs_dt = time() - ffs_start + push!(runtimes, "force_free_states" => ffs_dt) + @info "Force-Free States completed in $(@sprintf("%.3f", ffs_dt)) s" # SLAYER tearing-mode analysis stage. Needs only equil + intr, so it runs in # both the force_termination=true path and the full pipeline. `pe_file` is the @@ -536,7 +545,9 @@ function main_from_inputs( slayer_start = time() result = Runner.run_slayer(equil, intr, slayer_ctrl; dir_path=intr.dir_path) - @info "SLAYER completed in $(@sprintf("%.3f", time() - slayer_start)) s" + slayer_dt = time() - slayer_start + push!(runtimes, "slayer" => slayer_dt) + @info "SLAYER completed in $(@sprintf("%.3f", slayer_dt)) s" h5_filename = pe_file === nothing ? ctrl.HDF5_filename : pe_file h5_path = joinpath(intr.dir_path, h5_filename) # Append the Tearing/ group; create the file if no prior stage wrote @@ -557,7 +568,10 @@ function main_from_inputs( # Early exit if user only requested force-free states (SLAYER still runs). if ctrl.force_termination slayer_result = _run_slayer_stage(nothing) - @info "\n$_BANNER\n GPEC completed successfully in $(@sprintf("%.3f", time() - total_start)) s\n$_BANNER" + total_dt = time() - total_start + push!(runtimes, "total" => total_dt) + _write_runtimes!(joinpath(intr.dir_path, ctrl.HDF5_filename), runtimes) + @info "\n$_BANNER\n GPEC completed successfully in $(@sprintf("%.3f", total_dt)) s\n$_BANNER" return (ctrl=ctrl, equil=equil, intr=intr, ffit=ffit, odet=odet, vac_data=ctrl.vac_flag ? vac_data : nothing, slayer=slayer_result) @@ -642,7 +656,9 @@ function main_from_inputs( end end - @info "Perturbed Equilibrium completed in $(@sprintf("%.3f", time() - pe_start)) s" + pe_dt = time() - pe_start + push!(runtimes, "perturbed_equilibrium" => pe_dt) + @info "Perturbed Equilibrium completed in $(@sprintf("%.3f", pe_dt)) s" # ---------------------------------------------------------------- # KineticForces (Neoclassical Toroidal Viscosity) @@ -670,7 +686,9 @@ function main_from_inputs( end end - @info "KineticForces completed in $(@sprintf("%.3f", time() - kf_start)) s" + kf_dt = time() - kf_start + push!(runtimes, "kinetic_forces" => kf_dt) + @info "KineticForces completed in $(@sprintf("%.3f", kf_dt)) s" end # ---------------------------------------------------------------- @@ -688,7 +706,10 @@ function main_from_inputs( # ---------------------------------------------------------------- # Done # ---------------------------------------------------------------- - @info "\n$_BANNER\n GPEC completed successfully in $(@sprintf("%.3f", time() - total_start)) s\n$_BANNER" + total_dt = time() - total_start + push!(runtimes, "total" => total_dt) + _write_runtimes!(joinpath(intr.dir_path, ctrl.HDF5_filename), runtimes) + @info "\n$_BANNER\n GPEC completed successfully in $(@sprintf("%.3f", total_dt)) s\n$_BANNER" # TODO: Do not allow perturbed equilibrium calculations if zero crossings are found @@ -1017,6 +1038,35 @@ function _write_coil_snapshot!(h5_path::String, coil_sets::Vector{ForcingTerms.C return nothing end +const _RUNTIME_LONG_NAMES = Dict( + "equilibrium" => "Wall-clock time of the Equilibrium construction stage", + "force_free_states" => "Wall-clock time of the ForceFreeStates stability stage", + "galerkin" => "Wall-clock time of the Galerkin outer-region solve", + "slayer" => "Wall-clock time of the SLAYER tearing-mode stage", + "perturbed_equilibrium" => "Wall-clock time of the PerturbedEquilibrium stage", + "kinetic_forces" => "Wall-clock time of the KineticForces (NTV) stage", + "total" => "Wall-clock time of the full GPEC run") + +""" + _write_runtimes!(h5_path::String, runtimes) + +Write the per-stage wall-clock seconds collected during a run into `Info/Runtimes/` of an +existing gpec.h5 file. `runtimes` iterates `stage => seconds` pairs; only the stages that ran +are recorded. These timings are informational only — machine- and load-dependent, never a +regression quantity. +""" +function _write_runtimes!(h5_path::String, runtimes) + isfile(h5_path) || return nothing + h5open(h5_path, "r+") do out_h5 + haskey(out_h5, H5_RUNTIMES) && HDF5.delete_object(out_h5, H5_RUNTIMES) + for (stage, dt) in runtimes + out_h5["$H5_RUNTIMES/$stage"] = dt + end + Utilities.HDF5Annotations.annotate!(out_h5, ["$H5_RUNTIMES/$stage" => (; long_name=_RUNTIME_LONG_NAMES[stage], units="s") for (stage, _) in runtimes]) + end + return nothing +end + """ write_imas(dd, result) diff --git a/test/runtests_fullruns.jl b/test/runtests_fullruns.jl index ce6470385..cb0adc19d 100644 --- a/test/runtests_fullruns.jl +++ b/test/runtests_fullruns.jl @@ -28,6 +28,9 @@ using HDF5 et = read(h5["ForceFreeStates/FreeBoundaryStability/eigenmode_energies"]) @test isfinite(real(et[1])) @test real(et[1]) > 0 + # Per-stage wall-clock records (informational, not regression quantities). + @test haskey(h5, "Info/Runtimes/total") && read(h5["Info/Runtimes/total"]) > 0 + @test haskey(h5, "Info/Runtimes/force_free_states") end rm(joinpath(ex3, "gpec.h5"); force=true) true From e695e890e4a7564f1dd3e05d4e7f2b42365a7c64 Mon Sep 17 00:00:00 2001 From: logan-nc <6198372+logan-nc@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:29:45 -0400 Subject: [PATCH 2/9] BENCHMARKS - IMPROVEMENT - Report per-stage runtime deltas from Info/Runtimes Collect Info/Runtimes inside the warm-run loop (each run overwrites gpec.h5), average over the stages present in every run, and print a per-stage delta table alongside the existing totals. Refs predating the record yield an empty dict and the table is skipped silently, so old-vs-new comparisons still work. Co-Authored-By: Claude Fable 5 --- benchmarks/benchmark_git_branches.jl | 58 ++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/benchmarks/benchmark_git_branches.jl b/benchmarks/benchmark_git_branches.jl index c8e017bc2..63367fb0e 100755 --- a/benchmarks/benchmark_git_branches.jl +++ b/benchmarks/benchmark_git_branches.jl @@ -43,6 +43,7 @@ For each branch/commit, reports: - Eigenmode energy (et[1]) from gpec.h5 - Integration steps from gpec.h5 - Runtime (averaged over warm runs) +- Per-stage runtime breakdown from `Info/Runtimes` (averaged over warm runs, when both refs record it) - Git commit hash # Example Output @@ -67,6 +68,12 @@ Comparison: Eigenmode Δ: +0.0067 (+0.4%) Steps Δ: -775 (-85.1%) Runtime Δ: -2.5 s (-1.7% faster) + +Per-stage runtime (avg over warm runs): + Stage Branch 1 Branch 2 Δ s Δ % + equilibrium 2.10s 2.08s -0.02 -1.0% + force_free_states 100.40s 98.10s -2.30 -2.3% + total 145.30s 142.80s -2.50 -1.7% ``` """ @@ -156,6 +163,28 @@ function checkout_ref(branch, commit) end end +# Read the per-stage wall-clock seconds a run recorded under Info/Runtimes. +# Returns an empty Dict for refs that predate the feature, so those still benchmark. +function read_stage_runtimes(gpec_path) + stages = Dict{String,Float64}() + isfile(gpec_path) || return stages + h5open(gpec_path, "r") do h5 + haskey(h5, "Info/Runtimes") || return + group = h5["Info/Runtimes"] + for stage in keys(group) + stages[stage] = read(group[stage]) + end + end + return stages +end + +# Average per-stage runtimes over the warm runs, keeping only stages present in every run. +function average_stage_runtimes(stage_times) + isempty(stage_times) && return Dict{String,Float64}() + shared = intersect(map(keys, stage_times)...) + return Dict(stage => sum(s[stage] for s in stage_times) / length(stage_times) for stage in shared) +end + # Run the example benchmark. # Each run is a fresh Julia subprocess so that git branch switches take effect # (using statement only fires once per process; re-checkout without subprocess restart @@ -176,19 +205,22 @@ function run_example_benchmark(example_path, num_runs) println("\n[1/$(num_runs+1)] First run (JIT compilation)...") run(`julia --project=$project_root $tmpscript $abs_example_path`) - # Warm runs for timing + # Warm runs for timing. Each run overwrites gpec.h5, so the per-stage + # Info/Runtimes record must be collected inside the loop, not after it. + gpec_path = joinpath(abs_example_path, "gpec.h5") runtimes = Float64[] + stage_times = Vector{Dict{String,Float64}}() for i in 1:num_runs println("\n[$((i+1))/$(num_runs+1)] Warm run $i...") t_start = time() run(`julia --project=$project_root $tmpscript $abs_example_path`) runtime = time() - t_start push!(runtimes, runtime) + push!(stage_times, read_stage_runtimes(gpec_path)) println(" Runtime: $(round(runtime, digits=2)) s") end # Extract metrics from gpec.h5 - gpec_path = joinpath(abs_example_path, "gpec.h5") if !isfile(gpec_path) error("gpec.h5 not found at $gpec_path") end @@ -203,13 +235,31 @@ function run_example_benchmark(example_path, num_runs) return ( eigenvalue=real(et[1]), steps=nsteps, - runtime=avg_runtime + runtime=avg_runtime, + stages=average_stage_runtimes(stage_times) ) finally rm(tmpscript; force=true) end end +# Per-stage runtime breakdown (Info/Runtimes), averaged over the warm runs of each branch. +# Silently skipped when either side lacks the record, so comparisons against older refs work. +function print_stage_comparison(r1, r2) + shared = intersect(keys(r1.metrics.stages), keys(r2.metrics.stages)) + isempty(shared) && return nothing + ordered = sort(collect(shared); by=stage -> (stage == "total", stage)) + + println("\nPer-stage runtime (avg over warm runs):") + @printf(" %-22s %10s %10s %10s %9s\n", "Stage", "Branch 1", "Branch 2", "Δ s", "Δ %") + for stage in ordered + t1 = r1.metrics.stages[stage] + t2 = r2.metrics.stages[stage] + @printf(" %-22s %9.2fs %9.2fs %+9.2f %+8.1f%%\n", stage, t1, t2, t2 - t1, 100 * (t2 - t1) / t1) + end + return nothing +end + # Main benchmarking function function benchmark_branches(options) println("="^60) @@ -293,6 +343,8 @@ function benchmark_branches(options) speedup_desc = time_delta < 0 ? "faster" : "slower" @printf(" Runtime Δ: %+.2f s (%+.1f%% %s)\n", time_delta, abs(time_pct), speedup_desc) + print_stage_comparison(r1, r2) + println("\n" * "="^60) # Write to output file if requested From a496b731e1831f0adddb65e28876d13919de2896 Mon Sep 17 00:00:00 2001 From: logan-nc <6198372+logan-nc@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:36:26 -0400 Subject: [PATCH 3/9] DOCS - HANDOFF - Add machine-switch handoff document (delete before merge) Temporary scaffolding for continuing this branch on another machine: what the branch does, the merge resolution against upstream's inline-path convention, verification already done, the wrong-baseline regression pitfall, and the outstanding work. Carries its own removal instructions. Co-Authored-By: Claude Fable 5 --- HANDOFF_h5_runtime_records.md | 133 ++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 HANDOFF_h5_runtime_records.md diff --git a/HANDOFF_h5_runtime_records.md b/HANDOFF_h5_runtime_records.md new file mode 100644 index 000000000..c0e60777d --- /dev/null +++ b/HANDOFF_h5_runtime_records.md @@ -0,0 +1,133 @@ +# HANDOFF — `feature/h5-runtime-records` + +**Temporary working document. Delete it before this PR merges — see "Remove this file" at the bottom.** + +Written 2026-08-14 when the working session moved to another machine. Everything described here is +pushed to `origin/feature/h5-runtime-records`. + +## What this branch does + +`main_from_inputs` already timed every pipeline stage but only logged the durations. This branch +persists them to `gpec.h5` under `Info/Runtimes/` (Float64 seconds, annotated with +`long_name` + `units="s"`), and teaches the branch-benchmark tool to report per-stage deltas. + +Stages recorded, only when they actually run: `equilibrium`, `galerkin`, `force_free_states`, +`slayer`, `perturbed_equilibrium`, `kinetic_forces`, plus `total`. + +**Runtimes are informational only** — machine- and load-dependent. They are not regression +quantities and must never be added to the regression harness. + +Target: PR into `refactor/hdf5-metadata` (stack: #363 → #364 → this). + +## Commits + +| Commit | Contents | +|---|---| +| `df82c9e1` | `GPEC - NEW FEATURE` — `_write_runtimes!` + `_RUNTIME_LONG_NAMES`, six stage captures, both h5-bearing exit paths, doc schema row, two `runtests_fullruns.jl` assertions | +| `e695e890` | `BENCHMARKS - IMPROVEMENT` — `read_stage_runtimes` / `average_stage_runtimes` / `print_stage_comparison` in `benchmarks/benchmark_git_branches.jl` | +| `eb944f9b` | Merge of `origin/refactor/hdf5-metadata` (branch was ~60 commits behind) | + +### Design points worth knowing + +- The banner at both exit paths was rewritten to reuse `total_dt`, so the log line and the recorded + value cannot disagree. +- `_write_runtimes!` guards on `isfile` (no-op when `write_outputs_to_HDF5=false`) and deletes an + existing `Info/Runtimes` group before writing, so reruns into an existing file stay idempotent. +- Annotation goes through the existing `Utilities.HDF5Annotations.annotate!` — no new annotation + path was written. `test/runtests_h5_schema.jl` enforces this automatically: its metadata walker + fails if any `Info/Runtimes/*` dataset lacks `long_name`/`units`. +- The equilibrium-only early exit is deliberately untouched — no h5 exists at that point. +- In the benchmark tool the `Info/Runtimes` read happens **inside** the warm-run loop, because each + run overwrites `gpec.h5`. Refs predating the feature yield an empty dict and the per-stage table + is skipped silently, so old-vs-new comparisons still work. + +### Merge resolution note + +Upstream `refactor/hdf5-metadata` removed the shared `H5_*` path consts from +`src/GeneralizedPerturbedEquilibrium.jl` in favour of inline literal paths with cross-reference +comments. The merge conflicted there; it was resolved **upstream's way** — the `H5_RUNTIMES` const +was dropped and `_write_runtimes!` now uses the `"Info/Runtimes"` literal directly. The second +conflict was `galerkin_solve`'s changed signature (`wv=` replaced `vac_data=`); upstream's call was +kept with the timing capture layered on top. + +## Verification already done + +All of the following ran **before** the upstream merge unless noted: + +| Check | Result | +|---|---| +| `runtests_h5_schema.jl` | 28/28 pass — confirms `Info/Runtimes/*` carries required metadata | +| `runtests_fullruns.jl` | 19/19 pass (17 before; +2 new runtime assertions) | +| Manual DIII-D run | all five recorded values match their `completed in` log lines exactly; `units`/`long_name` present; only stages that ran appear | +| Benchmark dry-check | real read, missing-group fallback, missing-file fallback, shared-key averaging, `total`-last ordering, silent skip on empty/disjoint sides — all correct | +| Regression harness, `diiid_n1`, `0ece9c4f` vs working tree | every physical quantity agrees to 1e-9–1e-16 (0.00%); see caveat below | +| Package load after merge | clean | + +### Regression-harness caveat — read before rerunning + +The first harness run was done against `origin/refactor/hdf5-metadata` and reported **38 changed +quantities**. That was a wrong-baseline artifact: at the time this branch's parent (`0ece9c4f`) was +~60 commits behind that ref, so the report measured *upstream* physics changes (auto psi-grid Δ′ +convergence fix, on-demand solution derivatives, coil `psilim` fix) — not this work. Rerun against +the branch's actual merge-base, not the remote branch tip. + +Against the correct baseline, 14 quantities were still flagged `** CHANGED **` but every one at +**0.00%** (1e-9 to 1e-16 — floating-point noise; the harness flags any last-bit difference, and the +profile "checksum" quantities flag on any bit change at all). Two residuals were being investigated +when the session ended: + +- `ODE steps (total)` 1960 → 1968 (0.41%); `ODE steps (saved)` identical at 1327. +- Three profile checksums (Mercier `D_I`, resistive interchange `D_R`, ballooning Δ′) differ. + +The diff cannot affect numerics — it computes `time()` differences and appends to the h5 *after* all +computation — so the working hypothesis is run-to-run nondeterminism (threaded BLAS reassociation) +and/or environment drift between the harness worktree and the local environment. A determinism check +was running at handoff time: two `--force` runs of the *same* commit `0ece9c4f`, diffed against each +other. **Its result was never seen.** Re-run it to close this out: + +```bash +julia --project=regression-harness regression-harness/regress.jl --cases diiid_n1 --refs 0ece9c4f --force > a.log 2>&1 +julia --project=regression-harness regression-harness/regress.jl --cases diiid_n1 --refs 0ece9c4f --force > b.log 2>&1 +diff a.log b.log # differences here ⇒ the case is nondeterministic ⇒ the residuals above are noise +``` + +## Outstanding work + +1. **Re-run the full test suite on the merged tree.** Only the package load and (in flight at + handoff) `runtests_h5_schema.jl` were checked after the merge. `runtests_fullruns.jl` (~23 min) + has not been re-run post-merge: + ```bash + julia --project=. test/runtests.jl runtests_h5_schema.jl runtests_fullruns.jl + ``` +2. **Close out the determinism check** above, and post the corrected regression table on the PR. +3. **Optional:** a real two-branch benchmark run to see the per-stage table print end-to-end. Only a + dry-check against synthetic and real h5 files was done, by explicit choice — the full run costs + ~20–40 min of compute. +4. **Remove this file** (below) and get human review. + +## Environment note + +If a `julia` invocation hits manifest errors on the new machine: +`julia --project=. -e 'using Pkg; Pkg.resolve(); Pkg.instantiate()'`. +**Never** remove a package from `Project.toml` — the developer works across several machines and +environment drift is expected; fix the environment, not the manifest. + +## Remove this file + +This document is scaffolding for a machine switch and must not land in `refactor/hdf5-metadata`: + +```bash +git rm HANDOFF_h5_runtime_records.md +git commit -m "DOCS - CLEANUP - Remove the machine-switch handoff document" +git push +``` + +Do this once the outstanding work above is finished and before the PR is approved for merge. + +--- + +# ⚠️ **MERGE GATE** ⚠️ + +# **NO PULL REQUEST IS EVER MERGED INTO `develop` WITHOUT A THIRD-PARTY HUMAN REVIEWER'S APPROVAL.** + +# **THIS IS NON-NEGOTIABLE. NO EXCEPTIONS.** From e232b870de580134babe0014c1e19467f2fc5e70 Mon Sep 17 00:00:00 2001 From: logan-nc <6198372+logan-nc@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:49:41 -0400 Subject: [PATCH 4/9] DOCS - HANDOFF - Record post-merge verification and the closed regression question Both test files pass on the merged tree, and the regression residuals are resolved: the case is deterministic, and parent-vs-feature run through the same worktree path shows 48 unchanged / 0 changed. Documents the commit-vs-local environment trap that produced the earlier spurious diffs. Co-Authored-By: Claude Fable 5 --- HANDOFF_h5_runtime_records.md | 48 +++++++++++++++-------------------- 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/HANDOFF_h5_runtime_records.md b/HANDOFF_h5_runtime_records.md index c0e60777d..0a8422e50 100644 --- a/HANDOFF_h5_runtime_records.md +++ b/HANDOFF_h5_runtime_records.md @@ -56,11 +56,11 @@ All of the following ran **before** the upstream merge unless noted: | Check | Result | |---|---| -| `runtests_h5_schema.jl` | 28/28 pass — confirms `Info/Runtimes/*` carries required metadata | -| `runtests_fullruns.jl` | 19/19 pass (17 before; +2 new runtime assertions) | +| `runtests_h5_schema.jl` | pass, **re-run after the merge** (14/14 + 6/6 group-name rule) | +| `runtests_fullruns.jl` | 19/19 pass, **re-run after the merge** (17 before; +2 new runtime assertions) | | Manual DIII-D run | all five recorded values match their `completed in` log lines exactly; `units`/`long_name` present; only stages that ran appear | | Benchmark dry-check | real read, missing-group fallback, missing-file fallback, shared-key averaging, `total`-last ordering, silent skip on empty/disjoint sides — all correct | -| Regression harness, `diiid_n1`, `0ece9c4f` vs working tree | every physical quantity agrees to 1e-9–1e-16 (0.00%); see caveat below | +| Regression harness, `diiid_n1`, `0ece9c4f` vs `e695e890` | **48 unchanged, 0 changed** — zero movement | | Package load after merge | clean | ### Regression-harness caveat — read before rerunning @@ -71,39 +71,31 @@ quantities**. That was a wrong-baseline artifact: at the time this branch's pare convergence fix, on-demand solution derivatives, coil `psilim` fix) — not this work. Rerun against the branch's actual merge-base, not the remote branch tip. -Against the correct baseline, 14 quantities were still flagged `** CHANGED **` but every one at -**0.00%** (1e-9 to 1e-16 — floating-point noise; the harness flags any last-bit difference, and the -profile "checksum" quantities flag on any bit change at all). Two residuals were being investigated -when the session ended: +There is a second, subtler trap: **comparing a ref against `local` compares environments as well as +code.** Non-`local` refs run in a freshly instantiated harness worktree; `local` runs the working +tree with its own resolved environment. Run `0ece9c4f` vs `local` and 14 quantities come back +flagged `** CHANGED **` — all at 0.00% (1e-9 to 1e-16), plus `ODE steps (total)` 1960 → 1968 and +three profile checksums. That is environment drift, not code. -- `ODE steps (total)` 1960 → 1968 (0.41%); `ODE steps (saved)` identical at 1327. -- Three profile checksums (Mercier `D_I`, resistive interchange `D_R`, ballooning Δ′) differ. +This was chased to ground and is now **closed**: -The diff cannot affect numerics — it computes `time()` differences and appends to the h5 *after* all -computation — so the working hypothesis is run-to-run nondeterminism (threaded BLAS reassociation) -and/or environment drift between the harness worktree and the local environment. A determinism check -was running at handoff time: two `--force` runs of the *same* commit `0ece9c4f`, diffed against each -other. **Its result was never seen.** Re-run it to close this out: +- *Is the case nondeterministic?* No. Two `--force` runs of the same commit `0ece9c4f` produced + byte-identical values for all 49 quantities; only the wall-clock line differed. +- *Do these commits move any number?* No. Running the parent and the feature tip through the **same** + worktree path — `--refs 0ece9c4f,e695e890` — gives **48 unchanged, 0 changed**. -```bash -julia --project=regression-harness regression-harness/regress.jl --cases diiid_n1 --refs 0ece9c4f --force > a.log 2>&1 -julia --project=regression-harness regression-harness/regress.jl --cases diiid_n1 --refs 0ece9c4f --force > b.log 2>&1 -diff a.log b.log # differences here ⇒ the case is nondeterministic ⇒ the residuals above are noise -``` +So: compare worktree-to-worktree (two commit refs), not commit-vs-`local`, whenever the numbers +need to be trusted at last-bit precision. ## Outstanding work -1. **Re-run the full test suite on the merged tree.** Only the package load and (in flight at - handoff) `runtests_h5_schema.jl` were checked after the merge. `runtests_fullruns.jl` (~23 min) - has not been re-run post-merge: - ```bash - julia --project=. test/runtests.jl runtests_h5_schema.jl runtests_fullruns.jl - ``` -2. **Close out the determinism check** above, and post the corrected regression table on the PR. -3. **Optional:** a real two-branch benchmark run to see the per-stage table print end-to-end. Only a +1. **Optional:** a real two-branch benchmark run to see the per-stage table print end-to-end. Only a dry-check against synthetic and real h5 files was done, by explicit choice — the full run costs ~20–40 min of compute. -4. **Remove this file** (below) and get human review. +2. **Remove this file** (below) and get human review. + +Everything else is done: both test files pass on the merged tree and the regression comparison is +clean. ## Environment note From 31285e193d986dd6e36ac90fe549512870884a0c Mon Sep 17 00:00:00 2001 From: logan-nc <6198372+logan-nc@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:50:25 -0400 Subject: [PATCH 5/9] DOCS - HANDOFF - Point the harness caveat at the documented Manifest-pinning fix Co-Authored-By: Claude Fable 5 --- HANDOFF_h5_runtime_records.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/HANDOFF_h5_runtime_records.md b/HANDOFF_h5_runtime_records.md index 0a8422e50..e676532e5 100644 --- a/HANDOFF_h5_runtime_records.md +++ b/HANDOFF_h5_runtime_records.md @@ -87,6 +87,12 @@ This was chased to ground and is now **closed**: So: compare worktree-to-worktree (two commit refs), not commit-vs-`local`, whenever the numbers need to be trusted at last-bit precision. +This is a known class of artifact — see `docs/development/regression-harness.md`, "Making source +code the only variable": an unpinned `Manifest.toml` lets a worktree resolve different package +versions, and the adaptive ODE step controller amplifies machine-epsilon library differences into +apparent regressions. The harness merged in from upstream now pins the working tree's Manifest into +every worktree; the misleading run above was made with the pre-merge harness, which predates that. + ## Outstanding work 1. **Optional:** a real two-branch benchmark run to see the per-stage table print end-to-end. Only a From c170b9684705e6323520329bde609bf59897e265 Mon Sep 17 00:00:00 2001 From: logan-nc Date: Sat, 15 Aug 2026 02:32:41 -0400 Subject: [PATCH 6/9] GPEC - IMPROVEMENT - Gate runtime records on this run writing the output file The isfile guard alone let a run with write_outputs_to_HDF5=false stamp its timings into a stale gpec.h5 left over from an earlier run. Gate both call sites on the run having produced the file, keeping the SLAYER-created case. Move the stage long_names into HDF5Schema.jl as RUNTIME_H5_ANNOTATIONS, next to the other Info/ metadata; annotate! skips the stages that did not run. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Dh1NCejnd3fYMmcRKoQRcG --- src/GeneralizedPerturbedEquilibrium.jl | 28 +++++++++++++------------- src/HDF5Schema.jl | 12 +++++++++++ 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/GeneralizedPerturbedEquilibrium.jl b/src/GeneralizedPerturbedEquilibrium.jl index 1359a7137..19306c7cd 100755 --- a/src/GeneralizedPerturbedEquilibrium.jl +++ b/src/GeneralizedPerturbedEquilibrium.jl @@ -556,7 +556,10 @@ function main_from_inputs( slayer_result = _run_slayer_stage(nothing) total_dt = time() - total_start push!(runtimes, "total" => total_dt) - _write_runtimes!(joinpath(intr.dir_path, ctrl.HDF5_filename), runtimes) + # A returned SLAYER result means that stage created or appended to the file itself. + if ctrl.write_outputs_to_HDF5 || slayer_result !== nothing + _write_runtimes!(joinpath(intr.dir_path, ctrl.HDF5_filename), runtimes) + end @info "\n$_BANNER\n GPEC completed successfully in $(@sprintf("%.3f", total_dt)) s\n$_BANNER" return (ctrl=ctrl, equil=equil, intr=intr, ffit=ffit, odet=odet, free_energies=free_energies, @@ -694,7 +697,10 @@ function main_from_inputs( # ---------------------------------------------------------------- total_dt = time() - total_start push!(runtimes, "total" => total_dt) - _write_runtimes!(joinpath(intr.dir_path, ctrl.HDF5_filename), runtimes) + # A returned SLAYER result means that stage created or appended to the file itself. + if ctrl.write_outputs_to_HDF5 || slayer_result !== nothing + _write_runtimes!(joinpath(intr.dir_path, ctrl.HDF5_filename), runtimes) + end @info "\n$_BANNER\n GPEC completed successfully in $(@sprintf("%.3f", total_dt)) s\n$_BANNER" # TODO: Do not allow perturbed equilibrium calculations if zero crossings are found @@ -1030,22 +1036,16 @@ function _write_coil_snapshot!(h5_path::String, coil_sets::Vector{ForcingTerms.C return nothing end -const _RUNTIME_LONG_NAMES = Dict( - "equilibrium" => "Wall-clock time of the Equilibrium construction stage", - "force_free_states" => "Wall-clock time of the ForceFreeStates stability stage", - "galerkin" => "Wall-clock time of the Galerkin outer-region solve", - "slayer" => "Wall-clock time of the SLAYER tearing-mode stage", - "perturbed_equilibrium" => "Wall-clock time of the PerturbedEquilibrium stage", - "kinetic_forces" => "Wall-clock time of the KineticForces (NTV) stage", - "total" => "Wall-clock time of the full GPEC run") - """ _write_runtimes!(h5_path::String, runtimes) Write the per-stage wall-clock seconds collected during a run into `Info/Runtimes/` of an existing gpec.h5 file. `runtimes` iterates `stage => seconds` pairs; only the stages that ran -are recorded. These timings are informational only — machine- and load-dependent, never a -regression quantity. +are recorded. Metadata comes from `RUNTIME_H5_ANNOTATIONS`, which skips the absent stages. +These timings are informational only — machine- and load-dependent, never a regression quantity. + +Call it only when this run produced the file; the `isfile` guard alone would also stamp a +stale gpec.h5 left over from an earlier run. """ function _write_runtimes!(h5_path::String, runtimes) isfile(h5_path) || return nothing @@ -1054,7 +1054,7 @@ function _write_runtimes!(h5_path::String, runtimes) for (stage, dt) in runtimes out_h5["Info/Runtimes/$stage"] = dt end - Utilities.HDF5Annotations.annotate!(out_h5, ["Info/Runtimes/$stage" => (; long_name=_RUNTIME_LONG_NAMES[stage], units="s") for (stage, _) in runtimes]) + Utilities.HDF5Annotations.annotate!(out_h5, RUNTIME_H5_ANNOTATIONS) end return nothing end diff --git a/src/HDF5Schema.jl b/src/HDF5Schema.jl index 047b452fb..d704741ef 100644 --- a/src/HDF5Schema.jl +++ b/src/HDF5Schema.jl @@ -185,6 +185,18 @@ const MAIN_H5_ANNOTATIONS = [ "SurfaceGeometries/Wall/z" => (; long_name="Cartesian z of wall point cloud", units="m"), ] +# Per-stage wall-clock records written by `_write_runtimes!` after every stage has run. +# Stages that did not run leave their path absent and are skipped by `annotate!`. +const RUNTIME_H5_ANNOTATIONS = [ + "Info/Runtimes/equilibrium" => (; long_name="wall-clock time of the Equilibrium construction stage", units="s"), + "Info/Runtimes/galerkin" => (; long_name="wall-clock time of the Galerkin outer-region solve", units="s"), + "Info/Runtimes/force_free_states" => (; long_name="wall-clock time of the ForceFreeStates stability stage", units="s"), + "Info/Runtimes/slayer" => (; long_name="wall-clock time of the SLAYER tearing-mode stage", units="s"), + "Info/Runtimes/perturbed_equilibrium" => (; long_name="wall-clock time of the PerturbedEquilibrium stage", units="s"), + "Info/Runtimes/kinetic_forces" => (; long_name="wall-clock time of the KineticForces (NTV) stage", units="s"), + "Info/Runtimes/total" => (; long_name="wall-clock time of the full GPEC run", units="s"), +] + # Euler-Lagrange operator matrices: same wording per letter, Ideal/ and Kinetic/ variants. const _ELM_IDEAL_LETTERS = [ ("A", "Euler-Lagrange primitive coefficient matrix A"), From fe3127859e0b8bdc2d5d2c1e9c232a71ca222e31 Mon Sep 17 00:00:00 2001 From: logan-nc Date: Sat, 15 Aug 2026 02:32:46 -0400 Subject: [PATCH 7/9] DOCS - CLEANUP - Remove the machine-switch handoff document Temporary scaffolding for a machine switch; its contents are folded into the PR description. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Dh1NCejnd3fYMmcRKoQRcG --- HANDOFF_h5_runtime_records.md | 131 ---------------------------------- 1 file changed, 131 deletions(-) delete mode 100644 HANDOFF_h5_runtime_records.md diff --git a/HANDOFF_h5_runtime_records.md b/HANDOFF_h5_runtime_records.md deleted file mode 100644 index e676532e5..000000000 --- a/HANDOFF_h5_runtime_records.md +++ /dev/null @@ -1,131 +0,0 @@ -# HANDOFF — `feature/h5-runtime-records` - -**Temporary working document. Delete it before this PR merges — see "Remove this file" at the bottom.** - -Written 2026-08-14 when the working session moved to another machine. Everything described here is -pushed to `origin/feature/h5-runtime-records`. - -## What this branch does - -`main_from_inputs` already timed every pipeline stage but only logged the durations. This branch -persists them to `gpec.h5` under `Info/Runtimes/` (Float64 seconds, annotated with -`long_name` + `units="s"`), and teaches the branch-benchmark tool to report per-stage deltas. - -Stages recorded, only when they actually run: `equilibrium`, `galerkin`, `force_free_states`, -`slayer`, `perturbed_equilibrium`, `kinetic_forces`, plus `total`. - -**Runtimes are informational only** — machine- and load-dependent. They are not regression -quantities and must never be added to the regression harness. - -Target: PR into `refactor/hdf5-metadata` (stack: #363 → #364 → this). - -## Commits - -| Commit | Contents | -|---|---| -| `df82c9e1` | `GPEC - NEW FEATURE` — `_write_runtimes!` + `_RUNTIME_LONG_NAMES`, six stage captures, both h5-bearing exit paths, doc schema row, two `runtests_fullruns.jl` assertions | -| `e695e890` | `BENCHMARKS - IMPROVEMENT` — `read_stage_runtimes` / `average_stage_runtimes` / `print_stage_comparison` in `benchmarks/benchmark_git_branches.jl` | -| `eb944f9b` | Merge of `origin/refactor/hdf5-metadata` (branch was ~60 commits behind) | - -### Design points worth knowing - -- The banner at both exit paths was rewritten to reuse `total_dt`, so the log line and the recorded - value cannot disagree. -- `_write_runtimes!` guards on `isfile` (no-op when `write_outputs_to_HDF5=false`) and deletes an - existing `Info/Runtimes` group before writing, so reruns into an existing file stay idempotent. -- Annotation goes through the existing `Utilities.HDF5Annotations.annotate!` — no new annotation - path was written. `test/runtests_h5_schema.jl` enforces this automatically: its metadata walker - fails if any `Info/Runtimes/*` dataset lacks `long_name`/`units`. -- The equilibrium-only early exit is deliberately untouched — no h5 exists at that point. -- In the benchmark tool the `Info/Runtimes` read happens **inside** the warm-run loop, because each - run overwrites `gpec.h5`. Refs predating the feature yield an empty dict and the per-stage table - is skipped silently, so old-vs-new comparisons still work. - -### Merge resolution note - -Upstream `refactor/hdf5-metadata` removed the shared `H5_*` path consts from -`src/GeneralizedPerturbedEquilibrium.jl` in favour of inline literal paths with cross-reference -comments. The merge conflicted there; it was resolved **upstream's way** — the `H5_RUNTIMES` const -was dropped and `_write_runtimes!` now uses the `"Info/Runtimes"` literal directly. The second -conflict was `galerkin_solve`'s changed signature (`wv=` replaced `vac_data=`); upstream's call was -kept with the timing capture layered on top. - -## Verification already done - -All of the following ran **before** the upstream merge unless noted: - -| Check | Result | -|---|---| -| `runtests_h5_schema.jl` | pass, **re-run after the merge** (14/14 + 6/6 group-name rule) | -| `runtests_fullruns.jl` | 19/19 pass, **re-run after the merge** (17 before; +2 new runtime assertions) | -| Manual DIII-D run | all five recorded values match their `completed in` log lines exactly; `units`/`long_name` present; only stages that ran appear | -| Benchmark dry-check | real read, missing-group fallback, missing-file fallback, shared-key averaging, `total`-last ordering, silent skip on empty/disjoint sides — all correct | -| Regression harness, `diiid_n1`, `0ece9c4f` vs `e695e890` | **48 unchanged, 0 changed** — zero movement | -| Package load after merge | clean | - -### Regression-harness caveat — read before rerunning - -The first harness run was done against `origin/refactor/hdf5-metadata` and reported **38 changed -quantities**. That was a wrong-baseline artifact: at the time this branch's parent (`0ece9c4f`) was -~60 commits behind that ref, so the report measured *upstream* physics changes (auto psi-grid Δ′ -convergence fix, on-demand solution derivatives, coil `psilim` fix) — not this work. Rerun against -the branch's actual merge-base, not the remote branch tip. - -There is a second, subtler trap: **comparing a ref against `local` compares environments as well as -code.** Non-`local` refs run in a freshly instantiated harness worktree; `local` runs the working -tree with its own resolved environment. Run `0ece9c4f` vs `local` and 14 quantities come back -flagged `** CHANGED **` — all at 0.00% (1e-9 to 1e-16), plus `ODE steps (total)` 1960 → 1968 and -three profile checksums. That is environment drift, not code. - -This was chased to ground and is now **closed**: - -- *Is the case nondeterministic?* No. Two `--force` runs of the same commit `0ece9c4f` produced - byte-identical values for all 49 quantities; only the wall-clock line differed. -- *Do these commits move any number?* No. Running the parent and the feature tip through the **same** - worktree path — `--refs 0ece9c4f,e695e890` — gives **48 unchanged, 0 changed**. - -So: compare worktree-to-worktree (two commit refs), not commit-vs-`local`, whenever the numbers -need to be trusted at last-bit precision. - -This is a known class of artifact — see `docs/development/regression-harness.md`, "Making source -code the only variable": an unpinned `Manifest.toml` lets a worktree resolve different package -versions, and the adaptive ODE step controller amplifies machine-epsilon library differences into -apparent regressions. The harness merged in from upstream now pins the working tree's Manifest into -every worktree; the misleading run above was made with the pre-merge harness, which predates that. - -## Outstanding work - -1. **Optional:** a real two-branch benchmark run to see the per-stage table print end-to-end. Only a - dry-check against synthetic and real h5 files was done, by explicit choice — the full run costs - ~20–40 min of compute. -2. **Remove this file** (below) and get human review. - -Everything else is done: both test files pass on the merged tree and the regression comparison is -clean. - -## Environment note - -If a `julia` invocation hits manifest errors on the new machine: -`julia --project=. -e 'using Pkg; Pkg.resolve(); Pkg.instantiate()'`. -**Never** remove a package from `Project.toml` — the developer works across several machines and -environment drift is expected; fix the environment, not the manifest. - -## Remove this file - -This document is scaffolding for a machine switch and must not land in `refactor/hdf5-metadata`: - -```bash -git rm HANDOFF_h5_runtime_records.md -git commit -m "DOCS - CLEANUP - Remove the machine-switch handoff document" -git push -``` - -Do this once the outstanding work above is finished and before the PR is approved for merge. - ---- - -# ⚠️ **MERGE GATE** ⚠️ - -# **NO PULL REQUEST IS EVER MERGED INTO `develop` WITHOUT A THIRD-PARTY HUMAN REVIEWER'S APPROVAL.** - -# **THIS IS NON-NEGOTIABLE. NO EXCEPTIONS.** From 9bd0ef23cd9984278ad7817a8f168b1b0a58a0f3 Mon Sep 17 00:00:00 2001 From: logan-nc Date: Sat, 15 Aug 2026 09:13:41 -0400 Subject: [PATCH 8/9] TEST - BUGFIX - Exempt Info/Runtimes from the replay bit-for-bit comparison The replay guard compares every dataset in a source gpec.h5 against its rerun. Wall-clock seconds cannot repeat, so the new records broke it the same way Info/git_version would; skip the group alongside it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Dh1NCejnd3fYMmcRKoQRcG --- test/runtests_rerun_from_h5.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/runtests_rerun_from_h5.jl b/test/runtests_rerun_from_h5.jl index 2e364f2df..cb4420e6b 100644 --- a/test/runtests_rerun_from_h5.jl +++ b/test/runtests_rerun_from_h5.jl @@ -7,10 +7,11 @@ using TOML # Collect every leaf dataset path under an open HDF5 file, skipping the groups/paths that # legitimately differ between a source run and its replay (`Input/` is re-emitted with the -# rerun's own filename/TOML blob; `Info/git_version` reflects the running commit). +# rerun's own filename/TOML blob; `Info/git_version` reflects the running commit; +# `Info/Runtimes/` records wall-clock seconds, which never repeat). function _rerun_leaf_paths(h5) skip_toplevel = Set(["Input"]) - skip_paths = Set(["Info/git_version"]) + skip_paths = Set(["Info/git_version", "Info/Runtimes"]) paths = String[] function walk(node, prefix) for k in keys(node) From 40e550e3c9ab86306ffad10f2cd9ec366468b81a Mon Sep 17 00:00:00 2001 From: logan-nc Date: Sat, 15 Aug 2026 10:43:25 -0400 Subject: [PATCH 9/9] GPEC - IMPROVEMENT - Name the tearing runtime record and stamp the file the run wrote Four corrections to the Info/Runtimes records after the schema overhaul landed: - The stage token is `tearing`, not `slayer`. The `slayer/` group became `Tearing/` under the "name the physics, not the algorithm" rule, and every other token already mirrors its group. The solver is named in the long_name instead. - `perturbed_equilibrium` is recorded only when the section is present. It was pushed unconditionally, so a stability-only run stamped the stage at ~0 s for work that never happened. - `_write_runtimes!` now targets the file the run actually wrote. It hardcoded ctrl.HDF5_filename, but PerturbedEquilibrium, KineticForces and SLAYER each write their own filename, so timings were silently dropped whenever a stage other than ForceFreeStates produced the output. Each write site records its path in `written_h5` (last writer wins, matching where SLAYER appends) and both exit paths stamp that file, replacing the write_outputs_to_HDF5/slayer_result proxy with the fact it was approximating. - The benchmark per-stage table prints an em dash instead of Inf/NaN when the branch-1 baseline is zero. The galerkin and total long_names now state that the values nest rather than partition the run, so a reader does not sum them, and workflow.md's Info/ row mentions Runtimes/. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01VGmFuAw5JdYrAyBSXCVssR --- benchmarks/benchmark_git_branches.jl | 4 +++- docs/src/workflow.md | 2 +- src/GeneralizedPerturbedEquilibrium.jl | 25 +++++++++++++++++-------- src/HDF5Schema.jl | 7 ++++--- 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/benchmarks/benchmark_git_branches.jl b/benchmarks/benchmark_git_branches.jl index 63367fb0e..3daf54d5d 100755 --- a/benchmarks/benchmark_git_branches.jl +++ b/benchmarks/benchmark_git_branches.jl @@ -255,7 +255,9 @@ function print_stage_comparison(r1, r2) for stage in ordered t1 = r1.metrics.stages[stage] t2 = r2.metrics.stages[stage] - @printf(" %-22s %9.2fs %9.2fs %+9.2f %+8.1f%%\n", stage, t1, t2, t2 - t1, 100 * (t2 - t1) / t1) + # A stage that ran in no measurable time has no meaningful percentage baseline. + pct = t1 == 0 ? "—" : @sprintf("%+.1f%%", 100 * (t2 - t1) / t1) + @printf(" %-22s %9.2fs %9.2fs %+9.2f %9s\n", stage, t1, t2, t2 - t1, pct) end return nothing end diff --git a/docs/src/workflow.md b/docs/src/workflow.md index 228459ed3..a39cec8d8 100644 --- a/docs/src/workflow.md +++ b/docs/src/workflow.md @@ -174,7 +174,7 @@ All results are written to a single HDF5 file (default: `gpec.h5`). The top-leve | Group | Contents | |---|---| -| `Info/` | Run metadata: git version, mode-number ranges, ψ limit | +| `Info/` | Run metadata: git version, mode-number ranges, ψ limit, `Runtimes/` (per-stage wall-clock seconds) | | `Input/` | Self-contained rerun snapshot: merged TOML blob, raw equilibrium/forcing/coil inputs | | `Equilibrium/` | Equilibrium scalars (`beta_N`, `q_axis`, `q_95`, …), 1-D profiles (`Profiles/`), 2-D geometry (`Geometry/`) | | `ForceFreeStates/` | Stability solve: `Solutions/{ForwardIntegration,GalerkinIntegration}`, `EulerLagrangeMatrices/`, `FreeBoundaryStability/`, `EdgeScan/` | diff --git a/src/GeneralizedPerturbedEquilibrium.jl b/src/GeneralizedPerturbedEquilibrium.jl index b3e431315..17c3ab413 100755 --- a/src/GeneralizedPerturbedEquilibrium.jl +++ b/src/GeneralizedPerturbedEquilibrium.jl @@ -174,6 +174,9 @@ function main_from_inputs( total_start = time() # Per-stage wall-clock seconds, written to Info/Runtimes at the end of the run. runtimes = Vector{Pair{String,Float64}}() + # Output file this run actually wrote, last writer wins (matching where the SLAYER stage + # appends). `nothing` means no stage produced a file, so there is nothing to stamp. + written_h5 = nothing # ---------------------------------------------------------------- # Equilibrium @@ -508,6 +511,7 @@ function main_from_inputs( locstab=locstab, ballooning_boundary=ballooning_boundary ) + written_h5 = ctrl.HDF5_filename @info "Results written to $(ctrl.HDF5_filename)" end @@ -532,7 +536,7 @@ function main_from_inputs( result = Runner.run_slayer(equil, intr, slayer_ctrl; dir_path=intr.dir_path) slayer_dt = time() - slayer_start - push!(runtimes, "slayer" => slayer_dt) + push!(runtimes, "tearing" => slayer_dt) @info "SLAYER completed in $(@sprintf("%.3f", slayer_dt)) s" h5_filename = pe_file === nothing ? ctrl.HDF5_filename : pe_file h5_path = joinpath(intr.dir_path, h5_filename) @@ -541,6 +545,7 @@ function main_from_inputs( HDF5.h5open(h5_path, isfile(h5_path) ? "r+" : "w") do f Runner.write_slayer_hdf5!(f, result) end + written_h5 = h5_filename @info "SLAYER results written to $h5_filename" return result catch err @@ -556,9 +561,8 @@ function main_from_inputs( slayer_result = _run_slayer_stage(nothing) total_dt = time() - total_start push!(runtimes, "total" => total_dt) - # A returned SLAYER result means that stage created or appended to the file itself. - if ctrl.write_outputs_to_HDF5 || slayer_result !== nothing - _write_runtimes!(joinpath(intr.dir_path, ctrl.HDF5_filename), runtimes) + if written_h5 !== nothing + _write_runtimes!(joinpath(intr.dir_path, written_h5), runtimes) end @info "\n$_BANNER\n GPEC completed successfully in $(@sprintf("%.3f", total_dt)) s\n$_BANNER" return (ctrl=ctrl, equil=equil, intr=intr, ffit=ffit, odet=odet, @@ -635,6 +639,7 @@ function main_from_inputs( PerturbedEquilibrium.write_outputs_to_HDF5( pe_state, pe_intr, joinpath(intr.dir_path, output_file) ) + written_h5 = output_file @info "Results written to $output_file" end @@ -646,7 +651,11 @@ function main_from_inputs( end pe_dt = time() - pe_start - push!(runtimes, "perturbed_equilibrium" => pe_dt) + # Record only when the stage ran; an absent [PerturbedEquilibrium] section would otherwise + # stamp a ~0 s entry for work that never happened. + if "PerturbedEquilibrium" in keys(inputs) + push!(runtimes, "perturbed_equilibrium" => pe_dt) + end @info "Perturbed Equilibrium completed in $(@sprintf("%.3f", pe_dt)) s" # ---------------------------------------------------------------- @@ -672,6 +681,7 @@ function main_from_inputs( h5open(joinpath(intr.dir_path, kf_ctrl.HDF5_filename), "cw") do h5file KineticForces.write_to_hdf5!(h5file, kf_state; dVdpsi_spline=equil.profiles.dVdpsi_spline) end + written_h5 = kf_ctrl.HDF5_filename end end @@ -697,9 +707,8 @@ function main_from_inputs( # ---------------------------------------------------------------- total_dt = time() - total_start push!(runtimes, "total" => total_dt) - # A returned SLAYER result means that stage created or appended to the file itself. - if ctrl.write_outputs_to_HDF5 || slayer_result !== nothing - _write_runtimes!(joinpath(intr.dir_path, ctrl.HDF5_filename), runtimes) + if written_h5 !== nothing + _write_runtimes!(joinpath(intr.dir_path, written_h5), runtimes) end @info "\n$_BANNER\n GPEC completed successfully in $(@sprintf("%.3f", total_dt)) s\n$_BANNER" diff --git a/src/HDF5Schema.jl b/src/HDF5Schema.jl index ebb5f32e7..54b302cd7 100644 --- a/src/HDF5Schema.jl +++ b/src/HDF5Schema.jl @@ -245,12 +245,13 @@ const MAIN_H5_ANNOTATIONS = [ # Stages that did not run leave their path absent and are skipped by `annotate!`. const RUNTIME_H5_ANNOTATIONS = [ "Info/Runtimes/equilibrium" => (; long_name="wall-clock time of the Equilibrium construction stage", units="s"), - "Info/Runtimes/galerkin" => (; long_name="wall-clock time of the Galerkin outer-region solve", units="s"), + "Info/Runtimes/galerkin" => (; long_name="wall-clock time of the Galerkin outer-region solve (measured inside force_free_states, not additional to it)", units="s"), "Info/Runtimes/force_free_states" => (; long_name="wall-clock time of the ForceFreeStates stability stage", units="s"), - "Info/Runtimes/slayer" => (; long_name="wall-clock time of the SLAYER tearing-mode stage", units="s"), + "Info/Runtimes/tearing" => (; long_name="wall-clock time of the Tearing stage (SLAYER inner-layer solve)", units="s"), "Info/Runtimes/perturbed_equilibrium" => (; long_name="wall-clock time of the PerturbedEquilibrium stage", units="s"), "Info/Runtimes/kinetic_forces" => (; long_name="wall-clock time of the KineticForces (NTV) stage", units="s"), - "Info/Runtimes/total" => (; long_name="wall-clock time of the full GPEC run", units="s"), + "Info/Runtimes/total" => + (; long_name="wall-clock time of the full GPEC run (the per-stage values nest rather than partition it, so they do not sum to this)", units="s"), ] # Euler-Lagrange operator matrices: same wording per letter, Ideal/ and Kinetic/ variants.