From 7e7192e32537ac10fef61482f96b8df106a742d3 Mon Sep 17 00:00:00 2001 From: Michael Kraus Date: Thu, 20 Aug 2026 12:56:51 +0900 Subject: [PATCH 1/5] Import NeuralNetworkParameters explicitly for AbstractNeuralNetworks 0.7 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 0.7 moved the parameter container out to `NeuralNetworkParameters`, where the type is called `NetworkParameters`, and the alias left behind in `AbstractNeuralNetworks` is deliberately unexported so that every user of the name says where it came from. This package used the name in 31 places across 11 source files, plus its HDF5 extension and six test files, and re-exports it. Importing it explicitly next to the `using AbstractNeuralNetworks` leaves all of that untouched, and keeps the name exported from here, so nothing downstream of this package changes either. The HDF5 extension needs nothing: its `import AbstractNeuralNetworks: h5save, save, load` now resolves to upstream's generics, its recursive loader is the private `_gml_h5load` so it cannot clash with upstream's `h5load`, and its `h5save(::H5DataStore, ::StiefelManifold, …)` methods are more specific than upstream's `AbstractArray` method, so they still win. That is all still the type piracy D8 and the key-order guess D4 describe; Phase 3 is where that gets fixed. Co-Authored-By: Claude Opus 5 (1M context) --- Project.toml | 2 +- scripts/enzyme.jl | 39 ++++++ scripts/harmonic_oscillator_sympnet.jl | 58 ++++++++ scripts/sae_script2.jl | 181 +++++++++++++++++++++++++ scripts/zygote.jl | 20 +++ src/GeometricMachineLearning.jl | 5 + 6 files changed, 304 insertions(+), 1 deletion(-) create mode 100644 scripts/enzyme.jl create mode 100644 scripts/harmonic_oscillator_sympnet.jl create mode 100644 scripts/sae_script2.jl create mode 100644 scripts/zygote.jl diff --git a/Project.toml b/Project.toml index 203bef727..45d9f4089 100644 --- a/Project.toml +++ b/Project.toml @@ -30,7 +30,7 @@ HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" HDF5Ext = "HDF5" [compat] -AbstractNeuralNetworks = "0.6.4" +AbstractNeuralNetworks = "0.7" ChainRulesCore = "1" ChainRulesTestUtils = "1" Distances = "0.10" diff --git a/scripts/enzyme.jl b/scripts/enzyme.jl new file mode 100644 index 000000000..78612cd4a --- /dev/null +++ b/scripts/enzyme.jl @@ -0,0 +1,39 @@ +using BenchmarkTools +using Enzyme +using LinearAlgebra +using Zygote + + +loss(A,x) = norm(A*x) + +# function loss(A,x) +# y = zero(x) +# mul!(y,A,x) +# norm(y) +# end + +function test(n) + A = rand(n,n) + x = rand(n) + + l = a -> loss(a,x) + + dA = zero(A) + + println("\nn = $n") + + println("\nEnzyme (autodiff):") + @btime Enzyme.autodiff(Reverse, $l, Active, Duplicated($A, $dA)) + + println("\nEnzyme (gradient):") + @btime Enzyme.gradient(Reverse, $l, $A) + + println("\nZygote:") + @btime Zygote.gradient($l, $A)[1] + + println("") +end + +test(100) +test(1000) +test(10000) diff --git a/scripts/harmonic_oscillator_sympnet.jl b/scripts/harmonic_oscillator_sympnet.jl new file mode 100644 index 000000000..4f48ec5d6 --- /dev/null +++ b/scripts/harmonic_oscillator_sympnet.jl @@ -0,0 +1,58 @@ +using GeometricMachineLearning +using GeometricIntegrators: ImplicitMidpoint, integrate +import GeometricProblems.HarmonicOscillator as ho + +# the problem is the ODE of the harmonic oscillator +ho_problem = ho.hodeproblem(; tspan = 500) + +# integrate the system +solution = integrate(ho_problem, ImplicitMidpoint()) + +dl_raw = DataLoader(solution; suppress_info = true) + +# specify the data type and the backend +type = Float64 +backend = CPU() + +# we can then make a new instance of `DataLoader` with this backend and type. +dl = DataLoader(dl_raw, backend, type) + + +const upscaling_dimension = 2 +const nhidden = 1 +const activation = tanh +const n_layers = 4 # number of layers for the G-SympNet +const depth = 4 # number of layers in each linear block in the LA-SympNet + +# calling G-SympNet architecture +gsympnet = GSympNet(dl; upscaling_dimension = upscaling_dimension, + n_layers = n_layers, + activation = activation) + +# initialize the networks +g_nn = NeuralNetwork(gsympnet, backend, type) + +# set up optimizer; for this we first need to specify the optimization method +opt_method = AdamOptimizer(type) + +# we then call the optimizer struct which allocates the cache +g_opt = Optimizer(opt_method, g_nn) + +# determine the batch size (the number of samples in one batch) +const batch_size = 16 + +batch = Batch(batch_size) + +# number of training epochs +const nepochs = 100 + +# perform training (returns array that contains the total loss for each training step) +g_loss_array = g_opt(g_nn, dl, batch, nepochs; show_progress = false) + +ics = (q=dl.input.q[:, 1, 1], p=dl.input.p[:, 1, 1]) + +steps_to_plot = 1000 + +#predictions +g_trajectory = iterate(g_nn, ics; n_points = steps_to_plot) + diff --git a/scripts/sae_script2.jl b/scripts/sae_script2.jl new file mode 100644 index 000000000..902a8db61 --- /dev/null +++ b/scripts/sae_script2.jl @@ -0,0 +1,181 @@ +using GeometricIntegrators: integrate, ImplicitMidpoint +using GeometricMachineLearning +import Random # hide +import GeometricProblems.TodaLattice as tl +using JLD2 +using CairoMakie + +sae_dir = "animations" +mkpath(sae_dir) + +N = tl.Ñ # hide +Δx = 1. / (N - 1) # hide +Ω = -0.5 : Δx : 0.5 # hide +tl.μ + +# todo +#pr = tl.hodeproblem(; tspan = (0.0, 8.)) +pr = tl.hodeproblem(; tspan = (0.0, 800.)) +@time "FOM + Implicit Midpoint" sol = integrate(pr, ImplicitMidpoint()) + +dl_cpu = DataLoader(sol; autoencoder = true, suppress_info = true) + +const reduced_dim = 2 + +Random.seed!(123) # hide +sae_arch = SymplecticAutoencoder(dl_cpu.input_dim, reduced_dim; n_encoder_blocks = 4, + n_decoder_blocks = 4, + n_encoder_layers = 2, + n_decoder_layers = 2) + +const mtc = GeometricMachineLearning.map_to_cpu + +sae_trained_parameters = load("../docs/src/tutorials/sae_parameters.jld2")["sae_parameters"] +_nnp(ps::Tuple) = NeuralNetworkParameters{Tuple(Symbol("L$(i)") for i in 1:length(ps))}(ps) +sae_nn_cpu = NeuralNetwork(sae_arch, Chain(sae_arch), _nnp(sae_trained_parameters), CPU()) + +sae_rs = HRedSys(pr, encoder(sae_nn_cpu), decoder(sae_nn_cpu); integrator = ImplicitMidpoint()) + +# @time "FOM + Implicit Midpoint" sol_full = integrate_full_system(sae_rs) # hide +@time "SAE + Implicit Midpoint" sol_sae_reduced = integrate_reduced_system(sae_rs) # hide + + + + + +const T = Float32 +_T(qp::NamedTuple{(:q, :p)}) = (q = T.(qp.q), p = T.(qp.p)) + +dl_reduced = DataLoader(encoder(sae_nn_cpu)(_T(dl_cpu.input))) + +# lines(dl_reduced.input.q[1, :, 1], dl_reduced.input.p[1, :, 1]) + +# sympnet_arch = GSympNet(2; n_layers = 10) +# sympnet_nn = NeuralNetwork(sympnet_arch, T) +# o = Optimizer(AdamOptimizer(), sympnet_nn) +# o(sympnet_nn, dl_reduced, Batch(10), 500) + +morange = RGBf(255 / 256, 127 / 256, 14 / 256) +mred = RGBf(214 / 256, 39 / 256, 40 / 256) +mpurple = RGBf(148 / 256, 103 / 256, 189 / 256) +mblue = RGBf(31 / 256, 119 / 256, 180 / 256) +mgreen = RGBf(44 / 256, 160 / 256, 44 / 256) + +function plot_solution(time_step; theme = :light, framerate = 50) + textcolor = theme == :dark ? :white : :black + fig = Figure(size = (1000, 500), figure_padding = (5,50,5,10), fontsize = 24) + ax = Axis(fig[1, 1], backgroundcolor = :transparent, + bottomspinecolor = textcolor, + topspinecolor = textcolor, + leftspinecolor = textcolor, + rightspinecolor = textcolor, + xtickcolor = textcolor, + ytickcolor = textcolor, + xticklabelcolor = textcolor, + yticklabelcolor = textcolor, + xlabel=L"\omega", + ylabel=L"q", + xlabelcolor = textcolor, + ylabelcolor = textcolor) + lines!(ax, sol.s.q[time_step], label = rich("FOM + Implicit Midpoint"; color = textcolor), color = mblue) + lines!(ax, sae_rs.decoder((q = sol_sae_reduced.s.q[time_step], p = sol_sae_reduced.s.p[time_step])).q, label = rich("SAE + Implicit Midpoint"; color = textcolor), color = mgreen) + axislegend(ax; position = :rt) + xlims!(ax, 0, 200) + ylims!(ax, 0, 1) + fig +end + + +#### Transformer + +const seq_length = 4 +integrator_architecture = StandardTransformerIntegrator(reduced_dim; + transformer_dim = 20, + n_blocks = 3, + n_heads = 5, + L = 3, + upscaling_activation = tanh) + +nn_integrator_parameters = load("../docs/src/tutorials/integrator_parameters.jld2")["integrator_parameters"] # hide +integrator_nn = NeuralNetwork(integrator_architecture, Chain(integrator_architecture), _nnp(nn_integrator_parameters), CPU()) # hide + +# todo +#n_time_steps = 100 +n_time_steps = 10000 + +ics = (q = dl_reduced.input.q[:, 1:seq_length], p = dl_reduced.input.p[:, 1:seq_length]) +time_series = iterate(mtc(integrator_nn), ics; n_points = n_time_steps, prediction_window = seq_length) +function plot_solution2(time_step; theme = :light, framerate = 50) + textcolor = theme == :dark ? :white : :black + fig = Figure(size = (1000, 500), figure_padding = (5,50,5,10), fontsize = 24) + ax = Axis(fig[1, 1], backgroundcolor = :transparent, + bottomspinecolor = textcolor, + topspinecolor = textcolor, + leftspinecolor = textcolor, + rightspinecolor = textcolor, + xtickcolor = textcolor, + ytickcolor = textcolor, + xticklabelcolor = textcolor, + yticklabelcolor = textcolor, + xlabel=L"\omega", + ylabel=L"q", + xlabelcolor = textcolor, + ylabelcolor = textcolor) + lines!(ax, sol.s.q[time_step], label = rich("FOM + Implicit Midpoint"; color = textcolor), color = mblue) + # prediction = (q = time_series.q[:, end], p = time_series.p[:, end]) + prediction = (q = time_series.q[:, time_step], p = time_series.p[:, time_step]) + prediction_big = decoder(sae_nn_cpu)(prediction) + + lines!(ax, prediction_big.q; label = rich("SAE + Transformer"; color = textcolor), color = mpurple) + axislegend(ax; position = :rt) + xlims!(ax, 0, 200) + ylims!(ax, 0, 1) + fig +end + +# ics3 = (q = ics.q[:, 1], p = ics.p[:, 1]) +# +# time_series2 = iterate(sympnet_nn, ics3; n_points = n_time_steps) +# +# function plot_solution3(time_step; theme = :light, framerate = 50) +# textcolor = theme == :dark ? :white : :black +# fig = Figure(size = (1000, 500), figure_padding = (5,50,5,5), fontsize = 24) +# ax = Axis(fig[1, 1], backgroundcolor = :transparent, +# bottomspinecolor = textcolor, +# topspinecolor = textcolor, +# leftspinecolor = textcolor, +# rightspinecolor = textcolor, +# xtickcolor = textcolor, +# ytickcolor = textcolor, +# xticklabelcolor = textcolor, +# yticklabelcolor = textcolor, +# xlabel=L"\omega", +# ylabel=L"q", +# xlabelcolor = textcolor, +# ylabelcolor = textcolor) +# lines!(ax, sol.s.q[time_step], label = rich("FOM + Implicit Midpoint"; color = textcolor), color = mblue) +# time_series = iterate(sympnet_nn, ics3; n_points = time_step) +# # prediction = (q = time_series.q[:, end], p = time_series.p[:, end]) +# prediction = (q = time_series2.q[:, time_step], p = time_series2.p[:, time_step]) +# prediction_big = decoder(sae_nn_cpu)(prediction) +# +# lines!(ax, prediction_big.q; label = rich("SAE + SympNet"; color = textcolor), color = mpurple) +# axislegend(ax; position = :rt) +# xlims!(ax, 0, 200) +# fig +# end + +# todo +#time_steps = 1:5 # axes(time_series.q, 2) +time_steps = 1:500 # axes(time_series.q, 2) + +for time_step in time_steps + fig1 = plot_solution(time_step) + save(sae_dir * "/sae-midpoint-$(string(time_step, pad = 3)).pdf", fig1) + + fig2 = plot_solution2(time_step) + save(sae_dir * "/sae-transformer-$(string(time_step, pad = 3)).pdf", fig2) + +# fig3 = plot_solution3(time_step) +# save(sae_dir * "/sae-sympnet-$(string(time_step, pad = 3)).pdf", fig3) +end diff --git a/scripts/zygote.jl b/scripts/zygote.jl new file mode 100644 index 000000000..cde81e962 --- /dev/null +++ b/scripts/zygote.jl @@ -0,0 +1,20 @@ +using Zygote, Printf, LinearAlgebra + +const number_data_points = 1000 + +const data_input = [[i] for i in 1:number_data_points] + +function_to_be_differentiated(input, A) = norm(A*input) + +function gradient_eval(data, num, A = rand(100000,1)) + input = data[num] + @printf "First one: " + @time Zygote.gradient(A -> function_to_be_differentiated(input, A), A)[1] + @printf "Second one:" + @time Zygote.gradient(A -> function_to_be_differentiated(data[num], A), A)[1] + @printf "\n" +end + +for i in 1:5 + gradient_eval(data_input, Int(ceil(rand()*number_data_points))) +end diff --git a/src/GeometricMachineLearning.jl b/src/GeometricMachineLearning.jl index ae185a293..4268ee33a 100644 --- a/src/GeometricMachineLearning.jl +++ b/src/GeometricMachineLearning.jl @@ -1,6 +1,11 @@ module GeometricMachineLearning using AbstractNeuralNetworks +# `AbstractNeuralNetworks` 0.7 no longer exports `NeuralNetworkParameters`: the parameter container +# moved out to the package of that name, where the type is called `NetworkParameters`, and the alias +# left behind is deliberately unexported so that every user of it says where it came from. This +# package uses the name in 31 places and re-exports it below, so it is imported explicitly here. +import AbstractNeuralNetworks: NeuralNetworkParameters using ChainRulesCore # `sqeuclidean` is the default distance of every `TrainingMethod` in `src/training_method/`. using Distances From 451100cbdea12cd540a90614f04d1ea580fe2bcd Mon Sep 17 00:00:00 2001 From: Michael Kraus Date: Thu, 20 Aug 2026 13:25:35 +0900 Subject: [PATCH 2/5] Drop four scratch scripts staged by accident They were untracked files in the working tree, swept in by a `git add -A` in the previous commit. They are not part of this change; the diff of this branch against main is now just the import and the compat bound. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/enzyme.jl | 39 ------ scripts/harmonic_oscillator_sympnet.jl | 58 -------- scripts/sae_script2.jl | 181 ------------------------- scripts/zygote.jl | 20 --- 4 files changed, 298 deletions(-) delete mode 100644 scripts/enzyme.jl delete mode 100644 scripts/harmonic_oscillator_sympnet.jl delete mode 100644 scripts/sae_script2.jl delete mode 100644 scripts/zygote.jl diff --git a/scripts/enzyme.jl b/scripts/enzyme.jl deleted file mode 100644 index 78612cd4a..000000000 --- a/scripts/enzyme.jl +++ /dev/null @@ -1,39 +0,0 @@ -using BenchmarkTools -using Enzyme -using LinearAlgebra -using Zygote - - -loss(A,x) = norm(A*x) - -# function loss(A,x) -# y = zero(x) -# mul!(y,A,x) -# norm(y) -# end - -function test(n) - A = rand(n,n) - x = rand(n) - - l = a -> loss(a,x) - - dA = zero(A) - - println("\nn = $n") - - println("\nEnzyme (autodiff):") - @btime Enzyme.autodiff(Reverse, $l, Active, Duplicated($A, $dA)) - - println("\nEnzyme (gradient):") - @btime Enzyme.gradient(Reverse, $l, $A) - - println("\nZygote:") - @btime Zygote.gradient($l, $A)[1] - - println("") -end - -test(100) -test(1000) -test(10000) diff --git a/scripts/harmonic_oscillator_sympnet.jl b/scripts/harmonic_oscillator_sympnet.jl deleted file mode 100644 index 4f48ec5d6..000000000 --- a/scripts/harmonic_oscillator_sympnet.jl +++ /dev/null @@ -1,58 +0,0 @@ -using GeometricMachineLearning -using GeometricIntegrators: ImplicitMidpoint, integrate -import GeometricProblems.HarmonicOscillator as ho - -# the problem is the ODE of the harmonic oscillator -ho_problem = ho.hodeproblem(; tspan = 500) - -# integrate the system -solution = integrate(ho_problem, ImplicitMidpoint()) - -dl_raw = DataLoader(solution; suppress_info = true) - -# specify the data type and the backend -type = Float64 -backend = CPU() - -# we can then make a new instance of `DataLoader` with this backend and type. -dl = DataLoader(dl_raw, backend, type) - - -const upscaling_dimension = 2 -const nhidden = 1 -const activation = tanh -const n_layers = 4 # number of layers for the G-SympNet -const depth = 4 # number of layers in each linear block in the LA-SympNet - -# calling G-SympNet architecture -gsympnet = GSympNet(dl; upscaling_dimension = upscaling_dimension, - n_layers = n_layers, - activation = activation) - -# initialize the networks -g_nn = NeuralNetwork(gsympnet, backend, type) - -# set up optimizer; for this we first need to specify the optimization method -opt_method = AdamOptimizer(type) - -# we then call the optimizer struct which allocates the cache -g_opt = Optimizer(opt_method, g_nn) - -# determine the batch size (the number of samples in one batch) -const batch_size = 16 - -batch = Batch(batch_size) - -# number of training epochs -const nepochs = 100 - -# perform training (returns array that contains the total loss for each training step) -g_loss_array = g_opt(g_nn, dl, batch, nepochs; show_progress = false) - -ics = (q=dl.input.q[:, 1, 1], p=dl.input.p[:, 1, 1]) - -steps_to_plot = 1000 - -#predictions -g_trajectory = iterate(g_nn, ics; n_points = steps_to_plot) - diff --git a/scripts/sae_script2.jl b/scripts/sae_script2.jl deleted file mode 100644 index 902a8db61..000000000 --- a/scripts/sae_script2.jl +++ /dev/null @@ -1,181 +0,0 @@ -using GeometricIntegrators: integrate, ImplicitMidpoint -using GeometricMachineLearning -import Random # hide -import GeometricProblems.TodaLattice as tl -using JLD2 -using CairoMakie - -sae_dir = "animations" -mkpath(sae_dir) - -N = tl.Ñ # hide -Δx = 1. / (N - 1) # hide -Ω = -0.5 : Δx : 0.5 # hide -tl.μ - -# todo -#pr = tl.hodeproblem(; tspan = (0.0, 8.)) -pr = tl.hodeproblem(; tspan = (0.0, 800.)) -@time "FOM + Implicit Midpoint" sol = integrate(pr, ImplicitMidpoint()) - -dl_cpu = DataLoader(sol; autoencoder = true, suppress_info = true) - -const reduced_dim = 2 - -Random.seed!(123) # hide -sae_arch = SymplecticAutoencoder(dl_cpu.input_dim, reduced_dim; n_encoder_blocks = 4, - n_decoder_blocks = 4, - n_encoder_layers = 2, - n_decoder_layers = 2) - -const mtc = GeometricMachineLearning.map_to_cpu - -sae_trained_parameters = load("../docs/src/tutorials/sae_parameters.jld2")["sae_parameters"] -_nnp(ps::Tuple) = NeuralNetworkParameters{Tuple(Symbol("L$(i)") for i in 1:length(ps))}(ps) -sae_nn_cpu = NeuralNetwork(sae_arch, Chain(sae_arch), _nnp(sae_trained_parameters), CPU()) - -sae_rs = HRedSys(pr, encoder(sae_nn_cpu), decoder(sae_nn_cpu); integrator = ImplicitMidpoint()) - -# @time "FOM + Implicit Midpoint" sol_full = integrate_full_system(sae_rs) # hide -@time "SAE + Implicit Midpoint" sol_sae_reduced = integrate_reduced_system(sae_rs) # hide - - - - - -const T = Float32 -_T(qp::NamedTuple{(:q, :p)}) = (q = T.(qp.q), p = T.(qp.p)) - -dl_reduced = DataLoader(encoder(sae_nn_cpu)(_T(dl_cpu.input))) - -# lines(dl_reduced.input.q[1, :, 1], dl_reduced.input.p[1, :, 1]) - -# sympnet_arch = GSympNet(2; n_layers = 10) -# sympnet_nn = NeuralNetwork(sympnet_arch, T) -# o = Optimizer(AdamOptimizer(), sympnet_nn) -# o(sympnet_nn, dl_reduced, Batch(10), 500) - -morange = RGBf(255 / 256, 127 / 256, 14 / 256) -mred = RGBf(214 / 256, 39 / 256, 40 / 256) -mpurple = RGBf(148 / 256, 103 / 256, 189 / 256) -mblue = RGBf(31 / 256, 119 / 256, 180 / 256) -mgreen = RGBf(44 / 256, 160 / 256, 44 / 256) - -function plot_solution(time_step; theme = :light, framerate = 50) - textcolor = theme == :dark ? :white : :black - fig = Figure(size = (1000, 500), figure_padding = (5,50,5,10), fontsize = 24) - ax = Axis(fig[1, 1], backgroundcolor = :transparent, - bottomspinecolor = textcolor, - topspinecolor = textcolor, - leftspinecolor = textcolor, - rightspinecolor = textcolor, - xtickcolor = textcolor, - ytickcolor = textcolor, - xticklabelcolor = textcolor, - yticklabelcolor = textcolor, - xlabel=L"\omega", - ylabel=L"q", - xlabelcolor = textcolor, - ylabelcolor = textcolor) - lines!(ax, sol.s.q[time_step], label = rich("FOM + Implicit Midpoint"; color = textcolor), color = mblue) - lines!(ax, sae_rs.decoder((q = sol_sae_reduced.s.q[time_step], p = sol_sae_reduced.s.p[time_step])).q, label = rich("SAE + Implicit Midpoint"; color = textcolor), color = mgreen) - axislegend(ax; position = :rt) - xlims!(ax, 0, 200) - ylims!(ax, 0, 1) - fig -end - - -#### Transformer - -const seq_length = 4 -integrator_architecture = StandardTransformerIntegrator(reduced_dim; - transformer_dim = 20, - n_blocks = 3, - n_heads = 5, - L = 3, - upscaling_activation = tanh) - -nn_integrator_parameters = load("../docs/src/tutorials/integrator_parameters.jld2")["integrator_parameters"] # hide -integrator_nn = NeuralNetwork(integrator_architecture, Chain(integrator_architecture), _nnp(nn_integrator_parameters), CPU()) # hide - -# todo -#n_time_steps = 100 -n_time_steps = 10000 - -ics = (q = dl_reduced.input.q[:, 1:seq_length], p = dl_reduced.input.p[:, 1:seq_length]) -time_series = iterate(mtc(integrator_nn), ics; n_points = n_time_steps, prediction_window = seq_length) -function plot_solution2(time_step; theme = :light, framerate = 50) - textcolor = theme == :dark ? :white : :black - fig = Figure(size = (1000, 500), figure_padding = (5,50,5,10), fontsize = 24) - ax = Axis(fig[1, 1], backgroundcolor = :transparent, - bottomspinecolor = textcolor, - topspinecolor = textcolor, - leftspinecolor = textcolor, - rightspinecolor = textcolor, - xtickcolor = textcolor, - ytickcolor = textcolor, - xticklabelcolor = textcolor, - yticklabelcolor = textcolor, - xlabel=L"\omega", - ylabel=L"q", - xlabelcolor = textcolor, - ylabelcolor = textcolor) - lines!(ax, sol.s.q[time_step], label = rich("FOM + Implicit Midpoint"; color = textcolor), color = mblue) - # prediction = (q = time_series.q[:, end], p = time_series.p[:, end]) - prediction = (q = time_series.q[:, time_step], p = time_series.p[:, time_step]) - prediction_big = decoder(sae_nn_cpu)(prediction) - - lines!(ax, prediction_big.q; label = rich("SAE + Transformer"; color = textcolor), color = mpurple) - axislegend(ax; position = :rt) - xlims!(ax, 0, 200) - ylims!(ax, 0, 1) - fig -end - -# ics3 = (q = ics.q[:, 1], p = ics.p[:, 1]) -# -# time_series2 = iterate(sympnet_nn, ics3; n_points = n_time_steps) -# -# function plot_solution3(time_step; theme = :light, framerate = 50) -# textcolor = theme == :dark ? :white : :black -# fig = Figure(size = (1000, 500), figure_padding = (5,50,5,5), fontsize = 24) -# ax = Axis(fig[1, 1], backgroundcolor = :transparent, -# bottomspinecolor = textcolor, -# topspinecolor = textcolor, -# leftspinecolor = textcolor, -# rightspinecolor = textcolor, -# xtickcolor = textcolor, -# ytickcolor = textcolor, -# xticklabelcolor = textcolor, -# yticklabelcolor = textcolor, -# xlabel=L"\omega", -# ylabel=L"q", -# xlabelcolor = textcolor, -# ylabelcolor = textcolor) -# lines!(ax, sol.s.q[time_step], label = rich("FOM + Implicit Midpoint"; color = textcolor), color = mblue) -# time_series = iterate(sympnet_nn, ics3; n_points = time_step) -# # prediction = (q = time_series.q[:, end], p = time_series.p[:, end]) -# prediction = (q = time_series2.q[:, time_step], p = time_series2.p[:, time_step]) -# prediction_big = decoder(sae_nn_cpu)(prediction) -# -# lines!(ax, prediction_big.q; label = rich("SAE + SympNet"; color = textcolor), color = mpurple) -# axislegend(ax; position = :rt) -# xlims!(ax, 0, 200) -# fig -# end - -# todo -#time_steps = 1:5 # axes(time_series.q, 2) -time_steps = 1:500 # axes(time_series.q, 2) - -for time_step in time_steps - fig1 = plot_solution(time_step) - save(sae_dir * "/sae-midpoint-$(string(time_step, pad = 3)).pdf", fig1) - - fig2 = plot_solution2(time_step) - save(sae_dir * "/sae-transformer-$(string(time_step, pad = 3)).pdf", fig2) - -# fig3 = plot_solution3(time_step) -# save(sae_dir * "/sae-sympnet-$(string(time_step, pad = 3)).pdf", fig3) -end diff --git a/scripts/zygote.jl b/scripts/zygote.jl deleted file mode 100644 index cde81e962..000000000 --- a/scripts/zygote.jl +++ /dev/null @@ -1,20 +0,0 @@ -using Zygote, Printf, LinearAlgebra - -const number_data_points = 1000 - -const data_input = [[i] for i in 1:number_data_points] - -function_to_be_differentiated(input, A) = norm(A*input) - -function gradient_eval(data, num, A = rand(100000,1)) - input = data[num] - @printf "First one: " - @time Zygote.gradient(A -> function_to_be_differentiated(input, A), A)[1] - @printf "Second one:" - @time Zygote.gradient(A -> function_to_be_differentiated(data[num], A), A)[1] - @printf "\n" -end - -for i in 1:5 - gradient_eval(data_input, Int(ceil(rand()*number_data_points))) -end From 1d8e1b1935a6e1662f1bfa56f6d8b96fd846f06f Mon Sep 17 00:00:00 2001 From: Michael Kraus Date: Sat, 22 Aug 2026 22:41:29 +0900 Subject: [PATCH 3/5] Take the parameter container from its own package, and the traversal with it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `AbstractNeuralNetworks` 0.7 moved the container out to `NeuralNetworkParameters` and removed the old name entirely rather than aliasing it, so that one type has one name across the ecosystem. This package follows, at all 31 call sites and in its export list — `export NetworkParameters` where `export NeuralNetworkParameters` was. Same type object, so `::Type{}` dispatch, `<:` bounds and `{keys}(vals)` construction are unaffected; only the spelling changes. That is still a change to this package's own exports, hence 0.6.0 and a CHANGELOG entry. The HDF5 extension loses 73 of its 187 lines. Five `h5save` methods tagging a `gml_type` attribute, the `_gml_h5load` reader and the `_natural_sort_keys` key-order heuristic are gone, because each job belongs to a package that owns the pieces: * `NeuralNetworkParameters` walks the parameter set and writes it, recording each group's key order in a `keys` attribute. `_natural_sort_keys` was standing in for that and it *guessed* — sorting on a trailing integer when every name in the group had one, and falling back to lexicographic order otherwise, so a group whose names do not end in a digit came back in whatever order sorting gave. * `GeometricOptimizers` says where each structured matrix keeps its numbers, through `freeparameters`/`rebuild`, and registers the types so a file loads with no prototype. `StiefelManifold` and `SymmetricMatrix` are its types, not this package's, so the methods here were type piracy twice over — on `h5save` and on the type. The half that stays is the half that genuinely dispatches on `NeuralNetwork`: `save(h5, nn)` now hands the whole `NetworkParameters` to upstream rather than unwrapping it first, and `load(NeuralNetwork, h5, arch)` gained a prototype form that rebuilds against a parameter set of the right shape and skips the registry. Existing files still load. `test/hdf5_support.jl` writes one in the old layout by hand and reads it back, so the deletion cannot quietly make them unreadable. The five `changebackend` methods for the same types stay for now. They are the same ownership smell, but `changebackend` is `AbstractNeuralNetworks`' and the fix is a `GeometricOptimizers` extension on it — a separate change with its own release chain. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 44 +++++ Project.toml | 6 +- .../tutorials/adjusting_the_loss_function.md | 4 +- docs/src/tutorials/grassmann_layer.md | 2 +- ext/HDF5Ext.jl | 169 +++++------------- src/GeometricMachineLearning.jl | 12 +- src/architectures/autoencoder.jl | 4 +- .../neural_network_integrator.jl | 2 +- src/data_loader/data_loader.jl | 2 +- src/data_loader/optimize.jl | 4 +- src/loss/hnn_loss.jl | 4 +- src/loss/losses.jl | 14 +- src/map_to_cpu.jl | 4 +- src/optimizers/optimizer.jl | 12 +- src/pullbacks/zygote_pullback.jl | 12 +- src/utils.jl | 2 +- test/docstrings/layers_and_architectures.jl | 2 +- test/hamiltonian_neural_network_tests.jl | 2 +- test/hdf5_support.jl | 61 ++++++- .../optimizer_convergence_tests/psd_optim.jl | 2 +- .../optimizer_convergence_tests/svd_optim.jl | 2 +- .../multi_head_attention_stiefel_setup.jl | 2 +- 22 files changed, 200 insertions(+), 168 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ddad2dbcf..d1e9622ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,43 @@ breaking release). of the module already provides, so `GeometricMachineLearning.IdentityActivation` still resolves — the line was redundant, not load-bearing. +### Removed + +- **`NeuralNetworkParameters` is no longer exported; the name is `NetworkParameters`.** The parameter + container moved out of `AbstractNeuralNetworks` into + [NeuralNetworkParameters.jl](https://github.com/JuliaGNI/NeuralNetworkParameters.jl) in + `AbstractNeuralNetworks` 0.7, which removed the old name outright rather than leaving an alias, so + that one type has one name across the ecosystem. This package follows, at all 31 call sites and in + its export list. It is the same type object, so `::Type{}` dispatch, `<:` bounds and + `NetworkParameters{keys}(vals)` construction are unaffected; only the spelling changes. + + ```julia + # before + using GeometricMachineLearning # brought NeuralNetworkParameters into scope + # after + using GeometricMachineLearning # brings NetworkParameters into scope + ``` + +- **The HDF5 extension no longer carries its own traversal.** Five `h5save` methods tagging a + `gml_type` attribute, the `_gml_h5load` reader and the `_natural_sort_keys` key-order heuristic are + gone — 73 of the extension's 187 lines. Each job now sits with the package that owns the pieces: + + - `NeuralNetworkParameters` walks the parameter set and writes it, recording each group's key order + in a `keys` attribute. `_natural_sort_keys` was standing in for that, and it *guessed*: it sorted + on a trailing integer when every name in the group had one and fell back to lexicographic order + otherwise, so a group whose names do not end in a digit came back in whatever order sorting gave. + - `GeometricOptimizers` says where each structured matrix keeps its numbers, through + `freeparameters`/`rebuild`, and registers the types so a file loads with no prototype. + `StiefelManifold` and `SymmetricMatrix` are its types, not this package's, so the methods here + were type piracy twice over — on `h5save` and on the type. + + Existing files still load. `NeuralNetworkParameters` recognises the `gml_type` tag and rebuilds + through the same registry, and `test/hdf5_support.jl` now writes a file in the old layout by hand + and reads it back, so the deletion cannot quietly make old files unreadable. + + `load(NeuralNetwork, h5, arch)` additionally accepts a prototype parameter set — + `load(NeuralNetwork, h5, arch, prototype)` — which rebuilds against it and skips the registry. + ### Fixed - **Zygote 0.7 silently zeroed every gradient that flows through `assign_q_and_p`.** Its `rrule` @@ -104,6 +141,13 @@ breaking release). ### Dependencies +- **`NeuralNetworkParameters = "0.1"`** added, and **`AbstractNeuralNetworks = "0.7"`** (was + `"0.6.4"`). The parameter container is defined in the former as of the latter; see *Removed* above. + +- **`GeometricOptimizers = "0.4.1"`** (was `"0.4"`). 0.4.1 is the release that carries the + `NeuralNetworkParameters` leaf protocol for the manifolds, storage matrices and horizontal lifts, + which is what lets this package's HDF5 extension drop its own copy of the traversal. + - **`Zygote = "0.7"`** (was `"0.6"`). 0.7 replaced the eager unthunking in `wrap_chainrules_output` with `unthunk_tangent` at the `gradient`/`pullback` boundaries, which is what let thunks reach GML's `rrule`s and surfaced everything under *Fixed* above. Implicit parameters are deprecated in diff --git a/Project.toml b/Project.toml index 45d9f4089..0c22b0a4f 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "GeometricMachineLearning" uuid = "194d25b2-d3f5-49f0-af24-c124f4aa80cc" -version = "0.5.0" +version = "0.6.0" authors = ["Michael Kraus "] [deps] @@ -16,6 +16,7 @@ InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" +NeuralNetworkParameters = "67f4d93a-60e9-472b-8cdd-1ccf6005724a" ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SymbolicNeuralNetworks = "aed23131-dcd0-47ca-8090-d21e605652e3" @@ -38,11 +39,12 @@ ForwardDiff = "1" GeometricBase = "0.14" GeometricEquations = "0.21" GeometricIntegrators = "0.18.2" -GeometricOptimizers = "0.4" +GeometricOptimizers = "0.4.1" GeometricSolutions = "0.6" HDF5 = "0.16, 0.17" KernelAbstractions = "0.9" NNlib = "0.8, 0.9" +NeuralNetworkParameters = "0.1" ProgressMeter = "1" SafeTestsets = "0.1" SymbolicNeuralNetworks = "0.5" diff --git a/docs/src/tutorials/adjusting_the_loss_function.md b/docs/src/tutorials/adjusting_the_loss_function.md index 871572d20..7acf859dd 100644 --- a/docs/src/tutorials/adjusting_the_loss_function.md +++ b/docs/src/tutorials/adjusting_the_loss_function.md @@ -42,7 +42,7 @@ using LinearAlgebra: norm # hide # norm of parameters for single layer network_parameter_norm(params::NamedTuple) = sum([norm(params[i]) for i in 1:length(params)]) # norm of parameters for entire network -function network_parameter_norm(params::NeuralNetworkParameters) +function network_parameter_norm(params::NetworkParameters) sum([network_parameter_norm(params[key]) for key in keys(params)]) end @@ -60,7 +60,7 @@ struct CustomLoss <: GeometricMachineLearning.NetworkLoss end using GeometricMachineLearning: QPTOAT, AbstractExplicitLayer # hide const λ = .1 -function (loss::CustomLoss)(model::Union{AbstractExplicitLayer, Chain}, params::Union{NeuralNetworkParameters, NamedTuple}, input::QPTOAT, output::QPTOAT) +function (loss::CustomLoss)(model::Union{AbstractExplicitLayer, Chain}, params::Union{NetworkParameters, NamedTuple}, input::QPTOAT, output::QPTOAT) FeedForwardLoss()(model, params, input, output) + λ * network_parameter_norm(params) end nothing # hide diff --git a/docs/src/tutorials/grassmann_layer.md b/docs/src/tutorials/grassmann_layer.md index 5bbb90a7a..4def6c820 100644 --- a/docs/src/tutorials/grassmann_layer.md +++ b/docs/src/tutorials/grassmann_layer.md @@ -260,7 +260,7 @@ where `np` is the number of points in ``\mathcal{D}_2`` and ``W_2`` is the *Wass where ``\nabla{}W_2`` is equivalent to the function `compute_wasserstein_gradient`. ```@example rosenbrock -function compute_gradient(ps::NeuralNetworkParameters) +function compute_gradient(ps::NetworkParameters) samples = randn(2, size(xyz_points, 2)) estimate, nn_pullback = Zygote.pullback(ps -> model(samples, ps), ps) diff --git a/ext/HDF5Ext.jl b/ext/HDF5Ext.jl index 038263cb0..dec35464b 100644 --- a/ext/HDF5Ext.jl +++ b/ext/HDF5Ext.jl @@ -2,58 +2,43 @@ module HDF5Ext using HDF5 using GeometricMachineLearning -import AbstractNeuralNetworks: h5save, changebackend, NeuralNetworkBackend, save, load, - NeuralNetworkParameters, params, Architecture +import AbstractNeuralNetworks: changebackend, NeuralNetworkBackend, save, load, Architecture +import NeuralNetworkParameters: NetworkParameters, params # --------------------------------------------------------------------------- -# h5save — new methods for GML special array types +# The traversal is not here any more. # -# AbstractNeuralNetworks only defines h5save for AbstractArray and NamedTuple. -# PSDLayer stores StiefelManifold and LASympNet's LinearLayer stores -# SymmetricMatrix; without these methods h5save throws a MethodError on any -# network whose parameters include those types. +# This extension used to carry five `h5save` methods that tagged a `gml_type` attribute, plus +# `_gml_h5load` and `_natural_sort_keys` to read them back. All three jobs now belong to packages +# that own the pieces: +# +# * `NeuralNetworkParameters` walks the parameter set and writes it, recording each group's key +# order in a `keys` attribute — which is what the `_natural_sort_keys` heuristic here was +# standing in for, and it guessed rather than knowing. Names that do not end in a digit were +# sorted lexicographically and silently came back in the wrong order. +# +# * `GeometricOptimizers` says where each structured matrix keeps its numbers, through +# `freeparameters`/`rebuild`, and registers the types so a file loads with no prototype. +# `StiefelManifold` and `SymmetricMatrix` are its types, not this package's, so the methods +# were type piracy here — on `h5save` and on the type both. +# +# Files written by the old code still load: `NeuralNetworkParameters` recognises the `gml_type` +# tag and rebuilds through the same registry (see `test/hdf5_support.jl`). +# +# What is left is the two entry points that genuinely dispatch on this package's `NeuralNetwork`. # --------------------------------------------------------------------------- -function h5save(h5::HDF5.H5DataStore, Y::StiefelManifold, path::AbstractString) - group = haskey(h5, path) ? h5[path] : HDF5.create_group(h5, path) - HDF5.attributes(group)["gml_type"] = "StiefelManifold" - group["A"] = Array(Y.A) -end - -function h5save(h5::HDF5.H5DataStore, A::SymmetricMatrix, path::AbstractString) - group = haskey(h5, path) ? h5[path] : HDF5.create_group(h5, path) - HDF5.attributes(group)["gml_type"] = "SymmetricMatrix" - group["S"] = Array(A.S) - group["n"] = A.n -end - -function h5save(h5::HDF5.H5DataStore, A::SkewSymMatrix, path::AbstractString) - group = haskey(h5, path) ? h5[path] : HDF5.create_group(h5, path) - HDF5.attributes(group)["gml_type"] = "SkewSymMatrix" - group["S"] = Array(A.S) - group["n"] = A.n -end - -function h5save(h5::HDF5.H5DataStore, A::LowerTriangular, path::AbstractString) - group = haskey(h5, path) ? h5[path] : HDF5.create_group(h5, path) - HDF5.attributes(group)["gml_type"] = "LowerTriangular" - group["S"] = Array(A.S) - group["n"] = A.n -end - -function h5save(h5::HDF5.H5DataStore, A::UpperTriangular, path::AbstractString) - group = haskey(h5, path) ? h5[path] : HDF5.create_group(h5, path) - HDF5.attributes(group)["gml_type"] = "UpperTriangular" - group["S"] = Array(A.S) - group["n"] = A.n -end - # --------------------------------------------------------------------------- # changebackend — new methods for GML special array types # # AbstractNeuralNetworks.changebackend handles AbstractArray and NamedTuple. # Moving a NeuralNetwork between devices fails for parameters that include # StiefelManifold, SymmetricMatrix, or SkewSymMatrix without these methods. +# +# These are the same ownership smell as the `h5save` methods above — `changebackend` is +# `AbstractNeuralNetworks`', the types are `GeometricOptimizers`' — and they belong in a +# `GeometricOptimizers` extension on `AbstractNeuralNetworks`. That is a separate change with its +# own release chain, so they stay here for now. # --------------------------------------------------------------------------- function changebackend(backend::NeuralNetworkBackend, Y::StiefelManifold) @@ -77,110 +62,52 @@ function changebackend(backend::NeuralNetworkBackend, A::UpperTriangular) end # --------------------------------------------------------------------------- -# Internal recursive loader that reconstructs GML special types from the -# gml_type attribute written by h5save. Kept private; used only by the -# load methods below so that we do not shadow AbstractNeuralNetworks.h5load. -# --------------------------------------------------------------------------- - -_gml_h5load(ds::HDF5.Dataset) = read(ds) - -# HDF5 returns group keys alphabetically, so "L10" precedes "L2". Sort by the -# numeric suffix when all keys match the pattern , otherwise -# fall back to lexicographic order so per-layer NamedTuples (bias/scale/weight) -# are unaffected. -function _natural_sort_keys(ks) - if all(k -> occursin(r"^\D+\d+$", k), ks) - return sort(collect(ks), - by = k -> (m = match(r"^(\D+)(\d+)$", k); (m[1], parse(Int, m[2])))) - end - sort(collect(ks)) -end - -function _gml_h5load(group::HDF5.Group) - if haskey(HDF5.attributes(group), "gml_type") - gml_type = read(HDF5.attributes(group)["gml_type"]) - if gml_type == "StiefelManifold" - return StiefelManifold(read(group["A"])) - elseif gml_type == "SymmetricMatrix" - return SymmetricMatrix(read(group["S"]), read(group["n"])) - elseif gml_type == "SkewSymMatrix" - return SkewSymMatrix(read(group["S"]), read(group["n"])) - elseif gml_type == "LowerTriangular" - return LowerTriangular(read(group["S"]), read(group["n"])) - elseif gml_type == "UpperTriangular" - return UpperTriangular(read(group["S"]), read(group["n"])) - end - end - sorted_keys = _natural_sort_keys(keys(group)) - paramkeys = Tuple(Symbol.(sorted_keys)) - paramvals = Tuple(_gml_h5load(group[k]) for k in sorted_keys) - NamedTuple{paramkeys}(paramvals) -end - -# --------------------------------------------------------------------------- -# save — new dispatch on NeuralNetwork, mirroring the existing -# save(h5::H5DataStore, p::NeuralNetworkParameters) -# method in AbstractNeuralNetworks. +# save / load — dispatch on `NeuralNetwork`, alongside the `NetworkParameters` +# methods in `NeuralNetworkParameters`. # --------------------------------------------------------------------------- """ save(h5::HDF5.H5DataStore, nn::NeuralNetwork) + save(filename::AbstractString, nn::NeuralNetwork) -Save the parameters of `nn` into an already-open HDF5 store. +Save the parameters of `nn` to an already-open HDF5 store or to a file. -Extends `AbstractNeuralNetworks.save` with a dispatch on `NeuralNetwork`. -GML special array types (`StiefelManifold`, `SymmetricMatrix`, `SkewSymMatrix`, -`LowerTriangular`, `UpperTriangular`) are tagged with a `gml_type` attribute -so that [`load`](@ref) can reconstruct them faithfully. +Extends `save` with a dispatch on `NeuralNetwork`; the parameters themselves are written by +`NeuralNetworkParameters`, which tags each structured leaf with the type to rebuild it as and +records the key order of every group. """ -function save(h5::HDF5.H5DataStore, nn::NeuralNetwork) - h5save(h5, params(params(nn)), "/") -end +save(h5::HDF5.H5DataStore, nn::NeuralNetwork) = save(h5, params(nn)) -""" - save(filename::AbstractString, nn::NeuralNetwork) - -Convenience overload: open `filename` for writing, then call -`save(h5, nn)`. -""" function save(filename::AbstractString, nn::NeuralNetwork) HDF5.h5open(filename, "w") do h5 save(h5, nn) end + filename end -# --------------------------------------------------------------------------- -# load — new dispatch on NeuralNetwork, mirroring the existing -# load(::Type{NeuralNetworkParameters}, h5::H5DataStore) -# method in AbstractNeuralNetworks. -# --------------------------------------------------------------------------- - """ - load(::Type{NeuralNetwork}, h5::HDF5.H5DataStore, arch::Architecture; backend=CPU()) + load(::Type{NeuralNetwork}, h5, arch::Architecture; backend = CPU()) + load(::Type{NeuralNetwork}, h5, arch::Architecture, prototype; backend = CPU()) -Load network parameters from an already-open HDF5 store and return a -`NeuralNetwork` for `arch`. +Load parameters from an HDF5 store or file and return a `NeuralNetwork` for `arch`. -Extends `AbstractNeuralNetworks.load` with a dispatch on `NeuralNetwork`. -GML special array types are reconstructed from their `gml_type` attribute. -The element type is preserved as stored (Float32 files reload as Float32). +The element type is whatever the file holds, so a `Float32` network reloads as `Float32`. + +Structured parameters — `StiefelManifold`, `SymmetricMatrix` and the rest — are rebuilt from the +type each was stored under, which `GeometricOptimizers` registers with +`NeuralNetworkParameters.register_parameter_type!`. Pass a `prototype` parameter set of the right +shape to rebuild against it instead and skip the registry altogether. """ -function load(::Type{NeuralNetwork}, h5::HDF5.H5DataStore, arch::Architecture; +function load(::Type{NeuralNetwork}, h5::HDF5.H5DataStore, arch::Architecture, args...; backend::NeuralNetworkBackend = CPU()) - ps = NeuralNetworkParameters(_gml_h5load(h5["/"])) + ps = load(NetworkParameters, h5, args...) NeuralNetwork(arch, Chain(arch), ps, backend) end -""" - load(::Type{NeuralNetwork}, filename::AbstractString, arch::Architecture; backend=CPU()) - -Convenience overload: open `filename` for reading, then call -`load(NeuralNetwork, h5, arch; backend)`. -""" -function load(::Type{NeuralNetwork}, filename::AbstractString, arch::Architecture; +function load(::Type{NeuralNetwork}, filename::AbstractString, arch::Architecture, args...; backend::NeuralNetworkBackend = CPU()) HDF5.h5open(filename, "r") do h5 - load(NeuralNetwork, h5, arch; backend = backend) + load(NeuralNetwork, h5, arch, args...; backend = backend) end end diff --git a/src/GeometricMachineLearning.jl b/src/GeometricMachineLearning.jl index 4268ee33a..8ceaac034 100644 --- a/src/GeometricMachineLearning.jl +++ b/src/GeometricMachineLearning.jl @@ -1,11 +1,11 @@ module GeometricMachineLearning using AbstractNeuralNetworks -# `AbstractNeuralNetworks` 0.7 no longer exports `NeuralNetworkParameters`: the parameter container -# moved out to the package of that name, where the type is called `NetworkParameters`, and the alias -# left behind is deliberately unexported so that every user of it says where it came from. This -# package uses the name in 31 places and re-exports it below, so it is imported explicitly here. -import AbstractNeuralNetworks: NeuralNetworkParameters +# The parameter container lives in `NeuralNetworkParameters` as of `AbstractNeuralNetworks` 0.7, +# under the name `NetworkParameters`. The import is selective rather than a bare `using`: that +# package also exports `flatten`/`unflatten` and the leaf protocol, none of which this package +# extends — `GeometricOptimizers` carries the protocol for the structured matrices. +import NeuralNetworkParameters: NetworkParameters using ChainRulesCore # `sqeuclidean` is the default distance of every `TrainingMethod` in `src/training_method/`. using Distances @@ -97,7 +97,7 @@ export Chain, NeuralNetwork export Dense, Linear export initialparameters export parameterlength -export NeuralNetworkParameters +export NetworkParameters export σ, sigmoid, softmax diff --git a/src/architectures/autoencoder.jl b/src/architectures/autoencoder.jl index 67c7e44fa..2d92400c7 100644 --- a/src/architectures/autoencoder.jl +++ b/src/architectures/autoencoder.jl @@ -172,7 +172,7 @@ end function encoder_parameters(nn::NeuralNetwork{<:AutoEncoder}) n_encoder_layers = length(encoder_model(nn.architecture).layers) keys = Tuple(Symbol.(["L$(i)" for i in 1:n_encoder_layers])) - NeuralNetworkParameters(NamedTuple{keys}(Tuple([params(nn)[key] for key in keys]))) + NetworkParameters(NamedTuple{keys}(Tuple([params(nn)[key] for key in keys]))) end # """ @@ -188,7 +188,7 @@ function decoder_parameters(nn::NeuralNetwork{<:AutoEncoder}) n_keys = length(keys_old) # "new keys" are the ones describing the keys in the new NamedTuple keys_new = Tuple(Symbol.(["L$(i)" for i in 1:n_keys])) - NeuralNetworkParameters(NamedTuple{keys_new}(Tuple([params(nn)[key] for key in keys_old]))) + NetworkParameters(NamedTuple{keys_new}(Tuple([params(nn)[key] for key in keys_old]))) end function Chain(arch::AutoEncoder) diff --git a/src/architectures/neural_network_integrator.jl b/src/architectures/neural_network_integrator.jl index 02376eac8..1935d760b 100644 --- a/src/architectures/neural_network_integrator.jl +++ b/src/architectures/neural_network_integrator.jl @@ -73,7 +73,7 @@ using GeometricMachineLearning model = ResNet(3, 0, identity) weight = [1 0 0; 0 2 0; 0 0 1] bias = [0, 0, 1] -ps = NeuralNetworkParameters((L1 = (weight = weight, bias = bias), )) +ps = NetworkParameters((L1 = (weight = weight, bias = bias), )) nn = NeuralNetwork(model, Chain(model), ps, CPU()) ics = [1, 1, 1] diff --git a/src/data_loader/data_loader.jl b/src/data_loader/data_loader.jl index 265efdf24..48939fe94 100644 --- a/src/data_loader/data_loader.jl +++ b/src/data_loader/data_loader.jl @@ -458,7 +458,7 @@ Compute the accuracy of a neural network classifier. This needs an instance of [`DataLoader`](@ref) that stores the *test data*. """ -function accuracy(model::Chain, ps::NeuralNetworkParameters, dl::DataLoader{T,AT,BT}) where {T,T1<:Integer,AT<:AbstractArray{T},BT<:AbstractArray{T1}} +function accuracy(model::Chain, ps::NetworkParameters, dl::DataLoader{T,AT,BT}) where {T,T1<:Integer,AT<:AbstractArray{T},BT<:AbstractArray{T1}} output_tensor = model(dl.input, ps) output_estimate = assign_output_estimate(output_tensor, dl.output_time_steps) backend = networkbackend(output_estimate) diff --git a/src/data_loader/optimize.jl b/src/data_loader/optimize.jl index 72af33360..a479c5bf6 100644 --- a/src/data_loader/optimize.jl +++ b/src/data_loader/optimize.jl @@ -48,7 +48,7 @@ number_of_batches(dl, batch) ``` """ function optimize_for_one_epoch!( opt::Optimizer, - model, ps::Union{NeuralNetworkParameters, NamedTuple}, + model, ps::Union{NetworkParameters, NamedTuple}, dl::DataLoader{T}, batch::Batch, loss::NetworkLoss, @@ -58,7 +58,7 @@ end function optimize_for_one_epoch!( opt::Optimizer, model, - ps::Union{NeuralNetworkParameters, NamedTuple}, + ps::Union{NetworkParameters, NamedTuple}, dl::DataLoader{T}, batch::Batch, _pullback::AbstractPullback, diff --git a/src/loss/hnn_loss.jl b/src/loss/hnn_loss.jl index 397e0602d..50b318066 100644 --- a/src/loss/hnn_loss.jl +++ b/src/loss/hnn_loss.jl @@ -29,13 +29,13 @@ end AbstractNeuralNetworks.NetworkLoss(arch::HamiltonianArchitecture) = HNNLoss(arch) function (loss::HNNLoss)(::Union{Chain, AbstractExplicitLayer}, - ps::Union{NeuralNetworkParameters, NamedTuple}, + ps::Union{NetworkParameters, NamedTuple}, input::QPTOAT, output::QPTOAT) loss(ps, input, output) end -function (loss::HNNLoss)(ps::Union{NeuralNetworkParameters, NamedTuple}, +function (loss::HNNLoss)(ps::Union{NetworkParameters, NamedTuple}, input::QPTOAT, output::QPTOAT) norm(loss.hvf(input, ps) - output) / norm(output) diff --git a/src/loss/losses.jl b/src/loss/losses.jl index 6c97ab8ff..2354a1551 100644 --- a/src/loss/losses.jl +++ b/src/loss/losses.jl @@ -73,7 +73,7 @@ function crop_array_for_transformer_loss(nn_output::AT, end function (loss::TransformerLoss)(model::Union{Chain, AbstractExplicitLayer}, - ps::Union{NeuralNetworkParameters, NamedTuple}, input::AT, + ps::Union{NetworkParameters, NamedTuple}, input::AT, output::AT) where {T, AT <: AbstractArray{T, 3}} input_dim, input_seq_length = size(input) output_dim, output_prediction_window = size(output) @@ -88,13 +88,13 @@ function (loss::TransformerLoss)(model::Union{Chain, AbstractExplicitLayer}, end function (loss::TransformerLoss)(model::Union{Chain, AbstractExplicitLayer}, - ps::Union{NeuralNetworkParameters, NamedTuple}, input::AT, + ps::Union{NetworkParameters, NamedTuple}, input::AT, output::AT) where {T, AT <: AbstractArray{T, 2}} loss(model, ps, reshape(input, size(input)..., 1), reshape(output, size(output)..., 1)) end function (loss::TransformerLoss)(model::Union{Chain, AbstractExplicitLayer}, - ps::Union{NeuralNetworkParameters, NamedTuple}, + ps::Union{NetworkParameters, NamedTuple}, input::T, output::T) where {T <: QPT} loss(model, ps, vcat(input.q, input.p), vcat(output.q, output.p)) end @@ -120,7 +120,7 @@ end struct ClassificationTransformerLoss <: NetworkLoss end function (loss::ClassificationTransformerLoss)(model::Union{Chain, AbstractExplicitLayer}, - ps::Union{NeuralNetworkParameters, NamedTuple}, + ps::Union{NetworkParameters, NamedTuple}, input::AbstractArray, output::AbstractArray) predicted_output_uncropped = model(input, ps) # predicted_output_cropped = crop_array_for_transformer_loss(predicted_output_uncropped, output) @@ -179,12 +179,12 @@ function (loss::AutoEncoderLoss)(nn::NeuralNetwork, input::QPTOAT) end function (loss::AutoEncoderLoss)(model::Union{Chain, AbstractExplicitLayer}, - ps::Union{NeuralNetworkParameters, NamedTuple}, input::QPTOAT) + ps::Union{NetworkParameters, NamedTuple}, input::QPTOAT) loss(model, ps, input, input) end function (loss::AutoEncoderLoss)(model::Union{Chain, AbstractExplicitLayer}, - ps::Union{NeuralNetworkParameters, NamedTuple}, input::QPTOAT, output::QPTOAT) + ps::Union{NetworkParameters, NamedTuple}, input::QPTOAT, output::QPTOAT) FeedForwardLoss()(model, ps, input, output) end @@ -253,7 +253,7 @@ function ReducedLoss(autoencoder::NeuralNetwork{<:AutoEncoder}) ReducedLoss(encoder(autoencoder), decoder(autoencoder)) end -function (loss::ReducedLoss)(model::Chain, params::NeuralNetworkParameters, +function (loss::ReducedLoss)(model::Chain, params::NetworkParameters, input::CT, output::CT) where {CT <: QPTOAT} _compute_loss(loss.decoder(model(loss.encoder(input), params)), output) end diff --git a/src/map_to_cpu.jl b/src/map_to_cpu.jl index 8ca85e0de..fc5a1a1c3 100644 --- a/src/map_to_cpu.jl +++ b/src/map_to_cpu.jl @@ -1,5 +1,5 @@ -function map_to_cpu(ps::NeuralNetworkParameters) - NeuralNetworkParameters(NamedTuple{keys(ps)}(Tuple(map_to_cpu(ps[key]) for key in keys(ps)))) +function map_to_cpu(ps::NetworkParameters) + NetworkParameters(NamedTuple{keys(ps)}(Tuple(map_to_cpu(ps[key]) for key in keys(ps)))) end map_to_cpu(layer::NamedTuple) = apply_toNT(map_to_cpu, layer) diff --git a/src/optimizers/optimizer.jl b/src/optimizers/optimizer.jl index 93b766b48..5dbba5f17 100644 --- a/src/optimizers/optimizer.jl +++ b/src/optimizers/optimizer.jl @@ -1,8 +1,8 @@ # Optimizer machinery on top of GeometricOptimizers. # Kept out of `utils.jl` because it dispatches on `Manifold`, which is defined later. -# Extend GlobalSection so it works with NeuralNetworkParameters (wraps a NamedTuple). -GeometricOptimizers.GlobalSection(ps::NeuralNetworkParameters) = +# Extend GlobalSection so it works with NetworkParameters (wraps a NamedTuple). +GeometricOptimizers.GlobalSection(ps::NetworkParameters) = GeometricOptimizers.GlobalSection(params(ps)) # Backward-compat alias @@ -55,7 +55,7 @@ _use_go_cache(method, x) = function _make_optimizer_cache(method, x) if _use_go_cache(method, x) GeometricOptimizers.OptimizerCache(_adapt_method_to_T(method, _eltype(x)), x) - elseif x isa NamedTuple || x isa NeuralNetworkParameters + elseif x isa NamedTuple || x isa NetworkParameters NamedTuple{keys(x)}(Tuple(_make_optimizer_cache(method, x[k]) for k in keys(x))) else GMLEuclideanState(x) @@ -65,7 +65,7 @@ end function _make_optimizer_state(method, x) if _use_go_cache(method, x) GeometricOptimizers.OptimizerState(_adapt_method_to_T(method, _eltype(x)), x) - elseif x isa NamedTuple || x isa NeuralNetworkParameters + elseif x isa NamedTuple || x isa NetworkParameters NamedTuple{keys(x)}(Tuple(_make_optimizer_state(method, x[k]) for k in keys(x))) else GMLEuclideanState(x) @@ -129,7 +129,7 @@ function Optimizer(method::GeometricOptimizers.OptimizerMethod, nn::NeuralNetwor end function Optimizer(method::GeometricOptimizers.OptimizerMethod, - ps::Union{NamedTuple, NeuralNetworkParameters}; + ps::Union{NamedTuple, NetworkParameters}; retraction = GeometricOptimizers.cayley, step_size = _default_step_size(method)) Optimizer(method, _make_optimizer_cache(method, ps), _make_optimizer_state(method, ps), @@ -139,7 +139,7 @@ end # The keyword form, so that the `(algorithm, linesearch)` pairing `GeometricOptimizers` returns from # `AdamOptimizerWithDecay` splats in unchanged. `linesearch` is the step size under the name # upstream gives it; a `Static` carries its own `α`, which is then the fixed learning rate. -function Optimizer(nn_or_ps::Union{NeuralNetwork, NamedTuple, NeuralNetworkParameters}; +function Optimizer(nn_or_ps::Union{NeuralNetwork, NamedTuple, NetworkParameters}; algorithm::GeometricOptimizers.OptimizerMethod, linesearch = nothing, retraction = GeometricOptimizers.cayley, diff --git a/src/pullbacks/zygote_pullback.jl b/src/pullbacks/zygote_pullback.jl index dad6054c5..df882e224 100644 --- a/src/pullbacks/zygote_pullback.jl +++ b/src/pullbacks/zygote_pullback.jl @@ -39,9 +39,9 @@ Unwrap the single element `Zygote` may wrap a pullback result in. Together with [`_get_params`](@ref) this makes up [`_processing`](@ref). """ -_get_contents(nt::Union{NamedTuple, NeuralNetworkParameters}) = nt -_get_contents(nt::Tuple{<:Union{NamedTuple, NeuralNetworkParameters}}) = nt[1] -function _get_contents(nt::AbstractVector{<:Union{NamedTuple, NeuralNetworkParameters}}) +_get_contents(nt::Union{NamedTuple, NetworkParameters}) = nt +_get_contents(nt::Tuple{<:Union{NamedTuple, NetworkParameters}}) = nt[1] +function _get_contents(nt::AbstractVector{<:Union{NamedTuple, NetworkParameters}}) length(nt) == 1 || throw(ArgumentError( "the pullback returned $(length(nt)) parameter sets, expected one.")) nt[1] @@ -51,14 +51,14 @@ end _get_params(returned_pullback) Get the parameters out of a pullback result, whether they come as a -`NeuralNetworkParameters`, wrapped in a `NamedTuple` with a single `params` field, or bare. +`NetworkParameters`, wrapped in a `NamedTuple` with a single `params` field, or bare. Together with [`_get_contents`](@ref) this makes up [`_processing`](@ref). """ _get_params(nt::NamedTuple) = nt -_get_params(ps::NeuralNetworkParameters) = params(ps) +_get_params(ps::NetworkParameters) = params(ps) function _get_params(nt::NamedTuple{(:params,), Tuple{AT}}) where {AT} - @warn "This function was most likely called because @adjoint for `NeuralNetworkParameters` hasn't been implemented." + @warn "This function was most likely called because @adjoint for `NetworkParameters` hasn't been implemented." nt.params end diff --git a/src/utils.jl b/src/utils.jl index 9fa719360..d4d28a872 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -170,4 +170,4 @@ Base.:≈(qp₁::QPT, qp₂::QPT) = (qp₁.q ≈ qp₂.q) & (qp₁.p ≈ qp₂.p _eltype(x) = eltype(x) _eltype(ps::NamedTuple) = _eltype(ps[1]) _eltype(ps::Tuple) = _eltype(ps[1]) -_eltype(ps::NeuralNetworkParameters) = _eltype(params(ps)[1]) +_eltype(ps::NetworkParameters) = _eltype(params(ps)[1]) diff --git a/test/docstrings/layers_and_architectures.jl b/test/docstrings/layers_and_architectures.jl index 7205331b5..6d8415f7b 100644 --- a/test/docstrings/layers_and_architectures.jl +++ b/test/docstrings/layers_and_architectures.jl @@ -14,7 +14,7 @@ using GeometricMachineLearning: UnknownEncoder, params model = ResNet(3, 0, identity) weight = [1 0 0; 0 2 0; 0 0 1] bias = [0, 0, 1] - ps = NeuralNetworkParameters((L1 = (weight = weight, bias = bias),)) + ps = NetworkParameters((L1 = (weight = weight, bias = bias),)) nn = NeuralNetwork(model, Chain(model), ps, CPU()) @test iterate(nn, [1, 1, 1]; n_points = 4) == [1 2 4 8; 1 3 9 27; 1 3 7 15] diff --git a/test/hamiltonian_neural_network_tests.jl b/test/hamiltonian_neural_network_tests.jl index 778b89fa8..2f976f6a0 100644 --- a/test/hamiltonian_neural_network_tests.jl +++ b/test/hamiltonian_neural_network_tests.jl @@ -37,7 +37,7 @@ function test_hnn_loss_derivative( dim::Integer = 2, activation::GMLA = GeometricMachineLearning.SigmoidActivation()) nn, loss, dl = allocate_network_and_data_loader(dim, width, nhidden, activation) dp = Zygote.gradient(ps -> loss(ps, dl.input, dl.output), nn.params)[1] - @test typeof(dp) <: NeuralNetworkParameters + @test typeof(dp) <: NetworkParameters @test keys(dp) == keys(nn.params) end diff --git a/test/hdf5_support.jl b/test/hdf5_support.jl index 94a3cdf02..ad0f17479 100644 --- a/test/hdf5_support.jl +++ b/test/hdf5_support.jl @@ -4,6 +4,7 @@ using HDF5 using LinearAlgebra: qr import Random import AbstractNeuralNetworks: params, changebackend +import NeuralNetworkParameters: NetworkParameters Random.seed!(42) @@ -21,11 +22,41 @@ function _ps_eq(a::NamedTuple, b::NamedTuple) Set(keys(a)) == Set(keys(b)) || return false all(_ps_eq(a[k], b[k]) for k in keys(a)) end -function _ps_eq(a::NeuralNetworkParameters, b::NeuralNetworkParameters) +function _ps_eq(a::NetworkParameters, b::NetworkParameters) keys(a) == keys(b) || return false all(_ps_eq(a[k], b[k]) for k in keys(a)) end +# Reproduces the on-disk layout this package wrote before the traversal moved out: plain nested +# groups with no `kind`/`keys` attributes, and each structured matrix tagged `gml_type`. +_legacy_group(h5, path) = path == "/" ? h5 : + (haskey(h5, path) ? h5[path] : HDF5.create_group(h5, path)) + +function _write_legacy(h5, nt::NamedTuple, path::AbstractString) + g = _legacy_group(h5, path) + for (k, v) in pairs(nt) + _write_legacy(g, v, String(k)) + end +end + +_write_legacy(h5, x::AbstractArray, path::AbstractString) = (h5[path] = Array(x); nothing) + +function _write_legacy(h5, Y::StiefelManifold, path::AbstractString) + g = HDF5.create_group(h5, path) + HDF5.attributes(g)["gml_type"] = "StiefelManifold" + g["A"] = Array(Y.A) +end + +for (T, name) in ((:SymmetricMatrix, "SymmetricMatrix"), (:SkewSymMatrix, "SkewSymMatrix"), + (:LowerTriangular, "LowerTriangular"), (:UpperTriangular, "UpperTriangular")) + @eval function _write_legacy(h5, A::$T, path::AbstractString) + g = HDF5.create_group(h5, path) + HDF5.attributes(g)["gml_type"] = $name + g["S"] = Array(A.S) + g["n"] = A.n + end +end + # --------------------------------------------------------------------------- # save / load roundtrip — one testset per architecture # --------------------------------------------------------------------------- @@ -136,6 +167,34 @@ end end end +# --------------------------------------------------------------------------- +# Files written before the traversal moved out of this package +# --------------------------------------------------------------------------- + +# This package used to write each structured matrix itself, as a group tagged `gml_type` holding the +# fields under their own names and recording no key order. `NeuralNetworkParameters` recognises the +# tag and rebuilds through the registry `GeometricOptimizers` fills, so those files still load — +# which is the whole reason the duplicated reader here could be deleted rather than kept alongside. +@testset "a file in the old gml_type layout still loads" begin + arch = LASympNet(4) # LinearLayer → SymmetricMatrix, the tagged case + nn = NeuralNetwork(arch) + ps = params(nn) + x = rand(4) + y = nn(x) + + mktempdir() do dir + path = joinpath(dir, "legacy.h5") + HDF5.h5open(path, "w") do h5 + _write_legacy(h5, params(ps), "/") + end + nn2 = load(NeuralNetwork, path, arch) + + @test keys(params(nn2)) == keys(ps) + @test _ps_eq(ps, params(nn2)) + @test nn2(x) ≈ y + end +end + # --------------------------------------------------------------------------- # changebackend — new methods for GML special array types # --------------------------------------------------------------------------- diff --git a/test/optimizers/optimizer_convergence_tests/psd_optim.jl b/test/optimizers/optimizer_convergence_tests/psd_optim.jl index b58d66f2c..e0fd5b68c 100644 --- a/test/optimizers/optimizer_convergence_tests/psd_optim.jl +++ b/test/optimizers/optimizer_convergence_tests/psd_optim.jl @@ -63,7 +63,7 @@ function svd_test(A, n, train_steps=1000, tol=1e-1; retraction=cayley) @test norm((err₃ - err_best)/err_best) < tol end -function train_network!(o::Optimizer, model::Chain, ps::NeuralNetworkParameters, A::AbstractMatrix, train_steps, tol) +function train_network!(o::Optimizer, model::Chain, ps::NetworkParameters, A::AbstractMatrix, train_steps, tol) error(ps) = norm(A - model(A, ps)) for _ in 1:train_steps diff --git a/test/optimizers/optimizer_convergence_tests/svd_optim.jl b/test/optimizers/optimizer_convergence_tests/svd_optim.jl index 0ea4705a1..02be1f449 100644 --- a/test/optimizers/optimizer_convergence_tests/svd_optim.jl +++ b/test/optimizers/optimizer_convergence_tests/svd_optim.jl @@ -59,7 +59,7 @@ function svd_test(A, n, train_steps=1000, tol=1e-1; retraction=cayley) @test norm((err₃ - err_best)/err_best) < tol end -function train_network!(o::Optimizer, model::Chain, ps::NeuralNetworkParameters, A::AbstractMatrix, train_steps, tol) +function train_network!(o::Optimizer, model::Chain, ps::NetworkParameters, A::AbstractMatrix, train_steps, tol) error(ps) = norm(A - model(A, ps)) for _ in 1:train_steps diff --git a/test/transformer_related/multi_head_attention_stiefel_setup.jl b/test/transformer_related/multi_head_attention_stiefel_setup.jl index 6573c146a..181bfc866 100644 --- a/test/transformer_related/multi_head_attention_stiefel_setup.jl +++ b/test/transformer_related/multi_head_attention_stiefel_setup.jl @@ -13,7 +13,7 @@ function check_setup(A::AbstractMatrix{T}, tol=T(10)*eps(T)) where T @test check(A) < tol end check_setup(ps::NamedTuple) = apply_toNT(check_setup, ps) -check_setup(ps::NeuralNetworkParameters) = check_setup(GeometricMachineLearning.params(ps)) +check_setup(ps::NetworkParameters) = check_setup(GeometricMachineLearning.params(ps)) @doc raw""" This checks for an arbitrary matrix ``B\in\mathbb{R}^{N\times{}N}`` if ``B\in\mathfrak{g}^\mathrm{hor}``. From c86e8bb99d0013dd7d65b5829ad2e9306a9e5e57 Mon Sep 17 00:00:00 2001 From: Michael Kraus Date: Sun, 23 Aug 2026 14:18:36 +0900 Subject: [PATCH 4/5] Address review: the compat chain, the docs build, and four smaller things MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `SymbolicNeuralNetworks = "0.5"` caps `AbstractNeuralNetworks` at `0.6.4 - 0.6`, so together with `AbstractNeuralNetworks = "0.7"` the `[compat]` block was not merely pointing at unregistered versions, it was unsatisfiable. The bound is `"0.6"`, and the CHANGELOG now spells out the release chain the branch waits on: ANN 0.7.0 registered, then GeometricOptimizers 0.4.1 tagged and registered, then SymbolicNeuralNetworks fixed — its `abstractneuralnetworks-0.7` branch still imports `AbstractNeuralNetworks.QPTOAT`, which 0.7 replaced — bumped to 0.6.0 and registered. The docs build was broken: `abstract_neural_networks.md` lists four `@docs` signatures and this branch had merged four docstrings into two, so Documenter would have errored on `save(::AbstractString, ::NeuralNetwork)` and `load(::Type{NeuralNetwork}, ::AbstractString, ::Architecture)` — `docs/make.jl` passes `HDF5Ext` in `modules` and sets no `warnonly`. Each of the four methods carries its own docstring again, and the prose above them says who does the work now instead of claiming this package still handles the structured types itself. Also: * `load(NeuralNetwork, …, args...)` took untyped varargs where one optional prototype was meant. Explicit methods instead, so a wrong arity is a `MethodError` at the call site and the `@docs` signatures name real methods. * the prototype form had no test, and the old-layout test covered only `SymmetricMatrix` — `GeometricOptimizers` normalises the two old shapes through different helpers, so the `StiefelManifold` leg is read back too. * `save`/`load` are `NeuralNetworkParameters`' generics; 0.7 only re-binds them. Imported from the owner, in `src/` and in the extension. * seven `NeuralNetworkParameters` the rename missed, in `scripts/`. * `save(filename, nn)` returning `filename`, and the new prototype `load`, recorded in the CHANGELOG under *Changed* and *Added* rather than left unmentioned and filed under *Removed*. The `RecurrentNeuralNetwork`/`LSTMNeuralNetwork` removal this commit originally carried is gone: #247 landed the same removal on main, more thoroughly — it also drops the `IdentityActivation, ZeroVector` import the removal left dead and updates the docs footnote — so the rebase takes main's. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 40 ++++++-- Project.toml | 2 +- .../architectures/abstract_neural_networks.md | 2 +- ext/HDF5Ext.jl | 93 ++++++++++--------- scripts/test_attention.jl | 8 +- .../test_double_multiplication_derivative.jl | 6 +- src/GeometricMachineLearning.jl | 4 +- test/hdf5_support.jl | 73 +++++++++++---- 8 files changed, 150 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d1e9622ca..fee17f450 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,15 +36,14 @@ breaking release). of the module already provides, so `GeometricMachineLearning.IdentityActivation` still resolves — the line was redundant, not load-bearing. -### Removed - - **`NeuralNetworkParameters` is no longer exported; the name is `NetworkParameters`.** The parameter container moved out of `AbstractNeuralNetworks` into [NeuralNetworkParameters.jl](https://github.com/JuliaGNI/NeuralNetworkParameters.jl) in `AbstractNeuralNetworks` 0.7, which removed the old name outright rather than leaving an alias, so - that one type has one name across the ecosystem. This package follows, at all 31 call sites and in - its export list. It is the same type object, so `::Type{}` dispatch, `<:` bounds and - `NetworkParameters{keys}(vals)` construction are unaffected; only the spelling changes. + that one type has one name across the ecosystem. This package follows, at every call site — 31 in + `src/`, `docs/` and `test/`, seven more in `scripts/` — and in its export list. It is the same type + object, so `::Type{}` dispatch, `<:` bounds and `NetworkParameters{keys}(vals)` construction are + unaffected; only the spelling changes. ```julia # before @@ -70,9 +69,6 @@ breaking release). through the same registry, and `test/hdf5_support.jl` now writes a file in the old layout by hand and reads it back, so the deletion cannot quietly make old files unreadable. - `load(NeuralNetwork, h5, arch)` additionally accepts a prototype parameter set — - `load(NeuralNetwork, h5, arch, prototype)` — which rebuilds against it and skips the registry. - ### Fixed - **Zygote 0.7 silently zeroed every gradient that flows through `assign_q_and_p`.** Its `rrule` @@ -139,6 +135,18 @@ breaking release). it sits in, on a signature it did not need — the body ignores its argument and returns `ZeroTangent()` regardless. +- **`save(filename, nn)` returns `filename`.** It used to return whatever the `h5open` block left + behind — the value of the innermost `h5save`, an implementation detail of the traversal. Returning + the path is what `NeuralNetworkParameters.save(filename, ps)` does, so the two now agree. + +### Added + +- **`load(NeuralNetwork, h5, arch, prototype)`** — a parameter set of the right shape to rebuild the + structured leaves against. It is the form that needs no registration: `rebuild` has a prototype to + take the non-differentiable fields from, so the file's type tags and + `NeuralNetworkParameters.register_parameter_type!` are not consulted at all. Both the store and the + filename overloads take it. + ### Dependencies - **`NeuralNetworkParameters = "0.1"`** added, and **`AbstractNeuralNetworks = "0.7"`** (was @@ -148,6 +156,22 @@ breaking release). `NeuralNetworkParameters` leaf protocol for the manifolds, storage matrices and horizontal lifts, which is what lets this package's HDF5 extension drop its own copy of the traversal. +- **`SymbolicNeuralNetworks = "0.6"`** (was `"0.5"`). 0.5 caps `AbstractNeuralNetworks` at `"0.6.4 - + 0.6"`, so leaving the bound would have made this package's `[compat]` unsatisfiable rather than + merely unresolved. 0.6 is the release that follows the container out to `NeuralNetworkParameters`. + + > **Merge order.** Three of these four bounds point at releases that do not exist in the General + > registry yet, so this cannot be merged before them, in this order: + > + > 1. `AbstractNeuralNetworks` 0.7.0 — tagged, awaiting registration. + > 2. `GeometricOptimizers` 0.4.1 — the `NeuralNetworkParameters` extension is on `main`; needs a + > version bump, a tag and registration. + > 3. `SymbolicNeuralNetworks` 0.6.0 — the `abstractneuralnetworks-0.7` branch still says `0.5.0`, + > and still imports `AbstractNeuralNetworks.QPTOAT`, which 0.7 replaced with + > `ArrayOrNamedTuple`; it does not load as it stands. + > + > Until then CI here fails at `Pkg.instantiate`. That is expected, not a regression. + - **`Zygote = "0.7"`** (was `"0.6"`). 0.7 replaced the eager unthunking in `wrap_chainrules_output` with `unthunk_tangent` at the `gradient`/`pullback` boundaries, which is what let thunks reach GML's `rrule`s and surfaced everything under *Fixed* above. Implicit parameters are deprecated in diff --git a/Project.toml b/Project.toml index 0c22b0a4f..462957bc8 100644 --- a/Project.toml +++ b/Project.toml @@ -47,7 +47,7 @@ NNlib = "0.8, 0.9" NeuralNetworkParameters = "0.1" ProgressMeter = "1" SafeTestsets = "0.1" -SymbolicNeuralNetworks = "0.5" +SymbolicNeuralNetworks = "0.6" Symbolics = "7" TimerOutputs = "0.5, 1" Zygote = "0.7" diff --git a/docs/src/architectures/abstract_neural_networks.md b/docs/src/architectures/abstract_neural_networks.md index 5abfb6a51..26f3a1427 100644 --- a/docs/src/architectures/abstract_neural_networks.md +++ b/docs/src/architectures/abstract_neural_networks.md @@ -47,7 +47,7 @@ and we see that it consists of two layers: a [`GradientLayerQ`](@ref) and a [`Gr ## Saving and Loading -`GeometricMachineLearning` extends `AbstractNeuralNetworks` with HDF5-backed save and load methods for `NeuralNetwork`. These handle GML-specific parameter types (`StiefelManifold`, `SymmetricMatrix`, `SkewSymMatrix`) transparently. +`GeometricMachineLearning` adds HDF5-backed `save` and `load` methods for `NeuralNetwork` to the generics `NeuralNetworkParameters` defines. Writing and reading the parameter set itself belongs to that package; the structured parameter types (`StiefelManifold`, `SymmetricMatrix`, `SkewSymMatrix`, …) come back as themselves because `GeometricOptimizers`, which owns them, registers how each is rebuilt. Passing a prototype parameter set to `load` rebuilds against it and needs no registration at all. ```@docs save(::HDF5.H5DataStore, ::NeuralNetwork) diff --git a/ext/HDF5Ext.jl b/ext/HDF5Ext.jl index dec35464b..0853d4257 100644 --- a/ext/HDF5Ext.jl +++ b/ext/HDF5Ext.jl @@ -2,31 +2,10 @@ module HDF5Ext using HDF5 using GeometricMachineLearning -import AbstractNeuralNetworks: changebackend, NeuralNetworkBackend, save, load, Architecture -import NeuralNetworkParameters: NetworkParameters, params - -# --------------------------------------------------------------------------- -# The traversal is not here any more. -# -# This extension used to carry five `h5save` methods that tagged a `gml_type` attribute, plus -# `_gml_h5load` and `_natural_sort_keys` to read them back. All three jobs now belong to packages -# that own the pieces: -# -# * `NeuralNetworkParameters` walks the parameter set and writes it, recording each group's key -# order in a `keys` attribute — which is what the `_natural_sort_keys` heuristic here was -# standing in for, and it guessed rather than knowing. Names that do not end in a digit were -# sorted lexicographically and silently came back in the wrong order. -# -# * `GeometricOptimizers` says where each structured matrix keeps its numbers, through -# `freeparameters`/`rebuild`, and registers the types so a file loads with no prototype. -# `StiefelManifold` and `SymmetricMatrix` are its types, not this package's, so the methods -# were type piracy here — on `h5save` and on the type both. -# -# Files written by the old code still load: `NeuralNetworkParameters` recognises the `gml_type` -# tag and rebuilds through the same registry (see `test/hdf5_support.jl`). -# -# What is left is the two entry points that genuinely dispatch on this package's `NeuralNetwork`. -# --------------------------------------------------------------------------- +import AbstractNeuralNetworks: changebackend, NeuralNetworkBackend, Architecture +# `save`, `load`, `params` and the parameter container are `NeuralNetworkParameters`' as of +# `AbstractNeuralNetworks` 0.7, which only re-binds them; reach for them where they are defined. +import NeuralNetworkParameters: NetworkParameters, params, save, load # --------------------------------------------------------------------------- # changebackend — new methods for GML special array types @@ -35,10 +14,10 @@ import NeuralNetworkParameters: NetworkParameters, params # Moving a NeuralNetwork between devices fails for parameters that include # StiefelManifold, SymmetricMatrix, or SkewSymMatrix without these methods. # -# These are the same ownership smell as the `h5save` methods above — `changebackend` is -# `AbstractNeuralNetworks`', the types are `GeometricOptimizers`' — and they belong in a -# `GeometricOptimizers` extension on `AbstractNeuralNetworks`. That is a separate change with its -# own release chain, so they stay here for now. +# `changebackend` is `AbstractNeuralNetworks`' and the types are `GeometricOptimizers`', so these +# methods are piracy the same way the `h5save` ones were before `GeometricOptimizers` took over the +# leaf protocol. They belong in a `GeometricOptimizers` extension on `AbstractNeuralNetworks`; that +# is a separate change with its own release chain, so they stay here for now. # --------------------------------------------------------------------------- function changebackend(backend::NeuralNetworkBackend, Y::StiefelManifold) @@ -62,22 +41,30 @@ function changebackend(backend::NeuralNetworkBackend, A::UpperTriangular) end # --------------------------------------------------------------------------- -# save / load — dispatch on `NeuralNetwork`, alongside the `NetworkParameters` -# methods in `NeuralNetworkParameters`. +# save / load — the entry points that dispatch on this package's `NeuralNetwork`. +# +# The traversal itself is not here. `NeuralNetworkParameters` walks the parameter set and records +# each group's key order; `GeometricOptimizers` says through `freeparameters`/`rebuild` where each +# structured matrix keeps its numbers, and registers the types so a file loads with no prototype. # --------------------------------------------------------------------------- """ save(h5::HDF5.H5DataStore, nn::NeuralNetwork) - save(filename::AbstractString, nn::NeuralNetwork) -Save the parameters of `nn` to an already-open HDF5 store or to a file. +Save the parameters of `nn` into an already-open HDF5 store. -Extends `save` with a dispatch on `NeuralNetwork`; the parameters themselves are written by +Extends `save` with a dispatch on `NeuralNetwork`. The parameters themselves are written by `NeuralNetworkParameters`, which tags each structured leaf with the type to rebuild it as and records the key order of every group. """ save(h5::HDF5.H5DataStore, nn::NeuralNetwork) = save(h5, params(nn)) +""" + save(filename::AbstractString, nn::NeuralNetwork) + +Convenience overload: open `filename` for writing, call [`save`](@ref) on the store, and return +`filename`. +""" function save(filename::AbstractString, nn::NeuralNetwork) HDF5.h5open(filename, "w") do h5 save(h5, nn) @@ -86,28 +73,46 @@ function save(filename::AbstractString, nn::NeuralNetwork) end """ - load(::Type{NeuralNetwork}, h5, arch::Architecture; backend = CPU()) - load(::Type{NeuralNetwork}, h5, arch::Architecture, prototype; backend = CPU()) + load(::Type{NeuralNetwork}, h5::HDF5.H5DataStore, arch::Architecture; backend = CPU()) + load(::Type{NeuralNetwork}, h5::HDF5.H5DataStore, arch::Architecture, prototype; backend = CPU()) -Load parameters from an HDF5 store or file and return a `NeuralNetwork` for `arch`. +Load network parameters from an already-open HDF5 store and return a `NeuralNetwork` for `arch`. The element type is whatever the file holds, so a `Float32` network reloads as `Float32`. Structured parameters — `StiefelManifold`, `SymmetricMatrix` and the rest — are rebuilt from the type each was stored under, which `GeometricOptimizers` registers with -`NeuralNetworkParameters.register_parameter_type!`. Pass a `prototype` parameter set of the right -shape to rebuild against it instead and skip the registry altogether. +`NeuralNetworkParameters.register_parameter_type!`. Pass `prototype`, a parameter set of the right +shape, to rebuild against it instead and skip the registry altogether. +""" +function load(::Type{NeuralNetwork}, h5::HDF5.H5DataStore, arch::Architecture; + backend::NeuralNetworkBackend = CPU()) + NeuralNetwork(arch, Chain(arch), load(NetworkParameters, h5), backend) +end + +function load(::Type{NeuralNetwork}, h5::HDF5.H5DataStore, arch::Architecture, prototype; + backend::NeuralNetworkBackend = CPU()) + NeuralNetwork(arch, Chain(arch), load(NetworkParameters, h5, prototype), backend) +end + +""" + load(::Type{NeuralNetwork}, filename::AbstractString, arch::Architecture; backend = CPU()) + load(::Type{NeuralNetwork}, filename::AbstractString, arch::Architecture, prototype; backend = CPU()) + +Convenience overload: open `filename` for reading, then call +[`load`](@ref) on the store. """ -function load(::Type{NeuralNetwork}, h5::HDF5.H5DataStore, arch::Architecture, args...; +function load(::Type{NeuralNetwork}, filename::AbstractString, arch::Architecture; backend::NeuralNetworkBackend = CPU()) - ps = load(NetworkParameters, h5, args...) - NeuralNetwork(arch, Chain(arch), ps, backend) + HDF5.h5open(filename, "r") do h5 + load(NeuralNetwork, h5, arch; backend = backend) + end end -function load(::Type{NeuralNetwork}, filename::AbstractString, arch::Architecture, args...; +function load(::Type{NeuralNetwork}, filename::AbstractString, arch::Architecture, prototype; backend::NeuralNetworkBackend = CPU()) HDF5.h5open(filename, "r") do h5 - load(NeuralNetwork, h5, arch, args...; backend = backend) + load(NeuralNetwork, h5, arch, prototype; backend = backend) end end diff --git a/scripts/test_attention.jl b/scripts/test_attention.jl index 5dac1fb19..37e7e98be 100644 --- a/scripts/test_attention.jl +++ b/scripts/test_attention.jl @@ -3,21 +3,21 @@ using GeometricMachineLearning: _custom_mul, _custom_transpose using LinearAlgebra: norm using Zygote: gradient -function symplectic_attention(z::NamedTuple{(:q, :p), Tuple{AT, AT}}, _ps::Union{NamedTuple, NeuralNetworkParameters}) where {AT<:AbstractArray} +function symplectic_attention(z::NamedTuple{(:q, :p), Tuple{AT, AT}}, _ps::Union{NamedTuple, NetworkParameters}) where {AT<:AbstractArray} expPAP = exp.(_custom_mul(_custom_mul(_custom_transpose(z.p), _ps.L1.A), z.p)) (q = z.q + _custom_mul(_custom_mul(_ps.L1.A, z.p), 2 * expPAP) / sum(expPAP), p = z.p) end -function symplectic_attention_simplified(z::NamedTuple{(:q, :p), Tuple{AT, AT}}, _ps::Union{NamedTuple, NeuralNetworkParameters}) where {AT<:AbstractArray} +function symplectic_attention_simplified(z::NamedTuple{(:q, :p), Tuple{AT, AT}}, _ps::Union{NamedTuple, NetworkParameters}) where {AT<:AbstractArray} (q = z.p + _custom_mul(_custom_mul(z.p, _ps.L1.A), z.p), p = z.p) end -function symplectic_linear_map(z::NamedTuple{(:q, :p), Tuple{AT, AT}}, _ps::Union{NamedTuple, NeuralNetworkParameters}) where {AT<:AbstractArray} +function symplectic_linear_map(z::NamedTuple{(:q, :p), Tuple{AT, AT}}, _ps::Union{NamedTuple, NetworkParameters}) where {AT<:AbstractArray} (q = z.q + _custom_mul(_ps.L1.A, z.p), p = z.p) end S = rand(SymmetricMatrix, 2) -ps = NeuralNetworkParameters((L1 = (A = S, ), )) +ps = NetworkParameters((L1 = (A = S, ), )) t = (q = rand(2, 2), p = rand(2, 2)) ∇₁ = gradient(_ps -> norm(symplectic_attention(t, _ps)), ps)[1] # this doesn't work diff --git a/scripts/test_double_multiplication_derivative.jl b/scripts/test_double_multiplication_derivative.jl index bf948defc..d7c909cce 100644 --- a/scripts/test_double_multiplication_derivative.jl +++ b/scripts/test_double_multiplication_derivative.jl @@ -3,16 +3,16 @@ using GeometricMachineLearning: _custom_mul, _custom_transpose using LinearAlgebra: norm using Zygote: gradient -function single_multiplication(a::AT, _ps::Union{NamedTuple, NeuralNetworkParameters}) where {AT<:AbstractArray} +function single_multiplication(a::AT, _ps::Union{NamedTuple, NetworkParameters}) where {AT<:AbstractArray} _custom_mul(a, _ps.L1.A) end -function double_multiplication(a::AT, _ps::Union{NamedTuple, NeuralNetworkParameters}) where {AT<:AbstractArray} +function double_multiplication(a::AT, _ps::Union{NamedTuple, NetworkParameters}) where {AT<:AbstractArray} _custom_mul(_ps.L1.A, _custom_mul(a, _ps.L1.A)) end S = rand(SymmetricMatrix, 4) -ps = NeuralNetworkParameters((L1 = (A = S, ), )) +ps = NetworkParameters((L1 = (A = S, ), )) t = rand(4, 4) ∇₁ = gradient(_ps -> norm(double_multiplication(t, _ps)), ps)[1] # this doesn't work diff --git a/src/GeometricMachineLearning.jl b/src/GeometricMachineLearning.jl index 8ceaac034..5311be0c9 100644 --- a/src/GeometricMachineLearning.jl +++ b/src/GeometricMachineLearning.jl @@ -85,7 +85,9 @@ import AbstractNeuralNetworks: GlorotUniform import AbstractNeuralNetworks: params, architecture, model, dim import AbstractNeuralNetworks: AbstractPullback, NetworkLoss, _compute_loss import AbstractNeuralNetworks: networkbackend -import AbstractNeuralNetworks: save, load +# `save` and `load` are `NeuralNetworkParameters`' generics; `AbstractNeuralNetworks` 0.7 only +# re-binds them. Reach for them where they are defined. +import NeuralNetworkParameters: save, load # export params, architetcure, model export dim import NNlib: σ, sigmoid, softmax diff --git a/test/hdf5_support.jl b/test/hdf5_support.jl index ad0f17479..e6b870f44 100644 --- a/test/hdf5_support.jl +++ b/test/hdf5_support.jl @@ -4,7 +4,6 @@ using HDF5 using LinearAlgebra: qr import Random import AbstractNeuralNetworks: params, changebackend -import NeuralNetworkParameters: NetworkParameters Random.seed!(42) @@ -167,6 +166,40 @@ end end end +# --------------------------------------------------------------------------- +# Loading against a prototype — the form that consults no registry +# --------------------------------------------------------------------------- + +# `load(NeuralNetwork, …, prototype)` rebuilds each structured leaf with `rebuild(prototype_leaf, +# storage)` instead of looking its stored type name up in `NeuralNetworkParameters`' registry. It is +# the path that works for a type nobody registered, so it needs a test of its own rather than riding +# on the roundtrips above, which all go through the registry. +@testset "save/load roundtrip: against a prototype parameter set" begin + arch = SymplecticAutoencoder(10, 4) + nn = NeuralNetwork(arch) + x = rand(10) + y_before = nn(x) + + # a second network of the same architecture: same shapes, different numbers + prototype = params(NeuralNetwork(arch)) + + mktempdir() do dir + path = joinpath(dir, "sae_prototype.h5") + save(path, nn) + nn2 = load(NeuralNetwork, path, arch, prototype) + + @test _ps_eq(params(nn), params(nn2)) + @test nn2(x) ≈ y_before + @test params(nn2)[5].weight isa StiefelManifold + + # and on an already-open store + nn3 = HDF5.h5open(path, "r") do h5 + load(NeuralNetwork, h5, arch, prototype) + end + @test nn3(x) ≈ y_before + end +end + # --------------------------------------------------------------------------- # Files written before the traversal moved out of this package # --------------------------------------------------------------------------- @@ -175,23 +208,31 @@ end # fields under their own names and recording no key order. `NeuralNetworkParameters` recognises the # tag and rebuilds through the registry `GeometricOptimizers` fills, so those files still load — # which is the whole reason the duplicated reader here could be deleted rather than kept alongside. +# `SymmetricMatrix` and `StiefelManifold` are the two shapes the old writer produced, and +# `GeometricOptimizers` normalises them through different helpers — `S`/`n` for a storage matrix, +# a bare `A` for a manifold element — so both legs need reading back. @testset "a file in the old gml_type layout still loads" begin - arch = LASympNet(4) # LinearLayer → SymmetricMatrix, the tagged case - nn = NeuralNetwork(arch) - ps = params(nn) - x = rand(4) - y = nn(x) - - mktempdir() do dir - path = joinpath(dir, "legacy.h5") - HDF5.h5open(path, "w") do h5 - _write_legacy(h5, params(ps), "/") + for (name, arch, dimin) in (("LASympNet (SymmetricMatrix)", LASympNet(4), 4), + ("SymplecticAutoencoder (StiefelManifold)", + SymplecticAutoencoder(10, 4), 10)) + @testset "$name" begin + nn = NeuralNetwork(arch) + ps = params(nn) + x = rand(dimin) + y = nn(x) + + mktempdir() do dir + path = joinpath(dir, "legacy.h5") + HDF5.h5open(path, "w") do h5 + _write_legacy(h5, params(ps), "/") + end + nn2 = load(NeuralNetwork, path, arch) + + @test keys(params(nn2)) == keys(ps) + @test _ps_eq(ps, params(nn2)) + @test nn2(x) ≈ y + end end - nn2 = load(NeuralNetwork, path, arch) - - @test keys(params(nn2)) == keys(ps) - @test _ps_eq(ps, params(nn2)) - @test nn2(x) ≈ y end end From c15cdb7c7e2cf2b30283eb073d59cb44335329e5 Mon Sep 17 00:00:00 2001 From: Michael Kraus Date: Sun, 23 Aug 2026 17:15:52 +0900 Subject: [PATCH 5/5] Correct the merge order: only SymbolicNeuralNetworks is outstanding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `AbstractNeuralNetworks` 0.7.0, `NeuralNetworkParameters` 0.1.1 and `GeometricOptimizers` 0.4.1 were all registered in General on 2026-08-23, at 04:00, 04:12 and 06:10 UTC. The note here still listed the first two as outstanding, because the Julia package server's snapshot of General lags the registry by hours and both CI and the local depot were resolving against a copy that predated them — the CI run at 05:37 still reported `AbstractNeuralNetworks … possible versions are: 0.1.0 - 0.6.4`. `SymbolicNeuralNetworks` 0.6.0 is the one release left. Against a current registry everything else resolves from it directly, and the suite passes with only that package `dev`'d. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fee17f450..bf48a4243 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -160,17 +160,13 @@ breaking release). 0.6"`, so leaving the bound would have made this package's `[compat]` unsatisfiable rather than merely unresolved. 0.6 is the release that follows the container out to `NeuralNetworkParameters`. - > **Merge order.** Three of these four bounds point at releases that do not exist in the General - > registry yet, so this cannot be merged before them, in this order: + > **Merge order.** `AbstractNeuralNetworks` 0.7.0, `GeometricOptimizers` 0.4.1 and + > `NeuralNetworkParameters` 0.1.1 are all in the General registry as of 2026-08-23. + > `SymbolicNeuralNetworks` 0.6.0 is not: its `abstractneuralnetworks-0.7` branch still says + > `0.5.0`, and still does `using AbstractNeuralNetworks: QPTOAT`, which 0.7 replaced with + > `ArrayOrNamedTuple`, so it does not load as it stands. That is the one release this waits on. > - > 1. `AbstractNeuralNetworks` 0.7.0 — tagged, awaiting registration. - > 2. `GeometricOptimizers` 0.4.1 — the `NeuralNetworkParameters` extension is on `main`; needs a - > version bump, a tag and registration. - > 3. `SymbolicNeuralNetworks` 0.6.0 — the `abstractneuralnetworks-0.7` branch still says `0.5.0`, - > and still imports `AbstractNeuralNetworks.QPTOAT`, which 0.7 replaced with - > `ArrayOrNamedTuple`; it does not load as it stands. - > - > Until then CI here fails at `Pkg.instantiate`. That is expected, not a regression. + > Until it lands, CI here fails at `Pkg.instantiate`. That is expected, not a regression. - **`Zygote = "0.7"`** (was `"0.6"`). 0.7 replaced the eager unthunking in `wrap_chainrules_output` with `unthunk_tangent` at the `gradient`/`pullback` boundaries, which is what let thunks reach