Fix distributed CSV comparison test

.github/workflows/documentation.yaml

@@ -0,0 +1,27 @@
+name: documentation
+
+on: [push, pull_request, workflow_dispatch]
+
+permissions:
+  contents: write
+
+jobs:
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+      - name: Install dependencies
+        run: |
+          pip install sphinx sphinx_rtd_theme myst_parser
+      - name: Sphinx build
+        run: |
+          sphinx-build doc _build
+      - name: Deploy to GitHub Pages
+        uses: peaceiris/actions-gh-pages@v3
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+        with:
+          publish_branch: gh-pages
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: _build/
+          force_orphan: true

CMakeLists.txt

@@ -23,7 +23,7 @@ cmake_policy(SET CMP0144 NEW) # ``PACKAGE_NAME''_ROOT
 
 project(HARD LANGUAGES C CXX)
 set(CXX_STANDARD_REQUIRED ON)
-set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_EXTENSIONS OFF)
 
 set(default_build_type "Debug")
@@ -102,6 +102,17 @@ option(ENABLE_RADIATION
   "Enable radiation in evolution" ON)
 mark_as_advanced(ENABLE_RADIATION)
 
+#------------------------------------------------------------------------------#
+# Enable flecsolve
+#------------------------------------------------------------------------------#
+
+option(ENABLE_FLECSOLVE
+       "Enable flecsolve for KSP solvers in MG" ON)
+# mark_as_advanced(ENABLE_FLECSOLVE)
+if(ENABLE_FLECSOLVE)
+       find_package(flecsolve REQUIRED)
+endif()
+
 #------------------------------------------------------------------------------#
 # Formatting
 #------------------------------------------------------------------------------#
@@ -142,20 +153,11 @@ if(ENABLE_DOCUMENTATION)
   )
 
   #----------------------------------------------------------------------------#
-  # Sphinx & Doxygen
+  # Sphinx
   #----------------------------------------------------------------------------#
 
-  include(FleCSI/doxygen)
   include(FleCSI/sphinx)
 
-  # Avoid race condition by proactively creating output directory
-  file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/.doxygen/xml)
-  flecsi_add_doxygen_target(hard
-    CONFIGS
-      ${CMAKE_SOURCE_DIR}/doc/doxygen/hard.conf.in
-      ${CMAKE_SOURCE_DIR}/doc/doxygen/conf.in
-  )
-
   file(COPY ${CMAKE_SOURCE_DIR}/doc/sphinx
     DESTINATION ${CMAKE_BINARY_DIR}
   )
@@ -226,3 +228,66 @@ add_subdirectory(spec)
 #------------------------------------------------------------------------------#
 
 add_subdirectory(app)
+
+#------------------------------------------------------------------------------#
+# Add apps.
+#------------------------------------------------------------------------------#
+
+option(ENABLE_VERIFICATION "Enable exactpack verification" OFF)
+mark_as_advanced(ENABLE_FORMAT)
+
+if(ENABLE_VERIFICATION OR ENABLE_UNIT_TESTS)
+
+  find_package(Python3 REQUIRED COMPONENTS Interpreter)
+  # We need the exactpack package
+  execute_process(
+    COMMAND ${Python3_EXECUTABLE} -c "import exactpack"
+    RESULT_VARIABLE EXACTPACK_FOUND
+    OUTPUT_QUIET
+    ERROR_QUIET
+  )
+  if(NOT EXACTPACK_FOUND EQUAL 0)
+    message(FATAL_ERROR "ExactPack Python module not found. Please install it (e.g., with 'pip install exactpack').")
+  endif()
+
+  execute_process(
+    COMMAND ${Python3_EXECUTABLE} -c "import matplotlib"
+    RESULT_VARIABLE MATPLOTLIB_FOUND
+    OUTPUT_QUIET
+    ERROR_QUIET
+  )
+  if(NOT EXACTPACK_FOUND EQUAL 0)
+    message(FATAL_ERROR "Matplotlib Python module not found. Please install it (e.g., with 'pip install matplotlib').")
+  endif()
+
+  execute_process(
+    COMMAND ${Python3_EXECUTABLE} -c "import pyyaml"
+    RESULT_VARIABLE PYYAML_FOUND
+    OUTPUT_QUIET
+    ERROR_QUIET
+  )
+  if(NOT EXACTPACK_FOUND EQUAL 0)
+    message(FATAL_ERROR "Yaml Python module not found. Please install it (e.g., with 'pip install pyyaml').")
+  endif()
+
+
+
+  set(CHECK_SCRIPT ${CMAKE_SOURCE_DIR}/tools/exactpack_verify.py)
+  set(CONVERGENCE_SCRIPT ${CMAKE_SOURCE_DIR}/tools/convergence_verify.py)
+
+  if(CONFIG STREQUAL "")
+    set(CONFIG sod.yaml)
+  endif()
+
+  add_custom_target(verify
+    COMMAND ${Python3_EXECUTABLE} ${CHECK_SCRIPT} \$\(CONFIG\) "--plot"
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    VERBATIM
+  )
+
+  add_custom_target(convergence
+    COMMAND ${Python3_EXECUTABLE} ${CONVERGENCE_SCRIPT} \$\(CONFIG\)
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    VERBATIM
+  )
+endif()

README.md

@@ -1,21 +1,22 @@
 # HARD: Hydrodynamics And Radiative Diffusion
 
-HARD is a radiation-hydrodynamics solver suite for the study of
-astrophysical phenomena.
+HARD is a radiation-hydrodynamics solver suite for the study of astrophysical phenomena.
 
-HARD is based on the FleCSI framework and implemented on top of FleCSI-SP (FleCSI Specialization project). 
+HARD is based on the FleCSI framework and implemented on top of FleCSI-SP (FleCSI Specialization project).
 
 # Copyright
 © 2024. Triad National Security, LLC. All rights reserved.
 This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. Department of Energy/National Nuclear Security Administration. All rights in the program are reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear Security Administration. The Government is granted for itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare. derivative works, distribute copies to the public, perform publicly and display publicly, and to permit others to do so (Copyright request O4795).
 
-# Spack
+# Spack, Darwin build
 
-The easiest way to build HARD is to use *spack*:
+The easiest way to build HARD is to use *spack*.
 
-Clone the spack repo and initialize:
+Clone the spack repo, pick the right version, and initialize:
 ```
 $ git clone git@github.com:spack/spack.git $HOME/.spack
+$ cd $HOME/.spack
+$ git checkout v0.23.1
 $ source $HOME/.spack/share/spack/setup-env.sh
 ```
 
@@ -37,33 +38,24 @@ Next, create a spack *environment* for HARD and activate it:
 $ spack env create hard
 $ spacktivate hard
 ```
-
-Once you are in the *hard* environment, you need to edit the
-environment config and replace the contents with the following, where
-*PATH-TO-HARD-CLONE* is the path to your clone of HARD:
-```
-spack:
-  specs:
-  - flecsi@2.3+flog+graphviz++hdf5++kokkos backend=mpi
-  - graphviz+libgd
-  - yaml-cpp
-  view: true
-  concretizer:
-    unify: true
-  repos:
-  - /PATH-TO-HARD-CLONE/spack-repo
+Once you are in the *hard* environment, you can specify the repository:
 ```
-
-To edit the environment config, execute the following, which will open
-an editor session (you can cut and paste the replacement contents
-from above):
+$ spack repo add /PATH-TO-HARD-CLONE/spack-repo
 ```
-$ spack config edit
+You can find the compiler we loaded earlier using:
+```
+$ spack compiler find
 ```
-**Note: You must be in an activated spack environment for this to
-work.**
 
-Save and quit your editor session. The next step is to *concretize* the
+You can see the different options available for *hard* by using:
+```
+$ spack info hard
+```
+And add *hard* to the environment:
+```
+$ spack add hard
+```
+The next step is to *concretize* the
 new environment. Concretization solves an optimization problem that
 takes all of the package dependencies in an environment and reconciles
 version and subdependency compatibility:
@@ -78,16 +70,16 @@ up-to-date config information.
 To finish up the spack portion of the build, we just need to *install*
 the packages:
 ```
-$ spack install
+$ spack install --only dependencies
 ```
-This will fetch all of the HARD dependencies and build them in your
+This will fetch all of the *hard* dependencies and build them in your
 local spack tree (under *$HOME/.spack/opt* to be precise). This step
 will take some time.
 
 # Build
 
 Now that we have setup our spack environment, the rest of the build is
-really easy. First, change directory into your HARD clone and create
+really easy. First, change directory into your *hard* clone and create
 a build directory:
 ```
 $ cd PATH-TO-HARD-CLONE

app/CMakeLists.txt

@@ -28,6 +28,14 @@ target_link_libraries(hard
   singularity-eos::singularity-eos
 )
 
+if(ENABLE_FLECSOLVE)
+  target_compile_definitions(hard
+    PRIVATE
+      USE_FLECSOLVE=1)
+
+  target_link_libraries(hard flecsolve::flecsolve)
+endif()
+
 
 if(ENABLE_CATALYST)
   target_compile_definitions(hard
@@ -60,17 +68,58 @@ if(ENABLE_UNIT_TESTS AND NOT ENABLE_CATALYST)
     ${CMAKE_BINARY_DIR}/tests/sod_3d
   )
 
-  add_test(NAME hard_rk_3d
-    COMMAND hard ${CMAKE_SOURCE_DIR}/configs/rk.yaml
-    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/tests/rk_3d)
-  add_test(NAME hard_sod_3d
-    COMMAND hard ${CMAKE_SOURCE_DIR}/configs/sod.yaml
-    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/tests/sod_3d)
+if(NOT ENABLE_RADIATION)
+  # Accuracy tests 
+  add_test(NAME accuracy_rk_1d
+    COMMAND ${CMAKE_SOURCE_DIR}/app/tests/test_verify.sh
+    ${MPIEXEC_EXECUTABLE} ${CMAKE_BINARY_DIR}/app/hard
+    ${CMAKE_SOURCE_DIR}/configs/ci_configs/rk.yaml -d1
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+  )
+  add_test(NAME accuracy_sod_1d
+    COMMAND ${CMAKE_SOURCE_DIR}/app/tests/test_verify.sh
+    ${MPIEXEC_EXECUTABLE} ${CMAKE_BINARY_DIR}/app/hard
+    ${CMAKE_SOURCE_DIR}/configs/ci_configs/sod.yaml -d1
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+  )
+  add_test(NAME accuracy_leblanc_1d
+    COMMAND ${CMAKE_SOURCE_DIR}/app/tests/test_verify.sh
+    ${MPIEXEC_EXECUTABLE} ${CMAKE_BINARY_DIR}/app/hard
+    ${CMAKE_SOURCE_DIR}/configs/ci_configs/leblanc.yaml -d1
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+  )
+  # add_test(NAME accuracy_sedov_1d
+  #   COMMAND ${CMAKE_SOURCE_DIR}/app/tests/test_verify.sh
+  #   ${MPIEXEC_EXECUTABLE} ${CMAKE_BINARY_DIR}/app/hard
+  #   ${CMAKE_SOURCE_DIR}/configs/ci_configs/sedov.yaml -d1
+  #   WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+  # )
+  add_test(NAME accuracy_acoustic_1d
+    COMMAND ${CMAKE_SOURCE_DIR}/app/tests/test_verify.sh
+    ${MPIEXEC_EXECUTABLE} ${CMAKE_BINARY_DIR}/app/hard
+    ${CMAKE_SOURCE_DIR}/configs/ci_configs/acoustic-wave.yaml -d1
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+  )
+endif()
 
+if(NOT ENABLE_RADIATION)
+  # Distributed tests 
   add_test(NAME hard-distributed
-    COMMAND ${CMAKE_SOURCE_DIR}/app/tests/test_distributed.sh 
-      ${MPIEXEC_EXECUTABLE} ${CMAKE_BINARY_DIR}/app/hard 
-      ${CMAKE_SOURCE_DIR}/configs/distr-rk.yaml -d1)
+    COMMAND ${CMAKE_SOURCE_DIR}/app/tests/test_distributed.sh
+    ${MPIEXEC_EXECUTABLE} ${CMAKE_BINARY_DIR}/app/hard
+    ${CMAKE_SOURCE_DIR}/configs/ci_configs/distr-rk.yaml -d1)
+endif()
+
+if(NOT ENABLE_RADIATION)
+  # Convergence tests
+  add_test(NAME convergence_testing
+    COMMAND ${CMAKE_SOURCE_DIR}/app/tests/test_convergence.sh
+    ${MPIEXEC_EXECUTABLE} ${CMAKE_BINARY_DIR}/app/hard
+    6 10
+    ${CMAKE_SOURCE_DIR}/configs/ci_configs/acoustic-wave_convergence.yaml -d1
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+  )
+endif()
 
 endif()