diff --git a/.github/workflows/documentation.yaml b/.github/workflows/documentation.yaml new file mode 100644 index 0000000..54f003c --- /dev/null +++ b/.github/workflows/documentation.yaml @@ -0,0 +1,27 @@ +name: documentation + +on: [push, pull_request, workflow_dispatch] + +permissions: + contents: write + +jobs: + docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + - name: Install dependencies + run: | + pip install sphinx sphinx_rtd_theme myst_parser + - name: Sphinx build + run: | + sphinx-build doc _build + - name: Deploy to GitHub Pages + uses: peaceiris/actions-gh-pages@v3 + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + with: + publish_branch: gh-pages + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: _build/ + force_orphan: true diff --git a/CMakeLists.txt b/CMakeLists.txt index 800e779..07b5194 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,7 +23,7 @@ cmake_policy(SET CMP0144 NEW) # ``PACKAGE_NAME''_ROOT project(HARD LANGUAGES C CXX) set(CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_EXTENSIONS OFF) set(default_build_type "Debug") @@ -102,6 +102,17 @@ option(ENABLE_RADIATION "Enable radiation in evolution" ON) mark_as_advanced(ENABLE_RADIATION) +#------------------------------------------------------------------------------# +# Enable flecsolve +#------------------------------------------------------------------------------# + +option(ENABLE_FLECSOLVE + "Enable flecsolve for KSP solvers in MG" ON) +# mark_as_advanced(ENABLE_FLECSOLVE) +if(ENABLE_FLECSOLVE) + find_package(flecsolve REQUIRED) +endif() + #------------------------------------------------------------------------------# # Formatting #------------------------------------------------------------------------------# @@ -142,20 +153,11 @@ if(ENABLE_DOCUMENTATION) ) #----------------------------------------------------------------------------# - # Sphinx & Doxygen + # Sphinx #----------------------------------------------------------------------------# - include(FleCSI/doxygen) include(FleCSI/sphinx) - # Avoid race condition by proactively creating output directory - file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/.doxygen/xml) - flecsi_add_doxygen_target(hard - CONFIGS - ${CMAKE_SOURCE_DIR}/doc/doxygen/hard.conf.in - ${CMAKE_SOURCE_DIR}/doc/doxygen/conf.in - ) - file(COPY ${CMAKE_SOURCE_DIR}/doc/sphinx DESTINATION ${CMAKE_BINARY_DIR} ) @@ -226,3 +228,66 @@ add_subdirectory(spec) #------------------------------------------------------------------------------# add_subdirectory(app) + +#------------------------------------------------------------------------------# +# Add apps. +#------------------------------------------------------------------------------# + +option(ENABLE_VERIFICATION "Enable exactpack verification" OFF) +mark_as_advanced(ENABLE_FORMAT) + +if(ENABLE_VERIFICATION OR ENABLE_UNIT_TESTS) + + find_package(Python3 REQUIRED COMPONENTS Interpreter) + # We need the exactpack package + execute_process( + COMMAND ${Python3_EXECUTABLE} -c "import exactpack" + RESULT_VARIABLE EXACTPACK_FOUND + OUTPUT_QUIET + ERROR_QUIET + ) + if(NOT EXACTPACK_FOUND EQUAL 0) + message(FATAL_ERROR "ExactPack Python module not found. Please install it (e.g., with 'pip install exactpack').") + endif() + + execute_process( + COMMAND ${Python3_EXECUTABLE} -c "import matplotlib" + RESULT_VARIABLE MATPLOTLIB_FOUND + OUTPUT_QUIET + ERROR_QUIET + ) + if(NOT EXACTPACK_FOUND EQUAL 0) + message(FATAL_ERROR "Matplotlib Python module not found. Please install it (e.g., with 'pip install matplotlib').") + endif() + + execute_process( + COMMAND ${Python3_EXECUTABLE} -c "import pyyaml" + RESULT_VARIABLE PYYAML_FOUND + OUTPUT_QUIET + ERROR_QUIET + ) + if(NOT EXACTPACK_FOUND EQUAL 0) + message(FATAL_ERROR "Yaml Python module not found. Please install it (e.g., with 'pip install pyyaml').") + endif() + + + + set(CHECK_SCRIPT ${CMAKE_SOURCE_DIR}/tools/exactpack_verify.py) + set(CONVERGENCE_SCRIPT ${CMAKE_SOURCE_DIR}/tools/convergence_verify.py) + + if(CONFIG STREQUAL "") + set(CONFIG sod.yaml) + endif() + + add_custom_target(verify + COMMAND ${Python3_EXECUTABLE} ${CHECK_SCRIPT} \$\(CONFIG\) "--plot" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM + ) + + add_custom_target(convergence + COMMAND ${Python3_EXECUTABLE} ${CONVERGENCE_SCRIPT} \$\(CONFIG\) + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM + ) +endif() diff --git a/README.md b/README.md index e11b490..3afd2dc 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,22 @@ # HARD: Hydrodynamics And Radiative Diffusion -HARD is a radiation-hydrodynamics solver suite for the study of -astrophysical phenomena. +HARD is a radiation-hydrodynamics solver suite for the study of astrophysical phenomena. -HARD is based on the FleCSI framework and implemented on top of FleCSI-SP (FleCSI Specialization project). +HARD is based on the FleCSI framework and implemented on top of FleCSI-SP (FleCSI Specialization project). # Copyright © 2024. Triad National Security, LLC. All rights reserved. This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. Department of Energy/National Nuclear Security Administration. All rights in the program are reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear Security Administration. The Government is granted for itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare. derivative works, distribute copies to the public, perform publicly and display publicly, and to permit others to do so (Copyright request O4795). -# Spack +# Spack, Darwin build -The easiest way to build HARD is to use *spack*: +The easiest way to build HARD is to use *spack*. -Clone the spack repo and initialize: +Clone the spack repo, pick the right version, and initialize: ``` $ git clone git@github.com:spack/spack.git $HOME/.spack +$ cd $HOME/.spack +$ git checkout v0.23.1 $ source $HOME/.spack/share/spack/setup-env.sh ``` @@ -37,33 +38,24 @@ Next, create a spack *environment* for HARD and activate it: $ spack env create hard $ spacktivate hard ``` - -Once you are in the *hard* environment, you need to edit the -environment config and replace the contents with the following, where -*PATH-TO-HARD-CLONE* is the path to your clone of HARD: -``` -spack: - specs: - - flecsi@2.3+flog+graphviz++hdf5++kokkos backend=mpi - - graphviz+libgd - - yaml-cpp - view: true - concretizer: - unify: true - repos: - - /PATH-TO-HARD-CLONE/spack-repo +Once you are in the *hard* environment, you can specify the repository: ``` - -To edit the environment config, execute the following, which will open -an editor session (you can cut and paste the replacement contents -from above): +$ spack repo add /PATH-TO-HARD-CLONE/spack-repo ``` -$ spack config edit +You can find the compiler we loaded earlier using: +``` +$ spack compiler find ``` -**Note: You must be in an activated spack environment for this to -work.** -Save and quit your editor session. The next step is to *concretize* the +You can see the different options available for *hard* by using: +``` +$ spack info hard +``` +And add *hard* to the environment: +``` +$ spack add hard +``` +The next step is to *concretize* the new environment. Concretization solves an optimization problem that takes all of the package dependencies in an environment and reconciles version and subdependency compatibility: @@ -78,16 +70,16 @@ up-to-date config information. To finish up the spack portion of the build, we just need to *install* the packages: ``` -$ spack install +$ spack install --only dependencies ``` -This will fetch all of the HARD dependencies and build them in your +This will fetch all of the *hard* dependencies and build them in your local spack tree (under *$HOME/.spack/opt* to be precise). This step will take some time. # Build Now that we have setup our spack environment, the rest of the build is -really easy. First, change directory into your HARD clone and create +really easy. First, change directory into your *hard* clone and create a build directory: ``` $ cd PATH-TO-HARD-CLONE diff --git a/app/CMakeLists.txt b/app/CMakeLists.txt index 9ddc5b0..57665dd 100644 --- a/app/CMakeLists.txt +++ b/app/CMakeLists.txt @@ -28,6 +28,14 @@ target_link_libraries(hard singularity-eos::singularity-eos ) +if(ENABLE_FLECSOLVE) + target_compile_definitions(hard + PRIVATE + USE_FLECSOLVE=1) + + target_link_libraries(hard flecsolve::flecsolve) +endif() + if(ENABLE_CATALYST) target_compile_definitions(hard @@ -60,17 +68,58 @@ if(ENABLE_UNIT_TESTS AND NOT ENABLE_CATALYST) ${CMAKE_BINARY_DIR}/tests/sod_3d ) - add_test(NAME hard_rk_3d - COMMAND hard ${CMAKE_SOURCE_DIR}/configs/rk.yaml - WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/tests/rk_3d) - add_test(NAME hard_sod_3d - COMMAND hard ${CMAKE_SOURCE_DIR}/configs/sod.yaml - WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/tests/sod_3d) +if(NOT ENABLE_RADIATION) + # Accuracy tests + add_test(NAME accuracy_rk_1d + COMMAND ${CMAKE_SOURCE_DIR}/app/tests/test_verify.sh + ${MPIEXEC_EXECUTABLE} ${CMAKE_BINARY_DIR}/app/hard + ${CMAKE_SOURCE_DIR}/configs/ci_configs/rk.yaml -d1 + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + ) + add_test(NAME accuracy_sod_1d + COMMAND ${CMAKE_SOURCE_DIR}/app/tests/test_verify.sh + ${MPIEXEC_EXECUTABLE} ${CMAKE_BINARY_DIR}/app/hard + ${CMAKE_SOURCE_DIR}/configs/ci_configs/sod.yaml -d1 + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + ) + add_test(NAME accuracy_leblanc_1d + COMMAND ${CMAKE_SOURCE_DIR}/app/tests/test_verify.sh + ${MPIEXEC_EXECUTABLE} ${CMAKE_BINARY_DIR}/app/hard + ${CMAKE_SOURCE_DIR}/configs/ci_configs/leblanc.yaml -d1 + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + ) + # add_test(NAME accuracy_sedov_1d + # COMMAND ${CMAKE_SOURCE_DIR}/app/tests/test_verify.sh + # ${MPIEXEC_EXECUTABLE} ${CMAKE_BINARY_DIR}/app/hard + # ${CMAKE_SOURCE_DIR}/configs/ci_configs/sedov.yaml -d1 + # WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + # ) + add_test(NAME accuracy_acoustic_1d + COMMAND ${CMAKE_SOURCE_DIR}/app/tests/test_verify.sh + ${MPIEXEC_EXECUTABLE} ${CMAKE_BINARY_DIR}/app/hard + ${CMAKE_SOURCE_DIR}/configs/ci_configs/acoustic-wave.yaml -d1 + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + ) +endif() +if(NOT ENABLE_RADIATION) + # Distributed tests add_test(NAME hard-distributed - COMMAND ${CMAKE_SOURCE_DIR}/app/tests/test_distributed.sh - ${MPIEXEC_EXECUTABLE} ${CMAKE_BINARY_DIR}/app/hard - ${CMAKE_SOURCE_DIR}/configs/distr-rk.yaml -d1) + COMMAND ${CMAKE_SOURCE_DIR}/app/tests/test_distributed.sh + ${MPIEXEC_EXECUTABLE} ${CMAKE_BINARY_DIR}/app/hard + ${CMAKE_SOURCE_DIR}/configs/ci_configs/distr-rk.yaml -d1) +endif() + +if(NOT ENABLE_RADIATION) + # Convergence tests + add_test(NAME convergence_testing + COMMAND ${CMAKE_SOURCE_DIR}/app/tests/test_convergence.sh + ${MPIEXEC_EXECUTABLE} ${CMAKE_BINARY_DIR}/app/hard + 6 10 + ${CMAKE_SOURCE_DIR}/configs/ci_configs/acoustic-wave_convergence.yaml -d1 + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + ) +endif() endif() diff --git a/app/actions.hh b/app/actions.hh index 16c328b..9169cca 100644 --- a/app/actions.hh +++ b/app/actions.hh @@ -5,7 +5,8 @@ #include "numerical_algorithms/time_stepper.hh" #include "rad.hh" #include "state.hh" -#include "tasks/boundary.hh" +#include "tasks/boundaries/boundary.hh" +#include "tasks/external_source.hh" #include "tasks/hydro/compute_interface_fluxes.hh" #include "tasks/hydro/cons2prim.hh" #include "tasks/hydro/maxcharspeed.hh" @@ -17,12 +18,15 @@ #include #include +#include "linsolve.hh" + namespace hard::actions { template void initialize_time_derivative(control_policy & cp) { auto & s = cp.state(); + flecsi::scheduler & sc = cp.scheduler(); #ifdef HARD_ENABLE_LEGION_TRACING // Legion tracing: Skip first iteration @@ -32,29 +36,38 @@ initialize_time_derivative(control_policy & cp) { cp.guard.emplace(cp.tracing); #endif - flecsi::execute( - s.dt(s.gt), s.dt_weighted(s.gt), hard::time_stepper::time_stepper_gamma); + sc.execute(flecsi::exec::on, + s.dt(*s.gt), + s.dt_weighted(*s.gt), + hard::time_stepper::time_stepper_gamma); // Set all dU_dt temporaries to zero before adding time derivative terms - flecsi::execute, flecsi::default_accelerator>(s.m, - s.dt_mass_density(s.m), - s.dt_momentum_density(s.m), - s.dt_total_energy_density(s.m), - s.dt_radiation_energy_density(s.m)); + sc.execute>(flecsi::exec::on, + *s.m, + s.dt_mass_density(*s.m), + s.dt_momentum_density(*s.m), + s.dt_total_energy_density(*s.m), + s.dt_radiation_energy_density(*s.m)); + sc.execute>(flecsi::exec::on, + *s.m, + s.dt_mass_density_2(*s.m), + s.dt_momentum_density_2(*s.m), + s.dt_total_energy_density_2(*s.m), + s.dt_radiation_energy_density_2(*s.m)); // Store the current state of evolved variables (U^n) before performing a time // step - flecsi::execute, flecsi::default_accelerator>( - s.m, - s.mass_density(s.m), - s.momentum_density(s.m), - s.total_energy_density(s.m), - s.radiation_energy_density(s.m), + sc.execute>(flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.momentum_density(*s.m), + s.total_energy_density(*s.m), + s.radiation_energy_density(*s.m), // - s.mass_density_n(s.m), - s.momentum_density_n(s.m), - s.total_energy_density_n(s.m), - s.radiation_energy_density_n(s.m)); + s.mass_density_n(*s.m), + s.momentum_density_n(*s.m), + s.total_energy_density_n(*s.m), + s.radiation_energy_density_n(*s.m)); } // -------------------------------------------------------------------- @@ -63,75 +76,213 @@ initialize_time_derivative(control_policy & cp) { template void -RK_advance(control_policy & cp) { +RK_advance(control_policy & cp, time_stepper::rk_stage Stage) { auto & s = cp.state(); + flecsi::scheduler & sc = cp.scheduler(); #ifdef ENABLE_RADIATION - flecsi::execute, flecsi::default_accelerator>( - s.m, s.radiation_energy_density(s.m), s.gradient_rad_energy(s.m)); - - flecsi::execute, flecsi::default_accelerator>(s.m, - s.mass_density(s.m), - s.radiation_energy_density(s.m), - s.gradient_rad_energy(s.m), - s.magnitude_gradient_rad_energy(s.m), - s.R_value(s.m), - s.lambda_bridge(s.m), - kappa(s.gt)); - - flecsi::execute, flecsi::default_accelerator>(s.m, - s.radiation_pressure_tensor(s.m), - s.radiation_energy_density(s.m), - s.gradient_rad_energy(s.m), - s.magnitude_gradient_rad_energy(s.m), - s.lambda_bridge(s.m), - s.R_value(s.m)); - - flecsi::execute, flecsi::default_accelerator>(s.m, - s.lambda_bridge(s.m), - s.gradient_rad_energy(s.m), - s.radiation_force(s.m)); - - flecsi::execute, flecsi::default_accelerator>( - s.m, s.velocity_gradient(s.m), s.velocity(s.m)); - - flecsi::execute, - flecsi::default_accelerator>(s.m, - s.velocity(s.m), - s.radiation_force(s.m), - s.radiation_pressure_tensor(s.m), - s.velocity_gradient(s.m), - // - s.dt_momentum_density(s.m), - s.dt_total_energy_density(s.m), - s.dt_radiation_energy_density(s.m)); + sc.execute>(flecsi::exec::on, + *s.m, + s.radiation_energy_density(*s.m), + s.gradient_rad_energy(*s.m)); + + // Standard (Constant) FLD formulation + + sc.execute>(flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.radiation_energy_density(*s.m), + s.gradient_rad_energy(*s.m), + s.magnitude_gradient_rad_energy(*s.m), + s.R_value(*s.m), + s.lambda_bridge(*s.m), + s.kappa(*s.gt), + s.limiter_id(*s.gt)); + + sc.execute>(flecsi::exec::on, + *s.m, + s.lambda_bridge(*s.m), + s.eddington_factor(*s.m), + s.limiter_id(*s.gt), + s.closure_id(*s.gt)); + + sc.execute>(flecsi::exec::on, + *s.m, + s.radiation_pressure_tensor(*s.m), + s.radiation_energy_density(*s.m), + s.gradient_rad_energy(*s.m), + s.magnitude_gradient_rad_energy(*s.m), + s.eddington_factor(*s.m)); + + sc.execute>(flecsi::exec::on, + *s.m, + s.lambda_bridge(*s.m), + s.gradient_rad_energy(*s.m), + s.radiation_force(*s.m)); + + sc.execute>( + flecsi::exec::on, *s.m, s.velocity_gradient(*s.m), s.velocity(*s.m)); + + if(Stage == time_stepper::rk_stage::First) { + sc.execute>(flecsi::exec::on, + *s.m, + s.velocity(*s.m), + s.radiation_force(*s.m), + s.radiation_pressure_tensor(*s.m), + s.velocity_gradient(*s.m), + // + s.dt_momentum_density(*s.m), + s.dt_total_energy_density(*s.m), + s.dt_radiation_energy_density(*s.m)); + } + else if(Stage == time_stepper::rk_stage::Second) { + sc.execute>(flecsi::exec::on, + *s.m, + s.velocity(*s.m), + s.radiation_force(*s.m), + s.radiation_pressure_tensor(*s.m), + s.velocity_gradient(*s.m), + // + s.dt_momentum_density_2(*s.m), + s.dt_total_energy_density_2(*s.m), + s.dt_radiation_energy_density_2(*s.m)); + } #endif + if(Stage == time_stepper::rk_stage::First) { + // RK Stage: 1 - Explicit source term (gravity) update for RT case in hydro + // file + sc.execute>(flecsi::exec::on, + *s.m, + s.velocity(*s.m), + s.gravity_force(*s.m), + // time-derivatives + s.dt_momentum_density(*s.m), + s.dt_total_energy_density(*s.m)); + + // We need update_u here before we compute fluxes in the presence of + // hydro/rad::explictSourceUpdate with body forces. See Moens'21 Eq. 24-26 + sc.execute>(flecsi::exec::on, + s.dt(*s.gt), + *s.m, + // + s.mass_density(*s.m), + s.momentum_density(*s.m), + s.total_energy_density(*s.m), + s.radiation_energy_density(*s.m), + // + s.dt_mass_density(*s.m), + s.dt_momentum_density(*s.m), + s.dt_total_energy_density(*s.m), + s.dt_radiation_energy_density(*s.m)); + // Perform primitive recovery + sc.execute>(flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.momentum_density(*s.m), + s.total_energy_density(*s.m), + s.velocity(*s.m), + s.pressure(*s.m), + s.specific_internal_energy(*s.m), + s.sound_speed(*s.m), + s.eos); + } + // RK Stage: 2 - Explicit source term (gravity) update for RT case in hydro + // file + else if(Stage == time_stepper::rk_stage::Second) { + sc.execute>(flecsi::exec::on, + *s.m, + s.velocity(*s.m), + s.gravity_force(*s.m), + // time-derivatives + s.dt_momentum_density_2(*s.m), + s.dt_total_energy_density_2(*s.m)); + + // We need update_u here before we compute fluxes in the presence of + // hydro/rad::explictSourceUpdate with body forces. See Moens'21 Eq. 24-26 + sc.execute>(flecsi::exec::on, + s.dt(*s.gt), + *s.m, + // + s.mass_density(*s.m), + s.momentum_density(*s.m), + s.total_energy_density(*s.m), + s.radiation_energy_density(*s.m), + // + s.dt_mass_density_2(*s.m), + s.dt_momentum_density_2(*s.m), + s.dt_total_energy_density_2(*s.m), + s.dt_radiation_energy_density_2(*s.m)); + + // Perform primitive recovery + sc.execute>(flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.momentum_density(*s.m), + s.total_energy_density(*s.m), + s.velocity(*s.m), + s.pressure(*s.m), + s.specific_internal_energy(*s.m), + s.sound_speed(*s.m), + s.eos); + } + + // + // Perform reconstruction on cell faces, compute face fluxes with a Riemann + // solver, and add the summation of dF^i/dx^i into (dU_dt)_explicit + // - using limiter = spec::limiters::ppm4; + using limiter = spec::limiters::weno5z; for(std::size_t axis = 0; axis < D; axis++) { // clang-format off - flecsi::execute, flecsi::default_accelerator>(axis, - s.m, s.mass_density(s.m), s.velocity(s.m), s.pressure(s.m), - s.specific_internal_energy(s.m), - s.sound_speed(s.m), - s.radiation_energy_density(s.m), - s.rTail(s.m), s.rHead(s.m), s.uTail(s.m), s.uHead(s.m), - s.pTail(s.m), s.pHead(s.m), s.eTail(s.m), s.eHead(s.m), s.cTail(s.m), s.cHead(s.m), s.EradTail(s.m), s.EradHead(s.m), - s.ruTail(s.m), s.ruHead(s.m), s.rETail(s.m), s.rEHead(s.m)); - - flecsi::execute, flecsi::default_accelerator>(axis, s.m, - s.rTail(s.m), s.rHead(s.m), s.uTail(s.m), s.uHead(s.m), - s.pTail(s.m), s.pHead(s.m), s.eTail(s.m), s.eHead(s.m), s.cTail(s.m), s.cHead(s.m), s.EradTail(s.m), s.EradHead(s.m), - s.ruTail(s.m), s.ruHead(s.m), - s.rETail(s.m), s.rEHead(s.m), - s.rF(s.m), s.ruF(s.m), s.rEF(s.m), s.EradF(s.m), - s.dt_mass_density(s.m), - s.dt_momentum_density(s.m), - s.dt_total_energy_density(s.m), - s.dt_radiation_energy_density(s.m)); - + sc.execute>( + flecsi::exec::on, + axis, + *s.m, s.mass_density(*s.m), s.velocity(*s.m), s.pressure(*s.m), + s.specific_internal_energy(*s.m), + s.sound_speed(*s.m), + s.radiation_energy_density(*s.m), + s.rTail(*s.m), s.rHead(*s.m), s.uTail(*s.m), s.uHead(*s.m), + s.pTail(*s.m), s.pHead(*s.m), s.eTail(*s.m), s.eHead(*s.m), s.cTail(*s.m), + s.cHead(*s.m), s.EradTail(*s.m), s.EradHead(*s.m), + s.ruTail(*s.m), s.ruHead(*s.m), s.rETail(*s.m), s.rEHead(*s.m)); + + if(Stage == time_stepper::rk_stage::First) { + // Calculate K1 and save it to dt_U + sc.execute>( + flecsi::exec::on, + axis, *s.m, + s.rTail(*s.m), s.rHead(*s.m), s.uTail(*s.m), s.uHead(*s.m), + s.pTail(*s.m), s.pHead(*s.m), s.cTail(*s.m), + s.cHead(*s.m), s.EradTail(*s.m), s.EradHead(*s.m), + s.ruTail(*s.m), s.ruHead(*s.m), + s.rETail(*s.m), s.rEHead(*s.m), + s.rF(*s.m), s.ruF(*s.m), s.rEF(*s.m), s.EradF(*s.m), + s.dt_mass_density(*s.m), + s.dt_momentum_density(*s.m), + s.dt_total_energy_density(*s.m), + s.dt_radiation_energy_density(*s.m), + s.gravity_acc(*s.gt)); + } + else if(Stage == time_stepper::rk_stage::Second) { + // Calculate K2 and save it to dt_U_2 + sc.execute>( + flecsi::exec::on, + axis, *s.m, + s.rTail(*s.m), s.rHead(*s.m), s.uTail(*s.m), s.uHead(*s.m), + s.pTail(*s.m), s.pHead(*s.m), s.cTail(*s.m), + s.cHead(*s.m), s.EradTail(*s.m), s.EradHead(*s.m), + s.ruTail(*s.m), s.ruHead(*s.m), + s.rETail(*s.m), s.rEHead(*s.m), + s.rF(*s.m), s.ruF(*s.m), s.rEF(*s.m), s.EradF(*s.m), + s.dt_mass_density_2(*s.m), + s.dt_momentum_density_2(*s.m), + s.dt_total_energy_density_2(*s.m), + s.dt_radiation_energy_density_2(*s.m), + s.gravity_acc(*s.gt)); + + } // clang-format on } } // RK_advance @@ -141,128 +292,128 @@ void update_vars(control_policy & cp, time_stepper::rk_stage Stage) { auto & s = cp.state(); + flecsi::scheduler & sc = cp.scheduler(); if(Stage == time_stepper::rk_stage::Second) { - const double coefficient{1}; - flecsi::execute, flecsi::default_accelerator>(s.dt(s.gt), - s.m, + // Apply K1 to U with a Forward Euler step, so we can use U for the + + // K2 calculation in the next RK advance + sc.execute>( + flecsi::exec::on, + s.dt(*s.gt), + *s.m, // - s.mass_density(s.m), - s.momentum_density(s.m), - s.total_energy_density(s.m), - s.radiation_energy_density(s.m), + s.mass_density_n(*s.m), + s.momentum_density_n(*s.m), + s.total_energy_density_n(*s.m), + s.radiation_energy_density_n(*s.m), // - s.dt_mass_density(s.m), - s.dt_momentum_density(s.m), - s.dt_total_energy_density(s.m), - s.dt_radiation_energy_density(s.m), - coefficient); + s.dt_mass_density(*s.m), + s.dt_momentum_density(*s.m), + s.dt_total_energy_density(*s.m), + s.dt_radiation_energy_density(*s.m), + // + s.mass_density(*s.m), + s.momentum_density(*s.m), + s.total_energy_density(*s.m), + s.radiation_energy_density(*s.m)); } else if(Stage == time_stepper::rk_stage::Update) { - const double coefficient{0.5}; - flecsi::execute, flecsi::default_accelerator>(s.dt(s.gt), - s.m, + // First compute K1' = (K1 + K2) * 0.5 + sc.execute>(flecsi::exec::on, + *s.m, + // + s.dt_mass_density(*s.m), + s.dt_momentum_density(*s.m), + s.dt_total_energy_density(*s.m), + s.dt_radiation_energy_density(*s.m), + // + s.dt_mass_density_2(*s.m), + s.dt_momentum_density_2(*s.m), + s.dt_total_energy_density_2(*s.m), + s.dt_radiation_energy_density_2(*s.m)); + + // Now get U_n(+1) = U_n + h * K1' + sc.execute>(flecsi::exec::on, + s.dt(*s.gt), + *s.m, // - s.mass_density_n(s.m), - s.momentum_density_n(s.m), - s.total_energy_density_n(s.m), - s.radiation_energy_density_n(s.m), + s.mass_density_n(*s.m), + s.momentum_density_n(*s.m), + s.total_energy_density_n(*s.m), + s.radiation_energy_density_n(*s.m), // - s.dt_mass_density(s.m), - s.dt_momentum_density(s.m), - s.dt_total_energy_density(s.m), - s.dt_radiation_energy_density(s.m), - coefficient); + s.dt_mass_density(*s.m), + s.dt_momentum_density(*s.m), + s.dt_total_energy_density(*s.m), + s.dt_radiation_energy_density(*s.m)); + + // Finish by updating the values stored in U_n to U + sc.execute>(flecsi::exec::on, + *s.m, + s.mass_density_n(*s.m), + s.momentum_density_n(*s.m), + s.total_energy_density_n(*s.m), + s.radiation_energy_density_n(*s.m), + // + s.mass_density(*s.m), + s.momentum_density(*s.m), + s.total_energy_density(*s.m), + s.radiation_energy_density(*s.m)); } // Perform primitive recovery - flecsi::execute, - flecsi::default_accelerator>(s.m, - s.mass_density(s.m), - s.momentum_density(s.m), - s.total_energy_density(s.m), - s.velocity(s.m), - s.pressure(s.m), - s.specific_internal_energy(s.m), - s.sound_speed(s.m), + sc.execute>(flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.momentum_density(*s.m), + s.total_energy_density(*s.m), + s.velocity(*s.m), + s.pressure(*s.m), + s.specific_internal_energy(*s.m), + s.sound_speed(*s.m), s.eos); // Update boundary cells - flecsi::execute, flecsi::default_accelerator>(s.m, - s.bmap(s.gt), - s.mass_density(s.m), - s.velocity(s.m), - s.pressure(s.m), - s.radiation_energy_density(s.m), - s.momentum_density(s.m), - s.total_energy_density(s.m)); + sc.execute>(flecsi::exec::on, + *s.m, + s.bmap(*s.gt), + std::vector{s.mass_density(*s.m), + s.pressure(*s.m), + s.specific_internal_energy(*s.m), + s.radiation_energy_density(*s.m), + s.total_energy_density(*s.m)}, + std::vector{s.velocity(*s.m), s.momentum_density(*s.m)}); if(s.mg) { // FIXME: figure out how not to use the hardcoded radiation temperature // boundary auto radiation_boundary_f = - flecsi::execute(s.t(s.gt), - time_boundary(s.dense_topology), - temperature_boundary(s.dense_topology)); - flecsi::execute, - flecsi::default_accelerator>( - s.m, s.radiation_energy_density(s.m), radiation_boundary_f); + sc.execute(flecsi::exec::on, + s.t(*s.gt), + s.time_boundary(*s.dense_topology), + s.temperature_boundary(*s.dense_topology)); + sc.execute>(flecsi::exec::on, + *s.m, + s.bmap(*s.gt), + std::vector{s.radiation_energy_density(*s.m)}, + radiation_boundary_f); } } // update_vars -// -// Perform reconstruction on cell faces, compute face fluxes with a Riemann -// solver, and add the summation of dF^i/dx^i into (dU_dt)_explicit -// -template -void -fluxes_terms(control_policy & cp) { - auto & s = cp.state(); - - using limiter = spec::limiters::ppm4; - - for(std::size_t axis = 0; axis < D; axis++) { - // clang-format off - flecsi::execute, - flecsi::default_accelerator>(axis, - s.m, s.mass_density(s.m), s.velocity(s.m), s.pressure(s.m), - s.specific_internal_energy(s.m), - s.sound_speed(s.m), - s.radiation_energy_density(s.m), - s.rTail(s.m), s.rHead(s.m), s.uTail(s.m), s.uHead(s.m), - s.pTail(s.m), s.pHead(s.m), s.eTail(s.m), s.eHead(s.m), s.cTail(s.m), - s.cHead(s.m), s.EradTail(s.m), s.EradHead(s.m), - s.ruTail(s.m), s.ruHead(s.m), s.rETail(s.m), s.rEHead(s.m)); - - flecsi::execute, - flecsi::default_accelerator>(axis, s.m, - s.rTail(s.m), s.rHead(s.m), s.uTail(s.m), s.uHead(s.m), - s.pTail(s.m), s.pHead(s.m), s.eTail(s.m), s.eHead(s.m), s.cTail(s.m), - s.cHead(s.m), s.EradTail(s.m), s.EradHead(s.m), - s.ruTail(s.m), s.ruHead(s.m), - s.rETail(s.m), s.rEHead(s.m), - s.rF(s.m), s.ruF(s.m), s.rEF(s.m), s.EradF(s.m), - s.dt_mass_density(s.m), - s.dt_momentum_density(s.m), - s.dt_total_energy_density(s.m), - s.dt_radiation_energy_density(s.m)); - // clang-format on - } -} // fluxes_terms - template void advection_advance(control_policy & cp) { - // First RK advance - RK_advance(cp); + // First RK advance (this gives us K1) + RK_advance(cp, time_stepper::rk_stage::First); - // Update variable for second advance + // Update variable for second advance (this is u0 + h * K1) update_vars(cp, time_stepper::rk_stage::Second); - // Second RK advance - RK_advance(cp); + // Second RK advance (this gives us K2) + RK_advance(cp, time_stepper::rk_stage::Second); // Final variable update update_vars(cp, time_stepper::rk_stage::Update); @@ -277,119 +428,149 @@ template void radiation_advance(control_policy & cp) { -#ifdef ENABLE_RADIATION - using namespace flecsi; auto & s = cp.state(); - - flecsi::execute, - flecsi::default_accelerator>(s.m, - s.mass_density(s.m), - s.velocity(s.m), - s.temperature(s.m), - s.total_energy_density(s.m), - s.radiation_energy_density(s.m), - kappa(s.gt), - particle_mass(s.gt), - s.dt_weighted(s.gt), + flecsi::scheduler & sc = cp.scheduler(); + + sc.execute>(flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.velocity(*s.m), + s.temperature(*s.m), + s.total_energy_density(*s.m), + s.radiation_energy_density(*s.m), + s.kappa(*s.gt), + s.dt_weighted(*s.gt), s.eos); - flecsi::execute, flecsi::default_accelerator>( - s.m, s.radiation_energy_density(s.m), s.gradient_rad_energy(s.m)); + sc.execute>(flecsi::exec::on, + *s.m, + s.radiation_energy_density(*s.m), + s.gradient_rad_energy(*s.m)); + + // Adaptive FLD Radiation Advance - flecsi::execute, flecsi::default_accelerator>(s.m, - s.mass_density(s.m), - s.radiation_energy_density(s.m), - s.gradient_rad_energy(s.m), - s.magnitude_gradient_rad_energy(s.m), - s.R_value(s.m), - s.lambda_bridge(s.m), - kappa(s.gt)); + sc.execute>(flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.radiation_energy_density(*s.m), + s.gradient_rad_energy(*s.m), + s.magnitude_gradient_rad_energy(*s.m), + s.R_value(*s.m), + s.lambda_bridge(*s.m), + s.kappa(*s.gt), + s.limiter_id(*s.gt)); - flecsi::execute, - flecsi::default_accelerator>(s.m, s.bmap(s.gt), s.lambda_bridge(s.m)); + sc.execute>( + flecsi::exec::on, *s.m, s.bmap(*s.gt), std::vector{s.lambda_bridge(*s.m)}); if(s.mg) { // FIXME: figure out how not to use the hardcoded radiation temperature // boundary auto radiation_boundary_f = - flecsi::execute(s.t(s.gt), - time_boundary(s.dense_topology), - temperature_boundary(s.dense_topology)); - flecsi::execute, - flecsi::default_accelerator>( - s.m, s.radiation_energy_density(s.m), radiation_boundary_f); + sc.execute(flecsi::exec::on, + s.t(*s.gt), + s.time_boundary(*s.dense_topology), + s.temperature_boundary(*s.dense_topology)); + sc.execute>(flecsi::exec::on, + *s.m, + s.bmap(*s.gt), + std::vector{s.radiation_energy_density(*s.m)}, + radiation_boundary_f); } - execute, flecsi::default_accelerator>( - s.m, s.mass_density(s.m), s.lambda_bridge(s.m), s.Diff(s.m), kappa(s.gt)); + sc.execute>(flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.lambda_bridge(*s.m), + s.Diff(*s.m), + s.kappa(*s.gt)); // Initialize the diffusion coefficient - execute, flecsi::default_accelerator>( - s.m, s.Diff(s.m), s.Df_x(s.m), s.Df_y(s.m), s.Df_z(s.m)); + sc.execute>(flecsi::exec::on, + *s.m, + s.Diff(*s.m), + s.Df_x(*s.m), + s.Df_y(*s.m), + s.Df_z(*s.m)); // Initialize the stencil - execute, flecsi::default_accelerator>( - s.m, s.Df_x(s.m), s.Df_y(s.m), s.Df_z(s.m), s.Ew(s.m), s.dt_weighted(s.gt)); + sc.execute>(flecsi::exec::on, + *s.m, + s.Df_x(*s.m), + s.Df_y(*s.m), + s.Df_z(*s.m), + s.Ew(*s.m), + s.dt_weighted(*s.gt)); // Initialize fields - flecsi::execute, flecsi::default_accelerator>( - s.m, s.radiation_energy_density(s.m), s.Ef(s.m)); - flecsi::execute, flecsi::default_accelerator>( - s.m, s.Esf(s.m), 0.0); - flecsi::execute, flecsi::default_accelerator>( - s.m, s.Esf(s.m, 1), 0.0); - flecsi::execute, flecsi::default_accelerator>( - s.m, s.Resf(s.m), 0.0); + sc.execute>( + flecsi::exec::on, *s.m, s.radiation_energy_density(*s.m), s.Ef(*s.m)); + sc.execute>( + flecsi::exec::on, *s.m, s.Esf(*s.m, 1), 0.0); + sc.execute>( + flecsi::exec::on, *s.m, s.Resf(*s.m), 0.0); + + std::chrono::time_point start_timer_rad = + std::chrono::system_clock::now(); + + hard::rad::linsolve(cp); - hard::rad::fmg(s); + std::chrono::time_point stop_timer_rad = + std::chrono::system_clock::now(); + + flog(info) << " Radiation Timing: " + << (stop_timer_rad - start_timer_rad).count() * 1e-9 << " [s] " + << std::endl; + + // Move solution from rad solver + sc.execute>( + flecsi::exec::on, *s.m, s.Uf(*s.m), s.radiation_energy_density(*s.m)); // Perform primitive recovery, since energy densities have changed - flecsi::execute, - flecsi::default_accelerator>(s.m, - s.mass_density(s.m), - s.momentum_density(s.m), - s.total_energy_density(s.m), - s.velocity(s.m), - s.pressure(s.m), - s.specific_internal_energy(s.m), - s.sound_speed(s.m), + sc.execute>(flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.momentum_density(*s.m), + s.total_energy_density(*s.m), + s.velocity(*s.m), + s.pressure(*s.m), + s.specific_internal_energy(*s.m), + s.sound_speed(*s.m), s.eos); // and also update boundary cells - flecsi::execute, flecsi::default_accelerator>(s.m, - s.bmap(s.gt), - s.mass_density(s.m), - s.velocity(s.m), - s.pressure(s.m), - s.radiation_energy_density(s.m), - s.momentum_density(s.m), - s.total_energy_density(s.m)); - -#endif - + sc.execute>(flecsi::exec::on, + *s.m, + s.bmap(*s.gt), + std::vector{s.mass_density(*s.m), + s.pressure(*s.m), + s.specific_internal_energy(*s.m), + s.radiation_energy_density(*s.m), + s.total_energy_density(*s.m)}, + std::vector{s.velocity(*s.m), s.momentum_density(*s.m)}); } // radiation_advance // ----------------------------------------------------------------------------- -// Compute max characteristic speeds, and determine dt_min() for the next time -// step. +// Compute max characteristic speeds, and determine dt_min() for the next +// time step. // ----------------------------------------------------------------------------- template void update_time_step_size(control_policy & cp) { auto & s = cp.state(); + flecsi::scheduler & sc = cp.scheduler(); - auto lmax_f = - flecsi::execute, - flecsi::default_accelerator>(s.m, - s.mass_density(s.m), - s.velocity(s.m), - s.pressure(s.m), - s.sound_speed(s.m)); + auto lmax_f = sc.execute>( + flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.velocity(*s.m), + s.sound_speed(*s.m)); s.dtmin_ = - flecsi::reduce, flecsi::exec::fold::min>( - s.m, lmax_f); + sc.reduce, flecsi::exec::fold::min>( + flecsi::exec::on, *s.m, lmax_f); #ifdef HARD_ENABLE_LEGION_TRACING cp.guard.reset(); diff --git a/app/advance.hh b/app/advance.hh index ddfbe44..820f8b0 100644 --- a/app/advance.hh +++ b/app/advance.hh @@ -28,6 +28,7 @@ inline control::action, cp::advance> inline control::action, cp::advance> initialize_time_derivative_3d; +#ifdef ENABLE_RADIATION // ---------------------------------------------------------------------------- // Solve radiation diffusion and source-sink terms // ---------------------------------------------------------------------------- @@ -44,6 +45,7 @@ inline const auto dep_radiation_advance_2d = radiation_advance_2d.add(initialize_time_derivative_2d); inline const auto dep_radiation_advance_3d = radiation_advance_3d.add(initialize_time_derivative_3d); +#endif // ---------------------------------------------------------------------------- // Solve advection @@ -55,13 +57,21 @@ inline control::action, cp::advance> advection_advance_2d; inline control::action, cp::advance> advection_advance_3d; +#ifdef ENABLE_RADIATION inline const auto dep_advection_advance_1d = advection_advance_1d.add(radiation_advance_1d); inline const auto dep_advection_advance_2d = advection_advance_2d.add(radiation_advance_2d); inline const auto dep_advection_advance_3d = advection_advance_3d.add(radiation_advance_3d); - +#else +inline const auto dep_advection_advance_1d = + advection_advance_1d.add(initialize_time_derivative_1d); +inline const auto dep_advection_advance_2d = + advection_advance_2d.add(initialize_time_derivative_2d); +inline const auto dep_advection_advance_3d = + advection_advance_3d.add(initialize_time_derivative_3d); +#endif // -------------------------------------------------------------------- // Update for next cycle // -------------------------------------------------------------------- diff --git a/app/analyze.hh b/app/analyze.hh index 4a37874..03f29ec 100644 --- a/app/analyze.hh +++ b/app/analyze.hh @@ -18,44 +18,61 @@ void analyze(control_policy & cp) { using namespace flecsi; auto & s = cp.state(); - auto lm = data::launch::make(s.m); + auto & sc = cp.scheduler(); + auto lm = data::launch::make(sc, *s.m); #ifndef HARD_BENCHMARK_MODE + +#if FLECSI_BACKEND == FLECSI_BACKEND_legion if(((cp.step() % cp.output_frequency()) == 0) or (cp.step() == cp.max_steps())) { +#else + if(((cp.step() % cp.output_frequency()) == 0) or + (cp.step() == cp.max_steps()) or (cp.time() == cp.max_time())) { +#endif - execute, mpi>(spec::io::name{"output-"} - << std::setfill('0') << std::setw(5) - << cp.step(), - s.t(s.gt), + execute, mpi>(flecsi::exec::on, + spec::io::name{""} << std::setfill('0') << std::setw(5) << cp.step(), + s.t(*s.gt), lm, - s.mass_density(lm), - s.pressure(lm), - s.sound_speed(lm), - s.specific_internal_energy(lm), - s.velocity(lm), - s.momentum_density(lm), - s.total_energy_density(lm), - s.radiation_energy_density(lm)); + std::vector{s.mass_density(lm), + s.pressure(lm), + s.sound_speed(lm), + s.specific_internal_energy(lm), + s.total_energy_density(lm), + s.radiation_energy_density(lm)}, + std::vector{s.velocity(lm), s.momentum_density(lm)}, + std::vector{"density", + "pressure", + "sound_speed", + "specific_internal_energy", + "total_energy_density", + "radiation_energy_density"}, + std::vector{"velocity", "momentum_density"}); #ifdef USE_CATALYST if constexpr(D == 3) { // First update fields (catalyst_attributes) for catalysts pipeline flog(info) << "analyze action: prepare catalyst data" << std::endl; - execute, mpi>(catalyst_data(pt), - s.t(s.gt), - s.m, - s.mass_density(s.m), - s.velocity(s.m), - s.pressure(s.m), - s.total_energy_density(s.m), - s.radiation_energy_density(s.m)); // <<< add variables here for catalyst + execute, mpi>(flecsi::exec::on, + s.catalyst_data(*s.pt), + s.t(*s.gt), + *s.m, + s.mass_density(*s.m), + s.velocity(*s.m), + s.pressure(*s.m), + s.total_energy_density(*s.m), + s.radiation_energy_density( + *s.m)); // <<< add variables here for catalyst // Then pass catalyst_data to catalyst pipeline flog(info) << "analyze action: execute catalyst" << std::endl; - execute( - catalyst_data(pt), cp.step(), s.t(s.gt), lattice); + execute(flecsi::exec::on, + s.catalyst_data(*s.pt), + cp.step(), + s.t(*s.gt), + lattice); } else { /* Do nothing, catalyst/paraview visualization only for 3D */ diff --git a/app/finalize.hh b/app/finalize.hh index 993dde1..7e4b5be 100644 --- a/app/finalize.hh +++ b/app/finalize.hh @@ -14,13 +14,14 @@ namespace hard::action { template void -finalize(control_policy &) { +finalize([[maybe_unused]] control_policy & cs) { using namespace flecsi; #ifdef USE_CATALYST - + auto & s = cs.state(); if constexpr(D == 3) { - execute(catalyst_data(pt)); + execute( + flecsi::exec::on, s.catalyst_data(*s.pt)); flog(info) << "finalize action: done" << std::endl; } else { diff --git a/app/flecsolvers.hh b/app/flecsolvers.hh new file mode 100644 index 0000000..35fc981 --- /dev/null +++ b/app/flecsolvers.hh @@ -0,0 +1,401 @@ +#ifndef FLECSOLVERS_HH +#define FLECSOLVERS_HH + +#include "flecsi/execution.hh" +#include "flecsi/flog.hh" + +#include "state.hh" +#include "tasks/boundaries/boundary.hh" +#include "tasks/rad.hh" +#include "types.hh" + +#if defined(USE_FLECSOLVE) && USE_FLECSOLVE +#include "flecsolve/operators/core.hh" +#include "flecsolve/solvers/cg.hh" +#include "flecsolve/solvers/factory.hh" +#include "flecsolve/solvers/gmres.hh" +#include "flecsolve/vectors/topo_view.hh" +#endif + +namespace hard::rad { + +template +struct solver_parameters { + std::reference_wrapper> s; + std::reference_wrapper sc; + double temp; +}; + +template +struct operator_t : flecsolve::op::base> { + using base = flecsolve::op::base>; + using base::params; + + operator_t(solver_parameters params) : base(std::move(params)) {} + + template + void apply(const Domain & x, Ranges & y) const { + flecsi::scheduler & sc = params.sc.get(); + sc.execute>(flecsi::exec::on, + y.data.topo(), + std::move(params.s.get().Ew(y.data.topo())), + y.data.ref(), + x.data.ref()); + // flecsi::execute>(flecsi::exec::on, + // y.data.topo(), + // y.data.ref(), + // params.temp); // HARD CODED VALUE for now + } +}; + +template +struct precond_parameters { + std::reference_wrapper> s; + std::reference_wrapper sc; + std::size_t index; + std::size_t nr_vcycles; + std::size_t jacobi_iterations; +}; + +template +struct v_cycle : flecsolve::op::base> { + using base = flecsolve::op::base>; + using base::params; + + v_cycle(precond_parameters params) : base(std::move(params)) {} + + template + void apply(const Domain & x, Range & y) const { + + flecsi::scheduler & sc = params.sc.get(); + + sc.execute>(flecsi::exec::on, + y.data.topo(), + x.data.ref(), + params.s.get().Ef_temp(x.data.topo())); + + // Zero solution vector + sc.execute>( + flecsi::exec::on, y.data.topo(), params.s.get().Esf(y.data.topo()), 0.0); + sc.execute>(flecsi::exec::on, + y.data.topo(), + params.s.get().Esf(y.data.topo(), 1), + 0.0); + sc.execute>( + flecsi::exec::on, y.data.topo(), params.s.get().Resf(y.data.topo()), 0.0); + + _vcycle(std::move(params.s), 0); + + sc.execute>(flecsi::exec::on, + y.data.topo(), + params.s.get().Esf(y.data.topo()), + y.data.ref()); + } + + void _vcycle(state & s, std::size_t index) const { + auto & mf = *s.mh[index]; + flecsi::scheduler & sc = params.sc.get(); + + // Find current level + std::size_t level{s.highest_level - index}; + + if(level == s.lowest_level) { + + for(std::size_t i{0}; i < params.jacobi_iterations; i++) { + s.Esf.flip(); + // NOTE: We are defaulting to damped_jacobi until gauss-seidel is + // parallelized + sc.execute>(flecsi::exec::on, + mf, + s.Ew(mf), + s.Esf(mf), + s.Esf(mf, 1), + s.Ef_temp(mf), + 0.8); + } // for + + // using namespace flecsolve; + // solver_parameters params{std::ref(s), std::ref(sc), 0.0}; + // op::core> so(params); + + // auto f = flecsolve::vec::make((s.Ef(mf))); + // auto u = flecsolve::vec::make((s.Esf(mf))); + + // std::size_t iter{0}; + // auto slv = + // flecsolve::cg::solver(s.solver_settings, + // flecsolve::cg::make_work(f))( + // op::ref(so), op::I, [&](auto &, double rnorm) { return false; }); + // auto info = slv(f, u); + // flog(info) << "coarse grid res norm " << info.res_norm_final + // << " iter: " << info.iters << std::endl; + } + else { + + auto & mc = *s.mh[index + 1]; + // Pre Smoothing + + for(std::size_t i{0}; i < s.mg_pre; ++i) { + s.Esf.flip(); + sc.execute>(flecsi::exec::on, + mf, + s.Ew(mf), + s.Esf(mf), + s.Esf(mf, 1), + s.Ef_temp(mf), + 0.8); + } // for + + // Set the diffusion coefficient and the stencil (TODO) + sc.execute>( + flecsi::exec::on, mf, mc, s.Df_x(mf), s.Df_x(mc)); + + sc.execute>( + flecsi::exec::on, mf, mc, s.Df_y(mf), s.Df_y(mc)); + + sc.execute>( + flecsi::exec::on, mf, mc, s.Df_z(mf), s.Df_z(mc)); + + sc.execute>(flecsi::exec::on, + mc, + s.Df_x(mc), + s.Df_y(mc), + s.Df_z(mc), + s.Ew(mc), + s.dt(*s.gt)); + + // Recursive solve + sc.execute>( + flecsi::exec::on, mf, s.Ew(mf), s.Esf(mf), s.Ef_temp(mf), s.Resf(mf)); + + sc.execute>( + flecsi::exec::on, mf, mc, s.Resf(mf), s.Ef_temp(mc)); + + // Initialize the solution fields for the coarser level + sc.execute>( + flecsi::exec::on, mc, s.Esf(mc), 0.0); + sc.execute>( + flecsi::exec::on, mc, s.Esf(mc, 1), 0.0); + + _vcycle(s, index + 1); + + sc.execute>( + flecsi::exec::on, mc, mf, s.Esf(mc), s.Errf(mf)); + + sc.execute>( + flecsi::exec::on, mf, s.Esf(mf), s.Errf(mf)); + + // Post Smoothing + for(std::size_t i{0}; i < s.mg_post; ++i) { + s.Esf.flip(); + sc.execute>(flecsi::exec::on, + mf, + s.Ew(mf), + s.Esf(mf), + s.Esf(mf, 1), + s.Ef_temp(mf), + 0.8); + } // for + } // if + } +}; + +template +struct f_mg : flecsolve::op::base> { + using base = flecsolve::op::base>; + using base::params; + + f_mg(precond_parameters params) : base(std::move(params)) {} + + template + void apply(const Domain & x, Range & y) const { + + flecsi::scheduler & sc = params.sc.get(); + + // Rhs = r // NO NEED + sc.execute>(flecsi::exec::on, + y.data.topo(), + x.data.ref(), + params.s.get().Ef_temp(x.data.topo())); + + // Zero solution vector + sc.execute>( + flecsi::exec::on, y.data.topo(), params.s.get().Esf(y.data.topo()), 0.0); + sc.execute>(flecsi::exec::on, + y.data.topo(), + params.s.get().Esf(y.data.topo(), 1), + 0.0); + sc.execute>( + flecsi::exec::on, y.data.topo(), params.s.get().Resf(y.data.topo()), 0.0); + + _fmg(std::move(params.s), 0); + + sc.execute>(flecsi::exec::on, + y.data.topo(), + params.s.get().Esf(y.data.topo()), + y.data.ref()); + } + + void _vcycle(state & s, std::size_t index) const { + auto & mf = *s.mh[index]; + flecsi::scheduler & sc = params.sc.get(); + + // Find current level + std::size_t level{s.highest_level - index}; + + if(level == s.lowest_level) { + + for(std::size_t i{0}; i < params.jacobi_iterations; i++) { + s.Esf.flip(); + // NOTE: We are defaulting to damped_jacobi until gauss-seidel is + // parallelized + sc.execute>(flecsi::exec::on, + mf, + s.Ew(mf), + s.Esf(mf), + s.Esf(mf, 1), + s.Ef_temp(mf), + 0.8); + } // for + + // using namespace flecsolve; + // solver_parameters params{std::ref(s), std::ref(sc), 0.0}; + // op::core> so(params); + + // auto f = flecsolve::vec::make((s.Ef(mf))); + // auto u = flecsolve::vec::make((s.Esf(mf))); + + // std::size_t iter{0}; + // auto slv = + // flecsolve::cg::solver(s.solver_settings, + // flecsolve::cg::make_work(f))( + // op::ref(so), op::I, [&](auto &, double rnorm) { return false; }); + // auto info = slv(f, u); + // flog(info) << "coarse grid res norm " << info.res_norm_final + // << " iter: " << info.iters << std::endl; + } + else { + + auto & mc = *s.mh[index + 1]; + // Pre Smoothing + + for(std::size_t i{0}; i < s.mg_pre; ++i) { + s.Esf.flip(); + sc.execute>(flecsi::exec::on, + mf, + s.Ew(mf), + s.Esf(mf), + s.Esf(mf, 1), + s.Ef_temp(mf), + 0.8); + } // for + + // Set the diffusion coefficient and the stencil (TODO) + sc.execute>( + flecsi::exec::on, mf, mc, s.Df_x(mf), s.Df_x(mc)); + + sc.execute>( + flecsi::exec::on, mf, mc, s.Df_y(mf), s.Df_y(mc)); + + sc.execute>( + flecsi::exec::on, mf, mc, s.Df_z(mf), s.Df_z(mc)); + + sc.execute>(flecsi::exec::on, + mc, + s.Df_x(mc), + s.Df_y(mc), + s.Df_z(mc), + s.Ew(mc), + s.dt(*s.gt)); + + // Recursive solve + sc.execute>( + flecsi::exec::on, mf, s.Ew(mf), s.Esf(mf), s.Ef_temp(mf), s.Resf(mf)); + + sc.execute>( + flecsi::exec::on, mf, mc, s.Resf(mf), s.Ef_temp(mc)); + + // Initialize the solution fields for the coarser level + sc.execute>( + flecsi::exec::on, mc, s.Esf(mc), 0.0); + sc.execute>( + flecsi::exec::on, mc, s.Esf(mc, 1), 0.0); + + _vcycle(s, index + 1); + + sc.execute>( + flecsi::exec::on, mc, mf, s.Esf(mc), s.Errf(mf)); + + sc.execute>( + flecsi::exec::on, mf, s.Esf(mf), s.Errf(mf)); + + // Post Smoothing + for(std::size_t i{0}; i < s.mg_post; ++i) { + s.Esf.flip(); + sc.execute>(flecsi::exec::on, + mf, + s.Ew(mf), + s.Esf(mf), + s.Esf(mf, 1), + s.Ef_temp(mf), + 0.8); + } // for + } // if + } + + void _fmg(state & s, std::size_t index) const { + auto & mf = *s.mh[index]; + flecsi::scheduler & sc = params.sc.get(); + + // Find current level + std::size_t level{s.highest_level - index}; + + // Deepest level + if(level == s.lowest_level) { + + _vcycle(s, index); + } + else { + auto & mc = *s.mh[index + 1]; + // Set the RHS and solution field + sc.execute>( + flecsi::exec::on, mf, mc, s.Ef(mf), s.Ef(mc)); + sc.execute>( + flecsi::exec::on, mf, mc, s.Esf(mf), s.Esf(mc)); + + // Set the diffusion coefficient and the stencil (TODO) + sc.execute>( + flecsi::exec::on, mf, mc, s.Df_x(mf), s.Df_x(mc)); + + sc.execute>( + flecsi::exec::on, mf, mc, s.Df_y(mf), s.Df_y(mc)); + + sc.execute>( + flecsi::exec::on, mf, mc, s.Df_z(mf), s.Df_z(mc)); + + sc.execute>(flecsi::exec::on, + mc, + s.Df_x(mc), + s.Df_y(mc), + s.Df_z(mc), + s.Ew(mc), + s.dt(*s.gt)); + + // Now call solve for one level deeper + _fmg(s, index + 1); + + // Interpolate solution back up (RHS does not change) + sc.execute>( + flecsi::exec::on, mc, mf, s.Esf(mc), s.Esf(mf)); + + // Do a V-Cycle + for(std::size_t i{0}; i < s.mg_cycles; ++i) { + _vcycle(s, index); + } // for + } // if + }; +}; + +} // namespace hard::rad + +#endif diff --git a/app/init.hh b/app/init.hh index 0afc44d..93b2e29 100644 --- a/app/init.hh +++ b/app/init.hh @@ -4,7 +4,7 @@ #include "options.hh" #include "spec/eos.hh" #include "state.hh" -#include "tasks/boundary.hh" +#include "tasks/boundaries/boundary.hh" #include "tasks/hydro/cons2prim.hh" #include "tasks/hydro/maxcharspeed.hh" #include "tasks/init.hh" @@ -12,6 +12,7 @@ #include "tasks/io.hh" #include "tasks/rad.hh" #include "types.hh" +#include "utils.hh" #ifdef USE_CATALYST #include "tasks/catalyst.hh" @@ -28,8 +29,39 @@ void initialize(control_policy & cp) { using namespace flecsi; auto & s = cp.state(); + flecsi::scheduler & sc = cp.scheduler(); + YAML::Node config = YAML::LoadFile(opt::config.value()); + /*--------------------------------------------------------------------------* + Solver. + *--------------------------------------------------------------------------*/ + +#if defined(USE_FLECSOLVE) && USE_FLECSOLVE + s.solver_settings.maxiter = config["linear_solver"]["maxiter"].IsDefined() + ? config["linear_solver"]["maxiter"].as() + : 50; + s.solver_settings.rtol = config["linear_solver"]["rtol"].IsDefined() + ? config["linear_solver"]["rtol"].as() + : 1e-12; + s.solver_settings.use_zero_guess = + config["linear_solver"]["use_zero_guess"].IsDefined() + ? config["linear_solver"]["use_zero_guess"].as() + : true; + s.flecsolve_coarse_grid = + config["linear_solver"]["flecsolve_coarse_grid"].IsDefined() + ? config["linear_solver"]["flecsolve_coarse_grid"].as() + : true; +#endif + s.jacobi_iterations = + config["linear_solver"]["jacobi_iterations"].IsDefined() + ? config["linear_solver"]["jacobi_iterations"].as() + : 100; + + s.full_multigrid = config["linear_solver"]["full_multigrid"].IsDefined() + ? config["linear_solver"]["full_multigrid"].as() + : true; + /*--------------------------------------------------------------------------* Global and color topology allocations. *--------------------------------------------------------------------------*/ @@ -60,12 +92,12 @@ initialize(control_policy & cp) { n_lines++; } - s.dense_topology.allocate(n_lines); + sc.allocate(s.dense_topology, n_lines); - s.gt.allocate({}); const auto num_colors = - opt::colors.value() == 0 ? flecsi::processes() : opt::colors.value(); - s.ct.allocate(num_colors); + opt::colors.value() == 0 ? sc.runtime().processes() : opt::colors.value(); + sc.allocate(s.gt, num_colors); + sc.allocate(s.ct, {num_colors}); /*--------------------------------------------------------------------------* Set boundaries. @@ -89,53 +121,103 @@ initialize(control_policy & cp) { utils::mesh_boundary(config["boundaries"]["zhigh"].as()); } // if - auto bf = execute>(s.bmap(s.gt), bnds); + auto bf = + execute>(flecsi::exec::on, s.bmap(*s.gt), bnds); /*--------------------------------------------------------------------------* T boundary. *--------------------------------------------------------------------------*/ - execute(time_boundary(s.dense_topology), time); execute( - temperature_boundary(s.dense_topology), temperature); + flecsi::exec::on, s.time_boundary(*s.dense_topology), time); + execute( + flecsi::exec::on, s.temperature_boundary(*s.dense_topology), temperature); if(config["problem"].as() == "implosion") - execute( - temperature_boundary(s.dense_topology), + execute(flecsi::exec::on, + s.temperature_boundary(*s.dense_topology), config["temperature_units"].as()); /*--------------------------------------------------------------------------* Kappa. *--------------------------------------------------------------------------*/ #ifdef ENABLE_RADIATION - execute(kappa(s.gt), config["kappa"].as()); + execute(s.kappa(*s.gt), config["kappa"].as()); #endif + + /*--------------------------------------------------------------------------* + Adaptive FLD Check, Closure ID and Limiter ID + *--------------------------------------------------------------------------*/ + +#ifdef ENABLE_RADIATION + // Default is limiter = 1 and closure = 3 + std::size_t ci = config["closure_id"].IsDefined() + ? config["closure_id"].as() + : 3; + std::size_t li = config["limiter_id"].IsDefined() + ? config["limiter_id"].as() + : 1; + sc.execute(s.closure_id(*s.gt), ci); + sc.execute(s.limiter_id(*s.gt), li); +#endif + + /*--------------------------------------------------------------------------* + Gravity Acceleration + *--------------------------------------------------------------------------*/ + + vec g(0.0); + if(config["gravity_acc"].IsDefined()) { + g[0] = config["gravity_acc"][0].as(); + if constexpr(D > 1) + g[1] = config["gravity_acc"][1].as(); + if constexpr(D > 2) + g[2] = config["gravity_acc"][2].as(); + } + execute>(s.gravity_acc(*s.gt), g); + /*--------------------------------------------------------------------------* Particle mass *--------------------------------------------------------------------------*/ execute( - particle_mass(s.gt), config["mean_molecular_weight"].as()); + s.particle_mass(*s.gt), config["mean_molecular_weight"].as()); /*--------------------------------------------------------------------------* Mesh topology allocation. *--------------------------------------------------------------------------*/ // Find out how many levels we can have. + auto get_resolution = [&config](const int dim) { + return opt::resolution.value() == 0 + ? config["levels"][dim].as() + : opt::resolution.value(); + }; // Record lowest level - s.lowest_level = config["lowest_level"].as(); + s.lowest_level = opt::resolution.value() == 0 + ? config["lowest_level"].as() + : opt::resolution.value(); // Find highest level - s.highest_level = config["levels"][0].as(); + s.highest_level = get_resolution(0); if(D == 2 || D == 3) { - s.highest_level = - std::min(s.highest_level, config["levels"][1].as()); + s.highest_level = get_resolution(1); } // if if(D == 3) { - s.highest_level = - std::min(s.highest_level, config["levels"][2].as()); + s.highest_level = get_resolution(2); } // if s.max_num_levels = s.highest_level - s.lowest_level + 1; + std::optional cd; + if(config["color_distribution"]) { + cd = [&sc, + cdcfg = config["color_distribution"].as()]() { + return (FLECSI_BACKEND == FLECSI_BACKEND_legion) || + + util::axes_colors(cdcfg) == sc.runtime().processes() + ? std::optional(cdcfg) + : std::nullopt; + }(); + } // if + { typename mesh::grect geom; geom[0][0] = config["coords"][0][0].as(); @@ -155,61 +237,34 @@ initialize(control_policy & cp) { for(std::size_t i{0}; i < s.max_num_levels; i++) { typename mesh::gcoord axis_extents(D); - axis_extents[ax::x] = 1 << (config["levels"][0].as() - i); + axis_extents[ax::x] = 1 << (get_resolution(0) - i); if(D == 2 || D == 3) { - axis_extents[ax::y] = 1 << (config["levels"][1].as() - i); + axis_extents[ax::y] = 1 << (get_resolution(1) - i); } // if if(D == 3) { - axis_extents[ax::z] = 1 << (config["levels"][2].as() - i); + axis_extents[ax::z] = 1 << (get_resolution(2) - i); } // if // Add a new grid - the finest grid is already there if(i > 0) { - s.mh.emplace_back(std::make_unique::slot>()); + s.mh.emplace_back(typename mesh::ptr()); } // if - // Allocate the grid - s.mh[i]->allocate( - typename mesh::mpi_coloring{num_colors, axis_extents, bf.get()}, - geom); + if(cd.has_value()) { + sc.allocate(s.mh[i], + typename mesh::mpi_coloring( + sc, cd.value(), axis_extents, bf.get()), + geom); + } + else { + sc.allocate(s.mh[i], + typename mesh::mpi_coloring( + sc, sc.runtime().processes(), axis_extents, bf.get()), + geom); + } } // for } // scope - /*--------------------------------------------------------------------------* - Fake initialization to avoid legion warnings. - *--------------------------------------------------------------------------*/ - - // clang-format off - execute>(s.m, - s.mass_density(s.m), s.momentum_density(s.m), s.total_energy_density(s.m), - s.velocity(s.m), s.pressure(s.m), - s.rTail(s.m), s.ruTail(s.m), s.rETail(s.m), s.uTail(s.m), s.pTail(s.m), - s.rHead(s.m), s.ruHead(s.m), s.rEHead(s.m), s.uHead(s.m), s.pHead(s.m), - s.rF(s.m), s.ruF(s.m), s.rEF(s.m) -#ifdef ENABLE_RADIATION - , s.radiation_energy_density(s.m), s.EradTail(s.m), s.EradHead(s.m), s.EradF(s.m), - s.Esf(s.m), s.Ef(s.m), s.Ew(s.m), s.Diff(s.m), s.Resf(s.m), s.Errf(s.m), - s.gradient_rad_energy(s.m), s.magnitude_gradient_rad_energy(s.m), - s.radiation_force(s.m), s.R_value(s.m), s.lambda_bridge(s.m), - s.velocity_gradient(s.m) -#endif - ); - execute>(s.m, - s.mass_density(s.m), s.momentum_density(s.m), s.total_energy_density(s.m), - s.velocity(s.m), s.pressure(s.m), - s.rTail(s.m), s.ruTail(s.m), s.rETail(s.m), s.uTail(s.m), s.pTail(s.m), - s.rHead(s.m), s.ruHead(s.m), s.rEHead(s.m), s.uHead(s.m), s.pHead(s.m), - s.rF(s.m), s.ruF(s.m), s.rEF(s.m) -#ifdef ENABLE_RADIATION - , s.radiation_energy_density(s.m), s.EradTail(s.m), s.EradHead(s.m), s.EradF(s.m), - s.Esf(s.m), s.Ef(s.m), s.Ew(s.m), s.Diff(s.m), s.Resf(s.m), s.Errf(s.m), - s.gradient_rad_energy(s.m), s.magnitude_gradient_rad_energy(s.m), - s.radiation_force(s.m), s.R_value(s.m), s.lambda_bridge(s.m), - s.velocity_gradient(s.m) -#endif - ); - // clang-format on - /*--------------------------------------------------------------------------* Equation of State *--------------------------------------------------------------------------*/ @@ -246,93 +301,206 @@ initialize(control_policy & cp) { Initialize problem state. *--------------------------------------------------------------------------*/ + execute>( + flecsi::exec::on, s.gravity_force(*s.m)); + if(config["problem"].as() == "sod") { + +#ifdef ENABLE_RADIATION + flog_fatal("Sod must be built with ENABLE_RADIATION=OFF"); +#endif + execute< - tasks::initial_data::shock, - flecsi::default_accelerator>(s.m, - s.mass_density(s.m), - s.momentum_density(s.m), - s.total_energy_density(s.m), - s.radiation_energy_density(s.m), + tasks::initial_data::shock>( + flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.momentum_density(*s.m), + s.total_energy_density(*s.m), + s.radiation_energy_density(*s.m), s.eos); } else if(config["problem"].as() == "rankine-hugoniot") { + +#ifdef ENABLE_RADIATION + flog_fatal("Rankine-Hugoniot must be built with ENABLE_RADIATION=OFF"); +#endif + execute, - flecsi::default_accelerator>(s.m, - s.mass_density(s.m), - s.momentum_density(s.m), - s.total_energy_density(s.m), - s.radiation_energy_density(s.m), + shock>( + flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.momentum_density(*s.m), + s.total_energy_density(*s.m), + s.radiation_energy_density(*s.m), + s.eos); + } + else if(config["problem"].as() == "leblanc") { + +#ifdef ENABLE_RADIATION + flog_fatal("Leblanc must be built with ENABLE_RADIATION=OFF"); +#endif + + execute< + tasks::initial_data::shock>( + flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.momentum_density(*s.m), + s.total_energy_density(*s.m), + s.radiation_energy_density(*s.m), s.eos); } - else if(config["problem"].as() == "sine-wave") { - execute, flecsi::default_accelerator>(s.m, - s.mass_density(s.m), - s.momentum_density(s.m), - s.total_energy_density(s.m), - s.radiation_energy_density(s.m), + else if(config["problem"].as() == "acoustic-wave") { + +#ifdef ENABLE_RADIATION + flog_fatal("Acoustic wave must be built with ENABLE_RADIATION=OFF"); +#endif + + execute>(flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.momentum_density(*s.m), + s.total_energy_density(*s.m), + s.radiation_energy_density(*s.m), s.eos); } else if(config["problem"].as() == "kh-test") { - execute>(s.m, - s.mass_density(s.m), - s.momentum_density(s.m), - s.total_energy_density(s.m), - s.radiation_energy_density(s.m), + +#ifdef ENABLE_RADIATION + flog_fatal( + "Kelvin-Helmholtz instability must be built with ENABLE_RADIATION=OFF"); +#endif + + execute>(flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.momentum_density(*s.m), + s.total_energy_density(*s.m), + s.radiation_energy_density(*s.m), + s.eos); + } + // Rayleigh-Taylor setup + else if(config["problem"].as() == "rt-test") { + +#ifdef ENABLE_RADIATION + flog_fatal( + "Rayleigh-Taylor instability must be built with ENABLE_RADIATION=OFF"); +#endif + + execute>(flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.momentum_density(*s.m), + s.gravity_force(*s.m), + s.gravity_acc(*s.gt), + s.total_energy_density(*s.m), + s.radiation_energy_density(*s.m), + s.eos); + } + // Ritchmyer-Meshkov works with both radiation on and off + else if(config["problem"].as() == "richtmyer-meshkov") { + + execute>(flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.momentum_density(*s.m), + s.total_energy_density(*s.m), + s.radiation_energy_density(*s.m), s.eos); } else if(config["problem"].as() == "heating_and_cooling") { if(config["eos"].as() != "ideal") flog_fatal("Heating and cooling test only supports Ideal Gas eos"); - execute>(s.m, - s.mass_density(s.m), - s.momentum_density(s.m), - s.total_energy_density(s.m), - s.radiation_energy_density(s.m), - particle_mass(s.gt), + execute>(flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.momentum_density(*s.m), + s.total_energy_density(*s.m), + s.radiation_energy_density(*s.m), + s.particle_mass(*s.gt), config["gamma"].as()); } - else if(config["problem"].as() == "sedov") { - execute>(s.m, - s.mass_density(s.m), - s.momentum_density(s.m), - s.total_energy_density(s.m), - s.radiation_energy_density(s.m), - particle_mass(s.gt), + // Heating and Cooling for AFLD + else if(config["problem"].as() == "heating-cooling-afld") { + if(config["eos"].as() != "ideal") + flog_fatal("Heating and cooling test only supports Ideal Gas eos"); + sc.execute>( + flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.momentum_density(*s.m), + s.total_energy_density(*s.m), + s.radiation_energy_density(*s.m), + s.particle_mass(*s.gt), config["gamma"].as()); } + else if(config["problem"].as() == "sedov") { + +#ifdef ENABLE_RADIATION + flog_fatal("Sedov blast must be built with ENABLE_RADIATION=OFF"); +#endif + + execute>(flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.momentum_density(*s.m), + s.total_energy_density(*s.m), + s.radiation_energy_density(*s.m)); + } else if(config["problem"].as() == "implosion") { - execute>(s.m, - s.mass_density(s.m), - s.momentum_density(s.m), - s.total_energy_density(s.m), - s.radiation_energy_density(s.m), - temperature_boundary(s.dense_topology), - particle_mass(s.gt), + execute>(flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.momentum_density(*s.m), + s.total_energy_density(*s.m), + s.radiation_energy_density(*s.m), + s.temperature_boundary(*s.dense_topology), + s.particle_mass(*s.gt), config["gamma"].as()); s.mg = true; } -#if 0 + // FIXME: This problem has not been tested for correctness else if(config["problem"].as() == "rad-rh") { execute>(s.m, - s.mass_density(s.m), - s.momentum_density(s.m), - s.total_energy_density(s.m), - s.radiation_energy_density(s.m), - gamma(s.gt), - particle_mass(s.gt)); + rad_RH>( + flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.momentum_density(*s.m), + s.total_energy_density(*s.m), + s.radiation_energy_density(*s.m), + config["gamma"].as(), + s.particle_mass(*s.gt)); } + // FIXME: This problem has not been tested for correctness else if(config["problem"].as() == "lw-implosion") { - execute>(s.m, - s.mass_density(s.m), - s.momentum_density(s.m), - s.total_energy_density(s.m), - s.radiation_energy_density(s.m), - gamma(s.gt)); - } + +#ifdef ENABLE_RADIATION + flog_fatal( + "Liska-Wendroff implosion must be built with ENABLE_RADIATION=OFF"); #endif + + execute>(flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.momentum_density(*s.m), + s.total_energy_density(*s.m), + s.radiation_energy_density(*s.m), + config["gamma"].as()); + } + // Kelvin Helmholtz with radiation setup + else if(config["problem"].as() == "kh-rad-test") { + + execute>(flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.momentum_density(*s.m), + s.total_energy_density(*s.m), + s.radiation_energy_density(*s.m), + s.eos); + } else { flog_fatal( "unsupported problem(" << config["problem"].as() << ")"); @@ -342,69 +510,79 @@ initialize(control_policy & cp) { Initialize time advance. *--------------------------------------------------------------------------*/ - execute, flecsi::default_accelerator>(s.m, - s.bmap(s.gt), - s.mass_density(s.m), - s.velocity(s.m), - s.pressure(s.m), - s.radiation_energy_density(s.m), - s.momentum_density(s.m), - s.total_energy_density(s.m)); - + sc.execute>(flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.momentum_density(*s.m), + s.total_energy_density(*s.m), + s.velocity(*s.m), + s.pressure(*s.m), + s.specific_internal_energy(*s.m), + s.sound_speed(*s.m), + s.eos); + auto lmax_f = sc.execute>( + flecsi::exec::on, + *s.m, + s.mass_density(*s.m), + s.velocity(*s.m), + s.sound_speed(*s.m)); + s.dtmin_ = reduce, exec::fold::min>( + flecsi::exec::on, *s.m, lmax_f); + + sc.execute>(flecsi::exec::on, + *s.m, + s.bmap(*s.gt), + std::vector{s.mass_density(*s.m), + s.pressure(*s.m), + s.specific_internal_energy(*s.m), + s.radiation_energy_density(*s.m), + s.total_energy_density(*s.m)}, + std::vector{s.velocity(*s.m), s.momentum_density(*s.m)}); if(s.mg) { // FIXME: figure out how not to use the hardcoded radiation temperature // boundary auto radiation_boundary_f = - flecsi::execute(s.t(s.gt), - time_boundary(s.dense_topology), - temperature_boundary(s.dense_topology)); - flecsi::execute, - flecsi::default_accelerator>( - s.m, s.radiation_energy_density(s.m), radiation_boundary_f); + sc.execute(flecsi::exec::on, + s.t(*s.gt), + s.time_boundary(*s.dense_topology), + s.temperature_boundary(*s.dense_topology)); + sc.execute>(flecsi::exec::on, + *s.m, + s.bmap(*s.gt), + std::vector{s.radiation_energy_density(*s.m)}, + radiation_boundary_f); } - execute, - flecsi::default_accelerator>(s.m, - s.mass_density(s.m), - s.momentum_density(s.m), - s.total_energy_density(s.m), - s.velocity(s.m), - s.pressure(s.m), - s.specific_internal_energy(s.m), - s.sound_speed(s.m), - s.eos); - auto lmax_f = execute, - flecsi::default_accelerator>(s.m, - s.mass_density(s.m), - s.velocity(s.m), - s.pressure(s.m), - s.sound_speed(s.m)); - s.dtmin_ = - reduce, exec::fold::min>(s.m, lmax_f); /*--------------------------------------------------------------------------* Initialize time to 0 *--------------------------------------------------------------------------*/ - flecsi::execute(s.t(s.gt), config["t0"].as()); + sc.execute( + flecsi::exec::on, s.t(*s.gt), config["t0"].as()); /*--------------------------------------------------------------------------* Save raw data after initialization, at t=t_i *--------------------------------------------------------------------------*/ #ifndef HARD_BENCHMARK_MODE - auto lm = data::launch::make(s.m); - execute, mpi>( - spec::io::name{"output-"} << std::setfill('0') << std::setw(5) << cp.step(), - s.t(s.gt), + auto lm = data::launch::make(sc, *s.m); + execute, mpi>(flecsi::exec::on, + spec::io::name{""} << std::setfill('0') << std::setw(5) << cp.step(), + s.t(*s.gt), lm, - s.mass_density(lm), - s.pressure(lm), - s.sound_speed(lm), - s.specific_internal_energy(lm), - s.velocity(lm), - s.momentum_density(lm), - s.total_energy_density(lm), - s.radiation_energy_density(lm)); - + std::vector{s.mass_density(lm), + s.pressure(lm), + s.sound_speed(lm), + s.specific_internal_energy(lm), + s.total_energy_density(lm), + s.radiation_energy_density(lm)}, + std::vector{s.velocity(lm), s.momentum_density(lm)}, + std::vector{"density", + "pressure", + "sound_speed", + "specific_internal_energy", + "total_energy_density", + "radiation_energy_density"}, + std::vector{"velocity", "momentum_density"}); #endif /*--------------------------------------------------------------------------* Initialize catalyst: Build the lattice, initialize adaptor, parse yaml file @@ -414,15 +592,16 @@ initialize(control_policy & cp) { #ifdef USE_CATALYST if constexpr(D == 3) { // Initialize catalyst - execute(); + execute(flecsi::exec::on); + sc.allocate(s.pt, num_colors); // Initialize lattice - execute, mpi>(s.m); + execute, mpi>(flecsi::exec::on, *s.m); flog(info) << "init action, catalyst: got lattice dimensions from hard: " << number_vertices[0] << " x " << number_vertices[1] << " x " << number_vertices[2] << " vertices." << std::endl; - execute, mpi>(s.m); + execute, mpi>(flecsi::exec::on, *s.m); flog(info) << "init action, catalyst: got color dimensions from hard: " << number_colors[0] << " x " << number_colors[1] << " x " << number_colors[2] << " color blocks." << std::endl; @@ -462,7 +641,8 @@ initialize(control_policy & cp) { */ // Initialize catalyst data structure - execute(catalyst_data(pt), + execute(flecsi::exec::on, + s.catalyst_data(*s.pt), lattice.get_number_of_points(), lattice.get_number_of_cells()); @@ -484,18 +664,19 @@ initialize(control_policy & cp) { } // Send initial problem state to catalyst - execute, mpi>(catalyst_data(pt), - s.t(s.gt), - s.m, - s.mass_density(s.m), - s.velocity(s.m), - s.pressure(s.m), - s.total_energy_density(s.m), - s.radiation_energy_density(s.m)); // <<< add variables here for catalyst + execute, mpi>(flecsi::exec::on, + s.catalyst_data(*s.pt), + s.t(*s.gt), + *s.m, + s.mass_density(*s.m), + s.velocity(*s.m), + s.pressure(*s.m), + s.total_energy_density(*s.m), + s.radiation_energy_density(*s.m)); // <<< add variables here for catalyst flog(info) << "init action, catalyst: execute catalyst for initial state" << std::endl; execute( - catalyst_data(pt), 0, s.t(s.gt), lattice); + flecsi::exec::on, s.catalyst_data(*s.pt), 0, s.t(*s.gt), lattice); } else { /* Do nothing, catalyst/paraview visualization only for 3D */ diff --git a/app/linsolve.hh b/app/linsolve.hh new file mode 100644 index 0000000..c5b73a9 --- /dev/null +++ b/app/linsolve.hh @@ -0,0 +1,95 @@ +#ifndef LINSOLVE_HH +#define LINSOLVE_HH + +#include "flecsi/execution.hh" +#include "flecsi/flog.hh" + +#include "rad.hh" +#include "state.hh" + +#ifdef USE_FLECSOLVE +#include "flecsolvers.hh" +#endif + +namespace hard::rad { + +#if defined(USE_FLECSOLVE) && USE_FLECSOLVE + +template +auto +make_solver(control_policy & cp) { + + auto & s = cp.state(); + using namespace flecsolve; + + // Solver parameters + double temp{0.0}; + + // Only pass cp instead? + solver_parameters params{ + std::ref(cp.state()), std::ref(cp.scheduler()), temp}; + op::core> so(params); + auto op_handle = op::ref(so); + + precond_parameters pparams{std::ref(s), + std::ref(cp.scheduler()), + 0, + s.jacobi_iterations, + s.nr_vcycles}; + op::core> po(pparams); + auto prec_handle = op::ref(po); + + auto f = flecsolve::vec::make(s.Ef(*s.m)); + + auto slv = flecsolve::bicgstab::solver( + s.solver_settings, flecsolve::bicgstab::make_work(f))( + op_handle, prec_handle, [&](auto &, double rnorm) { return false; }); + return slv; +} +#endif + +template +void +linsolve(control_policy & cp) { + + flecsi::scheduler & sc = cp.scheduler(); + auto & s = cp.state(); + auto & mf = *s.m; + + if(s.full_multigrid) { + fmg(cp); + +#if defined(USE_FLECSOLVE) && USE_FLECSOLVE + sc.execute>( + flecsi::exec::on, mf, s.Ew(mf), s.Esf(mf), s.Ef(mf), s.Resf(mf)); + auto r = flecsolve::vec::make(s.Resf(mf)); + flog(info) << "final res norm radiation: " << r.l2norm().get() << std::endl; +#endif + } + else { +#if defined(USE_FLECSOLVE) && USE_FLECSOLVE + // flecsolve vectors + auto f = flecsolve::vec::make(s.Ef(mf)); + auto u = flecsolve::vec::make(s.Uf(mf)); + + /* Sets up a new linear solver at each iteration (this should not be + * done).*/ + auto slv = make_solver(cp); + auto slv_info = slv(f, u); + auto iters = slv_info.iters; + auto res_norm_final = slv_info.res_norm_final; + + flog(info) << "final res norm radiation (flecsolve): " << res_norm_final + << " iter: " << iters << std::endl; + + sc.execute>( + flecsi::exec::on, mf, s.Ew(mf), s.Uf(mf), s.Ef(mf), s.Resf(mf)); + auto r = flecsolve::vec::make(s.Resf(mf)); + flog(info) << "final res norm radiation: " << r.l2norm().get() << std::endl; + +#endif + } +} // linsolve +} // namespace hard::rad + +#endif diff --git a/app/numerical_algorithms/riemann_solvers.hh b/app/numerical_algorithms/riemann_solvers.hh index 5da2b60..2e2b9fb 100644 --- a/app/numerical_algorithms/riemann_solvers.hh +++ b/app/numerical_algorithms/riemann_solvers.hh @@ -2,85 +2,206 @@ #pragma once #include -#include +#include #include namespace hard::numerical_algorithms { -// -// Returns HLL Riemann fluxes for the hydro part -// -template -FLECSI_INLINE_TARGET std::tuple, double> -hll_hydro( - // Evolved (conserved) variables - const double mass_density_left, - const double mass_density_right, - const spec::vec & momentum_density_left, - const spec::vec & momentum_density_right, - const double total_energy_density_left, - const double total_energy_density_right, - // Fluxes - const double flux_mass_density_left, - const double flux_mass_density_right, - const spec::vec & flux_momentum_density_left, - const spec::vec & flux_momentum_density_right, - const double flux_total_energy_density_left, - const double flux_total_energy_density_right, - // characteristic speeds from left and right states - double min_char_speed_left, - double max_char_speed_left, - double min_char_speed_right, - double max_char_speed_right) { - - std::tuple, double> result{}; - - const double lambda_max = - std::max(0.0, std::max(max_char_speed_left, max_char_speed_right)); - const double lambda_min = - std::min(0.0, std::min(min_char_speed_left, min_char_speed_right)); - - const double lambdas_product = lambda_max * lambda_min; - const double one_over_lambda_max_minus_lambda_min = - 1.0 / (lambda_max - lambda_min); - - get<0>(result) = - (lambda_max * flux_mass_density_left - - lambda_min * flux_mass_density_right + - lambdas_product * (mass_density_right - mass_density_left)) * - one_over_lambda_max_minus_lambda_min; - - // Note : vector operation is done here - get<1>(result) = - (lambda_max * flux_momentum_density_left - - lambda_min * flux_momentum_density_right + - lambdas_product * (momentum_density_right - momentum_density_left)) * - one_over_lambda_max_minus_lambda_min; - - get<2>(result) = (lambda_max * flux_total_energy_density_left - - lambda_min * flux_total_energy_density_right + - lambdas_product * (total_energy_density_right - - total_energy_density_left)) * - one_over_lambda_max_minus_lambda_min; - - return result; +// Applies hll fluxes to a homogeneous conservation equation +template +FLECSI_INLINE_TARGET T +advect_conserved(const T left, + const T right, + const T flux_left, + const T flux_right, + const double LminT, + const double LmaxT, + const double LminH, + const double LmaxH) { + + const double Lmax = std::max(0.0, std::max(LmaxT, LmaxH)); + const double Lmin = std::min(0.0, std::min(LminT, LminH)); + + return (Lmax * flux_left - Lmin * flux_right + Lmax * Lmin * (right - left)) / + (Lmax - Lmin); +} + +// Compute S* for HLLC flux computation +FLECSI_INLINE_TARGET std::tuple +compute_S_star(const double uL, + const double uR, + const double rhoL, + const double rhoR, + const double pL, + const double pR, + const double cL, + const double cR) { + + const double sqrt_rhoL = std::sqrt(rhoL); + const double sqrt_rhoR = std::sqrt(rhoR); + const double denom = sqrt_rhoL + sqrt_rhoR; + + const double u_hat = (uL * sqrt_rhoL + uR * sqrt_rhoR) / denom; + const double c_hat = std::sqrt( + (cL * cL * sqrt_rhoL + cR * cR * sqrt_rhoR) / denom + + (sqrt_rhoL * sqrt_rhoR) / (2.0 * denom * denom) * (uR - uL) * (uR - uL)); + + const double SL = std::min(uL - cL, u_hat - c_hat); + const double SR = std::max(uR + cR, u_hat + c_hat); + + const double S_star = + (pR - pL + rhoL * uL * (SL - uL) - rhoR * uR * (SR - uR)) / + (rhoL * (SL - uL) - rhoR * (SR - uR)); + + return std::make_tuple(SL, SR, S_star); +} + +// Compute U* for HLLC flux computation +template +FLECSI_INLINE_TARGET std::pair +compute_U_star(const double uL_n, + const double uR_n, + const vec ul_star, + const vec ur_star, + const double rhoL, + const double rhoR, + const double EL, + const double ER, + const double pL, + const double pR, + const double SL, + const double SR, + const double S_star, + const std::string & var_name) { + + T UL_star(0.0); + T UR_star(0.0); + + double const facL = rhoL * (SL - uL_n) / (SL - S_star); + double const facR = rhoR * (SR - uR_n) / (SR - S_star); + + if constexpr(std::is_same_v) { + if(var_name == "rho") { + UL_star = facL; + UR_star = facR; + } + else if(var_name == "E") { + UL_star = facL * (EL / rhoL + (S_star - uL_n) * + (S_star + pL / (rhoL * (SL - uL_n)))); + UR_star = facR * (ER / rhoR + (S_star - uR_n) * + (S_star + pR / (rhoR * (SR - uR_n)))); + } + else { + assert(false && "Invalid var_name for double star state"); + } + } + else if constexpr(std::is_same_v>) { + if(var_name == "rhou") { + UL_star = facL * ul_star; + UR_star = facR * ur_star; + } + else { + + assert(false && "Invalid var_name for double star state"); + } + } + return {UL_star, UR_star}; } -// Advection of radiation energy is perfomed using -// expressions 20 and 21 from -// Yang and Yuan, Publ. Astron. Soc. Japan 64, 69, 2012 August 25 -FLECSI_INLINE_TARGET static double -advect_Erad(const double Erad_left, - const double Erad_right, - const double speed_left, - const double speed_right) { - - const double speed_max = - std::max(std::abs(speed_left), std::abs(speed_right)); - const double f_Erad_left{Erad_left * speed_left}; - const double f_Erad_right{Erad_right * speed_right}; - - return 0.5 * - (f_Erad_right + f_Erad_left - speed_max * (Erad_right - Erad_left)); + +// Computes F* for HLLC flux computation +template +FLECSI_INLINE_TARGET T +compute_F_star(const T FL, + const T FR, + const T QL, + const T QR, + const T UL_star, + const T UR_star, + const double SL, + const double SR, + const double S_star) { + + // Star fluxes + const T FL_star = FL + SL * (UL_star - QL); + const T FR_star = FR + SR * (UR_star - QR); + + // Final flux selection + if(0.0 <= SL) { + return FL; + } + else if(SL <= 0.0 && 0.0 <= S_star) { + return FL_star; + } + else if(S_star <= 0.0 && 0.0 <= SR) { + return FR_star; + } + else { + return FR; + } +} + +// computes and applies HLLC fluxes to a homogeneous conservation equation +template +FLECSI_INLINE_TARGET T +compute_HLLC_fluxes(std::size_t fa, + const T QL, + const double rhoL, + const vec uL, + const double EL, + const double pL, + const double cL, + const T FL, + const T QR, + const double rhoR, + const vec uR, + const double ER, + const double pR, + const double cR, + const T FR, + const std::string & var_name) { + + double uL_n = 0.0, uR_n = 0.0; + + if(fa == 0) { + uL_n = uL.x(); + uR_n = uR.x(); + } + if constexpr(Dim > 1) + if(fa == 1) { + uL_n = uL.y(); + uR_n = uR.y(); + } + if constexpr(Dim > 2) + if(fa == 2) { + uL_n = uL.z(); + uR_n = uR.z(); + } + + auto [SL, SR, S_star] = + compute_S_star(uL_n, uR_n, rhoL, rhoR, pL, pR, cL, cR); + + vec ul_star = uL, ur_star = uR; + if(fa == 0) { + ul_star.x() = S_star; + ur_star.x() = S_star; + } + if constexpr(Dim > 1) + if(fa == 1) { + ul_star.y() = S_star; + ur_star.y() = S_star; + } + if constexpr(Dim > 2) + if(fa == 2) { + ul_star.z() = S_star; + ur_star.z() = S_star; + } + + // clang-format off + auto [UL_star, UR_star] = compute_U_star(uL_n, uR_n, + ul_star, ur_star, rhoL, rhoR, EL, ER, pL,pR, SL, SR, S_star, var_name); + // clang-format on + + return compute_F_star(FL, FR, QL, QR, UL_star, UR_star, SL, SR, S_star); } } // namespace hard::numerical_algorithms diff --git a/app/options.hh b/app/options.hh index 349d961..0dcad90 100644 --- a/app/options.hh +++ b/app/options.hh @@ -31,6 +31,11 @@ inline flecsi::program_option dimension("Hard Options", "Specify the dimension of the solver (default: 3).", {{flecsi::option_default, 3}}); +inline flecsi::program_option resolution("Hard Options", + "resolution,r", + "Specify the lowest level resolution for the grid.", + {{flecsi::option_default, 0}}); + inline flecsi::program_option source_fds("Hard Options", "fds_file", "Specify where the file lives (default: source_fds.txt)", diff --git a/app/rad.hh b/app/rad.hh index 0945c80..b90f17d 100644 --- a/app/rad.hh +++ b/app/rad.hh @@ -10,7 +10,9 @@ namespace hard::rad { template void -vcycle(state & s, std::size_t index) { +vcycle(control_policy & cp, std::size_t index) { + auto & s = cp.state(); + flecsi::scheduler & sc = cp.scheduler(); auto & mf = *s.mh[index]; // Find current level @@ -19,7 +21,8 @@ vcycle(state & s, std::size_t index) { if(level == s.lowest_level) { // FIXME: Remove when finished with debugging - // flog(warn) << "Direct solve level(index): " << level << "(" << index << ")" + // flog(warn) << "Direct solve level(index): " << level << "(" << index << + // ")" // << std::endl; // Direct solve for a single interior point @@ -27,10 +30,8 @@ vcycle(state & s, std::size_t index) { s.Esf.flip(); // NOTE: We are defaulting to damped_jacobi until gauss-seidel is // parallelized - flecsi::execute, flecsi::default_accelerator>( - mf, s.Ew(mf), s.Esf(mf), s.Esf(mf, 1), s.Ef(mf), 0.8); - // flecsi::execute>(mf, s.Esf(mf)); - // flecsi::execute>(mf, s.Esf(mf, 1)); + sc.execute>( + flecsi::exec::on, mf, s.Ew(mf), s.Esf(mf), s.Esf(mf, 1), s.Ef(mf), 0.8); } // for } else { @@ -44,51 +45,44 @@ vcycle(state & s, std::size_t index) { // Pre Smoothing for(std::size_t i{0}; i < s.mg_pre; ++i) { s.Esf.flip(); - flecsi::execute, flecsi::default_accelerator>( - mf, s.Ew(mf), s.Esf(mf), s.Esf(mf, 1), s.Ef(mf), 0.8); - // flecsi::execute>(mf, s.Esf(mf)); - // flecsi::execute>(mf, s.Esf(mf, 1)); + sc.execute>( + flecsi::exec::on, mf, s.Ew(mf), s.Esf(mf), s.Esf(mf, 1), s.Ef(mf), 0.8); } // for // Recursive solve - flecsi::execute, flecsi::default_accelerator>( - mf, s.Ew(mf), s.Esf(mf), s.Ef(mf), s.Resf(mf)); - // flecsi::execute>(mf, s.Resf(mf)); + sc.execute>( + flecsi::exec::on, mf, s.Ew(mf), s.Esf(mf), s.Ef(mf), s.Resf(mf)); - flecsi::execute, flecsi::default_accelerator>( - mf, mc, s.Resf(mf), s.Ef(mc)); - // flecsi::execute>(mc, s.Ef(mc)); + sc.execute>( + flecsi::exec::on, mf, mc, s.Resf(mf), s.Ef(mc)); // Initialize the solution fields for the coarser level - flecsi::execute, flecsi::default_accelerator>( - mc, s.Esf(mc), 0.0); - flecsi::execute, flecsi::default_accelerator>( - mc, s.Esf(mc, 1), 0.0); + sc.execute>(flecsi::exec::on, mc, s.Esf(mc), 0.0); + sc.execute>( + flecsi::exec::on, mc, s.Esf(mc, 1), 0.0); - vcycle(s, index + 1); + vcycle(cp, index + 1); - flecsi::execute, - flecsi::default_accelerator>(mc, mf, s.Esf(mc), s.Errf(mf)); - // flecsi::execute>(mf, s.Errf(mf)); + sc.execute>( + flecsi::exec::on, mc, mf, s.Esf(mc), s.Errf(mf)); - flecsi::execute, flecsi::default_accelerator>( - mf, s.Esf(mf), s.Errf(mf)); - // flecsi::execute>(mf, s.Errf(mf)); + sc.execute>( + flecsi::exec::on, mf, s.Esf(mf), s.Errf(mf)); // Post Smoothing for(std::size_t i{0}; i < s.mg_post; ++i) { s.Esf.flip(); - flecsi::execute, flecsi::default_accelerator>( - mf, s.Ew(mf), s.Esf(mf), s.Esf(mf, 1), s.Ef(mf), 0.8); - // flecsi::execute>(mf, s.Esf(mf)); - // flecsi::execute>(mf, s.Esf(mf, 1)); + sc.execute>( + flecsi::exec::on, mf, s.Ew(mf), s.Esf(mf), s.Esf(mf, 1), s.Ef(mf), 0.8); } // for } // if } // vcycle template void -fmg(state & s, std::size_t index = 0) { +fmg(control_policy & cp, std::size_t index = 0) { + auto & s = cp.state(); + flecsi::scheduler & sc = cp.scheduler(); auto & mf = *s.mh[index]; // The scheme requires: @@ -108,7 +102,7 @@ fmg(state & s, std::size_t index = 0) { // << std::endl; // If in the deepest level, the V-Cycle is already doing a direct solve - vcycle(s, index); + vcycle(cp, index); } else { @@ -118,41 +112,39 @@ fmg(state & s, std::size_t index = 0) { auto & mc = *s.mh[index + 1]; // Set the RHS and solution field - flecsi::execute, flecsi::default_accelerator>( - mf, mc, s.Ef(mf), s.Ef(mc)); - flecsi::execute, flecsi::default_accelerator>( - mf, mc, s.Esf(mf), s.Esf(mc)); - - // flecsi::execute>(mc, s.Ef(mc)); - // flecsi::execute>(mc, s.Esf(mc)); + sc.execute>( + flecsi::exec::on, mf, mc, s.Ef(mf), s.Ef(mc)); + sc.execute>( + flecsi::exec::on, mf, mc, s.Esf(mf), s.Esf(mc)); // Set the diffusion coefficient and the stencil (TODO) - flecsi::execute, flecsi::default_accelerator>( - mf, mc, s.Df_x(mf), s.Df_x(mc)); - // flecsi::execute>(mc, s.Df_x(mc)); + sc.execute>( + flecsi::exec::on, mf, mc, s.Df_x(mf), s.Df_x(mc)); - flecsi::execute, flecsi::default_accelerator>( - mf, mc, s.Df_y(mf), s.Df_y(mc)); - // flecsi::execute>(mc, s.Df_y(mc)); + sc.execute>( + flecsi::exec::on, mf, mc, s.Df_y(mf), s.Df_y(mc)); - flecsi::execute, flecsi::default_accelerator>( - mf, mc, s.Df_z(mf), s.Df_z(mc)); - // flecsi::execute>(mc, s.Df_z(mc)); + sc.execute>( + flecsi::exec::on, mf, mc, s.Df_z(mf), s.Df_z(mc)); - flecsi::execute, flecsi::default_accelerator>( - mc, s.Df_x(mc), s.Df_y(mc), s.Df_z(mc), s.Ew(mc), s.dt(s.gt)); + sc.execute>(flecsi::exec::on, + mc, + s.Df_x(mc), + s.Df_y(mc), + s.Df_z(mc), + s.Ew(mc), + s.dt(*s.gt)); // Now call solve for one level deeper - fmg(s, index + 1); + fmg(cp, index + 1); // Interpolate solution back up (RHS does not change) - flecsi::execute, - flecsi::default_accelerator>(mc, mf, s.Esf(mc), s.Esf(mf)); - // flecsi::execute>(mf, s.Esf(mf)); + sc.execute>( + flecsi::exec::on, mc, mf, s.Esf(mc), s.Esf(mf)); // Do a V-Cycle for(std::size_t i{0}; i < s.mg_cycles; ++i) { - vcycle(s, index); + vcycle(cp, index); } // for } // if } // fmg diff --git a/app/state.hh b/app/state.hh index 3bc0db9..ff051d3 100644 --- a/app/state.hh +++ b/app/state.hh @@ -7,18 +7,15 @@ #include "catalyst/types.hh" #endif // USE_CATALYST -namespace hard { - -/*----------------------------------------------------------------------------* - Global parameters. - *----------------------------------------------------------------------------*/ - -#ifdef ENABLE_RADIATION -inline const single::definition kappa; +#if defined(USE_FLECSOLVE) && USE_FLECSOLVE +#include "flecsolve/operators/core.hh" +#include "flecsolve/solvers/cg.hh" +#include "flecsolve/solvers/factory.hh" +#include "flecsolve/solvers/gmres.hh" +#include "flecsolve/vectors/topo_view.hh" #endif -inline const single::definition particle_mass; -inline const field::definition time_boundary; -inline const field::definition temperature_boundary; + +namespace hard { /*----------------------------------------------------------------------------* Problem state. @@ -38,31 +35,51 @@ struct state { /*--------------------------------------------------------------------------* Topology slots. *--------------------------------------------------------------------------*/ - index::slot ct; /* Color topology. */ - flecsi::topo::global::slot gt; - flecsi::topo::global::slot dense_topology; + flecsi::topo::index::ptr ct; /* Color topology. */ + flecsi::topo::global::ptr gt; + flecsi::topo::global::ptr dense_topology; - // Vector for meshes in multigrid - std::vector::slot>> mh; +#ifdef USE_CATALYST + flecsi::topo::index::ptr pt; + /*----------------------------------------------------------------------------* + Register the catalyst_data_structure on the "index" topology. + *----------------------------------------------------------------------------*/ + static inline single::definition + catalyst_data; + +#endif // USE_CATALYST + + // Deque for meshes in multigrid + std::deque::ptr> mh; // Define the fine grid - typename mesh::slot & m = - *mh.emplace_back(std::make_unique::slot>()); + typename mesh::ptr & m = mh.emplace_back(typename mesh::ptr()); /*--------------------------------------------------------------------------* Global parameters. *--------------------------------------------------------------------------*/ - static inline const single::bmap>::template definition< - global> + static inline const typename single< + typename mesh::bmap>::template definition bmap; +#ifdef ENABLE_RADIATION + static inline const single::definition kappa; + static inline const single::definition limiter_id; + static inline const single::definition closure_id; +#endif + static inline const single::definition particle_mass; + static inline const field::definition time_boundary; + static inline const field::definition temperature_boundary; + + static inline const typename single>::template definition + gravity_acc; /*--------------------------------------------------------------------------* Color parameters (One per color using an index topology instance). *--------------------------------------------------------------------------*/ /* Maximum characteristic speed for a color. */ - static inline const single>::template definition lmax; + static inline const typename single>::template definition lmax; static inline const single::template definition dt, t, dt_weighted; @@ -74,7 +91,8 @@ struct state { // Conserved quantities. static inline const field::definition, is::cells> mass_density; - static inline const field>::template definition, is::cells> + static inline const typename field>::template definition, + is::cells> momentum_density; static inline const field::definition, is::cells> total_energy_density; @@ -82,7 +100,8 @@ struct state { radiation_energy_density; // Primitives. - static inline const field>::template definition, is::cells> + static inline const typename field>::template definition, + is::cells> velocity; // u static inline const field::definition, is::cells> pressure; // p @@ -91,16 +110,18 @@ struct state { static inline const field::definition, is::cells> sound_speed; // c static inline const field::definition, is::cells> - temperature; // c + temperature; // t // Faces. static inline const field::definition, is::cells> eTail; static inline const field::definition, is::cells> cTail; static inline const field::definition, is::cells> rTail; - static inline const field>::template definition, is::cells> + static inline const typename field>::template definition, + is::cells> ruTail; static inline const field::definition, is::cells> rETail; - static inline const field>::template definition, is::cells> + static inline const typename field>::template definition, + is::cells> uTail; static inline const field::definition, is::cells> pTail; static inline const field::definition, is::cells> EradTail; @@ -108,31 +129,39 @@ struct state { static inline const field::definition, is::cells> eHead; static inline const field::definition, is::cells> cHead; static inline const field::definition, is::cells> rHead; - static inline const field>::template definition, is::cells> + static inline const typename field>::template definition, + is::cells> ruHead; static inline const field::definition, is::cells> rEHead; - static inline const field>::template definition, is::cells> + static inline const typename field>::template definition, + is::cells> uHead; static inline const field::definition, is::cells> pHead; static inline const field::definition, is::cells> EradHead; // Riemann fluxes. static inline const field::definition, is::cells> rF; - static inline const field>::template definition, is::cells> + static inline const typename field>::template definition, + is::cells> ruF; static inline const field::definition, is::cells> rEF; static inline const field::definition, is::cells> EradF; // Radiation pressure (P^{ij}) - static inline const field>::template definition, is::cells> radiation_pressure_tensor; // Variables related to the diffusion (multigrid) solver - static inline dual_field Esf; // solution field + static inline dual_field Esf; // Temp solution field in multigrid + static inline const field::template definition, is::cells> + Uf; // Outer solution field static inline const field::template definition, is::cells> Ef; // RHS of Au=f - static inline const field>::template definition, is::cells> + static inline const field::template definition, is::cells> + Ef_temp; // RHS of Au=f in multigrid precond + static inline const typename field>::template definition, + is::cells> Ew; // stencil weights static inline const field::template definition, is::cells> r; // rho @@ -167,27 +196,41 @@ struct state { // Cycles std::size_t mg_cycles{1}; + // Jacobi iterations in mg coarse grid + std::size_t jacobi_iterations; + // Gradient of a radiation energy density - static inline const field>::template definition, is::cells> + static inline const typename field>::template definition, + is::cells> gradient_rad_energy; // Magnitude of the gradient of the radiation energy density static inline const field::definition, is::cells> magnitude_gradient_rad_energy; + // Gravity force + static inline const typename field>::template definition, + is::cells> + gravity_force; + // Radiation force - static inline const field>::template definition, is::cells> + static inline const typename field>::template definition, + is::cells> radiation_force; // Dimensionless quantitiy, R static inline const field::definition, is::cells> R_value; - // Flux limiter + // Flux limiter (standard/adaptive) static inline const field::definition, is::cells> lambda_bridge; + // Eddington Factor + static inline const field::definition, is::cells> + eddington_factor; + // Gradient of velocity - static inline const field>::template definition, is::cells> velocity_gradient; @@ -198,7 +241,8 @@ struct state { // - For RK substep 1 static inline const field::definition, is::cells> dt_mass_density; - static inline const field>::template definition, is::cells> + static inline const typename field>::template definition, + is::cells> dt_momentum_density; static inline const field::definition, is::cells> dt_total_energy_density; @@ -211,7 +255,8 @@ struct state { // - For RK substep 2 static inline const field::definition, is::cells> dt_mass_density_2; - static inline const field>::template definition, is::cells> + static inline const typename field>::template definition, + is::cells> dt_momentum_density_2; static inline const field::definition, is::cells> dt_total_energy_density_2; @@ -225,7 +270,8 @@ struct state { // Storing variables at U^n during performing time update static inline const field::definition, is::cells> mass_density_n; - static inline const field>::template definition, is::cells> + static inline const typename field>::template definition, + is::cells> momentum_density_n; static inline const field::definition, is::cells> total_energy_density_n; @@ -236,26 +282,18 @@ struct state { // multigroup implementation (mostly boundaries) bool mg = false; -}; // struct state - -#ifdef USE_CATALYST - -/*----------------------------------------------------------------------------* - Convenience reference of the flecsi pre-defined process_topology slot. The - process_topology slot has the same number of colors as there are processes. - This means that it can be used as an "mpi" topology, i.e., the number of - colors will always match the number of mpi ranks. In this example each - process/color will have its own instance of a type that is registered on it. - *----------------------------------------------------------------------------*/ -inline flecsi::topo::index::slot & pt = flecsi::process_topology; + /*--------------------------------------------------------------------------* + FleCSolve + *--------------------------------------------------------------------------*/ -/*----------------------------------------------------------------------------* - Register the catalyst_data_structure on the "index" topology. - *----------------------------------------------------------------------------*/ -inline const single::definition - catalyst_data; +#if defined(USE_FLECSOLVE) && USE_FLECSOLVE + flecsolve::bicgstab::settings solver_settings; + bool flecsolve_coarse_grid; + std::size_t nr_vcycles = 1; +#endif + bool full_multigrid = false; -#endif // USE_CATALYST +}; // struct state } // namespace hard diff --git a/app/tasks/boundaries/boundary.hh b/app/tasks/boundaries/boundary.hh new file mode 100644 index 0000000..e2d556a --- /dev/null +++ b/app/tasks/boundaries/boundary.hh @@ -0,0 +1,248 @@ +#ifndef TASKS_BOUNDARIES_HH +#define TASKS_BOUNDARIES_HH + +#include +#include + +#include "dirichlet.hh" +#include "flow.hh" +#include "reflective.hh" + +namespace hard::tasks { + +using hard::tasks::util::bl; + +/*----------------------------------------------------------------------------* + Boundary input translation. + *----------------------------------------------------------------------------*/ + +template +typename mesh::periodic_axes +init_boundaries(flecsi::exec::cpu, + typename single::bmap>::template accessor bmap_a, + std::array, D> bnds) { + auto & bmap = *bmap_a; + constexpr auto periodic = mesh::boundary_type::periodic; + + typename mesh::periodic_axes p; + p[ax::x] = false; + if constexpr(D == 2 || D == 3) { + p[ax::y] = false; + } + else if constexpr(D == 3) { + p[ax::z] = false; + } // if + + bmap[ax::x][bl::low] = bnds[ax::x][bl::low]; + bmap[ax::x][bl::high] = bnds[ax::x][bl::high]; + + if((bnds[ax::x][bl::low] == periodic && bnds[ax::x][bl::high] != periodic) || + (bnds[ax::x][bl::low] != periodic && bnds[ax::x][bl::high] == periodic)) + flog_fatal("Wrong periodicity in the boundaries."); + p[ax::x] = + bnds[ax::x][bl::low] == periodic && bnds[ax::x][bl::high] == periodic; + + if constexpr(D == 2 || D == 3) { + bmap[ax::y][bl::low] = bnds[ax::y][bl::low]; + bmap[ax::y][bl::high] = bnds[ax::y][bl::high]; + if((bnds[ax::y][bl::low] == periodic && + bnds[ax::y][bl::high] != periodic) || + (bnds[ax::y][bl::low] != periodic && bnds[ax::y][bl::high] == periodic)) + flog_fatal("Wrong periodicity in the boundaries."); + + p[ax::y] = + bnds[ax::y][bl::low] == periodic && bnds[ax::y][bl::high] == periodic; + } // if + + if constexpr(D == 3) { + bmap[ax::z][bl::low] = bnds[ax::z][bl::low]; + bmap[ax::z][bl::high] = bnds[ax::z][bl::high]; + if((bnds[ax::z][bl::low] == periodic && + bnds[ax::z][bl::high] != periodic) || + (bnds[ax::z][bl::low] != periodic && bnds[ax::z][bl::high] == periodic)) + p[ax::z] = + bnds[ax::z][bl::low] == periodic && bnds[ax::z][bl::high] == periodic; + } // if + + return p; +} // boundaries + +template +void +apply_boundary(flecsi::exec::accelerator & s, + typename mesh::template accessor & m, + typename single::bmap>::template accessor & bmap_a, + std::vector::template accessor> & f_a, + double value = 0) noexcept { + + using hard::tasks::util::get_mdiota_policy; + + const size_t ghost_zone_size = m.ghost_zone_size(); + + const flow f(ghost_zone_size); + const reflective r(ghost_zone_size); + const dirichlet d(ghost_zone_size); + + for(auto f_c : f_a) { + auto f_acc = m.template mdcolex(f_c); + if constexpr(D == 1) { + std::array levels = { + m.template is_low() ? bl::low : bl::none, + m.template is_high() ? bl::high : bl::none}; + for(auto l : levels) + if(l != bl::none) { + std::size_t i = l == bl::low ? 0 : m.template size(); + flecsi::util::iota_view policy{0, 1}; // default execution space + s.executor().forall(x, policy) { + const typename mesh::bmap & bm = *bmap_a; + (void)x; // remove compiler warning for unused + if(bm[0][l] == bd::flow) + f(f_acc, i, l); + if(bm[0][l] == bd::reflecting) + r(f_acc, i, l); + if(bm[0][l] == bd::dirichlet) + d(f_acc, value, i, l); + }; + } + } + else if constexpr(D == 2) { + std::array levels_x = { + m.template is_low() ? bl::low : bl::none, + m.template is_high() ? bl::high : bl::none}; + std::array levels_y = { + m.template is_low() ? bl::low : bl::none, + m.template is_high() ? bl::high : bl::none}; + for(auto l : levels_x) + if(l != bl::none) { + std::size_t i = l == bl::low ? 0 : m.template size(); + s.executor().forall(j, (m.template cells())) { + const typename mesh::bmap & bm = *bmap_a; + if(bm[0][l] == bd::flow) + f(ax::x, f_acc, i, j, l); + if(bm[0][l] == bd::reflecting) + r(ax::x, f_acc, i, j, l); + if(bm[0][l] == bd::dirichlet) + d(ax::x, f_acc, value, i, j, l); + }; + } + for(auto l : levels_y) + if(l != bl::none) { + std::size_t j = l == bl::low ? 0 : m.template size(); + s.executor().forall(i, (m.template cells())) { + const typename mesh::bmap & bm = *bmap_a; + if(bm[1][l] == bd::flow) + f(ax::y, f_acc, i, j, l); + if(bm[1][l] == bd::reflecting) + r(ax::y, f_acc, i, j, l); + if(bm[1][l] == bd::dirichlet) + d(ax::y, f_acc, value, i, j, l); + }; // forall + } + } + else if constexpr(D == 3) { + std::array levels_x = { + m.template is_low() ? bl::low : bl::none, + m.template is_high() ? bl::high : bl::none}; + std::array levels_y = { + m.template is_low() ? bl::low : bl::none, + m.template is_high() ? bl::high : bl::none}; + std::array levels_z = { + m.template is_low() ? bl::low : bl::none, + m.template is_high() ? bl::high : bl::none}; + for(auto l : levels_x) + if(l != bl::none) { + auto mdpolicy_zy = get_mdiota_policy(f_acc, + m.template cells(), + m.template cells()); + std::size_t i = l == bl::low ? 0 : m.template size(); + s.executor().forall(kj, mdpolicy_zy) { + const typename mesh::bmap & bm = *bmap_a; + auto [k, j] = kj; + if(bm[0][l] == bd::flow) + f(ax::x, f_acc, i, j, k, l); + if(bm[0][l] == bd::reflecting) + r(ax::x, f_acc, i, j, k, l); + if(bm[0][l] == bd::dirichlet) + d(ax::x, f_acc, value, i, j, k, l); + }; + } + for(auto l : levels_y) + if(l != bl::none) { + auto mdpolicy_zx = get_mdiota_policy(f_acc, + m.template cells(), + m.template cells()); + std::size_t j = l == bl::low ? 0 : m.template size(); + s.executor().forall(ki, mdpolicy_zx) { + const typename mesh::bmap & bm = *bmap_a; + auto [k, i] = ki; + if(bm[1][l] == bd::flow) + f(ax::y, f_acc, i, j, k, l); + if(bm[1][l] == bd::reflecting) + r(ax::y, f_acc, i, j, k, l); + if(bm[1][l] == bd::dirichlet) + d(ax::y, f_acc, value, i, j, k, l); + }; + } + for(auto l : levels_z) + if(l != bl::none) { + auto mdpolicy_yx = get_mdiota_policy(f_acc, + m.template cells(), + m.template cells()); + std::size_t k = l == bl::low ? 0 : m.template size(); + s.executor().forall(ji, mdpolicy_yx) { + const typename mesh::bmap & bm = *bmap_a; + auto [j, i] = ji; + if(bm[2][l] == bd::flow) + f(ax::z, f_acc, i, j, k, l); + if(bm[2][l] == bd::reflecting) + r(ax::z, f_acc, i, j, k, l); + if(bm[2][l] == bd::dirichlet) + d(ax::z, f_acc, value, i, j, k, l); + }; + } + } + } +} + +template +void +apply_boundaries(flecsi::exec::accelerator s, + typename mesh::template accessor m, + typename single::bmap>::template accessor bmap_a, + std::vector::accessor> f_a, + std::vector>::template accessor> + fv_a) noexcept { + apply_boundary(s, m, bmap_a, f_a); + apply_boundary>(s, m, bmap_a, fv_a); +} + +template +void +apply_boundaries_scalar(flecsi::exec::accelerator s, + typename mesh::template accessor m, + typename single::bmap>::template accessor bmap_a, + std::vector::accessor> f_a) noexcept { + apply_boundary(s, m, bmap_a, f_a); +} + +template +void +apply_boundaries_vector(flecsi::exec::accelerator s, + typename mesh::template accessor m, + typename single::bmap>::template accessor bmap_a, + std::vector>::template accessor> f_a) noexcept { + apply_boundary>(s, m, bmap_a, f_a); +} + +template +void +apply_dirichlet_boundaries(flecsi::exec::accelerator s, + typename mesh::template accessor m, + typename single::bmap>::template accessor bmap_a, + std::vector::accessor> f_a, + flecsi::future value) noexcept { + apply_boundary(s, m, bmap_a, f_a, value.get()); +} + +} // namespace hard::tasks +#endif diff --git a/app/tasks/boundaries/dirichlet.hh b/app/tasks/boundaries/dirichlet.hh new file mode 100644 index 0000000..bd59b7d --- /dev/null +++ b/app/tasks/boundaries/dirichlet.hh @@ -0,0 +1,103 @@ +#ifndef TASKS_BOUNDARIES_DIRICHLET_HH +#define TASKS_BOUNDARIES_DIRICHLET_HH + +#include "../../tasks/utils.hh" +#include "../../types.hh" + +namespace hard::tasks { + +using hard::tasks::util::bl; + +template +struct dirichlet {}; + +template<> +struct dirichlet<1> { + + FLECSI_INLINE_TARGET dirichlet<1>(int gzs) : ghost_zone_size(gzs) {} + + template + FLECSI_INLINE_TARGET void + operator()(flecsi::util::mdcolex a, double v, int i, int level) const { + if(level == bl::low) + for(int m = 0; m < ghost_zone_size; ++m) + a(m) = v; + if(level == bl::high) + for(int m = 0; m < ghost_zone_size; ++m) + a(i - 1 - m) = v; + } + int ghost_zone_size; +}; + +template<> +struct dirichlet<2> { + + FLECSI_INLINE_TARGET dirichlet<2>(int gzs) : ghost_zone_size(gzs) {} + + template + FLECSI_INLINE_TARGET void operator()(int axis, + flecsi::util::mdcolex a, + double v, + int i, + int j, + int level) const { + if(level == bl::low) { + if(axis == ax::x) + for(int m = 0; m < ghost_zone_size; ++m) + a(m, j) = v; + if(axis == ax::y) + for(int m = 0; m < ghost_zone_size; ++m) + a(i, m) = v; + } + if(level == bl::high) { + if(axis == ax::x) + for(int m = 0; m < ghost_zone_size; ++m) + a(i - 1 - m, j) = v; + if(axis == ax::y) + for(int m = 0; m < ghost_zone_size; ++m) + a(i, j - 1 - m) = v; + } + } + int ghost_zone_size; +}; + +template<> +struct dirichlet<3> { + + FLECSI_INLINE_TARGET dirichlet<3>(int gzs) : ghost_zone_size(gzs) {} + + template + FLECSI_INLINE_TARGET void operator()(int axis, + flecsi::util::mdcolex a, + double v, + int i, + int j, + int k, + int level) const { + if(level == bl::low) { + if(axis == ax::x) + for(int m = 0; m < ghost_zone_size; ++m) + a(m, j, k) = v; + if(axis == ax::y) + for(int m = 0; m < ghost_zone_size; ++m) + a(i, m, k) = v; + if(axis == ax::z) + for(int m = 0; m < ghost_zone_size; ++m) + a(i, j, m) = v; + } + if(level == bl::high) { + if(axis == ax::x) + for(int m = 0; m < ghost_zone_size; ++m) + a(i - 1 - m, j, k) = v; + if(axis == ax::y) + for(int m = 0; m < ghost_zone_size; ++m) + a(i, j - 1 - m, k) = v; + if(axis == ax::z) + for(int m = 0; m < ghost_zone_size; ++m) + a(i, j, k - 1 - m) = v; + } + } + int ghost_zone_size; +}; +} // namespace hard::tasks +#endif diff --git a/app/tasks/boundaries/flow.hh b/app/tasks/boundaries/flow.hh new file mode 100644 index 0000000..70d190f --- /dev/null +++ b/app/tasks/boundaries/flow.hh @@ -0,0 +1,102 @@ +#ifndef TASKS_BOUNDARIES_FLOW_HH +#define TASKS_BOUNDARIES_FLOW_HH + +#include "../../tasks/utils.hh" +#include "../../types.hh" + +namespace hard::tasks { + +using hard::tasks::util::bl; + +template +struct flow {}; + +template<> +struct flow<1> { + + FLECSI_INLINE_TARGET flow<1>(int gzs) : ghost_zone_size(gzs){}; + + template + FLECSI_INLINE_TARGET void + operator()(flecsi::util::mdcolex a, int i, int level) const { + if(level == bl::low) + for(int m = 0; m < ghost_zone_size; ++m) + a(m) = a(ghost_zone_size); + + if(level == bl::high) + for(int m = 0; m < ghost_zone_size; ++m) + a(i - 1 - m) = a(i - 1 - ghost_zone_size); + } + int ghost_zone_size; +}; + +template<> +struct flow<2> { + + FLECSI_INLINE_TARGET flow<2>(int gzs) : ghost_zone_size(gzs){}; + + template + FLECSI_INLINE_TARGET void operator()(int axis, + flecsi::util::mdcolex a, + int i, + int j, + int level) const { + if(level == bl::low) { + if(axis == ax::x) + for(int m = 0; m < ghost_zone_size; ++m) + a(m, j) = a(ghost_zone_size, j); + if(axis == ax::y) + for(int m = 0; m < ghost_zone_size; ++m) + a(i, m) = a(i, ghost_zone_size); + } + if(level == bl::high) { + if(axis == ax::x) + for(int m = 0; m < ghost_zone_size; ++m) + a(i - 1 - m, j) = a(i - 1 - ghost_zone_size, j); + if(axis == ax::y) + for(int m = 0; m < ghost_zone_size; ++m) + a(i, j - 1 - m) = a(i, j - 1 - ghost_zone_size); + } + } + int ghost_zone_size; +}; + +template<> +struct flow<3> { + + FLECSI_INLINE_TARGET flow<3>(int gzs) : ghost_zone_size(gzs){}; + + template + FLECSI_INLINE_TARGET void operator()(int axis, + flecsi::util::mdcolex a, + int i, + int j, + int k, + int level) const { + if(level == bl::low) { + if(axis == ax::x) + for(int m = 0; m < ghost_zone_size; ++m) + a(m, j, k) = a(ghost_zone_size, j, k); + if(axis == ax::y) + for(int m = 0; m < ghost_zone_size; ++m) + a(i, m, k) = a(i, ghost_zone_size, k); + if(axis == ax::z) + for(int m = 0; m < ghost_zone_size; ++m) + a(i, j, m) = a(i, j, ghost_zone_size); + } + if(level == bl::high) { + if(axis == ax::x) + for(int m = 0; m < ghost_zone_size; ++m) + a(i - 1 - m, j, k) = a(i - 1 - ghost_zone_size, j, k); + if(axis == ax::y) + for(int m = 0; m < ghost_zone_size; ++m) + a(i, j - 1 - m, k) = a(i, j - 1 - ghost_zone_size, k); + if(axis == ax::z) + for(int m = 0; m < ghost_zone_size; ++m) + a(i, j, k - 1 - m) = a(i, j, k - 1 - ghost_zone_size); + } + } + int ghost_zone_size; +}; +} // namespace hard::tasks +#endif diff --git a/app/tasks/boundaries/reflective.hh b/app/tasks/boundaries/reflective.hh new file mode 100644 index 0000000..7f7fa86 --- /dev/null +++ b/app/tasks/boundaries/reflective.hh @@ -0,0 +1,178 @@ +#ifndef TASKS_BOUNDARIES_REFLECTIVE_HH +#define TASKS_BOUNDARIES_REFLECTIVE_HH + +#include "../../tasks/utils.hh" +#include "../../types.hh" + +namespace hard::tasks { + +using hard::tasks::util::bl; + +template +struct reflective {}; + +template<> +struct reflective<1> { + + FLECSI_INLINE_TARGET reflective<1>(int gzs) : ghost_zone_size(gzs){}; + + template + FLECSI_INLINE_TARGET void + operator()(flecsi::util::mdcolex a, int i, int level) const { + if(level == bl::low) { + for(int m = 0; m < ghost_zone_size; ++m) + if constexpr(std::is_same_v) + a(m) = a(2 * ghost_zone_size - 1 - m); + else + a(m) = -1 * a(2 * ghost_zone_size - 1 - m); + } + if(level == bl::high) { + for(int m = 0; m < ghost_zone_size; ++m) + if constexpr(std::is_same_v) + a(i - 1 - m) = a(i - 2 * ghost_zone_size + m); + else + a(i - 1 - m) = -1 * a(i - 2 * ghost_zone_size + m); + } + } + int ghost_zone_size; +}; + +template<> +struct reflective<2> { + + FLECSI_INLINE_TARGET reflective<2>(int gzs) : ghost_zone_size(gzs){}; + + template + FLECSI_INLINE_TARGET void operator()(int axis, + flecsi::util::mdcolex a, + int i, + int j, + int level) const { + if(level == bl::low) { + if(axis == ax::x) { + for(int m = 0; m < ghost_zone_size; ++m) + if constexpr(std::is_same_v) + a(m, j) = a(2 * ghost_zone_size - 1 - m, j); + else { + a(m, j).x() = -1 * a(2 * ghost_zone_size - 1 - m, j).x(); + a(m, j).y() = a(2 * ghost_zone_size - 1 - m, j).y(); + } + } + if(axis == ax::y) { + for(int m = 0; m < ghost_zone_size; ++m) + if constexpr(std::is_same_v) + a(i, m) = a(i, 2 * ghost_zone_size - 1 - m); + else { + a(i, m).x() = a(i, 2 * ghost_zone_size - 1 - m).x(); + a(i, m).y() = -1 * a(i, 2 * ghost_zone_size - 1 - m).y(); + } + } + } + if(level == bl::high) { + if(axis == ax::x) { + for(int m = 0; m < ghost_zone_size; ++m) + if constexpr(std::is_same_v) + a(i - 1 - m, j) = a(i - 2 * ghost_zone_size + m, j); + else { + a(i - 1 - m, j).x() = -1 * a(i - 2 * ghost_zone_size + m, j).x(); + a(i - 1 - m, j).y() = a(i - 2 * ghost_zone_size + m, j).y(); + } + if(axis == ax::y) { + for(int m = 0; m < ghost_zone_size; ++m) + if constexpr(std::is_same_v) + a(i, j - 1 - m) = a(i, j - 2 * ghost_zone_size + m); + else { + a(i, j - 1 - m).x() = a(i, j - 2 * ghost_zone_size + m).y(); + a(i, j - 1 - m).y() = -1 * a(i, j - 2 * ghost_zone_size + m).y(); + } + } + } + } + } + int ghost_zone_size; +}; + +template<> +struct reflective<3> { + + FLECSI_INLINE_TARGET reflective<3>(int gzs) : ghost_zone_size(gzs){}; + + template + FLECSI_INLINE_TARGET void operator()(int axis, + flecsi::util::mdcolex a, + int i, + int j, + int k, + int level) const { + if(level == bl::low) { + if(axis == ax::x) { + for(int m = 0; m < ghost_zone_size; ++m) + if constexpr(std::is_same_v) + a(m, j, k) = a(2 * ghost_zone_size - 1 - m, j, k); + else { + a(m, j, k).x() = -1 * a(2 * ghost_zone_size - 1 - m, j, k).x(); + a(m, j, k).y() = a(2 * ghost_zone_size - 1 - m, j, k).y(); + a(m, j, k).z() = a(2 * ghost_zone_size - 1 - m, j, k).z(); + } + } + if(axis == ax::y) { + for(int m = 0; m < ghost_zone_size; ++m) + if constexpr(std::is_same_v) + a(i, m, k) = a(i, 2 * ghost_zone_size - 1 - m, k); + else { + a(i, m, k).x() = a(i, 2 * ghost_zone_size - 1 - m, k).x(); + a(i, m, k).y() = -1 * a(i, 2 * ghost_zone_size - 1 - m, k).y(); + a(i, m, k).z() = a(i, 2 * ghost_zone_size - 1 - m, k).z(); + } + } + if(axis == ax::z) { + for(int m = 0; m < ghost_zone_size; ++m) + if constexpr(std::is_same_v) + a(i, j, m) = a(i, j, 2 * ghost_zone_size - 1 - m); + else { + a(i, j, m).x() = a(i, j, 2 * ghost_zone_size - 1 - m).x(); + a(i, j, m).y() = a(i, j, 2 * ghost_zone_size - 1 - m).y(); + a(i, j, m).z() = -1 * a(i, j, 2 * ghost_zone_size - 1 - m).z(); + } + } + } + if(level == bl::high) { + if(axis == ax::x) { + for(int m = 0; m < ghost_zone_size; ++m) + if constexpr(std::is_same_v) + a(i - 1 - m, j, k) = a(i - 2 * ghost_zone_size + m, j, k); + else { + a(i - 1 - m, j, k).x() = + -1 * a(i - 2 * ghost_zone_size + m, j, k).x(); + a(i - 1 - m, j, k).y() = a(i - 2 * ghost_zone_size + m, j, k).y(); + a(i - 1 - m, j, k).z() = a(i - 2 * ghost_zone_size + m, j, k).z(); + } + } + if(axis == ax::y) { + for(int m = 0; m < ghost_zone_size; ++m) + if constexpr(std::is_same_v) + a(i, j - 1 - m, k) = a(i, j - 2 * ghost_zone_size + m, k); + else { + a(i, j - 1 - m, k).x() = a(i, j - 2 * ghost_zone_size + m, k).x(); + a(i, j - 1 - m, k).y() = + -1 * a(i, j - 2 * ghost_zone_size + m, k).y(); + a(i, j - 1 - m, k).z() = a(i, j - 2 * ghost_zone_size + m, k).z(); + } + } + if(axis == ax::z) { + for(int m = 0; m < ghost_zone_size; ++m) + if constexpr(std::is_same_v) + a(i, j, k - 1 - m) = a(i, j, k - 2 * ghost_zone_size + m); + else { + a(i, j, k - 1 - m).x() = a(i, j, k - 2 * ghost_zone_size + m).x(); + a(i, j, k - 1 - m).y() = a(i, j, k - 2 * ghost_zone_size + m).y(); + a(i, j, k - 1 - m).z() = + -1 * a(i, j, k - 2 * ghost_zone_size + m).z(); + } + } + } + } + int ghost_zone_size; +}; +} // namespace hard::tasks +#endif diff --git a/app/tasks/boundary.hh b/app/tasks/boundary.hh deleted file mode 100644 index 4d9195e..0000000 --- a/app/tasks/boundary.hh +++ /dev/null @@ -1,758 +0,0 @@ - -#pragma once - -#include "../tasks/utils.hh" -#include "../types.hh" -#include - -#define LOW 0 -#define HIGH 1 - -namespace hard::tasks { - -template -void -apply_boundaries(typename mesh::template accessor m, - typename single::bmap>::template accessor bmap_a, - // primitive variables - field::accessor r_a, - typename field>::template accessor v_a, - field::accessor p_a, - field::accessor Erad_a, - // conservative variables - typename field>::template accessor ru_a, - field::accessor rE_a) { - using hard::tasks::util::get_mdiota_policy; - - auto r = m.template mdcolex(r_a); - auto v = m.template mdcolex(v_a); - auto p = m.template mdcolex(p_a); - auto Erad = m.template mdcolex(Erad_a); - - auto ru = m.template mdcolex(ru_a); - auto rE = m.template mdcolex(rE_a); - - const size_t ghost_zone_size = m.ghost_zone_size(); - - if(m.template is_low()) { // NOTE: true only for boundary colors - if constexpr(D == 1) { - flecsi::util::iota_view policy{0, 1}; // default execution space - forall(x, policy, "bd_x") { - (void)x; // remove compiler warning for unused - - const typename mesh::bmap & bm = *bmap_a; - const auto xlow = bm[0][LOW]; - if(xlow == bd::flow) { // NOTE: Same for all threads/warps in a color. - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(m) = r(ghost_zone_size); - v(m) = v(ghost_zone_size); - p(m) = p(ghost_zone_size); - - ru(m) = ru(ghost_zone_size); - rE(m) = rE(ghost_zone_size); - Erad(m) = Erad(ghost_zone_size); - } - } - else if(xlow == bd::reflecting) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(m) = r(2 * ghost_zone_size - 1 - m); - v(m).x = -v(2 * ghost_zone_size - 1 - m).x; - p(m) = p(2 * ghost_zone_size - 1 - m); - - ru(m).x = -ru(2 * ghost_zone_size - 1 - m).x; - rE(m) = rE(2 * ghost_zone_size - 1 - m); - Erad(m) = Erad(2 * ghost_zone_size - 1 - m); - } - } - }; - } - else if constexpr(D == 2) { - forall(j, (m.template cells()), "bd_x") { - const typename mesh::bmap & bm = *bmap_a; - const auto xlow = bm[0][LOW]; - if(xlow == bd::flow) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(m, j) = r(ghost_zone_size, j); - v(m, j) = v(ghost_zone_size, j); - p(m, j) = p(ghost_zone_size, j); - - ru(m, j) = ru(ghost_zone_size, j); - rE(m, j) = rE(ghost_zone_size, j); - Erad(m, j) = Erad(ghost_zone_size, j); - } - } - else if(xlow == bd::reflecting) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(m, j) = r(2 * ghost_zone_size - 1 - m, j); - v(m, j).x = -v(2 * ghost_zone_size - 1 - m, j).x; - v(m, j).y = v(2 * ghost_zone_size - 1 - m, j).y; - p(m, j) = p(2 * ghost_zone_size - 1 - m, j); - - ru(m, j).x = -ru(2 * ghost_zone_size - 1 - m, j).x; - ru(m, j).y = ru(2 * ghost_zone_size - 1 - m, j).y; - rE(m, j) = rE(2 * ghost_zone_size - 1 - m, j); - Erad(m, j) = Erad(2 * ghost_zone_size - 1 - m, j); - } - } - }; // for - } - else /* D == 3 */ { - auto mdpolicy_zy = get_mdiota_policy(r, - m.template cells(), - m.template cells()); - - forall(kj, mdpolicy_zy, "flow_x_3d") { - auto [k, j] = kj; - const typename mesh::bmap & bm = *bmap_a; - const auto xlow = bm[0][LOW]; - if(xlow == bd::flow) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(m, j, k) = r(ghost_zone_size, j, k); - v(m, j, k) = v(ghost_zone_size, j, k); - p(m, j, k) = p(ghost_zone_size, j, k); - - ru(m, j, k) = ru(ghost_zone_size, j, k); - rE(m, j, k) = rE(ghost_zone_size, j, k); - Erad(m, j, k) = Erad(ghost_zone_size, j, k); - } - } - else if(xlow == bd::reflecting) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(m, j, k) = r(2 * ghost_zone_size - 1 - m, j, k); - v(m, j, k).x = -v(2 * ghost_zone_size - 1 - m, j, k).x; - v(m, j, k).y = v(2 * ghost_zone_size - 1 - m, j, k).y; - v(m, j, k).z = v(2 * ghost_zone_size - 1 - m, j, k).z; - p(m, j, k) = p(2 * ghost_zone_size - 1 - m, j, k); - - ru(m, j, k).x = -ru(2 * ghost_zone_size - 1 - m, j, k).x; - ru(m, j, k).y = ru(2 * ghost_zone_size - 1 - m, j, k).y; - ru(m, j, k).z = ru(2 * ghost_zone_size - 1 - m, j, k).z; - rE(m, j, k) = rE(2 * ghost_zone_size - 1 - m, j, k); - Erad(m, j, k) = Erad(2 * ghost_zone_size - 1 - m, j, k); - } - } - }; // forall - } // if - } - if(m.template is_high()) { - const std::size_t i = m.template size(); - if constexpr(D == 1) { - flecsi::util::iota_view policy{0, 1}; // default execution space - forall(x, policy, "flow_x_1d") { - (void)x; // remove compiler warning for unused - - const typename mesh::bmap & bm = *bmap_a; - const auto xhigh = bm[0][HIGH]; - if(xhigh == bd::flow) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i - 1 - m) = r(i - 1 - ghost_zone_size); - v(i - 1 - m) = v(i - 1 - ghost_zone_size); - p(i - 1 - m) = p(i - 1 - ghost_zone_size); - - ru(i - 1 - m) = ru(i - 1 - ghost_zone_size); - rE(i - 1 - m) = rE(i - 1 - ghost_zone_size); - Erad(i - 1 - m) = Erad(i - 1 - ghost_zone_size); - } - } - else if(xhigh == bd::reflecting) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i - 1 - m) = r(i - 2 * ghost_zone_size + m); - v(i - 1 - m) = v(i - 2 * ghost_zone_size + m); - p(i - 1 - m) = p(i - 2 * ghost_zone_size + m); - - ru(i - 1 - m) = ru(i - 2 * ghost_zone_size + m); - rE(i - 1 - m) = rE(i - 2 * ghost_zone_size + m); - Erad(i - 1 - m) = Erad(i - 2 * ghost_zone_size + m); - } - } - }; - } - else if constexpr(D == 2) { - forall(j, (m.template cells()), "flow") { - const typename mesh::bmap & bm = *bmap_a; - const auto xhigh = bm[0][HIGH]; - if(xhigh == bd::flow) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i - 1 - m, j) = r(i - 1 - ghost_zone_size, j); - v(i - 1 - m, j) = v(i - 1 - ghost_zone_size, j); - p(i - 1 - m, j) = p(i - 1 - ghost_zone_size, j); - - ru(i - 1 - m, j) = ru(i - 1 - ghost_zone_size, j); - rE(i - 1 - m, j) = rE(i - 1 - ghost_zone_size, j); - Erad(i - 1 - m, j) = Erad(i - 1 - ghost_zone_size, j); - } - } - else if(xhigh == bd::reflecting) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i - 1 - m, j) = r(i - 2 * ghost_zone_size + m, j); - v(i - 1 - m, j).x = -v(i - 2 * ghost_zone_size + m, j).x; - v(i - 1 - m, j).y = v(i - 2 * ghost_zone_size + m, j).y; - p(i - 1 - m, j) = p(i - 2 * ghost_zone_size + m, j); - - ru(i - 1 - m, j).x = -ru(i - 2 * ghost_zone_size + m, j).x; - ru(i - 1 - m, j).y = ru(i - 2 * ghost_zone_size + m, j).y; - rE(i - 1 - m, j) = rE(i - 2 * ghost_zone_size + m, j); - Erad(i - 1 - m, j) = Erad(i - 2 * ghost_zone_size + m, j); - } - } - }; // forall - } - else /* D == 3 */ { - auto mdpolicy_kj = get_mdiota_policy(r, - m.template cells(), - m.template cells()); - - forall(kj, mdpolicy_kj, "flow_3d") { - auto [k, j] = kj; - const typename mesh::bmap & bm = *bmap_a; - const auto xhigh = bm[0][HIGH]; - if(xhigh == bd::flow) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i - 1 - m, j, k) = r(i - 1 - ghost_zone_size, j, k); - v(i - 1 - m, j, k) = v(i - 1 - ghost_zone_size, j, k); - p(i - 1 - m, j, k) = p(i - 1 - ghost_zone_size, j, k); - - ru(i - 1 - m, j, k) = ru(i - 1 - ghost_zone_size, j, k); - rE(i - 1 - m, j, k) = rE(i - 1 - ghost_zone_size, j, k); - Erad(i - 1 - m, j, k) = Erad(i - 1 - ghost_zone_size, j, k); - } - } - else if(xhigh == bd::reflecting) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i - 1 - m, j, k) = r(i - 2 * ghost_zone_size + m, j, k); - v(i - 1 - m, j, k).x = -v(i - 2 * ghost_zone_size + m, j, k).x; - v(i - 1 - m, j, k).y = v(i - 2 * ghost_zone_size + m, j, k).y; - v(i - 1 - m, j, k).z = v(i - 2 * ghost_zone_size + m, j, k).z; - p(i - 1 - m, j, k) = p(i - 2 * ghost_zone_size + m, j, k); - - ru(i - 1 - m, j, k).x = -ru(i - 2 * ghost_zone_size + m, j, k).x; - ru(i - 1 - m, j, k).y = ru(i - 2 * ghost_zone_size + m, j, k).y; - ru(i - 1 - m, j, k).z = ru(i - 2 * ghost_zone_size + m, j, k).z; - rE(i - 1 - m, j, k) = rE(i - 2 * ghost_zone_size + m, j, k); - Erad(i - 1 - m, j, k) = Erad(i - 2 * ghost_zone_size + m, j, k); - } - } - - }; // forall - } // if - } // if - - if constexpr(D == 2 || D == 3) { - if(m.template is_low()) { - if constexpr(D == 2) { - forall(i, (m.template cells()), "flow_y") { - const typename mesh::bmap & bm = *bmap_a; - const auto ylow = bm[1][LOW]; - if(ylow == bd::flow) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i, m) = r(i, ghost_zone_size); - v(i, m) = v(i, ghost_zone_size); - p(i, m) = p(i, ghost_zone_size); - - ru(i, m) = ru(i, ghost_zone_size); - rE(i, m) = rE(i, ghost_zone_size); - Erad(i, m) = Erad(i, ghost_zone_size); - } - } - else if(ylow == bd::reflecting) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i, m) = r(i, 2 * ghost_zone_size - 1 - m); - v(i, m).x = v(i, 2 * ghost_zone_size - 1 - m).x; - v(i, m).y = -v(i, 2 * ghost_zone_size - 1 - m).y; - p(i, m) = p(i, 2 * ghost_zone_size - 1 - m); - - ru(i, m).x = ru(i, 2 * ghost_zone_size - 1 - m).x; - ru(i, m).y = -ru(i, 2 * ghost_zone_size - 1 - m).y; - rE(i, m) = rE(i, 2 * ghost_zone_size - 1 - m); - Erad(i, m) = Erad(i, 2 * ghost_zone_size - 1 - m); - } - } - }; // forall - } - else /* D == 3 */ { - auto mdpolicy_zx = get_mdiota_policy(r, - m.template cells(), - m.template cells()); - forall(ki, mdpolicy_zx, "flow_y") { - auto [k, i] = ki; - - const typename mesh::bmap & bm = *bmap_a; - const auto ylow = bm[1][LOW]; - if(ylow == bd::flow) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i, m, k) = r(i, ghost_zone_size, k); - v(i, m, k) = v(i, ghost_zone_size, k); - p(i, m, k) = p(i, ghost_zone_size, k); - - ru(i, m, k) = ru(i, ghost_zone_size, k); - rE(i, m, k) = rE(i, ghost_zone_size, k); - Erad(i, m, k) = Erad(i, ghost_zone_size, k); - } - } - else if(ylow == bd::reflecting) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i, m, k) = r(i, 2 * ghost_zone_size - 1 - m, k); - v(i, m, k).x = v(i, 2 * ghost_zone_size - 1 - m, k).x; - v(i, m, k).y = -v(i, 2 * ghost_zone_size - 1 - m, k).y; - v(i, m, k).z = v(i, 2 * ghost_zone_size - 1 - m, k).z; - p(i, m, k) = p(i, 2 * ghost_zone_size - 1 - m, k); - - ru(i, m, k).x = ru(i, 2 * ghost_zone_size - 1 - m, k).x; - ru(i, m, k).y = -ru(i, 2 * ghost_zone_size - 1 - m, k).y; - ru(i, m, k).z = ru(i, 2 * ghost_zone_size - 1 - m, k).z; - rE(i, m, k) = rE(i, 2 * ghost_zone_size - 1 - m, k); - Erad(i, m, k) = Erad(i, 2 * ghost_zone_size - 1 - m, k); - } - } - }; // forall - } // if - } - if(m.template is_high()) { - const std::size_t j = m.template size(); - if constexpr(D == 2) { - forall(i, (m.template cells()), "flow_y_high") { - const typename mesh::bmap & bm = *bmap_a; - const auto yhigh = bm[1][HIGH]; - if(yhigh == bd::flow) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i, j - 1 - m) = r(i, j - 1 - ghost_zone_size); - v(i, j - 1 - m) = v(i, j - 1 - ghost_zone_size); - p(i, j - 1 - m) = p(i, j - 1 - ghost_zone_size); - - ru(i, j - 1 - m) = ru(i, j - 1 - ghost_zone_size); - rE(i, j - 1 - m) = rE(i, j - 1 - ghost_zone_size); - Erad(i, j - 1 - m) = Erad(i, j - 1 - ghost_zone_size); - } - } - else if(yhigh == bd::reflecting) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i, j - 1 - m) = r(i, j - 2 * ghost_zone_size + m); - v(i, j - 1 - m).x = v(i, j - 2 * ghost_zone_size + m).x; - v(i, j - 1 - m).y = -v(i, j - 2 * ghost_zone_size + m).y; - p(i, j - 1 - m) = p(i, j - 2 * ghost_zone_size + m); - - ru(i, j - 1 - m).x = ru(i, j - 2 * ghost_zone_size + m).x; - ru(i, j - 1 - m).y = -ru(i, j - 2 * ghost_zone_size + m).y; - rE(i, j - 1 - m) = rE(i, j - 2 * ghost_zone_size + m); - Erad(i, j - 1 - m) = Erad(i, j - 2 * ghost_zone_size + m); - } - } - }; // forall - } - else /* D == 3 */ { - auto mdpolicy_zx = get_mdiota_policy(r, - m.template cells(), - m.template cells()); - forall(ki, mdpolicy_zx, "flow_y_high") { - auto [k, i] = ki; - const typename mesh::bmap & bm = *bmap_a; - const auto yhigh = bm[1][HIGH]; - if(yhigh == bd::flow) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i, j - 1 - m, k) = r(i, j - 1 - ghost_zone_size, k); - v(i, j - 1 - m, k) = v(i, j - 1 - ghost_zone_size, k); - p(i, j - 1 - m, k) = p(i, j - 1 - ghost_zone_size, k); - - ru(i, j - 1 - m, k) = ru(i, j - 1 - ghost_zone_size, k); - rE(i, j - 1 - m, k) = rE(i, j - 1 - ghost_zone_size, k); - Erad(i, j - 1 - m, k) = Erad(i, j - 1 - ghost_zone_size, k); - } - } - else if(yhigh == bd::reflecting) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i, j - 1 - m, k) = r(i, j - 2 * ghost_zone_size + m, k); - v(i, j - 1 - m, k).x = v(i, j - 2 * ghost_zone_size + m, k).x; - v(i, j - 1 - m, k).y = -v(i, j - 2 * ghost_zone_size + m, k).y; - v(i, j - 1 - m, k).z = v(i, j - 2 * ghost_zone_size + m, k).z; - p(i, j - 1 - m, k) = p(i, j - 2 * ghost_zone_size + m, k); - - ru(i, j - 1 - m, k).x = ru(i, j - 2 * ghost_zone_size + m, k).x; - ru(i, j - 1 - m, k).y = -ru(i, j - 2 * ghost_zone_size + m, k).y; - ru(i, j - 1 - m, k).z = ru(i, j - 2 * ghost_zone_size + m, k).z; - rE(i, j - 1 - m, k) = rE(i, j - 2 * ghost_zone_size + m, k); - Erad(i, j - 1 - m, k) = Erad(i, j - 2 * ghost_zone_size + m, k); - } - } - }; // forall - } // if - } // if - } - if constexpr(D == 3) { - if(m.template is_low()) { - auto mdpolicy_yx = get_mdiota_policy(r, - m.template cells(), - m.template cells()); - forall(ji, mdpolicy_yx, "flow_z_low") { - auto [j, i] = ji; - const typename mesh::bmap & bm = *bmap_a; - const auto zlow = bm[2][LOW]; - if(zlow == bd::flow) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i, j, m) = r(i, j, ghost_zone_size); - v(i, j, m) = v(i, j, ghost_zone_size); - p(i, j, m) = p(i, j, ghost_zone_size); - - ru(i, j, m) = ru(i, j, ghost_zone_size); - rE(i, j, m) = rE(i, j, ghost_zone_size); - Erad(i, j, m) = Erad(i, j, ghost_zone_size); - } - } - else if(zlow == bd::reflecting) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i, j, m) = r(i, j, 2 * ghost_zone_size - 1 - m); - v(i, j, m).x = v(i, j, 2 * ghost_zone_size - 1 - m).x; - v(i, j, m).y = v(i, j, 2 * ghost_zone_size - 1 - m).y; - v(i, j, m).z = -v(i, j, 2 * ghost_zone_size - 1 - m).z; - p(i, j, m) = p(i, j, 2 * ghost_zone_size - 1 - m); - - ru(i, j, m).x = ru(i, j, 2 * ghost_zone_size - 1 - m).x; - ru(i, j, m).y = ru(i, j, 2 * ghost_zone_size - 1 - m).y; - ru(i, j, m).z = -ru(i, j, 2 * ghost_zone_size - 1 - m).z; - rE(i, j, m) = rE(i, j, 2 * ghost_zone_size - 1 - m); - Erad(i, j, m) = Erad(i, j, 2 * ghost_zone_size - 1 - m); - } - } - }; // forall - } - if(m.template is_high()) { - const std::size_t k = m.template size(); - auto mdpolicy_yx = get_mdiota_policy(r, - m.template cells(), - m.template cells()); - forall(ji, mdpolicy_yx, "flow_3d_high") { - auto [j, i] = ji; - const typename mesh::bmap & bm = *bmap_a; - const auto zhigh = bm[2][HIGH]; - if(zhigh == bd::flow) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i, j, k - 1 - m) = r(i, j, k - 1 - ghost_zone_size); - v(i, j, k - 1 - m) = v(i, j, k - 1 - ghost_zone_size); - p(i, j, k - 1 - m) = p(i, j, k - 1 - ghost_zone_size); - - ru(i, j, k - 1 - m) = ru(i, j, k - 1 - ghost_zone_size); - rE(i, j, k - 1 - m) = rE(i, j, k - 1 - ghost_zone_size); - Erad(i, j, k - 1 - m) = Erad(i, j, k - 1 - ghost_zone_size); - } - } - else if(zhigh == bd::reflecting) { - - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i, j, k - 1 - m) = r(i, j, k - 2 * ghost_zone_size + m); - v(i, j, k - 1 - m).x = v(i, j, k - 2 * ghost_zone_size + m).x; - v(i, j, k - 1 - m).y = v(i, j, k - 2 * ghost_zone_size + m).y; - v(i, j, k - 1 - m).z = -v(i, j, k - 2 * ghost_zone_size + m).z; - p(i, j, k - 1 - m) = p(i, j, k - 2 * ghost_zone_size + m); - - ru(i, j, k - 1 - m).x = ru(i, j, k - 2 * ghost_zone_size + m).x; - ru(i, j, k - 1 - m).y = ru(i, j, k - 2 * ghost_zone_size + m).y; - ru(i, j, k - 1 - m).z = -ru(i, j, k - 2 * ghost_zone_size + m).z; - rE(i, j, k - 1 - m) = rE(i, j, k - 2 * ghost_zone_size + m); - Erad(i, j, k - 1 - m) = Erad(i, j, k - 2 * ghost_zone_size + m); - } - } - }; // forall - } // if - } // if -} // flow - -// TODO: Undo the hardcoded boundary -template -void -apply_radiation_boundary(typename mesh::template accessor m, - field::accessor r_a, - flecsi::future radiation_boundary_f) { - using hard::tasks::util::get_mdiota_policy; - auto r = m.template mdcolex(r_a); - - const size_t ghost_zone_size = m.ghost_zone_size(); - double radiation_boundary = radiation_boundary_f.get(); - - if(m.template is_high()) { - const std::size_t i = m.template size(); - if constexpr(D == 1) { - flecsi::util::iota_view policy{0, 1}; // default execution space - forall(x, policy, "dirichlet_x_high_1D") { - (void)x; - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i - 1 - m) = radiation_boundary; - } - }; // forall - } - else if constexpr(D == 2) { - forall( - j, (m.template cells()), "dirichlet_x_high_2D") { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i - 1 - m, j) = radiation_boundary; - } - }; // /forall - } - else /* D == 3 */ { - auto mdpolicy_zy = get_mdiota_policy(r, - m.template cells(), - m.template cells()); - - forall(kj, mdpolicy_zy, "dirichlet_x_high_3D") { - auto [k, j] = kj; - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i - 1 - m, j, k) = radiation_boundary; - } - }; // forall - } // if - } // if -} // flow - -// TODO: Refactor. apply_boundary overload for just one field. -template -void -apply_boundary_single_field(typename mesh::template accessor m, - typename single::bmap>::template accessor bmap_a, - field::accessor r_a) { - using hard::tasks::util::get_mdiota_policy; - auto r = m.template mdcolex(r_a); - - const size_t ghost_zone_size = m.ghost_zone_size(); - - if(m.template is_low()) { - if constexpr(D == 1) { - flecsi::util::iota_view policy{0, 1}; // default execution space - forall(x, policy, "bd_x") { - (void)x; // remove compiler warning for unused - - const typename mesh::bmap & bm = *bmap_a; - const auto xlow = bm[0][LOW]; - if(xlow == bd::flow) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(m) = r(ghost_zone_size); - } - } - else if(xlow == bd::reflecting) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(m) = r(2 * ghost_zone_size - 1 - m); - } - } - }; - } - else if constexpr(D == 2) { - forall(j, (m.template cells()), "bd_x") { - const typename mesh::bmap & bm = *bmap_a; - const auto xlow = bm[0][LOW]; - if(xlow == bd::flow) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(m, j) = r(ghost_zone_size, j); - } - } - else if(xlow == bd::reflecting) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(m, j) = r(2 * ghost_zone_size - 1 - m, j); - } - } - }; // for - } - else /* D == 3 */ { - auto mdpolicy_zy = get_mdiota_policy(r, - m.template cells(), - m.template cells()); - - forall(kj, mdpolicy_zy, "flow_x_3d") { - auto [k, j] = kj; - const typename mesh::bmap & bm = *bmap_a; - const auto xlow = bm[0][LOW]; - if(xlow == bd::flow) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(m, j, k) = r(ghost_zone_size, j, k); - } - } - else if(xlow == bd::reflecting) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(m, j, k) = r(2 * ghost_zone_size - 1 - m, j, k); - } - } - }; // forall - } // if - } - if(m.template is_high()) { - const std::size_t i = m.template size(); - if constexpr(D == 1) { - flecsi::util::iota_view policy{0, 1}; // default execution space - forall(x, policy, "flow_x_1d") { - (void)x; // remove compiler warning for unused - - const typename mesh::bmap & bm = *bmap_a; - const auto xhigh = bm[0][HIGH]; - if(xhigh == bd::flow) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i - 1 - m) = r(i - 1 - ghost_zone_size); - } - } - else if(xhigh == bd::reflecting) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i - 1 - m) = r(i - 2 * ghost_zone_size + m); - } - } - }; - } - else if constexpr(D == 2) { - forall(j, (m.template cells()), "flow") { - const typename mesh::bmap & bm = *bmap_a; - const auto xhigh = bm[0][HIGH]; - if(xhigh == bd::flow) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i - 1 - m, j) = r(i - 1 - ghost_zone_size, j); - } - } - else if(xhigh == bd::reflecting) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i - 1 - m, j) = r(i - 2 * ghost_zone_size + m, j); - } - } - }; // forall - } - else /* D == 3 */ { - auto mdpolicy_kj = get_mdiota_policy(r, - m.template cells(), - m.template cells()); - - forall(kj, mdpolicy_kj, "flow_3d") { - auto [k, j] = kj; - const typename mesh::bmap & bm = *bmap_a; - const auto xhigh = bm[0][HIGH]; - if(xhigh == bd::flow) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i - 1 - m, j, k) = r(i - 1 - ghost_zone_size, j, k); - } - } - else if(xhigh == bd::reflecting) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i - 1 - m, j, k) = r(i - 2 * ghost_zone_size + m, j, k); - } - } - }; // forall - } // if - } // if - - if constexpr(D == 2 || D == 3) { - if(m.template is_low()) { - if constexpr(D == 2) { - forall(i, (m.template cells()), "flow_y") { - const typename mesh::bmap & bm = *bmap_a; - const auto ylow = bm[1][LOW]; - if(ylow == bd::flow) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i, m) = r(i, ghost_zone_size); - } - } - else if(ylow == bd::reflecting) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i, m) = r(i, 2 * ghost_zone_size - 1 - m); - } - } - }; // forall - } - else /* D == 3 */ { - auto mdpolicy_zx = get_mdiota_policy(r, - m.template cells(), - m.template cells()); - forall(ki, mdpolicy_zx, "flow_y") { - auto [k, i] = ki; - - const typename mesh::bmap & bm = *bmap_a; - const auto ylow = bm[1][LOW]; - if(ylow == bd::flow) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i, m, k) = r(i, ghost_zone_size, k); - } - } - else if(ylow == bd::reflecting) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i, m, k) = r(i, 2 * ghost_zone_size - 1 - m, k); - } - } - }; // forall - } // if - } - if(m.template is_high()) { - const std::size_t j = m.template size(); - if constexpr(D == 2) { - forall(i, (m.template cells()), "flow_y_high") { - const typename mesh::bmap & bm = *bmap_a; - const auto yhigh = bm[1][HIGH]; - if(yhigh == bd::flow) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i, j - 1 - m) = r(i, j - 1 - ghost_zone_size); - } - } - else if(yhigh == bd::reflecting) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i, j - 1 - m) = r(i, j - 2 * ghost_zone_size + m); - } - } - }; // forall - } - else /* D == 3 */ { - auto mdpolicy_zx = get_mdiota_policy(r, - m.template cells(), - m.template cells()); - forall(ki, mdpolicy_zx, "flow_y_high") { - auto [k, i] = ki; - const typename mesh::bmap & bm = *bmap_a; - const auto yhigh = bm[1][HIGH]; - if(yhigh == bd::flow) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i, j - 1 - m, k) = r(i, j - 1 - ghost_zone_size, k); - } - } - else if(yhigh == bd::reflecting) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i, j - 1 - m, k) = r(i, j - 2 * ghost_zone_size + m, k); - } - } - }; // forall - } // if - } // if - } - if constexpr(D == 3) { - if(m.template is_low()) { - auto mdpolicy_yx = get_mdiota_policy(r, - m.template cells(), - m.template cells()); - forall(ji, mdpolicy_yx, "flow_z_low") { - auto [j, i] = ji; - const typename mesh::bmap & bm = *bmap_a; - const auto zlow = bm[2][LOW]; - if(zlow == bd::flow) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i, j, m) = r(i, j, ghost_zone_size); - } - } - else if(zlow == bd::reflecting) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i, j, m) = r(i, j, 2 * ghost_zone_size - 1 - m); - } - } - }; // forall - } - if(m.template is_high()) { - const std::size_t k = m.template size(); - auto mdpolicy_yx = get_mdiota_policy(r, - m.template cells(), - m.template cells()); - forall(ji, mdpolicy_yx, "flow_3d_high") { - auto [j, i] = ji; - const typename mesh::bmap & bm = *bmap_a; - const auto zhigh = bm[2][HIGH]; - if(zhigh == bd::flow) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i, j, k - 1 - m) = r(i, j, k - 1 - ghost_zone_size); - } - } - else if(zhigh == bd::reflecting) { - for(size_t m = 0; m < ghost_zone_size; ++m) { - r(i, j, k - 1 - m) = r(i, j, k - 2 * ghost_zone_size + m); - } - } - }; // forall - } // if - } // if -} // flow - -} // namespace hard::tasks diff --git a/app/tasks/catalyst.hh b/app/tasks/catalyst.hh index 339c0de..ae4043c 100644 --- a/app/tasks/catalyst.hh +++ b/app/tasks/catalyst.hh @@ -18,7 +18,7 @@ namespace hard::tasks::external { *--------------------------------------------------------------------------*/ template inline void -get_lattice_size(typename mesh::template accessor m) { +get_lattice_size(flecsi::exec::cpu, typename mesh::template accessor m) { if constexpr(D == 3) { number_vertices[0] = m.template size() + 1; // 1 more vertex than cell in simple cubic lattice @@ -32,7 +32,7 @@ get_lattice_size(typename mesh::template accessor m) { template inline void -get_color_data(typename mesh::template accessor m) { +get_color_data(flecsi::exec::cpu s, typename mesh::template accessor m) { if constexpr(D == 3) { // get number of color in each direction auto ccolors = m.axis_colors(); @@ -63,7 +63,7 @@ get_color_data(typename mesh::template accessor m) { color_cell_id_begin[2] = ax_z.global_id(*cell_it_z.begin()); color_cell_id_end[2] = ax_z.global_id(*cell_it_z.end() - 1); - my_rank = flecsi::process(); + my_rank = s.launch().index; } else { /* Do nothing, catalyst/paraview visualization only for 3D */ @@ -71,7 +71,8 @@ get_color_data(typename mesh::template accessor m) { } inline void -init_attributes(single::accessor c_a, +init_attributes(flecsi::exec::cpu, + single::accessor c_a, std::size_t pt, std::size_t cl) { /* Need to use the '->' operator to "dereference" 'single' accessor types. */ @@ -86,7 +87,8 @@ init_attributes(single::accessor c_a, *--------------------------------------------------------------------------*/ template inline void -update_attributes(single::accessor c_a, +update_attributes(flecsi::exec::cpu, + single::accessor c_a, single::accessor time_a, typename mesh::template accessor m, field::accessor r_a, @@ -127,7 +129,7 @@ update_attributes(single::accessor c_a, Esf(i, j, k)); // <<< add variables here for catalyst // vector data for catalyst/paraview stored non-interleaved: first all // x, then all y, etc. - velocity_vals.push_back(u(i, j, k).x); + velocity_vals.push_back(u(i, j, k).x()); } // for } // for } // for @@ -135,7 +137,7 @@ update_attributes(single::accessor c_a, for(auto j : m.template cells()) { for(auto i : m.template cells()) { velocity_vals.push_back( - u(i, j, k).y); // run loop again to add y components + u(i, j, k).y()); // run loop again to add y components } // for } // for } // for @@ -143,7 +145,7 @@ update_attributes(single::accessor c_a, for(auto j : m.template cells()) { for(auto i : m.template cells()) { velocity_vals.push_back( - u(i, j, k).z); // run loop again to add z components + u(i, j, k).z()); // run loop again to add z components } // for } // for } // for @@ -166,7 +168,8 @@ update_attributes(single::accessor c_a, 'execute' task. *--------------------------------------------------------------------------*/ inline void -execute_catalyst(single::accessor c_a, +execute_catalyst(flecsi::exec::cpu, + single::accessor c_a, size_t step, single::accessor time, simple_cubic & lattice) { @@ -180,7 +183,7 @@ execute_catalyst(single::accessor c_a, the 'initialize' task. *--------------------------------------------------------------------------*/ inline void -initialize() { +initialize(flecsi::exec::cpu) { catalyst_adaptor::initialize(); flog(info) << "Catalyst Initialize, ok" << std::endl; } @@ -191,7 +194,7 @@ initialize() { the 'finalize' task. *--------------------------------------------------------------------------*/ inline void -finalize(single::accessor c_a) { +finalize(flecsi::exec::cpu, single::accessor c_a) { catalyst_adaptor::finalize(); c_a->finalize(); flog(info) << "Catalyst Finalize, ok" << std::endl; diff --git a/app/tasks/external_source.hh b/app/tasks/external_source.hh new file mode 100644 index 0000000..e6595da --- /dev/null +++ b/app/tasks/external_source.hh @@ -0,0 +1,88 @@ + +#pragma once + +#include "../numerical_algorithms/riemann_solvers.hh" +#include "../types.hh" +#include "utils.hh" +#include + +namespace hard::tasks { + +using hard::tasks::util::get_mdiota_policy; + +// Gravity force and work terms. (Explicit source terms) (for RT instability +// test case) +template +void +externalSource(flecsi::exec::accelerator s, + typename mesh::template accessor m, + // Primitive variables + typename field>::template accessor velocity_a, + // Terms required for computing radiation force and photon tiring + typename field>::template accessor gravity_force_a, + // time derivative + typename field>::template accessor dt_momentum_density_a, + field::accessor dt_total_energy_density_a) noexcept { + + auto velocity = m.template mdcolex(velocity_a); + auto fg = m.template mdcolex(gravity_force_a); + auto dt_momentum_density = + m.template mdcolex(dt_momentum_density_a); + auto dt_total_energy_density = + m.template mdcolex(dt_total_energy_density_a); + // const double radiation_constant = hard::constants::cgs::radiation_constant; + + if constexpr(D == 1) { + s.executor().forall(i, (m.template cells())) { + + // Explicitly updating conservative variables with S_ex + // Adding the radiation force term to the momentum density + dt_momentum_density(i) += fg(i); + + // Updating the total gas energy density: Adding contribution from the + // work done by the radiative force: vdot_fr(i) = u(i).x * fr(i).x + dt_total_energy_density(i) += velocity(i).x() * fg(i).x(); + }; + } + else if constexpr(D == 2) { + auto mdpolicy_qq = get_mdiota_policy(velocity, + m.template cells(), + m.template cells()); + + s.executor().forall(ji, mdpolicy_qq) { + auto [j, i] = ji; + + // Explicitly updating conservative variables with S_ex + // Adding the radiation force term to the momentum density + dt_momentum_density(i, j) += fg(i, j); + + // Updating the total gas energy density: Adding contribution from the + // work done by the radiative force: vdot_fr(i) = u(i).x * fr(i).x + dt_total_energy_density(i, j) += + velocity(i, j).x() * fg(i, j).x() + velocity(i, j).y() * fg(i, j).y(); + }; + } + else { + auto mdpolicy_qqq = get_mdiota_policy(velocity, + m.template cells(), + m.template cells(), + m.template cells()); + + s.executor().forall(kji, mdpolicy_qqq) { + auto [k, j, i] = kji; + + // Explicitly updating conservative variables with S_ex + // Adding the radiation force term to the momentum density + dt_momentum_density(i, j, k) += fg(i, j, k); + + // Updating the total gas energy density: Adding contribution from the + // work done by the radiative force: vdot_fr(i) = u(i).x * fr(i).x + dt_total_energy_density(i, j, k) += + velocity(i, j, k).x() * fg(i, j, k).x() + + velocity(i, j, k).y() * fg(i, j, k).y() + + velocity(i, j, k).z() * fg(i, j, k).z(); + }; // forall + } +} // explicitSourceUpdate + +} // namespace hard::tasks diff --git a/app/tasks/hydro/compute_interface_fluxes.hh b/app/tasks/hydro/compute_interface_fluxes.hh index 26f1d8b..23fd412 100644 --- a/app/tasks/hydro/compute_interface_fluxes.hh +++ b/app/tasks/hydro/compute_interface_fluxes.hh @@ -14,7 +14,8 @@ namespace hard::tasks::hydro { // template void -compute_interface_fluxes(std::size_t face_axis, +compute_interface_fluxes(flecsi::exec::cpu s, + std::size_t face_axis, typename mesh::template accessor m, field::accessor rTail_a, field::accessor rHead_a, @@ -22,8 +23,6 @@ compute_interface_fluxes(std::size_t face_axis, typename field>::template accessor uHead_a, field::accessor pTail_a, field::accessor pHead_a, - field::accessor eTail_a, - field::accessor eHead_a, field::accessor cTail_a, field::accessor cHead_a, field::accessor @@ -57,16 +56,15 @@ compute_interface_fluxes(std::size_t face_axis, #ifdef ENABLE_RADIATION dt_radiation_energy_density_a #endif -) { - + , + typename single>::template accessor g_acc) noexcept { + auto g = g_acc.get(); auto rTail = m.template mdcolex(rTail_a); auto rHead = m.template mdcolex(rHead_a); auto uTail = m.template mdcolex(uTail_a); auto uHead = m.template mdcolex(uHead_a); auto pTail = m.template mdcolex(pTail_a); auto pHead = m.template mdcolex(pHead_a); - auto eTail = m.template mdcolex(eTail_a); - auto eHead = m.template mdcolex(eHead_a); auto cTail = m.template mdcolex(cTail_a); auto cHead = m.template mdcolex(cHead_a); @@ -116,45 +114,62 @@ compute_interface_fluxes(std::size_t face_axis, flog_fatal("1D executable can only compute fluxes along X axis"); } #endif - forall(i, (m.template cells()), "compute_flux_1d") { - // min/max characteristic speeds on left + s.executor().forall(i, (m.template cells())) { + const auto dx = m.template delta(); + + // Fluxes from left and right state + const double pTail_wave = pTail(i - 1) + 0.5 * rTail(i - 1) * g.x() * dx; + const double pHead_wave = pHead(i) - 0.5 * rHead(i) * g.x() * dx; + const double f_r_T{ruTail(i - 1).x()}; + const double f_r_H{ruHead(i).x()}; + const vec<1> f_ru_T{ruTail(i - 1).x() * uTail(i - 1).x() + pTail_wave}; + const vec<1> f_ru_H{ruHead(i).x() * uHead(i).x() + pHead_wave}; + const double f_rE_T{(rETail(i - 1) + pTail_wave) * uTail(i - 1).x()}; + const double f_rE_H{(rEHead(i) + pHead_wave) * uHead(i).x()}; + + // clang-format off + rF(i) = + numerical_algorithms::compute_HLLC_fluxes(face_axis, + rTail(i-1), rTail(i-1), uTail(i-1), rETail(i-1), pTail_wave, cTail(i-1), f_r_T, + rHead(i), rHead(i), uHead(i), rEHead(i), pHead_wave, cHead(i), f_r_H, + "rho"); + ruF(i) = + numerical_algorithms::compute_HLLC_fluxes>(face_axis, + ruTail(i-1), rTail(i-1), uTail(i-1), rETail(i-1), pTail_wave, cTail(i-1), f_ru_T, + ruHead(i), rHead(i), uHead(i), rEHead(i), pHead_wave, cHead(i), f_ru_H, + "rhou"); + rEF(i) = + numerical_algorithms::compute_HLLC_fluxes(face_axis, + rETail(i-1), rTail(i-1), uTail(i-1), rETail(i-1), pTail_wave, cTail(i-1), f_rE_T, + rEHead(i), rHead(i), uHead(i), rEHead(i), pHead_wave, cHead(i), f_rE_H, + "E"); + // clang-format on + +#ifdef ENABLE_RADIATION - const double cT = cTail(i - 1); - const double LminT = uTail(i - 1).x - cT; - const double LmaxT = uTail(i - 1).x + cT; + // min/max characteristic speeds on left + const double cT{cTail(i - 1)}; + const double LminT{uTail(i - 1).x() - cT}; + const double LmaxT{uTail(i - 1).x() + cT}; // min/max characteristic speeds on right - const double cH = cHead(i); - const double LminH = uHead(i).x - cH; - const double LmaxH = uHead(i).x + cH; + const double cH{cHead(i)}; + const double LminH{uHead(i).x() - cH}; + const double LmaxH{uHead(i).x() + cH}; - // Fluxes from left and right state - const double f_r_T{ruTail(i - 1).x}; - const double f_r_H{ruHead(i).x}; - const vec<1> f_ru_T{ruTail(i - 1).x * uTail(i - 1).x + pTail(i - 1)}; - const vec<1> f_ru_H{ruHead(i).x * uHead(i).x + pHead(i)}; - const double f_rE_T{(rETail(i - 1) + pTail(i - 1)) * uTail(i - 1).x}; - const double f_rE_H{(rEHead(i) + pHead(i)) * uHead(i).x}; + const double f_Erad_T{EradTail(i - 1) * uTail(i - 1).x()}; + const double f_Erad_H{EradHead(i) * uHead(i).x()}; // clang-format off - const auto hll_fluxes = numerical_algorithms::hll_hydro( - rTail(i - 1), rHead(i), ruTail(i - 1), ruHead(i), rETail(i - 1), rEHead(i), - f_r_T, f_r_H, f_ru_T, f_ru_H, f_rE_T, f_rE_H, + EradF(i) = numerical_algorithms::advect_conserved( + EradTail(i - 1), EradHead(i), f_Erad_T, f_Erad_H, LminT, LmaxT, LminH, LmaxH); // clang-format on - - rF(i) = get<0>(hll_fluxes); - ruF(i) = get<1>(hll_fluxes); - rEF(i) = get<2>(hll_fluxes); - -#ifdef ENABLE_RADIATION - EradF(i) = numerical_algorithms::advect_Erad( - EradTail(i - 1), EradHead(i), uTail(i - 1).x, uHead(i).x); #endif }; // forall // Store dF^x/dx into du_dt - forall(i, (m.template cells()), "update_flux_1d") { + s.executor().forall(i, (m.template cells())) { dt_mass_density(i) += one_over_dx_i[0] * (rF(i) - rF(i + 1)); dt_momentum_density(i) += one_over_dx_i[0] * (ruF(i) - ruF(i + 1)); dt_total_energy_density(i) += one_over_dx_i[0] * (rEF(i) - rEF(i + 1)); @@ -175,46 +190,62 @@ compute_interface_fluxes(std::size_t face_axis, m.template cells(), m.template cells()); - forall(ji, mdpolicy_qc, "compute_flux_2dx") { + s.executor().forall(ji, mdpolicy_qc) { auto [j, i] = ji; - // min/max characteristic speeds on left - const double cT = cTail(i - 1, j); - const double LminT = uTail(i - 1, j).x - cT; - const double LmaxT = uTail(i - 1, j).x + cT; - - // min/max characteristic speeds on right - const double cH = cHead(i, j); - const double LminH = uHead(i, j).x - cH; - const double LmaxH = uHead(i, j).x + cH; // Fluxes from left and right state - const double f_r_T{ruTail(i - 1, j).x}; - const double f_r_H{ruHead(i, j).x}; + const double f_r_T{ruTail(i - 1, j).x()}; + const double f_r_H{ruHead(i, j).x()}; const vec<2> f_ru_T{ - ruTail(i - 1, j).x * uTail(i - 1, j).x + pTail(i - 1, j), - ruTail(i - 1, j).x * uTail(i - 1, j).y}; - const vec<2> f_ru_H{ruHead(i, j).x * uHead(i, j).x + pHead(i, j), - ruHead(i, j).x * uHead(i, j).y}; + + ruTail(i - 1, j).x() * uTail(i - 1, j).x() + pTail(i - 1, j), + ruTail(i - 1, j).x() * uTail(i - 1, j).y()}; + const vec<2> f_ru_H{ruHead(i, j).x() * uHead(i, j).x() + pHead(i, j), + ruHead(i, j).x() * uHead(i, j).y()}; const double f_rE_T{ - (rETail(i - 1, j) + pTail(i - 1, j)) * uTail(i - 1, j).x}; - const double f_rE_H{(rEHead(i, j) + pHead(i, j)) * uHead(i, j).x}; + (rETail(i - 1, j) + pTail(i - 1, j)) * uTail(i - 1, j).x()}; + const double f_rE_H{(rEHead(i, j) + pHead(i, j)) * uHead(i, j).x()}; // clang-format off - const auto hll_fluxes = - numerical_algorithms::hll_hydro( - rTail(i - 1, j), rHead(i, j), ruTail(i - 1, j), ruHead(i, j), rETail(i - 1, j), rEHead(i, j), - f_r_T, f_r_H, f_ru_T, f_ru_H, f_rE_T, f_rE_H, - LminT, LmaxT, LminH, LmaxH); + const double pTail_wave = pTail(i-1,j) - 0.5 * rTail(i-1,j) * g.x() / one_over_dx_i[0]; + const double pHead_wave = pHead(i,j) - 0.5 * rHead(i,j) * g.x() / one_over_dx_i[0]; + rF(i, j) = + numerical_algorithms::compute_HLLC_fluxes(face_axis, + rTail(i-1,j), rTail(i-1,j), uTail(i-1,j), rETail(i-1,j), pTail_wave, cTail(i-1,j), f_r_T, + rHead(i,j), rHead(i,j), uHead(i,j), rEHead(i,j), pHead_wave, cHead(i,j), f_r_H, + "rho" ); + ruF(i, j) = + numerical_algorithms::compute_HLLC_fluxes>(face_axis, + ruTail(i-1,j), rTail(i-1,j), uTail(i-1,j), rETail(i-1,j), pTail_wave, cTail(i-1,j), f_ru_T, + ruHead(i,j), rHead(i,j), uHead(i,j), rEHead(i,j), pHead_wave, cHead(i,j), f_ru_H, + "rhou" ); + rEF(i, j) = + numerical_algorithms::compute_HLLC_fluxes(face_axis, + rETail(i-1,j), rTail(i-1,j), uTail(i-1,j), rETail(i-1,j), pTail_wave, cTail(i-1,j), f_rE_T, + rEHead(i,j), rHead(i,j), uHead(i,j), rEHead(i,j), pHead_wave, cHead(i,j), f_rE_H, + "E" ); // clang-format on - rF(i, j) = get<0>(hll_fluxes); - ruF(i, j) = get<1>(hll_fluxes); - rEF(i, j) = get<2>(hll_fluxes); - #ifdef ENABLE_RADIATION - EradF(i, j) = numerical_algorithms::advect_Erad( - EradTail(i - 1, j), EradHead(i, j), uTail(i - 1, j).x, uHead(i, j).x); + // min/max characteristic speeds on left + const double cT = cTail(i - 1, j); + const double LminT = uTail(i - 1, j).x() - cT; + const double LmaxT = uTail(i - 1, j).x() + cT; + + // min/max characteristic speeds on right + const double cH = cHead(i, j); + const double LminH = uHead(i, j).x() - cH; + const double LmaxH = uHead(i, j).x() + cH; + + const double f_Erad_T{EradTail(i - 1, j) * uTail(i - 1, j).x()}; + const double f_Erad_H{EradHead(i, j) * uHead(i, j).x()}; + + // clang-format off + EradF(i, j) = numerical_algorithms::advect_conserved( + EradTail(i - 1, j), EradHead(i, j), f_Erad_T, f_Erad_H, LminT, + LmaxT, LminH, LmaxH); + // clang-format on #endif }; // forall @@ -224,7 +255,7 @@ compute_interface_fluxes(std::size_t face_axis, m.template cells(), m.template cells()); - forall(ji, mdpolicy_qq, "update_flux_2dx") { + s.executor().forall(ji, mdpolicy_qq) { auto [j, i] = ji; dt_mass_density(i, j) += one_over_dx_i[0] * (rF(i, j) - rF(i + 1, j)); dt_momentum_density(i, j) += @@ -243,45 +274,61 @@ compute_interface_fluxes(std::size_t face_axis, m.template cells(), m.template cells()); - forall(ji, mdpolicy_cq, "compute_flux_2dy") { + s.executor().forall(ji, mdpolicy_cq) { auto [j, i] = ji; + + // Fluxes from left and right state + const double f_r_T{ruTail(i, j - 1).y()}; + const double f_r_H{ruHead(i, j).y()}; + const vec<2> f_ru_T{ruTail(i, j - 1).y() * uTail(i, j - 1).x(), + ruTail(i, j - 1).y() * uTail(i, j - 1).y() + pTail(i, j - 1)}; + const vec<2> f_ru_H{ruHead(i, j).y() * uHead(i, j).x(), + ruHead(i, j).y() * uHead(i, j).y() + pHead(i, j)}; + const double f_rE_T{ + (rETail(i, j - 1) + pTail(i, j - 1)) * uTail(i, j - 1).y()}; + const double f_rE_H{(rEHead(i, j) + pHead(i, j)) * uHead(i, j).y()}; + + // clang-format off + const double pTail_wave = pTail(i,j-1) - 0.5 * rTail(i,j-1) * g.y() / one_over_dx_i[1]; + const double pHead_wave = pHead(i,j) - 0.5 * rHead(i,j) * g.y() / one_over_dx_i[1]; + rF(i, j) = + numerical_algorithms::compute_HLLC_fluxes(face_axis, + rTail(i,j-1), rTail(i,j-1), uTail(i,j-1), rETail(i,j-1), pTail_wave, cTail(i,j-1), f_r_T, + rHead(i,j), rHead(i,j), uHead(i,j), rEHead(i,j), pHead_wave, cHead(i,j), f_r_H, + "rho" ); + ruF(i, j) = + numerical_algorithms::compute_HLLC_fluxes>(face_axis, + ruTail(i,j-1), rTail(i,j-1), uTail(i,j-1), rETail(i,j-1), pTail_wave, cTail(i,j-1), f_ru_T, + ruHead(i,j), rHead(i,j), uHead(i,j), rEHead(i,j), pHead_wave, cHead(i,j), f_ru_H, + "rhou" ); + rEF(i, j) = + numerical_algorithms::compute_HLLC_fluxes(face_axis, + rETail(i,j-1), rTail(i,j-1), uTail(i,j-1), rETail(i,j-1), pTail_wave, cTail(i,j-1), f_rE_T, + rEHead(i,j), rHead(i,j), uHead(i,j), rEHead(i,j), pHead_wave, cHead(i,j), f_rE_H, + "E" ); + // clang-format on + +#ifdef ENABLE_RADIATION + // min/max characteristic speeds on left const double cT = cTail(i, j - 1); - const double LminT = uTail(i, j - 1).y - cT; - const double LmaxT = uTail(i, j - 1).y + cT; + const double LminT = uTail(i, j - 1).y() - cT; + const double LmaxT = uTail(i, j - 1).y() + cT; // min/max characteristic speeds on right const double cH = cHead(i, j); - const double LminH = uHead(i, j).y - cH; - const double LmaxH = uHead(i, j).y + cH; + const double LminH = uHead(i, j).y() - cH; + const double LmaxH = uHead(i, j).y() + cH; - // Fluxes from left and right state - const double f_r_T{ruTail(i, j - 1).y}; - const double f_r_H{ruHead(i, j).y}; - const vec<2> f_ru_T{ruTail(i, j - 1).y * uTail(i, j - 1).x, - ruTail(i, j - 1).y * uTail(i, j - 1).y + pTail(i, j - 1)}; - const vec<2> f_ru_H{ruHead(i, j).y * uHead(i, j).x, - ruHead(i, j).y * uHead(i, j).y + pHead(i, j)}; - const double f_rE_T{ - (rETail(i, j - 1) + pTail(i, j - 1)) * uTail(i, j - 1).y}; - const double f_rE_H{(rEHead(i, j) + pHead(i, j)) * uHead(i, j).y}; + const double f_Erad_T{EradTail(i, j - 1) * uTail(i, j - 1).y()}; + const double f_Erad_H{EradHead(i, j) * uHead(i, j).y()}; // clang-format off - const auto hll_fluxes = - numerical_algorithms::hll_hydro( - rTail(i, j - 1), rHead(i, j), ruTail(i, j - 1), ruHead(i, j), rETail(i, j - 1), rEHead(i, j), - f_r_T, f_r_H, f_ru_T, f_ru_H, f_rE_T, f_rE_H, - LminT, LmaxT, LminH, LmaxH); + EradF(i, j) = numerical_algorithms::advect_conserved( + EradTail(i, j - 1), EradHead(i, j), f_Erad_T, f_Erad_H, + LminT, LmaxT, LminH, LmaxH); // clang-format on - - rF(i, j) = get<0>(hll_fluxes); - ruF(i, j) = get<1>(hll_fluxes); - rEF(i, j) = get<2>(hll_fluxes); - -#ifdef ENABLE_RADIATION - EradF(i, j) = numerical_algorithms::advect_Erad( - EradTail(i, j - 1), EradHead(i, j), uTail(i, j - 1).y, uHead(i, j).y); #endif }; // forall @@ -291,7 +338,7 @@ compute_interface_fluxes(std::size_t face_axis, m.template cells(), m.template cells()); - forall(ji, mdpolicy_qq, "update_flux_2dy") { + s.executor().forall(ji, mdpolicy_qq) { auto [j, i] = ji; dt_mass_density(i, j) += one_over_dx_i[1] * (rF(i, j) - rF(i, j + 1)); dt_momentum_density(i, j) += @@ -314,55 +361,67 @@ compute_interface_fluxes(std::size_t face_axis, m.template cells(), m.template cells()); - forall(kji, mdpolicy_qqc, "compute_flux_3dx") { + s.executor().forall(kji, mdpolicy_qqc) { auto [k, j, i] = kji; - // min/max characteristic speeds on left - const double cT = cTail(i - 1, j, k); - const double LminT = uTail(i - 1, j, k).x - cT; - const double LmaxT = uTail(i - 1, j, k).x + cT; - - // min/max characteristic speeds on right - const double cH = cHead(i, j, k); - const double LminH = uHead(i, j, k).x - cH; - const double LmaxH = uHead(i, j, k).x + cH; // Fluxes from left and right state - const double f_r_T{ruTail(i - 1, j, k).x}; - const double f_r_H{ruHead(i, j, k).x}; + const double f_r_T{ruTail(i - 1, j, k).x()}; + const double f_r_H{ruHead(i, j, k).x()}; const vec<3> f_ru_T{ - ruTail(i - 1, j, k).x * uTail(i - 1, j, k).x + pTail(i - 1, j, k), - ruTail(i - 1, j, k).x * uTail(i - 1, j, k).y, - ruTail(i - 1, j, k).x * uTail(i - 1, j, k).z}; + ruTail(i - 1, j, k).x() * uTail(i - 1, j, k).x() + pTail(i - 1, j, k), + ruTail(i - 1, j, k).x() * uTail(i - 1, j, k).y(), + ruTail(i - 1, j, k).x() * uTail(i - 1, j, k).z()}; const vec<3> f_ru_H{ - ruHead(i, j, k).x * uHead(i, j, k).x + pHead(i, j, k), - ruHead(i, j, k).x * uHead(i, j, k).y, - ruHead(i, j, k).x * uHead(i, j, k).z}; + ruHead(i, j, k).x() * uHead(i, j, k).x() + pHead(i, j, k), + ruHead(i, j, k).x() * uHead(i, j, k).y(), + ruHead(i, j, k).x() * uHead(i, j, k).z()}; const double f_rE_T{ - (rETail(i - 1, j, k) + pTail(i - 1, j, k)) * uTail(i - 1, j, k).x}; + (rETail(i - 1, j, k) + pTail(i - 1, j, k)) * uTail(i - 1, j, k).x()}; const double f_rE_H{ - (rEHead(i, j, k) + pHead(i, j, k)) * uHead(i, j, k).x}; + (rEHead(i, j, k) + pHead(i, j, k)) * uHead(i, j, k).x()}; + // Advect conserved quantities // clang-format off - const auto hll_fluxes = - numerical_algorithms::hll_hydro( - rTail(i - 1, j, k), rHead(i, j, k), - ruTail(i - 1, j, k), ruHead(i, j, k), - rETail(i - 1, j, k), rEHead(i, j, k), - f_r_T, f_r_H, f_ru_T, f_ru_H, f_rE_T, f_rE_H, - LminT, LmaxT, LminH, LmaxH); + const double pTail_wave = pTail(i-1,j,k) + 0.5 * rTail(i-1,j,k) * g.x() / one_over_dx_i[0]; + const double pHead_wave = pHead(i,j,k) - 0.5 * rHead(i,j,k) * g.x() / one_over_dx_i[0]; + rF(i, j, k) = + numerical_algorithms::compute_HLLC_fluxes(face_axis, + rTail(i-1,j,k), rTail(i-1,j,k), uTail(i-1,j,k), rETail(i-1,j,k), pTail_wave, cTail(i-1,j,k), f_r_T, + rHead(i,j,k), rHead(i,j,k), uHead(i,j,k), rEHead(i,j,k), pHead_wave, cHead(i,j,k), f_r_H, + "rho" ); + ruF(i, j, k) = + numerical_algorithms::compute_HLLC_fluxes>(face_axis, + ruTail(i-1,j,k), rTail(i-1,j,k), uTail(i-1,j,k), rETail(i-1,j,k), pTail_wave, cTail(i-1,j,k), f_ru_T, + ruHead(i,j,k), rHead(i,j,k), uHead(i,j,k), rEHead(i,j,k), pHead_wave, cHead(i,j,k), f_ru_H, + "rhou" ); + rEF(i, j,k) = + numerical_algorithms::compute_HLLC_fluxes(face_axis, + rETail(i-1,j,k), rTail(i-1,j,k), uTail(i-1,j,k), rETail(i-1,j,k), pTail_wave, cTail(i-1,j,k), f_rE_T, + rEHead(i,j,k), rHead(i,j,k), uHead(i,j,k), rEHead(i,j,k), pHead_wave, cHead(i,j,k), f_rE_H, + "E" ); // clang-format on - rF(i, j, k) = get<0>(hll_fluxes); - ruF(i, j, k) = get<1>(hll_fluxes); - rEF(i, j, k) = get<2>(hll_fluxes); - #ifdef ENABLE_RADIATION - EradF(i, j, k) = - numerical_algorithms::advect_Erad(EradTail(i - 1, j, k), - EradHead(i, j, k), - uTail(i - 1, j, k).x, - uHead(i, j, k).x); + + // min/max characteristic speeds on left + const double cT = cTail(i - 1, j, k); + const double LminT = uTail(i - 1, j, k).x() - cT; + const double LmaxT = uTail(i - 1, j, k).x() + cT; + + // min/max characteristic speeds on right + const double cH = cHead(i, j, k); + const double LminH = uHead(i, j, k).x() - cH; + const double LmaxH = uHead(i, j, k).x() + cH; + + const double f_Erad_T{EradTail(i - 1, j, k) * uTail(i - 1, j, k).x()}; + const double f_Erad_H{EradHead(i, j, k) * uHead(i, j, k).x()}; + + // clang-format off + EradF(i, j, k) = numerical_algorithms::advect_conserved( + EradTail(i - 1, j, k), EradHead(i, j, k), f_Erad_T, f_Erad_H, + LminT, LmaxT, LminH, LmaxH); + // clang-format on #endif }; // forall @@ -374,7 +433,7 @@ compute_interface_fluxes(std::size_t face_axis, m.template cells(), m.template cells()); - forall(kji, mdpolicy_qqq, "update_flux_3dx") { + s.executor().forall(kji, mdpolicy_qqq) { auto [k, j, i] = kji; dt_mass_density(i, j, k) += one_over_dx_i[0] * (rF(i, j, k) - rF(i + 1, j, k)); @@ -394,53 +453,66 @@ compute_interface_fluxes(std::size_t face_axis, m.template cells(), m.template cells()); - forall(kji, mdpolicy_qcq, "compute_flux_3dy") { + s.executor().forall(kji, mdpolicy_qcq) { auto [k, j, i] = kji; - // min/max characteristic speeds on left - const double cT = cTail(i, j - 1, k); - const double LminT = uTail(i, j - 1, k).y - cT; - const double LmaxT = uTail(i, j - 1, k).y + cT; - - // min/max characteristic speeds on right - const double cH = cHead(i, j, k); - const double LminH = uHead(i, j, k).y - cH; - const double LmaxH = uHead(i, j, k).y + cH; // Fluxes from left and right state - const double f_r_T{ruTail(i, j - 1, k).y}; - const double f_r_H{ruHead(i, j, k).y}; - const vec<3> f_ru_T{ruTail(i, j - 1, k).y * uTail(i, j - 1, k).x, - ruTail(i, j - 1, k).y * uTail(i, j - 1, k).y + pTail(i, j - 1, k), - ruTail(i, j - 1, k).y * uTail(i, j - 1, k).z}; - const vec<3> f_ru_H{ruHead(i, j, k).y * uHead(i, j, k).x, - ruHead(i, j, k).y * uHead(i, j, k).y + pHead(i, j, k), - ruHead(i, j, k).y * uHead(i, j, k).z}; + const double f_r_T{ruTail(i, j - 1, k).y()}; + const double f_r_H{ruHead(i, j, k).y()}; + const vec<3> f_ru_T{ruTail(i, j - 1, k).y() * uTail(i, j - 1, k).x(), + ruTail(i, j - 1, k).y() * uTail(i, j - 1, k).y() + pTail(i, j - 1, k), + ruTail(i, j - 1, k).y() * uTail(i, j - 1, k).z()}; + const vec<3> f_ru_H{ruHead(i, j, k).y() * uHead(i, j, k).x(), + ruHead(i, j, k).y() * uHead(i, j, k).y() + pHead(i, j, k), + ruHead(i, j, k).y() * uHead(i, j, k).z()}; const double f_rE_T{ - (rETail(i, j - 1, k) + pTail(i, j - 1, k)) * uTail(i, j - 1, k).y}; + (rETail(i, j - 1, k) + pTail(i, j - 1, k)) * uTail(i, j - 1, k).y()}; const double f_rE_H{ - (rEHead(i, j, k) + pHead(i, j, k)) * uHead(i, j, k).y}; + (rEHead(i, j, k) + pHead(i, j, k)) * uHead(i, j, k).y()}; + + // Advect conserved quantities // clang-format off - const auto hll_fluxes = - numerical_algorithms::hll_hydro( - rTail(i, j - 1, k), rHead(i, j, k), - ruTail(i, j - 1, k), ruHead(i, j, k), - rETail(i, j - 1, k), rEHead(i, j, k), - f_r_T, f_r_H, f_ru_T, f_ru_H, f_rE_T, f_rE_H, - LminT, LmaxT, LminH, LmaxH); + const double pTail_wave = pTail(i,j-1,k) + 0.5 * rTail(i,j-1,k) * g.y() / one_over_dx_i[1]; + const double pHead_wave = pHead(i,j,k) - 0.5 * rHead(i,j,k) * g.y() / one_over_dx_i[1]; + rF(i, j, k) = + numerical_algorithms::compute_HLLC_fluxes(face_axis, + rTail(i,j-1,k), rTail(i,j-1,k), uTail(i,j-1,k), rETail(i,j-1,k), pTail_wave, cTail(i,j-1,k), f_r_T, + rHead(i,j,k), rHead(i,j,k), uHead(i,j,k), rEHead(i,j,k), pHead_wave, cHead(i,j,k), f_r_H, + "rho" ); + ruF(i, j, k) = + numerical_algorithms::compute_HLLC_fluxes>(face_axis, + ruTail(i,j-1,k), rTail(i,j-1,k), uTail(i,j-1,k), rETail(i,j-1,k), pTail_wave, cTail(i,j-1,k), f_ru_T, + ruHead(i,j,k), rHead(i,j,k), uHead(i,j,k), rEHead(i,j,k), pHead_wave, cHead(i,j,k), f_ru_H, + "rhou" ); + rEF(i, j, k) = + numerical_algorithms::compute_HLLC_fluxes(face_axis, + rETail(i,j-1,k), rTail(i,j-1,k), uTail(i,j-1,k), rETail(i,j-1,k), pTail_wave, cTail(i,j-1,k), f_rE_T, + rEHead(i,j,k), rHead(i,j,k), uHead(i,j,k), rEHead(i,j,k), pHead_wave, cHead(i,j,k), f_rE_H, + "E" ); // clang-format on - rF(i, j, k) = get<0>(hll_fluxes); - ruF(i, j, k) = get<1>(hll_fluxes); - rEF(i, j, k) = get<2>(hll_fluxes); - #ifdef ENABLE_RADIATION - EradF(i, j, k) = - numerical_algorithms::advect_Erad(EradTail(i, j - 1, k), - EradHead(i, j, k), - uTail(i, j - 1, k).y, - uHead(i, j, k).y); + + // min/max characteristic speeds on left + const double cT = cTail(i, j - 1, k); + const double LminT = uTail(i, j - 1, k).y() - cT; + const double LmaxT = uTail(i, j - 1, k).y() + cT; + + // min/max characteristic speeds on right + const double cH = cHead(i, j, k); + const double LminH = uHead(i, j, k).y() - cH; + const double LmaxH = uHead(i, j, k).y() + cH; + + const double f_Erad_T{EradTail(i, j - 1, k) * uTail(i, j - 1, k).y()}; + const double f_Erad_H{EradHead(i, j, k) * uHead(i, j, k).y()}; + + // clang-format off + EradF(i, j, k) = numerical_algorithms::advect_conserved( + EradTail(i, j - 1, k), EradHead(i, j, k), f_Erad_T, f_Erad_H, LminT, + LmaxT, LminH, LmaxH); + // clang-format on #endif }; // forall @@ -451,7 +523,7 @@ compute_interface_fluxes(std::size_t face_axis, m.template cells(), m.template cells()); - forall(kji, mdpolicy_qqq, "update_flux_3dy") { + s.executor().forall(kji, mdpolicy_qqq) { auto [k, j, i] = kji; dt_mass_density(i, j, k) += one_over_dx_i[1] * (rF(i, j, k) - rF(i, j + 1, k)); @@ -471,53 +543,67 @@ compute_interface_fluxes(std::size_t face_axis, m.template cells(), m.template cells()); - forall(kji, mdpolicy_cqq, "compute_flux_3dz") { + s.executor().forall(kji, mdpolicy_cqq) { auto [k, j, i] = kji; - // min/max characteristic speeds on left - const double cT = cTail(i, j, k - 1); - const double LminT = uTail(i, j, k - 1).z - cT; - const double LmaxT = uTail(i, j, k - 1).z + cT; - - // min/max characteristic speeds on right - const double cH = cHead(i, j, k); - const double LminH = uHead(i, j, k).z - cH; - const double LmaxH = uHead(i, j, k).z + cH; // Fluxes from left and right state - const double f_r_T{ruTail(i, j, k - 1).z}; - const double f_r_H{ruHead(i, j, k).z}; - const vec<3> f_ru_T{ruTail(i, j, k - 1).z * uTail(i, j, k - 1).x, - ruTail(i, j, k - 1).z * uTail(i, j, k - 1).y, - ruTail(i, j, k - 1).z * uTail(i, j, k - 1).z + pTail(i, j, k - 1)}; - const vec<3> f_ru_H{ruHead(i, j, k).z * uHead(i, j, k).x, - ruHead(i, j, k).z * uHead(i, j, k).y, - ruHead(i, j, k).z * uHead(i, j, k).z + pHead(i, j, k)}; + const double f_r_T{ruTail(i, j, k - 1).z()}; + const double f_r_H{ruHead(i, j, k).z()}; + const vec<3> f_ru_T{ruTail(i, j, k - 1).z() * uTail(i, j, k - 1).x(), + ruTail(i, j, k - 1).z() * uTail(i, j, k - 1).y(), + ruTail(i, j, k - 1).z() * uTail(i, j, k - 1).z() + + pTail(i, j, k - 1)}; + const vec<3> f_ru_H{ruHead(i, j, k).z() * uHead(i, j, k).x(), + ruHead(i, j, k).z() * uHead(i, j, k).y(), + ruHead(i, j, k).z() * uHead(i, j, k).z() + pHead(i, j, k)}; const double f_rE_T{ - (rETail(i, j, k - 1) + pTail(i, j, k - 1)) * uTail(i, j, k - 1).z}; + (rETail(i, j, k - 1) + pTail(i, j, k - 1)) * uTail(i, j, k - 1).z()}; const double f_rE_H{ - (rEHead(i, j, k) + pHead(i, j, k)) * uHead(i, j, k).z}; + (rEHead(i, j, k) + pHead(i, j, k)) * uHead(i, j, k).z()}; + + // Advect conserved quantities // clang-format off - const auto hll_fluxes = - numerical_algorithms::hll_hydro( - rTail(i, j, k - 1), rHead(i, j, k), - ruTail(i, j, k - 1), ruHead(i, j, k), - rETail(i, j, k - 1), rEHead(i, j, k), - f_r_T, f_r_H, f_ru_T, f_ru_H, f_rE_T, f_rE_H, - LminT, LmaxT, LminH, LmaxH); + const double pTail_wave = pTail(i,j,k-1) + 0.5 * rTail(i,j,k-1) * g.z() / one_over_dx_i[2]; + const double pHead_wave = pHead(i,j,k) - 0.5 * rHead(i,j,k) * g.z() / one_over_dx_i[2]; + rF(i, j, k) = + numerical_algorithms::compute_HLLC_fluxes(face_axis, + rTail(i,j,k-1), rTail(i,j,k-1), uTail(i,j,k-1), rETail(i,j,k-1), pTail_wave, cTail(i,j,k-1), f_r_T, + rHead(i,j,k), rHead(i,j,k), uHead(i,j,k), rEHead(i,j,k), pHead_wave, cHead(i,j,k), f_r_H, + "rho" ); + ruF(i, j, k) = + numerical_algorithms::compute_HLLC_fluxes>(face_axis, + ruTail(i,j,k-1), rTail(i,j,k-1), uTail(i,j,k-1), rETail(i,j,k-1), pTail_wave, cTail(i,j,k-1), f_ru_T, + ruHead(i,j,k), rHead(i,j,k), uHead(i,j,k), rEHead(i,j,k), pHead_wave, cHead(i,j,k), f_ru_H, + "rhou" ); + rEF(i, j, k) = + numerical_algorithms::compute_HLLC_fluxes(face_axis, + rETail(i,j,k-1), rTail(i,j,k-1), uTail(i,j,k-1), rETail(i,j,k-1), pTail_wave, cTail(i,j,k-1), f_rE_T, + rEHead(i,j,k), rHead(i,j,k), uHead(i,j,k), rEHead(i,j,k), pHead_wave, cHead(i,j,k), f_rE_H, + "E" ); // clang-format on - rF(i, j, k) = get<0>(hll_fluxes); - ruF(i, j, k) = get<1>(hll_fluxes); - rEF(i, j, k) = get<2>(hll_fluxes); - #ifdef ENABLE_RADIATION - EradF(i, j, k) = - numerical_algorithms::advect_Erad(EradTail(i, j, k - 1), - EradHead(i, j, k), - uTail(i, j, k - 1).z, - uHead(i, j, k).z); + + // min/max characteristic speeds on left + const double cT = cTail(i, j, k - 1); + const double LminT = uTail(i, j, k - 1).z() - cT; + const double LmaxT = uTail(i, j, k - 1).z() + cT; + + // min/max characteristic speeds on right + const double cH = cHead(i, j, k); + const double LminH = uHead(i, j, k).z() - cH; + const double LmaxH = uHead(i, j, k).z() + cH; + + const double f_Erad_T{EradTail(i, j, k - 1) * uTail(i, j, k - 1).z()}; + const double f_Erad_H{EradHead(i, j, k) * uHead(i, j, k).z()}; + + // clang-format off + EradF(i, j, k) = numerical_algorithms::advect_conserved( + EradTail(i, j, k - 1), EradHead(i, j, k), f_Erad_T, f_Erad_H, + LminT, LmaxT, LminH, LmaxH); + // clang-format on #endif }; // forall @@ -528,7 +614,7 @@ compute_interface_fluxes(std::size_t face_axis, m.template cells(), m.template cells()); - forall(kji, mdpolicy_qqq, "update_flux_3dz") { + s.executor().forall(kji, mdpolicy_qqq) { auto [k, j, i] = kji; dt_mass_density(i, j, k) += one_over_dx_i[2] * (rF(i, j, k) - rF(i, j, k + 1)); diff --git a/app/tasks/hydro/cons2prim.hh b/app/tasks/hydro/cons2prim.hh index 8e52d4c..ffde3e7 100644 --- a/app/tasks/hydro/cons2prim.hh +++ b/app/tasks/hydro/cons2prim.hh @@ -10,7 +10,8 @@ namespace hard::tasks::hydro { template void -conservative_to_primitive(typename mesh::template accessor m, +conservative_to_primitive(flecsi::exec::accelerator s, + typename mesh::template accessor m, field::accessor mass_density_a, typename field>::template accessor momentum_density_a, field::accessor total_energy_density_a, @@ -18,7 +19,8 @@ conservative_to_primitive(typename mesh::template accessor m, field::accessor pressure_a, field::accessor specific_internal_energy_a, field::accessor soundspeed_a, - eos::eos_wrapper const & eos) { + eos::eos_wrapper const & eos) noexcept { + using hard::tasks::util::get_mdiota_policy; auto mass_density = m.template mdcolex(mass_density_a); @@ -32,7 +34,7 @@ conservative_to_primitive(typename mesh::template accessor m, auto soundspeed = m.template mdcolex(soundspeed_a); if constexpr(Dim == 1) { - forall(i, (m.template cells()), "conv2prim") { + s.executor().forall(i, (m.template cells())) { velocity(i) = momentum_density(i) / mass_density(i); specific_internal_energy(i) = (total_energy_density(i) - @@ -47,7 +49,7 @@ conservative_to_primitive(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(ji, mdpolicy_yx, "conv2prim") { + s.executor().forall(ji, mdpolicy_yx) { auto [j, i] = ji; velocity(i, j) = momentum_density(i, j) / mass_density(i, j); specific_internal_energy(i, j) = @@ -65,7 +67,7 @@ conservative_to_primitive(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(kji, mdpolicy_zyx, "conv2prim") { + s.executor().forall(kji, mdpolicy_zyx) { auto [k, j, i] = kji; velocity(i, j, k) = momentum_density(i, j, k) / mass_density(i, j, k); specific_internal_energy(i, j, k) = diff --git a/app/tasks/hydro/maxcharspeed.hh b/app/tasks/hydro/maxcharspeed.hh index 7257e79..c7b316e 100644 --- a/app/tasks/hydro/maxcharspeed.hh +++ b/app/tasks/hydro/maxcharspeed.hh @@ -7,26 +7,21 @@ namespace hard::tasks::hydro { template double -update_max_characteristic_speed(typename mesh::template accessor m, +update_max_characteristic_speed(flecsi::exec::accelerator s, + typename mesh::template accessor m, field::accessor r_a, typename field>::template accessor u_a, - field::accessor p_a, - field::accessor c_a) { + field::accessor c_a) noexcept { using hard::tasks::util::get_mdiota_policy; namespace fold = flecsi::exec::fold; auto density = m.template mdcolex(r_a); auto velocity = m.template mdcolex(u_a); - auto pressure = m.template mdcolex(p_a); auto soundspeed = m.template mdcolex(c_a); if constexpr(D == 1) { - return reduceall(i, - up, - (m.template cells()), - fold::max, - double, - "max_char_1d") { + return s.executor().reduceall( + i, up, (m.template cells()), fold::max, double) { up(velocity(i).norm() + soundspeed(i)); }; } @@ -35,7 +30,7 @@ update_max_characteristic_speed(typename mesh::template accessor m, m.template cells(), m.template cells()); - return reduceall(ji, up, mdpolicy_qq, fold::max, double, "max_char_2d") { + return s.executor().reduceall(ji, up, mdpolicy_qq, fold::max, double) { auto [j, i] = ji; up(velocity(i, j).norm() + soundspeed(i, j)); }; @@ -46,7 +41,7 @@ update_max_characteristic_speed(typename mesh::template accessor m, m.template cells(), m.template cells()); - return reduceall(kji, up, mdpolicy_qqq, fold::max, double, "max_char_3d") { + return s.executor().reduceall(kji, up, mdpolicy_qqq, fold::max, double) { auto [k, j, i] = kji; up(velocity(i, j, k).norm() + soundspeed(i, j, k)); }; diff --git a/app/tasks/hydro/reconstruct.hh b/app/tasks/hydro/reconstruct.hh index 6dfdd66..7f4e548 100644 --- a/app/tasks/hydro/reconstruct.hh +++ b/app/tasks/hydro/reconstruct.hh @@ -9,13 +9,14 @@ namespace hard::tasks::hydro { template struct stencil { template - auto operator()(const T & i, const A & acc) { + FLECSI_INLINE_TARGET auto operator()(const T & i, const A & acc) { return Limiter::reconstruct( acc(i - 2), acc(i - 1), acc(i), acc(i + 1), acc(i + 2)); } template - auto operator()(const int & x, const T & i, const T & j, const A & acc) { + FLECSI_INLINE_TARGET auto + operator()(const int & x, const T & i, const T & j, const A & acc) { if(x == 0) return Limiter::reconstruct( acc(i - 2, j), acc(i - 1, j), acc(i, j), acc(i + 1, j), acc(i + 2, j)); @@ -24,7 +25,7 @@ struct stencil { } template - auto operator()(const int & x, + FLECSI_INLINE_TARGET auto operator()(const int & x, const T & i, const T & j, const T & k, @@ -49,6 +50,17 @@ struct stencil { } }; +namespace utils { + +template +FLECSI_INLINE_TARGET void +tie(const std::tuple & tup, T & a, T & b) noexcept { + a = std::get<0>(tup); + b = std::get<1>(tup); +} + +} // namespace utils + // // Perform reconstruction of primitive variables on cell interfaces, calculate // corresponding conservative variables on faces, and store them into `*Head` @@ -64,7 +76,8 @@ struct stencil { // template void -reconstruct(std::size_t reconstruction_axis, +reconstruct(flecsi::exec::accelerator s, + std::size_t reconstruction_axis, typename mesh::template accessor m, // cell-centered primitive varibles field::accessor mass_density_a, @@ -102,7 +115,7 @@ reconstruct(std::size_t reconstruction_axis, typename field>::template accessor ruTail_a, typename field>::template accessor ruHead_a, field::accessor rETail_a, - field::accessor rEHead_a) { + field::accessor rEHead_a) noexcept { auto mass_density = m.template mdcolex(mass_density_a); auto velocity = m.template mdcolex(velocity_a); @@ -141,14 +154,14 @@ reconstruct(std::size_t reconstruction_axis, if constexpr(Dim == 1) { - forall(i, (m.template cells()), "reconstruct_1d") { + s.executor().forall(i, (m.template cells())) { + utils::tie(stencil()(i, mass_density), rHead(i), rTail(i)); + utils::tie(stencil()(i, pressure), pHead(i), pTail(i)); + utils::tie( + stencil()(i, specific_internal_energy), eHead(i), eTail(i)); + utils::tie(stencil()(i, soundspeed), cHead(i), cTail(i)); + utils::tie(stencil()(i, velocity), uHead(i), uTail(i)); - std::tie(rHead(i), rTail(i)) = stencil()(i, mass_density); - std::tie(uHead(i), uTail(i)) = stencil()(i, velocity); - std::tie(pHead(i), pTail(i)) = stencil()(i, pressure); - std::tie(eHead(i), eTail(i)) = - stencil()(i, specific_internal_energy); - std::tie(cHead(i), cTail(i)) = stencil()(i, soundspeed); // Compute conservative variables ruHead(i) = rHead(i) * uHead(i); ruTail(i) = rTail(i) * uTail(i); @@ -158,9 +171,11 @@ reconstruct(std::size_t reconstruction_axis, rTail(i) * eTail(i) + 0.5 * rTail(i) * uTail(i).norm_squared(); #ifdef ENABLE_RADIATION - std::tie(EradHead(i), EradTail(i)) = - stencil()(i, radiation_energy_density); + utils::tie(stencil()(i, radiation_energy_density), + EradHead(i), + EradTail(i)); #endif + }; // forall } else if constexpr(Dim == 2) { @@ -169,24 +184,25 @@ reconstruct(std::size_t reconstruction_axis, m.template cells(), m.template cells()); - forall(ji, mdpolicy_pp, "reconstruct_2d") { - + s.executor().forall(ji, mdpolicy_pp) { auto [j, i] = ji; - std::tie(rHead(i, j), rTail(i, j)) = - stencil()(ra, i, j, mass_density); - std::tie(uHead(i, j), uTail(i, j)) = - stencil()(ra, i, j, velocity); - std::tie(pHead(i, j), pTail(i, j)) = - stencil()(ra, i, j, pressure); - std::tie(eHead(i, j), eTail(i, j)) = - stencil()(ra, i, j, specific_internal_energy); - std::tie(cHead(i, j), cTail(i, j)) = - stencil()(ra, i, j, soundspeed); + utils::tie( + stencil()(ra, i, j, mass_density), rHead(i, j), rTail(i, j)); + utils::tie( + stencil()(ra, i, j, pressure), pHead(i, j), pTail(i, j)); + utils::tie(stencil()(ra, i, j, specific_internal_energy), + eHead(i, j), + eTail(i, j)); + utils::tie( + stencil()(ra, i, j, soundspeed), cHead(i, j), cTail(i, j)); + utils::tie( + stencil()(ra, i, j, velocity), uHead(i, j), uTail(i, j)); #ifdef ENABLE_RADIATION - std::tie(EradHead(i, j), EradTail(i, j)) = - stencil()(ra, i, j, radiation_energy_density); + utils::tie(stencil()(ra, i, j, radiation_energy_density), + EradHead(i, j), + EradTail(i, j)); #endif // Compute conservative variables @@ -196,7 +212,6 @@ reconstruct(std::size_t reconstruction_axis, 0.5 * rHead(i, j) * uHead(i, j).norm_squared(); rETail(i, j) = rTail(i, j) * eTail(i, j) + 0.5 * rTail(i, j) * uTail(i, j).norm_squared(); - }; // forall } else { // Dim == 3 @@ -206,23 +221,29 @@ reconstruct(std::size_t reconstruction_axis, m.template cells(), m.template cells()); - forall(kji, mdpolicy_ppp, "reconstruct_3d") { + s.executor().forall(kji, mdpolicy_ppp) { auto [k, j, i] = kji; - std::tie(rHead(i, j, k), rTail(i, j, k)) = - stencil()(ra, i, j, k, mass_density); - std::tie(uHead(i, j, k), uTail(i, j, k)) = - stencil()(ra, i, j, k, velocity); - std::tie(pHead(i, j, k), pTail(i, j, k)) = - stencil()(ra, i, j, k, pressure); - std::tie(eHead(i, j, k), eTail(i, j, k)) = - stencil()(ra, i, j, k, specific_internal_energy); - std::tie(cHead(i, j, k), cTail(i, j, k)) = - stencil()(ra, i, j, k, soundspeed); + utils::tie(stencil()(ra, i, j, k, mass_density), + rHead(i, j, k), + rTail(i, j, k)); + utils::tie(stencil()(ra, i, j, k, pressure), + pHead(i, j, k), + pTail(i, j, k)); + utils::tie(stencil()(ra, i, j, k, specific_internal_energy), + eHead(i, j, k), + eTail(i, j, k)); + utils::tie(stencil()(ra, i, j, k, soundspeed), + cHead(i, j, k), + cTail(i, j, k)); + utils::tie(stencil()(ra, i, j, k, velocity), + uHead(i, j, k), + uTail(i, j, k)); #ifdef ENABLE_RADIATION - std::tie(EradHead(i, j, k), EradTail(i, j, k)) = - stencil()(ra, i, j, k, radiation_energy_density); + utils::tie(stencil()(ra, i, j, k, radiation_energy_density), + EradHead(i, j, k), + EradTail(i, j, k)); #endif // Compute conservative variables on faces @@ -232,7 +253,6 @@ reconstruct(std::size_t reconstruction_axis, 0.5 * rHead(i, j, k) * uHead(i, j, k).norm_squared(); rETail(i, j, k) = eTail(i, j, k) * eTail(i, j, k) + 0.5 * rTail(i, j, k) * uTail(i, j, k).norm_squared(); - }; // forall } } diff --git a/app/tasks/init.hh b/app/tasks/init.hh index 159bd81..0bec396 100644 --- a/app/tasks/init.hh +++ b/app/tasks/init.hh @@ -10,14 +10,32 @@ namespace hard { namespace tasks::init { -void inline compute_dt_weighted(single::accessor dt, +template +void inline initialize_gravity_force(flecsi::exec::cpu, + typename field>::template accessor gravity_force_a) { + auto gf = gravity_force_a.span(); + // initialize gravity force with zero + std::fill(gf.begin(), gf.end(), vec{0.0}); +} // initialize_gravity_force + +template +void inline intialize_gravity_acc( + typename single>::template accessor gravity_acc_a, + vec k) { + (*gravity_acc_a) = k; +} // initialize_gravity_acc + +void inline compute_dt_weighted(flecsi::exec::cpu, + single::accessor dt, single::accessor dt_w, - double tsg) { + double tsg) noexcept { dt_w = *dt * tsg; } inline void -init_time(single::accessor time, double vtime) { +init_time(flecsi::exec::cpu, + single::accessor time, + double vtime) noexcept { *time = vtime; } @@ -25,7 +43,8 @@ init_time(single::accessor time, double vtime) { Temperature boundaries. *----------------------------------------------------------------------------*/ -void inline set_t_boundary(field::accessor t_boundary, +void inline set_t_boundary(flecsi::exec::cpu, + field::accessor t_boundary, std::vector copy_values) { for(std::size_t i{0}; i < t_boundary.span().size(); i++) { @@ -33,11 +52,28 @@ void inline set_t_boundary(field::accessor t_boundary, } } // t_boundary +/*----------------------------------------------------------------------------* + Adaptive Limiter Check, Closure ID, Limiter ID + *----------------------------------------------------------------------------*/ + +// Simple opacity that is used across all +void inline closure_id(single::accessor closure_id_a, + std::size_t clid) noexcept { + (*closure_id_a) = clid; +} // closure_id + +// Simple opacity that is used across all +void inline limiter_id(single::accessor limiter_id_a, + std::size_t lmid) noexcept { + (*limiter_id_a) = lmid; +} // limiter_id + /*----------------------------------------------------------------------------* Temperature unit conversion from eV (or other) to Kelvin *----------------------------------------------------------------------------*/ -void inline convert_temperature(field::accessor temperature, +void inline convert_temperature(flecsi::exec::cpu, + field::accessor temperature, std::string const & unit) { assert((unit == "Kelvin" || unit == "eV") && "Unsupported temperature unit"); @@ -89,7 +125,7 @@ void inline kappaR(single::accessor kappaR_a, double kR) { void inline particle_mass(single::accessor particle_mass_a, double mean_molecular_weight) { (*particle_mass_a) = mean_molecular_weight * constants::cgs::proton_mass; -} // kappa +} // particle_mass /*----------------------------------------------------------------------------* Fake initialization tasks to avoid legion errors. @@ -137,6 +173,7 @@ touch(typename mesh::template accessor, // m, typename field>::template accessor, // radiation_force_a, field::accessor, // R_value_a, field::accessor, // lambda_bridge_a, + field::accessor, // eddington_factor_a, typename field>::template accessor // velocity_gradient_a #endif @@ -185,55 +222,13 @@ touch1(typename mesh::template accessor, // m, typename field>::template accessor, // radiation_force_a, field::accessor, // R_value_a, field::accessor, // lambda_bridge_a, + field::accessor, // eddington_factor_a, typename field>::template accessor // velocity_gradient_a #endif ) { } // touch -/*----------------------------------------------------------------------------* - Boundary input translation. - *----------------------------------------------------------------------------*/ - -template -mesh::periodic_axes -boundaries( - typename single::bmap>::template accessor bmap_a, - std::array, D> bnds) { - auto & bmap = *bmap_a; - constexpr auto periodic = mesh::boundary_type::periodic; - - typename mesh::periodic_axes p; - p[ax::x] = false; - if constexpr(D == 2 || D == 3) { - p[ax::y] = false; - } - else if constexpr(D == 3) { - p[ax::z] = false; - } // if - - bmap[ax::x][bd::low] = bnds[ax::x][bd::low]; - bmap[ax::x][bd::high] = bnds[ax::x][bd::high]; - p[ax::x] = - bnds[ax::x][bd::low] == periodic && bnds[ax::x][bd::high] == periodic; - - if constexpr(D == 2 || D == 3) { - bmap[ax::y][bd::low] = bnds[ax::y][bd::low]; - bmap[ax::y][bd::high] = bnds[ax::y][bd::high]; - p[ax::y] = - bnds[ax::y][bd::low] == periodic && bnds[ax::y][bd::high] == periodic; - } // if - - if constexpr(D == 3) { - bmap[ax::z][bd::low] = bnds[ax::z][bd::low]; - bmap[ax::z][bd::high] = bnds[ax::z][bd::high]; - p[ax::z] = - bnds[ax::z][bd::low] == periodic && bnds[ax::z][bd::high] == periodic; - } // if - - return p; -} // boundaries - } // namespace tasks::init } // namespace hard diff --git a/app/tasks/initial_data/acoustic_wave.hh b/app/tasks/initial_data/acoustic_wave.hh new file mode 100644 index 0000000..4afef5a --- /dev/null +++ b/app/tasks/initial_data/acoustic_wave.hh @@ -0,0 +1,86 @@ + +#pragma once + +#include "../../options.hh" +#include "../../types.hh" +#include "../utils.hh" +#include +#include +#include +#include + +namespace hard::tasks::initial_data { + +// +// An acoustic wave set-up +// +template +auto +acoustic_wave(flecsi::exec::cpu s, + typename mesh::template accessor m, + field::accessor mass_density_a, + typename field>::template accessor momentum_density_a, + field::accessor total_energy_density_a, + field::accessor radiation_energy_density_a, + const eos::eos_wrapper & eos) { + + auto mass_density = m.template mdcolex(mass_density_a); + auto momentum_density = m.template mdcolex(momentum_density_a); + auto total_energy_density = + m.template mdcolex(total_energy_density_a); + auto radiation_energy_density = + m.template mdcolex(radiation_energy_density_a); + + YAML::Node config = YAML::LoadFile(opt::config.value()); + + // Problem parameters + // Equilibrium values + const double r0{ + config["problem_parameters"]["r0"].as()}; // Equilibrium density + const double p0{ + config["problem_parameters"]["p0"].as()}; // Equilibrium pressure + + // Perturbation amplitudes + const double rA{ + config["problem_parameters"]["amplitude"].as()}; // Density + const double uA{ + config["problem_parameters"]["amplitude"].as()}; // Velocity + + // Sound speed + const double cs{sqrt(config["gamma"].as() * p0 / r0)}; + + // Define the inverse of the wave number + const double scale{config["problem_parameters"]["scale"].as()}; + + // sine_quad is the volume average of the sine per cell, + // for sine wave fvm initialization + const double k{2 * M_PI * scale}; + auto sine_quad = [k](double x0, double x1) { + return (cos(k * x0) - cos(k * x1)) / (k * (x1 - x0)); + }; + + // + // Only 1D version has been implemented. + // + if constexpr(Dim == 1) { + s.executor().forall(i, (m.template cells())) { + const auto x0{m.template head(i)}; + const auto x1{m.template tail(i)}; + const double ux{cs * uA * sine_quad(x0, x1)}; + mass_density(i) = r0 * (1 + rA * sine_quad(x0, x1)); + + momentum_density(i).x() = mass_density(i) * ux; + const double e = util::find_sie(eos, mass_density(i), p0); + total_energy_density(i) = + mass_density(i) * e + 0.5 * mass_density(i) * utils::sqr(ux); + + radiation_energy_density(i) = 0.0; + + }; // forall + } + else { + flog_fatal("Acoustic wave problem for D >= 2 is not implemented") + } +} // acoustic_wave + +} // namespace hard::tasks::initial_data diff --git a/app/tasks/initial_data/all_initial_data.hh b/app/tasks/initial_data/all_initial_data.hh index c10a30d..429c126 100644 --- a/app/tasks/initial_data/all_initial_data.hh +++ b/app/tasks/initial_data/all_initial_data.hh @@ -1,11 +1,15 @@ #pragma once +#include "acoustic_wave.hh" #include "heating_and_cooling.hh" +#include "heating_and_cooling_afld.hh" #include "implosion_forced_T.hh" #include "kelvin_helm.hh" +#include "kelvin_helm_rad.hh" #include "lw_implosion.hh" #include "rad_rank_hugoniot.hh" +#include "rayleigh_taylor.hh" +#include "richtmyer_meshkov.hh" #include "sedov.hh" #include "shock_tube.hh" -#include "sine_wave.hh" diff --git a/app/tasks/initial_data/heating_and_cooling.hh b/app/tasks/initial_data/heating_and_cooling.hh index b76d426..b917d48 100644 --- a/app/tasks/initial_data/heating_and_cooling.hh +++ b/app/tasks/initial_data/heating_and_cooling.hh @@ -22,7 +22,8 @@ namespace hard::tasks::initial_data { // template auto -heating_and_cooling(typename mesh::template accessor m, +heating_and_cooling(flecsi::exec::cpu s, + typename mesh::template accessor m, field::accessor mass_density_a, typename field>::template accessor momentum_density_a, field::accessor total_energy_density_a, @@ -57,13 +58,13 @@ heating_and_cooling(typename mesh::template accessor m, spec::utils::sqr(spec::utils::sqr(radiation_temperature)); if constexpr(Dim == 1) { - for(auto i : m.template cells()) { + s.executor().forall(i, (m.template cells())) { mass_density(i) = mass_density_v; - momentum_density(i).x = 0.0; + momentum_density(i).x() = 0.0; total_energy_density(i) = fluid_internal_energy_density; radiation_energy_density(i) = radiation_energy_density_v; - } + }; } else { flog_fatal( diff --git a/app/tasks/initial_data/heating_and_cooling_afld.hh b/app/tasks/initial_data/heating_and_cooling_afld.hh new file mode 100644 index 0000000..f47e48e --- /dev/null +++ b/app/tasks/initial_data/heating_and_cooling_afld.hh @@ -0,0 +1,75 @@ + +#pragma once + +#include "../../constants.hh" +#include "../../options.hh" +#include "../../types.hh" +#include +#include +#include + +namespace hard::tasks::initial_data { + +// +// A test problem demonstrating fluid and radiation reaching to a thermal +// equilibrium. +// +// The fluid is uniform and initially at rest everywhere. Radiation energy +// density is also uniform everywhere. +// +// Ideally, since E_rad = const. and nonzero everywhere, this problem should run +// without any issues even if every radiation-hydro tasks are switched on. +// +template +auto +heating_and_cooling_afld(flecsi::exec::accelerator s, + typename mesh::template accessor m, + field::accessor mass_density_a, + typename field>::template accessor momentum_density_a, + field::accessor total_energy_density_a, + field::accessor radiation_energy_density_a, + single::accessor particle_mass_a, + const double gamma) noexcept { + + auto mass_density = m.template mdcolex(mass_density_a); + auto momentum_density = m.template mdcolex(momentum_density_a); + auto total_energy_density = + m.template mdcolex(total_energy_density_a); + auto radiation_energy_density = + m.template mdcolex(radiation_energy_density_a); + auto const particle_mass = *particle_mass_a; + + // Parse input parameters + YAML::Node config = YAML::LoadFile(opt::config.value()); + + const double mass_density_v = + config["problem_parameters"]["fluid_mass_density"].as(); + const double fluid_temperature = + config["problem_parameters"]["fluid_temperature"].as(); + const double radiation_temperature = + config["problem_parameters"]["radiation_temperature"].as(); + + // Note : assuming ideal gas EOS + const double fluid_internal_energy_density = + mass_density_v * constants::cgs::boltzmann_constant * fluid_temperature / + ((gamma - 1.0) * particle_mass); + const double radiation_energy_density_v = + constants::cgs::radiation_constant * + spec::utils::sqr(spec::utils::sqr(radiation_temperature)); + + if constexpr(Dim == 1) { + s.executor().forall(i, (m.template cells())) { + mass_density(i) = mass_density_v; + momentum_density(i).x() = 0.0; + + total_energy_density(i) = fluid_internal_energy_density; + radiation_energy_density(i) = radiation_energy_density_v; + }; + } + else { + flog_fatal( + "Radiative heating-cooling problem is only implemented for D = 1") + } +} + +} // namespace hard::tasks::initial_data diff --git a/app/tasks/initial_data/implosion_forced_T.hh b/app/tasks/initial_data/implosion_forced_T.hh index 729cab1..201dfcf 100644 --- a/app/tasks/initial_data/implosion_forced_T.hh +++ b/app/tasks/initial_data/implosion_forced_T.hh @@ -23,7 +23,8 @@ struct implosion { template auto -implosion_forced_T(typename mesh::template accessor m, +implosion_forced_T(flecsi::exec::cpu s, + typename mesh::template accessor m, field::accessor mass_density_a, typename field>::template accessor momentum_density_a, field::accessor total_energy_density_a, @@ -71,13 +72,13 @@ implosion_forced_T(typename mesh::template accessor m, radiation_constant * sqr(sqr(radiation_temperature)); if constexpr(Dim == 1) { - for(auto i : m.template cells()) { + s.executor().forall(i, (m.template cells())) { mass_density(i) = mass_density_v; - momentum_density(i).x = 0.0; + momentum_density(i).x() = 0.0; total_energy_density(i) = fluid_internal_energy_density; radiation_energy_density(i) = radiation_energy_density_v; - } + }; } else { flog_fatal("Implosion problem is only implemented for D == 1") diff --git a/app/tasks/initial_data/kelvin_helm.hh b/app/tasks/initial_data/kelvin_helm.hh index d3d81b7..a645b07 100644 --- a/app/tasks/initial_data/kelvin_helm.hh +++ b/app/tasks/initial_data/kelvin_helm.hh @@ -13,7 +13,8 @@ namespace hard::tasks::initial_data { template auto -kh_instability(typename mesh::template accessor m, +kh_instability(flecsi::exec::cpu s, + typename mesh::template accessor m, field::accessor mass_density_a, typename field>::template accessor momentum_density_a, field::accessor total_energy_density_a, @@ -44,7 +45,7 @@ kh_instability(typename mesh::template accessor m, "Kelvin-Helmholtz instability problem for D == 1 is not implemented") } else if constexpr(D == 2) { - forall(j, (m.template cells()), "init_kh_2d") { + s.executor().forall(j, (m.template cells())) { for(auto i : m.template cells()) { const auto x = m.template center(i); const auto y = m.template center(j); @@ -52,15 +53,15 @@ kh_instability(typename mesh::template accessor m, // initialize two different density and velocity fluids if(std::abs(y - 0.5) > 0.25) { mass_density(i, j) = rL; - momentum_density(i, j).x = rL * uL; - momentum_density(i, j).y = rL * vL; + momentum_density(i, j).x() = rL * uL; + momentum_density(i, j).y() = rL * vL; const double e = util::find_sie(eos, rL, pL); total_energy_density(i, j) = rL * e + 0.5 * rL * (vL * vL); } else { mass_density(i, j) = rH; - momentum_density(i, j).x = rH * uH; - momentum_density(i, j).y = rH * vH; + momentum_density(i, j).x() = rH * uH; + momentum_density(i, j).y() = rH * vH; const double e = util::find_sie(eos, rH, pH); total_energy_density(i, j) = rH * e + 0.5 * rH * (vH * vH); } // if @@ -69,10 +70,10 @@ kh_instability(typename mesh::template accessor m, // velocity perturbations in the Y-direction if(std::abs(y - 0.25) < 0.1) { - momentum_density(i, j).y = 0.05 * sin(N * wavenumber * x); + momentum_density(i, j).y() = 0.05 * sin(N * wavenumber * x); } if(std::abs(y - 0.75) < 0.1) { - momentum_density(i, j).y = 0.05 * sin(N * wavenumber * x); + momentum_density(i, j).y() = 0.05 * sin(N * wavenumber * x); } } // for }; // forall diff --git a/app/tasks/initial_data/kelvin_helm_rad.hh b/app/tasks/initial_data/kelvin_helm_rad.hh new file mode 100644 index 0000000..27a3629 --- /dev/null +++ b/app/tasks/initial_data/kelvin_helm_rad.hh @@ -0,0 +1,180 @@ +#pragma once + +#include "../../constants.hh" +#include "../../options.hh" +#include "../../types.hh" +#include +#include +#include +#include + +namespace hard::tasks::initial_data { + +/*----------------------------------------------------------------------------* + Kelvin-Helmholtz Instability with Radiation Enabled + *----------------------------------------------------------------------------*/ + +template +auto +kh_instability_rad(flecsi::exec::cpu s, + typename mesh::template accessor m, + field::accessor mass_density_a, + typename field>::template accessor momentum_density_a, + field::accessor total_energy_density_a, + field::accessor radiation_energy_density_a, + const eos::eos_wrapper & eos) noexcept { + + auto mass_density = m.template mdcolex(mass_density_a); + auto momentum_density = m.template mdcolex(momentum_density_a); + auto total_energy_density = + m.template mdcolex(total_energy_density_a); + auto radiation_energy_density = + m.template mdcolex(radiation_energy_density_a); + + // constants required from constants.hh + const double a = hard::constants::cgs::radiation_constant; + + // Parse input parameters + YAML::Node config = YAML::LoadFile(opt::config.value()); + + // Domain Parameters - + // X and Z dimensions not used currently so not called here + const double y_min = config["coords"][0][1].as(); + const double y_max = config["coords"][1][1].as(); + const double L_y = std::abs(y_max - y_min); + + // setting density and velocity + // H is the heavier fluid at bottom + const double rH = + config["problem_parameters"]["fluid_mass_density_high"].as(); + const double pH = + config["problem_parameters"]["fluid_pressure_high"].as(); + const double uH = + config["problem_parameters"]["fluid_x_velocity_high"].as(); + const double vH = + config["problem_parameters"]["fluid_y_velocity_high"].as(); + double wH{0}; + + // L is the lighter fluid at top + const double rL = + config["problem_parameters"]["fluid_mass_density_low"].as(); + const double pL = + config["problem_parameters"]["fluid_pressure_low"].as(); + const double uL = + config["problem_parameters"]["fluid_x_velocity_low"].as(); + const double vL = + config["problem_parameters"]["fluid_y_velocity_low"].as(); + double wL{0}; + + if constexpr(D == 3) { + // setting density and velocity + // H is the heavier fluid at bottom + wH = config["problem_parameters"]["fluid_z_velocity_high"].as(); + // L is the lighter fluid at top + wL = config["problem_parameters"]["fluid_z_velocity_low"].as(); + } + + // setting fluid separation and velocity perturbation fractions and + // location(s) (fraction so as to make actual location agnostic of domain + // dimensions) + const double y_sep_f = + config["problem_parameters"]["fluid_sep_y_frac"].as(); + const double y_vp_f = + config["problem_parameters"]["vel_per_y_frac"].as(); + const double y_sep = y_sep_f * L_y; + const double y_vp = y_vp_f * L_y; + + // setting perturbation parameters + // constants for perturbation + const double N = config["problem_parameters"]["perturb_N"].as(); + + const double wavenumber = 2.0 * M_PI * N; + + // setting temperature + double rad_temp = 0; +#ifdef ENABLE_RADIATION + rad_temp = config["problem_parameters"]["rad_temp"].as(); +#endif + + if constexpr(D == 1) { + assert( + false && + "Kelvin-Helmholtz instability problem for D == 1 is not implemented"); + } + else if constexpr(D == 2) { + s.executor().forall(j, (m.template cells())) { + for(auto i : m.template cells()) { + const auto x = m.template center(i); + const auto y = m.template center(j); + + // initialize two different density and velocity fluids + if(std::abs(y - (L_y / 2)) > y_sep) { + mass_density(i, j) = rL; + momentum_density(i, j).x() = rL * uL; + momentum_density(i, j).y() = rL * vL; + const double e = util::find_sie(eos, rL, pL); + total_energy_density(i, j) = rL * e + 0.5 * rL * (vL * vL); + } + else { + mass_density(i, j) = rH; + momentum_density(i, j).x() = rH * uH; + momentum_density(i, j).y() = rH * vH; + const double e = util::find_sie(eos, rH, pH); + total_energy_density(i, j) = rH * e + 0.5 * rH * (vH * vH); + } // if + + radiation_energy_density(i, j) = + a * spec::utils::sqr(spec::utils::sqr(rad_temp)); + + // velocity perturbations in the Y-direction + if(std::abs(y - (L_y / 4)) < y_vp) { + momentum_density(i, j).y() = 0.05 * sin(wavenumber * x); + } + if(std::abs(y - (3 * L_y / 4)) < y_vp) { + momentum_density(i, j).y() = 0.05 * sin(wavenumber * x); + } + } // for + }; // forall + } + else /* D == 3 */ { + s.executor().forall(k, (m.template cells())) { + for(auto j : m.template cells()) { + for(auto i : m.template cells()) { + const auto x = m.template center(i); + const auto y = m.template center(j); + + // initialize two different density and velocity fluids + if(std::abs(y - (L_y / 2)) > y_sep) { + mass_density(i, j, k) = rL; + momentum_density(i, j, k).x() = rL * uL; + momentum_density(i, j, k).y() = rL * vL; + momentum_density(i, j, k).z() = rL * wL; + const double e = util::find_sie(eos, rL, pL); + total_energy_density(i, j, k) = rL * e + 0.5 * rL * (vL * vL); + } + else { + mass_density(i, j, k) = rH; + momentum_density(i, j, k).x() = rH * uH; + momentum_density(i, j, k).y() = rH * vH; + momentum_density(i, j, k).z() = rH * wH; + const double e = util::find_sie(eos, rH, pH); + total_energy_density(i, j, k) = rH * e + 0.5 * rH * (vH * vH); + } // if + + radiation_energy_density(i, j, k) = + a * spec::utils::sqr(spec::utils::sqr(rad_temp)); + + // velocity perturbations in the Y-direction + if(std::abs(y - (L_y / 4)) < y_vp) { + momentum_density(i, j, k).y() = 0.05 * sin(wavenumber * x); + } + if(std::abs(y - (3 * L_y / 4)) < y_vp) { + momentum_density(i, j, k).y() = 0.05 * sin(wavenumber * x); + } + } // for + } + }; // forall + } // if +} // kh_instability + +} // namespace hard::tasks::initial_data diff --git a/app/tasks/initial_data/lw_implosion.hh b/app/tasks/initial_data/lw_implosion.hh index 5993f85..f3268d6 100644 --- a/app/tasks/initial_data/lw_implosion.hh +++ b/app/tasks/initial_data/lw_implosion.hh @@ -1,6 +1,6 @@ - #pragma once +#include "../../types.hh" #include #include @@ -14,16 +14,17 @@ namespace hard::tasks::initial_data { // < 0.15, density=0.125 and pressure=0.14. Otherwise denisty and pressure are // both set to 1.0. Initial velocity is zero everywhere. // -template +template auto -lw_implosion(typename mesh::template accessor m, - field::accessor mass_density_a, - typename field>::template accessor momentum_density_a, - field::accessor total_energy_density_a, - field::accessor radiation_energy_density_a, - single::accessor gamma_a) { +lw_implosion(flecsi::exec::cpu s, + typename mesh::template accessor m, + field::accessor mass_density_a, + typename field>::template accessor momentum_density_a, + field::accessor total_energy_density_a, + field::accessor radiation_energy_density_a, + const double gamma) { - if constexpr(Dim == 2) { + if constexpr(D == 2) { auto mass_density = m.template mdcolex(mass_density_a); auto momentum_density = m.template mdcolex(momentum_density_a); @@ -32,10 +33,9 @@ lw_implosion(typename mesh::template accessor m, auto radiation_energy_density = m.template mdcolex(radiation_energy_density_a); - auto const gamma = *gamma_a; const double mult = 1.0 / (gamma - 1.0); - forall(j, (m.template cells()), "init_sedov_2d") { + s.executor().forall(j, (m.template cells())) { for(auto i : m.template cells()) { const auto x = m.template center(i); const auto y = m.template center(j); @@ -49,9 +49,7 @@ lw_implosion(typename mesh::template accessor m, total_energy_density(i, j) = mult * 0.14; } - momentum_density(i, j).x = 0.0; - momentum_density(i, j).y = 0.0; - + momentum_density(i, j) = vec(0.0); radiation_energy_density(i, j) = 0.0; } // for diff --git a/app/tasks/initial_data/rad_rank_hugoniot.hh b/app/tasks/initial_data/rad_rank_hugoniot.hh index 0a92bd4..965817c 100644 --- a/app/tasks/initial_data/rad_rank_hugoniot.hh +++ b/app/tasks/initial_data/rad_rank_hugoniot.hh @@ -42,18 +42,18 @@ struct rad_rankine_hugoniot { template auto -rad_RH(typename mesh::template accessor m, - field::accessor r_a, - typename field>::template accessor ru_a, - field::accessor rE_a, - field::accessor Erad_a, - single::accessor gamma_a, +rad_RH(flecsi::exec::cpu s, + typename mesh::template accessor m, + field::accessor r_a, + typename field>::template accessor ru_a, + field::accessor rE_a, + field::accessor Erad_a, + const double gamma, single::accessor particle_mass_a) { auto r = m.template mdcolex(r_a); auto ru = m.template mdcolex(ru_a); auto rE = m.template mdcolex(rE_a); auto Erad = m.template mdcolex(Erad_a); - auto const gamma = *gamma_a; auto const particle_mass = *particle_mass_a; const double mult = 1.0 / (gamma - 1.0); @@ -61,89 +61,79 @@ rad_RH(typename mesh::template accessor m, const double a = hard::constants::cgs::radiation_constant; if constexpr(D == 1) { - for(auto i : m.template cells()) { + s.executor().forall(i, (m.template cells())) { const auto x = m.template center(i); if(x < T::x0) { r(i) = T::rL; - ru(i).x = T::rL * T::uL; + ru(i).x() = T::rL * T::uL; rE(i) = mult * kb * T::TL * T::rL / (particle_mass) + (0.5 * T::rL * T::uL * T::uL); Erad(i) = a * T::TL * T::TL * T::TL * T::TL; } else { r(i) = T::rR; - ru(i).x = T::rR * T::uR; - // rE(i) = T::ER; + ru(i).x() = T::rR * T::uR; rE(i) = mult * kb * T::TR * T::rR / (particle_mass) + (0.5 * T::rR * T::uR * T::uR); Erad(i) = a * T::TR * T::TR * T::TR * T::TR; - // Erad(i) = T::EradR; } // if if(rE(i) <= 0) { std::cout << "TotalE is negative for i = " << i << std::endl; } - } // for + }; // for } else if constexpr(D == 2) { - for(auto j : m.template cells()) { + s.executor().forall(j, (m.template cells())) { for(auto i : m.template cells()) { const auto x = m.template center(i); if(x < T::x0) { r(i, j) = T::rL; - ru(i, j).x = T::rL * T::uL; - ru(i, j).y = T::rL * T::vL; - // rE(i, j) = T::EL; - // Erad(i, j) = T::EradL; + ru(i, j).x() = T::rL * T::uL; + ru(i, j).y() = T::rL * T::vL; rE(i, j) = mult * kb * T::TL * T::rL / (particle_mass) + (0.5 * T::rL * T::uL * T::uL); Erad(i, j) = a * T::TL * T::TL * T::TL * T::TL; } else { r(i, j) = T::rR; - ru(i, j).x = T::rR * T::uR; - ru(i, j).y = T::rR * T::vR; - // rE(i, j) = T::ER; - // Erad(i, j) = T::EradR; + ru(i, j).x() = T::rR * T::uR; + ru(i, j).y() = T::rR * T::vR; rE(i, j) = mult * kb * T::TR * T::rR / (particle_mass) + (0.5 * T::rR * T::uR * T::uR); Erad(i, j) = a * T::TR * T::TR * T::TR * T::TR; } // if } // for - } // for + }; // for } else /* D == 3 */ { - for(auto k : m.template cells()) { + s.executor().forall(k, (m.template cells())) { for(auto j : m.template cells()) { for(auto i : m.template cells()) { const auto x = m.template center(i); if(x < T::x0) { r(i, j, k) = T::rL; - ru(i, j, k).x = T::rL * T::uL; - ru(i, j, k).y = T::rL * T::vL; - ru(i, j, k).z = T::rL * T::wL; - // rE(i, j, k) = T::EL; - // Erad(i, j, k) = T::EradL; + ru(i, j, k).x() = T::rL * T::uL; + ru(i, j, k).y() = T::rL * T::vL; + ru(i, j, k).z() = T::rL * T::wL; rE(i, j, k) = mult * kb * T::TL * T::rL / (particle_mass) + (0.5 * T::rL * T::uL * T::uL); Erad(i, j, k) = a * T::TL * T::TL * T::TL * T::TL; } else { r(i, j, k) = T::rR; - ru(i, j, k).x = T::rR * T::uR; - ru(i, j, k).y = T::rR * T::vR; - ru(i, j, k).z = T::rR * T::wR; - // rE(i, j, k) = T::ER; - // Erad(i, j, k) = T::EradR; + ru(i, j, k).x() = T::rR * T::uR; + ru(i, j, k).y() = T::rR * T::vR; + ru(i, j, k).z() = T::rR * T::wR; rE(i, j, k) = mult * kb * T::TR * T::rR / (particle_mass) + (0.5 * T::rR * T::uR * T::uR); Erad(i, j, k) = a * T::TR * T::TR * T::TR * T::TR; } // if } // for } // for - } // for + }; // for } // if } // rad_RH diff --git a/app/tasks/initial_data/rayleigh_taylor.hh b/app/tasks/initial_data/rayleigh_taylor.hh new file mode 100644 index 0000000..2a1343d --- /dev/null +++ b/app/tasks/initial_data/rayleigh_taylor.hh @@ -0,0 +1,170 @@ +#pragma once + +#include "../../types.hh" +#include +#include + +namespace hard::tasks::initial_data { + +/* +Rayleigh-Taylor Instability +Reference: https://www.astro.princeton.edu/~jstone/Athena/tests/rt/rt.html +Currently this file is a copy of input file for Kelvin-Helmholtz Instability*/ + +template +auto +rt_instability(flecsi::exec::cpu s, + typename mesh::template accessor m, + field::accessor mass_density_a, + typename field>::template accessor momentum_density_a, + typename field>::template accessor gravity_force_a, + typename single>::template accessor gravity_acc_a, + field::accessor total_energy_density_a, + field::accessor radiation_energy_density_a, + const eos::eos_wrapper & eos) noexcept { + auto mass_density = m.template mdcolex(mass_density_a); + auto gravity_force = m.template mdcolex(gravity_force_a); + auto g = gravity_acc_a.get(); + auto momentum_density = m.template mdcolex(momentum_density_a); + auto total_energy_density = + m.template mdcolex(total_energy_density_a); + auto radiation_energy_density = + m.template mdcolex(radiation_energy_density_a); + + // fixed values to be used + + const double p0 = + 2.5; // For hydrostatic equilibrium pressure, add g*rho*y to p0 + const double v0 = 0.05; // For perturbed velocity, multiply + // ((1 + cos (4*pi*x))/2)*((1 + cos (3*pi*y))/2) + // density and x-velocity for two fluids + // L is the bottom fluid + const double rL = 1.0; + const double uL = 0.0; + + // H is the upper fluid + const double rH = 2.0; + const double uH = 0.0; + + // initialize variables to be used at location considered + if constexpr(D == 1) { + s.executor().forall(i, (m.template cells())) { + + const auto x = m.template center(i); + double r, u, v, p; + // set the density and x-velocity based on y location + if(x >= 0.75) { + // upper heavy fluid + r = rH; + u = 0.0; + p = p0 + rL * g.x() * 0.75 + + g.x() * r * (x - 0.75); // Hydrostatic Equilibrium pressure + } + else { + // bottom lighter fluid + r = rL; + u = 0.0; + p = p0 + rL * g.x() * x; + } // if + + // Gaussian-localized x-velocity perturbation + v = v0 * 1 / 2 * std::exp(-std::pow((x - 0.75) / 0.05, 2)); + + // Initialize state variables at location + mass_density(i) = r; + gravity_force(i).x() = r * g.x(); + momentum_density(i).x() = r * u; + const double e = util::find_sie(eos, r, p); + total_energy_density(i) = r * e + 0.5 * r * (u * u + v * v); + radiation_energy_density(i) = 0.0; + }; // forall + } + else if constexpr(D == 2) { + s.executor().forall(j, (m.template cells())) { + for(auto i : m.template cells()) { + const auto x = m.template center(i); + const auto y = m.template center(j); + + double r, u, v, p; + // set the density and x-velocity based on y location + if(y > 0.75) { + // upper heavy fluid + r = rH; + u = uH; + p = p0 + rL * g.y() * 0.75 + + (g.y() * r * (y - 0.75)); // Hydrostatic Equilibrium pressure + } + else { + // bottom lighter fluid + r = rL; + u = uL; + p = p0 + rL * g.y() * y; + } // if + + // Gaussian-localized y-velocity perturbation + v = v0 * (1 + cos(4 * M_PI * (x - 0.25))) / 2 * + std::exp(-std::pow((y - 0.75) / 0.05, 2)); + + // Initialize state variables at location + mass_density(i, j) = r; + gravity_force(i, j).x() = r * g.x(); + gravity_force(i, j).y() = r * g.y(); + momentum_density(i, j).x() = r * u; + momentum_density(i, j).y() = r * v; + const double e = util::find_sie(eos, r, p); + total_energy_density(i, j) = r * e + 0.5 * r * (u * u + v * v); + radiation_energy_density(i, j) = 0.0; + + } // for + }; // forall + } + else /* D == 3 */ { + s.executor().forall(k, (m.template cells())) { + for(auto i : m.template cells()) { + for(auto j : m.template cells()) { + const auto x = m.template center(i); + const auto y = m.template center(j); + const auto z = m.template center(k); + + double r, u, v, w, p; + // set the density and x-velocity based on y location + if(y > 0.75) { + // upper heavy fluid + r = rH; + u = uH; + w = 0.0; + p = p0 + rL * g.y() * 0.75 + + (g.y() * r * (y - 0.75)); // Hydrostatic Equilibrium pressure + } + else { + // bottom lighter fluid + r = rL; + u = uL; + w = 0.0; + p = p0 + rL * g.y() * y; + } // if + + // Gaussian-localized y-velocity perturbation + v = v0 * + (1 + cos(4 * M_PI * (x - 0.75)) * (cos(4 * M_PI * (z - 0.75)))) / + 2 * std::exp(-std::pow((y - 0.75) / 0.05, 2)); + + // Initialize state variables at location + mass_density(i, j, k) = r; + gravity_force(i, j, k).x() = r * g.x(); + gravity_force(i, j, k).y() = r * g.y(); + gravity_force(i, j, k).z() = r * g.z(); + momentum_density(i, j, k).x() = r * u; + momentum_density(i, j, k).y() = r * v; + momentum_density(i, j, k).z() = r * w; + const double e = util::find_sie(eos, r, p); + total_energy_density(i, j, k) = + r * e + 0.5 * r * (u * u + v * v + w * w); + radiation_energy_density(i, j, k) = 0.0; + } // for + } // for + }; // forall + } // if +} // rt_instability + +} // namespace hard::tasks::initial_data diff --git a/app/tasks/initial_data/richtmyer_meshkov.hh b/app/tasks/initial_data/richtmyer_meshkov.hh new file mode 100644 index 0000000..c01a13e --- /dev/null +++ b/app/tasks/initial_data/richtmyer_meshkov.hh @@ -0,0 +1,115 @@ +#pragma once + +#include "../../constants.hh" +#include "../../options.hh" +#include "../../types.hh" +#include "../utils.hh" +#include +#include +#include + +namespace hard::tasks::initial_data { + +/*----------------------------------------------------------------------------* + Richtmyer-Meshkov Instability. + *----------------------------------------------------------------------------*/ + +template +auto +richtmyer_meshkov(flecsi::exec::cpu s, + typename mesh::template accessor m, + field::accessor mass_density_a, + typename field>::template accessor momentum_density_a, + field::accessor total_energy_density_a, + field::accessor radiation_energy_density_a, + const eos::eos_wrapper & eos) { + auto mass_density = m.template mdcolex(mass_density_a); + auto momentum_density = m.template mdcolex(momentum_density_a); + auto total_energy_density = + m.template mdcolex(total_energy_density_a); + auto radiation_energy_density = + m.template mdcolex(radiation_energy_density_a); + + YAML::Node config = YAML::LoadFile(opt::config.value()); + + const double rAs = + config["problem_parameters"]["density_above_shock"].as(); + const double pAs = + config["problem_parameters"]["pressure_above_shock"].as(); + const double vAs = + config["problem_parameters"]["velocity_above_shock"].as(); + const double rL = config["problem_parameters"]["density_low"].as(); + const double rH = config["problem_parameters"]["density_high"].as(); + const double p0 = config["problem_parameters"]["pressure"].as(); + const double amp = config["problem_parameters"]["perturbation"].as(); + const double interface = + config["problem_parameters"]["interface"].as(); + // for single mode perturbation, wavelength = domain width. + const double wavelength = config["coords"][1][0].as(); + const double wavenumber = 2.0 * M_PI; + + // for defining shock + const double y_shock = interface + 0.01; + + // constant for the radiation energy + const double a = hard::constants::cgs::radiation_constant; + + // temperature for radiation + const double rad_temp = + config["radiation_parameters"]["rad_temp"].as(); + flog(info) << "radiation temperature is " << rad_temp << std::endl; + + if constexpr(D == 2) { + s.executor().forall(j, (m.template cells())) { + for(auto i : m.template cells()) { + const double x = m.template center(i); + const double y = m.template center(j); + + // define the initial perturbation at the interface + const double interface_y = + interface + amp * (1 - std::cos(wavenumber * x / wavelength)) / 2; + + // place holder values for imposing initial conditions + double rho, vy = 0, vx = 0.0, p; + + // lighter fluid above shock + if(y >= y_shock) { + rho = rAs; + p = pAs; + vy = vAs; + } + // lighter fluid below shock + else if(y >= interface_y && y < y_shock) { + rho = rL; // 0.125 + p = p0; // 0.1 + vy = 0.0; + } + // heavy fluid + else { + rho = rH; + p = p0; + vy = 0.0; + } + // Assign fields + mass_density(i, j) = rho; + momentum_density(i, j) = vec(0.0); + momentum_density(i, j).x() = rho * vx; + momentum_density(i, j).y() = rho * vy; + const double e = util::find_sie(eos, rho, p); + const double kinetic = 0.5 * rho * vy * vy; + total_energy_density(i, j) = rho * e + kinetic; + radiation_energy_density(i, j) = 0; + +#ifdef ENABLE_RADIATION + radiation_energy_density(i, j) = + a * spec::utils::sqr(spec::utils::sqr(rad_temp)); +#endif + } + }; + } + else { + flog_fatal("Richtmyer-Meshkov problem is only implemented for D == 2"); + } +} + +} // namespace hard::tasks::initial_data diff --git a/app/tasks/initial_data/sedov.hh b/app/tasks/initial_data/sedov.hh index 35dc7e7..4d876b8 100644 --- a/app/tasks/initial_data/sedov.hh +++ b/app/tasks/initial_data/sedov.hh @@ -15,15 +15,14 @@ namespace hard::tasks::initial_data { // A hotspot of high temperature is added at the center of a // domain with constant density, zero velocity, and low temperature. // -template +template auto -sedov_blast(typename mesh::template accessor m, +sedov_blast(flecsi::exec::cpu s, + typename mesh::template accessor m, field::accessor mass_density_a, - typename field>::template accessor momentum_density_a, + typename field>::template accessor momentum_density_a, field::accessor total_energy_density_a, - field::accessor radiation_energy_density_a, - single::accessor particle_mass_a, - const double gamma) { + field::accessor radiation_energy_density_a) { auto mass_density = m.template mdcolex(mass_density_a); auto momentum_density = m.template mdcolex(momentum_density_a); @@ -32,15 +31,6 @@ sedov_blast(typename mesh::template accessor m, auto radiation_energy_density = m.template mdcolex(radiation_energy_density_a); - // adiabatic index, assuming ideal gas EOS - // auto const gamma = *gamma_a; - auto const particle_mass = *particle_mass_a; - const double mult = 1.0 / (gamma - 1.0); - - // physical constants in cgs units - const double & kb = hard::constants::cgs::boltzmann_constant; - const double & a = hard::constants::cgs::radiation_constant; - // Parse input parameters YAML::Node config = YAML::LoadFile(opt::config.value()); @@ -52,88 +42,55 @@ sedov_blast(typename mesh::template accessor m, config["problem_parameters"]["hotspot_position"][0][2].as(); const double radius = config["problem_parameters"]["hotspot_radius"].as(); + const double E_0 = config["problem_parameters"]["E_0"].as(); // some problem parameters: for now hardwired - const double density = 1.0e-2; - const double temperature_inside = 8.0e6; - const double temperature_outside = 1.0e2; + const double density = 1.0; + + const double volume = + pow(radius, D) * (D > 1 ? M_PI : 1.) * (D > 2 ? 4. / 3. : 1.); + const double rE_inside = E_0 / volume; + const double rE_outside = 1.0e-5; using spec::utils::sqr; - if constexpr(Dim == 1) { - forall(i, (m.template cells()), "init_sedov_1d") { + if constexpr(D == 1) { + s.executor().forall(i, (m.template cells())) { const auto x = m.template center(i); - double distance = std::abs(x - x0); - mass_density(i) = density; - momentum_density(i).x = 0.0; // velocity is zero. - if(distance < radius) { - double pressure = kb * temperature_inside * density / (particle_mass); - total_energy_density(i) = mult * pressure; - radiation_energy_density(i) = a * sqr(sqr(temperature_inside)); - } - else { - double pressure = kb * temperature_outside * density / (particle_mass); - total_energy_density(i) = mult * pressure; - radiation_energy_density(i) = a * sqr(sqr(temperature_outside)); - } + momentum_density(i).x() = 0.0; + radiation_energy_density(i) = 0.0; + total_energy_density(i) = distance < radius ? rE_inside : rE_outside; }; // forall } - else if constexpr(Dim == 2) { - forall(j, (m.template cells()), "init_sedov_2d") { + else if constexpr(D == 2) { + s.executor().forall(j, (m.template cells())) { for(auto i : m.template cells()) { const auto x = m.template center(i); const auto y = m.template center(j); - double distance = std::sqrt((x - x0) * (x - x0) + (y - y0) * (y - y0)); - mass_density(i, j) = density; - momentum_density(i, j).x = 0.0; - momentum_density(i, j).y = 0.0; - if(distance < radius) { - double pressure = kb * temperature_inside * density / (particle_mass); - total_energy_density(i, j) = mult * pressure; - radiation_energy_density(i, j) = a * sqr(sqr(temperature_inside)); - } - else { - double pressure = - kb * temperature_outside * density / (particle_mass); - total_energy_density(i, j) = mult * pressure; - radiation_energy_density(i, j) = a * sqr(sqr(temperature_outside)); - } + radiation_energy_density(i, j) = 0.0; + momentum_density(i, j) = vec(0.0); + total_energy_density(i, j) = distance < radius ? rE_inside : rE_outside; } // for }; // forall } - else if constexpr(Dim == 3) { - forall(k, (m.template cells()), "init_sedov_3d") { + else if constexpr(D == 3) { + s.executor().forall(k, (m.template cells())) { for(auto j : m.template cells()) { for(auto i : m.template cells()) { const auto x = m.template center(i); const auto y = m.template center(j); const auto z = m.template center(k); - double distance = std::sqrt( (x - x0) * (x - x0) + (y - y0) * (y - y0) + (z - z0) * (z - z0)); - mass_density(i, j, k) = density; - momentum_density(i, j, k).x = 0.0; - momentum_density(i, j, k).y = 0.0; - momentum_density(i, j, k).z = 0.0; - if(distance < radius) { - double pressure = - kb * temperature_inside * density / (particle_mass); - total_energy_density(i, j, k) = mult * pressure; - radiation_energy_density(i, j, k) = - a * sqr(sqr(temperature_inside)); - } - else { - double pressure = - kb * temperature_outside * density / (particle_mass); - total_energy_density(i, j, k) = mult * pressure; - radiation_energy_density(i, j, k) = - a * sqr(sqr(temperature_outside)); - } + radiation_energy_density(i, j, k) = 0.0; + momentum_density(i, j, k) = vec(0.0); + total_energy_density(i, j, k) = + distance < radius ? rE_inside : rE_outside; } // for } }; // forall diff --git a/app/tasks/initial_data/shock_tube.hh b/app/tasks/initial_data/shock_tube.hh index 1ec3793..a300a51 100644 --- a/app/tasks/initial_data/shock_tube.hh +++ b/app/tasks/initial_data/shock_tube.hh @@ -11,39 +11,41 @@ namespace hard::tasks::initial_data { namespace shock_tubes { struct rankine_hugoniot { - static constexpr double rL = 2.299156e-01; - static constexpr double uL = 1.270775e+00; - static constexpr double vL = 0.0; - static constexpr double wL = 0.0; - static constexpr double pL = 6.700238e-01; - - static constexpr double rR = 1.250000e-01; - static constexpr double uR = 0.000000e+00; - static constexpr double vR = 0.0; - static constexpr double wR = 0.0; - static constexpr double pR = 2.276638e-01; + static constexpr double rL = 1.0; + static constexpr double uL = 0.0; + static constexpr double pL = 1.0; + + static constexpr double rR = 0.25; + static constexpr double uR = 0.0; + static constexpr double pR = 0.1795; static constexpr double x0 = 0.5; - static constexpr double y0 = 0.5; }; // struct rankine_hugoniot struct sod { static constexpr double rL = 1.0; static constexpr double uL = 0.0; - static constexpr double vL = 0.0; - static constexpr double wL = 0.0; static constexpr double pL = 1.0; static constexpr double rR = 0.125; static constexpr double uR = 0.0; - static constexpr double vR = 0.0; - static constexpr double wR = 0.0; static constexpr double pR = 0.1; static constexpr double x0 = 0.5; - static constexpr double y0 = 0.5; }; // struct sod +struct leblanc { + static constexpr double rL = 1.0; + static constexpr double uL = 0.0; + static constexpr double pL = 0.1; + + static constexpr double rR = 1.0e-3; + static constexpr double uR = 0.0; + static constexpr double pR = 1.0e-10; + + static constexpr double x0 = 0.5; +}; // struct leblanc + } // namespace shock_tubes /*----------------------------------------------------------------------------* @@ -52,7 +54,8 @@ struct sod { template auto -shock(typename mesh::template accessor m, +shock(flecsi::exec::cpu s, + typename mesh::template accessor m, field::accessor mass_density_a, typename field>::template accessor momentum_density_a, field::accessor total_energy_density_a, @@ -68,18 +71,18 @@ shock(typename mesh::template accessor m, m.template mdcolex(radiation_energy_density_a); if constexpr(D == 1) { - forall(i, (m.template cells()), "init_1d") { + s.executor().forall(i, (m.template cells())) { const auto x = m.template head(i); if(x < T::x0) { mass_density(i) = T::rL; - momentum_density(i).x = T::rL * T::uL; + momentum_density(i).x() = T::rL * T::uL; const double e = util::find_sie(eos, T::rL, T::pL); total_energy_density(i) = T::rL * e + 0.5 * T::rL * (T::uL * T::uL); } else { mass_density(i) = T::rR; - momentum_density(i).x = T::rR * T::uR; + momentum_density(i).x() = T::rR * T::uR; const double e = util::find_sie(eos, T::rR, T::pR); total_energy_density(i) = T::rR * e + 0.5 * T::rR * (T::uR * T::uR); } // if @@ -93,26 +96,19 @@ shock(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(ji, mdpolicy_yx, "init_shock") { + s.executor().forall(ji, mdpolicy_yx) { auto [j, i] = ji; const auto x = m.template head(i); - const auto y = m.template head(j); - - double radius = - sqrt((x - T::x0) * (x - T::x0) + (y - T::y0) * (y - T::y0)); - // if(x < T::x0) { - if(radius < 0.2) { + if(x < T::x0) { mass_density(i, j) = T::rL; - momentum_density(i, j).x = T::rL * T::uL; - momentum_density(i, j).y = T::rL * T::vL; + momentum_density(i, j) = vec(T::rL * T::uL); const double e = util::find_sie(eos, T::rL, T::pL); total_energy_density(i, j) = T::rL * e + 0.5 * T::rL * (T::uL * T::uL); } else { mass_density(i, j) = T::rR; - momentum_density(i, j).x = T::rR * T::uR; - momentum_density(i, j).y = T::rR * T::vR; + momentum_density(i, j) = vec(T::rR * T::uR); const double e = util::find_sie(eos, T::rR, T::pR); total_energy_density(i, j) = T::rR * e + 0.5 * T::rR * (T::uR * T::uR); } // if @@ -126,24 +122,20 @@ shock(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(kji, mdpolicy_zyx, "init_shock") { + s.executor().forall(kji, mdpolicy_zyx) { auto [k, j, i] = kji; const auto x = m.template head(i); if(x < T::x0) { mass_density(i, j, k) = T::rL; - momentum_density(i, j, k).x = T::rL * T::uL; - momentum_density(i, j, k).y = T::rL * T::vL; - momentum_density(i, j, k).z = T::rL * T::wL; + momentum_density(i, j, k) = vec(T::rL * T::uL); const double e = util::find_sie(eos, T::rL, T::pL); total_energy_density(i, j, k) = T::rL * e + 0.5 * T::rL * (T::uL * T::uL); } else { mass_density(i, j, k) = T::rR; - momentum_density(i, j, k).x = T::rR * T::uR; - momentum_density(i, j, k).y = T::rR * T::vR; - momentum_density(i, j, k).z = T::rR * T::wR; + momentum_density(i, j, k) = vec(T::rR * T::uR); const double e = util::find_sie(eos, T::rR, T::pR); total_energy_density(i, j, k) = T::rR * e + 0.5 * T::rR * (T::uR * T::uR); diff --git a/app/tasks/initial_data/sine_wave.hh b/app/tasks/initial_data/sine_wave.hh deleted file mode 100644 index cec59fc..0000000 --- a/app/tasks/initial_data/sine_wave.hh +++ /dev/null @@ -1,61 +0,0 @@ - -#pragma once - -#include "../../types.hh" -#include "../utils.hh" -#include -#include - -namespace hard::tasks::initial_data { - -// -// A sinusoidal density profile being advected with a constant velocity. -// This test problem can be used to check the numerical convergence -// -template -auto -sine_wave(typename mesh::template accessor m, - field::accessor mass_density_a, - typename field>::template accessor momentum_density_a, - field::accessor total_energy_density_a, - field::accessor radiation_energy_density_a, - const eos::eos_wrapper & eos) { - - auto mass_density = m.template mdcolex(mass_density_a); - auto momentum_density = m.template mdcolex(momentum_density_a); - auto total_energy_density = - m.template mdcolex(total_energy_density_a); - auto radiation_energy_density = - m.template mdcolex(radiation_energy_density_a); - - // Problem parameters - const double mass_density_amplitude = 0.2; - const double wavenumber = 2.0 * M_PI; // wavelength = 1.0 - const double velocity_x = - 0.2; // later we might want to extend this to multi-D - const double pressure = 1.0; - - // - // Only 1D version has been implemented. - // - if constexpr(Dim == 1) { - forall( - i, (m.template cells()), "init_sine_wave_1d") { - const auto x = m.template center(i); - - mass_density(i) = 1.0 + mass_density_amplitude * sin(wavenumber * x); - momentum_density(i).x = mass_density(i) * velocity_x; - const double e = util::find_sie(eos, mass_density(i), pressure); - total_energy_density(i) = - mass_density(i) * e + 0.5 * mass_density(i) * utils::sqr(velocity_x); - - radiation_energy_density(i) = 0.0; - - }; // forall - } - else { - flog_fatal("Sine wave problem for D >= 2 is not implemented") - } -} - -} // namespace hard::tasks::initial_data diff --git a/app/tasks/io.hh b/app/tasks/io.hh index 951fe9f..ac5c0fa 100644 --- a/app/tasks/io.hh +++ b/app/tasks/io.hh @@ -10,161 +10,119 @@ namespace hard::tasks::io { template -void inline raw(spec::io::name const & base, +void inline csv(flecsi::exec::cpu s, + spec::io::name const & base, single::accessor time, multi::template accessor> mm, - multi::accessor> r_ma, - multi::accessor> p_ma, - multi::accessor> c_ma, - multi::accessor> e_ma, - multi>::template accessor> u_ma, - multi>::template accessor> ru_ma, - multi::accessor> rE_ma, - multi::accessor> radE_ma) { + std::vector::accessor>> field_ma, + std::vector>::template accessor>> + field_vector_ma, + std::vector field_name, + std::vector field_vector_name) { for(uint32_t i{0}; i < mm.depth(); ++i) { const auto m = mm.accessors()[i]; - - auto r_a = r_ma.accessors()[i]; - auto p_a = p_ma.accessors()[i]; - auto c_a = c_ma.accessors()[i]; - auto e_a = e_ma.accessors()[i]; - auto u_a = u_ma.accessors()[i]; - auto ru_a = ru_ma.accessors()[i]; - auto rE_a = rE_ma.accessors()[i]; - auto radE_a = radE_ma.accessors()[i]; - - auto r = m.template mdcolex(r_a); - auto p = m.template mdcolex(p_a); - auto c = m.template mdcolex(c_a); - auto e = m.template mdcolex(e_a); - auto u = m.template mdcolex(u_a); - auto ru = m.template mdcolex(ru_a); - auto rE = m.template mdcolex(rE_a); - auto radE = m.template mdcolex(radE_a); + std::vector> fields; + std::vector, D>> fields_vectors; + + std::transform(field_ma.begin(), + field_ma.end(), + std::back_inserter(fields), + [&m, &i](const multi::accessor> & ma) + -> flecsi::util::mdcolex { + return m.template mdcolex(ma.accessors()[i]); + }); + std::transform(field_vector_ma.begin(), + field_vector_ma.end(), + std::back_inserter(fields_vectors), + [&m, &i]( + const multi>::template accessor> & ma) + -> flecsi::util::mdcolex, D> { + return m.template mdcolex(ma.accessors()[i]); + }); if constexpr(D == 1) { - std::ofstream file( - base.str() + "-1D-" + std::to_string(flecsi::process()) + ".raw"); - - file << "#" << m.template size() << std::endl; - file << "#" << m.template size() << std::endl; - - { - auto ccoords = m.color_indeces(); - file << "#" << ccoords[ax::x] << std::endl; - auto ccolors = m.axis_colors(); - file << "#" << ccolors[ax::x] << std::endl; - } // scope - - // File format, columns: - // time, cellid_x, coord_x, density, pressure, velocity, - // fluid total energy density, radiation energy density - file << "#time\tcellid_x\tcoord_" - "x\tdensity\tpressure\tvelocity\ttotalE\tRadE\tsoundspeed\tsie" - << std::endl; + std::ofstream file("output-1D-" + std::to_string(s.launch().index) + "-" + + base.str() + ".csv"); + + file << "time,cellid_x,coord_x"; + for(auto n : field_name) + file << "," << n; + for(auto n : field_vector_name) + file << "," << n << "_x"; + file << std::endl; file << std::fixed; // just to get a uniform file format + for(auto i : m.template cells()) { - if(std::isnan(r(i)) || std::isnan(p(i)) || std::isnan(rE(i)) || - std::isnan(radE(i)) || std::isnan(c(i)) || std::isnan(e(i))) { - flog_fatal( - "NAN Variable: r: " - << std::isnan(r(i)) << " p: " << std::isnan(p(i)) - << " rE: " << std::isnan(rE(i)) << " radE: " << std::isnan(radE(i)) - << " c: " << std::isnan(c(i)) << " e: " << std::isnan(e(i))); - } - file << std::setprecision(6) << std::scientific << time << "\t" << i - << "\t" << m.template center(i) << std::setprecision(12) - << "\t" << r(i) << "\t" << p(i) << "\t" << u(i).x << "\t" << rE(i) - << "\t" << radE(i) << "\t" << c(i) << "\t" << e(i) << std::endl; - } // for - file << std::endl; // empty line needed for gnuplot's splot + file << std::setprecision(6) << std::scientific << time << "," << i + << "," << m.template center(i) << std::setprecision(12); + for(auto f_ma : fields) + file << "," << f_ma(i); + for(auto f_ma : fields_vectors) + file << "," << f_ma(i)[0]; + file << std::endl; + } } else if constexpr(D == 2) { - std::ofstream file( - base.str() + "-2D-" + std::to_string(flecsi::process()) + ".raw"); - - file << "#" << m.template size() << " " - << m.template size() << std::endl; - file << "#" << m.template size() << " " - << m.template size() << std::endl; - - { - auto ccoords = m.color_indeces(); - file << "#" << ccoords[ax::x] << " " << ccoords[ax::y] << std::endl; - auto ccolors = m.axis_colors(); - file << "#" << ccolors[ax::x] << " " << ccolors[ax::y] << std::endl; - } // scope - - // File format, columns: - // time, cellid_x, cellid_y, coord_x, coord_y, density, pressure, - // velocity, fluid total energy density, radiation energy density - file << "#time\tcellidx\tcellidy\tx\ty\tdensity\tpressure\tvx\tvy\ttotalE" - "\tRadE" - << std::endl; + + std::ofstream file("output-2D-" + std::to_string(s.launch().index) + "-" + + base.str() + ".csv"); + + file << "time,cellid_x,cellid_y,coord_x,coord_y"; + for(auto n : field_name) + file << "," << n; + for(auto n : field_vector_name) + file << "," << n << "_x," << n << "_y"; + file << std::endl; file << std::fixed; // just to get a uniform file format + for(auto i : m.template cells()) { for(auto j : m.template cells()) { - file << std::setprecision(6) << std::scientific << time << "\t" << i - << "\t" << j << "\t" << m.template center(i) << "\t" - << m.template center(j) << std::setprecision(9) << "\t" - << r(i, j) << "\t" << p(i, j) << "\t" << u(i, j).x << "\t" - << u(i, j).y << "\t" << rE(i, j) << "\t" << radE(i, j) - << std::endl; - } // for - file << std::endl; // empty line needed for gnuplot's splot - } // for + + file << std::setprecision(6) << std::scientific << time << "," << i + << "," << j << "," << m.template center(i) << "," + << m.template center(j) << std::setprecision(12); + for(auto f_ma : fields) + file << "," << f_ma(i, j); + for(auto f_ma : fields_vectors) + file << "," << f_ma(i, j)[0] << "," << f_ma(i, j)[1]; + file << std::endl; + } + } } else /* D == 3 */ { - std::ofstream file( - base.str() + "-" + std::to_string(flecsi::process()) + ".raw"); - - file << m.template size() << " " - << m.template size() << " " - << m.template size() << std::endl; - file << m.template size() << " " - << m.template size() << " " - << m.template size() << std::endl; - - { - auto ccoords = m.color_indeces(); - file << ccoords[ax::x] << " " << ccoords[ax::y] << " " << ccoords[ax::z] - << std::endl; - auto ccolors = m.axis_colors(); - file << ccolors[ax::x] << " " << ccolors[ax::y] << " " << ccolors[ax::z] - << std::endl; - } // scope - - // Density - for(auto k : m.template cells()) { - for(auto j : m.template cells()) { - for(auto i : m.template cells()) { - file << r(i, j, k) << std::endl; - } // for - } // for - } // for - - // Momentum - for(auto k : m.template cells()) { - for(auto j : m.template cells()) { - for(auto i : m.template cells()) { - file << ru(i, j, k).x << " " << ru(i, j, k).y << " " - << ru(i, j, k).z << std::endl; - } // for - } // for - } // for - - // Total Energy - for(auto k : m.template cells()) { + + std::ofstream file("output-3D-" + std::to_string(s.launch().index) + "-" + + base.str() + ".csv"); + + file << "time,cellid_x,cellid_y,cellid_z,coord_x,coord_y,coord_z"; + for(auto n : field_name) + file << "," << n; + for(auto n : field_vector_name) + file << "," << n << "_x," << n << "_y," << n << "_z"; + file << std::endl; + file << std::fixed; // just to get a uniform file format + + for(auto i : m.template cells()) { for(auto j : m.template cells()) { - for(auto i : m.template cells()) { - file << rE(i, j, k) << std::endl; - } // for - } // for - } // for - } // for - } // if -} // raw + for(auto k : m.template cells()) { + + file << std::setprecision(6) << std::scientific << time << "," << i + << "," << j << "," << k << "," << m.template center(i) + << "," << m.template center(j) << "," + << m.template center(k) << std::setprecision(12); + for(auto f_ma : fields) + file << "," << f_ma(i, j, k); + for(auto f_ma : fields_vectors) + file << "," << f_ma(i, j, k)[0] << "," << f_ma(i, j, k)[1] << "," + << f_ma(i, j, k)[2]; + file << std::endl; + } + } + } + } // if + } // for +} // csv } // namespace hard::tasks::io diff --git a/app/tasks/rad.hh b/app/tasks/rad.hh index 653daae..14455f1 100644 --- a/app/tasks/rad.hh +++ b/app/tasks/rad.hh @@ -9,23 +9,143 @@ namespace hard::task::rad { +// Castro/Source/radiation/fluxlimiter.H/Edd_factor +FLECSI_INLINE_TARGET double +AFLDEddFactor(double lam, + std::size_t limiter_id, + std::size_t closure_id) noexcept { + double f = 0.0; + + switch(closure_id) { + case 0: // f = λ + f = lam; + break; + case 1: // f = 1/3 + f = 1.0 / 3.0; + break; + case 2: // f = 1 - 2λ + f = 1.0 - 2.0 * lam; + break; + case 3: // f = λ + (λR)^2 with different limiters + // lambda + (lambda*R)**2 + switch(limiter_id) { + case 0: + f = 1.0 / 3.0; + break; + case 1: { + double temp = + 0.5 * std::max(0.0, 1.0 - 3.0 * lam) + + std::sqrt(std::max(0.0, (1.0 - 3.0 * lam) * (1.0 + 5.0 * lam))); + f = lam + temp * temp; // approximate LP, [123] + break; + } + case 2: + f = 1.0 - 5.0 * lam + 9.0 * lam * lam; // Bruenn, 1[123] + break; + case 3: + f = 1.0 + lam - 9.0 * lam * lam; // Larsen's square root, 2[123] + break; + case 4: + if(lam > 2.0 / 9.0) // Minerbo + f = 1.0 / 3.0; + else + f = 1.0 + 3.0 * lam - 2.0 * std::sqrt(2.0 * lam); + break; + default: + assert(false && "Invalid Limiter ID (Closure 3)"); + return -1.0; + } + break; + case 4: // f = 1/3 + (2/3)*(λR)^2 with different limiters + switch(limiter_id) { + case 0: + f = 1.0 / 3.0; // no limiter + break; + case 1: { + double temp = + std::max(0.0, 1.0 - 3.0 * lam) + + std::sqrt(std::max(0.0, (1.0 - 3.0 * lam) * (1.0 + 5.0 * lam))); + f = 1.0 / 3.0 + (temp * temp / 6.0); // approximate LP, [123] + break; + } + case 2: + f = 1.0 / 3.0 + + 2.0 * (1.0 - 6.0 * lam + 9.0 * lam * lam) / 3.0; // Bruenn, 1[123] + break; + case 3: + f = 1.0 / 3.0 + 2.0 * (1.0 - 9.0 * lam * lam) / + 3.0; // Larsen's square root, 2[123] + break; + case 4: + if(lam > 2.0 / 9.0) + f = 5.0 / 9.0 - (2.0 * lam / 3.0); // Minerbo + else + f = 1.0 / 3.0 + + (2.0 * (1.0 + 2.0 * lam - 2.0 * std::sqrt(2.0 * lam)) / 3.0); + break; + default: + assert(false && "Invalid Limiter ID (Closure 4)"); + return -1.0; + } + break; + default: + assert(false && "Invalid Closure ID"); + return -1.0; + } + + return f; +} + +// Castro/Source/radiation/rad_util.H/FLDlambda +FLECSI_INLINE_TARGET double +AFLDlambda(double r, std::size_t limiter_id) noexcept { + double l = 0.0; + + switch(limiter_id) { + case 0: + l = 1.0 / 3.0; // no limiter + break; + case 1: + l = (2.0 + r) / (6.0 + 3.0 * r + r * r); // approximate LP + break; + case 2: + l = 1.0 / (3.0 + r); // Bruenn + break; + case 3: + l = 1.0 / std::sqrt(9.0 + r * r); // Larsen's square root + break; + case 4: + if(r < 1.5) + l = 2.0 / (3.0 + std::sqrt(9.0 + 12.0 * r * r)); // Minerbo + else + l = 1.0 / (1.0 + r + std::sqrt(1.0 + 2.0 * r)); + break; + default: + assert(false && "Invalid Limiter ID"); + return -1.0; + } + + return l; +} + template double -update_dtmin(typename mesh::template accessor m, - flecsi::future lmax_f) { +update_dtmin(flecsi::exec::cpu, + typename mesh::template accessor m, + flecsi::future lmax_f) noexcept { double lmax = lmax_f.get(); if constexpr(D == 1) { return m.template delta() / lmax; } else if constexpr(D == 2) { - return std::min( - m.template delta() / lmax, m.template delta() / lmax); + return std::min(m.template delta(), m.template delta()) / + (D * lmax); } else { - return std::min(m.template delta() / lmax, - std::min( - m.template delta() / lmax, m.template delta() / lmax)); + return std::min(m.template delta(), + std::min(m.template delta(), m.template delta())) / + (D * lmax); } // if } // update_dtmin @@ -34,10 +154,11 @@ using hard::tasks::util::get_mdiota_policy; // Get the gradient of the velocity using a 5-point stencil. template void -getGradV(typename mesh::template accessor m, +getGradV(flecsi::exec::accelerator s, + typename mesh::template accessor m, typename field>::template accessor gradV_a, - typename field>::template accessor u_a) { + typename field>::template accessor u_a) noexcept { auto u = m.template mdcolex(u_a); auto gradV = m.template mdcolex(gradV_a); @@ -45,10 +166,10 @@ getGradV(typename mesh::template accessor m, if constexpr(D == 1) { const double one_over_12dx = 1.0 / (12.0 * m.template delta()); - forall(i, (m.template cells()), "getGradV") { - gradV(i).xx = - (u(i - 2).x - 8.0 * u(i - 1).x + 8.0 * u(i + 1).x - u(i + 2).x) * - one_over_12dx; + s.executor().forall(i, (m.template cells())) { + gradV(i).xx = (u(i - 2).x() - 8.0 * u(i - 1).x() + 8.0 * u(i + 1).x() - + u(i + 2).x()) * + one_over_12dx; }; } else if constexpr(D == 2) { @@ -59,21 +180,21 @@ getGradV(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(ji, mdpolicy_qq, "getGradV") { + s.executor().forall(ji, mdpolicy_qq) { auto [j, i] = ji; - gradV(i, j).xx = (u(i - 2, j).x - 8.0 * u(i - 1, j).x + - 8.0 * u(i + 1, j).x - u(i + 2, j).x) * + gradV(i, j).xx = (u(i - 2, j).x() - 8.0 * u(i - 1, j).x() + + 8.0 * u(i + 1, j).x() - u(i + 2, j).x()) * one_over_12dx; - gradV(i, j).xy = (u(i, j - 2).x - 8.0 * u(i, j - 1).x + - 8.0 * u(i, j + 1).x - u(i, j + 2).x) * + gradV(i, j).xy = (u(i, j - 2).x() - 8.0 * u(i, j - 1).x() + + 8.0 * u(i, j + 1).x() - u(i, j + 2).x()) * one_over_12dy; - gradV(i, j).yx = (u(i - 2, j).y - 8.0 * u(i - 1, j).y + - 8.0 * u(i + 1, j).y - u(i + 2, j).y) * + gradV(i, j).yx = (u(i - 2, j).y() - 8.0 * u(i - 1, j).y() + + 8.0 * u(i + 1, j).y() - u(i + 2, j).y()) * one_over_12dx; - gradV(i, j).yy = (u(i, j - 2).y - 8.0 * u(i, j - 1).y + - 8.0 * u(i, j + 1).y - u(i, j + 2).y) * + gradV(i, j).yy = (u(i, j - 2).y() - 8.0 * u(i, j - 1).y() + + 8.0 * u(i, j + 1).y() - u(i, j + 2).y()) * one_over_12dy; }; } @@ -87,70 +208,107 @@ getGradV(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(kji, mdpolicy_qqq, "getGradV") { + s.executor().forall(kji, mdpolicy_qqq) { auto [k, j, i] = kji; - gradV(i, j, k).xx = (u(i - 2, j, k).x - 8.0 * u(i - 1, j, k).x + - 8.0 * u(i + 1, j, k).x - u(i + 2, j, k).x) * + gradV(i, j, k).xx = (u(i - 2, j, k).x() - 8.0 * u(i - 1, j, k).x() + + 8.0 * u(i + 1, j, k).x() - u(i + 2, j, k).x()) * one_over_12dx; - gradV(i, j, k).xy = (u(i, j - 2, k).x - 8.0 * u(i, j - 1, k).x + - 8.0 * u(i, j + 1, k).x - u(i, j + 2, k).x) * + gradV(i, j, k).xy = (u(i, j - 2, k).x() - 8.0 * u(i, j - 1, k).x() + + 8.0 * u(i, j + 1, k).x() - u(i, j + 2, k).x()) * one_over_12dy; - gradV(i, j, k).xz = (u(i, j, k - 2).x - 8.0 * u(i, j, k - 1).x + - 8.0 * u(i, j, k + 1).x - u(i, j, k + 2).x) * + gradV(i, j, k).xz = (u(i, j, k - 2).x() - 8.0 * u(i, j, k - 1).x() + + 8.0 * u(i, j, k + 1).x() - u(i, j, k + 2).x()) * one_over_12dz; - gradV(i, j, k).yx = (u(i - 2, j, k).y - 8.0 * u(i - 1, j, k).y + - 8.0 * u(i + 1, j, k).y - u(i + 2, j, k).y) * + gradV(i, j, k).yx = (u(i - 2, j, k).y() - 8.0 * u(i - 1, j, k).y() + + 8.0 * u(i + 1, j, k).y() - u(i + 2, j, k).y()) * one_over_12dx; - gradV(i, j, k).yy = (u(i, j - 2, k).y - 8.0 * u(i, j - 1, k).y + - 8.0 * u(i, j + 1, k).y - u(i, j + 2, k).y) * + gradV(i, j, k).yy = (u(i, j - 2, k).y() - 8.0 * u(i, j - 1, k).y() + + 8.0 * u(i, j + 1, k).y() - u(i, j + 2, k).y()) * one_over_12dy; - gradV(i, j, k).yz = (u(i, j, k - 2).y - 8.0 * u(i, j, k - 1).y + - 8.0 * u(i, j, k + 1).y - u(i, j, k + 2).y) * + gradV(i, j, k).yz = (u(i, j, k - 2).y() - 8.0 * u(i, j, k - 1).y() + + 8.0 * u(i, j, k + 1).y() - u(i, j, k + 2).y()) * one_over_12dz; - gradV(i, j, k).zx = (u(i - 2, j, k).z - 8.0 * u(i - 1, j, k).z + - 8.0 * u(i + 1, j, k).z - u(i + 2, j, k).z) * + gradV(i, j, k).zx = (u(i - 2, j, k).z() - 8.0 * u(i - 1, j, k).z() + + 8.0 * u(i + 1, j, k).z() - u(i + 2, j, k).z()) * one_over_12dx; - gradV(i, j, k).zy = (u(i, j - 2, k).z - 8.0 * u(i, j - 1, k).z + - 8.0 * u(i, j + 1, k).z - u(i, j + 2, k).z) * + gradV(i, j, k).zy = (u(i, j - 2, k).z() - 8.0 * u(i, j - 1, k).z() + + 8.0 * u(i, j + 1, k).z() - u(i, j + 2, k).z()) * one_over_12dy; - gradV(i, j, k).zz = (u(i, j, k - 2).z - 8.0 * u(i, j, k - 1).z + - 8.0 * u(i, j, k + 1).z - u(i, j, k + 2).z) * + gradV(i, j, k).zz = (u(i, j, k - 2).z() - 8.0 * u(i, j, k - 1).z() + + 8.0 * u(i, j, k + 1).z() - u(i, j, k + 2).z()) * one_over_12dz; }; } } // getGradV +// Get Eddington Factor based on either constant or adaptive limiter FLD +template +void +getEddFactor(flecsi::exec::accelerator s, + typename mesh::template accessor m, + field::accessor lambda_a, + field::accessor edd_factor_a, + single::accessor limiter_id_a, + single::accessor closure_id_a) noexcept { + + auto lambda = m.template mdcolex(lambda_a); + auto edd_factor = m.template mdcolex(edd_factor_a); + + if constexpr(D == 1) { + s.executor().forall(i, (m.template cells())) { + edd_factor(i) = AFLDEddFactor(lambda(i), *limiter_id_a, *closure_id_a); + }; + } + else if constexpr(D == 2) { + auto mdpolicy_qq = get_mdiota_policy(lambda, + m.template cells(), + m.template cells()); + s.executor().forall(ji, mdpolicy_qq) { + auto [j, i] = ji; + edd_factor(i, j) = + AFLDEddFactor(lambda(i, j), *limiter_id_a, *closure_id_a); + }; + } + else { + auto mdpolicy_qqq = get_mdiota_policy(lambda, + m.template cells(), + m.template cells(), + m.template cells()); + s.executor().forall(kji, mdpolicy_qqq) { + auto [k, j, i] = kji; + edd_factor(i, j, k) = + AFLDEddFactor(lambda(i, j, k), *limiter_id_a, *closure_id_a); + }; + } + +} // getEddFactor + // Get the radiation pressure tensor P +// Modified so that it takes the eddington factor field template void -getTensorP(typename mesh::template accessor m, +getTensorP(flecsi::exec::accelerator s, + typename mesh::template accessor m, typename field>::template accessor P_tensor_a, field::accessor Esf_a, typename field>::template accessor gradEsf_a, field::accessor gradE_mag_a, - field::accessor lambda_a, - field::accessor R_a) { + field::accessor edd_factor_a) noexcept { auto P_tensor = m.template mdcolex(P_tensor_a); auto Esf = m.template mdcolex(Esf_a); auto gradEsf = m.template mdcolex(gradEsf_a); auto gradE_mag = m.template mdcolex(gradE_mag_a); - auto lambda = m.template mdcolex(lambda_a); - auto R = m.template mdcolex(R_a); + auto edd_factor = m.template mdcolex(edd_factor_a); - const double eps = 1.0e-15; - - const auto compute_eddington_factor = []( - double lambda_value, double R_value) { - return lambda_value + spec::utils::sqr(lambda_value * R_value); - }; + const double zero_guard = 1.0e-15; if constexpr(D == 1) { - forall(i, (m.template cells()), "getTensorP") { - const double f = compute_eddington_factor(lambda(i), R(i)); + s.executor().forall(i, (m.template cells())) { + const double f = edd_factor(i); P_tensor(i).xx = (0.5 * (1 - f) + 0.5 * (3 * f - 1)) * Esf(i); }; } @@ -159,13 +317,13 @@ getTensorP(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(ji, mdpolicy_qq, "getTensorP") { + s.executor().forall(ji, mdpolicy_qq) { auto [j, i] = ji; - const double nx = gradEsf(i, j).x / (gradE_mag(i, j) + eps); - const double ny = gradEsf(i, j).y / (gradE_mag(i, j) + eps); + const double nx = gradEsf(i, j).x() / (gradE_mag(i, j) + zero_guard); + const double ny = gradEsf(i, j).y() / (gradE_mag(i, j) + zero_guard); - const double f = compute_eddington_factor(lambda(i, j), R(i, j)); + const double f = edd_factor(i, j); P_tensor(i, j).xx = (0.5 * (1 - f) + 0.5 * (3 * f - 1) * nx * nx) * Esf(i, j); @@ -181,14 +339,17 @@ getTensorP(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(kji, mdpolicy_qqq, "getTensorP") { + s.executor().forall(kji, mdpolicy_qqq) { auto [k, j, i] = kji; - const double nx = gradEsf(i, j, k).x / (gradE_mag(i, j, k) + eps); - const double ny = gradEsf(i, j, k).y / (gradE_mag(i, j, k) + eps); - const double nz = gradEsf(i, j, k).z / (gradE_mag(i, j, k) + eps); + const double nx = + gradEsf(i, j, k).x() / (gradE_mag(i, j, k) + zero_guard); + const double ny = + gradEsf(i, j, k).y() / (gradE_mag(i, j, k) + zero_guard); + const double nz = + gradEsf(i, j, k).z() / (gradE_mag(i, j, k) + zero_guard); - const double f = compute_eddington_factor(lambda(i, j, k), R(i, j, k)); + const double f = edd_factor(i, j, k); P_tensor(i, j, k).xx = (0.5 * (1 - f) + 0.5 * (3 * f - 1) * nx * nx) * Esf(i, j, k); @@ -213,9 +374,10 @@ getTensorP(typename mesh::template accessor m, // for the Legion tracing so I am using `` for now. template void -getGradE(typename mesh::template accessor m, +getGradE(flecsi::exec::accelerator s, + typename mesh::template accessor m, field::accessor Esf_a, - typename field>::template accessor gradEsf_a) { + typename field>::template accessor gradEsf_a) noexcept { auto Esf = m.template mdcolex(Esf_a); auto gradEsf = m.template mdcolex(gradEsf_a); @@ -223,8 +385,8 @@ getGradE(typename mesh::template accessor m, const double one_over_12dx = 1.0 / (12.0 * m.template delta()); // Application of the 5-stencil central differencing: - forall(i, (m.template cells()), "getGradE") { - gradEsf(i).x = + s.executor().forall(i, (m.template cells())) { + gradEsf(i).x() = (Esf(i - 2) - 8.0 * Esf(i - 1) + 8.0 * Esf(i + 1) - Esf(i + 2)) * one_over_12dx; }; // for @@ -237,16 +399,16 @@ getGradE(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(ji, mdpolicy_qq, "getGradE") { + s.executor().forall(ji, mdpolicy_qq) { auto [j, i] = ji; // Application of the 5-stencil central differencing: - gradEsf(i, j).x = (Esf(i - 2, j) - 8.0 * Esf(i - 1, j) + - 8.0 * Esf(i + 1, j) - Esf(i + 2, j)) * - one_over_12dx; - gradEsf(i, j).y = (Esf(i, j - 2) - 8.0 * Esf(i, j - 1) + - 8.0 * Esf(i, j + 1) - Esf(i, j + 2)) * - one_over_12dy; + gradEsf(i, j).x() = (Esf(i - 2, j) - 8.0 * Esf(i - 1, j) + + 8.0 * Esf(i + 1, j) - Esf(i + 2, j)) * + one_over_12dx; + gradEsf(i, j).y() = (Esf(i, j - 2) - 8.0 * Esf(i, j - 1) + + 8.0 * Esf(i, j + 1) - Esf(i, j + 2)) * + one_over_12dy; }; // forall } else { @@ -259,19 +421,19 @@ getGradE(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(kji, mdpolicy_qqq, "getGradE") { + s.executor().forall(kji, mdpolicy_qqq) { auto [k, j, i] = kji; // Application of the 5-stencil central differencing: - gradEsf(i, j, k).x = (Esf(i - 2, j, k) - 8.0 * Esf(i - 1, j, k) + - 8.0 * Esf(i + 1, j, k) - Esf(i + 2, j, k)) * - one_over_12dx; - gradEsf(i, j, k).y = (Esf(i, j - 2, k) - 8.0 * Esf(i, j - 1, k) + - 8.0 * Esf(i, j + 1, k) - Esf(i, j + 2, k)) * - one_over_12dy; - gradEsf(i, j, k).z = (Esf(i, j, k - 2) - 8.0 * Esf(i, j, k - 1) + - 8.0 * Esf(i, j, k + 1) - Esf(i, j, k + 2)) * - one_over_12dz; + gradEsf(i, j, k).x() = (Esf(i - 2, j, k) - 8.0 * Esf(i - 1, j, k) + + 8.0 * Esf(i + 1, j, k) - Esf(i + 2, j, k)) * + one_over_12dx; + gradEsf(i, j, k).y() = (Esf(i, j - 2, k) - 8.0 * Esf(i, j - 1, k) + + 8.0 * Esf(i, j + 1, k) - Esf(i, j + 2, k)) * + one_over_12dy; + gradEsf(i, j, k).z() = (Esf(i, j, k - 2) - 8.0 * Esf(i, j, k - 1) + + 8.0 * Esf(i, j, k + 1) - Esf(i, j, k + 2)) * + one_over_12dz; }; } } // getGradE @@ -280,23 +442,23 @@ getGradE(typename mesh::template accessor m, // Compute // 1) magnitude of grad(E) // 2) variable R (Eq 16 of Moens 2022 paper) -// 3) flux limiter function `lambda` +// 3) flux limiter function `lambda` - can either be a constant or adaptive // // Note : Lambda is only computed on the main grid (dm::quantities), then the // outermost values are copied into ghost zones. // template void -getLambda(typename mesh::template accessor m, +getLambda(flecsi::exec::accelerator s, + typename mesh::template accessor m, field::accessor r_a, field::accessor Esf_a, typename field>::template accessor gradEsf_a, field::accessor gradE_mag_a, field::accessor R_a, field::accessor lambda_a, - single::accessor kappa_a) { - - // TODO: applying boundary condition should be separated to new task + single::accessor kappa_a, + single::accessor limiter_id_a) noexcept { auto r = m.template mdcolex(r_a); auto Esf = m.template mdcolex(Esf_a); @@ -306,28 +468,27 @@ getLambda(typename mesh::template accessor m, auto lambda = m.template mdcolex(lambda_a); if constexpr(D == 1) { - forall(i, (m.template cells()), "getLambda") { + s.executor().forall(i, (m.template cells())) { auto const kappa = *kappa_a; - const double eps = 1.0e-30; + const double zero_guard = 1.0e-15; - gradE_mag(i) = std::abs(gradEsf(i).x); - R(i) = gradE_mag(i) / (kappa * r(i) * Esf(i) + eps); - lambda(i) = (2.0 + R(i)) / (6.0 + 3.0 * R(i) + R(i) * R(i)); + gradE_mag(i) = std::abs(gradEsf(i).x()); + R(i) = gradE_mag(i) / (kappa * r(i) * Esf(i) + zero_guard); + lambda(i) = AFLDlambda(R(i), *limiter_id_a); }; } else if constexpr(D == 2) { auto mdpolicy_qq = get_mdiota_policy(Esf, m.template cells(), m.template cells()); - forall(ji, mdpolicy_qq, "getLambda") { + s.executor().forall(ji, mdpolicy_qq) { auto const kappa = *kappa_a; - const double eps = 1.0e-30; + const double zero_guard = 1.0e-15; auto [j, i] = ji; gradE_mag(i, j) = gradEsf(i, j).norm(); - R(i, j) = gradE_mag(i, j) / (kappa * r(i, j) * Esf(i, j) + eps); - lambda(i, j) = - (2.0 + R(i, j)) / (6.0 + 3.0 * R(i, j) + R(i, j) * R(i, j)); + R(i, j) = gradE_mag(i, j) / (kappa * r(i, j) * Esf(i, j) + zero_guard); + lambda(i, j) = AFLDlambda(R(i, j), *limiter_id_a); }; } else { @@ -335,16 +496,15 @@ getLambda(typename mesh::template accessor m, m.template cells(), m.template cells(), m.template cells()); - forall(kji, mdpolicy_qqq, "getLambda") { + s.executor().forall(kji, mdpolicy_qqq) { auto const kappa = *kappa_a; - const double eps = 1.0e-30; + const double zero_guard = 1.0e-15; auto [k, j, i] = kji; gradE_mag(i, j, k) = gradEsf(i, j, k).norm(); R(i, j, k) = - gradE_mag(i, j, k) / (kappa * r(i, j, k) * Esf(i, j, k) + eps); - lambda(i, j, k) = - (2.0 + R(i, j, k)) / (6.0 + 3.0 * R(i, j, k) + R(i, j, k) * R(i, j, k)); + gradE_mag(i, j, k) / (kappa * r(i, j, k) * Esf(i, j, k) + zero_guard); + lambda(i, j, k) = AFLDlambda(R(i, j, k), *limiter_id_a); }; } } // getLambda @@ -352,18 +512,19 @@ getLambda(typename mesh::template accessor m, // Get the radiation force using the FLD approximation template void -getRadForce(typename mesh::template accessor m, +getRadForce(flecsi::exec::accelerator s, + typename mesh::template accessor m, field::accessor lambda_a, typename field>::template accessor gradEsf_a, - typename field>::template accessor fr_a) { + typename field>::template accessor fr_a) noexcept { auto lambda = m.template mdcolex(lambda_a); auto gradEsf = m.template mdcolex(gradEsf_a); auto fr = m.template mdcolex(fr_a); if constexpr(D == 1) { - forall(i, (m.template cells()), "getRadForce") { - fr(i).x = -lambda(i) * gradEsf(i).x; + s.executor().forall(i, (m.template cells())) { + fr(i).x() = -lambda(i) * gradEsf(i).x(); }; } else if constexpr(D == 2) { @@ -371,10 +532,10 @@ getRadForce(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(ji, mdpolicy_qq, "getRadForce") { + s.executor().forall(ji, mdpolicy_qq) { auto [j, i] = ji; - fr(i, j).x = -lambda(i, j) * gradEsf(i, j).x; - fr(i, j).y = -lambda(i, j) * gradEsf(i, j).y; + fr(i, j).x() = -lambda(i, j) * gradEsf(i, j).x(); + fr(i, j).y() = -lambda(i, j) * gradEsf(i, j).y(); }; } else { @@ -382,11 +543,11 @@ getRadForce(typename mesh::template accessor m, m.template cells(), m.template cells(), m.template cells()); - forall(kji, mdpolicy_qqq, "getRadForce") { + s.executor().forall(kji, mdpolicy_qqq) { auto [k, j, i] = kji; - fr(i, j, k).x = -lambda(i, j, k) * gradEsf(i, j, k).x; - fr(i, j, k).y = -lambda(i, j, k) * gradEsf(i, j, k).y; - fr(i, j, k).z = -lambda(i, j, k) * gradEsf(i, j, k).z; + fr(i, j, k).x() = -lambda(i, j, k) * gradEsf(i, j, k).x(); + fr(i, j, k).y() = -lambda(i, j, k) * gradEsf(i, j, k).y(); + fr(i, j, k).z() = -lambda(i, j, k) * gradEsf(i, j, k).z(); }; } } // getRadForce @@ -395,7 +556,8 @@ getRadForce(typename mesh::template accessor m, // Moens2022) template void -explicitSourceUpdate(typename mesh::template accessor m, +explicitSourceUpdate(flecsi::exec::accelerator s, + typename mesh::template accessor m, // Primitive variables typename field>::template accessor velocity_a, // Terms required for computing radiation force and photon tiring @@ -407,7 +569,7 @@ explicitSourceUpdate(typename mesh::template accessor m, // time derivative typename field>::template accessor dt_momentum_density_a, field::accessor dt_total_energy_density_a, - field::accessor dt_radiation_energy_density_a) { + field::accessor dt_radiation_energy_density_a) noexcept { auto velocity = m.template mdcolex(velocity_a); auto fr = m.template mdcolex(fr_a); @@ -422,16 +584,15 @@ explicitSourceUpdate(typename mesh::template accessor m, // const double radiation_constant = hard::constants::cgs::radiation_constant; if constexpr(D == 1) { - forall( - i, (m.template cells()), "explicitSourceUpdate") { + s.executor().forall(i, (m.template cells())) { // Explicitly updating conservative variables with S_ex // Adding the radiation force term to the momentum density dt_momentum_density(i) += fr(i); // Updating the total gas energy density: Adding contribution from the - // work done by the radiative force: vdot_fr(i) = u(i).x * fr(i).x - dt_total_energy_density(i) += velocity(i).x * fr(i).x; + // work done by the radiative force: vdot_fr(i) = u(i).x() * fr(i).x() + dt_total_energy_density(i) += velocity(i).x() * fr(i).x(); // Subtracting the photon tiring term, (P::gradV), from the radiation // energy density in each cell. See Eq(34) in Moens2022. @@ -449,7 +610,7 @@ explicitSourceUpdate(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(ji, mdpolicy_qq, "explicitSourceUpdate") { + s.executor().forall(ji, mdpolicy_qq) { auto [j, i] = ji; // Explicitly updating conservative variables with S_ex @@ -457,9 +618,9 @@ explicitSourceUpdate(typename mesh::template accessor m, dt_momentum_density(i, j) += fr(i, j); // Updating the total gas energy density: Adding contribution from the - // work done by the radiative force: vdot_fr(i) = u(i).x * fr(i).x + // work done by the radiative force: vdot_fr(i) = u(i).x() * fr(i).x() dt_total_energy_density(i, j) += - velocity(i, j).x * fr(i, j).x + velocity(i, j).y * fr(i, j).y; + velocity(i, j).x() * fr(i, j).x() + velocity(i, j).y() * fr(i, j).y(); // Subtracting the photon tiring term, (P::gradV), from the radiation // energy density in each cell. See Eq(34) in Moens2022. @@ -476,7 +637,7 @@ explicitSourceUpdate(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(kji, mdpolicy_qqq, "explicitSourceUpdate") { + s.executor().forall(kji, mdpolicy_qqq) { auto [k, j, i] = kji; // Explicitly updating conservative variables with S_ex @@ -484,10 +645,11 @@ explicitSourceUpdate(typename mesh::template accessor m, dt_momentum_density(i, j, k) += fr(i, j, k); // Updating the total gas energy density: Adding contribution from the - // work done by the radiative force: vdot_fr(i) = u(i).x * fr(i).x - dt_total_energy_density(i, j, k) += velocity(i, j, k).x * fr(i, j, k).x + - velocity(i, j, k).y * fr(i, j, k).y + - velocity(i, j, k).z * fr(i, j, k).z; + // work done by the radiative force: vdot_fr(i) = u(i).x() * fr(i).x() + dt_total_energy_density(i, j, k) += + velocity(i, j, k).x() * fr(i, j, k).x() + + velocity(i, j, k).y() * fr(i, j, k).y() + + velocity(i, j, k).z() * fr(i, j, k).z(); // Subtracting the photon tiring term, (P::gradV), from the radiation // energy density in each cell. See Eq(34) in Moens et al. 2022. @@ -508,18 +670,19 @@ explicitSourceUpdate(typename mesh::template accessor m, // Compute the diffusion coefficients D template void -getDiff(typename mesh::template accessor m, +getDiff(flecsi::exec::accelerator s, + typename mesh::template accessor m, field::accessor r_a, field::accessor lambda_a, field::accessor Diff_a, - single::accessor kappa_a) { + single::accessor kappa_a) noexcept { auto r = m.template mdcolex(r_a); auto lambda = m.template mdcolex(lambda_a); auto Diff = m.template mdcolex(Diff_a); if constexpr(D == 1) { - forall(i, (m.template cells()), "getDiff") { + s.executor().forall(i, (m.template cells())) { auto const kappa = *kappa_a; const double clight = hard::constants::cgs::speed_of_light; Diff(i) = clight * lambda(i) / (kappa * r(i)); @@ -530,7 +693,7 @@ getDiff(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(ji, mdpolicy_aa, "getDiff") { + s.executor().forall(ji, mdpolicy_aa) { auto const kappa = *kappa_a; const double clight = hard::constants::cgs::speed_of_light; auto [j, i] = ji; @@ -543,7 +706,7 @@ getDiff(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(kji, mdpolicy_aaa, "getDiff") { + s.executor().forall(kji, mdpolicy_aaa) { auto const kappa = *kappa_a; const double clight = hard::constants::cgs::speed_of_light; auto [k, j, i] = kji; @@ -555,11 +718,12 @@ getDiff(typename mesh::template accessor m, // Compute the radiation diffusion coefficients D on faces template void -diffusion_init(typename mesh::template accessor m, +diffusion_init(flecsi::exec::accelerator s, + typename mesh::template accessor m, typename field::template accessor Diff_a, typename field::template accessor Df_xa, typename field::template accessor Df_ya, - typename field::template accessor Df_za) { + typename field::template accessor Df_za) noexcept { auto Diff = m.template mdcolex(Diff_a); auto Df_x = m.template mdcolex(Df_xa); @@ -567,7 +731,7 @@ diffusion_init(typename mesh::template accessor m, auto Df_z = m.template mdcolex(Df_za); if constexpr(D == 1) { - forall(i, (m.template cells()), "diffusion_init") { + s.executor().forall(i, (m.template cells())) { Df_x(i) = 2 * Diff(i) * Diff(i - 1) / (Diff(i) + Diff(i - 1)); }; } @@ -577,7 +741,7 @@ diffusion_init(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(ji, mdpolicy_cc, "explicitSourceUpdate") { + s.executor().forall(ji, mdpolicy_cc) { auto [j, i] = ji; Df_x(i, j) = 2 * Diff(i, j) * Diff(i - 1, j) / (Diff(i, j) + Diff(i - 1, j)); @@ -591,7 +755,7 @@ diffusion_init(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(kji, mdpolicy_ccc, "diffusion_init") { + s.executor().forall(kji, mdpolicy_ccc) { auto [k, j, i] = kji; Df_x(i, j, k) = 2 * Diff(i, j, k) * Diff(i - 1, j, k) / (Diff(i, j, k) + Diff(i - 1, j, k)); @@ -605,14 +769,15 @@ diffusion_init(typename mesh::template accessor m, template void -const_init(typename mesh::template accessor m, +const_init(flecsi::exec::accelerator s, + typename mesh::template accessor m, typename field::template accessor f_a, - double w) { + double w) noexcept { auto f = m.template mdcolex(f_a); if constexpr(D == 1) { - forall(i, (m.template cells()), "const_init") { + s.executor().forall(i, (m.template cells())) { f(i) = w; }; // for } @@ -621,7 +786,7 @@ const_init(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(ji, mdpolicy_aa, "const_init") { + s.executor().forall(ji, mdpolicy_aa) { auto [j, i] = ji; f(i, j) = w; }; @@ -633,7 +798,7 @@ const_init(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(kji, mdpolicy_aaa, "const_init") { + s.executor().forall(kji, mdpolicy_aaa) { auto [k, j, i] = kji; f(i, j, k) = w; }; @@ -642,102 +807,51 @@ const_init(typename mesh::template accessor m, template void -initialize_Ef(typename mesh::template accessor m, - typename field::template accessor Erad_a, - typename field::template accessor Ef_a) { +copy_field(flecsi::exec::accelerator s, + typename mesh::template accessor m, + typename field::template accessor from_a, + typename field::template accessor to_a) noexcept { - auto Erad = m.template mdcolex(Erad_a); - auto Ef = m.template mdcolex(Ef_a); + auto from = m.template mdcolex(from_a); + auto to = m.template mdcolex(to_a); if constexpr(D == 1) { - forall(i, (m.template cells()), "initialize_Ef") { - Ef(i) = Erad(i); + s.executor().forall(i, (m.template cells())) { + to(i) = from(i); }; // forall } else if constexpr(D == 2) { - auto mdpolicy_qq = get_mdiota_policy(Ef, + auto mdpolicy_qq = get_mdiota_policy(to, m.template cells(), m.template cells()); - forall(ji, mdpolicy_qq, "initialize_Ef") { + s.executor().forall(ji, mdpolicy_qq) { auto [j, i] = ji; - Ef(i, j) = Erad(i, j); + to(i, j) = from(i, j); }; // forall } else if constexpr(D == 3) { - auto mdpolicy_qqq = get_mdiota_policy(Ef, + auto mdpolicy_qqq = get_mdiota_policy(to, m.template cells(), m.template cells(), m.template cells()); - forall(kji, mdpolicy_qqq, "initialize_Ef") { + s.executor().forall(kji, mdpolicy_qqq) { auto [k, j, i] = kji; - Ef(i, j, k) = Erad(i, j, k); + to(i, j, k) = from(i, j, k); }; // forall } -} // initialize_Ef +} // copy_field template void -store_du_dt_implicit_from_diffusion(typename mesh::template accessor m, - typename field::template accessor Esf_a, - typename field::template accessor Ef_a, - single::accessor dtw_a, - typename field::template accessor - dt_radiation_energy_density_implicit_a) { - - auto Esf = m.template mdcolex(Esf_a); - auto Ef = m.template mdcolex(Ef_a); - auto dt_radiation_energy_density_implicit = - m.template mdcolex(dt_radiation_energy_density_implicit_a); - - if constexpr(D == 1) { - forall(i, - (m.template cells()), - "store_du_dt_implicit_from_diffusion") { - auto & dt_weighted = *dtw_a; - const double one_over_aii_dt{1.0 / dt_weighted}; - dt_radiation_energy_density_implicit(i) += - (Esf(i) - Ef(i)) * one_over_aii_dt; - }; // for - } - else if constexpr(D == 2) { - auto mdpolicy_qq = get_mdiota_policy(Ef, - m.template cells(), - m.template cells()); - - forall(ji, mdpolicy_qq, "initialize_Ef") { - auto [j, i] = ji; - auto & dt_weighted = *dtw_a; - const double one_over_aii_dt{1.0 / dt_weighted}; - dt_radiation_energy_density_implicit(i, j) += - (Esf(i, j) - Ef(i, j)) * one_over_aii_dt; - }; // for - } - else if constexpr(D == 3) { - auto mdpolicy_qqq = get_mdiota_policy(Ef, - m.template cells(), - m.template cells(), - m.template cells()); - - forall(kji, mdpolicy_qqq, "initialize_Ef") { - auto [k, j, i] = kji; - auto & dt_weighted = *dtw_a; - const double one_over_aii_dt{1.0 / dt_weighted}; - dt_radiation_energy_density_implicit(i, j, k) += - (Esf(i, j, k) - Ef(i, j, k)) * one_over_aii_dt; - }; // for - } -} // copy_esf_into_ef - -template -void -stencil_init(typename mesh::template accessor m, +stencil_init(flecsi::exec::accelerator s, + typename mesh::template accessor m, typename field::template accessor Df_xa, typename field::template accessor Df_ya, typename field::template accessor Df_za, typename field>::template accessor Ew_a, - single::accessor dt_a) { + single::accessor dt_a) noexcept { // TODO: Stencil, Ew ghosts can be `na` (?) auto Df_x = m.template mdcolex(Df_xa); @@ -748,7 +862,7 @@ stencil_init(typename mesh::template accessor m, if constexpr(D == 1) { const double dx{m.template delta()}; - forall(i, (m.template cells()), "stencil_init") { + s.executor().forall(i, (m.template cells())) { auto const dt = *dt_a; const double wx{dt / pow(dx, 2)}; Ew(i)[dirs::c] = 1.0 + wx * (Df_x(i + 1) + Df_x(i)); @@ -763,7 +877,7 @@ stencil_init(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(ji, mdpolicy_qq, "explicitSourceUpdate") { + s.executor().forall(ji, mdpolicy_qq) { auto [j, i] = ji; auto const dt = *dt_a; const double wx{dt / pow(dx, 2)}; @@ -784,7 +898,7 @@ stencil_init(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(kji, mdpolicy_qqq, "stencil_init") { + s.executor().forall(kji, mdpolicy_qqq) { auto [k, j, i] = kji; auto const dt = *dt_a; const double wx{dt / pow(dx, 2)}; @@ -802,34 +916,18 @@ stencil_init(typename mesh::template accessor m, template void -apply_BC(typename mesh::template accessor m, - typename field::template accessor E_a) { - - auto E = m.template mdcolex(E_a); - // TODO: Other D? - if constexpr(D == 1) { - const std::size_t i = m.template size(); - // periodic boundary for 1D - E(0) = E(i - 4); - E(1) = E(i - 3); - E(i - 2) = E(2); - E(i - 1) = E(3); - } -} // apply_BC - -template -void -full_weighting(typename mesh::template accessor mf, +full_weighting(flecsi::exec::accelerator s, + typename mesh::template accessor mf, typename mesh::template accessor mc, typename field::template accessor rfa, - typename field::template accessor fca) { + typename field::template accessor fca) noexcept { // TODO: fca could be , since only writing quantities auto rf = mf.template mdcolex(rfa); auto fc = mc.template mdcolex(fca); if constexpr(D == 1) { - forall(i, (mc.template cells()), "full_weighting") { + s.executor().forall(i, (mc.template cells())) { auto fi = 2 * i; fc(i) = 0.125 * (3.0 * (rf(fi - 2) + rf(fi - 1)) + rf(fi - 3) + rf(fi)); }; // for @@ -839,7 +937,7 @@ full_weighting(typename mesh::template accessor mf, mc.template cells(), mc.template cells()); - forall(ji, mdpolicy_qq, "full_weighting") { + s.executor().forall(ji, mdpolicy_qq) { auto [j, i] = ji; auto fj = 2 * j; auto fi = 2 * i; @@ -861,7 +959,7 @@ full_weighting(typename mesh::template accessor mf, mc.template cells(), mc.template cells()); - forall(kji, mdpolicy_qqq, "full_weighting") { + s.executor().forall(kji, mdpolicy_qqq) { auto [k, j, i] = kji; auto fk = 2 * k; auto fj = 2 * j; @@ -913,19 +1011,18 @@ full_weighting(typename mesh::template accessor mf, template void -nlinear_interpolation(typename mesh::template accessor mc, +nlinear_interpolation(flecsi::exec::accelerator s, + typename mesh::template accessor mc, typename mesh::template accessor mf, typename field::template accessor cfa, - typename field::template accessor ffa) { + typename field::template accessor ffa) noexcept { // TODO: As above, ffa could be , since only writing quantities auto cf = mc.template mdcolex(cfa); auto ff = mf.template mdcolex(ffa); if constexpr(D == 1) { - forall(i, - (mf.template cells()), - "nlinear_interpolation") { + s.executor().forall(i, (mf.template cells())) { auto ci = i / 2 + 1; auto di = +1; if((i + 1) % 2) { @@ -939,7 +1036,7 @@ nlinear_interpolation(typename mesh::template accessor mc, mc.template cells(), mc.template cells()); - forall(ji, mdpolicy_qq, "full_weighting") { + s.executor().forall(ji, mdpolicy_qq) { auto [j, i] = ji; auto cj = j / 2 + 1; auto dj = 1; @@ -961,7 +1058,7 @@ nlinear_interpolation(typename mesh::template accessor mc, mc.template cells(), mc.template cells()); - forall(kji, mdpolicy_qqq, "full_weighting") { + s.executor().forall(kji, mdpolicy_qqq) { auto [k, j, i] = kji; auto ck = k / 2 + 1; auto dk = 1; @@ -990,261 +1087,237 @@ nlinear_interpolation(typename mesh::template accessor mc, template void -damped_jacobi(typename mesh::template accessor m, - typename field>::template accessor Ew_a, - typename field::template accessor ua_new, - typename field::template accessor ua_old, - typename field::template accessor fa, - double omega) { +cell_centered_weighting(flecsi::exec::accelerator s, + typename mesh::template accessor mf, + typename mesh::template accessor mc, + field::accessor rfa, + field::accessor fca) noexcept { - auto Ew = m.template mdcolex(Ew_a); - auto u_new = m.template mdcolex(ua_new); - auto u_old = m.template mdcolex(ua_old); - auto f = m.template mdcolex(fa); + auto rf = mf.template mdcolex(rfa); + auto fc = mc.template mdcolex(fca); if constexpr(D == 1) { - forall(i, (m.template cells()), "damped_jacobi") { - const double z = (Ew(i)[dirs::w] * u_old(i - 1) + - Ew(i + 1)[dirs::w] * u_old(i + 1) + f(i)) / - Ew(i)[dirs::c]; - - u_new(i) = u_old(i) + omega * (z - u_old(i)); + s.executor().forall(i, (mc.template cells())) { + auto fi = 2 * i - mc.ghost_zone_size(); + fc(i) = 0.5 * (rf(fi) + rf(fi + 1)); }; // for } else if constexpr(D == 2) { - auto mdpolicy_qq = get_mdiota_policy(u_new, - m.template cells(), - m.template cells()); + auto mdpolicy_qq = get_mdiota_policy(fc, + mc.template cells(), + mc.template cells()); - forall(ji, mdpolicy_qq, "damped_jacobi") { + s.executor().forall(ji, mdpolicy_qq) { auto [j, i] = ji; - const double z = (Ew(i, j)[dirs::w] * u_old(i - 1, j) + - Ew(i + 1, j)[dirs::w] * u_old(i + 1, j) + - Ew(i, j)[dirs::s] * u_old(i, j - 1) + - Ew(i, j + 1)[dirs::s] * u_old(i, j + 1) + f(i, j)) / - Ew(i, j)[dirs::c]; + auto fj = 2 * j - mc.ghost_zone_size(); + auto fi = 2 * i - mc.ghost_zone_size(); - u_new(i, j) = u_old(i, j) + omega * (z - u_old(i, j)); + fc(i, j) = 0.25 * (rf(fi, fj) + rf(fi + 1, fj) + rf(fi, fj + 1) + + rf(fi + 1, fj + 1)); }; // forall } else /* D == 3 */ { - auto mdpolicy_qqq = get_mdiota_policy(u_new, - m.template cells(), - m.template cells(), - m.template cells()); + auto mdpolicy_qqq = get_mdiota_policy(fc, + mc.template cells(), + mc.template cells(), + mc.template cells()); - forall(kji, mdpolicy_qqq, "damped_jacobi") { + s.executor().forall(kji, mdpolicy_qqq) { auto [k, j, i] = kji; - const double z = - (Ew(i, j, k)[dirs::w] * u_old(i - 1, j, k) + - Ew(i + 1, j, k)[dirs::w] * u_old(i + 1, j, k) + - Ew(i, j, k)[dirs::s] * u_old(i, j - 1, k) + - Ew(i, j + 1, k)[dirs::s] * u_old(i, j + 1, k) + - Ew(i, j, k)[dirs::d] * u_old(i, j, k - 1) + - Ew(i, j, k + 1)[dirs::d] * u_old(i, j, k + 1) + f(i, j, k)) / - Ew(i, j, k)[dirs::c]; + auto fk = 2 * k - mc.ghost_zone_size(); + auto fj = 2 * j - mc.ghost_zone_size(); + auto fi = 2 * i - mc.ghost_zone_size(); - u_new(i, j, k) = u_old(i, j, k) + omega * (z - u_old(i, j, k)); - }; // forall + fc(i, j, k) = + 0.125 * + (rf(fi, fj, fk) + rf(fi + 1, fj, fk) + rf(fi, fj + 1, fk) + + rf(fi, fj, fk + 1) + rf(fi + 1, fj + 1, fk) + rf(fi + 1, fj, fk + 1) + + rf(fi, fj + 1, fk + 1) + rf(fi + 1, fj + 1, fk + 1)); + + }; // for } // if -} // damped_jacobi +} // full_weighting -// Gauss-Seidel template void -gauss_seidel(typename mesh::template accessor m, - typename field>::template accessor Ew_a, - typename field::template accessor ua, - typename field::template accessor fa) { - - // TODO: Parallelize Gauss seidel +cell_centered_interpolation(flecsi::exec::accelerator s, + typename mesh::template accessor mc, + typename mesh::template accessor mf, + field::accessor cfa, + field::accessor ffa) noexcept { - auto Ew = m.template mdcolex(Ew_a); - auto u = m.template mdcolex(ua); - auto f = m.template mdcolex(fa); + auto cf = mc.template mdcolex(cfa); + auto ff = mf.template mdcolex(ffa); if constexpr(D == 1) { - for(auto i : m.template cells()) { - u(i) = - (Ew(i)[dirs::w] * u(i - 1) + Ew(i + 1)[dirs::w] * u(i + 1) + f(i)) / - Ew(i)[dirs::c]; - - } // for + s.executor().forall(i, (mc.template cells())) { + auto fi = 2 * i - mc.ghost_zone_size(); + ff(i) = cf(fi); + ff(i + 1) = cf(fi); + }; // for } else if constexpr(D == 2) { - for(auto j : m.template cells()) { - for(auto i : m.template cells()) { - u(i, j) = (Ew(i, j)[dirs::w] * u(i - 1, j) + - Ew(i + 1, j)[dirs::w] * u(i + 1, j) + - Ew(i, j)[dirs::s] * u(i, j - 1) + - Ew(i, j + 1)[dirs::s] * u(i, j + 1) + f(i, j)) / - Ew(i, j)[dirs::c]; - - } // for - } // for + auto mdpolicy_qq = get_mdiota_policy(cf, + mc.template cells(), + mc.template cells()); + + s.executor().forall(ji, mdpolicy_qq) { + auto [j, i] = ji; + auto fj = 2 * j - mc.ghost_zone_size(); + auto fi = 2 * i - mc.ghost_zone_size(); + + ff(fi, fj) = cf(i, j); + ff(fi + 1, fj) = cf(i, j); + ff(fi, fj + 1) = cf(i, j); + ff(fi + 1, fj + 1) = cf(i, j); + + }; // for } else /* D == 3 */ { - for(auto k : m.template cells()) { - for(auto j : m.template cells()) { - for(auto i : m.template cells()) { - u(i, j, k) = - (Ew(i, j, k)[dirs::w] * u(i - 1, j, k) + - Ew(i + 1, j, k)[dirs::w] * u(i + 1, j, k) + - Ew(i, j, k)[dirs::s] * u(i, j - 1, k) + - Ew(i, j + 1, k)[dirs::s] * u(i, j + 1, k) + - Ew(i, j, k)[dirs::d] * u(i, j, k - 1) + - Ew(i, j, k + 1)[dirs::d] * u(i, j, k + 1) + f(i, j, k)) / - Ew(i, j, k)[dirs::c]; - - } // for - } // for - } // for + auto mdpolicy_qqq = get_mdiota_policy(cf, + mc.template cells(), + mc.template cells(), + mc.template cells()); + + s.executor().forall(kji, mdpolicy_qqq) { + auto [k, j, i] = kji; + auto fj = 2 * j - mc.ghost_zone_size(); + auto fi = 2 * i - mc.ghost_zone_size(); + auto fk = 2 * k - mc.ghost_zone_size(); + + ff(fi, fj, fk) = cf(i, j, k); + ff(fi + 1, fj, fk) = cf(i, j, k); + ff(fi, fj + 1, fk) = cf(i, j, k); + ff(fi, fj, fk + 1) = cf(i, j, k); + ff(fi + 1, fj + 1, fk) = cf(i, j, k); + ff(fi + 1, fj + 1, fk + 1) = cf(i, j, k); + ff(fi + 1, fj, fk + 1) = cf(i, j, k); + ff(fi, fj + 1, fk + 1) = cf(i, j, k); + }; } // if -} // gauss_seidel +} -// Red Gauss-Seidel template void -red_gauss_seidel(typename mesh::template accessor m, +apply_operator(flecsi::exec::accelerator s, + typename mesh::template accessor m, typename field>::template accessor Ew_a, - typename field::template accessor ua, - typename field::template accessor fa) { + field::accessor ua_new, + field::accessor ua_old) noexcept { auto Ew = m.template mdcolex(Ew_a); - auto u = m.template mdcolex(ua); - auto f = m.template mdcolex(fa); + auto u_new = m.template mdcolex(ua_new); + auto u_old = m.template mdcolex(ua_old); if constexpr(D == 1) { - for(auto i : m.template cells()) { - - unsigned int id_i = i; - - if((id_i % 2) == 0) { - u(i) = - (Ew(i)[dirs::w] * u(i - 1) + Ew(i + 1)[dirs::w] * u(i + 1) + f(i)) / - Ew(i)[dirs::c]; - } - - } // for + s.executor().forall(i, (m.template cells())) { + u_new(i) = (Ew(i)[dirs::c] * u_old(i) - Ew(i)[dirs::w] * u_old(i - 1) - + Ew(i + 1)[dirs::w] * u_old(i + 1)); + }; } else if constexpr(D == 2) { - for(auto j : m.template cells()) { - for(auto i : m.template cells()) { - - unsigned int id_i = i; - unsigned int id_j = j; - - if(((id_i + id_j) % 2) == 0) { - u(i, j) = (Ew(i, j)[dirs::w] * u(i - 1, j) + - Ew(i + 1, j)[dirs::w] * u(i + 1, j) + - Ew(i, j)[dirs::s] * u(i, j - 1) + - Ew(i, j + 1)[dirs::s] * u(i, j + 1) + f(i, j)) / - Ew(i, j)[dirs::c]; - } + auto mdpolicy_qq = get_mdiota_policy(u_new, + m.template cells(), + m.template cells()); - } // for - } // for + s.executor().forall(ji, mdpolicy_qq) { + auto [j, i] = ji; + u_new(i, j) = + (Ew(i, j)[dirs::c] * u_old(i, j) - Ew(i, j)[dirs::w] * u_old(i - 1, j) - + Ew(i + 1, j)[dirs::w] * u_old(i + 1, j) - + Ew(i, j)[dirs::s] * u_old(i, j - 1) - + Ew(i, j + 1)[dirs::s] * u_old(i, j + 1)); + }; // forall } else /* D == 3 */ { - for(auto k : m.template cells()) { - for(auto j : m.template cells()) { - for(auto i : m.template cells()) { - - unsigned int id_i = i; - unsigned int id_j = j; - unsigned int id_k = k; - - if(((id_i + id_j + id_k) % 2) == 0) { - u(i, j, k) = - (Ew(i, j, k)[dirs::w] * u(i - 1, j, k) + - Ew(i + 1, j, k)[dirs::w] * u(i + 1, j, k) + - Ew(i, j, k)[dirs::s] * u(i, j - 1, k) + - Ew(i, j + 1, k)[dirs::s] * u(i, j + 1, k) + - Ew(i, j, k)[dirs::d] * u(i, j, k - 1) + - Ew(i, j, k + 1)[dirs::d] * u(i, j, k + 1) + f(i, j, k)) / - Ew(i, j, k)[dirs::c]; - } - - } // for - } // for - } // for + auto mdpolicy_qqq = get_mdiota_policy(u_new, + m.template cells(), + m.template cells(), + m.template cells()); + + s.executor().forall(kji, mdpolicy_qqq) { + auto [k, j, i] = kji; + u_new(i, j, k) = (Ew(i, j, k)[dirs::c] * u_old(i, j, k) - + Ew(i, j, k)[dirs::w] * u_old(i - 1, j, k) - + Ew(i + 1, j, k)[dirs::w] * u_old(i + 1, j, k) - + Ew(i, j, k)[dirs::s] * u_old(i, j - 1, k) - + Ew(i, j + 1, k)[dirs::s] * u_old(i, j + 1, k) - + Ew(i, j, k)[dirs::d] * u_old(i, j, k - 1) - + Ew(i, j, k + 1)[dirs::d] * u_old(i, j, k + 1)); + }; // forall } // if -} // red_gauss_seidel +} // Ax_op -// Black Gauss-Seidel template void -black_gauss_seidel(typename mesh::template accessor m, +damped_jacobi(flecsi::exec::accelerator s, + typename mesh::template accessor m, typename field>::template accessor Ew_a, - typename field::template accessor ua, - typename field::template accessor fa) { + typename field::template accessor ua_new, + typename field::template accessor ua_old, + typename field::template accessor fa, + double omega) noexcept { auto Ew = m.template mdcolex(Ew_a); - auto u = m.template mdcolex(ua); + auto u_new = m.template mdcolex(ua_new); + auto u_old = m.template mdcolex(ua_old); auto f = m.template mdcolex(fa); if constexpr(D == 1) { - for(auto i : m.template cells()) { - - unsigned int id_i = i; - - if((id_i % 2) == 1) { - u(i) = - (Ew(i)[dirs::w] * u(i - 1) + Ew(i + 1)[dirs::w] * u(i + 1) + f(i)) / - Ew(i)[dirs::c]; - } + s.executor().forall(i, (m.template cells())) { + const double z = (Ew(i)[dirs::w] * u_old(i - 1) + + Ew(i + 1)[dirs::w] * u_old(i + 1) + f(i)) / + Ew(i)[dirs::c]; - } // for + u_new(i) = u_old(i) + omega * (z - u_old(i)); + }; // for } else if constexpr(D == 2) { - for(auto j : m.template cells()) { - for(auto i : m.template cells()) { - - unsigned int id_i = i; - unsigned int id_j = j; - - if(((id_i + id_j) % 2) == 1) { - u(i, j) = (Ew(i, j)[dirs::w] * u(i - 1, j) + - Ew(i + 1, j)[dirs::w] * u(i + 1, j) + - Ew(i, j)[dirs::s] * u(i, j - 1) + - Ew(i, j + 1)[dirs::s] * u(i, j + 1) + f(i, j)) / - Ew(i, j)[dirs::c]; - } - } // for - } // for + auto mdpolicy_qq = get_mdiota_policy(u_new, + m.template cells(), + m.template cells()); + + s.executor().forall(ji, mdpolicy_qq) { + auto [j, i] = ji; + const double z = (Ew(i, j)[dirs::w] * u_old(i - 1, j) + + Ew(i + 1, j)[dirs::w] * u_old(i + 1, j) + + Ew(i, j)[dirs::s] * u_old(i, j - 1) + + Ew(i, j + 1)[dirs::s] * u_old(i, j + 1) + f(i, j)) / + Ew(i, j)[dirs::c]; + + u_new(i, j) = u_old(i, j) + omega * (z - u_old(i, j)); + }; // forall } else /* D == 3 */ { - for(auto k : m.template cells()) { - for(auto j : m.template cells()) { - for(auto i : m.template cells()) { - - unsigned int id_i = i; - unsigned int id_j = j; - unsigned int id_k = k; - - if(((id_i + id_j + id_k) % 2) == 1) { - u(i, j, k) = - (Ew(i, j, k)[dirs::w] * u(i - 1, j, k) + - Ew(i + 1, j, k)[dirs::w] * u(i + 1, j, k) + - Ew(i, j, k)[dirs::s] * u(i, j - 1, k) + - Ew(i, j + 1, k)[dirs::s] * u(i, j + 1, k) + - Ew(i, j, k)[dirs::d] * u(i, j, k - 1) + - Ew(i, j, k + 1)[dirs::d] * u(i, j, k + 1) + f(i, j, k)) / - Ew(i, j, k)[dirs::c]; - } - } // for - } // for - } // for + auto mdpolicy_qqq = get_mdiota_policy(u_new, + m.template cells(), + m.template cells(), + m.template cells()); + + s.executor().forall(kji, mdpolicy_qqq) { + auto [k, j, i] = kji; + const double z = + (Ew(i, j, k)[dirs::w] * u_old(i - 1, j, k) + + Ew(i + 1, j, k)[dirs::w] * u_old(i + 1, j, k) + + Ew(i, j, k)[dirs::s] * u_old(i, j - 1, k) + + Ew(i, j + 1, k)[dirs::s] * u_old(i, j + 1, k) + + Ew(i, j, k)[dirs::d] * u_old(i, j, k - 1) + + Ew(i, j, k + 1)[dirs::d] * u_old(i, j, k + 1) + f(i, j, k)) / + Ew(i, j, k)[dirs::c]; + + u_new(i, j, k) = u_old(i, j, k) + omega * (z - u_old(i, j, k)); + }; // forall } // if -} // black_gauss_seidel +} // damped_jacobi template void -residual(typename mesh::template accessor m, +residual(flecsi::exec::accelerator s, + typename mesh::template accessor m, typename field>::template accessor Ew_a, typename field::template accessor ua, typename field::template accessor fa, - typename field::template accessor ra) { + typename field::template accessor ra) noexcept { // TODO: Not using the ghost cells of fa here, are we? auto Ew = m.template mdcolex(Ew_a); @@ -1253,7 +1326,7 @@ residual(typename mesh::template accessor m, auto r = m.template mdcolex(ra); if constexpr(D == 1) { - forall(i, (m.template cells()), "residual") { + s.executor().forall(i, (m.template cells())) { r(i) = f(i) - (Ew(i)[dirs::c] * u(i) - Ew(i)[dirs::w] * u(i - 1) - Ew(i + 1)[dirs::w] * u(i + 1)); }; @@ -1263,7 +1336,7 @@ residual(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(ji, mdpolicy_qq, "residual") { + s.executor().forall(ji, mdpolicy_qq) { auto [j, i] = ji; r(i, j) = f(i, j) - (Ew(i, j)[dirs::c] * u(i, j) - Ew(i, j)[dirs::w] * u(i - 1, j) - @@ -1278,7 +1351,7 @@ residual(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(kji, mdpolicy_qqq, "residual") { + s.executor().forall(kji, mdpolicy_qqq) { auto [k, j, i] = kji; r(i, j, k) = f(i, j, k) - (Ew(i, j, k)[dirs::c] * u(i, j, k) - Ew(i, j, k)[dirs::w] * u(i - 1, j, k) - @@ -1293,16 +1366,17 @@ residual(typename mesh::template accessor m, template void -correction(typename mesh::template accessor m, +correction(flecsi::exec::accelerator s, + typename mesh::template accessor m, typename field::template accessor ua, - typename field::template accessor ea) { + typename field::template accessor ea) noexcept { // TODO: Looks like ea should be auto u = m.template mdcolex(ua); auto e = m.template mdcolex(ea); if constexpr(D == 1) { - forall(i, (m.template cells()), "correction") { + s.executor().forall(i, (m.template cells())) { u(i) += e(i); }; // for } @@ -1311,7 +1385,7 @@ correction(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(ji, mdpolicy_qq, "full_weighting") { + s.executor().forall(ji, mdpolicy_qq) { auto [j, i] = ji; u(i, j) += e(i, j); }; // for @@ -1322,7 +1396,7 @@ correction(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(kji, mdpolicy_qqq, "full_weighting") { + s.executor().forall(kji, mdpolicy_qqq) { auto [k, j, i] = kji; u(i, j, k) += e(i, j, k); }; // for @@ -1331,9 +1405,10 @@ correction(typename mesh::template accessor m, // Linear interpolation for boundary temperature double -interp_e_boundary(typename single::accessor t, - typename field::accessor time_boundary, - typename field::accessor temperature_boundary) { +interp_e_boundary(flecsi::exec::cpu, + single::accessor t, + field::accessor time_boundary, + field::accessor temperature_boundary) noexcept { auto get_energy = [](const double & temperature) -> double { return constants::cgs::radiation_constant * @@ -1342,10 +1417,12 @@ interp_e_boundary(typename single::accessor t, // Is it the first or last value? std::size_t i_end{time_boundary.span().size()}; - if(t <= time_boundary[0]) + if(t <= time_boundary[0]) { return get_energy(temperature_boundary[0]); - if(t >= time_boundary[i_end - 1]) + } + if(t >= time_boundary[i_end - 1]) { return get_energy(temperature_boundary[i_end - 1]); + } for(std::size_t i{0}; i < (i_end - 1); i++) { if((t >= time_boundary[i]) && (t <= time_boundary[i + 1])) { @@ -1359,8 +1436,8 @@ interp_e_boundary(typename single::accessor t, } // We should never reach this line - flog_fatal("Linear interpolation failed"); - return -1.0; // Point never found + assert(false && "Linear interpolation failed"); + return 0; } // interp_e_boundary } // namespace hard::task::rad diff --git a/app/tasks/rad_root.hh b/app/tasks/rad_root.hh index 2daf5c1..0942937 100644 --- a/app/tasks/rad_root.hh +++ b/app/tasks/rad_root.hh @@ -12,16 +12,16 @@ namespace hard::task::rad_root { // template void -update_energy_density(typename mesh::template accessor m, +update_energy_density(flecsi::exec::accelerator s, + typename mesh::template accessor m, field::accessor r_a, typename field>::template accessor u_a, field::accessor temperature_a, field::accessor rE_a, field::accessor radiation_energy_density_a, single::accessor kappa_a, - single::accessor particle_mass_a, single::accessor dt_a, - eos::eos_wrapper const & eos) { + eos::eos_wrapper const & eos) noexcept { using hard::tasks::util::get_mdiota_policy; @@ -32,7 +32,6 @@ update_energy_density(typename mesh::template accessor m, auto radiation_energy_density = m.template mdcolex(radiation_energy_density_a); auto const kappa = *kappa_a; - auto const particle_mass = *particle_mass_a; auto & dt_weighted = *dt_a; // Timestep constant for energy update @@ -42,7 +41,7 @@ update_energy_density(typename mesh::template accessor m, const double rad_c{hard::constants::cgs::radiation_constant}; // Define the En update lambda - auto get_up_En = [&eos, r, dt_c, rad_c](double_t t, double_t En) { + auto get_up_En = [eos, dt_c, rad_c](double_t t, double_t En) { return (En + dt_c * rad_c * pow(t, 4.0)) / (1 + dt_c); }; @@ -51,8 +50,7 @@ update_energy_density(typename mesh::template accessor m, Compute the updated gas energy and raditaion energy. *------------------------------------------------------------------------*/ - forall( - i, (m.template cells()), "upd_energy_density") { + s.executor().forall(i, (m.template cells())) { auto & dt_weighted = *dt_a; // NOTE: a1 and a2 are constants defined in the paper Moens et al (2022) // FIXME: The variables "a1" and "a2" do not exist, remove comment @@ -64,12 +62,13 @@ update_energy_density(typename mesh::template accessor m, const double ke = 0.5 * r(i) * u(i).norm_squared(); // kinetic energy const double en = rE(i) - ke; // internal energy + assert(en >= 0 && "Internal energy is negative"); // getting temperature from EOS, this should be inital guess temperature(i) = eos.tRhoSie(r(i), en); const double En = radiation_energy_density(i); // radiation energy - assert(En >= 0); + assert(En >= 0 && "Radiation energy is negative"); // Find the next temperature with root finding const double up_Tn{tasks::util::find_temp( @@ -77,7 +76,11 @@ update_energy_density(typename mesh::template accessor m, // Update En with the formula and en with conservation of energy const double up_En{get_up_En(up_Tn, En)}; + assert(up_En >= 0 && "Updated radiation energy is negative"); + const double up_en{en - (up_En - En)}; + assert(up_en >= 0 && "Updated internal energy is negative"); + radiation_energy_density(i) = up_En; rE(i) = up_en + ke; temperature(i) = up_Tn; @@ -89,25 +92,30 @@ update_energy_density(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(ji, mdpolicy_qq, "upd_energy_density") { + s.executor().forall(ji, mdpolicy_qq) { auto [j, i] = ji; const double ke = 0.5 * r(i, j) * u(i, j).norm_squared(); // kinetic energy const double en = rE(i, j) - ke; // internal energy + assert(en >= 0 && "Internal energy is negative"); temperature(i, j) = eos.tRhoSie(r(i, j), en); const double En = radiation_energy_density(i, j); // radiation energy - assert(En >= 0); + assert(En >= 0 && "Radiation energy is negative"); // Find the next temperature with root finding const double up_Tn{tasks::util::find_temp( eos, r(i, j), en, temperature(i, j), kappa, En, dt_weighted)}; const double up_En{get_up_En(up_Tn, En)}; + assert(up_En >= 0 && "Updated radiation energy is negative"); + const double up_en{en - (up_En - En)}; + assert(up_en >= 0 && "Updated internal energy is negative"); + radiation_energy_density(i, j) = up_En; rE(i, j) = up_en + ke; temperature(i, j) = up_Tn; @@ -119,24 +127,29 @@ update_energy_density(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(kji, mdpolicy_qqq, "upd_energy_density") { + s.executor().forall(kji, mdpolicy_qqq) { auto [k, j, i] = kji; const double ke = 0.5 * r(i, j, k) * u(i, j, k).norm_squared(); // kinetic energy const double en = rE(i, j, k) - ke; // internal energy + assert(en >= 0 && "Internal energy is negative"); temperature(i, j, k) = eos.tRhoSie(r(i, j, k), en); const double En = radiation_energy_density(i, j, k); // radiation energy - assert(En >= 0); + assert(En >= 0 && "Radiation energy is negative"); // Find the next temperature with root finding const double up_Tn{tasks::util::find_temp( eos, r(i, j, k), en, temperature(i, j, k), kappa, En, dt_weighted)}; const double up_En{get_up_En(up_Tn, En)}; + assert(up_En >= 0 && "Updated radiation energy is negative"); + const double up_en{en - (up_En - En)}; + assert(up_en >= 0 && "Updated internal energy is negative"); + radiation_energy_density(i, j, k) = up_En; rE(i, j, k) = up_en + ke; temperature(i, j, k) = up_Tn; diff --git a/app/tasks/time_derivative.hh b/app/tasks/time_derivative.hh index 6e9dd59..cbc3bf8 100644 --- a/app/tasks/time_derivative.hh +++ b/app/tasks/time_derivative.hh @@ -11,11 +11,12 @@ namespace hard::tasks { template void -set_dudt_to_zero(typename mesh::template accessor m, +set_dudt_to_zero(flecsi::exec::accelerator s, + typename mesh::template accessor m, field::accessor dt_mass_density_a, typename field>::template accessor dt_momentum_density_a, field::accessor dt_total_energy_density_a, - field::accessor dt_radiation_energy_density_a) { + field::accessor dt_radiation_energy_density_a) noexcept { auto dt_mass_density = m.template mdcolex(dt_mass_density_a); auto dt_momentum_density = @@ -27,10 +28,9 @@ set_dudt_to_zero(typename mesh::template accessor m, using hard::tasks::util::get_mdiota_policy; if constexpr(Dim == 1) { - forall( - i, (m.template cells()), "set_dudt_to_zero_1d") { + s.executor().forall(i, (m.template cells())) { dt_mass_density(i) = 0.0; - dt_momentum_density(i).x = 0.0; + dt_momentum_density(i).x() = 0.0; dt_total_energy_density(i) = 0.0; dt_radiation_energy_density(i) = 0.0; }; // forall @@ -40,11 +40,11 @@ set_dudt_to_zero(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(ji, mdpolicy_qq, "set_dudt_to_zero_2d") { + s.executor().forall(ji, mdpolicy_qq) { auto [j, i] = ji; dt_mass_density(i, j) = 0.0; - dt_momentum_density(i, j).x = 0.0; - dt_momentum_density(i, j).y = 0.0; + dt_momentum_density(i, j).x() = 0.0; + dt_momentum_density(i, j).y() = 0.0; dt_total_energy_density(i, j) = 0.0; #ifdef ENABLE_RADIATION dt_radiation_energy_density(i, j) = 0.0; @@ -58,12 +58,12 @@ set_dudt_to_zero(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(kji, mdpolicy_qqq, "set_dudt_to_zero_3d") { + s.executor().forall(kji, mdpolicy_qqq) { auto [k, j, i] = kji; dt_mass_density(i, j, k) = 0.0; - dt_momentum_density(i, j, k).x = 0.0; - dt_momentum_density(i, j, k).y = 0.0; - dt_momentum_density(i, j, k).z = 0.0; + dt_momentum_density(i, j, k).x() = 0.0; + dt_momentum_density(i, j, k).y() = 0.0; + dt_momentum_density(i, j, k).z() = 0.0; dt_total_energy_density(i, j, k) = 0.0; #ifdef ENABLE_RADIATION dt_radiation_energy_density(i, j, k) = 0.0; @@ -78,7 +78,8 @@ set_dudt_to_zero(typename mesh::template accessor m, // template void -store_current_state(typename mesh::template accessor m, +store_current_state(flecsi::exec::accelerator s, + typename mesh::template accessor m, // Copied from field::accessor mass_density_a, typename field>::template accessor momentum_density_a, @@ -96,7 +97,7 @@ store_current_state(typename mesh::template accessor m, #ifdef ENABLE_RADIATION radiation_energy_density_n_a #endif -) { + ) noexcept { auto mass_density = m.template mdcolex(mass_density_a); auto mass_density_n = m.template mdcolex(mass_density_n_a); @@ -115,7 +116,7 @@ store_current_state(typename mesh::template accessor m, using hard::tasks::util::get_mdiota_policy; if constexpr(Dim == 1) { - forall(i, (m.template cells()), "store_state_1d") { + s.executor().forall(i, (m.template cells())) { mass_density_n(i) = mass_density(i); momentum_density_n(i) = momentum_density(i); total_energy_density_n(i) = total_energy_density(i); @@ -129,7 +130,7 @@ store_current_state(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(ji, mdpolicy_qq, "store_state_2d") { + s.executor().forall(ji, mdpolicy_qq) { auto [j, i] = ji; mass_density_n(i, j) = mass_density(i, j); momentum_density_n(i, j) = momentum_density(i, j); @@ -146,7 +147,7 @@ store_current_state(typename mesh::template accessor m, m.template cells(), m.template cells()); - forall(kji, mdpolicy_qqq, "store_state_3d") { + s.executor().forall(kji, mdpolicy_qqq) { auto [k, j, i] = kji; mass_density_n(i, j, k) = mass_density(i, j, k); momentum_density_n(i, j, k) = momentum_density(i, j, k); @@ -160,93 +161,196 @@ store_current_state(typename mesh::template accessor m, } // -// Prepare evolved variables to U^1 or U^2 after implicit steps (radiative -// heating / diffusion) and storing (dU_dt)_implicit have been done. +// Used for the RK substeps // template void -compute_u_after_implicit_solve(typename mesh::template accessor m, - single::accessor dtw_a, +update_u(flecsi::exec::accelerator s, + single::accessor dt_a, + typename mesh::template accessor m, + // U^n we want to update + field::accessor mass_density_a, + typename field>::template accessor momentum_density_a, field::accessor total_energy_density_a, field::accessor #ifdef ENABLE_RADIATION radiation_energy_density_a #endif , - field::accessor dt_total_energy_density_implicit_a, + // Time derivatives for the state U^1 + field::accessor dt_mass_density_a, + typename field>::template accessor dt_momentum_density_a, + field::accessor dt_total_energy_density_a, field::accessor #ifdef ENABLE_RADIATION - dt_radiation_energy_density_implicit_a + dt_radiation_energy_density_a #endif -) { + ) noexcept { + auto mass_density = m.template mdcolex(mass_density_a); + auto momentum_density = m.template mdcolex(momentum_density_a); auto total_energy_density = m.template mdcolex(total_energy_density_a); #ifdef ENABLE_RADIATION - auto radiation_energy_density = m.template mdcolex(radiation_energy_density_a); #endif - auto dt_total_energy_density_implicit = - m.template mdcolex(dt_total_energy_density_implicit_a); + + auto dt_mass_density = m.template mdcolex(dt_mass_density_a); + auto dt_momentum_density = + m.template mdcolex(dt_momentum_density_a); + auto dt_total_energy_density = + m.template mdcolex(dt_total_energy_density_a); #ifdef ENABLE_RADIATION - auto dt_radiation_energy_density_implicit = - m.template mdcolex(dt_radiation_energy_density_implicit_a); + auto dt_radiation_energy_density = + m.template mdcolex(dt_radiation_energy_density_a); #endif using hard::tasks::util::get_mdiota_policy; + if constexpr(Dim == 1) { - forall(i, (m.template cells()), "compute_u_1d") { - auto & dt_weighted = *dtw_a; - total_energy_density(i) += - dt_weighted * dt_total_energy_density_implicit(i); + s.executor().forall(i, (m.template cells())) { + auto h = *dt_a; + mass_density(i) += h * dt_mass_density(i); + momentum_density(i) += h * dt_momentum_density(i); + total_energy_density(i) += h * dt_total_energy_density(i); + #ifdef ENABLE_RADIATION - radiation_energy_density(i) += - dt_weighted * dt_radiation_energy_density_implicit(i); + radiation_energy_density(i) += h * dt_radiation_energy_density(i); #endif }; // forall } else if constexpr(Dim == 2) { - auto mdpolicy_qq = get_mdiota_policy(total_energy_density, + auto mdpolicy_qq = get_mdiota_policy(mass_density, m.template cells(), m.template cells()); - forall(ji, mdpolicy_qq, "compute_u_2d") { + s.executor().forall(ji, mdpolicy_qq) { + auto h = *dt_a; auto [j, i] = ji; - auto & dt_weighted = *dtw_a; - total_energy_density(i, j) += - dt_weighted * dt_total_energy_density_implicit(i, j); + // Weights + mass_density(i, j) += h * dt_mass_density(i, j); + momentum_density(i, j) += h * dt_momentum_density(i, j); + total_energy_density(i, j) += h * dt_total_energy_density(i, j); #ifdef ENABLE_RADIATION - radiation_energy_density(i, j) += - dt_weighted * dt_radiation_energy_density_implicit(i, j); + radiation_energy_density(i, j) += h * dt_radiation_energy_density(i, j); #endif }; // forall } else { - auto mdpolicy_qqq = get_mdiota_policy(total_energy_density, + auto mdpolicy_qqq = get_mdiota_policy(mass_density, m.template cells(), m.template cells(), m.template cells()); - forall(kji, mdpolicy_qqq, "compute_u_3d") { + s.executor().forall(kji, mdpolicy_qqq) { + auto h = *dt_a; auto [k, j, i] = kji; - auto & dt_weighted = *dtw_a; - total_energy_density(i, j, k) += - dt_weighted * dt_total_energy_density_implicit(i, j, k); + + mass_density(i, j, k) += h * dt_mass_density(i, j, k); + momentum_density(i, j, k) += h * dt_momentum_density(i, j, k); + total_energy_density(i, j, k) += h * dt_total_energy_density(i, j, k); #ifdef ENABLE_RADIATION radiation_energy_density(i, j, k) += - dt_weighted * dt_radiation_energy_density_implicit(i, j, k); + h * dt_radiation_energy_density(i, j, k); +#endif + }; // forall + } +} + +template +void +add_k1_k2(flecsi::exec::accelerator s, + typename mesh::template accessor m, + // K1 + field::accessor dt_mass_density_a, + typename field>::template accessor dt_momentum_density_a, + field::accessor dt_total_energy_density_a, + field::accessor +#ifdef ENABLE_RADIATION + dt_radiation_energy_density_a +#endif + , + // K2 + field::accessor dt_mass_density_2_a, + typename field>::template accessor dt_momentum_density_2_a, + field::accessor dt_total_energy_density_2_a, + field::accessor +#ifdef ENABLE_RADIATION + dt_radiation_energy_density_2_a +#endif + ) noexcept { + // K1 + auto dt_r = m.template mdcolex(dt_mass_density_a); + auto dt_ru = m.template mdcolex(dt_momentum_density_a); + auto dt_te = m.template mdcolex(dt_total_energy_density_a); +#ifdef ENABLE_RADIATION + auto dt_re = m.template mdcolex(dt_radiation_energy_density_a); +#endif + + // K2 + auto dt_r2 = m.template mdcolex(dt_mass_density_2_a); + auto dt_ru2 = m.template mdcolex(dt_momentum_density_2_a); + auto dt_te2 = m.template mdcolex(dt_total_energy_density_2_a); +#ifdef ENABLE_RADIATION + auto dt_re2 = m.template mdcolex(dt_radiation_energy_density_2_a); +#endif + + using hard::tasks::util::get_mdiota_policy; + + if constexpr(Dim == 1) { + s.executor().forall(i, (m.template cells())) { + dt_r(i) = (dt_r(i) + dt_r2(i)) * 0.5; + dt_ru(i) = (dt_ru(i) + dt_ru2(i)) * 0.5; + dt_te(i) = (dt_te(i) + dt_te2(i)) * 0.5; + +#ifdef ENABLE_RADIATION + dt_re(i) = (dt_re(i) + dt_re2(i)) * 0.5; +#endif + }; // forall + } + else if constexpr(Dim == 2) { + auto mdpolicy_qq = get_mdiota_policy(dt_r, + m.template cells(), + m.template cells()); + + s.executor().forall(ji, mdpolicy_qq) { + auto [j, i] = ji; + // Weights + dt_r(i, j) = (dt_r(i, j) + dt_r2(i, j)) * 0.5; + dt_ru(i, j) = (dt_ru(i, j) + dt_ru2(i, j)) * 0.5; + dt_te(i, j) = (dt_te(i, j) + dt_te2(i, j)) * 0.5; +#ifdef ENABLE_RADIATION + dt_re(i, j) = (dt_re(i, j) + dt_re2(i, j)) * 0.5; +#endif + }; // forall + } + else { + auto mdpolicy_qqq = get_mdiota_policy(dt_r, + m.template cells(), + m.template cells(), + m.template cells()); + + s.executor().forall(kji, mdpolicy_qqq) { + auto [k, j, i] = kji; + + dt_r(i, j, k) = (dt_r(i, j, k) + dt_r2(i, j, k)) * 0.5; + dt_ru(i, j, k) = (dt_ru(i, j, k) + dt_ru2(i, j, k)) * 0.5; + dt_te(i, j, k) = (dt_te(i, j, k) + dt_te2(i, j, k)) * 0.5; +#ifdef ENABLE_RADIATION + dt_re(i, j, k) = (dt_re(i, j, k) + dt_re2(i, j, k)) * 0.5; #endif }; // forall } } // -// Used for the RK substeps +// Used for the RK stage updates // -template +template void -update_u(single::accessor dt_a, +update_u_stage(flecsi::exec::cpu s, + single::accessor dt_a, typename mesh::template accessor m, // U^n we want to update field::accessor mass_density_a, @@ -266,7 +370,15 @@ update_u(single::accessor dt_a, dt_radiation_energy_density_a #endif , - const double coefficient) { + // U^n+1 updated after stage + field::accessor mass_density_b, + typename field>::template accessor momentum_density_b, + field::accessor total_energy_density_b, + field::accessor +#ifdef ENABLE_RADIATION + radiation_energy_density_b +#endif + ) noexcept { auto mass_density = m.template mdcolex(mass_density_a); auto momentum_density = m.template mdcolex(momentum_density_a); @@ -277,6 +389,15 @@ update_u(single::accessor dt_a, m.template mdcolex(radiation_energy_density_a); #endif + auto mass_density_new = m.template mdcolex(mass_density_b); + auto momentum_density_new = m.template mdcolex(momentum_density_b); + auto total_energy_density_new = + m.template mdcolex(total_energy_density_b); +#ifdef ENABLE_RADIATION + auto radiation_energy_density_new = + m.template mdcolex(radiation_energy_density_b); +#endif + auto dt_mass_density = m.template mdcolex(dt_mass_density_a); auto dt_momentum_density = m.template mdcolex(dt_momentum_density_a); @@ -287,17 +408,20 @@ update_u(single::accessor dt_a, m.template mdcolex(dt_radiation_energy_density_a); #endif - auto h = *dt_a * coefficient; + auto h = *dt_a; using hard::tasks::util::get_mdiota_policy; if constexpr(Dim == 1) { - forall(i, (m.template cells()), "update_u_1d") { - mass_density(i) += h * dt_mass_density(i); - momentum_density(i) += h * dt_momentum_density(i); - total_energy_density(i) += h * dt_total_energy_density(i); + s.executor().forall(i, (m.template cells())) { + mass_density_new(i) = mass_density(i) + h * dt_mass_density(i); + momentum_density_new(i) = + momentum_density(i) + h * dt_momentum_density(i); + total_energy_density_new(i) = + total_energy_density(i) + h * dt_total_energy_density(i); #ifdef ENABLE_RADIATION - radiation_energy_density(i) += h * dt_radiation_energy_density(i); + radiation_energy_density_new(i) = + radiation_energy_density(i) + h * dt_radiation_energy_density(i); #endif }; // forall } @@ -306,14 +430,17 @@ update_u(single::accessor dt_a, m.template cells(), m.template cells()); - forall(ji, mdpolicy_qq, "update_u_2d") { + s.executor().forall(ji, mdpolicy_qq) { auto [j, i] = ji; - // Weights - mass_density(i, j) += h * dt_mass_density(i, j); - momentum_density(i, j) += h * dt_momentum_density(i, j); - total_energy_density(i, j) += h * dt_total_energy_density(i, j); + mass_density_new(i, j) = mass_density(i, j) + h * dt_mass_density(i, j); + momentum_density_new(i, j) = + momentum_density(i, j) + h * dt_momentum_density(i, j); + total_energy_density_new(i, j) = + total_energy_density(i, j) + h * dt_total_energy_density(i, j); + #ifdef ENABLE_RADIATION - radiation_energy_density(i, j) += h * dt_radiation_energy_density(i, j); + radiation_energy_density_new(i, j) = + radiation_energy_density(i, j) + h * dt_radiation_energy_density(i, j); #endif }; // forall } @@ -323,14 +450,18 @@ update_u(single::accessor dt_a, m.template cells(), m.template cells()); - forall(kji, mdpolicy_qqq, "update_u_3d") { + s.executor().forall(kji, mdpolicy_qqq) { auto [k, j, i] = kji; + mass_density_new(i, j, k) = + mass_density(i, j, k) + h * dt_mass_density(i, j, k); + momentum_density_new(i, j, k) = + momentum_density(i, j, k) + h * dt_momentum_density(i, j, k); + total_energy_density_new(i, j, k) = + total_energy_density(i, j, k) + h * dt_total_energy_density(i, j, k); - mass_density(i, j, k) += h * dt_mass_density(i, j, k); - momentum_density(i, j, k) += h * dt_momentum_density(i, j, k); - total_energy_density(i, j, k) += h * dt_total_energy_density(i, j, k); #ifdef ENABLE_RADIATION - radiation_energy_density(i, j, k) += + radiation_energy_density_new(i, j, k) = + radiation_energy_density(i, j, k) + h * dt_radiation_energy_density(i, j, k); #endif }; // forall diff --git a/app/tasks/utils.hh b/app/tasks/utils.hh index 1e8527b..5a2779c 100644 --- a/app/tasks/utils.hh +++ b/app/tasks/utils.hh @@ -9,12 +9,14 @@ namespace hard::tasks::util { +enum bl { low, high, none }; + /* Find the specific internal energy consitent with the given density and pressure. */ template -auto +FLECSI_INLINE_TARGET auto find_sie(E const & eos, const double_t r, const double_t p, @@ -25,10 +27,9 @@ find_sie(E const & eos, const double_t min{eos.eRhoT(r, 1.0e-50)}; const double_t max{eos.eRhoT(r, 1.0e20)}; g = g == std::numeric_limits::min() ? sqrt((max * min)) : g; - auto s = regula_falsi(kernel, p, g, min, max, 1.0e-12, 1.0e-12, sie); - flog_assert(s == Status::SUCCESS, - "specific internal energy root finder failed for r = " << r - << " and p = " << p); + [[maybe_unused]] auto s = + regula_falsi(kernel, p, g, min, max, 1.0e-12, 1.0e-12, sie); + assert(s == Status::SUCCESS && "specific internal energy root finder failed"); return sie; } // find_sie @@ -37,7 +38,7 @@ find_sie(E const & eos, // F(T) = e(rho, T) - e^n - a / (1 + a) * (ar * T^4 - En) // where a = dt * kappa * c template -auto +auto FLECSI_INLINE_TARGET find_temp(E const & eos, const double_t r, const double_t e, @@ -57,17 +58,17 @@ find_temp(E const & eos, return eos.eRhoT(r, t) + a / (1 + a) * (ar * pow(t, 4.0) - En); }; - auto s = regula_falsi(kernel, e, gt, min, max, 1.0e-12, 1.0e-12, t); - flog_assert(s == Status::SUCCESS, - "specific internal energy root finder failed for r = " << r - << " and t = " << t); + [[maybe_unused]] auto s = + regula_falsi(kernel, e, gt, min, max, 1.0e-12, 1.0e-12, t); + assert(s == Status::SUCCESS && "specific internal energy root finder failed"); return t; } // find_temp template inline void -print_conserved(typename mesh::template accessor m, +print_conserved(flecsi::exec::cpu, + typename mesh::template accessor m, field::accessor r_a, typename field>::template accessor ru_a, field::accessor rE_a, @@ -115,7 +116,8 @@ print_conserved(typename mesh::template accessor m, template inline void -print_vec_field(typename mesh::template accessor m, +print_vec_field(flecsi::exec::cpu, + typename mesh::template accessor m, typename field>::template accessor u_a) { std::stringstream ss; @@ -141,7 +143,8 @@ print_vec_field(typename mesh::template accessor m, template inline void -print_scal_field(typename mesh::template accessor m, +print_scal_field(flecsi::exec::cpu, + typename mesh::template accessor m, field::accessor u_a) { std::stringstream ss; @@ -167,7 +170,8 @@ print_scal_field(typename mesh::template accessor m, template inline void -print_primitives(typename mesh::template accessor m, +print_primitives(flecsi::exec::cpu, + typename mesh::template accessor m, typename field>::template accessor u_a, field::accessor p_a, flecsi::util::id zslice) { diff --git a/app/tests/test_convergence.sh b/app/tests/test_convergence.sh new file mode 100755 index 0000000..d5744ac --- /dev/null +++ b/app/tests/test_convergence.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -euo pipefail + +# Path to the executable +mpi_executable="$1" +hard_executable="$2" +start="$3" +stop="$4" +config_file="$5" +additional_args="$6" + +rm -rf output* +for INDX in $(seq $start $stop) +do + mkdir output_$INDX + cd output_$INDX + $mpi_executable -np 1 $hard_executable $config_file $additional_args -r $INDX + cd .. +done +make convergence CONFIG=$config_file + +rm -rf output* + diff --git a/app/tests/test_distributed.sh b/app/tests/test_distributed.sh index 33e12d2..c0c4fdd 100755 --- a/app/tests/test_distributed.sh +++ b/app/tests/test_distributed.sh @@ -7,41 +7,37 @@ config_file="$3" additional_args="$4" # Run the executable twice -rm *.raw -ls +rm -f *.csv echo "Starting.." OMP_NUM_THREADS=1 "$mpi_executable" -np 1 "$hard_executable" "$config_file" $additional_args echo "Mid..." # rename before getting overwritten by next command -mv output-00002-*-0.raw r1.raw -ls +mv output-*-0-00002.csv r1.csv OMP_NUM_THREADS=1 "$mpi_executable" -np 16 "$hard_executable" "$config_file" $additional_args -ls -# Stitch together 16 raw outputs to 1 (mp.raw) -> mp.raw +# Stitch together 16 csv outputs to 1 (mp.csv) +> mp.csv for i in {0..15}; do # Exclude header and first 3 columns - sed '1,5d;/^$/d' "output-00002-1D-${i}.raw" | awk 'BEGIN {FS="\t"; OFS="\t"} {print $4, $5, $6, $7}' >> mp.raw + sed '1d;/^$/d' "output-1D-${i}-00002.csv" | awk 'BEGIN {FS=","; OFS=","} {print $4, $5, $6, $7}' >> mp.csv done -# Create a temporary version of r1.raw excluding the header lines -sed '1,5d;/^$/d' r1.raw | awk 'BEGIN {FS="\t"; OFS="\t"} {print $4, $5, $6, $7}' > r1_temp.raw +# Create a temporary version of r1.csv excluding the header lines +sed '1d;/^$/d' r1.csv | awk 'BEGIN {FS=","; OFS=","} {print $4, $5, $6, $7}' > r1_temp.csv -diff mp.raw r1_temp.raw > /dev/null +diff mp.csv r1_temp.csv > /dev/null exit_status=$? if [ $exit_status -eq 0 ]; then echo "Files are the same." else echo "Files are different." - diff mp.raw r1_temp.raw + diff -u mp.csv r1_temp.csv fi # Clean up temporary files -#rm *.raw +rm -f *.csv # Exit with the status from diff (0 for no differences, 1 for differences) exit $exit_status - diff --git a/app/tests/test_verify.sh b/app/tests/test_verify.sh new file mode 100755 index 0000000..3fff25b --- /dev/null +++ b/app/tests/test_verify.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -euo pipefail + +# Path to the executable +mpi_executable="$1" +hard_executable="$2" +config_file="$3" +additional_args="$4" + +rm -rf *.csv +$mpi_executable -np 1 $hard_executable $config_file $additional_args && \ + make verify CONFIG="$config_file" +rm -rf *.csv + +if [ -d ./artifacts ]; then +# Save the plots as artifacts + mv *.png ./artifacts +fi diff --git a/app/types.hh b/app/types.hh index bb06c10..820d80f 100644 --- a/app/types.hh +++ b/app/types.hh @@ -17,6 +17,7 @@ using control = flecsi::run::control>; template typename S, std::size_t D> using control_policy = spec::control_policy; +using color_distribution = flecsi::topo::narray_impl::colors; using cp = spec::cp; template @@ -56,8 +57,8 @@ NS_WARN_SUPPRESS(bd, bd::low); More concise privileges. *----------------------------------------------------------------------------*/ -inline constexpr flecsi::partition_privilege_t na = spec::na, ro = spec::ro, - wo = spec::wo, rw = spec::rw; +inline constexpr flecsi::privilege na = spec::na, ro = spec::ro, wo = spec::wo, + rw = spec::rw; /*----------------------------------------------------------------------------* Dual field. @@ -65,7 +66,7 @@ inline constexpr flecsi::partition_privilege_t na = spec::na, ro = spec::ro, template struct dual_field { - using type = const field::template definition, is::cells>; + using type = const typename field::template definition, is::cells>; auto flip() { ++flip_; diff --git a/app/utils.hh b/app/utils.hh new file mode 100644 index 0000000..a3f309d --- /dev/null +++ b/app/utils.hh @@ -0,0 +1,32 @@ +#ifndef HARD_UTILS_HH +#define HARD_UTILS_HH + +#include "types.hh" + +namespace hard::util { + +/*! + Return the total number of colors across all axes from the given @em colors + reference. + + @tparam D The spatial dimension. + + @param cd The @em colors data structure of axis colors. + */ +template +std::size_t +axes_colors(color_distribution const & cd) { + if(D == 1) { + return cd[0]; + } + else if(D == 2) { + return cd[0] * cd[1]; + } + else /* D == 3 */ { + return cd[0] * cd[1] * cd[2]; + } +} // colors + +} // namespace hard::util + +#endif // HARD_UTILS_HH diff --git a/configs/scaling/size/rk-hydro-16.yaml b/configs/acoustic-wave.yaml similarity index 50% rename from configs/scaling/size/rk-hydro-16.yaml rename to configs/acoustic-wave.yaml index 4b4d895..aaa0535 100644 --- a/configs/scaling/size/rk-hydro-16.yaml +++ b/configs/acoustic-wave.yaml @@ -1,23 +1,29 @@ --- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 5 +problem: acoustic-wave gamma: 1.4 +kappa: 1.0 +mean_molecular_weight: 1.0 +problem_parameters: + scale: 1 + amplitude: 1e-5 + r0: 1.0 + p0: 1.0 +eos: ideal t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 +tf: 0.01 +max_steps: 100 +cfl: 0.5 max_dt: 1.0 -log_frequency: 5 -levels: [6, 5, 5] +log_frequency: 1 +output_frequency: 1 +levels: [12, 1, 1] +lowest_level: 12 coords: - [0.0, 0.0, 0.0] - [1.0, 1.0, 1.0] boundaries: - xlow: reflecting - xhigh: reflecting + xlow: periodic + xhigh: periodic ylow: flow yhigh: flow zlow: flow diff --git a/configs/heating-cooling-afld.yaml b/configs/heating-cooling-afld.yaml new file mode 100644 index 0000000..db0455d --- /dev/null +++ b/configs/heating-cooling-afld.yaml @@ -0,0 +1,40 @@ +--- + +problem: heating_and_cooling + +problem_parameters: + fluid_mass_density: 1.0e-11 + fluid_temperature: 1000 + radiation_temperature: 3000 + +kappa: 1.0 +gamma: 1.4 +mean_molecular_weight: 1.0 +eos: ideal +limiter_id: 3 +closure_id: 2 + +t0: 0.0 +tf: 40 +max_steps: 200 +cfl: 0.3 +max_dt: 1.0 +log_frequency: 1 +output_frequency: 1 +levels: [2, 1, 1] +coords: + - [0.0, 0.0, 0.0] + - [1e7, 1.0, 1.0] +boundaries: + xlow: periodic + xhigh: periodic + ylow: periodic + yhigh: periodic + zlow: periodic + zhigh: periodic +catalyst: + script: /path + implementation: paraview + implementation_directory: /path + +lowest_level: 2 diff --git a/configs/heating-cooling.yaml b/configs/heating-cooling.yaml index 3afceb7..f2d8e45 100644 --- a/configs/heating-cooling.yaml +++ b/configs/heating-cooling.yaml @@ -20,6 +20,7 @@ max_dt: 1.0 log_frequency: 1 output_frequency: 1 levels: [2, 1, 1] +lowest_level: 2 coords: - [0.0, 0.0, 0.0] - [1e7, 1.0, 1.0] @@ -33,6 +34,4 @@ boundaries: catalyst: script: /path implementation: paraview - implementation_directory: /path - -lowest_level: 2 + implementation_directory: /path \ No newline at end of file diff --git a/configs/kh.yaml b/configs/kh.yaml index 03a3ec9..1d287e8 100644 --- a/configs/kh.yaml +++ b/configs/kh.yaml @@ -3,6 +3,7 @@ problem: kh-test gamma: 1.6667 kappa: 0.0 mean_molecular_weight: 1.0 +eos: ideal t0: 0.0 tf: 1.0 max_steps: 10000 diff --git a/configs/kh_rad.yaml b/configs/kh_rad.yaml new file mode 100644 index 0000000..f4b3b68 --- /dev/null +++ b/configs/kh_rad.yaml @@ -0,0 +1,53 @@ +--- +problem: kh-rad-test + +problem_parameters: + fluid_mass_density_high: 2.0 + fluid_pressure_high: 2.5 + fluid_x_velocity_high: 0.5 + fluid_y_velocity_high: 0.0 + fluid_z_velocity_high: 0.0 + fluid_mass_density_low: 1.0 + fluid_pressure_low: 2.5 + fluid_x_velocity_low: -0.5 + fluid_y_velocity_low: 0.0 + fluid_z_velocity_low: 0.0 + fluid_sep_y_frac: 0.25 # discontinuity(ies) location + vel_per_y_frac: 0.1 # bound of domain which has velocity perturbation + perturb_N: 4.0 + rad_temp: 3000 +linear_solver: + maxiter: 100 + rtol: 1.0e-7 + use_zero_guess: True + +gamma: 1.6667 +adaptive_check: False +limiter_id: 0 +closure_id: 0 +kappa: 1.0 +mean_molecular_weight: 1.0 +eos: ideal +t0: 0.0 +tf: 1.0 +max_steps: 10000 +cfl: 0.3 +max_dt: 5.0e-2 +log_frequency: 1 +output_frequency: 100 +levels: [6, 6, 6] +lowest_level: 6 +coords: + - [0.0, 0.0, 0.0] + - [1.0, 1.0, 1.0] +boundaries: + xlow: periodic + xhigh: periodic + ylow: periodic + yhigh: periodic + zlow: flow + zhigh: flow +catalyst: + script: /path + implementation: paraview + implementation_directory: /path diff --git a/configs/scaling/size/rk-hydro-24.yaml b/configs/leblanc.yaml similarity index 63% rename from configs/scaling/size/rk-hydro-24.yaml rename to configs/leblanc.yaml index 97f5633..c32ef65 100644 --- a/configs/scaling/size/rk-hydro-24.yaml +++ b/configs/leblanc.yaml @@ -1,23 +1,24 @@ --- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 8 +problem: leblanc gamma: 1.4 +eos: ideal +kappa: 1.0e-100 +mean_molecular_weight: 1.0 t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 +tf: 0.2 +max_steps: 500 +cfl: 0.3 max_dt: 1.0 -log_frequency: 5 -levels: [8, 8, 8] +log_frequency: 500 +output_frequency: 500 +levels: [8, 3, 3] +lowest_level: 8 coords: - [0.0, 0.0, 0.0] - [1.0, 1.0, 1.0] boundaries: - xlow: reflecting - xhigh: reflecting + xlow: flow + xhigh: flow ylow: flow yhigh: flow zlow: flow diff --git a/configs/liska-wendroff.yaml b/configs/liska-wendroff.yaml index 0119897..50e5dec 100644 --- a/configs/liska-wendroff.yaml +++ b/configs/liska-wendroff.yaml @@ -3,6 +3,7 @@ problem: lw-implosion gamma: 1.4 kappa: 1.0e-100 mean_molecular_weight: 1.0 +eos: ideal t0: 0.0 tf: 10.0 max_steps: 20000 @@ -23,4 +24,4 @@ boundaries: catalyst: script: /path implementation: paraview - implementation_directory: /path \ No newline at end of file + implementation_directory: /path diff --git a/configs/rk.yaml b/configs/rk.yaml index d7a3964..82dd343 100644 --- a/configs/rk.yaml +++ b/configs/rk.yaml @@ -1,17 +1,18 @@ --- problem: rankine-hugoniot +eos: ideal gamma: 1.4 kappa: 1.0e-100 mean_molecular_weight: 1.0 t0: 0.0 -tf: 1.0 -max_steps: 2 +tf: 0.2 +max_steps: 500 cfl: 0.3 max_dt: 1.0 -log_frequency: 1 -output_frequency: 10 -levels: [3, 3, 3] -lowest_level: 3 +log_frequency: 1000 +output_frequency: 1000 +levels: [8, 3, 3] +lowest_level: 8 coords: - [0.0, 0.0, 0.0] - [1.0, 1.0, 1.0] diff --git a/configs/rm.yaml b/configs/rm.yaml new file mode 100644 index 0000000..646b1f7 --- /dev/null +++ b/configs/rm.yaml @@ -0,0 +1,48 @@ +--- +problem: richtmyer-meshkov +gamma: 1.4 +kappa: 1.0 +adaptive_check: False +limiter_id: 0 +closure_id: 0 +mean_molecular_weight: 1.0 +eos: ideal +t0: 0.0 +tf: 1.0e-2 +max_steps: 50000 +cfl: 0.5 +max_dt: 1.0e-2 +log_frequency: 1000 +output_frequency: 1000 +levels: [7,11,4] +lowest_level: 9 +#color_distribution: [8,16,1] +linear_solver: + maxiter: 600 + rtol: 1.0e-10 + use_zero_guess: True +problem_parameters: + density_above_shock: 1.5894 + pressure_above_shock: 153338.5 + velocity_above_shock: -105.7312 + density_low: 1.1847 + density_high: 5.494 + pressure: 101325 + interface: 0.08 + perturbation: -5.0e-4 +radiation_parameters: + rad_temp: 3000.0 +coords: + - [0.0, 0.0, 0.0] + - [0.01, 0.16, 0.0] +boundaries: + xlow: periodic + xhigh: periodic + ylow: reflecting + yhigh: flow + zlow: flow + zhigh: flow +catalyst: + script: ../tools/gridwriter.py + implementation: paraview + implementation_directory: /vast/home/icbannerman/.spack/var/spack/environments/hard_paraview/.spack-env/view/lib64/catalyst \ No newline at end of file diff --git a/configs/rt.yaml b/configs/rt.yaml new file mode 100644 index 0000000..50ede57 --- /dev/null +++ b/configs/rt.yaml @@ -0,0 +1,33 @@ +--- +problem: rt-test +gamma: 1.4 +kappa: 0.0 +adaptive_check: False +limiter_id: 0 +closure_id: 0 +gravity_acc: [0.0, -0.1, 0.0] +mean_molecular_weight: 1.0 +eos: ideal +t0: 0.0 +tf: 50.0 +max_steps: 100000 +cfl: 0.75 +max_dt: 5.0e-2 +log_frequency: 1 +output_frequency: 10000 +levels: [8, 8, 4] +lowest_level: 8 +coords: + - [0.0, 0.0, 0.0] # - [-0.25, -0.75, 0.0] + - [0.5, 1.5, 1.0] # - [0.25, 0.75, 1.0] +boundaries: + xlow: periodic # needs to be changed to Homogeneous Neumann + xhigh: periodic # needs to be changed to Homogeneous Neumann + ylow: reflecting + yhigh: reflecting + zlow: flow + zhigh: flow +catalyst: + script: /path + implementation: paraview + implementation_directory: /path diff --git a/configs/scaling/size/rk-hydro-15.yaml b/configs/scaling/size/rk-hydro-15.yaml deleted file mode 100644 index 5993f1b..0000000 --- a/configs/scaling/size/rk-hydro-15.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 5 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [5, 5, 5] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-hydro-17.yaml b/configs/scaling/size/rk-hydro-17.yaml deleted file mode 100644 index 35d8eed..0000000 --- a/configs/scaling/size/rk-hydro-17.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 5 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [6, 6, 5] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-hydro-18.yaml b/configs/scaling/size/rk-hydro-18.yaml deleted file mode 100644 index fc06233..0000000 --- a/configs/scaling/size/rk-hydro-18.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 6 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [6, 6, 6] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-hydro-19.yaml b/configs/scaling/size/rk-hydro-19.yaml deleted file mode 100644 index 9de3a9f..0000000 --- a/configs/scaling/size/rk-hydro-19.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 6 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [7, 6, 6] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-hydro-20.yaml b/configs/scaling/size/rk-hydro-20.yaml deleted file mode 100644 index ba1a54c..0000000 --- a/configs/scaling/size/rk-hydro-20.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 6 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [7, 7, 6] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-hydro-21.yaml b/configs/scaling/size/rk-hydro-21.yaml deleted file mode 100644 index 84cf908..0000000 --- a/configs/scaling/size/rk-hydro-21.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 7 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [7, 7, 7] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-hydro-22.yaml b/configs/scaling/size/rk-hydro-22.yaml deleted file mode 100644 index 521a8bb..0000000 --- a/configs/scaling/size/rk-hydro-22.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 7 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [8, 7, 7] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-hydro-23.yaml b/configs/scaling/size/rk-hydro-23.yaml deleted file mode 100644 index 82a94ea..0000000 --- a/configs/scaling/size/rk-hydro-23.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 7 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [8, 8, 7] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-hydro-25.yaml b/configs/scaling/size/rk-hydro-25.yaml deleted file mode 100644 index 39a5859..0000000 --- a/configs/scaling/size/rk-hydro-25.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 8 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [9, 8, 8] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-hydro-26.yaml b/configs/scaling/size/rk-hydro-26.yaml deleted file mode 100644 index d966a9f..0000000 --- a/configs/scaling/size/rk-hydro-26.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 8 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [9, 9, 8] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-hydro-27.yaml b/configs/scaling/size/rk-hydro-27.yaml deleted file mode 100644 index 35a8a45..0000000 --- a/configs/scaling/size/rk-hydro-27.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 9 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [9, 9, 9] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-hydro-28.yaml b/configs/scaling/size/rk-hydro-28.yaml deleted file mode 100644 index a86c42f..0000000 --- a/configs/scaling/size/rk-hydro-28.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 9 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [10, 9, 9] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-hydro-29.yaml b/configs/scaling/size/rk-hydro-29.yaml deleted file mode 100644 index e0d670d..0000000 --- a/configs/scaling/size/rk-hydro-29.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 9 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [10, 10, 9] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-hydro-30.yaml b/configs/scaling/size/rk-hydro-30.yaml deleted file mode 100644 index dae655c..0000000 --- a/configs/scaling/size/rk-hydro-30.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 10 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [10, 10, 10] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-rad-15.yaml b/configs/scaling/size/rk-rad-15.yaml deleted file mode 100644 index a068efe..0000000 --- a/configs/scaling/size/rk-rad-15.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [5, 5, 5] -lowest_level: 5 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-rad-16.yaml b/configs/scaling/size/rk-rad-16.yaml deleted file mode 100644 index 97f6948..0000000 --- a/configs/scaling/size/rk-rad-16.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [6, 5, 5] -lowest_level: 6 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-rad-17.yaml b/configs/scaling/size/rk-rad-17.yaml deleted file mode 100644 index e6d96a3..0000000 --- a/configs/scaling/size/rk-rad-17.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [6, 6, 5] -lowest_level: 6 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-rad-18.yaml b/configs/scaling/size/rk-rad-18.yaml deleted file mode 100644 index fbaebf1..0000000 --- a/configs/scaling/size/rk-rad-18.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [6, 6, 6] -lowest_level: 6 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-rad-19.yaml b/configs/scaling/size/rk-rad-19.yaml deleted file mode 100644 index 34e9449..0000000 --- a/configs/scaling/size/rk-rad-19.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [7, 6, 6] -lowest_level: 7 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-rad-20.yaml b/configs/scaling/size/rk-rad-20.yaml deleted file mode 100644 index 04209ff..0000000 --- a/configs/scaling/size/rk-rad-20.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [7, 7, 6] -lowest_level: 7 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-rad-21.yaml b/configs/scaling/size/rk-rad-21.yaml deleted file mode 100644 index 70c9cfa..0000000 --- a/configs/scaling/size/rk-rad-21.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [7, 7, 7] -lowest_level: 7 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-rad-22.yaml b/configs/scaling/size/rk-rad-22.yaml deleted file mode 100644 index d43a4be..0000000 --- a/configs/scaling/size/rk-rad-22.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [8, 7, 7] -lowest_level: 8 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-rad-23.yaml b/configs/scaling/size/rk-rad-23.yaml deleted file mode 100644 index 3ea8756..0000000 --- a/configs/scaling/size/rk-rad-23.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [8, 8, 7] -lowest_level: 8 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-rad-24.yaml b/configs/scaling/size/rk-rad-24.yaml deleted file mode 100644 index 4e19cf9..0000000 --- a/configs/scaling/size/rk-rad-24.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [8, 8, 8] -lowest_level: 8 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-rad-25.yaml b/configs/scaling/size/rk-rad-25.yaml deleted file mode 100644 index 2c389e7..0000000 --- a/configs/scaling/size/rk-rad-25.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [9, 8, 8] -lowest_level: 9 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-rad-26.yaml b/configs/scaling/size/rk-rad-26.yaml deleted file mode 100644 index 2122251..0000000 --- a/configs/scaling/size/rk-rad-26.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [9, 9, 8] -lowest_level: 9 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-rad-27.yaml b/configs/scaling/size/rk-rad-27.yaml deleted file mode 100644 index c576712..0000000 --- a/configs/scaling/size/rk-rad-27.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [9, 9, 9] -lowest_level: 9 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-rad-28.yaml b/configs/scaling/size/rk-rad-28.yaml deleted file mode 100644 index 04a1573..0000000 --- a/configs/scaling/size/rk-rad-28.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [10, 9, 9] -lowest_level: 10 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-rad-29.yaml b/configs/scaling/size/rk-rad-29.yaml deleted file mode 100644 index 12a025a..0000000 --- a/configs/scaling/size/rk-rad-29.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [10, 10, 9] -lowest_level: 10 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/size/rk-rad-30.yaml b/configs/scaling/size/rk-rad-30.yaml deleted file mode 100644 index 4ff4364..0000000 --- a/configs/scaling/size/rk-rad-30.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [10, 10, 10] -lowest_level: 10 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/strong/rk-1d-hydro.yaml b/configs/scaling/strong/rk-1d-hydro.yaml deleted file mode 100644 index 3023d69..0000000 --- a/configs/scaling/strong/rk-1d-hydro.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 30 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [30, 1, 1] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/strong/rk-1d-rad.yaml b/configs/scaling/strong/rk-1d-rad.yaml deleted file mode 100644 index b6aff40..0000000 --- a/configs/scaling/strong/rk-1d-rad.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [27, 1, 1] -lowest_level: 27 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/strong/rk-2d-hydro.yaml b/configs/scaling/strong/rk-2d-hydro.yaml deleted file mode 100644 index 451cfcd..0000000 --- a/configs/scaling/strong/rk-2d-hydro.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 14 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [15, 15, 1] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/strong/rk-2d-rad.yaml b/configs/scaling/strong/rk-2d-rad.yaml deleted file mode 100644 index ef41746..0000000 --- a/configs/scaling/strong/rk-2d-rad.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [14, 13, 1] -lowest_level: 1 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/strong/rk-3d-hydro.yaml b/configs/scaling/strong/rk-3d-hydro.yaml deleted file mode 100644 index dae655c..0000000 --- a/configs/scaling/strong/rk-3d-hydro.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 10 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [10, 10, 10] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/strong/rk-3d-rad.yaml b/configs/scaling/strong/rk-3d-rad.yaml deleted file mode 100644 index fea3033..0000000 --- a/configs/scaling/strong/rk-3d-rad.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [9, 9, 9] -lowest_level: 9 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/weak/rk-hydro-1.yaml b/configs/scaling/weak/rk-hydro-1.yaml deleted file mode 100644 index 97f5633..0000000 --- a/configs/scaling/weak/rk-hydro-1.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 8 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [8, 8, 8] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/weak/rk-hydro-1024.yaml b/configs/scaling/weak/rk-hydro-1024.yaml deleted file mode 100644 index 61d7a96..0000000 --- a/configs/scaling/weak/rk-hydro-1024.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 11 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [12, 11, 11] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/weak/rk-hydro-128.yaml b/configs/scaling/weak/rk-hydro-128.yaml deleted file mode 100644 index 96a836f..0000000 --- a/configs/scaling/weak/rk-hydro-128.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 10 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [11, 10, 10] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/weak/rk-hydro-16.yaml b/configs/scaling/weak/rk-hydro-16.yaml deleted file mode 100644 index a0e3c66..0000000 --- a/configs/scaling/weak/rk-hydro-16.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 5 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [10, 9, 9] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/weak/rk-hydro-2.yaml b/configs/scaling/weak/rk-hydro-2.yaml deleted file mode 100644 index 39a5859..0000000 --- a/configs/scaling/weak/rk-hydro-2.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 8 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [9, 8, 8] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/weak/rk-hydro-256.yaml b/configs/scaling/weak/rk-hydro-256.yaml deleted file mode 100644 index dd3b95c..0000000 --- a/configs/scaling/weak/rk-hydro-256.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 10 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [11, 11, 10] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/weak/rk-hydro-32.yaml b/configs/scaling/weak/rk-hydro-32.yaml deleted file mode 100644 index e0d670d..0000000 --- a/configs/scaling/weak/rk-hydro-32.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 9 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [10, 10, 9] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/weak/rk-hydro-4.yaml b/configs/scaling/weak/rk-hydro-4.yaml deleted file mode 100644 index d966a9f..0000000 --- a/configs/scaling/weak/rk-hydro-4.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 8 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [9, 9, 8] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/weak/rk-hydro-512.yaml b/configs/scaling/weak/rk-hydro-512.yaml deleted file mode 100644 index 3af5ed2..0000000 --- a/configs/scaling/weak/rk-hydro-512.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 11 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [11, 11, 11] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/weak/rk-hydro-64.yaml b/configs/scaling/weak/rk-hydro-64.yaml deleted file mode 100644 index dae655c..0000000 --- a/configs/scaling/weak/rk-hydro-64.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 10 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [10, 10, 10] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/weak/rk-hydro-8.yaml b/configs/scaling/weak/rk-hydro-8.yaml deleted file mode 100644 index 35a8a45..0000000 --- a/configs/scaling/weak/rk-hydro-8.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rankine-hugoniot -kappa: 0.0 -mean_molecular_weight: 1.0 -output_frequency: 10 -lowest_level: 9 -gamma: 1.4 -t0: 0.0 -tf: 1.0 -max_steps: 5 -cfl: 0.6 -max_dt: 1.0 -log_frequency: 5 -levels: [9, 9, 9] -coords: - - [0.0, 0.0, 0.0] - - [1.0, 1.0, 1.0] -boundaries: - xlow: reflecting - xhigh: reflecting - ylow: flow - yhigh: flow - zlow: flow - zhigh: flow -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/weak/rk-rad-1.yaml b/configs/scaling/weak/rk-rad-1.yaml deleted file mode 100644 index 46743d1..0000000 --- a/configs/scaling/weak/rk-rad-1.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [8, 8, 8] -lowest_level: 8 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/weak/rk-rad-1024.yaml b/configs/scaling/weak/rk-rad-1024.yaml deleted file mode 100644 index 60465cf..0000000 --- a/configs/scaling/weak/rk-rad-1024.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [12, 11, 11] -lowest_level: 12 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/weak/rk-rad-128.yaml b/configs/scaling/weak/rk-rad-128.yaml deleted file mode 100644 index 28b72e3..0000000 --- a/configs/scaling/weak/rk-rad-128.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [11, 10, 10] -lowest_level: 11 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/weak/rk-rad-16.yaml b/configs/scaling/weak/rk-rad-16.yaml deleted file mode 100644 index ac867f6..0000000 --- a/configs/scaling/weak/rk-rad-16.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [10, 9, 9] -lowest_level: 10 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/weak/rk-rad-2.yaml b/configs/scaling/weak/rk-rad-2.yaml deleted file mode 100644 index 21a9bd0..0000000 --- a/configs/scaling/weak/rk-rad-2.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [9, 8, 8] -lowest_level: 9 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/weak/rk-rad-256.yaml b/configs/scaling/weak/rk-rad-256.yaml deleted file mode 100644 index cae0dd2..0000000 --- a/configs/scaling/weak/rk-rad-256.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [11, 11, 10] -lowest_level: 11 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/weak/rk-rad-32.yaml b/configs/scaling/weak/rk-rad-32.yaml deleted file mode 100644 index 0355144..0000000 --- a/configs/scaling/weak/rk-rad-32.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [10, 10, 9] -lowest_level: 10 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/weak/rk-rad-4.yaml b/configs/scaling/weak/rk-rad-4.yaml deleted file mode 100644 index 6ccbf3a..0000000 --- a/configs/scaling/weak/rk-rad-4.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [9, 9, 8] -lowest_level: 9 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/weak/rk-rad-512.yaml b/configs/scaling/weak/rk-rad-512.yaml deleted file mode 100644 index 29bf9dc..0000000 --- a/configs/scaling/weak/rk-rad-512.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [11, 11, 11] -lowest_level: 11 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/weak/rk-rad-64.yaml b/configs/scaling/weak/rk-rad-64.yaml deleted file mode 100644 index 02367ec..0000000 --- a/configs/scaling/weak/rk-rad-64.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [10, 10, 10] -lowest_level: 10 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/scaling/weak/rk-rad-8.yaml b/configs/scaling/weak/rk-rad-8.yaml deleted file mode 100644 index fea3033..0000000 --- a/configs/scaling/weak/rk-rad-8.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -problem: rad-rh -gamma: 1.4 -kappa: 0.4 -mean_molecular_weight: 0.5 -t0: 0.0 -tf: 5.0e-3 -max_steps: 5 -cfl: 0.3 -max_dt: 5.0e-13 -log_frequency: 1 -output_frequency: 10000 -levels: [9, 9, 9] -lowest_level: 9 -coords: - - [0.0, 0.0, 0.0] - - [1.0e5, 1.0e5, 1.0e5] -boundaries: - xlow: flow - xhigh: flow - ylow: periodic - yhigh: periodic - zlow: periodic - zhigh: periodic -catalyst: - script: /path - implementation: paraview - implementation_directory: /path diff --git a/configs/sedov.yaml b/configs/sedov.yaml index 6ad203e..13c7862 100644 --- a/configs/sedov.yaml +++ b/configs/sedov.yaml @@ -5,24 +5,25 @@ kappa: 1.0e0 mean_molecular_weight: 0.5 eos: ideal t0: 0.0 -tf: 1.e1 -max_steps: 10 +tf: 0.5 +max_steps: 500 cfl: 0.3 -max_dt: 5.0e-08 -log_frequency: 1 -output_frequency: 1 -levels: [8, 8, 1] -lowest_level: 8 +max_dt: 1.0 +log_frequency: 1000 +output_frequency: 1000 +levels: [10, 1, 1] +lowest_level: 10 problem_parameters: + E_0: 0.0673185 hotspot_position: - - [2.0e5, 2.0e5, 0.0] - hotspot_radius: 2.0e4 + - [0., 0., 0.] + hotspot_radius: 0.01 coords: - [0.0, 0.0, 0.0] - - [4.0e5, 4.0e5, 1.0e4] + - [1.0, 1.0, 1.0] boundaries: - xlow: flow - xhigh: flow + xlow: reflecting + xhigh: reflecting ylow: flow yhigh: flow zlow: flow diff --git a/configs/sod.yaml b/configs/sod.yaml index 4912166..83e285e 100644 --- a/configs/sod.yaml +++ b/configs/sod.yaml @@ -2,25 +2,23 @@ problem: sod gamma: 1.4 eos: ideal -spiner_file: titanium.sp5 -spiner_matid: 2961 kappa: 1.0e-100 mean_molecular_weight: 1.0 t0: 0.0 -tf: 1.0 -max_steps: 5000 +tf: 0.2 +max_steps: 500 cfl: 0.3 max_dt: 1.0 -log_frequency: 1000 -output_frequency: 1000 -levels: [10, 5, 5] -lowest_level: 10 +log_frequency: 500 +output_frequency: 500 +levels: [8, 3, 3] +lowest_level: 8 coords: - [0.0, 0.0, 0.0] - [1.0, 1.0, 1.0] boundaries: - xlow: reflecting - xhigh: reflecting + xlow: flow + xhigh: flow ylow: flow yhigh: flow zlow: flow diff --git a/configs/distr-rk.yaml b/configs/sod_tabulated.yaml similarity index 65% rename from configs/distr-rk.yaml rename to configs/sod_tabulated.yaml index 4ec3a7c..4bb4cfd 100644 --- a/configs/distr-rk.yaml +++ b/configs/sod_tabulated.yaml @@ -1,16 +1,18 @@ --- -problem: rankine-hugoniot -gamma: 1.4 -kappa: 1.0 -mean_molecular_weight: 1.0 +problem: sod +eos: spiner +spiner_file: air.sp5 +spiner_matid: 5030 +kappa: 1.0e-100 +mean_molecular_weight: 28.96 t0: 0.0 tf: 1.0 -max_steps: 2 +max_steps: 100 cfl: 0.3 max_dt: 1.0 log_frequency: 1 -output_frequency: 2 -levels: [5, 5, 5] +output_frequency: 1 +levels: [5, 3, 3] lowest_level: 5 coords: - [0.0, 0.0, 0.0] diff --git a/doc/doxygen/common.conf b/doc/doxygen/common.conf deleted file mode 100644 index b095c27..0000000 --- a/doc/doxygen/common.conf +++ /dev/null @@ -1,37 +0,0 @@ -ALWAYS_DETAILED_SEC = YES -JAVADOC_AUTOBRIEF = YES -QT_AUTOBRIEF = YES -DISTRIBUTE_GROUP_DOC = YES -EXTRACT_PRIV_VIRTUAL = YES -HIDE_UNDOC_MEMBERS = YES -HIDE_UNDOC_CLASSES = YES -HIDE_SCOPE_NAMES = YES -FORCE_LOCAL_INCLUDES = YES -INLINE_INFO = NO -SORT_MEMBER_DOCS = NO -SORT_BRIEF_DOCS = YES -MAX_INITIALIZER_LINES = 0 -SHOW_USED_FILES = NO -SHOW_FILES = NO -QUIET = YES -WARN_IF_UNDOCUMENTED = NO -RECURSIVE = YES -VERBATIM_HEADERS = NO -ALPHABETICAL_INDEX = NO -DISABLE_INDEX = YES -GENERATE_HTML = NO -GENERATE_XML = YES -XML_PROGRAMLISTING = YES -XML_NS_MEMB_FILE_SCOPE = NO -GENERATE_TREEVIEW = YES -GENERATE_LATEX = NO -MACRO_EXPANSION = YES -EXPAND_ONLY_PREDEF = YES -HAVE_DOT = YES -CLASS_GRAPH = NO -COLLABORATION_GRAPH = NO -GROUP_GRAPHS = NO -INCLUDE_GRAPH = NO -INCLUDED_BY_GRAPH = NO -GRAPHICAL_HIERARCHY = NO -DIRECTORY_GRAPH = NO diff --git a/doc/doxygen/conf.in b/doc/doxygen/conf.in deleted file mode 100644 index aa533cb..0000000 --- a/doc/doxygen/conf.in +++ /dev/null @@ -1,25 +0,0 @@ -#------------------------------------------------------------------------------# -# Values specific to this project. -#------------------------------------------------------------------------------# - -PROJECT_LOGO = @PROJECT_SOURCE_DIR@/doc/doxygen/flecsi.png -PROJECT_NUMBER = "Version: ${${PROJECT_NAME}_VERSION}" -OUTPUT_DIRECTORY = @CMAKE_BINARY_DIR@/.doxygen -XML_OUTPUT = xml -IMAGE_PATH = @PROJECT_SOURCE_DIR@/doc/doxygen/images -STRIP_FROM_PATH = @PROJECT_SOURCE_DIR@/spec \ - @CMAKE_BINARY_DIR@ -STRIP_FROM_INC_PATH = @PROJECT_SOURCE_DIR@/spec \ - @CMAKE_BINARY_DIR@ -INPUT = @PROJECT_SOURCE_DIR@/spec \ - @PROJECT_SOURCE_DIR@/doc/doxygen/mainpage.md -USE_MDFILE_AS_MAINPAGE = @PROJECT_SOURCE_DIR@/doc/doxygen/mainpage.md - -EXCLUDE_PATTERNS = @PROJECT_SOURCE_DIR@/spec/*.md \ - */test/* -PREDEFINED = DOXYGEN \ - FLECSI_INLINE_TARGET= -WARN_LOGFILE = @CMAKE_BINARY_DIR@/api-@name@.out - -@INCLUDE = @PROJECT_SOURCE_DIR@/doc/doxygen/common.conf -@INCLUDE = @CMAKE_BINARY_DIR@/.doxygen/@name@/@name@.conf diff --git a/doc/doxygen/flastro.conf.in b/doc/doxygen/flastro.conf.in deleted file mode 100644 index 1c1cea0..0000000 --- a/doc/doxygen/flastro.conf.in +++ /dev/null @@ -1,5 +0,0 @@ -#------------------------------------------------------------------------------# -# Values specific to this target. -#------------------------------------------------------------------------------# - -PROJECT_NAME = "HARD User API" diff --git a/doc/doxygen/flecsi.png b/doc/doxygen/flecsi.png deleted file mode 100644 index 626427a..0000000 Binary files a/doc/doxygen/flecsi.png and /dev/null differ diff --git a/doc/doxygen/images/narray-layout.svg b/doc/doxygen/images/narray-layout.svg deleted file mode 100644 index 3533eb4..0000000 --- a/doc/doxygen/images/narray-layout.svg +++ /dev/null @@ -1,202 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - low - - - - - - - - - - logical [begin, end) - - - - extended [begin, end) - - - - extents - - - - boundary - - - - ghost - - - - global offset - - - - interior - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - logical [begin, end) - - - - extended [begin, end) - - - - extents - - - - ghost - - - - ghost - - - - global offset - - - - high - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - logical [begin, end) - - - - extended [begin, end) - - - - extents - - - - ghost - - - - boundary - - - - global offset - - - - - - logical - - - - extended - - - - boundary - - - - ghost - - - - diff --git a/doc/doxygen/mainpage.md b/doc/doxygen/mainpage.md deleted file mode 100644 index 516409f..0000000 --- a/doc/doxygen/mainpage.md +++ /dev/null @@ -1,3 +0,0 @@ -This document describes the FVM specialization API. - - diff --git a/doc/fmg.svg b/doc/fmg.svg new file mode 100644 index 0000000..9e383c1 --- /dev/null +++ b/doc/fmg.svg @@ -0,0 +1 @@ +Full multigrid \ No newline at end of file diff --git a/doc/sphinx/_static/hard-sp.png b/doc/sphinx/_static/hard-sp.png new file mode 100644 index 0000000..fffdf6f Binary files /dev/null and b/doc/sphinx/_static/hard-sp.png differ diff --git a/doc/sphinx/conf.py.in b/doc/sphinx/conf.py.in index b57a83c..ef487d6 100644 --- a/doc/sphinx/conf.py.in +++ b/doc/sphinx/conf.py.in @@ -3,14 +3,9 @@ # Sphinx extensions as list of strings extensions = [ 'sphinx.ext.mathjax', - 'sphinx.ext.githubpages', - 'breathe' + 'sphinx.ext.githubpages' ] -# Breathe configuration -breathe_projects = {"doxygen": "@CMAKE_BINARY_DIR@/.doxygen/xml"} -breathe_default_project = "doxygen" - # Add any paths that contain templates templates_path = ['_templates'] @@ -76,7 +71,6 @@ html_theme_options = { 'canonical_url': 'http://www.flecsi.org/', 'analytics_id': '', 'logo_only': True, - 'display_version': True, 'prev_next_buttons_location': 'bottom', 'style_external_links': False, @@ -88,7 +82,7 @@ html_theme_options = { 'titles_only': False } -html_logo = '_static/flecsi-sp.png' +html_logo = '_static/hard-sp.png' html_favicon = '_static/favicon.ico' html_sidebars = { diff --git a/doc/sphinx/hard/afld.rst b/doc/sphinx/hard/afld.rst new file mode 100644 index 0000000..684ff51 --- /dev/null +++ b/doc/sphinx/hard/afld.rst @@ -0,0 +1,193 @@ +.. |br| raw:: html + +
+ +.. _hard_afld: + +AFLD +**** + +**Author:** Moon Bakaya Hazarika +**Date:** August 2025 + +Overview of Variables +===================== + +Three user-input variables have been implemented as part of AFLD: + +1. :code:`limiter_id`: Integer varying from 0 to 4. Corresponds to the limiter relation to be used. The default is 1. +2. :code:`closure_id`: Integer varying from 0 to 4. Corresponds to the closure relation to be used. The default is 3. + +Besides this, the **Eddington Factor** field was implemented to store the values returned by invoking the relation specified in :code:`closure_id`. + +Limiter Relations +===================== + +Here :math:`R` is the energy gradient ratio given as input to the function, along with the :code:`limiter_id`. +:math:`\lambda` is the limited flux value returned as output by the function. + +.. math:: + + R = \frac{|\nabla E_r|}{\kappa \rho E_r} + +``limiter_id = 0`` +------------------ + +Fixed value :math:`\frac{1}{3}` + +.. math:: + + \lambda = \frac{1}{3} + +``limiter_id = 1`` +------------------ + +Approximate Levermore–Pomraning limiter: + +.. math:: + + \lambda = \frac{2 + R}{6 + 3R + R^2} + +``limiter_id = 2`` +------------------ + +Bruenn limiter: + +.. math:: + + \lambda = \frac{1}{3 + R} + +``limiter_id = 3`` +------------------ + +Larsen’s Square Root limiter: + +.. math:: + + \lambda = \frac{1}{\sqrt{9 + R^2}} + +``limiter_id = 4`` +------------------ + +Minerbo limiter: + +.. math:: + + \lambda = + \begin{cases} + \dfrac{2}{3 + \sqrt{9 + 12R^2}}, & R < 1.5 \\ + \dfrac{1}{1 + R + \sqrt{1 + 2R}}, & \text{otherwise} + \end{cases} + +Closure Relations +===================== + +Here :math:`\lambda` is the flux value given as input to the function, along with the :code:`limiter_id` and :code:`closure_id`. +:math:`E` is the Eddington Factor returned as output. + +``closure_id = 0`` +------------------ + +.. math:: + + E = \lambda + +``closure_id = 1`` +------------------ + +.. math:: + + E = \frac{1}{3} + +``closure_id = 2`` +------------------ + +.. math:: + + E = 1 - 2\lambda + +``closure_id = 3`` +------------------ + +Analogous to Levermore–Pomraning type, i.e. :math:`E = \lambda + (\lambda R)^2` + +- ``limiter_id = 0`` + +.. math:: + + E = \frac{1}{3} + +- ``limiter_id = 1`` + +.. math:: + + t = \frac{1}{2}\max(0, 1-3\lambda) + \sqrt{\max(0, (1-3\lambda)(1+5\lambda))} + +.. math:: + + E = \lambda + t^2 + +- ``limiter_id = 2`` + +.. math:: + + E = 1 - 5\lambda + 9\lambda^2 + +- ``limiter_id = 3`` + +.. math:: + + E = 1 + \lambda - 9\lambda^2 + +- ``limiter_id = 4`` + +.. math:: + + E = + \begin{cases} + 1 + 3\lambda - 2\sqrt{2\lambda}, & E < \frac{2}{9} \\ + \dfrac{1}{3}, & \text{otherwise} + \end{cases} + +``closure_id = 4`` +------------------ + +Modification of Levermore–Pomraning type: :math:`E = \frac{1}{3} + \frac{2}{3}(\lambda R)^2` + +- ``limiter_id = 0`` + +.. math:: + + E = \frac{1}{3} + +- ``limiter_id = 1`` + +.. math:: + + t = \max(0, 1-3\lambda) + \sqrt{\max(0, (1-3\lambda)(1+5\lambda))} + +.. math:: + + E = \frac{1}{3} + \frac{t^2}{6} + +- ``limiter_id = 2`` + +.. math:: + + E = \frac{1}{3} + \frac{2}{3}(1 - 6\lambda + 9\lambda^2) + +- ``limiter_id = 3`` + +.. math:: + + E = \frac{1}{3} + \frac{2}{3}(1 - 9\lambda^2) + +- ``limiter_id = 4`` + +.. math:: + + E = + \begin{cases} + \dfrac{1}{3} + \dfrac{2}{3}\left(1 + 2\lambda - 2\sqrt{2\lambda}\right), & E < \frac{2}{9} \\ + \dfrac{5}{9} - \dfrac{2}{3}\lambda, & \text{otherwise} + \end{cases} diff --git a/doc/sphinx/hard/api.rst b/doc/sphinx/hard/api.rst deleted file mode 100644 index a6c1757..0000000 --- a/doc/sphinx/hard/api.rst +++ /dev/null @@ -1,13 +0,0 @@ -.. |br| raw:: html - -
- -.. _hard_api: - -HARD Specialization API Reference -************************************ - -.. doxygenstruct:: spec::mesh - :members: - -.. vim: set tabstop=2 shiftwidth=2 expandtab fo=cqt tw=72 : diff --git a/doc/sphinx/hard/equations.rst b/doc/sphinx/hard/equations.rst new file mode 100644 index 0000000..f06131f --- /dev/null +++ b/doc/sphinx/hard/equations.rst @@ -0,0 +1,107 @@ +.. |br| raw:: html + +
+ +.. _hard_equations: + + +Basic Equations +*************** + +We follow the comoving frame formulation of the radiation hydrodynamics equations in the diffusion limit, which can be written in Einstein notation in the form + +.. math:: + \partial_t \mathbf{U} + \partial_j \mathbf{F}^j = \mathbf{S} + +with the evolved variables + +.. math:: + \mathbf{U} = \left[ \rho, \rho v^i, e + \frac{1}{2} \rho v^2, E \right]^T, + +fluxes + +.. math:: + \mathbf{F}^j = \left[ \begin{matrix} + \rho v^j \\ + \rho v^i v^j + p \delta_{ij} \\ + \left(e + \frac{1}{2} \rho v^2 + p\right) v^j \\ + E v^j \\ + \end{matrix} \right], + +and source terms + +.. math:: + \mathbf{S} = \left[ \begin{matrix} + 0 \\ + f^i \\ + f^i v_i + \dot{q} \\ + - \partial_i F_r^i - P_r^{ij} \nabla_i v_j - \dot{q} \\ + \end{matrix} \right], + +where :math:`\rho` is the fluid mass density, :math:`v^i` is the component *i* of the fluid velocity vector, :math:`e` is the fluid internal energy density, :math:`p` is the fluid pressure, :math:`E` is radiation energy density, :math:`F_r^i` is the *i*-th component of the radiation energy flux vector, :math:`P_r^{ij}` is the *i,j* element of the radiation pressure tensor, :math:`f^i` is the *i*-th component of the radiation force density vector + +.. math:: + f^i = \frac{\kappa \rho}{c} F_r^i , + +and :math:`\dot{q}` is the radiative heating rate + +.. math:: + \dot{q} = \frac{dq}{dt} = c \kappa \rho (E - a T^4) , + +where :math:`\kappa` is mean opacity, :math:`c` is the speed of light, :math:`a` is the radiation constant, and :math:`T` is the fluid temperature. +For closure of the evolution system for the fluid, we need to specify the functional form of the fluid pressure :math:`p`. Here we adopt the singularity-EOS, allowing us to use any material in evolution. + +Flux Limited Diffusion +~~~~~~~~~~~~~~~~~~~~~~~~ + +The radiative transfer equation — the last component of the evolution equation :math:`\partial_t \mathbf{U} + \partial_j \mathbf{F}^j = \mathbf{S}` — requires additional closures for radiation energy flux density :math:`F^i` and radiation pressure :math:`P_r^{ij}`. + +In the optically thick diffusion limit, they are given by: + +.. math:: + F^i = - \frac{c}{3\kappa\rho} \nabla E, + +.. math:: + P_r^{ij} = \frac{1}{3} E \delta_{ij}, + +provided the mean free path of photons is much shorter than the length scale of interest. + +Evolution equations are valid in the diffusion (optically thick) limit. However, it is possible to introduce a bridge law so that the radiation energy flux density :math:`F_r^i` recovers the correct norm in optically thin regions. + +.. math:: + F_r^i = - \lambda \frac{c}{\kappa \rho} \nabla E + +:math:`\lambda` is a radiation energy flux limiter function: + +.. math:: + \lambda = \frac{2 + R}{6 + 3R + R^2} + +.. math:: + R \equiv \frac{|\nabla E|}{\rho \kappa E} + +.. math:: + \frac{P^{ij}}{E} = \left(\frac{1 - f}{2}\right) \delta_{ij} + + \left(\frac{3f - 1}{2}\right) n^i n^j + +where :math:`n^i \equiv \nabla E / |\nabla E|` and the quantity :math:`f` is defined as: + +.. math:: + f = \lambda + \lambda^2 R^2 + +In the optically thick limit, :math:`R \to 0`, :math:`\lambda \to 1/3`, :math:`f \to 1/3`, and the Eddington approximation :math:`P^{ij} = (E/3)\delta_{ij}` is recovered. + +In the optically thin limit, the photon field streams toward the direction of the local radiation energy density gradient at the speed of light. + +These prescriptions are not a physically exact implementation of realistic light-matter interactions but serve as a computationally efficient proxy for bridging optically thin and thick limits. + +As a result, the radiative transfer equation becomes: + +.. math:: + \partial_t E + \partial_j (Ev^j) = + \partial_i(D\, \partial_i E) - P_r^{ij} \nabla_i v_j - \dot{q}, + +with :math:`D = c\lambda / (\kappa\rho)`. + +.. vim: set tabstop=2 shiftwidth=2 expandtab fo=cqt tw=72 : + + diff --git a/doc/sphinx/hard/inputfiles.rst b/doc/sphinx/hard/inputfiles.rst new file mode 100644 index 0000000..46ea1dd --- /dev/null +++ b/doc/sphinx/hard/inputfiles.rst @@ -0,0 +1,114 @@ +.. |br| raw:: html + +
+ +.. _hard_input_files: + +HARD YAML Input Files +************************************ + +HARD uses YAML files as input to describe the different problems and their configurations. +These files can be found in the `configs` directory at the root. +A subdirectory `ci_configs` is used to run the regression tests in the CI and should not be altered. + +Unless specified, the units are in CGS. + + +Here are details on each parts of the yaml files: + +`problem` + This is the string that is picked up by the `init.hh` file to configure the problem. For now these options are available: + * `sod` + * `rankine-hugoniot` + * `leblanc` + * `acoustic-wave` + * `kh-test` + * `heating_and_cooling` + * `sedov` + * `implosion` + * `rad-rh` + * `lw-implosion` + +Physics quantities + Input that represent physical quantities: `gamma`, `kappa`, `mean_molecular_weight` + +`eos` + Define which EOS to be used from Singularity EOS + * `ideal` + * `spiner` (tabulated EOS, not tested) + * `gruneisen` (not tested) + +time and steps + These variables handle the time and timestep of the simulation: + * `t0`, start time for the simulation + * `tf`, final time for the simulation (not supported for Legion backend) + * `cfl`, Courant–Friedrichs–Lewy coefficient + * `max_dt`, Maximum dt to be used in the simulation + * `max_steps`, Maximum number of iterations to run the simulation for + +logging and output + * `log_frequency`, Set the frequency of iterations outputted on the terminal + * `output_frequency`, Set the frequency of the output files + +`color_distribution` + An array representing how many colors per dimensions to assign. + +`levels` + This is an array representing the grid size as a power of two per dimension. + As an example [8,8,4] would mean a grid of 2^8 * 2^8 * 2^4. + If the code runs with the -d command line option, the extra dimensions are ignored. +`lowest_level` + This defines the lowest level of resolution for the Geometric Multigrid. + All dimensions decrease levels at the same rate, so the lowest resolution corresponds to whichever has one dimension arriving to the lowest level first +`coords` + Coordinates of the lowest and highest points of the domain. + We set two array per dimensions as, for example, a square: + - [0.0, 0.0, 0.0] + - [1.0, 1.0, 1.0] + +`boundaries` + The boundaries are defined per dimensions `xlow`, `xhigh`, `ylow`, `yhigh`, `zlow`, and `zhigh` For each several options are possible: + * `periodic` + * `flow` + * `reflecting` + * `dirichlet` + +`catalyst` + All these options are needed under catalyst to specify the location of the script and the paraview library. + * `script`, path to the python script that interface the output data and catalyst. An example is provided in `tools/gridwriter.py` + * `implementation`, only value for now is `paraview` + * `implementation_directory`, path to the catalyst library compiled using the implementation. Something like `libcatalyst_paraview.so` + + +Some problems have specific input information. These are all defined under the `problem_parameters` group. + + * `acoustic_wave`: + + * `problem_parameters`, a list containing: + * `r0`, the equilibrium density + * `p0`, the equilibrium pressure + * `amplitude`, the perturbation amplitudes + * `scale`, the inverse of the wave number + * `heating_and_cooling`: + + * `problem_parameters`, a list containing: + * `fluid_mass_density` + * `fluid_temperature` + * `radiation_temperature` + * `implosion`: + + * `problem_parameters`, a list containing: + * `fluid_mass_density` + * `fluid_temperature` + * `radiation_temperature` + * `sedov`: + + * `problem_parameters`, a list containing: + * `hotspot_position`, an array of the position of the hotspot + * `hotspot_radius`, the radius of the hotspot + * `E_0`, the initial energy injected at the hotspot in Ergs + +The other tests have setting hardcoded in their initialization files. +They can be found in `app/tasks/initial_data/`. + +.. vim: set tabstop=2 shiftwidth=2 expandtab fo=cqt tw=72 : diff --git a/doc/sphinx/hard/method.rst b/doc/sphinx/hard/method.rst index cac9992..84a9c74 100644 --- a/doc/sphinx/hard/method.rst +++ b/doc/sphinx/hard/method.rst @@ -2,95 +2,16 @@
-.. _hard_problem: +.. _hard_method: -Example Problem -*************** - -The application codes in this example solve the *Euler* equations of -compressible gas dynamics. In one spatial dimension, the Euler equations -are given by - -.. math:: - - \left[ - \begin{gathered} - \rho \\ - \rho u \\ - E - \end{gathered} - \right]_t - + - \left[ - \begin{gathered} - \rho u \\ - \rho u^2 + p \\ - \left(E + p\right)u - \end{gathered} - \right]_x - = 0, - -where :math:`\rho` is the density, :math:`u` is the velocity, :math:`E` is the -total engery, and :math:`p` is the pressure. This is generally referred to as -the *conservation* form, as it emphasises the property that the Euler equations -are mass, momentum, and total energy conserving. - -In three dimensions, the conservation form is - -.. math:: - - q_t + f(q)_x + g(q)_y + h(q)_w = 0, - -where - -.. math:: - - q = - \left[ - \begin{gathered} - \rho \\ - \rho u \\ - \rho v \\ - \rho w \\ - E - \end{gathered} - \right], - f(q) = - \left[ - \begin{gathered} - \rho u \\ - \rho u^2 + p \\ - \rho uv \\ - \rho uw \\ - \left(E+p\right)u - \end{gathered} - \right], - g(q) = - \left[ - \begin{gathered} - \rho v \\ - \rho uv \\ - \rho v^2 + p \\ - \rho vw \\ - \left(E+p\right)v - \end{gathered} - \right], - h(q) = - \left[ - \begin{gathered} - \rho w \\ - \rho uw \\ - \rho vw \\ - \rho w^2 + p \\ - \left(E+p\right)w - \end{gathered} - \right]. +Numerical Schemes +***************** MUSCL Scheme ~~~~~~~~~~~~ The MUSCL scheme (*Monotonic Upstream-centered Scheme for Conservation -Laws*) is a second-order (in space) finite volume method that is *total +Laws*) is a second-order in space finite volume method that is *total variation diminishing (TVD)* and can provide accurate simulations of shock dynamics. A succinct, semi-discrete representation of the scheme can be written @@ -100,7 +21,7 @@ can be written \frac{du_i}{dt} + \frac{1}{\Delta x_i} \left[ F^{*}_{i+1/2} - F^{*}_{i-1/2} - \right],\label{semi}\tag{1} + \right] = 0,\label{semi}\tag{1} where :math:`F^{*}_{i\pm 1/2}` are numerical fluxes that are a non-linear mix of first and second-order slope reconstructions depending @@ -130,22 +51,31 @@ one, which will select the higher-order approximation. Near a shock, :math:`r_i` will approach infinity and :math:`\phi_{mm}(r_i)` will be close to zero and the low-order approximation will be selected. -Hancock Time Evolution -~~~~~~~~~~~~~~~~~~~~~~ +Runge-Kutta Time Evolution for Explicit Update +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To fully discretize (:math:`\ref{semi}`), we can use the Hancock method, -which is second-order accurate in time. Hancock is a predictor-corrector -method that uses the averages of the reconstructed input quantities to -approximate the flux for :math:`t+\Delta t/2` in the predictor -step. These intermediate quantities are then reconstructed to the faces -again, but a *Riemann* solver is used to compute the corrector step. +To fully discretize (:math:`\ref{semi}`), we can use `Heun's method +`__ to approximate the +time derivative. This method is second-order accurate in time. Heun's +method is a Runge-Kutta method with the following Butcher tableau: .. note:: - The MUSCL-Hancock method documented here is second-order accurate in - space and time. + The method documented here is second-order accurate in space and + time. + ++-----+-----+-----+ +| 0 | | | ++-----+-----+-----+ +| 1 | 1 | | ++-----+-----+-----+ +| | 1/2 | 1/2 | ++-----+-----+-----+ -Quantity reconstruction (extrapolation) to faces uses the rule +Each step is preceded by a reconstruction of primitive variables and a +flux calculation given by a *Riemann* solver. + +The quantity reconstruction to faces uses the rule .. math:: @@ -155,23 +85,14 @@ Quantity reconstruction (extrapolation) to faces uses the rule where :math:`\partial q^n_i/\partial s` is the flux limited slope. -The predictor step computes intermediate quantities :math:`q^{*}` using +The calculation of a RK slope :math:`K` is then .. math:: - q^{*}_i = q^n_i - \frac{\Delta t}{2\Delta x_i} - \left(q^n_{i-1/2} - q^n_{i+1/2}\right). + K_i = \frac{F_{i-1/2} - F_{i+1/2}}{\Delta x}, -These quantities are then reconstructed to the faces again by applying -(:math:`\ref{recon}`). The corrector step is - -.. math:: - - q^{n+1}_i = q^n_i + \frac{\Delta t}{\Delta x} - \left( F_{i-1/2} - F_{i+1/2} \right), - -where :math:`F()` is a Riemann solver. The code in this specialization -example uses the HLL approximate Riemann sovler. +where the :math:`F_i\pm1/2` are obtained from a Riemann solver. The code +in this specialization example uses the HLL approximate Riemann sovler. .. math:: @@ -180,10 +101,61 @@ example uses the HLL approximate Riemann sovler. {S_R F_L - S_L F_R + S_R S_L\left(U_R - U_L\right)} {S_R - S_L} -where :math:`S_T` and :math:`S_H` are the min and max wave speeds taken -at the tail and head of the cell interfaces respectively, :math:`U_L` -and :math:`U_R` are the respective intermediate quantities, and -:math:`F_L` and :math:`F_R` are the flux functions evaluated at the +where :math:`S_T` and :math:`S_H` are the min and max characteristic +speeds taken at the tail and head of the cell interfaces respectively, +:math:`U_L` and :math:`U_R` are the respective intermediate quantities, +and :math:`F_L` and :math:`F_R` are the flux functions evaluated at the respective intermediate quantities. +The RK slope is then used to advance the conserved variables, with which +we can calculate the primitive variables and the next RK slope. + +Geometric Multigrid Solver for Implicit Update +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A geometric multigrid solver is a linear solver that uses the fact that +relaxation methods such as the Jacobi or the Gauss-Seidel method +converge faster in coarser domains or grids, and then the solutions in +those coarser grids can be used as initial guesses of the relaxation +methods applied in finer grids. + +The method begins by applying a relaxation method to the system + +.. math:: + + Ax = b. + +After a few iterations of the relaxation method we obtain the +approximate solution :math:`x'`. The residual :math:`r` is then computed +with :math:`r = b - Ax'`. This gives raise to the equation :math:`Ay = +r`, such that :math:`y` is a correction to the unkown array :math:`x` +and the improved approximation :math:`x''` is given by + +.. math:: + + x'' = x' + y \implies Ax'' = A(x' + y) = Ax' + r = b,\label{residual}\tag{3} + +which only works if the operator :math:`A` is linear. + +To solve :math:`Ay = r`, the equation can be transported to a coarser +grid where relaxation methods are more effective. This is done by +applying a restriction operator on :math:`r \rightarrow r_c`. In the +coarser grid, the system :math:`Ay_c = r_c` is solved by applying the +method described above, with an initial solution of :math:`y_{c0} = 0`. + +Once obtained, the solution :math:`y_c` is extended with a prolongation +operator to the finer grid, where the correction +(:math:`\ref{residual}`) is applied. In the coarsest level an exact +solution to the system is ideally calculated, but an approximated +solution can be used as well. + +The algorithm described above is named `v-cycle` due to its diagram: + +.. image:: ../../../doc/v-cycle.svg + +A full multigrid (`FMG`) is the chaining of successive such `v-cycles` +where a finer grid is added after each `v-cycle` is completed: + +.. image:: ../../../doc/fmg.svg + .. vim: set tabstop=2 shiftwidth=2 expandtab fo=cqt tw=72 : diff --git a/doc/sphinx/hard/solver.rst b/doc/sphinx/hard/solver.rst new file mode 100644 index 0000000..d3fb88d --- /dev/null +++ b/doc/sphinx/hard/solver.rst @@ -0,0 +1,35 @@ +.. |br| raw:: html + +
+ +.. _hard_solver_input: + +HARD Solver Details +******************* + +TBD + +HARD Solver Parameters +********************** + +All the options below are present under `linear_solver`: + +`maxiter` + Maximum number of iterations for the solver. Default is `50`. + +`rtol` + The residual tolerance for the solver. Default is `1e-12`. + +`use_zero_guess` + Boolean to define if the initial guess is 0. Default is `true`. + +`flecsolve_coarse_grid` + Boolean to define if we use the coarse grid in flecsolve. Default is `true`. + +`jocabi_iterations` + Define the maximum number of Jacobi iterations. Default is `100`. + +`full_multigrid` + Boolean to define if we use the full multigrid solver. Default is `true`. + +.. vim: set tabstop=2 shiftwidth=2 expandtab fo=cqt tw=72 : diff --git a/doc/sphinx/hard/test_problems.rst b/doc/sphinx/hard/test_problems.rst new file mode 100644 index 0000000..e9b65fe --- /dev/null +++ b/doc/sphinx/hard/test_problems.rst @@ -0,0 +1,108 @@ +.. |br| raw:: html + +
+ +.. _hard_test_problems: + +Test Problems +************************************ + +We describe several test problems that users can test with HARD + +Sod Shock Tube +~~~~~~~~~~~~~~~~~ + +The Sod shock tube is a standard test with a classical Riemann problem with the following initial parameters: + +.. math:: + + (\rho, v, p)_{t=0} = + \begin{cases} + (1.0, 0.0, 1.0) & \text{if} \;\; 0.0 < x \leq 0.5 \\ + (0.125, 0.0, 0.1) & \text{if} \;\; 0.5 < x < 1.0. + \end{cases} + +This leads to the development of a shock front, which propagates from high-density into low-density regions, and is followed by a contact discontinuity. A density rarefaction wave propagates into the high-density region. + +Heating-Cooling +~~~~~~~~~~~~~~~~~ + + + +Temperature Induced Shock +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +Rayleigh-Taylor Instability +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Rayleigh–Taylor Instability consists of a dense fluid over a lighter fluid in a gravitational field, and is perturbed to initiate the instability. + +- **Gravitational acceleration** acts in the x-direction: ``g = g_x`` +- **Pressure at interface**: :math:`p_0 = 2.5` +- **Velocity perturbation (in the y-direction)**: + + .. math:: + + v(x, y) = v_0 \cdot \left( \frac{1 + \cos(4\pi x)}{2} \right) \left( \frac{1 + \cos(3\pi y)}{2} \right), \quad v_0 = 0.05 + +- **Bottom fluid (light)**: + - Density: :math:`\rho_L = 1.0` + - x-velocity: :math:`u_L = 0.0` + +- **Top fluid (heavy)**: + - Density: :math:`\rho_H = 2.0` + - x-velocity: :math:`u_H = 0.0` + +The pressure is initialized to maintain hydrostatic equilibrium across the fluid interface. The pressure :math:`p(x)` is given by: + +.. math:: + + p(x) = + \begin{cases} + p_0 + \rho_L g x, & \text{for } x < 0.75 \\ + p_0 + \rho_L g \cdot 0.75 + \rho_H g (x - 0.75), & \text{for } x \geq 0.75 + \end{cases} + + +At each point in the domain, the following quantities are initialized: + +- **Density**: + + .. math:: + \rho(x) = + \begin{cases} + \rho_L, & x < 0.75 \\ + \rho_H, & x \geq 0.75 + \end{cases} + +- **x-Momentum density**: + + .. math:: + (\rho u)(x) = \rho(x) \cdot u(x) = 0 + +- **Gravitational source term (x-direction)**: + + .. math:: + (\rho g)(x) = \rho(x) \cdot g + +- **Total energy density**: + + .. math:: + + E(x) = \rho \cdot e(\rho, p) + \frac{1}{2} \rho \left( u^2 + v^2 \right) + + where :math:`e(\rho, p)` is the specific internal energy computed from the equation of state. + +- **Radiation energy density**: + + .. math:: + E_{\text{rad}} = 0 + +The setup supports 1D, 2D, and 3D simulations. A cosine-modulated perturbation in the y-velocity initiates the RTI at the fluid interface. The pressure profile ensures that the fluid is initially in hydrostatic equilibrium. + + +Kelvin-Helmholtz Instability +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. vim: set tabstop=2 shiftwidth=2 expandtab fo=cqt tw=72 : diff --git a/doc/sphinx/hard/visualization.rst b/doc/sphinx/hard/visualization.rst new file mode 100644 index 0000000..15af4a7 --- /dev/null +++ b/doc/sphinx/hard/visualization.rst @@ -0,0 +1,268 @@ +.. |br| raw:: html + +
+ +.. _flastro_visualization: + +Visualization +************* + +FlAstro has an integrated Catalyst +(https://www.paraview.org/Wiki/ParaView/Catalyst/Overview; https://docs.paraview.org/en/latest/Catalyst/index.html) +adaptor that creates output that can directly be opened in ParaView +(https://www.paraview.org/; https://docs.paraview.org/en/latest/UsersGuide/introduction.html). +**Note**: The Catalyst adaptor is only activated for 3D simulations. For 2D or 1D simulations, +FlAstro currently outputs raw text data files that can be read into gnuplot, for example. + +**How to build and run flastro with catalyst** + +To build flastro with catalyst, set ``ENABLE_CATALYST=ON``. +Run flastro as you would w/o creating visualzation output, for example: + +``mpirun -n 4 ./flastro -d 3 ../../configs/XYZ.yaml`` + +In the ``XYZ.yaml`` file you secify the physical dimension and resolution of the simulation box, the simulation time +or maximum number of steps, etc. To create output for paraview, you need to provide the path for the catalyst implementation +and specify a catalyst/paraview script, like ``gridwriter.py`` that is provided in the ``../../tools/`` directory along with +other example scripts that directly render pngs for different observables. To add further observables/fields to the pipeline +for visualization, see below. + +3D visualizations with catalyst/paraview +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Example**: *Evolution of speed (magnitude of velocity) in a 3D Sod shock tube during the first 0.5 seconds.* + +FlAstro's catalyst adaptor will put all visualization files into a directory ``datasets``. +It can be directly opened from your local paraview client. This is a visualization of an example run with +48 x 12 x12 cells, using ``gridwriter.py``. + +2D visualizations using raw output and gnuplot/pm3d +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Flastro creates raw output files in ``tsv`` format that can be read into spead sheets or by +tools like ``gnuplot``. Animations can be produced by plotting the data from each time step. + +**Example**: Evolution of density in a 2D Sod shock tube during the first approx. 0.5 seconds. + +**HowTo**: + +#. Process all raw output files from a 2D-FlAstro run with ``gnuplot``: + + * the script ``make_png_from_raw_2D.sh`` in the ``../../tools/`` directory will create a png file from + every data file in a specified range + + * Example was made with: ``make_png_from_raw_2D.sh 1 200 128 4`` (for help, run script w/o arguments) + +#. Combine pictures into an animation, for example using QuickTime or ImageMagick + + * QuickTime Player: load all image files ("Open image Sequence ...") and save as movie + + * ImageMagick (available on Darwin): ``convert *png 2d-flastro-sod-128cells.gif`` + + +1D visualizations using raw output and gnuplot/pm3d +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To create a plot of an observable at a certain time, you can directly plot columns from the raw output files. + +**Example**: Density in a 1D Sod shock tube at time :math:`t = 0.2` +(https://en.wikipedia.org/wiki/Sod_shock_tube). + +To create a 3D plot of the time evolution of a scalar observable: + +**Example**: Evolution of density in a 1D Sod shock tube during the first 0.2 seconds. + +**HowTo**: + +#. Concatenate all raw output files from a 1D-FlAstro run: ``cat output*raw > output_time_evolution.raw`` +#. Run ``gnuplot``: + +.. code-block:: + + user@host:.../flastro/build/app$ gnuplot + + gnuplot> splot 'output_time_evolution.raw' u 1:($2/128):3 w pm3d at bs title "flastro 1D, 128 cells, sod evolution 0.0 < t < 0.2" + gnuplot> set xlabel "time" + gnuplot> set ylabel "x" + gnuplot> set zlabel "density" + + [alternatively to all of the above:] + gnuplot> load "output_time_evolution.gnuplot" + + [use mouse to rotate to desired perspective] + [to output png:] + gnuplot> set term png + gnuplot> set output "output_time_evolution.png" + gnuplot> replot + gnuplot> quit + + +How to add other fields/observables to the catalyst output +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Adding new data to the catalyst/paraview pipeline requires editing all files in the catalyst module +(``catalyst/adaptor.[cc|hh]`` and ``catalyst/types[cc|hh]``), the catalyst tasks file (``tasks/catalyst.hh``), +and the files where you call catalyst tasks (``init.hh`` and ``analyze.hh``, in this example). + +**Example: Adding the total energy** + +**Note:** All necessary additions/changes for this example are available in the code repository, included in the commit +“How to add total energy to catalyst pipeline; documentation example”, and marked in the code with the ``<<< add total energy`` +tag. + +1. Physics fields/observables are referred to as “attributes” in the catalyst language. +So we have to create a new attribute and update it along with all the others. This is +done in ``catalyst/types.[hh|cc]`` and the ``update_attributes`` method in ``tasks/catalyst.hh``. +Note that total energy is a scalar field; in particular, it does not depend on the dimension. +(For vector fields, use ``velocity`` as an example accordingly). + +``catalyst/types.hh``: + +.. code-block:: + + class catalyst_attributes + public: + … + void update_fields(double * cell_vector_vals1, + double * cell_scalar_vals1, + double * cell_scalar_vals2, + double * cell_scalar_vals3); // <<< add total energy + … + + private: + … + std::vector tot_energy; // <<< add total energy + … + }; + +``catalyst/types.cc``: + +.. code-block:: + + void catalyst_attributes::initialize(size_t number_of_points, size_t number_of_cells) + { + … + this->tot_energy.resize(number_of_cells); // <<< add total energy + } + + void catalyst_attributes::update_fields(double * cell_vector_vals1, + double * cell_scalar_vals1, + double * cell_scalar_vals2, + double * cell_scalar_vals3) { // <<< add total energy + for(size_t cell_i = 0; cell_i < num_cells_; cell_i++) { + … + this->tot_energy[cell_i] = cell_scalar_vals3[cell_i]; // <<< add total energy + } + … + } // end update_fields + + +``task/catalyst.hh`` + +.. code-block:: + + template + inline void update_attributes(single::accessor c_a, + … + field::accessor rE_a) // <<< add total energy + { + if constexpr(D == 3) { + … + auto rE = m.template mdcolex(rE_a); // <<< add total energy + … + std::vector tot_energy_vals; // <<< add total energy + + for(auto k : m.template cells()) { + for(auto j : m.template cells()) { + for(auto i : m.template cells()) { + // update all scalar fields + … + tot_energy_vals.push_back(rE(i,j,k)); // <<< add total energy + … + } // for + } // for + } // for + … + c_a->update_fields(velocity_vals.data(), + density_vals.data(), + pressure_vals.data(), + tot_energy_vals.data()); // <<< add total energy + } + … + } + + +Finally, pass the new field along when the ``update_attributes`` task is executed. + +``init.hh`` and ``analyze.hh`` + +.. code-block:: + + // Send initial problem state to catalyst + execute, mpi>( + catalyst_data(pt), 0.0, s.m, s.r(s.m), s.u(s.m), s.p(s.m), + s.rE(s.m)); // <<< add total energy + +.. code-block:: + + execute, mpi>( + catalyst_data(pt), cp.time(), s.m, s.r(s.m), s.u(s.m), s.p(s.m), + s.rE(s.m)); // <<< add total energy + +2. Now the new attribute has to be included in the conduit node that is created for catalyst/paraview. +This node is defined in ``catalyst/adaptor.cc``. Note that the total energy is “cell data” +(cells are called “elements” in catalyst language). We have to also add the new attribute to +the ``catalyst_attributes::execute`` method in ``catalyst/types.cc``. + +``catalyst/adaptor.hh`` + +.. code-block:: + + void execute(int cycle, double time, flastro::simple_cubic& grid, + … + double* tot_energy_array); // <<< add total energy + + +``catalyst/adaptor.cc`` + +.. code-block:: + + void catalyst_adaptor::execute(int cycle, double time, simple_cubic& grid, + … + double* tot_energy_array) // <<< add total energy + { + … + // add total energy >>> + // total energy is cell-data + fields["total_energy/association"].set("element"); + fields["total_energy/topology"].set("mesh"); + fields["total_energy/volume_dependent"].set("false"); + fields["total_energy/values"].set_external(tot_energy_array, grid.get_number_of_cells()); + // <<< add total energy + … + } + +``catalyst/types.cc`` + +.. code-block:: + + void catalyst_attributes::execute(size_t step, double time, simple_cubic& lattice) + { + … + catalyst_adaptor::execute(step, time, lattice, &this->velocity[0], + … + &this->tot_energy[0]); // <<< add total energy + } + +Example for config file: + +.. code-block:: + + catalyst: + script: gridwriter.py + implementation: paraview + implementation_directory: /vast/home/thomasvogel/spack/var/spack/environments/vtk_viz/.spack-env/view/lib64/catalyst/ + + + +Feel free to contact Thomas (thomasvogel@lanl.gov) directly if you have any further questions. diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst index 205b3a4..3ac5d6e 100644 --- a/doc/sphinx/index.rst +++ b/doc/sphinx/index.rst @@ -1,16 +1,19 @@ HARD Application ******************* -HARD is an application for simulation of radiation hydrodynamics -phenomena. +HARD is a radiation-hydrodynamics solver suite for the study of astrophysical phenomena. -This is a test. +HARD is based on the FleCSI framework and implemented on top of FleCSI-SP (FleCSI Specialization project). .. toctree:: :caption: Contents: + hard/equations hard/method - hard/api + hard/inputfiles + hard/solver.rst + hard/test_problems hard/visualization + hard/afld.rst .. vim: set tabstop=2 shiftwidth=2 expandtab fo=cqt tw=72 : diff --git a/doc/v-cycle.svg b/doc/v-cycle.svg new file mode 100644 index 0000000..1ad0a39 --- /dev/null +++ b/doc/v-cycle.svg @@ -0,0 +1,126 @@ + + diff --git a/spack-repo/packages/flecsi/package.py b/spack-repo/packages/flecsi/package.py new file mode 100644 index 0000000..1e5465b --- /dev/null +++ b/spack-repo/packages/flecsi/package.py @@ -0,0 +1,8 @@ +from spack.package import * +from spack_repo.builtin.packages.flecsi.package import Flecsi + +class Flecsi(Flecsi): + """ + Additional named versions for FleCSI + """ + version("2.4.0", commit="ff213702c291c303add0ed94eaf58c138af67f39") diff --git a/spack-repo/packages/flecsolve/package.py b/spack-repo/packages/flecsolve/package.py new file mode 100644 index 0000000..fa7c744 --- /dev/null +++ b/spack-repo/packages/flecsolve/package.py @@ -0,0 +1,33 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack.package import * + +class Flecsolve(CMakePackage): + """Solvers package built on top of FleCSI""" + + homepage="https://github.com/lanl/flecsolve.git" + git = "https://github.com/lanl/flecsolve.git" + # TO DO update to stable when available + version("main", branch="flecsi-2.4") + + variant("tests", default=False, description="Enable unit tests") + variant("standard", default=False, description="Standard setup for flecsolve") + + depends_on('flecsi@2.4:') + depends_on("c", type="build") + depends_on("cxx", type="build") + depends_on("fortran", type="build") + #depends_on('amp+hypre', when="+standard") # Might want to enable at some point. + #depends_on('stacktrace+shared', when="+standard") + #depends_on('lapackwrappers@main', when="+standard") + + def cmake_args(self): + args = [ + self.define_from_variant("FLECSOLVE_ENABLE_UNIT_TESTS", "tests"), + self.define_from_variant("FLECSOLVE_ENABLE_AMP", "standard"), + ] + return args + + diff --git a/spack-repo/packages/hard/package.py b/spack-repo/packages/hard/package.py index af84f6f..eed5729 100644 --- a/spack-repo/packages/hard/package.py +++ b/spack-repo/packages/hard/package.py @@ -4,7 +4,7 @@ from spack.package import * -class Hard(CMakePackage): +class Hard(CMakePackage, CudaPackage): """A FleCSI-based radiation-hydrodynamics solver suite for the study of astrophysical phenomena""" @@ -15,18 +15,45 @@ class Hard(CMakePackage): variant("catalyst", default=False, description="Enable catalyst for paraview interface") variant("radiation", default=True, description="Enable support for radiation physics") + variant("flecsolve", default=True, description="Enable flecsolve linear solvers in the radiation solver") variant("tests", default=False, description="Enable unit tests") + variant("format", default=False, description="Enable format target") - depends_on("flecsi@2.3.0: +flog") + depends_on("flecsi@2.4.0: +flog") + depends_on("flecsi +cuda", when="+cuda") + depends_on("flecsolve", when="+flecsolve") depends_on("libcatalyst", when="+catalyst") + #depends_on("paraview@5.12.1+libcatalyst+python", when="+catalyst") depends_on("yaml-cpp@0.8:") - depends_on("singularity-eos@main +hdf5 +spiner +eospac build_extra=sesame") + depends_on("c", type="build") + depends_on("cxx", type="build") + + depends_on("singularity-eos@1.9.2: +hdf5 +spiner build_extra=sesame") + depends_on("singularity-eos@1.9.2.1 ~eospac+kokkos+kokkos-kernels+cuda", when="+cuda") + + depends_on("llvm@13.0.0", type="build", when="+format") + depends_on("python", when="+tests") + depends_on("py-numpy", when="+tests") + depends_on("py-yamlreader", when="+tests") + depends_on("py-scipy", when="+tests") + depends_on("py-exactpack", when="+tests") + depends_on("py-matplotlib", when="+tests") + + + requires("%clang@17:", when="+cuda", msg="CUDA version only supports Clang compiler") + + # Propagate cuda_arch requirement to dependencies + for _flag in CudaPackage.cuda_arch_values: + requires(f"+cuda cuda_arch={_flag}", when=f"^kokkos +cuda cuda_arch={_flag}") + depends_on(f"kokkos cuda_arch={_flag}", when=f"+cuda cuda_arch={_flag}") + depends_on(f"singularity-eos cuda_arch={_flag}", when=f"+cuda cuda_arch={_flag}") def cmake_args(self): options = [ self.define_from_variant("ENABLE_UNIT_TESTS", "tests"), self.define_from_variant("ENABLE_CATALYST", "catalyst"), - self.define("DISABLE_RADIATION", self.spec.satisfies("~radiation")), + self.define_from_variant("ENABLE_RADIATION", "radiation"), + self.define_from_variant("ENABLE_FLECSOLVE", "flecsolve") ] return options diff --git a/spack-repo/packages/mpark-variant/package.py b/spack-repo/packages/mpark-variant/package.py index cb081ae..828e7a8 100644 --- a/spack-repo/packages/mpark-variant/package.py +++ b/spack-repo/packages/mpark-variant/package.py @@ -18,6 +18,8 @@ class MparkVariant(CMakePackage): version("1.4.0", sha256="8f6b28ab3640b5d76d5b6664dda7257a4405ce59179220431b8fd196c79b2ecb") version("1.3.0", sha256="d0f7e41f818fcc839797a8017e76b8b66b323651c304cff641a83a56ae9943c6") + depends_on("c", type="build") + depends_on("cxx", type="build") # Ref.: https://github.com/mpark/variant/pull/73 patch("nvcc.patch", when="@:1.4.0") diff --git a/spack-repo/packages/ports-of-call/packages.py b/spack-repo/packages/ports-of-call/packages.py index 6fb1a60..80dc9b0 100644 --- a/spack-repo/packages/ports-of-call/packages.py +++ b/spack-repo/packages/ports-of-call/packages.py @@ -52,6 +52,8 @@ class PortsOfCall(CMakePackage): when="@1.6.1: +test", ) + depends_on("c", type="build") + depends_on("cxx", type="build") depends_on("cmake@3.12:") depends_on("catch2@3.0.1:", when="+test") depends_on("kokkos", when="+test test_portability_strategy=Kokkos") diff --git a/spack-repo/packages/py-exactpack/package.py b/spack-repo/packages/py-exactpack/package.py new file mode 100644 index 0000000..01d2fb4 --- /dev/null +++ b/spack-repo/packages/py-exactpack/package.py @@ -0,0 +1,17 @@ + +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack.package import * + + +class PyExactpack(PythonPackage): + """ExactPack Python package""" + + git = "https://github.com/lanl/exactpack" + + version("1.7.10", commit="dc331d9ac450c0ebc94ada908a275ff26949e0e5") + + depends_on("python@3.6:", type=("build", "run")) + depends_on("py-setuptools", type="build") diff --git a/spack-repo/packages/singularity-eos/package.py b/spack-repo/packages/singularity-eos/package.py index 5dbff1b..cadb76d 100644 --- a/spack-repo/packages/singularity-eos/package.py +++ b/spack-repo/packages/singularity-eos/package.py @@ -38,12 +38,9 @@ class SingularityEos(CMakePackage, CudaPackage, ROCmPackage): # allow `main` version for development version("main", branch="main") - version("1.9.1", sha256="148889e1b2d5bdc3d59c5fd6a6b5da25bb4f4f0f4343c57b3ccaf96691c93aff") - version("1.9.0", sha256="460b36a8311df430e6d4cccf3e72a6b3afda7db8d092b4a0a4259c4363c4dbde") - version("1.8.0", sha256="1f1ec496f714aa23cc7003c88a85bd10d0e53e37659ba7310541248e48a66558") - version("1.7.0", sha256="ce0825db2e9d079503e98cecf1c565352be696109042b3a0941762b35f36dc49") - version("1.6.2", sha256="9c85fca679139a40cc9c72fcaeeca78a407cc1ca184734785236042de364b942", deprecated=True) - version("1.6.1", sha256="c6d92dfecf9689ffe2df615791c039f7e527e9f47799a862e26fa4e3420fe5d7", deprecated=True) + version("1.9.2.1", commit="bef20b27935b563aee5e63096fc141e44accf3a5") + version("1.9.2", commit="9e7de3eccd610e0654e9a05d673ef7b24d32cf31") + version("1.9.1", commit="1be18426a2c9c26f969bc14a73b482d5beca0217") # build with kokkos, kokkos-kernels for offloading support variant("kokkos", default=False, description="Enable kokkos") @@ -73,8 +70,8 @@ class SingularityEos(CMakePackage, CudaPackage, ROCmPackage): # link to EOSPAC for table reads variant("eospac", default=True, description="Pull in EOSPAC") - # enable/disable HDF5 - used to control upstream `spiner` - # configuration + # enable/disable HDF5 - used to control upstream `spiner` + # configuration variant("hdf5", default=False, description="Use hdf5") variant("spiner", default=True, description="Use Spiner") @@ -104,6 +101,9 @@ class SingularityEos(CMakePackage, CudaPackage, ROCmPackage): depends_on("python@3:", when="+python") depends_on("py-numpy", when="+python+tests") depends_on("py-pybind11@2.9.1:", when="+python") + depends_on("c", type="build") + depends_on("cxx", type="build") + depends_on("fortran", type="build") # linear algebra when not using GPUs # TODO we can do LA with +kokkos+kokkos-kernels~cuda, @@ -112,7 +112,7 @@ class SingularityEos(CMakePackage, CudaPackage, ROCmPackage): requires("+kokkos-kernels", when="+cuda") requires("+kokkos-kernels", when="+rocm") - # eospac when asked for + # eospac when asked for depends_on("eospac", when="+eospac") depends_on("ports-of-call@1.4.2,1.5.2:", when="@:1.7.0") @@ -120,21 +120,19 @@ class SingularityEos(CMakePackage, CudaPackage, ROCmPackage): depends_on("ports-of-call@1.6.0:", when="@1.9.0:") # request HEAD of main branch depends_on("ports-of-call@main", when="@main") - + depends_on("spiner +kokkos", when="+kokkos+spiner") - # tell spiner to use HDF5 + # tell spiner to use HDF5 depends_on("spiner +hdf5", when="+hdf5+spiner") - depends_on("spiner@:1.6.0", when="@:1.7.0 +spiner") - depends_on("spiner@1.6.1:", when="@1.7.1: +spiner") #TODO version - depends_on("spiner@main", when="@main +spiner") + depends_on("spiner@1.6.3", when="+spiner") depends_on("mpark-variant") depends_on( "mpark-variant", patches=patch( "https://raw.githubusercontent.com/lanl/singularity-eos/refs/heads/main/utils/gpu_compatibility.patch", - sha256="c803670cbd95f9b97458fb4ef403de30229ec81566a5b8e5ccb75ad9d0b22541" + sha256="592e64ceccd2822ec1cc7eb01ac3fcad620551940beab793003afb6b5366dad8" ), when="+cuda", ) @@ -142,7 +140,7 @@ class SingularityEos(CMakePackage, CudaPackage, ROCmPackage): "mpark-variant", patches=patch( "https://raw.githubusercontent.com/lanl/singularity-eos/refs/heads/main/utils/gpu_compatibility.patch", - sha256="c803670cbd95f9b97458fb4ef403de30229ec81566a5b8e5ccb75ad9d0b22541", + sha256="592e64ceccd2822ec1cc7eb01ac3fcad620551940beab793003afb6b5366dad8", ), when="+rocm", ) @@ -166,7 +164,7 @@ class SingularityEos(CMakePackage, CudaPackage, ROCmPackage): # specfic specs when using GPU/cuda offloading # TODO remove +wrapper for clang builds # TODO version guard +cuda_lambda - depends_on("kokkos +wrapper+cuda_lambda", when="+cuda+kokkos") + depends_on("kokkos +cuda_lambda", when="+cuda+kokkos") # fix for older spacks if spack.version.Version(spack.spack_version) >= spack.version.Version("0.17"): @@ -200,7 +198,10 @@ class SingularityEos(CMakePackage, CudaPackage, ROCmPackage): depends_on("py-h5py" + _flag, when="@:1.6.2 " + _flag) # depends_on("kokkos-nvcc-wrapper" + _flag, when="+cuda+kokkos" + _flag) - # TODO some options are now version specific. For now it should be + patch("true.patch") + + + # TODO some options are now version specific. For now it should be # benign, but good practice to do some version guards. def cmake_args(self): args = [ diff --git a/spack-repo/packages/singularity-eos/true.patch b/spack-repo/packages/singularity-eos/true.patch new file mode 100644 index 0000000..c4fe3d6 --- /dev/null +++ b/spack-repo/packages/singularity-eos/true.patch @@ -0,0 +1,13 @@ +diff --git a/singularity-eos/base/fast-math/logs.hpp b/singularity-eos/base/fast-math/logs.hpp +index 28849233..c311c27b 100644 +--- a/singularity-eos/base/fast-math/logs.hpp ++++ b/singularity-eos/base/fast-math/logs.hpp +@@ -59,6 +59,8 @@ + // TODO(JMM): All the math here is all assuming inputs are + // doubles. Should add single-precision overloads. + ++#undef TRUE ++ + namespace singularity { + namespace FastMath { + diff --git a/spack-repo/packages/singularity-opac_deps/package.py b/spack-repo/packages/singularity-opac_deps/package.py new file mode 100644 index 0000000..1057900 --- /dev/null +++ b/spack-repo/packages/singularity-opac_deps/package.py @@ -0,0 +1,21 @@ +# dependency package for singulary-opac + +import os +from spack import * +from spack.package import * + +class SingularityOpacDeps(CMakePackage): + homepage = "https://github.com/lanl/singularity-opac" + git = "git@github.com:lanl/singularity-opac.git" + + version("main", branch="main") + + #depends_on("cmake") + #depends_on("hdf5") + #depends_on("kokkos") + + #phases=["install"] + + def install(self, spec, prefix): + install_tree(join_path(self.stage.source_path, "include"), prefix.include) + diff --git a/spack-repo/packages/spiner/package.py b/spack-repo/packages/spiner/package.py index e9b3235..953b727 100644 --- a/spack-repo/packages/spiner/package.py +++ b/spack-repo/packages/spiner/package.py @@ -15,25 +15,7 @@ class Spiner(CMakePackage): maintainers = ["rbberger"] - version("main", branch="main") - version("1.6.2", sha256="91fb403ce3b151fbdf8b6ff5aed0d8dde1177749f5633951027b100ebc7080d3") - version("1.6.1", sha256="52774322571d3b9b0dc3c6b255257de9af0e8e6170834360f2252c1ac272cbe7") - version("1.6.0", sha256="afa5526d87c78c1165ead06c09c5c2b9e4a913687443e5adff7b709ea4dd7edf") - version( - "1.5.1", - sha256="dd1cada84446443e8925438b8da53ab5a6cb9f373f1a993905ef0bf51f48223c", - deprecated=True, - ) - version( - "1.5.0", - sha256="b27ddabc0d21870b845444c24307d3a0c1b175483e72cc138139d6e0dd29b244", - deprecated=True, - ) - version( - "1.4.0", - sha256="c3801b9eab26feabec33ff8c59e4056f384287f407d23faba010d354766f3ac5", - deprecated=True, - ) + version("1.6.3", commit="c379f5d2cce9625a60b149192ec43c8cff7b74b1") # When overriding/overloading varaints, the last variant is always used, except for # "when" clauses. Therefore, call the whens FIRST then the non-whens. @@ -53,15 +35,15 @@ class Spiner(CMakePackage): depends_on("cmake@3.12:", when="@:1.5.1") depends_on("cmake@3.19:", when="@1.6.0:") - depends_on("catch2@3.0.1:", when="@main +test") + depends_on("catch2@3.0.1:", when="@1.6.3: +test") depends_on("catch2@2.13.4:2.13.9", when="@:1.6.2 +test") depends_on("ports-of-call@1.2.0:", when="@:1.5.1") depends_on("ports-of-call@1.5.1:", when="@1.6.0:") depends_on("ports-of-call@main", when="@main") + depends_on("c", type="build") + depends_on("cxx", type="build") # Currently the raw cuda backend of ports-of-call is not supported. - depends_on("ports-of-call portability_strategy=Kokkos", when="@:1.5.1 +kokkos") - depends_on("ports-of-call portability_strategy=None", when="@:1.5.1 ~kokkos") depends_on("kokkos@3.3.00:", when="+kokkos") requires("^kokkos+cuda_lambda+cuda_constexpr", when="+kokkos ^kokkos+cuda") diff --git a/spack-repo/packages/yaml-cpp/package.py b/spack-repo/packages/yaml-cpp/package.py index 72e621d..e2d0111 100644 --- a/spack-repo/packages/yaml-cpp/package.py +++ b/spack-repo/packages/yaml-cpp/package.py @@ -1,5 +1,5 @@ from spack.package import * -from spack.pkg.builtin.yaml_cpp import YamlCpp +from spack_repo.builtin.packages.yaml_cpp.package import YamlCpp class YamlCpp(YamlCpp): """ diff --git a/spec/control.hh b/spec/control.hh index d1780c7..f6f0b86 100644 --- a/spec/control.hh +++ b/spec/control.hh @@ -64,8 +64,9 @@ struct control_policy : flecsi::run::control_base { double max_dt, std::size_t log_frequency, std::size_t output_frequency) - : t0_(t0), tf_(tf), max_steps_(max_steps), cfl_(cfl), max_dt_(max_dt), - log_frequency_(log_frequency), output_frequency_(output_frequency) {} + : t0_(t0), tf_(tf), t_(t0), max_steps_(max_steps), cfl_(cfl), + max_dt_(max_dt), log_frequency_(log_frequency), + output_frequency_(output_frequency) {} S & state() { return state_; @@ -83,6 +84,14 @@ struct control_policy : flecsi::run::control_base { return max_steps_; } + auto time() const { + return t_; + } + + auto max_time() const { + return tf_; + } + static void compute_dt( typename single::template accessor t, typename single::template accessor dt, @@ -96,6 +105,20 @@ struct control_policy : flecsi::run::control_base { t += dt; } + static std::tuple compute_dt_mpi( + typename single::template accessor t, + typename single::template accessor dt, + flecsi::future dtmin, + double tf, + double max_dt, + double cfl) { + dt = cfl * dtmin.get(); + dt = t + dt > tf ? tf - t : dt; + dt = std::min(*dt, max_dt); + t += dt; + return std::make_tuple(t, dt); + } + static bool cycle_control(control_policy & cp) { #ifdef HARD_BENCHMARK_MODE @@ -132,13 +155,33 @@ struct control_policy : flecsi::run::control_base { bool exec_cycle = cp.step_ < cp.max_steps_; auto & s = cp.state(); + +#if FLECSI_BACKEND == FLECSI_BACKEND_legion flecsi::execute( - s.t(s.gt), s.dt(s.gt), s.dtmin_, cp.tf_, cp.max_dt_, cp.cfl_); + s.t(*s.gt), s.dt(*s.gt), s.dtmin_, cp.tf_, cp.max_dt_, cp.cfl_); if((cp.step_ % cp.log_frequency_) == 0 || cp.step_ == cp.max_steps_) { flog(info) << "step: " << cp.step_ << "/" << cp.max_steps_ << std::endl; flecsi::flog::flush(); } // if + +#else + auto [t, dt] = flecsi::execute( + s.t(*s.gt), s.dt(*s.gt), s.dtmin_, cp.tf_, cp.max_dt_, cp.cfl_) + .get(); + cp.t_ = t; + + if((cp.step_ % cp.log_frequency_) == 0 || cp.step_ == cp.max_steps_ || + cp.t_ == cp.tf_) { + flog(info) << "step: " << cp.step_ << " time: " << cp.t_ << " dt: " << dt + << std::endl; + flecsi::flog::flush(); + } // if + + exec_cycle = exec_cycle && t <= cp.tf_ && dt != 0.0; + +#endif + ++cp.step_; return exec_cycle; } // cycle_control @@ -152,6 +195,7 @@ private: std::size_t step_{0}; double t0_; double tf_; + double t_; std::size_t max_steps_; double cfl_; double max_dt_; diff --git a/spec/limiter.hh b/spec/limiter.hh index 56cdc57..791e097 100644 --- a/spec/limiter.hh +++ b/spec/limiter.hh @@ -86,8 +86,8 @@ struct ppm4 { for(int i = 0; i < T::Dim; ++i) { if(delta_u_plus[i] * delta_u_minus[i] < 0.0) { - get<0>(res)[i] = u_0[i]; - get<1>(res)[i] = u_0[i]; + std::get<0>(res)[i] = u_0[i]; + std::get<1>(res)[i] = u_0[i]; } else { if(std::abs(delta_u_plus[i]) >= 2.0 * std::abs(delta_u_minus[i])) { @@ -96,8 +96,8 @@ struct ppm4 { if(std::abs(delta_u_minus[i]) >= 2.0 * std::abs(delta_u_plus[i])) { u_minus_12[i] = u_0[i] - 2.0 * delta_u_plus[i]; } - get<0>(res)[i] = u_minus_12[i]; - get<1>(res)[i] = u_plus_12[i]; + std::get<0>(res)[i] = u_minus_12[i]; + std::get<1>(res)[i] = u_plus_12[i]; } } return res; diff --git a/spec/mesh.hh b/spec/mesh.hh index 9938108..ac8eaa3 100644 --- a/spec/mesh.hh +++ b/spec/mesh.hh @@ -42,6 +42,64 @@ struct help<3> { double zdelta; }; }; + +struct srange { + std::size_t beg, end; + std::size_t size() const { + return end - beg; + } +}; + +FLECSI_INLINE_TARGET static std::size_t +digit(flecsi::util::id & x, std::size_t d) { + std::size_t ret = x % d; + x /= d; + return ret; +} + +FLECSI_INLINE_TARGET flecsi::util::id +translate(flecsi::util::id & x, + std::size_t & stride, + const srange & sub, + std::size_t extent) { + flecsi::util::id ret = digit(x, sub.size()) + sub.beg; + + ret *= stride; + stride *= extent; + + return ret; +} + +template +flecsi::util::id +translate_index(flecsi::util::id x, + const std::array & subrange, + const std::array & extents, + std::index_sequence) { + std::size_t stride = 1; + return (translate(x, stride, subrange[Index], extents[Index]) + ...); +} + +template +std::size_t +subrange_size(std::array & subrange, + std::index_sequence) { + return (subrange[Index].size() * ...); +} + +template +inline auto +make_subrange_ids(std::array subrange, + std::array extents) { + return flecsi::util::transform_view( + flecsi::util::iota_view( + 0, subrange_size(subrange, std::make_index_sequence())), + [=](const auto & x) { + return flecsi::topo::id( + translate_index(x, subrange, extents, std::make_index_sequence())); + }); +} + } // namespace detail /*! @@ -284,16 +342,42 @@ struct mesh : flecsi::topo::specialization> { } // if } // axis_colors + template + auto dofs() { + return detail::make_subrange_ids( + interior_subrange(axes()), extents_array(axes())); + } + + protected: + template + detail::srange interior_subrange() { + const auto & l = axis().layout; + + return {l.template logical<0>(), l.template logical<1>()}; + } + + template + auto interior_subrange(flecsi::util::constants) { + return std::array{ + {interior_subrange()...}}; + } + + template + auto extents_array(flecsi::util::constants) { + return std::array{ + axis().layout.extent()...}; + } + }; // interface /*--------------------------------------------------------------------------* Color Task. *--------------------------------------------------------------------------*/ - static coloring - color(std::size_t num_colors, gcoord axis_extents, periodic_axes p) { + template + static auto index_colors(T & num, gcoord & axis_extents, periodic_axes & p) { index_definition idef; - idef.axes = mesh::base::make_axes(num_colors, axis_extents); + idef.axes = mesh::base::make_axes(num, axis_extents); std::size_t ai{0}; for(auto & a : idef.axes) { @@ -302,6 +386,19 @@ struct mesh : flecsi::topo::specialization> { a.periodic = p[ai++]; } // for + return idef; + } + + static coloring + color(flecsi::Color num_colors, gcoord axis_extents, periodic_axes p) { + index_definition idef = index_colors(num_colors, axis_extents, p); + return {{idef}}; + } + + static coloring color(typename mesh::base::colors const & color_distribution, + gcoord axis_extents, + periodic_axes p) { + index_definition idef = index_colors(color_distribution, axis_extents, p); return {{idef}}; } // color @@ -309,8 +406,9 @@ struct mesh : flecsi::topo::specialization> { Initialization. *--------------------------------------------------------------------------*/ - static void set_geometry(mesh::template accessor m, - grect const & g) { + static void set_geometry(flecsi::exec::cpu, + typename mesh::template accessor m, + grect const & g) noexcept { if constexpr(D == 1) { m.set_geometry( (g[0][1] - g[0][0]) / m.template size()); @@ -326,10 +424,11 @@ struct mesh : flecsi::topo::specialization> { } } // set_geometry - static void initialize(flecsi::data::topology_slot & s, + static void initialize(flecsi::scheduler & s, + typename mesh::topology & m, coloring const &, grect const & geometry) { - flecsi::execute(s, geometry); + s.execute(flecsi::exec::on, m, geometry); } // initialize }; // struct mesh diff --git a/spec/runtime.hh b/spec/runtime.hh index 11ac827..d4419c9 100644 --- a/spec/runtime.hh +++ b/spec/runtime.hh @@ -19,7 +19,7 @@ dispatch(flecsi::runtime & r, std::size_t d, std::index_sequence, AA &&... aa) { - bool ret; + bool ret = false; std::initializer_list({( d == DD ? (ret = r.control>(std::forward(aa)...)), 0 : 0)...}); return ret; diff --git a/spec/types.hh b/spec/types.hh index f6a476d..f6b4efd 100644 --- a/spec/types.hh +++ b/spec/types.hh @@ -5,8 +5,8 @@ #include namespace spec { -inline constexpr flecsi::partition_privilege_t na = flecsi::na, ro = flecsi::ro, - wo = flecsi::wo, rw = flecsi::rw; +inline constexpr flecsi::privilege na = flecsi::na, ro = flecsi::ro, + wo = flecsi::wo, rw = flecsi::rw; using flecsi::topo::global; using flecsi::topo::index; @@ -20,53 +20,49 @@ using flecsi::data::multi; // vec type definitions // ---------------------------------------------------------------------------- template -requires(D <= 3) struct vec { -}; +// requires(D <= 3) +struct vec {}; template<> struct vec<1> { constexpr static std::size_t Dim = 1; - FLECSI_INLINE_TARGET vec<1>() {} + FLECSI_INLINE_TARGET vec<1>() = default; + FLECSI_INLINE_TARGET ~vec<1>() = default; - FLECSI_INLINE_TARGET vec<1>(double v) { - x = v; + FLECSI_INLINE_TARGET vec<1>(double x) : vx(x) {} + FLECSI_INLINE_TARGET double & operator[](std::size_t) { + return vx; } - - FLECSI_INLINE_TARGET vec<1>(const vec<1> & v) { - x = v.x; + FLECSI_INLINE_TARGET double operator[](std::size_t) const { + return vx; } - FLECSI_INLINE_TARGET double & operator[](std::size_t d) noexcept { - return components[d]; + FLECSI_INLINE_TARGET double & x() { + return vx; } - - FLECSI_INLINE_TARGET double operator[](std::size_t d) const noexcept { - return components[d]; + FLECSI_INLINE_TARGET double x() const { + return vx; } - union - { - struct { - double x; - }; - double components[1]; - }; + double vx; FLECSI_INLINE_TARGET double norm_squared() const { - return x * x; + return vx * vx; } FLECSI_INLINE_TARGET double norm() const { - return x; + return std::abs(vx); } - FLECSI_INLINE_TARGET double & get(const size_t idx) { + FLECSI_INLINE_TARGET double & get([[maybe_unused]] const size_t idx) { assert(idx == 0 && "Invalid access for 1d vector"); - return x; + return vx; } - FLECSI_INLINE_TARGET const double & get(const size_t idx) const { + FLECSI_INLINE_TARGET const double & get( + [[maybe_unused]] const size_t idx) const { assert(idx == 0 && "Invalid access for 1d vector"); - return x; + + return vx; } }; @@ -75,53 +71,54 @@ struct vec<2> { constexpr static std::size_t Dim = 2; - union - { - struct { - double x; - double y; - }; - double components[2]; - }; - - FLECSI_INLINE_TARGET vec<2>() {} + std::array v; - FLECSI_INLINE_TARGET vec<2>(double v) { - x = v; - y = v; + FLECSI_INLINE_TARGET vec<2>(double x) { + v[0] = x; + v[1] = x; } - FLECSI_INLINE_TARGET vec<2>(double x, double y) { - x = x; - y = y; + v[0] = x; + v[1] = y; } - FLECSI_INLINE_TARGET vec<2>(const vec<2> & v) { - x = v.x; - y = v.y; - } + FLECSI_INLINE_TARGET vec<2>() = default; + FLECSI_INLINE_TARGET ~vec<2>() = default; FLECSI_INLINE_TARGET double & operator[](std::size_t d) noexcept { - return components[d]; + return v[d]; } FLECSI_INLINE_TARGET double operator[](std::size_t d) const noexcept { - return components[d]; + return v[d]; + } + + FLECSI_INLINE_TARGET double & x() { + return v[0]; + } + FLECSI_INLINE_TARGET double & y() { + return v[1]; + } + FLECSI_INLINE_TARGET double x() const { + return v[0]; + } + FLECSI_INLINE_TARGET double y() const { + return v[1]; } FLECSI_INLINE_TARGET double norm_squared() const { - return x * x + y * y; + return v[0] * v[0] + v[1] * v[1]; } FLECSI_INLINE_TARGET double norm() const { return std::sqrt(norm_squared()); } FLECSI_INLINE_TARGET double & get(const size_t idx) { assert(idx <= 1 && "Invalid access for 2d vector"); - return components[idx]; + return v[idx]; } FLECSI_INLINE_TARGET const double & get(const size_t idx) const { assert(idx <= 1 && "Invalid access for 2d vector"); - return components[idx]; + return v[idx]; } }; @@ -130,57 +127,63 @@ struct vec<3> { constexpr static std::size_t Dim = 3; - union - { - struct { - double x; - double y; - double z; - }; - double components[3]; - }; + std::array v; - FLECSI_INLINE_TARGET vec<3>() {} - - FLECSI_INLINE_TARGET vec<3>(double v) { - x = v; - y = v; - z = v; + FLECSI_INLINE_TARGET vec<3>(double x) { + v[0] = x; + v[1] = x; + v[2] = x; } - FLECSI_INLINE_TARGET vec<3>(double x, double y, double z) { - x = x; - y = y; - z = z; + v[0] = x; + v[1] = y; + v[2] = z; } - FLECSI_INLINE_TARGET vec<3>(const vec<3> & v) { - x = v.x; - y = v.y; - z = v.z; - } + FLECSI_INLINE_TARGET vec<3>() = default; + FLECSI_INLINE_TARGET ~vec<3>() = default; FLECSI_INLINE_TARGET double & operator[](std::size_t d) noexcept { - return components[d]; + return v[d]; } FLECSI_INLINE_TARGET double operator[](std::size_t d) const noexcept { - return components[d]; + return v[d]; + } + + FLECSI_INLINE_TARGET double & x() { + return v[0]; + } + FLECSI_INLINE_TARGET double & y() { + return v[1]; + } + FLECSI_INLINE_TARGET double & z() { + return v[2]; + } + + FLECSI_INLINE_TARGET double x() const { + return v[0]; + } + FLECSI_INLINE_TARGET double y() const { + return v[1]; + } + FLECSI_INLINE_TARGET double z() const { + return v[2]; } FLECSI_INLINE_TARGET double norm_squared() const { - return x * x + y * y + z * z; + return v[0] * v[0] + v[1] * v[1] + v[2] * v[2]; } FLECSI_INLINE_TARGET double norm() const { return std::sqrt(norm_squared()); } FLECSI_INLINE_TARGET double & get(const size_t idx) { assert(idx <= 2 && "Invalid access for 3d vector"); - return components[idx]; + return v[idx]; } FLECSI_INLINE_TARGET const double & get(const size_t idx) const { assert(idx <= 2 && "Invalid access for 3d vector"); - return components[idx]; + return v[idx]; } }; @@ -192,11 +195,11 @@ template std::ostream & operator<<(std::ostream & s, vec const & v) { if constexpr(D == 1) - s << "(" << v.x << ")"; + s << "(" << v[0] << ")"; else if constexpr(D == 2) - s << "(" << v.x << ", " << v.y << ")"; + s << "(" << v[0] << ", " << v[1] << ")"; else - s << "(" << v.x << ", " << v.y << ", " << v.z << ")"; + s << "(" << v[0] << ", " << v[1] << ", " << v[2] << ")"; return s; } @@ -204,11 +207,11 @@ template FLECSI_INLINE_TARGET vec operator+(const vec & a, const vec & b) { vec result; - result.x = a.x + b.x; + result[0] = a[0] + b[0]; if constexpr(D > 1) - result.y = a.y + b.y; + result[1] = a[1] + b[1]; if constexpr(D > 2) - result.z = a.z + b.z; + result[2] = a[2] + b[2]; return result; } @@ -217,11 +220,11 @@ template FLECSI_INLINE_TARGET vec operator+(const vec & a, double b) { vec result; - result.x = a.x + b; + result[0] = a[0] + b; if constexpr(D > 1) - result.y = a.y + b; + result[1] = a[1] + b; if constexpr(D > 2) - result.z = a.z + b; + result[2] = a[2] + b; return result; } @@ -230,11 +233,11 @@ template FLECSI_INLINE_TARGET vec operator+(double b, const vec & a) { vec result; - result.x = a.x + b; + result[0] = a[0] + b; if constexpr(D > 1) - result.y = a.y + b; + result[1] = a[1] + b; if constexpr(D > 2) - result.z = a.z + b; + result[2] = a[2] + b; return result; } @@ -243,63 +246,63 @@ template FLECSI_INLINE_TARGET vec operator-(const vec & a, const vec & b) { vec result; - result.x = a.x - b.x; + result[0] = a[0] - b[0]; if constexpr(D > 1) - result.y = a.y - b.y; + result[1] = a[1] - b[1]; if constexpr(D > 2) - result.z = a.z - b.z; + result[2] = a[2] - b[2]; return result; } template FLECSI_INLINE_TARGET void operator+=(vec & a, const vec & b) { - a.x += b.x; + a[0] += b[0]; if constexpr(D > 1) - a.y += b.y; + a[1] += b[1]; if constexpr(D > 2) - a.z += b.z; + a[2] += b[2]; } template FLECSI_INLINE_TARGET void operator+=(vec & a, double b) { - a.x += b; + a[0] += b; if constexpr(D > 1) - a.y += b; + a[1] += b; if constexpr(D > 2) - a.z += b; + a[2] += b; } template FLECSI_INLINE_TARGET void operator+=(double b, vec & a) { - a.x += b; + a[0] += b; if constexpr(D > 1) - a.y += b; + a[1] += b; if constexpr(D > 2) - a.z += b; + a[2] += b; } template FLECSI_INLINE_TARGET void operator-=(vec & a, const vec & b) { - a.x -= b.x; + a[0] -= b[0]; if constexpr(D > 1) - a.y -= b.y; + a[1] -= b[1]; if constexpr(D > 2) - a.z -= b.z; + a[2] -= b[2]; } template FLECSI_INLINE_TARGET vec operator*(const vec & a, double s) { vec result; - result.x = a.x * s; + result[0] = a[0] * s; if constexpr(D > 1) - result.y = a.y * s; + result[1] = a[1] * s; if constexpr(D > 2) - result.z = a.z * s; + result[2] = a[2] * s; return result; } @@ -313,11 +316,11 @@ template FLECSI_INLINE_TARGET vec operator*(const vec & a, const vec & b) { vec result; - result.x = a.x * b.x; + result[0] = a[0] * b[0]; if constexpr(D > 1) - result.y = a.y * b.y; + result[1] = a[1] * b[1]; if constexpr(D > 2) - result.z = a.z * b.z; + result[2] = a[2] * b[2]; return result; } @@ -325,11 +328,11 @@ template FLECSI_INLINE_TARGET vec operator/(const vec & a, double s) { vec result; - result.x = a.x / s; + result[0] = a[0] / s; if constexpr(D > 1) - result.y = a.y / s; + result[1] = a[1] / s; if constexpr(D > 2) - result.z = a.z / s; + result[2] = a[2] / s; return result; } @@ -337,11 +340,11 @@ template FLECSI_INLINE_TARGET vec operator/(const vec & a, const vec & b) { vec result; - result.x = a.x / b.x; + result[0] = a[0] / b[0]; if constexpr(D > 1) - result.y = a.y / b.y; + result[1] = a[1] / b[1]; if constexpr(D > 2) - result.z = a.z / b.z; + result[2] = a[2] / b[2]; return result; } @@ -349,11 +352,11 @@ template FLECSI_INLINE_TARGET vec abs(const vec & a) { vec result; - result.x = std::abs(a.x); + result[0] = std::abs(a[0]); if constexpr(D > 1) - result.y = std::abs(a.y); + result[1] = std::abs(a[1]); if constexpr(D > 2) - result.z = std::abs(a.z); + result[2] = std::abs(a[2]); return result; } diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 0000000..88711cf --- /dev/null +++ b/tools/README.md @@ -0,0 +1,46 @@ +# Darwin + +Get a `scaling` node on Darwin: + +``` +$ salloc -p scaling -t 01:00:00 +``` + +Follow the instructions on the main README.md. +The following packages should be loaded in the environment before concretizing: + +You can load the following compilers/mpi, and load them in Spack: + +``` +$ module load openmpi/5.0.2-gcc_13.2.0 +$ spack compiler find +``` + +You can add the openmpi by adding it in the `package.py` file: + +``` shell +$ vim ~/.spack/packages.py +``` +And add: + +``` yaml +packages: + openmpi: + externals: + - spec: openmpi@5.0.2 + prefix: /projects/opt/rhel8/x86_64/openmpi/5.0.2-gcc_13.2.0/ +``` + +You can then run the concretizer. + +## Format + Python + +In order to use the python script and the clang format I would recommand loading the packages in the following order: + +``` +module load clang/13.0.0 +spack env activate hard +module load miniconda3 +``` + +# Chicoma diff --git a/tools/acoustic_solution.py b/tools/acoustic_solution.py new file mode 100644 index 0000000..431017d --- /dev/null +++ b/tools/acoustic_solution.py @@ -0,0 +1,72 @@ +import numpy as np +from numpy.typing import NDArray + + +class Acoustic(object): + """ + Class for acoustic wave solution + """ + + def __init__(self, gamma: float, x0: float, x1: float, + problem_dict: dict[str, str]) -> None: + + self.gamma = gamma + self.x0 = x0 + self.x1 = x1 + self.problem_dict = problem_dict + + self.r0 = float(self.problem_dict["r0"]) + self.p0 = float(self.problem_dict["p0"]) + self.cs = np.sqrt(gamma * self.p0 / self.r0) + + self.amplitude = float(self.problem_dict["amplitude"]) + self.scale = 2 * np.pi * float(self.problem_dict["scale"]) + + def __perturbation(self): + """ + Perturbation function shape + """ + + return lambda x: np.sin(self.scale * x) * self.amplitude + + def __call__(self, x: NDArray | float, t: float) -> dict[str, NDArray]: + """ + Return named tuple with what the density, pressure and velocity are at + time t + + Assume periodic boundary conditions + """ + + # Take the initial solution and transport it by cs * t, assuming + # periodic boundary conditions + + perturbation = self.__perturbation() + span = abs(self.x1 - self.x0) + + density = self.r0 + pressure = self.p0 + + # Positive movement + x_init = (x - self.cs * t - self.x0) % span + self.x0 + + density += perturbation(x_init) * 0.5 + pressure += perturbation(x_init) * 0.5 * self.cs ** 2 + velocity = perturbation(x_init) * 0.5 * self.cs + + # Entropic mode (static density) + density += perturbation(x) + + # Negative movement + x_init = (x + self.cs * t - self.x0) % span + self.x0 + + density -= perturbation(x_init) * 0.5 + pressure -= perturbation(x_init) * 0.5 * self.cs ** 2 + velocity += perturbation(x_init) * 0.5 * self.cs + + solution = { + "density": density, + "pressure": pressure, + "velocity": velocity + } + + return solution diff --git a/tools/convergence_verify.py b/tools/convergence_verify.py new file mode 100644 index 0000000..10a3ac0 --- /dev/null +++ b/tools/convergence_verify.py @@ -0,0 +1,101 @@ +import glob +import os +from collections.abc import Callable + +import matplotlib.pyplot as plt +import numpy as np +from acoustic_solution import Acoustic +from verify_lib import (compute_l1_error_fvm, find_last_output, parse_cli, + parse_config, wrapFunction) + + +def main() -> None: + """ + Test convergence of the code + """ + + # Slope tolerance + tol = 1e-1 + + # Convergence order + ord = 3 + + # Get all files + yaml_file, out_dir, _, make_plot = parse_cli(get_file=False) + problem, gamma, x0, x1, problem_dict = parse_config(yaml_file) + + # Find every example and build the dx and L1 error arrays + if out_dir is None: + pattern = "output_*" + else: + pattern = os.path.join(out_dir, "output_*") + + dirs = glob.glob(pattern) + dirs.sort(key=lambda x: int(x.split("_")[-1])) + dx = [] + l1err = [] + + acoustic_instance: wrapFunction + u_exact: Callable + + first_loop = True + for dir in dirs: + last_output = find_last_output(dir=dir) + assert last_output is not None + + out_tuple = np.loadtxt(last_output, delimiter=",", + skiprows=1, + usecols=(0, 2, 3, 4, 9)).T + + # Extract physical quantities from tuple + t_arr, x_num, rho_num, p_num, u_num = out_tuple + time = t_arr[0] + + # Instantiate our solution class in the first loop + if first_loop: + acoustic_instance = wrapFunction( + Acoustic(gamma, x0, x1, problem_dict), time, + ["density", "pressure", "velocity"]) + u_exact = acoustic_instance.velocity + + first_loop = False + + dx.append(x_num[1] - x_num[0]) + l1err.append(compute_l1_error_fvm(x_num, u_num, u_exact)) + + if make_plot: + def plot_order(dx, l1err, ord): + plt.plot(dx, np.array(dx) ** ord * + l1err[0] / dx[0] ** ord, "--", label=f"order {ord}") + + plt.plot(dx, l1err, "o-") + plot_order(dx, l1err, 3) + plot_order(dx, l1err, 2) + plot_order(dx, l1err, 1) + + plt.xscale("log") + plt.yscale("log") + + plt.xlabel("dx") + plt.ylabel("L1 error") + plt.title("Error convergence") + + plt.legend() + plt.savefig("convergence.pdf") + + # Find slope of logarithms for convergence test + logE = np.log(l1err) + logDX = np.log(dx) + slope = (logE[1:] - logE[:-1]) / (logDX[1:] - logDX[:-1]) + + # Assert that slope is expected order + try: + assert abs(abs(np.mean(slope)) - ord) < tol + except AssertionError: + s = f"Mean of slope is {np.mean(slope)}" + print(s) + raise + + +if __name__ == "__main__": + main() diff --git a/tools/exactpack_verify.py b/tools/exactpack_verify.py new file mode 100644 index 0000000..38d3698 --- /dev/null +++ b/tools/exactpack_verify.py @@ -0,0 +1,290 @@ +import os +import sys +from collections.abc import Callable + +import matplotlib.pyplot as plt +import numpy as np +from acoustic_solution import Acoustic +from numpy.typing import NDArray +from verify_lib import (compute_l1_error_fvm, parse_cli, parse_config, + wrapFunction) + + +class ProblemData(object): + """ + Define the quantities for each problem + """ + + def __init__(self, name: str): + """ + name - problem name + """ + + # Define names for Riemann problems + self.riemann_problems = ["sod", "leblanc", "rankine-hugoniot"] + + # Default values + self.name = name + self.usecols = [0, 2, 3, 4, 9] + self.extract = ["density", "pressure", "velocity"] + self.labels = ["Density", "Pressure", "Velocity"] + self.tolerances = [1e-2, 1e-2, 5e-1] + self.function: Callable + + # Define specifics per problem and analytical solutions + if self.name in self.riemann_problems: + self.function = self.__riemann_analytic_solution + elif self.name == "sedov": + self.function = self.__sedov_analytic_solution + elif self.name == "acoustic-wave": + self.function = self.__acoustic_analytic_solution + else: + sys.exit(f"Unsupported problem type '{name}'") + + # For riemann problems, define left and right states + if self.name in self.riemann_problems: + if self.name == "sod": + self.left_state = (1.0, 0.0, 1.0) + self.right_state = (0.125, 0.0, 0.1) + elif self.name == "leblanc": + self.left_state = (1.0, 0.0, 0.1) + self.right_state = (1e-3, 0.0, 1e-10) + elif self.name == "rankine-hugoniot": + self.left_state = (1.0, 0.0, 1.0) + self.right_state = (0.25, 0.0, 0.1795) + + def __acoustic_analytic_solution(self, gamma: float, t: float, x0: float, + x1: float, problem_dict: dict[str, str] + ) -> tuple[Callable, Callable, Callable]: + """ + Return the acoustic analytical solution + """ + + result = wrapFunction( + Acoustic(gamma, x0, x1, problem_dict), t, self.extract) + + return result.density, result.pressure, result.velocity + + def __sedov_analytic_solution(self, x: NDArray, t: float, gamma: float + ) -> tuple[Callable, Callable, Callable]: + + from exactpack.solvers.sedov import Sedov + + solver = Sedov(gamma=gamma, geometry=1, eblast=0.0673185) + result = wrapFunction(solver, t, self.extract) + + return result.density, result.pressure, result.velocity + + def __riemann_analytic_solution(self, x: NDArray, t: float, gamma: float, + left_state: tuple[float, float, float], + right_state: tuple[float, float, float] + ) -> tuple[Callable, Callable, Callable]: + + try: + from exactpack.solvers.riemann.ep_riemann import IGEOS_Solver + except ImportError: + sys.exit("Error: ExactPack Riemann solver not found.") + + rho_l, u_l, p_l = left_state + rho_r, u_r, p_r = right_state + + solver = IGEOS_Solver( + rl=rho_l, ul=u_l, pl=p_l, gl=gamma, + rr=rho_r, ur=u_r, pr=p_r, gr=gamma, + xmin=min(x), xd0=0.5 * (min(x) + max(x)), xmax=max(x), t=t + ) + + sol = wrapFunction(solver, t, self.extract) + return sol.density, sol.pressure, sol.velocity + + +class Problem(object): + """ + Class to hold each type of problem parameters + """ + + def __init__(self, yaml_file: str) -> None: + """ + yaml_file - the yaml_file with the problem configuration + """ + + name, gamma, x0, x1, problem_dict = parse_config(yaml_file) + + self.name = name + self.gamma = gamma + self.x0 = x0 + self.x1 = x1 + self.problem_dict = problem_dict + + # Initialize quantities + self.data = ProblemData(self.name) + + # Variables to be defined later + self.time: float + self.x_arr: NDArray + self.numerical: list[NDArray] + self.references: list[NDArray] + self.analytical: list[Callable] + self.errors: list[float] + self.status: list[str] + + def load_csv_file(self, csv_file: str) -> None: + """ + Load the last csv file + """ + self.csv_file = csv_file + csv_out = np.loadtxt(csv_file, delimiter=",", + skiprows=1, usecols=self.data.usecols).T + + # Extract time and x_arr + t_index = self.data.usecols.index(0) + x_index = self.data.usecols.index(2) + self.time = csv_out[t_index][0] + self.x_arr = csv_out[x_index] + + # Extract the other quantities + self.numerical = [x for i, x in enumerate( + csv_out) if i not in [t_index, x_index]] + + def get_exact_solutions(self) -> None: + """ + Get the exact solutions depending on the problem + """ + + assert self.x_arr is not None + assert self.time is not None + + # Create input interator + if self.name in self.data.riemann_problems: + input = [self.x_arr, self.time, self.gamma, + self.data.left_state, self.data.right_state] + elif self.name == "sedov": + input = [self.x_arr, self.time, self.gamma] + elif self.name == "acoustic-wave": + input = [self.gamma, self.time, + self.x0, self.x1, self.problem_dict] + elif self.name == "su-olson": + input = [self.x_arr, self.time, self.data.extract] + + # Save the exact output functions + self.analytical = self.data.function(*input) + self.references = [f(self.x_arr) for f in self.analytical] + + def __plot_comparison(self, num_vals: NDArray, exact_vals: NDArray, + label: str, tag: str) -> None: + """ + Plot the numerical output and the reference + """ + + plt.figure() + plt.plot(self.x_arr, exact_vals, label="Analytic", linestyle="--") + plt.plot(self.x_arr, num_vals, label="Simulation", marker='o', + linestyle='none', markersize=4) + + plt.xlabel("x") + plt.ylabel(label) + plt.title(f"{label} at t = {self.time:.4f}") + + plt.legend() + plt.grid(True) + plt.tight_layout() + + filename = f"{label.lower()}_comparison_{tag}.png" + plt.savefig(filename) + plt.close() + + print(f"Saved plot: {filename}") + + def make_plot(self) -> None: + """ + Create plot with references + """ + + # Return the references + tag = os.path.splitext(os.path.basename(self.csv_file))[0].replace( + "output-", "").replace(".csv", "") + + for lab, num, ref in zip(self.data.labels, self.numerical, + self.references): + self.__plot_comparison(num, ref, lab, tag) + + def sedov_cutoff(self, cutoff: float = 0.2) -> None: + """ + Remove the values up to cutoff for the sedov solution + """ + + mask = self.x_arr > cutoff + self.x_arr = self.x_arr[mask] + self.numerical = [x[mask] for x in self.numerical] + self.references = [x[mask] for x in self.references] + + def compute_errors(self) -> None: + """ + Compute all the errors + """ + + self.errors = [compute_l1_error_fvm(self.x_arr, num, exact) + for num, exact in zip(self.numerical, self.analytical)] + + def __color_text(self, text: str, status: str) -> str: + if status == "PASS": + return f"\033[92m{text}\033[0m" # green + elif status == "FAIL": + return f"\033[91m{text}\033[0m" # red + else: + return text + + def check_error_status(self) -> None: + """ + Get status for every error + """ + + tols = self.data.tolerances + failed = [False if x < y else True for x, y in zip(self.errors, tols)] + statuses = ["FAIL" if x else "PASS" for x in failed] + + print(f"Problem type: {self.name}") + print(f"Compared solution at t = {self.time:.4f}") + print("L1 Errors and Status:") + for label, err, stat in zip(self.data.labels, self.errors, statuses): + print(f" {label} : {err:.6e} [{self.__color_text(stat, stat)}]") + + if any(failed): + s = "Test FAILED: relative L1 error exceeds tolerance of " + s += " ".join([f"{tol:.2e}" for tol in tols]) + sys.exit(s) + else: + print("Test PASSED") + + +def main() -> None: + + # Get the values + yaml_file, _, csv_file, make_plot = parse_cli() + assert csv_file is not None + + # Instantiate problem object + problem = Problem(yaml_file) + + # Load csv file + problem.load_csv_file(csv_file) + + # Get exact solutions + problem.get_exact_solutions() + + if make_plot: + problem.make_plot() + + # For Sedov: only compare for x > 0.2 + if problem == "sedov": + problem.sedov_cutoff(cutoff=0.2) + + # Compute errors + problem.compute_errors() + + # Error status + problem.check_error_status() + + +if __name__ == "__main__": + main() diff --git a/tools/verify_lib.py b/tools/verify_lib.py new file mode 100644 index 0000000..8cf939d --- /dev/null +++ b/tools/verify_lib.py @@ -0,0 +1,136 @@ +import glob +import os +import sys +from collections.abc import Callable + +import numpy as np +import yaml +from numpy.typing import NDArray + + +class wrapFunction(object): + + # Declaring the attributes to provide mypy with enough information + density: Callable + pressure: Callable + velocity: Callable + + def __init__(self, solver: Callable, t: float, extract: list[str]) -> None: + """ + Create the wrapper for each attribute + """ + + for name in extract: + self.__doWrapping(solver, t, name) + + def __doWrapping(self, f: Callable, t: float, name: str) -> None: + """ + Transform the ExactPack array tuple into a Callable tuple + """ + + def wrap(x): + return f(x, t)[name] + + self.__dict__[name] = wrap + + +def parse_cli(get_file: bool = True) -> tuple[ + str, + str | None, + str | None, + bool +]: + """ + Parse command line input + """ + + if len(sys.argv) < 2: + s = f"Usage: python {sys.argv[0]} " + s += " [output_dir | output.csv]" + s += " [--plot]" + sys.exit(s) + + # Read in the yaml file + yaml_file = sys.argv[1] + problem, gamma, x0, x1, problem_dict = parse_config(yaml_file) + + csv_file = None + out_dir = None + make_plot = False + + for arg in sys.argv[2:]: + if arg.endswith(".csv"): + csv_file = arg + elif arg == "--plot": + make_plot = True + else: + out_dir = arg + + if get_file: + + # If no file is passed, select the latest available output + if csv_file is None: + csv_file = find_last_output(dir=out_dir) + assert csv_file is not None + + print(f"Auto-selected input file: {csv_file}") + + return yaml_file, out_dir, csv_file, make_plot + + +def simple_quad(f: Callable, x0: float, x1: float, deg=10) -> float: + """ + Use a Gauss-Legendre quadrature of degree deg + """ + + points, weights = np.polynomial.legendre.leggauss(deg) + def transform(x): return ((x1 - x0) * x + x1 + x0) * 0.5 + + return np.sum(f(transform(points)) * weights) * 0.5 * (x1 - x0) + + +def compute_l1_error_fvm(x_num: NDArray, numerical: NDArray, + analytical: Callable) -> float: + + # For every cell, calculate the "volume" integral + dx = x_num[1] - x_num[0] + error = 0 + + for i, x in enumerate(x_num): + x1 = x + dx * 0.5 + x0 = x - dx * 0.5 + + error += abs(dx * numerical[i] - simple_quad(analytical, x0, x1)) + + return error + + +def find_last_output(pattern: str = "output-1D-0-*.csv", + dir: str | None = None) -> str | None: + + if dir is not None: + pattern = os.path.join(dir, pattern) + + files = glob.glob(pattern) + if not files: + print("No matching output files found.") + return None + + # Sort the files so they are in order + files.sort() + + # Return first and last + return files[-1] + + +def parse_config(yaml_file: str + ) -> tuple[str, float, float, float, dict[str, str]]: + with open(yaml_file, 'r') as f: + config = yaml.safe_load(f) + gamma = float(config.get('gamma', 1.4)) + x0 = float(config['coords'][0][0]) + x1 = float(config['coords'][1][0]) + problem = config['problem'] + problem_dict = config.get("problem_parameters") + + return problem, gamma, x0, x1, problem_dict