From 21c5d476a9cc40d362fb2a827f6c619c4c26f081 Mon Sep 17 00:00:00 2001 From: Prabhat Kumar Date: Tue, 16 Jul 2024 10:14:29 -0700 Subject: [PATCH 01/13] Reaction Diffusion Tutorial --- .../Reaction-Diffusion/CMakeLists.txt | 15 ++ .../Reaction-Diffusion/Exec/GNUmakefile | 51 ++++ .../Reaction-Diffusion/Exec/README_sundials | 85 +++++++ .../Exec/inputs_forward_euler | 14 ++ .../Reaction-Diffusion/Exec/inputs_rk3 | 24 ++ .../Reaction-Diffusion/Exec/inputs_sundials | 44 ++++ .../Reaction-Diffusion/Source/Make.package | 2 + .../Reaction-Diffusion/Source/main.cpp | 233 ++++++++++++++++++ .../Reaction-Diffusion/Source/myfunc.H | 6 + 9 files changed, 474 insertions(+) create mode 100644 ExampleCodes/SUNDIALS/Reaction-Diffusion/CMakeLists.txt create mode 100644 ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/GNUmakefile create mode 100644 ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/README_sundials create mode 100644 ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_forward_euler create mode 100644 ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_rk3 create mode 100644 ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials create mode 100644 ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/Make.package create mode 100644 ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp create mode 100644 ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/myfunc.H diff --git a/ExampleCodes/SUNDIALS/Reaction-Diffusion/CMakeLists.txt b/ExampleCodes/SUNDIALS/Reaction-Diffusion/CMakeLists.txt new file mode 100644 index 00000000..71eb16ae --- /dev/null +++ b/ExampleCodes/SUNDIALS/Reaction-Diffusion/CMakeLists.txt @@ -0,0 +1,15 @@ +if (AMReX_SPACEDIM EQUAL 1) + return() +endif () + +# List of source files +set(_sources main.cpp myfunc.H) +list(TRANSFORM _sources PREPEND "Source/") + +# List of input files +file( GLOB_RECURSE _input_files LIST_DIRECTORIES false Exec/input* ) + +setup_tutorial(_sources _input_files) + +unset( _sources ) +unset( _input_files ) diff --git a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/GNUmakefile b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/GNUmakefile new file mode 100644 index 00000000..7947b0b7 --- /dev/null +++ b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/GNUmakefile @@ -0,0 +1,51 @@ +# AMREX_HOME defines the directory in which we will find all the AMReX code. +AMREX_HOME ?= ../../../../../amrex + +DEBUG = FALSE +USE_MPI = TRUE +USE_OMP = FALSE +COMP = gnu +DIM = 3 +USE_RPATH = TRUE +USE_SUNDIALS = TRUE + +include $(AMREX_HOME)/Tools/GNUMake/Make.defs + +include ../Source/Make.package +VPATH_LOCATIONS += ../Source +INCLUDE_LOCATIONS += ../Source + +ifeq ($(USE_SUNDIALS),TRUE) + ifeq ($(USE_CUDA),TRUE) + SUNDIALS_ROOT ?= $(TOP)../../../../../sundials/instdir_cuda + else + SUNDIALS_ROOT ?= $(TOP)../../../../../sundials/instdir + endif + ifeq ($(NERSC_HOST),perlmutter) + SUNDIALS_LIB_DIR ?= $(SUNDIALS_ROOT)/lib64 + else + SUNDIALS_LIB_DIR ?= $(SUNDIALS_ROOT)/lib + endif + + USE_CVODE_LIBS ?= TRUE + USE_ARKODE_LIBS ?= TRUE + + DEFINES += -DAMREX_USE_SUNDIALS + INCLUDE_LOCATIONS += $(SUNDIALS_ROOT)/include + LIBRARY_LOCATIONS += $(SUNDIALS_LIB_DIR) + + LIBRARIES += -L$(SUNDIALS_LIB_DIR) -lsundials_cvode + LIBRARIES += -L$(SUNDIALS_LIB_DIR) -lsundials_arkode + LIBRARIES += -L$(SUNDIALS_LIB_DIR) -lsundials_nvecmanyvector + LIBRARIES += -L$(SUNDIALS_LIB_DIR) -lsundials_core + + ifeq ($(USE_CUDA),TRUE) + LIBRARIES += -L$(SUNDIALS_LIB_DIR) -lsundials_nveccuda + endif + +endif + +include $(AMREX_HOME)/Src/Base/Make.package + +include $(AMREX_HOME)/Tools/GNUMake/Make.rules + diff --git a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/README_sundials b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/README_sundials new file mode 100644 index 00000000..c0f315aa --- /dev/null +++ b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/README_sundials @@ -0,0 +1,85 @@ +SUNDIALS installation guide: +https://computing.llnl.gov/projects/sundials/faq#inst + +Installation + +# Download sundials-x.y.z.tar.gz file for SUNDIALS and extract it at the same level as amrex. +# Update 6/11/24 - v7.0.0. tested +https://computing.llnl.gov/projects/sundials/sundials-software + +>> tar -xzvf sundials-x.y.z.tar.gz # where x.y.z is the version of sundials + +# at the same level that amrex is cloned, do: + +>> mkdir sundials +>> cd sundials + +###################### +HOST BUILD +###################### + +>> mkdir instdir +>> mkdir builddir +>> cd builddir + +>> cmake -DCMAKE_INSTALL_PREFIX=/pathto/sundials/instdir -DEXAMPLES_INSTALL_PATH=/pathto/sundials/instdir/examples -DENABLE_MPI=ON ../../sundials-x.y.z + +###################### +NVIDIA/CUDA BUILD +###################### + +# Navigate back to the 'sundials' directory and do: + +>> mkdir instdir_cuda +>> mkdir builddir_cuda +>> cd builddir_cuda + +>> cmake -DCMAKE_INSTALL_PREFIX=/pathto/sundials/instdir_cuda -DEXAMPLES_INSTALL_PATH=/pathto/sundials/instdir_cuda/examples -DENABLE_CUDA=ON -DENABLE_MPI=ON ../../sundials-x.y.z + +###################### + +>> make -j4 +>> make install + +# in your .bashrc or preferred configuration file, add the following (and then "source ~/.bashrc") + +# If you have a CPU build: + +>> export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/pathto/sundials/instdir/lib/ + +# If you have a NVIDIA/CUDA build: + +>> export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/pathto/sundials/instdir_cuda/lib/ + +# now you are ready to compile amrex-tutorials/ExampleCodes/SUNDIALS/Exec with: + +>> make -j4 # optional to have 'USE_CUDA=TRUE' as well + +# in your inputs file, you will need to have: + +# INTEGRATION +## integration.type can take on the following values: +## 0 or "ForwardEuler" => Native AMReX Forward Euler integrator +## 1 or "RungeKutta" => Native AMReX Explicit Runge Kutta controlled by integration.rk.type +## 2 or "SUNDIALS" => SUNDIALS backend controlled by integration.sundials.strategy +integration.type = + +## Native AMReX Explicit Runge-Kutta parameters +# +## integration.rk.type can take the following values: +### 0 = User-specified Butcher Tableau +### 1 = Forward Euler +### 2 = Trapezoid Method +### 3 = SSPRK3 Method +### 4 = RK4 Method +integration.rk.type = 3 + +## If using the SUNDIALS Submodule, then +## compile with USE_SUNDIALS=TRUE or AMReX_SUNDIALS=ON and +## set strategy here: +# +## integration.sundials.strategy can take the following values: +### ERK = ERKStep from ARKode with SSPRK3 Method +### MRI = MRIStep from ARKode with Explict Trapezoid Method +### MRITEST = MRIStep from ARKode modified to use no-op inner f0 +integration.sundials.strategy = ERK diff --git a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_forward_euler b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_forward_euler new file mode 100644 index 00000000..39b09830 --- /dev/null +++ b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_forward_euler @@ -0,0 +1,14 @@ +n_cell = 32 +max_grid_size = 16 + +nsteps = 1000 +plot_int = 100 + +dt = 1.e-5 + +# INTEGRATION +## integration.type can take on the following values: +## 0 or "ForwardEuler" => Native AMReX Forward Euler integrator +## 1 or "RungeKutta" => Native AMReX Explicit Runge Kutta controlled by integration.rk.type +## 2 or "SUNDIALS" => SUNDIALS backend controlled by integration.sundials.strategy +integration.type = ForwardEuler diff --git a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_rk3 b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_rk3 new file mode 100644 index 00000000..6eb60137 --- /dev/null +++ b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_rk3 @@ -0,0 +1,24 @@ +n_cell = 32 +max_grid_size = 16 + +nsteps = 1000 +plot_int = 100 + +dt = 1.e-5 + +# INTEGRATION +## integration.type can take on the following values: +## 0 or "ForwardEuler" => Native AMReX Forward Euler integrator +## 1 or "RungeKutta" => Native AMReX Explicit Runge Kutta controlled by integration.rk.type +## 2 or "SUNDIALS" => SUNDIALS backend controlled by integration.sundials.strategy +integration.type = RungeKutta + +## Native AMReX Explicit Runge-Kutta parameters +# +## integration.rk.type can take the following values: +### 0 = User-specified Butcher Tableau +### 1 = Forward Euler +### 2 = Trapezoid Method +### 3 = SSPRK3 Method +### 4 = RK4 Method +integration.rk.type = 3 diff --git a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials new file mode 100644 index 00000000..bbdf2c69 --- /dev/null +++ b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials @@ -0,0 +1,44 @@ +n_cell = 32 +max_grid_size = 16 + +nsteps = 1000 +plot_int = 100 + +dt = 1.e-5 + +# Use adaptive time stepping and set integrator relative and absolute tolerances +# adapt_dt = true +# reltol = 1.0e-4 +# abstol = 1.0e-9 + +# INTEGRATION +# integration.type can take on the following values: +# 0 or "ForwardEuler" => Native AMReX Forward Euler integrator +# 1 or "RungeKutta" => Native AMReX Explicit Runge Kutta controlled by integration.rk.type +# 2 or "SUNDIALS" => SUNDIALS backend controlled by integration.sundials.type +# +# If using the SUNDIALS Submodule, then compile with USE_SUNDIALS=TRUE or +# AMReX_SUNDIALS=ON +integration.type = SUNDIALS + +# Set the SUNDIALS method type: +# ERK = Explicit Runge-Kutta method +# DIRK = Diagonally Implicit Runge-Kutta method +# IMEX-RK = Implicit-Explicit Additive Runge-Kutta method +# EX-MRI = Explicit Multirate Infatesimal method +# IM-MRI = Implicit Multirate Infatesimal method +# IMEX-MRI = Implicit-Explicit Multirate Infatesimal method +# +# Optionally select a specific SUNDIALS method by name, see the SUNDIALS +# documentation for the supported method names + +# Use the SUNDIALS default method for the chosen type (fixed or adaptive step sizes) +integration.sundials.type = ERK + +# Use forward Euler (fixed step sizes only) +# integration.sundials.type = ERK +# integration.sundials.method = ARKODE_FORWARD_EULER_1_1 + +# Use backward Euler (fixed step sizes only) +# integration.sundials.type = DIRK +# integration.sundials.method = ARKODE_BACKWARD_EULER_1_1 diff --git a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/Make.package b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/Make.package new file mode 100644 index 00000000..fa194a35 --- /dev/null +++ b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/Make.package @@ -0,0 +1,2 @@ +CEXE_sources += main.cpp +CEXE_headers += myfunc.H diff --git a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp new file mode 100644 index 00000000..4eab09d4 --- /dev/null +++ b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp @@ -0,0 +1,233 @@ +#include +#include +#include + +#include "myfunc.H" + +using namespace amrex; + +int main (int argc, char* argv[]) +{ + amrex::Initialize(argc,argv); + + main_main(); + + amrex::Finalize(); + return 0; +} + +void main_main () +{ + + // ********************************** + // SIMULATION PARAMETERS + + // number of cells on each side of the domain + int n_cell; + + // size of each box (or grid) + int max_grid_size; + + // total steps in simulation + int nsteps; + + // how often to write a plotfile + int plot_int; + + // time step + Real dt; + + // use adaptive time step (dt used to set output times) + bool adapt_dt = false; + + // adaptive time step relative and absolute tolerances + Real reltol = 1.0e-4; + Real abstol = 1.0e-9; + + // inputs parameters + { + // ParmParse is way of reading inputs from the inputs file + // pp.get means we require the inputs file to have it + // pp.query means we optionally need the inputs file to have it - but we must supply a default here + ParmParse pp; + + // We need to get n_cell from the inputs file - this is the number of cells on each side of + // a square (or cubic) domain. + pp.get("n_cell",n_cell); + + // The domain is broken into boxes of size max_grid_size + pp.get("max_grid_size",max_grid_size); + + // Default nsteps to 10, allow us to set it to something else in the inputs file + nsteps = 10; + pp.query("nsteps",nsteps); + + // Default plot_int to -1, allow us to set it to something else in the inputs file + // If plot_int < 0 then no plot files will be written + plot_int = -1; + pp.query("plot_int",plot_int); + + // time step + pp.get("dt",dt); + + // use adaptive step sizes + pp.query("adapt_dt",adapt_dt); + + // adaptive step tolerances + pp.query("reltol",reltol); + pp.query("abstol",abstol); + + } + + // ********************************** + // SIMULATION SETUP + + // make BoxArray and Geometry + // ba will contain a list of boxes that cover the domain + // geom contains information such as the physical domain size, + // number of points in the domain, and periodicity + BoxArray ba; + Geometry geom; + + // AMREX_D_DECL means "do the first X of these, where X is the dimensionality of the simulation" + IntVect dom_lo(AMREX_D_DECL( 0, 0, 0)); + IntVect dom_hi(AMREX_D_DECL(n_cell-1, n_cell-1, n_cell-1)); + + // Make a single box that is the entire domain + Box domain(dom_lo, dom_hi); + + // Initialize the boxarray "ba" from the single box "domain" + ba.define(domain); + + // Break up boxarray "ba" into chunks no larger than "max_grid_size" along a direction + ba.maxSize(max_grid_size); + + // This defines the physical box, [0,1] in each direction. + RealBox real_box({AMREX_D_DECL( 0., 0., 0.)}, + {AMREX_D_DECL( 1., 1., 1.)}); + + // periodic in all direction + Array is_periodic{AMREX_D_DECL(1,1,1)}; + + // This defines a Geometry object + geom.define(domain, real_box, CoordSys::cartesian, is_periodic); + + // extract dx from the geometry object + GpuArray dx = geom.CellSizeArray(); + + // Nghost = number of ghost cells for each array + int Nghost = 1; + + // Ncomp = number of components for each array + int Ncomp = 1; + + // How Boxes are distrubuted among MPI processes + DistributionMapping dm(ba); + + // we allocate two phi multifabs; one will store the old state, the other the new. + MultiFab phi(ba, dm, Ncomp, Nghost); + + // time = starting time in the simulation + Real time = 0.0; + + // ********************************** + // INITIALIZE DATA + + // loop over boxes + for (MFIter mfi(phi); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.validbox(); + + const Array4& phi_array = phi.array(mfi); + + // set phi = 1 + e^(-(r-0.5)^2) + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) + { + Real x = (i+0.5) * dx[0]; + Real y = (j+0.5) * dx[1]; +#if (AMREX_SPACEDIM == 2) + Real rsquared = ((x-0.5)*(x-0.5)+(y-0.5)*(y-0.5))/0.01; +#elif (AMREX_SPACEDIM == 3) + Real z= (k+0.5) * dx[2]; + Real rsquared = ((x-0.5)*(x-0.5)+(y-0.5)*(y-0.5)+(z-0.5)*(z-0.5))/0.01; +#endif + phi_array(i,j,k) = 1. + std::exp(-rsquared); + }); + } + + // Write a plotfile of the initial data if plot_int > 0 + if (plot_int > 0) + { + int step = 0; + const std::string& pltfile = amrex::Concatenate("plt",step,5); + WriteSingleLevelPlotfile(pltfile, phi, {"phi"}, geom, time, 0); + } + + auto pre_rhs_function = [&](MultiFab& S_data, const Real /* time */) { + // fill periodic ghost cells + S_data.FillBoundary(geom.periodicity()); + }; + + auto rhs_function = [&](MultiFab& S_rhs, const MultiFab& S_data, + const Real /* time */) { + + // loop over boxes + auto& phi_data = S_data; + auto& phi_rhs = S_rhs; + + //Reaction and Diffusion Coefficients + Real reaction_coef = 0.0; + Real diffusion_coef = 1.0; + ParmParse pp; + pp.query("reaction_coef",reaction_coef); + pp.query("diffusion_coef",diffusion_coef); + + for ( MFIter mfi(phi_data); mfi.isValid(); ++mfi ) + { + const Box& bx = mfi.validbox(); + + const Array4& phi_array = phi_data.array(mfi); + const Array4& phi_rhs_array = phi_rhs.array(mfi); + + // fill the right-hand-side for phi + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) + { + phi_rhs_array(i,j,k) = diffusion_coef*( (phi_array(i+1,j,k) - 2.*phi_array(i,j,k) + phi_array(i-1,j,k)) / (dx[0]*dx[0]) + +(phi_array(i,j+1,k) - 2.*phi_array(i,j,k) + phi_array(i,j-1,k)) / (dx[1]*dx[1]) +#if (AMREX_SPACEDIM == 3) + +(phi_array(i,j,k+1) - 2.*phi_array(i,j,k) + phi_array(i,j,k-1)) / (dx[2]*dx[2]) ) +#endif + + reaction_coef*phi_array(i,j,k); + }); + } + }; + + TimeIntegrator integrator(phi, time); + integrator.set_pre_rhs_action(pre_rhs_function); + integrator.set_rhs(rhs_function); + if (adapt_dt) { + integrator.set_adaptive_step(); + integrator.set_tolerances(reltol, abstol); + } else { + integrator.set_time_step(dt); + } + + for (int step = 1; step <= nsteps; ++step) + { + // Set time to evolve to + time += dt; + + // Advance to output time + integrator.evolve(phi, time); + + // Tell the I/O Processor to write out which step we're doing + amrex::Print() << "Advanced step " << step << "\n"; + + // Write a plotfile of the current data (plot_int was defined in the inputs file) + if (plot_int > 0 && step%plot_int == 0) + { + const std::string& pltfile = amrex::Concatenate("plt",step,5); + WriteSingleLevelPlotfile(pltfile, phi, {"phi"}, geom, time, step); + } + } +} diff --git a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/myfunc.H b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/myfunc.H new file mode 100644 index 00000000..0f828c4e --- /dev/null +++ b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/myfunc.H @@ -0,0 +1,6 @@ +#ifndef MYFUNC_H_ +#define MYFUNC_H_ + +void main_main (); + +#endif From ffa94a0e83e283390ad3f8faa447d222a3539ce4 Mon Sep 17 00:00:00 2001 From: prkkumar Date: Wed, 17 Jul 2024 09:45:32 -0700 Subject: [PATCH 02/13] fast rhs fun --- .../Reaction-Diffusion/Source/main.cpp | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp index 4eab09d4..4add8ebd 100644 --- a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp +++ b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp @@ -202,9 +202,43 @@ void main_main () } }; + auto rhs_fast_function = [&](MultiFab& S_rhs, const MultiFab& S_data, + const Real /* time */) { + + // loop over boxes + auto& phi_data = S_data; + auto& phi_rhs = S_rhs; + + //Reaction and Diffusion Coefficients + Real reaction_coef = 0.0; + Real diffusion_coef = 1.0; + ParmParse pp; + pp.query("reaction_coef",reaction_coef); + pp.query("diffusion_coef",diffusion_coef); + + for ( MFIter mfi(phi_data); mfi.isValid(); ++mfi ) + { + const Box& bx = mfi.validbox(); + + const Array4& phi_array = phi_data.array(mfi); + const Array4& phi_rhs_array = phi_rhs.array(mfi); + + // fill the right-hand-side for phi + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) + { + phi_rhs_array(i,j,k) = diffusion_coef*( (phi_array(i+1,j,k) - 2.*phi_array(i,j,k) + phi_array(i-1,j,k)) / (dx[0]*dx[0]) + +(phi_array(i,j+1,k) - 2.*phi_array(i,j,k) + phi_array(i,j-1,k)) / (dx[1]*dx[1]) +#if (AMREX_SPACEDIM == 3) + +(phi_array(i,j,k+1) - 2.*phi_array(i,j,k) + phi_array(i,j,k-1)) / (dx[2]*dx[2]) ); +#endif + }); + } + }; + TimeIntegrator integrator(phi, time); integrator.set_pre_rhs_action(pre_rhs_function); integrator.set_rhs(rhs_function); + integrator.set_fast_rhs(rhs_fast_function); if (adapt_dt) { integrator.set_adaptive_step(); integrator.set_tolerances(reltol, abstol); @@ -212,6 +246,10 @@ void main_main () integrator.set_time_step(dt); } + // This sets the ratio of slow timestep size to fast timestep size as an integer, + // or equivalently, the number of fast timesteps per slow timestep. + integrator.set_slow_fast_timestep_ratio(10); + for (int step = 1; step <= nsteps; ++step) { // Set time to evolve to From a906cb80b80dff83d2cfbdcf858bb6eac36b4aba Mon Sep 17 00:00:00 2001 From: prkkumar Date: Wed, 17 Jul 2024 12:23:42 -0700 Subject: [PATCH 03/13] remove set_slow_fast_timestep_ratio and add input file --- .../Exec/inputs_forward_euler | 14 ----------- .../Reaction-Diffusion/Exec/inputs_rk3 | 24 ------------------- .../{inputs_sundials => inputs_sundials_mri} | 14 ++++++----- .../Reaction-Diffusion/Source/main.cpp | 4 ---- 4 files changed, 8 insertions(+), 48 deletions(-) delete mode 100644 ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_forward_euler delete mode 100644 ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_rk3 rename ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/{inputs_sundials => inputs_sundials_mri} (79%) diff --git a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_forward_euler b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_forward_euler deleted file mode 100644 index 39b09830..00000000 --- a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_forward_euler +++ /dev/null @@ -1,14 +0,0 @@ -n_cell = 32 -max_grid_size = 16 - -nsteps = 1000 -plot_int = 100 - -dt = 1.e-5 - -# INTEGRATION -## integration.type can take on the following values: -## 0 or "ForwardEuler" => Native AMReX Forward Euler integrator -## 1 or "RungeKutta" => Native AMReX Explicit Runge Kutta controlled by integration.rk.type -## 2 or "SUNDIALS" => SUNDIALS backend controlled by integration.sundials.strategy -integration.type = ForwardEuler diff --git a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_rk3 b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_rk3 deleted file mode 100644 index 6eb60137..00000000 --- a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_rk3 +++ /dev/null @@ -1,24 +0,0 @@ -n_cell = 32 -max_grid_size = 16 - -nsteps = 1000 -plot_int = 100 - -dt = 1.e-5 - -# INTEGRATION -## integration.type can take on the following values: -## 0 or "ForwardEuler" => Native AMReX Forward Euler integrator -## 1 or "RungeKutta" => Native AMReX Explicit Runge Kutta controlled by integration.rk.type -## 2 or "SUNDIALS" => SUNDIALS backend controlled by integration.sundials.strategy -integration.type = RungeKutta - -## Native AMReX Explicit Runge-Kutta parameters -# -## integration.rk.type can take the following values: -### 0 = User-specified Butcher Tableau -### 1 = Forward Euler -### 2 = Trapezoid Method -### 3 = SSPRK3 Method -### 4 = RK4 Method -integration.rk.type = 3 diff --git a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials_mri similarity index 79% rename from ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials rename to ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials_mri index bbdf2c69..fed23dc7 100644 --- a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials +++ b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials_mri @@ -1,3 +1,4 @@ +## n_cell = 32 max_grid_size = 16 @@ -6,6 +7,10 @@ plot_int = 100 dt = 1.e-5 +# To replicate heat equation +diffusion_coef = 1.0 +reaction_coef = 1.0 + # Use adaptive time stepping and set integrator relative and absolute tolerances # adapt_dt = true # reltol = 1.0e-4 @@ -33,12 +38,9 @@ integration.type = SUNDIALS # documentation for the supported method names # Use the SUNDIALS default method for the chosen type (fixed or adaptive step sizes) -integration.sundials.type = ERK +integration.sundials.type = EX-MRI # all the types listed above work with respective default methods -# Use forward Euler (fixed step sizes only) +# The following combination of type and method works as well # integration.sundials.type = ERK -# integration.sundials.method = ARKODE_FORWARD_EULER_1_1 +#integration.sundials.method = ARKODE_MRI_GARK_FORWARD_EULER #ARKODE_FORWARD_EULER_1_1 -# Use backward Euler (fixed step sizes only) -# integration.sundials.type = DIRK -# integration.sundials.method = ARKODE_BACKWARD_EULER_1_1 diff --git a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp index 4add8ebd..cda2385e 100644 --- a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp +++ b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp @@ -246,10 +246,6 @@ void main_main () integrator.set_time_step(dt); } - // This sets the ratio of slow timestep size to fast timestep size as an integer, - // or equivalently, the number of fast timesteps per slow timestep. - integrator.set_slow_fast_timestep_ratio(10); - for (int step = 1; step <= nsteps; ++step) { // Set time to evolve to From 2155e513a904f7cddfeebe1fce9250be539c3abd Mon Sep 17 00:00:00 2001 From: prkkumar Date: Wed, 17 Jul 2024 12:25:57 -0700 Subject: [PATCH 04/13] remove unnecessary options from input --- .../SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials_mri | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials_mri b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials_mri index fed23dc7..17b382b5 100644 --- a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials_mri +++ b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials_mri @@ -41,6 +41,6 @@ integration.type = SUNDIALS integration.sundials.type = EX-MRI # all the types listed above work with respective default methods # The following combination of type and method works as well -# integration.sundials.type = ERK -#integration.sundials.method = ARKODE_MRI_GARK_FORWARD_EULER #ARKODE_FORWARD_EULER_1_1 +# integration.sundials.type = EX-MRI +# integration.sundials.method = ARKODE_MRI_GARK_FORWARD_EULER From 50b7f7436f99a1b202c9cf97150b49996f4a5d81 Mon Sep 17 00:00:00 2001 From: prkkumar Date: Wed, 28 Aug 2024 10:39:28 -0700 Subject: [PATCH 05/13] remove set_pre_rhs_action --- .../Reaction-Diffusion/Source/main.cpp | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp index cda2385e..504469fd 100644 --- a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp +++ b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp @@ -163,13 +163,11 @@ void main_main () WriteSingleLevelPlotfile(pltfile, phi, {"phi"}, geom, time, 0); } - auto pre_rhs_function = [&](MultiFab& S_data, const Real /* time */) { + auto rhs_fast_function = [&](MultiFab& S_rhs, MultiFab& S_data, + const Real /* time */) { + // fill periodic ghost cells S_data.FillBoundary(geom.periodicity()); - }; - - auto rhs_function = [&](MultiFab& S_rhs, const MultiFab& S_data, - const Real /* time */) { // loop over boxes auto& phi_data = S_data; @@ -195,16 +193,18 @@ void main_main () phi_rhs_array(i,j,k) = diffusion_coef*( (phi_array(i+1,j,k) - 2.*phi_array(i,j,k) + phi_array(i-1,j,k)) / (dx[0]*dx[0]) +(phi_array(i,j+1,k) - 2.*phi_array(i,j,k) + phi_array(i,j-1,k)) / (dx[1]*dx[1]) #if (AMREX_SPACEDIM == 3) - +(phi_array(i,j,k+1) - 2.*phi_array(i,j,k) + phi_array(i,j,k-1)) / (dx[2]*dx[2]) ) + +(phi_array(i,j,k+1) - 2.*phi_array(i,j,k) + phi_array(i,j,k-1)) / (dx[2]*dx[2]) ); #endif - + reaction_coef*phi_array(i,j,k); }); } }; - auto rhs_fast_function = [&](MultiFab& S_rhs, const MultiFab& S_data, + auto rhs_function = [&](MultiFab& S_rhs, MultiFab& S_data, const Real /* time */) { + // fill periodic ghost cells + S_data.FillBoundary(geom.periodicity()); + // loop over boxes auto& phi_data = S_data; auto& phi_rhs = S_rhs; @@ -226,17 +226,13 @@ void main_main () // fill the right-hand-side for phi amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) { - phi_rhs_array(i,j,k) = diffusion_coef*( (phi_array(i+1,j,k) - 2.*phi_array(i,j,k) + phi_array(i-1,j,k)) / (dx[0]*dx[0]) - +(phi_array(i,j+1,k) - 2.*phi_array(i,j,k) + phi_array(i,j-1,k)) / (dx[1]*dx[1]) -#if (AMREX_SPACEDIM == 3) - +(phi_array(i,j,k+1) - 2.*phi_array(i,j,k) + phi_array(i,j,k-1)) / (dx[2]*dx[2]) ); -#endif + phi_rhs_array(i,j,k) = -1.*reaction_coef*phi_array(i,j,k); }); } }; + TimeIntegrator integrator(phi, time); - integrator.set_pre_rhs_action(pre_rhs_function); integrator.set_rhs(rhs_function); integrator.set_fast_rhs(rhs_fast_function); if (adapt_dt) { From e2923dbaa73b0881b27bd3e45d9d33dc3faa40f4 Mon Sep 17 00:00:00 2001 From: prkkumar Date: Wed, 28 Aug 2024 10:58:43 -0700 Subject: [PATCH 06/13] add set_fast_time_step for mri --- ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp index 504469fd..3a77eee4 100644 --- a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp +++ b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp @@ -242,6 +242,8 @@ void main_main () integrator.set_time_step(dt); } + integrator.set_fast_time_step(0.1*dt); + for (int step = 1; step <= nsteps; ++step) { // Set time to evolve to From 1dcdf2805a6c2a22975d51ac4d12fa3481613bd3 Mon Sep 17 00:00:00 2001 From: prkkumar Date: Wed, 28 Aug 2024 16:12:37 -0700 Subject: [PATCH 07/13] input file and some minor tweaks --- .../Exec/inputs_sundials_mri | 28 +++++++++++++------ .../Reaction-Diffusion/Source/main.cpp | 28 ++++++++++++++++--- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials_mri b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials_mri index 17b382b5..96860e0d 100644 --- a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials_mri +++ b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials_mri @@ -2,14 +2,18 @@ n_cell = 32 max_grid_size = 16 -nsteps = 1000 -plot_int = 100 +nsteps = 12 +plot_int = 2 -dt = 1.e-5 +dt = 8.5e-3 # To replicate heat equation diffusion_coef = 1.0 -reaction_coef = 1.0 +reaction_coef = 1.e-4 + +# MRI parameters +use_MRI = true +fast_dt_ratio = 0.02 # Use adaptive time stepping and set integrator relative and absolute tolerances # adapt_dt = true @@ -36,11 +40,17 @@ integration.type = SUNDIALS # # Optionally select a specific SUNDIALS method by name, see the SUNDIALS # documentation for the supported method names +integration.sundials.type = EX-MRI +integration.sundials.fast_type = ERK -# Use the SUNDIALS default method for the chosen type (fixed or adaptive step sizes) -integration.sundials.type = EX-MRI # all the types listed above work with respective default methods +## *** Select a specific SUNDIALS ERK method *** +#integration.sundials.method = ARKODE_BOGACKI_SHAMPINE_4_2_3 +# +## *** Select a specific SUNDIALS ImEx method *** +#integration.sundials.method_i = ARKODE_ARK2_DIRK_3_1_2 +#integration.sundials.method_e = ARKODE_ARK2_ERK_3_1_2 -# The following combination of type and method works as well -# integration.sundials.type = EX-MRI -# integration.sundials.method = ARKODE_MRI_GARK_FORWARD_EULER +# *** Select a specific SUNDIALS MRI method *** +integration.sundials.method = ARKODE_MIS_KW3 +integration.sundials.fast_method = ARKODE_KNOTH_WOLKE_3_3 diff --git a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp index 3a77eee4..f4f6552b 100644 --- a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp +++ b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp @@ -40,6 +40,10 @@ void main_main () // use adaptive time step (dt used to set output times) bool adapt_dt = false; + // use MRI + bool use_MRI = false; + Real fast_dt_ratio = 0.1; + // adaptive time step relative and absolute tolerances Real reltol = 1.0e-4; Real abstol = 1.0e-9; @@ -73,6 +77,10 @@ void main_main () // use adaptive step sizes pp.query("adapt_dt",adapt_dt); + // use MRI + pp.query("use_MRI",use_MRI); + pp.query("fast_dt_ratio",fast_dt_ratio); + // adaptive step tolerances pp.query("reltol",reltol); pp.query("abstol",abstol); @@ -177,7 +185,6 @@ void main_main () Real reaction_coef = 0.0; Real diffusion_coef = 1.0; ParmParse pp; - pp.query("reaction_coef",reaction_coef); pp.query("diffusion_coef",diffusion_coef); for ( MFIter mfi(phi_data); mfi.isValid(); ++mfi ) @@ -226,7 +233,16 @@ void main_main () // fill the right-hand-side for phi amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) { - phi_rhs_array(i,j,k) = -1.*reaction_coef*phi_array(i,j,k); + if (use_MRI) { + phi_rhs_array(i,j,k) = -1.*reaction_coef*phi_array(i,j,k); + } else { + phi_rhs_array(i,j,k) = diffusion_coef*( (phi_array(i+1,j,k) - 2.*phi_array(i,j,k) + phi_array(i-1,j,k)) / (dx[0]*dx[0]) + +(phi_array(i,j+1,k) - 2.*phi_array(i,j,k) + phi_array(i,j-1,k)) / (dx[1]*dx[1]) +#if (AMREX_SPACEDIM == 3) + +(phi_array(i,j,k+1) - 2.*phi_array(i,j,k) + phi_array(i,j,k-1)) / (dx[2]*dx[2]) ) +#endif + -1.*reaction_coef*phi_array(i,j,k); + } }); } }; @@ -234,7 +250,9 @@ void main_main () TimeIntegrator integrator(phi, time); integrator.set_rhs(rhs_function); - integrator.set_fast_rhs(rhs_fast_function); + if (use_MRI) { + integrator.set_fast_rhs(rhs_fast_function); + } if (adapt_dt) { integrator.set_adaptive_step(); integrator.set_tolerances(reltol, abstol); @@ -242,7 +260,9 @@ void main_main () integrator.set_time_step(dt); } - integrator.set_fast_time_step(0.1*dt); + if (use_MRI) { + integrator.set_fast_time_step(fast_dt_ratio*dt); + } for (int step = 1; step <= nsteps; ++step) { From e06553564021aebcd84f2a3023c167cb7246ce34 Mon Sep 17 00:00:00 2001 From: prkkumar Date: Wed, 28 Aug 2024 16:24:24 -0700 Subject: [PATCH 08/13] add theoretical dt constraint as comments --- .../SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials_mri | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials_mri b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials_mri index 96860e0d..19e147d2 100644 --- a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials_mri +++ b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials_mri @@ -1,4 +1,11 @@ ## +# Theoretical forward euler explicit time step constraint for each process is based +# on the diffusion and reaction coefficients. +# For diffusion term +# dt < dx^2 /(2*DIM*diffusion_coef) = (1/32)^2/(2*3*1) = 1.63e-4 +# For reaction term +# dt <= 2/reaction_coef + n_cell = 32 max_grid_size = 16 From 7c92469a03de9dc37d2519158c2b4b6a7d1b4a78 Mon Sep 17 00:00:00 2001 From: prkkumar Date: Wed, 28 Aug 2024 16:34:29 -0700 Subject: [PATCH 09/13] test results as comments --- .../Reaction-Diffusion/Exec/inputs_sundials_mri | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials_mri b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials_mri index 19e147d2..e380326c 100644 --- a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials_mri +++ b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/inputs_sundials_mri @@ -9,10 +9,16 @@ n_cell = 32 max_grid_size = 16 -nsteps = 12 -plot_int = 2 +nsteps = 12 # 60 # 600 +plot_int = 2 # 10 # 100 -dt = 8.5e-3 +dt = 8.5e-3 # 1.7e-3 #1.7e-4 + +# With integration.sundials.type = ERK, dt = 1.6e4 is stable, dt = 1.7e-4 is unstable +# With integration.sundials.type = EX-MRI, dt = 1.7e-4 with fast_dt_ratio = 0.1 is stable +# With integration.sundials.type = EX-MRI, dt = 1.7e-3 with fast_dt_ratio = 0.1 is stable +# With integration.sundials.type = EX-MRI, dt = 8.5e-3 with fast_dt_ratio = 0.1 FAILS +# With integration.sundials.type = EX-MRI, dt = 8.5e-3 with fast_dt_ratio = 0.02 is stable # To replicate heat equation diffusion_coef = 1.0 @@ -22,6 +28,7 @@ reaction_coef = 1.e-4 use_MRI = true fast_dt_ratio = 0.02 + # Use adaptive time stepping and set integrator relative and absolute tolerances # adapt_dt = true # reltol = 1.0e-4 From ffe9236e7737c7c46b95481f09517e211eedce9a Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 29 Aug 2024 16:03:23 -0700 Subject: [PATCH 10/13] CI: Python Install Dependencies Update the Python install dependencies according to latest docs. Setuptools had a breaking change for Python 3.12 and we modernized accordingly. --- .github/workflows/dependencies/dependencies.sh | 3 ++- .github/workflows/dependencies/dependencies_clang6.sh | 3 ++- .github/workflows/dependencies/dependencies_gcc10.sh | 3 ++- .github/workflows/dependencies/dependencies_nofortran.sh | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dependencies/dependencies.sh b/.github/workflows/dependencies/dependencies.sh index ecaa6eca..35825b96 100755 --- a/.github/workflows/dependencies/dependencies.sh +++ b/.github/workflows/dependencies/dependencies.sh @@ -17,4 +17,5 @@ sudo apt-get install -y --no-install-recommends\ python3 \ python3-pip -python3 -m pip install -U pip setuptools wheel +python3 -m pip install -U pip +python3 -m pip install -U build packaging setuptools wheel diff --git a/.github/workflows/dependencies/dependencies_clang6.sh b/.github/workflows/dependencies/dependencies_clang6.sh index 596dd8c3..ad18689d 100755 --- a/.github/workflows/dependencies/dependencies_clang6.sh +++ b/.github/workflows/dependencies/dependencies_clang6.sh @@ -15,4 +15,5 @@ sudo apt-get install -y \ python3 \ python3-pip -python3 -m pip install -U pip setuptools wheel +python3 -m pip install -U pip +python3 -m pip install -U build packaging setuptools wheel diff --git a/.github/workflows/dependencies/dependencies_gcc10.sh b/.github/workflows/dependencies/dependencies_gcc10.sh index 66573abc..3ea17c85 100755 --- a/.github/workflows/dependencies/dependencies_gcc10.sh +++ b/.github/workflows/dependencies/dependencies_gcc10.sh @@ -18,4 +18,5 @@ sudo apt-get install -y --no-install-recommends \ python3 \ python3-pip -python3 -m pip install -U pip setuptools wheel +python3 -m pip install -U pip +python3 -m pip install -U build packaging setuptools wheel diff --git a/.github/workflows/dependencies/dependencies_nofortran.sh b/.github/workflows/dependencies/dependencies_nofortran.sh index d00c1ff8..65eb13f9 100755 --- a/.github/workflows/dependencies/dependencies_nofortran.sh +++ b/.github/workflows/dependencies/dependencies_nofortran.sh @@ -21,4 +21,5 @@ sudo apt-get install -y --no-install-recommends\ python3 \ python3-pip -python3 -m pip install -U pip setuptools wheel +python3 -m pip install -U pip +python3 -m pip install -U build packaging setuptools wheel From 0bb992c2bce9d555befb8dee5a69cb9f0520879d Mon Sep 17 00:00:00 2001 From: Yoann Launay <75337357+YL-codehub@users.noreply.github.com> Date: Mon, 2 Sep 2024 19:35:35 +0100 Subject: [PATCH 11/13] fix(SWFFT/Poisson) (#133) - rank-mapping didn't have to be modified from fortran to C order - already done so in SWFFT/Simple. --- ExampleCodes/SWFFT/SWFFT_poisson/swfft_solver.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/ExampleCodes/SWFFT/SWFFT_poisson/swfft_solver.cpp b/ExampleCodes/SWFFT/SWFFT_poisson/swfft_solver.cpp index a12f4c87..f4c81deb 100644 --- a/ExampleCodes/SWFFT/SWFFT_poisson/swfft_solver.cpp +++ b/ExampleCodes/SWFFT/SWFFT_poisson/swfft_solver.cpp @@ -55,12 +55,7 @@ swfft_solver(MultiFab& rhs, MultiFab& soln, Geometry& geom, int verbose) int j = ba[ib].smallEnd(1) / ny; int k = ba[ib].smallEnd(2) / nz; - // This would be the "correct" local index if the data wasn't being transformed - // int local_index = k*nbx*nby + j*nbx + i; - - // This is what we pass to dfft to compensate for the Fortran ordering - // of amrex data in MultiFabs. - int local_index = i*nby*nbz + j*nbz + k; + int local_index = k*nbx*nby + j*nbx + i; rank_mapping[local_index] = dmap[ib]; if (verbose) From 72792b71822ebd32727b814de9baeefc3d19dbab Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Thu, 5 Sep 2024 09:42:34 -0700 Subject: [PATCH 12/13] fix trailing whitespace --- ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp index f4f6552b..cb009cd3 100644 --- a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp +++ b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp @@ -242,7 +242,7 @@ void main_main () +(phi_array(i,j,k+1) - 2.*phi_array(i,j,k) + phi_array(i,j,k-1)) / (dx[2]*dx[2]) ) #endif -1.*reaction_coef*phi_array(i,j,k); - } + } }); } }; From 49d1d368851ce3481d2dccb8f2d97bd419d6bbdb Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Thu, 5 Sep 2024 11:28:03 -0700 Subject: [PATCH 13/13] update README_sundials to match the other tutorial add timers --- .../Reaction-Diffusion/Exec/README_sundials | 49 ++++++------------- .../Reaction-Diffusion/Source/main.cpp | 19 +++++-- 2 files changed, 28 insertions(+), 40 deletions(-) diff --git a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/README_sundials b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/README_sundials index c0f315aa..bf0e47c1 100644 --- a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/README_sundials +++ b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Exec/README_sundials @@ -3,13 +3,19 @@ https://computing.llnl.gov/projects/sundials/faq#inst Installation -# Download sundials-x.y.z.tar.gz file for SUNDIALS and extract it at the same level as amrex. -# Update 6/11/24 - v7.0.0. tested -https://computing.llnl.gov/projects/sundials/sundials-software - +# You need SUNDIALS v7.1.1 or later. +# Check https://computing.llnl.gov/projects/sundials/sundials-software to see if it's available for download. +# If so, download sundials-x.y.z.tar.gz and extract it at the same level as amrex using >> tar -xzvf sundials-x.y.z.tar.gz # where x.y.z is the version of sundials +>> mv sundials-x.y.z sundials-src + +# If v7.1.1. is not available on the website, clone the git repo directly and use the latest version +# At the same level that amrex is cloned, do: + +>> git clone https://github.com/LLNL/sundials.git +>> mv sundials sundials-src -# at the same level that amrex is cloned, do: +# Next >> mkdir sundials >> cd sundials @@ -22,7 +28,7 @@ HOST BUILD >> mkdir builddir >> cd builddir ->> cmake -DCMAKE_INSTALL_PREFIX=/pathto/sundials/instdir -DEXAMPLES_INSTALL_PATH=/pathto/sundials/instdir/examples -DENABLE_MPI=ON ../../sundials-x.y.z +>> cmake -DCMAKE_INSTALL_PREFIX=/pathto/sundials/instdir -DEXAMPLES_INSTALL_PATH=/pathto/sundials/instdir/examples -DENABLE_MPI=ON ../../sundials-src ###################### NVIDIA/CUDA BUILD @@ -34,7 +40,7 @@ NVIDIA/CUDA BUILD >> mkdir builddir_cuda >> cd builddir_cuda ->> cmake -DCMAKE_INSTALL_PREFIX=/pathto/sundials/instdir_cuda -DEXAMPLES_INSTALL_PATH=/pathto/sundials/instdir_cuda/examples -DENABLE_CUDA=ON -DENABLE_MPI=ON ../../sundials-x.y.z +>> cmake -DCMAKE_INSTALL_PREFIX=/pathto/sundials/instdir_cuda -DEXAMPLES_INSTALL_PATH=/pathto/sundials/instdir_cuda/examples -DENABLE_CUDA=ON -DENABLE_MPI=ON ../../sundials-src ###################### @@ -55,31 +61,4 @@ NVIDIA/CUDA BUILD >> make -j4 # optional to have 'USE_CUDA=TRUE' as well -# in your inputs file, you will need to have: - -# INTEGRATION -## integration.type can take on the following values: -## 0 or "ForwardEuler" => Native AMReX Forward Euler integrator -## 1 or "RungeKutta" => Native AMReX Explicit Runge Kutta controlled by integration.rk.type -## 2 or "SUNDIALS" => SUNDIALS backend controlled by integration.sundials.strategy -integration.type = - -## Native AMReX Explicit Runge-Kutta parameters -# -## integration.rk.type can take the following values: -### 0 = User-specified Butcher Tableau -### 1 = Forward Euler -### 2 = Trapezoid Method -### 3 = SSPRK3 Method -### 4 = RK4 Method -integration.rk.type = 3 - -## If using the SUNDIALS Submodule, then -## compile with USE_SUNDIALS=TRUE or AMReX_SUNDIALS=ON and -## set strategy here: -# -## integration.sundials.strategy can take the following values: -### ERK = ERKStep from ARKode with SSPRK3 Method -### MRI = MRIStep from ARKode with Explict Trapezoid Method -### MRITEST = MRIStep from ARKode modified to use no-op inner f0 -integration.sundials.strategy = ERK +# Refer to inputs to see how to enable different integrators diff --git a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp index cb009cd3..32089415 100644 --- a/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp +++ b/ExampleCodes/SUNDIALS/Reaction-Diffusion/Source/main.cpp @@ -252,28 +252,33 @@ void main_main () integrator.set_rhs(rhs_function); if (use_MRI) { integrator.set_fast_rhs(rhs_fast_function); + integrator.set_fast_time_step(fast_dt_ratio*dt); } - if (adapt_dt) { + + if (adapt_dt && !use_MRI) { integrator.set_adaptive_step(); integrator.set_tolerances(reltol, abstol); } else { integrator.set_time_step(dt); } - if (use_MRI) { - integrator.set_fast_time_step(fast_dt_ratio*dt); - } + Real evolution_start_time = ParallelDescriptor::second(); for (int step = 1; step <= nsteps; ++step) { // Set time to evolve to time += dt; + Real step_start_time = ParallelDescriptor::second(); + // Advance to output time integrator.evolve(phi, time); + Real step_stop_time = ParallelDescriptor::second() - step_start_time; + ParallelDescriptor::ReduceRealMax(step_stop_time); + // Tell the I/O Processor to write out which step we're doing - amrex::Print() << "Advanced step " << step << "\n"; + amrex::Print() << "Advanced step " << step << " in " << step_stop_time << " seconds; dt = " << dt << " time = " << time << "\n"; // Write a plotfile of the current data (plot_int was defined in the inputs file) if (plot_int > 0 && step%plot_int == 0) @@ -282,4 +287,8 @@ void main_main () WriteSingleLevelPlotfile(pltfile, phi, {"phi"}, geom, time, step); } } + + Real evolution_stop_time = ParallelDescriptor::second() - evolution_start_time; + ParallelDescriptor::ReduceRealMax(evolution_stop_time); + amrex::Print() << "Total evolution time = " << evolution_stop_time << " seconds\n"; }