From e60fb75798d737114ae43f14296272436a5e23af Mon Sep 17 00:00:00 2001
From: Andy Nonaka <AJNonaka@lbl.gov>
Date: Wed, 12 Jun 2024 09:17:59 -0700
Subject: [PATCH] readme and updates to makefile for SUNDIALS

---
 ExampleCodes/SUNDIALS/Exec/GNUmakefile     | 49 +++++++------
 ExampleCodes/SUNDIALS/Exec/README_sundials | 85 ++++++++++++++++++++++
 2 files changed, 112 insertions(+), 22 deletions(-)
 create mode 100644 ExampleCodes/SUNDIALS/Exec/README_sundials

diff --git a/ExampleCodes/SUNDIALS/Exec/GNUmakefile b/ExampleCodes/SUNDIALS/Exec/GNUmakefile
index bee147e2..448b77a0 100644
--- a/ExampleCodes/SUNDIALS/Exec/GNUmakefile
+++ b/ExampleCodes/SUNDIALS/Exec/GNUmakefile
@@ -1,12 +1,13 @@
 # AMREX_HOME defines the directory in which we will find all the AMReX code.
-AMREX_HOME ?= ../../../../../amrex
+AMREX_HOME ?= ../../../../amrex
 
 DEBUG        = FALSE
-USE_MPI      = FALSE
+USE_MPI      = TRUE
 USE_OMP      = FALSE
 COMP         = gnu
 DIM          = 3
 USE_RPATH    = TRUE
+USE_SUNDIALS = TRUE
 
 include $(AMREX_HOME)/Tools/GNUMake/Make.defs
 
@@ -14,29 +15,33 @@ include ../Source/Make.package
 VPATH_LOCATIONS  += ../Source
 INCLUDE_LOCATIONS += ../Source
 
-
 ifeq ($(USE_SUNDIALS),TRUE)
-SUNDIALS_ROOT ?= $(TOP)/../../../../../sundials/instdir
-SUNDIALS_LIB_DIR ?= $(SUNDIALS_ROOT)/lib
-
-USE_CVODE_LIBS ?= TRUE
-USE_ARKODE_LIBS ?= TRUE
-
-DEFINES += -DAMREX_USE_SUNDIALS
-INCLUDE_LOCATIONS += $(SUNDIALS_ROOT)/include
-LIBRARY_LOCATIONS += $(SUNDIALS_LIB_DIR)
-
-LIBRARIES += -lsundials_cvode
-LIBRARIES += -lsundials_arkode
-LIBRARIES += -lsundials_nvecmanyvector
-
-ifneq (,$(wildcard $(SUNDIALS_LIB_DIR)/libsundials_core*))
+  ifeq ($(USE_CUDA),TRUE)
+    SUNDIALS_ROOT ?= $(TOP)../../../../sundials/instdir_cuda
+  else
+    SUNDIALS_ROOT ?= $(TOP)../../../../sundials/instdir
+  endif
+  ifeq ($(NERSC_HOST),perlmutter)
+    SUNDIALS_LIB_DIR ?= $(SUNDIALS_ROOT)/lib64
+  else
+    SUNDIALS_LIB_DIR ?= $(SUNDIALS_ROOT)/lib
+  endif
+
+  USE_CVODE_LIBS ?= TRUE
+  USE_ARKODE_LIBS ?= TRUE
+
+  DEFINES += -DAMREX_USE_SUNDIALS
+  INCLUDE_LOCATIONS += $(SUNDIALS_ROOT)/include
+  LIBRARY_LOCATIONS += $(SUNDIALS_LIB_DIR)
+
+  LIBRARIES += -L$(SUNDIALS_LIB_DIR) -lsundials_cvode
+  LIBRARIES += -L$(SUNDIALS_LIB_DIR) -lsundials_arkode
+  LIBRARIES += -L$(SUNDIALS_LIB_DIR) -lsundials_nvecmanyvector
   LIBRARIES += -L$(SUNDIALS_LIB_DIR) -lsundials_core
-endif
 
-ifeq ($(USE_CUDA),TRUE)
-  LIBRARIES += -lsundials_nveccuda
-endif
+  ifeq ($(USE_CUDA),TRUE)
+    LIBRARIES += -L$(SUNDIALS_LIB_DIR) -lsundials_nveccuda
+  endif
 
 endif
 
diff --git a/ExampleCodes/SUNDIALS/Exec/README_sundials b/ExampleCodes/SUNDIALS/Exec/README_sundials
new file mode 100644
index 00000000..c0f315aa
--- /dev/null
+++ b/ExampleCodes/SUNDIALS/Exec/README_sundials
@@ -0,0 +1,85 @@
+SUNDIALS installation guide:
+https://computing.llnl.gov/projects/sundials/faq#inst
+
+Installation
+
+# Download sundials-x.y.z.tar.gz file for SUNDIALS and extract it at the same level as amrex.
+# Update 6/11/24 - v7.0.0. tested
+https://computing.llnl.gov/projects/sundials/sundials-software
+
+>> tar -xzvf sundials-x.y.z.tar.gz # where x.y.z is the version of sundials
+
+# at the same level that amrex is cloned, do:
+
+>> mkdir sundials
+>> cd sundials
+
+######################
+HOST BUILD 
+######################
+
+>> mkdir instdir
+>> mkdir builddir
+>> cd builddir
+
+>> cmake -DCMAKE_INSTALL_PREFIX=/pathto/sundials/instdir -DEXAMPLES_INSTALL_PATH=/pathto/sundials/instdir/examples -DENABLE_MPI=ON ../../sundials-x.y.z
+
+######################
+NVIDIA/CUDA BUILD
+######################
+
+# Navigate back to the 'sundials' directory and do:
+
+>> mkdir instdir_cuda
+>> mkdir builddir_cuda
+>> cd builddir_cuda
+
+>> cmake -DCMAKE_INSTALL_PREFIX=/pathto/sundials/instdir_cuda -DEXAMPLES_INSTALL_PATH=/pathto/sundials/instdir_cuda/examples -DENABLE_CUDA=ON -DENABLE_MPI=ON ../../sundials-x.y.z
+
+######################
+
+>> make -j4
+>> make install
+
+# in your .bashrc or preferred configuration file, add the following (and then "source ~/.bashrc")
+
+# If you have a CPU build:
+
+>> export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/pathto/sundials/instdir/lib/
+
+# If you have a NVIDIA/CUDA build:
+
+>> export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/pathto/sundials/instdir_cuda/lib/
+
+# now you are ready to compile amrex-tutorials/ExampleCodes/SUNDIALS/Exec with:
+
+>> make -j4 # optional to have 'USE_CUDA=TRUE' as well
+
+# in your inputs file, you will need to have:
+
+# INTEGRATION
+## integration.type can take on the following values:
+## 0 or "ForwardEuler" => Native AMReX Forward Euler integrator
+## 1 or "RungeKutta"   => Native AMReX Explicit Runge Kutta controlled by integration.rk.type
+## 2 or "SUNDIALS"     => SUNDIALS backend controlled by integration.sundials.strategy
+integration.type = <pick a value>
+
+## Native AMReX Explicit Runge-Kutta parameters
+#
+## integration.rk.type can take the following values:
+### 0 = User-specified Butcher Tableau
+### 1 = Forward Euler
+### 2 = Trapezoid Method
+### 3 = SSPRK3 Method
+### 4 = RK4 Method
+integration.rk.type = 3
+
+## If using the SUNDIALS Submodule, then
+## compile with USE_SUNDIALS=TRUE or AMReX_SUNDIALS=ON and
+## set strategy here:
+#
+## integration.sundials.strategy can take the following values:
+### ERK     = ERKStep from ARKode with SSPRK3 Method
+### MRI     = MRIStep from ARKode with Explict Trapezoid Method
+### MRITEST = MRIStep from ARKode modified to use no-op inner f0
+integration.sundials.strategy = ERK