From 5417ba8f26b8d60d1c268ad19bb64f88a530e83c Mon Sep 17 00:00:00 2001
From: Abishek Gopal <agopal@tamu.edu>
Date: Thu, 19 Aug 2021 08:20:39 -0500
Subject: [PATCH] Support for Grace. And new GoM LR domain

---
 config/cesm/config_grids.xml              | 24 +++++++++
 config/cesm/machines/config_batch.xml     | 19 +++++++
 config/cesm/machines/config_compilers.xml | 37 +++++++++++++
 config/cesm/machines/config_machines.xml  | 65 ++++++++++++++++++++++-
 4 files changed, 144 insertions(+), 1 deletion(-)

diff --git a/config/cesm/config_grids.xml b/config/cesm/config_grids.xml
index c0f1c0bd8f4..7e4296370a6 100644
--- a/config/cesm/config_grids.xml
+++ b/config/cesm/config_grids.xml
@@ -995,6 +995,12 @@
       <grid name="ocnice">gom3x</grid>
     </model_grid>
 
+    <model_grid alias="txlw27k_gom27k">
+      <grid name="atm">txlw27k</grid>
+      <grid name="lnd">txlw27k</grid>
+      <grid name="ocnice">gom27k</grid>
+    </model_grid>
+
   </grids>
 
   <!-- ======================================================== -->
@@ -1468,6 +1474,17 @@
       <desc>3km Gulf of Mexico for Wrf, to pair with gom3x in xRoms</desc>
     </domain>
 
+    <domain name="txlw27k">
+      <nx>89</nx>  <ny>79</ny>
+      <file grid="atm|lnd">$DIN_LOC_ROOT/share/domains/domain.atmlnd.txlw27k_gom27k.210706.nc</file>
+      <desc>27km Gulf of Mexico for Wrf, to pair with gom27k in xRoms</desc>
+    </domain>
+    <domain name="gom27k">
+      <nx>273</nx>  <ny>242</ny>
+      <file grid="ocnice">$DIN_LOC_ROOT/share/domains/gom_lr_docn_grd.210705.nc</file>
+      <desc>27km Gulf of Mexico grid for xRoms</desc>
+    </domain>
+
     <!-- GLC domain -->
 
     <domain name="gland20">
@@ -1837,6 +1854,13 @@
       <map name="OCN2ATM_SMAPNAME">atm/wrf/tx3k_g3x/map_o2a_aave_Rmask_1909.nc</map>
     </gridmap>
 
+    <gridmap atm_grid="txlw27k" ocn_grid="gom27k">
+      <map name="ATM2OCN_FMAPNAME">atm/wrf/tx27k_g27x/map_a2o_aave.nc</map>
+      <map name="ATM2OCN_SMAPNAME">atm/wrf/tx27k_g27x/map_a2o_aave.nc</map>
+      <map name="ATM2OCN_VMAPNAME">atm/wrf/tx27k_g27x/map_a2o_aave.nc</map>
+      <map name="OCN2ATM_FMAPNAME">atm/wrf/tx27k_g27x/map_o2a_aave.nc</map>
+      <map name="OCN2ATM_SMAPNAME">atm/wrf/tx27k_g27x/map_o2a_aave.nc</map>
+    </gridmap>
     <!--  QL, 150525, wav to ocn, atm, ice mapping files  -->
 
     <gridmap ocn_grid="gx3v7" wav_grid="ww3a">
diff --git a/config/cesm/machines/config_batch.xml b/config/cesm/machines/config_batch.xml
index 46e5a32c0c5..46e069ccd9e 100755
--- a/config/cesm/machines/config_batch.xml
+++ b/config/cesm/machines/config_batch.xml
@@ -509,6 +509,25 @@
     </queues>
   </batch_system>
 
+<batch_system MACH="grace" type="slurm" >
+    <batch_submit>sbatch</batch_submit>
+    <submit_args>
+      <arg flag="--time" name="$JOB_WALLCLOCK_TIME"/>
+      <arg flag="-p" name="$JOB_QUEUE"/>
+    </submit_args>
+    <directives>
+    <directive>--mem=360GB </directive>
+    </directives>
+    <queues>
+      <queue walltimemax="02:00:00" nodemin="1" nodemax="32">short</queue>
+      <queue walltimemax="24:00:00" nodemin="1" nodemax="128" default="true">medium</queue>
+      <queue walltimemax="168:00:00" nodemin="1" nodemax="64" >long</queue>
+      <queue walltimemax="504:00:00" nodemin="1" nodemax="32" >xlong</queue>
+      <queue walltimemax="24:00:00" nodemin="1" nodemax="800" >special</queue>
+    </queues>
+  </batch_system>
+
+
   <batch_system MACH="ada" type="lsf">
     <queues>
       <queue walltimemax="168:00" nodemin="101" nodemax="811">special</queue>
diff --git a/config/cesm/machines/config_compilers.xml b/config/cesm/machines/config_compilers.xml
index a73e93c5a2a..b13b69a8b71 100644
--- a/config/cesm/machines/config_compilers.xml
+++ b/config/cesm/machines/config_compilers.xml
@@ -1160,6 +1160,43 @@ using a fortran linker.
   </SLIBS>
 </compiler>
 
+<compiler MACH="grace" COMPILER="intel">
+  <CPPDEFS>
+    <append> -DHAVE_NANOTIME -DNO_R16 </append>
+  </CPPDEFS>
+  <CFLAGS>
+    <append></append>
+  </CFLAGS>
+  <FFLAGS>
+    <append></append>
+  </FFLAGS>
+  <MPIFC>mpiifort</MPIFC>
+  <MPICC>mpiicc</MPICC>
+  <MPICXX>mpiicpc</MPICXX>
+  <SFC>ifort</SFC>
+  <SCC>icc</SCC>
+  <SCXX>icpc</SCXX>
+ <!-- <NETCDF_PATH>/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/netCDF/4.3.3.1-intel-2018b</NETCDF_PATH>
+  <NETCDF_C_PATH>/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/netCDF/4.3.3.1-intel-2018b</NETCDF_C_PATH>
+  <PNETCDF_PATH>/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/PnetCDF/1.8.1-intel-2018b</PNETCDF_PATH>
+  <NETCDF_FORTRAN_PATH>/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/netCDF-Fortran/4.4.2-intel-2018b</NETCDF_FORTRAN_PATH> -->
+ <!-- <NETCDF_PATH>/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/netCDF/4.3.3.1-intel-2020b</NETCDF_PATH>
+  <NETCDF_C_PATH>/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/netCDF/4.3.3.1-intel-2020b</NETCDF_C_PATH>
+  <PNETCDF_PATH>/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/PnetCDF/1.8.1-intel-2020b</PNETCDF_PATH>
+  <NETCDF_FORTRAN_PATH>/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/netCDF-Fortran/4.4.2-intel-2020b</NETCDF_FORTRAN_PATH> --> 
+ <NETCDF_PATH>/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/netCDF/4.7.4-iimpi-2019b</NETCDF_PATH>
+  <NETCDF_C_PATH>/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/netCDF/4.7.4-iimpi-2019b</NETCDF_C_PATH>
+  <NETCDF_FORTRAN_PATH>/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/netCDF-Fortran/4.5.3-iimpi-2019b</NETCDF_FORTRAN_PATH>
+  <PNETCDF_PATH>/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/PnetCDF/1.12.1-iimpi-2019b</PNETCDF_PATH>
+  <SLIBS>
+    <append>-L$(NETCDF_C_PATH)/lib -Wl,-rpath,$(NETCDF_PATH)/lib -lnetcdf</append>
+    <append>-L$(NETCDF_FORTRAN_PATH)/lib -Wl,-rpath,$(NETCDF_FORTRAN_PATH)/lib -lnetcdff</append>
+    <append>-L$(PNETCDF_PATH)/lib -Wl,-rpath,$(PNETCDF_PATH)/lib -lpnetcdf</append>
+  </SLIBS>
+  <PIO_FILESYSTEM_HINTS>lustre</PIO_FILESYSTEM_HINTS>
+</compiler>
+
+
 <compiler MACH="ada" COMPILER="intel">
   <CPPDEFS>
     <append> -DHAVE_NANOTIME -DNO_R16 </append>
diff --git a/config/cesm/machines/config_machines.xml b/config/cesm/machines/config_machines.xml
index 86c43f31c3e..3e814e8eec3 100644
--- a/config/cesm/machines/config_machines.xml
+++ b/config/cesm/machines/config_machines.xml
@@ -2344,6 +2344,69 @@ This allows using a different mpirun command to launch unit tests
     </module_system>
   </machine>
 
+
+<machine MACH="grace">
+    <DESC>Intel Xeon 6248R 3.0 GHz ("Cascade Lake"),48 cores on two sockets (24 cores/socket) , batch system is SLURM</DESC>
+    <NODENAME_REGEX>.*grace</NODENAME_REGEX>
+    <OS>LINUX</OS>
+    <COMPILERS>intel</COMPILERS>
+    <MPILIBS>impi</MPILIBS>
+    <!--<CIME_OUTPUT_ROOT>$ENV{SCRATCH}</CIME_OUTPUT_ROOT>-->
+    <CIME_OUTPUT_ROOT>/ihesp/shared/rundir/rcesm</CIME_OUTPUT_ROOT>
+    <DIN_LOC_ROOT>/ihesp/obs_root/inputdata</DIN_LOC_ROOT>
+    <DIN_LOC_ROOT_CLMFORC>/ihesp/obs_root/inputdata/lmwg</DIN_LOC_ROOT_CLMFORC>
+    <DOUT_S_ROOT>$ENV{SCRATCH}/archive/$CASE</DOUT_S_ROOT>
+    <BASELINE_ROOT>/ihesp/obs_root/inputdata/cesm_baselines</BASELINE_ROOT>
+    <CCSM_CPRNC>/ihesp/software/cesm/cime/tools/cprnc/cprnc</CCSM_CPRNC>
+    <GMAKE_J>4</GMAKE_J>
+    <BATCH_SYSTEM>slurm</BATCH_SYSTEM>
+    <SUPPORTED_BY>agopal</SUPPORTED_BY>
+    <MAX_TASKS_PER_NODE>96</MAX_TASKS_PER_NODE>
+    <MAX_MPITASKS_PER_NODE>48</MAX_MPITASKS_PER_NODE>
+    <mpirun mpilib="impi">
+      <executable>mpirun</executable>
+      <arguments>
+         <arg name="num_tasks"> -np $TOTALPES</arg>
+      </arguments>
+    </mpirun>
+    <module_system type="module">
+      <init_path lang="perl">/sw/lmod/lmod/init/perl</init_path>
+      <init_path lang="python">/sw/lmod/lmod/init/env_modules_python.py</init_path>
+      <init_path lang="sh">/sw/lmod/lmod/init/sh</init_path>
+      <init_path lang="csh">/sw/lmod/lmod/init/csh</init_path>
+      <cmd_path lang="perl">/sw/lmod/lmod/libexec/lmod perl</cmd_path>
+      <cmd_path lang="python">/sw/lmod/lmod/libexec/lmod python</cmd_path>
+      <cmd_path lang="sh">module</cmd_path>
+      <cmd_path lang="csh">module</cmd_path>
+      <modules>
+        <command name="purge"></command>
+        <command name="unuse">/sw/eb/mods/all/MPI/intel/2019.5.281/impi/2018.5.288:/sw/eb/mods/all/Compiler/intel/2019.5.281:/sw/eb/mods/all/Compiler/GCCcore/8.3.0:/sw/hprc/mods/all/Core:/sw/eb/mods/all/Core:/sw/lmod/hprc/mods/Linux:/sw/lmod/8.4.3/modulefiles/Core</command>
+        <command name="use">/scratch/group/ihesp/software/easybuild/eb/x86_64/mods/all/Core</command>
+        <command name="load">intel/2019b</command>
+        <command name="load">CMake/3.15.3</command>
+        <command name="load">cURL/7.66.0</command>
+        <command name="load">Python/2.7.16</command>
+        <command name="load">XML-LibXML/2.0201</command>
+        <command name="load">PnetCDF/1.12.1</command>
+        <command name="load">netCDF/4.7.4</command>
+        <command name="load">netCDF-Fortran/4.5.3</command>
+      </modules>
+   </module_system>
+    <environment_variables>
+      <env name="OMP_STACKSIZE">256M</env>
+    <!--  <env name="I_MPI_PMI_LIBRARY">/usr/lib64/libpmi.so</env> -->
+      <env name="I_MPI_FABRICS">shm:ofi</env>
+      <!--<env name="I_MPI_DEBUG">4</env>-->
+    </environment_variables>
+    <environment_variables>
+      <env name="NETCDF_PATH">/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/netCDF/4.7.4-iimpi-2019b</env>
+      <env name="PNETCDF_PATH">/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/PnetCDF/1.12.1-iimpi-2019b</env>
+      <env name="NETCDFF_INCDIR">/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/netCDF-Fortran/4.5.3-iimpi-2019b/include</env>
+    </environment_variables>
+
+  </machine>
+
+
   <machine MACH="ada">
     <DESC>TAMU IBM NextScale Cluster, os is Linux, 20 pes/node, batch system is LSF</DESC>
     <NODENAME_REGEX>.*.ada.tamu.edu</NODENAME_REGEX>
@@ -2388,7 +2451,7 @@ This allows using a different mpirun command to launch unit tests
       <cmd_path lang="csh">module</cmd_path>
       <modules>
         <command name="purge"/>
-        <command name="load">python/2.7.10-intel-2015B</command>
+        <command name="load">Python/2.7.12-intel-2017A</command>
       </modules>
       <modules compiler="intel">
         <command name="load">cesm-deps/intel-2015B-pn181</command>