Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support for Grace. And new GoM LR domain #2

Open
wants to merge 1 commit into
base: ihesp-regional-master
Choose a base branch
from
Open

Support for Grace. And new GoM LR domain #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions config/cesm/config_grids.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -995,6 +995,12 @@
<grid name="ocnice">gom3x</grid>
</model_grid>

<model_grid alias="txlw27k_gom27k">
<grid name="atm">txlw27k</grid>
<grid name="lnd">txlw27k</grid>
<grid name="ocnice">gom27k</grid>
</model_grid>

</grids>

<!-- ======================================================== -->
Expand Down Expand Up @@ -1468,6 +1474,17 @@
<desc>3km Gulf of Mexico for Wrf, to pair with gom3x in xRoms</desc>
</domain>

<domain name="txlw27k">
<nx>89</nx> <ny>79</ny>
<file grid="atm|lnd">$DIN_LOC_ROOT/share/domains/domain.atmlnd.txlw27k_gom27k.210706.nc</file>
<desc>27km Gulf of Mexico for Wrf, to pair with gom27k in xRoms</desc>
</domain>
<domain name="gom27k">
<nx>273</nx> <ny>242</ny>
<file grid="ocnice">$DIN_LOC_ROOT/share/domains/gom_lr_docn_grd.210705.nc</file>
<desc>27km Gulf of Mexico grid for xRoms</desc>
</domain>

<!-- GLC domain -->

<domain name="gland20">
Expand Down Expand Up @@ -1837,6 +1854,13 @@
<map name="OCN2ATM_SMAPNAME">atm/wrf/tx3k_g3x/map_o2a_aave_Rmask_1909.nc</map>
</gridmap>

<gridmap atm_grid="txlw27k" ocn_grid="gom27k">
<map name="ATM2OCN_FMAPNAME">atm/wrf/tx27k_g27x/map_a2o_aave.nc</map>
<map name="ATM2OCN_SMAPNAME">atm/wrf/tx27k_g27x/map_a2o_aave.nc</map>
<map name="ATM2OCN_VMAPNAME">atm/wrf/tx27k_g27x/map_a2o_aave.nc</map>
<map name="OCN2ATM_FMAPNAME">atm/wrf/tx27k_g27x/map_o2a_aave.nc</map>
<map name="OCN2ATM_SMAPNAME">atm/wrf/tx27k_g27x/map_o2a_aave.nc</map>
</gridmap>
<!-- QL, 150525, wav to ocn, atm, ice mapping files -->

<gridmap ocn_grid="gx3v7" wav_grid="ww3a">
Expand Down
19 changes: 19 additions & 0 deletions config/cesm/machines/config_batch.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -509,6 +509,25 @@
</queues>
</batch_system>

<batch_system MACH="grace" type="slurm" >
<batch_submit>sbatch</batch_submit>
<submit_args>
<arg flag="--time" name="$JOB_WALLCLOCK_TIME"/>
<arg flag="-p" name="$JOB_QUEUE"/>
</submit_args>
<directives>
<directive>--mem=360GB </directive>
</directives>
<queues>
<queue walltimemax="02:00:00" nodemin="1" nodemax="32">short</queue>
<queue walltimemax="24:00:00" nodemin="1" nodemax="128" default="true">medium</queue>
<queue walltimemax="168:00:00" nodemin="1" nodemax="64" >long</queue>
<queue walltimemax="504:00:00" nodemin="1" nodemax="32" >xlong</queue>
<queue walltimemax="24:00:00" nodemin="1" nodemax="800" >special</queue>
</queues>
</batch_system>


<batch_system MACH="ada" type="lsf">
<queues>
<queue walltimemax="168:00" nodemin="101" nodemax="811">special</queue>
Expand Down
37 changes: 37 additions & 0 deletions config/cesm/machines/config_compilers.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1160,6 +1160,43 @@ using a fortran linker.
</SLIBS>
</compiler>

<compiler MACH="grace" COMPILER="intel">
<CPPDEFS>
<append> -DHAVE_NANOTIME -DNO_R16 </append>
</CPPDEFS>
<CFLAGS>
<append></append>
</CFLAGS>
<FFLAGS>
<append></append>
</FFLAGS>
<MPIFC>mpiifort</MPIFC>
<MPICC>mpiicc</MPICC>
<MPICXX>mpiicpc</MPICXX>
<SFC>ifort</SFC>
<SCC>icc</SCC>
<SCXX>icpc</SCXX>
<!-- <NETCDF_PATH>/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/netCDF/4.3.3.1-intel-2018b</NETCDF_PATH>
<NETCDF_C_PATH>/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/netCDF/4.3.3.1-intel-2018b</NETCDF_C_PATH>
<PNETCDF_PATH>/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/PnetCDF/1.8.1-intel-2018b</PNETCDF_PATH>
<NETCDF_FORTRAN_PATH>/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/netCDF-Fortran/4.4.2-intel-2018b</NETCDF_FORTRAN_PATH> -->
<!-- <NETCDF_PATH>/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/netCDF/4.3.3.1-intel-2020b</NETCDF_PATH>
<NETCDF_C_PATH>/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/netCDF/4.3.3.1-intel-2020b</NETCDF_C_PATH>
<PNETCDF_PATH>/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/PnetCDF/1.8.1-intel-2020b</PNETCDF_PATH>
<NETCDF_FORTRAN_PATH>/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/netCDF-Fortran/4.4.2-intel-2020b</NETCDF_FORTRAN_PATH> -->
<NETCDF_PATH>/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/netCDF/4.7.4-iimpi-2019b</NETCDF_PATH>
<NETCDF_C_PATH>/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/netCDF/4.7.4-iimpi-2019b</NETCDF_C_PATH>
<NETCDF_FORTRAN_PATH>/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/netCDF-Fortran/4.5.3-iimpi-2019b</NETCDF_FORTRAN_PATH>
<PNETCDF_PATH>/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/PnetCDF/1.12.1-iimpi-2019b</PNETCDF_PATH>
<SLIBS>
<append>-L$(NETCDF_C_PATH)/lib -Wl,-rpath,$(NETCDF_PATH)/lib -lnetcdf</append>
<append>-L$(NETCDF_FORTRAN_PATH)/lib -Wl,-rpath,$(NETCDF_FORTRAN_PATH)/lib -lnetcdff</append>
<append>-L$(PNETCDF_PATH)/lib -Wl,-rpath,$(PNETCDF_PATH)/lib -lpnetcdf</append>
</SLIBS>
<PIO_FILESYSTEM_HINTS>lustre</PIO_FILESYSTEM_HINTS>
</compiler>


<compiler MACH="ada" COMPILER="intel">
<CPPDEFS>
<append> -DHAVE_NANOTIME -DNO_R16 </append>
Expand Down
65 changes: 64 additions & 1 deletion config/cesm/machines/config_machines.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2344,6 +2344,69 @@ This allows using a different mpirun command to launch unit tests
</module_system>
</machine>


<machine MACH="grace">
<DESC>Intel Xeon 6248R 3.0 GHz ("Cascade Lake"),48 cores on two sockets (24 cores/socket) , batch system is SLURM</DESC>
<NODENAME_REGEX>.*grace</NODENAME_REGEX>
<OS>LINUX</OS>
<COMPILERS>intel</COMPILERS>
<MPILIBS>impi</MPILIBS>
<!--<CIME_OUTPUT_ROOT>$ENV{SCRATCH}</CIME_OUTPUT_ROOT>-->
<CIME_OUTPUT_ROOT>/ihesp/shared/rundir/rcesm</CIME_OUTPUT_ROOT>
<DIN_LOC_ROOT>/ihesp/obs_root/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>/ihesp/obs_root/inputdata/lmwg</DIN_LOC_ROOT_CLMFORC>
<DOUT_S_ROOT>$ENV{SCRATCH}/archive/$CASE</DOUT_S_ROOT>
<BASELINE_ROOT>/ihesp/obs_root/inputdata/cesm_baselines</BASELINE_ROOT>
<CCSM_CPRNC>/ihesp/software/cesm/cime/tools/cprnc/cprnc</CCSM_CPRNC>
<GMAKE_J>4</GMAKE_J>
<BATCH_SYSTEM>slurm</BATCH_SYSTEM>
<SUPPORTED_BY>agopal</SUPPORTED_BY>
<MAX_TASKS_PER_NODE>96</MAX_TASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE>48</MAX_MPITASKS_PER_NODE>
<mpirun mpilib="impi">
<executable>mpirun</executable>
<arguments>
<arg name="num_tasks"> -np $TOTALPES</arg>
</arguments>
</mpirun>
<module_system type="module">
<init_path lang="perl">/sw/lmod/lmod/init/perl</init_path>
<init_path lang="python">/sw/lmod/lmod/init/env_modules_python.py</init_path>
<init_path lang="sh">/sw/lmod/lmod/init/sh</init_path>
<init_path lang="csh">/sw/lmod/lmod/init/csh</init_path>
<cmd_path lang="perl">/sw/lmod/lmod/libexec/lmod perl</cmd_path>
<cmd_path lang="python">/sw/lmod/lmod/libexec/lmod python</cmd_path>
<cmd_path lang="sh">module</cmd_path>
<cmd_path lang="csh">module</cmd_path>
<modules>
<command name="purge"></command>
<command name="unuse">/sw/eb/mods/all/MPI/intel/2019.5.281/impi/2018.5.288:/sw/eb/mods/all/Compiler/intel/2019.5.281:/sw/eb/mods/all/Compiler/GCCcore/8.3.0:/sw/hprc/mods/all/Core:/sw/eb/mods/all/Core:/sw/lmod/hprc/mods/Linux:/sw/lmod/8.4.3/modulefiles/Core</command>
<command name="use">/scratch/group/ihesp/software/easybuild/eb/x86_64/mods/all/Core</command>
<command name="load">intel/2019b</command>
<command name="load">CMake/3.15.3</command>
<command name="load">cURL/7.66.0</command>
<command name="load">Python/2.7.16</command>
<command name="load">XML-LibXML/2.0201</command>
<command name="load">PnetCDF/1.12.1</command>
<command name="load">netCDF/4.7.4</command>
<command name="load">netCDF-Fortran/4.5.3</command>
</modules>
</module_system>
<environment_variables>
<env name="OMP_STACKSIZE">256M</env>
<!-- <env name="I_MPI_PMI_LIBRARY">/usr/lib64/libpmi.so</env> -->
<env name="I_MPI_FABRICS">shm:ofi</env>
<!--<env name="I_MPI_DEBUG">4</env>-->
</environment_variables>
<environment_variables>
<env name="NETCDF_PATH">/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/netCDF/4.7.4-iimpi-2019b</env>
<env name="PNETCDF_PATH">/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/PnetCDF/1.12.1-iimpi-2019b</env>
<env name="NETCDFF_INCDIR">/scratch/group/ihesp/software/easybuild/eb/x86_64/sw/netCDF-Fortran/4.5.3-iimpi-2019b/include</env>
</environment_variables>

</machine>


<machine MACH="ada">
<DESC>TAMU IBM NextScale Cluster, os is Linux, 20 pes/node, batch system is LSF</DESC>
<NODENAME_REGEX>.*.ada.tamu.edu</NODENAME_REGEX>
Expand Down Expand Up @@ -2388,7 +2451,7 @@ This allows using a different mpirun command to launch unit tests
<cmd_path lang="csh">module</cmd_path>
<modules>
<command name="purge"/>
<command name="load">python/2.7.10-intel-2015B</command>
<command name="load">Python/2.7.12-intel-2017A</command>
</modules>
<modules compiler="intel">
<command name="load">cesm-deps/intel-2015B-pn181</command>
Expand Down