Merge pull request #39 from mvertens/feature/major_update_to_ccs_config

major update to ccs config for machines
NorESMhub · May 12, 2024 · 10df097 · 10df097
2 parents ff0d5f1 + 6074740
commit 10df097
Show file tree

Hide file tree

Showing 82 changed files with 3,617 additions and 4,109 deletions.
diff --git a/component_grids_nuopc.xml b/component_grids_nuopc.xml
@@ -89,6 +89,13 @@
   </domain>
 
   <!-- CAM/SE meshes -->
+  <domain name="ne3np4">
+    <nx>488</nx> <ny>1</ny>
+    <!-- This is needed for PTS_MODE - it specifies the xml variable PTS_DOMAINFILE -->
+    <file>$DIN_LOC_ROOT/share/domains/domain.lnd.ne3np4_gx3v7.230718.nc</file>
+    <mesh>$DIN_LOC_ROOT/share/meshes/ne3np4_ESMFmesh_c230714_cdf5.nc</mesh>
+    <desc>ne3np4 is Spectral Elem 10-deg grid:</desc>
+  </domain>
   <domain name="ne3np4.pg3">
     <nx>486</nx> <ny>1</ny>
     <mesh>$DIN_LOC_ROOT/share/meshes/ne3pg3_ESMFmesh_c221214_cdf5.nc</mesh>
@@ -241,6 +248,13 @@
     <support>Test support only</support>
   </domain>
 
+  <domain name="ne0np4.POLARCAP.ne30x4">
+    <nx>186194</nx> <ny>1</ny>
+    <mesh>$DIN_LOC_ROOT/share/meshes/POLARCAP_ne30x4_np4_ESMFmesh_cdf5_c20240222.nc</mesh>
+    <desc>ne0np4.POLARCAP.ne30x4 is a Spectral Elem 1-deg grid with a 1/4 deg refined region over the Arctic and Antarctica:</desc>
+    <support>Test support only</support>
+  </domain>
+
   <!-- CAM/MPAS meshes -->
   <domain name="mpasa480">
     <nx>2562</nx> <ny>1</ny>
@@ -320,6 +334,12 @@
     <desc>tx0.66v1 is tripole v1 0.66-deg MOM6 grid:</desc>
     <support>Experimental for MOM6 experiments</support>
   </domain>
+  <domain name="tx2_3v2">
+    <nx>540</nx> <ny>480</ny>
+    <mesh>$DIN_LOC_ROOT/share/meshes/tx2_3v2_230415_ESMFmesh.nc</mesh>
+    <desc>tx2_3v2 is tripole v2 2/3-deg MOM6 grid:</desc>
+    <support>Experimental for MOM6 experiments</support>
+  </domain>
   <domain name="tx0.25v1">
     <nx>1440</nx> <ny>1080</ny>
     <mesh>$DIN_LOC_ROOT/share/meshes/tx0.25v1_190204_ESMFmesh.nc</mesh>

diff --git a/machines/Depends.cray b/machines/Depends.cray
@@ -1,4 +1,18 @@
-NOOPTOBJS= ice_boundary.o dyn_comp.o unicon.o
+NOOPTOBJS= ice_boundary.o dyn_comp.o unicon.o SnowHydrologyMod.o
+
+# RRTMGP contains openmp directives for running on GPUs.  These directives need to be
+# disabled to allow building CAM with threading on CPUs enabled.
+RRTMGP_OBJS=\
+mo_fluxes_byband.o                       mo_rrtmgp_clr_all_sky.o \
+mo_zenith_angle_spherical_correction.o   mo_gas_concentrations.o \
+mo_aerosol_optics_rrtmgp_merra.o         mo_cloud_optics_rrtmgp.o \
+mo_gas_optics_rrtmgp.o                   mo_gas_optics_rrtmgp_kernels.o \
+mo_rte_sw.o                              mo_rte_lw.o \
+mo_rte_util_array_validation.o           mo_rte_util_array.o \
+mo_fluxes_broadband_kernels.o
 
 $(NOOPTOBJS): %.o: %.F90
 	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FFLAGS_NOOPT) $(FREEFLAGS) $<
+
+$(RRTMGP_OBJS): %.o: %.F90
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -h noomp $<
diff --git a/machines/Depends.gnu b/machines/Depends.gnu
@@ -1,2 +1,16 @@
 geopk.o:geopk.F90
 	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -fcray-pointer $<
+
+# RRTMGP contains openmp directives for running on GPUs.  These directives need to be
+# disabled to allow building CAM with threading on CPUs enabled.
+RRTMGP_OBJS=\
+mo_fluxes_byband.o                       mo_rrtmgp_clr_all_sky.o \
+mo_zenith_angle_spherical_correction.o   mo_gas_concentrations.o \
+mo_aerosol_optics_rrtmgp_merra.o         mo_cloud_optics_rrtmgp.o \
+mo_gas_optics_rrtmgp.o                   mo_gas_optics_rrtmgp_kernels.o \
+mo_rte_sw.o                              mo_rte_lw.o \
+mo_rte_util_array_validation.o           mo_rte_util_array.o \
+mo_fluxes_broadband_kernels.o
+
+$(RRTMGP_OBJS): %.o: %.F90
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -fno-openmp $<
diff --git a/machines/Depends.intel b/machines/Depends.intel
@@ -28,11 +28,21 @@ kissvec.o
 
 PUMAS_MG_OBJS=\
 micro_mg1_0.o \
-micro_mg3_0.o \
+micro_pumas_v1.o \
 micro_pumas_data.o \
 micro_pumas_utils.o \
 wv_sat_methods.o
 
+# RRTMGP contains openmp directives for running on GPUs.  These directives need to be
+# disabled to allow building CAM with threading on CPUs enabled.
+RRTMGP_OBJS=\
+mo_fluxes_byband.o                       mo_rrtmgp_clr_all_sky.o \
+mo_zenith_angle_spherical_correction.o   mo_gas_concentrations.o \
+mo_aerosol_optics_rrtmgp_merra.o         mo_cloud_optics_rrtmgp.o \
+mo_gas_optics_rrtmgp.o                   mo_gas_optics_rrtmgp_kernels.o \
+mo_rte_sw.o                              mo_rte_lw.o \
+mo_rte_util_array_validation.o           mo_rte_util_array.o \
+mo_fluxes_broadband_kernels.o
 
 ifeq ($(DEBUG),FALSE)
   $(PERFOBJS): %.o: %.F90
@@ -47,5 +57,7 @@ ifeq ($(DEBUG),FALSE)
 	  $(CC) -c $(INCLDIR) $(INCS) $(CFLAGS) -O3 -fp-model fast $<
   $(PUMAS_MG_OBJS): %.o: %.F90
 	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -no-fma -ftz -no-prec-sqrt -qoverride-limits -no-inline-max-total-size -inline-factor=200 -qopt-report=5 $<
-
 endif
+
+$(RRTMGP_OBJS): %.o: %.F90
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -qno-openmp $<
diff --git a/machines/Depends.intel-oneapi b/machines/Depends.intel-oneapi
@@ -0,0 +1,51 @@
+#
+PERFOBJS=\
+prim_advection_mod.o \
+edge_mod.o \
+derivative_mod.o \
+bndry_mod.o \
+prim_advance_mod.o
+
+# CLM's SatellitePhenologyMod is compiled incorrectly with intel 15.0.0 at -O2
+REDUCED_OPT_OBJS=\
+SatellitePhenologyMod.o 
+
+# shr_wv_sat_mod does not need to have better than ~0.1% precision, and benefits
+# enormously from a lower precision in the vector functions.
+REDUCED_PRECISION_OBJS=\
+shr_wv_sat_mod.o
+
+SHR_RANDNUM_FORT_OBJS=\
+kissvec_mod.o \
+mersennetwister_mod.o \
+dSFMT_interface.o \
+shr_RandNum_mod.o
+
+SHR_RANDNUM_C_OBJS=\
+dSFMT.o \
+dSFMT_utils.o \
+kissvec.o
+
+PUMAS_MG_OBJS=\
+micro_mg1_0.o \
+micro_pumas_v1.o \
+micro_pumas_data.o \
+micro_pumas_utils.o \
+wv_sat_methods.o
+
+
+ifeq ($(DEBUG),FALSE)
+  $(PERFOBJS): %.o: %.F90
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS)  -O3  -no-prec-div $<
+  $(REDUCED_OPT_OBJS): %.o: %.F90
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS)  -O1 $<
+  $(REDUCED_PRECISION_OBJS): %.o: %.F90
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -fimf-precision=low -fp-model fast $<
+  $(SHR_RANDNUM_FORT_OBJS): %.o: %.F90
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fp-model fast -no-prec-div -no-prec-sqrt -qoverride-limits $<
+  $(SHR_RANDNUM_C_OBJS): %.o: %.c
+	  $(CC) -c $(INCLDIR) $(INCS) $(CFLAGS) -O3 -fp-model fast $<
+  $(PUMAS_MG_OBJS): %.o: %.F90
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -no-fma -ftz -no-prec-sqrt -qoverride-limits -no-inline-max-total-size -inline-factor=200 -qopt-report=5 $<
+
+endif
diff --git a/machines/Depends.nvhpc b/machines/Depends.nvhpc
@@ -1,7 +1,78 @@
 # 10/13/2022 nvhpc compiler on gust.hpc.ucar.edu produced incorrect mpitask cound when esm.F90 was optimized
+# 04/05/2023 do not apply optimization flags to "SHR_RANDNUM_C_OBJS" for the nvhpc compiler, which will fail the ensemble consistency test
 #
+# Apply a different optimization level consistent with the Intel compiler
+PERFOBJS=\
+prim_advection_mod.o \
+edge_mod.o \
+derivative_mod.o \
+bndry_mod.o \
+prim_advance_mod.o
+
 REDUCED_OPT_OBJS=\
+SatellitePhenologyMod.o \
 esm.o
 
-$(REDUCED_OPT_OBJS): %.o: %.F90
-		     $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS)  -O1 $<
+REDUCED_PRECISION_OBJS=\
+shr_wv_sat_mod.o
+
+SHR_RANDNUM_FORT_OBJS=\
+kissvec_mod.o \
+mersennetwister_mod.o \
+dSFMT_interface.o \
+shr_RandNum_mod.o
+
+SHR_RANDNUM_C_OBJS=\
+dSFMT.o \
+dSFMT_utils.o \
+kissvec.o
+
+# Apply GPU flags to the following code
+PUMAS_OBJS=\
+micro_mg1_0.o \
+micro_pumas_v1.o \
+micro_pumas_data.o \
+micro_pumas_utils.o \
+pumas_stochastic_collect_tau.o \
+micro_pumas_cam.o \
+wv_sat_methods.o \
+wv_saturation.o \
+macrop_driver.o \
+shr_spfn_mod.o
+
+RRTMGP_OBJS=\
+rrtmgp_allsky.o \
+rrtmgp_rfmip_lw.o \
+rrtmgp_rfmip_sw.o \
+mo_fluxes_byband.o \
+mo_zenith_angle_spherical_correction.o \
+mo_rrtmgp_clr_all_sky.o \
+mo_gas_concentrations.o \
+mo_aerosol_optics_rrtmgp_merra.o \
+mo_cloud_optics_rrtmgp.o \
+mo_gas_optics_rrtmgp.o \
+mo_gas_optics_rrtmgp_kernels.o \
+mo_rte_lw.o \
+mo_rte_sw.o \
+mo_rte_util_array_validation.o \
+mo_rte_util_array.o \
+mo_fluxes_broadband_kernels.o \
+mo_rte_solver_kernels.o \
+mo_optical_props_kernels.o
+
+ifeq ($(DEBUG),FALSE)
+  $(PERFOBJS): %.o: %.F90
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -Mfprelaxed=div $<
+  $(REDUCED_OPT_OBJS): %.o: %.F90
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O1 $<
+  $(REDUCED_PRECISION_OBJS): %.o: %.F90
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -fast $<
+  $(SHR_RANDNUM_FORT_OBJS): %.o: %.F90
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fast -Mfprelaxed=div,sqrt $<
+#  $(SHR_RANDNUM_C_OBJS): %.o: %.c
+#	  $(CC) -c $(INCLDIR) $(INCS) $(CFLAGS) -O3 -fast $<
+  $(PUMAS_OBJS): %.o: %.F90
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt $(GPUFLAGS) $<
+  $(RRTMGP_OBJS): %.o: %.F90
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(GPUFLAGS) $<
+endif
diff --git a/machines/Depends.nvhpc-gpu b/machines/Depends.nvhpc-gpu
diff --git a/machines/Depends.pgi-gpu b/machines/Depends.pgi-gpu
diff --git a/machines/aleph/config_machines.xml b/machines/aleph/config_machines.xml
@@ -0,0 +1,64 @@
+<machine MACH="aleph">
+  <DESC>XC50 SkyLake, os is CNL, 40 pes/node, batch system is PBSPro</DESC>
+  <OS>CNL</OS>
+  <COMPILERS>intel,gnu,cray</COMPILERS>
+  <MPILIBS>mpt,mpi-serial</MPILIBS>
+  <CIME_OUTPUT_ROOT>/proj/$ENV{USER}</CIME_OUTPUT_ROOT>
+  <DIN_LOC_ROOT>$ENV{DIN_LOC_ROOT}</DIN_LOC_ROOT>
+  <DIN_LOC_ROOT_CLMFORC>$DIN_LOC_ROOT</DIN_LOC_ROOT_CLMFORC>
+  <DOUT_S_ROOT>${CIME_OUTPUT_ROOT}/archive/$CASE</DOUT_S_ROOT>
+  <BASELINE_ROOT>${CIME_OUTPUT_ROOT}/cesm_baselines</BASELINE_ROOT>
+  <GMAKE_J>8</GMAKE_J>
+  <BATCH_SYSTEM>pbs</BATCH_SYSTEM>
+  <SUPPORTED_BY> @ pusan.ac.kr</SUPPORTED_BY>
+  <MAX_TASKS_PER_NODE>40</MAX_TASKS_PER_NODE>
+  <MAX_MPITASKS_PER_NODE>40</MAX_MPITASKS_PER_NODE>
+  <mpirun mpilib="default">
+    <executable>aprun</executable>
+    <arguments>
+      <arg name="hyperthreading" default="1"> -j {{ hyperthreading }}</arg>
+      <arg name="num_tasks"> -n {{ total_tasks }}</arg>
+      <arg name="tasks_per_node"> -N $MAX_MPITASKS_PER_NODE</arg>
+      <arg name="tasks_per_numa" > -S {{ tasks_per_numa }}</arg>
+      <arg name="thread_count"> -d $ENV{OMP_NUM_THREADS}</arg>
+      <arg name="env_thread_count">--mpmd-env OMP_NUM_THREADS=$OMP_NUM_THREADS</arg>
+    </arguments>
+  </mpirun>
+  <module_system type="module">
+    <init_path lang="perl">/opt/modules/default/init/perl.pm</init_path>
+    <init_path lang="python">/opt/modules/default/init/python.py</init_path>
+    <init_path lang="sh">/opt/modules/default/init/sh</init_path>
+    <init_path lang="csh">/opt/modules/default/init/csh</init_path>
+    <cmd_path lang="perl">/opt/modules/default/bin/modulecmd perl</cmd_path>
+    <cmd_path lang="python">/opt/modules/default/bin/modulecmd python</cmd_path>
+    <cmd_path lang="sh">module</cmd_path>
+    <cmd_path lang="csh">module</cmd_path>
+    <modules>
+      <command name="rm">craype-x86-skylake</command>
+      <command name="rm">PrgEnv-pgi</command>
+      <command name="rm">PrgEnv-intel</command>
+      <command name="rm">PrgEnv-cray</command>
+      <command name="rm">PrgEnv-gnu</command>
+      <command name="rm">cray-netcdf</command>
+      <command name="rm">cray-hdf5</command>
+      <command name="rm">cray-parallel-netcdf</command>
+      <command name="rm">papi</command>
+    </modules>
+    <modules compiler="intel">
+      <command name="load">PrgEnv-intel</command>
+      <command name="load">craype-x86-skylake</command>
+      <command name="load">craype-hugepages2M</command>
+      <command name="rm">perftools-base/7.0.4</command>
+      <command name="load">cray-netcdf/4.6.1.3</command>
+      <command name="load">cray-hdf5/1.10.2.0</command>
+      <command name="load">cray-parallel-netcdf/1.11.1.1</command>
+      <command name="load">papi/5.6.0.4</command>
+      <command name="load">gridftp/6.0</command>
+      <command name="load">cray-python/3.6.5.1</command>
+    </modules>
+  </module_system>
+  <environment_variables>
+    <env name="OMP_STACKSIZE">256M</env>
+    <env name="POSTPROCESS_PATH">/home/jedwards/workflow/CESM_postprocessing</env>
+  </environment_variables>
+</machine>