Merge pull request #43 from mvdebolskiy/update-betzy

Update dependencies and modules for IntelOneApi to work with NorESM/CESM

config_archive.xml

@@ -1,48 +1,5 @@
 <components version="2.0">
 
-  <comp_archive_spec compname="pop" compclass="ocn">
-    <rest_file_extension>r</rest_file_extension>
-    <rest_file_extension>r[ho]</rest_file_extension>
-    <rest_file_extension>rh.*</rest_file_extension>
-    <hist_file_extension>h\d*.*\.nc$</hist_file_extension>
-    <hist_file_extension>d[dovt]</hist_file_extension>
-    <rest_history_varname>unset</rest_history_varname>
-    <rpointer>
-      <rpointer_file>rpointer.ocn$NINST_STRING.restart</rpointer_file>
-      <rpointer_content>./$CASE.pop$NINST_STRING.r.$DATENAME.nc,RESTART_FMT=nc</rpointer_content>
-    </rpointer>
-    <rpointer>
-      <rpointer_file>rpointer.ocn$NINST_STRING.ovf</rpointer_file>
-      <rpointer_content>./$CASE.pop$NINST_STRING.ro.$DATENAME</rpointer_content>
-    </rpointer>
-    <rpointer>
-      <rpointer_file>rpointer.ocn$NINST_STRING.tavg</rpointer_file>
-      <rpointer_content>./$CASE.pop$NINST_STRING.rh.$DATENAME.nc</rpointer_content>
-    </rpointer>
-    <test_file_names>
-      <tfile disposition="copy">rpointer.pop</tfile>
-      <tfile disposition="copy">casename.pop_0001.r.1976-01-01-00000.nc</tfile>
-      <tfile disposition="copy">casename.pop.r.1976-01-01-00000.nc</tfile>
-      <tfile disposition="move">casename.pop.h.1976-01-01-00000.nc</tfile>
-      <tfile disposition="move">casename.pop.h.1975-02-01-00000.nc</tfile>
-      <tfile disposition="move">casename.pop.h0.1976-01-01-00000.nc</tfile>
-      <tfile disposition="move">casename.pop.dd.1976-01-01-00000</tfile>
-      <tfile disposition="copy">casename.pop.rh.ecosys.nyear1.1976-01-01-00000.nc</tfile>
-      <tfile disposition="ignore">casename.pop.r.1975-01-01-00000.nc</tfile>
-      <tfile disposition="ignore">anothercasename.pop.r.1976-01-01-00000.nc</tfile>
-    </test_file_names>
-  </comp_archive_spec>
-
-  <comp_archive_spec compname="ww3" compclass="wav">
-    <rest_file_extension>r</rest_file_extension>
-    <hist_file_extension>hi.*\.nc$</hist_file_extension>
-    <rest_history_varname>unset</rest_history_varname>
-    <rpointer>
-      <rpointer_file>rpointer.wav$NINST_STRING</rpointer_file>
-      <rpointer_content>unset</rpointer_content>
-    </rpointer>
-    </comp_archive_spec>
-
     <comp_archive_spec compclass="esp" compname="dart">
       <rest_file_extension>r</rest_file_extension>
       <rest_file_extension>rh\d?</rest_file_extension>

machines/betzy/config_machines.xml

@@ -1,8 +1,8 @@
 <machine MACH="betzy">
   <DESC> BullSequana XH2000 AMD® Epyc™ "Rome" 2.2GHz, 128-way nodes, os is Linux, batch system is SLURM</DESC>
   <OS>LINUX</OS>
-  <COMPILERS>intel</COMPILERS>
-  <MPILIBS compiler="intel" >openmpi,impi</MPILIBS>
+  <COMPILERS>intel-oneapi,intel</COMPILERS>
+  <MPILIBS>openmpi,impi</MPILIBS>
   <CIME_OUTPUT_ROOT>/cluster/work/users/$USER/noresm</CIME_OUTPUT_ROOT>
   <DIN_LOC_ROOT>/cluster/shared/noresm/inputdata</DIN_LOC_ROOT>
   <DIN_LOC_ROOT_CLMFORC>/cluster/shared/noresm/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
@@ -21,7 +21,7 @@
   <mpirun mpilib="default">
     <executable>srun</executable>
   </mpirun>
-  <module_system type="module">
+  <module_system type="module" allow_error="true">
     <init_path lang="perl">$ENV{LMOD_PKG}/init/perl</init_path>
     <init_path lang="python">$ENV{LMOD_PKG}/init/env_modules_python.py</init_path>
     <init_path lang="csh">$ENV{LMOD_PKG}/init/csh</init_path>
@@ -30,14 +30,22 @@
     <cmd_path lang="python">$ENV{LMOD_PKG}/libexec/lmod python</cmd_path>
     <cmd_path lang="sh">module</cmd_path>
     <cmd_path lang="csh">module</cmd_path>
+    <modules compiler="intel-oneapi" mpilib='openmpi'>
+      <command name="--force purge"></command>
+      <command name="load">StdEnv</command>
+      <command name="use">/cluster/shared/noresm/eb_mods/modules/all</command>
+      <command name="load">ESMF/8.4.2-iomkl-2022a-ParallelIO-2.5.10</command>
+      <command name="load">Python/3.11.3-GCCcore-12.3.0</command>
+      <command name="load">CMake/3.26.3-GCCcore-12.3.0</command>
+      <command name="load">ParMETIS/4.0.3-iompi-2022a</command>
+    </modules>
     <modules compiler="intel" mpilib="openmpi">
       <command name="--force purge"></command>
       <command name="load">StdEnv</command>
       <command name="use">/cluster/shared/noresm/eb_mods/modules/all</command>
       <command name="load">ESMF/8.4.1-iomkl-2021b-ParallelIO-2.5.10</command>
       <command name="load">CMake/3.21.1-GCCcore-11.2.0</command>
       <command name="load">Python/3.9.6-GCCcore-11.2.0</command>
-
       <command name="load">ParMETIS/4.0.3-iompi-2021b</command>
     </modules>
     <modules compiler="intel" mpilib="impi">

machines/cmake_macros/intel-oneapi_betzy.cmake

@@ -0,0 +1,32 @@
+set(PIO_FILESYSTEM_HINTS "lustre")
+string(APPEND CFLAGS " -qopt-report -march=core-avx2")
+string(APPEND FFLAGS " -qopt-report -march=core-avx2")
+set(NETCDF_C_PATH	$ENV{EBROOTNETCDF})
+set(NETCDF_FORTRAN_PATH $ENV{EBROOTNETCDFMINFORTRAN})
+set(PNETCDF_PATH	$ENV{PNETCDF})
+
+if(DEFINED ENV{PIO})
+  set(PIO_LIBDIR "$ENV{PIO}/lib")
+  set(PIO_INCDIR "$ENV{PIO}/include")
+endif()
+
+string(REPLACE "-mkl=cluster" "" SLIBS "${SLIBS}")
+string(APPEND CPPDEFS " -DHAVE_GETTID")
+
+if (COMP_WAV STREQUAL ww3)
+  if (MPILIB STREQUAL openmpi)
+     string(APPEND SLIBS "  -l:libparmetis.a -lmetis ")
+  endif()
+endif() 
+
+if (MPILIB STREQUAL impi)
+ set(MPICC "mpiicc")
+ set(MPICXX "mpiicpc")
+ set(MPIFC "mpiifort")
+endif()
+
+if (MPILIB STREQUAL openmpi)
+ set(MPICC "mpicc")
+ set(MPICXX "mpicpc")
+ set(MPIFC "mpifort")
+endif()

machines/cmake_macros/ubuntu-latest.cmake

@@ -1 +1 @@
-set(PFUNIT_PATH "$GITHUB_WORKSPACE/pfunit/build")
+set(PFUNIT_PATH "$GITHUB_WORKSPACE/pfunit/build")