Merge pull request #139 from xylar/add-pm-gpu

Many updates to supported Perlmutter compilers

mache/machines/pm-gpu.cfg

@@ -0,0 +1,84 @@
+# Options related to deploying an e3sm-unified conda environment on supported
+# machines
+[e3sm_unified]
+
+# the unix group for permissions for the e3sm-unified conda environment
+group = e3sm
+
+# the compiler set to use for system libraries and MPAS builds
+compiler = nvidiagpu
+
+# the system MPI library to use for intel18 compiler
+mpi = mpich
+
+# the path to the directory where activation scripts, the base environment, and
+# system libraries will be deployed
+base_path = /global/common/software/e3sm/anaconda_envs
+
+# whether to use system modules for hdf5, netcdf-c, netcdf-fortran and pnetcdf
+# (spack modules are used otherwise)
+use_system_hdf5_netcdf = True
+
+
+# config options related to data needed by diagnostics software such as
+# e3sm_diags and MPAS-Analysis
+[diagnostics]
+
+# The base path to the diagnostics directory
+base_path = /global/cfs/cdirs/e3sm/diagnostics
+
+# the unix group for permissions for diagnostics
+group = e3sm
+
+
+# config options associated with web portals
+[web_portal]
+
+# The path to the base of the web portals
+base_path = /global/cfs/cdirs/e3sm/www
+
+# The base URL that corresponds to the base path
+base_url = https://portal.nersc.gov/cfs/e3sm
+
+
+# The parallel section describes options related to running jobs in parallel
+[parallel]
+
+# parallel system of execution: slurm, cobalt or single_node
+system = slurm
+
+# whether to use mpirun or srun to run a task
+parallel_executable = srun
+
+# cores per node on the machine
+cores_per_node = 256
+
+# account for running diagnostics jobs
+account = e3sm
+
+# available constraint(s) (default is the first)
+constraints = gpu
+
+# quality of service (default is the first)
+qos = regular, debug, premium
+
+# Config options related to spack environments
+[spack]
+
+# whether to load modules from the spack yaml file before loading the spack
+# environment
+modules_before = False
+
+# whether to load modules from the spack yaml file after loading the spack
+# environment
+modules_after = False
+
+# whether the machine uses cray compilers
+cray_compilers = True
+
+
+# config options related to synchronizing files
+[sync]
+
+# the full hostname of the machine
+hostname = perlmutter-p1.nersc.gov

mache/spack/pm-cpu_gnu_mpich.csh

@@ -21,10 +21,10 @@ module load craype-accel-host
 {% if e3sm_lapack %}
 module load cray-libsci/23.02.1.1
 {% endif %}
-module load craype/2.7.19
+module load craype/2.7.20
 module rm cray-mpich &> /dev/null
 module load libfabric/1.15.2.0
-module load cray-mpich/8.1.24
+module load cray-mpich/8.1.25
 {% if e3sm_hdf5_netcdf %}
 module rm cray-hdf5-parallel &> /dev/null
 module rm cray-netcdf-hdf5parallel &> /dev/null

mache/spack/pm-cpu_gnu_mpich.sh

@@ -21,10 +21,10 @@ module load craype-accel-host
 {% if e3sm_lapack %}
 module load cray-libsci/23.02.1.1
 {% endif %}
-module load craype/2.7.19
+module load craype/2.7.20
 module rm cray-mpich &> /dev/null
 module load libfabric/1.15.2.0
-module load cray-mpich/8.1.24
+module load cray-mpich/8.1.25
 {% if e3sm_hdf5_netcdf %}
 module rm cray-hdf5-parallel &> /dev/null
 module rm cray-netcdf-hdf5parallel &> /dev/null

mache/spack/pm-cpu_gnu_mpich.yaml

@@ -18,7 +18,7 @@ spack:
     all:
       compiler: [[email protected]]
       providers:
-        mpi: [[email protected].24]
+        mpi: [[email protected].25]
 {% if e3sm_lapack %}
         lapack: [[email protected]]
 {% endif %}
@@ -32,21 +32,46 @@ spack:
       - spec: [email protected]
         prefix: /usr
       buildable: false
+    cmake:
+      externals:
+      - spec: [email protected]
+        prefix: /global/common/software/nersc/pm-2022q4/spack/linux-sles15-zen/cmake-3.24.3-k5msymx/
+      buildable: false
     gettext:
       externals:
       - spec: [email protected]
         prefix: /usr
       buildable: false
+    gmake:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    libuv:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
     libxml2:
       externals:
       - spec: [email protected]
         prefix: /usr
       buildable: false
+    m4:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
     ncurses:
       externals:
       - spec: [email protected]
         prefix: /usr
       buildable: false
+    ninja:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
     openssl:
       externals:
       - spec: [email protected]
@@ -81,16 +106,16 @@ spack:
         - PrgEnv-gnu/8.3.3
         - gcc/11.2.0
         - craype-accel-host
-        - craype/2.7.19
+        - craype/2.7.20
         - libfabric/1.15.2.0
       buildable: false
     cray-mpich:
       externals:
-      - spec: [email protected].24
-        prefix: /opt/cray/pe/mpich/8.1.24/ofi/gnu/9.1
+      - spec: [email protected].25
+        prefix: /opt/cray/pe/mpich/8.1.25/ofi/gnu/9.1
         modules:
         - libfabric/1.15.2.0
-        - cray-mpich/8.1.24
+        - cray-mpich/8.1.25
       buildable: false
     libfabric:
       externals:
@@ -108,35 +133,11 @@ spack:
         - cray-libsci/23.02.1.1
       buildable: false
 {% endif %}
-{% if e3sm_hdf5_netcdf %}
-    hdf5:
-      externals:
-      - spec: [email protected]~cxx+fortran+hl~java+mpi+shared
-        prefix: /opt/cray/pe/hdf5-parallel/1.12.2.3/GNU/9.1
-      buildable: false
-    parallel-netcdf:
-      externals:
-      - spec: [email protected]+cxx+fortran+pic+shared
-        prefix: /opt/cray/pe/parallel-netcdf/1.12.3.3/GNU/9.1/
-      buildable: false
-    netcdf-c:
-      externals:
-      - spec: [email protected]+mpi~parallel-netcdf
-        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/GNU/9.1
-      buildable: false
-    netcdf-fortran:
-      externals:
-      - spec: [email protected] ^netcdf-c+mpi
-        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/GNU/9.1
-      buildable: false
-{% endif %}
-{% if system_hdf5_netcdf %}
+{% if e3sm_hdf5_netcdf or system_hdf5_netcdf %}
     hdf5:
       externals:
       - spec: [email protected]~cxx+fortran+hl~java+mpi+shared
         prefix: /opt/cray/pe/hdf5-parallel/1.12.2.3/GNU/9.1
-      - spec: [email protected]~cxx+fortran+hl~java~mpi+shared
-        prefix: /opt/cray/pe/hdf5/1.12.2.3/GNU/9.1
       buildable: false
     parallel-netcdf:
       externals:
@@ -147,15 +148,11 @@ spack:
       externals:
       - spec: [email protected]+mpi~parallel-netcdf
         prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/GNU/9.1
-      - spec: [email protected]~mpi~parallel-netcdf
-        prefix: /opt/cray/pe/netcdf/4.9.0.3/GNU/9.1
       buildable: false
     netcdf-fortran:
       externals:
-      - spec: [email protected] ^netcdf-c+mpi
+      - spec: [email protected]
         prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/GNU/9.1
-      - spec: [email protected] ^netcdf-c~mpi
-        prefix: /opt/cray/pe/netcdf/4.9.0.3/GNU/9.1
       buildable: false
 {% endif %}
   config:
@@ -175,8 +172,8 @@ spack:
       - PrgEnv-gnu/8.3.3
       - gcc/11.2.0
       - craype-accel-host
-      - craype/2.7.19
+      - craype/2.7.20
       - libfabric/1.15.2.0
       environment:
         prepend_path:
-          PKG_CONFIG_PATH: "/opt/cray/xpmem/2.5.2-2.4_3.30__gd0f7936.shasta/lib64/pkgconfig"
+          PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.33__gd067c3f.shasta/lib64/pkgconfig"

mache/spack/pm-cpu_intel_mpich.csh

@@ -18,10 +18,9 @@ module rm darshan &> /dev/null
 module load PrgEnv-intel/8.3.3
 module load intel/2023.1.0
 module load craype-accel-host
-module load craype/2.7.19
+module load craype/2.7.20
 module rm cray-mpich &> /dev/null
-module load libfabric/1.15.2.0
-module load cray-mpich/8.1.24
+module load cray-mpich/8.1.25
 {% if e3sm_hdf5_netcdf %}
 module rm cray-hdf5-parallel &> /dev/null
 module rm cray-netcdf-hdf5parallel &> /dev/null

mache/spack/pm-cpu_intel_mpich.sh

@@ -18,10 +18,9 @@ module rm darshan &> /dev/null
 module load PrgEnv-intel/8.3.3
 module load intel/2023.1.0
 module load craype-accel-host
-module load craype/2.7.19
+module load craype/2.7.20
 module rm cray-mpich &> /dev/null
-module load libfabric/1.15.2.0
-module load cray-mpich/8.1.24
+module load cray-mpich/8.1.25
 {% if e3sm_hdf5_netcdf %}
 module rm cray-hdf5-parallel &> /dev/null
 module rm cray-netcdf-hdf5parallel &> /dev/null