From 9fddf4b45b772c1bc401c778d7a11a4742685755 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sun, 20 Oct 2024 21:31:42 -0700 Subject: [PATCH 01/14] Update libfabric --- mache/spack/pm-cpu_gnu_mpich.csh | 2 +- mache/spack/pm-cpu_gnu_mpich.sh | 2 +- mache/spack/pm-cpu_gnu_mpich.yaml | 13 +++---------- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/mache/spack/pm-cpu_gnu_mpich.csh b/mache/spack/pm-cpu_gnu_mpich.csh index 14e3845f..41542c44 100644 --- a/mache/spack/pm-cpu_gnu_mpich.csh +++ b/mache/spack/pm-cpu_gnu_mpich.csh @@ -27,7 +27,7 @@ module load cray-libsci/23.02.1.1 {% endif %} module load craype/2.7.20 module rm cray-mpich &> /dev/null -module load libfabric/1.15.2.0 +module load libfabric/1.20.1 module load cray-mpich/8.1.25 {% if e3sm_hdf5_netcdf %} module rm cray-hdf5-parallel &> /dev/null diff --git a/mache/spack/pm-cpu_gnu_mpich.sh b/mache/spack/pm-cpu_gnu_mpich.sh index 75fb3870..53600b30 100644 --- a/mache/spack/pm-cpu_gnu_mpich.sh +++ b/mache/spack/pm-cpu_gnu_mpich.sh @@ -27,7 +27,7 @@ module load cray-libsci/23.02.1.1 {% endif %} module load craype/2.7.20 module rm cray-mpich &> /dev/null -module load libfabric/1.15.2.0 +module load libfabric/1.20.1 module load cray-mpich/8.1.25 {% if e3sm_hdf5_netcdf %} module rm cray-hdf5-parallel &> /dev/null diff --git a/mache/spack/pm-cpu_gnu_mpich.yaml b/mache/spack/pm-cpu_gnu_mpich.yaml index 1a8b5d6d..3d809b39 100644 --- a/mache/spack/pm-cpu_gnu_mpich.yaml +++ b/mache/spack/pm-cpu_gnu_mpich.yaml @@ -107,23 +107,16 @@ spack: - gcc/12.2.0 - craype-accel-host - craype/2.7.20 - - libfabric/1.15.2.0 + - libfabric/1.20.1 buildable: false cray-mpich: externals: - spec: cray-mpich@8.1.25 prefix: /opt/cray/pe/mpich/8.1.25/ofi/gnu/9.1 modules: - - libfabric/1.15.2.0 + - libfabric/1.20.1 - cray-mpich/8.1.25 buildable: false - libfabric: - externals: - - spec: libfabric@1.15.2.0 - prefix: /opt/cray/libfabric/1.15.2.0 - modules: - - libfabric/1.15.2.0 - buildable: false {% if e3sm_lapack %} cray-libsci: externals: @@ -173,7 +166,7 @@ spack: - gcc/12.2.0 - craype-accel-host - craype/2.7.20 - - libfabric/1.15.2.0 + - libfabric/1.20.1 environment: prepend_path: PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.33__gd067c3f.shasta/lib64/pkgconfig" From 5bce2536038ef69113f2824bbc45356e26fdadf7 Mon Sep 17 00:00:00 2001 From: Althea Denlinger Date: Tue, 22 Oct 2024 09:59:51 -0700 Subject: [PATCH 02/14] Require `cray-libsci` as part of compiler stack --- mache/spack/pm-cpu_gnu_mpich.csh | 4 +--- mache/spack/pm-cpu_gnu_mpich.sh | 4 +--- mache/spack/pm-cpu_gnu_mpich.yaml | 10 ++-------- mache/spack/pm-cpu_nvidia_mpich.csh | 4 +--- mache/spack/pm-cpu_nvidia_mpich.sh | 4 +--- mache/spack/pm-cpu_nvidia_mpich.yaml | 10 ++-------- mache/spack/pm-gpu_gnugpu_mpich.csh | 4 +--- mache/spack/pm-gpu_gnugpu_mpich.sh | 4 +--- mache/spack/pm-gpu_gnugpu_mpich.yaml | 10 ++-------- mache/spack/pm-gpu_nvidiagpu_mpich.csh | 4 +--- mache/spack/pm-gpu_nvidiagpu_mpich.sh | 4 +--- mache/spack/pm-gpu_nvidiagpu_mpich.yaml | 10 ++-------- 12 files changed, 16 insertions(+), 56 deletions(-) diff --git a/mache/spack/pm-cpu_gnu_mpich.csh b/mache/spack/pm-cpu_gnu_mpich.csh index 41542c44..8205a1af 100644 --- a/mache/spack/pm-cpu_gnu_mpich.csh +++ b/mache/spack/pm-cpu_gnu_mpich.csh @@ -22,9 +22,7 @@ module rm darshan &> /dev/null module load PrgEnv-gnu/8.5.0 module load gcc/12.2.0 module load craype-accel-host -{% if e3sm_lapack %} -module load cray-libsci/23.02.1.1 -{% endif %} +module load cray-libsci/23.12.5 module load craype/2.7.20 module rm cray-mpich &> /dev/null module load libfabric/1.20.1 diff --git a/mache/spack/pm-cpu_gnu_mpich.sh b/mache/spack/pm-cpu_gnu_mpich.sh index 53600b30..bda35395 100644 --- a/mache/spack/pm-cpu_gnu_mpich.sh +++ b/mache/spack/pm-cpu_gnu_mpich.sh @@ -22,9 +22,7 @@ module rm darshan &> /dev/null module load PrgEnv-gnu/8.5.0 module load gcc/12.2.0 module load craype-accel-host -{% if e3sm_lapack %} -module load cray-libsci/23.02.1.1 -{% endif %} +module load cray-libsci/23.12.5 module load craype/2.7.20 module rm cray-mpich &> /dev/null module load libfabric/1.20.1 diff --git a/mache/spack/pm-cpu_gnu_mpich.yaml b/mache/spack/pm-cpu_gnu_mpich.yaml index 3d809b39..5f045345 100644 --- a/mache/spack/pm-cpu_gnu_mpich.yaml +++ b/mache/spack/pm-cpu_gnu_mpich.yaml @@ -2,9 +2,7 @@ spack: specs: - gcc - cray-mpich -{% if e3sm_lapack %} - cray-libsci -{% endif %} {% if e3sm_hdf5_netcdf %} - hdf5 - netcdf-c @@ -19,9 +17,7 @@ spack: compiler: [gcc@12.2.0] providers: mpi: [cray-mpich@8.1.25] -{% if e3sm_lapack %} - lapack: [cray-libsci@23.02.1.1] -{% endif %} + lapack: [cray-libsci@23.12.5] bzip2: externals: - spec: bzip2@1.0.6 @@ -117,15 +113,13 @@ spack: - libfabric/1.20.1 - cray-mpich/8.1.25 buildable: false -{% if e3sm_lapack %} cray-libsci: externals: - - spec: cray-libsci@23.02.1.1 + - spec: cray-libsci@23.12.5 prefix: /opt/cray/pe/libsci/23.02.1.1/GNU/9.1/x86_64 modules: - cray-libsci/23.02.1.1 buildable: false -{% endif %} {% if e3sm_hdf5_netcdf %} hdf5: externals: diff --git a/mache/spack/pm-cpu_nvidia_mpich.csh b/mache/spack/pm-cpu_nvidia_mpich.csh index 87ff8220..a9ef0190 100644 --- a/mache/spack/pm-cpu_nvidia_mpich.csh +++ b/mache/spack/pm-cpu_nvidia_mpich.csh @@ -27,9 +27,7 @@ module load craype-accel-host module load craype/2.7.20 module rm cray-mpich &> /dev/null module load cray-mpich/8.1.25 -{% if e3sm_lapack %} -module load cray-libsci/23.02.1.1 -{% endif %} +module load cray-libsci/23.12.5 {% if e3sm_hdf5_netcdf %} module rm cray-hdf5-parallel &> /dev/null module rm cray-netcdf-hdf5parallel &> /dev/null diff --git a/mache/spack/pm-cpu_nvidia_mpich.sh b/mache/spack/pm-cpu_nvidia_mpich.sh index f28d4d23..af25835c 100644 --- a/mache/spack/pm-cpu_nvidia_mpich.sh +++ b/mache/spack/pm-cpu_nvidia_mpich.sh @@ -27,9 +27,7 @@ module load craype-accel-host module load craype/2.7.20 module rm cray-mpich &> /dev/null module load cray-mpich/8.1.25 -{% if e3sm_lapack %} -module load cray-libsci/23.02.1.1 -{% endif %} +module load cray-libsci/23.12.5 {% if e3sm_hdf5_netcdf %} module rm cray-hdf5-parallel &> /dev/null module rm cray-netcdf-hdf5parallel &> /dev/null diff --git a/mache/spack/pm-cpu_nvidia_mpich.yaml b/mache/spack/pm-cpu_nvidia_mpich.yaml index 52b02d73..779d8a48 100644 --- a/mache/spack/pm-cpu_nvidia_mpich.yaml +++ b/mache/spack/pm-cpu_nvidia_mpich.yaml @@ -1,9 +1,7 @@ spack: specs: - cray-mpich -{% if e3sm_lapack %} - cray-libsci -{% endif %} {% if e3sm_hdf5_netcdf %} - hdf5 - netcdf-c @@ -18,9 +16,7 @@ spack: compiler: [nvhpc@22.7] providers: mpi: [cray-mpich@8.1.25] -{% if e3sm_lapack %} - lapack: [cray-libsci@23.02.1.1] -{% endif %} + lapack: [cray-libsci@23.12.5] bzip2: externals: - spec: bzip2@1.0.6 @@ -113,13 +109,11 @@ spack: modules: - libfabric/1.15.2.0 buildable: false -{% if e3sm_lapack %} cray-libsci: externals: - - spec: cray-libsci@23.02.1.1 + - spec: cray-libsci@23.12.5 prefix: /opt/cray/pe/libsci/23.02.1.1/NVIDIA/20.7/x86_64 buildable: false -{% endif %} {% if e3sm_hdf5_netcdf %} hdf5: externals: diff --git a/mache/spack/pm-gpu_gnugpu_mpich.csh b/mache/spack/pm-gpu_gnugpu_mpich.csh index 53e03d55..62f93822 100644 --- a/mache/spack/pm-gpu_gnugpu_mpich.csh +++ b/mache/spack/pm-gpu_gnugpu_mpich.csh @@ -26,9 +26,7 @@ module load craype-accel-nvidia80 module load craype/2.7.20 module rm cray-mpich &> /dev/null module load cray-mpich/8.1.25 -{% if e3sm_lapack %} -module load cray-libsci/23.02.1.1 -{% endif %} +module load cray-libsci/23.12.5 {% if e3sm_hdf5_netcdf %} module rm cray-hdf5-parallel &> /dev/null module rm cray-netcdf-hdf5parallel &> /dev/null diff --git a/mache/spack/pm-gpu_gnugpu_mpich.sh b/mache/spack/pm-gpu_gnugpu_mpich.sh index 3ce26746..dbfe5481 100644 --- a/mache/spack/pm-gpu_gnugpu_mpich.sh +++ b/mache/spack/pm-gpu_gnugpu_mpich.sh @@ -26,9 +26,7 @@ module load craype-accel-nvidia80 module load craype/2.7.20 module rm cray-mpich &> /dev/null module load cray-mpich/8.1.25 -{% if e3sm_lapack %} -module load cray-libsci/23.02.1.1 -{% endif %} +module load cray-libsci/23.12.5 {% if e3sm_hdf5_netcdf %} module rm cray-hdf5-parallel &> /dev/null module rm cray-netcdf-hdf5parallel &> /dev/null diff --git a/mache/spack/pm-gpu_gnugpu_mpich.yaml b/mache/spack/pm-gpu_gnugpu_mpich.yaml index 985ceddb..172125ca 100644 --- a/mache/spack/pm-gpu_gnugpu_mpich.yaml +++ b/mache/spack/pm-gpu_gnugpu_mpich.yaml @@ -2,9 +2,7 @@ spack: specs: - gcc - cray-mpich -{% if e3sm_lapack %} - cray-libsci -{% endif %} {% if e3sm_hdf5_netcdf %} - hdf5 - netcdf-c @@ -19,9 +17,7 @@ spack: compiler: [gcc@11.2.0] providers: mpi: [cray-mpich@8.1.24] -{% if e3sm_lapack %} - lapack: [cray-libsci@23.02.1.1] -{% endif %} + lapack: [cray-libsci@23.12.5] bzip2: externals: - spec: bzip2@1.0.6 @@ -137,15 +133,13 @@ spack: modules: - libfabric/1.15.2.0 buildable: false -{% if e3sm_lapack %} cray-libsci: externals: - - spec: cray-libsci@23.02.1.1 + - spec: cray-libsci@23.12.5 prefix: /opt/cray/pe/libsci/23.02.1.1/GNU/9.1/x86_64 modules: - cray-libsci/23.02.1.1 buildable: false -{% endif %} {% if e3sm_hdf5_netcdf %} hdf5: externals: diff --git a/mache/spack/pm-gpu_nvidiagpu_mpich.csh b/mache/spack/pm-gpu_nvidiagpu_mpich.csh index 5b474a81..24019f90 100644 --- a/mache/spack/pm-gpu_nvidiagpu_mpich.csh +++ b/mache/spack/pm-gpu_nvidiagpu_mpich.csh @@ -27,9 +27,7 @@ module load gcc-mixed/11.2.0 module load craype/2.7.20 module rm cray-mpich &> /dev/null module load cray-mpich/8.1.25 -{% if e3sm_lapack %} -module load cray-libsci/23.02.1.1 -{% endif %} +module load cray-libsci/23.12.5 {% if e3sm_hdf5_netcdf %} module rm cray-hdf5-parallel &> /dev/null module rm cray-netcdf-hdf5parallel &> /dev/null diff --git a/mache/spack/pm-gpu_nvidiagpu_mpich.sh b/mache/spack/pm-gpu_nvidiagpu_mpich.sh index ae949ff8..986993ed 100644 --- a/mache/spack/pm-gpu_nvidiagpu_mpich.sh +++ b/mache/spack/pm-gpu_nvidiagpu_mpich.sh @@ -27,9 +27,7 @@ module load gcc-mixed/11.2.0 module load craype/2.7.20 module rm cray-mpich &> /dev/null module load cray-mpich/8.1.25 -{% if e3sm_lapack %} -module load cray-libsci/23.02.1.1 -{% endif %} +module load cray-libsci/23.12.5 {% if e3sm_hdf5_netcdf %} module rm cray-hdf5-parallel &> /dev/null module rm cray-netcdf-hdf5parallel &> /dev/null diff --git a/mache/spack/pm-gpu_nvidiagpu_mpich.yaml b/mache/spack/pm-gpu_nvidiagpu_mpich.yaml index 5c1338df..be8bb68a 100644 --- a/mache/spack/pm-gpu_nvidiagpu_mpich.yaml +++ b/mache/spack/pm-gpu_nvidiagpu_mpich.yaml @@ -1,9 +1,7 @@ spack: specs: - cray-mpich -{% if e3sm_lapack %} - cray-libsci -{% endif %} {% if e3sm_hdf5_netcdf %} - hdf5 - netcdf-c @@ -18,9 +16,7 @@ spack: compiler: [nvhpc@22.7] providers: mpi: [cray-mpich@8.1.25] -{% if e3sm_lapack %} - lapack: [cray-libsci@23.02.1.1] -{% endif %} + lapack: [cray-libsci@23.12.5] bzip2: externals: - spec: bzip2@1.0.6 @@ -126,15 +122,13 @@ spack: modules: - libfabric/1.15.2.0 buildable: false -{% if e3sm_lapack %} cray-libsci: externals: - - spec: cray-libsci@23.02.1.1 + - spec: cray-libsci@23.12.5 prefix: /opt/cray/pe/libsci/23.02.1.1/NVIDIA/20.7/x86_64 modules: - cray-libsci/23.02.1.1 buildable: false -{% endif %} {% if e3sm_hdf5_netcdf %} hdf5: externals: From ac0efad6f4dc5487a8fd1b01f3633e0cfecfa177 Mon Sep 17 00:00:00 2001 From: Althea Denlinger Date: Tue, 22 Oct 2024 09:20:26 -0700 Subject: [PATCH 03/14] Combine module load and rm commands --- mache/spack/pm-cpu_gnu_mpich.csh | 65 ++++++++++++------------- mache/spack/pm-cpu_gnu_mpich.sh | 65 ++++++++++++------------- mache/spack/pm-cpu_intel_mpich.csh | 63 ++++++++++++------------ mache/spack/pm-cpu_intel_mpich.sh | 63 ++++++++++++------------ mache/spack/pm-cpu_nvidia_mpich.csh | 67 ++++++++++++-------------- mache/spack/pm-cpu_nvidia_mpich.sh | 67 ++++++++++++-------------- mache/spack/pm-gpu_gnugpu_mpich.csh | 65 ++++++++++++------------- mache/spack/pm-gpu_gnugpu_mpich.sh | 65 ++++++++++++------------- mache/spack/pm-gpu_nvidiagpu_mpich.csh | 67 ++++++++++++-------------- mache/spack/pm-gpu_nvidiagpu_mpich.sh | 63 ++++++++++++------------ 10 files changed, 310 insertions(+), 340 deletions(-) diff --git a/mache/spack/pm-cpu_gnu_mpich.csh b/mache/spack/pm-cpu_gnu_mpich.csh index 8205a1af..67e2fe7c 100644 --- a/mache/spack/pm-cpu_gnu_mpich.csh +++ b/mache/spack/pm-cpu_gnu_mpich.csh @@ -1,39 +1,36 @@ -module rm cray-hdf5-parallel &> /dev/null -module rm cray-netcdf-hdf5parallel &> /dev/null -module rm cray-parallel-netcdf &> /dev/null -module rm PrgEnv-gnu &> /dev/null -module rm PrgEnv-intel &> /dev/null -module rm PrgEnv-nvidia &> /dev/null -module rm PrgEnv-cray &> /dev/null -module rm PrgEnv-aocc &> /dev/null -module rm gcc-native &> /dev/null -module rm intel &> /dev/null -module rm intel-oneapi &> /dev/null -module rm cudatoolkit &> /dev/null -module rm climate-utils &> /dev/null -module rm cray-libsci &> /dev/null -module rm matlab &> /dev/null -module rm craype-accel-nvidia80 &> /dev/null -module rm craype-accel-host &> /dev/null -module rm perftools-base &> /dev/null -module rm perftools &> /dev/null -module rm darshan &> /dev/null +module rm cray-hdf5-parallel \ + cray-netcdf-hdf5parallel \ + cray-parallel-netcdf \ + PrgEnv-gnu \ + PrgEnv-intel \ + PrgEnv-nvidia \ + PrgEnv-cray \ + PrgEnv-aocc \ + gcc-native \ + intel \ + intel-oneapi \ + cudatoolkit \ + climate-utils \ + cray-libsci \ + matlab \ + craype-accel-nvidia80 \ + craype-accel-host \ + perftools-base \ + perftools \ + darshan \ + cray-mpich &> /dev/null -module load PrgEnv-gnu/8.5.0 -module load gcc/12.2.0 -module load craype-accel-host -module load cray-libsci/23.12.5 -module load craype/2.7.20 -module rm cray-mpich &> /dev/null -module load libfabric/1.20.1 -module load cray-mpich/8.1.25 +module load PrgEnv-gnu/8.5.0 \ + gcc/12.2.0 \ + craype-accel-host \ + cray-libsci/23.12.5 \ + craype/2.7.20 \ + libfabric/1.20.1 \ + cray-mpich/8.1.25 {% if e3sm_hdf5_netcdf %} -module rm cray-hdf5-parallel &> /dev/null -module rm cray-netcdf-hdf5parallel &> /dev/null -module rm cray-parallel-netcdf &> /dev/null -module load cray-hdf5-parallel/1.12.2.3 -module load cray-netcdf-hdf5parallel/4.9.0.3 -module load cray-parallel-netcdf/1.12.3.3 +module load cray-hdf5-parallel/1.12.2.3 \ + cray-netcdf-hdf5parallel/4.9.0.3 \ + cray-parallel-netcdf/1.12.3.3 {% endif %} {% if e3sm_hdf5_netcdf %} diff --git a/mache/spack/pm-cpu_gnu_mpich.sh b/mache/spack/pm-cpu_gnu_mpich.sh index bda35395..9543f87d 100644 --- a/mache/spack/pm-cpu_gnu_mpich.sh +++ b/mache/spack/pm-cpu_gnu_mpich.sh @@ -1,39 +1,36 @@ -module rm cray-hdf5-parallel &> /dev/null -module rm cray-netcdf-hdf5parallel &> /dev/null -module rm cray-parallel-netcdf &> /dev/null -module rm PrgEnv-gnu &> /dev/null -module rm PrgEnv-intel &> /dev/null -module rm PrgEnv-nvidia &> /dev/null -module rm PrgEnv-cray &> /dev/null -module rm PrgEnv-aocc &> /dev/null -module rm gcc-native &> /dev/null -module rm intel &> /dev/null -module rm intel-oneapi &> /dev/null -module rm cudatoolkit &> /dev/null -module rm climate-utils &> /dev/null -module rm cray-libsci &> /dev/null -module rm matlab &> /dev/null -module rm craype-accel-nvidia80 &> /dev/null -module rm craype-accel-host &> /dev/null -module rm perftools-base &> /dev/null -module rm perftools &> /dev/null -module rm darshan &> /dev/null +module rm cray-hdf5-parallel \ + cray-netcdf-hdf5parallel \ + cray-parallel-netcdf \ + PrgEnv-gnu \ + PrgEnv-intel \ + PrgEnv-nvidia \ + PrgEnv-cray \ + PrgEnv-aocc \ + gcc-native \ + intel \ + intel-oneapi \ + cudatoolkit \ + climate-utils \ + cray-libsci \ + matlab \ + craype-accel-nvidia80 \ + craype-accel-host \ + perftools-base \ + perftools \ + darshan \ + cray-mpich &> /dev/null -module load PrgEnv-gnu/8.5.0 -module load gcc/12.2.0 -module load craype-accel-host -module load cray-libsci/23.12.5 -module load craype/2.7.20 -module rm cray-mpich &> /dev/null -module load libfabric/1.20.1 -module load cray-mpich/8.1.25 +module load PrgEnv-gnu/8.5.0 \ + gcc/12.2.0 \ + craype-accel-host \ + cray-libsci/23.12.5 \ + craype/2.7.20 \ + libfabric/1.20.1 \ + cray-mpich/8.1.25 {% if e3sm_hdf5_netcdf %} -module rm cray-hdf5-parallel &> /dev/null -module rm cray-netcdf-hdf5parallel &> /dev/null -module rm cray-parallel-netcdf &> /dev/null -module load cray-hdf5-parallel/1.12.2.3 -module load cray-netcdf-hdf5parallel/4.9.0.3 -module load cray-parallel-netcdf/1.12.3.3 +module load cray-hdf5-parallel/1.12.2.3 \ + cray-netcdf-hdf5parallel/4.9.0.3 \ + cray-parallel-netcdf/1.12.3.3 {% endif %} {% if e3sm_hdf5_netcdf %} diff --git a/mache/spack/pm-cpu_intel_mpich.csh b/mache/spack/pm-cpu_intel_mpich.csh index d6eda3d1..665dd3d4 100644 --- a/mache/spack/pm-cpu_intel_mpich.csh +++ b/mache/spack/pm-cpu_intel_mpich.csh @@ -1,38 +1,35 @@ -module rm cray-hdf5-parallel &> /dev/null -module rm cray-netcdf-hdf5parallel &> /dev/null -module rm cray-parallel-netcdf &> /dev/null -module rm PrgEnv-gnu &> /dev/null -module rm PrgEnv-intel &> /dev/null -module rm PrgEnv-nvidia &> /dev/null -module rm PrgEnv-cray &> /dev/null -module rm PrgEnv-aocc &> /dev/null -module rm gcc-native &> /dev/null -module rm intel &> /dev/null -module rm intel-oneapi &> /dev/null -module rm cudatoolkit &> /dev/null -module rm climate-utils &> /dev/null -module rm cray-libsci &> /dev/null -module rm matlab &> /dev/null -module rm craype-accel-nvidia80 &> /dev/null -module rm craype-accel-host &> /dev/null -module rm perftools-base &> /dev/null -module rm perftools &> /dev/null -module rm darshan &> /dev/null +module rm cray-hdf5-parallel \ + cray-netcdf-hdf5parallel \ + cray-parallel-netcdf \ + PrgEnv-gnu \ + PrgEnv-intel \ + PrgEnv-nvidia \ + PrgEnv-cray \ + PrgEnv-aocc \ + gcc-native \ + intel \ + intel-oneapi \ + cudatoolkit \ + climate-utils \ + cray-libsci \ + matlab \ + craype-accel-nvidia80 \ + craype-accel-host \ + perftools-base \ + perftools \ + darshan \ + cray-mpich &> /dev/null -module load PrgEnv-intel/8.5.0 -module load intel/2023.2.0 -module load craype-accel-host -module load craype/2.7.30 -module load libfabric/1.15.2.0 -module rm cray-mpich &> /dev/null -module load cray-mpich/8.1.28 +module load PrgEnv-intel/8.5.0 \ + intel/2023.2.0 \ + craype-accel-host \ + craype/2.7.30 \ + libfabric/1.15.2.0 \ + cray-mpich/8.1.28 {% if e3sm_hdf5_netcdf %} -module rm cray-hdf5-parallel &> /dev/null -module rm cray-netcdf-hdf5parallel &> /dev/null -module rm cray-parallel-netcdf &> /dev/null -module load cray-hdf5-parallel/1.12.2.9 -module load cray-netcdf-hdf5parallel/4.9.0.9 -module load cray-parallel-netcdf/1.12.3.9 +module load cray-hdf5-parallel/1.12.2.9 \ + cray-netcdf-hdf5parallel/4.9.0.9 \ + cray-parallel-netcdf/1.12.3.9 {% endif %} {% if e3sm_hdf5_netcdf %} diff --git a/mache/spack/pm-cpu_intel_mpich.sh b/mache/spack/pm-cpu_intel_mpich.sh index f997eaca..0c9e550b 100644 --- a/mache/spack/pm-cpu_intel_mpich.sh +++ b/mache/spack/pm-cpu_intel_mpich.sh @@ -1,38 +1,35 @@ -module rm cray-hdf5-parallel &> /dev/null -module rm cray-netcdf-hdf5parallel &> /dev/null -module rm cray-parallel-netcdf &> /dev/null -module rm PrgEnv-gnu &> /dev/null -module rm PrgEnv-intel &> /dev/null -module rm PrgEnv-nvidia &> /dev/null -module rm PrgEnv-cray &> /dev/null -module rm PrgEnv-aocc &> /dev/null -module rm gcc-native &> /dev/null -module rm intel &> /dev/null -module rm intel-oneapi &> /dev/null -module rm cudatoolkit &> /dev/null -module rm climate-utils &> /dev/null -module rm cray-libsci &> /dev/null -module rm matlab &> /dev/null -module rm craype-accel-nvidia80 &> /dev/null -module rm craype-accel-host &> /dev/null -module rm perftools-base &> /dev/null -module rm perftools &> /dev/null -module rm darshan &> /dev/null +module rm cray-hdf5-parallel \ + cray-netcdf-hdf5parallel \ + cray-parallel-netcdf \ + PrgEnv-gnu \ + PrgEnv-intel \ + PrgEnv-nvidia \ + PrgEnv-cray \ + PrgEnv-aocc \ + gcc-native \ + intel \ + intel-oneapi \ + cudatoolkit \ + climate-utils \ + cray-libsci \ + matlab \ + craype-accel-nvidia80 \ + craype-accel-host \ + perftools-base \ + perftools \ + darshan \ + cray-mpich &> /dev/null -module load PrgEnv-intel/8.5.0 -module load intel/2023.2.0 -module load craype-accel-host -module load craype/2.7.30 -module load libfabric/1.15.2.0 -module rm cray-mpich &> /dev/null -module load cray-mpich/8.1.28 +module load PrgEnv-intel/8.5.0 \ + intel/2023.2.0 \ + craype-accel-host \ + craype/2.7.30 \ + libfabric/1.15.2.0 \ + cray-mpich/8.1.28 {% if e3sm_hdf5_netcdf %} -module rm cray-hdf5-parallel &> /dev/null -module rm cray-netcdf-hdf5parallel &> /dev/null -module rm cray-parallel-netcdf &> /dev/null -module load cray-hdf5-parallel/1.12.2.9 -module load cray-netcdf-hdf5parallel/4.9.0.9 -module load cray-parallel-netcdf/1.12.3.9 +module load cray-hdf5-parallel/1.12.2.9 \ + cray-netcdf-hdf5parallel/4.9.0.9 \ + cray-parallel-netcdf/1.12.3.9 {% endif %} {% if e3sm_hdf5_netcdf %} diff --git a/mache/spack/pm-cpu_nvidia_mpich.csh b/mache/spack/pm-cpu_nvidia_mpich.csh index a9ef0190..eb87c86a 100644 --- a/mache/spack/pm-cpu_nvidia_mpich.csh +++ b/mache/spack/pm-cpu_nvidia_mpich.csh @@ -1,40 +1,37 @@ -module rm cray-hdf5-parallel &> /dev/null -module rm cray-netcdf-hdf5parallel &> /dev/null -module rm cray-parallel-netcdf &> /dev/null -module rm PrgEnv-gnu &> /dev/null -module rm PrgEnv-intel &> /dev/null -module rm PrgEnv-nvidia &> /dev/null -module rm PrgEnv-cray &> /dev/null -module rm PrgEnv-aocc &> /dev/null -module rm gcc-native &> /dev/null -module rm intel &> /dev/null -module rm intel-oneapi &> /dev/null -module rm cudatoolkit &> /dev/null -module rm climate-utils &> /dev/null -module rm cray-libsci &> /dev/null -module rm matlab &> /dev/null -module rm craype-accel-nvidia80 &> /dev/null -module rm craype-accel-host &> /dev/null -module rm perftools-base &> /dev/null -module rm perftools &> /dev/null -module rm darshan &> /dev/null +module rm cray-hdf5-parallel \ + cray-netcdf-hdf5parallel \ + cray-parallel-netcdf \ + PrgEnv-gnu \ + PrgEnv-intel \ + PrgEnv-nvidia \ + PrgEnv-cray \ + PrgEnv-aocc \ + gcc-native \ + intel \ + intel-oneapi \ + cudatoolkit \ + climate-utils \ + cray-libsci \ + matlab \ + craype-accel-nvidia80 \ + craype-accel-host \ + perftools-base \ + perftools \ + darshan \ + cray-mpich &> /dev/null -module load PrgEnv-nvidia -module load nvidia/22.7 -module load craype-x86-milan -module load libfabric/1.15.2.0 -module load craype-accel-host -module load craype/2.7.20 -module rm cray-mpich &> /dev/null -module load cray-mpich/8.1.25 -module load cray-libsci/23.12.5 +module load PrgEnv-nvidia \ + nvidia/22.7 \ + craype-x86-milan \ + libfabric/1.15.2.0 \ + craype-accel-host \ + craype/2.7.20 \ + cray-mpich/8.1.25 \ + cray-libsci/23.12.5 {% if e3sm_hdf5_netcdf %} -module rm cray-hdf5-parallel &> /dev/null -module rm cray-netcdf-hdf5parallel &> /dev/null -module rm cray-parallel-netcdf &> /dev/null -module load cray-hdf5-parallel/1.12.2.3 -module load cray-netcdf-hdf5parallel/4.9.0.3 -module load cray-parallel-netcdf/1.12.3.3 +module load cray-hdf5-parallel/1.12.2.3 \ + cray-netcdf-hdf5parallel/4.9.0.3 \ + cray-parallel-netcdf/1.12.3.3 {% endif %} {% if e3sm_hdf5_netcdf %} diff --git a/mache/spack/pm-cpu_nvidia_mpich.sh b/mache/spack/pm-cpu_nvidia_mpich.sh index af25835c..eced7fff 100644 --- a/mache/spack/pm-cpu_nvidia_mpich.sh +++ b/mache/spack/pm-cpu_nvidia_mpich.sh @@ -1,40 +1,37 @@ -module rm cray-hdf5-parallel &> /dev/null -module rm cray-netcdf-hdf5parallel &> /dev/null -module rm cray-parallel-netcdf &> /dev/null -module rm PrgEnv-gnu &> /dev/null -module rm PrgEnv-intel &> /dev/null -module rm PrgEnv-nvidia &> /dev/null -module rm PrgEnv-cray &> /dev/null -module rm PrgEnv-aocc &> /dev/null -module rm gcc-native &> /dev/null -module rm intel &> /dev/null -module rm intel-oneapi &> /dev/null -module rm cudatoolkit &> /dev/null -module rm climate-utils &> /dev/null -module rm cray-libsci &> /dev/null -module rm matlab &> /dev/null -module rm craype-accel-nvidia80 &> /dev/null -module rm craype-accel-host &> /dev/null -module rm perftools-base &> /dev/null -module rm perftools &> /dev/null -module rm darshan &> /dev/null +module rm cray-hdf5-parallel \ + cray-netcdf-hdf5parallel \ + cray-parallel-netcdf \ + PrgEnv-gnu \ + PrgEnv-intel \ + PrgEnv-nvidia \ + PrgEnv-cray \ + PrgEnv-aocc \ + gcc-native \ + intel \ + intel-oneapi \ + cudatoolkit \ + climate-utils \ + cray-libsci \ + matlab \ + craype-accel-nvidia80 \ + craype-accel-host \ + perftools-base \ + perftools \ + darshan \ + cray-mpich &> /dev/null -module load PrgEnv-nvidia -module load nvidia/22.7 -module load craype-x86-milan -module load libfabric/1.15.2.0 -module load craype-accel-host -module load craype/2.7.20 -module rm cray-mpich &> /dev/null -module load cray-mpich/8.1.25 -module load cray-libsci/23.12.5 +module load PrgEnv-nvidia \ + nvidia/22.7 \ + craype-x86-milan \ + libfabric/1.15.2.0 \ + craype-accel-host \ + craype/2.7.20 \ + cray-mpich/8.1.25 \ + cray-libsci/23.12.5 {% if e3sm_hdf5_netcdf %} -module rm cray-hdf5-parallel &> /dev/null -module rm cray-netcdf-hdf5parallel &> /dev/null -module rm cray-parallel-netcdf &> /dev/null -module load cray-hdf5-parallel/1.12.2.3 -module load cray-netcdf-hdf5parallel/4.9.0.3 -module load cray-parallel-netcdf/1.12.3.3 +module load cray-hdf5-parallel/1.12.2.3 \ + cray-netcdf-hdf5parallel/4.9.0.3 \ + cray-parallel-netcdf/1.12.3.3 {% endif %} {% if e3sm_hdf5_netcdf %} diff --git a/mache/spack/pm-gpu_gnugpu_mpich.csh b/mache/spack/pm-gpu_gnugpu_mpich.csh index 62f93822..93b07ddf 100644 --- a/mache/spack/pm-gpu_gnugpu_mpich.csh +++ b/mache/spack/pm-gpu_gnugpu_mpich.csh @@ -1,39 +1,36 @@ -module rm cray-hdf5-parallel &> /dev/null -module rm cray-netcdf-hdf5parallel &> /dev/null -module rm cray-parallel-netcdf &> /dev/null -module rm PrgEnv-gnu &> /dev/null -module rm PrgEnv-intel &> /dev/null -module rm PrgEnv-nvidia &> /dev/null -module rm PrgEnv-cray &> /dev/null -module rm PrgEnv-aocc &> /dev/null -module rm intel &> /dev/null -module rm intel-oneapi &> /dev/null -module rm cudatoolkit &> /dev/null -module rm climate-utils &> /dev/null -module rm matlab &> /dev/null -module rm craype-accel-nvidia80 &> /dev/null -module rm craype-accel-host &> /dev/null -module rm perftools-base &> /dev/null -module rm perftools &> /dev/null -module rm darshan &> /dev/null +module rm cray-hdf5-parallel \ + cray-netcdf-hdf5parallel \ + cray-parallel-netcdf \ + PrgEnv-gnu \ + PrgEnv-intel \ + PrgEnv-nvidia \ + PrgEnv-cray \ + PrgEnv-aocc \ + intel \ + intel-oneapi \ + cudatoolkit \ + climate-utils \ + matlab \ + craype-accel-nvidia80 \ + craype-accel-host \ + perftools-base \ + perftools \ + darshan \ + cray-mpich &> /dev/null -module load PrgEnv-gnu/8.3.3 -module load gcc/11.2.0 -module load craype-x86-milan -module load libfabric/1.15.2.0 -module load cudatoolkit/11.7 -module load craype-accel-nvidia80 -module load craype/2.7.20 -module rm cray-mpich &> /dev/null -module load cray-mpich/8.1.25 -module load cray-libsci/23.12.5 +module load PrgEnv-gnu/8.3.3 \ + gcc/11.2.0 \ + craype-x86-milan \ + libfabric/1.15.2.0 \ + cudatoolkit/11.7 \ + craype-accel-nvidia80 \ + craype/2.7.20 \ + cray-mpich/8.1.25 \ + cray-libsci/23.12.5 {% if e3sm_hdf5_netcdf %} -module rm cray-hdf5-parallel &> /dev/null -module rm cray-netcdf-hdf5parallel &> /dev/null -module rm cray-parallel-netcdf &> /dev/null -module load cray-hdf5-parallel/1.12.2.3 -module load cray-netcdf-hdf5parallel/4.9.0.3 -module load cray-parallel-netcdf/1.12.3.3 +module load cray-hdf5-parallel/1.12.2.3 \ + cray-netcdf-hdf5parallel/4.9.0.3 \ + cray-parallel-netcdf/1.12.3.3 {% endif %} {% if e3sm_hdf5_netcdf %} diff --git a/mache/spack/pm-gpu_gnugpu_mpich.sh b/mache/spack/pm-gpu_gnugpu_mpich.sh index dbfe5481..d56e32f1 100644 --- a/mache/spack/pm-gpu_gnugpu_mpich.sh +++ b/mache/spack/pm-gpu_gnugpu_mpich.sh @@ -1,39 +1,36 @@ -module rm cray-hdf5-parallel &> /dev/null -module rm cray-netcdf-hdf5parallel &> /dev/null -module rm cray-parallel-netcdf &> /dev/null -module rm PrgEnv-gnu &> /dev/null -module rm PrgEnv-intel &> /dev/null -module rm PrgEnv-nvidia &> /dev/null -module rm PrgEnv-cray &> /dev/null -module rm PrgEnv-aocc &> /dev/null -module rm climate-utils &> /dev/null -module rm matlab &> /dev/null -module rm intel &> /dev/null -module rm intel-oneapi &> /dev/null -module rm cudatoolkit &> /dev/null -module rm craype-accel-nvidia80 &> /dev/null -module rm craype-accel-host &> /dev/null -module rm perftools-base &> /dev/null -module rm perftools &> /dev/null -module rm darshan &> /dev/null +module rm cray-hdf5-parallel \ + cray-netcdf-hdf5parallel \ + cray-parallel-netcdf \ + PrgEnv-gnu \ + PrgEnv-intel \ + PrgEnv-nvidia \ + PrgEnv-cray \ + PrgEnv-aocc \ + climate-utils \ + matlab \ + intel \ + intel-oneapi \ + cudatoolkit \ + craype-accel-nvidia80 \ + craype-accel-host \ + perftools-base \ + perftools \ + darshan \ + cray-mpich &> /dev/null -module load PrgEnv-gnu/8.3.3 -module load gcc/11.2.0 -module load craype-x86-milan -module load libfabric/1.15.2.0 -module load cudatoolkit/11.7 -module load craype-accel-nvidia80 -module load craype/2.7.20 -module rm cray-mpich &> /dev/null -module load cray-mpich/8.1.25 -module load cray-libsci/23.12.5 +module load PrgEnv-gnu/8.3.3 \ + gcc/11.2.0 \ + craype-x86-milan \ + libfabric/1.15.2.0 \ + cudatoolkit/11.7 \ + craype-accel-nvidia80 \ + craype/2.7.20 \ + cray-mpich/8.1.25 \ + cray-libsci/23.12.5 {% if e3sm_hdf5_netcdf %} -module rm cray-hdf5-parallel &> /dev/null -module rm cray-netcdf-hdf5parallel &> /dev/null -module rm cray-parallel-netcdf &> /dev/null -module load cray-hdf5-parallel/1.12.2.3 -module load cray-netcdf-hdf5parallel/4.9.0.3 -module load cray-parallel-netcdf/1.12.3.3 +module load cray-hdf5-parallel/1.12.2.3 \ + cray-netcdf-hdf5parallel/4.9.0.3 \ + cray-parallel-netcdf/1.12.3.3 {% endif %} {% if e3sm_hdf5_netcdf %} diff --git a/mache/spack/pm-gpu_nvidiagpu_mpich.csh b/mache/spack/pm-gpu_nvidiagpu_mpich.csh index 24019f90..d18b6ee8 100644 --- a/mache/spack/pm-gpu_nvidiagpu_mpich.csh +++ b/mache/spack/pm-gpu_nvidiagpu_mpich.csh @@ -1,40 +1,37 @@ -module rm cray-hdf5-parallel &> /dev/null -module rm cray-netcdf-hdf5parallel &> /dev/null -module rm cray-parallel-netcdf &> /dev/null -module rm PrgEnv-gnu &> /dev/null -module rm PrgEnv-intel &> /dev/null -module rm PrgEnv-nvidia &> /dev/null -module rm PrgEnv-cray &> /dev/null -module rm PrgEnv-aocc &> /dev/null -module rm intel &> /dev/null -module rm intel-oneapi &> /dev/null -module rm cudatoolkit &> /dev/null -module rm climate-utils &> /dev/null -module rm matlab &> /dev/null -module rm craype-accel-nvidia80 &> /dev/null -module rm craype-accel-host &> /dev/null -module rm perftools-base &> /dev/null -module rm perftools &> /dev/null -module rm darshan &> /dev/null +module rm cray-hdf5-parallel \ + cray-netcdf-hdf5parallel \ + cray-parallel-netcdf \ + PrgEnv-gnu \ + PrgEnv-intel \ + PrgEnv-nvidia \ + PrgEnv-cray \ + PrgEnv-aocc \ + intel \ + intel-oneapi \ + cudatoolkit \ + climate-utils \ + matlab \ + craype-accel-nvidia80 \ + craype-accel-host \ + perftools-base \ + perftools \ + darshan \ + cray-mpich &> /dev/null -module load PrgEnv-nvidia -module load nvidia/22.7 -module load craype-x86-milan -module load libfabric/1.15.2.0 -module load cudatoolkit/11.7 -module load craype-accel-nvidia80 -module load gcc-mixed/11.2.0 -module load craype/2.7.20 -module rm cray-mpich &> /dev/null -module load cray-mpich/8.1.25 -module load cray-libsci/23.12.5 +module load PrgEnv-nvidia \ + nvidia/22.7 \ + craype-x86-milan \ + libfabric/1.15.2.0 \ + cudatoolkit/11.7 \ + craype-accel-nvidia80 \ + gcc-mixed/11.2.0 \ + craype/2.7.20 \ + cray-mpich/8.1.25 \ + cray-libsci/23.12.5 {% if e3sm_hdf5_netcdf %} -module rm cray-hdf5-parallel &> /dev/null -module rm cray-netcdf-hdf5parallel &> /dev/null -module rm cray-parallel-netcdf &> /dev/null -module load cray-hdf5-parallel/1.12.2.3 -module load cray-netcdf-hdf5parallel/4.9.0.3 -module load cray-parallel-netcdf/1.12.3.3 +module load cray-hdf5-parallel/1.12.2.3 \ + cray-netcdf-hdf5parallel/4.9.0.3 \ + cray-parallel-netcdf/1.12.3.3 {% endif %} {% if e3sm_hdf5_netcdf %} diff --git a/mache/spack/pm-gpu_nvidiagpu_mpich.sh b/mache/spack/pm-gpu_nvidiagpu_mpich.sh index 986993ed..6d61f96e 100644 --- a/mache/spack/pm-gpu_nvidiagpu_mpich.sh +++ b/mache/spack/pm-gpu_nvidiagpu_mpich.sh @@ -1,40 +1,37 @@ -module rm cray-hdf5-parallel &> /dev/null -module rm cray-netcdf-hdf5parallel &> /dev/null -module rm cray-parallel-netcdf &> /dev/null -module rm PrgEnv-gnu &> /dev/null -module rm PrgEnv-intel &> /dev/null -module rm PrgEnv-nvidia &> /dev/null -module rm PrgEnv-cray &> /dev/null -module rm PrgEnv-aocc &> /dev/null -module rm intel &> /dev/null -module rm intel-oneapi &> /dev/null -module rm cudatoolkit &> /dev/null -module rm climate-utils &> /dev/null -module rm matlab &> /dev/null -module rm craype-accel-nvidia80 &> /dev/null -module rm craype-accel-host &> /dev/null -module rm perftools-base &> /dev/null -module rm perftools &> /dev/null -module rm darshan &> /dev/null +module rm cray-hdf5-parallel \ + cray-netcdf-hdf5parallel \ + cray-parallel-netcdf \ + PrgEnv-gnu \ + PrgEnv-intel \ + PrgEnv-nvidia \ + PrgEnv-cray \ + PrgEnv-aocc \ + intel \ + intel-oneapi \ + cudatoolkit \ + climate-utils \ + matlab \ + craype-accel-nvidia80 \ + craype-accel-host \ + perftools-base \ + perftools \ + darshan \ + cray-mpich &> /dev/null module load PrgEnv-nvidia -module load nvidia/22.7 -module load craype-x86-milan -module load libfabric/1.15.2.0 -module load cudatoolkit/11.7 -module load craype-accel-nvidia80 -module load gcc-mixed/11.2.0 -module load craype/2.7.20 -module rm cray-mpich &> /dev/null -module load cray-mpich/8.1.25 -module load cray-libsci/23.12.5 + nvidia/22.7 + craype-x86-milan + libfabric/1.15.2.0 + cudatoolkit/11.7 + craype-accel-nvidia80 + gcc-mixed/11.2.0 + craype/2.7.20 + cray-mpich/8.1.25 + cray-libsci/23.12.5 {% if e3sm_hdf5_netcdf %} -module rm cray-hdf5-parallel &> /dev/null -module rm cray-netcdf-hdf5parallel &> /dev/null -module rm cray-parallel-netcdf &> /dev/null module load cray-hdf5-parallel/1.12.2.3 -module load cray-netcdf-hdf5parallel/4.9.0.3 -module load cray-parallel-netcdf/1.12.3.3 + cray-netcdf-hdf5parallel/4.9.0.3 + cray-parallel-netcdf/1.12.3.3 {% endif %} {% if e3sm_hdf5_netcdf %} From eb7680c56d8268c0a2b97b79e9f262a792ca2109 Mon Sep 17 00:00:00 2001 From: Althea Denlinger Date: Tue, 22 Oct 2024 12:57:15 -0700 Subject: [PATCH 04/14] Update `config_machines.xml` --- mache/cime_machine_config/config_machines.xml | 567 ++++++------------ 1 file changed, 175 insertions(+), 392 deletions(-) diff --git a/mache/cime_machine_config/config_machines.xml b/mache/cime_machine_config/config_machines.xml index ff0c583f..593acc1b 100644 --- a/mache/cime_machine_config/config_machines.xml +++ b/mache/cime_machine_config/config_machines.xml @@ -190,6 +190,7 @@ module + cpe cray-hdf5-parallel cray-netcdf-hdf5parallel cray-parallel-netcdf @@ -218,8 +219,8 @@ PrgEnv-gnu/8.5.0 - gcc/12.2.0 - cray-libsci/23.02.1.1 + gcc-native/12.3 + cray-libsci/23.12.5 @@ -229,35 +230,23 @@ PrgEnv-nvidia - nvidia/22.7 - cray-libsci/23.02.1.1 + nvidia/24.5 + cray-libsci/23.12.5 PrgEnv-aocc - aocc/4.0.0 - cray-libsci/23.02.1.1 + aocc/4.1.0 + cray-libsci/23.12.5 - + craype-accel-host craype/2.7.30 cray-mpich/8.1.28 cray-hdf5-parallel/1.12.2.9 cray-netcdf-hdf5parallel/4.9.0.9 cray-parallel-netcdf/1.12.3.9 - - - - craype-accel-host - craype/2.7.20 - cray-mpich/8.1.25 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 - - - cmake/3.24.3 evp-patch @@ -367,6 +356,7 @@ module + cpe cray-hdf5-parallel cray-netcdf-hdf5parallel cray-parallel-netcdf @@ -377,12 +367,14 @@ PrgEnv-nvidia PrgEnv-cray PrgEnv-aocc + gcc-native intel intel-oneapi nvidia aocc cudatoolkit climate-utils + cray-libsci matlab craype-accel-nvidia80 craype-accel-host @@ -392,24 +384,24 @@ - PrgEnv-gnu/8.3.3 - gcc/11.2.0 + PrgEnv-gnu/8.5.0 + gcc-native/12.3 PrgEnv-nvidia - nvidia/22.7 + nvidia/24.5 - cudatoolkit/11.7 + cudatoolkit/12.2 craype-accel-nvidia80 - cudatoolkit/11.7 + cudatoolkit/12.2 craype-accel-nvidia80 - gcc-mixed/11.2.0 + gcc-native-mixed/12.3 @@ -421,12 +413,12 @@ - cray-libsci/23.02.1.1 - craype/2.7.20 - cray-mpich/8.1.25 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 + cray-libsci/23.12.5 + craype/2.7.30 + cray-mpich/8.1.28 + cray-hdf5-parallel/1.12.2.9 + cray-netcdf-hdf5parallel/4.9.0.9 + cray-parallel-netcdf/1.12.3.9 cmake/3.24.3 @@ -434,6 +426,7 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 + 0.20 1 @@ -454,6 +447,9 @@ 1 + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/gnugpu ; else echo "$MOAB_ROOT"; fi} + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/gcc-11.2.0; else echo "$ADIOS2_ROOT"; fi} @@ -588,10 +584,9 @@ /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch software MPI_Bcast - $SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/albany-e3sm-serial-release-gcc; else echo "$Albany_ROOT"; fi} - $SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/trilinos-e3sm-serial-release-gcc; else echo "$Trilinos_ROOT"; fi} $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + 4000MB $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/intel-2023.1.0; else echo "$ADIOS2_ROOT"; fi} @@ -599,6 +594,8 @@ $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/gcc-11.2.0; else echo "$ADIOS2_ROOT"; fi} Generic + $SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/albany/2024.03.26/gcc/11.2.0; else echo "$Albany_ROOT"; fi} + $SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/trilinos/15.1.1/gcc/11.2.0; else echo "$Trilinos_ROOT"; fi} $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/nvidia-22.7; else echo "$ADIOS2_ROOT"; fi} @@ -614,6 +611,13 @@ $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/aocc-4.0.0; else echo "$ADIOS2_ROOT"; fi} + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/intel; else echo "$MOAB_ROOT"; fi} + + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/gnu; else echo "$MOAB_ROOT"; fi} + + -1 @@ -680,12 +684,14 @@ PrgEnv-nvidia PrgEnv-cray PrgEnv-aocc + gcc-native intel intel-oneapi nvidia aocc cudatoolkit climate-utils + cray-libsci matlab craype-accel-nvidia80 craype-accel-host @@ -695,26 +701,24 @@ - PrgEnv-gnu/8.3.3 - gcc/11.2.0 - + PrgEnv-gnu/8.5.0 + gcc-native/12.3 PrgEnv-nvidia - nvidia/23.9 + nvidia/24.5 - cudatoolkit/11.7 - + cudatoolkit/12.2 craype-accel-nvidia80 - cudatoolkit/11.7 + cudatoolkit/12.2 craype-accel-nvidia80 + gcc-native-mixed/12.3 @@ -726,26 +730,20 @@ - cray-libsci/23.02.1.1 - craype/2.7.20 - cray-mpich/8.1.25 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 - + cray-parallel-netcdf/1.12.3.9 cmake/3.24.3 - evp-patch $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 + 0.20 1 @@ -766,6 +764,9 @@ 1 + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/gnugpu ; else echo "$MOAB_ROOT"; fi} + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/gcc-11.2.0; else echo "$ADIOS2_ROOT"; fi} @@ -823,6 +824,7 @@ module + cpe cray-hdf5-parallel cray-netcdf-hdf5parallel cray-parallel-netcdf @@ -850,36 +852,35 @@ - PrgEnv-gnu - gcc-native - cray-libsci + PrgEnv-gnu/8.5.0 + gcc-native/13.2 + cray-libsci/24.03.0 - PrgEnv-intel - intel + PrgEnv-intel/8.5.0 + intel/2024.1.0 PrgEnv-nvidia nvidia/24.5 - cray-libsci + cray-libsci/24.03.0 PrgEnv-aocc - aocc/4.0.1 - cray-libsci + aocc/4.1.0 + cray-libsci/24.03.0 craype-accel-host - cray-libsci - craype/2.7.30 - cray-mpich/8.1.28 - cray-hdf5-parallel/1.12.2.9 - cray-netcdf-hdf5parallel/4.9.0.9 - cray-parallel-netcdf/1.12.3.9 + craype/2.7.31.11 + cray-mpich/8.1.29 + cray-hdf5-parallel/1.12.2.11 + cray-netcdf-hdf5parallel/4.9.0.11 + cray-parallel-netcdf/1.12.3.11 cmake/3.24.3 @@ -887,6 +888,7 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 + 0.20 1 @@ -901,12 +903,44 @@ MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + 4000MB + + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/intel-2023.1.0; else echo "$ADIOS2_ROOT"; fi} + + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/gcc-11.2.0; else echo "$ADIOS2_ROOT"; fi} + Generic + $SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/albany/2024.03.26/gcc/11.2.0; else echo "$Albany_ROOT"; fi} + $SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/trilinos/15.1.1/gcc/11.2.0; else echo "$Trilinos_ROOT"; fi} + + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/nvidia-22.7; else echo "$ADIOS2_ROOT"; fi} + + + $SHELL{if [ -z "$BLAS_ROOT" ]; then echo $NVIDIA_PATH/compilers; else echo "$BLAS_ROOT"; fi} + $SHELL{if [ -z "$LAPACK_ROOT" ]; then echo $NVIDIA_PATH/compilers; else echo "$LAPACK_ROOT"; fi} + NVHPC + + + Intel10_64_dyn + + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/aocc-4.0.0; else echo "$ADIOS2_ROOT"; fi} + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/intel; else echo "$MOAB_ROOT"; fi} + + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/gnu; else echo "$MOAB_ROOT"; fi} + + -1 + Spock. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. .*spock.* @@ -1115,306 +1149,11 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss --> - - Crusher. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. 192 AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs - .*crusher.* - Linux - crayclang,gnu,amdclang,gnugpu,crayclanggpu,amdclanggpu - mpich - cli115 - /lustre/orion/cli115/world-shared/crusher - .* - /lustre/orion/$PROJECT/proj-shared/$ENV{USER}/e3sm_scratch/crusher - /lustre/orion/cli115/world-shared/e3sm/inputdata - /lustre/orion/cli115/world-shared/e3sm/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - /lustre/orion/cli115/world-shared/e3sm/baselines/crusher/$COMPILER - /lustre/orion/cli115/world-shared/e3sm/tools/cprnc/cprnc - 8 - 1 - slurm - e3sm - 56 - 56 - 8 - 8 - 8 - TRUE - - srun - - -l -K -n {{ total_tasks }} -N {{ num_nodes }} - -c $ENV{OMP_NUM_THREADS} - $ENV{NTASKS_PER_GPU} - $ENV{GPU_BIND_ARGS} - - - - /usr/share/lmod/lmod/init/sh - /usr/share/lmod/lmod/init/csh - /usr/share/lmod/lmod/init/perl - /usr/share/lmod/lmod/init/env_modules_python.py - /usr/share/lmod/lmod/libexec/lmod perl - module - module - /usr/share/lmod/lmod/libexec/lmod python - - - PrgEnv-cray PrgEnv-cray/8.3.3 - cce cce/15.0.1 - - - craype craype/2.7.20 - - - craype-accel-amd-gfx90a - rocm/5.4.0 - - - - PrgEnv-cray PrgEnv-amd/8.3.3 - amd amd/5.4.0 - - - craype-accel-amd-gfx90a - - - - PrgEnv-cray PrgEnv-gnu/8.3.3 - gcc gcc/11.2.0 - - - craype-accel-amd-gfx90a - rocm/5.4.0 - - - cray-python/3.9.13.1 - subversion/1.14.1 - git/2.36.1 - cmake/3.21.3 - zlib/1.2.11 - cray-hdf5-parallel/1.12.2.1 - cray-netcdf-hdf5parallel/4.9.0.1 - cray-parallel-netcdf/1.12.3.1 - - - $CIME_OUTPUT_ROOT/$CASE/run - $CIME_OUTPUT_ROOT/$CASE/bld - 0.1 - 0.25 - 0 - - $ENV{NETCDF_DIR} - $ENV{PNETCDF_DIR} - - - - - $ENV{CRAY_LIBSCI_DIR}/amd/4.0/x86_64/lib:$ENV{LD_LIBRARY_PATH} - - - --ntasks-per-gpu=$SHELL{echo "`./xmlquery --value MAX_MPITASKS_PER_NODE`/8"|bc} - --gpu-bind=closest - romio_cb_read=disable - 0 - - - 10 - 3 - - - 128M - spread - threads - - - - - Crusher. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. 192 AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs - .*crusher.* - Linux - crayclang-scream - mpich - CLI115 - /lustre/orion/cli133/proj-shared/$ENV{USER}/e3sm_scratch/crusher - /lustre/orion/cli115/world-shared/e3sm/inputdata - /lustre/orion/cli115/world-shared/e3sm/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - /lustre/orion/cli133/world-shared/e3sm/baselines/$COMPILER - /lustre/orion/cli115/world-shared/e3sm/tools/cprnc/cprnc - 8 - 1 - slurm - e3sm - 56 - 56 - TRUE - - - srun - - -l -K -n {{ total_tasks }} -N {{ num_nodes }} - - - --threads-per-core=1 - -c $ENV{OMP_NUM_THREADS} - -m *:block - - - - - - /usr/share/lmod/lmod/init/sh - /usr/share/lmod/lmod/init/csh - /usr/share/lmod/lmod/init/perl - /usr/share/lmod/lmod/init/env_modules_python.py - /usr/share/lmod/lmod/libexec/lmod perl - module - module - /usr/share/lmod/lmod/libexec/lmod python - - - - PrgEnv-cray PrgEnv-cray/8.3.3 - cce cce/14.0.0 - - - - - PrgEnv-cray PrgEnv-amd/8.3.3 - amd amd/5.1.0 - - - - - PrgEnv-cray PrgEnv-gnu/8.3.3 - - - cray-mpich/8.1.12 - cray-python/3.9.4.2 - subversion/1.14.0 - git/2.31.1 - cmake/3.21.3 - zlib/1.2.11 - cray-libsci/21.08.1.2 - cray-hdf5-parallel/1.12.1.1 - cray-netcdf-hdf5parallel/4.8.1.1 - cray-parallel-netcdf/1.12.1.7 - - - - $CIME_OUTPUT_ROOT/$CASE/run - $CIME_OUTPUT_ROOT/$CASE/bld - 0.1 - - $ENV{NETCDF_DIR} - $ENV{PNETCDF_DIR} - - - - $ENV{CRAY_LIBSCI_DIR}/amd/4.0/x86_64/lib:$ENV{LD_LIBRARY_PATH} - - - - 0 - - - - 128M - spread - threads - - - - - Crusher. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. 192 AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs - .*crusher.* - Linux - crayclang-scream - mpich - CLI115 - /lustre/orion/cli133/proj-shared/$ENV{USER}/e3sm_scratch/crusher - /lustre/orion/cli115/world-shared/e3sm/inputdata - /lustre/orion/cli115/world-shared/e3sm/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - /lustre/orion/cli133/world-shared/e3sm/baselines/$COMPILER - /lustre/orion/cli115/world-shared/e3sm/tools/cprnc/cprnc - 8 - 1 - slurm - e3sm - 64 - 8 - TRUE - - - srun - - -l -K -n {{ total_tasks }} -N {{ num_nodes }} - - - --gpus-per-node=8 --gpu-bind=closest - -c $ENV{OMP_NUM_THREADS} - - - - - - - /usr/share/lmod/lmod/init/sh - /usr/share/lmod/lmod/init/csh - /usr/share/lmod/lmod/init/perl - /usr/share/lmod/lmod/init/env_modules_python.py - /usr/share/lmod/lmod/libexec/lmod perl - module - module - /usr/share/lmod/lmod/libexec/lmod python - - - PrgEnv-cray - - craype-accel-amd-gfx90a - rocm/5.1.0 - - cce/14.0.3 - - - cray-python/3.9.4.2 - subversion/1.14.0 - git/2.31.1 - cmake/3.21.3 - zlib/1.2.11 - cray-hdf5-parallel/1.12.2.1 - cray-netcdf-hdf5parallel/4.9.0.1 - cray-parallel-netcdf/1.12.3.1 - - - - $CIME_OUTPUT_ROOT/$CASE/run - $CIME_OUTPUT_ROOT/$CASE/bld - 0.1 - 0 - - $ENV{NETCDF_DIR} - $ENV{PNETCDF_DIR} - 0 - - 1 - romio_cb_read=disable - - - - 128M - spread - threads - - - Frontier. AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs. .*frontier.* CNL - crayclang-scream + craygnuamdgpu,crayclang-scream mpich cli115 /lustre/orion/proj-shared/cli115 @@ -1451,17 +1190,34 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss module module /usr/share/lmod/lmod/libexec/lmod python + + + + PrgEnv-gnu + cpe/24.07 + libfabric/1.15.2.0 + craype-accel-amd-gfx90a + rocm/6.2.0 + libunwind + cray-python + subversion + git + cmake + cray-hdf5-parallel + cray-netcdf-hdf5parallel + cray-parallel-netcdf + darshan-runtime + - PrgEnv-cray + cpe/22.12 craype-accel-amd-gfx90a rocm/5.4.0 libunwind/1.6.2 - - cce/15.0.1 - craype craype/2.7.20 - cray-mpich cray-mpich/8.1.26 + libfabric/1.15.2.0 + craype/2.7.20 + cray-mpich/8.1.26 cray-python/3.9.13.1 subversion/1.14.1 git/2.36.1 @@ -1495,9 +1251,13 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss threads + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.10/install/craygnuamdgppu/cpe-24.07/libfabric-1.15.2.0; else echo "$ADIOS2_ROOT"; fi} + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.9.1/cray-mpich-8.1.26/crayclang-scream-14.0.0; else echo "$ADIOS2_ROOT"; fi} + @@ -3044,6 +2804,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6 /lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/netcdf-fortran/4.6.1b/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/netcdf-c/4.9.2b/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/openmpi/4.1.6/gcc-12.3.0/bin:/lcrc/group/e3sm/soft/perl/improv/bin:$ENV{PATH} $SHELL{lp=/lcrc/group/e3sm/soft/improv/netlib-lapack/3.12.0/gcc-12.3.0:/lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6/lib:/lcrc/group/e3sm/soft/improv/netcdf-fortran/4.6.1b/gcc-12.3.0/openmpi-4.1.6/lib:/lcrc/group/e3sm/soft/improv/netcdf-c/4.9.2b/gcc-12.3.0/openmpi-4.1.6/lib:/opt/pbs/lib:/lcrc/group/e3sm/soft/improv/openmpi/4.1.6/gcc-12.3.0/lib; if [ -z "$LD_LIBRARY_PATH" ]; then echo $lp; else echo "$lp:$LD_LIBRARY_PATH"; fi} + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /lcrc/soft/climate/moab/improv/gnu; else echo "$MOAB_ROOT"; fi} ^lockedfile @@ -3065,9 +2826,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /usr/workspace/e3sm/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER - /usr/workspace/e3sm/tools/cprnc + /usr/workspace/e3sm/apps/cprnc 8 - lc_slurm + slurm boutte3 -at- llnl.gov 56 56 @@ -3076,8 +2837,16 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss srun + + --mpi=pmi2 + --export=ALL + -n {{ total_tasks }} -N {{ num_nodes }} + -c 1 + --cpu_bind=cores + -m plane={{ tasks_per_node }} + - + /usr/share/lmod/lmod/init/env_modules_python.py /usr/share/lmod/lmod/init/perl /usr/share/lmod/lmod/init/sh @@ -3089,24 +2858,27 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss python/3.9.12 git + subversion + cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic - mvapich2/2.3.7 - cmake/3.19.2 - /usr/workspace/e3sm/install/quartz/modulefiles - hdf5/1.12.2 - netcdf-c/4.9.0 - netcdf-fortran/4.6.0 - parallel-netcdf/1.12.3 - screamML-venv/0.0.1 - subversion + /usr/workspace/e3sm/spack/modules/ruby/linux-rhel8-x86_64/Core + mvapich2/2.3.7-ll7cmqm + hdf5/1.10.7-ewjpbjd + netcdf-c/4.4.1.1-vaxofek + netcdf-fortran/4.4.4-3pzbx2u + parallel-netcdf/1.11.0-tzgdala $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /usr/workspace/e3sm/install/quartz/netcdf-fortran/ - /usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0 + 128M + FALSE + /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/hdf5-1.10.7-ewjpbjdhjgjzrzjcvwyjyuulaesbsjhg + /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-c-4.4.1.1-vaxofekwvnvngh7wptmzkwdb7tkzvesn + /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-fortran-4.4.4-3pzbx2unddhladhubaahhhysjmprzqi2 + /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/parallel-netcdf-1.11.0-tzgdalakmem7tod6cruhqyeackeix5q5 @@ -3121,9 +2893,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /usr/workspace/e3sm/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER - /usr/workspace/e3sm/tools/cprnc + /usr/workspace/e3sm/apps/cprnc 8 - lc_slurm + slurm boutte3 -at- llnl.gov 224 112 @@ -3132,8 +2904,16 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss srun + + --mpi=pmi2 + --export=ALL + -n {{ total_tasks }} -N {{ num_nodes }} + -c 1 + --cpu_bind=cores + -m plane={{ tasks_per_node }} + - + /usr/share/lmod/lmod/init/env_modules_python.py /usr/share/lmod/lmod/init/perl /usr/share/lmod/lmod/init/sh @@ -3145,24 +2925,27 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss python/3.9.12 git + subversion mkl/2022.1.0 intel-classic/2021.6.0-magic - mvapich2/2.3.7 cmake/3.19.2 - /usr/workspace/e3sm/install/quartz/modulefiles - hdf5/1.12.2 - netcdf-c/4.9.0 - netcdf-fortran/4.6.0 - parallel-netcdf/1.12.3 - screamML-venv/0.0.1 - subversion + /usr/workspace/e3sm/spack/modules/dane/linux-rhel8-x86_64/Core + mvapich2/2.3.7-27jao34 + hdf5/1.10.7-766kapa + netcdf-c/4.4.1.1-2uznnlw + netcdf-fortran/4.4.4-itpstyo + parallel-netcdf/1.11.0-26sxm4m $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /usr/workspace/e3sm/install/quartz/netcdf-fortran/ - /usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0 + 128M + FALSE + /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/hdf5-1.10.7-766kapalbrdntu2pcgdgbhg2ch26gsuv + /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-c-4.4.1.1-2uznnlwgiezxute6iyqzqjrpolokeaib + /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-fortran-4.4.4-itpstyordbern7vlulmlnt47eeeokzfp + /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/parallel-netcdf-1.11.0-26sxm4mormsglmhi24poix7sugbigkck From 526266b387e515f36a2fc80efeea856156947076 Mon Sep 17 00:00:00 2001 From: Althea Denlinger Date: Tue, 22 Oct 2024 17:01:16 -0700 Subject: [PATCH 05/14] Update `libfabric` for `pm-cpu` nvidia files --- mache/spack/pm-cpu_nvidia_mpich.csh | 2 +- mache/spack/pm-cpu_nvidia_mpich.sh | 2 +- mache/spack/pm-cpu_nvidia_mpich.yaml | 9 +-------- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/mache/spack/pm-cpu_nvidia_mpich.csh b/mache/spack/pm-cpu_nvidia_mpich.csh index eb87c86a..e87ae2f2 100644 --- a/mache/spack/pm-cpu_nvidia_mpich.csh +++ b/mache/spack/pm-cpu_nvidia_mpich.csh @@ -23,7 +23,7 @@ module rm cray-hdf5-parallel \ module load PrgEnv-nvidia \ nvidia/22.7 \ craype-x86-milan \ - libfabric/1.15.2.0 \ + libfabric/1.20.1 \ craype-accel-host \ craype/2.7.20 \ cray-mpich/8.1.25 \ diff --git a/mache/spack/pm-cpu_nvidia_mpich.sh b/mache/spack/pm-cpu_nvidia_mpich.sh index eced7fff..1cd675ba 100644 --- a/mache/spack/pm-cpu_nvidia_mpich.sh +++ b/mache/spack/pm-cpu_nvidia_mpich.sh @@ -23,7 +23,7 @@ module rm cray-hdf5-parallel \ module load PrgEnv-nvidia \ nvidia/22.7 \ craype-x86-milan \ - libfabric/1.15.2.0 \ + libfabric/1.20.1 \ craype-accel-host \ craype/2.7.20 \ cray-mpich/8.1.25 \ diff --git a/mache/spack/pm-cpu_nvidia_mpich.yaml b/mache/spack/pm-cpu_nvidia_mpich.yaml index 779d8a48..884f87ea 100644 --- a/mache/spack/pm-cpu_nvidia_mpich.yaml +++ b/mache/spack/pm-cpu_nvidia_mpich.yaml @@ -99,16 +99,9 @@ spack: - spec: cray-mpich@8.1.25 prefix: /opt/cray/pe/mpich/8.1.25/ofi/nvidia/20.7 modules: - - libfabric/1.15.2.0 + - libfabric/1.20.1 - cray-mpich/8.1.25 buildable: false - libfabric: - externals: - - spec: libfabric@1.15.2.0 - prefix: /opt/cray/libfabric/1.15.2.0 - modules: - - libfabric/1.15.2.0 - buildable: false cray-libsci: externals: - spec: cray-libsci@23.12.5 From f22e3540ddb0fcc682f750c4b6320b6cb310ad02 Mon Sep 17 00:00:00 2001 From: Althea Denlinger Date: Tue, 22 Oct 2024 17:08:51 -0700 Subject: [PATCH 06/14] Update `libfabric` on `pm-gpu` gnu & nvidia --- mache/spack/pm-gpu_gnugpu_mpich.csh | 2 +- mache/spack/pm-gpu_gnugpu_mpich.sh | 2 +- mache/spack/pm-gpu_gnugpu_mpich.yaml | 15 ++++----------- mache/spack/pm-gpu_nvidiagpu_mpich.csh | 2 +- mache/spack/pm-gpu_nvidiagpu_mpich.sh | 2 +- mache/spack/pm-gpu_nvidiagpu_mpich.yaml | 9 +-------- 6 files changed, 9 insertions(+), 23 deletions(-) diff --git a/mache/spack/pm-gpu_gnugpu_mpich.csh b/mache/spack/pm-gpu_gnugpu_mpich.csh index 93b07ddf..43cf40eb 100644 --- a/mache/spack/pm-gpu_gnugpu_mpich.csh +++ b/mache/spack/pm-gpu_gnugpu_mpich.csh @@ -21,7 +21,7 @@ module rm cray-hdf5-parallel \ module load PrgEnv-gnu/8.3.3 \ gcc/11.2.0 \ craype-x86-milan \ - libfabric/1.15.2.0 \ + libfabric/1.20.1 \ cudatoolkit/11.7 \ craype-accel-nvidia80 \ craype/2.7.20 \ diff --git a/mache/spack/pm-gpu_gnugpu_mpich.sh b/mache/spack/pm-gpu_gnugpu_mpich.sh index d56e32f1..cbb692ec 100644 --- a/mache/spack/pm-gpu_gnugpu_mpich.sh +++ b/mache/spack/pm-gpu_gnugpu_mpich.sh @@ -21,7 +21,7 @@ module rm cray-hdf5-parallel \ module load PrgEnv-gnu/8.3.3 \ gcc/11.2.0 \ craype-x86-milan \ - libfabric/1.15.2.0 \ + libfabric/1.20.1 \ cudatoolkit/11.7 \ craype-accel-nvidia80 \ craype/2.7.20 \ diff --git a/mache/spack/pm-gpu_gnugpu_mpich.yaml b/mache/spack/pm-gpu_gnugpu_mpich.yaml index 172125ca..cb978f44 100644 --- a/mache/spack/pm-gpu_gnugpu_mpich.yaml +++ b/mache/spack/pm-gpu_gnugpu_mpich.yaml @@ -104,7 +104,7 @@ spack: - cudatoolkit/11.7 - craype-accel-nvidia80 - craype/2.7.19 - - libfabric/1.15.2.0 + - libfabric/1.20.1 buildable: false cuda: externals: @@ -116,23 +116,16 @@ spack: - cudatoolkit/11.7 - craype-accel-nvidia80 - craype/2.7.19 - - libfabric/1.15.2.0 + - libfabric/1.20.1 buildable: false cray-mpich: externals: - spec: cray-mpich@8.1.25 prefix: /opt/cray/pe/mpich/8.1.25/ofi/gnu/9.1 modules: - - libfabric/1.15.2.0 + - libfabric/1.20.1 - cray-mpich/8.1.25 buildable: false - libfabric: - externals: - - spec: libfabric@1.15.2.0 - prefix: /opt/cray/libfabric/1.15.2.0 - modules: - - libfabric/1.15.2.0 - buildable: false cray-libsci: externals: - spec: cray-libsci@23.12.5 @@ -181,7 +174,7 @@ spack: - cudatoolkit/11.7 - craype-accel-nvidia80 - craype/2.7.20 - - libfabric/1.15.2.0 + - libfabric/1.20.1 environment: prepend_path: PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.33__gd067c3f.shasta/lib64/pkgconfig" diff --git a/mache/spack/pm-gpu_nvidiagpu_mpich.csh b/mache/spack/pm-gpu_nvidiagpu_mpich.csh index d18b6ee8..2b76014a 100644 --- a/mache/spack/pm-gpu_nvidiagpu_mpich.csh +++ b/mache/spack/pm-gpu_nvidiagpu_mpich.csh @@ -21,7 +21,7 @@ module rm cray-hdf5-parallel \ module load PrgEnv-nvidia \ nvidia/22.7 \ craype-x86-milan \ - libfabric/1.15.2.0 \ + libfabric/1.20.1 \ cudatoolkit/11.7 \ craype-accel-nvidia80 \ gcc-mixed/11.2.0 \ diff --git a/mache/spack/pm-gpu_nvidiagpu_mpich.sh b/mache/spack/pm-gpu_nvidiagpu_mpich.sh index 6d61f96e..9d6dd998 100644 --- a/mache/spack/pm-gpu_nvidiagpu_mpich.sh +++ b/mache/spack/pm-gpu_nvidiagpu_mpich.sh @@ -21,7 +21,7 @@ module rm cray-hdf5-parallel \ module load PrgEnv-nvidia nvidia/22.7 craype-x86-milan - libfabric/1.15.2.0 + libfabric/1.20.1 cudatoolkit/11.7 craype-accel-nvidia80 gcc-mixed/11.2.0 diff --git a/mache/spack/pm-gpu_nvidiagpu_mpich.yaml b/mache/spack/pm-gpu_nvidiagpu_mpich.yaml index be8bb68a..7d0f9026 100644 --- a/mache/spack/pm-gpu_nvidiagpu_mpich.yaml +++ b/mache/spack/pm-gpu_nvidiagpu_mpich.yaml @@ -112,16 +112,9 @@ spack: - spec: cray-mpich@8.1.25 prefix: /opt/cray/pe/mpich/8.1.25/ofi/nvidia/20.7 modules: - - libfabric/1.15.2.0 + - libfabric/1.20.1 - cray-mpich/8.1.25 buildable: false - libfabric: - externals: - - spec: libfabric@1.15.2.0 - prefix: /opt/cray/libfabric/1.15.2.0 - modules: - - libfabric/1.15.2.0 - buildable: false cray-libsci: externals: - spec: cray-libsci@23.12.5 From a694923b9b809c7b48d4cad81571655e3133e2ff Mon Sep 17 00:00:00 2001 From: Althea Denlinger Date: Tue, 22 Oct 2024 17:25:52 -0700 Subject: [PATCH 07/14] Update libfabric in `pm-cpu` intel --- mache/spack/pm-cpu_intel_mpich.csh | 2 +- mache/spack/pm-cpu_intel_mpich.sh | 2 +- mache/spack/pm-cpu_intel_mpich.yaml | 13 +++---------- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/mache/spack/pm-cpu_intel_mpich.csh b/mache/spack/pm-cpu_intel_mpich.csh index 665dd3d4..636b38b4 100644 --- a/mache/spack/pm-cpu_intel_mpich.csh +++ b/mache/spack/pm-cpu_intel_mpich.csh @@ -24,7 +24,7 @@ module load PrgEnv-intel/8.5.0 \ intel/2023.2.0 \ craype-accel-host \ craype/2.7.30 \ - libfabric/1.15.2.0 \ + libfabric/1.20.1 \ cray-mpich/8.1.28 {% if e3sm_hdf5_netcdf %} module load cray-hdf5-parallel/1.12.2.9 \ diff --git a/mache/spack/pm-cpu_intel_mpich.sh b/mache/spack/pm-cpu_intel_mpich.sh index 0c9e550b..88efb102 100644 --- a/mache/spack/pm-cpu_intel_mpich.sh +++ b/mache/spack/pm-cpu_intel_mpich.sh @@ -24,7 +24,7 @@ module load PrgEnv-intel/8.5.0 \ intel/2023.2.0 \ craype-accel-host \ craype/2.7.30 \ - libfabric/1.15.2.0 \ + libfabric/1.20.1 \ cray-mpich/8.1.28 {% if e3sm_hdf5_netcdf %} module load cray-hdf5-parallel/1.12.2.9 \ diff --git a/mache/spack/pm-cpu_intel_mpich.yaml b/mache/spack/pm-cpu_intel_mpich.yaml index 97846bdd..aec3302f 100644 --- a/mache/spack/pm-cpu_intel_mpich.yaml +++ b/mache/spack/pm-cpu_intel_mpich.yaml @@ -101,7 +101,7 @@ spack: - intel/2023.2.0 - craype-accel-host - craype/2.7.30 - - libfabric/1.15.2.0 + - libfabric/1.20.1 buildable: false cray-mpich: externals: @@ -109,14 +109,7 @@ spack: prefix: /opt/cray/pe/mpich/8.1.28/ofi/intel/19.0 modules: - cray-mpich/8.1.28 - - libfabric/1.15.2.0 - buildable: false - libfabric: - externals: - - spec: libfabric@1.15.2.0 - prefix: /opt/cray/libfabric/1.15.2.0 - modules: - - libfabric/1.15.2.0 + - libfabric/1.20.1 buildable: false {% if e3sm_hdf5_netcdf %} hdf5: @@ -158,7 +151,7 @@ spack: - intel/2023.2.0 - craype-accel-host - craype/2.7.30 - - libfabric/1.15.2.0 + - libfabric/1.20.1 environment: prepend_path: PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.33__gd067c3f.shasta/lib64/pkgconfig" From 5adb32c881c3af702e272ff57658e9f767b65e13 Mon Sep 17 00:00:00 2001 From: Althea Denlinger Date: Tue, 22 Oct 2024 17:32:32 -0700 Subject: [PATCH 08/14] Fix some leftover cray-libsci versions --- mache/spack/pm-cpu_gnu_mpich.yaml | 2 +- mache/spack/pm-gpu_gnugpu_mpich.yaml | 2 +- mache/spack/pm-gpu_nvidiagpu_mpich.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mache/spack/pm-cpu_gnu_mpich.yaml b/mache/spack/pm-cpu_gnu_mpich.yaml index 5f045345..b174d3d2 100644 --- a/mache/spack/pm-cpu_gnu_mpich.yaml +++ b/mache/spack/pm-cpu_gnu_mpich.yaml @@ -118,7 +118,7 @@ spack: - spec: cray-libsci@23.12.5 prefix: /opt/cray/pe/libsci/23.02.1.1/GNU/9.1/x86_64 modules: - - cray-libsci/23.02.1.1 + - cray-libsci/23.12.5 buildable: false {% if e3sm_hdf5_netcdf %} hdf5: diff --git a/mache/spack/pm-gpu_gnugpu_mpich.yaml b/mache/spack/pm-gpu_gnugpu_mpich.yaml index cb978f44..a76a1e49 100644 --- a/mache/spack/pm-gpu_gnugpu_mpich.yaml +++ b/mache/spack/pm-gpu_gnugpu_mpich.yaml @@ -131,7 +131,7 @@ spack: - spec: cray-libsci@23.12.5 prefix: /opt/cray/pe/libsci/23.02.1.1/GNU/9.1/x86_64 modules: - - cray-libsci/23.02.1.1 + - cray-libsci/23.12.5 buildable: false {% if e3sm_hdf5_netcdf %} hdf5: diff --git a/mache/spack/pm-gpu_nvidiagpu_mpich.yaml b/mache/spack/pm-gpu_nvidiagpu_mpich.yaml index 7d0f9026..9980f7c9 100644 --- a/mache/spack/pm-gpu_nvidiagpu_mpich.yaml +++ b/mache/spack/pm-gpu_nvidiagpu_mpich.yaml @@ -120,7 +120,7 @@ spack: - spec: cray-libsci@23.12.5 prefix: /opt/cray/pe/libsci/23.02.1.1/NVIDIA/20.7/x86_64 modules: - - cray-libsci/23.02.1.1 + - cray-libsci/23.12.5 buildable: false {% if e3sm_hdf5_netcdf %} hdf5: From 70e5caf46827e8122a5b0acb386a44c2205d1050 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 22 Oct 2024 19:40:27 -0600 Subject: [PATCH 09/14] Update pm spack based on config_machines.xml --- mache/spack/pm-cpu_gnu_mpich.csh | 17 ++++---- mache/spack/pm-cpu_gnu_mpich.sh | 17 ++++---- mache/spack/pm-cpu_gnu_mpich.yaml | 38 ++++++++--------- mache/spack/pm-cpu_intel_mpich.csh | 3 +- mache/spack/pm-cpu_intel_mpich.sh | 3 +- mache/spack/pm-cpu_intel_mpich.yaml | 3 +- mache/spack/pm-cpu_nvidia_mpich.csh | 20 ++++----- mache/spack/pm-cpu_nvidia_mpich.sh | 20 ++++----- mache/spack/pm-cpu_nvidia_mpich.yaml | 43 ++++++++++--------- mache/spack/pm-gpu_gnugpu_mpich.csh | 26 +++++------ mache/spack/pm-gpu_gnugpu_mpich.sh | 30 +++++++------ mache/spack/pm-gpu_gnugpu_mpich.yaml | 51 +++++++++++----------- mache/spack/pm-gpu_nvidiagpu_mpich.csh | 26 +++++------ mache/spack/pm-gpu_nvidiagpu_mpich.sh | 30 +++++++------ mache/spack/pm-gpu_nvidiagpu_mpich.yaml | 57 ++++++++++++------------- 15 files changed, 196 insertions(+), 188 deletions(-) diff --git a/mache/spack/pm-cpu_gnu_mpich.csh b/mache/spack/pm-cpu_gnu_mpich.csh index 67e2fe7c..b7aecff8 100644 --- a/mache/spack/pm-cpu_gnu_mpich.csh +++ b/mache/spack/pm-cpu_gnu_mpich.csh @@ -1,4 +1,5 @@ -module rm cray-hdf5-parallel \ +module rm cpe \ + cray-hdf5-parallel \ cray-netcdf-hdf5parallel \ cray-parallel-netcdf \ PrgEnv-gnu \ @@ -21,16 +22,16 @@ module rm cray-hdf5-parallel \ cray-mpich &> /dev/null module load PrgEnv-gnu/8.5.0 \ - gcc/12.2.0 \ - craype-accel-host \ + gcc-native/12.3 \ cray-libsci/23.12.5 \ - craype/2.7.20 \ + craype-accel-host \ + craype/2.7.30 \ libfabric/1.20.1 \ - cray-mpich/8.1.25 + cray-mpich/8.1.28 {% if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.3 \ - cray-netcdf-hdf5parallel/4.9.0.3 \ - cray-parallel-netcdf/1.12.3.3 +module load cray-hdf5-parallel/1.12.2.9 \ + cray-netcdf-hdf5parallel/4.9.0.9 \ + cray-parallel-netcdf/1.12.3.9 {% endif %} {% if e3sm_hdf5_netcdf %} diff --git a/mache/spack/pm-cpu_gnu_mpich.sh b/mache/spack/pm-cpu_gnu_mpich.sh index 9543f87d..c163d90a 100644 --- a/mache/spack/pm-cpu_gnu_mpich.sh +++ b/mache/spack/pm-cpu_gnu_mpich.sh @@ -1,4 +1,5 @@ -module rm cray-hdf5-parallel \ +module rm cpe \ + cray-hdf5-parallel \ cray-netcdf-hdf5parallel \ cray-parallel-netcdf \ PrgEnv-gnu \ @@ -21,16 +22,16 @@ module rm cray-hdf5-parallel \ cray-mpich &> /dev/null module load PrgEnv-gnu/8.5.0 \ - gcc/12.2.0 \ - craype-accel-host \ + gcc-native/12.3 \ cray-libsci/23.12.5 \ - craype/2.7.20 \ + craype-accel-host \ + craype/2.7.30 \ libfabric/1.20.1 \ - cray-mpich/8.1.25 + cray-mpich/8.1.28 {% if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.3 \ - cray-netcdf-hdf5parallel/4.9.0.3 \ - cray-parallel-netcdf/1.12.3.3 +module load cray-hdf5-parallel/1.12.2.9 \ + cray-netcdf-hdf5parallel/4.9.0.9 \ + cray-parallel-netcdf/1.12.3.9 {% endif %} {% if e3sm_hdf5_netcdf %} diff --git a/mache/spack/pm-cpu_gnu_mpich.yaml b/mache/spack/pm-cpu_gnu_mpich.yaml index b174d3d2..7fd22e6f 100644 --- a/mache/spack/pm-cpu_gnu_mpich.yaml +++ b/mache/spack/pm-cpu_gnu_mpich.yaml @@ -14,9 +14,9 @@ spack: unify: when_possible packages: all: - compiler: [gcc@12.2.0] + compiler: [gcc@12.3] providers: - mpi: [cray-mpich@8.1.25] + mpi: [cray-mpich@8.1.28] lapack: [cray-libsci@23.12.5] bzip2: externals: @@ -97,56 +97,55 @@ spack: buildable: false gcc: externals: - - spec: gcc@12.2.0 + - spec: gcc@12.3 modules: - PrgEnv-gnu/8.5.0 - - gcc/12.2.0 + - gcc-native/12.3 + - cray-libsci/23.12.5 - craype-accel-host - - craype/2.7.20 + - craype/2.7.30 - libfabric/1.20.1 buildable: false cray-mpich: externals: - - spec: cray-mpich@8.1.25 - prefix: /opt/cray/pe/mpich/8.1.25/ofi/gnu/9.1 + - spec: cray-mpich@8.1.28 modules: - libfabric/1.20.1 - - cray-mpich/8.1.25 + - cray-mpich/8.1.28 buildable: false cray-libsci: externals: - spec: cray-libsci@23.12.5 - prefix: /opt/cray/pe/libsci/23.02.1.1/GNU/9.1/x86_64 modules: - cray-libsci/23.12.5 buildable: false {% if e3sm_hdf5_netcdf %} hdf5: externals: - - spec: hdf5@1.12.2.3~cxx+fortran+hl~java+mpi+shared - prefix: /opt/cray/pe/hdf5-parallel/1.12.2.3/GNU/9.1 + - spec: hdf5@1.12.2.9~cxx+fortran+hl~java+mpi+shared + prefix: /opt/cray/pe/hdf5-parallel/1.12.2.9/gnu/12.3 buildable: false parallel-netcdf: externals: - - spec: parallel-netcdf@1.12.3.3+cxx+fortran+pic+shared - prefix: /opt/cray/pe/parallel-netcdf/1.12.3.3/GNU/9.1/ + - spec: parallel-netcdf@1.12.3.9+cxx+fortran+pic+shared + prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/gnu/12.3 buildable: false netcdf-c: externals: - - spec: netcdf-c@4.9.0.3+mpi~parallel-netcdf - prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/GNU/9.1 + - spec: netcdf-c@4.9.0.9+mpi~parallel-netcdf + prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/gnu/12.3 buildable: false netcdf-fortran: externals: - spec: netcdf-fortran@4.5.3 - prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/GNU/9.1 + prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/gnu/12.3 buildable: false {% endif %} config: install_missing_compilers: false compilers: - compiler: - spec: gcc@12.2.0 + spec: gcc@12.3 paths: cc: cc cxx: CC @@ -157,9 +156,10 @@ spack: target: x86_64 modules: - PrgEnv-gnu/8.5.0 - - gcc/12.2.0 + - gcc-native/12.3 + - cray-libsci/23.12.5 - craype-accel-host - - craype/2.7.20 + - craype/2.7.30 - libfabric/1.20.1 environment: prepend_path: diff --git a/mache/spack/pm-cpu_intel_mpich.csh b/mache/spack/pm-cpu_intel_mpich.csh index 636b38b4..c281b8a7 100644 --- a/mache/spack/pm-cpu_intel_mpich.csh +++ b/mache/spack/pm-cpu_intel_mpich.csh @@ -1,4 +1,5 @@ -module rm cray-hdf5-parallel \ +module rm cpe \ + cray-hdf5-parallel \ cray-netcdf-hdf5parallel \ cray-parallel-netcdf \ PrgEnv-gnu \ diff --git a/mache/spack/pm-cpu_intel_mpich.sh b/mache/spack/pm-cpu_intel_mpich.sh index 88efb102..77c7ab52 100644 --- a/mache/spack/pm-cpu_intel_mpich.sh +++ b/mache/spack/pm-cpu_intel_mpich.sh @@ -1,4 +1,5 @@ -module rm cray-hdf5-parallel \ +module rm cpe \ + cray-hdf5-parallel \ cray-netcdf-hdf5parallel \ cray-parallel-netcdf \ PrgEnv-gnu \ diff --git a/mache/spack/pm-cpu_intel_mpich.yaml b/mache/spack/pm-cpu_intel_mpich.yaml index aec3302f..0a2d34fc 100644 --- a/mache/spack/pm-cpu_intel_mpich.yaml +++ b/mache/spack/pm-cpu_intel_mpich.yaml @@ -106,7 +106,6 @@ spack: cray-mpich: externals: - spec: cray-mpich@8.1.28 - prefix: /opt/cray/pe/mpich/8.1.28/ofi/intel/19.0 modules: - cray-mpich/8.1.28 - libfabric/1.20.1 @@ -120,7 +119,7 @@ spack: parallel-netcdf: externals: - spec: parallel-netcdf@1.12.3.9+cxx+fortran+pic+shared - prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/intel/19.0/ + prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/intel/19.0 buildable: false netcdf-c: externals: diff --git a/mache/spack/pm-cpu_nvidia_mpich.csh b/mache/spack/pm-cpu_nvidia_mpich.csh index e87ae2f2..90ded235 100644 --- a/mache/spack/pm-cpu_nvidia_mpich.csh +++ b/mache/spack/pm-cpu_nvidia_mpich.csh @@ -1,4 +1,5 @@ -module rm cray-hdf5-parallel \ +module rm cpe \ + cray-hdf5-parallel \ cray-netcdf-hdf5parallel \ cray-parallel-netcdf \ PrgEnv-gnu \ @@ -21,17 +22,16 @@ module rm cray-hdf5-parallel \ cray-mpich &> /dev/null module load PrgEnv-nvidia \ - nvidia/22.7 \ - craype-x86-milan \ - libfabric/1.20.1 \ + nvidia/24.5 \ + cray-libsci/23.12.5 \ craype-accel-host \ - craype/2.7.20 \ - cray-mpich/8.1.25 \ - cray-libsci/23.12.5 + craype/2.7.30 \ + libfabric/1.20.1 \ + cray-mpich/8.1.28 {% if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.3 \ - cray-netcdf-hdf5parallel/4.9.0.3 \ - cray-parallel-netcdf/1.12.3.3 +module load cray-hdf5-parallel/1.12.2.9 \ + cray-netcdf-hdf5parallel/4.9.0.9 \ + cray-parallel-netcdf/1.12.3.9 {% endif %} {% if e3sm_hdf5_netcdf %} diff --git a/mache/spack/pm-cpu_nvidia_mpich.sh b/mache/spack/pm-cpu_nvidia_mpich.sh index 1cd675ba..a0c54d0d 100644 --- a/mache/spack/pm-cpu_nvidia_mpich.sh +++ b/mache/spack/pm-cpu_nvidia_mpich.sh @@ -1,4 +1,5 @@ -module rm cray-hdf5-parallel \ +module rm cpe \ + cray-hdf5-parallel \ cray-netcdf-hdf5parallel \ cray-parallel-netcdf \ PrgEnv-gnu \ @@ -21,17 +22,16 @@ module rm cray-hdf5-parallel \ cray-mpich &> /dev/null module load PrgEnv-nvidia \ - nvidia/22.7 \ - craype-x86-milan \ - libfabric/1.20.1 \ + nvidia/24.5 \ + cray-libsci/23.12.5 \ craype-accel-host \ - craype/2.7.20 \ - cray-mpich/8.1.25 \ - cray-libsci/23.12.5 + craype/2.7.30 \ + libfabric/1.20.1 \ + cray-mpich/8.1.28 {% if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.3 \ - cray-netcdf-hdf5parallel/4.9.0.3 \ - cray-parallel-netcdf/1.12.3.3 +module load cray-hdf5-parallel/1.12.2.9 \ + cray-netcdf-hdf5parallel/4.9.0.9 \ + cray-parallel-netcdf/1.12.3.9 {% endif %} {% if e3sm_hdf5_netcdf %} diff --git a/mache/spack/pm-cpu_nvidia_mpich.yaml b/mache/spack/pm-cpu_nvidia_mpich.yaml index 884f87ea..92789bee 100644 --- a/mache/spack/pm-cpu_nvidia_mpich.yaml +++ b/mache/spack/pm-cpu_nvidia_mpich.yaml @@ -13,9 +13,9 @@ spack: unify: when_possible packages: all: - compiler: [nvhpc@22.7] + compiler: [nvhpc@24.5] providers: - mpi: [cray-mpich@8.1.25] + mpi: [cray-mpich@8.1.28] lapack: [cray-libsci@23.12.5] bzip2: externals: @@ -96,58 +96,59 @@ spack: buildable: false cray-mpich: externals: - - spec: cray-mpich@8.1.25 - prefix: /opt/cray/pe/mpich/8.1.25/ofi/nvidia/20.7 + - spec: cray-mpich@8.1.28 modules: - libfabric/1.20.1 - - cray-mpich/8.1.25 + - cray-mpich/8.1.28 buildable: false cray-libsci: externals: - spec: cray-libsci@23.12.5 - prefix: /opt/cray/pe/libsci/23.02.1.1/NVIDIA/20.7/x86_64 + module: + - cray-libsci/23.12.5 buildable: false {% if e3sm_hdf5_netcdf %} hdf5: externals: - - spec: hdf5@1.12.2.3~cxx+fortran+hl~java+mpi+shared - prefix: /opt/cray/pe/hdf5-parallel/1.12.2.3/nvidia/20.7 + - spec: hdf5@1.12.2.9~cxx+fortran+hl~java+mpi+shared + prefix: /opt/cray/pe/hdf5-parallel/1.12.2.9/nvidia/23.3 buildable: false parallel-netcdf: externals: - - spec: parallel-netcdf@1.12.3.3+cxx+fortran+pic+shared - prefix: /opt/cray/pe/parallel-netcdf/1.12.3.3/nvidia/20.7 + - spec: parallel-netcdf@1.12.3.9+cxx+fortran+pic+shared + prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/nvidia/23.3 buildable: false netcdf-c: externals: - - spec: netcdf-c@4.9.0.3+mpi~parallel-netcdf - prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/nvidia/20.7 + - spec: netcdf-c@4.9.0.9+mpi~parallel-netcdf + prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/nvidia/23.3 buildable: false netcdf-fortran: externals: - spec: netcdf-fortran@4.5.3 - prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/nvidia/20.7 + prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/nvidia/23.3 buildable: false {% endif %} config: install_missing_compilers: false compilers: - compiler: - spec: nvhpc@22.7 + spec: nvhpc@24.5 paths: - cc: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvc - cxx: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvc++ - f77: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvfortran - fc: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvfortran + cc: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvc + cxx: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvc++ + f77: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvfortran + fc: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvfortran flags: {} operating_system: sles15 target: any modules: - PrgEnv-nvidia - - nvidia/22.7 - - craype-x86-milan - - libfabric + - nvidia/24.5 + - cray-libsci/23.12.5 - craype-accel-host + - craype/2.7.30 + - libfabric/1.20 environment: prepend_path: PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.33__gd067c3f.shasta/lib64/pkgconfig" diff --git a/mache/spack/pm-gpu_gnugpu_mpich.csh b/mache/spack/pm-gpu_gnugpu_mpich.csh index 43cf40eb..1041043a 100644 --- a/mache/spack/pm-gpu_gnugpu_mpich.csh +++ b/mache/spack/pm-gpu_gnugpu_mpich.csh @@ -1,4 +1,5 @@ -module rm cray-hdf5-parallel \ +module rm cpe \ + cray-hdf5-parallel \ cray-netcdf-hdf5parallel \ cray-parallel-netcdf \ PrgEnv-gnu \ @@ -6,10 +7,12 @@ module rm cray-hdf5-parallel \ PrgEnv-nvidia \ PrgEnv-cray \ PrgEnv-aocc \ + gcc-native \ intel \ intel-oneapi \ cudatoolkit \ climate-utils \ + cray-libsci \ matlab \ craype-accel-nvidia80 \ craype-accel-host \ @@ -18,19 +21,18 @@ module rm cray-hdf5-parallel \ darshan \ cray-mpich &> /dev/null -module load PrgEnv-gnu/8.3.3 \ - gcc/11.2.0 \ - craype-x86-milan \ - libfabric/1.20.1 \ - cudatoolkit/11.7 \ +module load PrgEnv-gnu/8.5.0 \ + gcc-native/12.3 \ + cudatoolkit/12.2 \ craype-accel-nvidia80 \ - craype/2.7.20 \ - cray-mpich/8.1.25 \ - cray-libsci/23.12.5 + cray-libsci/23.12.5 \ + libfabric/1.20.1 \ + craype/2.7.30 \ + cray-mpich/8.1.28 {% if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.3 \ - cray-netcdf-hdf5parallel/4.9.0.3 \ - cray-parallel-netcdf/1.12.3.3 +module load cray-hdf5-parallel/1.12.2.9 \ + cray-netcdf-hdf5parallel/4.9.0.9 \ + cray-parallel-netcdf/1.12.3.9 {% endif %} {% if e3sm_hdf5_netcdf %} diff --git a/mache/spack/pm-gpu_gnugpu_mpich.sh b/mache/spack/pm-gpu_gnugpu_mpich.sh index cbb692ec..228ac3ca 100644 --- a/mache/spack/pm-gpu_gnugpu_mpich.sh +++ b/mache/spack/pm-gpu_gnugpu_mpich.sh @@ -1,4 +1,5 @@ -module rm cray-hdf5-parallel \ +module rm cpe \ + cray-hdf5-parallel \ cray-netcdf-hdf5parallel \ cray-parallel-netcdf \ PrgEnv-gnu \ @@ -6,11 +7,13 @@ module rm cray-hdf5-parallel \ PrgEnv-nvidia \ PrgEnv-cray \ PrgEnv-aocc \ - climate-utils \ - matlab \ + gcc-native \ intel \ intel-oneapi \ cudatoolkit \ + climate-utils \ + cray-libsci \ + matlab \ craype-accel-nvidia80 \ craype-accel-host \ perftools-base \ @@ -18,19 +21,18 @@ module rm cray-hdf5-parallel \ darshan \ cray-mpich &> /dev/null -module load PrgEnv-gnu/8.3.3 \ - gcc/11.2.0 \ - craype-x86-milan \ - libfabric/1.20.1 \ - cudatoolkit/11.7 \ +module load PrgEnv-gnu/8.5.0 \ + gcc-native/12.3 \ + cudatoolkit/12.2 \ craype-accel-nvidia80 \ - craype/2.7.20 \ - cray-mpich/8.1.25 \ - cray-libsci/23.12.5 + cray-libsci/23.12.5 \ + libfabric/1.20.1 \ + craype/2.7.30 \ + cray-mpich/8.1.28 {% if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.3 \ - cray-netcdf-hdf5parallel/4.9.0.3 \ - cray-parallel-netcdf/1.12.3.3 +module load cray-hdf5-parallel/1.12.2.9 \ + cray-netcdf-hdf5parallel/4.9.0.9 \ + cray-parallel-netcdf/1.12.3.9 {% endif %} {% if e3sm_hdf5_netcdf %} diff --git a/mache/spack/pm-gpu_gnugpu_mpich.yaml b/mache/spack/pm-gpu_gnugpu_mpich.yaml index a76a1e49..37be1b20 100644 --- a/mache/spack/pm-gpu_gnugpu_mpich.yaml +++ b/mache/spack/pm-gpu_gnugpu_mpich.yaml @@ -14,9 +14,9 @@ spack: unify: when_possible packages: all: - compiler: [gcc@11.2.0] + compiler: [gcc@12.3] providers: - mpi: [cray-mpich@8.1.24] + mpi: [cray-mpich@8.1.28] lapack: [cray-libsci@23.12.5] bzip2: externals: @@ -97,69 +97,68 @@ spack: buildable: false gcc: externals: - - spec: gcc@11.2.0 + - spec: gcc@12.3 modules: - - PrgEnv-gnu/8.3.3 - - gcc/11.2.0 - - cudatoolkit/11.7 + - PrgEnv-gnu/8.5.0 + - gcc-native/12.3 + - cudatoolkit/12.2 - craype-accel-nvidia80 - - craype/2.7.19 + - cray-libsci/23.12.5 - libfabric/1.20.1 + - craype/2.7.30 buildable: false cuda: externals: - - spec: cuda@11.7 - prefix: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7 + - spec: cuda@12.2 modules: - - PrgEnv-gnu/8.3.3 - - gcc/11.2.0 - - cudatoolkit/11.7 + - PrgEnv-gnu/8.5.0 + - gcc-native/12.3 + - cudatoolkit/12.2 - craype-accel-nvidia80 - - craype/2.7.19 + - cray-libsci/23.12.5 - libfabric/1.20.1 + - craype/2.7.30 buildable: false cray-mpich: externals: - - spec: cray-mpich@8.1.25 - prefix: /opt/cray/pe/mpich/8.1.25/ofi/gnu/9.1 + - spec: cray-mpich@8.1.28 modules: - libfabric/1.20.1 - - cray-mpich/8.1.25 + - cray-mpich/8.1.28 buildable: false cray-libsci: externals: - spec: cray-libsci@23.12.5 - prefix: /opt/cray/pe/libsci/23.02.1.1/GNU/9.1/x86_64 modules: - cray-libsci/23.12.5 buildable: false {% if e3sm_hdf5_netcdf %} hdf5: externals: - - spec: hdf5@1.12.2.3~cxx+fortran+hl~java+mpi+shared - prefix: /opt/cray/pe/hdf5-parallel/1.12.2.3/GNU/9.1 + - spec: hdf5@1.12.2.9~cxx+fortran+hl~java+mpi+shared + prefix: /opt/cray/pe/hdf5-parallel/1.12.2.9/gnu/12.3 buildable: false parallel-netcdf: externals: - - spec: parallel-netcdf@1.12.3.3+cxx+fortran+pic+shared - prefix: /opt/cray/pe/parallel-netcdf/1.12.3.3/GNU/9.1/ + - spec: parallel-netcdf@1.12.3.9+cxx+fortran+pic+shared + prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/gnu/12.3 buildable: false netcdf-c: externals: - - spec: netcdf-c@4.9.0.3+mpi~parallel-netcdf - prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/GNU/9.1 + - spec: netcdf-c@4.9.0.9+mpi~parallel-netcdf + prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/gnu/12.3 buildable: false netcdf-fortran: externals: - spec: netcdf-fortran@4.5.3 - prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/GNU/9.1 + prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/gnu/12.3 buildable: false {% endif %} config: install_missing_compilers: false compilers: - compiler: - spec: gcc@11.2.0 + spec: gcc@12.3 paths: cc: cc cxx: CC @@ -170,7 +169,7 @@ spack: target: any modules: - PrgEnv-gnu/8.3.3 - - gcc/11.2.0 + - gcc/12.3 - cudatoolkit/11.7 - craype-accel-nvidia80 - craype/2.7.20 diff --git a/mache/spack/pm-gpu_nvidiagpu_mpich.csh b/mache/spack/pm-gpu_nvidiagpu_mpich.csh index 2b76014a..4cf585b1 100644 --- a/mache/spack/pm-gpu_nvidiagpu_mpich.csh +++ b/mache/spack/pm-gpu_nvidiagpu_mpich.csh @@ -1,4 +1,5 @@ -module rm cray-hdf5-parallel \ +module rm cpe \ + cray-hdf5-parallel \ cray-netcdf-hdf5parallel \ cray-parallel-netcdf \ PrgEnv-gnu \ @@ -6,10 +7,12 @@ module rm cray-hdf5-parallel \ PrgEnv-nvidia \ PrgEnv-cray \ PrgEnv-aocc \ + gcc-native \ intel \ intel-oneapi \ cudatoolkit \ climate-utils \ + cray-libsci \ matlab \ craype-accel-nvidia80 \ craype-accel-host \ @@ -19,19 +22,18 @@ module rm cray-hdf5-parallel \ cray-mpich &> /dev/null module load PrgEnv-nvidia \ - nvidia/22.7 \ - craype-x86-milan \ - libfabric/1.20.1 \ - cudatoolkit/11.7 \ + nvidia/24.5 \ + cudatoolkit/12.2 \ craype-accel-nvidia80 \ - gcc-mixed/11.2.0 \ - craype/2.7.20 \ - cray-mpich/8.1.25 \ - cray-libsci/23.12.5 + gcc-native-mixed/12.3 \ + cray-libsci/23.12.5 \ + libfabric/1.20.1 \ + craype/2.7.30 \ + cray-mpich/8.1.28 {% if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.3 \ - cray-netcdf-hdf5parallel/4.9.0.3 \ - cray-parallel-netcdf/1.12.3.3 +module load cray-hdf5-parallel/1.12.2.9 \ + cray-netcdf-hdf5parallel/4.9.0.9 \ + cray-parallel-netcdf/1.12.3.9 {% endif %} {% if e3sm_hdf5_netcdf %} diff --git a/mache/spack/pm-gpu_nvidiagpu_mpich.sh b/mache/spack/pm-gpu_nvidiagpu_mpich.sh index 9d6dd998..e4054cb6 100644 --- a/mache/spack/pm-gpu_nvidiagpu_mpich.sh +++ b/mache/spack/pm-gpu_nvidiagpu_mpich.sh @@ -1,4 +1,5 @@ -module rm cray-hdf5-parallel \ +module rm cpe \ + cray-hdf5-parallel \ cray-netcdf-hdf5parallel \ cray-parallel-netcdf \ PrgEnv-gnu \ @@ -6,10 +7,12 @@ module rm cray-hdf5-parallel \ PrgEnv-nvidia \ PrgEnv-cray \ PrgEnv-aocc \ + gcc-native \ intel \ intel-oneapi \ cudatoolkit \ climate-utils \ + cray-libsci \ matlab \ craype-accel-nvidia80 \ craype-accel-host \ @@ -18,20 +21,19 @@ module rm cray-hdf5-parallel \ darshan \ cray-mpich &> /dev/null -module load PrgEnv-nvidia - nvidia/22.7 - craype-x86-milan - libfabric/1.20.1 - cudatoolkit/11.7 - craype-accel-nvidia80 - gcc-mixed/11.2.0 - craype/2.7.20 - cray-mpich/8.1.25 - cray-libsci/23.12.5 +module load PrgEnv-nvidia \ + nvidia/24.5 \ + cudatoolkit/12.2 \ + craype-accel-nvidia80 \ + gcc-native-mixed/12.3 \ + cray-libsci/23.12.5 \ + libfabric/1.20.1 \ + craype/2.7.30 \ + cray-mpich/8.1.28 {% if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 +module load cray-hdf5-parallel/1.12.2.9 \ + cray-netcdf-hdf5parallel/4.9.0.9 \ + cray-parallel-netcdf/1.12.3.9 {% endif %} {% if e3sm_hdf5_netcdf %} diff --git a/mache/spack/pm-gpu_nvidiagpu_mpich.yaml b/mache/spack/pm-gpu_nvidiagpu_mpich.yaml index 9980f7c9..1b843b6f 100644 --- a/mache/spack/pm-gpu_nvidiagpu_mpich.yaml +++ b/mache/spack/pm-gpu_nvidiagpu_mpich.yaml @@ -13,9 +13,9 @@ spack: unify: when_possible packages: all: - compiler: [nvhpc@22.7] + compiler: [nvhpc@24.5] providers: - mpi: [cray-mpich@8.1.25] + mpi: [cray-mpich@8.1.28] lapack: [cray-libsci@23.12.5] bzip2: externals: @@ -96,75 +96,72 @@ spack: buildable: false cuda: externals: - - spec: cuda@11.7 - prefix: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7 + - spec: cuda@12.2 modules: - PrgEnv-nvidia - - nvidia/22.7 - - cudatoolkit/11.7 + - nvidia/24.5 + - cudatoolkit/12.2 - craype-accel-nvidia80 - - gcc-mixed/11.2.0 - - craype-x86-milan - - libfabric + - cray-libsci/23.12.5 + - libfabric/1.20.1 + - craype/2.7.30 buildable: false cray-mpich: externals: - - spec: cray-mpich@8.1.25 - prefix: /opt/cray/pe/mpich/8.1.25/ofi/nvidia/20.7 + - spec: cray-mpich@8.1.28 modules: - libfabric/1.20.1 - - cray-mpich/8.1.25 + - cray-mpich/8.1.28 buildable: false cray-libsci: externals: - spec: cray-libsci@23.12.5 - prefix: /opt/cray/pe/libsci/23.02.1.1/NVIDIA/20.7/x86_64 modules: - cray-libsci/23.12.5 buildable: false {% if e3sm_hdf5_netcdf %} hdf5: externals: - - spec: hdf5@1.12.2.3~cxx+fortran+hl~java+mpi+shared - prefix: /opt/cray/pe/hdf5-parallel/1.12.2.3/nvidia/20.7 + - spec: hdf5@1.12.2.9~cxx+fortran+hl~java+mpi+shared + prefix: /opt/cray/pe/hdf5-parallel/1.12.2.9/nvidia/23.3 buildable: false parallel-netcdf: externals: - - spec: parallel-netcdf@1.12.3.3+cxx+fortran+pic+shared - prefix: /opt/cray/pe/parallel-netcdf/1.12.3.3/nvidia/20.7 + - spec: parallel-netcdf@1.12.3.9+cxx+fortran+pic+shared + prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/nvidia/23.3 buildable: false netcdf-c: externals: - - spec: netcdf-c@4.9.0.3+mpi~parallel-netcdf - prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/nvidia/20.7 + - spec: netcdf-c@4.9.0.9+mpi~parallel-netcdf + prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/nvidia/23.3 buildable: false netcdf-fortran: externals: - spec: netcdf-fortran@4.5.3 - prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/nvidia/20.7 + prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/nvidia/23.3 buildable: false {% endif %} config: install_missing_compilers: false compilers: - compiler: - spec: nvhpc@22.7 + spec: nvhpc@24.5 paths: - cc: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvc - cxx: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvc++ - f77: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvfortran - fc: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvfortran + cc: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvc + cxx: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvc++ + f77: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvfortran + fc: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvfortran flags: {} operating_system: sles15 target: any modules: - PrgEnv-nvidia - - nvidia/22.7 - - cudatoolkit/11.7 + - nvidia/24.5 + - cudatoolkit/12.2 - craype-accel-nvidia80 - - gcc-mixed/11.2.0 - - craype-x86-milan - - libfabric + - cray-libsci/23.12.5 + - libfabric/1.20.1 + - craype/2.7.30 environment: prepend_path: PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.33__gd067c3f.shasta/lib64/pkgconfig" From be430e65e746bd238dc6ef138752809388f0acb4 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 22 Oct 2024 20:07:43 -0600 Subject: [PATCH 10/14] Fix some missing environment variables --- mache/spack/pm-cpu_gnu_mpich.csh | 1 + mache/spack/pm-cpu_gnu_mpich.sh | 1 + mache/spack/pm-cpu_intel_mpich.csh | 1 + mache/spack/pm-cpu_intel_mpich.sh | 2 +- mache/spack/pm-cpu_nvidia_mpich.csh | 2 ++ mache/spack/pm-cpu_nvidia_mpich.sh | 2 ++ mache/spack/pm-gpu_gnugpu_mpich.csh | 2 ++ mache/spack/pm-gpu_gnugpu_mpich.sh | 2 ++ mache/spack/pm-gpu_nvidiagpu_mpich.csh | 2 ++ mache/spack/pm-gpu_nvidiagpu_mpich.sh | 2 ++ 10 files changed, 16 insertions(+), 1 deletion(-) diff --git a/mache/spack/pm-cpu_gnu_mpich.csh b/mache/spack/pm-cpu_gnu_mpich.csh index b7aecff8..7b9b8498 100644 --- a/mache/spack/pm-cpu_gnu_mpich.csh +++ b/mache/spack/pm-cpu_gnu_mpich.csh @@ -51,6 +51,7 @@ setenv HDF5_USE_FILE_LOCKING FALSE ## Not needed # setenv PERL5LIB /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch setenv FI_CXI_RX_MATCH_MODE software +setenv FI_MR_CACHE_MONITOR kdreg2 setenv MPICH_COLL_SYNC MPI_Bcast setenv GATOR_INITIAL_MB 4000MB setenv BLA_VENDOR Generic diff --git a/mache/spack/pm-cpu_gnu_mpich.sh b/mache/spack/pm-cpu_gnu_mpich.sh index c163d90a..d4580d8d 100644 --- a/mache/spack/pm-cpu_gnu_mpich.sh +++ b/mache/spack/pm-cpu_gnu_mpich.sh @@ -51,6 +51,7 @@ export HDF5_USE_FILE_LOCKING=FALSE ## Not needed # export PERL5LIB=/global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch export FI_CXI_RX_MATCH_MODE=software +export FI_MR_CACHE_MONITOR=kdreg2 if [ -z "${NERSC_HOST:-}" ]; then # happens when building spack environment diff --git a/mache/spack/pm-cpu_intel_mpich.csh b/mache/spack/pm-cpu_intel_mpich.csh index c281b8a7..52de4034 100644 --- a/mache/spack/pm-cpu_intel_mpich.csh +++ b/mache/spack/pm-cpu_intel_mpich.csh @@ -50,6 +50,7 @@ setenv HDF5_USE_FILE_LOCKING FALSE ## Not needed # setenv PERL5LIB /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch setenv FI_CXI_RX_MATCH_MODE software +setenv FI_MR_CACHE_MONITOR kdreg2 setenv MPICH_COLL_SYNC MPI_Bcast setenv GATOR_INITIAL_MB 4000MB setenv BLA_VENDOR Intel10_64_dyn diff --git a/mache/spack/pm-cpu_intel_mpich.sh b/mache/spack/pm-cpu_intel_mpich.sh index 77c7ab52..5d4e3b85 100644 --- a/mache/spack/pm-cpu_intel_mpich.sh +++ b/mache/spack/pm-cpu_intel_mpich.sh @@ -50,12 +50,12 @@ export HDF5_USE_FILE_LOCKING=FALSE ## Not needed # export PERL5LIB=/global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch export FI_CXI_RX_MATCH_MODE=software +export FI_MR_CACHE_MONITOR=kdreg2 if [ -z "${NERSC_HOST:-}" ]; then # happens when building spack environment export NERSC_HOST="perlmutter" fi -export FI_CXI_RX_MATCH_MODE=software export MPICH_COLL_SYNC=MPI_Bcast export GATOR_INITIAL_MB=4000MB export BLA_VENDOR=Intel10_64_dyn diff --git a/mache/spack/pm-cpu_nvidia_mpich.csh b/mache/spack/pm-cpu_nvidia_mpich.csh index 90ded235..d410cac6 100644 --- a/mache/spack/pm-cpu_nvidia_mpich.csh +++ b/mache/spack/pm-cpu_nvidia_mpich.csh @@ -51,6 +51,8 @@ setenv HDF5_USE_FILE_LOCKING FALSE ## Not needed # setenv PERL5LIB /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch setenv MPICH_GPU_SUPPORT_ENABLED 1 +setenv FI_CXI_RX_MATCH_MODE software +setenv FI_MR_CACHE_MONITOR kdreg2 setenv MPICH_COLL_SYNC MPI_Bcast setenv GATOR_INITIAL_MB 4000MB setenv BLA_VENDOR NVHPC diff --git a/mache/spack/pm-cpu_nvidia_mpich.sh b/mache/spack/pm-cpu_nvidia_mpich.sh index a0c54d0d..7a1624e7 100644 --- a/mache/spack/pm-cpu_nvidia_mpich.sh +++ b/mache/spack/pm-cpu_nvidia_mpich.sh @@ -56,6 +56,8 @@ if [ -z "${NERSC_HOST:-}" ]; then # happens when building spack environment export NERSC_HOST="perlmutter" fi +export FI_CXI_RX_MATCH_MODE=software +export FI_MR_CACHE_MONITOR=kdreg2 export MPICH_COLL_SYNC=MPI_Bcast export GATOR_INITIAL_MB=4000MB export BLA_VENDOR=NVHPC diff --git a/mache/spack/pm-gpu_gnugpu_mpich.csh b/mache/spack/pm-gpu_gnugpu_mpich.csh index 1041043a..10e54ac1 100644 --- a/mache/spack/pm-gpu_gnugpu_mpich.csh +++ b/mache/spack/pm-gpu_gnugpu_mpich.csh @@ -51,4 +51,6 @@ setenv MPICH_MPIIO_DVS_MAXNODES 1 setenv HDF5_USE_FILE_LOCKING FALSE ## Not needed # setenv PERL5LIB /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch +setenv MPICH_COLL_SYNC MPI_Bcast +setenv FI_MR_CACHE_MONITOR kdreg2 setenv MPICH_GPU_SUPPORT_ENABLED 1 diff --git a/mache/spack/pm-gpu_gnugpu_mpich.sh b/mache/spack/pm-gpu_gnugpu_mpich.sh index 228ac3ca..1a96b61a 100644 --- a/mache/spack/pm-gpu_gnugpu_mpich.sh +++ b/mache/spack/pm-gpu_gnugpu_mpich.sh @@ -51,6 +51,8 @@ export MPICH_VERSION_DISPLAY=1 export HDF5_USE_FILE_LOCKING=FALSE ## Not needed # export PERL5LIB=/global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch +export MPICH_COLL_SYNC=MPI_Bcast +export FI_MR_CACHE_MONITOR=kdreg2 export MPICH_GPU_SUPPORT_ENABLED=1 if [ -z "${NERSC_HOST:-}" ]; then diff --git a/mache/spack/pm-gpu_nvidiagpu_mpich.csh b/mache/spack/pm-gpu_nvidiagpu_mpich.csh index 4cf585b1..c5a71fc7 100644 --- a/mache/spack/pm-gpu_nvidiagpu_mpich.csh +++ b/mache/spack/pm-gpu_nvidiagpu_mpich.csh @@ -50,6 +50,8 @@ setenv MPICH_MPIIO_DVS_MAXNODES 1 # setenv OMP_PROC_BIND spread # setenv OMP_PLACES threads setenv HDF5_USE_FILE_LOCKING FALSE +setenv MPICH_COLL_SYNC MPI_Bcast +setenv FI_MR_CACHE_MONITOR kdreg2 ## Not needed # setenv PERL5LIB /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch setenv MPICH_GPU_SUPPORT_ENABLED 1 diff --git a/mache/spack/pm-gpu_nvidiagpu_mpich.sh b/mache/spack/pm-gpu_nvidiagpu_mpich.sh index e4054cb6..17e0f8dd 100644 --- a/mache/spack/pm-gpu_nvidiagpu_mpich.sh +++ b/mache/spack/pm-gpu_nvidiagpu_mpich.sh @@ -52,6 +52,8 @@ export MPICH_MPIIO_DVS_MAXNODES=1 export HDF5_USE_FILE_LOCKING=FALSE ## Not needed # export PERL5LIB=/global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch +export MPICH_COLL_SYNC=MPI_Bcast +export FI_MR_CACHE_MONITOR=kdreg2 export MPICH_GPU_SUPPORT_ENABLED=1 if [ -z "${NERSC_HOST:-}" ]; then From f20f030f18899b136aa0e2ad8a19ce6e2706c7c8 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 23 Oct 2024 12:26:52 -0700 Subject: [PATCH 11/14] Try prefix for cray-mpich in gnugpu --- mache/spack/pm-gpu_gnugpu_mpich.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mache/spack/pm-gpu_gnugpu_mpich.yaml b/mache/spack/pm-gpu_gnugpu_mpich.yaml index 37be1b20..25115d47 100644 --- a/mache/spack/pm-gpu_gnugpu_mpich.yaml +++ b/mache/spack/pm-gpu_gnugpu_mpich.yaml @@ -122,9 +122,7 @@ spack: cray-mpich: externals: - spec: cray-mpich@8.1.28 - modules: - - libfabric/1.20.1 - - cray-mpich/8.1.28 + prefix: /opt/cray/pe/mpich/8.1.28/ofi/gnu/12.3 buildable: false cray-libsci: externals: From 3ceae78673b23b9405d92360e0eaea55c6070dd0 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 23 Oct 2024 15:03:02 -0700 Subject: [PATCH 12/14] Load cudatoolkit last Also, fix gcc module definitions --- mache/spack/pm-gpu_gnugpu_mpich.yaml | 13 +++++++------ mache/spack/pm-gpu_nvidiagpu_mpich.yaml | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/mache/spack/pm-gpu_gnugpu_mpich.yaml b/mache/spack/pm-gpu_gnugpu_mpich.yaml index 25115d47..ebd03a26 100644 --- a/mache/spack/pm-gpu_gnugpu_mpich.yaml +++ b/mache/spack/pm-gpu_gnugpu_mpich.yaml @@ -101,11 +101,11 @@ spack: modules: - PrgEnv-gnu/8.5.0 - gcc-native/12.3 - - cudatoolkit/12.2 - craype-accel-nvidia80 - cray-libsci/23.12.5 - libfabric/1.20.1 - craype/2.7.30 + - cudatoolkit/12.2 buildable: false cuda: externals: @@ -113,11 +113,11 @@ spack: modules: - PrgEnv-gnu/8.5.0 - gcc-native/12.3 - - cudatoolkit/12.2 - craype-accel-nvidia80 - cray-libsci/23.12.5 - libfabric/1.20.1 - craype/2.7.30 + - cudatoolkit/12.2 buildable: false cray-mpich: externals: @@ -166,12 +166,13 @@ spack: operating_system: sles15 target: any modules: - - PrgEnv-gnu/8.3.3 - - gcc/12.3 - - cudatoolkit/11.7 + - PrgEnv-gnu/8.5.0 + - gcc-native/12.3 - craype-accel-nvidia80 - - craype/2.7.20 + - cray-libsci/23.12.5 - libfabric/1.20.1 + - craype/2.7.30 + - cudatoolkit/12.2 environment: prepend_path: PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.33__gd067c3f.shasta/lib64/pkgconfig" diff --git a/mache/spack/pm-gpu_nvidiagpu_mpich.yaml b/mache/spack/pm-gpu_nvidiagpu_mpich.yaml index 1b843b6f..b380f878 100644 --- a/mache/spack/pm-gpu_nvidiagpu_mpich.yaml +++ b/mache/spack/pm-gpu_nvidiagpu_mpich.yaml @@ -100,11 +100,11 @@ spack: modules: - PrgEnv-nvidia - nvidia/24.5 - - cudatoolkit/12.2 - craype-accel-nvidia80 - cray-libsci/23.12.5 - libfabric/1.20.1 - craype/2.7.30 + - cudatoolkit/12.2 buildable: false cray-mpich: externals: @@ -157,11 +157,11 @@ spack: modules: - PrgEnv-nvidia - nvidia/24.5 - - cudatoolkit/12.2 - craype-accel-nvidia80 - cray-libsci/23.12.5 - libfabric/1.20.1 - craype/2.7.30 + - cudatoolkit/12.2 environment: prepend_path: PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.33__gd067c3f.shasta/lib64/pkgconfig" From 921a1ea23186653c35cde636a7a2872f498b1f1a Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 24 Oct 2024 08:52:45 -0700 Subject: [PATCH 13/14] Update to cudatoolkit 12.4 This is needed for Trilinos --- mache/spack/pm-gpu_gnugpu_mpich.csh | 2 +- mache/spack/pm-gpu_gnugpu_mpich.sh | 2 +- mache/spack/pm-gpu_gnugpu_mpich.yaml | 8 ++++---- mache/spack/pm-gpu_nvidiagpu_mpich.csh | 2 +- mache/spack/pm-gpu_nvidiagpu_mpich.sh | 2 +- mache/spack/pm-gpu_nvidiagpu_mpich.yaml | 6 +++--- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mache/spack/pm-gpu_gnugpu_mpich.csh b/mache/spack/pm-gpu_gnugpu_mpich.csh index 10e54ac1..725bfd01 100644 --- a/mache/spack/pm-gpu_gnugpu_mpich.csh +++ b/mache/spack/pm-gpu_gnugpu_mpich.csh @@ -23,7 +23,7 @@ module rm cpe \ module load PrgEnv-gnu/8.5.0 \ gcc-native/12.3 \ - cudatoolkit/12.2 \ + cudatoolkit/12.4 \ craype-accel-nvidia80 \ cray-libsci/23.12.5 \ libfabric/1.20.1 \ diff --git a/mache/spack/pm-gpu_gnugpu_mpich.sh b/mache/spack/pm-gpu_gnugpu_mpich.sh index 1a96b61a..b81aa9b5 100644 --- a/mache/spack/pm-gpu_gnugpu_mpich.sh +++ b/mache/spack/pm-gpu_gnugpu_mpich.sh @@ -23,7 +23,7 @@ module rm cpe \ module load PrgEnv-gnu/8.5.0 \ gcc-native/12.3 \ - cudatoolkit/12.2 \ + cudatoolkit/12.4 \ craype-accel-nvidia80 \ cray-libsci/23.12.5 \ libfabric/1.20.1 \ diff --git a/mache/spack/pm-gpu_gnugpu_mpich.yaml b/mache/spack/pm-gpu_gnugpu_mpich.yaml index ebd03a26..0f18d918 100644 --- a/mache/spack/pm-gpu_gnugpu_mpich.yaml +++ b/mache/spack/pm-gpu_gnugpu_mpich.yaml @@ -105,11 +105,11 @@ spack: - cray-libsci/23.12.5 - libfabric/1.20.1 - craype/2.7.30 - - cudatoolkit/12.2 + - cudatoolkit/12.4 buildable: false cuda: externals: - - spec: cuda@12.2 + - spec: cuda@12.4 modules: - PrgEnv-gnu/8.5.0 - gcc-native/12.3 @@ -117,7 +117,7 @@ spack: - cray-libsci/23.12.5 - libfabric/1.20.1 - craype/2.7.30 - - cudatoolkit/12.2 + - cudatoolkit/12.4 buildable: false cray-mpich: externals: @@ -172,7 +172,7 @@ spack: - cray-libsci/23.12.5 - libfabric/1.20.1 - craype/2.7.30 - - cudatoolkit/12.2 + - cudatoolkit/12.4 environment: prepend_path: PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.33__gd067c3f.shasta/lib64/pkgconfig" diff --git a/mache/spack/pm-gpu_nvidiagpu_mpich.csh b/mache/spack/pm-gpu_nvidiagpu_mpich.csh index c5a71fc7..7220e0a8 100644 --- a/mache/spack/pm-gpu_nvidiagpu_mpich.csh +++ b/mache/spack/pm-gpu_nvidiagpu_mpich.csh @@ -23,7 +23,7 @@ module rm cpe \ module load PrgEnv-nvidia \ nvidia/24.5 \ - cudatoolkit/12.2 \ + cudatoolkit/12.4 \ craype-accel-nvidia80 \ gcc-native-mixed/12.3 \ cray-libsci/23.12.5 \ diff --git a/mache/spack/pm-gpu_nvidiagpu_mpich.sh b/mache/spack/pm-gpu_nvidiagpu_mpich.sh index 17e0f8dd..03cde3dc 100644 --- a/mache/spack/pm-gpu_nvidiagpu_mpich.sh +++ b/mache/spack/pm-gpu_nvidiagpu_mpich.sh @@ -23,7 +23,7 @@ module rm cpe \ module load PrgEnv-nvidia \ nvidia/24.5 \ - cudatoolkit/12.2 \ + cudatoolkit/12.4 \ craype-accel-nvidia80 \ gcc-native-mixed/12.3 \ cray-libsci/23.12.5 \ diff --git a/mache/spack/pm-gpu_nvidiagpu_mpich.yaml b/mache/spack/pm-gpu_nvidiagpu_mpich.yaml index b380f878..863652f5 100644 --- a/mache/spack/pm-gpu_nvidiagpu_mpich.yaml +++ b/mache/spack/pm-gpu_nvidiagpu_mpich.yaml @@ -96,7 +96,7 @@ spack: buildable: false cuda: externals: - - spec: cuda@12.2 + - spec: cuda@12.4 modules: - PrgEnv-nvidia - nvidia/24.5 @@ -104,7 +104,7 @@ spack: - cray-libsci/23.12.5 - libfabric/1.20.1 - craype/2.7.30 - - cudatoolkit/12.2 + - cudatoolkit/12.4 buildable: false cray-mpich: externals: @@ -161,7 +161,7 @@ spack: - cray-libsci/23.12.5 - libfabric/1.20.1 - craype/2.7.30 - - cudatoolkit/12.2 + - cudatoolkit/12.4 environment: prepend_path: PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.33__gd067c3f.shasta/lib64/pkgconfig" From f94d4349aa488bb4d3aada0f136d6495f67d1117 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 24 Oct 2024 10:20:40 -0700 Subject: [PATCH 14/14] Load cmake --- mache/spack/pm-cpu_gnu_mpich.csh | 3 ++- mache/spack/pm-cpu_gnu_mpich.sh | 3 ++- mache/spack/pm-cpu_nvidia_mpich.csh | 3 ++- mache/spack/pm-cpu_nvidia_mpich.sh | 3 ++- mache/spack/pm-gpu_gnugpu_mpich.csh | 3 ++- mache/spack/pm-gpu_gnugpu_mpich.sh | 3 ++- mache/spack/pm-gpu_nvidiagpu_mpich.csh | 3 ++- mache/spack/pm-gpu_nvidiagpu_mpich.sh | 3 ++- 8 files changed, 16 insertions(+), 8 deletions(-) diff --git a/mache/spack/pm-cpu_gnu_mpich.csh b/mache/spack/pm-cpu_gnu_mpich.csh index 7b9b8498..174ef24f 100644 --- a/mache/spack/pm-cpu_gnu_mpich.csh +++ b/mache/spack/pm-cpu_gnu_mpich.csh @@ -27,7 +27,8 @@ module load PrgEnv-gnu/8.5.0 \ craype-accel-host \ craype/2.7.30 \ libfabric/1.20.1 \ - cray-mpich/8.1.28 + cray-mpich/8.1.28 \ + cmake/3.24.3 {% if e3sm_hdf5_netcdf %} module load cray-hdf5-parallel/1.12.2.9 \ cray-netcdf-hdf5parallel/4.9.0.9 \ diff --git a/mache/spack/pm-cpu_gnu_mpich.sh b/mache/spack/pm-cpu_gnu_mpich.sh index d4580d8d..b824d419 100644 --- a/mache/spack/pm-cpu_gnu_mpich.sh +++ b/mache/spack/pm-cpu_gnu_mpich.sh @@ -27,7 +27,8 @@ module load PrgEnv-gnu/8.5.0 \ craype-accel-host \ craype/2.7.30 \ libfabric/1.20.1 \ - cray-mpich/8.1.28 + cray-mpich/8.1.28 \ + cmake/3.24.3 {% if e3sm_hdf5_netcdf %} module load cray-hdf5-parallel/1.12.2.9 \ cray-netcdf-hdf5parallel/4.9.0.9 \ diff --git a/mache/spack/pm-cpu_nvidia_mpich.csh b/mache/spack/pm-cpu_nvidia_mpich.csh index d410cac6..6a0f0732 100644 --- a/mache/spack/pm-cpu_nvidia_mpich.csh +++ b/mache/spack/pm-cpu_nvidia_mpich.csh @@ -27,7 +27,8 @@ module load PrgEnv-nvidia \ craype-accel-host \ craype/2.7.30 \ libfabric/1.20.1 \ - cray-mpich/8.1.28 + cray-mpich/8.1.28 \ + cmake/3.24.3 {% if e3sm_hdf5_netcdf %} module load cray-hdf5-parallel/1.12.2.9 \ cray-netcdf-hdf5parallel/4.9.0.9 \ diff --git a/mache/spack/pm-cpu_nvidia_mpich.sh b/mache/spack/pm-cpu_nvidia_mpich.sh index 7a1624e7..90f1d967 100644 --- a/mache/spack/pm-cpu_nvidia_mpich.sh +++ b/mache/spack/pm-cpu_nvidia_mpich.sh @@ -27,7 +27,8 @@ module load PrgEnv-nvidia \ craype-accel-host \ craype/2.7.30 \ libfabric/1.20.1 \ - cray-mpich/8.1.28 + cray-mpich/8.1.28 \ + cmake/3.24.3 {% if e3sm_hdf5_netcdf %} module load cray-hdf5-parallel/1.12.2.9 \ cray-netcdf-hdf5parallel/4.9.0.9 \ diff --git a/mache/spack/pm-gpu_gnugpu_mpich.csh b/mache/spack/pm-gpu_gnugpu_mpich.csh index 725bfd01..b13ef5b3 100644 --- a/mache/spack/pm-gpu_gnugpu_mpich.csh +++ b/mache/spack/pm-gpu_gnugpu_mpich.csh @@ -28,7 +28,8 @@ module load PrgEnv-gnu/8.5.0 \ cray-libsci/23.12.5 \ libfabric/1.20.1 \ craype/2.7.30 \ - cray-mpich/8.1.28 + cray-mpich/8.1.28 \ + cmake/3.24.3 {% if e3sm_hdf5_netcdf %} module load cray-hdf5-parallel/1.12.2.9 \ cray-netcdf-hdf5parallel/4.9.0.9 \ diff --git a/mache/spack/pm-gpu_gnugpu_mpich.sh b/mache/spack/pm-gpu_gnugpu_mpich.sh index b81aa9b5..009ffd41 100644 --- a/mache/spack/pm-gpu_gnugpu_mpich.sh +++ b/mache/spack/pm-gpu_gnugpu_mpich.sh @@ -28,7 +28,8 @@ module load PrgEnv-gnu/8.5.0 \ cray-libsci/23.12.5 \ libfabric/1.20.1 \ craype/2.7.30 \ - cray-mpich/8.1.28 + cray-mpich/8.1.28 \ + cmake/3.24.3 {% if e3sm_hdf5_netcdf %} module load cray-hdf5-parallel/1.12.2.9 \ cray-netcdf-hdf5parallel/4.9.0.9 \ diff --git a/mache/spack/pm-gpu_nvidiagpu_mpich.csh b/mache/spack/pm-gpu_nvidiagpu_mpich.csh index 7220e0a8..4aca5458 100644 --- a/mache/spack/pm-gpu_nvidiagpu_mpich.csh +++ b/mache/spack/pm-gpu_nvidiagpu_mpich.csh @@ -29,7 +29,8 @@ module load PrgEnv-nvidia \ cray-libsci/23.12.5 \ libfabric/1.20.1 \ craype/2.7.30 \ - cray-mpich/8.1.28 + cray-mpich/8.1.28 \ + cmake/3.24.3 {% if e3sm_hdf5_netcdf %} module load cray-hdf5-parallel/1.12.2.9 \ cray-netcdf-hdf5parallel/4.9.0.9 \ diff --git a/mache/spack/pm-gpu_nvidiagpu_mpich.sh b/mache/spack/pm-gpu_nvidiagpu_mpich.sh index 03cde3dc..0b88e212 100644 --- a/mache/spack/pm-gpu_nvidiagpu_mpich.sh +++ b/mache/spack/pm-gpu_nvidiagpu_mpich.sh @@ -29,7 +29,8 @@ module load PrgEnv-nvidia \ cray-libsci/23.12.5 \ libfabric/1.20.1 \ craype/2.7.30 \ - cray-mpich/8.1.28 + cray-mpich/8.1.28 \ + cmake/3.24.3 {% if e3sm_hdf5_netcdf %} module load cray-hdf5-parallel/1.12.2.9 \ cray-netcdf-hdf5parallel/4.9.0.9 \