Add pm-gpu gnugpu as a supported compiler

E3SM-Project · Jan 26, 2024 · 4068572 · 4068572
1 parent 1664e2d
commit 4068572
Show file tree

Hide file tree

Showing 3 changed files with 292 additions and 0 deletions.
diff --git a/mache/spack/pm-gpu_gnugpu_mpich.csh b/mache/spack/pm-gpu_gnugpu_mpich.csh
@@ -0,0 +1,53 @@
+module rm cray-hdf5-parallel &> /dev/null
+module rm cray-netcdf-hdf5parallel &> /dev/null
+module rm cray-parallel-netcdf &> /dev/null
+module rm PrgEnv-gnu &> /dev/null
+module rm PrgEnv-intel &> /dev/null
+module rm PrgEnv-nvidia &> /dev/null
+module rm PrgEnv-cray &> /dev/null
+module rm PrgEnv-aocc &> /dev/null
+module rm intel &> /dev/null
+module rm intel-oneapi &> /dev/null
+module rm cudatoolkit &> /dev/null
+module rm craype-accel-nvidia80 &> /dev/null
+module rm craype-accel-host &> /dev/null
+module rm perftools-base &> /dev/null
+module rm perftools &> /dev/null
+module rm darshan &> /dev/null
+
+module load PrgEnv-gnu/8.3.3
+module load gcc/11.2.0
+module load craype-x86-milan
+module load libfabric/1.15.2.0
+module load cudatoolkit/11.7
+module load craype-accel-nvidia80
+module load craype/2.7.20
+module rm cray-mpich &> /dev/null
+module load cray-mpich/8.1.25
+{% if e3sm_lapack %}
+module load cray-libsci/23.02.1.1
+{% endif %}
+{% if e3sm_hdf5_netcdf %}
+module rm cray-hdf5-parallel &> /dev/null
+module rm cray-netcdf-hdf5parallel &> /dev/null
+module rm cray-parallel-netcdf &> /dev/null
+module load cray-hdf5-parallel/1.12.2.3
+module load cray-netcdf-hdf5parallel/4.9.0.3
+module load cray-parallel-netcdf/1.12.3.3
+{% endif %}
+
+{% if e3sm_hdf5_netcdf %}
+setenv NETCDF_C_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX
+setenv NETCDF_FORTRAN_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX
+setenv PNETCDF_PATH $CRAY_PARALLEL_NETCDF_PREFIX
+{% endif %}
+setenv MPICH_ENV_DISPLAY 1
+setenv MPICH_VERSION_DISPLAY 1
+## purposefully omitting OMP variables that cause trouble in ESMF
+# setenv OMP_STACKSIZE 128M
+# setenv OMP_PROC_BIND spread
+# setenv OMP_PLACES threads
+setenv HDF5_USE_FILE_LOCKING FALSE
+## Not needed
+# setenv PERL5LIB /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch
+setenv MPICH_GPU_SUPPORT_ENABLED 1
diff --git a/mache/spack/pm-gpu_gnugpu_mpich.sh b/mache/spack/pm-gpu_gnugpu_mpich.sh
@@ -0,0 +1,58 @@
+module rm cray-hdf5-parallel &> /dev/null
+module rm cray-netcdf-hdf5parallel &> /dev/null
+module rm cray-parallel-netcdf &> /dev/null
+module rm PrgEnv-gnu &> /dev/null
+module rm PrgEnv-intel &> /dev/null
+module rm PrgEnv-nvidia &> /dev/null
+module rm PrgEnv-cray &> /dev/null
+module rm PrgEnv-aocc &> /dev/null
+module rm intel &> /dev/null
+module rm intel-oneapi &> /dev/null
+module rm cudatoolkit &> /dev/null
+module rm craype-accel-nvidia80 &> /dev/null
+module rm craype-accel-host &> /dev/null
+module rm perftools-base &> /dev/null
+module rm perftools &> /dev/null
+module rm darshan &> /dev/null
+
+module load PrgEnv-gnu/8.3.3
+module load gcc/11.2.0
+module load craype-x86-milan
+module load libfabric/1.15.2.0
+module load cudatoolkit/11.7
+module load craype-accel-nvidia80
+module load craype/2.7.20
+module rm cray-mpich &> /dev/null
+module load cray-mpich/8.1.25
+{% if e3sm_lapack %}
+module load cray-libsci/23.02.1.1
+{% endif %}
+{% if e3sm_hdf5_netcdf %}
+module rm cray-hdf5-parallel &> /dev/null
+module rm cray-netcdf-hdf5parallel &> /dev/null
+module rm cray-parallel-netcdf &> /dev/null
+module load cray-hdf5-parallel/1.12.2.3
+module load cray-netcdf-hdf5parallel/4.9.0.3
+module load cray-parallel-netcdf/1.12.3.3
+{% endif %}
+
+{% if e3sm_hdf5_netcdf %}
+export NETCDF_C_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX
+export NETCDF_FORTRAN_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX
+export PNETCDF_PATH=$CRAY_PARALLEL_NETCDF_PREFIX
+{% endif %}
+export MPICH_ENV_DISPLAY=1
+export MPICH_VERSION_DISPLAY=1
+## purposefully omitting OMP variables that cause trouble in ESMF
+# export OMP_STACKSIZE=128M
+# export OMP_PROC_BIND=spread
+# export OMP_PLACES=threads
+export HDF5_USE_FILE_LOCKING=FALSE
+## Not needed
+# export PERL5LIB=/global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch
+export MPICH_GPU_SUPPORT_ENABLED=1
+
+if [ -z "${NERSC_HOST:-}" ]; then
+  # happens when building spack environment
+  export NERSC_HOST="perlmutter"
+fi
diff --git a/mache/spack/pm-gpu_gnugpu_mpich.yaml b/mache/spack/pm-gpu_gnugpu_mpich.yaml
@@ -0,0 +1,181 @@
+spack:
+  specs:
+  - gcc
+  - cray-mpich
+{% if e3sm_lapack %}
+  - cray-libsci
+{% endif %}
+{% if e3sm_hdf5_netcdf %}
+  - hdf5
+  - netcdf-c
+  - netcdf-fortran
+  - parallel-netcdf
+{% endif %}
+{{ specs }}
+  concretizer:
+    unify: when_possible
+  packages:
+    all:
+      compiler: [[email protected]]
+      providers:
+        mpi: [[email protected]]
+{% if e3sm_lapack %}
+        lapack: [[email protected]]
+{% endif %}
+    bzip2:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    curl:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    cmake:
+      externals:
+      - spec: [email protected]
+        prefix: /global/common/software/nersc/pm-2022q4/spack/linux-sles15-zen/cmake-3.24.3-k5msymx/
+      buildable: false
+    gettext:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    gmake:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    libuv:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    libxml2:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    m4:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    ncurses:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    ninja:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    openssl:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    perl:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    python:
+      externals:
+      - spec: [email protected]
+        prefix: /global/common/software/nersc/pm-2022q3/sw/python/3.9-anaconda-2021.11
+        modules:
+        - python/3.9-anaconda-2021.11
+      buildable: false
+    tar:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    xz:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    gcc:
+      externals:
+      - spec: [email protected]
+        modules:
+        - PrgEnv-gnu/8.3.3
+        - gcc/11.2.0
+        - craype-accel-host
+        - craype/2.7.19
+        - libfabric/1.15.2.0
+      buildable: false
+    cray-mpich:
+      externals:
+      - spec: [email protected]
+        prefix: /opt/cray/pe/mpich/8.1.24/ofi/gnu/9.1
+        modules:
+        - libfabric/1.15.2.0
+        - cray-mpich/8.1.24
+      buildable: false
+    libfabric:
+      externals:
+      - spec: [email protected]
+        prefix: /opt/cray/libfabric/1.15.2.0
+        modules:
+        - libfabric/1.15.2.0
+      buildable: false
+{% if e3sm_lapack %}
+    cray-libsci:
+      externals:
+      - spec: [email protected]
+        prefix: /opt/cray/pe/libsci/23.02.1.1/GNU/9.1/x86_64
+        modules:
+        - cray-libsci/23.02.1.1
+      buildable: false
+{% endif %}
+{% if e3sm_hdf5_netcdf or system_hdf5_netcdf %}
+    hdf5:
+      externals:
+      - spec: [email protected]~cxx+fortran+hl~java+mpi+shared
+        prefix: /opt/cray/pe/hdf5-parallel/1.12.2.3/GNU/9.1
+      buildable: false
+    parallel-netcdf:
+      externals:
+      - spec: [email protected]+cxx+fortran+pic+shared
+        prefix: /opt/cray/pe/parallel-netcdf/1.12.3.3/GNU/9.1/
+      buildable: false
+    netcdf-c:
+      externals:
+      - spec: [email protected]+mpi~parallel-netcdf
+        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/GNU/9.1
+      buildable: false
+    netcdf-fortran:
+      externals:
+      - spec: [email protected]
+        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/GNU/9.1
+      buildable: false
+{% endif %}
+  config:
+    install_missing_compilers: false
+  compilers:
+  - compiler:
+      spec: [email protected]
+      paths:
+        cc: cc
+        cxx: CC
+        f77: ftn
+        fc: ftn
+      flags: {}
+      operating_system: sles15
+      target: any
+      modules:
+      - PrgEnv-gnu/8.3.3
+      - gcc/11.2.0
+      - craype-x86-milan
+      - cudatoolkit/11.7
+      - craype-accel-nvidia80
+      - craype/2.7.19
+      - libfabric/1.15.2.0
+      environment:
+        prepend_path:
+          PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.33__gd067c3f.shasta/lib64/pkgconfig"