Add configuration for Sunspot. (sandialabs#521)

configs/base/concretizer.yaml

@@ -1,2 +1,3 @@
 concretizer:
-  unify: true
+  unify: false
+  reuse: false

configs/sunspot/compilers.yaml

@@ -0,0 +1,19 @@
+compilers:
+  - compiler:
+      spec: [email protected]
+      paths:
+        cc: /soft/restricted/CNDA/updates/2022.12.30.001/oneapi/compiler/trunk-20230201/compiler/linux/bin/icx
+        cxx: /soft/restricted/CNDA/updates/2022.12.30.001/oneapi/compiler/trunk-20230201/compiler/linux/bin/icpx
+        f77: /soft/restricted/CNDA/updates/2022.12.30.001/oneapi/compiler/trunk-20230201/compiler/linux/bin/ifx
+        fc: /soft/restricted/CNDA/updates/2022.12.30.001/oneapi/compiler/trunk-20230201/compiler/linux/bin/ifx
+      flags: {}
+      operating_system: sles15
+      target: x86_64
+      modules: 
+        - gcc/11.2.0
+        - oneapi/eng-compiler/2022.12.30.003
+        - mpich/51.2/icc-all-pmix-gpu
+        - libfabric/1.15.2.0
+        - intel_compute_runtime/release/agama-devel-551
+      environment: {}
+      extra_rpaths: []

configs/sunspot/config.yaml

@@ -0,0 +1,2 @@
+config:
+  build_jobs: 32

configs/sunspot/packages.yaml

@@ -0,0 +1,35 @@
+packages:
+  mpich:
+    require: "@51.2"
+    buildable: false
+    externals:
+      - spec: "[email protected]"
+        prefix: /soft/restricted/CNDA/updates/mpich/51.2/mpich-ofi-all-icc-default-pmix-gpu-drop51
+        modules:
+          - mpich/51.2/icc-all-pmix-gpu
+  m4:
+    require: "@1.4.19"
+    buildable: false
+    externals:
+      - spec: "[email protected]"
+        prefix: /usr
+  libffi:
+    require: "@3.2.1"
+    buildable: false
+    externals:
+      - spec: "[email protected]"
+        prefix: /usr
+  nalu-wind:
+    require: "~trilinos-solvers"
+  trilinos:
+    require: "+tpetra+kokkos"
+  boost:
+    version: [1.78.0]
+    variants: ~shared+pic
+  all:
+    compiler: [intel]
+    variants: build_type=Release +mpi
+    providers:
+      mpi: [mpich]
+      blas: [netlib-lapack]
+      lapack: [netlib-lapack]

env-templates/exawind_sunspot.yaml

@@ -0,0 +1,9 @@
+spack:
+  include:
+  - include.yaml
+  concretizer:
+    unify: false
+    reuse: false
+  view: false
+  specs:
+    - 'exawind+ninja+hypre+amr_wind_gpu~nalu_wind_gpu+sycl'

repos/exawind/packages/amr-wind/package.py

@@ -22,12 +22,15 @@ class AmrWind(SMCMakeExtension, bAmrWind):
             description="Enable HDF5 plots with ZFP compression")
     variant("umpire", default=False,
             description="Enable Umpire")
+    variant("sycl", default=False,
+            description="Enable SYCL backend")
 
     depends_on("hdf5~mpi", when="+hdf5~mpi")
     depends_on("hdf5+mpi", when="+hdf5+mpi")
     depends_on("h5z-zfp", when="+hdf5")
     depends_on("zfp", when="+hdf5")
     depends_on("hypre+umpire", when="+umpire")
+    depends_on("hypre+sycl", when="+sycl")
 
     def setup_build_environment(self, env):
         if "+asan" in self.spec:
@@ -75,6 +78,16 @@ def cmake_args(self):
             cmake_options.append("-DAMDGPU_TARGETS=" + ";".join(str(x) for x in targets))
             cmake_options.append("-DGPU_TARGETS=" + ";".join(str(x) for x in targets))
 
+        if "+sycl" in self.spec:
+            cmake_options.append("-DAMReX_GPU_BACKEND=SYCL")
+            # SYCL GPU backend only supported with Intel's oneAPI or DPC++ compilers
+            sycl_compatible_compilers = ["dpcpp", "icpx"]
+            if not (os.path.basename(self.compiler.cxx) in sycl_compatible_compilers):
+                raise InstallError(
+                    "AMReX's SYCL GPU Backend requires DPC++ (dpcpp)"
+                    + " or the oneAPI CXX (icpx) compiler."
+                )
+
         if "+tests" in spec:
             spack_manager_local_golds = os.path.join(os.getenv("SPACK_MANAGER"), "golds")
             spack_manager_golds_dir = os.getenv("SPACK_MANAGER_GOLDS_DIR", default=spack_manager_local_golds)

repos/exawind/packages/exawind/package.py

@@ -44,9 +44,12 @@ class Exawind(SMCMakeExtension, CudaPackage, ROCmPackage):
             description="Enable Umpire")
     variant("tiny_profile", default=False,
             description="Turn on AMR-wind with tiny profile")
+    variant("sycl", default=False,
+            description="Enable SYCL backend for AMR-Wind")
 
-    conflicts("+amr_wind_gpu", when="~cuda~rocm")
+    conflicts("+amr_wind_gpu", when="~cuda~rocm~sycl")
     conflicts("+nalu_wind_gpu", when="~cuda~rocm")
+    conflicts("+nalu_wind_gpu", when="+sycl")
 
     for arch in CudaPackage.cuda_arch_values:
         depends_on("amr-wind+cuda cuda_arch=%s" % arch, when="+amr_wind_gpu+cuda cuda_arch=%s" % arch)
@@ -67,16 +70,17 @@ class Exawind(SMCMakeExtension, CudaPackage, ROCmPackage):
     depends_on("[email protected]:", when="+openfast")
     depends_on("[email protected]:", when="^nalu-wind+openfast")
     depends_on("[email protected]:", when="^amr-wind+openfast")
-    depends_on("nalu-wind+hypre", when="+hypre+amr_wind_gpu+nalu_wind_gpu")
-    depends_on("nalu-wind+hypre", when="+hypre~amr_wind_gpu~nalu_wind_gpu")
-    depends_on("nalu-wind+hypre2", when="+hypre~amr_wind_gpu+nalu_wind_gpu")
-    depends_on("nalu-wind+hypre2", when="+hypre+amr_wind_gpu~nalu_wind_gpu")
+    depends_on("nalu-wind+hypre~hypre2", when="+hypre+amr_wind_gpu+nalu_wind_gpu")
+    depends_on("nalu-wind+hypre~hypre2", when="+hypre~amr_wind_gpu~nalu_wind_gpu")
+    depends_on("nalu-wind+hypre2~hypre", when="+hypre~amr_wind_gpu+nalu_wind_gpu")
+    depends_on("nalu-wind+hypre2~hypre", when="+hypre+amr_wind_gpu~nalu_wind_gpu")
     depends_on("amr-wind+hypre", when="+hypre")
     depends_on("amr-wind~hypre", when="~hypre")
     depends_on("nalu-wind~hypre", when="~hypre")
     depends_on("trilinos+ninja", when="+ninja")
     depends_on("nalu-wind+ninja", when="+ninja")
     depends_on("amr-wind+ninja", when="+ninja")
+    depends_on("amr-wind+sycl", when="+amr_wind_gpu+sycl")
     # not required but added so these get picked up as a
     # direct dependency when creating snapshots
     depends_on("trilinos")

scripts/install-exawind.sh

@@ -42,6 +42,7 @@ if [ "${SPACK_MANAGER_MACHINE}" == 'eagle' ] || \
    [ "${SPACK_MANAGER_MACHINE}" == 'spock' ] || \
    [ "${SPACK_MANAGER_MACHINE}" == 'crusher' ] || \
    [ "${SPACK_MANAGER_MACHINE}" == 'frontier' ] || \
+   [ "${SPACK_MANAGER_MACHINE}" == 'sunspot' ] || \
    [ "${SPACK_MANAGER_MACHINE}" == 'azure' ]; then
   cmd "spack manager create-env -y ${SPACK_MANAGER}/env-templates/exawind_${SPACK_MANAGER_MACHINE}.yaml -d ${SPACK_MANAGER}/environments/exawind-${SPACK_MANAGER_MACHINE}"
 else

spack-scripting/scripting/cmd/manager_cmds/find_machine.py

@@ -78,8 +78,11 @@ def is_e4s():
     "frontier": MachineData(
         lambda: os.environ["LMOD_SYSTEM_NAME"] == "frontier", "frontier.olcf.ornl.gov"
     ),
-    # JLSE
+    # ANL
     "arcticus": MachineData(lambda: is_jlse(socket.gethostname()), "arcticus.alcf.anl.gov"),
+    "sunspot": MachineData(
+        lambda: os.environ["AURORA_BASE_ENV"] == "2023.0", "sunspot.alcf.anl.gov"
+    ),
     # E4S
     "e4s": MachineData(lambda: is_e4s(), "e4s.nodomain.gov"),
     # Azure