Merge pull request #355 from lanl/ci_upgrade

More CI changes
lanl · Apr 8, 2024 · db010dd · db010dd
2 parents 522c517 + ca22266
commit db010dd
Showing 1 changed file with 44 additions and 33 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -8,20 +8,22 @@ stages:
 
 variables:
   GIT_SUBMODULE_STRATEGY: normal
+  XCAP_PROJECT_DIR: "/usr/projects/xcap/oss"
+  TESTING_CLUSTER: darwin
   SINGULARITY_GOLDFILES_VERSION: "goldfiles-1.8.0"
   SINGULARITY_EOS_GCC_VERSION: "10.3.0"
   SINGULARITY_EOS_OPENMPI_VERSION: "4.1.1"
   SINGULARITY_EOS_CUDA_VERSION: "11.6.0"
   SINGULARITY_EOS_BUILD_TYPE: "RelWithDebInfo"
-  SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+spiner+hdf5+python+tests%gcc@${SINGULARITY_EOS_GCC_VERSION} ^openmpi@${SINGULARITY_EOS_OPENMPI_VERSION}"
+  SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+spiner+hdf5+python+tests%gcc@=${SINGULARITY_EOS_GCC_VERSION} ^openmpi@${SINGULARITY_EOS_OPENMPI_VERSION}"
   COLOR_CYAN: "\e[1;36m"
   COLOR_PLAIN: "\e[0m"
-  CLUSTER: darwin
-  # set to different spack upstream for testing
-  SINGULARITY_EOS_TEMP_SPACK_DIR: ""
-  # uncomment for debugging the CI and persist the Spack installation
-  # SPACK_USER_CACHE_PATH: "${CI_PROJECT_DIR}/spack-local"
-  # DEVEL_SPACK_CHECKOUT: "${CI_PROJECT_DIR}/spack"
+  # uncomment to have the CI Spack installation for debugging
+  # PROJECT_TMP_CI_DIR: "${CI_PROJECT_DIR}"
+  # uncomment & set to different spack upstream for testing
+  # PROJECT_TMP_SPACK_DIR: "deployment/CI/spack-v0.20.1-8"
+  # uncomment for spack debug output
+  # PROJECT_SPACK_DEBUG_FLAG: "-d"
 
 workflow:
   rules:
@@ -53,7 +55,7 @@ default:
 
 .gpu: &gpu
   SINGULARITY_USE_CUDA: "true"
-  SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+tests+cuda+kokkos+spiner+hdf5 cuda_arch=${SINGULARITY_CUDA_ARCH} +kokkos-kernels%gcc@${SINGULARITY_EOS_GCC_VERSION} ^kokkos std=14 ^cuda@${SINGULARITY_EOS_CUDA_VERSION} ^openmpi@${SINGULARITY_EOS_OPENMPI_VERSION}"
+  SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+tests+cuda+kokkos+spiner+hdf5 cuda_arch=${SINGULARITY_CUDA_ARCH} +kokkos-kernels%gcc@=${SINGULARITY_EOS_GCC_VERSION} ^kokkos std=14 ^cuda@${SINGULARITY_EOS_CUDA_VERSION} ^openmpi@${SINGULARITY_EOS_OPENMPI_VERSION}"
 
 .fort: &fort
   SINGULARITY_USE_FORTRAN: "true"
@@ -78,9 +80,9 @@ default:
     - rocinante
     - batch
   variables:
-    CLUSTER: rocinante
+    TESTING_CLUSTER: rocinante
     SINGULARITY_EOS_GCC_VERSION: "12.2.0"
-    SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+spiner+hdf5+python+tests%gcc@${SINGULARITY_EOS_GCC_VERSION} ^cray-mpich"
+    SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+spiner+hdf5+python+tests%gcc@=${SINGULARITY_EOS_GCC_VERSION} ^cray-mpich"
     SCHEDULER_PARAMETERS: "--nodes=1 --time 2:00:00 -L none@slurmdb --account=asc-ci --qos=ci --reservation=ci --partition=ci"
     HDF5_USE_FILE_LOCKING: "FALSE"
 
@@ -114,27 +116,38 @@ default:
     - section() { echo $'\e[0K'"section_$1:$(date +%s):$2"$'\r\e[0K'"${3+${COLOR_CYAN}$3${COLOR_PLAIN}}"; }
     - export PYTHONNOUSERSITE=1
     - export SPACK_DISABLE_LOCAL_CONFIG=true
-    - export SPACK_USER_CACHE_PATH=${SPACK_USER_CACHE_PATH:-/tmp/spack-local}
-    - export DEVEL_SPACK_CHECKOUT=${DEVEL_SPACK_CHECKOUT:-/tmp/spack}
-    - export SINGULARITY_EOS_SPACK_DIR=${SINGULARITY_EOS_TEMP_SPACK_DIR:=$XCAP_OSS_SPACK_DIR}
-    - section start "spack_init[collapsed=true]" "Creating Spack instance at ${DEVEL_SPACK_CHECKOUT}"
-    - echo "Upstream ${SINGULARITY_EOS_SPACK_DIR}"
+    - export SPACK_SKIP_MODULES=true
+    - export TMP_USER_PROJECT_DIR="/tmp/${USER}/${CI_PROJECT_NAME}/${CI_JOB_NAME}"
+    - export PROJECT_TMP_CI_DIR=${PROJECT_TMP_CI_DIR:-${TMP_USER_PROJECT_DIR}}
+    - export SPACK_USER_CACHE_PATH="${PROJECT_TMP_CI_DIR}/spack-local"
+    - export DEVEL_SPACK_ROOT="${PROJECT_TMP_CI_DIR}/spack"
+    - export DEVEL_SPACK_MIRROR="${CI_PROJECT_DIR}/spack-mirror"
+    - rm -rf ${SPACK_USER_CACHE_PATH} ${DEVEL_SPACK_ROOT}
+    - |
+      if [[ -z "${PROJECT_TMP_SPACK_DIR}" ]]; then
+        export PROJECT_SPACK_ROOT=${XCAP_PROJECT_DIR}/spack
+      else
+        export PROJECT_SPACK_ROOT="${XCAP_PROJECT_DIR}/${PROJECT_TMP_SPACK_DIR}-${TESTING_CLUSTER}"
+      fi
+    - section start "spack_init[collapsed=true]" "Creating Spack instance at ${DEVEL_SPACK_ROOT}"
+    - echo "Upstream ${PROJECT_SPACK_ROOT}"
+    - mkdir -m 2770 -p ${PROJECT_TMP_CI_DIR}
     - |
       rsync -rpl \
           --exclude=".git" \
-          --exclude="etc/spack/repos.yaml" \
-          --exclude="etc/spack/packages.yaml" \
+          --exclude={"*__pycache__*","*.pyc"} \
           --include="etc/spack/**" \
           --include="lib/spack/**" \
-          --exclude-from=${SINGULARITY_EOS_SPACK_DIR}/.gitignore \
-          ${SINGULARITY_EOS_SPACK_DIR}/ ${DEVEL_SPACK_CHECKOUT}/
-    - source ${DEVEL_SPACK_CHECKOUT}/share/spack/setup-env.sh
+          --exclude-from=${PROJECT_SPACK_ROOT}/.gitignore \
+          ${PROJECT_SPACK_ROOT}/ ${DEVEL_SPACK_ROOT}/
+    - sed -i "s;xcap-admin;xcap;g" "${DEVEL_SPACK_ROOT}/etc/spack/packages.yaml"
+    - source ${DEVEL_SPACK_ROOT}/share/spack/setup-env.sh
     - spack --version
     - spack compiler list
     - section end spack_init
   script:
     - |
-      if [[ "${CLUSTER}" = "rocinante" ]]; then
+      if [[ "${TESTING_CLUSTER}" = "rocinante" ]]; then
         module load PrgEnv-gnu
         module load gcc/${SINGULARITY_EOS_GCC_VERSION}
       else
@@ -146,29 +159,27 @@ default:
       if [[ "${CI_COMMIT_BRANCH}" == "${CI_DEFAULT_BRANCH}" ]];
       then
         section start "spack_build[collapsed=true]" "Building via Spack"
-        mkdir -p spack_build_env
         spack env create spack_build_env
         spack env activate spack_build_env
-        spack repo add ${DEVEL_SPACK_CHECKOUT}/var/spack/repos/xcap_deployment
+        spack repo add ${DEVEL_SPACK_ROOT}/var/spack/repos/xcap_deployment
         spack repo add spack-repo
-        spack develop -p ${CI_PROJECT_DIR}/utils/spiner spiner@main
-        spack develop -p ${CI_PROJECT_DIR}/utils/ports-of-call ports-of-call@main
-        spack develop -p ${CI_PROJECT_DIR} singularity-eos@main
-        spack config add upstreams:default:install_tree:${SINGULARITY_EOS_SPACK_DIR}/opt/spack/
+        spack develop --no-clone -p ${CI_PROJECT_DIR}/utils/spiner spiner@main
+        spack develop --no-clone -p ${CI_PROJECT_DIR}/utils/ports-of-call ports-of-call@main
+        spack develop --no-clone -p ${CI_PROJECT_DIR} singularity-eos@main
+        spack config add upstreams:default:install_tree:${PROJECT_SPACK_ROOT}/opt/spack/
         spack dev-build -j $(nproc) ${SINGULARITY_EOS_SPACK_SPEC}
         spack env deactivate
         section end spack_build
       fi
     - section start "spack_env[collapsed=true]" "Creating Spack environment"
-    - mkdir -p spack_env
     - spack env create spack_env
     - spack env activate spack_env
-    - spack repo add ${DEVEL_SPACK_CHECKOUT}/var/spack/repos/xcap_deployment
+    - spack repo add ${DEVEL_SPACK_ROOT}/var/spack/repos/xcap_deployment
     - spack repo add spack-repo
-    - spack develop -p ${CI_PROJECT_DIR}/utils/spiner spiner@main
-    - spack develop -p ${CI_PROJECT_DIR}/utils/ports-of-call ports-of-call@main
-    - spack develop -p ${CI_PROJECT_DIR} singularity-eos@main
-    - spack config add upstreams:default:install_tree:${SINGULARITY_EOS_SPACK_DIR}/opt/spack/
+    - spack develop --no-clone -p ${CI_PROJECT_DIR}/utils/spiner spiner@main
+    - spack develop --no-clone -p ${CI_PROJECT_DIR}/utils/ports-of-call ports-of-call@main
+    - spack develop --no-clone -p ${CI_PROJECT_DIR} singularity-eos@main
+    - spack config add upstreams:default:install_tree:${PROJECT_SPACK_ROOT}/opt/spack/
     - spack spec -I ${SINGULARITY_EOS_SPACK_SPEC}
     - spack add ${SINGULARITY_EOS_SPACK_SPEC}
     - spack install -j $(nproc) --show-log-on-error --no-checksum --yes-to-all -u cmake