Bring Spack CI to release branh

.gitlab/gitlab-ci-spack.yml

@@ -0,0 +1,130 @@
+#  This file is used to define the GitLab CI/CD pipeline for the ADIOS2 project.
+default:
+  interruptible: true
+  tags:
+    - medium
+    - uo
+    - public
+    - x86_64
+
+#  This file is used to define the GitLab CI/CD pipeline for the ADIOS2 project.
+.common-sanitizer:
+  rules:
+    - if: '$CI_PIPELINE_SOURCE != "schedule"'
+      when: on_success
+    - when: never
+  variables:
+    GITLAB_SITE: "UO CI (gitlab.spack.io)"
+    CI_BIN_DIR:  "$CI_PROJECT_DIR/build"
+
+.common-build:
+  extends:
+    - .common-sanitizer
+  stage: build
+  script:
+    - bash scripts/ci/gitlab-ci/run.sh update
+    - bash scripts/ci/gitlab-ci/run.sh configure
+    - bash scripts/ci/gitlab-ci/run.sh build
+  artifacts:
+    expire_in: 1 hours
+    when: always
+    paths:
+      - $CI_BIN_DIR/bin/
+      - $CI_BIN_DIR/lib/
+      - $CI_BIN_DIR/lib64/
+      - $CI_BIN_DIR/thirdparty/**/lib64/
+      - $CI_BIN_DIR/testing/
+      - $CI_BIN_DIR/source/**/*.h
+      - $CI_BIN_DIR/source/**/*.in
+      - $CI_BIN_DIR/adios2_reorganize_wrapper
+
+      # CTest and CMake install files.
+      - $CI_BIN_DIR/CMakeCache.txt
+      - $CI_BIN_DIR/**/*.cmake
+      - $CI_BIN_DIR/Testing/
+
+      # CDash files.
+      - $CI_BIN_DIR/DartConfiguration.tcl
+
+.common-test:
+  extends:
+    - .common-sanitizer
+  stage: test
+  script:
+    - bash scripts/ci/gitlab-ci/run.sh memcheck
+
+#===============================================================================
+
+build:uo-sanitizer-asan:
+  image: ghcr.io/ornladios/adios2:ci-opensuse-tw-asan-20241205
+  extends: [.common-build]
+
+build:uo-sanitizer-msan:
+  image: ghcr.io/ornladios/adios2:ci-opensuse-tw-msan-20241205
+  extends: [.common-build]
+
+build:uo-sanitizer-tsan:
+  image: ghcr.io/ornladios/adios2:ci-opensuse-tw-tsan-20241205
+  extends: [.common-build]
+
+build:uo-sanitizer-ubsan:
+  image: ghcr.io/ornladios/adios2:ci-opensuse-tw-ubsan-20241205
+  extends: [.common-build]
+
+test:uo-sanitizer-asan:
+  image: ghcr.io/ornladios/adios2:ci-opensuse-tw-asan-20241205
+  extends: [.common-test]
+  needs: [build:uo-sanitizer-asan]
+
+test:uo-sanitizer-msan:
+  image: ghcr.io/ornladios/adios2:ci-opensuse-tw-msan-20241205
+  extends: [.common-test]
+  needs: [build:uo-sanitizer-msan]
+
+test:uo-sanitizer-tsan:
+  image: ghcr.io/ornladios/adios2:ci-opensuse-tw-tsan-20241205
+  extends: [.common-test]
+  needs: [build:uo-sanitizer-tsan]
+
+test:uo-sanitizer-ubsan:
+  image: ghcr.io/ornladios/adios2:ci-opensuse-tw-ubsan-20241205
+  extends: [.common-test]
+  needs: [build:uo-sanitizer-ubsan]
+
+deploy:sync-ornl:
+  environment: ornl
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+      when: on_success
+    - when: never
+  image: ornladios/adios2:sync-20241206-b
+  script: >
+    .gitlab/config/SpackCIBridge.py
+    ornladios/ADIOS2
+    [email protected]:ecpcitest/adios2.git
+    https://code.ornl.gov/
+    ecpcitest/adios2
+    --prereq-check=format
+    --prereq-check=git_checks
+    --disable-status-post
+  dependencies: []
+  needs: []
+
+deploy:sync-spack:
+  environment: spackio
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+      when: on_success
+    - when: never
+  image: ornladios/adios2:sync-20241206-b
+  script: >
+    .gitlab/config/SpackCIBridge.py
+    ornladios/ADIOS2
+    [email protected]:adios2/adios2.git
+    https://gitlab.spack.io/
+    adios2/ADIOS2
+    --prereq-check=format
+    --prereq-check=git_checks
+    --status-context="Frank CI (sanitizers)"
+  dependencies: []
+  needs: []

scripts/ci/cmake/ci-common.cmake

@@ -18,10 +18,15 @@ if(NOT CTEST_BUILD_FLAGS)
     set(CTEST_BUILD_FLAGS "-j${N2CPUS}")
   endif()
 endif()
+
 if(NOT PARALLEL_LEVEL IN_LIST CTEST_TEST_ARGS)
   list(APPEND CTEST_TEST_ARGS PARALLEL_LEVEL ${N2CPUS})
 endif()
 
+if(NOT PARALLEL_LEVEL IN_LIST CTEST_MEMCHECK_ARGS)
+  list(APPEND CTEST_MEMCHECK_ARGS PARALLEL_LEVEL ${N2CPUS})
+endif()
+
 if(NOT dashboard_model)
   set(dashboard_model Experimental)
 endif()

scripts/ci/cmake/ci-el7-spack.cmake

@@ -1,4 +1,4 @@
-# Client maintainer: chuck.atkins@kitware.com
+# Client maintainer: vicente.bolea@kitware.com
 
 include(ProcessorCount)
 ProcessorCount(NCPUS)

scripts/ci/cmake/ci-el7.cmake

@@ -1,4 +1,4 @@
-# Client maintainer: chuck.atkins@kitware.com
+# Client maintainer: vicente.bolea@kitware.com
 
 set(ENV{CC}  gcc)
 set(ENV{CXX} g++)

scripts/ci/cmake/ci-fedora-asan.cmake

@@ -1,4 +1,4 @@
-# Client maintainer: chuck.atkins@kitware.com
+# Client maintainer: vicente.bolea@kitware.com
 
 set(ENV{CC}  clang)
 set(ENV{CXX} clang++)
@@ -8,6 +8,8 @@ set(ENV{LSAN_OPTIONS} "suppressions=$ENV{CI_SOURCE_DIR}/scripts/ci/cmake/adios-a
 set(ENV{CFLAGS}   "${ASAN_FLAGS}")
 set(ENV{CXXFLAGS} "${ASAN_FLAGS}")
 
+set(CTEST_MEMORYCHECK_SANITIZER_OPTIONS "detect_odr_violation=0")
+
 set(dashboard_cache "
 BUILD_TESTING:BOOL=ON
 ADIOS2_BUILD_EXAMPLES:BOOL=ON
@@ -24,6 +26,13 @@ set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
 set(CTEST_BUILD_FLAGS "-k -j4")
 set(CTEST_MEMORYCHECK_TYPE "AddressSanitizer")
 
+list(APPEND EXCLUDE_EXPRESSIONS
+  "Install.*"
+  "Unit.FileTransport.FailOnEOF.Serial"
+  )
+list(JOIN EXCLUDE_EXPRESSIONS "|" TEST_EXCLUDE_STRING)
+set(CTEST_MEMCHECK_ARGS EXCLUDE "${TEST_EXCLUDE_STRING}")
+
 set(ADIOS_TEST_REPEAT 0)
 list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")
 include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake)

scripts/ci/cmake/ci-fedora-msan.cmake

@@ -1,4 +1,7 @@
-# Client maintainer: [email protected]
+# Client maintainer: [email protected]
+
+set(ENV{CC}  clang)
+set(ENV{CXX} clang++)
 
 set(dashboard_cache "
 BUILD_TESTING:BOOL=ON
@@ -15,10 +18,19 @@ HDF5_DIFF_EXECUTABLE:FILEPATH=/opt/msan/bin/h5diff
 ")
 
 set(dashboard_track "Analysis")
-set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
-set(CTEST_BUILD_FLAGS "-k -j4")
+set(CTEST_CMAKE_GENERATOR "Ninja")
 set(CTEST_MEMORYCHECK_TYPE "MemorySanitizer")
 
+list(APPEND EXCLUDE_EXPRESSIONS
+  "Install.*"
+  "Staging.1x1DefSync.BP3"
+  "Engine.DataMan.DataManEngineTest"
+  "Unit.FileTransport.FailOnEOF.Serial"
+  "Engine.BP.BPBufferSizeTest.SyncDeferredIdenticalUsage.*.Serial"
+  )
+list(JOIN EXCLUDE_EXPRESSIONS "|" TEST_EXCLUDE_STRING)
+set(CTEST_MEMCHECK_ARGS EXCLUDE "${TEST_EXCLUDE_STRING}")
+
 set(ADIOS_TEST_REPEAT 0)
 list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")
 include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake)

scripts/ci/cmake/ci-fedora-tsan.cmake

@@ -1,4 +1,4 @@
-# Client maintainer: chuck.atkins@kitware.com
+# Client maintainer: vicente.bolea@kitware.com
 
 set(dashboard_cache "
 BUILD_TESTING:BOOL=ON
@@ -19,6 +19,14 @@ set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
 set(CTEST_BUILD_FLAGS "-k -j4")
 set(CTEST_MEMORYCHECK_TYPE "ThreadSanitizer")
 
+list(APPEND EXCLUDE_EXPRESSIONS
+  "Install.*"
+  "Unit.FileTransport.FailOnEOF.Serial"
+  "Engine.BP.BPBufferSizeTest.SyncDeferredIdenticalUsage.*.Serial"
+  )
+list(JOIN EXCLUDE_EXPRESSIONS "|" TEST_EXCLUDE_STRING)
+set(CTEST_MEMCHECK_ARGS EXCLUDE "${TEST_EXCLUDE_STRING}")
+
 set(ADIOS_TEST_REPEAT 0)
 list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")
 include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake)

scripts/ci/cmake/ci-fedora-ubsan.cmake

@@ -1,4 +1,4 @@
-# Client maintainer: chuck.atkins@kitware.com
+# Client maintainer: vicente.bolea@kitware.com
 
 set(ENV{CC}  gcc)
 set(ENV{CXX} g++)
@@ -7,14 +7,15 @@ set(UBSAN_FLAGS "-fsanitize=undefined -fno-sanitize-recover=all -pthread")
 set(ENV{CFLAGS}   "${UBSAN_FLAGS}")
 set(ENV{CXXFLAGS} "${UBSAN_FLAGS}")
 set(ENV{FFLAGS}   "${UBSAN_FLAGS}")
-set(ENV{UBSAN_OPTIONS} "print_stacktrace=1")
+set(CTEST_MEMORYCHECK_SANITIZER_OPTIONS "print_stacktrace=1")
 
 set(dashboard_cache "
 BUILD_TESTING:BOOL=ON
 ADIOS2_BUILD_EXAMPLES:BOOL=ON
 
 ADIOS2_USE_HDF5:STRING=ON
 ADIOS2_USE_MPI:STRING=OFF
+ADIOS2_USE_Python:STRING=OFF
 
 HDF5_C_COMPILER_EXECUTABLE:FILEPATH=/usr/bin/h5cc
 HDF5_DIFF_EXECUTABLE:FILEPATH=/usr/bin/h5diff
@@ -25,6 +26,13 @@ set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
 set(CTEST_BUILD_FLAGS "-k -j2")
 set(CTEST_MEMORYCHECK_TYPE "UndefinedBehaviorSanitizer")
 
+list(APPEND EXCLUDE_EXPRESSIONS
+  "Install.*"
+  "Unit.FileTransport.FailOnEOF.Serial"
+  )
+list(JOIN EXCLUDE_EXPRESSIONS "|" TEST_EXCLUDE_STRING)
+set(CTEST_MEMCHECK_ARGS EXCLUDE "${TEST_EXCLUDE_STRING}")
+
 set(ADIOS_TEST_REPEAT 0)
 list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")
 include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake)

scripts/ci/images/opensuse-tw/build-images-opensuse-tw

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# This script is used to build the openSUSE Tumbleweed images
+
+set -xeuo pipefail
+
+declare -r DOCKER_REPOSITORY=ghcr.io/ornladios/adios2
+declare -u ENABLE_PUSH=
+
+#=============================================================================
+# Functions
+
+# Function to build a docker image
+function docker_build() {
+  local docker_file=$1
+  local docker_tag=$2
+  local args=(--progress=plain --rm)
+
+  if [[ -n "${ENABLE_PUSH}" ]]; then
+    args+=("--push")
+  fi
+
+  sudo docker build "${args[@]}"  \
+    --file "${docker_file}" \
+    --tag "${DOCKER_REPOSITORY}:${docker_tag}-$(date +"%Y%m%d")" \
+    .
+}
+
+function usage() {
+  echo "Usage: $0 [-p <PROJECT>] [-m <MILESTONE>]"
+  echo "    -p    Push to the docker registry"
+  exit 0
+}
+
+#=============================================================================
+# Main
+while getopts "p" o; do
+  case "${o}" in
+    p) ENABLE_PUSH=true ;;
+    *) usage ;;
+  esac
+done
+
+docker_build opensuse-tw-sanitizer-base.dockerfile ci-opensuse-tw-sanitizer-base
+docker_build opensuse-tw-full-stack-onbuild.dockerfile ci-opensuse-tw-full-stack-onbuild
+docker_build opensuse-tw-asan.dockerfile ci-opensuse-tw-asan
+docker_build opensuse-tw-msan.dockerfile ci-opensuse-tw-msan
+docker_build opensuse-tw-tsan.dockerfile ci-opensuse-tw-tsan
+docker_build opensuse-tw-ubsan.dockerfile ci-opensuse-tw-ubsan