From 367341feb5a993aaa2d0034d43089c3515ca6ad6 Mon Sep 17 00:00:00 2001 From: Pierre Pebay Date: Thu, 12 Dec 2024 16:57:36 +0100 Subject: [PATCH] #2302: Modify compiler selection for metric testing workflows --- ci/azure/azure-clang-17-ubuntu-mpich-perf.yml | 218 ++++++++++++++++++ ...api.yml => azure-clang-18-ubuntu-papi.yml} | 12 +- ...-perf.yml => azure-gcc-14-ubuntu-perf.yml} | 10 +- scripts/workflows-azure.ini | 38 ++- 4 files changed, 245 insertions(+), 33 deletions(-) create mode 100644 ci/azure/azure-clang-17-ubuntu-mpich-perf.yml rename ci/azure/{azure-gcc-12-ubuntu-papi.yml => azure-clang-18-ubuntu-papi.yml} (98%) rename ci/azure/{azure-gcc-12-ubuntu-perf.yml => azure-gcc-14-ubuntu-perf.yml} (98%) diff --git a/ci/azure/azure-clang-17-ubuntu-mpich-perf.yml b/ci/azure/azure-clang-17-ubuntu-mpich-perf.yml new file mode 100644 index 0000000000..a01722fca1 --- /dev/null +++ b/ci/azure/azure-clang-17-ubuntu-mpich-perf.yml @@ -0,0 +1,218 @@ +############################################################################### +############## Warning this is a generated file---do not modify ############### +############################################################################### + +name: PR tests (clang-17, ubuntu, mpich, perf) + +trigger: + branches: + include: + - develop + - main + +pr: + drafts: false + autoCancel: true + branches: + include: + - '*' + + +resources: +- repo: self + +variables: + tag: '$(Build.BuildId)' + REPO: lifflander1/vt + ARCH: amd64 + UBUNTU: 22.04 + COMPILER_TYPE: clang + COMPILER: clang-17 + HOST_COMPILER: clang-17 + BUILD_TYPE: release + ULIMIT_CORE: 0 + VT_CODE_COVERAGE: 0 + VT_LB: 1 + VT_TRACE: 0 + VT_TRACE_RT: 0 + VT_TRACE_ONLY: 1 + VT_MIMALLOC: 0 + VT_DOCS: 0 + VT_ASAN: 0 + VT_UBSAN: 0 + VT_WERROR: 1 + VT_POOL: 1 + VT_EXTENDED_TESTS: 1 + VT_TESTS_NUM_NODES: 2 + VT_EXTERNAL_FMT: 0 + VT_UNITY_BUILD: 1 + VT_PRODUCTION_BUILD: 0 + VT_FCONTEXT: 0 + VT_ZOLTAN: 0 + VT_CI_BUILD: 1 + VT_DIAGNOSTICS: 1 + VT_NO_COLOR: 1 + VT_BUILD_SHARED_LIBS: 0 + VT_INCLUSION: TPL + VT_DEBUG_VERBOSE: 0 + VT_KOKKOS_ENABLED: 0 + VT_TV_ENABLED: 0 + VT_PERF_ENABLED: 1 + VT_PAPI_ENABLED: 0 + CMAKE_CXX_STANDARD: 17 + VT_CI_TEST_LB_SCHEMA: 0 + RUN_FULL_CI: true + CACHE: "$(Agent.TempDirectory)/cache/" + cache_name: ubuntu-clang-17-cache + volume_name: ubuntu-cpp + build_root: "$(CACHE)$(ARCH)-ubuntu-$(UBUNTU)-$(HOST_COMPILER)-$(COMPILER)-cache/" + TS: 0 + TS_YEAR: 0 + TS_MONTH: 0 + TS_DAY: 0 + + + +stages: +- stage: Build + displayName: Build image + jobs: + - job: Build + displayName: Build + pool: + vmImage: 'ubuntu-22.04' + timeoutInMinutes: 180 + steps: + - checkout: self + fetchDepth: 0 + - task: Bash@3 + displayName: Build timestamp for caching + continueOnError: true + condition: eq(variables['RUN_FULL_CI'], 'true') + inputs: + targetType: 'inline' + script: | + echo 'string(TIMESTAMP current_date "%H;%M;%S" UTC)' > script + echo 'execute_process(COMMAND ${CMAKE_COMMAND} -E echo "${current_date}")' >> script + val=$(cmake -P script) + echo "##vso[task.setvariable variable=TS]$val" + echo 'string(TIMESTAMP current_date "%Y" UTC)' > script + echo 'execute_process(COMMAND ${CMAKE_COMMAND} -E echo "${current_date}")' >> script + val=$(cmake -P script) + echo "##vso[task.setvariable variable=TS_YEAR]$val" + echo 'string(TIMESTAMP current_date "%m" UTC)' > script + echo 'execute_process(COMMAND ${CMAKE_COMMAND} -E echo "${current_date}")' >> script + val=$(cmake -P script) + echo "##vso[task.setvariable variable=TS_MONTH]$val" + echo 'string(TIMESTAMP current_date "%d" UTC)' > script + echo 'execute_process(COMMAND ${CMAKE_COMMAND} -E echo "${current_date}")' >> script + val=$(cmake -P script) + echo "##vso[task.setvariable variable=TS_DAY]$val" + - task: Bash@3 + displayName: Output timestamp for caching + continueOnError: true + condition: and(eq(variables['RUN_FULL_CI'], 'true'), in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues')) + inputs: + targetType: 'inline' + script: 'echo "my pipeline variable is $(TS) $(TS_YEAR) $(TS_MONTH) $(TS_DAY)"' + - task: Cache@2 + displayName: Update cache + continueOnError: true + condition: and(eq(variables['RUN_FULL_CI'], 'true'), in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues')) + inputs: + securityNamespace: cache + key: $(Agent.OS) | "$(cache_name)" | $(TS_YEAR) | $(TS_MONTH) | $(TS_DAY) | $(TS) + path: '$(build_root)$(volume_name)/ccache' + restoreKeys: | + $(Agent.OS) | "$(cache_name)" | $(TS_YEAR) | $(TS_MONTH) | $(TS_DAY) + $(Agent.OS) | "$(cache_name)" | $(TS_YEAR) | $(TS_MONTH) + $(Agent.OS) | "$(cache_name)" | $(TS_YEAR) + $(Agent.OS) | "$(cache_name)" + - task: Bash@3 + displayName: Check for changes in containers + condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues') + inputs: + targetType: 'inline' + script: | + val=$(./scripts/check_containers.sh) + echo "##vso[task.setvariable variable=compose_command]$val" + - task: DockerCompose@1 + displayName: Pull container + inputs: + projectName: darma-tasking-vt + containerregistrytype: 'Container Registry' + dockerComposeFile: '**/docker-compose.yml' + action: 'Run a Docker Compose command' + dockerComposeCommand: '$(compose_command) ubuntu-cpp-clean' + - task: DockerCompose@1 + displayName: Build container + inputs: + projectName: darma-tasking-vt + containerregistrytype: 'Container Registry' + dockerComposeFile: '**/docker-compose.yml' + action: 'Run a Docker Compose command' + dockerComposeCommand: 'run ubuntu-cpp-clean' + env: + CODECOV_TOKEN: $(codecov_token) + - task: Bash@3 + displayName: Put compilation's and tests' logs in PR comment + continueOnError: true + condition: and(eq(variables['RUN_FULL_CI'], 'true'), and(succeededOrFailed(), eq(variables['Build.Reason'], 'PullRequest'))) + inputs: + targetType: 'inline' + script: | + ./scripts/report_logs_in_comment.sh \ + "$(build_root)$(volume_name)/vt/compilation_errors_warnings.out" \ + "$(build_root)$(volume_name)/vt/cmake-output.log" \ + "$(Build.BuildNumber)" \ + "$(System.PullRequest.PullRequestNumber)" \ + "$(Build.Repository.Name)" \ + "$GITHUB_PAT" \ + "$(Build.BuildId)" \ + "$(System.JobId)" \ + "$(Agent.JobStatus)" + env: + GITHUB_PAT: $(github_pat) + - task: DockerCompose@1 + displayName: Push container to registry + continueOnError: true + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/develop')) + inputs: + projectName: darma-tasking-vt + containerregistrytype: 'Container Registry' + dockerRegistryEndpoint: 'dockerRegistryConnection1' + dockerComposeFile: '**/docker-compose.yml' + action: 'Run a Docker Compose command' + dockerComposeCommand: 'push ubuntu-cpp-clean' + - task: Bash@3 + displayName: Create artifacts + continueOnError: true + condition: and(eq(variables['RUN_FULL_CI'], 'true'), in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues')) + inputs: + targetType: 'inline' + script: | + zip -j $(Agent.TempDirectory)/LastTest.log.gz $(build_root)$(volume_name)/vt/Testing/Temporary/LastTest.log + zip -j $(Agent.TempDirectory)/cmake-output.log.gz $(build_root)$(volume_name)/vt/cmake-output.log + - task: PublishPipelineArtifact@1 + displayName: Upload CMake test output artifact + continueOnError: true + condition: and(eq(variables['RUN_FULL_CI'], 'true'), in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues')) + inputs: + targetPath: '$(Agent.TempDirectory)/LastTest.log.gz' + artifact: 'CMakeLastTestLog' + publishLocation: 'pipeline' + - task: PublishPipelineArtifact@1 + displayName: Upload CMake full output artifact + continueOnError: true + condition: and(eq(variables['RUN_FULL_CI'], 'true'), in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues')) + inputs: + targetPath: '$(Agent.TempDirectory)/cmake-output.log.gz' + artifact: 'CMakeOutputLog' + publishLocation: 'pipeline' + - task: Bash@3 + displayName: Finish pipeline + condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues') + inputs: + targetType: 'inline' + script: | + echo Done diff --git a/ci/azure/azure-gcc-12-ubuntu-papi.yml b/ci/azure/azure-clang-18-ubuntu-papi.yml similarity index 98% rename from ci/azure/azure-gcc-12-ubuntu-papi.yml rename to ci/azure/azure-clang-18-ubuntu-papi.yml index 7697f7d2d9..6b149996f8 100644 --- a/ci/azure/azure-gcc-12-ubuntu-papi.yml +++ b/ci/azure/azure-clang-18-ubuntu-papi.yml @@ -2,7 +2,7 @@ ############## Warning this is a generated file---do not modify ############### ############################################################################### -name: PR tests (gcc-12, ubuntu, papi) +name: PR tests (clang-18, ubuntu, papi) trigger: branches: @@ -25,10 +25,10 @@ variables: tag: '$(Build.BuildId)' REPO: lifflander1/vt ARCH: amd64 - UBUNTU: 22.04 - COMPILER_TYPE: gnu - COMPILER: gcc-12 - HOST_COMPILER: gcc-12 + UBUNTU: 24.04 + COMPILER_TYPE: clang + COMPILER: clang-18 + HOST_COMPILER: clang-18 BUILD_TYPE: release ULIMIT_CORE: 0 VT_CODE_COVERAGE: 0 @@ -63,7 +63,7 @@ variables: VT_CI_TEST_LB_SCHEMA: 0 RUN_FULL_CI: true CACHE: "$(Agent.TempDirectory)/cache/" - cache_name: ubuntu-gcc-12-cache + cache_name: ubuntu-clang-18-cache volume_name: ubuntu-cpp build_root: "$(CACHE)$(ARCH)-ubuntu-$(UBUNTU)-$(HOST_COMPILER)-$(COMPILER)-cache/" TS: 0 diff --git a/ci/azure/azure-gcc-12-ubuntu-perf.yml b/ci/azure/azure-gcc-14-ubuntu-perf.yml similarity index 98% rename from ci/azure/azure-gcc-12-ubuntu-perf.yml rename to ci/azure/azure-gcc-14-ubuntu-perf.yml index f9ddcbc314..196033d7e2 100644 --- a/ci/azure/azure-gcc-12-ubuntu-perf.yml +++ b/ci/azure/azure-gcc-14-ubuntu-perf.yml @@ -2,7 +2,7 @@ ############## Warning this is a generated file---do not modify ############### ############################################################################### -name: PR tests (gcc-12, ubuntu, perf) +name: PR tests (gcc-14, ubuntu, perf) trigger: branches: @@ -25,10 +25,10 @@ variables: tag: '$(Build.BuildId)' REPO: lifflander1/vt ARCH: amd64 - UBUNTU: 22.04 + UBUNTU: 24.04 COMPILER_TYPE: gnu - COMPILER: gcc-12 - HOST_COMPILER: gcc-12 + COMPILER: gcc-14 + HOST_COMPILER: gcc-14 BUILD_TYPE: release ULIMIT_CORE: 0 VT_CODE_COVERAGE: 0 @@ -63,7 +63,7 @@ variables: VT_CI_TEST_LB_SCHEMA: 0 RUN_FULL_CI: true CACHE: "$(Agent.TempDirectory)/cache/" - cache_name: ubuntu-gcc-12-cache + cache_name: ubuntu-gcc-14-cache volume_name: ubuntu-cpp build_root: "$(CACHE)$(ARCH)-ubuntu-$(UBUNTU)-$(HOST_COMPILER)-$(COMPILER)-cache/" TS: 0 diff --git a/scripts/workflows-azure.ini b/scripts/workflows-azure.ini index 744f6d9f20..3f1369378e 100644 --- a/scripts/workflows-azure.ini +++ b/scripts/workflows-azure.ini @@ -245,32 +245,26 @@ output_name = ci/azure/azure-gcc-12-ubuntu-vttv.yml vt_tv_enabled = 1 vt_trace_only = 0 -[PR-tests-gcc-12-perf] -test_configuration = "gcc-12, ubuntu, perf" +[PR-tests-gcc-14-perf] +test_configuration = "gcc-14, ubuntu, perf" compiler_type = gnu -compiler = gcc-12 -distro = 22.04 -output_name = ci/azure/azure-gcc-12-ubuntu-perf.yml +compiler = gcc-14 +distro = 24.04 +output_name = ci/azure/azure-gcc-14-ubuntu-perf.yml vt_perf_enabled = 1 -[PR-tests-clang-14-perf] -test_configuration = "clang-14, ubuntu, mpich, perf" +[PR-tests-clang-17-perf] +test_configuration = "clang-17, ubuntu, mpich, perf" compiler_type = clang -compiler = clang-14 -distro = 22.04 -output_name = ci/azure/azure-clang-14-ubuntu-mpich-perf.yml - -[PR-tests-gcc-12-papi] -test_configuration = "gcc-12, ubuntu, papi" -compiler_type = gnu -compiler = gcc-12 +compiler = clang-17 distro = 22.04 -output_name = ci/azure/azure-gcc-12-ubuntu-papi.yml -vt_papi_enabled = 1 +output_name = ci/azure/azure-clang-17-ubuntu-mpich-perf.yml +vt_perf_enabled = 1 -[PR-tests-clang-14] -test_configuration = "clang-14, ubuntu, mpich, papi" +[PR-tests-clang-18-papi] +test_configuration = "clang-18, ubuntu, papi" compiler_type = clang -compiler = clang-14 -distro = 22.04 -output_name = ci/azure/azure-clang-14-ubuntu-mpich-papi.yml +compiler = clang-18 +distro = 24.04 +output_name = ci/azure/azure-clang-18-ubuntu-papi.yml +vt_papi_enabled = 1