#2121: Generalize names of named volumes

DARMA-tasking · Nov 29, 2023 · 511daee · 511daee
1 parent 65e7ebc
commit 511daee
Show file tree

Hide file tree

Showing 18 changed files with 134 additions and 56 deletions.
diff --git a/ci/azure/azure-clang-10-ubuntu-mpich.yml b/ci/azure/azure-clang-10-ubuntu-mpich.yml
@@ -127,6 +127,13 @@ stages:
           script: |
             val=$(./scripts/check_containers.sh)
             echo "##vso[task.setvariable variable=compose_command]$val"
+      - task: Bash@3
+        displayName: Create directory for docker volume
+        condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues')
+        inputs:
+          targetType: 'inline'
+          script: |
+            mkdir -p "$(build_root)"
       - task: DockerCompose@0
         displayName: Pull container
         inputs:

diff --git a/ci/azure/azure-clang-11-ubuntu-mpich.yml b/ci/azure/azure-clang-11-ubuntu-mpich.yml
@@ -127,6 +127,13 @@ stages:
           script: |
             val=$(./scripts/check_containers.sh)
             echo "##vso[task.setvariable variable=compose_command]$val"
+      - task: Bash@3
+        displayName: Create directory for docker volume
+        condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues')
+        inputs:
+          targetType: 'inline'
+          script: |
+            mkdir -p "$(build_root)"
       - task: DockerCompose@0
         displayName: Pull container
         inputs:

diff --git a/ci/azure/azure-clang-12-ubuntu-mpich.yml b/ci/azure/azure-clang-12-ubuntu-mpich.yml
@@ -127,6 +127,13 @@ stages:
           script: |
             val=$(./scripts/check_containers.sh)
             echo "##vso[task.setvariable variable=compose_command]$val"
+      - task: Bash@3
+        displayName: Create directory for docker volume
+        condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues')
+        inputs:
+          targetType: 'inline'
+          script: |
+            mkdir -p "$(build_root)"
       - task: DockerCompose@0
         displayName: Pull container
         inputs:

diff --git a/ci/azure/azure-clang-13-ubuntu-mpich.yml b/ci/azure/azure-clang-13-ubuntu-mpich.yml
@@ -127,6 +127,13 @@ stages:
           script: |
             val=$(./scripts/check_containers.sh)
             echo "##vso[task.setvariable variable=compose_command]$val"
+      - task: Bash@3
+        displayName: Create directory for docker volume
+        condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues')
+        inputs:
+          targetType: 'inline'
+          script: |
+            mkdir -p "$(build_root)"
       - task: DockerCompose@0
         displayName: Pull container
         inputs:

diff --git a/ci/azure/azure-clang-14-ubuntu-mpich.yml b/ci/azure/azure-clang-14-ubuntu-mpich.yml
@@ -127,6 +127,13 @@ stages:
           script: |
             val=$(./scripts/check_containers.sh)
             echo "##vso[task.setvariable variable=compose_command]$val"
+      - task: Bash@3
+        displayName: Create directory for docker volume
+        condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues')
+        inputs:
+          targetType: 'inline'
+          script: |
+            mkdir -p "$(build_root)"
       - task: DockerCompose@0
         displayName: Pull container
         inputs:

diff --git a/ci/azure/azure-clang-9-ubuntu-mpich.yml b/ci/azure/azure-clang-9-ubuntu-mpich.yml
@@ -127,6 +127,13 @@ stages:
           script: |
             val=$(./scripts/check_containers.sh)
             echo "##vso[task.setvariable variable=compose_command]$val"
+      - task: Bash@3
+        displayName: Create directory for docker volume
+        condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues')
+        inputs:
+          targetType: 'inline'
+          script: |
+            mkdir -p "$(build_root)"
       - task: DockerCompose@0
         displayName: Pull container
         inputs:

diff --git a/ci/azure/azure-clang-alpine-mpich.yml b/ci/azure/azure-clang-alpine-mpich.yml
@@ -127,6 +127,13 @@ stages:
           script: |
             val=$(./scripts/check_containers.sh)
             echo "##vso[task.setvariable variable=compose_command]$val"
+      - task: Bash@3
+        displayName: Create directory for docker volume
+        condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues')
+        inputs:
+          targetType: 'inline'
+          script: |
+            mkdir -p "$(build_root)"
       - task: DockerCompose@0
         displayName: Pull container
         inputs:

diff --git a/ci/azure/azure-gcc-10-ubuntu-openmpi.yml b/ci/azure/azure-gcc-10-ubuntu-openmpi.yml
@@ -127,6 +127,13 @@ stages:
           script: |
             val=$(./scripts/check_containers.sh)
             echo "##vso[task.setvariable variable=compose_command]$val"
+      - task: Bash@3
+        displayName: Create directory for docker volume
+        condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues')
+        inputs:
+          targetType: 'inline'
+          script: |
+            mkdir -p "$(build_root)"
       - task: DockerCompose@0
         displayName: Pull container
         inputs:

diff --git a/ci/azure/azure-gcc-11-ubuntu-mpich.yml b/ci/azure/azure-gcc-11-ubuntu-mpich.yml
@@ -127,6 +127,13 @@ stages:
           script: |
             val=$(./scripts/check_containers.sh)
             echo "##vso[task.setvariable variable=compose_command]$val"
+      - task: Bash@3
+        displayName: Create directory for docker volume
+        condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues')
+        inputs:
+          targetType: 'inline'
+          script: |
+            mkdir -p "$(build_root)"
       - task: DockerCompose@0
         displayName: Pull container
         inputs:

diff --git a/ci/azure/azure-gcc-12-ubuntu-mpich.yml b/ci/azure/azure-gcc-12-ubuntu-mpich.yml
@@ -127,6 +127,13 @@ stages:
           script: |
             val=$(./scripts/check_containers.sh)
             echo "##vso[task.setvariable variable=compose_command]$val"
+      - task: Bash@3
+        displayName: Create directory for docker volume
+        condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues')
+        inputs:
+          targetType: 'inline'
+          script: |
+            mkdir -p "$(build_root)"
       - task: DockerCompose@0
         displayName: Pull container
         inputs:

diff --git a/ci/azure/azure-gcc-8-ubuntu-mpich.yml b/ci/azure/azure-gcc-8-ubuntu-mpich.yml
@@ -127,6 +127,13 @@ stages:
           script: |
             val=$(./scripts/check_containers.sh)
             echo "##vso[task.setvariable variable=compose_command]$val"
+      - task: Bash@3
+        displayName: Create directory for docker volume
+        condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues')
+        inputs:
+          targetType: 'inline'
+          script: |
+            mkdir -p "$(build_root)"
       - task: DockerCompose@0
         displayName: Pull container
         inputs:

diff --git a/ci/azure/azure-gcc-9-ubuntu-mpich.yml b/ci/azure/azure-gcc-9-ubuntu-mpich.yml
@@ -127,6 +127,13 @@ stages:
           script: |
             val=$(./scripts/check_containers.sh)
             echo "##vso[task.setvariable variable=compose_command]$val"
+      - task: Bash@3
+        displayName: Create directory for docker volume
+        condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues')
+        inputs:
+          targetType: 'inline'
+          script: |
+            mkdir -p "$(build_root)"
       - task: DockerCompose@0
         displayName: Pull container
         inputs:

diff --git a/ci/azure/azure-intel-oneapi-icpc-ubuntu-mpich.yml b/ci/azure/azure-intel-oneapi-icpc-ubuntu-mpich.yml
@@ -127,6 +127,13 @@ stages:
           script: |
             val=$(./scripts/check_containers.sh)
             echo "##vso[task.setvariable variable=compose_command]$val"
+      - task: Bash@3
+        displayName: Create directory for docker volume
+        condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues')
+        inputs:
+          targetType: 'inline'
+          script: |
+            mkdir -p "$(build_root)"
       - task: DockerCompose@0
         displayName: Pull container
         inputs:

diff --git a/ci/azure/azure-intel-oneapi-icpx-ubuntu-mpich.yml b/ci/azure/azure-intel-oneapi-icpx-ubuntu-mpich.yml
@@ -127,6 +127,13 @@ stages:
           script: |
             val=$(./scripts/check_containers.sh)
             echo "##vso[task.setvariable variable=compose_command]$val"
+      - task: Bash@3
+        displayName: Create directory for docker volume
+        condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues')
+        inputs:
+          targetType: 'inline'
+          script: |
+            mkdir -p "$(build_root)"
       - task: DockerCompose@0
         displayName: Pull container
         inputs:

diff --git a/ci/azure/azure-nvidia-11-2-ubuntu-mpich.yml b/ci/azure/azure-nvidia-11-2-ubuntu-mpich.yml
@@ -127,6 +127,13 @@ stages:
           script: |
             val=$(./scripts/check_containers.sh)
             echo "##vso[task.setvariable variable=compose_command]$val"
+      - task: Bash@3
+        displayName: Create directory for docker volume
+        condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues')
+        inputs:
+          targetType: 'inline'
+          script: |
+            mkdir -p "$(build_root)"
       - task: DockerCompose@0
         displayName: Pull container
         inputs:

diff --git a/ci/azure/azure-nvidia-12-ubuntu-mpich.yml b/ci/azure/azure-nvidia-12-ubuntu-mpich.yml
@@ -127,6 +127,13 @@ stages:
           script: |
             val=$(./scripts/check_containers.sh)
             echo "##vso[task.setvariable variable=compose_command]$val"
+      - task: Bash@3
+        displayName: Create directory for docker volume
+        condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues')
+        inputs:
+          targetType: 'inline'
+          script: |
+            mkdir -p "$(build_root)"
       - task: DockerCompose@0
         displayName: Pull container
         inputs:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -65,61 +65,20 @@
 # Need verision >= 3.5 for the features in use
 version: '3.8'
 
-# Named volumes must be predefined according the docker compose rules. Many
-# combinations have already been added, but if a needed configuration is missing
-# add it to this list. For example, for ARM64v8 on Ubuntu 20.04 with clang-9,
-# add `arm64v8-ubuntu-20.04-clang-9-clang-9-cache`.
+# Volume will be created automatically for each combination of parameters.
 volumes:
-  amd64-ubuntu-18.04-clang-8-clang-8-cache:
-  amd64-ubuntu-18.04-clang-9-clang-9-cache:
-  amd64-ubuntu-20.04-clang-10-clang-10-cache:
-  amd64-ubuntu-22.04-clang-11-clang-11-cache:
-  amd64-ubuntu-18.04-gcc-8-gcc-8-cache:
-  amd64-ubuntu-20.04-gcc-9-gcc-9-cache:
-  amd64-ubuntu-20.04-gcc-10-gcc-10-cache:
-  amd64-ubuntu-22.04-gcc-12-gcc-12-cache:
-  amd64-ubuntu-18.04-icpx-icpx-cache:
-  amd64-ubuntu-18.04-icpc-icpc-cache:
-  amd64-ubuntu-20.04-gcc-9-11.0.3-cache:
-  amd64-ubuntu-20.04-gcc-9-11.2.0-cache:
-  amd64-alpine-clang-8-clang-8-cache:
-  amd64-alpine-clang-9-clang-9-cache:
-  amd64-alpine-clang-10-clang-10-cache:
-  amd64-alpine-clang-11-clang-11-cache:
-  amd64-alpine-clang-13-clang-13-cache:
-  amd64-alpine-gcc-8-gcc-8-cache:
-  amd64-alpine-gcc-9-gcc-9-cache:
-  amd64-alpine-gcc-10-gcc-10-cache:
-  amd64-alpine-gcc-12-gcc-12-cache:
-  amd64-alpine-icpx-icpx-cache:
-  amd64-alpine-icpc-icpc-cache:
-  amd64-alpine-gcc-9-11.0.3-cache:
-  amd64-alpine-gcc-9-11.2.0-cache:
-  arm64v8-ubuntu-18.04-clang-8-clang-8-cache:
-  arm64v8-ubuntu-18.04-clang-9-clang-9-cache:
-  arm64v8-ubuntu-20.04-clang-10-clang-10-cache:
-  arm64v8-ubuntu-22.04-clang-11-clang-11-cache:
-  arm64v8-ubuntu-18.04-gcc-8-gcc-8-cache:
-  arm64v8-ubuntu-20.04-gcc-9-gcc-9-cache:
-  arm64v8-ubuntu-20.04-gcc-10-gcc-10-cache:
-  arm64v8-ubuntu-22.04-gcc-12-gcc-12-cache:
-  arm64v8-ubuntu-18.04-icpx-icpx-cache:
-  arm64v8-ubuntu-18.04-icpc-icpc-cache:
-  arm64v8-ubuntu-20.04-gcc-9-11.0.3-cache:
-  arm64v8-ubuntu-20.04-gcc-9-11.2.0-cache:
-  arm64v8-alpine-clang-8-clang-8-cache:
-  arm64v8-alpine-clang-9-clang-9-cache:
-  arm64v8-alpine-clang-10-clang-10-cache:
-  arm64v8-alpine-clang-11-clang-11-cache:
-  arm64v8-alpine-clang-13-clang-13-cache:
-  arm64v8-alpine-gcc-8-gcc-8-cache:
-  arm64v8-alpine-gcc-9-gcc-9-cache:
-  arm64v8-alpine-gcc-10-gcc-10-cache:
-  arm64v8-alpine-gcc-12-gcc-12-cache:
-  arm64v8-alpine-icpx-icpx-cache:
-  arm64v8-alpine-icpc-icpc-cache:
-  arm64v8-alpine-gcc-9-11.0.3-cache:
-  arm64v8-alpine-gcc-9-11.2.0-cache:
+  ubuntu-cpp:
+    name: "${ARCH}-ubuntu-${UBUNTU}-${HOST_COMPILER}-${COMPILER}-cache"
+    driver_opts:
+      type: "none"
+      o: "bind"
+      device: "${CACHE}${ARCH}-ubuntu-${UBUNTU}-${HOST_COMPILER}-${COMPILER}-cache"
+  alpine-cpp:
+    name: "${ARCH}-alpine-${HOST_COMPILER}-${COMPILER}-cache"
+    driver_opts:
+      type: "none"
+      o: "bind"
+      device: "${CACHE}${ARCH}-alpine-${HOST_COMPILER}-${COMPILER}-cache"
 
 # Define basic rules for ccache used across multiple services. The beauty of
 # docker compose with cached volumes is that similarly configured builds will
@@ -195,7 +154,7 @@ services:
       <<: [*ccache, *vtopts]
     volumes: &ubuntu-volumes
       - .:/vt:delegated
-      - ${CACHE}${ARCH}-ubuntu-${UBUNTU}-${HOST_COMPILER}-${COMPILER}-cache:/build:delegated
+      - ubuntu-cpp:/build:delegated
     command: &vt-cpp-command >
       /bin/bash -c "
         /vt/ci/build_cpp.sh /vt /build &&
@@ -425,7 +384,7 @@ services:
       <<: [*ccache, *vtopts]
     volumes: &alpine-volumes
       - .:/vt:delegated
-      - ${CACHE}${ARCH}-alpine-${HOST_COMPILER}-${COMPILER}-cache:/build:delegated
+      - alpine-cpp:/build:delegated
     command: *vt-cpp-command
 
   ##############################################################################

diff --git a/scripts/workflow-azure-template.yml b/scripts/workflow-azure-template.yml
@@ -115,6 +115,13 @@ stages:
           script: |
             val=$(./scripts/check_containers.sh)
             echo "##vso[task.setvariable variable=compose_command]$val"
+      - task: Bash@3
+        displayName: Create directory for docker volume
+        condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues')
+        inputs:
+          targetType: 'inline'
+          script: |
+            mkdir -p "$(build_root)"
       - task: DockerCompose@0
         displayName: Pull container
         inputs: