add snakemake image test matrix (#20)

Snakemake-Profiles · Mar 22, 2024 · fd09713 · fd09713
1 parent bc9dbd4
commit fd09713
Show file tree

Hide file tree

Showing 6 changed files with 23 additions and 39 deletions.
diff --git a/.github/workflows/htcondor.yaml b/.github/workflows/htcondor.yaml
@@ -1,16 +1,19 @@
 name: Test SnakemakeProfiles/htcondor
-env:
-  SNAKEMAKE_IMAGE: quay.io/biocontainers/snakemake:7.32.4--hdfd78af_1
-  HTCONDOR_IMAGE: htcondor/mini:23.0-el8
-  DOCKER_COMPOSE: tests/docker-compose.yaml
-
 on: [push, pull_request]
 
 jobs:
   htcondortest:
     name: Test htcondor profile in docker containers
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    strategy:
+      matrix:
+        snakemake_image:
+          - "quay.io/biocontainers/snakemake:7.32.4--hdfd78af_1"
+          - "quay.io/biocontainers/snakemake:8.9.0--hdfd78af_0"
+        htcondor_image:
+          - "htcondor/mini:23.5.2-el8"
+
     steps:
       - uses: actions/checkout@v2
       - run: mkdir -p ~/image-cache
@@ -28,7 +31,7 @@ jobs:
           CACHE_NUMBER: 0
         with:
           path: ~/image-cache
-          key: image-cache-${{ runner.os }}-${{ env.CACHE_NUMBER }}-${{ env.SNAKEMAKE_IMAGE }}-${{ env.HTCONDOR_IMAGE }}
+          key: image-cache-${{ runner.os }}-${{ env.CACHE_NUMBER }}-${{ matrix.snakemake_image }}-${{ matrix.htcondor_image }}
 
       - name: install miniconda
         uses: conda-incubator/setup-miniconda@v2
@@ -49,16 +52,16 @@ jobs:
       - name: docker deploy
         shell: bash -l {0}
         env:
-          DOCKER_COMPOSE: ${{ env.DOCKER_COMPOSE }}
-          SNAKEMAKE_IMAGE: ${{ env.SNAKEMAKE_IMAGE }}
-          HTCONDOR_IMAGE: ${{ env.HTCONDOR_IMAGE }}
+          DOCKER_COMPOSE: "tests/docker-compose.yaml"
+          SNAKEMAKE_IMAGE: ${{ matrix.snakemake_image }}
+          HTCONDOR_IMAGE: ${{ matrix.htcondor_image }}
         run: ./tests/deploystack.sh
 
       - if: steps.cache-images.outputs.cache-hit != 'true'
-        run: docker save -o ~/image-cache/snakemake.tar ${{ env.SNAKEMAKE_IMAGE }}
+        run: docker save -o ~/image-cache/snakemake.tar ${{ matrix.snakemake_image }}
 
       - if: steps.cache-images.outputs.cache-hit != 'true'
-        run: docker save -o ~/image-cache/htcondor.tar ${{ env.HTCONDOR_IMAGE }}
+        run: docker save -o ~/image-cache/htcondor.tar ${{ matrix.htcondor_image }}
 
       - name: run tests
         shell: bash -l {0}

diff --git a/test-environment.yml b/test-environment.yml
@@ -15,3 +15,4 @@ dependencies:
   - pyflakes
   - urllib3
   - cryptography
+  - psutil
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -118,7 +118,7 @@ def htcondor(request):
     """HTCondor fixture
 
     Return relevant container depending on environment. First look for
-    sbatch command to determine whether we are on a system running the
+    condor_q command to determine whether we are on a system running the
     HTCondor scheduler. Second, try deploying a docker stack to run htcondor
     locally.
 

diff --git a/tests/deploystack.sh b/tests/deploystack.sh
@@ -7,7 +7,7 @@ DOCKER_COMPOSE=${DOCKER_COMPOSE:=docker-compose.yaml}
 
 # Images
 SNAKEMAKE_IMAGE=${SNAKEMAKE_IMAGE:=quay.io/biocontainers/snakemake:7.32.4--hdfd78af_1}
-HTCONDORIMAGE=${HTCONDORIMAGE:=htcondor/mini:23.0-el8}
+HTCONDORIMAGE=${HTCONDORIMAGE:=htcondor/mini:23.5.2-el8}
 
 docker pull $SNAKEMAKE_IMAGE
 docker pull $HTCONDORIMAGE
@@ -72,4 +72,4 @@ fi
 
 # Add htcondor to snakemake
 CONTAINER=$(docker ps | grep cookiecutter-htcondor_snakemake | awk '{print $1}')
-docker exec $CONTAINER pip install htcondor
+docker exec $CONTAINER pip install htcondor==23.5.2
diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml
@@ -1,7 +1,7 @@
 version: '3'
 services:
   snakemake:
-    image: quay.io/biocontainers/snakemake:7.3.2--hdfd78af_0
+    image: ${SNAKEMAKE_IMAGE}
     hostname: htcondorctl
     command: /bin/bash
     deploy:
@@ -19,7 +19,7 @@ services:
       - usr:/usr/
 
   htcondor:
-    image: htcondor/mini:el7
+    image: ${HTCONDOR_IMAGE}
     hostname: htcondorctl
     stdin_open: true
     tty: true
@@ -33,5 +33,6 @@ services:
       # Mount snakemake image usr volume to opt
       - usr:/opt/
       - /tmp:/tmp
+
 volumes:
   usr:
diff --git a/tests/wrapper.py b/tests/wrapper.py
@@ -211,28 +211,6 @@ def output(self):
             return "\n".join(self._output)
         return self._output
 
-    @property
-    def external_jobid(self):
-        if len(self._external_jobid) == 0:
-            try:
-                m = re.findall(self._jobid_regex, self.output)
-                if m is not None:
-                    self._external_jobid = [int(x) for y in m for x in y if x]
-            except Exception as e:
-                print(e)
-            finally:
-                (_, out) = self.exec_run('squeue -h -o "%.50j,%.10i"', stream=False)
-                try:
-                    for res in out.decode().split("\n"):
-                        if self.jobname in res:
-                            self._external_jobid.append(
-                                re.search(r" (\d+)$", res.strip()).group(1)
-                            )
-                except Exception as e:
-                    print(e)
-
-        return self._external_jobid
-
     def wait_while_status(self, status, timeout=60, tdelta=10, verbose=False):
         """Wait for status to change"""
         t = 0
@@ -292,9 +270,10 @@ def external_jobid(self):
             except Exception as e:
                 print(e)
             finally:
-                (_, out) = self.exec_run("condor_q --json", stream=False)
+                (_, out) = self.exec_run("condor_q --allusers --json", stream=False)
                 try:
                     jobinfos = json.loads(out)
+                    print(jobinfos)
                 except json.decoder.JSONDecodeError:
                     return []
                 for job in jobinfos:
-Original file line number
+Diff line change
@@ Expand Up / @@ -15,3 +15,4 @@ dependencies: @@
       - pyflakes
       - urllib3
       - cryptography
+      - psutil