diff --git a/.github/workflows/htcondor.yaml b/.github/workflows/htcondor.yaml index 86a5cad..bd1c6e9 100644 --- a/.github/workflows/htcondor.yaml +++ b/.github/workflows/htcondor.yaml @@ -1,9 +1,4 @@ name: Test SnakemakeProfiles/htcondor -env: - SNAKEMAKE_IMAGE: quay.io/biocontainers/snakemake:7.32.4--hdfd78af_1 - HTCONDOR_IMAGE: htcondor/mini:23.0-el8 - DOCKER_COMPOSE: tests/docker-compose.yaml - on: [push, pull_request] jobs: @@ -11,6 +6,14 @@ jobs: name: Test htcondor profile in docker containers runs-on: ubuntu-latest timeout-minutes: 30 + strategy: + matrix: + snakemake_image: + - "quay.io/biocontainers/snakemake:7.32.4--hdfd78af_1" + - "quay.io/biocontainers/snakemake:8.9.0--hdfd78af_0" + htcondor_image: + - "htcondor/mini:23.5.2-el8" + steps: - uses: actions/checkout@v2 - run: mkdir -p ~/image-cache @@ -28,7 +31,7 @@ jobs: CACHE_NUMBER: 0 with: path: ~/image-cache - key: image-cache-${{ runner.os }}-${{ env.CACHE_NUMBER }}-${{ env.SNAKEMAKE_IMAGE }}-${{ env.HTCONDOR_IMAGE }} + key: image-cache-${{ runner.os }}-${{ env.CACHE_NUMBER }}-${{ matrix.snakemake_image }}-${{ matrix.htcondor_image }} - name: install miniconda uses: conda-incubator/setup-miniconda@v2 @@ -49,16 +52,16 @@ jobs: - name: docker deploy shell: bash -l {0} env: - DOCKER_COMPOSE: ${{ env.DOCKER_COMPOSE }} - SNAKEMAKE_IMAGE: ${{ env.SNAKEMAKE_IMAGE }} - HTCONDOR_IMAGE: ${{ env.HTCONDOR_IMAGE }} + DOCKER_COMPOSE: "tests/docker-compose.yaml" + SNAKEMAKE_IMAGE: ${{ matrix.snakemake_image }} + HTCONDOR_IMAGE: ${{ matrix.htcondor_image }} run: ./tests/deploystack.sh - if: steps.cache-images.outputs.cache-hit != 'true' - run: docker save -o ~/image-cache/snakemake.tar ${{ env.SNAKEMAKE_IMAGE }} + run: docker save -o ~/image-cache/snakemake.tar ${{ matrix.snakemake_image }} - if: steps.cache-images.outputs.cache-hit != 'true' - run: docker save -o ~/image-cache/htcondor.tar ${{ env.HTCONDOR_IMAGE }} + run: docker save -o ~/image-cache/htcondor.tar ${{ matrix.htcondor_image }} - name: run tests shell: bash -l {0} diff --git a/test-environment.yml b/test-environment.yml index 03280ac..1c6d127 100644 --- a/test-environment.yml +++ b/test-environment.yml @@ -15,3 +15,4 @@ dependencies: - pyflakes - urllib3 - cryptography + - psutil \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 2b3600f..b280591 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -118,7 +118,7 @@ def htcondor(request): """HTCondor fixture Return relevant container depending on environment. First look for - sbatch command to determine whether we are on a system running the + condor_q command to determine whether we are on a system running the HTCondor scheduler. Second, try deploying a docker stack to run htcondor locally. diff --git a/tests/deploystack.sh b/tests/deploystack.sh index 8fdde9a..b5931bb 100755 --- a/tests/deploystack.sh +++ b/tests/deploystack.sh @@ -7,7 +7,7 @@ DOCKER_COMPOSE=${DOCKER_COMPOSE:=docker-compose.yaml} # Images SNAKEMAKE_IMAGE=${SNAKEMAKE_IMAGE:=quay.io/biocontainers/snakemake:7.32.4--hdfd78af_1} -HTCONDORIMAGE=${HTCONDORIMAGE:=htcondor/mini:23.0-el8} +HTCONDORIMAGE=${HTCONDORIMAGE:=htcondor/mini:23.5.2-el8} docker pull $SNAKEMAKE_IMAGE docker pull $HTCONDORIMAGE @@ -72,4 +72,4 @@ fi # Add htcondor to snakemake CONTAINER=$(docker ps | grep cookiecutter-htcondor_snakemake | awk '{print $1}') -docker exec $CONTAINER pip install htcondor +docker exec $CONTAINER pip install htcondor==23.5.2 diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index bf3efc2..e47ed61 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -1,7 +1,7 @@ version: '3' services: snakemake: - image: quay.io/biocontainers/snakemake:7.3.2--hdfd78af_0 + image: ${SNAKEMAKE_IMAGE} hostname: htcondorctl command: /bin/bash deploy: @@ -19,7 +19,7 @@ services: - usr:/usr/ htcondor: - image: htcondor/mini:el7 + image: ${HTCONDOR_IMAGE} hostname: htcondorctl stdin_open: true tty: true @@ -33,5 +33,6 @@ services: # Mount snakemake image usr volume to opt - usr:/opt/ - /tmp:/tmp + volumes: usr: diff --git a/tests/wrapper.py b/tests/wrapper.py index bd609b5..6a734f5 100644 --- a/tests/wrapper.py +++ b/tests/wrapper.py @@ -211,28 +211,6 @@ def output(self): return "\n".join(self._output) return self._output - @property - def external_jobid(self): - if len(self._external_jobid) == 0: - try: - m = re.findall(self._jobid_regex, self.output) - if m is not None: - self._external_jobid = [int(x) for y in m for x in y if x] - except Exception as e: - print(e) - finally: - (_, out) = self.exec_run('squeue -h -o "%.50j,%.10i"', stream=False) - try: - for res in out.decode().split("\n"): - if self.jobname in res: - self._external_jobid.append( - re.search(r" (\d+)$", res.strip()).group(1) - ) - except Exception as e: - print(e) - - return self._external_jobid - def wait_while_status(self, status, timeout=60, tdelta=10, verbose=False): """Wait for status to change""" t = 0 @@ -292,9 +270,10 @@ def external_jobid(self): except Exception as e: print(e) finally: - (_, out) = self.exec_run("condor_q --json", stream=False) + (_, out) = self.exec_run("condor_q --allusers --json", stream=False) try: jobinfos = json.loads(out) + print(jobinfos) except json.decoder.JSONDecodeError: return [] for job in jobinfos: