Skip to content

Commit

Permalink
Merge branch 'main' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
mr-c committed Nov 11, 2024
2 parents 83a49f1 + ee30368 commit 61ad022
Show file tree
Hide file tree
Showing 109 changed files with 1,086 additions and 1,107 deletions.
101 changes: 101 additions & 0 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
version: 2.1

parameters:
REF:
type: string
default: ''
description: Optional tag to build

jobs:
arm-wheels:
parameters:
build:
type: string
image:
type: string

machine:
image: ubuntu-2404:current
resource_class: arm.medium # 2 vCPUs

environment:
CIBW_ARCHS: "aarch64"
CIBW_MANYLINUX_AARCH64_IMAGE: "<< parameters.image >>"
CIBW_MUSLLINUX_AARCH64_IMAGE: "<< parameters.image >>"
CIBW_BUILD: "<< parameters.build >>"

steps:
- checkout
- when:
condition: << pipeline.parameters.REF >>
steps:
- run:
name: Checkout branch/tag << pipeline.parameters.REF >>
command: |
echo "Switching to branch/tag << pipeline.parameters.REF >> if it exists"
git checkout << pipeline.parameters.REF >> || true
git pull origin << pipeline.parameters.REF >> || true
- run:
name: install cibuildwheel and other build reqs
command: |
python3 -m pip install --upgrade pip setuptools setuptools_scm[toml]
python3 -m pip install -rcibw-requirements.txt
- run:
name: pip freeze
command: |
python3 -m pip freeze
- run:
name: list wheels
command: |
python3 -m cibuildwheel . --print-build-identifiers
- run:
name: cibuildwheel
command: |
python3 -m cibuildwheel .
- store_test_results:
path: test-results/

- store_artifacts:
path: wheelhouse/

# - when:
# condition:
# or:
# - matches:
# pattern: ".+"
# value: "<< pipeline.git.tag >>"
# - << pipeline.parameters.REF >>
# steps:
# - run:
# environment:
# TWINE_NONINTERACTIVE: "1"
# command: |
# python3 -m pip install twine
# python3 -m twine upload --verbose --skip-existing wheelhouse/*

workflows:
wheels: # This is the name of the workflow, feel free to change it to better match your workflow.
# Inside the workflow, you define the jobs you want to run.
jobs:
- arm-wheels:
name: arm-wheels-manylinux_2_28
filters:
branches:
only: main
tags:
only: /.*/
build: "*manylinux*"
image: quay.io/pypa/manylinux_2_28_aarch64
- arm-wheels:
name: arm-wheels-musllinux_1_2
filters:
branches:
only: main
tags:
only: /.*/
build: "*musllinux*"
image: quay.io/pypa/musllinux_1_2_aarch64
19 changes: 10 additions & 9 deletions .github/workflows/ci-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ jobs:
strategy:
matrix:
py-ver-major: [3]
py-ver-minor: [8, 9, 10, 11, 12]
py-ver-minor: [9, 10, 11, 12, 13]
step: [lint, unit, bandit, mypy]

env:
Expand All @@ -44,11 +44,11 @@ jobs:
with:
fetch-depth: 0

- name: Set up Singularity
- name: Set up Singularity and environment-modules
if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }}
run: |
wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-focal_amd64.deb
sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb
sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb environment-modules
- name: Give the test runner user a name to make provenance happy.
if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }}
Expand Down Expand Up @@ -132,10 +132,10 @@ jobs:
with:
fetch-depth: 0

- name: Set up Singularity
- name: Set up Singularity and environment-modules
run: |
wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-focal_amd64.deb
sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb
sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb environment-modules
- name: Give the test runner user a name to make provenance happy.
run: sudo usermod -c 'CI Runner' "$(whoami)"
Expand Down Expand Up @@ -165,6 +165,7 @@ jobs:
runs-on: ubuntu-22.04

strategy:
fail-fast: false
matrix:
cwl-version: [v1.0, v1.1, v1.2]
container: [docker, singularity, podman]
Expand All @@ -179,11 +180,11 @@ jobs:
steps:
- uses: actions/checkout@v4

- name: Set up Singularity
- name: Set up Singularity and environment-modules
if: ${{ matrix.container == 'singularity' }}
run: |
wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-jammy_amd64.deb
sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb
sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb environment-modules
- name: Singularity cache
if: ${{ matrix.container == 'singularity' }}
Expand Down Expand Up @@ -228,10 +229,10 @@ jobs:
steps:
- uses: actions/checkout@v4

- name: Set up Singularity
- name: Set up Singularity and environment-modules
run: |
wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-jammy_amd64.deb
sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb
sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb environment-modules
- name: Set up Python
uses: actions/setup-python@v5
Expand Down
127 changes: 127 additions & 0 deletions .github/workflows/wheels.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
name: Python package build and publish

on:
release:
types: [published]
workflow_dispatch: {}
repository_dispatch: {}
push:
branches:
- main

concurrency:
group: wheels-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true

jobs:
build_wheels:
name: ${{ matrix.image }} wheels
runs-on: ubuntu-24.04
strategy:
matrix:
include:
- image: manylinux_2_28_x86_64
build: "*manylinux*"
- image: musllinux_1_2_x86_64
build: "*musllinux*"

steps:
- uses: actions/checkout@v4
if: ${{ github.event_name != 'repository_dispatch' }}
with:
fetch-depth: 0 # slow, but gets all the tags
- uses: actions/checkout@v4
if: ${{ github.event_name == 'repository_dispatch' }}
with:
fetch-depth: 0 # slow, but gets all the tags
ref: ${{ github.event.client_payload.ref }}

# - name: Set up QEMU
# if: runner.os == 'Linux'
# uses: docker/setup-qemu-action@v2
# with:
# platforms: all

- name: Build wheels
uses: pypa/[email protected]
env:
CIBW_BUILD: ${{ matrix.build }}
CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/${{ matrix.image }}
CIBW_MUSLLINUX_X86_64_IMAGE: quay.io/pypa/${{ matrix.image }}
# configure cibuildwheel to build native 64-bit archs ('auto64'), and some
# emulated ones
# Linux arm64 wheels are built on circleci
CIBW_ARCHS_LINUX: auto64 # ppc64le s390x

- uses: actions/upload-artifact@v4
with:
name: artifact-${{ matrix.image }}
path: ./wheelhouse/*.whl

build_sdist:
name: Build source distribution
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v4
if: ${{ github.event_name != 'repository_dispatch' }}
with:
fetch-depth: 0 # slow, but gets all the tags
- uses: actions/checkout@v4
if: ${{ github.event_name == 'repository_dispatch' }}
with:
fetch-depth: 0 # slow, but gets all the tags
ref: ${{ github.event.client_payload.ref }}

- name: Build sdist
run: pipx run build --sdist

- uses: actions/upload-artifact@v4
with:
name: artifact-source
path: dist/*.tar.gz

build_wheels_macos:
name: Build wheels on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
# macos-13 is an intel runner, macos-14 is apple silicon
os: [macos-13, macos-14]
steps:
- uses: actions/checkout@v4
if: ${{ github.event_name != 'repository_dispatch' }}
with:
fetch-depth: 0 # slow, but gets all the tags
- uses: actions/checkout@v4
if: ${{ github.event_name == 'repository_dispatch' }}
with:
fetch-depth: 0 # slow, but gets all the tags
ref: ${{ github.event.client_payload.ref }}

- name: Build wheels
uses: pypa/[email protected]

- uses: actions/upload-artifact@v4
with:
name: artifact-${{ matrix.os }}-${{ strategy.job-index }}
path: ./wheelhouse/*.whl

# upload_pypi:
# needs: [build_wheels, build_sdist]
# runs-on: ubuntu-24.04
# environment: deploy
# permissions:
# id-token: write
# if: (github.event_name == 'release' && github.event.action == 'published') || (github.event_name == 'repository_dispatch' && github.event.client_payload.publish_wheel == true)
# steps:
# - uses: actions/download-artifact@v4
# with:
# # unpacks default artifact into dist/
# pattern: artifact-*
# merge-multiple: true
# path: dist

# - name: Publish package distributions to PyPI
# uses: pypa/gh-action-pypi-publish@release/v1
# with:
# skip-existing: true
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
# Generated during tests
pytestdebug.log
tmp/
*.sif
involucro

# Python temps
__pycache__/
Expand Down Expand Up @@ -59,4 +61,3 @@ cwltool/_version.py
cwltool_deps
docs/_build/
docs/autoapi/

1 change: 1 addition & 0 deletions MANIFEST.in
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ include tests/reloc/dir2/*
include tests/checker_wf/*
include tests/subgraph/*
include tests/input_deps/*
recursive-include tests/test_deps_env
include tests/trs/*
include tests/wf/generator/*
include cwltool/py.typed
Expand Down
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ MODULE=cwltool

# `SHELL=bash` doesn't work for some, so don't use BASH-isms like
# `[[` conditional expressions.
PYSOURCES=$(wildcard ${MODULE}/**.py cwltool/cwlprov/*.py tests/*.py) setup.py
PYSOURCES=$(wildcard ${MODULE}/**.py cwltool/cwlprov/*.py tests/*.py tests/cwl-conformance/*.py) setup.py
DEVPKGS=diff_cover pylint pep257 pydocstyle 'tox<4' tox-pyenv auto-walrus \
isort wheel autoflake pyupgrade bandit -rlint-requirements.txt\
-rtest-requirements.txt -rmypy-requirements.txt -rdocs/requirements.txt
Expand Down Expand Up @@ -190,7 +190,7 @@ shellcheck: FORCE
cwltool-in-docker.sh

pyupgrade: $(PYSOURCES)
pyupgrade --exit-zero-even-if-changed --py38-plus $^
pyupgrade --exit-zero-even-if-changed --py39-plus $^
auto-walrus $^

release-test: FORCE
Expand Down
16 changes: 16 additions & 0 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,22 @@ and ``--tmp-outdir-prefix`` to somewhere under ``/Users``::

$ cwl-runner --tmp-outdir-prefix=/Users/username/project --tmpdir-prefix=/Users/username/project wc-tool.cwl wc-job.json


Docker default platform on macOS with Apple Silicon
===================================================

If macOS users want to run CWL tools/workflows using ``cwltool`` with Docker and their software containers only have support for amd64 (64-bit x86) CPUs, but they have an Apple Silicon (aarch64/arm64) CPU,
they run into the error:

WARNING: The requested image's platform (linux/amd64) does not match the detected host platform (linux/arm64/v8) and no specific platform was requested.

To fix this, export the following environment variable before executing `cwltool`:

``export DOCKER_DEFAULT_PLATFORM=linux/amd64``

To automatically have this variable set in the future, add the same command to ones respective shell profile (e.g. ``~/.zshrc``, ``~/.bash_profile``).


Using uDocker
=============

Expand Down
1 change: 1 addition & 0 deletions cibw-requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
cibuildwheel==2.21.3
Loading

0 comments on commit 61ad022

Please sign in to comment.