Switch eccodes python to wheelchains

ecmwf · Dec 16, 2024 · 2d1d0ce · 2d1d0ce
1 parent 6e01ef6
commit 2d1d0ce
Show file tree

Hide file tree

Showing 6 changed files with 104 additions and 256 deletions.
diff --git a/.github/workflows/build-wheel-linux.yml b/.github/workflows/build-wheel-linux.yml
@@ -7,6 +7,8 @@
 # nor does it submit to any jurisdiction.
 
 
+name: Build Python Wheel for Linux
+
 name: Build Linux
 
 on:
@@ -16,137 +18,36 @@ on:
   # Allow to be called from another workflow
   workflow_call: ~
 
+  # TODO automation trigger
   # repository_dispatch:
   #   types: [eccodes-updated]
 
-  push:
-    tags-ignore:
-    - '**'
-    paths:
-    - 'scripts/common.sh'
-    - 'scripts/select-python-linux.sh'
-    - 'scripts/wheel-linux.sh'
-    - 'scripts/build-linux.sh'
-    - 'scripts/test-linux.sh'
-    - 'scripts/copy-licences.py'
-    - '.github/workflows/build-wheel-linux.yml'
-
-# to allow the action to run on the manylinux docker image based on CentOS 7
-env:
-  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
-
 jobs:
-
   build:
-
-    # if: false  # for temporarily disabling for debugging
-
+    name: Build manylinux_2_28
     runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6]
-    container: dockcross/manylinux2014-x64:latest
-
-    name: Build manylinux2014
-
+    # TODO which manylinux do we want to build for? 2014? 2_28? 2_34? Matrix?
+    container:
+      image: eccr.ecmwf.int/wheelmaker/2_28:latest
+      credentials:
+        username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }}
+        password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }}
     steps:
-      - uses: actions/checkout@v2
-
-      - run: ./scripts/build-linux.sh
-
-      ################################################################
-      - run: ./scripts/wheel-linux.sh 3.8
-      - uses: actions/upload-artifact@v4
-        name: Upload wheel 3.8
-        with:
-          name: wheel-manylinux2014-3.8
-          path: wheelhouse/*.whl
-
-      # ################################################################
-      - run: ./scripts/wheel-linux.sh 3.9
-      - uses: actions/upload-artifact@v4
-        name: Upload wheel 3.9
-        with:
-          name: wheel-manylinux2014-3.9
-          path: wheelhouse/*.whl
-
-      # ################################################################
-      - run: ./scripts/wheel-linux.sh 3.10
-      - uses: actions/upload-artifact@v4
-        name: Upload wheel 3.10
-        with:
-          name: wheel-manylinux2014-3.10
-          path: wheelhouse/*.whl
-
-      # ################################################################
-      - run: ./scripts/wheel-linux.sh 3.11
-      - uses: actions/upload-artifact@v4
-        name: Upload wheel 3.11
-        with:
-          name: wheel-manylinux2014-3.11
-          path: wheelhouse/*.whl
-
-      # ################################################################
-      - run: ./scripts/wheel-linux.sh 3.12
-      - uses: actions/upload-artifact@v4
-        name: Upload wheel 3.12
-        with:
-          name: wheel-manylinux2014-3.12
-          path: wheelhouse/*.whl
-
-  test:
-
-    needs: build
-
-    strategy:
-      fail-fast: false
-      matrix: # We don't test 3.6, as it is not supported anymore by github actions
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
-
-    runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6]
-
-    name: Test with ${{ matrix.python-version }}
-
-    steps:
-
-      - uses: actions/checkout@v2
-
-      - uses: actions/download-artifact@v4
-        with:
-          name: wheel-manylinux2014-${{ matrix.python-version }}
-
-      - run: ./scripts/test-linux.sh  ${{ matrix.python-version }}
-
-
-  deploy:
-
-    if: ${{ github.ref_type == 'tag' || github.event_name == 'release' }}
-
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
-
-    needs: [test, build]
-
-    name: Deploy wheel ${{ matrix.python-version }}
-
-    runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6]
-
-
-    steps:
-
-      - run: mkdir artifact-${{ matrix.python-version }}
-
-      - uses: actions/checkout@v2
-
-      - uses: actions/download-artifact@v4
-        with:
-          name: wheel-manylinux2014-${{ matrix.python-version }}
-          path: artifact-${{ matrix.python-version }}
-
+        # TODO convert this to be matrix-friendly. Note it's a bit tricky since
+        # we'd ideally not reexecute the compile step multiple times, but it
+        # (non-essentially) depends on a matrix-based step
+        # NOTE we dont use action checkout because it doesnt cleanup after itself correctly
+      - run: git clone --depth=1 --branch="${GITHUB_REF#refs/heads/}" https://github.com/$GITHUB_REPOSITORY /proj
+      - run: cd /proj && /buildscripts/prepare_deps.sh ./buildconfig 3.11
       - run: |
-          source ./scripts/select-python-linux.sh 3.10
-          pip3 install twine
-          ls -l artifact-${{ matrix.python-version }}/*.whl
-          twine upload artifact-${{ matrix.python-version }}/*.whl
+           cd /proj
+           export PYTHONPATH=/buildscripts/; export LIBDIR=/tmp/prereqs/eccodeslib/lib64; export INCDIR=/tmp/prereqs/eccodeslib/include
+           uv run --python python3.11 python -m build --installer uv --wheel .
+      - run: mkdir -p /build/wheel && mv /proj/dist/*whl /build/wheel
+      - run: cd /proj && /buildscripts/test-wheel.sh ./python_wrapper/buildconfig 3.11 /build/wheel/*whl
+      - run: cd /proj && /buildscripts/upload-pypi.sh /build/wheel/*whl
         env:
           TWINE_USERNAME: __token__
-          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+        # NOTE temporary thing until all the mess gets cleared
+      - run: rm -rf ./* ./.git ./.github
diff --git a/MANIFEST.in b/MANIFEST.in
diff --git a/buildconfig b/buildconfig
@@ -0,0 +1,14 @@
+# (C) Copyright 2024- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+# NOTE since eccodes-python uses only parts of the Wheelmaker tooling, this file
+# is only correspondingly partial. Namely, `compile` and `wheel-linux` options
+# are not present, and only `prepare_deps` and `test-wheel` invocation is assumed
+
+DEPENDENCIES='["eccodeslib"]'
+NAME="eccodes"
diff --git a/gribapi/bindings.py b/gribapi/bindings.py
@@ -27,11 +27,6 @@
 
 LOG = logging.getLogger(__name__)
 
-_MAP = {
-    "grib_api": "eccodes",
-    "gribapi": "eccodes",
-}
-
 EXTENSIONS = {
     "darwin": ".dylib",
     "win32": ".dll",
@@ -43,16 +38,13 @@
     LOG.addHandler(logging.StreamHandler())
 
 
-def _lookup(name):
-    return _MAP.get(name, name)
-
-
-def find_binary_libs(name):
-    name = _lookup(name)
+def _find_eccodes_custom() -> str|None:
+    # TODO deprecate this method in favour of findlibs only
+    name = "eccodes"
     env_var = "ECCODES_PYTHON_USE_FINDLIBS"
     if int(os.environ.get(env_var, "0")):
         LOG.debug(f"{name} lib search: {env_var} set, so using findlibs")
-
+        return None
     else:
         LOG.debug(f"{name} lib search: trying to find binary wheel")
         here = os.path.dirname(__file__)
@@ -90,17 +82,14 @@ def find_binary_libs(name):
         LOG.debug(
             f"{name} lib search: did not find library from wheel; try to find as separate lib"
         )
+        return None
 
-    # if did not find the binary wheel, or the env var is set, fall back to findlibs
-    import findlibs
-
-    foundlib = findlibs.find(name)
-    LOG.debug(f"{name} lib search: findlibs returned {foundlib}")
-    return foundlib
-
-
-library_path = find_binary_libs("eccodes")
 
+library_path = _find_eccodes_custom()
+if library_path is None:
+    import findlibs
+    library_path = findlibs.find("eccodes")
+    LOG.debug(f"eccodes lib search: findlibs returned {library_path}")
 if library_path is None:
     raise RuntimeError("Cannot find the ecCodes library")
 

diff --git a/setup.cfg b/setup.cfg
@@ -1,3 +1,23 @@
+[metadata]
+description = "eccodes"
+long_description = file: README.rst
+long_description_content_type = text/rst
+author = "European Centre for Medium-Range Weather Forecasts (ECMWF)"
+author_email = "[email protected]"
+url = "https://github.com/ecmwf/eccodes-python"
+keywords = ecCodes, GRIB, BUFR
+classifiers = 
+    Development Status :: 4 - Beta
+    Intended Audience :: Developers
+    License :: OSI Approved :: Apache Software License
+    Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.11
+    Programming Language :: Python :: Implementation :: CPython
+    Programming Language :: Python :: Implementation :: PyPy
+    Operating System :: OS Independent
+
 [aliases]
 test = pytest