diff --git a/.github/workflows/build-wheel-linux.yml b/.github/workflows/build-wheel-linux.yml index 500bf3c..137e485 100644 --- a/.github/workflows/build-wheel-linux.yml +++ b/.github/workflows/build-wheel-linux.yml @@ -7,6 +7,8 @@ # nor does it submit to any jurisdiction. +name: Build Python Wheel for Linux + name: Build Linux on: @@ -16,137 +18,36 @@ on: # Allow to be called from another workflow workflow_call: ~ + # TODO automation trigger # repository_dispatch: # types: [eccodes-updated] - push: - tags-ignore: - - '**' - paths: - - 'scripts/common.sh' - - 'scripts/select-python-linux.sh' - - 'scripts/wheel-linux.sh' - - 'scripts/build-linux.sh' - - 'scripts/test-linux.sh' - - 'scripts/copy-licences.py' - - '.github/workflows/build-wheel-linux.yml' - -# to allow the action to run on the manylinux docker image based on CentOS 7 -env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - jobs: - build: - - # if: false # for temporarily disabling for debugging - + name: Build manylinux_2_28 runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] - container: dockcross/manylinux2014-x64:latest - - name: Build manylinux2014 - + # TODO which manylinux do we want to build for? 2014? 2_28? 2_34? Matrix? + container: + image: eccr.ecmwf.int/wheelmaker/2_28:latest + credentials: + username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} + password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} steps: - - uses: actions/checkout@v2 - - - run: ./scripts/build-linux.sh - - ################################################################ - - run: ./scripts/wheel-linux.sh 3.8 - - uses: actions/upload-artifact@v4 - name: Upload wheel 3.8 - with: - name: wheel-manylinux2014-3.8 - path: wheelhouse/*.whl - - # ################################################################ - - run: ./scripts/wheel-linux.sh 3.9 - - uses: actions/upload-artifact@v4 - name: Upload wheel 3.9 - with: - name: wheel-manylinux2014-3.9 - path: wheelhouse/*.whl - - # ################################################################ - - run: ./scripts/wheel-linux.sh 3.10 - - uses: actions/upload-artifact@v4 - name: Upload wheel 3.10 - with: - name: wheel-manylinux2014-3.10 - path: wheelhouse/*.whl - - # ################################################################ - - run: ./scripts/wheel-linux.sh 3.11 - - uses: actions/upload-artifact@v4 - name: Upload wheel 3.11 - with: - name: wheel-manylinux2014-3.11 - path: wheelhouse/*.whl - - # ################################################################ - - run: ./scripts/wheel-linux.sh 3.12 - - uses: actions/upload-artifact@v4 - name: Upload wheel 3.12 - with: - name: wheel-manylinux2014-3.12 - path: wheelhouse/*.whl - - test: - - needs: build - - strategy: - fail-fast: false - matrix: # We don't test 3.6, as it is not supported anymore by github actions - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - - runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] - - name: Test with ${{ matrix.python-version }} - - steps: - - - uses: actions/checkout@v2 - - - uses: actions/download-artifact@v4 - with: - name: wheel-manylinux2014-${{ matrix.python-version }} - - - run: ./scripts/test-linux.sh ${{ matrix.python-version }} - - - deploy: - - if: ${{ github.ref_type == 'tag' || github.event_name == 'release' }} - - strategy: - fail-fast: false - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - - needs: [test, build] - - name: Deploy wheel ${{ matrix.python-version }} - - runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] - - - steps: - - - run: mkdir artifact-${{ matrix.python-version }} - - - uses: actions/checkout@v2 - - - uses: actions/download-artifact@v4 - with: - name: wheel-manylinux2014-${{ matrix.python-version }} - path: artifact-${{ matrix.python-version }} - + # TODO convert this to be matrix-friendly. Note it's a bit tricky since + # we'd ideally not reexecute the compile step multiple times, but it + # (non-essentially) depends on a matrix-based step + # NOTE we dont use action checkout because it doesnt cleanup after itself correctly + - run: git clone --depth=1 --branch="${GITHUB_REF#refs/heads/}" https://github.com/$GITHUB_REPOSITORY /proj + - run: cd /proj && /buildscripts/prepare_deps.sh ./buildconfig 3.11 - run: | - source ./scripts/select-python-linux.sh 3.10 - pip3 install twine - ls -l artifact-${{ matrix.python-version }}/*.whl - twine upload artifact-${{ matrix.python-version }}/*.whl + cd /proj + export PYTHONPATH=/buildscripts/; export LIBDIR=/tmp/prereqs/eccodeslib/lib64; export INCDIR=/tmp/prereqs/eccodeslib/include + uv run --python python3.11 python -m build --installer uv --wheel . + - run: mkdir -p /build/wheel && mv /proj/dist/*whl /build/wheel + - run: cd /proj && /buildscripts/test-wheel.sh ./python_wrapper/buildconfig 3.11 /build/wheel/*whl + - run: cd /proj && /buildscripts/upload-pypi.sh /build/wheel/*whl env: TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} \ No newline at end of file + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + # NOTE temporary thing until all the mess gets cleared + - run: rm -rf ./* ./.git ./.github diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index ebd3b9e..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,20 +0,0 @@ -include .dockerignore -include *.rst -include *.yml -include Dockerfile -include LICENSE -include Makefile -include tox.ini -include *.py -recursive-include ci *.in -recursive-include ci *.txt -recursive-include ci *.yml -recursive-include ci *.ps1 -recursive-include docs *.gitkeep -recursive-include docs *.py -recursive-include docs *.rst -recursive-include gribapi *.h -recursive-include tests *.grib2 -recursive-include tests *.grib -recursive-include tests *.ipynb -recursive-include tests *.py diff --git a/buildconfig b/buildconfig new file mode 100644 index 0000000..7ad99f6 --- /dev/null +++ b/buildconfig @@ -0,0 +1,14 @@ +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +# NOTE since eccodes-python uses only parts of the Wheelmaker tooling, this file +# is only correspondingly partial. Namely, `compile` and `wheel-linux` options +# are not present, and only `prepare_deps` and `test-wheel` invocation is assumed + +DEPENDENCIES='["eccodeslib"]' +NAME="eccodes" diff --git a/gribapi/bindings.py b/gribapi/bindings.py index 7b86e10..4f5301e 100644 --- a/gribapi/bindings.py +++ b/gribapi/bindings.py @@ -27,11 +27,6 @@ LOG = logging.getLogger(__name__) -_MAP = { - "grib_api": "eccodes", - "gribapi": "eccodes", -} - EXTENSIONS = { "darwin": ".dylib", "win32": ".dll", @@ -43,16 +38,13 @@ LOG.addHandler(logging.StreamHandler()) -def _lookup(name): - return _MAP.get(name, name) - - -def find_binary_libs(name): - name = _lookup(name) +def _find_eccodes_custom() -> str|None: + # TODO deprecate this method in favour of findlibs only + name = "eccodes" env_var = "ECCODES_PYTHON_USE_FINDLIBS" if int(os.environ.get(env_var, "0")): LOG.debug(f"{name} lib search: {env_var} set, so using findlibs") - + return None else: LOG.debug(f"{name} lib search: trying to find binary wheel") here = os.path.dirname(__file__) @@ -90,17 +82,14 @@ def find_binary_libs(name): LOG.debug( f"{name} lib search: did not find library from wheel; try to find as separate lib" ) + return None - # if did not find the binary wheel, or the env var is set, fall back to findlibs - import findlibs - - foundlib = findlibs.find(name) - LOG.debug(f"{name} lib search: findlibs returned {foundlib}") - return foundlib - - -library_path = find_binary_libs("eccodes") +library_path = _find_eccodes_custom() +if library_path is None: + import findlibs + library_path = findlibs.find("eccodes") + LOG.debug(f"eccodes lib search: findlibs returned {library_path}") if library_path is None: raise RuntimeError("Cannot find the ecCodes library") diff --git a/setup.cfg b/setup.cfg index 2bd557e..c8cf29b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,23 @@ +[metadata] +description = "eccodes" +long_description = file: README.rst +long_description_content_type = text/rst +author = "European Centre for Medium-Range Weather Forecasts (ECMWF)" +author_email = "software.support@ecmwf.int" +url = "https://github.com/ecmwf/eccodes-python" +keywords = ecCodes, GRIB, BUFR +classifiers = + Development Status :: 4 - Beta + Intended Audience :: Developers + License :: OSI Approved :: Apache Software License + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Programming Language :: Python :: Implementation :: CPython + Programming Language :: Python :: Implementation :: PyPy + Operating System :: OS Independent + [aliases] test = pytest diff --git a/setup.py b/setup.py index 5a435e2..924cd8e 100644 --- a/setup.py +++ b/setup.py @@ -1,125 +1,69 @@ -#!/usr/bin/env python -# -# (C) Copyright 2017- ECMWF. +# (C) Copyright 2024- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# # In applying this licence, ECMWF does not waive the privileges and immunities -# granted to it by virtue of its status as an intergovernmental organisation nor -# does it submit to any jurisdiction. -# +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +# TODO ideally merge this with wheelmaker's setup_utils somehow import io import os import re import sys - import setuptools +from setup_utils import parse_dependencies, ext_kwargs -def read(path): - file_path = os.path.join(os.path.dirname(__file__), *path.split("/")) - return io.open(file_path, encoding="utf-8").read() +if sys.version_info < (3, 7): + install_requires = ["numpy<1.20"] +elif sys.version_info < (3, 8): + install_requires = ["numpy<1.22"] +elif sys.version_info < (3, 9): + install_requires = ["numpy<1.25"] +else: + install_requires = ["numpy"] +install_requires += ["attrs", "cffi", "findlibs"] +ext_modules = [ + setuptools.Extension( + "eccodes._eccodes", + sources=["eccodes/_eccodes.cc"], + language="c++", + libraries=["eccodes"], + library_dirs=[os.environ["LIBDIR"]], + include_dirs=[os.environ["INCDIR"]], + ) +] -# single-sourcing the package version using method 1 of: -# https://packaging.python.org/guides/single-sourcing-package-version/ -def parse_version_from(path): +def get_version() -> str: version_pattern = ( r"^__version__ = [\"\'](.*)[\"\']" # More permissive regex pattern ) - version_file = read(path) + file_path = os.path.join(os.path.dirname(__file__), "gribapi", "bindings.py") + version_file = io.open(file_path, encoding="utf-8").read() version_match = re.search(version_pattern, version_file, re.M) if version_match is None or len(version_match.groups()) > 1: raise ValueError("couldn't parse version") return version_match.group(1) - -# for the binary wheel -libdir = os.path.realpath("install/lib") -incdir = os.path.realpath("install/include") -libs = ["eccodes"] - -if "--binary-wheel" in sys.argv: - sys.argv.remove("--binary-wheel") - - # https://setuptools.pypa.io/en/latest/userguide/ext_modules.html - ext_modules = [ - setuptools.Extension( - "eccodes._eccodes", - sources=["eccodes/_eccodes.cc"], - language="c++", - libraries=libs, - library_dirs=[libdir], - include_dirs=[incdir], - extra_link_args=["-Wl,-rpath," + libdir], - ) - ] - - def shared(directory): - result = [] - for path, dirs, files in os.walk(directory): - for f in files: - result.append(os.path.join(path, f)) - return result - - # Paths must be relative to package directory... - shared_files = ["versions.txt"] - shared_files += [x[len("eccodes/") :] for x in shared("eccodes/copying")] - - if os.name == "nt": - for n in os.listdir("eccodes"): - if n.endswith(".dll"): - shared_files.append(n) - -else: - ext_modules = [] - shared_files = [] - - -install_requires = ["numpy"] -if sys.version_info < (3, 7): - install_requires = ["numpy<1.20"] -elif sys.version_info < (3, 8): - install_requires = ["numpy<1.22"] -elif sys.version_info < (3, 9): - install_requires = ["numpy<1.25"] - -install_requires += ["attrs", "cffi", "findlibs"] - setuptools.setup( name="eccodes", - version=parse_version_from("gribapi/bindings.py"), - description="Python interface to the ecCodes GRIB and BUFR decoder/encoder", - long_description=read("README.rst") + read("CHANGELOG.rst"), - author="European Centre for Medium-Range Weather Forecasts (ECMWF)", - author_email="software.support@ecmwf.int", - license="Apache License Version 2.0", - url="https://github.com/ecmwf/eccodes-python", + version=get_version(), packages=setuptools.find_packages(), - include_package_data=True, - package_data={"": shared_files}, - install_requires=install_requires, + package_data={"": ["**/*.h"]}, + install_requires=parse_dependencies() + install_requires, + **ext_kwargs[sys.platform], + # NOTE what is this? Setuptools 75.6.0 doesnt recognize. Move to extras? tests_require=[ "pytest", "pytest-cov", "pytest-flakes", ], + # NOTE what is this? Setuptools 75.6.0 doesnt recognize. + tests_require=[ test_suite="tests", zip_safe=True, - keywords="ecCodes GRIB BUFR", - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Operating System :: OS Independent", - ], ext_modules=ext_modules, )