From 7b9b9be131af4486be0a7f9a5641a5958aa84165 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Wed, 20 Nov 2024 08:56:51 -0800 Subject: [PATCH] [DevOps]: CDAT Migration: Replace `setup.py` with `pyproject.toml` for modern Python packaging (#895) --- .github/workflows/build_workflow.yml | 6 +- .pre-commit-config.yaml | 2 +- conda-env/ci.yml | 2 +- conda-env/dev.yml | 2 +- pyproject.toml | 154 +++++++++++++++++++++++- setup.py | 174 --------------------------- tbump.toml | 3 - 7 files changed, 159 insertions(+), 184 deletions(-) delete mode 100644 setup.py diff --git a/.github/workflows/build_workflow.yml b/.github/workflows/build_workflow.yml index 5fe6d13f7..da1ac3b16 100644 --- a/.github/workflows/build_workflow.yml +++ b/.github/workflows/build_workflow.yml @@ -29,10 +29,10 @@ jobs: uses: actions/checkout@v3 - if: ${{ steps.skip_check.outputs.should_skip != 'true' }} - name: Set up Python 3.10 + name: Set up Python 3.11 uses: actions/setup-python@v3 with: - python-version: "3.10" + python-version: "3.11" - if: ${{ steps.skip_check.outputs.should_skip != 'true' }} # Run all pre-commit hooks on all the files. @@ -50,7 +50,7 @@ jobs: shell: bash -l {0} strategy: matrix: - python-version: ["3.9", "3.10"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - id: skip_check uses: fkirc/skip-duplicate-actions@master diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1cfd94353..f4ec0222d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ exclude: "docs|node_modules|migrations|.git|.tox|examples|analysis_data_preprocess|auxiliary_tools|conda/meta.yaml|e3sm_diags/driver/utils/zwf_functions.py" -default_stages: [commit] +default_stages: [pre-commit] fail_fast: true repos: diff --git a/conda-env/ci.yml b/conda-env/ci.yml index 29a0ec6b8..54a42cbae 100644 --- a/conda-env/ci.yml +++ b/conda-env/ci.yml @@ -18,7 +18,7 @@ dependencies: - esmpy >=8.4.0 - lxml - mache >=0.15.0 - - matplotlib-base + - matplotlib-base >=3.8.2 - netcdf4 - output_viewer >=1.3.0 - numpy >=2.0.0,<3.0.0 diff --git a/conda-env/dev.yml b/conda-env/dev.yml index ec0e1ec68..a5bec13be 100644 --- a/conda-env/dev.yml +++ b/conda-env/dev.yml @@ -16,7 +16,7 @@ dependencies: - esmpy >=8.4.0 - lxml - mache >=0.15.0 - - matplotlib-base + - matplotlib-base >=3.8.2 - netcdf4 - output_viewer >=1.3.0 - numpy >=2.0.0,<3.0.0 diff --git a/pyproject.toml b/pyproject.toml index 09d466ab5..26a0803df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,155 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "e3sm_diags" +dynamic = ["version"] +description = "E3SM Diagnostics" +authors = [ + { name = "Chengzhu (Jill) Zhang", email = "zhang40@llnl.gov" }, + { name = "Tom Vo" }, + { name = "Ryan Forsyth" }, + { name = "Chris Golaz" }, + { name = "Zeshawn Shaheen" }, +] +license = { text = "BSD 3-Clause" } +readme = "README.md" +requires-python = ">=3.9" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD 3-Clause License", + "Natural Language :: English", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +dependencies = [ + "beautifulsoup4", + "cartopy >=0.17.0", + # This package is not available on PyPI. + # "cartopy_offlinedata", + "cf-units", + "dask", + "esmpy >=8.4.0", + "lxml", + "mache >=0.15.0", + "matplotlib >=3.8.2", + "netcdf4", + "output_viewer >=1.3.0", + "numpy >=2.0.0,<3.0.0", + "shapely >=2.0.0,<3.0.0", + "xarray >=2024.03.0", + "xcdat >=0.7.3,<1.0.0", + "xesmf >=0.8.7", + "xskillscore >=0.0.20", +] + +[project.optional-dependencies] +test = ["pytest", "pytest-cov"] +docs = ["sphinx", "sphinx_rtd_theme", "sphinx-multiversion"] +dev = [ + "black", + "flake8", + "flake8-isort", + "isort", + "mypy", + "pre-commit", + "types-PyYAML", +] + +[project.urls] +Documentation = "https://docs.e3sm.org/e3sm_diags/_build/html/main/index.html" +"Issue Tracker" = "https://github.com/E3SM-Project/e3sm_diags/issues" + +[project.scripts] +e3sm_diags = "e3sm_diags.e3sm_diags_driver:main" +e3sm_diags_vars = "e3sm_diags.e3sm_diags_vars:main" + +[tool.setuptools.packages.find] +include = ["e3sm_diags", "e3sm_diags.*"] + +[tool.setuptools.dynamic] +version = { attr = "e3sm_diags.__version__" } + +[tool.setuptools.data-files] +"share/e3sm_diags" = [ + "e3sm_diags/driver/acme_ne30_ocean_land_mask.nc", + "misc/e3sm_logo.png", +] + +"share/e3sm_diags/viewer" = ["e3sm_diags/viewer/index_template.html"] + +"share/e3sm_diags/zonal_mean_xy" = [ + "e3sm_diags/driver/default_diags/zonal_mean_xy*", + "e3sm_diags/driver/default_diags/legacy_diags/zonal_mean_xy*", +] +"share/e3sm_diags/zonal_mean_2d" = [ + "e3sm_diags/driver/default_diags/zonal_mean_2d_model*", + "e3sm_diags/driver/default_diags/legacy_diags/zonal_mean_2d*", +] +"share/e3sm_diags/zonal_mean_2d_stratosphere" = [ + "e3sm_diags/driver/default_diags/zonal_mean_2d_stratosphere*", +] +"share/e3sm_diags/meridional_mean_2d" = [ + "e3sm_diags/driver/default_diags/meridional_mean_2d*", +] +"share/e3sm_diags/lat_lon" = [ + "e3sm_diags/driver/default_diags/lat_lon*", + "e3sm_diags/driver/default_diags/legacy_diags/lat_lon*", +] +"share/e3sm_diags/polar" = [ + "e3sm_diags/driver/default_diags/polar*", + "e3sm_diags/driver/default_diags/legacy_diags/polar*", +] +"share/e3sm_diags/lat_lon_vector" = [ + "e3sm_diags/driver/default_diags/lat_lon_vector*", +] +"share/e3sm_diags/lat_lon_land" = [ + "e3sm_diags/driver/default_diags/lat_lon_land*", +] +"share/e3sm_diags/lat_lon_river" = [ + "e3sm_diags/driver/default_diags/lat_lon_river*", +] +"share/e3sm_diags/cosp_histogram" = [ + "e3sm_diags/driver/default_diags/cosp_histogram*", + "e3sm_diags/driver/default_diags/legacy_diags/cosp_histogram*", +] +"share/e3sm_diags/area_mean_time_series" = [ + "e3sm_diags/driver/default_diags/area_mean_time_series*", +] +"share/e3sm_diags/enso_diags" = ["e3sm_diags/driver/default_diags/enso_*"] +"share/e3sm_diags/qbo" = ["e3sm_diags/driver/default_diags/qbo*"] +"share/e3sm_diags/streamflow" = ["e3sm_diags/driver/default_diags/streamflow*"] +"share/e3sm_diags/diurnal_cycle" = [ + "e3sm_diags/driver/default_diags/diurnal_cycle_*", +] +"share/e3sm_diags/arm_diags" = ["e3sm_diags/driver/default_diags/arm_diags_*"] +"share/e3sm_diags/tc_analysis" = [ + "e3sm_diags/driver/default_diags/tc_analysis_*", +] +"share/e3sm_diags/annual_cycle_zonal_mean" = [ + "e3sm_diags/driver/default_diags/annual_cycle_zonal_mean_*", +] +"share/e3sm_diags/aerosol_aeronet" = [ + "e3sm_diags/driver/default_diags/aerosol_aeronet*cfg", + "e3sm_diags/driver/default_diags/aerosol_aeronet_data/*.txt", +] +"share/e3sm_diags/aerosol_budget" = [ + "e3sm_diags/driver/default_diags/aerosol_budget*cfg", +] +"share/e3sm_diags/mp_partition" = [ + "e3sm_diags/driver/default_diags/mp_partition*cfg", +] +"share/e3sm_diags/tropical_subseasonal" = [ + "e3sm_diags/driver/default_diags/tropical_subseasonal*cfg", +] +"share/e3sm_diags/colormaps" = ["e3sm_diags/plot/colormaps/*.rgb"] +"share/e3sm_diags/control_runs" = ["e3sm_diags/driver/control_runs/*"] + [tool.black] # Docs: https://black.readthedocs.io/en/stable/usage_and_configuration/the_basics.html line-length = 88 @@ -20,7 +172,7 @@ exclude = ''' | config | conda | analysis_data_preprocess - )/ +)/ ''' [tool.isort] diff --git a/setup.py b/setup.py deleted file mode 100644 index bbd683fd5..000000000 --- a/setup.py +++ /dev/null @@ -1,174 +0,0 @@ -import glob -import os - -from setuptools import find_packages, setup - - -def get_all_files_in_dir(directory, pattern): - return glob.glob(os.path.join(directory, pattern)) - - -zonal_mean_xy_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "zonal_mean_xy*" -) -zonal_mean_xy_files += get_all_files_in_dir( - "e3sm_diags/driver/default_diags/legacy_diags", "zonal_mean_xy*" -) - -zonal_mean_2d_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "zonal_mean_2d_model*" -) -zonal_mean_2d_files += get_all_files_in_dir( - "e3sm_diags/driver/default_diags/legacy_diags", "zonal_mean_2d*" -) -zonal_mean_2d_stratosphere_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "zonal_mean_2d_stratosphere*" -) - -meridional_mean_2d_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "meridional_mean_2d*" -) - -lat_lon_files = get_all_files_in_dir("e3sm_diags/driver/default_diags", "lat_lon*") -lat_lon_files += get_all_files_in_dir( - "e3sm_diags/driver/default_diags/legacy_diags", "lat_lon*" -) - -lat_lon_vector_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "lat_lon_vector*" -) - -lat_lon_land_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "lat_lon_land*" -) -lat_lon_river_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "lat_lon_river*" -) - -polar_files = get_all_files_in_dir("e3sm_diags/driver/default_diags", "polar*") -polar_files += get_all_files_in_dir( - "e3sm_diags/driver/default_diags/legacy_diags", "polar*" -) - -cosp_histogram_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "cosp_histogram*" -) -cosp_histogram_files += get_all_files_in_dir( - "e3sm_diags/driver/default_diags/legacy_diags", "cosp_histogram*" -) - -area_mean_time_series = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "area_mean_time_series*" -) -qbo = get_all_files_in_dir("e3sm_diags/driver/default_diags", "qbo*") -streamflow = get_all_files_in_dir("e3sm_diags/driver/default_diags", "streamflow*") -enso_diags_files = get_all_files_in_dir("e3sm_diags/driver/default_diags", "enso_*") -diurnal_cycle_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "diurnal_cycle_*" -) -arm_diags_files = get_all_files_in_dir("e3sm_diags/driver/default_diags", "arm_diags_*") -tc_analysis_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "tc_analysis_*" -) -annual_cycle_zonal_mean_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "annual_cycle_zonal_mean_*" -) -aerosol_aeronet_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "aerosol_aeronet*cfg" -) -aerosol_aeronet_files += get_all_files_in_dir( - "e3sm_diags/driver/default_diags/aerosol_aeronet_data", "*.txt" -) -aerosol_budget_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "aerosol_budget*cfg" -) -mp_partition_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "mp_partition*cfg" -) -tropical_subseasonal_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "tropical_subseasonal*cfg" -) - -rgb_files = get_all_files_in_dir("e3sm_diags/plot/colormaps", "*.rgb") -control_runs_files = get_all_files_in_dir("e3sm_diags/driver/control_runs", "*") - -INSTALL_PATH = "share/e3sm_diags/" - -data_files = [ - (os.path.join(INSTALL_PATH, "zonal_mean_xy"), zonal_mean_xy_files), - (os.path.join(INSTALL_PATH, "zonal_mean_2d"), zonal_mean_2d_files), - ( - os.path.join(INSTALL_PATH, "zonal_mean_2d_stratosphere"), - zonal_mean_2d_stratosphere_files, - ), - ( - os.path.join(INSTALL_PATH, "meridional_mean_2d"), - meridional_mean_2d_files, - ), - (os.path.join(INSTALL_PATH, "lat_lon"), lat_lon_files), - (os.path.join(INSTALL_PATH, "polar"), polar_files), - (os.path.join(INSTALL_PATH, "lat_lon_vector"), lat_lon_vector_files), - (os.path.join(INSTALL_PATH, "lat_lon_land"), lat_lon_land_files), - (os.path.join(INSTALL_PATH, "lat_lon_river"), lat_lon_river_files), - (os.path.join(INSTALL_PATH, "cosp_histogram"), cosp_histogram_files), - ( - os.path.join(INSTALL_PATH, "area_mean_time_series"), - area_mean_time_series, - ), - (os.path.join(INSTALL_PATH, "enso_diags"), enso_diags_files), - (os.path.join(INSTALL_PATH, "qbo"), qbo), - (os.path.join(INSTALL_PATH, "streamflow"), streamflow), - (os.path.join(INSTALL_PATH, "diurnal_cycle"), diurnal_cycle_files), - (os.path.join(INSTALL_PATH, "arm_diags"), arm_diags_files), - (os.path.join(INSTALL_PATH, "tc_analysis"), tc_analysis_files), - ( - os.path.join(INSTALL_PATH, "annual_cycle_zonal_mean"), - annual_cycle_zonal_mean_files, - ), - ( - INSTALL_PATH, - [ - "e3sm_diags/driver/acme_ne30_ocean_land_mask.nc", - "misc/e3sm_logo.png", - ], - ), - (os.path.join(INSTALL_PATH, "aerosol_aeronet"), aerosol_aeronet_files), - (os.path.join(INSTALL_PATH, "aerosol_budget"), aerosol_budget_files), - (os.path.join(INSTALL_PATH, "mp_partition"), mp_partition_files), - (os.path.join(INSTALL_PATH, "tropical_subseasonal"), tropical_subseasonal_files), - (os.path.join(INSTALL_PATH, "colormaps"), rgb_files), - (os.path.join(INSTALL_PATH, "control_runs"), control_runs_files), - ( - os.path.join(INSTALL_PATH, "viewer"), - ["e3sm_diags/viewer/index_template.html"], - ), -] - -setup( - python_requires=">=3.9", - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "License :: OSI Approved :: BSD 3-Clause License", - "Natural Language :: English", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - ], - name="e3sm_diags", - version="2.12.0", - author="Chengzhu (Jill) Zhang, Tom Vo, Ryan Forsyth, Chris Golaz and Zeshawn Shaheen", - author_email="zhang40@llnl.gov", - description="E3SM Diagnostics", - license="BSD 3-Clause", - scripts=["e3sm_diags/e3sm_diags_driver.py"], - packages=find_packages(include=["e3sm_diags", "e3sm_diags.*"]), - data_files=data_files, - test_suite="tests", - entry_points={ - "console_scripts": [ - "e3sm_diags=e3sm_diags.e3sm_diags_driver:main", - "e3sm_diags_vars=e3sm_diags.e3sm_diags_vars:main", - ] - }, -) diff --git a/tbump.toml b/tbump.toml index 095774281..680e4ac2a 100644 --- a/tbump.toml +++ b/tbump.toml @@ -22,9 +22,6 @@ tag_template = "v{new_version}" # For each file to patch, add a [[file]] config # section containing the path of the file, relative to the # tbump.toml location. -[[file]] -src = "setup.py" - [[file]] src = "e3sm_diags/__init__.py"