diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4c1ac5208..f86079c00 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,33 +4,27 @@ fail_fast: true repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v5.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml - - repo: https://github.com/psf/black - rev: 23.9.1 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.9.2 hooks: - - id: black - - - repo: https://github.com/PyCQA/isort - rev: 5.12.0 - hooks: - - id: isort - - # Need to use flake8 GitHub mirror due to CentOS git issue with GitLab - # https://github.com/pre-commit/pre-commit/issues/1206 - - repo: https://github.com/pycqa/flake8 - rev: 6.1.0 - hooks: - - id: flake8 - args: [--config=setup.cfg] - additional_dependencies: [flake8-isort==6.1.0] + # Sort the imports + - id: ruff + name: ruff-sort-imports + args: [--select, I, --fix] + # Run the linter. + - id: ruff + args: [--fix] + # Run the formatter. + - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.5.1 + rev: v1.14.1 hooks: - id: mypy args: [--config=pyproject.toml] diff --git a/.vscode/e3sm_diags.code-workspace b/.vscode/e3sm_diags.code-workspace index 7cf614852..5ddbb4a2e 100644 --- a/.vscode/e3sm_diags.code-workspace +++ b/.vscode/e3sm_diags.code-workspace @@ -25,11 +25,9 @@ "editor.rulers": [80, 88, 120], "editor.wordWrap": "wordWrapColumn", "editor.wordWrapColumn": 120, - "editor.defaultFormatter": "ms-python.black-formatter" + "editor.defaultFormatter": "charliermarsh.ruff" }, - // Code Formatting and Linting - // --------------------------- - "flake8.args": ["--config=setup.cfg"], + "ruff.importStrategy": "fromEnvironment", // Type checking // --------------------------- "mypy-type-checker.args": ["--config=pyproject.toml"], diff --git a/Makefile b/Makefile index 9736f7704..0ea712d9e 100644 --- a/Makefile +++ b/Makefile @@ -68,8 +68,12 @@ clean-test-int-data: # remove integration test data and images (expected) -- us pre-commit: # run pre-commit quality assurance checks pre-commit run --all-files -lint: ## check style with flake8 - flake8 e3sm_diags tests +lint: ## check style ruff + ruff check --select I --fix + ruff check --fix + +format: ## format code using ruff + ruff format test: ## run tests quickly with the default Python and produces code coverage report pytest diff --git a/README.md b/README.md index 8907bf652..d8066d55e 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,13 @@ # E3SM Diagnostics Package -[![Anaconda Version](https://anaconda.org/conda-forge/e3sm_diags/badges/version.svg)](https://anaconda.org/e3sm/e3sm_diags) -[![Anaconda Downloads](https://anaconda.org/conda-forge/e3sm_diags/badges/downloads.svg)](https://anaconda.org/e3sm/e3sm_diags) +[![Anaconda Version](https://anaconda.org/conda-forge/e3sm_diags/badges/version.svg)](https://anaconda.org/conda-forge/e3sm_diags) +[![Anaconda Downloads](https://anaconda.org/conda-forge/e3sm_diags/badges/downloads.svg)](https://anaconda.org/conda-forge/e3sm_diags) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.1036779.svg)](https://doi.org/10.5281/zenodo.1036779) [![CI/CD Build Workflow](https://github.com/E3SM-Project/e3sm_diags/actions/workflows/build_workflow.yml/badge.svg)](https://github.com/E3SM-Project/e3sm_diags/actions/workflows/build_workflow.yml) [![CI/CD Release Workflow](https://github.com/E3SM-Project/e3sm_diags/actions/workflows/release_workflow.yml/badge.svg)](https://github.com/E3SM-Project/e3sm_diags/actions/workflows/release_workflow.yml) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) -[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) -[![flake8](https://img.shields.io/badge/flake8-enabled-green)](https://github.com/PyCQA/flake8) +[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) [![Checked with mypy](http://www.mypy-lang.org/static/mypy_badge.svg)](http://mypy-lang.org/) ## Table of Contents @@ -48,26 +47,28 @@ This diagnostics package is constructed for supporting the diagnostics task of D E3SM Diags is modeled after the National Center for Atmospheric Research (NCAR) Atmosphere Model Working Group (AMWG) diagnostics package. In its version 1 release, E3SM Diags included a set of core essential diagnostics to evaluate the mean physical climate from model simulations. As of version 2, more process-oriented and phenomenon-based evaluation diagnostics have been implemented, as listed below: ### New Feature added during v2 development -| Feature name
(set name) | Brief Introduction | Developers Contributors* | Released version | -|--- | ---------------- | --- | --- | -| Wavenumber frequency analysis (tropical_subseasonal) | The wavenumber frequency analysis (Wheeler-Kiladis Diagram) for Tropical subseasonal analysis | Jim Benedict, Brian Medeiros, Jill Zhang, Tom Vo | 2.12.0 -| T5050 diagnostic /Mixed phase partition (mp_partition) | Temperature at which cloud top is 50% ice, 50% liquid, following McCoy et al. (2015). | Yuying Zhang, Jill Zhang, Jiwen Fan, Yunpeng Shan | 2.9.0 | -| ARM diagnostics v3 (arm_diags) | Enhanced ARM diagnostics with new aerosol-cloud-interaction and aerosol activation metrics | Xiaojian Zheng, Jill Zhang, Cheng Tao, Shaocheng Xie | 2.9.0 | -| Aerosol budget diagnostics (aerosol_budget) | Global mean burdens, source/sink budgets and lifetimes for aerosol species. | Jill Zhang, Susannah Burrows, Naser Mahfouz, Tom Vo, Kai Zhang, Taufiq Hassan Mozumder, Xue Zheng, Ziming Ke, Yan Feng, Manish Shrivastava, Hailong Wang, Mingxuan Wu | 2.8.0 | -| Aerosol evaluation with AERONET data (aerosol_aeronet) | Scatter plots for comparing annual mean long-term Absorption Optical Depth (AOD) Aerosol Absorption Optical Depth (AAOD) at 550 nm from AERONET measurements. | Jill Zhang, Yan Feng | 2.7.0 | -| CMIP6 comparison (with lat_lon) | Box whisker plot to compare with key fields from CMIP6 models | Chris Golaz, Jill Zhang | 2.7.0 | -| Zonal mean 2D over stratosphere (zonal_mean
_2d_stratosphere) | Pressure–latitude zonal-mean contour plots of seasonal mean | Jill Zhang | 2.6.0 | -| TC Analysis (tc_analysis) | Bar plots for TC frequency and accumulated cyclone energy distributed by ocean basin; line plots for TC intensity and seasonality for each ocean basin; maps for TC density and African easterly waves | Karthik Balaguru, Jill Zhang, Paul Ullrich | 2.5.0 | -| Annual cycle zonal mean (annual
_cycle_zonal_
mean) | Latitude zonal-mean–month box plot (including Stratospheric and Tropospheric column Ozone) | Jill Zhang, Qi Tang | 2.5.0 | -| ARM diagnostics v2 (arm_diags) | Seasonal mean, annual cycle, diurnal cycle and convection onset metrics over DOE's ARM ground-based facilities | Jill Zhang, Tao Chen, Shaocheng Xie, Todd Emmenegger, David Neelin, Kathleen Schiro, Yi-Hung Kuo | 2.4.0 | -| Streamflow diags (streamflow) | Seasonality map, annual-mean map/scatterplots based on gauge station data from GSIM | Ryan Forsyth, Tian Zhou | 2.3.0 | -| Diurnal cycle (diurnal_cycle) | Amplitude and phase map of the seasonal-mean diurnal cycle of precipitation compare with TRMM 3hourly data. | Jill Zhang, Charlie Zender, Wuyin Lin, Shaocheng Xie | 2.3.0 | -| Quasi-biennial oscillation analysis (qbo) | Quasi-biennial oscillation (QBO) analysis between 5S and 5N , including monthly mean zonal-mean zonal wind as a function of pressure and time and the power spectrum and amplitude | Chris Terai, Jack Chen, Jadwiga (Yaga) Richter, Sasha Glanville, Chris Golaz | 2.1.0 | -| ENSO diagnostics (enso_diags) | Maps of the regression coefficient of atmospheric fields over the SST anomaly, scatterplots of atmospheric feedback on the SST anomaly | Ryan Forsyth, Salil Mahajan | 2.1.0 | - -*: The code infrastructure and maintenance are supported by the core developer group including: Jill Zhang, Chris Golaz, Tom Vo, Ryan Forsyth, Xylar Asay-Davis. + +| Feature name
(set name) | Brief Introduction | Developers Contributors\* | Released version | +| ------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | +| Wavenumber frequency analysis (tropical_subseasonal) | The wavenumber frequency analysis (Wheeler-Kiladis Diagram) for Tropical subseasonal analysis | Jim Benedict, Brian Medeiros, Jill Zhang, Tom Vo | 2.12.0 | +| T5050 diagnostic /Mixed phase partition (mp_partition) | Temperature at which cloud top is 50% ice, 50% liquid, following McCoy et al. (2015). | Yuying Zhang, Jill Zhang, Jiwen Fan, Yunpeng Shan | 2.9.0 | +| ARM diagnostics v3 (arm_diags) | Enhanced ARM diagnostics with new aerosol-cloud-interaction and aerosol activation metrics | Xiaojian Zheng, Jill Zhang, Cheng Tao, Shaocheng Xie | 2.9.0 | +| Aerosol budget diagnostics (aerosol_budget) | Global mean burdens, source/sink budgets and lifetimes for aerosol species. | Jill Zhang, Susannah Burrows, Naser Mahfouz, Tom Vo, Kai Zhang, Taufiq Hassan Mozumder, Xue Zheng, Ziming Ke, Yan Feng, Manish Shrivastava, Hailong Wang, Mingxuan Wu | 2.8.0 | +| Aerosol evaluation with AERONET data (aerosol_aeronet) | Scatter plots for comparing annual mean long-term Absorption Optical Depth (AOD) Aerosol Absorption Optical Depth (AAOD) at 550 nm from AERONET measurements. | Jill Zhang, Yan Feng | 2.7.0 | +| CMIP6 comparison (with lat_lon) | Box whisker plot to compare with key fields from CMIP6 models | Chris Golaz, Jill Zhang | 2.7.0 | +| Zonal mean 2D over stratosphere (zonal_mean
\_2d_stratosphere) | Pressure–latitude zonal-mean contour plots of seasonal mean | Jill Zhang | 2.6.0 | +| TC Analysis (tc_analysis) | Bar plots for TC frequency and accumulated cyclone energy distributed by ocean basin; line plots for TC intensity and seasonality for each ocean basin; maps for TC density and African easterly waves | Karthik Balaguru, Jill Zhang, Paul Ullrich | 2.5.0 | +| Annual cycle zonal mean (annual
_cycle_zonal_
mean) | Latitude zonal-mean–month box plot (including Stratospheric and Tropospheric column Ozone) | Jill Zhang, Qi Tang | 2.5.0 | +| ARM diagnostics v2 (arm_diags) | Seasonal mean, annual cycle, diurnal cycle and convection onset metrics over DOE's ARM ground-based facilities | Jill Zhang, Tao Chen, Shaocheng Xie, Todd Emmenegger, David Neelin, Kathleen Schiro, Yi-Hung Kuo | 2.4.0 | +| Streamflow diags (streamflow) | Seasonality map, annual-mean map/scatterplots based on gauge station data from GSIM | Ryan Forsyth, Tian Zhou | 2.3.0 | +| Diurnal cycle (diurnal_cycle) | Amplitude and phase map of the seasonal-mean diurnal cycle of precipitation compare with TRMM 3hourly data. | Jill Zhang, Charlie Zender, Wuyin Lin, Shaocheng Xie | 2.3.0 | +| Quasi-biennial oscillation analysis (qbo) | Quasi-biennial oscillation (QBO) analysis between 5S and 5N , including monthly mean zonal-mean zonal wind as a function of pressure and time and the power spectrum and amplitude | Chris Terai, Jack Chen, Jadwiga (Yaga) Richter, Sasha Glanville, Chris Golaz | 2.1.0 | +| ENSO diagnostics (enso_diags) | Maps of the regression coefficient of atmospheric fields over the SST anomaly, scatterplots of atmospheric feedback on the SST anomaly | Ryan Forsyth, Salil Mahajan | 2.1.0 | + +\*: The code infrastructure and maintenance are supported by the core developer group including: Jill Zhang, Chris Golaz, Tom Vo, Ryan Forsyth, Xylar Asay-Davis. ### Core sets established in v1 development + Algorithm and visualization codes for lat-lon contour plots, polar contour plots, zonal mean 2d plots, zonal mean line plots, 2d joint histogram for COSP cloud simulator output, tables and Taylor Diagrams summarizing metrics for climatology seasonal means, are implemented as core sets of this diagnostics package. Example of a complete set of diagnostics can be found [here](https://web.lcrc.anl.gov/public/e3sm/e3sm_diags_test_data/unit_test_complete_run/expected/all_sets/viewer/), and as shown below. The package features built-in user diagnostics, by specifying user desired diagnostics regions and pressure levels for variables with the vertical dimension. @@ -77,16 +78,13 @@ In addition to default model versus observation comparison, the package also pro Example-sets
Example plots including core sets from E3SM Diags v1 and process-oriented sets from v2
- - ## Reference Zhang, C., Golaz, J.-C., Forsyth, R., Vo, T., Xie, S., Shaheen, Z., Potter, G. L., Asay-Davis, X. S., Zender, C. S., Lin, W., Chen, C.-C., Terai, C. R., Mahajan, S., Zhou, T., Balaguru, K., Tang, Q., Tao, C., Zhang, Y., Emmenegger, T., Burrows, S., and Ullrich, P. A.: The E3SM Diagnostics Package (E3SM Diags v2.7): a Python-based diagnostics package for Earth system model evaluation, Geosci. Model Dev., 15, 9031–9056, https://doi.org/10.5194/gmd-15-9031-2022, 2022. - ## Acknowledgement -The work is performed for the [E3SM](https://e3sm.org/) project, which is sponsored by Earth System Model Development ([ESMD](https://climatemodeling.science.energy.gov/program/earth-system-model-development)) program. ESMD is a program area for the Earth and Environmental Systems Sciences Division ([EESSD](https://science.osti.gov/ber/Research/eessd)) in the Office of Biological and Environmental Research ([BER](https://science.osti.gov/ber)) within the [Department of Energy](https://www.energy.gov/)'s [Office of Science](https://science.osti.gov). Contributors' effort for this work are partially supported by the US DOE ARM program, RGMA program, EAGLES project, as well as by NSF. +The work is performed for the [E3SM](https://e3sm.org/) project, which is sponsored by Earth System Model Development ([ESMD](https://climatemodeling.science.energy.gov/program/earth-system-model-development)) program. ESMD is a program area for the Earth and Environmental Systems Sciences Division ([EESSD](https://science.osti.gov/ber/Research/eessd)) in the Office of Biological and Environmental Research ([BER](https://science.osti.gov/ber)) within the [Department of Energy](https://www.energy.gov/)'s [Office of Science](https://science.osti.gov). Contributors' effort for this work are partially supported by the US DOE ARM program, RGMA program, EAGLES project, as well as by NSF. ## License diff --git a/analysis_data_preprocess/fix_units_1.5hr_time_shift_forTRMM.py b/analysis_data_preprocess/fix_units_1.5hr_time_shift_forTRMM.py index 3e094a09c..6ad1c497a 100644 --- a/analysis_data_preprocess/fix_units_1.5hr_time_shift_forTRMM.py +++ b/analysis_data_preprocess/fix_units_1.5hr_time_shift_forTRMM.py @@ -1,5 +1,6 @@ -import cdms2 from pathlib import Path + +import cdms2 import MV2 datapath='/p/user_pub/e3sm/e3sm_diags_data/obs_for_e3sm_diags/climatology/TRMM-3B43v-7_3hr/' diff --git a/auxiliary_tools/LCF_mixed-phase_partition_diagram.ipynb b/auxiliary_tools/LCF_mixed-phase_partition_diagram.ipynb index 1d67fc56f..a89ca6f15 100644 --- a/auxiliary_tools/LCF_mixed-phase_partition_diagram.ipynb +++ b/auxiliary_tools/LCF_mixed-phase_partition_diagram.ipynb @@ -7,13 +7,14 @@ "metadata": {}, "outputs": [], "source": [ - "import xarray as xr\n", - "import numpy\n", + "import json\n", + "import math\n", + "\n", "import matplotlib.pyplot as plt\n", - "from scipy.stats import binned_statistic\n", + "import numpy\n", "import scipy.io\n", - "import math\n", - "import json" + "import xarray as xr\n", + "from scipy.stats import binned_statistic" ] }, { diff --git a/auxiliary_tools/aerosol_budget.py b/auxiliary_tools/aerosol_budget.py index 8a5c04c2a..1ccf1399c 100644 --- a/auxiliary_tools/aerosol_budget.py +++ b/auxiliary_tools/aerosol_budget.py @@ -1,15 +1,15 @@ # NOTE: This module uses the deprecated e3sm_diags.driver.utils.dataset.Dataset # class, which was replaced by e3sm_diags.driver.utils.dataset_xr.Dataset. -import e3sm_diags -from e3sm_diags.driver import utils +import csv +import json +import os + import cdms2 import cdutil import numpy -import os -import json - -import csv +import e3sm_diags +from e3sm_diags.driver import utils from e3sm_diags.parameter.core_parameter import CoreParameter diff --git a/auxiliary_tools/cdat_regression_testing/654-zonal_mean_xy/654-zonal_mean_xy_cdat_regression_test_netcdf.ipynb b/auxiliary_tools/cdat_regression_testing/654-zonal_mean_xy/654-zonal_mean_xy_cdat_regression_test_netcdf.ipynb index 5593b6685..c97566d9b 100644 --- a/auxiliary_tools/cdat_regression_testing/654-zonal_mean_xy/654-zonal_mean_xy_cdat_regression_test_netcdf.ipynb +++ b/auxiliary_tools/cdat_regression_testing/654-zonal_mean_xy/654-zonal_mean_xy_cdat_regression_test_netcdf.ipynb @@ -49,8 +49,8 @@ } ], "source": [ - "from collections import defaultdict\n", "import glob\n", + "from collections import defaultdict\n", "\n", "import numpy as np\n", "import xarray as xr\n", diff --git a/auxiliary_tools/cdat_regression_testing/654-zonal_mean_xy/debug_654-zonal_mean_xy_cdat_regression_test_netcdf_debug.ipynb b/auxiliary_tools/cdat_regression_testing/654-zonal_mean_xy/debug_654-zonal_mean_xy_cdat_regression_test_netcdf_debug.ipynb index 4feb1bbf9..90b8ee5dc 100644 --- a/auxiliary_tools/cdat_regression_testing/654-zonal_mean_xy/debug_654-zonal_mean_xy_cdat_regression_test_netcdf_debug.ipynb +++ b/auxiliary_tools/cdat_regression_testing/654-zonal_mean_xy/debug_654-zonal_mean_xy_cdat_regression_test_netcdf_debug.ipynb @@ -41,8 +41,8 @@ "metadata": {}, "outputs": [], "source": [ - "from collections import defaultdict\n", "import glob\n", + "from collections import defaultdict\n", "\n", "import numpy as np\n", "import xarray as xr\n", diff --git a/auxiliary_tools/cdat_regression_testing/655-zonal-mean-2d/655_zonal_mean_2d_regression_test_netcdf.ipynb b/auxiliary_tools/cdat_regression_testing/655-zonal-mean-2d/655_zonal_mean_2d_regression_test_netcdf.ipynb index 4157e1d8a..9302f1745 100644 --- a/auxiliary_tools/cdat_regression_testing/655-zonal-mean-2d/655_zonal_mean_2d_regression_test_netcdf.ipynb +++ b/auxiliary_tools/cdat_regression_testing/655-zonal-mean-2d/655_zonal_mean_2d_regression_test_netcdf.ipynb @@ -41,8 +41,8 @@ "metadata": {}, "outputs": [], "source": [ - "from collections import defaultdict\n", "import glob\n", + "from collections import defaultdict\n", "\n", "import numpy as np\n", "import xarray as xr\n", diff --git a/auxiliary_tools/cdat_regression_testing/655-zonal-mean-2d/655_zonal_mean_2d_stratosphere_regression_test_netcdf.ipynb b/auxiliary_tools/cdat_regression_testing/655-zonal-mean-2d/655_zonal_mean_2d_stratosphere_regression_test_netcdf.ipynb index 522f24a56..f3f210222 100644 --- a/auxiliary_tools/cdat_regression_testing/655-zonal-mean-2d/655_zonal_mean_2d_stratosphere_regression_test_netcdf.ipynb +++ b/auxiliary_tools/cdat_regression_testing/655-zonal-mean-2d/655_zonal_mean_2d_stratosphere_regression_test_netcdf.ipynb @@ -41,8 +41,8 @@ "metadata": {}, "outputs": [], "source": [ - "from collections import defaultdict\n", "import glob\n", + "from collections import defaultdict\n", "\n", "import numpy as np\n", "import xarray as xr\n", diff --git a/auxiliary_tools/cdat_regression_testing/657-meridional-mean-2d/657_meridional_mean_2d_cdat_regression_test_netcdf.ipynb b/auxiliary_tools/cdat_regression_testing/657-meridional-mean-2d/657_meridional_mean_2d_cdat_regression_test_netcdf.ipynb index 3e1f841a2..929585474 100644 --- a/auxiliary_tools/cdat_regression_testing/657-meridional-mean-2d/657_meridional_mean_2d_cdat_regression_test_netcdf.ipynb +++ b/auxiliary_tools/cdat_regression_testing/657-meridional-mean-2d/657_meridional_mean_2d_cdat_regression_test_netcdf.ipynb @@ -45,8 +45,8 @@ "\n", "import numpy as np\n", "import xarray as xr\n", - "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", + "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", "# TODO: Update SET_NAME and SET_DIR\n", "SET_NAME = \"meridional_mean_2d\"\n", diff --git a/auxiliary_tools/cdat_regression_testing/659-polar/659_polar_cdat_regression_test_netcdf.ipynb b/auxiliary_tools/cdat_regression_testing/659-polar/659_polar_cdat_regression_test_netcdf.ipynb index 7e9666cc5..16670d88b 100644 --- a/auxiliary_tools/cdat_regression_testing/659-polar/659_polar_cdat_regression_test_netcdf.ipynb +++ b/auxiliary_tools/cdat_regression_testing/659-polar/659_polar_cdat_regression_test_netcdf.ipynb @@ -45,8 +45,8 @@ "\n", "import numpy as np\n", "import xarray as xr\n", - "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", + "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", "# TODO: Update SET_NAME and SET_DIR\n", "SET_NAME = \"polar\"\n", diff --git a/auxiliary_tools/cdat_regression_testing/660-cosp-histogram/660_cosp_histogram_regression_test_netcdf.ipynb b/auxiliary_tools/cdat_regression_testing/660-cosp-histogram/660_cosp_histogram_regression_test_netcdf.ipynb index a9bb918e7..9987860a1 100644 --- a/auxiliary_tools/cdat_regression_testing/660-cosp-histogram/660_cosp_histogram_regression_test_netcdf.ipynb +++ b/auxiliary_tools/cdat_regression_testing/660-cosp-histogram/660_cosp_histogram_regression_test_netcdf.ipynb @@ -46,8 +46,8 @@ "metadata": {}, "outputs": [], "source": [ - "from collections import defaultdict\n", "import glob\n", + "from collections import defaultdict\n", "\n", "import numpy as np\n", "import xarray as xr\n", diff --git a/auxiliary_tools/cdat_regression_testing/662-area-mean-time-series/regression-test-json.ipynb b/auxiliary_tools/cdat_regression_testing/662-area-mean-time-series/regression-test-json.ipynb index 4025d7012..9fe5c47b4 100644 --- a/auxiliary_tools/cdat_regression_testing/662-area-mean-time-series/regression-test-json.ipynb +++ b/auxiliary_tools/cdat_regression_testing/662-area-mean-time-series/regression-test-json.ipynb @@ -55,8 +55,8 @@ "from auxiliary_tools.cdat_regression_testing.utils import (\n", " get_num_metrics_above_diff_thres,\n", " get_rel_diffs,\n", - " update_diffs_to_pct,\n", " highlight_large_diffs,\n", + " update_diffs_to_pct,\n", ")\n", "\n", "SET_NAME = \"area_mean_time_series\"\n", diff --git a/auxiliary_tools/cdat_regression_testing/662-area-mean-time-series/regression-test-netcdf.ipynb b/auxiliary_tools/cdat_regression_testing/662-area-mean-time-series/regression-test-netcdf.ipynb index cd7a385f2..bc4113599 100644 --- a/auxiliary_tools/cdat_regression_testing/662-area-mean-time-series/regression-test-netcdf.ipynb +++ b/auxiliary_tools/cdat_regression_testing/662-area-mean-time-series/regression-test-netcdf.ipynb @@ -41,14 +41,14 @@ "metadata": {}, "outputs": [], "source": [ - "from collections import defaultdict\n", "import glob\n", + "from collections import defaultdict\n", "from typing import Tuple\n", "\n", "import numpy as np\n", "import xarray as xr\n", - "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", + "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", "# TODO: Update SET_NAME and SET_DIR\n", "SET_NAME = \"area_mean_time_series\"\n", diff --git a/auxiliary_tools/cdat_regression_testing/663-enso-diags/662-area-mean-time-series-rerun/regression-test-json.ipynb b/auxiliary_tools/cdat_regression_testing/663-enso-diags/662-area-mean-time-series-rerun/regression-test-json.ipynb index cc27db393..f2d35e7f8 100644 --- a/auxiliary_tools/cdat_regression_testing/663-enso-diags/662-area-mean-time-series-rerun/regression-test-json.ipynb +++ b/auxiliary_tools/cdat_regression_testing/663-enso-diags/662-area-mean-time-series-rerun/regression-test-json.ipynb @@ -55,8 +55,8 @@ "from auxiliary_tools.cdat_regression_testing.utils import (\n", " get_num_metrics_above_diff_thres,\n", " get_rel_diffs,\n", - " update_diffs_to_pct,\n", " highlight_large_diffs,\n", + " update_diffs_to_pct,\n", ")\n", "\n", "SET_NAME = \"area_mean_time_series\"\n", diff --git a/auxiliary_tools/cdat_regression_testing/663-enso-diags/662-area-mean-time-series-rerun/regression-test-netcdf.ipynb b/auxiliary_tools/cdat_regression_testing/663-enso-diags/662-area-mean-time-series-rerun/regression-test-netcdf.ipynb index 11714aecf..1d58c94c0 100644 --- a/auxiliary_tools/cdat_regression_testing/663-enso-diags/662-area-mean-time-series-rerun/regression-test-netcdf.ipynb +++ b/auxiliary_tools/cdat_regression_testing/663-enso-diags/662-area-mean-time-series-rerun/regression-test-netcdf.ipynb @@ -41,14 +41,14 @@ "metadata": {}, "outputs": [], "source": [ - "from collections import defaultdict\n", "import glob\n", + "from collections import defaultdict\n", "from typing import Tuple\n", "\n", "import numpy as np\n", "import xarray as xr\n", - "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", + "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", "# TODO: Update SET_NAME and SET_DIR\n", "SET_NAME = \"area_mean_time_series\"\n", diff --git a/auxiliary_tools/cdat_regression_testing/663-enso-diags/regression_test.ipynb b/auxiliary_tools/cdat_regression_testing/663-enso-diags/regression_test.ipynb index c74f5b2b3..c540d83fa 100644 --- a/auxiliary_tools/cdat_regression_testing/663-enso-diags/regression_test.ipynb +++ b/auxiliary_tools/cdat_regression_testing/663-enso-diags/regression_test.ipynb @@ -45,6 +45,7 @@ "\n", "import numpy as np\n", "import xarray as xr\n", + "\n", "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", "SET_NAME = \"enso_diags\"\n", diff --git a/auxiliary_tools/cdat_regression_testing/664-qbo/regression_test.ipynb b/auxiliary_tools/cdat_regression_testing/664-qbo/regression_test.ipynb index 7340b1394..e127ac4a9 100644 --- a/auxiliary_tools/cdat_regression_testing/664-qbo/regression_test.ipynb +++ b/auxiliary_tools/cdat_regression_testing/664-qbo/regression_test.ipynb @@ -45,6 +45,7 @@ "\n", "import numpy as np\n", "import xarray as xr\n", + "\n", "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", "SET_NAME = \"qbo\"\n", @@ -332,8 +333,8 @@ "metadata": {}, "outputs": [], "source": [ - "import xcdat as xc\n", "import xarray as xr\n", + "import xcdat as xc\n", "\n", "ds_xc = xc.open_dataset(\n", " \"/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/664-qbo/qbo/QBO-ERA-Interim/qbo_diags_qbo_ref.nc\"\n", diff --git a/auxiliary_tools/cdat_regression_testing/666-diurnal-cycle/regression_test.ipynb b/auxiliary_tools/cdat_regression_testing/666-diurnal-cycle/regression_test.ipynb index 1b889e1b8..218a01c72 100644 --- a/auxiliary_tools/cdat_regression_testing/666-diurnal-cycle/regression_test.ipynb +++ b/auxiliary_tools/cdat_regression_testing/666-diurnal-cycle/regression_test.ipynb @@ -45,8 +45,8 @@ "\n", "import numpy as np\n", "import xarray as xr\n", - "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", + "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", "# TODO: Update SET_NAME and SET_DIR\n", "SET_NAME = \"diurnal_cycle\"\n", diff --git a/auxiliary_tools/cdat_regression_testing/669-annual_cycle_zonal_mean/669-annual_cycle_zonal_mean_cdat_regression_test_netcdf.ipynb b/auxiliary_tools/cdat_regression_testing/669-annual_cycle_zonal_mean/669-annual_cycle_zonal_mean_cdat_regression_test_netcdf.ipynb index 3c8200b54..588f68964 100644 --- a/auxiliary_tools/cdat_regression_testing/669-annual_cycle_zonal_mean/669-annual_cycle_zonal_mean_cdat_regression_test_netcdf.ipynb +++ b/auxiliary_tools/cdat_regression_testing/669-annual_cycle_zonal_mean/669-annual_cycle_zonal_mean_cdat_regression_test_netcdf.ipynb @@ -49,8 +49,8 @@ } ], "source": [ - "from collections import defaultdict\n", "import glob\n", + "from collections import defaultdict\n", "\n", "import numpy as np\n", "import xarray as xr\n", diff --git a/auxiliary_tools/cdat_regression_testing/669-annual_cycle_zonal_mean/re-run-lat-lon-with-var-subset/669_lat_lon_cdat_regression_test_json.ipynb b/auxiliary_tools/cdat_regression_testing/669-annual_cycle_zonal_mean/re-run-lat-lon-with-var-subset/669_lat_lon_cdat_regression_test_json.ipynb index cde6823aa..998939001 100644 --- a/auxiliary_tools/cdat_regression_testing/669-annual_cycle_zonal_mean/re-run-lat-lon-with-var-subset/669_lat_lon_cdat_regression_test_json.ipynb +++ b/auxiliary_tools/cdat_regression_testing/669-annual_cycle_zonal_mean/re-run-lat-lon-with-var-subset/669_lat_lon_cdat_regression_test_json.ipynb @@ -59,20 +59,20 @@ } ], "source": [ - "from collections import defaultdict\n", "import glob\n", + "from collections import defaultdict\n", "\n", - "import pandas as pd\n", "import numpy as np\n", + "import pandas as pd\n", "import xarray as xr\n", "\n", "from auxiliary_tools.cdat_regression_testing.utils import (\n", " get_metrics,\n", " get_num_metrics_above_diff_thres,\n", " get_rel_diffs,\n", + " highlight_large_diffs,\n", " sort_columns,\n", " update_diffs_to_pct,\n", - " highlight_large_diffs,\n", ")\n", "\n", "# TODO: Update SET_NAME and SET_DIR\n", diff --git a/auxiliary_tools/cdat_regression_testing/669-annual_cycle_zonal_mean/re-run-lat-lon-with-var-subset/669_lat_lon_cdat_regression_test_netcdf.ipynb b/auxiliary_tools/cdat_regression_testing/669-annual_cycle_zonal_mean/re-run-lat-lon-with-var-subset/669_lat_lon_cdat_regression_test_netcdf.ipynb index 5e8eb5c0b..206c7636a 100644 --- a/auxiliary_tools/cdat_regression_testing/669-annual_cycle_zonal_mean/re-run-lat-lon-with-var-subset/669_lat_lon_cdat_regression_test_netcdf.ipynb +++ b/auxiliary_tools/cdat_regression_testing/669-annual_cycle_zonal_mean/re-run-lat-lon-with-var-subset/669_lat_lon_cdat_regression_test_netcdf.ipynb @@ -45,8 +45,8 @@ "\n", "import numpy as np\n", "import xarray as xr\n", - "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", + "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", "# TODO: Update SET_NAME and SET_DIR\n", "SET_NAME = \"lat_lon\"\n", diff --git a/auxiliary_tools/cdat_regression_testing/671-lat-lon/12-4-23-qa-no-cdms-slice.ipynb b/auxiliary_tools/cdat_regression_testing/671-lat-lon/12-4-23-qa-no-cdms-slice.ipynb index 7a8d8ebae..5c206df3b 100644 --- a/auxiliary_tools/cdat_regression_testing/671-lat-lon/12-4-23-qa-no-cdms-slice.ipynb +++ b/auxiliary_tools/cdat_regression_testing/671-lat-lon/12-4-23-qa-no-cdms-slice.ipynb @@ -24,12 +24,12 @@ "metadata": {}, "outputs": [], "source": [ + "import glob\n", + "\n", "import pandas as pd\n", "\n", - "import glob\n", "from auxiliary_tools.cdat_regression_testing.utils import get_metrics, get_rel_diffs\n", "\n", - "\n", "# TODO: Update DEV_RESULTS and MAIN_RESULTS to your diagnostic sets.\n", "DEV_PATH = \"/global/cfs/cdirs/e3sm/www/vo13/examples_658/ex1_modTS_vs_modTS_3years/lat_lon/model_vs_model\"\n", "MAIN_PATH = \"/global/cfs/cdirs/e3sm/www/vo13/examples_main_w_slice/ex1_modTS_vs_modTS_3years/lat_lon/model_vs_model\"\n", diff --git a/auxiliary_tools/cdat_regression_testing/671-lat-lon/671-lat-lon.ipynb b/auxiliary_tools/cdat_regression_testing/671-lat-lon/671-lat-lon.ipynb index 9a7add496..12ab12e53 100644 --- a/auxiliary_tools/cdat_regression_testing/671-lat-lon/671-lat-lon.ipynb +++ b/auxiliary_tools/cdat_regression_testing/671-lat-lon/671-lat-lon.ipynb @@ -48,18 +48,19 @@ "outputs": [], "source": [ "import glob\n", + "\n", + "import pandas as pd\n", + "\n", "from auxiliary_tools.cdat_regression_testing.utils import (\n", + " PERCENTAGE_COLUMNS,\n", " get_metrics,\n", - " get_rel_diffs,\n", " get_num_metrics_above_diff_thres,\n", + " get_rel_diffs,\n", " highlight_large_diffs,\n", " sort_columns,\n", " update_diffs_to_pct,\n", - " PERCENTAGE_COLUMNS,\n", ")\n", "\n", - "import pandas as pd\n", - "\n", "# TODO: Update DEV_RESULTS and MAIN_RESULTS to your diagnostic sets.\n", "DEV_PATH = \"/global/cfs/cdirs/e3sm/www/vo13/examples_658/ex1_modTS_vs_modTS_3years/lat_lon/model_vs_model\"\n", "MAIN_PATH = \"/global/cfs/cdirs/e3sm/www/vo13/examples/ex1_modTS_vs_modTS_3years/lat_lon/model_vs_model\"\n", diff --git a/auxiliary_tools/cdat_regression_testing/759-slice-flag/regression-test-json.ipynb b/auxiliary_tools/cdat_regression_testing/759-slice-flag/regression-test-json.ipynb index 602d1120c..9cdb8865d 100644 --- a/auxiliary_tools/cdat_regression_testing/759-slice-flag/regression-test-json.ipynb +++ b/auxiliary_tools/cdat_regression_testing/759-slice-flag/regression-test-json.ipynb @@ -59,21 +59,20 @@ } ], "source": [ - "from typing import List\n", "import glob\n", + "from typing import List\n", "\n", "import pandas as pd\n", "\n", "from auxiliary_tools.cdat_regression_testing.utils import (\n", - " get_rel_diffs,\n", + " PERCENTAGE_COLUMNS,\n", " get_num_metrics_above_diff_thres,\n", + " get_rel_diffs,\n", " highlight_large_diffs,\n", " sort_columns,\n", " update_diffs_to_pct,\n", - " PERCENTAGE_COLUMNS,\n", ")\n", "\n", - "\n", "SET_NAME = \"lat_lon\"\n", "SET_DIR = \"759-slice-flag\"\n", "\n", diff --git a/auxiliary_tools/cdat_regression_testing/759-slice-flag/regression-test-netcdf.ipynb b/auxiliary_tools/cdat_regression_testing/759-slice-flag/regression-test-netcdf.ipynb index cd7a385f2..bc4113599 100644 --- a/auxiliary_tools/cdat_regression_testing/759-slice-flag/regression-test-netcdf.ipynb +++ b/auxiliary_tools/cdat_regression_testing/759-slice-flag/regression-test-netcdf.ipynb @@ -41,14 +41,14 @@ "metadata": {}, "outputs": [], "source": [ - "from collections import defaultdict\n", "import glob\n", + "from collections import defaultdict\n", "from typing import Tuple\n", "\n", "import numpy as np\n", "import xarray as xr\n", - "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", + "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", "# TODO: Update SET_NAME and SET_DIR\n", "SET_NAME = \"area_mean_time_series\"\n", diff --git a/auxiliary_tools/cdat_regression_testing/792-lat-lon-run-script/792_lat_lon_cdat_regression_test_json.ipynb b/auxiliary_tools/cdat_regression_testing/792-lat-lon-run-script/792_lat_lon_cdat_regression_test_json.ipynb index cde6823aa..998939001 100644 --- a/auxiliary_tools/cdat_regression_testing/792-lat-lon-run-script/792_lat_lon_cdat_regression_test_json.ipynb +++ b/auxiliary_tools/cdat_regression_testing/792-lat-lon-run-script/792_lat_lon_cdat_regression_test_json.ipynb @@ -59,20 +59,20 @@ } ], "source": [ - "from collections import defaultdict\n", "import glob\n", + "from collections import defaultdict\n", "\n", - "import pandas as pd\n", "import numpy as np\n", + "import pandas as pd\n", "import xarray as xr\n", "\n", "from auxiliary_tools.cdat_regression_testing.utils import (\n", " get_metrics,\n", " get_num_metrics_above_diff_thres,\n", " get_rel_diffs,\n", + " highlight_large_diffs,\n", " sort_columns,\n", " update_diffs_to_pct,\n", - " highlight_large_diffs,\n", ")\n", "\n", "# TODO: Update SET_NAME and SET_DIR\n", diff --git a/auxiliary_tools/cdat_regression_testing/792-lat-lon-run-script/792_lat_lon_cdat_regression_test_netcdf.ipynb b/auxiliary_tools/cdat_regression_testing/792-lat-lon-run-script/792_lat_lon_cdat_regression_test_netcdf.ipynb index 1796d0439..80ebcf685 100644 --- a/auxiliary_tools/cdat_regression_testing/792-lat-lon-run-script/792_lat_lon_cdat_regression_test_netcdf.ipynb +++ b/auxiliary_tools/cdat_regression_testing/792-lat-lon-run-script/792_lat_lon_cdat_regression_test_netcdf.ipynb @@ -45,8 +45,8 @@ "\n", "import numpy as np\n", "import xarray as xr\n", - "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", + "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", "# TODO: Update SET_NAME and SET_DIR\n", "SET_NAME = \"lat_lon\"\n", diff --git a/auxiliary_tools/cdat_regression_testing/843-migration-phase3/ex-scripts/examples_regression_test_netcdf.ipynb b/auxiliary_tools/cdat_regression_testing/843-migration-phase3/ex-scripts/examples_regression_test_netcdf.ipynb index 5a01eb2d0..858ba0621 100644 --- a/auxiliary_tools/cdat_regression_testing/843-migration-phase3/ex-scripts/examples_regression_test_netcdf.ipynb +++ b/auxiliary_tools/cdat_regression_testing/843-migration-phase3/ex-scripts/examples_regression_test_netcdf.ipynb @@ -41,15 +41,15 @@ "metadata": {}, "outputs": [], "source": [ - "from collections import defaultdict\n", "import glob\n", "import os\n", + "from collections import defaultdict\n", "from typing import Tuple\n", "\n", "import numpy as np\n", "import xarray as xr\n", - "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", + "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", "DEV_PATH = f\"/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/examples-dev/**\"\n", "\n", diff --git a/auxiliary_tools/cdat_regression_testing/843-migration-phase3/ex-scripts/fix_isccpcosp_ann_climo_nc_calendar.py b/auxiliary_tools/cdat_regression_testing/843-migration-phase3/ex-scripts/fix_isccpcosp_ann_climo_nc_calendar.py index dee0ccc97..c94f2c9e8 100644 --- a/auxiliary_tools/cdat_regression_testing/843-migration-phase3/ex-scripts/fix_isccpcosp_ann_climo_nc_calendar.py +++ b/auxiliary_tools/cdat_regression_testing/843-migration-phase3/ex-scripts/fix_isccpcosp_ann_climo_nc_calendar.py @@ -1,12 +1,11 @@ # %% from datetime import datetime -from dateutil import parser -from dateutil import relativedelta as rd import numpy as np import xarray as xr import xcdat as xc - +from dateutil import parser +from dateutil import relativedelta as rd # %% # Reproduce error: ValueError: Non-integer years and months are ambiguous and not currently supported. diff --git a/auxiliary_tools/cdat_regression_testing/843-migration-phase3/run-script-model-vs-obs/model_vs_obs_png.ipynb b/auxiliary_tools/cdat_regression_testing/843-migration-phase3/run-script-model-vs-obs/model_vs_obs_png.ipynb index ce18996b3..d3f45038e 100644 --- a/auxiliary_tools/cdat_regression_testing/843-migration-phase3/run-script-model-vs-obs/model_vs_obs_png.ipynb +++ b/auxiliary_tools/cdat_regression_testing/843-migration-phase3/run-script-model-vs-obs/model_vs_obs_png.ipynb @@ -39,7 +39,6 @@ "\n", "from auxiliary_tools.cdat_regression_testing.utils import get_image_diffs\n", "\n", - "\n", "DEV_DIR = \"843-migration-phase3-model-vs-obs\"\n", "DEV_PATH = f\"/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/{DEV_DIR}/\"\n", "\n", diff --git a/auxiliary_tools/cdat_regression_testing/843-migration-phase3/run-script-model-vs-obs/run_script.py b/auxiliary_tools/cdat_regression_testing/843-migration-phase3/run-script-model-vs-obs/run_script.py index 15965e0a2..645d64bdc 100644 --- a/auxiliary_tools/cdat_regression_testing/843-migration-phase3/run-script-model-vs-obs/run_script.py +++ b/auxiliary_tools/cdat_regression_testing/843-migration-phase3/run-script-model-vs-obs/run_script.py @@ -18,6 +18,7 @@ # flake8: noqa E501 import os +import timeit from typing import Tuple, TypedDict from mache import MachineInfo @@ -38,7 +39,6 @@ ZonalMean2dStratosphereParameter, ) from e3sm_diags.run import runner -import timeit class MachinePaths(TypedDict): diff --git a/auxiliary_tools/cdat_regression_testing/843-migration-phase3/run-script-model-vs-obs/run_script_main.py b/auxiliary_tools/cdat_regression_testing/843-migration-phase3/run-script-model-vs-obs/run_script_main.py index 9b8a107b4..e85ac11a1 100644 --- a/auxiliary_tools/cdat_regression_testing/843-migration-phase3/run-script-model-vs-obs/run_script_main.py +++ b/auxiliary_tools/cdat_regression_testing/843-migration-phase3/run-script-model-vs-obs/run_script_main.py @@ -18,6 +18,7 @@ # flake8: noqa E501 import os +import timeit from typing import Tuple, TypedDict from mache import MachineInfo @@ -38,7 +39,6 @@ ZonalMean2dStratosphereParameter, ) from e3sm_diags.run import runner -import timeit class MachinePaths(TypedDict): diff --git a/auxiliary_tools/cdat_regression_testing/850-qbo-wavelet/regression_test.ipynb b/auxiliary_tools/cdat_regression_testing/850-qbo-wavelet/regression_test.ipynb index 03f47e094..a949a43a0 100644 --- a/auxiliary_tools/cdat_regression_testing/850-qbo-wavelet/regression_test.ipynb +++ b/auxiliary_tools/cdat_regression_testing/850-qbo-wavelet/regression_test.ipynb @@ -45,6 +45,7 @@ "\n", "import numpy as np\n", "import xarray as xr\n", + "\n", "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", "SET_NAME = \"qbo\"\n", @@ -337,8 +338,8 @@ "metadata": {}, "outputs": [], "source": [ - "import xcdat as xc\n", "import xarray as xr\n", + "import xcdat as xc\n", "\n", "ds_xc = xc.open_dataset(\n", " \"/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/664-qbo/qbo/QBO-ERA-Interim/qbo_diags_qbo_ref.nc\"\n", diff --git a/auxiliary_tools/cdat_regression_testing/855-perf-benchmark/run-script-model-vs-obs/run_script.py b/auxiliary_tools/cdat_regression_testing/855-perf-benchmark/run-script-model-vs-obs/run_script.py index 239867ba4..d337c5f51 100644 --- a/auxiliary_tools/cdat_regression_testing/855-perf-benchmark/run-script-model-vs-obs/run_script.py +++ b/auxiliary_tools/cdat_regression_testing/855-perf-benchmark/run-script-model-vs-obs/run_script.py @@ -19,6 +19,7 @@ import datetime import os +import timeit from typing import Tuple, TypedDict from mache import MachineInfo @@ -39,7 +40,6 @@ ZonalMean2dStratosphereParameter, ) from e3sm_diags.run import runner -import timeit class MachinePaths(TypedDict): diff --git a/auxiliary_tools/cdat_regression_testing/855-perf-benchmark/run-script-model-vs-obs/run_script_main.py b/auxiliary_tools/cdat_regression_testing/855-perf-benchmark/run-script-model-vs-obs/run_script_main.py index 55b0568f8..10252d931 100644 --- a/auxiliary_tools/cdat_regression_testing/855-perf-benchmark/run-script-model-vs-obs/run_script_main.py +++ b/auxiliary_tools/cdat_regression_testing/855-perf-benchmark/run-script-model-vs-obs/run_script_main.py @@ -19,6 +19,7 @@ import datetime import os +import timeit from typing import Tuple, TypedDict from mache import MachineInfo @@ -39,7 +40,6 @@ ZonalMean2dStratosphereParameter, ) from e3sm_diags.run import runner -import timeit class MachinePaths(TypedDict): diff --git a/auxiliary_tools/cdat_regression_testing/861-time-series-multiple/all-ts-datasets/png.ipynb b/auxiliary_tools/cdat_regression_testing/861-time-series-multiple/all-ts-datasets/png.ipynb index ce18996b3..d3f45038e 100644 --- a/auxiliary_tools/cdat_regression_testing/861-time-series-multiple/all-ts-datasets/png.ipynb +++ b/auxiliary_tools/cdat_regression_testing/861-time-series-multiple/all-ts-datasets/png.ipynb @@ -39,7 +39,6 @@ "\n", "from auxiliary_tools.cdat_regression_testing.utils import get_image_diffs\n", "\n", - "\n", "DEV_DIR = \"843-migration-phase3-model-vs-obs\"\n", "DEV_PATH = f\"/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/{DEV_DIR}/\"\n", "\n", diff --git a/auxiliary_tools/cdat_regression_testing/861-time-series-multiple/all-ts-datasets/specific_sets/diurnal_cycle_nc.ipynb b/auxiliary_tools/cdat_regression_testing/861-time-series-multiple/all-ts-datasets/specific_sets/diurnal_cycle_nc.ipynb index d32568ffd..ea01266e1 100644 --- a/auxiliary_tools/cdat_regression_testing/861-time-series-multiple/all-ts-datasets/specific_sets/diurnal_cycle_nc.ipynb +++ b/auxiliary_tools/cdat_regression_testing/861-time-series-multiple/all-ts-datasets/specific_sets/diurnal_cycle_nc.ipynb @@ -45,8 +45,8 @@ "\n", "import numpy as np\n", "import xarray as xr\n", - "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", + "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", "# TODO: Update SET_NAME and SET_DIR\n", "SET_NAME = \"diurnal_cycle\"\n", diff --git a/auxiliary_tools/cdat_regression_testing/861-time-series-multiple/all-ts-datasets/specific_sets/enso_diags_nc.ipynb b/auxiliary_tools/cdat_regression_testing/861-time-series-multiple/all-ts-datasets/specific_sets/enso_diags_nc.ipynb index 9be2f7ad2..c02d2e1e6 100644 --- a/auxiliary_tools/cdat_regression_testing/861-time-series-multiple/all-ts-datasets/specific_sets/enso_diags_nc.ipynb +++ b/auxiliary_tools/cdat_regression_testing/861-time-series-multiple/all-ts-datasets/specific_sets/enso_diags_nc.ipynb @@ -45,6 +45,7 @@ "\n", "import numpy as np\n", "import xarray as xr\n", + "\n", "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", "SET_NAME = \"enso_diags\"\n", diff --git a/auxiliary_tools/cdat_regression_testing/861-time-series-multiple/all-ts-datasets/specific_sets/qbo_nc.ipynb b/auxiliary_tools/cdat_regression_testing/861-time-series-multiple/all-ts-datasets/specific_sets/qbo_nc.ipynb index 834a49e36..2b21179ff 100644 --- a/auxiliary_tools/cdat_regression_testing/861-time-series-multiple/all-ts-datasets/specific_sets/qbo_nc.ipynb +++ b/auxiliary_tools/cdat_regression_testing/861-time-series-multiple/all-ts-datasets/specific_sets/qbo_nc.ipynb @@ -45,6 +45,7 @@ "\n", "import numpy as np\n", "import xarray as xr\n", + "\n", "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", "SET_NAME = \"qbo\"\n", @@ -335,8 +336,8 @@ "metadata": {}, "outputs": [], "source": [ - "import xcdat as xc\n", "import xarray as xr\n", + "import xcdat as xc\n", "\n", "ds_xc = xc.open_dataset(\n", " \"/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/664-qbo/qbo/QBO-ERA-Interim/qbo_diags_qbo_ref.nc\"\n", diff --git a/auxiliary_tools/cdat_regression_testing/861-time-series-multiple/run_script.py b/auxiliary_tools/cdat_regression_testing/861-time-series-multiple/run_script.py index 0f0e31acb..0a749a949 100644 --- a/auxiliary_tools/cdat_regression_testing/861-time-series-multiple/run_script.py +++ b/auxiliary_tools/cdat_regression_testing/861-time-series-multiple/run_script.py @@ -1,12 +1,12 @@ # %% import os + import numpy + from e3sm_diags.parameter.core_parameter import CoreParameter from e3sm_diags.parameter.enso_diags_parameter import EnsoDiagsParameter from e3sm_diags.parameter.streamflow_parameter import StreamflowParameter from e3sm_diags.parameter.tc_analysis_parameter import TCAnalysisParameter - - from e3sm_diags.run import runner short_name = "v3.LR.historical_0051" diff --git a/auxiliary_tools/cdat_regression_testing/863-z-bounds-fix/run_script.py b/auxiliary_tools/cdat_regression_testing/863-z-bounds-fix/run_script.py index 7ca32ced9..b240c62ae 100644 --- a/auxiliary_tools/cdat_regression_testing/863-z-bounds-fix/run_script.py +++ b/auxiliary_tools/cdat_regression_testing/863-z-bounds-fix/run_script.py @@ -1,4 +1,5 @@ import os + from e3sm_diags.parameter.core_parameter import CoreParameter from e3sm_diags.run import runner diff --git a/auxiliary_tools/cdat_regression_testing/877-attr-error/run_script.py b/auxiliary_tools/cdat_regression_testing/877-attr-error/run_script.py index eeb452f91..e566a850e 100644 --- a/auxiliary_tools/cdat_regression_testing/877-attr-error/run_script.py +++ b/auxiliary_tools/cdat_regression_testing/877-attr-error/run_script.py @@ -1,4 +1,5 @@ import os + from e3sm_diags.parameter.core_parameter import CoreParameter from e3sm_diags.run import runner diff --git a/auxiliary_tools/cdat_regression_testing/886-jjb/regression_test_netcdf.ipynb b/auxiliary_tools/cdat_regression_testing/886-jjb/regression_test_netcdf.ipynb index 8ad14d6e4..58ec0a6ea 100644 --- a/auxiliary_tools/cdat_regression_testing/886-jjb/regression_test_netcdf.ipynb +++ b/auxiliary_tools/cdat_regression_testing/886-jjb/regression_test_netcdf.ipynb @@ -45,8 +45,8 @@ "\n", "import numpy as np\n", "import xarray as xr\n", - "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", + "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", "# TODO: Update SET_NAME and SET_DIR\n", "SET_NAME = \"tropical_subseasonal\"\n", diff --git a/auxiliary_tools/cdat_regression_testing/886-jjb/regression_test_png.ipynb b/auxiliary_tools/cdat_regression_testing/886-jjb/regression_test_png.ipynb index b06ebbe19..fc11faef3 100644 --- a/auxiliary_tools/cdat_regression_testing/886-jjb/regression_test_png.ipynb +++ b/auxiliary_tools/cdat_regression_testing/886-jjb/regression_test_png.ipynb @@ -35,8 +35,8 @@ "outputs": [], "source": [ "import glob\n", - "from auxiliary_tools.cdat_regression_testing.utils import get_image_diffs\n", "\n", + "from auxiliary_tools.cdat_regression_testing.utils import get_image_diffs\n", "\n", "SET_NAME = \"tropical_subseasonal\"\n", "SET_DIR = \"886-jjb\"\n", diff --git a/auxiliary_tools/cdat_regression_testing/886-jjb/run_script.py b/auxiliary_tools/cdat_regression_testing/886-jjb/run_script.py index 14570aaf7..dc19f1ea6 100644 --- a/auxiliary_tools/cdat_regression_testing/886-jjb/run_script.py +++ b/auxiliary_tools/cdat_regression_testing/886-jjb/run_script.py @@ -1,4 +1,5 @@ import os + from e3sm_diags.parameter.tropical_subseasonal_parameter import ( TropicalSubseasonalParameter, ) diff --git a/auxiliary_tools/cdat_regression_testing/892-bottleneck/run_script.py b/auxiliary_tools/cdat_regression_testing/892-bottleneck/run_script.py index 452901cd6..4d3dbad20 100644 --- a/auxiliary_tools/cdat_regression_testing/892-bottleneck/run_script.py +++ b/auxiliary_tools/cdat_regression_testing/892-bottleneck/run_script.py @@ -1,5 +1,6 @@ -import sys import os +import sys + from e3sm_diags.parameter.core_parameter import CoreParameter from e3sm_diags.run import runner diff --git a/auxiliary_tools/cdat_regression_testing/892-bottleneck/xr_mvce_gh.py b/auxiliary_tools/cdat_regression_testing/892-bottleneck/xr_mvce_gh.py index 2baf2ef65..d459ffa84 100644 --- a/auxiliary_tools/cdat_regression_testing/892-bottleneck/xr_mvce_gh.py +++ b/auxiliary_tools/cdat_regression_testing/892-bottleneck/xr_mvce_gh.py @@ -1,10 +1,10 @@ # %% -import numpy as np -import pandas as pd -import xarray as xr import timeit import dask.array as da +import numpy as np +import pandas as pd +import xarray as xr # %% # Define the dimensions diff --git a/auxiliary_tools/cdat_regression_testing/906-v2_complete_run/regression_png.ipynb b/auxiliary_tools/cdat_regression_testing/906-v2_complete_run/regression_png.ipynb index ea0c3eb78..fc4f6144b 100644 --- a/auxiliary_tools/cdat_regression_testing/906-v2_complete_run/regression_png.ipynb +++ b/auxiliary_tools/cdat_regression_testing/906-v2_complete_run/regression_png.ipynb @@ -39,7 +39,6 @@ "\n", "from auxiliary_tools.cdat_regression_testing.utils import get_image_diffs\n", "\n", - "\n", "DEV_DIR = \"25-01-10-branch-907-v2-data\"\n", "DEV_PATH = f\"/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/{DEV_DIR}/\"\n", "\n", diff --git a/auxiliary_tools/cdat_regression_testing/906-v2_complete_run/run_script.py b/auxiliary_tools/cdat_regression_testing/906-v2_complete_run/run_script.py index e7575d1de..162414fd9 100644 --- a/auxiliary_tools/cdat_regression_testing/906-v2_complete_run/run_script.py +++ b/auxiliary_tools/cdat_regression_testing/906-v2_complete_run/run_script.py @@ -18,6 +18,7 @@ # flake8: noqa E501 import os +import timeit from typing import Tuple, TypedDict from mache import MachineInfo @@ -38,7 +39,6 @@ ZonalMean2dStratosphereParameter, ) from e3sm_diags.run import runner -import timeit class MachinePaths(TypedDict): diff --git a/auxiliary_tools/cdat_regression_testing/906-v3_complete_run/debug-missing-arm-files/get_diff_arm_diags.py b/auxiliary_tools/cdat_regression_testing/906-v3_complete_run/debug-missing-arm-files/get_diff_arm_diags.py index d441c3d4a..1d0cade2d 100644 --- a/auxiliary_tools/cdat_regression_testing/906-v3_complete_run/debug-missing-arm-files/get_diff_arm_diags.py +++ b/auxiliary_tools/cdat_regression_testing/906-v3_complete_run/debug-missing-arm-files/get_diff_arm_diags.py @@ -1,5 +1,5 @@ -import os import filecmp +import os def get_png_files(directory): diff --git a/auxiliary_tools/cdat_regression_testing/906-v3_complete_run/regression_png.ipynb b/auxiliary_tools/cdat_regression_testing/906-v3_complete_run/regression_png.ipynb index 563d9fc12..e46677bbf 100644 --- a/auxiliary_tools/cdat_regression_testing/906-v3_complete_run/regression_png.ipynb +++ b/auxiliary_tools/cdat_regression_testing/906-v3_complete_run/regression_png.ipynb @@ -39,7 +39,6 @@ "\n", "from auxiliary_tools.cdat_regression_testing.utils import get_image_diffs\n", "\n", - "\n", "DEV_DIR = \"25-01-15-branch-907\"\n", "DEV_PATH = f\"/global/cfs/cdirs/e3sm/www/e3sm_diags/complete_run/{DEV_DIR}/\"\n", "\n", diff --git a/auxiliary_tools/cdat_regression_testing/915-units-attr/run_script.py b/auxiliary_tools/cdat_regression_testing/915-units-attr/run_script.py index 539829a07..5a7994204 100644 --- a/auxiliary_tools/cdat_regression_testing/915-units-attr/run_script.py +++ b/auxiliary_tools/cdat_regression_testing/915-units-attr/run_script.py @@ -1,9 +1,9 @@ import os + import numpy + from e3sm_diags.parameter.core_parameter import CoreParameter from e3sm_diags.parameter.lat_lon_land_parameter import LatLonLandParameter - - from e3sm_diags.run import runner short_name = "v3.LR.historical_0051" diff --git a/auxiliary_tools/cdat_regression_testing/base_run_script.py b/auxiliary_tools/cdat_regression_testing/base_run_script.py index 0aaa6a70a..6fff17557 100644 --- a/auxiliary_tools/cdat_regression_testing/base_run_script.py +++ b/auxiliary_tools/cdat_regression_testing/base_run_script.py @@ -6,7 +6,7 @@ import os import sys -from typing import Literal, List, Tuple, TypedDict +from typing import List, Literal, Tuple, TypedDict from mache import MachineInfo diff --git a/auxiliary_tools/cdat_regression_testing/template_cdat_regression_test_json.ipynb b/auxiliary_tools/cdat_regression_testing/template_cdat_regression_test_json.ipynb index e782d1869..cab74a9e9 100644 --- a/auxiliary_tools/cdat_regression_testing/template_cdat_regression_test_json.ipynb +++ b/auxiliary_tools/cdat_regression_testing/template_cdat_regression_test_json.ipynb @@ -59,20 +59,20 @@ } ], "source": [ - "from collections import defaultdict\n", "import glob\n", + "from collections import defaultdict\n", "\n", - "import pandas as pd\n", "import numpy as np\n", + "import pandas as pd\n", "import xarray as xr\n", "\n", "from auxiliary_tools.cdat_regression_testing.utils import (\n", " get_metrics,\n", " get_num_metrics_above_diff_thres,\n", " get_rel_diffs,\n", + " highlight_large_diffs,\n", " sort_columns,\n", " update_diffs_to_pct,\n", - " highlight_large_diffs,\n", ")\n", "\n", "# TODO: Update SET_NAME and SET_DIR\n", diff --git a/auxiliary_tools/cdat_regression_testing/template_cdat_regression_test_netcdf.ipynb b/auxiliary_tools/cdat_regression_testing/template_cdat_regression_test_netcdf.ipynb index 37528e228..846d3a2f6 100644 --- a/auxiliary_tools/cdat_regression_testing/template_cdat_regression_test_netcdf.ipynb +++ b/auxiliary_tools/cdat_regression_testing/template_cdat_regression_test_netcdf.ipynb @@ -41,14 +41,14 @@ "metadata": {}, "outputs": [], "source": [ - "from collections import defaultdict\n", "import glob\n", + "from collections import defaultdict\n", "from typing import Tuple\n", "\n", "import numpy as np\n", "import xarray as xr\n", - "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", + "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", "\n", "# TODO: Update SET_NAME and SET_DIR\n", "SET_NAME = \"cosp_histogram\"\n", diff --git a/auxiliary_tools/image_checker.py b/auxiliary_tools/image_checker.py index e1db49378..84a3d9e56 100644 --- a/auxiliary_tools/image_checker.py +++ b/auxiliary_tools/image_checker.py @@ -1,6 +1,8 @@ import os + from PIL import Image, ImageChops + def images_are_identical(img1_path, img2_path): # Open both images with Image.open(img1_path) as img1, Image.open(img2_path) as img2: diff --git a/auxiliary_tools/run_tc_analysis.py b/auxiliary_tools/run_tc_analysis.py index 0fc344c9b..625c2e3d3 100644 --- a/auxiliary_tools/run_tc_analysis.py +++ b/auxiliary_tools/run_tc_analysis.py @@ -1,4 +1,5 @@ import os + from e3sm_diags.parameter.core_parameter import CoreParameter from e3sm_diags.parameter.tc_analysis_parameter import TCAnalysisParameter from e3sm_diags.run import runner diff --git a/auxiliary_tools/summarize_e3sm_diags.py b/auxiliary_tools/summarize_e3sm_diags.py index 768a16448..03dbb6464 100755 --- a/auxiliary_tools/summarize_e3sm_diags.py +++ b/auxiliary_tools/summarize_e3sm_diags.py @@ -1,10 +1,21 @@ #!/usr/bin/env python -import sys, string, copy, glob, os, json, argparse, textwrap, functools +import argparse +import copy +import functools +import glob +import json +import os +import string +import sys +import textwrap + import matplotlib + matplotlib.use('AGG') import matplotlib.pyplot as pl + def is_none(a): return a == None def is_empty(a): return len(a) == 0 def is_zero(a): return a == 0 diff --git a/auxiliary_tools/tropical_subseasonal_diags/run_tropical_subseasonal.py b/auxiliary_tools/tropical_subseasonal_diags/run_tropical_subseasonal.py index 45b6d6f61..c4b7ff53d 100644 --- a/auxiliary_tools/tropical_subseasonal_diags/run_tropical_subseasonal.py +++ b/auxiliary_tools/tropical_subseasonal_diags/run_tropical_subseasonal.py @@ -1,5 +1,8 @@ import os -from e3sm_diags.parameter.tropical_subseasonal_parameter import TropicalSubseasonalParameter + +from e3sm_diags.parameter.tropical_subseasonal_parameter import ( + TropicalSubseasonalParameter, +) from e3sm_diags.run import runner param = TropicalSubseasonalParameter() diff --git a/auxiliary_tools/tropical_subseasonal_diags/tropical_subseasonal_driver.py b/auxiliary_tools/tropical_subseasonal_diags/tropical_subseasonal_driver.py index 1bcadd3ec..64c8f40e1 100755 --- a/auxiliary_tools/tropical_subseasonal_diags/tropical_subseasonal_driver.py +++ b/auxiliary_tools/tropical_subseasonal_diags/tropical_subseasonal_driver.py @@ -7,19 +7,18 @@ import os from typing import TYPE_CHECKING # , Optional +import matplotlib.pyplot as plt import numpy as np import xarray as xr +from matplotlib.colors import BoundaryNorm, ListedColormap +from tropical_subseasonal_plot import plot +from tropical_subseasonal_viewer import create_viewer +from zwf import zwf_functions as wf import e3sm_diags from e3sm_diags.driver import utils from e3sm_diags.logger import custom_logger from e3sm_diags.plot.mp_partition_plot import plot -import matplotlib.pyplot as plt -from matplotlib.colors import ListedColormap, BoundaryNorm - -from tropical_subseasonal_plot import plot -from tropical_subseasonal_viewer import create_viewer -from zwf import zwf_functions as wf logger = custom_logger(__name__) diff --git a/auxiliary_tools/tropical_subseasonal_diags/tropical_subseasonal_plot.py b/auxiliary_tools/tropical_subseasonal_diags/tropical_subseasonal_plot.py index 7a096684a..4cf2786d1 100755 --- a/auxiliary_tools/tropical_subseasonal_diags/tropical_subseasonal_plot.py +++ b/auxiliary_tools/tropical_subseasonal_diags/tropical_subseasonal_plot.py @@ -1,19 +1,19 @@ from __future__ import annotations +import os from typing import TYPE_CHECKING import matplotlib -import xarray as xr - -from e3sm_diags.logger import custom_logger -from e3sm_diags.parameter.core_parameter import CoreParameter import matplotlib.pyplot as plt -from matplotlib.colors import ListedColormap, BoundaryNorm import numpy as np -import os +import xarray as xr +from matplotlib.colors import BoundaryNorm, ListedColormap + #from e3sm_diags.plot.utils import _add_colormap, _save_plot from zwf import zwf_functions as wf +from e3sm_diags.logger import custom_logger +from e3sm_diags.parameter.core_parameter import CoreParameter if TYPE_CHECKING: from e3sm_diags.driver.lat_lon_driver import MetricsDict diff --git a/auxiliary_tools/tropical_subseasonal_diags/tropical_subseasonal_viewer.py b/auxiliary_tools/tropical_subseasonal_diags/tropical_subseasonal_viewer.py index 30475c1c0..813d528a1 100755 --- a/auxiliary_tools/tropical_subseasonal_diags/tropical_subseasonal_viewer.py +++ b/auxiliary_tools/tropical_subseasonal_diags/tropical_subseasonal_viewer.py @@ -5,7 +5,6 @@ from e3sm_diags.logger import custom_logger - logger = custom_logger(__name__) diff --git a/auxiliary_tools/tropical_subseasonal_diags/zwf/zwf_functions.py b/auxiliary_tools/tropical_subseasonal_diags/zwf/zwf_functions.py index 1b5f6ed81..33b8a9264 100755 --- a/auxiliary_tools/tropical_subseasonal_diags/zwf/zwf_functions.py +++ b/auxiliary_tools/tropical_subseasonal_diags/zwf/zwf_functions.py @@ -1,8 +1,10 @@ +import logging import sys -import xarray as xr + import numpy as np +import xarray as xr from scipy.signal import convolve2d, detrend -import logging + logging.basicConfig(level=logging.INFO) diff --git a/auxiliary_tools/tropical_subseasonal_diags/zwf/zwf_plot_DIFF.py b/auxiliary_tools/tropical_subseasonal_diags/zwf/zwf_plot_DIFF.py index d56fd3f1d..e66c1aa3d 100755 --- a/auxiliary_tools/tropical_subseasonal_diags/zwf/zwf_plot_DIFF.py +++ b/auxiliary_tools/tropical_subseasonal_diags/zwf/zwf_plot_DIFF.py @@ -1,12 +1,13 @@ import sys + +import matplotlib.pyplot as plt import numpy as np import xarray as xr -import matplotlib.pyplot as plt + # our local module: import zwf_functions as wf - - + def plot_diff_spectrum(z, ofil=None, clevs=None, do_zoom=False, titleStr="Main title", leftStr="Left title", rightStr="Right title", cbar_label="cbar_label", disp_col='black', disp_thk=1.5, disp_alpha=0.60, diff --git a/conda-env/dev.yml b/conda-env/dev.yml index 339e0c2ec..8c009787d 100644 --- a/conda-env/dev.yml +++ b/conda-env/dev.yml @@ -38,13 +38,10 @@ dependencies: # ======================= # Run `pre-commit autoupdate` to get the latest pinned versions of 'rev' in # `.pre-commit.config.yaml`, then update the pinned versions here. - - black=23.9.1 - - flake8=6.1.0 - - flake8-isort=6.1.0 - - isort=5.12.0 - - mypy=1.5.1 - pre-commit >=3.0.0 - types-PyYAML >=6.0.0 + - ruff=0.9.2 + - mypy=1.14.1 # Developer Tools # ======================= - tbump=6.9.0 diff --git a/docs/source/dev_guide/project-standards.rst b/docs/source/dev_guide/project-standards.rst index 4d10c615b..504c3d498 100644 --- a/docs/source/dev_guide/project-standards.rst +++ b/docs/source/dev_guide/project-standards.rst @@ -68,7 +68,7 @@ Manually run all pre-commit hooks :: Run individual hook :: - # Available hook ids: trailing-whitespace, end-of-file-fixer, check-yaml, black, isort, flake8, mypy + # Available hook ids: trailing-whitespace, end-of-file-fixer, check-yaml, ruff, and mypy pre-commit run Squash and Rebase Commits @@ -131,9 +131,8 @@ Code Quality Assurance This project uses several tools for code formatting, linting, and type checking listed below. -- Code Formatting: `black `__ -- Linting: `flake8 `__, `isort `__ -- Optional Type Checking: `mypy `__ +- `Ruff `__ for code formatting and linting. +- `mypy `_` for type checking.` You can run them as hooks manually/automatically when committing using ``pre-commit``, or manually through the terminal or IDE/text editor. @@ -143,7 +142,7 @@ Helpful Commands Run a tool :: - # Available tool names: black, flake8, isort, mypy + # Available tool names: ruff format, ruff check, mypy . .. _ci-cd: @@ -174,7 +173,7 @@ This project uses `GitHub Actions `__. +This project follows the Ruff code style. Please read about it more `here `__. API Documentation ----------------- diff --git a/e3sm_diags/derivations/derivations.py b/e3sm_diags/derivations/derivations.py index b4da31044..77f6dfbda 100644 --- a/e3sm_diags/derivations/derivations.py +++ b/e3sm_diags/derivations/derivations.py @@ -12,6 +12,7 @@ 3. 'PRECC' and 'PRECL' are used to derive `PRECT` by passing the data for these variables to the formula function 'prect()'. """ + from collections import OrderedDict from typing import Callable, Dict, Tuple, Union diff --git a/e3sm_diags/derivations/formulas.py b/e3sm_diags/derivations/formulas.py index f6200986f..843fef624 100644 --- a/e3sm_diags/derivations/formulas.py +++ b/e3sm_diags/derivations/formulas.py @@ -5,6 +5,7 @@ the arithmetic should be wrapped with `with xr.set_options(keep_attrs=True)` to keep attributes on the resultant `xr.DataArray`. """ + from typing import List import numpy as np diff --git a/e3sm_diags/derivations/formulas_cosp.py b/e3sm_diags/derivations/formulas_cosp.py index 0131d6e6c..7f55d5975 100644 --- a/e3sm_diags/derivations/formulas_cosp.py +++ b/e3sm_diags/derivations/formulas_cosp.py @@ -213,7 +213,7 @@ def _get_prs_subset_range( """ act_range = (prs[0].item(), prs[-1].item()) range: List[float] = [] - + # FIXME: B905: zip() without an explicit strict= parameter for act_val, adj_val in zip(act_range, prs_adj_range): if adj_val is not None: if prs.name in PRS_UNIT_ADJ_MAP.keys() and prs.max().item() > 1000: @@ -259,7 +259,7 @@ def _get_tau_subset_range_and_str( final_range = tuple(range) - return final_range, range_str # type: ignore + return final_range, range_str def _get_prs_and_tau_cond( diff --git a/e3sm_diags/derivations/utils.py b/e3sm_diags/derivations/utils.py index 05111037d..e2ec84174 100644 --- a/e3sm_diags/derivations/utils.py +++ b/e3sm_diags/derivations/utils.py @@ -2,6 +2,7 @@ This module defines general utilities for deriving variables, including unit conversion functions, renaming variables, etc. """ + import cf_units import xarray as xr diff --git a/e3sm_diags/driver/aerosol_budget_driver.py b/e3sm_diags/driver/aerosol_budget_driver.py index faf0c4005..dc8ff4e08 100644 --- a/e3sm_diags/driver/aerosol_budget_driver.py +++ b/e3sm_diags/driver/aerosol_budget_driver.py @@ -215,7 +215,7 @@ def _calc_column_integral( # if ABURDEN terms are available, use them var_key = f"ABURDEN{aerosol_name}" burden = test_ds.get_climo_dataset(var_key, season)[var_key] - except IOError: + except IOError as err: # if not, use the Mass_ terms and integrate over the column var_key = f"Mass_{aerosol}" ds_mass = test_ds.get_climo_dataset(var_key, season) @@ -226,7 +226,7 @@ def _calc_column_integral( f"ERROR in aerosol_budget_driver/calc_column_integral!" f"Mass_{aerosol} at season {season} has units {mass.units}." f"But kg/kg units were expected." - ) + ) from err pressure_levs = _hybrid_to_pressure( ds_mass, var_key, p0=100000.0, a_key="hyai", b_key="hybi" diff --git a/e3sm_diags/driver/annual_cycle_zonal_mean_driver.py b/e3sm_diags/driver/annual_cycle_zonal_mean_driver.py index 672282547..2dcf64ce0 100755 --- a/e3sm_diags/driver/annual_cycle_zonal_mean_driver.py +++ b/e3sm_diags/driver/annual_cycle_zonal_mean_driver.py @@ -139,9 +139,14 @@ def _run_diags_annual_cycle( parameter.regrid_method, ) - test_zonal_mean: xr.DataArray = spatial_avg(ds_test, var_key, axis=["X"], as_list=False) # type: ignore + test_zonal_mean: xr.DataArray = spatial_avg( + ds_test, var_key, axis=["X"], as_list=False + ) # type: ignore test_reg_zonal_mean: xr.DataArray = spatial_avg( - ds_test_reg, var_key, axis=["X"], as_list=False # type: ignore + ds_test_reg, + var_key, + axis=["X"], + as_list=False, # type: ignore ) if ( @@ -156,13 +161,19 @@ def _run_diags_annual_cycle( else: ref_zonal_mean = spatial_avg(ds_ref, var_key, axis=["X"], as_list=False) # type: ignore ref_reg_zonal_mean = spatial_avg( - ds_ref_reg, var_key, axis=["X"], as_list=False # type: ignore + ds_ref_reg, + var_key, + axis=["X"], + as_list=False, # type: ignore ) else: ref_zonal_mean = spatial_avg(ds_ref, var_key, axis=["X"], as_list=False) # type: ignore ref_reg_zonal_mean = spatial_avg( - ds_ref_reg, var_key, axis=["X"], as_list=False # type: ignore + ds_ref_reg, + var_key, + axis=["X"], + as_list=False, # type: ignore ) # Make a copy of dataset to preserve time dimension diff --git a/e3sm_diags/driver/arm_diags_driver.py b/e3sm_diags/driver/arm_diags_driver.py index aa185bbe5..8abaf0518 100644 --- a/e3sm_diags/driver/arm_diags_driver.py +++ b/e3sm_diags/driver/arm_diags_driver.py @@ -161,7 +161,7 @@ def _run_diag_diurnal_cycle_zt(parameter: ARMDiagsParameter) -> ARMDiagsParamete ref_path = parameter.reference_data_path seasons = SEASONS_BY_DIAG["diurnal_cycle_zt"] - plevs = list(np.linspace(100, 1000, 37)) + plevs: List[float] = list(np.linspace(100, 1000, 37)) for region in regions: logger.info(f"Selected region: {region}") diff --git a/e3sm_diags/driver/cosp_histogram_driver.py b/e3sm_diags/driver/cosp_histogram_driver.py index 3de596487..afed83fa5 100755 --- a/e3sm_diags/driver/cosp_histogram_driver.py +++ b/e3sm_diags/driver/cosp_histogram_driver.py @@ -137,6 +137,7 @@ def _align_test_to_ref_dims( # NOTE: This logic assumes that prs and tau are in the same order for # the test and ref variables. If they are not, then this will break or # perform incorrect arithmetic. + # FIXME: B905: zip() without an explicit strict= parameter da_test_new = da_test_new.rename( {dim1: dim2 for dim1, dim2 in zip(da_test.dims, da_ref.dims)} ) diff --git a/e3sm_diags/driver/diurnal_cycle_driver.py b/e3sm_diags/driver/diurnal_cycle_driver.py index 83f2a66fc..8e5070f18 100644 --- a/e3sm_diags/driver/diurnal_cycle_driver.py +++ b/e3sm_diags/driver/diurnal_cycle_driver.py @@ -76,16 +76,12 @@ def run_diag(parameter: DiurnalCycleParameter) -> DiurnalCycleParameter: test_cmean, test_amplitude, test_maxtime, - ) = composite_diurnal_cycle( - test_domain, var_key, season - ) # type: ignore + ) = composite_diurnal_cycle(test_domain, var_key, season) # type: ignore ( ref_cmean, ref_amplitude, ref_maxtime, - ) = composite_diurnal_cycle( - ref_domain, var_key, season - ) # type: ignore + ) = composite_diurnal_cycle(ref_domain, var_key, season) # type: ignore parameter.var_region = region diff --git a/e3sm_diags/driver/enso_diags_driver.py b/e3sm_diags/driver/enso_diags_driver.py index dd376dc4f..b7846c6a9 100644 --- a/e3sm_diags/driver/enso_diags_driver.py +++ b/e3sm_diags/driver/enso_diags_driver.py @@ -384,9 +384,9 @@ def calculate_nino_index_obs( try: start_ind = np.where(sst_years == start)[0][0] end_ind = np.where(sst_years == end)[0][0] - except Exception: + except Exception as err: msg = "Requested years are outside of available sst obs records." - raise RuntimeError(msg) + raise RuntimeError(msg) from err sst = sst_orig[start_ind : end_ind + 1, 1:] @@ -608,4 +608,4 @@ def _get_contour_levels(metrics_dict: MetricsDictMap) -> List[float]: contour_levels = list(np.arange(lower_bound, upper_bound + 1, step_size)) - return contour_levels + return contour_levels # type: ignore diff --git a/e3sm_diags/driver/mp_partition_driver.py b/e3sm_diags/driver/mp_partition_driver.py index f1602ce0b..e7ebb4942 100644 --- a/e3sm_diags/driver/mp_partition_driver.py +++ b/e3sm_diags/driver/mp_partition_driver.py @@ -4,6 +4,7 @@ Zhang and Yuying Zhang, with contribution from Yunpeng Shan, Jiwen Fan, Xue Zheng and Susannah Burrows. """ + from __future__ import annotations import glob diff --git a/e3sm_diags/driver/qbo_driver.py b/e3sm_diags/driver/qbo_driver.py index 3379f4c46..b92092762 100644 --- a/e3sm_diags/driver/qbo_driver.py +++ b/e3sm_diags/driver/qbo_driver.py @@ -269,10 +269,10 @@ def _get_power_spectral_density(var_avg: xr.DataArray): # Average over vertical try: average = var_avg.sel({z_dim: slice(level_top, level_bottom)}) - except Exception: + except Exception as err: raise Exception( "No levels found between {}hPa and {}hPa".format(level_top, level_bottom) - ) + ) from err x0 = np.nanmean(average.values, axis=1) diff --git a/e3sm_diags/driver/tc_analysis_driver.py b/e3sm_diags/driver/tc_analysis_driver.py index f59045d20..2be70e281 100644 --- a/e3sm_diags/driver/tc_analysis_driver.py +++ b/e3sm_diags/driver/tc_analysis_driver.py @@ -14,6 +14,7 @@ if TYPE_CHECKING: from numpy.ma.core import MaskedArray + from e3sm_diags.parameter.tc_analysis_parameter import TCAnalysisParameter from e3sm_diags.logger import custom_logger diff --git a/e3sm_diags/driver/tropical_subseasonal_driver.py b/e3sm_diags/driver/tropical_subseasonal_driver.py index acea8d54d..d336836ff 100644 --- a/e3sm_diags/driver/tropical_subseasonal_driver.py +++ b/e3sm_diags/driver/tropical_subseasonal_driver.py @@ -6,6 +6,7 @@ Authors: Jim Benedict and Brian Medeiros Modified by Jill Zhang to integrate into E3SM Diags. """ + from __future__ import annotations import glob diff --git a/e3sm_diags/driver/utils/dataset_xr.py b/e3sm_diags/driver/utils/dataset_xr.py index b740a8ec1..aa9aad5b8 100644 --- a/e3sm_diags/driver/utils/dataset_xr.py +++ b/e3sm_diags/driver/utils/dataset_xr.py @@ -192,7 +192,7 @@ def _get_derived_vars_map(self) -> DerivedVariablesMap: function. """ dvars: DerivedVariablesMap = DERIVED_VARIABLES.copy() - user_dvars: DerivedVariablesMap = getattr(self.parameter, "derived_variables") + user_dvars: DerivedVariablesMap = self.parameter.derived_variables # If the user-defined derived vars already exist, create a # new OrderedDict that combines the user-defined entries with the @@ -201,7 +201,7 @@ def _get_derived_vars_map(self) -> DerivedVariablesMap: if user_dvars is not None: for key, ordered_dict in user_dvars.items(): if key in dvars.keys(): - dvars[key] = collections.OrderedDict(**ordered_dict, **dvars[key]) + dvars[key] = collections.OrderedDict(**ordered_dict, **dvars[key]) # type: ignore else: dvars[key] = ordered_dict @@ -496,7 +496,7 @@ def _open_climo_dataset(self, filepath: str) -> xr.Dataset: if "dimension 'time' already exists as a scalar variable" in msg: ds = xc.open_mfdataset(**args, drop_variables=["time"]) else: - raise ValueError(msg) + raise ValueError(msg) from e if "time" not in ds.coords: ds["time"] = xr.DataArray( @@ -1483,7 +1483,7 @@ def _get_land_sea_mask_dataset(self, season: str) -> xr.Dataset | None: ocn_keys = FRAC_REGION_KEYS["ocean"] datasets = [] - + # FIXME: B905: zip() without an explicit strict= parameter for land_key, ocn_key in zip(land_keys, ocn_keys): try: ds_land = self.get_climo_dataset(land_key, season) # type: ignore diff --git a/e3sm_diags/driver/utils/diurnal_cycle_xr.py b/e3sm_diags/driver/utils/diurnal_cycle_xr.py index 14706f4e5..c7c89c95d 100644 --- a/e3sm_diags/driver/utils/diurnal_cycle_xr.py +++ b/e3sm_diags/driver/utils/diurnal_cycle_xr.py @@ -174,11 +174,11 @@ def _get_time(ds: xr.Dataset, var_key: str) -> xr.DataArray: try: time = xc.get_dim_coords(ds_decoded, axis="T") - except (ValueError, KeyError): + except (ValueError, KeyError) as err: raise KeyError( f"This variable ({var_key}) does not have a time axis. " "Climatology cannot be run on this variable." - ) + ) from err return time diff --git a/e3sm_diags/driver/utils/general.py b/e3sm_diags/driver/utils/general.py index 168201444..fab0ad0eb 100644 --- a/e3sm_diags/driver/utils/general.py +++ b/e3sm_diags/driver/utils/general.py @@ -8,8 +8,10 @@ def monotonic(L): def _monotonically_decreasing(L): + # FIXME: B905: zip() without an explicit strict= parameter return all(x >= y for x, y in zip(L, L[1:])) def _monotonically_increasing(L): + # FIXME: B905: zip() without an explicit strict= parameter return all(x <= y for x, y in zip(L, L[1:])) diff --git a/e3sm_diags/driver/utils/io.py b/e3sm_diags/driver/utils/io.py index 13bf5dc16..0e48dc88f 100644 --- a/e3sm_diags/driver/utils/io.py +++ b/e3sm_diags/driver/utils/io.py @@ -239,6 +239,6 @@ def _get_output_dir(parameter: CoreParameter): # For parallel runs, raise errors for all cases except when a # process already created the directory. if e.errno != errno.EEXIST: - raise OSError(e) + raise OSError(e) from e return dir_path diff --git a/e3sm_diags/e3sm_diags_driver.py b/e3sm_diags/e3sm_diags_driver.py index 57ed8aed9..b1726718c 100644 --- a/e3sm_diags/e3sm_diags_driver.py +++ b/e3sm_diags/e3sm_diags_driver.py @@ -171,7 +171,10 @@ def save_provenance(results_dir, parser): _save_python_script(results_dir, parser) -def get_parameters(parser=CoreParser()): +# FIXME: B008 Do not perform function call `CoreParser` in argument defaults; +# instead, perform the call within the function, or read the default from a +# module-level singleton variable +def get_parameters(parser=CoreParser()): # noqa B008 """ Get the parameters from the parser. """ @@ -348,7 +351,8 @@ def _collapse_results(parameters: List[List[CoreParameter]]) -> List[CoreParamet return output_parameters -def main(parameters=[]) -> List[CoreParameter]: +# FIXME: B006 Do not use mutable data structures for argument defaults +def main(parameters=[]) -> List[CoreParameter]: # noqa B006 # Get the diagnostic run parameters # --------------------------------- parser = CoreParser() diff --git a/e3sm_diags/logger.py b/e3sm_diags/logger.py index 8b393f474..947b0508d 100644 --- a/e3sm_diags/logger.py +++ b/e3sm_diags/logger.py @@ -1,4 +1,5 @@ """Logger module for setting up a custom logger.""" + import logging import logging.handlers import os diff --git a/e3sm_diags/metrics/metrics.py b/e3sm_diags/metrics/metrics.py index d98fe519d..2c4fda1c7 100644 --- a/e3sm_diags/metrics/metrics.py +++ b/e3sm_diags/metrics/metrics.py @@ -265,11 +265,11 @@ def _get_z_weights(ds: xr.Dataset, var_key: str) -> xr.DataArray: """ try: z_bnds = ds.bounds.get_bounds("Z", var_key) - except KeyError: + except KeyError as err: raise RuntimeError( f"The dataset for {var_key} has no Z axis bounds to get weights for the Z " "axis." - ) + ) from err weights = np.abs(z_bnds[:, 1] - z_bnds[:, 0]) weights = weights.fillna(0) diff --git a/e3sm_diags/parser/area_mean_time_series_parser.py b/e3sm_diags/parser/area_mean_time_series_parser.py index 2b46e7455..25166448f 100644 --- a/e3sm_diags/parser/area_mean_time_series_parser.py +++ b/e3sm_diags/parser/area_mean_time_series_parser.py @@ -6,7 +6,9 @@ class AreaMeanTimeSeriesParser(CoreParser): def __init__(self, *args, **kwargs): - super().__init__(parameter_cls=AreaMeanTimeSeriesParameter, *args, **kwargs) # type: ignore + # FIXME: B026 Star-arg unpacking after a keyword argument is strongly + # discouraged + super().__init__(parameter_cls=AreaMeanTimeSeriesParameter, *args, **kwargs) # type: ignore # noqa: B026 def add_arguments(self): super().add_arguments() diff --git a/e3sm_diags/parser/arm_diags_parser.py b/e3sm_diags/parser/arm_diags_parser.py index 5209b6873..ef2d986d9 100644 --- a/e3sm_diags/parser/arm_diags_parser.py +++ b/e3sm_diags/parser/arm_diags_parser.py @@ -4,7 +4,8 @@ class ARMDiagsParser(CoreParser): def __init__(self, *args, **kwargs): - super().__init__(parameter_cls=ARMDiagsParameter, *args, **kwargs) # type: ignore + # FIXME: B026 Star-arg unpacking after a keyword argument is strongly discouraged + super().__init__(parameter_cls=ARMDiagsParameter, *args, **kwargs) # type: ignore # noqa B026 def add_arguments(self): super().add_arguments() diff --git a/e3sm_diags/parser/core_parser.py b/e3sm_diags/parser/core_parser.py index d823de253..37a7e5a0e 100644 --- a/e3sm_diags/parser/core_parser.py +++ b/e3sm_diags/parser/core_parser.py @@ -22,7 +22,9 @@ def __init__(self, parameter_cls=CoreParameter, *args, **kwargs): # "resolve" allows arguments to be overriden. conflict_handler="resolve", formatter_class=argparse.ArgumentDefaultsHelpFormatter, - *args, + # FIXME: B026 Star-arg unpacking after a keyword argument + # is strongly discouraged + *args, # noqa: B026 **kwargs, ) self.add_arguments() @@ -608,7 +610,7 @@ def parse_args( namespace, _ = self.parser.parse_known_args(args, namespace) - return namespace + return namespace # type: ignore def _remove_ipykernel_args(self) -> List[str]: """Removes `sys.argv` arguments set by `ipykernel`. @@ -664,7 +666,8 @@ def get_parameters( self, cmdline_parameters=None, orig_parameters=None, - other_parameters=[], + # FIXME: B006 Do not use mutable data structures for argument defaults + other_parameters=[], # noqa: B006 default_vars=True, cmd_default_vars=True, *args, @@ -754,7 +757,8 @@ def get_orig_parameters(self, check_values=False, argparse_vals_only=True): def get_cfg_parameters( self, - files_to_open: List[str] = [], + # FIXME: B006 Do not use mutable data structures for argument defaults + files_to_open: List[str] = [], # noqa: B006 check_values: bool = False, argparse_vals_only: bool = True, ) -> List[CoreParameter]: @@ -1065,7 +1069,8 @@ def _combine_params( cmdline_parameters=None, orig_parameters=None, other_parameters=None, - vars_to_ignore=[], + # FIXME: B006 Do not use mutable data structures for argument defaults + vars_to_ignore=[], # noqa: B006 default_vars=False, cmd_default_vars=False, ): diff --git a/e3sm_diags/parser/diurnal_cycle_parser.py b/e3sm_diags/parser/diurnal_cycle_parser.py index 719e27c8a..7194947d9 100644 --- a/e3sm_diags/parser/diurnal_cycle_parser.py +++ b/e3sm_diags/parser/diurnal_cycle_parser.py @@ -4,7 +4,8 @@ class DiurnalCycleParser(CoreParser): def __init__(self, *args, **kwargs): - super().__init__(parameter_cls=DiurnalCycleParameter, *args, **kwargs) # type: ignore + # FIXME: B026 Star-arg unpacking after a keyword argument is strongly discouraged + super().__init__(parameter_cls=DiurnalCycleParameter, *args, **kwargs) # type: ignore # noqa: B026 def add_arguments(self): super().add_arguments() diff --git a/e3sm_diags/parser/enso_diags_parser.py b/e3sm_diags/parser/enso_diags_parser.py index 9bfbc76a0..d6f973631 100644 --- a/e3sm_diags/parser/enso_diags_parser.py +++ b/e3sm_diags/parser/enso_diags_parser.py @@ -4,7 +4,8 @@ class EnsoDiagsParser(CoreParser): def __init__(self, *args, **kwargs): - super().__init__(parameter_cls=EnsoDiagsParameter, *args, **kwargs) # type: ignore + # FIXME: B026 Star-arg unpacking after a keyword argument is strongly discouraged + super().__init__(parameter_cls=EnsoDiagsParameter, *args, **kwargs) # type: ignore # noqa: B026 def add_arguments(self): super().add_arguments() diff --git a/e3sm_diags/parser/meridional_mean_2d_parser.py b/e3sm_diags/parser/meridional_mean_2d_parser.py index 6eb22f7d5..e31facf85 100644 --- a/e3sm_diags/parser/meridional_mean_2d_parser.py +++ b/e3sm_diags/parser/meridional_mean_2d_parser.py @@ -3,7 +3,8 @@ class MeridionalMean2dParser(CoreParser): def __init__(self, *args, **kwargs): - super().__init__(parameter_cls=MeridionalMean2dParser, *args, **kwargs) # type: ignore + # FIXME: B026 Star-arg unpacking after a keyword argument is strongly discouraged + super().__init__(parameter_cls=MeridionalMean2dParser, *args, **kwargs) # type: ignore # noqa: B026 def add_arguments(self): super().add_arguments() diff --git a/e3sm_diags/parser/mp_partition_parser.py b/e3sm_diags/parser/mp_partition_parser.py index 41eb3e248..9757b4eb5 100644 --- a/e3sm_diags/parser/mp_partition_parser.py +++ b/e3sm_diags/parser/mp_partition_parser.py @@ -4,9 +4,8 @@ class MPpartitionParser(CoreParser): def __init__(self, *args, **kwargs): - super().__init__( - parameter_cls=MPpartitionParameter, *args, **kwargs - ) # type:ignore + # FIXME: B026 Star-arg unpacking after a keyword argument is strongly discouraged + super().__init__(parameter_cls=MPpartitionParameter, *args, **kwargs) # type: ignore # noqa: B026 def add_arguments(self): super().add_arguments() diff --git a/e3sm_diags/parser/qbo_parser.py b/e3sm_diags/parser/qbo_parser.py index 3fe69dbdb..95d930ce1 100644 --- a/e3sm_diags/parser/qbo_parser.py +++ b/e3sm_diags/parser/qbo_parser.py @@ -4,7 +4,8 @@ class QboParser(CoreParser): def __init__(self, *args, **kwargs): - super().__init__(parameter_cls=QboParameter, *args, **kwargs) # type:ignore + # FIXME: B026 Star-arg unpacking after a keyword argument is strongly discouraged + super().__init__(parameter_cls=QboParameter, *args, **kwargs) # type:ignore # noqa: B026 def add_arguments(self): super().add_arguments() diff --git a/e3sm_diags/parser/streamflow_parser.py b/e3sm_diags/parser/streamflow_parser.py index cd7d25e86..80df64a15 100644 --- a/e3sm_diags/parser/streamflow_parser.py +++ b/e3sm_diags/parser/streamflow_parser.py @@ -4,7 +4,8 @@ class StreamflowParser(CoreParser): def __init__(self, *args, **kwargs): - super().__init__(parameter_cls=StreamflowParameter, *args, **kwargs) # type: ignore + # FIXME: B026 Star-arg unpacking after a keyword argument is strongly discouraged + super().__init__(parameter_cls=StreamflowParameter, *args, **kwargs) # type: ignore # noqa: B026 def add_arguments(self): super().add_arguments() diff --git a/e3sm_diags/parser/tc_analysis_parser.py b/e3sm_diags/parser/tc_analysis_parser.py index b58be62e7..b0ecafada 100644 --- a/e3sm_diags/parser/tc_analysis_parser.py +++ b/e3sm_diags/parser/tc_analysis_parser.py @@ -4,4 +4,5 @@ class TCAnalysisParser(CoreParser): def __init__(self, *args, **kwargs): - super().__init__(parameter_cls=TCAnalysisParameter, *args, **kwargs) # type: ignore + # FIXME: B026 Star-arg unpacking after a keyword argument is strongly discouraged + super().__init__(parameter_cls=TCAnalysisParameter, *args, **kwargs) # type: ignore # noqa: B026 diff --git a/e3sm_diags/parser/tropical_subseasonal_parser.py b/e3sm_diags/parser/tropical_subseasonal_parser.py index 7e95b6fae..aab8153aa 100644 --- a/e3sm_diags/parser/tropical_subseasonal_parser.py +++ b/e3sm_diags/parser/tropical_subseasonal_parser.py @@ -6,9 +6,8 @@ class TropicalSubseasonalParser(CoreParser): def __init__(self, *args, **kwargs): - super().__init__( - parameter_cls=TropicalSubseasonalParameter, *args, **kwargs - ) # type:ignore + # FIXME: B026 Star-arg unpacking after a keyword argument is strongly discouraged + super().__init__(parameter_cls=TropicalSubseasonalParameter, *args, **kwargs) # type: ignore # noqa: B026 def add_arguments(self): super().add_arguments() diff --git a/e3sm_diags/parser/zonal_mean_2d_parser.py b/e3sm_diags/parser/zonal_mean_2d_parser.py index 88728bf14..900e28add 100644 --- a/e3sm_diags/parser/zonal_mean_2d_parser.py +++ b/e3sm_diags/parser/zonal_mean_2d_parser.py @@ -4,7 +4,8 @@ class ZonalMean2dParser(CoreParser): def __init__(self, *args, **kwargs): - super().__init__(parameter_cls=ZonalMean2dParameter, *args, **kwargs) # type: ignore + # FIXME: B026 Star-arg unpacking after a keyword argument is strongly discouraged + super().__init__(parameter_cls=ZonalMean2dParameter, *args, **kwargs) # type: ignore # noqa: B026 def add_arguments(self): super().add_arguments() diff --git a/e3sm_diags/parser/zonal_mean_2d_stratosphere_parser.py b/e3sm_diags/parser/zonal_mean_2d_stratosphere_parser.py index e3aae1ec9..e014a9fde 100644 --- a/e3sm_diags/parser/zonal_mean_2d_stratosphere_parser.py +++ b/e3sm_diags/parser/zonal_mean_2d_stratosphere_parser.py @@ -6,9 +6,12 @@ class ZonalMean2dStratosphereParser(CoreParser): def __init__(self, *args, **kwargs): - super().__init__( # type: ignore - parameter_cls=ZonalMean2dStratosphereParameter, *args, **kwargs - ) + super().__init__( + parameter_cls=ZonalMean2dStratosphereParameter, + # FIXME: B026 Star-arg unpacking after a keyword argument is strongly discouraged + *args, # noqa: B026 + **kwargs, + ) # type: ignore def add_arguments(self): super().add_arguments() diff --git a/e3sm_diags/plot/streamflow_plot_map.py b/e3sm_diags/plot/streamflow_plot_map.py index 394da4c8c..1f7bcdf67 100644 --- a/e3sm_diags/plot/streamflow_plot_map.py +++ b/e3sm_diags/plot/streamflow_plot_map.py @@ -258,6 +258,7 @@ def _plot_gauges( color_list : List[str] The list of colors to use for markers. """ + # FIXME: B905: zip() without an explicit strict= parameter for gauge, i in zip(export_data, range(len(export_data))): if panel_type == "test": value = gauge[1] diff --git a/e3sm_diags/plot/taylor_diagram.py b/e3sm_diags/plot/taylor_diagram.py index ec42526fa..0ef793849 100644 --- a/e3sm_diags/plot/taylor_diagram.py +++ b/e3sm_diags/plot/taylor_diagram.py @@ -35,7 +35,7 @@ def __init__(self, refstd, fig=None, rect=111, label="_"): tr = PolarAxes.PolarTransform() # Correlation labels - rlocs: np.ndarray = np.concatenate((np.arange(10) / 10.0, [0.95, 0.99])) # type: ignore + rlocs: np.ndarray = np.concatenate((np.arange(10) / 10.0, [0.95, 0.99])) tlocs = np.arccos(rlocs) # Conversion to polar angles # type: ignore gl1 = GF.FixedLocator(tlocs) # Positions gl2_num = np.linspace(0, 1.5, 7) @@ -43,13 +43,14 @@ def __init__(self, refstd, fig=None, rect=111, label="_"): gl2_ticks = [(x, str(x)) for x in gl2_num] gl2_ticks[-1] = [gl2_num[-1], ""] # type: ignore gl2_ticks[0] = [gl2_num[0], "0"] # type: ignore + # FIXME: B905: zip() without an explicit strict= parameter tf1 = GF.DictFormatter(dict(list(zip(tlocs, list(map(str, rlocs)))))) tf2 = GF.DictFormatter(dict(gl2_ticks)) # Standard deviation axis extent - self.smin = min(gl2_num) + self.smin = min(gl2_num) # type: ignore # self.smax = 1.5*self.refstd - self.smax = max(gl2_num) + self.smax = max(gl2_num) # type: ignore ghelper = FA.GridHelperCurveLinear( tr, @@ -97,7 +98,8 @@ def __init__(self, refstd, fig=None, rect=111, label="_"): # Add reference point and stddev contour # print "Reference std:", self.refstd - (l,) = self.ax.plot([0], self.refstd, "k*", ls="", ms=10, label=label) + # FIXME: E741 Ambiguous variable name: `l` + (l,) = self.ax.plot([0], self.refstd, "k*", ls="", ms=10, label=label) # noqa: E741 t = np.linspace(0, np.pi / 2) r = np.zeros_like(t) + self.refstd self.ax.plot(t, r, "k--", label="_") @@ -109,8 +111,8 @@ def add_sample(self, stddev, corrcoef, *args, **kwargs): """Add sample (stddev,corrcoeff) to the Taylor diagram. args and kwargs are directly propagated to the Figure.plot command.""" - - (l,) = self.ax.plot( + # FIXME: E741 Ambiguous variable name: `l` + (l,) = self.ax.plot( # noqa: E741 np.arccos(corrcoef), stddev, *args, **kwargs ) # (theta,radius) self.samplePoints.append(l) diff --git a/e3sm_diags/plot/tropical_subseasonal_plot.py b/e3sm_diags/plot/tropical_subseasonal_plot.py index 59bbf0bc8..fe6cbed55 100755 --- a/e3sm_diags/plot/tropical_subseasonal_plot.py +++ b/e3sm_diags/plot/tropical_subseasonal_plot.py @@ -401,14 +401,14 @@ def _wave_frequency_plot( # noqa: C901 z = np.log10(z) # type: ignore z.attrs["long_name"] = PlotDesc[var_name]["long_name_desc"] - z.attrs[ - "method" - ] = f"Follows {PlotDesc[var_name]['ref_fig_num']} methods of Wheeler and Kiladis (1999; https://doi.org/10.1175/1520-0469(1999)056<0374:CCEWAO>2.0.CO;2)" + z.attrs["method"] = ( + f"Follows {PlotDesc[var_name]['ref_fig_num']} methods of Wheeler and Kiladis (1999; https://doi.org/10.1175/1520-0469(1999)056<0374:CCEWAO>2.0.CO;2)" + ) if "spec_raw" in var_name and subplot_num < 2: - z.attrs[ - "ew_ratio_method" - ] = "Sum of raw (not log10) symmetric spectral power for ZWNs +/- 1-3, periods 24-96 days" + z.attrs["ew_ratio_method"] = ( + "Sum of raw (not log10) symmetric spectral power for ZWNs +/- 1-3, periods 24-96 days" + ) z.attrs["east_power"] = east_power.values z.attrs["west_power"] = west_power.values z.attrs["ew_ratio"] = ew_ratio.values diff --git a/e3sm_diags/viewer/default_viewer.py b/e3sm_diags/viewer/default_viewer.py index f96e9581b..0594ef32a 100644 --- a/e3sm_diags/viewer/default_viewer.py +++ b/e3sm_diags/viewer/default_viewer.py @@ -75,13 +75,13 @@ def create_viewer(root_dir, parameters): # } # } # } - ROW_INFO: OrderedDict[ - str, Dict[str, Dict[str, Dict[str, Dict[str, str]]]] - ] = collections.OrderedDict() + ROW_INFO: OrderedDict[str, Dict[str, Dict[str, Dict[str, Dict[str, str]]]]] = ( + collections.OrderedDict() + ) # A similar dict, but for creating the lat-lon tables. - LAT_LON_TABLE_INFO: OrderedDict[ - str, Dict[str, Dict[str, Dict[str, str]]] - ] = collections.OrderedDict() + LAT_LON_TABLE_INFO: OrderedDict[str, Dict[str, Dict[str, Dict[str, str]]]] = ( + collections.OrderedDict() + ) # Since we're only having one set for each # create_viewer() call, this works. @@ -158,16 +158,16 @@ def create_viewer(root_dir, parameters): if set_name not in ROW_INFO: ROW_INFO[set_name] = collections.OrderedDict() if parameter.case_id not in ROW_INFO[set_name]: - ROW_INFO[set_name][ - parameter.case_id - ] = collections.OrderedDict() + ROW_INFO[set_name][parameter.case_id] = ( + collections.OrderedDict() + ) if row_name not in ROW_INFO[set_name][parameter.case_id]: - ROW_INFO[set_name][parameter.case_id][ - row_name - ] = collections.OrderedDict() - ROW_INFO[set_name][parameter.case_id][row_name][ - "descr" - ] = _get_description(var, parameter) + ROW_INFO[set_name][parameter.case_id][row_name] = ( + collections.OrderedDict() + ) + ROW_INFO[set_name][parameter.case_id][row_name]["descr"] = ( + _get_description(var, parameter) + ) # Each season has a image_path and metadata linked to it, thus we use a dict. ROW_INFO[set_name][parameter.case_id][row_name][season] = {} # Format the filename to support relative paths. diff --git a/e3sm_diags/viewer/mp_partition_viewer.py b/e3sm_diags/viewer/mp_partition_viewer.py index 4ceeba0e1..dde526d84 100644 --- a/e3sm_diags/viewer/mp_partition_viewer.py +++ b/e3sm_diags/viewer/mp_partition_viewer.py @@ -36,7 +36,8 @@ def create_viewer(root_dir, parameters): image_relative_path = "{}.{}".format(relative_path, ext) if param.print_statements: logger.info("image_relative_path: {}".format(image_relative_path)) - for var in param.variables: + + for _ in param.variables: viewer.add_row("LCF") # Adding the description for this var to the current row. # This was obtained and stored in the driver for this plotset. diff --git a/examples/e3sm_diags_for_cmip/cmip6_comparison.py b/examples/e3sm_diags_for_cmip/cmip6_comparison.py index 6982dce8e..c418e4bee 100644 --- a/examples/e3sm_diags_for_cmip/cmip6_comparison.py +++ b/examples/e3sm_diags_for_cmip/cmip6_comparison.py @@ -6,12 +6,13 @@ """ import glob -import matplotlib.pyplot as plt +import os + import matplotlib.cbook as cbook -import matplotlib.transforms as mtransforms +import matplotlib.pyplot as plt import numpy as np import numpy.ma as ma -import os + # --- Function to read E3SM Diags metrics for CMIP6 models --- def read_e3sm_diags_metrics(path, variables, seasons, names=None): @@ -97,7 +98,7 @@ def write_csv(path, metrics): for iseason in range(len(metric['seasons'])): line.append( f"{metric['data'][imodel,ivariable,iseason]}" ) writer.writerow(line) - + return # --- Main --- diff --git a/examples/e3sm_diags_for_cmip/cmip6_e3sm_diags.py b/examples/e3sm_diags_for_cmip/cmip6_e3sm_diags.py index 43e1c6eee..665cba091 100644 --- a/examples/e3sm_diags_for_cmip/cmip6_e3sm_diags.py +++ b/examples/e3sm_diags_for_cmip/cmip6_e3sm_diags.py @@ -1,20 +1,19 @@ #!/usr/bin/env python3 -import numpy.ma as ma -import numpy as np -import matplotlib.pyplot as plt import matplotlib.cbook as cbook -import matplotlib.transforms as mtransforms +import matplotlib.pyplot as plt +import numpy as np +import numpy.ma as ma -# This script is to compare rmse from e3sm_diags tables and those from cmip6 models in a pre-compiled csv file. -# The script was written by Chris Golaz (Golaz et al. 2022 E3SM v2 overview paper) and adapted by Jill Zhang for implementation in e3sm_diags. +# This script is to compare rmse from e3sm_diags tables and those from cmip6 models in a pre-compiled csv file. +# The script was written by Chris Golaz (Golaz et al. 2022 E3SM v2 overview paper) and adapted by Jill Zhang for implementation in e3sm_diags. # --- Function to read CMIP6 model metrics --- -def read_cmip6_metrics_from_csv(path, variables, seasons): +def read_cmip6_metrics_from_csv(path, variables, seasons): models=[] - + with open (path, 'r') as fin: - # skip 3 header lines and last 2 E3SMv2 composites + # skip 3 header lines and last 2 E3SMv2 composites rmse = fin.readlines()[3:-2] nmodels = len(rmse) nvariables = len(variables) @@ -43,7 +42,7 @@ def read_cmip6_metrics_from_csv(path, variables, seasons): d['models'] = models.copy() d['variables'] = variables.copy() d['seasons'] = seasons.copy() - + return d # --- Function to read E3SM Diags metrics --- @@ -208,8 +207,8 @@ def read_e3sm_diags_metrics(path, variables, seasons, names=None): fig.savefig("cmip6.pdf",bbox_inches='tight') - - - - - + + + + + diff --git a/examples/e3sm_diags_for_cmip/generate_page.py b/examples/e3sm_diags_for_cmip/generate_page.py index ff216f5b3..a270b262c 100755 --- a/examples/e3sm_diags_for_cmip/generate_page.py +++ b/examples/e3sm_diags_for_cmip/generate_page.py @@ -1,14 +1,12 @@ # Script to generate html summary page of E3SM Diags applied to CMIP6 models import collections -import jinja2 import glob import os.path + +import jinja2 import pandas as pd -import pprint -import shlex -import subprocess -import utils + def tree(): return collections.defaultdict(tree) @@ -107,8 +105,8 @@ def table_elements(search): n_simulations = n_simulations - 1 print(n_simulations, ' number of simulations') pass - - + + """ # Print for verification @@ -117,7 +115,7 @@ def table_elements(search): print(s['xmls']) pp.pprint(s['data']) """ - + # Output portion to html page. # Data should be separated into header and content # - header should be a list @@ -144,7 +142,7 @@ def table_elements(search): for variable in variables: header.extend(["ANN", "DJF", "MAM", "JJA", "SON"]) - # Table content + # Table content ncols = len(header) #nrows = len(simulations) print(n_simulations, ' number of simulations') diff --git a/examples/e3sm_diags_for_cmip/generate_xmls.py b/examples/e3sm_diags_for_cmip/generate_xmls.py index 2a715361b..74bf09c51 100644 --- a/examples/e3sm_diags_for_cmip/generate_xmls.py +++ b/examples/e3sm_diags_for_cmip/generate_xmls.py @@ -4,9 +4,9 @@ import os.path import shlex import subprocess - from datetime import date + def run_command(command): p1 = subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = p1.communicate() diff --git a/examples/e3sm_diags_for_cmip/run_e3sm_diags.py b/examples/e3sm_diags_for_cmip/run_e3sm_diags.py index c5a5c41c8..6dc38fe87 100644 --- a/examples/e3sm_diags_for_cmip/run_e3sm_diags.py +++ b/examples/e3sm_diags_for_cmip/run_e3sm_diags.py @@ -1,10 +1,9 @@ # Script to generate e3sm_diags batch script for CMIP6 model output -import jinja2 import glob import os.path -import shlex -import subprocess + +import jinja2 import utils # Location of xml files @@ -37,7 +36,7 @@ c['experiment'] = p[-2] c['model'] = p[-3] c['institution'] = p[-4] - + # Create script #scriptFile = 'e3sm_diags_%s.bash' % (c['model']) scriptFile = 'e3sm_diags_{}_{}_{}.bash'.format(c['model'], c['experiment'], c['realization']) diff --git a/examples/e3sm_diags_for_cmip/utils.py b/examples/e3sm_diags_for_cmip/utils.py index c8c044de1..1f4eb6be8 100644 --- a/examples/e3sm_diags_for_cmip/utils.py +++ b/examples/e3sm_diags_for_cmip/utils.py @@ -1,7 +1,7 @@ -import os import shlex import time -from subprocess import Popen, PIPE +from subprocess import PIPE, Popen + # ----------------------------------------------------------------------------- def submitScript(scriptFile, export='ALL'): diff --git a/examples/e3sm_diags_for_eamxx/run_e3sm_diags_1996.py b/examples/e3sm_diags_for_eamxx/run_e3sm_diags_1996.py index 3a44ebd06..3b6204194 100644 --- a/examples/e3sm_diags_for_eamxx/run_e3sm_diags_1996.py +++ b/examples/e3sm_diags_for_eamxx/run_e3sm_diags_1996.py @@ -1,4 +1,5 @@ import os + from e3sm_diags.parameter.core_parameter import CoreParameter from e3sm_diags.run import runner @@ -8,7 +9,7 @@ #param.test_data_path = '/global/cfs/cdirs/e3sm/zhang40/e3sm_diags_for_EAMxx/data/Cess' #param.reference_data_path = '/global/cfs/cdirs/e3sm/diagnostics/observations/Atm/climatology' param.reference_data_path = '/global/cfs/cdirs/e3sm/diagnostics/observations/Atm/time-series' -param.test_data_path = '/global/cfs/cdirs/e3sm/chengzhu/eamxx/post/data/rgr' +param.test_data_path = '/global/cfs/cdirs/e3sm/chengzhu/eamxx/post/data/rgr' param.test_name = 'eamxx_decadal' param.seasons = ["ANN"] #param.save_netcdf = True diff --git a/examples/e3sm_diags_for_eamxx/run_e3sm_diags_climo.py b/examples/e3sm_diags_for_eamxx/run_e3sm_diags_climo.py index f8a84fe1d..43ba1f1b4 100644 --- a/examples/e3sm_diags_for_eamxx/run_e3sm_diags_climo.py +++ b/examples/e3sm_diags_for_eamxx/run_e3sm_diags_climo.py @@ -1,4 +1,5 @@ import os + from e3sm_diags.parameter.core_parameter import CoreParameter from e3sm_diags.run import runner @@ -7,7 +8,7 @@ #param.reference_data_path = '/global/cfs/cdirs/e3sm/diagnostics/observations/Atm/climatology' #param.test_data_path = '/global/cfs/cdirs/e3sm/zhang40/e3sm_diags_for_EAMxx/data/Cess' param.reference_data_path = '/global/cfs/cdirs/e3sm/diagnostics/observations/Atm/climatology' -param.test_data_path = '/global/cfs/cdirs/e3sm/chengzhu/eamxx/post/data/rgr' +param.test_data_path = '/global/cfs/cdirs/e3sm/chengzhu/eamxx/post/data/rgr' param.test_name = 'eamxx_decadal' param.seasons = ["ANN"] #param.save_netcdf = True diff --git a/examples/run_all_sets_E3SM_machines.py b/examples/run_all_sets_E3SM_machines.py index 37f43bc48..6b0f74b1c 100644 --- a/examples/run_all_sets_E3SM_machines.py +++ b/examples/run_all_sets_E3SM_machines.py @@ -17,7 +17,6 @@ """ # flake8: noqa E501 -import os from typing import Tuple, TypedDict from mache import MachineInfo diff --git a/examples/tutorials2024/e3sm_diags_core_sets.py b/examples/tutorials2024/e3sm_diags_core_sets.py index 8f3663bb6..a09108953 100644 --- a/examples/tutorials2024/e3sm_diags_core_sets.py +++ b/examples/tutorials2024/e3sm_diags_core_sets.py @@ -50,6 +50,6 @@ "meridional_mean_2d", "aerosol_aeronet", "aerosol_budget", - + ] runner.run_diags([param]) diff --git a/pyproject.toml b/pyproject.toml index d0b85f4a5..d4c85c1ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,13 +51,10 @@ dependencies = [ test = ["pytest", "pytest-cov"] docs = ["sphinx", "sphinx_rtd_theme", "sphinx-multiversion"] dev = [ - "black", - "flake8", - "flake8-isort", - "isort", - "mypy", "pre-commit", "types-PyYAML", + "ruff", + "mypy", ] [project.urls] @@ -149,39 +146,61 @@ version = { attr = "e3sm_diags.__version__" } "share/e3sm_diags/colormaps" = ["e3sm_diags/plot/colormaps/*.rgb"] "share/e3sm_diags/control_runs" = ["e3sm_diags/driver/control_runs/*"] -[tool.black] -# Docs: https://black.readthedocs.io/en/stable/usage_and_configuration/the_basics.html -line-length = 88 -target-version = ["py39", "py310"] -include = '\.pyi?$' -exclude = ''' -/( - \.eggs - | \.git - | \.hg - | \.mypy_cache - | \.tox - | \.idea - | venv - | _build - | buck-out - | build - | dist - | docs - | config - | conda - | analysis_data_preprocess -)/ -''' - -[tool.isort] -# Docs: https://pycqa.github.io/isort/docs/configuration/options.html#example-pyprojecttoml_4 -multi_line_output = 3 -include_trailing_comma = true -force_grid_wrap = 0 -use_parentheses = true -line_length = 88 -skip = "e3sm_diags/e3sm_diags_driver.py" +[tool.ruff] +# Exclude a variety of commonly ignored directories. +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "docs", + "node_modules", + "site-packages", + "venv", + "auxiliary_tools", + "analysis_data_preprocess", + "model_data_preprocess", +] + +[tool.ruff.lint] +# E4 - whitespace +# E7 - multiple-statements +# E9 - trailing-whitespace +# F - Enable Pyflakes +# B - Enable flake8-bugbear +# W - Enable pycodestyle +# C901 - complex-structure +# D - Enable flake8-docstrings +select = ["E4", "E7", "E9", "F", "B", "W", "C901"] + +# E501 - line-too-long +ignore = ["E501"] + +[tool.ruff.lint.mccabe] +# Flag errors (`C901`) whenever the complexity level exceeds 5. +max-complexity = 18 + +[tool.ruff.lint.pydocstyle] +convention = "numpy" [tool.pytest.ini_options] # Docs: https://docs.pytest.org/en/7.2.x/reference/customize.html#configuration diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 50b647306..000000000 --- a/setup.cfg +++ /dev/null @@ -1,30 +0,0 @@ -# This file is used for configuring flake8, which does not currently support pyproject.toml - -[flake8] -# https://pep8.readthedocs.io/en/latest/intro.html#error-codes -ignore = - # whitespace before ‘:’ - E203 - # line too long (82 > 79 characters) - E501 - # line break before operator - W503 -# Max width of Github code review is 119 characters -max-line-length = 119 -max-complexity = 18 -per-file-ignores = - e3sm_diags/e3sm_diags_driver.py:E402 -exclude = - .tox, - .git, - */migrations/*, - */static/CACHE/*, - docs, - node_modules, - .idea, - .mypy_cache, - .pytest_cache, - *__init__.py, - venv, - analysis_data_preprocess - model_data_preprocess diff --git a/setup.py b/setup.py deleted file mode 100644 index 696ccd781..000000000 --- a/setup.py +++ /dev/null @@ -1,174 +0,0 @@ -import glob -import os - -from setuptools import find_packages, setup - - -def get_all_files_in_dir(directory, pattern): - return glob.glob(os.path.join(directory, pattern)) - - -zonal_mean_xy_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "zonal_mean_xy*" -) -zonal_mean_xy_files += get_all_files_in_dir( - "e3sm_diags/driver/default_diags/legacy_diags", "zonal_mean_xy*" -) - -zonal_mean_2d_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "zonal_mean_2d_model*" -) -zonal_mean_2d_files += get_all_files_in_dir( - "e3sm_diags/driver/default_diags/legacy_diags", "zonal_mean_2d*" -) -zonal_mean_2d_stratosphere_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "zonal_mean_2d_stratosphere*" -) - -meridional_mean_2d_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "meridional_mean_2d*" -) - -lat_lon_files = get_all_files_in_dir("e3sm_diags/driver/default_diags", "lat_lon*") -lat_lon_files += get_all_files_in_dir( - "e3sm_diags/driver/default_diags/legacy_diags", "lat_lon*" -) - -lat_lon_vector_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "lat_lon_vector*" -) - -lat_lon_land_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "lat_lon_land*" -) -lat_lon_river_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "lat_lon_river*" -) - -polar_files = get_all_files_in_dir("e3sm_diags/driver/default_diags", "polar*") -polar_files += get_all_files_in_dir( - "e3sm_diags/driver/default_diags/legacy_diags", "polar*" -) - -cosp_histogram_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "cosp_histogram*" -) -cosp_histogram_files += get_all_files_in_dir( - "e3sm_diags/driver/default_diags/legacy_diags", "cosp_histogram*" -) - -area_mean_time_series = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "area_mean_time_series*" -) -qbo = get_all_files_in_dir("e3sm_diags/driver/default_diags", "qbo*") -streamflow = get_all_files_in_dir("e3sm_diags/driver/default_diags", "streamflow*") -enso_diags_files = get_all_files_in_dir("e3sm_diags/driver/default_diags", "enso_*") -diurnal_cycle_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "diurnal_cycle_*" -) -arm_diags_files = get_all_files_in_dir("e3sm_diags/driver/default_diags", "arm_diags_*") -tc_analysis_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "tc_analysis_*" -) -annual_cycle_zonal_mean_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "annual_cycle_zonal_mean_*" -) -aerosol_aeronet_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "aerosol_aeronet*cfg" -) -aerosol_aeronet_files += get_all_files_in_dir( - "e3sm_diags/driver/default_diags/aerosol_aeronet_data", "*.txt" -) -aerosol_budget_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "aerosol_budget*cfg" -) -mp_partition_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "mp_partition*cfg" -) -tropical_subseasonal_files = get_all_files_in_dir( - "e3sm_diags/driver/default_diags", "tropical_subseasonal*cfg" -) - -rgb_files = get_all_files_in_dir("e3sm_diags/plot/colormaps", "*.rgb") -control_runs_files = get_all_files_in_dir("e3sm_diags/driver/control_runs", "*") - -INSTALL_PATH = "share/e3sm_diags/" - -data_files = [ - (os.path.join(INSTALL_PATH, "zonal_mean_xy"), zonal_mean_xy_files), - (os.path.join(INSTALL_PATH, "zonal_mean_2d"), zonal_mean_2d_files), - ( - os.path.join(INSTALL_PATH, "zonal_mean_2d_stratosphere"), - zonal_mean_2d_stratosphere_files, - ), - ( - os.path.join(INSTALL_PATH, "meridional_mean_2d"), - meridional_mean_2d_files, - ), - (os.path.join(INSTALL_PATH, "lat_lon"), lat_lon_files), - (os.path.join(INSTALL_PATH, "polar"), polar_files), - (os.path.join(INSTALL_PATH, "lat_lon_vector"), lat_lon_vector_files), - (os.path.join(INSTALL_PATH, "lat_lon_land"), lat_lon_land_files), - (os.path.join(INSTALL_PATH, "lat_lon_river"), lat_lon_river_files), - (os.path.join(INSTALL_PATH, "cosp_histogram"), cosp_histogram_files), - ( - os.path.join(INSTALL_PATH, "area_mean_time_series"), - area_mean_time_series, - ), - (os.path.join(INSTALL_PATH, "enso_diags"), enso_diags_files), - (os.path.join(INSTALL_PATH, "qbo"), qbo), - (os.path.join(INSTALL_PATH, "streamflow"), streamflow), - (os.path.join(INSTALL_PATH, "diurnal_cycle"), diurnal_cycle_files), - (os.path.join(INSTALL_PATH, "arm_diags"), arm_diags_files), - (os.path.join(INSTALL_PATH, "tc_analysis"), tc_analysis_files), - ( - os.path.join(INSTALL_PATH, "annual_cycle_zonal_mean"), - annual_cycle_zonal_mean_files, - ), - ( - INSTALL_PATH, - [ - "e3sm_diags/driver/acme_ne30_ocean_land_mask.nc", - "misc/e3sm_logo.png", - ], - ), - (os.path.join(INSTALL_PATH, "aerosol_aeronet"), aerosol_aeronet_files), - (os.path.join(INSTALL_PATH, "aerosol_budget"), aerosol_budget_files), - (os.path.join(INSTALL_PATH, "mp_partition"), mp_partition_files), - (os.path.join(INSTALL_PATH, "tropical_subseasonal"), tropical_subseasonal_files), - (os.path.join(INSTALL_PATH, "colormaps"), rgb_files), - (os.path.join(INSTALL_PATH, "control_runs"), control_runs_files), - ( - os.path.join(INSTALL_PATH, "viewer"), - ["e3sm_diags/viewer/index_template.html"], - ), -] - -setup( - python_requires=">=3.9", - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "License :: OSI Approved :: BSD 3-Clause License", - "Natural Language :: English", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - ], - name="e3sm_diags", - version="2.12.1", - author="Chengzhu (Jill) Zhang, Tom Vo, Ryan Forsyth, Chris Golaz and Zeshawn Shaheen", - author_email="zhang40@llnl.gov", - description="E3SM Diagnostics", - license="BSD 3-Clause", - scripts=["e3sm_diags/e3sm_diags_driver.py"], - packages=find_packages(include=["e3sm_diags", "e3sm_diags.*"]), - data_files=data_files, - test_suite="tests", - entry_points={ - "console_scripts": [ - "e3sm_diags=e3sm_diags.e3sm_diags_driver:main", - "e3sm_diags_vars=e3sm_diags.e3sm_diags_vars:main", - ] - }, -) diff --git a/tests/e3sm_diags/driver/utils/test_dataset_xr.py b/tests/e3sm_diags/driver/utils/test_dataset_xr.py index d64dc443e..8408e752e 100644 --- a/tests/e3sm_diags/driver/utils/test_dataset_xr.py +++ b/tests/e3sm_diags/driver/utils/test_dataset_xr.py @@ -241,7 +241,8 @@ def test_sets_drived_vars_map_with_existing_entry(self): # The expected `derived_vars_map` result. expected = DERIVED_VARIABLES.copy() expected["PRECT"] = OrderedDict( - **parameter.derived_variables["PRECT"], **expected["PRECT"] + **parameter.derived_variables["PRECT"], + **expected["PRECT"], # type: ignore ) assert ds.derived_vars_map == expected diff --git a/tests/e3sm_diags/test_run.py b/tests/e3sm_diags/test_run.py index 3421b0aaf..317124c6a 100644 --- a/tests/e3sm_diags/test_run.py +++ b/tests/e3sm_diags/test_run.py @@ -89,7 +89,7 @@ def test_all_sets_and_all_seasons(self): season_counter["ANN"] -= set_counter["enso_diags"] season_counter["ANN"] -= set_counter["streamflow"] - for season, count in season_counter.items(): + for _, count in season_counter.items(): if count != season_counter["ANN"]: self.fail( "In .cfg files, at least one season does not match the count for ANN: {}".format( diff --git a/tests/integration/complete_run.py b/tests/integration/complete_run.py index 5d62836c7..595503ab8 100644 --- a/tests/integration/complete_run.py +++ b/tests/integration/complete_run.py @@ -10,6 +10,7 @@ - source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh - Or: source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_anvil.sh """ + import os from e3sm_diags.parameter.annual_cycle_zonal_mean_parameter import ACzonalmeanParameter diff --git a/tests/integration/test_all_sets_image_diffs.py b/tests/integration/test_all_sets_image_diffs.py index 0e15be4e8..b62bdba41 100644 --- a/tests/integration/test_all_sets_image_diffs.py +++ b/tests/integration/test_all_sets_image_diffs.py @@ -336,7 +336,7 @@ def _check_streamflow_plots(self): case_id_lower = case_id.lower() set_name = "streamflow" variables = ["RIVER_DISCHARGE_OVER_LAND_LIQ"] - for variable in variables: + for _ in variables: for plot_type in [ "seasonality_map", "annual_map",