Adopt ruff as the central tool for linting and formatting (#920)

* Remove `setup.py` introduced accidentally by rebasing on `main` * Fix `conda-forge` badge links to point to correct page
E3SM-Project · Jan 21, 2025 · df39bf4 · df39bf4
1 parent 5a1ac38
commit df39bf4
Show file tree

Hide file tree

Showing 125 changed files with 400 additions and 530 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -4,33 +4,27 @@ fail_fast: true
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-yaml
 
-  - repo: https://github.com/psf/black
-    rev: 23.9.1
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.2
     hooks:
-      - id: black
-
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
-    hooks:
-      - id: isort
-
-  # Need to use flake8 GitHub mirror due to CentOS git issue with GitLab
-  # https://github.com/pre-commit/pre-commit/issues/1206
-  - repo: https://github.com/pycqa/flake8
-    rev: 6.1.0
-    hooks:
-      - id: flake8
-        args: [--config=setup.cfg]
-        additional_dependencies: [flake8-isort==6.1.0]
+      # Sort the imports
+      - id: ruff
+        name: ruff-sort-imports
+        args: [--select, I, --fix]
+      # Run the linter.
+      - id: ruff
+        args: [--fix]
+      # Run the formatter.
+      - id: ruff-format
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.5.1
+    rev: v1.14.1
     hooks:
       - id: mypy
         args: [--config=pyproject.toml]

diff --git a/.vscode/e3sm_diags.code-workspace b/.vscode/e3sm_diags.code-workspace
@@ -25,11 +25,9 @@
       "editor.rulers": [80, 88, 120],
       "editor.wordWrap": "wordWrapColumn",
       "editor.wordWrapColumn": 120,
-      "editor.defaultFormatter": "ms-python.black-formatter"
+      "editor.defaultFormatter": "charliermarsh.ruff"
     },
-    // Code Formatting and Linting
-    // ---------------------------
-    "flake8.args": ["--config=setup.cfg"],
+    "ruff.importStrategy": "fromEnvironment",
     // Type checking
     // ---------------------------
     "mypy-type-checker.args": ["--config=pyproject.toml"],

diff --git a/Makefile b/Makefile
@@ -68,8 +68,12 @@ clean-test-int-data:  # remove integration test data and images (expected) -- us
 pre-commit:  # run pre-commit quality assurance checks
 	pre-commit run --all-files
 
-lint: ## check style with flake8
-	flake8 e3sm_diags tests
+lint: ## check style ruff
+	ruff check --select I --fix
+	ruff check --fix
+
+format: ## format code using ruff
+	ruff format
 
 test: ## run tests quickly with the default Python and produces code coverage report
 	pytest

diff --git a/README.md b/README.md
diff --git a/analysis_data_preprocess/fix_units_1.5hr_time_shift_forTRMM.py b/analysis_data_preprocess/fix_units_1.5hr_time_shift_forTRMM.py
@@ -1,5 +1,6 @@
-import cdms2
 from pathlib import Path
+
+import cdms2
 import MV2
 
 datapath='/p/user_pub/e3sm/e3sm_diags_data/obs_for_e3sm_diags/climatology/TRMM-3B43v-7_3hr/'

diff --git a/auxiliary_tools/LCF_mixed-phase_partition_diagram.ipynb b/auxiliary_tools/LCF_mixed-phase_partition_diagram.ipynb
@@ -7,13 +7,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import xarray as xr\n",
-    "import numpy\n",
+    "import json\n",
+    "import math\n",
+    "\n",
     "import matplotlib.pyplot as plt\n",
-    "from scipy.stats import binned_statistic\n",
+    "import numpy\n",
     "import scipy.io\n",
-    "import math\n",
-    "import json"
+    "import xarray as xr\n",
+    "from scipy.stats import binned_statistic"
    ]
   },
   {

diff --git a/auxiliary_tools/aerosol_budget.py b/auxiliary_tools/aerosol_budget.py
@@ -1,15 +1,15 @@
 # NOTE: This module uses the deprecated e3sm_diags.driver.utils.dataset.Dataset
 # class, which was replaced by e3sm_diags.driver.utils.dataset_xr.Dataset.
-import e3sm_diags
-from e3sm_diags.driver import utils
+import csv
+import json
+import os
+
 import cdms2
 import cdutil
 import numpy
-import os
-import json
-
-import csv
 
+import e3sm_diags
+from e3sm_diags.driver import utils
 from e3sm_diags.parameter.core_parameter import CoreParameter
 
 

diff --git a/..._regression_testing/654-zonal_mean_xy/654-zonal_mean_xy_cdat_regression_test_netcdf.ipynb b/..._regression_testing/654-zonal_mean_xy/654-zonal_mean_xy_cdat_regression_test_netcdf.ipynb
@@ -49,8 +49,8 @@
     }
    ],
    "source": [
-    "from collections import defaultdict\n",
     "import glob\n",
+    "from collections import defaultdict\n",
     "\n",
     "import numpy as np\n",
     "import xarray as xr\n",

diff --git a/...testing/654-zonal_mean_xy/debug_654-zonal_mean_xy_cdat_regression_test_netcdf_debug.ipynb b/...testing/654-zonal_mean_xy/debug_654-zonal_mean_xy_cdat_regression_test_netcdf_debug.ipynb
@@ -41,8 +41,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from collections import defaultdict\n",
     "import glob\n",
+    "from collections import defaultdict\n",
     "\n",
     "import numpy as np\n",
     "import xarray as xr\n",

diff --git a/.../cdat_regression_testing/655-zonal-mean-2d/655_zonal_mean_2d_regression_test_netcdf.ipynb b/.../cdat_regression_testing/655-zonal-mean-2d/655_zonal_mean_2d_regression_test_netcdf.ipynb
@@ -41,8 +41,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from collections import defaultdict\n",
     "import glob\n",
+    "from collections import defaultdict\n",
     "\n",
     "import numpy as np\n",
     "import xarray as xr\n",

diff --git a/...ion_testing/655-zonal-mean-2d/655_zonal_mean_2d_stratosphere_regression_test_netcdf.ipynb b/...ion_testing/655-zonal-mean-2d/655_zonal_mean_2d_stratosphere_regression_test_netcdf.ipynb
@@ -41,8 +41,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from collections import defaultdict\n",
     "import glob\n",
+    "from collections import defaultdict\n",
     "\n",
     "import numpy as np\n",
     "import xarray as xr\n",

diff --git a/...n_testing/657-meridional-mean-2d/657_meridional_mean_2d_cdat_regression_test_netcdf.ipynb b/...n_testing/657-meridional-mean-2d/657_meridional_mean_2d_cdat_regression_test_netcdf.ipynb
@@ -45,8 +45,8 @@
     "\n",
     "import numpy as np\n",
     "import xarray as xr\n",
-    "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n",
     "\n",
+    "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n",
     "\n",
     "# TODO: Update SET_NAME and SET_DIR\n",
     "SET_NAME = \"meridional_mean_2d\"\n",

diff --git a/...liary_tools/cdat_regression_testing/659-polar/659_polar_cdat_regression_test_netcdf.ipynb b/...liary_tools/cdat_regression_testing/659-polar/659_polar_cdat_regression_test_netcdf.ipynb
@@ -45,8 +45,8 @@
     "\n",
     "import numpy as np\n",
     "import xarray as xr\n",
-    "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n",
     "\n",
+    "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n",
     "\n",
     "# TODO: Update SET_NAME and SET_DIR\n",
     "SET_NAME = \"polar\"\n",

diff --git a/...dat_regression_testing/660-cosp-histogram/660_cosp_histogram_regression_test_netcdf.ipynb b/...dat_regression_testing/660-cosp-histogram/660_cosp_histogram_regression_test_netcdf.ipynb
@@ -46,8 +46,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from collections import defaultdict\n",
     "import glob\n",
+    "from collections import defaultdict\n",
     "\n",
     "import numpy as np\n",
     "import xarray as xr\n",

diff --git a/auxiliary_tools/cdat_regression_testing/662-area-mean-time-series/regression-test-json.ipynb b/auxiliary_tools/cdat_regression_testing/662-area-mean-time-series/regression-test-json.ipynb
@@ -55,8 +55,8 @@
     "from auxiliary_tools.cdat_regression_testing.utils import (\n",
     "    get_num_metrics_above_diff_thres,\n",
     "    get_rel_diffs,\n",
-    "    update_diffs_to_pct,\n",
     "    highlight_large_diffs,\n",
+    "    update_diffs_to_pct,\n",
     ")\n",
     "\n",
     "SET_NAME = \"area_mean_time_series\"\n",

diff --git a/...iary_tools/cdat_regression_testing/662-area-mean-time-series/regression-test-netcdf.ipynb b/...iary_tools/cdat_regression_testing/662-area-mean-time-series/regression-test-netcdf.ipynb
@@ -41,14 +41,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from collections import defaultdict\n",
     "import glob\n",
+    "from collections import defaultdict\n",
     "from typing import Tuple\n",
     "\n",
     "import numpy as np\n",
     "import xarray as xr\n",
-    "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n",
     "\n",
+    "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n",
     "\n",
     "# TODO: Update SET_NAME and SET_DIR\n",
     "SET_NAME = \"area_mean_time_series\"\n",

diff --git a/...ression_testing/663-enso-diags/662-area-mean-time-series-rerun/regression-test-json.ipynb b/...ression_testing/663-enso-diags/662-area-mean-time-series-rerun/regression-test-json.ipynb
@@ -55,8 +55,8 @@
     "from auxiliary_tools.cdat_regression_testing.utils import (\n",
     "    get_num_metrics_above_diff_thres,\n",
     "    get_rel_diffs,\n",
-    "    update_diffs_to_pct,\n",
     "    highlight_large_diffs,\n",
+    "    update_diffs_to_pct,\n",
     ")\n",
     "\n",
     "SET_NAME = \"area_mean_time_series\"\n",

diff --git a/...ssion_testing/663-enso-diags/662-area-mean-time-series-rerun/regression-test-netcdf.ipynb b/...ssion_testing/663-enso-diags/662-area-mean-time-series-rerun/regression-test-netcdf.ipynb
@@ -41,14 +41,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from collections import defaultdict\n",
     "import glob\n",
+    "from collections import defaultdict\n",
     "from typing import Tuple\n",
     "\n",
     "import numpy as np\n",
     "import xarray as xr\n",
-    "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n",
     "\n",
+    "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n",
     "\n",
     "# TODO: Update SET_NAME and SET_DIR\n",
     "SET_NAME = \"area_mean_time_series\"\n",

diff --git a/auxiliary_tools/cdat_regression_testing/663-enso-diags/regression_test.ipynb b/auxiliary_tools/cdat_regression_testing/663-enso-diags/regression_test.ipynb
@@ -45,6 +45,7 @@
     "\n",
     "import numpy as np\n",
     "import xarray as xr\n",
+    "\n",
     "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n",
     "\n",
     "SET_NAME = \"enso_diags\"\n",

diff --git a/auxiliary_tools/cdat_regression_testing/664-qbo/regression_test.ipynb b/auxiliary_tools/cdat_regression_testing/664-qbo/regression_test.ipynb
@@ -45,6 +45,7 @@
     "\n",
     "import numpy as np\n",
     "import xarray as xr\n",
+    "\n",
     "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n",
     "\n",
     "SET_NAME = \"qbo\"\n",
@@ -332,8 +333,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import xcdat as xc\n",
     "import xarray as xr\n",
+    "import xcdat as xc\n",
     "\n",
     "ds_xc = xc.open_dataset(\n",
     "    \"/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/664-qbo/qbo/QBO-ERA-Interim/qbo_diags_qbo_ref.nc\"\n",

diff --git a/auxiliary_tools/cdat_regression_testing/666-diurnal-cycle/regression_test.ipynb b/auxiliary_tools/cdat_regression_testing/666-diurnal-cycle/regression_test.ipynb
@@ -45,8 +45,8 @@
     "\n",
     "import numpy as np\n",
     "import xarray as xr\n",
-    "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n",
     "\n",
+    "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n",
     "\n",
     "# TODO: Update SET_NAME and SET_DIR\n",
     "SET_NAME = \"diurnal_cycle\"\n",

diff --git a/...669-annual_cycle_zonal_mean/669-annual_cycle_zonal_mean_cdat_regression_test_netcdf.ipynb b/...669-annual_cycle_zonal_mean/669-annual_cycle_zonal_mean_cdat_regression_test_netcdf.ipynb
@@ -49,8 +49,8 @@
     }
    ],
    "source": [
-    "from collections import defaultdict\n",
     "import glob\n",
+    "from collections import defaultdict\n",
     "\n",
     "import numpy as np\n",
     "import xarray as xr\n",

diff --git a/...cle_zonal_mean/re-run-lat-lon-with-var-subset/669_lat_lon_cdat_regression_test_json.ipynb b/...cle_zonal_mean/re-run-lat-lon-with-var-subset/669_lat_lon_cdat_regression_test_json.ipynb
@@ -59,20 +59,20 @@
     }
    ],
    "source": [
-    "from collections import defaultdict\n",
     "import glob\n",
+    "from collections import defaultdict\n",
     "\n",
-    "import pandas as pd\n",
     "import numpy as np\n",
+    "import pandas as pd\n",
     "import xarray as xr\n",
     "\n",
     "from auxiliary_tools.cdat_regression_testing.utils import (\n",
     "    get_metrics,\n",
     "    get_num_metrics_above_diff_thres,\n",
     "    get_rel_diffs,\n",
+    "    highlight_large_diffs,\n",
     "    sort_columns,\n",
     "    update_diffs_to_pct,\n",
-    "    highlight_large_diffs,\n",
     ")\n",
     "\n",
     "# TODO: Update SET_NAME and SET_DIR\n",

diff --git a/...e_zonal_mean/re-run-lat-lon-with-var-subset/669_lat_lon_cdat_regression_test_netcdf.ipynb b/...e_zonal_mean/re-run-lat-lon-with-var-subset/669_lat_lon_cdat_regression_test_netcdf.ipynb
@@ -45,8 +45,8 @@
     "\n",
     "import numpy as np\n",
     "import xarray as xr\n",
-    "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n",
     "\n",
+    "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n",
     "\n",
     "# TODO: Update SET_NAME and SET_DIR\n",
     "SET_NAME = \"lat_lon\"\n",

diff --git a/auxiliary_tools/cdat_regression_testing/671-lat-lon/12-4-23-qa-no-cdms-slice.ipynb b/auxiliary_tools/cdat_regression_testing/671-lat-lon/12-4-23-qa-no-cdms-slice.ipynb
@@ -24,12 +24,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "import glob\n",
+    "\n",
     "import pandas as pd\n",
     "\n",
-    "import glob\n",
     "from auxiliary_tools.cdat_regression_testing.utils import get_metrics, get_rel_diffs\n",
     "\n",
-    "\n",
     "# TODO: Update DEV_RESULTS and MAIN_RESULTS to your diagnostic sets.\n",
     "DEV_PATH = \"/global/cfs/cdirs/e3sm/www/vo13/examples_658/ex1_modTS_vs_modTS_3years/lat_lon/model_vs_model\"\n",
     "MAIN_PATH = \"/global/cfs/cdirs/e3sm/www/vo13/examples_main_w_slice/ex1_modTS_vs_modTS_3years/lat_lon/model_vs_model\"\n",

diff --git a/auxiliary_tools/cdat_regression_testing/671-lat-lon/671-lat-lon.ipynb b/auxiliary_tools/cdat_regression_testing/671-lat-lon/671-lat-lon.ipynb
@@ -48,18 +48,19 @@
    "outputs": [],
    "source": [
     "import glob\n",
+    "\n",
+    "import pandas as pd\n",
+    "\n",
     "from auxiliary_tools.cdat_regression_testing.utils import (\n",
+    "    PERCENTAGE_COLUMNS,\n",
     "    get_metrics,\n",
-    "    get_rel_diffs,\n",
     "    get_num_metrics_above_diff_thres,\n",
+    "    get_rel_diffs,\n",
     "    highlight_large_diffs,\n",
     "    sort_columns,\n",
     "    update_diffs_to_pct,\n",
-    "    PERCENTAGE_COLUMNS,\n",
     ")\n",
     "\n",
-    "import pandas as pd\n",
-    "\n",
     "# TODO: Update DEV_RESULTS and MAIN_RESULTS to your diagnostic sets.\n",
     "DEV_PATH = \"/global/cfs/cdirs/e3sm/www/vo13/examples_658/ex1_modTS_vs_modTS_3years/lat_lon/model_vs_model\"\n",
     "MAIN_PATH = \"/global/cfs/cdirs/e3sm/www/vo13/examples/ex1_modTS_vs_modTS_3years/lat_lon/model_vs_model\"\n",

diff --git a/auxiliary_tools/cdat_regression_testing/759-slice-flag/regression-test-json.ipynb b/auxiliary_tools/cdat_regression_testing/759-slice-flag/regression-test-json.ipynb
@@ -59,21 +59,20 @@
     }
    ],
    "source": [
-    "from typing import List\n",
     "import glob\n",
+    "from typing import List\n",
     "\n",
     "import pandas as pd\n",
     "\n",
     "from auxiliary_tools.cdat_regression_testing.utils import (\n",
-    "    get_rel_diffs,\n",
+    "    PERCENTAGE_COLUMNS,\n",
     "    get_num_metrics_above_diff_thres,\n",
+    "    get_rel_diffs,\n",
     "    highlight_large_diffs,\n",
     "    sort_columns,\n",
     "    update_diffs_to_pct,\n",
-    "    PERCENTAGE_COLUMNS,\n",
     ")\n",
     "\n",
-    "\n",
     "SET_NAME = \"lat_lon\"\n",
     "SET_DIR = \"759-slice-flag\"\n",
     "\n",