diff --git a/auxiliary_tools/cdat_regression_testing/894-regression-test/netcdf.ipynb b/auxiliary_tools/cdat_regression_testing/894-regression-test/netcdf.ipynb new file mode 100644 index 000000000..bee5e3776 --- /dev/null +++ b/auxiliary_tools/cdat_regression_testing/894-regression-test/netcdf.ipynb @@ -0,0 +1,746 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# CDAT Migration Regression Testing Notebook (`.nc` files)\n", + "\n", + "This notebook is used to perform regression testing between the development and\n", + "production versions of a diagnostic set.\n", + "\n", + "## How it works\n", + "\n", + "It compares the relative differences (%) between ref and test variables between\n", + "the dev and `main` branches.\n", + "\n", + "## How to use\n", + "\n", + "PREREQUISITE: The diagnostic set's netCDF stored in `.json` files in two directories\n", + "(dev and `main` branches).\n", + "\n", + "1. Make a copy of this notebook under `auxiliary_tools/cdat_regression_testing/`.\n", + "2. Run `mamba create -n cdat_regression_test -y -c conda-forge \"python<3.12\" xarray netcdf4 dask pandas matplotlib-base ipykernel`\n", + "3. Run `mamba activate cdat_regression_test`\n", + "4. Update `SET_DIR` and `SET_NAME` in the copy of your notebook.\n", + "5. Run all cells IN ORDER.\n", + "6. Review results for any outstanding differences (>=1e-5 relative tolerance).\n", + " - Debug these differences (e.g., bug in metrics functions, incorrect variable references, etc.)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup Code\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import glob\n", + "from typing import List\n", + "\n", + "import numpy as np\n", + "import xarray as xr\n", + "\n", + "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", + "\n", + "DEV_DIR = \"24-12-09-main\"\n", + "DEV_DIR = f\"/global/cfs/cdirs/e3sm/www/e3sm_diags/complete_run/{DEV_DIR}/\"\n", + "DEV_GLOB = sorted(glob.glob(DEV_DIR + \"**/**/*.nc\"))\n", + "DEV_NUM_FILES = len(DEV_GLOB)\n", + "\n", + "MAIN_DIR = \"v2.12.1\"\n", + "MAIN_PATH = f\"/global/cfs/cdirs/e3sm/www/e3sm_diags/complete_run/{MAIN_DIR}/\"\n", + "MAIN_GLOB = sorted(glob.glob(MAIN_DIR + \"**/**/*.nc\"))\n", + "MAIN_NUM_FILES = len(MAIN_GLOB)\n", + "\n", + "\n", + "# def _remove_unwanted_files(file_glob: List[str]) -> List[str]:\n", + "# \"\"\"Remove files that we don't want to compare.\n", + "\n", + "# * area_mean_time_series -- `main` does not generate netCDF\n", + "# * enso_diags -- `main` does not generate netCDF\n", + "# * qbo -- variable name differs\n", + "# * diurnal_cycle -- variable name differs\n", + "# * diff -- comparing the difference between regridded files is not helpful\n", + "# between branches because of the influence in floating point errors.\n", + "# * ERA5_ext-U10-ANN-global_ref and ERA5_ext-U10-JJA-global_ref -- dev\n", + "# branch does not generate these files because it is a model-only run.\n", + "\n", + "# Parameters\n", + "# ----------\n", + "# file_glob : List[str]\n", + "# _description_\n", + "\n", + "# Returns\n", + "# -------\n", + "# List[str]\n", + "# _description_\n", + "# \"\"\"\n", + "\n", + "# new_glob = []\n", + "\n", + "# for fp in file_glob:\n", + "# if (\n", + "# \"area_mean_time_series\" in fp\n", + "# or \"enso_diags\" in fp\n", + "# or \"qbo\" in fp\n", + "# or \"diurnal_cycle\" in fp\n", + "# or \"diff\" in fp\n", + "# or \"ERA5_ext-U10-ANN-global_ref\" in fp\n", + "# or \"ERA5_ext-U10-JJA-global_ref\" in fp\n", + "# ):\n", + "# continue\n", + "\n", + "# new_glob.append(fp)\n", + "\n", + "# return new_glob\n", + "\n", + "\n", + "# DEV_GLOB = _remove_unwanted_files(DEV_GLOB)\n", + "# MAIN_GLOB = _remove_unwanted_files(MAIN_GLOB)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "MAIN_NUM_FILES" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "MAIN_GLOB" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "MAIN_NUM_FILES" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def _get_relative_diffs():\n", + " # We are mainly focusing on relative tolerance here (in percentage terms).\n", + " atol = 0\n", + " rtol = 1e-4\n", + "\n", + " results = {\n", + " \"missing_files\": [],\n", + " \"missing_vars\": [],\n", + " \"matching_files\": [],\n", + " \"mismatch_errors\": [],\n", + " \"not_equal_errors\": [],\n", + " \"key_errors\": [],\n", + " }\n", + "\n", + " for fp_main in MAIN_GLOB:\n", + " fp_dev = fp_main.replace(MAIN_DIR, DEV_DIR)\n", + "\n", + " print(\"Comparing:\")\n", + " print(f\" * {fp_dev}\")\n", + " print(f\" * {fp_main}\")\n", + "\n", + " if \"annual_cycle_zonal_mean\" in fp_main:\n", + " if \"test.nc\" in fp_main:\n", + " fp_dev = fp_dev.replace(\"test.nc\", \"ref.nc\")\n", + " elif \"ref.nc\" in fp_main:\n", + " fp_dev = fp_dev.replace(\"ref.nc\", \"test.nc\")\n", + "\n", + " try:\n", + " ds1 = xr.open_dataset(fp_dev)\n", + " ds2 = xr.open_dataset(fp_main)\n", + " except FileNotFoundError as e:\n", + " print(f\" {e}\")\n", + "\n", + " if isinstance(e, FileNotFoundError) or isinstance(e, OSError):\n", + " results[\"missing_files\"].append(fp_dev)\n", + "\n", + " continue\n", + "\n", + " var_key = fp_main.split(\"-\")[-3]\n", + "\n", + " # for 3d vars such as T-200\n", + " var_key.isdigit()\n", + " if var_key.isdigit():\n", + " var_key = fp_main.split(\"-\")[-4]\n", + "\n", + " dev_data = _get_var_data(ds1, var_key)\n", + " main_data = _get_var_data(ds2, var_key)\n", + "\n", + " print(f\" * var_key: {var_key}\")\n", + "\n", + " if dev_data is None or main_data is None:\n", + " if dev_data is None:\n", + " results[\"missing_vars\"].append(fp_dev)\n", + " elif main_data is None:\n", + " results[\"missing_vars\"].append(fp_main)\n", + "\n", + " print(\" * Could not find variable key in the dataset(s)\")\n", + "\n", + " continue\n", + "\n", + " try:\n", + " np.testing.assert_allclose(\n", + " dev_data,\n", + " main_data,\n", + " atol=atol,\n", + " rtol=rtol,\n", + " )\n", + " results[\"matching_files\"].append(fp_main)\n", + " except (KeyError, AssertionError) as e:\n", + " msg = str(e)\n", + "\n", + " print(f\" {msg}\")\n", + "\n", + " if \"mismatch\" in msg:\n", + " results[\"mismatch_errors\"].append(fp_dev)\n", + " elif \"Not equal to tolerance\" in msg:\n", + " results[\"not_equal_errors\"].append(fp_dev)\n", + " else:\n", + " print(f\" * All close and within relative tolerance ({rtol})\")\n", + "\n", + " return results\n", + "\n", + "\n", + "def _get_var_data(ds: xr.Dataset, var_key: str) -> np.ndarray:\n", + " \"\"\"Get the variable data using a list of matching keys.\n", + "\n", + " The `main` branch saves the dataset using the original variable name,\n", + " while the dev branch saves the variable with the derived variable name.\n", + " The dev branch is performing the expected behavior here.\n", + "\n", + " Parameters\n", + " ----------\n", + " ds : xr.Dataset\n", + " _description_\n", + " var_key : str\n", + " _description_\n", + "\n", + " Returns\n", + " -------\n", + " np.ndarray\n", + " _description_\n", + " \"\"\"\n", + "\n", + " data = None\n", + "\n", + " try:\n", + " data = ds[var_key].values\n", + " except KeyError:\n", + " try:\n", + " var_keys = DERIVED_VARIABLES[var_key].keys()\n", + " except KeyError:\n", + " var_keys = DERIVED_VARIABLES[var_key.upper()].keys()\n", + "\n", + " var_keys = [var_key] + list(sum(var_keys, ()))\n", + "\n", + " for key in var_keys:\n", + " if key in ds.data_vars.keys():\n", + " data = ds[key].values\n", + " break\n", + "\n", + " return data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def _check_if_files_found():\n", + " if DEV_NUM_FILES == 0 or MAIN_NUM_FILES == 0:\n", + " raise IOError(\n", + " \"No files found at DEV_DIR and/or MAIN_DIR. \"\n", + " f\"Please check {DEV_DIR} and {MAIN_DIR}.\"\n", + " )\n", + "\n", + "\n", + "def _check_if_matching_filecount():\n", + " if DEV_NUM_FILES != MAIN_NUM_FILES:\n", + " raise IOError(\n", + " \"Number of files do not match at DEV_DIR and MAIN_DIR \"\n", + " f\"({DEV_NUM_FILES} vs. {MAIN_NUM_FILES}).\"\n", + " )\n", + "\n", + " print(f\"Matching file count ({DEV_NUM_FILES} and {MAIN_NUM_FILES}).\")\n", + "\n", + "\n", + "def _check_if_missing_files():\n", + " missing_dev_files = []\n", + " missing_main_files = []\n", + "\n", + " for fp_main in MAIN_GLOB:\n", + " fp_dev = fp_main.replace(MAIN_DIR, DEV_DIR)\n", + "\n", + " if fp_dev not in DEV_GLOB:\n", + " missing_dev_files.append(fp_dev)\n", + "\n", + " for fp_dev in DEV_GLOB:\n", + " fp_main = fp_dev.replace(DEV_DIR, MAIN_DIR)\n", + "\n", + " if fp_main not in MAIN_GLOB:\n", + " missing_main_files.append(fp_main)\n", + "\n", + " return missing_dev_files, missing_main_files" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Check for matching and equal number of files\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "_check_if_files_found()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "DEV_GLOB = [fp for fp in DEV_GLOB if \"diff.nc\" not in fp]\n", + "MAIN_GLOB = [fp for fp in MAIN_GLOB if \"diff.nc\" not in fp]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "len(DEV_GLOB), len(MAIN_GLOB)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "missing_dev_files, missing_main_files = _check_if_missing_files()\n", + "\n", + "print(f\"Missing dev files: {len(missing_dev_files)}\")\n", + "print(f\"Missing main files: {len(missing_main_files)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Check missing main files (not concerned)\n", + "\n", + "Results:\n", + "\n", + "- The missing files are due to a recent .cfg update in [PR #830](https://github.com/E3SM-Project/e3sm_diags/pull/830)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "missing_main_files" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Check missing dev files:\n", + "\n", + "Results:\n", + "\n", + "- The missing reference files are due to not saving them out to netCDF since they are the same as the test files (skipped, model-only run)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "missing_dev_files" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2 Compare the netCDF files between branches\n", + "\n", + "- Compare \"ref\" and \"test\" files\n", + "- \"diff\" files are ignored because getting relative diffs for these does not make sense (relative diff will be above tolerance)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "results = _get_relative_diffs()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "# Statistics\n", + "(\n", + " missing_files,\n", + " missing_vars,\n", + " matching_files,\n", + " mismatch_errors,\n", + " not_equal_errors,\n", + " key_errors,\n", + ") = results.values()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Missing Files\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "missing_files" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- AOD_550 files were retired (https://github.com/E3SM-Project/e3sm_diags/issues/852)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "missing_files = []" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### `NaN` Mismatching Errors\n", + "\n", + "I found these `nan` mismatch errors occur due to either:\n", + "\n", + "1. Regional subsetting on \"ccb\" flag in CDAT adding a coordinate points -- removing these coordinates results in matching results\n", + "2. Slightly different masking in the data between xCDAT and CDAT via xESMF/ESMF -- same number of nans just slightly shifted over some coordinates points\n", + "\n", + "- Refer to PR [#794](https://github.com/E3SM-Project/e3sm_diags/pull/794)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "mismatch_errors = [\n", + " f\n", + " for f in mismatch_errors\n", + " # https://github.com/E3SM-Project/e3sm_diags/pull/794\n", + " if \"TAUXY\" not in f and \"ERA5-TREFHT\" not in f and \"MERRA2-TREFHT\" not in f\n", + " # https://github.com/E3SM-Project/e3sm_diags/pull/798#issuecomment-2251287986\n", + " and \"ceres_ebaf_toa_v4.1-ALBEDO\" not in f\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "mismatch_errors" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Not Equal Errors\n", + "\n", + "- Note, some files are omitted due to known root causes to the diffs (not a concern)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "not_equal_errors = [\n", + " f\n", + " for f in not_equal_errors\n", + " # https://github.com/E3SM-Project/e3sm_diags/issues/797\n", + " if \"MISRCOSP-CLDLOW_TAU1.3_9.4_MISR\" not in f\n", + " and \"MISRCOSP-CLDLOW_TAU1.3_MISR\" not in f\n", + " and \"MISRCOSP-CLDLOW_TAU9.4_MISR\" not in f\n", + " # only 1 mismatching element with 1e-4 tolerance\n", + " and \"ERA5-OMEGA-JJA\" not in f and \"MERRA2-OMEGA-JJA\" not in f\n", + " # https://github.com/E3SM-Project/e3sm_diags/issues/787\n", + " and \"MERRA2-U\" not in f\n", + " # https://github.com/E3SM-Project/e3sm_diags/issues/852\n", + " and \"AOD_550\" not in f\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "not_equal_errors" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note, the two files have less than 10 elements that mismatch.\n", + "This does not seem to be of a concern since the rtol is 0.04% and 1.2%.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```python\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/843-migration-phase3-model-vs-obs/lat_lon/Cloud MISR/MISRCOSP-CLDTOT_TAU1.3_9.4_MISR-ANN-global_test.nc\n", + " * /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main/lat_lon/Cloud MISR/MISRCOSP-CLDTOT_TAU1.3_9.4_MISR-ANN-global_test.nc\n", + " * var_key: CLDTOT_TAU1.3_9.4_MISR\n", + "\n", + "Not equal to tolerance rtol=0.0001, atol=0\n", + "\n", + "Mismatched elements: 7 / 64800 (0.0108%)\n", + "Max absolute difference: 0.0970192\n", + "Max relative difference: 0.00456364\n", + " x: array([[ 4.128816, 4.128816, 4.128816, ..., 7.77411 , 7.77411 ,\n", + " 7.77411 ],\n", + " [ 4.164101, 4.164073, 4.164018, ..., 7.538528, 7.538158,...\n", + " y: array([[ 4.128816, 4.128816, 4.128816, ..., 7.774109, 7.774109,\n", + " 7.774109],\n", + " [ 4.164101, 4.164073, 4.164018, ..., 7.538528, 7.538157,...\n", + "\n", + "Not equal to tolerance rtol=0.0001, atol=0\n", + "\n", + "Mismatched elements: 8 / 64800 (0.0123%)\n", + "Max absolute difference: 0.0970192\n", + "Max relative difference: 0.01244658\n", + " x: array([[ 0.056777, 0.056777, 0.056777, ..., 1.274017, 1.274017,\n", + " 1.274017],\n", + " [ 0.207892, 0.207774, 0.207536, ..., 1.675944, 1.676576,...\n", + " y: array([[ 0.056777, 0.056777, 0.056777, ..., 1.274017, 1.274017,\n", + " 1.274017],\n", + " [ 0.207892, 0.207774, 0.207536, ..., 1.675944, 1.676576,...\n", + "```\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ds1 = xr.open_dataset(\n", + " \"/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/843-migration-phase3-model-vs-obs/lat_lon/Cloud MISR/MISRCOSP-CLDTOT_TAU1.3_9.4_MISR-ANN-global_test.nc\"\n", + ")\n", + "ds2 = xr.open_dataset(\n", + " \"/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main/lat_lon/Cloud MISR/MISRCOSP-CLDTOT_TAU1.3_9.4_MISR-ANN-global_test.nc\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "np.testing.assert_allclose(\n", + " ds1[\"CLDTOT_TAU1.3_9.4_MISR\"], ds2[\"CLDTOT_TAU1.3_9.4_MISR\"], atol=0, rtol=1e-4\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ds1[\"CLDTOT_TAU1.3_9.4_MISR\"].values.sum()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ds2[\"CLDTOT_TAU1.3_9.4_MISR\"].values.sum()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Assuming these variables are defined in your notebook\n", + "matching_files_count = len(matching_files)\n", + "missing_vars_count = len(missing_vars)\n", + "mismatch_errors_count = len(mismatch_errors)\n", + "not_equal_errors_count = len(not_equal_errors)\n", + "key_errors_count = len(key_errors)\n", + "missing_files_count = len(missing_files)\n", + "\n", + "sum_files_compared = (\n", + " matching_files_count\n", + " + missing_vars_count\n", + " + mismatch_errors_count\n", + " + not_equal_errors_count\n", + " + key_errors_count\n", + " + missing_files_count\n", + ")\n", + "\n", + "pct_match = (matching_files_count / sum_files_compared) * 100\n", + "\n", + "# Collect statistics into a dictionary\n", + "statistics = {\n", + " \"stat_name\": [\n", + " \"matching_files_count\",\n", + " \"missing_vars_count\",\n", + " \"mismatch_errors_count\",\n", + " \"not_equal_errors_count\",\n", + " \"key_errors_count\",\n", + " \"missing_files_count\",\n", + " ],\n", + " \"value\": [\n", + " matching_files_count,\n", + " missing_vars_count,\n", + " mismatch_errors_count,\n", + " not_equal_errors_count,\n", + " key_errors_count,\n", + " missing_files_count,\n", + " ],\n", + " \"pct\": [\n", + " matching_files_count / sum_files_compared,\n", + " missing_vars_count / sum_files_compared,\n", + " mismatch_errors_count / sum_files_compared,\n", + " not_equal_errors_count / sum_files_compared,\n", + " key_errors_count / sum_files_compared,\n", + " missing_files_count / sum_files_compared,\n", + " ],\n", + "}\n", + "\n", + "# Convert the dictionary to a pandas DataFrame\n", + "df = pd.DataFrame(statistics)\n", + "\n", + "# Display the DataFrame\n", + "print(df)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/complete_run_script.py b/auxiliary_tools/complete_run_script.py similarity index 98% rename from tests/complete_run_script.py rename to auxiliary_tools/complete_run_script.py index 3a6192856..9e7ebb102 100644 --- a/tests/complete_run_script.py +++ b/auxiliary_tools/complete_run_script.py @@ -48,7 +48,7 @@ short_name = "v3.LR.historical_0101" # TODO: Update `MAIN_DIR` as needed. -MAIN_DIR = "24-12-09-main" +MAIN_DIR = "v2.12.1" results_dir = f"/global/cfs/cdirs/e3sm/www/e3sm_diags/{MAIN_DIR}/" test_climo = "/global/cfs/cdirs/e3sm/chengzhu/tutorial2024/v3.LR.historical_0101/post/atm/180x360_aave/clim/15yr" @@ -215,5 +215,4 @@ "tropical_subseasonal", ] -if __name__ == "__main__": - runner.run_diags(params) +runner.run_diags(params) diff --git a/tests/test_regression.py b/tests/test_regression.py index af04ded2c..0221ad146 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -7,9 +7,9 @@ import pytest import xarray as xr +from auxiliary_tools.complete_run_script import params, runner from e3sm_diags.derivations.derivations import DERIVED_VARIABLES from e3sm_diags.logger import custom_logger -from tests.complete_run_script import params, runner logger = custom_logger(__name__)