Starter code for fixing integration tests

Fix 3D variable bugs
- Ignore adding bounds if user defined plevs is <= 1 element
- Fix setting time series attributes in `dataset_xr.py` for climatology datasets
- Update `_convert_units_to_mb()` to handle hPa units
- Fix subsetting on `ilev` in `_run_3d_diags()`
- Refactor `test_all_sets.py` for readability and use pytest

Fix regridding tests due to bounds not being converted
- Update workspace debugger settings to function work with testing api

Uncomment integration test related lines

Add comments in `test_all_sets.py`

update comment

Remove repeated call to save plots and metrics
- This call resulted in a bug where duplicate metrics and plot files were created without proper names

Fix incorrect import of `_get_output_dir()

Revert VS Code debugger settings

Update fixture in `test_diags.py` to only run once

Update order of methods

Fix conditional for bbox

Uncomment pytest testspaths config

Update regression testing notebooks with `util.py` imports
- Add separate lat_lon testing directory

Add function to print variable keys and num largest diffs
- Remove diff of differences comparison

Update notebooks and move scripts

auxiliary_tools/cdat_regression_testing/671-lat-lon/11_28_23_qa_diffs.py

@@ -0,0 +1,69 @@
+"""
+QA diffs
+
+* NET_FLUX_SRF - test and ref
+* RESTOM - test and ref
+
+"""
+# %%
+import os
+
+from e3sm_diags.parameter.core_parameter import CoreParameter
+from e3sm_diags.run import runner
+
+param = CoreParameter()
+
+# Location of the data.
+param.test_data_path = "/global/cfs/cdirs/e3sm/e3sm_diags/test_model_data_for_acme_diags/time-series/E3SM_v1"
+param.reference_data_path = "/global/cfs/cdirs/e3sm/e3sm_diags/test_model_data_for_acme_diags/time-series/E3SM_v1"
+
+# Variables
+param.variables = ["NET_FLUX_SRF"]
+# param.variables = ["RESTOM"]
+
+# Set this parameter to True.
+# By default, e3sm_diags expects the test data to be climo data.
+param.test_timeseries_input = True
+# Years to slice the test data, base this off the years in the filenames.
+param.test_start_yr = "2011"
+param.test_end_yr = "2013"
+
+# Set this parameter to True.
+# By default, e3sm_diags expects the ref data to be climo data.
+param.ref_timeseries_input = True
+# Years to slice the ref data, base this off the years in the filenames
+param.ref_start_yr = "1850"
+param.ref_end_yr = "1852"
+
+# When running with time-series data, you don't need to specify the name of the data.
+# But you should, otherwise nothing is displayed when the test/ref name is needed.
+param.short_test_name = "historical_H1"
+param.short_ref_name = "historical_H1"
+
+# This parameter modifies the software to accommodate model vs model runs.
+# The default setting for run_type is 'model_vs_obs'.
+param.run_type = "model_vs_model"
+# Name of the folder where the results are stored.
+# Change `prefix` to use your directory.
+prefix = "/global/cfs/cdirs/e3sm/www/vo13/examples"
+param.results_dir = os.path.join(prefix, "run_refactor_single_param")
+
+# Below are more optional arguments.
+
+# What plotsets to run the diags on.
+# If not defined, then all available sets are used.
+param.sets = ["lat_lon"]
+# What seasons to run the diags on.
+# If not defined, diags are run on ['ANN', 'DJF', 'MAM', 'JJA', 'SON'].
+param.seasons = ["ANN"]
+# Title of the difference plots.
+param.diff_title = "Model (2011-2013) - Model (1850-1852)"
+
+# For running with multiprocessing.
+param.multiprocessing = False
+# param.num_workers = 24
+
+# %%
+runner.run_diags([param])
+
+# %%

auxiliary_tools/cdat_regression_testing/671-lat-lon/658-diags.cfg

@@ -0,0 +1,15 @@
+[#]
+sets = ["lat_lon"]
+case_id = "model_vs_model"
+variables = ["NET_FLUX_SRF"]
+seasons = ["ANN", "DJF", "MAM", "JJA", "SON"]
+contour_levels = [-200, -160, -120, -80, -40, 0, 40, 80, 120, 160, 200]
+diff_levels = [-75, -50, -25, -10, -5, -2, 2, 5, 10, 25, 50, 75]
+
+[#]
+sets = ["lat_lon"]
+case_id = "model_vs_model"
+variables = ["RESTOM"]
+seasons = ["ANN", "DJF", "MAM", "JJA", "SON"]
+contour_levels = [-120, -100, -80, -60, -40, -20, 0, 20, 40, 60, 80]
+diff_levels = [-30, -25, -20, -15, -10, -5, -2, 2, 5, 10, 15, 20, 25, 30]

auxiliary_tools/cdat_regression_testing/671-lat-lon/ex1.py

@@ -0,0 +1,61 @@
+# %%
+import os
+
+from e3sm_diags.parameter.core_parameter import CoreParameter
+from e3sm_diags.run import runner
+
+param = CoreParameter()
+
+# Location of the data.
+param.test_data_path = "/global/cfs/cdirs/e3sm/e3sm_diags/test_model_data_for_acme_diags/time-series/E3SM_v1"
+param.reference_data_path = "/global/cfs/cdirs/e3sm/e3sm_diags/test_model_data_for_acme_diags/time-series/E3SM_v1"
+
+# Variables
+param.variables = ["PRECT"]
+
+# Set this parameter to True.
+# By default, e3sm_diags expects the test data to be climo data.
+param.test_timeseries_input = True
+# Years to slice the test data, base this off the years in the filenames.
+param.test_start_yr = "2011"
+param.test_end_yr = "2013"
+
+# Set this parameter to True.
+# By default, e3sm_diags expects the ref data to be climo data.
+param.ref_timeseries_input = True
+# Years to slice the ref data, base this off the years in the filenames
+param.ref_start_yr = "1850"
+param.ref_end_yr = "1852"
+
+# When running with time-series data, you don't need to specify the name of the data.
+# But you should, otherwise nothing is displayed when the test/ref name is needed.
+param.short_test_name = "historical_H1"
+param.short_ref_name = "historical_H1"
+
+# This parameter modifies the software to accommodate model vs model runs.
+# The default setting for run_type is 'model_vs_obs'.
+param.run_type = "model_vs_model"
+# Name of the folder where the results are stored.
+# Change `prefix` to use your directory.
+prefix = "/global/cfs/cdirs/e3sm/www/vo13/examples"
+param.results_dir = os.path.join(prefix, "run_refactor_single_param")
+
+# Below are more optional arguments.
+
+# What plotsets to run the diags on.
+# If not defined, then all available sets are used.
+param.sets = ["lat_lon"]
+# What seasons to run the diags on.
+# If not defined, diags are run on ['ANN', 'DJF', 'MAM', 'JJA', 'SON'].
+param.seasons = ["ANN"]
+# Title of the difference plots.
+param.diff_title = "Model (2011-2013) - Model (1850-1852)"
+
+# For running with multiprocessing.
+param.multiprocessing = False
+# param.num_workers = 24
+
+# %%
+runner.run_diags([param])
+
+# %%

auxiliary_tools/cdat_regression_testing/671-lat-lon/ex1_3d.py

@@ -0,0 +1,41 @@
+# %%
+import os
+
+from e3sm_diags.parameter.core_parameter import CoreParameter
+from e3sm_diags.run import runner
+
+param = CoreParameter()
+
+# %%
+param.sets = ["lat_lon"]
+param.case_id = "ERA-Interim"
+param.variables = ["T"]
+param.seasons = ["ANN"]
+param.plevs = [850.0]
+param.contour_levels = [240, 245, 250, 255, 260, 265, 270, 275, 280, 285, 290, 295]
+param.diff_levels = [-10, -7.5, -5, -4, -3, -2, -1, -0.5, 0.5, 1, 2, 3, 4, 5, 7.5, 10]
+
+param.test_name = "system tests"
+param.short_test_name = "short_system tests"
+param.ref_name = "ERA-Interim"
+param.reference_name = "ERA-Interim Reanalysis 1979-2015"
+param.reference_data_path = (
+    "/global/u2/v/vo13/E3SM-Project/e3sm_diags/tests/integration/integration_test_data"
+)
+param.ref_file = "ta_ERA-Interim_ANN_198001_201401_climo.nc"
+param.test_data_path = (
+    "/global/u2/v/vo13/E3SM-Project/e3sm_diags/tests/integration/integration_test_data"
+)
+param.test_file = "T_20161118.beta0.FC5COSP.ne30_ne30.edison_ANN_climo.nc"
+
+param.backend = "mpl"
+prefix = "/global/cfs/cdirs/e3sm/www/vo13/examples"
+param.results_dir = os.path.join(prefix, "lat_lon_3d_var_test")
+param.debug = True
+param.multiprocessing = False
+
+
+# %%
+runner.run_diags([param])
+
+# %%