diff --git a/cupid/run_timeseries.py b/cupid/run_timeseries.py index 6164162..c02b419 100755 --- a/cupid/run_timeseries.py +++ b/cupid/run_timeseries.py @@ -112,7 +112,10 @@ def run_timeseries( if isinstance(timeseries_params["case_name"], list): ts_input_dirs = [] for cname in timeseries_params["case_name"]: - ts_input_dirs.append(global_params["CESM_output_dir"]+"/"+cname+f"/{component}/hist/") + if cname == global_params["base_case_name"]: + ts_input_dirs.append(global_params["base_case_output_dir"]+"/"+cname+f"/{component}/hist/") + else: + ts_input_dirs.append(global_params["CESM_output_dir"]+"/"+cname+f"/{component}/hist/") else: ts_input_dirs = [ global_params["CESM_output_dir"] + "/" + diff --git a/cupid/timeseries.py b/cupid/timeseries.py index db14e6d..4f509c2 100644 --- a/cupid/timeseries.py +++ b/cupid/timeseries.py @@ -111,7 +111,7 @@ def create_time_series( # Check that path actually exists: if not starting_location.is_dir(): - emsg = "Provided 'cam_hist_loc' directory '{starting_location}' not found." + emsg = f"Provided 'cam_hist_loc' directory '{starting_location}' not found." emsg += " Script is ending here." raise FileNotFoundError(emsg) diff --git a/examples/key_metrics/config.yml b/examples/key_metrics/config.yml index 802ba49..5f3999c 100644 --- a/examples/key_metrics/config.yml +++ b/examples/key_metrics/config.yml @@ -142,7 +142,7 @@ compute_notebooks: Greenland_SMB_visual_compare_obs: parameter_groups: none: - obs_path: '/glade/u/home/gunterl/obs_diagnostic_cesm/' + obs_path: '/glade/campaign/cesm/development/cross-wg/diagnostic_framework/SMB_data' obs_name: 'GrIS_MARv3.12_climo_1960_1999.nc' climo_nyears: 40 diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py index d110f18..a74f1a3 100755 --- a/helper_scripts/generate_adf_config_file.py +++ b/helper_scripts/generate_adf_config_file.py @@ -25,10 +25,10 @@ def _parse_args(): ) # Command line argument for CUPiD example from which to get config.yml parser.add_argument( - "--cupid-example", + "--cupid-config-loc", action="store", - dest="cupid_example", - default="external_diag_packages", + dest="cupid_config_loc", + default=None, help="CUPiD example to use as template for config.yml", ) parser.add_argument( @@ -46,27 +46,20 @@ def _parse_args(): return parser.parse_args() -def generate_adf_config(cesm_root, cupid_example, adf_file, out_file): - """Use cupid config file (YAML) from cupid_example and adf_file (YAML) +def generate_adf_config(cesm_root, cupid_config_loc, adf_file, out_file): + """Use cupid config file (YAML) from cupid_config_loc and adf_file (YAML) to produce out_file by modifying adf_file with data from cupid config file. """ sys.path.append(os.path.join(cesm_root, "cime")) - # Is cupid_example a valid value? cupid_root = os.path.join(cesm_root, "tools", "CUPiD") - cupid_examples = os.path.join(cupid_root, "examples") - valid_examples = [ - example - for example in next(os.walk(cupid_examples))[1] - if example not in ["ilamb"] - ] - if cupid_example not in valid_examples: - error_msg = f"argument --cupid-example: invalid choice '{cupid_example}'" - raise KeyError( - f"{error_msg} (choose from subdirectories of {cupid_examples}: {valid_examples})", - ) + # Is cupid_config_loc a valid value? + if cupid_config_loc is None: + cupid_config_loc = os.path.join(cupid_root, "examples", "key_metrics") + if not os.path.exists(os.path.join(cupid_config_loc, "config.yml")): + raise KeyError(f"Can not find config.yml in {cupid_config_loc}") - with open(os.path.join(cupid_root, "examples", cupid_example, "config.yml")) as c: + with open(os.path.join(cupid_config_loc, "config.yml")) as c: c_dict = yaml.safe_load(c) with open(adf_file, encoding="UTF-8") as a: a_dict = yaml.safe_load(a) @@ -141,9 +134,9 @@ def generate_adf_config(cesm_root, cupid_example, adf_file, out_file): ts_case_names.index(base_case_name) if base_case_name in ts_case_names else None ) - base_case_output_dir = c_dict["global_params"].get( - "base_case_output_dir", - DOUT + "/" + base_case_name, + base_case_output_dir = os.path.join( + c_dict["global_params"].get("base_case_output_dir", DOUT), + base_case_name, ) base_start_date = get_date_from_ts( c_ts["atm"], @@ -208,9 +201,7 @@ def generate_adf_config(cesm_root, cupid_example, adf_file, out_file): "regrid", ) # This is where ADF will make "regrid" files a_dict["diag_basic_info"]["cam_diag_plot_loc"] = os.path.join( - cupid_root, - "examples", - cupid_example, + cupid_config_loc, "ADF_output", ) # this is where ADF will put plots, and "website" directory a_dict["user"] = os.environ["USER"] @@ -255,10 +246,10 @@ def generate_adf_config(cesm_root, cupid_example, adf_file, out_file): f.write( "# This file has been auto-generated using generate_adf_config_file.py\n", ) - f.write(f"# It is based off of examples/{cupid_example}/config.yml\n") + f.write(f"# It is based off of {cupid_config_loc}/config.yml\n") f.write("# Arguments:\n") f.write(f"# {cesm_root=}\n") - f.write(f"# {cupid_example=}\n") + f.write(f"# {cupid_config_loc=}\n") f.write(f"# {adf_file=}\n") f.write(f"# Output: {out_file=}\n") # enter in each element of the dictionary into the new file @@ -284,7 +275,7 @@ def get_date_from_ts(data: dict, keyname: str, listindex: int, default=None): print(args) generate_adf_config( args["cesm_root"], - args["cupid_example"], + args["cupid_config_loc"], args["adf_template"], args["out_file"], ) diff --git a/helper_scripts/generate_cupid_config_for_cesm_case.py b/helper_scripts/generate_cupid_config_for_cesm_case.py index 74edfce..d02f75f 100755 --- a/helper_scripts/generate_cupid_config_for_cesm_case.py +++ b/helper_scripts/generate_cupid_config_for_cesm_case.py @@ -108,10 +108,11 @@ def generate_cupid_config(case_root, cesm_root, cupid_example): dout_s_root = cesm_case.get_value("DOUT_S_ROOT") # Additional options we need to get from env_cupid.xml + base_case = "b.e23_alpha17f.BLT1850.ne30_t232.092" nyears = 1 start_date = "0001-01-01" end_date = f"{nyears+1:04d}-01-01" - climo_nyears = 1 + climo_nyears = nyears base_case_output_dir = "/glade/campaign/cesm/development/cross-wg/diagnostic_framework/CESM_output_for_testing" base_nyears = 100 base_end_date = f"{base_nyears+1:04d}-01-01" @@ -120,18 +121,40 @@ def generate_cupid_config(case_root, cesm_root, cupid_example): with open(os.path.join(cupid_root, "examples", cupid_example, "config.yml")) as f: my_dict = yaml.safe_load(f) + my_dict["data_sources"]["nb_path_root"] = os.path.join( + cesm_root, + "tools", + "CUPiD", + "nblibrary", + ) my_dict["global_params"]["case_name"] = case my_dict["global_params"]["start_date"] = start_date my_dict["global_params"]["end_date"] = end_date - my_dict["global_params"]["climo_nyears"] = climo_nyears + my_dict["global_params"]["base_case_name"] = base_case my_dict["global_params"]["base_case_output_dir"] = base_case_output_dir my_dict["global_params"]["base_end_date"] = base_end_date - my_dict["global_params"]["base_climo_nyears"] = base_climo_nyears - my_dict["timeseries"]["case_name"] = case - my_dict["timeseries"]["atm"]["end_years"] = [nyears, base_nyears] + my_dict["timeseries"]["case_name"] = [case, base_case] + + for component in my_dict["timeseries"]: + if ( + isinstance(my_dict["timeseries"][component], dict) + and "end_years" in my_dict["timeseries"][component] + ): + my_dict["timeseries"][component]["end_years"] = [nyears, base_nyears] + if "link_to_ADF" in my_dict["compute_notebooks"]["atm"]: + my_dict["compute_notebooks"]["atm"]["link_to_ADF"]["parameter_groups"]["none"][ + "adf_root" + ] = os.path.join(case_root, "ADF_output") + if "Greenland_SMB_visual_compare_obs" in my_dict["compute_notebooks"]["glc"]: + my_dict["compute_notebooks"]["glc"]["Greenland_SMB_visual_compare_obs"][ + "parameter_groups" + ]["none"]["climo_nyears"] = climo_nyears + my_dict["compute_notebooks"]["glc"]["Greenland_SMB_visual_compare_obs"][ + "parameter_groups" + ]["none"]["base_climo_nyears"] = base_climo_nyears # replace with environment variable - my_dict["global_params"]["CESM_output_dir"] = dout_s_root + my_dict["global_params"]["CESM_output_dir"] = os.path.dirname(dout_s_root) # create new file, make it writeable with open("config.yml", "w") as f: diff --git a/nblibrary/atm/Global_PSL_NMSE_compare_obs_lens.ipynb b/nblibrary/atm/Global_PSL_NMSE_compare_obs_lens.ipynb index 7909c8d..dbd3252 100644 --- a/nblibrary/atm/Global_PSL_NMSE_compare_obs_lens.ipynb +++ b/nblibrary/atm/Global_PSL_NMSE_compare_obs_lens.ipynb @@ -204,7 +204,7 @@ " fix_time_dim(\n", " xr.open_mfdataset(f\"{base_file_path}/*PSL*.nc\", decode_times=False)\n", " )\n", - " .sel(time=slice(start_date, end_date))\n", + " .sel(time=slice(base_start_date, base_end_date))\n", " .assign_coords({\"lon\": lon, \"lat\": lat})\n", " .PSL\n", " / 100.0\n", diff --git a/nblibrary/atm/averaging_utils.py b/nblibrary/atm/averaging_utils.py index b98625b..96a44e9 100644 --- a/nblibrary/atm/averaging_utils.py +++ b/nblibrary/atm/averaging_utils.py @@ -36,7 +36,10 @@ def seasonal_climatology_weighted(dat): datw_am = dat * wgts_am ds_season = ( - datw.rolling(min_periods=3, center=True, time=3).sum().dropna("time", how="all") + datw.load() + .rolling(min_periods=3, center=True, time=3) + .sum() + .dropna("time", how="all") ) dat_djf = ds_season.where(ds_season.time.dt.month == 1, drop=True).mean("time") dat_mam = ds_season.where(ds_season.time.dt.month == 4, drop=True).mean("time") diff --git a/nblibrary/glc/Greenland_SMB_visual_compare_obs.ipynb b/nblibrary/glc/Greenland_SMB_visual_compare_obs.ipynb index 5065ebe..0d4c584 100644 --- a/nblibrary/glc/Greenland_SMB_visual_compare_obs.ipynb +++ b/nblibrary/glc/Greenland_SMB_visual_compare_obs.ipynb @@ -36,6 +36,8 @@ "outputs": [], "source": [ "# Import packages\n", + "import os\n", + "\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import matplotlib.cm as mcm\n", @@ -121,7 +123,7 @@ { "cell_type": "code", "execution_count": null, - "id": "cef60ddb-9ff4-4a14-a8ea-0d5740b6c18a", + "id": "08146fd8-d631-46c5-abc3-04fc1a004b77", "metadata": { "editable": true, "slideshow": { @@ -135,18 +137,27 @@ "source": [ "last_year = int(end_date.split(\"-\")[0])\n", "\n", - "case_init_file = f\"{CESM_output_dir}/{case_name}/glc/hist/{case_name}.cism.gris.initial_hist.0001-01-01-00000.nc\" # name of glc file output\n", + "case_init_file = os.path.join(\n", + " obs_path, \"cism.gris.initial_hist.0001-01-01-00000.nc\"\n", + ") # name of glc file output\n", "\n", - "case_path = f\"{CESM_output_dir}/{case_name}/cpl/hist\" # path to glc output\n", - "case_file = f\"{case_path}/{case_name}.cpl.hx.1yr2glc.{last_year:04d}-01-01-00000.nc\" # name of glc file output\n", - "obs_file = f\"{obs_path}/{obs_name}\" # name of observed dataset file\n", + "case_path = os.path.join(\n", + " CESM_output_dir, case_name, \"cpl\", \"hist\"\n", + ") # path to glc output\n", + "case_file = os.path.join(\n", + " case_path, f\"{case_name}.cpl.hx.1yr2glc.{last_year:04d}-01-01-00000.nc\"\n", + ") # name of glc file output\n", + "obs_file = os.path.join(obs_path, obs_name) # name of observed dataset file\n", "\n", "if base_case_name is not None:\n", " base_last_year = int(base_end_date.split(\"-\")[0])\n", - " base_case_path = (\n", - " f\"{base_case_output_dir}/{base_case_name}/cpl/hist\" # path to cpl output\n", - " )\n", - " base_file = f\"{base_case_path}/{base_case_name}.cpl.hx.1yr2glc.{base_last_year:04d}-01-01-00000.nc\" # name of last cpl simulation output" + " base_case_path = os.path.join(\n", + " base_case_output_dir, base_case_name, \"cpl\", \"hist\"\n", + " ) # path to cpl output\n", + " base_file = os.path.join(\n", + " base_case_path,\n", + " f\"{base_case_name}.cpl.hx.1yr2glc.{base_last_year:04d}-01-01-00000.nc\",\n", + " ) # name of last cpl simulation output" ] }, { diff --git a/nblibrary/lnd/Global_TerrestrialCouplingIndex_VisualCompareObs.ipynb b/nblibrary/lnd/Global_TerrestrialCouplingIndex_VisualCompareObs.ipynb index a505626..a1ab61e 100755 --- a/nblibrary/lnd/Global_TerrestrialCouplingIndex_VisualCompareObs.ipynb +++ b/nblibrary/lnd/Global_TerrestrialCouplingIndex_VisualCompareObs.ipynb @@ -84,7 +84,16 @@ "# obsDir = '/glade/campaign/cgd/tss/people/mdfowler/FLUXNET2015/' ## Need to copy into CUPiD Data\n", "\n", "## Where CESM timeseries data is stored\n", - "CESM_output_dir = \"/glade/campaign/cesm/development/cross-wg/diagnostic_framework/CESM_output_for_testing/\"\n", + "CESM_output_dir = os.path.join(\n", + " os.path.sep,\n", + " \"glade\",\n", + " \"campaign\",\n", + " \"cesm\",\n", + " \"development\",\n", + " \"cross-wg\",\n", + " \"diagnostic_framework\",\n", + " \"CESM_output_for_testing\",\n", + ")\n", "\n", "\n", "## Full casenames that are present in CESM_output_dir and in individual filenames\n", @@ -123,6 +132,14 @@ "## - - - - - - - - - - - - - - - - - - - - - -\n", "## Settings for computing coupling index\n", "## - - - - - - - - - - - - - - - - - - - - - -\n", + "\n", + "# Set up directory for aux output like coupling index file\n", + "if \"SCRATCH\" in os.environ:\n", + " cupid_temp = os.path.join(os.path.sep, os.environ[\"SCRATCH\"], \"CUPiD_scratch\")\n", + " os.makedirs(cupid_temp, exist_ok=True)\n", + "else:\n", + " cupid_temp = \".\"\n", + "\n", "startYrs = [start_date.split(\"-\")[0]]\n", "endYrs = [f\"{int(end_date.split('-')[0])-1:04d}\"]\n", "\n", @@ -200,10 +217,8 @@ "source": [ "for iCase in range(len(caseNames)):\n", " ## Check first if coupling index has already been created:\n", - " TCI_filePath = (\n", - " \"/glade/derecho/scratch/mdfowler/\"\n", - " + caseNames[0]\n", - " + \"_TerrestrialCouplingIndex_SHvsSM.nc\"\n", + " TCI_filePath = os.path.join(\n", + " cupid_temp, f\"{caseNames[iCase]}_TerrestrialCouplingIndex_SHvsSM.nc\"\n", " )\n", "\n", " if os.path.exists(TCI_filePath): # Use previously computed TCI\n", @@ -280,14 +295,8 @@ " yDS = shflx_DS\n", "\n", " couplingInd = compute_couplingIndex_cesm(xname, yname, xDS, yDS)\n", - "\n", - " filePath = (\n", - " \"/glade/derecho/scratch/mdfowler/\"\n", - " + caseNames[0]\n", - " + \"_TerrestrialCouplingIndex_SHvsSM.nc\"\n", - " )\n", - " couplingInd.to_netcdf(filePath)\n", - " print(\"File created: \", filePath)" + " couplingInd.to_netcdf(TCI_filePath)\n", + " print(\"File created: \", TCI_filePath)" ] }, { @@ -295,7 +304,7 @@ "id": "5f8fba2a-98d2-4e94-9d71-3b2625e16032", "metadata": {}, "source": [ - "### 1.1 Read in FLUXNET data if requested" + "### 2.1 Read in FLUXNET data if requested" ] }, { @@ -659,7 +668,7 @@ "id": "bc387253-cdd7-4a36-956b-8ce548e963bd", "metadata": {}, "source": [ - "## 2. Make plots" + "## 3. Make plots" ] }, { @@ -701,12 +710,11 @@ "outputs": [], "source": [ "for iCase in range(len(caseNames)):\n", - " filePath = (\n", - " \"/glade/derecho/scratch/mdfowler/\"\n", - " + caseNames[iCase]\n", - " + \"_TerrestrialCouplingIndex_SHvsSM.nc\"\n", + " TCI_filePath = os.path.join(\n", + " cupid_temp, f\"{caseNames[iCase]}_TerrestrialCouplingIndex_SHvsSM.nc\"\n", " )\n", - " couplingIndex_case = uxr.open_dataset(gridFile, filePath)\n", + "\n", + " couplingIndex_case = uxr.open_dataset(gridFile, TCI_filePath)\n", " # Rename the variable:\n", " couplingIndex_case = couplingIndex_case.rename(\n", " {\"__xarray_dataarray_variable__\": \"CouplingIndex\"}\n",