Fixed parsing errors and incorrectly set namelist parameters (HPSCTer…

…rSys#6)

Commit summary:

- f0fc1a6 Set default case_name to filename; bumped script version to 0.4
- 96ec507 Fixed eclm runtime errors due to namelist errors
- 95dfb47 Renamed cesm.exe to eclm.exe
- 8405cba Allowed stream file paths to be modified by user
- 128117b Added namelist validator script
- 5737064 Fixed discrepancies in drv_in
- 1fcc7ab Fixed discrepancies in datm, mosart, and modelio namelists

build_tools/build.juwels-centos8

@@ -2,9 +2,9 @@
 
 # ------- User-specified variables ------------
 ARCH="juwels-centos8"
-CLM5_TSMP_ROOT=$(git rev-parse --show-toplevel)
-BUILD_FOLDER="$CLM5_TSMP_ROOT/outputs/$ARCH/build"
-INSTALL_PATH="$CLM5_TSMP_ROOT/outputs/$ARCH/run"
+ECLM_ROOT=$(git rev-parse --show-toplevel)
+BUILD_FOLDER="$ECLM_ROOT/outputs/$ARCH/build"
+INSTALL_PATH="$ECLM_ROOT/outputs/$ARCH/run"
 if [ "$1" = "--debug" ] || [ "$1" = "-d" ]; then
   BUILD_TYPE="DEBUG"
 else
@@ -32,7 +32,7 @@ echo "NetCDF_Fortran_PATH=$EBROOTNETCDFMINFORTRAN"
 echo "PnetCDF_C_PATH=$EBROOTPARALLELMINNETCDF"
 GENF90_PATH=$(pwd)
 rm -rf $BUILD_FOLDER
-cmake -S "$CLM5_TSMP_ROOT/src" \
+cmake -S "$ECLM_ROOT/src" \
       -B "$BUILD_FOLDER" \
       -DCMAKE_INSTALL_PREFIX="$INSTALL_PATH" \
       -DCMAKE_MODULE_PATH="$(pwd)/cmake" \

build_tools/build.ubuntu-20.04LTS

@@ -2,9 +2,9 @@
 
 # ------- User-specified variables ------------
 ARCH="ubuntu-20.04LTS"
-CLM5_TSMP_ROOT=$(git rev-parse --show-toplevel)
-BUILD_FOLDER="$CLM5_TSMP_ROOT/outputs/$ARCH/build"
-INSTALL_PATH="$CLM5_TSMP_ROOT/outputs/$ARCH/run"
+ECLM_ROOT=$(git rev-parse --show-toplevel)
+BUILD_FOLDER="$ECLM_ROOT/outputs/$ARCH/build"
+INSTALL_PATH="$ECLM_ROOT/outputs/$ARCH/run"
 if [ "$1" = "--debug" ] || [ "$1" = "-d" ]; then
   BUILD_TYPE="DEBUG"
 else
@@ -15,7 +15,7 @@ fi
 NetCDF_ROOT=/opt/custom
 GENF90_PATH=$(pwd)
 rm -rf $BUILD_FOLDER
-cmake -S "$CLM5_TSMP_ROOT/src" \
+cmake -S "$ECLM_ROOT/src" \
       -B "$BUILD_FOLDER" \
       -DCMAKE_INSTALL_PREFIX="$INSTALL_PATH" \
       -DCMAKE_MODULE_PATH="$(pwd)/cmake" \

build_tools/cmake/SetBuildOptions.cmake

@@ -39,4 +39,4 @@ endif()
 message(STATUS " ******* ${CMAKE_PROJECT_NAME} build options ******* ")
 message(STATUS " Build type = '${CMAKE_BUILD_TYPE}'")
 message(STATUS " Compiler = '${COMPILER}'")
-message(STATUS " *************************************** ")
+message(STATUS " ********************************** ")

namelist_generator/clm5nl-check

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+"""clm5nl-check - CLM5 namelist validator
+
+Checks if the specified directory contains valid CLM5 namelists.
+
+Usage: 
+  clm5nl-check [DIR]
+
+Arguments:
+  DIR            Directory to check. Defaults to current directory
+                 if not specified.
+
+Options:
+  -h --help      Show this screen.
+  -v --version   Show version.
+"""
+from docopt import docopt
+import os, re, sys
+
+__version__ = "0.1"
+args = docopt(__doc__, version="clm5nl-check v" + __version__)
+
+errors = []
+case_dir = os.path.abspath(args["DIR"]) if args["DIR"] else os.getcwd()
+if os.path.isdir(case_dir):
+  os.chdir(case_dir)
+  nl_pio = [f"{c}_modelio.nml" for c in ["atm", "cpl", "esp", "glc", "ice", "lnd", "ocn", "rof", "wav"]]
+  nl_all = ["lnd_in", "datm_in", "drv_in", "drv_flds_in", "mosart_in", "seq_maps.rc"]
+  nl_all.extend(nl_pio)
+  if os.path.isfile("datm_in"):
+    s_params = "".join(l for l in open("datm_in", "r").readlines() if "datm.streams.txt" in l)
+    s_files = re.compile(r"[^('|\")]*datm.streams.txt[^\s]*").findall(s_params, re.MULTILINE)
+    if s_files: nl_all.extend(f.strip() for f in s_files)
+  for nl in nl_all:
+    if not os.path.isfile(nl):
+      errors.append(f"'{nl}' is missing")
+else:
+  errors.append(f"{case_dir} does not exist")
+
+if len(errors) > 0:
+  print("clm5nl-check errors:")
+  for msg in errors: print(f"  {msg}")
+  sys.exit(1)
+
+
+

namelist_generator/clm5nl-gen

@@ -24,7 +24,7 @@ import toml
 from docopt import docopt
 from clm5nl.generators import build_namelist
 
-__version__ = "0.3"
+__version__ = "0.4"
 
 env_var_regex = re.compile(r"\$\{.*\}")
 _missing_envs = set()
@@ -57,11 +57,16 @@ def generate_namelists(params_file, out_dir: str = ""):
   print(f"Namelists will be saved to {out_dir}")
   os.makedirs(out_dir, exist_ok=True)   
   for nl, opts in nl_opts.items():
-    if nl == "lnd_in":
+    if nl == "drv_in":
+      if "case_name" not in opts:
+        opts["case_name"] = os.path.splitext(os.path.basename(params_file))[0]
+    elif nl == "lnd_in":
       opts["drv_in.start_type"] = nl_opts["drv_in"]["clm_start_type"]
     elif nl == "drv_flds_in":
       opts["lnd_in.clm_accelerated_spinup"] = nl_opts["lnd_in"]["clm_accelerated_spinup"]
       opts["lnd_in.use_fates"] = nl_opts["lnd_in"]["use_fates"]
+    elif nl == "modelio_nml":
+      opts["MAX_MPITASKS_PER_NODE"] = nl_opts["drv_in"]["ntasks"]
     success, msg = build_namelist(nl, opts, out_dir)
     if not success: print(f'ERROR in build_namelist("{nl}"): {msg}', file=sys.stderr) ; return 3
 
@@ -70,7 +75,7 @@ def generate_namelists(params_file, out_dir: str = ""):
   for nl in namelists:
     if nl == "mosart_in":
       #TODO: Throw error if ${CESMDATAROOT} is missing
-      opts["frivinp_rtm"] = resolve_env_vars("${CESMDATAROOT}/inputdata") 
+      opts["frivinp_rtm"] = resolve_env_vars("${CESMDATAROOT}") 
     success, msg = build_namelist(nl, opts, out_dir)
     if not success: print(f'ERROR in build_namelist("{nl}"): {msg}', file=sys.stderr) ; return 3
   print("Success!")

namelist_generator/clm5nl/generators/constants_datm.py

@@ -94,7 +94,7 @@
   "DOMAIN_FILE_NAMES":  "topodata_0.9x1.25_USGS_070110_stream_c151201.nc",
   "FIELD_VAR_NAMES"  :  "TOPO    topo",
   "FIELD_FILE_PATH"  :  "$DIN_LOC_ROOT/atm/datm7/topo_forcing",
-  "FIELD_FILE_NAMES" :  "topodata_0.9x1.25_USGS_070110_stream_c151201.n"
+  "FIELD_FILE_NAMES" :  "topodata_0.9x1.25_USGS_070110_stream_c151201.nc"
 }
 
 # Store indent spaces for multiline

namelist_generator/clm5nl/generators/gen_datm_in.py

@@ -13,8 +13,8 @@
 
 __all__ = ['build_datm_in']
 
-_user_nl = {}
 _opts = {}
+_user_nl = {}
 _nl = datm_in()
 
 def build_datm_in(opts: dict = None, nl_file: str = "datm_in"):
@@ -37,12 +37,11 @@ def build_datm_in(opts: dict = None, nl_file: str = "datm_in"):
     # Write to file
     if nl_file and Path(nl_file).name.strip() != "": 
         _nl.write(nl_file, ["datm_nml", "shr_strdata_nml"])
-
-        # Generate stream files
-        out_dir = Path(nl_file).parent.absolute() if nl_file else Path.cwd()
-        create_stream_files(_opts["datm_in.streams"], out_dir)
-        generated_files = [Path(out_dir, s) for s in _opts["datm_in.streams"].values()]
-        generated_files.append(Path(nl_file))
+        generated_files = [Path(nl_file)]
+        if "streams" not in _user_nl:
+            # generate default stream files if no streams were specified
+            out_dir = Path(nl_file).parent.absolute()
+            generated_files.extend(Path(out_dir, s) for s in create_stream_files(out_dir))
         return True, generated_files
     else:
         return True, ""
@@ -65,23 +64,32 @@ def shr_strdata_nml():
     with _nl.shr_strdata_nml as n:
         n.datamode   = "CLMNCEP" 
         n.domainfile = _opts["domainfile"] # not optional!
-        n.streams    = generate_streams()
+        if "streams" in _user_nl:
+            n.streams = _user_nl["streams"]
+        else:
+            n.streams = generate_streams()
         num_streams  = len(n.streams)
 
         # array params w/ length = # stream files
-        n.dtlimit    = [_user_nl.get("dtlimit"  , 1.5)]        * num_streams # see line 3040
-        n.fillalgo   = [_user_nl.get("fillalgo" , "nn")]       * num_streams
-        n.fillmask   = [_user_nl.get("fillmask" , "nomask")]   * num_streams
-        n.fillread   = [_user_nl.get("fillread" , "NOT_SET")]  * num_streams
-        n.fillwrite  = [_user_nl.get("fillwrite", "NOT_SET")]  * num_streams
-        n.mapalgo    = [_user_nl.get("mapalgo"  , "bilinear")] * num_streams # stream-dependent; see L2884
-        n.mapmask    = [_user_nl.get("mapmask"  , "nomask")]   * num_streams
-        n.mapread    = [_user_nl.get("mapread"  , "NOT_SET")]  * num_streams
-        n.mapwrite   = [_user_nl.get("mapwrite" , "NOT_SET")]  * num_streams
-        n.readmode   = [_user_nl.get("readmode" , "single")]   * num_streams
-        n.taxmode    = [_user_nl.get("taxmode"  , "cycle")]    * num_streams # stream-dependent; see L2994
-        n.tintalgo   = [_user_nl.get("tintalgo" , "nearest")]  * num_streams # stream-dependent; see L2938
-        n.vectors    = [_user_nl.get("vectors"  , "null")]     * num_streams # depends on datm_mode; see L3065
+        n.dtlimit    = set_array_values("dtlimit"  , 1.5,        num_streams)  # see line 3040
+        n.fillalgo   = set_array_values("fillalgo" , "nn",       num_streams)      
+        n.fillmask   = set_array_values("fillmask" , "nomask",   num_streams)  
+        n.fillread   = set_array_values("fillread" , "NOT_SET",  num_streams)
+        n.fillwrite  = set_array_values("fillwrite", "NOT_SET",  num_streams)
+        n.mapalgo    = set_array_values("mapalgo"  , "bilinear", num_streams)  # stream-dependent; see L2884
+        n.mapmask    = set_array_values("mapmask"  , "nomask",   num_streams)  
+        n.mapread    = set_array_values("mapread"  , "NOT_SET",  num_streams) 
+        n.mapwrite   = set_array_values("mapwrite" , "NOT_SET",  num_streams) 
+        n.readmode   = set_array_values("readmode" , "single",   num_streams)  
+        n.taxmode    = set_array_values("taxmode"  , "cycle",    num_streams)  # stream-dependent; see L2994
+        n.tintalgo   = set_array_values("tintalgo" , "nearest",  num_streams)  # stream-dependent; see L2938
+        n.vectors    = set_array_values("vectors"  , "null",     num_streams)  # depends on datm_mode; see L3065
+
+def set_array_values(key, default_val, array_len):
+    if key in _user_nl:
+        return _user_nl[key]
+    else:
+        return [default_val] * array_len
 
 def generate_streams():
     datm_mode = _opts["datm_mode"]
@@ -103,26 +111,30 @@ def generate_streams():
              f'{s_files["presaero"]} {PRESAERO_YR_PARAMS.get(datm_presaero)}',
              f'{s_files["topo"]} 1 1 1']
 
-    _opts["datm_in.streams"] = s_files
     return s_nml
 
-def create_stream_files(stream_files : dict, out_dir : str):
+def create_stream_files(out_dir : str):
     s_template = Template(STREAM_FILE_TEMPLATE)
     s_vars = {}
-
-    for s_type, s_file in stream_files.items():
-        # Populate stream variables 
+    s_files = []
+    for stream in _nl.shr_strdata_nml.streams:
+        s_file, s_type = parse_stream_param(stream)
+        s_files.append(s_file)
         if s_type == "presaero":
             s_vars = deepcopy(PRESAERO_STREAM_DEFAULTS)
+            s_vars["DOMAIN_FILE_PATH"]  = Path(_opts["domainfile"]).parent.absolute()
+            s_vars["FIELD_FILE_PATH"]   = Path(_opts["domainfile"]).parent.absolute()
         elif s_type == "topo":
             s_vars = deepcopy(TOPO_STREAM_DEFAULTS)
+            s_vars["DOMAIN_FILE_PATH"]  = Path(_opts["domainfile"]).parent.absolute()
+            s_vars["FIELD_FILE_PATH"]   = Path(_opts["domainfile"]).parent.absolute()
         else:
             s_vars["DOMAIN_FILE_PATH"]  = Path(_opts["domainfile"]).parent.absolute()
             s_vars["DOMAIN_FILE_NAMES"] = Path(_opts["domainfile"]).name
-            s_vars["FIELD_FILE_PATH"]   = _opts["domainfile"]
+            s_vars["FIELD_FILE_PATH"]   = _opts.get("stream_root_dir", "")
             s_vars["DOMAIN_VAR_NAMES"]  = DATM_STREAM_DEFAULTS["DOMAIN_VAR_NAMES"]
-            s_vars["FIELD_VAR_NAMES"]   = DATM_STREAM_DEFAULTS["FIELD_VAR_NAMES"][s_type]
-            s_vars["FIELD_FILE_NAMES"]  = _opts["stream_files"]
+            s_vars["FIELD_VAR_NAMES"]   = DATM_STREAM_DEFAULTS["FIELD_VAR_NAMES"].get(s_type, "")
+            s_vars["FIELD_FILE_NAMES"]  = _opts.get("stream_files", "")
 
         # Convert list variables into multiline strings with proper indentation
         for key, val in s_vars.items():
@@ -134,6 +146,19 @@ def create_stream_files(stream_files : dict, out_dir : str):
         # Write stream file to disk
         with open(Path(out_dir, s_file), "w+") as output:
             output.write(s_template.safe_substitute(s_vars))
+    return s_files
+
+def parse_stream_param(stream):
+    if len(stream.split(" ")) == 4:
+        stream_file = stream.split(" ")[0]
+        stream_type = stream_file.split(".")[-1]
+        return stream_file, stream_type
+    else:
+        error(f"""
+               Invalid stream namelist parameter '{line}'.
+               The correct syntax is:
+               <stream_file> <year_align> <year_first> <year_last>
+               """)
 
 if __name__ == "__main__":
     """
@@ -150,7 +175,7 @@ def create_stream_files(stream_files : dict, out_dir : str):
     opts["stream_year_align"] = "2017"
     opts["stream_year_first"] = "2017"
     opts["stream_year_last"] = "2017"
-    opts["stream_root_folder"] = "/p/scratch/nrw_test_case/COSMOREA6/forcings"
+    opts["stream_root_dir"] = "/p/scratch/nrw_test_case/COSMOREA6/forcings"
     opts["stream_files"] = ["2017-{}.nc".format(str(month).zfill(2)) for month in range(1,13)]
     opts["domainfile"] = "/p/scratch/nrw_test_case/domain.lnd.300x300_NRW_300x300_NRW.190619.nc"
     build_datm_in(opts, nl, "datm_in_test")