clm5nl-gen v0.5 (HPSCTerrSys#7)

Commit summary:

- 8577cfa Standardized location of version string to `clm5nl/_version.py`. Bumped to v0.5.
- a700656 Don't set default flanduse_timeseries if not specified in model params file; also fixed incorrect parsing of stream types in datm_in
- 52be297 Auto create folders required by drv_in and modelio; also fixed pio_stride not being updated in modelio

namelist_generator/clm5nl-check

@@ -14,11 +14,11 @@ Options:
   -h --help      Show this screen.
   -v --version   Show version.
 """
-from docopt import docopt
 import os, re, sys
+from docopt import docopt
+from clm5nl._version import __version__
 
-__version__ = "0.1"
-args = docopt(__doc__, version="clm5nl-check v" + __version__)
+args = docopt(__doc__, version=__version__)
 
 errors = []
 case_dir = os.path.abspath(args["DIR"]) if args["DIR"] else os.getcwd()
@@ -28,8 +28,8 @@ if os.path.isdir(case_dir):
   nl_all = ["lnd_in", "datm_in", "drv_in", "drv_flds_in", "mosart_in", "seq_maps.rc"]
   nl_all.extend(nl_pio)
   if os.path.isfile("datm_in"):
-    s_params = "".join(l for l in open("datm_in", "r").readlines() if "datm.streams.txt" in l)
-    s_files = re.compile(r"[^('|\")]*datm.streams.txt[^\s]*").findall(s_params, re.MULTILINE)
+    s_params = "".join(l for l in open("datm_in", "r").readlines() if "datm.streams" in l)
+    s_files = re.compile(r"[^('|\")]*datm.streams[^\s]*").findall(s_params, re.MULTILINE)
     if s_files: nl_all.extend(f.strip() for f in s_files)
   for nl in nl_all:
     if not os.path.isfile(nl):

namelist_generator/clm5nl-gen

@@ -23,29 +23,26 @@ import os, re, sys
 import toml
 from docopt import docopt
 from clm5nl.generators import build_namelist
-
-__version__ = "0.4"
+from clm5nl._version import __version__
 
 env_var_regex = re.compile(r"\$\{.*\}")
 _missing_envs = set()
 
 def generate_namelists(params_file, out_dir: str = ""):
   """
-  Generate CLM5 namelists from a .toml parameter file.
+  Generates Fortran namelists from a model parameters file.
   """
   print(f"Reading model parameters file from {params_file}")
 
   # Parse model parameters file
-  model_config = toml.loads(open(params_file, "r").read())
+  model_params = toml.loads(open(params_file, "r").read())
   namelists = ["drv_in", "lnd_in", "drv_flds_in", "datm_in", "mosart_in", "modelio_nml", "seq_maps.rc"]
-  nl_opts = {}
-  for section in model_config:
+  nl_opts = dict.fromkeys(namelists, {})
+  for section in model_params:
     if section in namelists:
-      nl_opts[section] = {k:resolve_env_vars(v) for k, v in model_config[section].items()}     
-      namelists.remove(section)
+      nl_opts[section] = {k:resolve_env_vars(v) for k, v in model_params[section].items()}
     else:
-
-      print(f"***WARNING*** Section '{section}' in config file is not recognized.")
+      print(f"***WARNING*** Unrecognized section in params file: '{section}'")
 
   # Print warnings
   if has_warnings():
@@ -65,19 +62,13 @@ def generate_namelists(params_file, out_dir: str = ""):
     elif nl == "drv_flds_in":
       opts["lnd_in.clm_accelerated_spinup"] = nl_opts["lnd_in"]["clm_accelerated_spinup"]
       opts["lnd_in.use_fates"] = nl_opts["lnd_in"]["use_fates"]
+    elif nl == "mosart_in":
+      opts["frivinp_rtm"] = resolve_env_vars("${CESMDATAROOT}")
     elif nl == "modelio_nml":
-      opts["MAX_MPITASKS_PER_NODE"] = nl_opts["drv_in"]["ntasks"]
+      opts["drv_in.ntasks"] = nl_opts["drv_in"]["ntasks"]
     success, msg = build_namelist(nl, opts, out_dir)
     if not success: print(f'ERROR in build_namelist("{nl}"): {msg}', file=sys.stderr) ; return 3
 
-  # Namelists not mentioned in config file should still be generated
-  opts = {}
-  for nl in namelists:
-    if nl == "mosart_in":
-      #TODO: Throw error if ${CESMDATAROOT} is missing
-      opts["frivinp_rtm"] = resolve_env_vars("${CESMDATAROOT}") 
-    success, msg = build_namelist(nl, opts, out_dir)
-    if not success: print(f'ERROR in build_namelist("{nl}"): {msg}', file=sys.stderr) ; return 3
   print("Success!")
   return 0
 
@@ -121,7 +112,7 @@ def has_warnings():
     return False
 
 if __name__ == "__main__":
-  args = docopt(__doc__, version="clm5nl-gen v" + __version__)
+  args = docopt(__doc__, version=__version__)
   params_file = os.path.abspath((args["PARAMFILE"]))
   if (not os.path.exists(params_file)):
     print(f"clm5nl-gen error: Model parameter file '{params_file}' not found.", file=sys.stderr)

namelist_generator/clm5nl/_version.py

@@ -0,0 +1 @@
+__version__ = "0.5"

namelist_generator/clm5nl/generators/gen_datm_in.py

@@ -151,7 +151,19 @@ def create_stream_files(out_dir : str):
 def parse_stream_param(stream):
     if len(stream.split(" ")) == 4:
         stream_file = stream.split(" ")[0]
-        stream_type = stream_file.split(".")[-1]
+        sf = stream_file.lower()
+        if "solar" in sf:
+            stream_type = "Solar"
+        elif "precip" in sf:
+            stream_type = "Precip"
+        elif "tpqw" in sf:
+            stream_type = "TPQW"
+        elif "presaero" in sf:
+            stream_type = "presaero"
+        elif "topo" in sf:
+            stream_type = "topo"
+        else:
+            stream_type = "Unknown"
         return stream_file, stream_type
     else:
         error(f"""

namelist_generator/clm5nl/generators/gen_drv_in.py

@@ -61,6 +61,8 @@ def build_drv_in(opts: dict = None, nl_file: str = "drv_in"):
     #Write to file
     if nl_file and Path(nl_file).name.strip() != "":
         _nl.write(nl_file)
+        Path(_nl.seq_infodata_inparm.timing_dir).mkdir(parents=True, exist_ok=True)
+        Path(_nl.seq_infodata_inparm.tchkpt_dir).mkdir(parents=True, exist_ok=True)
         return True, Path(nl_file)
     else:
         return True, ""

namelist_generator/clm5nl/generators/gen_lnd_in.py

@@ -620,7 +620,8 @@ def setup_logic_initial_conditions():
 
 def setup_logic_dynamic_subgrid():
     # Options controlling which parts of flanduse_timeseries to use
-    _nl.dynamic_subgrid.flanduse_timeseries = _user_nl.get("flanduse_timeseries", None)
+    if "flanduse_timeseries" in _user_nl:
+        _nl.dynamic_subgrid.flanduse_timeseries = _user_nl["flanduse_timeseries"]
     setup_logic_do_transient_pfts()
     setup_logic_do_transient_crops()
     setup_logic_do_harvest()

namelist_generator/clm5nl/generators/gen_modelio_nml.py

@@ -9,51 +9,46 @@
 
 __all__ = ['build_modelio_nml']
 
-# _in_nl = {}
-# _in_opts = {}
-# _nl = datm_in()
-
 def build_modelio_nml(opts: dict = None, out_dir: str = None):
-    # Global vars aren't necessary for now
-    # global _in_opts, _in_nl, _nl
-    #_in_opts = opts
-    #_in_nl = nl
-
     _nl = modelio_in()
 
     with _nl.modelio as n:
         n.diri = "UNUSED"
-        n.diro = opts.get("run_dir", Path(out_dir).absolute())
+        n.diro = opts.get("log_dir", Path(out_dir, "logs").absolute())
 
     with _nl.pio_inparm as n:
         n.pio_netcdf_format = "64bit_offset"
         n.pio_numiotasks = -99
         n.pio_rearranger = 1
         n.pio_root = 1
-        n.pio_stride = opts.get("MAX_MPITASKS_PER_NODE", 8)
+        n.pio_stride = opts.get("drv_in.ntasks", 1)
         n.pio_typename = "netcdf"
 
-    if out_dir is None: out_dir = Path.cwd()
-    components = ["atm", "cpl", "glc", "ice", "lnd", "ocn", "rof", "wav"]
-    nl_files = {}
-    for comp in components:
-        nl_files[comp] = Path(out_dir, f"{comp}_modelio.nml")
-        _nl.modelio.logfile = "{}.log.{}".format(comp, datetime.now().strftime("%Y-%m-%d_%H%M%S"))
-        _nl.write(nl_files[comp], ["modelio", "pio_inparm"])
-
-    # Assign different set of defaults for esp
-    with _nl.pio_inparm as n:
-        n.pio_netcdf_format = ""
-        n.pio_rearranger = -99
-        n.pio_root = -99
-        n.pio_stride = -99
-        n.pio_typename = "nothing"
+    if out_dir is not None:
+        # Generate modelio namelists
+        components = ["atm", "cpl", "glc", "ice", "lnd", "ocn", "rof", "wav"]
+        nl_files = {}
+        for comp in components:
+            nl_files[comp] = Path(out_dir, f"{comp}_modelio.nml")
+            _nl.modelio.logfile = "{}.log.{}".format(comp, datetime.now().strftime("%Y-%m-%d_%H%M%S"))
+            _nl.write(nl_files[comp], ["modelio", "pio_inparm"])
 
-    nl_files["esp"] = Path(out_dir, "esp_modelio.nml")
-    _nl.modelio.logfile = "esp.log.{}".format(datetime.now().strftime("%Y-%m-%d_%H%M%S"))
-    _nl.write(nl_files["esp"], ["modelio", "pio_inparm"])
+        # Assign different set of defaults for esp
+        with _nl.pio_inparm as n:
+            n.pio_netcdf_format = ""
+            n.pio_rearranger = -99
+            n.pio_root = -99
+            n.pio_stride = -99
+            n.pio_typename = "nothing"
+        nl_files["esp"] = Path(out_dir, "esp_modelio.nml")
+        _nl.modelio.logfile = "esp.log.{}".format(datetime.now().strftime("%Y-%m-%d_%H%M%S"))
+        _nl.write(nl_files["esp"], ["modelio", "pio_inparm"])
 
-    return True, [Path(nl) for nl in nl_files.values()]
+        # Automatically create log directory
+        Path(_nl.modelio.diro).mkdir(parents=True, exist_ok=True)
+        return True, [Path(nl) for nl in nl_files.values()]
+    else:
+        return True, ""
 
 if __name__ == "__main__":
     """

namelist_generator/setup.cfg

@@ -3,7 +3,6 @@
 
 [metadata]
 name = clm5nl
-version = 0.4
 description = CLM5 namelist generator
 author = kvrigor
 author_email = [email protected]

namelist_generator/setup.py

@@ -1,2 +1,11 @@
 import setuptools
-setuptools.setup()
+import site, sys
+
+exec(open('clm5nl/_version.py').read())
+if "--user" in sys.argv[1:]:
+    # Enables --editable install with --user
+    # https://github.com/pypa/pip/issues/7953#issuecomment-645133255
+    site.ENABLE_USER_SITE = True
+    __version__ += ".dev"
+
+setuptools.setup(version = __version__)