WIP: allocator work

python/lsst/ctrl/execute/allocator.py

@@ -33,6 +33,7 @@
 from lsst.ctrl.execute.allocationConfig import AllocationConfig
 from lsst.ctrl.execute.condorInfoConfig import CondorInfoConfig
 from lsst.ctrl.execute.templateWriter import TemplateWriter
+from lsst.resources import ResourcePath
 
 _LOG = logging.getLogger(__name__)
 
@@ -48,11 +49,18 @@ class Allocator:
         the name of the platform to execute on
     opts : `Config`
         Config object containing options
-    condorInfoFileName : `str`
+    condorInfoFileName : `str | lsst.resources.ResourcePath`
         Name of the file containing Config information
+
+    Raises
+    ------
+    TypeError
+        If the condorInfoFileName is the wrong type.
     """
 
-    def __init__(self, platform, opts, configuration, condorInfoFileName):
+    def __init__(
+        self, platform: str, opts, configuration, condorInfoFileName: str | ResourcePath
+    ):
         """Constructor
         @param platform: target platform for PBS submission
         @param opts: options to override
@@ -61,9 +69,14 @@ def __init__(self, platform, opts, configuration, condorInfoFileName):
         self.defaults = {}
         self.configuration = configuration
 
-        fileName = envString.resolve(condorInfoFileName)
         condorInfoConfig = CondorInfoConfig()
-        condorInfoConfig.load(fileName)
+        if isinstance(condorInfoFileName, str):
+            fileName = envString.resolve(condorInfoFileName)
+            condorInfoConfig.load(fileName)
+        elif isinstance(condorInfoFileName, ResourcePath):
+            condorInfoConfig.loadFromStream(condorInfoFileName.read())
+        else:
+            raise TypeError("Wrong type of condor info file provided to allocator.")
 
         self.platform = platform
 
@@ -152,15 +165,18 @@ def load(self):
         )
         self.defaults["SCHEDULER"] = self.configuration.platform.scheduler
 
-    def loadAllocationConfig(self, name, suffix):
+    def loadAllocationConfig(self, name: str | ResourcePath, suffix):
         """Loads all values from allocationConfig and command line overrides
         into data structures suitable for use by the TemplateWriter object.
         """
-        resolvedName = envString.resolve(name)
         allocationConfig = AllocationConfig()
-        if not os.path.exists(resolvedName):
-            raise RuntimeError("%s was not found." % resolvedName)
-        allocationConfig.load(resolvedName)
+        if isinstance(name, str):
+            resolvedName = envString.resolve(name)
+            if not os.path.exists(resolvedName):
+                raise RuntimeError("%s was not found." % resolvedName)
+            allocationConfig.load(resolvedName)
+        elif isinstance(name, ResourcePath):
+            allocationConfig.loadFromStream(name.read())
 
         self.defaults["QUEUE"] = allocationConfig.platform.queue
         self.defaults["EMAIL_NOTIFICATION"] = allocationConfig.platform.email
@@ -460,3 +476,7 @@ def runCommand(self, cmd, verbose):
         # high order bits are status, low order bits are signal.
         exitCode = (status & 0xFF00) >> 8
         return exitCode
+
+    def submit(self):
+        """Submit the glidein jobs to the Batch system."""
+        raise NotImplementedError

python/lsst/ctrl/execute/allocatorParser.py

@@ -41,12 +41,9 @@ def __init__(self, basename):
         """
 
         self.defaults = {}
-
-        self.args = []
-
         self.args = self.parseArgs(basename)
 
-    def parseArgs(self, basename):
+    def parseArgs(self, basename) -> argparse.Namespace:
         """Parse command line, and test for required arguments
 
         Parameters
@@ -60,7 +57,9 @@ def parseArgs(self, basename):
         """
 
         parser = argparse.ArgumentParser(prog=basename)
-        parser.add_argument("platform", help="node allocation platform")
+        parser.add_argument(
+            "platform", type=str, default="s3df", help="node allocation platform"
+        )
         parser.add_argument(
             "--auto",
             action="store_true",
@@ -214,7 +213,7 @@ def getArgs(self):
 
         Returns
         -------
-        args: `list`
+        args: `argparse.Namespace`
             remaining command line arguments
         """
         return self.args

python/lsst/ctrl/execute/envString.py

@@ -24,23 +24,33 @@
 
 import os
 import re
-import sys
 
-# Given a string, look for any $ prefixed word, attempt to substitute
-# an environment variable with that name.
-# @throw exception if the environment variable doesn't exist
-# @return the resulting string
 
+def resolve(input: str) -> str:
+    """Render a string with any `$`-prefixed words substituted with a matching
+    environment variable.
 
-def resolve(strVal):
+    FIXME: this reimplements the `os.path.expandvars()` function with the
+           exception of raising errors on unresolved variables.
+
+    Parameters
+    ----------
+    input : str | lsst.resources.ResourcePath
+        The string or object that can be cast as a string containing
+        environment variables to resolve.
+
+    Raises
+    ------
+    RuntimeError
+        If the environment variable does not exist
+    """
     p = re.compile(r"\$[a-zA-Z0-9_]+")
-    retVal = strVal
+    retVal = input
     exprs = p.findall(retVal)
     for i in exprs:
         var = i[1:]
         val = os.getenv(var, None)
         if val is None:
             raise RuntimeError("couldn't find environment variable " + i)
-            sys.exit(120)
         retVal = p.sub(val, retVal, 1)
     return retVal

python/lsst/ctrl/execute/findPackageFile.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+
+#
+# LSST Data Management System
+# Copyright 2008-2016 LSST Corporation.
+#
+# This product includes software developed by the
+# LSST Project (http://www.lsst.org/).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
+# see <http://www.lsstcorp.org/LegalNotices/>.
+#
+import os
+
+import lsst.utils
+from lsst.ctrl.execute.envString import resolve
+from lsst.resources import ResourcePath
+
+
+def find_package_file(
+    filename: str, kind: str = "config", platform: str = "s3df"
+) -> ResourcePath:
+    """Find a package file from a set of candidate locations.
+
+    The candidate locations are, in descending order of preference:
+    - An `.lsst` directory in the user's home directory.
+    - An `lsst` directory in the user's `$XDG_CONFIG_HOME` directory
+    - An `etc/{kind}` directory in the stack environment for the platform
+    - An `etc/{kind}` directory in an installed `lsst.ctrl.platform.*` package
+    - An `etc/{kind}` directory in the `lsst.ctrl.execute` package.
+
+    Raises
+    ------
+    IndexError
+        If a requested file object cannot be located in the candidate hierarchy
+    """
+    _filename = resolve(filename)
+    home_dir = os.getenv("HOME", "/")
+    xdg_config_home = os.getenv("XDG_CONFIG_HOME", f"{home_dir}/.config")
+    try:
+        platform_pkg_dir = lsst.utils.getPackageDir(f"ctrl_platform_{platform}")
+    except (LookupError, ValueError):
+        platform_pkg_dir = None
+
+    file_candidates = [
+        ResourcePath(f"file://{home_dir}/.lsst/{_filename}"),
+        ResourcePath(f"file://{xdg_config_home}/lsst/{_filename}"),
+        (
+            ResourcePath(f"file://{platform_pkg_dir}/etc/{kind}/{_filename}")
+            if platform_pkg_dir
+            else None
+        ),
+        ResourcePath(
+            f"resource://lsst.ctrl.platform.{platform}/etc/{kind}/{_filename}"
+        ),
+        ResourcePath(f"resource://lsst.ctrl.execute/etc/{kind}/{_filename}"),
+    ]
+    try:
+        found_file: ResourcePath = [
+            c for c in file_candidates if c is not None and c.exists()
+        ][0]
+    except IndexError:
+        raise
+    return found_file

python/lsst/ctrl/execute/libexec/allocateNodes.py

@@ -23,14 +23,13 @@
 #
 
 import logging
-import os
 import sys
 from typing import Any
 
-import lsst.utils
-from lsst.ctrl.execute import envString
+from lsst.ctrl.execute.allocator import Allocator
 from lsst.ctrl.execute.allocatorParser import AllocatorParser
 from lsst.ctrl.execute.condorConfig import CondorConfig
+from lsst.ctrl.execute.findPackageFile import find_package_file
 from lsst.ctrl.execute.namedClassFactory import NamedClassFactory
 
 _LOG = logging.getLogger("lsst.ctrl.execute")
@@ -70,12 +69,9 @@ def main():
     platform = p.getPlatform()
 
     # load the CondorConfig file
-    platformPkgDir = lsst.utils.getPackageDir("ctrl_platform_" + platform)
-    execConfigName = os.path.join(platformPkgDir, "etc", "config", "execConfig.py")
-
-    resolvedName = envString.resolve(execConfigName)
+    execConfigName = find_package_file("execConfig.py", platform=platform)
     configuration = CondorConfig()
-    configuration.load(resolvedName)
+    configuration.loadFromStream(execConfigName.read())
 
     # create the plugin class
     schedulerName = configuration.platform.scheduler
@@ -84,12 +80,13 @@ def main():
     )
 
     # create the plugin
-    scheduler = schedulerClass(
-        platform, p.getArgs(), configuration, "$HOME/.lsst/condor-info.py"
+    condor_info_file = find_package_file("condor-info.py", platform=platform)
+    scheduler: Allocator = schedulerClass(
+        platform, p.getArgs(), configuration, condor_info_file
     )
 
     # submit the request
-    scheduler.submit(platform, platformPkgDir)
+    scheduler.submit()
 
 
 if __name__ == "__main__":