From dece688477c48b65c30083dc0a8a813f36a738c8 Mon Sep 17 00:00:00 2001 From: Todor Ivanov Date: Fri, 21 Jun 2024 16:09:18 +0200 Subject: [PATCH] Add first draft of wmagent-component-standalone script Move module imports out of main Load all submodules found at component source Find all class definitions local to the modules && Create instances. Add modReload function Add file logger handler && Add pkl file option for ExecuteDao Move arg parsing and pkl file loading in global space Fix exit call Add short arguments. Pylint fixes Add loadEnvFile && Add help and usage strings to wmagent-component-standalone Moving loadEnvFile to Utils && Properly setting the env at both ExecuteDAO and wmagent-component-standalone Explicitely avoid checking subprocess status for loadEnvFile && Return default logger option to ExecuteDAO constructor. --- bin/wmagent-component-standalone | 241 +++++++++++++++++++++++ src/python/Utils/FileTools.py | 28 +++ src/python/WMCore/Database/ExecuteDAO.py | 111 +++++++---- 3 files changed, 342 insertions(+), 38 deletions(-) create mode 100644 bin/wmagent-component-standalone mode change 100644 => 100755 src/python/WMCore/Database/ExecuteDAO.py diff --git a/bin/wmagent-component-standalone b/bin/wmagent-component-standalone new file mode 100644 index 0000000000..8d77e9599a --- /dev/null +++ b/bin/wmagent-component-standalone @@ -0,0 +1,241 @@ +#!/usr/bin/env python +""" +wmagent-component-standalone + +Utility script for running a WMAgent component standalone. + +Example usage: + +interactive mode: +ipython -i $WMA_DEPLOY_DIR/bin/wmagent-component-standalone -- -c $WMA_CONFIG_FILE -p JobAccountant -e $WMA_ENV_FILE + +daemon mode: +wmagent-component-standalone -c $WMA_CONFIG_FILE -p JobAccountant -e $WMA_ENV_FILE +""" + +import os +import sys +import logging +import importlib +import runpy +import pkgutil +import inspect +import subprocess +from pprint import pprint,pformat +from distutils.sysconfig import get_python_lib +from argparse import ArgumentParser, RawDescriptionHelpFormatter +from WMCore.Configuration import loadConfigurationFile +from WMCore.ResourceControl.ResourceControl import ResourceControl +from WMCore.Services.CRIC.CRIC import CRIC +from WMCore.WMInit import connectToDB +from Utils.FileTools import loadEnvFile + +def createOptionParser(): + """ + _createOptionParser_ + + Creates an option parser to setup the component run parameters + """ + exampleStr = """ + Examples: + + * interactive mode: + ipython -i $WMA_DEPLOY_DIR/bin/wmagent-component-standalone -- -c $WMA_CONFIG_FILE -p JobAccountant -e $WMA_ENV_FILE + + * daemon mode: + wmagent-component-standalone -c $WMA_CONFIG_FILE -p JobAccountant -e $WMA_ENV_FILE + """ + helpStr = """ + Utility script for running a WMAgent component standalone. It supports two modes: + * interactive - it loads all possible sub modules defined within the component's source area + and tries to create a proper set of object instances for each of them + in the global scope of the script, such that they could be used interactively + * daemon - it mimics the normal behavior of a regular component run in the background, + while at the same time giving the ability to overwrite some of the component's configuration parameters + e.g. loglevel or provide an alternative WMA_CONFIG_FILE and/or WMA_ENV_FILE. + """ + optParser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter, + description=helpStr, + epilog=exampleStr) + optParser.add_argument("-c", "--config", dest="wmaConfigPath", help="WMAgent configuration file.", + default=os.environ.get("WMA_CONFIG_FILE", None)) + optParser.add_argument("-e", "--envfile", dest="wmaEnvFilePath", help="WMAgent environment file.", + default=os.environ.get("WMA_ENV_FILE", None)) + optParser.add_argument("-d", "--daemon", dest="daemon", + default=False, action="store_true", + help="The component would be run as a daemon, similar to how it is run through the standard agent manage scripts.") + optParser.add_argument("-p", "--component", dest="wmaComponentName", + default=None, + help="The component/package to be loaded.") + optParser.add_argument("-l", "--loglevel", dest="logLevel", + default='INFO', + help="The loglevel at which the component should be run,") + + return optParser + + +def main(): + """ + _main_ + The main function to be used for starting the chosen component as a daemon + """ + # NOTE: There are two methods for running the wmcore daemon + # * The first method forks and assosiates a new shell to the running process + # os.system(f"wmcoreD --start --component {wmaComponentName}") + # * The second method executes the daemon in the very same interpretter: + # runpy("wmcoreD") + + wmaDaemonPath = os.environ.get("WMA_DEPLOY_DIR", None) + "/bin/wmcoreD" + + # Run the daemon: + logger.info("Starting the component daemon") + sys.argv = ["--", "--restart", "--component", wmaComponentName] + runpy.run_path(wmaDaemonPath) + + return + +if __name__ == '__main__': + optParser = createOptionParser() + (options, args) = optParser.parse_known_args() + FORMAT = "%(asctime)s:%(levelname)s:%(module)s:%(funcName)s(): %(message)s" + LOGLEVEL = logging.INFO + # add logfile handler as well: + logFilePath = os.path.join(os.getenv('WMA_INSTALL_DIR', '/tmp'), f'{options.wmaComponentName}/ComponentLogStandalone') + logFileHandler = logging.FileHandler(logFilePath) + logStdoutHandler = logging.StreamHandler(sys.stdout) + + logging.basicConfig(handlers=[logStdoutHandler, logFileHandler], format=FORMAT, level=LOGLEVEL) + logger = logging.getLogger(__name__) + # logger.setLevel(LOGLEVEL) + + # sourcing $WMA_ENV_FILE explicitely + if not options.wmaEnvFilePath or not os.path.exists(options.wmaEnvFilePath): + msg = "Missing WMAgent environment file! One may expect component misbehaviour!" + logger.warning(msg) + else: + msg = "Trying to source explicitely the WMAgent environment file: %s" + logger.info(msg, options.wmaEnvFilePath) + try: + loadEnvFile(options.wmaEnvFilePath) + except Exception as ex: + logger.error("Failed to load wmaEnvFile: %s", options.wmaEnvFilePath) + raise + + # checking the existence of wmaConfig file + if not options.wmaConfigPath or not os.path.exists(options.wmaConfigPath): + msg = "Missing WMAgent config file! One may expect component failure" + logger.warning(msg) + else: + # resetting the configuration in the env (if the default is overwritten through args) + os.environ['WMAGENT_CONFIG'] = options.wmaConfigPath + os.environ['WMA_CONFIG_FILE'] = options.wmaConfigPath + + wmaConfig = loadConfigurationFile(options.wmaConfigPath) + logger.info(f"wmaEnvFilePath: {options.wmaEnvFilePath}") + logger.info(f"wmaConfigPath: {options.wmaConfigPath}") + logger.info(f"wmaComponent: {options.wmaComponentName}") + logger.info(f"logLevel: {options.logLevel}") + logger.info(f"daemon: {options.daemon}") + + connectToDB() + + logger.info(f"Creating default component objects.") + resourceControl = ResourceControl(config=wmaConfig) + wmaComponentName = options.wmaComponentName + wmaComponentModule = f"WMComponent.{wmaComponentName}" + # wmaComponent = importlib.import_module(wmaComponentModule + "." + wmaComponentName) + + logger.info("Importing all possible modules found for this component") + + # First find all possible locations for the component source + # NOTE: We always consider PYTHONPATH first + pythonLibPaths = os.getenv('PYTHONPATH', '').split(':') + pythonLibPaths.append(get_python_lib()) + # Normalize paths and remove empty ones + pythonLibPaths = [x for x in pythonLibPaths if x] + for index, libPath in enumerate(pythonLibPaths): + pythonLibPaths[index] = os.path.normpath(libPath) + + wmaComponentPaths = [] + for comPath in pythonLibPaths: + comPath = f"{comPath}/WMComponent/{wmaComponentName}" + comPath = os.path.normpath(comPath) + if os.path.exists(comPath): + wmaComponentPaths.append(comPath) + + modules = {} + classDefs = {} + # Then try to load all possible modules and submodules under the component's source path + # for pkgSource, pkgName, _ in pkgutil.iter_modules(wmaComponentPaths): + for pkgSource, pkgName, _ in pkgutil.walk_packages(wmaComponentPaths): + fullPkgName = f"{wmaComponentModule}.{pkgName}" + if pkgName == "DefaultConfig": + continue + logger.info("Loading package: %s", fullPkgName) + modSpec = pkgSource.find_spec(fullPkgName) + module = importlib.util.module_from_spec(modSpec) + modSpec.loader.exec_module(module) + modules[pkgName] = module + + # Emulating `from module import *`" + logger.info(f"Populating the namespace with all definitions from {pkgName}") + if "__all__" in module.__dict__: + names = module.__dict__["__all__"] + else: + names = [x for x in module.__dict__ if not x.startswith("_")] + globals().update({k: getattr(module, k) for k in names}) + + # Creating instances only for class definitions local to pkgName + + # Note: The method bellow for separating local definitions from imported ones + # won't work for decorated classes, since the module name of the + # decorated class defintion is considered to be the fully qulified + # module name/path of the decorator rather than the package where the + # the class definition exists: + # e.g.: + # In [10]: modules['DataCollectAPI'].__name__ + # Out[10]: 'WMComponent.AnalyticsDataCollector.DataCollectAPI' + # + # In [11]: classDefs['DataCollectAPI']['LocalCouchDBData'] + # Out[11]: WMComponent.AnalyticsDataCollector.DataCollectorEmulatorSwitch.emulatorHook..EmulatorWrapper + # + # In [12]: classDefs['DataCollectAPI']['LocalCouchDBData'].__name__ + # Out[12]: 'EmulatorWrapper' + # + # In [13]: classDefs['DataCollectAPI']['LocalCouchDBData'].__module__ + # Out[13]: 'WMComponent.AnalyticsDataCollector.DataCollectorEmulatorSwitch' + # + # In [14]: modules['DataCollectAPI'].__name__ + # Out[14]: 'WMComponent.AnalyticsDataCollector.DataCollectAPI' + + logger.info("Trying to create an instance of all class definitions inside: %s", pkgName) + classDefs[pkgName] = {} + for objName, obj in inspect.getmembers(module): + if inspect.isclass(obj) and obj.__module__ == modules[pkgName].__name__: + classDefs[pkgName][objName]=obj + + for className in classDefs[pkgName]: + logger.info(f"{fullPkgName}:{className}") + try: + exec(f"{className.lower()} = {className}()") + except TypeError: + try: + exec(f"{className.lower()} = {className}(wmaConfig)") + except TypeError: + try: + exec(f"{className.lower()} = {className}(wmaConfig, logger=logger)") + except TypeError: + logger.warning(f"We did our best to create an instance of: {pkgName}. Giving up now!") + + def modReload(): + for module in modules: + modName = modules[module].__name__ + modInst = sys.modules.get(modName) + if modInst: + logger.info(f"Reloading module:{modName}") + importlib.reload(modInst) + else: + logger.warning(f"Cannot find module: {modName} in sys.modules") + + if options.daemon: + main() diff --git a/src/python/Utils/FileTools.py b/src/python/Utils/FileTools.py index 93ce7509b9..12a67b64b8 100644 --- a/src/python/Utils/FileTools.py +++ b/src/python/Utils/FileTools.py @@ -11,6 +11,7 @@ import subprocess import time import zlib +import logging from Utils.Utilities import decodeBytesToUnicode @@ -152,3 +153,30 @@ def getFullPath(name, envPath="PATH"): if os.path.exists(fullPath): return fullPath return None + + +def loadEnvFile(wmaEnvFilePath, logger=None): + """ + _loadEnvFile_ + A simple function to load an additional bash env file into the current script + runtime environment + :param wmaEnvFilePath: The path to the environment file to be loaded + :return: True if the script has loaded successfully, False otherwise. + """ + if not logger: + logger = logging.getLogger() + subProc = subprocess.run(['bash', '-c', f'source {wmaEnvFilePath} && python -c "import os; print(repr(os.environ.copy()))" '], + capture_output=True, check=False) + if subProc.returncode == 0: + newEnv = eval(subProc.stdout) + os.environ.update(newEnv) + if subProc.stderr: + logger.warning("Environment file: %s loaded with errors:", wmaEnvFilePath) + logger.warning(subProc.stderr.decode()) + else: + logger.info("Environment file: %s loaded successfully", wmaEnvFilePath) + return True + else: + logger.error("Failed to load environment file: %s", wmaEnvFilePath) + logger.error(subProc.stderr.decode()) + return False diff --git a/src/python/WMCore/Database/ExecuteDAO.py b/src/python/WMCore/Database/ExecuteDAO.py old mode 100644 new mode 100755 index ea21405f6a..07341f4812 --- a/src/python/WMCore/Database/ExecuteDAO.py +++ b/src/python/WMCore/Database/ExecuteDAO.py @@ -24,16 +24,17 @@ import sys import os import re -import ast + import threading import logging import argparse +import pickle from pprint import pformat from WMCore.DAOFactory import DAOFactory from WMCore.WMInit import WMInit from WMCore.Agent.Configuration import Configuration, loadConfigurationFile - +from Utils.FileTools import loadEnvFile def parseArgs(): """ @@ -44,11 +45,18 @@ def parseArgs(): formatter_class=argparse.RawTextHelpFormatter, description=__doc__) - parser.add_argument('-c', '--config', required=True, + parser.add_argument('-c', '--config', required=False, + default=os.environ.get("WMA_CONFIG_FILE", None), help="""\ - The path to WMAGENT_CONFIG to be used, e.g. - for production: /data/srv/wmagent/current/config/wmagent/config.py - for tier0: /data/tier0/srv/wmagent/current/config/tier0/config.py""") + The WMAgent config file to be used for the this execution. Default is taken from + the current's shell environment variable $WMA_CONFIG_FILE + """) + parser.add_argument('-e', '--envFile', required=False, + default=os.environ.get("WMA_ENV_FILE", None), + help=""" + The WMAgent environment file to be used for the this execution. Default is taken from + the current's shell environment variable $WMA_ENV_FILE + """) parser.add_argument('-p', '--package', required=True, help="""\ The package from which the DAO factory to be created for this execution, e.g. WMCore.WMBS or WMComponent.DBS3Buffer""") @@ -70,25 +78,32 @@ def parseArgs(): parser.add_argument('sqlArgs', nargs=argparse.REMAINDER, default=(), help="""\ -- Positional parameters to be forwarded to the DAO execute method and used as SQL arguments in the query.""") + parser.add_argument('-f', '--pklFile', default=None, + help="""\ + An extra *.pkl file containing any additional python objects needed for the given dao + e.g. WMCore.WMBS.Files.AddRunLumi. + The object is always loaded under the name `pklFile`. One can access the contents of the so loaded pkl file + during the dao execution trough the -s arguent e.g.: + ExecuteDAO.py -p WMCore.WMBS -m Files.AddRunLumi -c $WMA_CONFIG_FILE -f runLumiBinds_2035-4016.pkl -s "{'file': pklFile['data']} + """) + currArgs = parser.parse_args() - args = parser.parse_args() - - return args + return currArgs def loggerSetup(logLevel=logging.INFO): """ Return a logger which writes everything to stdout. """ - logger = logging.getLogger() + currLogger = logging.getLogger() outHandler = logging.StreamHandler(sys.stdout) outHandler.setFormatter(logging.Formatter("%(asctime)s:%(levelname)s:%(module)s: %(message)s")) outHandler.setLevel(logLevel) - if logger.handlers: - logger.handlers.clear() - logger.addHandler(outHandler) - logger.setLevel(logLevel) - return logger + if currLogger.handlers: + currLogger.handlers.clear() + currLogger.addHandler(outHandler) + currLogger.setLevel(logLevel) + return currLogger def getBackendFromDbURL(dburl): @@ -110,15 +125,12 @@ class ExecuteDAO(): """ A generic class to create the DAO Factory and execute the DAO module. """ - def __init__(self, logger=None, configFile=None, - connectUrl=None, socket=None, - package=None, daoModule=None): + def __init__(self, connectUrl=None, socket=None, configFile=None, + package=None, daoModule=None, logger=None): """ __init__ The ExecuteDAO constructor method. - :param logger: The logger instance. :param package: The Package from which the DAO factory to be initialised. - :param configFile: Path to WMAgent configuration file. :param connectUrl: Database connection URL (overwrites the connectUrl param from configFile if both present) :param socket: Database connection URL (overwrites the socket param from configFile if both present) :param module: The DAO module to be executed. @@ -215,7 +227,13 @@ def __call__(self, *sqlArgs, dryRun=False, daoHelp=False, **sqlKwArgs): self.logger.info("DAO SQL arguments provided:\n%s, %s", pformat(sqlArgs), pformat(sqlKwArgs)) else: results = self.dao.execute(*sqlArgs, **sqlKwArgs) - self.logger.info("DAO Results:\n%s", pformat(results if isinstance(results, dict) else list(results))) + # self.logger.info("DAO Results:\n%s", pformat(results if isinstance(results, dict) else list(results))) + if isinstance(results, dict): + self.logger.info("DAO Results:\n%s", pformat(results)) + elif isinstance(results, bool): + self.logger.info("DAO Results:\n%s", results) + else: + self.logger.info("DAO Results:\n%s", list(results)) return results def getSqlQuery(self): @@ -244,21 +262,19 @@ def strToDict(dString, logger=None): :param dString: The dictionary string to be parsed. Possible formats are either a string of multiple space separated named values of the form 'name=value': or a srting fully defining the dictionary itself. - :param logger: A logger object to be used for Error message printout :return: The constructed dictionary """ - logger = logger or logging.getLogger() - result = ast.literal_eval(dString) + if not logger: + logger = logging.getLogger() + # result = ast.literal_eval(dString) + result = eval(dString) if not isinstance(result, dict): logger.error("The Query named arguments need to be provided as a dictionary. WRONG option: %s", pformat(dString)) raise TypeError(pformat(dString)) return result -def main(): - """ - An Utility to construct a DAO Factory and execute the DAO requested. - """ +if __name__ == '__main__': args = parseArgs() if args.debug: @@ -266,6 +282,17 @@ def main(): else: logger = loggerSetup() + # Create an instance of the *.pkl file provided with the dao call, if any. + if args.pklFile: + pklFilePath = os.path.normpath(args.pklFile) + if not os.path.exists(pklFilePath): + logger.error("Cannot find the pkl file: %s. Exit!", pklFilePath) + sys.exit(1) + with open(pklFilePath, 'rb') as fd: + pklFile = pickle.load(fd) + logger.info('PklFile: %s loaded as: `pklFile`. You can refer to its content through the -s argument.', pklFilePath) + # logger.info(pformat(pklFile)) + # Remove leading double slash if present: if args.sqlArgs and args.sqlArgs[0] == '--': args.sqlArgs = args.sqlArgs[1:] @@ -276,17 +303,25 @@ def main(): # Parse named arguments to a proper dictionary: if not isinstance(args.sqlKwArgs, dict): - args.sqlKwArgs = strToDict(args.sqlKwArgs, logger=logger) + args.sqlKwArgs = strToDict(args.sqlKwArgs) - # Try to set the wmagent configuration in the environment - if 'WMAGENT_CONFIG' not in os.environ: + # Trying to load WMA_ENV_FILE + if not args.envFile or not os.path.exists(args.envFile): + logger.warning("Missing WMAgent environment file! One may expect DAO misbehavior!") + else: + logger.info("Trying to source explicitely the WMAgent environment file: %s", args.envFile) + try: + loadEnvFile(args.envFile) + except Exception as ex: + logger.error("Failed to load wmaEnvFile: %s", args.envFile) + raise + + if not args.config or not os.path.exists(args.config): + logger.warning("Missing WMAgent config file! One may expect DAO failure") + else: + # resetting the configuration file in the env (if the default is overwritten through args) os.environ['WMAGENT_CONFIG'] = args.config - configFile = os.environ['WMAGENT_CONFIG'] + os.environ['WMA_CONFIG_FILE'] = args.config - daoObject = ExecuteDAO(logger=logger, configFile=configFile, - package=args.package, daoModule=args.module) + daoObject = ExecuteDAO(package=args.package, daoModule=args.module, configFile=args.config) daoObject(*args.sqlArgs, dryRun=args.dryRun, daoHelp=True, **args.sqlKwArgs) - - -if __name__ == '__main__': - main()