From 38d6dcf134b9f765a01be99852d4572afc4977c7 Mon Sep 17 00:00:00 2001 From: Jhonatan Amado Date: Sun, 27 Mar 2022 21:17:52 -0500 Subject: [PATCH 1/4] Assign the correct activity for Tape_Test RSEs Add the proper activity for the T0-Agent when is subscribing data to Tape_Test rses. --- src/python/WMComponent/RucioInjector/RucioInjectorPoller.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/python/WMComponent/RucioInjector/RucioInjectorPoller.py b/src/python/WMComponent/RucioInjector/RucioInjectorPoller.py index 08ac0901bd..e9a5f0f60a 100644 --- a/src/python/WMComponent/RucioInjector/RucioInjectorPoller.py +++ b/src/python/WMComponent/RucioInjector/RucioInjectorPoller.py @@ -465,7 +465,7 @@ def insertContainerRules(self): grouping="ALL", comment=ruleComment, meta=self.metaData) - if not rseName.endswith("_Tape"): + if not rseName.endswith(("_Tape", "_Tape_Test")): # add extra parameters to the Disk rule as defined in the component configuration ruleKwargs.update(self.containerDiskRuleParams) @@ -529,7 +529,7 @@ def _activityMap(self, rseName): """ if not self.isT0agent and not rseName.endswith("_Tape"): return "Production Output" - elif self.isT0agent and rseName.endswith("_Tape"): + elif self.isT0agent and rseName.endswith(("_Tape", "_Tape_Test")): return "T0 Tape" elif self.isT0agent: return "T0 Export" From 97ea894aad29be62afaf7b7c1c75350941fd113d Mon Sep 17 00:00:00 2001 From: Alan Malta Rodrigues Date: Fri, 8 Apr 2022 21:25:34 -0400 Subject: [PATCH 2/4] If job requires rhel8 container, use slc7 python libraries Trying the new map of OS_ARCH replace slc7 by rhel7 apply Valentins suggestion --- etc/submit_py3.sh | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/etc/submit_py3.sh b/etc/submit_py3.sh index f5968a533c..527afbae7d 100644 --- a/etc/submit_py3.sh +++ b/etc/submit_py3.sh @@ -54,6 +54,7 @@ PY3_FUTURE_VERSION=0.18.2 # Saving START_TIME and when job finishes END_TIME. WMA_MIN_JOB_RUNTIMESECS=300 START_TIME=$(date +%s) +WMA_DEFAULT_OS=rhel7 # assign arguments SANDBOX=$1 INDEX=$2 @@ -132,20 +133,17 @@ echo "======== WMAgent COMP Python bootstrap starting at $(TZ=GMT date) ======== # slc6_ppc64le_gcc493 (from CMSSW): py2-future/0.18.2 python/2.7.15 py3-future/0.18.2 python3/3.8.2 # # NOTE: all the ppc64le ScramArchs are actually pointing to: slc7_ppc64le_gcc820 +### UPDATE on 11 April, 2022: See a new map of CVMFS packages in: +# https://github.com/dmwm/WMCore/pull/11077#issuecomment-1094814966 + # First, decide which COMP ScramArch to use based on the required OS and Architecture THIS_ARCH=`uname -m` # if it's PowerPC, it returns `ppc64le` -if [ "$THIS_ARCH" = "x86_64" ] -then - THIS_ARCH="amd64" -fi -if [ "$REQUIRED_OS" = "rhel7" ] -then - WMA_SCRAM_ARCH=slc7_${THIS_ARCH}_gcc630 -elif [ "$THIS_ARCH" = "amd64" ] +# if this job can run at any OS, then use rhel7 as default +if [ "$REQUIRED_OS" = "any" ] then - WMA_SCRAM_ARCH=slc6_${THIS_ARCH}_gcc700 + WMA_SCRAM_ARCH=${WMA_DEFAULT_OS}_${THIS_ARCH} else - WMA_SCRAM_ARCH=slc6_${THIS_ARCH}_gcc493 + WMA_SCRAM_ARCH=${REQUIRED_OS}_${THIS_ARCH} fi echo "Job requires OS: $REQUIRED_OS, thus setting ScramArch to: $WMA_SCRAM_ARCH" From 6cb9fc7ec5b5c34cc0fb4d2b0364a2d659d497d3 Mon Sep 17 00:00:00 2001 From: Alan Malta Rodrigues Date: Mon, 11 Apr 2022 15:59:53 -0400 Subject: [PATCH 3/4] When ScramArch is empty str/list/None, return any as required_os --- src/python/WMCore/BossAir/Plugins/BasePlugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/WMCore/BossAir/Plugins/BasePlugin.py b/src/python/WMCore/BossAir/Plugins/BasePlugin.py index 4eedec0d83..f6981b2d27 100644 --- a/src/python/WMCore/BossAir/Plugins/BasePlugin.py +++ b/src/python/WMCore/BossAir/Plugins/BasePlugin.py @@ -138,7 +138,7 @@ def scramArchtoRequiredOS(scramArch=None): string to be matched for OS requirements for job """ requiredOSes = set() - if scramArch is None: + if not scramArch: requiredOSes.add('any') elif isinstance(scramArch, (str, bytes)): for arch, validOSes in viewitems(ARCH_TO_OS): From 4d43873e979109483ccb8ba78630281e7820adcf Mon Sep 17 00:00:00 2001 From: Alan Malta Rodrigues Date: Tue, 12 Apr 2022 08:38:17 -0400 Subject: [PATCH 4/4] Enhance logic to map ScramArch to OS typo in the docstring --- .../WMCore/BossAir/Plugins/BasePlugin.py | 36 +++++++++---------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/src/python/WMCore/BossAir/Plugins/BasePlugin.py b/src/python/WMCore/BossAir/Plugins/BasePlugin.py index f6981b2d27..0dfbab16a2 100644 --- a/src/python/WMCore/BossAir/Plugins/BasePlugin.py +++ b/src/python/WMCore/BossAir/Plugins/BasePlugin.py @@ -6,7 +6,7 @@ """ from builtins import object, str, bytes -from future.utils import viewitems, viewvalues +from future.utils import viewvalues from Utils.Utilities import decodeBytesToUnicode from WMCore.WMException import WMException @@ -130,27 +130,25 @@ def updateSiteInformation(self, jobs, siteName, excludeSite): @staticmethod def scramArchtoRequiredOS(scramArch=None): """ + Matches a ScramArch - or a list of it - against a map of Scram + to Operating System - Args: - scramArch: string or list of scramArches that are acceptable for the job - - Returns: - string to be matched for OS requirements for job + :param scramArch: string or list of scramArches defined for a given job + :return: a string with the required OS to use """ - requiredOSes = set() + defaultValue = 'any' if not scramArch: - requiredOSes.add('any') - elif isinstance(scramArch, (str, bytes)): - for arch, validOSes in viewitems(ARCH_TO_OS): - if arch in scramArch: - requiredOSes.update(validOSes) - elif isinstance(scramArch, list): - for validArch in scramArch: - for arch, validOSes in viewitems(ARCH_TO_OS): - if arch in validArch: - requiredOSes.update(validOSes) - else: - requiredOSes.add('any') + return defaultValue + + requiredOSes = set() + if isinstance(scramArch, (str, bytes)): + scramArch = [scramArch] + elif not isinstance(scramArch, (list, tuple)): + return defaultValue + + for validArch in scramArch: + scramOS = validArch.split("_")[0] + requiredOSes.update(ARCH_TO_OS.get(scramOS, [])) return ','.join(sorted(requiredOSes))