From 283fa16437a328e7e506368e16fa6f29b730b563 Mon Sep 17 00:00:00 2001 From: Alan Malta Rodrigues Date: Tue, 6 Aug 2024 13:19:13 -0400 Subject: [PATCH] WorkQueue: convert PNN to PSN for pileup location --- src/python/WMCore/WorkQueue/DataLocationMapper.py | 6 ++++-- .../WMCore/WorkQueue/Policy/Start/StartPolicyInterface.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/python/WMCore/WorkQueue/DataLocationMapper.py b/src/python/WMCore/WorkQueue/DataLocationMapper.py index 1d62b73666..3025e66523 100644 --- a/src/python/WMCore/WorkQueue/DataLocationMapper.py +++ b/src/python/WMCore/WorkQueue/DataLocationMapper.py @@ -136,7 +136,7 @@ def locationsFromMSPileup(self, dataItems, dbsUrl): :param dbsUrl: str, dbs url to check which dbs server :return: dict, dict of pileup name keys with location set values """ - self.logger.info(f'Fetching locations from MSPileup for {len(dataItems)}') + self.logger.info(f'Fetching locations from MSPileup for {len(dataItems)} containers') result = defaultdict(set) # TODO: Fetch multiple pileups in single request @@ -148,7 +148,8 @@ def locationsFromMSPileup(self, dataItems, dbsUrl): msPileupUrl = f"https://cmsweb{pileupInstance}.cern.ch/ms-pileup/data/pileup" doc = getPileupDocs(msPileupUrl, queryDict, method='POST')[0] self.logger.info(f'locationsFromPileup - name: {dataItem}, currentRSEs: {doc["currentRSEs"]}, containerFraction: {doc["containerFraction"]}') - result[dataItem] = doc['currentRSEs'] + # resolve PNNs into PSNs + result[dataItem] = self.cric.PNNstoPSNs(doc['currentRSEs']) except IndexError: self.logger.error('Did not find any pileup document for query: %s', queryDict['query']) except Exception as ex: @@ -249,6 +250,7 @@ def updatePileupLocation(self): for dataMapping in listvalues(dataLocations): for data, locations in viewitems(dataMapping): elements = self.backend.getElementsForPileupData(data) + self.logger.info("Found %d elements using pileup: %s", len(elements), data) for element in elements: if element.get('NoPileupUpdate', False): continue diff --git a/src/python/WMCore/WorkQueue/Policy/Start/StartPolicyInterface.py b/src/python/WMCore/WorkQueue/Policy/Start/StartPolicyInterface.py index e7b844e0a1..2b5a5aa94a 100644 --- a/src/python/WMCore/WorkQueue/Policy/Start/StartPolicyInterface.py +++ b/src/python/WMCore/WorkQueue/Policy/Start/StartPolicyInterface.py @@ -290,7 +290,7 @@ def getDatasetLocationsFromMSPileup(self, datasetsWithDbsURL): 'filters': ['expectedRSEs', 'currentRSEs', 'pileupName', 'containerFraction', 'ruleIds']} try: doc = getPileupDocs(msPileupUrl, queryDict, method='POST')[0] - currentRSEs = doc['currentRSEs'] + currentRSEs = self.cric.PNNstoPSNs(doc['currentRSEs']) self.logger.debug(f'Retrieved MSPileup document: {doc}') if len(currentRSEs) == 0: self.logger.warning(f'No RSE has a copy of the desired pileup dataset. Expected RSEs: {doc["expectedRSEs"]}')