Merge pull request #6478 from HassenRiahi/FixRegexSupport

1.0.12_crab version of Fix regex support

src/python/WMCore/Storage/TrivialFileCatalog.py

@@ -85,12 +85,18 @@ def _doMatch(self, protocol, path, style, caller):
                     path = caller(mapping["chain"], path)
                     if not path:
                         continue
-                try:
-                    splitPath = mapping['path-match-expr'].split(path, 1)[1]
-                except IndexError:
+                splitList = []
+                if len(mapping['path-match-expr'].split(path, 1)) > 1 :
+                    for split in range(len(mapping['path-match-expr'].split(path, 1))):
+                        s = mapping['path-match-expr'].split(path, 1)[split]
+                        if s:
+                            splitList.append(s)
+                else:
                     path = oldpath
                     continue
-                result = mapping['result'].replace("$1", splitPath)
+                result = mapping['result']
+                for split in range(len(splitList)):
+                    result = result.replace("$" + str(split + 1), splitList[split])
                 return result
 
         return None

test/python/WMCore_t/Storage_t/T2_CH_CERNBOX_TrivialFileCatalog.xml

@@ -0,0 +1,5 @@
+<storage-mapping>
+
+        <lfn-to-pfn protocol="srmv2" path-match="/+store/user/(.)(.*)" result="root://eosuser.cern.ch/eos/user/$1/$1$2"/>
+
+</storage-mapping>

test/python/WMCore_t/Storage_t/T2_ES_IFCA_TrivialFileCatalog.xml

@@ -0,0 +1,17 @@
+<storage-mapping>
+
+   <lfn-to-pfn protocol="direct" path-match=".*/LoadTest07_IFCA_(.*)_.*_.*" result="/cms/store/PhEDEx_LoadTest07/LoadTest07_Debug_IFCA/LoadTest07_IFCA_$1" />
+   <lfn-to-pfn protocol="direct" path-match=".*/LoadTest07_IFCATMP_(.*)_.*_.*" result="/cms/store/PhEDEx_LoadTest07/LoadTest07_Debug_IFCA/LoadTest07_IFCA_$1" />
+   <lfn-to-pfn protocol="direct" path-match="/+store/test/xrootd/T2_ES_IFCA/store/(.*)" result="/cms/store/$1" />
+   <lfn-to-pfn protocol="direct" path-match="/+store/user/(.*)" result="/cms/store/user/$1" />
+   <lfn-to-pfn protocol="direct" path-match="/+store/(.*)" result="/cms/store/$1" />
+   <lfn-to-pfn protocol="xrootd" path-match="/+store/(.*)" destination-match=".*" result="root://xrootd-cms.infn.it//store/$1" />
+   <lfn-to-pfn protocol="file" path-match="/+(.*)" chain="direct" result="/gpfs/gaes/$1" />
+   <lfn-to-pfn protocol="srmv2" path-match=".*/+SAM-srm01.ifca.es/(.*)" chain="direct" result="srm://srm01.ifca.es:8444/srm/managerv2?SFN=/cms/store/unmerged/SAM/testSRM/SAM-srm01.ifca.es/$1" />
+   <lfn-to-pfn protocol="srmv2" path-match=".*/+store/user/(.*)" chain="direct" result="srm://srm01.ifca.es:8444/srm/managerv2?SFN=/cms/store/user/$1" />
+   <lfn-to-pfn protocol="srmv2" path-match="(.*)" chain="direct" result="srm://storm.ifca.es:8444/srm/managerv2?SFN=$1" />
+   <pfn-to-lfn protocol="direct" path-match="/+cms/store/(.*)" result="/store/$1" />
+   <pfn-to-lfn protocol="file" path-match="/gpfs/gaes/(.*)" chain="direct" result="/$1" />
+   <pfn-to-lfn protocol="srmv2" path-match=".*\?SFN=(.*)" chain="direct" result="$1" />
+
+</storage-mapping>

test/python/WMCore_t/Storage_t/T2_PT_NCG_Lisbon_TrivialFileCatalog.xml

@@ -0,0 +1,20 @@
+<storage-mapping>
+
+   <lfn-to-pfn protocol="direct" path-match="/+store/test/xrootd/T2_PT_NCG_Lisbon/store/(mc/SAM/GenericTTbar/GEN-SIM-RECO/CMSSW_5_3_1_START53_V5-v1/.*)" destination-match=".*" result="/gstore/t2cms/store/$1" />
+   <lfn-to-pfn protocol="direct" path-match=".*LoadTest07_NCG_Lisbon_(.*)_.*_.*" result="/store/PhEDEx_LoadTest07/LoadTest07_Debug_NCG_Lisbon/LoadTest07_NCG_Lisbon_$1" />
+   <lfn-to-pfn protocol="direct" path-match="/+store(.*)" result="/store$1" />
+   <lfn-to-pfn protocol="xrootd" path-match="/+store/(.*)" destination-match=".*" result="root://xrootd-cms.infn.it//store/$1" />
+   <lfn-to-pfn protocol="file" path-match="/+store/user(.*)" chain="direct" result="/gstore/t3cms/store/user$1" />
+   <lfn-to-pfn protocol="file" path-match="/+store(.*)" chain="direct" result="/gstore/t2cms/cms/store$1" />
+   <lfn-to-pfn protocol="srmv2" path-match=".*/LoadTest07_NCG_Lisbon_(.*)_.*_.*" result="srm://srm01.ncg.ingrid.pt:8444/srm/managerv2?SFN=/cms/store/PhEDEx_LoadTest07/LoadTest07_Debug_NCG_Lisbon/LoadTest07_NCG_Lisbon_$1" />
+   <lfn-to-pfn protocol="srmv2" path-match="/+store/user(.*)" chain="direct" result="srm://srm01.ncg.ingrid.pt:8444/srm/managerv2?SFN=/cmst3/store/user$1" />
+   <lfn-to-pfn protocol="srmv2" path-match="/+store(.*)" chain="direct" result="srm://srm01.ncg.ingrid.pt:8444/srm/managerv2?SFN=/cms/store$1" />
+   <pfn-to-lfn protocol="direct" path-match="/+store(.*)" result="/store$1" />
+   <pfn-to-lfn protocol="file" path-match="/gstore/t3cms(.*)" chain="direct" result="$1" />
+   <pfn-to-lfn protocol="file" path-match="/gstore/t2cms(.*)" chain="direct" result="$1" />
+   <pfn-to-lfn protocol="srmv2" path-match=".*\?SFN=/cmst3/store/user(.*)" chain="direct" result="/store/user$1" />
+   <pfn-to-lfn protocol="srmv2" path-match=".*\?SFN=/cms/store(.*)" chain="direct" result="/store$1" />
+
+</storage-mapping>
+
+

test/python/WMCore_t/Storage_t/T2_US_Florida_TrivialFileCatalog.xml

@@ -0,0 +1,54 @@
+<storage-mapping>
+
+   <lfn-to-pfn protocol="direct" path-match=".*/LoadTest07_Florida_(.*)_.*_.*" destination-match=".*" result="/cms/data/loadtest/store/PhEDEx_LoadTest07/LoadTest07_Debug_Florida/LoadTest07_Florida_$1" />
+   <lfn-to-pfn protocol="direct" path-match=".*/MonarcTest_Florida_(.*)_.*_.*" destination-match=".*" result="/cmsdata/store/phedex_monarctest/Florida/MonarcTest_Florida_$1" />
+   <lfn-to-pfn protocol="direct" path-match="/+LoadTest/(.*)" destination-match=".*" result="/cmsdata/SC4/phedex_loadtest/$1" />
+   <lfn-to-pfn protocol="direct" path-match="/+store/(unmerged/SAM/testSRM/SAM-srm.ihepa.ufl.edu.*)" destination-match=".*" result="/cmsdata/data/store/$1" />
+   <lfn-to-pfn protocol="direct" path-match="/+store/(unmerged/SAM/testSRM/SAM-srm1.ihepa.ufl.edu.*)" destination-match=".*" result="/cmsdata/data/store/$1" />
+   <lfn-to-pfn protocol="direct" path-match="/+store/(user/test/SAM-srm.ihepa.ufl.edu.*)" destination-match=".*" result="/cmsdata/data/store/$1" />
+   <lfn-to-pfn protocol="direct" path-match="/+store/(user/test/SAM-srm1.ihepa.ufl.edu.*)" destination-match=".*" result="/cmsdata/data/store/$1" />
+   <lfn-to-pfn protocol="direct" path-match="/+store/test/xrootd/T2_US_Florida/store/(mc/JobRobot/RelValProdTTbar/GEN-SIM-DIGI-RECO/MC_3XY_V24_JobRobot-v1/.*)" destination-match=".*" result="/cms/data/store/$1" />
+   <lfn-to-pfn protocol="direct" path-match="/+store/test/xrootd/T2_US_Florida/store/(mc/SAM/GenericTTbar/GEN-SIM-RECO/CMSSW_5_3_1_START53_V5-v1/.*)" destination-match=".*" result="/cms/data/store/$1" />
+   <lfn-to-pfn protocol="direct" path-match="/+store/(PhEDEx_LoadTest07/.*)" destination-match=".*" result="/cms/data/loadtest/store/$1" />
+   <lfn-to-pfn protocol="direct" path-match="/+store/test/xrootd/T2_US_Florida/(store/.*)" destination-match=".*" result="/cms/data/$1" />
+   <lfn-to-pfn protocol="direct" path-match="/+store/(.*)" destination-match=".*" result="/cms/data/store/$1" />
+   <lfn-to-pfn protocol="direct" path-match="/+store/(temp/user/test/SAM-srmb.*)" destination-match=".*" result="/cms/data/store/$1" />
+   <lfn-to-pfn protocol="direct" path-match="/+store/(express/.*)" destination-match=".*" result="/cms/data/store/$1" />
+   <lfn-to-pfn protocol="direct" path-match="/+store/(relval/.*)" destination-match=".*" result="/cms/data/store/$1" />
+   <lfn-to-pfn protocol="direct" path-match="/+store/(local/.*)" destination-match=".*" result="/cms/data/store/$1" />
+   <lfn-to-pfn protocol="direct" path-match="/+store/(results/.*)" destination-match=".*" result="/cms/data/store/$1" />
+   <lfn-to-pfn protocol="direct" path-match="/+store/(user/.*)" destination-match=".*" result="/cms/data/store/$1" />
+   <lfn-to-pfn protocol="direct" path-match="/+store/(group/.*)" destination-match=".*" result="/cms/data/store/$1" />
+   <lfn-to-pfn protocol="direct" path-match="/+store/(data/.*)" destination-match=".*" result="/cms/data/store/$1" />
+   <lfn-to-pfn protocol="direct" path-match="/+(.*)" destination-match=".*" result="/$1" />
+   <lfn-to-pfn protocol="xrootd" path-match="/+store/(.*)" destination-match=".*" result="root://cmsxrootd.fnal.gov//store/$1?tried=xrootd1.ihepa.ufl.edu,xrootd2.ihepa.ufl.edu,gridftp3.ihepa.ufl.edu,cmsio1.rc.ufl.edu,cmsio2.rc.ufl.edu,cmsio3.rc.ufl.edu,cmsio4.rc.ufl.edu,cmsio5.rc.ufl.edu,cmsio6.rc.ufl.edu,cmsio7.rc.ufl.edu,cmsio8.rc.ufl.edu,cmsio9.rc.ufl.edu,cmsio10.rc.ufl.edu" />
+   <lfn-to-pfn protocol="gsiftp" path-match="/+(.*)" chain="direct" destination-match=".*" result="gsiftp://gridftp1.hpc.ufl.edu/$1" />
+   <lfn-to-pfn protocol="srm" path-match="/+cms/data(/.*)" chain="direct" destination-match=".*" result="srm://srm.ihepa.ufl.edu:8443/srm/v2/server?SFN=/cms/data$1" />
+   <lfn-to-pfn protocol="srm" path-match="/+(.*)" chain="direct" destination-match=".*" result="srm://srm.ihepa.ufl.edu:8443/srm/v2/server?SFN=/$1" />
+   <lfn-to-pfn protocol="myread" path-match="/+store/(unmerged/SAM/testSRM/SAM-srm.ihepa.ufl.edu.*)" result="file:///cmsdata/data/store/$1" />
+   <lfn-to-pfn protocol="myread" path-match="/+store/(user/test/SAM-srm.ihepa.ufl.edu.*)" result="file:///cmsdata/data/store/$1" />
+   <lfn-to-pfn protocol="myread" path-match="/+store/(unmerged/SAM/testSRM/SAM-srm1.ihepa.ufl.edu.*)" result="file:///cmsdata/data/store/$1" />
+   <lfn-to-pfn protocol="myread" path-match="/+store/(user/test/SAM-srm1.ihepa.ufl.edu.*)" result="file:///cmsdata/data/store/$1" />
+   <lfn-to-pfn protocol="myread" path-match="/+(.*)" chain="direct" destination-match=".*" result="file:///$1" />
+   <lfn-to-pfn protocol="myread" path-match="/+store/(temp/user/test/SAM-srmb.*)" result="file:///cms/data/store/$1" />
+   <lfn-to-pfn protocol="myread" path-match="/+store/(express/.*)" result="file:///cms/data/store/$1" />
+   <lfn-to-pfn protocol="myread" path-match="/+store/(relval/.*)" result="file:///cms/data/store/$1" />
+   <lfn-to-pfn protocol="myread" path-match="/+store/(local/.*)" result="file:///cms/data/store/$1" />
+   <lfn-to-pfn protocol="myread" path-match="/+store/(results/.*)" result="file:///cms/data/store/$1" />
+   <lfn-to-pfn protocol="myread" path-match="/+store/(user/.*)" result="file:///cms/data/store/$1" />
+   <lfn-to-pfn protocol="myread" path-match="/+store/(group/.*)" result="file:///cms/data/store/$1" />
+   <lfn-to-pfn protocol="myread" path-match="/+store/(data/.*)" result="file:///cms/data/store/$1" />
+   <lfn-to-pfn protocol="myread" path-match="/+(.*)" chain="direct" destination-match=".*" result="file:///cms/data/$1" />
+   <lfn-to-pfn protocol="srmv2" path-match="/+cmsdata(/.*)" chain="direct" destination-match=".*" result="srm://srm.ihepa.ufl.edu:8443/srm/v2/server?SFN=/cmsdata$1" />
+   <lfn-to-pfn protocol="srmv2" path-match="/+cms/data(/.*)" chain="direct" destination-match=".*" result="srm://srm.ihepa.ufl.edu:8443/srm/v2/server?SFN=/cms/data$1" />
+   <lfn-to-pfn protocol="srmv2" path-match="/+(.*)" chain="direct" destination-match=".*" result="srm://srm.ihepa.ufl.edu:8443/srm/v2/server?SFN=/$1" />
+   <pfn-to-lfn protocol="direct" path-match="/+cmsdata/SC4/phedex_loadtest/(.*)" destination-match=".*" result="/LoadTest/$1" />
+   <pfn-to-lfn protocol="gsiftp" path-match="[a-z]+://[-a-z0-9:.]+/(.*)" chain="direct" destination-match=".*" result="$1" />
+   <pfn-to-lfn protocol="srm" path-match=".*\?SFN=/+cmsdata(/.*)" result="$1" />
+   <pfn-to-lfn protocol="srm" path-match=".*\?SFN=/+raid(/.*)" result="$1" />
+   <pfn-to-lfn protocol="srmv2" path-match=".*\?SFN=/+lustre/raidl(/.*)" result="$1" />
+   <pfn-to-lfn protocol="srmv2" path-match=".*\?SFN=/+cms/data(/.*)" result="$1" />
+   <pfn-to-lfn protocol="srmv2" path-match=".*\?SFN=/+cmsdata(/.*)" result="$1" />
+   <pfn-to-lfn protocol="srmv2" path-match=".*\?SFN=/+raid(/.*)" result="$1" />
+
+</storage-mapping>

test/python/WMCore_t/Storage_t/T2_US_Nebraska_TrivialFileCatalog.xml

@@ -0,0 +1,21 @@
+<storage-mapping>
+
+   <lfn-to-pfn protocol="fdt" path-match="(.*)" chain="direct" destination-match=".*" result="fdt://red-gridftp11.unl.edu:54321/$1" />
+   <lfn-to-pfn protocol="direct" path-match=".*/LoadTest07_Nebraska_(.*)_.*_.*" destination-match=".*" result="/mnt/hadoop/user/uscms01/pnfs/unl\.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_$1" />
+   <lfn-to-pfn protocol="direct" path-match="/+store/(.*)" destination-match=".*" result="/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/$1" />
+   <lfn-to-pfn protocol="xrootd" path-match="/+store/(.*)" destination-match=".*" result="root://xrootd.unl.edu//store/$1" />
+   <lfn-to-pfn protocol="hadoop" path-match="/+store/(.*)" destination-match=".*" result="/user/uscms01/pnfs/unl.edu/data4/cms/store/$1" />
+   <lfn-to-pfn protocol="srmv2" path-match=".*/LoadTest07_Nebraska_(.*)_.*_.*" chain="direct" destination-match=".*" result="srm://dcache07.unl.edu:8443/srm/v2/server?SFN=$1" />
+   <lfn-to-pfn protocol="srmv2" path-match="/store/PhEDEx_LoadTest07/LoadTest07_Debug_.*/Nebraska/.*/(.*)" destination-match=".*" result="srm://dcache07.unl.edu:8443/srm/v2/server?SFN=/mnt/hadoop/user/uscms01/LoadTestDownload/$1" />
+   <lfn-to-pfn protocol="srmv2" path-match="(.*/+store/unmerged/SAM/testSRM/SAM-dcache07.unl.edu.*)" chain="direct" destination-match=".*" result="srm://dcache07.unl.edu:8443/srm/v2/server?SFN=$1" />
+   <lfn-to-pfn protocol="srmv2" path-match="(.*/+store/unmerged/SAM/testSRM/SAM-red-srm1.unl.edu.*)" chain="direct" destination-match=".*" result="srm://red-srm1.unl.edu:8443/srm/v2/server?SFN=$1" />
+   <lfn-to-pfn protocol="srmv2" path-match="(.*/+store/.*user/test/SAM-srm.unl.edu.*)" chain="direct" destination-match=".*" result="srm://srm.unl.edu:8443/srm/v2/server?SFN=$1" />
+   <lfn-to-pfn protocol="srmv2" path-match="(.*/store/unmerged/testLFN)" chain="direct" destination-match=".*" result="srm://srm.unl.edu:8443/srm/v2/server?SFN=$1" />
+   <lfn-to-pfn protocol="srmv2" path-match="(.*/store/unmerged/SAM.*)" chain="direct" destination-match=".*" result="srm://srm.unl.edu:8443/srm/v2/server?SFN=$1" />
+   <lfn-to-pfn protocol="srmv2" path-match="(.*/store/mc/SAM.*)" chain="direct" destination-match=".*" result="srm://srm.unl.edu:8443/srm/v2/server?SFN=$1" />
+   <lfn-to-pfn protocol="srmv2" path-match="(.*)" chain="direct" destination-match=".*" result="srm://dcache07.unl.edu:8443/srm/v2/server?SFN=$1" />
+   <pfn-to-lfn protocol="direct" path-match="/mnt/hadoop/user/uscms01/pnfs/unl\.edu/data4/cms/(.*)" destination-match=".*" result="/$1" />
+   <pfn-to-lfn protocol="hadoop" path-match="/user/uscms01/pnfs/unl.edu/data4/cms/(.*)" destination-match=".*" result="/$1" />
+   <pfn-to-lfn protocol="srmv2" path-match=".*\?SFN=(.*)" chain="direct" destination-match=".*" result="$1" />
+
+</storage-mapping>

test/python/WMCore_t/Storage_t/TrivialFileCatalog_t.py

@@ -170,6 +170,50 @@ def testSaveTFC(self):
         self.assertEqual(pfn, out_pfn, "Error: incorrect matching")
         f.close()
 
+    def testMultipleRegexMatch(self):
+        # Check that an lfn is converted to the right pfn
+        in_lfn = '/store/user/fred/data'
+
+        tfc_file = os.path.join(getTestBase(),
+                                "WMCore_t/Storage_t",
+                                "T2_CH_CERNBOX_TrivialFileCatalog.xml")
+        tfc = readTFC(tfc_file)
+        out_pfn = "root://eosuser.cern.ch/eos/user/f/fred/data"
+        pfn = tfc.matchLFN('srmv2', in_lfn)
+        self.assertEqual(out_pfn, pfn)
+
+        tfc_file = os.path.join(getTestBase(),
+                                "WMCore_t/Storage_t",
+                                "T2_PT_NCG_Lisbon_TrivialFileCatalog.xml")
+        tfc = readTFC(tfc_file)
+        out_pfn = "srm://srm01.ncg.ingrid.pt:8444/srm/managerv2?SFN=/cmst3/store/user/fred/data"
+        pfn = tfc.matchLFN('srmv2', in_lfn)
+        self.assertEqual(out_pfn, pfn)
+
+        tfc_file = os.path.join(getTestBase(),
+                                "WMCore_t/Storage_t",
+                                "T2_US_Florida_TrivialFileCatalog.xml")
+        tfc = readTFC(tfc_file)
+        out_pfn = "srm://srm.ihepa.ufl.edu:8443/srm/v2/server?SFN=/cms/data/store/user/fred/data"
+        pfn = tfc.matchLFN('srmv2', in_lfn)
+        self.assertEqual(out_pfn, pfn)
+
+        tfc_file = os.path.join(getTestBase(),
+                                "WMCore_t/Storage_t",
+                                "T2_ES_IFCA_TrivialFileCatalog.xml")
+        tfc = readTFC(tfc_file)
+        out_pfn = "srm://srm01.ifca.es:8444/srm/managerv2?SFN=/cms/store/user/fred/data"
+        pfn = tfc.matchLFN('srmv2', in_lfn)
+        self.assertEqual(out_pfn, pfn)
+
+        tfc_file = os.path.join(getTestBase(),
+                                "WMCore_t/Storage_t",
+                                "T2_US_Nebraska_TrivialFileCatalog.xml")
+        tfc = readTFC(tfc_file)
+        out_pfn = "srm://dcache07.unl.edu:8443/srm/v2/server?SFN=/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/user/fred/data"
+        pfn = tfc.matchLFN('srmv2', in_lfn)
+        self.assertEqual(out_pfn, pfn)
+
 
 if __name__ == "__main__":
     unittest.main()