Merge pull request yuanxingyang#3 from xuqi2024/jfrog_upload_feature

Jfrog upload feature

.gitignore

@@ -9,3 +9,4 @@ __pycache__/
 /dist/
 /htmlcov
 /pym/bob/version.py
+**/__pycache__/

doc/manual/configuration.rst

@@ -2067,6 +2067,12 @@ the following table for supported backends and their configuration.
 Backend     Description
 =========== ===================================================================
 none        Do not use a binary repository (default).
+artifactory JFrog Artifactory backend. Use the ``url`` keyword to provide the
+            repository url. Use optional keys ``username`` to specify the user
+            and ``key`` for the API-key or password. Without ``username`` and
+            ``key`` either no authentication or the global artifactory
+            configuration file is used. See dohq-artifactory documentation for
+            details.
 azure       Microsoft Azure Blob storage backend. The account must be specified
             in the ``account`` key. Either a ``key`` or a ``sasToken`` may
             be set to authenticate, otherwise an anonymous access is used.
@@ -2089,8 +2095,8 @@ shell       This backend can be used to execute commands that do the actual up-
             example below for a possible use with ``scp``.
 =========== ===================================================================
 
-The directory layouts of the ``azure``, ``file``, ``http`` and ``shell``
-(``$BOB_REMOTE_ARTIFACT``) backends are compatible. If multiple download
+The directory layouts of the ``artifactory``, ``azure``, ``file``, ``http`` and
+``shell`` (``$BOB_REMOTE_ARTIFACT``) backends are compatible. If multiple download
 backends are available they will be tried in order until a matching artifact is
 found. All available upload backends are used for uploading artifacts. Any
 failing upload will fail the whole build.

pym/bob/archive.py

@@ -24,7 +24,7 @@
 from .tty import stepAction, stepMessage, \
     SKIPPED, EXECUTED, WARNING, INFO, TRACE, ERROR, IMPORTANT
 from .utils import asHexStr, removePath, isWindows, sslNoVerifyContext, \
-    getBashPath
+    getBashPath, hashFile
 from shlex import quote
 from tempfile import mkstemp, NamedTemporaryFile, TemporaryFile, gettempdir
 import argparse
@@ -405,7 +405,7 @@ def _downloadPackage(self, buildId, suffix, audit, content, caches, workspace):
                     self._extract(fo, audit, content)
             return (True, None, None)
         except ArtifactNotFoundError:
-            return (False, "not found", WARNING)
+            return (False, "Not found in the backend", WARNING)
         except ArtifactDownloadError as e:
             return (False, e.reason, WARNING)
         except BuildError as e:
@@ -437,10 +437,11 @@ def _downloadLocalFile(self, key, suffix):
         # Set default signal handler so that KeyboardInterrupt is raised.
         # Needed to gracefully handle ctrl+c.
         signal.signal(signal.SIGINT, signal.default_int_handler)
-
         try:
             with self._openDownloadFile(key, suffix) as (name, fileobj):
                 ret = readFileOrHandle(name, fileobj)
+
+
             return (ret, None, None)
         except ArtifactNotFoundError:
             return (None, "not found", WARNING)
@@ -450,6 +451,8 @@ def _downloadLocalFile(self, key, suffix):
             raise
         except OSError as e:
             raise BuildError("Cannot download file: " + str(e))
+        except Exception as e:
+            raise BuildError("Other  " + str(e))
         finally:
             # Restore signals to default so that Ctrl+C kills process. Needed
             # to prevent ugly backtraces when user presses ctrl+c.
@@ -1104,6 +1107,235 @@ def __upload(self):
         except AzureError as e:
             raise ArtifactUploadError(str(e))
 
+class ArtifactoryArchive(BaseArchive):
+    def __init__(self, spec=None):
+        if spec:
+            super().__init__(spec)
+            self.__url = spec['url']
+            self.__username = spec.get('username', None)
+            self.__key = spec.get('key', None)
+        try:
+            from artifactory import ArtifactoryPath
+        except ImportError:
+            raise BuildError("dohq-artifactory Python3 library not installed!")
+
+    def setArgs(self, args):
+        self.__url = args.url
+        self.__username = args.username
+        self.__key = args.key
+
+    @staticmethod
+    def __makeBlobName(buildId, suffix):
+        packageResultId = buildIdToName(buildId)
+        return "/".join([packageResultId[0:2], packageResultId[2:4],
+            packageResultId[4:] + suffix])
+
+    def _remoteName(self, buildId, suffix):
+        a = "{}/{}".format(self.__url, self.__makeBlobName(buildId, suffix))
+        return "{}/{}".format(self.__url, self.__makeBlobName(buildId, suffix))
+
+    def _makeArtifactoryPath(self, blobName):
+        from artifactory import ArtifactoryPath
+        if self.__username and self.__key:
+            return ArtifactoryPath(
+                "{}/{}".format(self.__url, blobName.replace(os.sep, '_')),
+                auth=(self.__username, self.__key))
+        elif self.__key:
+            return ArtifactoryPath(
+                "{}/{}".format(self.__url, blobName.replace(os.sep, '_')),
+                apikey=self.__key)
+        else:
+            return ArtifactoryPath(
+                "{}/{}".format(self.__url, blobName.replace(os.sep, '_')))
+
+    def _openDownloadFile(self, buildId, suffix):
+        from artifactory import ArtifactoryPath
+        try:
+            fd = self._makeArtifactoryPath(self.__makeBlobName(buildId,suffix))
+
+            if not fd.exists():
+                raise ArtifactNotFoundError() 
+            fd = fd.open()
+            return ArtifactoryDownloader(fd)
+        except RuntimeError as e:
+            raise ArtifactDownloadError(str(e))
+
+    def _openUploadFile(self, buildId, suffix ,overwrite):
+        blobName = self.__makeBlobName(buildId,suffix)
+        from artifactory import ArtifactoryPath
+        try:
+            fd = self._makeArtifactoryPath(blobName)
+            try:
+
+                if not overwrite and fd.exists():
+                    raise ArtifactExistsError()
+            except RuntimeError:
+                pass
+        except RuntimeError as e:
+            raise ArtifactUploadError(str(e))
+        (tmpFd, tmpName) = mkstemp()
+        os.close(tmpFd)
+        return ArtifactoryUploader(self, fd, tmpName, blobName, overwrite)
+
+    def upload(self, step, buildIdFile, fingerprintFile, tgzFile):
+        if not self.canUploadJenkins():
+            return ""
+
+        args = []
+        if self.__key: args.append("--key=" + self.__key)
+        if self.__username: args.append("--username=" + self.__username)
+
+        return "\n" + textwrap.dedent("""\
+            # upload artifact
+            cd $WORKSPACE
+            bob _upload artifactory {ARGS} {URL} {BUILDID} {SUFFIX} {RESULT}{FIXUP}
+            """.format(ARGS=" ".join(map(quote, args)),
+                       URL=self.__url, BUILDID=quote(buildIdFile),
+                       RESULT=quote(tgzFile),
+                       FIXUP=" || echo Upload failed: $?" if self._ignoreErrors() else "",
+                       SUFFIX=artifactSuffixJenkins(fingerprintFile)))
+
+    def download(self, step, buildIdFile, fingerprintFile, tgzFile):
+        if not self.canDownloadJenkins():
+            return ""
+
+        args = []
+        if self.__key: args.append("--key=" + self.__key)
+        if self.__username: args.append("--username=" + self.__username)
+
+        return "\n" + textwrap.dedent("""\
+            if [[ ! -e {RESULT} ]] ; then
+                bob _download artifactory {ARGS} {URL} {BUILDID} {SUFFIX} {RESULT} || echo Download failed: $?
+            fi
+            """.format(ARGS=" ".join(map(quote, args)),
+                       URL=self.__url, BUILDID=quote(buildIdFile),
+                       RESULT=quote(tgzFile), SUFFIX=artifactSuffixJenkins(fingerprintFile)))
+
+    def uploadJenkinsLiveBuildId(self, step, liveBuildId, buildId):
+        if not self.canUploadJenkins():
+            return ""
+
+        args = []
+        if self.__key: args.append("--key=" + self.__key)
+        if self.__username: args.append("--username=" + self.__username)
+
+        return "\n" + textwrap.dedent("""\
+            # upload live build-id
+            cd $WORKSPACE
+            bob _upload artifactory {ARGS} {URL} {LIVEBUILDID} {SUFFIX} {BUILDID}{FIXUP}
+            """.format(ARGS=" ".join(map(quote, args)),
+                       URL=self.__url, LIVEBUILDID=quote(liveBuildId),
+                       BUILDID=quote(buildId),
+                       FIXUP=" || echo Upload failed: $?" if self._ignoreErrors() else "",
+                       SUFFIX=BUILDID_SUFFIX))
+
+    @staticmethod
+    def scriptDownload(args):
+        archive, remoteBlob, localFile = ArtifactoryArchive.scriptGetService(args)
+
+        from artifactory import ArtifactoryPath
+
+        # Download into temporary file and rename if downloaded successfully
+        tmpName = None
+        try:
+            (tmpFd, tmpName) = mkstemp(dir=".")
+
+            path = archive._makeArtifactoryPath(remoteBlob)
+
+            with path.open() as fd:
+                os.write(tmpFd, fd.read())
+
+            os.close(tmpFd)
+            os.rename(tmpName, localFile)
+            tmpName = None
+        except (OSError, RuntimeError) as e:
+            raise BuildError("Download failed: " + str(e))
+        finally:
+            if tmpName is not None: os.unlink(tmpName)
+
+    @staticmethod
+    def scriptUpload(args):
+        archive, remoteBlob, localFile = ArtifactoryArchive.scriptGetService(args)
+
+        from artifactory import ArtifactoryPath, md5sum, sha1sum
+        try:
+            sha1 = sha1sum(localFile)
+            md5 = md5sum(localFile)
+            with open(localFile, 'rb') as f:
+                fd = archive._makeArtifactoryPath(remoteBlob)
+                if fd.exists:
+                    print("skipped")
+                else:
+                    fd.deploy(f, sha1=sha1, md5=md5);
+            print("OK")
+        except (OSError, RuntimeError) as e:
+            raise BuildError("Upload failed: " + str(e))
+
+    @staticmethod
+    def scriptGetService(args):
+        parser = argparse.ArgumentParser()
+        parser.add_argument('url')
+        parser.add_argument('buildid')
+        parser.add_argument('suffix')
+        parser.add_argument('file')
+        parser.add_argument('--key')
+        parser.add_argument('--username')
+        args = parser.parse_args(args)
+
+        try:
+            from artifactory import ArtifactoryPath
+        except ImportError:
+            raise BuildError("artifactory Python3 library not installed!")
+
+        try:
+            with open(args.buildid, 'rb') as f:
+                remoteBlob = ArtifactoryArchive.__makeBlobName(f.read(), args.suffix)
+        except OSError as e:
+            raise BuildError(str(e))
+
+        archive = ArtifactoryArchive()
+        archive.setArgs(args)
+
+        return (archive, remoteBlob, args.file)
+
+class ArtifactoryDownloader:
+    def __init__(self, fd):
+        self.fd = fd
+    def __enter__(self):
+        return (None, self.fd)
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.fd.close()
+        return False
+
+class ArtifactoryUploader:
+    def __init__(self, artifactory, fd, name, remoteName, overwrite):
+        self.__artifactory = artifactory
+        self.__name = name
+        self.__remoteName = remoteName
+        self.__overwrite = overwrite
+
+    def __enter__(self):
+        return (self.__name, None)
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        try:
+            if exc_type is None:
+                self.__upload()
+        finally:
+            os.unlink(self.__name)
+        return False
+
+    def __upload(self):
+        from artifactory import ArtifactoryPath, md5sum, sha1sum
+
+        sha1 = sha1sum(self.__name)
+        md5 = md5sum(self.__name)
+        try:
+            with open(self.__name, 'rb') as f:
+                fd = self.__artifactory._makeArtifactoryPath(self.__remoteName)
+                fd.deploy(f, sha1=sha1, md5=md5);
+        except RuntimeError as e:
+            raise ArtifactUploadError("upload failed  "+ str(e))
 
 class MultiArchive:
     def __init__(self, archives):
@@ -1177,6 +1409,8 @@ def getSingleArchiver(recipes, archiveSpec):
         return CustomArchive(archiveSpec, recipes.envWhiteList())
     elif archiveBackend == "azure":
         return AzureArchive(archiveSpec)
+    elif archiveBackend == "artifactory":
+        return ArtifactoryArchive(archiveSpec)
     elif archiveBackend == "none":
         return DummyArchive()
     elif archiveBackend == "__jenkins":
@@ -1203,3 +1437,21 @@ def getArchiver(recipes, jenkins=None):
             return MultiArchive([ getSingleArchiver(recipes, i) for i in archiveSpec ])
     else:
         return getSingleArchiver(recipes, archiveSpec)
+
+def doDownload(args, bobRoot):
+    archiveBackend = args[0]
+    if archiveBackend == "azure":
+        AzureArchive.scriptDownload(args[1:])
+    elif archiveBackend == "artifactory":
+        ArtifactoryArchive.scriptDownload(args[1:])
+    else:
+        raise BuildError("Invalid archive backend: "+archiveBackend)
+
+def doUpload(args, bobRoot):
+    archiveBackend = args[0]
+    if archiveBackend == "azure":
+        AzureArchive.scriptUpload(args[1:])
+    elif archiveBackend == "artifactory":
+        ArtifactoryArchive.scriptUpload(args[1:])
+    else:
+        raise BuildError("Invalid archive backend: "+archiveBackend)