Plugin System

.bandit

@@ -0,0 +1,43 @@
+[tool.bandit]
+skips = [B101, B102, B105, B106, B107, B113, B202, B401, B402, B403, B404, B405, B406, B407, B408, B409, B410, B413, B307, B311, B507, B602, B603, B605, B607, B610, B611, B703]
+
+[tool.bandit.any_other_function_with_shell_equals_true]
+no_shell = [
+  "os.execl",
+  "os.execle",
+  "os.execlp",
+  "os.execlpe",
+  "os.execv",
+  "os.execve",
+  "os.execvp",
+  "os.execvpe",
+  "os.spawnl",
+  "os.spawnle",
+  "os.spawnlp",
+  "os.spawnlpe",
+  "os.spawnv",
+  "os.spawnve",
+  "os.spawnvp",
+  "os.spawnvpe",
+  "os.startfile"
+]
+shell = [
+  "os.system",
+  "os.popen",
+  "os.popen2",
+  "os.popen3",
+  "os.popen4",
+  "popen2.popen2",
+  "popen2.popen3",
+  "popen2.popen4",
+  "popen2.Popen3",
+  "popen2.Popen4",
+  "commands.getoutput",
+  "commands.getstatusoutput"
+]
+subprocess = [
+  "subprocess.Popen",
+  "subprocess.call",
+  "subprocess.check_call",
+  "subprocess.check_output"
+]

README.md

@@ -68,6 +68,14 @@ You can create custom nodes in python and make them available in Meshroom using
 In a standard precompiled version of Meshroom, you can also directly add custom nodes in `lib/meshroom/nodes`.
 To be recognized by Meshroom, a custom folder with nodes should be a Python module (an `__init__.py` file is needed).
 
+### Plugins
+
+Meshroom supports installing containerised plugins via Docker (with the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html)) or [Anaconda](https://docs.anaconda.com/free/miniconda/index.html).
+
+To do so, make sure docker or anaconda is installed properly and available from the command line.
+Then click on `File > Advanced > Install Plugin From URL` or `File > Advanced > Install Plugin From Local Folder` to begin the installation.
+
+To learn more about using or creating plugins, check the explanations [here](meshroom/plugins/README.md).
 
 ## License
 

meshroom/core/__init__.py

@@ -19,20 +19,27 @@
     pass
 
 from meshroom.core.submitter import BaseSubmitter
-from . import desc
+from meshroom.core import desc
 
 # Setup logging
 logging.basicConfig(format='[%(asctime)s][%(levelname)s] %(message)s', level=logging.INFO)
 
 # make a UUID based on the host ID and current time
 sessionUid = str(uuid.uuid1())
 
-cacheFolderName = 'MeshroomCache'
-defaultCacheFolder = os.environ.get('MESHROOM_CACHE', os.path.join(tempfile.gettempdir(), cacheFolderName))
 nodesDesc = {}
 submitters = {}
 pipelineTemplates = {}
 
+#meshroom paths
+meshroomFolder = os.path.dirname(os.path.dirname(__file__))
+cacheFolderName = 'MeshroomCache'
+defaultCacheFolder = os.environ.get('MESHROOM_CACHE', os.path.join(tempfile.gettempdir(), cacheFolderName))
+
+#plugin paths
+pluginsNodesFolder = os.path.join(meshroomFolder, "plugins")
+pluginsPipelinesFolder = os.path.join(meshroomFolder, "pipelines")
+pluginCatalogFile = os.path.join(meshroomFolder, "plugins", "catalog.json")
 
 def hashValue(value):
     """ Hash 'value' using sha1. """
@@ -329,29 +336,26 @@ def loadPipelineTemplates(folder):
 
 
 def initNodes():
-    meshroomFolder = os.path.dirname(os.path.dirname(__file__))
     additionalNodesPath = os.environ.get("MESHROOM_NODES_PATH", "").split(os.pathsep)
     # filter empty strings
     additionalNodesPath = [i for i in additionalNodesPath if i]
-    nodesFolders = [os.path.join(meshroomFolder, 'nodes')] + additionalNodesPath
+    nodesFolders = [os.path.join(meshroomFolder, 'nodes')] + additionalNodesPath + [pluginsNodesFolder]
     for f in nodesFolders:
         loadAllNodes(folder=f)
 
 
 def initSubmitters():
-    meshroomFolder = os.path.dirname(os.path.dirname(__file__))
     subs = loadSubmitters(os.environ.get("MESHROOM_SUBMITTERS_PATH", meshroomFolder), 'submitters')
     for sub in subs:
         registerSubmitter(sub())
 
 
 def initPipelines():
-    meshroomFolder = os.path.dirname(os.path.dirname(__file__))
     # Load pipeline templates: check in the default folder and any folder the user might have
     # added to the environment variable
     additionalPipelinesPath = os.environ.get("MESHROOM_PIPELINE_TEMPLATES_PATH", "").split(os.pathsep)
     additionalPipelinesPath = [i for i in additionalPipelinesPath if i]
-    pipelineTemplatesFolders = [os.path.join(meshroomFolder, 'pipelines')] + additionalPipelinesPath
+    pipelineTemplatesFolders = [os.path.join(meshroomFolder, 'pipelines')] + additionalPipelinesPath + [pluginsPipelinesFolder]
     for f in pipelineTemplatesFolders:
         if os.path.isdir(f):
             loadPipelineTemplates(f)

meshroom/core/desc.py

@@ -717,9 +717,56 @@
     documentation = ''
     category = 'Other'
 
+    _isPlugin = True
+
     def __init__(self):
         super(Node, self).__init__()
         self.hasDynamicOutputAttribute = any(output.isDynamicValue for output in self.outputs)
+        try:
+            self.envFile 
+            self.envType
+        except:
+            self._isPlugin=False
+
+    @property
+    def envType(cls):
+        from core.plugin import EnvType #lazy import for plugin to avoid circular dependency
+        return EnvType.NONE
+
+    @property
+    def envFile(cls):
+        """
+        Env file used to build the environement, you may overwrite this to custom the behaviour
+        """
+        raise NotImplementedError("You must specify an env file")
+
+    @property
+    def _envName(cls):
+        """
+        Get the env name by hashing the env files, overwrite this to use a custom pre-build env 
+        """
+        from meshroom.core.plugin import getEnvName, EnvType  #lazy import as to avoid circular dep
+        if cls.envType.value == EnvType.REZ.value:
+            return cls.envFile
+        with open(cls.envFile, 'r') as file:
+            envContent = file.read()
+
+        return getEnvName(envContent)
+
+    @property
+    def isPlugin(self):
+        """
+        Tests if the node is a valid plugin node
+        """
+        return self._isPlugin
+
+    @property
+    def isBuilt(self):
+        """
+        Tests if the environnement is built
+        """
+        from meshroom.core.plugin import isBuilt
+        return self._isPlugin and isBuilt(self)
 
     def upgradeAttributeValues(self, attrValues, fromVersion):
         return attrValues
@@ -806,7 +853,17 @@
         if chunk.node.isParallelized and chunk.node.size > 1:
             cmdSuffix = ' ' + self.commandLineRange.format(**chunk.range.toDict())
 
-        return cmdPrefix + chunk.node.nodeDesc.commandLine.format(**chunk.node._cmdVars) + cmdSuffix
+        cmd=cmdPrefix + chunk.node.nodeDesc.commandLine.format(**chunk.node._cmdVars) + cmdSuffix
+
+        #the process in Popen does not seem to use the right python, even if meshroom_compute is called within the env
+        #so in the case of command line using python, we have to make sure it is using the correct python
+        from meshroom.core.plugin import EnvType, getVenvPath, getVenvExe  #lazy import to prevent circular dep
+        if self.isPlugin and self.envType == EnvType.VENV:
+            envPath = getVenvPath(self._envName)
+            envExe = getVenvExe(envPath)
+            cmd=cmd.replace("python", envExe)
+
+        return cmd
 
     def stopProcess(self, chunk):
         # The same node could exists several times in the graph and

meshroom/core/node.py

@@ -21,7 +21,6 @@
 from meshroom.core.attribute import attributeFactory, ListAttribute, GroupAttribute, Attribute
 from meshroom.core.exception import NodeUpgradeError, UnknownNodeTypeError
 
-
 def getWritingFilepath(filepath):
     return filepath + '.writing.' + str(uuid.uuid4())
 
@@ -51,6 +50,8 @@
     KILLED = 5
     SUCCESS = 6
     INPUT = 7  # Special status for input nodes
+    BUILD = 8
+    FIRST_RUN = 9
 
 
 class ExecMode(Enum):
@@ -380,16 +381,16 @@
         renameWritingToFinalPath(statisticsFilepathWriting, statisticsFilepath)
 
     def isAlreadySubmitted(self):
-        return self._status.status in (Status.SUBMITTED, Status.RUNNING)
+        return self._status.status in (Status.SUBMITTED, Status.RUNNING, Status.BUILD, Status.FIRST_RUN)
 
     def isAlreadySubmittedOrFinished(self):
-        return self._status.status in (Status.SUBMITTED, Status.RUNNING, Status.SUCCESS)
+        return self._status.status in (Status.SUBMITTED, Status.RUNNING, Status.SUCCESS, Status.BUILD, Status.FIRST_RUN)
 
     def isFinishedOrRunning(self):
-        return self._status.status in (Status.SUCCESS, Status.RUNNING)
+        return self._status.status in (Status.SUCCESS, Status.RUNNING, Status.BUILD, Status.FIRST_RUN)
 
     def isRunning(self):
-        return self._status.status == Status.RUNNING
+        return self._status.status in (Status.RUNNING, Status.BUILD, Status.FIRST_RUN)
 
     def isStopped(self):
         return self._status.status == Status.STOPPED
@@ -401,36 +402,57 @@
         if not forceCompute and self._status.status == Status.SUCCESS:
             logging.info("Node chunk already computed: {}".format(self.name))
             return
-        global runningProcesses
-        runningProcesses[self.name] = self
-        self._status.initStartCompute()
-        exceptionStatus = None
-        startTime = time.time()
-        self.upgradeStatusTo(Status.RUNNING)
-        self.statThread = stats.StatisticsThread(self)
-        self.statThread.start()
-        try:
-            self.node.nodeDesc.processChunk(self)
-        except Exception:
-            if self._status.status != Status.STOPPED:
-                exceptionStatus = Status.ERROR
-            raise
-        except (KeyboardInterrupt, SystemError, GeneratorExit):
-            exceptionStatus = Status.STOPPED
-            raise
-        finally:
-            self._status.initEndCompute()
-            self._status.elapsedTime = time.time() - startTime
-            if exceptionStatus is not None:
-                self.upgradeStatusTo(exceptionStatus)
-            logging.info(" - elapsed time: {}".format(self._status.elapsedTimeStr))
-            # Ask and wait for the stats thread to stop
-            self.statThread.stopRequest()
-            self.statThread.join()
-            self.statistics = stats.Statistics()
-            del runningProcesses[self.name]
-
-        self.upgradeStatusTo(Status.SUCCESS)
+
+	    #if plugin node and if first call call meshroom_compute inside the env on 'host' so that the processchunk 
+        # of the node will be ran into the env
+        if self.node.nodeDesc.isPlugin and self._status.status!=Status.FIRST_RUN:
+            try:
+                from meshroom.core.plugin import isBuilt, build, getCommandLine #lazy import to avoid circular dep
+                if not isBuilt(self.node.nodeDesc):
+                    self.upgradeStatusTo(Status.BUILD)
+                    build(self.node.nodeDesc)
+                self.upgradeStatusTo(Status.FIRST_RUN)
+                command = getCommandLine(self)
+                #NOTE: docker returns 0 even if mount fail (it fails on the deamon side)
+                logging.info("Running plugin node with "+command)
+                status = os.system(command) 
+                if status != 0:
+                    raise RuntimeError("Error in node execution")
+                self.updateStatusFromCache()
+            except Exception as ex:
+                self.logger.exception(ex)
+                self.upgradeStatusTo(Status.ERROR)
+        else:
+            global runningProcesses
+            runningProcesses[self.name] = self
+            self._status.initStartCompute()
+            exceptionStatus = None
+            startTime = time.time()
+            self.upgradeStatusTo(Status.RUNNING)
+            self.statThread = stats.StatisticsThread(self)
+            self.statThread.start()
+            try:
+                self.node.nodeDesc.processChunk(self)
+            except Exception:
+                if self._status.status != Status.STOPPED:
+                    exceptionStatus = Status.ERROR
+                raise
+            except (KeyboardInterrupt, SystemError, GeneratorExit):
+                exceptionStatus = Status.STOPPED
+                raise
+            finally:
+                self._status.initEndCompute()
+                self._status.elapsedTime = time.time() - startTime
+                if exceptionStatus is not None:
+                    self.upgradeStatusTo(exceptionStatus)
+                logging.info(" - elapsed time: {}".format(self._status.elapsedTimeStr))
+                # Ask and wait for the stats thread to stop
+                self.statThread.stopRequest()
+                self.statThread.join()
+                self.statistics = stats.Statistics()
+                del runningProcesses[self.name]
+
+            self.upgradeStatusTo(Status.SUCCESS)
 
     def stopProcess(self):
         if not self.isExtern():
@@ -460,6 +482,7 @@
     statusNodeName = Property(str, lambda self: self._status.nodeName, constant=True)
 
     elapsedTime = Property(float, lambda self: self._status.elapsedTime, notify=statusChanged)
+
 
 
 # Simple structure for storing node position
@@ -1130,8 +1153,8 @@
             return Status.INPUT
         chunksStatus = [chunk.status.status for chunk in self._chunks]
 
-        anyOf = (Status.ERROR, Status.STOPPED, Status.KILLED,
-                 Status.RUNNING, Status.SUBMITTED)
+        anyOf = (Status.ERROR, Status.STOPPED, Status.KILLED, Status.RUNNING, Status.BUILD, Status.FIRST_RUN, 
+                 Status.SUBMITTED,)
         allOf = (Status.SUCCESS,)
 
         for status in anyOf:
@@ -1394,6 +1417,12 @@
     hasSequenceOutput = Property(bool, hasSequenceOutputAttribute, notify=outputAttrEnabledChanged)
     has3DOutput = Property(bool, has3DOutputAttribute, notify=outputAttrEnabledChanged)
 
+    isPlugin = Property(bool, lambda self: self.nodeDesc.isPlugin if self.nodeDesc is not None else False, constant=True)
+
+    isEnvBuild = (not isPlugin) #init build status false its not a plugin  
+    buildStatusChanged = Signal() #event to notify change in status
+    isBuiltStatus  = Property(bool, lambda self: self.isEnvBuild,  notify = buildStatusChanged)
+    # isBuiltStatus  = Property(bool, lambda self: self.nodeDesc.isBuilt,  constant=True)
 
 class Node(BaseNode):
     """