first iteration in secrets implementation

Signed-off-by: Jorge Aguilera <[email protected]>
nextflow-io · Jul 22, 2024 · df053ed · df053ed
1 parent 88474c9
commit df053ed
Show file tree

Hide file tree

Showing 8 changed files with 130 additions and 3 deletions.
diff --git a/buildSrc/src/main/groovy/nextflow/gradle/plugins/GenerateIdxTask.groovy b/buildSrc/src/main/groovy/nextflow/gradle/plugins/GenerateIdxTask.groovy
@@ -34,6 +34,7 @@ abstract class GenerateIdxTask extends DefaultTask{
 
         def matcher = new SourcesMatcher(project)
         def extensionsClassName = matcher.pluginExtensions
+        extensionsClassName += matcher.providers
         def traceClassName = matcher.traceObservers
         def all = extensionsClassName+traceClassName
         output.text = all.join('\n')

diff --git a/buildSrc/src/main/groovy/nextflow/gradle/plugins/SourcesMatcher.groovy b/buildSrc/src/main/groovy/nextflow/gradle/plugins/SourcesMatcher.groovy
@@ -21,6 +21,11 @@ class SourcesMatcher {
                 findSources(/class (\w+) extends Executor implements ExtensionPoint/)
     }
 
+    List<String> getProviders(){
+        return findSources(/class (\w+) implements (.+)Provider/)
+    }
+
+
     List<String> getTraceObservers(){
         return findSources(/class (\w+) implements TraceObserverFactory/)
     }

diff --git a/plugins/nf-nomad/src/main/nextflow/nomad/NomadPlugin.groovy b/plugins/nf-nomad/src/main/nextflow/nomad/NomadPlugin.groovy
@@ -21,6 +21,7 @@ import groovy.transform.CompileStatic
 import nextflow.nomad.executor.TaskDirectives
 import nextflow.plugin.BasePlugin
 import nextflow.script.ProcessConfig
+import nextflow.secret.SecretsLoader
 import org.pf4j.PluginWrapper
 
 /**
@@ -35,6 +36,7 @@ class NomadPlugin extends BasePlugin {
     NomadPlugin(PluginWrapper wrapper) {
         super(wrapper)
         addCustomDirectives()
+        SecretsLoader.instance.reset()
     }
 
     private static void addCustomDirectives() {

diff --git a/plugins/nf-nomad/src/main/nextflow/nomad/NomadSecretProvider.groovy b/plugins/nf-nomad/src/main/nextflow/nomad/NomadSecretProvider.groovy
@@ -0,0 +1,60 @@
+package nextflow.nomad
+
+import groovy.util.logging.Slf4j
+import nextflow.plugin.Priority
+import nextflow.secret.Secret
+import nextflow.secret.SecretsProvider
+
+@Slf4j
+@Priority(-100) // high priority
+class NomadSecretProvider implements SecretsProvider, Closeable{
+
+    @Override
+    void close() throws IOException {
+    }
+
+    @Override
+    boolean activable() {
+        return true
+    }
+
+    @Override
+    SecretsProvider load() {
+        this
+    }
+
+    @Override
+    Secret getSecret(String name) {
+        log.error("NomadSecretProvider can't get secret, use nomad cli or disable it")
+        null
+    }
+
+    @Override
+    String getSecretsEnv(List<String> secretNames) {
+        log.error("NomadSecretProvider can't get secret, use nomad cli or disable it")
+        null
+    }
+
+    @Override
+    String getSecretsEnv() {
+        log.error("NomadSecretProvider can't get secret, use nomad cli or disable it")
+        null
+    }
+
+    @Override
+    void putSecret(String name, String value) {
+        throw new UnsupportedOperationException("NomadSecretProvider can't put secret, use nomad cli")
+    }
+
+    @Override
+    void removeSecret(String name) {
+        throw new UnsupportedOperationException("NomadSecretProvider can't remove secret, use nomad cli")
+    }
+
+    @Override
+    Set<String> listSecretsNames() {
+        log.error("NomadSecretProvider can't get secret, use nomad cli or disable it")
+        null
+    }
+
+}
diff --git a/plugins/nf-nomad/src/main/nextflow/nomad/executor/NomadService.groovy b/plugins/nf-nomad/src/main/nextflow/nomad/executor/NomadService.groovy
@@ -20,6 +20,7 @@ package nextflow.nomad.executor
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
 import io.nomadproject.client.ApiClient
+import io.nomadproject.client.ApiException
 import io.nomadproject.client.api.JobsApi
 import io.nomadproject.client.model.*
 import nextflow.nomad.models.ConstraintsBuilder
@@ -110,6 +111,9 @@ class NomadService implements Closeable{
         try {
             JobRegisterResponse jobRegisterResponse = jobsApi.registerJob(jobRegisterRequest, config.jobOpts().region, config.jobOpts().namespace, null, null)
             jobRegisterResponse.evalID
+        } catch( ApiException apiException){
+            log.debug("[NOMAD] Failed to submit ${job.name} -- Cause: ${apiException.responseBody ?: apiException}", apiException)
+            throw new ProcessSubmitException("[NOMAD] Failed to submit ${job.name} -- Cause: ${apiException.responseBody ?: apiException}", apiException)
         } catch (Throwable e) {
             log.debug("[NOMAD] Failed to submit ${job.name} -- Cause: ${e.message ?: e}", e)
             throw new ProcessSubmitException("[NOMAD] Failed to submit ${job.name} -- Cause: ${e.message ?: e}", e)
@@ -186,7 +190,7 @@ class NomadService implements Closeable{
         affinity(task, taskDef)
         constraint(task, taskDef)
         constraints(task, taskDef)
-
+        secrets(task, taskDef)
         return taskDef
     }
 
@@ -276,7 +280,18 @@ class NomadService implements Closeable{
         taskDef
     }
 
-
+    protected Task secrets(TaskRun task, Task taskDef){
+        def secrets = task.processor?.config?.get(TaskDirectives.SECRETS)
+        if( secrets ){
+            Template template = new Template(envvars: true, destPath: "/secrets/nf-nomad")
+            String tmpl = secrets.collect{ String name->
+                "${name}={{ with nomadVar \"secrets/${name}\" }}{{ .${name} }}{{ end }}"
+            }.join('\n').stripIndent()
+            template.embeddedTmpl(tmpl)
+            taskDef.addTemplatesItem(template)
+        }
+        taskDef
+    }
 
     protected Job assignDatacenters(TaskRun task, Job job){
         def datacenters = task.processor?.config?.get(TaskDirectives.DATACENTERS)

diff --git a/plugins/nf-nomad/src/main/nextflow/nomad/executor/TaskDirectives.groovy b/plugins/nf-nomad/src/main/nextflow/nomad/executor/TaskDirectives.groovy
@@ -6,8 +6,11 @@ class TaskDirectives {
 
     public static final String CONSTRAINTS = "constraints"
 
+    public static final String SECRETS = "secret"
+
     public static final List<String> ALL = [
             DATACENTERS,
-            CONSTRAINTS
+            CONSTRAINTS,
+            SECRETS
     ]
 }
diff --git a/validation/secrets/main.nf b/validation/secrets/main.nf
@@ -0,0 +1,20 @@
+#!/usr/bin/env nextflow
+
+process sayHello {
+    container   'ubuntu:20.04'
+    secret 'MY_ACCESS_KEY'
+    secret 'MY_SECRET_KEY'
+
+    input:
+    val x
+    output:
+    stdout
+
+    """
+    echo $x world! the access \$MY_ACCESS_KEY and the secret \$MY_SECRET_KEY
+    """
+}
+
+workflow {
+    Channel.of('Bonjour', 'Ciao', 'Hello', 'Hola') | sayHello | view
+}
diff --git a/validation/secrets/nextflow.config b/validation/secrets/nextflow.config
@@ -0,0 +1,21 @@
+plugins {
+    id "nf-nomad@${System.getenv("NOMAD_PLUGIN_VERSION") ?: "latest"}"
+}
+
+process {
+    executor = "nomad"
+}
+
+nomad {
+
+    client {
+        address = "http://localhost:4646"
+    }
+
+    jobs {
+        deleteOnCompletion = false
+        volume = { type "host" name "scratchdir" }
+    }
+
+}
+